From 73a31daa59c767b1451395068103b4d4b33e0cf3 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sat, 9 May 2026 06:03:01 +0000 Subject: [PATCH 01/44] NMFW-464 phase 1 hypercomm expert groups --- megatron/core/hyper_comm_grid.py | 204 +++++++++++++++++- megatron/core/models/mimo/optimizer.py | 33 ++- megatron/core/process_groups_config.py | 63 ++++++ .../models/test_mimo_1f1b_schedule.py | 145 +++++++++++-- .../models/test_mimo_colocated_correctness.py | 84 ++++++++ tests/unit_tests/test_hyper_comm_grid.py | 128 +++++++++++ .../unit_tests/test_process_groups_config.py | 107 +++++++++ 7 files changed, 730 insertions(+), 34 deletions(-) diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py index 4b860396c4e..4f9a8d7251e 100644 --- a/megatron/core/hyper_comm_grid.py +++ b/megatron/core/hyper_comm_grid.py @@ -1,6 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import os +from dataclasses import dataclass, field from operator import itemgetter from typing import Any, Optional, Tuple, Union @@ -30,6 +31,19 @@ HAVE_ABSL = False +@dataclass +class _GridLayout: + """Rank layout owned by a HyperCommGrid. + + The base layout is the original Cartesian grid. Registered layouts are + alternate factorizations over the same rank span. + """ + + shape: list[int] + dim_names: list[str] + aliases: dict[str, list[str]] = field(default_factory=dict) + + class HyperCommGrid: r"""N-dimensional communication grid. @@ -116,6 +130,94 @@ def __init__( self.dim_names = dim_names[:] self.backend = backend self._pgs: dict[str, dist.ProcessGroup] = {} + self._layouts: dict[str, _GridLayout] = {"base": _GridLayout(self.shape, self.dim_names)} + self._aliases: dict[str, tuple[str, list[str]]] = {} + + def register_layout( + self, + name: str, + shape: list[int], + dim_names: list[str], + aliases: Optional[dict[str, list[str]]] = None, + ) -> None: + """Register an alternate rank layout over this grid's rank span. + + Registered layouts are useful when the same module rank universe has + more than one valid factorization, such as dense + ``tp/cp/dp/pp`` groups and expert ``expt_tp/ep/expt_dp/pp`` + groups. The original constructor remains a single Cartesian grid. + + Args: + name: Unique name for the alternate layout. + shape: Shape of the alternate layout. Its product must equal the + base grid size. + dim_names: Dimension names for the alternate layout. + aliases: Optional names for composite groups in this layout. + For example, ``{"tp_ep": ["expt_tp", "ep"]}``. + """ + if name == "base": + raise ValueError("'base' is reserved for the default HyperCommGrid layout") + if name in self._layouts: + raise ValueError(f"Layout {name!r} is already registered") + if len(shape) != len(dim_names): + raise ValueError(f"len(shape) {shape} != len(dim_names) {dim_names}") + if len(set(dim_names)) != len(dim_names): + raise ValueError(f"Layout {name!r} has duplicate dim_names: {dim_names}") + if np.prod(shape) != self.size: + raise ValueError( + f"Layout {name!r} shape {shape} has size {np.prod(shape)}, " + f"but base grid size is {self.size}" + ) + + layout = _GridLayout(shape[:], dim_names[:]) + + for dim in set(dim_names).intersection(self.dim_names): + base_enum = self._gen_rank_enum_for_layout([dim], "base") + layout_enum = self._gen_rank_enum_for_layout([dim], None, layout) + if base_enum != layout_enum: + raise ValueError( + f"Layout {name!r} dimension {dim!r} collides with the base layout " + "but has different rank enumeration" + ) + + aliases = aliases or {} + for alias_name, alias_dims in aliases.items(): + if alias_name in self._aliases: + raise ValueError(f"Alias {alias_name!r} is already registered") + if alias_name in self.dim_names or alias_name in dim_names: + raise ValueError(f"Alias {alias_name!r} conflicts with an existing dimension name") + if "-" in alias_name: + raise ValueError( + f"Alias {alias_name!r} cannot contain '-' because process group keys use '-'" + ) + if len(set(alias_dims)) != len(alias_dims): + raise ValueError(f"Alias {alias_name!r} has duplicate dimensions: {alias_dims}") + missing_dims = [dim for dim in alias_dims if dim not in dim_names] + if missing_dims: + raise ValueError( + f"Alias {alias_name!r} references dimensions not in layout {name!r}: " + f"{missing_dims}" + ) + layout.aliases[alias_name] = alias_dims[:] + + self._layouts[name] = layout + for alias_name, alias_dims in layout.aliases.items(): + self._aliases[alias_name] = (name, alias_dims[:]) + + def has_layout(self, name: str) -> bool: + """Return whether a named layout is registered.""" + return name in self._layouts + + def has_alias(self, name: str) -> bool: + """Return whether an alias is registered.""" + return name in self._aliases + + def get_alias_dims(self, name: str) -> list[str]: + """Return a copy of the dimensions referenced by an alias.""" + if name not in self._aliases: + raise KeyError(f"Alias {name!r} is not registered") + _, alias_dims = self._aliases[name] + return alias_dims[:] def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessGroup | None: r"""Create a process group based on a list of dimension names @@ -145,8 +247,8 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG Raises: KeyError: If attempting to recreate a process group with an existing key. """ - # ordered_dims and unique_group_key will follow the reversed order of self.dim_names - ordered_dims, unique_group_key = self._order_dims(dims) + # ordered_dims follows the reversed order of the owning layout's dim_names. + layout_name, ordered_dims, unique_group_key = self._resolve_dims(dims) if unique_group_key in self._pgs: raise KeyError( @@ -155,10 +257,10 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG f"of returning the process group that has already been created before." ) - rank_enum = self._gen_rank_enum(ordered_dims) + rank_enum = self._gen_rank_enum_for_layout(ordered_dims, layout_name) pg, _ = dist.new_subgroups_by_enumeration(rank_enum, backend=self.backend, **kwargs) - if dist.get_rank() == 0: + if not dist.is_initialized() or dist.get_rank() == 0: logging.info( f"Generated process group for {unique_group_key} with enumeration {rank_enum}" ) @@ -178,7 +280,7 @@ def get_pg(self, dims: Union[str, list[str]]) -> dist.ProcessGroup: Args: dims: Name of leading dimensions to create process group """ - _, unique_group_key = self._order_dims(dims) + _, _, unique_group_key = self._resolve_dims(dims) if unique_group_key not in self._pgs: raise KeyError( @@ -187,7 +289,9 @@ def get_pg(self, dims: Union[str, list[str]]) -> dist.ProcessGroup: return self._pgs[unique_group_key] - def get_rank_enum(self, dims: Union[str, list[str]]) -> list[list[int]]: + def get_rank_enum( + self, dims: Union[str, list[str]], layout_name: Optional[str] = None + ) -> list[list[int]]: r"""Get the rank enumeration for the requested dimension(s). This is the exact enumeration that would be used by create_pg for the same @@ -196,14 +300,25 @@ def get_rank_enum(self, dims: Union[str, list[str]]) -> list[list[int]]: Args: dims: Dimension name or list of dimension names. + layout_name: Optional registered layout name. When unset, the + owning layout is inferred from dims or aliases. Returns: List of rank lists (one per subgroup). """ - ordered_dims, _ = self._order_dims(dims) - return self._gen_rank_enum(ordered_dims) + if layout_name is None: + layout_name, ordered_dims, _ = self._resolve_dims(dims) + else: + dims = self._expand_alias(dims, layout_name) + ordered_dims, _ = self._order_dims_for_layout(dims, layout_name) + return self._gen_rank_enum_for_layout(ordered_dims, layout_name) def _gen_rank_enum(self, dims: list[str]) -> list[list[int]]: + return self._gen_rank_enum_for_layout(dims, "base") + + def _gen_rank_enum_for_layout( + self, dims: list[str], layout_name: Optional[str], layout: Optional[_GridLayout] = None + ) -> list[list[int]]: r"""Generate rank enumeration before calling new_subgroups_by_enumeration This function returns ranks grouped by the specified dimensions, but in REVERSE order @@ -229,9 +344,12 @@ def _gen_rank_enum(self, dims: list[str]) -> list[list[int]]: raise RuntimeError( "einops is not installed. Please install it with `pip install einops`." ) + if layout is None: + assert layout_name is not None + layout = self._layouts[layout_name] # Need to reverse order of dim_names to match MCore convention - dim_names_reverse = self.dim_names[::-1] + dim_names_reverse = layout.dim_names[::-1] remaining_dims = [] for v in dim_names_reverse: @@ -243,17 +361,33 @@ def _gen_rank_enum(self, dims: list[str]) -> list[list[int]]: ) logging.debug(rearrange_str) - shape_dict = {d: s for d, s in zip(self.dim_names, self.shape)} + shape_dict = {d: s for d, s in zip(layout.dim_names, layout.shape)} return einops.rearrange( np.arange(self.rank_offset, self.rank_offset + self.size), rearrange_str, **shape_dict ).tolist() def _order_dims(self, dims: Union[str, list[str]]) -> Tuple[list[str], str]: + return self._order_dims_for_layout(dims, "base") + + def _order_dims_for_layout( + self, dims: Union[str, list[str]], layout_name: str + ) -> Tuple[list[str], str]: r"""Reorder dims based on the order of self.dim_names""" + layout = self._layouts[layout_name] if not isinstance(dims, list): + if dims not in layout.dim_names: + raise ValueError( + f"Dimension {dims!r} is not in layout {layout_name!r}: {layout.dim_names}" + ) ordered_dims = [dims] else: - dim_names_reverse = self.dim_names[::-1] + dim_names_reverse = layout.dim_names[::-1] + missing_dims = [d for d in dims if d not in dim_names_reverse] + if missing_dims: + raise ValueError( + f"Dimensions {missing_dims} are not in layout {layout_name!r}: " + f"{layout.dim_names}" + ) indices = sorted([dim_names_reverse.index(d) for d in dims]) if len(indices) == 1: ordered_dims = [dim_names_reverse[indices[0]]] @@ -263,6 +397,54 @@ def _order_dims(self, dims: Union[str, list[str]]) -> Tuple[list[str], str]: unique_group_key = "-".join(ordered_dims) return ordered_dims, unique_group_key + def _resolve_dims(self, dims: Union[str, list[str]]) -> Tuple[str, list[str], str]: + if isinstance(dims, str) and dims in self._aliases: + layout_name, alias_dims = self._aliases[dims] + ordered_dims, _ = self._order_dims_for_layout(alias_dims, layout_name) + return layout_name, ordered_dims, dims + + raw_dims = [dims] if isinstance(dims, str) else dims + + if all(dim in self.dim_names for dim in raw_dims): + ordered_dims, unique_group_key = self._order_dims_for_layout(raw_dims, "base") + return "base", ordered_dims, unique_group_key + + candidate_layouts = [ + name + for name, layout in self._layouts.items() + if name != "base" and all(dim in layout.dim_names for dim in raw_dims) + ] + if not candidate_layouts: + raise ValueError( + f"Dimensions {raw_dims} are not all present in a single registered layout" + ) + if len(candidate_layouts) > 1: + raise ValueError( + f"Dimensions {raw_dims} match multiple registered layouts: {candidate_layouts}" + ) + + layout_name = candidate_layouts[0] + if len(raw_dims) > 1: + aliases = sorted(self._layouts[layout_name].aliases) + raise ValueError( + f"Composite dimensions {raw_dims} from registered layout {layout_name!r} " + f"must use an explicit alias. Available aliases: {aliases}" + ) + + ordered_dims, unique_group_key = self._order_dims_for_layout(raw_dims, layout_name) + return layout_name, ordered_dims, unique_group_key + + def _expand_alias(self, dims: Union[str, list[str]], layout_name: str) -> Union[str, list[str]]: + if not isinstance(dims, str) or dims not in self._aliases: + return dims + + alias_layout_name, alias_dims = self._aliases[dims] + if alias_layout_name != layout_name: + raise ValueError( + f"Alias {dims!r} belongs to layout {alias_layout_name!r}, not {layout_name!r}" + ) + return alias_dims[:] + def is_current_rank_in_grid(self) -> bool: """Check if the current rank belongs to this grid. diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 1a79c1f91ff..456199c61cd 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -51,6 +51,7 @@ def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: Optimiz @torch.no_grad() def prepare_grads(self) -> bool: + """Prepare gradients for all active module optimizers.""" found_inf = False for opt in self._active_optimizers: found_inf |= opt.prepare_grads() @@ -72,6 +73,7 @@ def get_grad_norm(self) -> float: @torch.no_grad() def step(self) -> Tuple[bool, Optional[float], Optional[int]]: + """Run a synchronized optimizer step across active module optimizers.""" found_inf = self.prepare_grads() # Synchronize found_inf across all ranks to prevent deadlock: # if encoder ranks detect inf but LLM ranks don't, the early return @@ -104,21 +106,25 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: @torch.no_grad() def step_with_ready_grads(self) -> bool: + """Step active module optimizers after gradients have been prepared.""" success = True for opt in self._active_optimizers: success &= opt.step_with_ready_grads() return success def zero_grad(self, set_to_none: bool = True): + """Clear gradients on all active module optimizers.""" for opt in self._active_optimizers: opt.zero_grad(set_to_none) def get_loss_scale(self) -> torch.Tensor: + """Return the loss scale from the first active optimizer, or one for stubs.""" if self._active_optimizers: return self._active_optimizers[0].get_loss_scale() return torch.tensor([1.0], dtype=torch.float32, device="cuda") def count_zeros(self) -> int: + """Count zero gradients across all active module optimizers.""" return sum(opt.count_zeros() for opt in self._active_optimizers) @property @@ -132,6 +138,7 @@ def param_groups(self) -> List[dict]: # Checkpointing def state_dict(self): + """Return per-module optimizer state dicts.""" return { name: info.optimizer.state_dict() if info.is_active and info.optimizer else None for name, info in self.module_infos.items() @@ -183,6 +190,7 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, return sharded_state def reload_model_params(self, state_dict=None): + """Reload model parameters for all active module optimizers.""" for opt in self._active_optimizers: opt.reload_model_params(state_dict) @@ -286,15 +294,16 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: Only fetches process groups required by the optimizer. Assumes all groups are pre-created in the grid via grid.create_pg() - does not create any new groups. - The following groups must be pre-created in the grid before calling this function: + For extended HyperCommGrid instances with registered expert layouts, the following + groups must be pre-created in the grid before calling this function: grid.create_pg(["dp"]) grid.create_pg(["dp", "cp"]) grid.create_pg(["tp"]) grid.create_pg(["pp"]) grid.create_pg(["tp", "pp"]) - grid.create_pg(["tp", "ep", "pp"]) - grid.create_pg(["dp", "ep"]) - grid.create_pg(["tp", "cp", "ep", "pp", "dp"]) + grid.create_pg("tp_ep_pp") + grid.create_pg("expt_dp") + grid.create_pg(["tp", "cp", "dp", "pp"]) Args: grid: HyperCommGrid with pre-created process groups. @@ -308,6 +317,22 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: - tp_ep_pp: Expert tensor-model-pipeline group - expt_dp: Expert data parallel group """ + try: + return ProcessGroupCollection.from_hyper_comm_grid( + grid, + create=False, + required_pgs=['dp', 'dp_cp', 'tp', 'pp', 'mp', 'tp_ep_pp', 'expt_dp', 'intra_dist_opt'], + ) + except (KeyError, ValueError) as exc: + has_registered_expert_aliases = hasattr(grid, 'has_alias') and ( + grid.has_alias('tp_ep') or grid.has_alias('tp_ep_pp') + ) + if has_registered_expert_aliases: + raise exc + # Backward-compatible fallback for older tests/grids that encoded EP + # directly in the base Cartesian layout. + pass + pg = ProcessGroupCollection() # Core groups needed by optimizer and checkpointing diff --git a/megatron/core/process_groups_config.py b/megatron/core/process_groups_config.py index 6c1e3651387..58bae2dd1b7 100644 --- a/megatron/core/process_groups_config.py +++ b/megatron/core/process_groups_config.py @@ -259,6 +259,69 @@ def use_mpu_process_groups(cls, required_pgs: Optional[List[str]] = None): return cls(**init_dict) + @classmethod + def from_hyper_comm_grid( + cls, + grid, + create: bool = False, + required_pgs: Optional[List[str]] = None, + num_distributed_optimizer_instances: int = 1, + ): + """Build a ProcessGroupCollection from an extended HyperCommGrid. + + The grid must expose expert groups via registered layout dimensions + such as ``expt_tp``/``expt_dp`` and aliases such as ``tp_ep_pp``. + When ``create`` is True, the helper owns group creation in a + deterministic order. Otherwise it only reads groups that must already + exist on the grid. + """ + if num_distributed_optimizer_instances != 1: + raise ValueError( + "ProcessGroupCollection.from_hyper_comm_grid only supports " + "num_distributed_optimizer_instances == 1" + ) + + pg_specs = { + 'tp': 'tp', + 'cp': 'cp', + 'pp': 'pp', + 'dp': 'dp', + 'dp_cp': ['dp', 'cp'], + 'tp_cp': ['tp', 'cp'], + 'mp': ['tp', 'pp'], + 'tp_dp_cp': ['tp', 'dp', 'cp'], + 'ep': 'ep', + 'expt_tp': 'expt_tp', + 'expt_dp': 'expt_dp', + 'tp_ep': 'tp_ep', + 'tp_ep_pp': 'tp_ep_pp', + 'intra_dist_opt': grid.dim_names, + } + if required_pgs is None: + required_pgs = list(pg_specs) + + invalid_pgs = [pg for pg in required_pgs if pg not in pg_specs] + if invalid_pgs: + raise ValueError(f"Invalid process groups requested: {invalid_pgs}") + + if 'tp_ep_pp' in required_pgs and hasattr(grid, 'get_alias_dims'): + alias_dims = grid.get_alias_dims('tp_ep_pp') + if 'pp' not in alias_dims: + raise ValueError("tp_ep_pp alias must include the shared pipeline dimension 'pp'") + + def get_or_create(dims): + return grid.create_pg(dims) if create else grid.get_pg(dims) + + init_dict = {pg_name: get_or_create(pg_specs[pg_name]) for pg_name in required_pgs} + + if 'dp_cp' in init_dict: + init_dict.setdefault('intra_dp_cp', init_dict['dp_cp']) + if 'expt_dp' in init_dict: + init_dict.setdefault('intra_expt_dp', init_dict['expt_dp']) + init_dict.setdefault('inter_dist_opt', None) + + return cls(**init_dict) + @staticmethod def setup_process_groups_for_optimizer( pg_collection: Optional['ProcessGroupCollection'], diff --git a/tests/unit_tests/models/test_mimo_1f1b_schedule.py b/tests/unit_tests/models/test_mimo_1f1b_schedule.py index 836382b21cc..004ec52697f 100644 --- a/tests/unit_tests/models/test_mimo_1f1b_schedule.py +++ b/tests/unit_tests/models/test_mimo_1f1b_schedule.py @@ -81,26 +81,44 @@ def no_sync_func(): return no_sync_func -def create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1): +def create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1, ep=1, expt_tp=None, expt_dp=None): """Create a HyperCommGrid with specified parallelism.""" + expt_tp = tp if expt_tp is None else expt_tp + module_world_size = tp * cp * pp * dp + expert_model_size = expt_tp * ep * pp + if expt_dp is None: + assert module_world_size % expert_model_size == 0, ( + f"module_world_size ({module_world_size}) must be divisible by " + f"expt_tp*ep*pp ({expert_model_size})" + ) + expt_dp = module_world_size // expert_model_size + grid = HyperCommGrid( - shape=[tp, cp, pp, dp, 1, 1], # [tp, cp, pp, dp, ep, expt_dp] - dim_names=["tp", "cp", "pp", "dp", "ep", "expt_dp"], + shape=[tp, cp, dp, pp], + dim_names=["tp", "cp", "dp", "pp"], rank_offset=offset, backend="nccl", ) + grid.register_layout( + "expert", + [expt_tp, ep, expt_dp, pp], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, + ) grid.create_pg(["tp"]) grid.create_pg(["cp"]) grid.create_pg(["pp"]) grid.create_pg(["dp"]) grid.create_pg(["dp", "cp"]) + grid.create_pg(["tp", "cp"]) grid.create_pg(["ep"]) + grid.create_pg(["expt_tp"]) grid.create_pg(["expt_dp"]) - # Required by _get_pg_collection_for_optimizer grid.create_pg(["tp", "pp"]) - grid.create_pg(["tp", "ep", "pp"]) - grid.create_pg(["dp", "ep"]) - grid.create_pg(["tp", "cp", "ep", "pp", "dp"]) + grid.create_pg(["tp", "cp", "dp"]) + grid.create_pg(["tp", "cp", "pp", "dp"]) + grid.create_pg("tp_ep") + grid.create_pg("tp_ep_pp") _active_grids.append(grid) return grid @@ -116,15 +134,25 @@ def destroy_all_grids(): def get_pg_collection(grid): """Get ProcessGroupCollection from grid.""" - pg_collection = ProcessGroupCollection() - pg_collection.tp = grid.get_pg("tp") - pg_collection.cp = grid.get_pg("cp") - pg_collection.pp = grid.get_pg("pp") - pg_collection.ep = grid.get_pg("ep") - pg_collection.dp = grid.get_pg("dp") - pg_collection.dp_cp = grid.get_pg(["dp", "cp"]) - pg_collection.expt_dp = grid.get_pg("expt_dp") - return pg_collection + return ProcessGroupCollection.from_hyper_comm_grid( + grid, + required_pgs=[ + 'tp', + 'cp', + 'pp', + 'dp', + 'dp_cp', + 'tp_cp', + 'mp', + 'tp_dp_cp', + 'ep', + 'expt_tp', + 'expt_dp', + 'tp_ep', + 'tp_ep_pp', + 'intra_dist_opt', + ], + ) def create_all_embedding_groups(grids): @@ -214,6 +242,9 @@ def get_language_model_spec( bias=True, dropout=True, per_token_loss=False, + num_moe_experts=None, + moe_router_topk=1, + moe_grouped_gemm=False, ): """Get the language model spec. @@ -229,6 +260,8 @@ def get_language_model_spec( pp_rank = dist.get_rank(pg_collection.pp) pp_size = dist.get_world_size(pg_collection.pp) tp_size = pg_collection.tp.size() if pg_collection.tp is not None else 1 + ep_size = pg_collection.ep.size() if pg_collection.ep is not None else 1 + expt_tp_size = pg_collection.expt_tp.size() if pg_collection.expt_tp is not None else tp_size pipeline_dtype = torch.bfloat16 if bf16 else torch.float32 extra_kwargs = {} @@ -237,6 +270,17 @@ def get_language_model_spec( if not dropout: extra_kwargs['attention_dropout'] = 0.0 extra_kwargs['hidden_dropout'] = 0.0 + if num_moe_experts is not None: + extra_kwargs.update( + { + 'num_moe_experts': num_moe_experts, + 'moe_router_topk': moe_router_topk, + 'moe_router_pre_softmax': moe_router_topk == 1, + 'expert_model_parallel_size': ep_size, + 'expert_tensor_parallel_size': expt_tp_size, + 'moe_grouped_gemm': moe_grouped_gemm, + } + ) lm_config = TransformerConfig( num_layers=num_layers, @@ -258,7 +302,9 @@ def get_language_model_spec( module=GPTModel, params={ "config": lm_config, - "transformer_layer_spec": get_gpt_layer_with_transformer_engine_spec(), + "transformer_layer_spec": get_gpt_layer_with_transformer_engine_spec( + num_experts=num_moe_experts, moe_grouped_gemm=moe_grouped_gemm + ), "vocab_size": vocab_size, "max_sequence_length": seq_len, "pre_process": (pp_rank == 0), @@ -380,6 +426,9 @@ def get_mimo_model( bias=True, dropout=True, per_token_loss=False, + language_num_moe_experts=None, + language_moe_router_topk=1, + language_moe_grouped_gemm=False, ): """Create MIMO model with TransformerBlock encoder and GPTModel LLM. @@ -414,6 +463,9 @@ def get_mimo_model( bias=bias, dropout=dropout, per_token_loss=per_token_loss, + num_moe_experts=language_num_moe_experts, + moe_router_topk=language_moe_router_topk, + moe_grouped_gemm=language_moe_grouped_gemm, ) vision_submodule_spec = get_vision_submodules_spec( num_layers=num_layers, @@ -560,6 +612,15 @@ def run_mimo_1f1b_test( llm_pp, llm_dp, llm_offset, + encoder_ep=1, + encoder_expt_tp=None, + encoder_expt_dp=None, + llm_ep=1, + llm_expt_tp=None, + llm_expt_dp=None, + language_num_moe_experts=None, + language_moe_router_topk=1, + language_moe_grouped_gemm=False, hidden_size=256, num_layers=2, vocab_size=1000, @@ -579,9 +640,25 @@ def run_mimo_1f1b_test( encoder_name = "images" encoder_grid = create_hypercomm_grid( - offset=encoder_offset, tp=encoder_tp, cp=1, pp=encoder_pp, dp=encoder_dp + offset=encoder_offset, + tp=encoder_tp, + cp=1, + pp=encoder_pp, + dp=encoder_dp, + ep=encoder_ep, + expt_tp=encoder_expt_tp, + expt_dp=encoder_expt_dp, + ) + llm_grid = create_hypercomm_grid( + offset=llm_offset, + tp=llm_tp, + cp=1, + pp=llm_pp, + dp=llm_dp, + ep=llm_ep, + expt_tp=llm_expt_tp, + expt_dp=llm_expt_dp, ) - llm_grid = create_hypercomm_grid(offset=llm_offset, tp=llm_tp, cp=1, pp=llm_pp, dp=llm_dp) # Create all embedding PGs upfront — dist.new_group is a collective that # requires ALL ranks to participate, so we must create them before any @@ -598,6 +675,9 @@ def run_mimo_1f1b_test( num_layers=num_layers, vocab_size=vocab_size, seq_len=seq_length, + language_num_moe_experts=language_num_moe_experts, + language_moe_router_topk=language_moe_router_topk, + language_moe_grouped_gemm=language_moe_grouped_gemm, ) no_sync_func = build_no_sync_func(mimo_model) @@ -844,6 +924,33 @@ def test_full_pp_8gpu(self): num_microbatches=4, ) + def test_moe_lm_ep2_edp1_pp2_8gpu(self): + """MoE LLM uses EP=2/EDP=1 over a PP=2 language pipeline.""" + if self.world_size != 8: + pytest.skip(f"Requires 8 GPUs, got {self.world_size}") + + run_mimo_1f1b_test( + encoder_tp=2, + encoder_pp=2, + encoder_dp=1, + encoder_offset=0, + llm_tp=1, + llm_pp=2, + llm_dp=2, + llm_offset=4, + llm_ep=2, + llm_expt_tp=1, + llm_expt_dp=1, + language_num_moe_experts=4, + language_moe_router_topk=1, + hidden_size=128, + num_layers=2, + vocab_size=512, + seq_length=32, + micro_batch_size=2, + num_microbatches=2, + ) + def test_fan_in_dp4_to_dp1_llm_tp2_pp2_8gpu(self): """Fan-in 4→1: Encoder DP=4 → LLM TP=2 PP=2 DP=1, on 8 GPUs. diff --git a/tests/unit_tests/models/test_mimo_colocated_correctness.py b/tests/unit_tests/models/test_mimo_colocated_correctness.py index e2d91bdf83e..7a0e1a71499 100644 --- a/tests/unit_tests/models/test_mimo_colocated_correctness.py +++ b/tests/unit_tests/models/test_mimo_colocated_correctness.py @@ -1181,3 +1181,87 @@ def test_dist_matches_dp1_reference_post_step_weights( if failures: summary = "\n\n".join(f"== {oracle} ==\n{msg}" for oracle, msg in failures) raise AssertionError(f"{len(failures)} oracle(s) failed:\n{summary}") + + @pytest.mark.skipif( + version.parse(torch.__version__) < version.parse("2.3.0"), reason="Requires PyTorch 2.3+" + ) + def test_colocated_moe_lm_etp2_ep2_edp2_smoke(self): + """Colocated MIMO step with MoE LLM using ETP/EP/EDP registered expert groups.""" + if self.world_size != 8: + pytest.skip(f"Requires 8 GPUs, got {self.world_size}") + + _set_deterministic_env() + + encoder_name = "images" + hidden_size, seq_length, vocab_size = 128, 32, 512 + micro_batch_size = 1 + num_microbatches = 1 + + encoder_grid = create_hypercomm_grid(offset=0, tp=2, cp=1, pp=1, dp=4) + llm_grid = create_hypercomm_grid( + offset=0, tp=1, cp=1, pp=1, dp=8, ep=2, expt_tp=2, expt_dp=2 + ) + create_all_embedding_groups([encoder_grid, llm_grid]) + + ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=True, bucket_size=10000, use_distributed_optimizer=True + ) + + torch.manual_seed(12345) + mimo_model, _, _, language_pg, vision_pg = get_mimo_model( + encoder_name=encoder_name, + encoder_grid=encoder_grid, + llm_grid=llm_grid, + hidden_size=hidden_size, + num_layers=1, + vocab_size=vocab_size, + seq_len=seq_length, + ddp_config=ddp_config, + bf16=False, + bias=False, + dropout=False, + per_token_loss=True, + language_num_moe_experts=4, + language_moe_router_topk=1, + ) + mimo_model.model_type = ModelType.encoder_or_decoder + self._mimo_models.append(mimo_model) + _wire_training_hooks(mimo_model, language_pg, vision_pg) + + opt_config = OptimizerConfig( + optimizer='adam', + lr=1e-4, + weight_decay=0.01, + clip_grad=1.0, + bf16=False, + use_distributed_optimizer=True, + ) + optimizer = get_mimo_optimizer(mimo_model, opt_config) + + global_batches = _generate_and_broadcast_global_batches( + global_mbs=micro_batch_size * llm_grid.get_pg("dp").size(), + seq_length=seq_length, + hidden_size=hidden_size, + vocab_size=vocab_size, + encoder_name=encoder_name, + num_batches=num_microbatches, + mask_pattern="uniform", + ) + batches = [_slice_global_batch_for_dist(b, encoder_grid, llm_grid) for b in global_batches] + + optimizer.zero_grad() + losses = _run_forward_backward( + mimo_model=mimo_model, + batches=batches, + enc_grid=encoder_grid, + llm_grid=llm_grid, + encoder_name=encoder_name, + language_pg=language_pg, + micro_batch_size=micro_batch_size, + seq_length=seq_length, + num_microbatches=num_microbatches, + ) + success, grad_norm, _ = optimizer.step() + assert success, "MoE colocated optimizer step failed" + assert grad_norm is not None and grad_norm > 0 + assert losses diff --git a/tests/unit_tests/test_hyper_comm_grid.py b/tests/unit_tests/test_hyper_comm_grid.py index dd27f84f60d..0e89cf896c3 100644 --- a/tests/unit_tests/test_hyper_comm_grid.py +++ b/tests/unit_tests/test_hyper_comm_grid.py @@ -309,6 +309,134 @@ def test_rank_enumeration_correctness(self): expected_ab = [[0, 2, 1, 3], [4, 6, 5, 7]] assert rank_enum_ab == expected_ab + def test_register_layout_for_expert_groups(self, monkeypatch): + """Test alternate expert layout over the same rank span.""" + monkeypatch.setenv("WORLD_SIZE", "16") + grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) + + grid.register_layout( + "expert", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, + ) + + assert grid.has_layout("base") + assert grid.has_layout("expert") + assert grid.get_rank_enum("pp") == grid.get_rank_enum("pp", layout_name="expert") + assert grid.get_rank_enum("ep") == [ + [0, 1, 2, 3], + [4, 5, 6, 7], + [8, 9, 10, 11], + [12, 13, 14, 15], + ] + assert grid.get_rank_enum("expt_dp") == [ + [0, 4], + [1, 5], + [2, 6], + [3, 7], + [8, 12], + [9, 13], + [10, 14], + [11, 15], + ] + assert grid.get_rank_enum("tp_ep") == [ + [0, 1, 2, 3], + [4, 5, 6, 7], + [8, 9, 10, 11], + [12, 13, 14, 15], + ] + assert grid.get_rank_enum("tp_ep_pp") == [ + [0, 1, 2, 3, 8, 9, 10, 11], + [4, 5, 6, 7, 12, 13, 14, 15], + ] + + @patch('torch.distributed.get_rank', return_value=0) + @patch('torch.distributed.new_subgroups_by_enumeration') + def test_create_pg_from_registered_layout_and_alias( + self, mock_new_subgroups, _mock_get_rank, monkeypatch + ): + """Test create/get with registered expert dims and aliases.""" + monkeypatch.setenv("WORLD_SIZE", "16") + mock_ep_pg = MagicMock(spec=dist.ProcessGroup) + mock_tp_ep_pg = MagicMock(spec=dist.ProcessGroup) + mock_new_subgroups.side_effect = [(mock_ep_pg, None), (mock_tp_ep_pg, None)] + + grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) + grid.register_layout( + "expert", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"]}, + ) + + assert grid.create_pg("ep") == mock_ep_pg + assert grid.create_pg("tp_ep") == mock_tp_ep_pg + assert grid.get_pg("ep") == mock_ep_pg + assert grid.get_pg("tp_ep") == mock_tp_ep_pg + + assert "ep" in grid._pgs + assert "tp_ep" in grid._pgs + first_call_enum = mock_new_subgroups.call_args_list[0].args[0] + second_call_enum = mock_new_subgroups.call_args_list[1].args[0] + assert first_call_enum == grid.get_rank_enum("ep") + assert second_call_enum == grid.get_rank_enum("tp_ep") + + def test_registered_layout_rejects_invalid_shapes_and_collisions(self, monkeypatch): + """Test validation for registered layouts.""" + monkeypatch.setenv("WORLD_SIZE", "16") + grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) + + with pytest.raises(ValueError, match="base.*reserved"): + grid.register_layout("base", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) + + with pytest.raises(ValueError, match="base grid size"): + grid.register_layout("bad_size", [1, 2, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) + + with pytest.raises(ValueError, match="collides.*different rank enumeration"): + grid.register_layout("bad_tp", [1, 4, 2, 2], ["expt_tp", "tp", "expt_dp", "pp"]) + + with pytest.raises(ValueError, match="conflicts with an existing dimension"): + grid.register_layout( + "bad_alias", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp": ["expt_tp", "ep"]}, + ) + + with pytest.raises(ValueError, match="cannot contain '-'"): + grid.register_layout( + "bad_alias_key", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"dp-cp": ["expt_tp", "ep"]}, + ) + + with pytest.raises(ValueError, match="duplicate dimensions"): + grid.register_layout( + "bad_alias_dims", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"bad_dims": ["ep", "ep"]}, + ) + + def test_registered_layout_rejects_implicit_cross_layout_composites(self, monkeypatch): + """Composite expert groups must use registered aliases.""" + monkeypatch.setenv("WORLD_SIZE", "16") + grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) + grid.register_layout( + "expert", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"]}, + ) + + with pytest.raises(ValueError, match="must use an explicit alias"): + grid.get_rank_enum(["expt_tp", "ep"]) + + with pytest.raises(ValueError, match="single registered layout"): + grid.get_rank_enum(["tp", "ep"]) + class TestHyperCommGridIntegration: """Integration tests for HyperCommGrid with real distributed initialization.""" diff --git a/tests/unit_tests/test_process_groups_config.py b/tests/unit_tests/test_process_groups_config.py index b49962b1a5a..546eecc77bd 100644 --- a/tests/unit_tests/test_process_groups_config.py +++ b/tests/unit_tests/test_process_groups_config.py @@ -3,6 +3,7 @@ import pytest import torch.distributed as dist +from megatron.core.hyper_comm_grid import HyperCommGrid from megatron.core.process_groups_config import ProcessGroupCollection from tests.unit_tests.test_utilities import Utils @@ -104,6 +105,112 @@ def test_repr_with_list_process_groups(self, mocker): assert "ProcessGroupCollection(" in repr_str assert "hcp([2, 4])" in repr_str + def test_from_hyper_comm_grid_reads_required_groups(self, mocker): + """Test mapping from an extended HyperCommGrid to ProcessGroupCollection.""" + grid = mocker.Mock() + grid.dim_names = ["tp", "cp", "dp", "pp"] + + pgs = {} + + def pg_for(dims): + key = tuple(dims) if isinstance(dims, list) else dims + pgs.setdefault(key, mocker.Mock(spec=dist.ProcessGroup)) + return pgs[key] + + grid.get_pg.side_effect = pg_for + grid.get_alias_dims.return_value = ["expt_tp", "ep", "pp"] + + collection = ProcessGroupCollection.from_hyper_comm_grid( + grid, + required_pgs=['tp', 'pp', 'dp', 'dp_cp', 'mp', 'expt_dp', 'tp_ep_pp', 'intra_dist_opt'], + ) + + assert collection.tp is pgs['tp'] + assert collection.pp is pgs['pp'] + assert collection.dp is pgs['dp'] + assert collection.dp_cp is pgs[('dp', 'cp')] + assert collection.mp is pgs[('tp', 'pp')] + assert collection.expt_dp is pgs['expt_dp'] + assert collection.tp_ep_pp is pgs['tp_ep_pp'] + assert collection.intra_dist_opt is pgs[('tp', 'cp', 'dp', 'pp')] + assert collection.intra_dp_cp is collection.dp_cp + assert collection.intra_expt_dp is collection.expt_dp + assert collection.inter_dist_opt is None + + def test_from_hyper_comm_grid_rejects_multi_instance_distopt(self, mocker): + """Phase 1 helper does not support multiple distributed optimizer instances.""" + grid = mocker.Mock() + with pytest.raises(ValueError, match="num_distributed_optimizer_instances == 1"): + ProcessGroupCollection.from_hyper_comm_grid(grid, num_distributed_optimizer_instances=2) + + def test_from_hyper_comm_grid_creates_from_real_extended_grid(self, mocker, monkeypatch): + """Test helper against real HyperCommGrid alias resolution without distributed init.""" + monkeypatch.setenv("WORLD_SIZE", "16") + mocker.patch('torch.distributed.get_rank', return_value=0) + mock_new_subgroups = mocker.patch('torch.distributed.new_subgroups_by_enumeration') + + created = [] + + def make_pg(rank_enum, **_kwargs): + pg = mocker.Mock(spec=dist.ProcessGroup) + pg.size.return_value = len(rank_enum[0]) + created.append((rank_enum, pg)) + return pg, None + + mock_new_subgroups.side_effect = make_pg + + grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) + grid.register_layout( + "expert", + [1, 4, 2, 2], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, + ) + + collection = ProcessGroupCollection.from_hyper_comm_grid( + grid, + create=True, + required_pgs=[ + 'tp', + 'dp', + 'dp_cp', + 'mp', + 'ep', + 'expt_tp', + 'expt_dp', + 'tp_ep', + 'tp_ep_pp', + 'intra_dist_opt', + ], + ) + + assert collection.tp is grid.get_pg("tp") + assert collection.dp is grid.get_pg("dp") + assert collection.dp_cp is grid.get_pg(["dp", "cp"]) + assert collection.mp is grid.get_pg(["tp", "pp"]) + assert collection.ep is grid.get_pg("ep") + assert collection.expt_tp is grid.get_pg("expt_tp") + assert collection.expt_dp is grid.get_pg("expt_dp") + assert collection.tp_ep is grid.get_pg("tp_ep") + assert collection.tp_ep_pp is grid.get_pg("tp_ep_pp") + assert collection.intra_dist_opt is grid.get_pg(["tp", "cp", "dp", "pp"]) + assert collection.intra_dp_cp is collection.dp_cp + assert collection.intra_expt_dp is collection.expt_dp + assert collection.inter_dist_opt is None + + def test_from_hyper_comm_grid_rejects_tp_ep_pp_without_shared_pp(self, monkeypatch): + """tp_ep_pp must include the same pp dimension used by the base layout.""" + monkeypatch.setenv("WORLD_SIZE", "4") + grid = HyperCommGrid([2, 2], ["tp", "pp"]) + grid.register_layout( + "expert", [2, 2], ["ep", "expert_pp"], aliases={"tp_ep_pp": ["ep", "expert_pp"]} + ) + + with pytest.raises(ValueError, match="shared pipeline dimension 'pp'"): + ProcessGroupCollection.from_hyper_comm_grid( + grid, create=True, required_pgs=['tp_ep_pp'] + ) + class TestPGConfigDefaultInitialization: From 34ea2603f7eeab0220fad57d360e756c84f4f706 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sat, 9 May 2026 06:42:36 +0000 Subject: [PATCH 02/44] NMFW-464 phase 2 hetero mock training loop --- .../mimo/scripts/run_hetero_mock_train.sh | 18 + examples/mimo/train_hetero.py | 990 ++++++++++++++++++ megatron/core/hyper_comm_grid.py | 9 +- .../pipeline_parallel/bridge_communicator.py | 9 +- 4 files changed, 1024 insertions(+), 2 deletions(-) create mode 100755 examples/mimo/scripts/run_hetero_mock_train.sh create mode 100644 examples/mimo/train_hetero.py diff --git a/examples/mimo/scripts/run_hetero_mock_train.sh b/examples/mimo/scripts/run_hetero_mock_train.sh new file mode 100755 index 00000000000..62a9a5d1e1f --- /dev/null +++ b/examples/mimo/scripts/run_hetero_mock_train.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Run from the repository root: +# ./examples/mimo/scripts/run_hetero_mock_train.sh + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +TRAIN_ITERS=${TRAIN_ITERS:-2} + +uv run python -m torch.distributed.run \ + --standalone \ + --nproc-per-node "${GPUS_PER_NODE}" \ + examples/mimo/train_hetero.py \ + --train-iters "${TRAIN_ITERS}" \ + "$@" diff --git a/examples/mimo/train_hetero.py b/examples/mimo/train_hetero.py new file mode 100644 index 00000000000..583a44bca50 --- /dev/null +++ b/examples/mimo/train_hetero.py @@ -0,0 +1,990 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Standalone heterogeneous MIMO mock training loop. + +This entrypoint is intentionally separate from examples/mimo/train.py. The +standard Megatron pretrain path owns a single homogeneous model-parallel +topology, while this loop owns one HyperCommGrid per MIMO module and wires the +multi-module pipeline schedule directly. +""" + +import argparse +import os +import sys +from contextlib import ExitStack, contextmanager +from functools import partial +from typing import Optional + +import torch +import torch.distributed as dist + +import megatron.core.pipeline_parallel.schedules as schedule +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.distributed.finalize_model_grads import finalize_model_grads +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.models.mimo.config.base_configs import MimoModelConfig +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.models.mimo.optimizer import get_mimo_optimizer +from megatron.core.models.mimo.submodules.vision import VisionModalitySubmodules +from megatron.core.models.vision.multimodal_projector import MultimodalProjector +from megatron.core.optimizer.optimizer_config import OptimizerConfig +from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator +from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage +from megatron.core.process_groups_config import ( + MultiModuleProcessGroupCollection, + ProcessGroupCollection, +) +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + +try: + from megatron.core.extensions.transformer_engine import ( + TEColumnParallelLinear, + TERowParallelLinear, + ) +except ImportError: + TEColumnParallelLinear = None + TERowParallelLinear = None + + +_ACTIVE_GRIDS: list[HyperCommGrid] = [] +_EMBEDDING_PG_CACHE: dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] = {} + + +def parse_args() -> argparse.Namespace: + """Parse standalone hetero MIMO loop arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + + grid = parser.add_argument_group("module grids") + grid.add_argument("--encoder-offset", type=int, default=0) + grid.add_argument("--encoder-tp", type=int, default=2) + grid.add_argument("--encoder-cp", type=int, default=1) + grid.add_argument("--encoder-pp", type=int, default=2) + grid.add_argument("--encoder-dp", type=int, default=1) + grid.add_argument("--encoder-ep", type=int, default=1) + grid.add_argument("--encoder-expt-tp", type=int, default=None) + grid.add_argument("--encoder-expt-dp", type=int, default=None) + grid.add_argument("--llm-offset", type=int, default=4) + grid.add_argument("--llm-tp", type=int, default=1) + grid.add_argument("--llm-cp", type=int, default=1) + grid.add_argument("--llm-pp", type=int, default=2) + grid.add_argument("--llm-dp", type=int, default=2) + grid.add_argument("--llm-ep", type=int, default=2) + grid.add_argument("--llm-expt-tp", type=int, default=1) + grid.add_argument("--llm-expt-dp", type=int, default=1) + + model = parser.add_argument_group("model") + model.add_argument("--hidden-size", type=int, default=128) + model.add_argument("--num-layers", type=int, default=2) + model.add_argument("--num-attention-heads", type=int, default=8) + model.add_argument("--vocab-size", type=int, default=512) + model.add_argument("--seq-length", type=int, default=32) + model.add_argument("--image-seq-length", type=int, default=None) + model.add_argument("--image-token-id", type=int, default=50257) + model.add_argument("--num-moe-experts", type=int, default=4) + model.add_argument("--moe-router-topk", type=int, default=1) + model.add_argument("--moe-grouped-gemm", action="store_true") + model.add_argument( + "--fp32", action="store_true", help="Build and train in fp32 instead of bf16" + ) + + train = parser.add_argument_group("training") + train.add_argument("--micro-batch-size", type=int, default=2) + train.add_argument("--num-microbatches", type=int, default=2) + train.add_argument("--train-iters", type=int, default=2) + train.add_argument("--lr", type=float, default=1.0e-4) + train.add_argument("--weight-decay", type=float, default=0.01) + train.add_argument("--clip-grad", type=float, default=1.0) + train.add_argument("--seed", type=int, default=12345) + train.add_argument("--log-interval", type=int, default=1) + + return parser.parse_args() + + +def clear_transformer_engine_env() -> None: + """Clear attention backend overrides that can conflict with GPTModel construction.""" + os.environ.pop("NVTE_FLASH_ATTN", None) + os.environ.pop("NVTE_FUSED_ATTN", None) + os.environ.pop("NVTE_UNFUSED_ATTN", None) + + +def initialize_distributed() -> None: + """Initialize torch.distributed for torchrun.""" + clear_transformer_engine_env() + os.environ.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + dist.barrier() + + +def print_rank_0(message: str) -> None: + """Print only on global rank zero.""" + if not dist.is_initialized() or dist.get_rank() == 0: + sys.stdout.write(f"{message}\n") + sys.stdout.flush() + + +def debug_rank(message: str) -> None: + """Emit per-rank startup checkpoints when MIMO_HETERO_DEBUG is set.""" + if os.environ.get("MIMO_HETERO_DEBUG"): + rank = dist.get_rank() if dist.is_initialized() else 0 + sys.stderr.write(f"[rank {rank}] {message}\n") + sys.stderr.flush() + + +def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: + """Validate the Phase 2 non-colocated 1F1B mock-training layout.""" + if args.encoder_cp != 1 or args.llm_cp != 1: + raise ValueError("Phase 2 mock training currently supports CP=1 only") + if args.hidden_size % args.num_attention_heads != 0: + raise ValueError("--hidden-size must be divisible by --num-attention-heads") + if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: + raise ValueError("--num-moe-experts must be divisible by --llm-ep") + if args.log_interval < 1: + raise ValueError("--log-interval must be >= 1") + + image_seq_length = args.image_seq_length or args.seq_length // 2 + if image_seq_length >= args.seq_length: + raise ValueError("--image-seq-length must be smaller than --seq-length") + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") + + encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp + llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp + encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + + if not encoder_ranks.isdisjoint(llm_ranks): + raise ValueError( + "Phase 2 train_hetero.py supports non-colocated 1F1B only; " + f"module rank spans overlap at {sorted(encoder_ranks & llm_ranks)}" + ) + if encoder_ranks | llm_ranks != all_ranks: + raise ValueError( + "The non-colocated module grids must cover every torchrun rank exactly once; " + f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" + ) + + return encoder_size, llm_size + + +def is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Return whether pg is a real process group for this rank.""" + group_member = getattr(dist, "GroupMember", None) + non_member = getattr(group_member, "NON_GROUP_MEMBER", None) + return pg is not None and pg != non_member + + +def destroy_process_group_if_member(pg: Optional[dist.ProcessGroup]) -> None: + """Destroy pg when this rank owns a process-group handle.""" + if is_process_group_member(pg): + dist.destroy_process_group(pg) + + +def create_hypercomm_grid( + offset: int, + tp: int, + cp: int, + pp: int, + dp: int, + ep: int, + expt_tp: Optional[int], + expt_dp: Optional[int], +) -> HyperCommGrid: + """Create a dense grid plus expert layout and required process groups.""" + expt_tp = tp if expt_tp is None else expt_tp + module_world_size = tp * cp * pp * dp + expert_model_size = expt_tp * ep * pp + if module_world_size % expert_model_size != 0: + raise ValueError( + f"module_world_size ({module_world_size}) must be divisible by " + f"expt_tp*ep*pp ({expert_model_size})" + ) + if expt_dp is None: + expt_dp = module_world_size // expert_model_size + if expt_tp * ep * expt_dp * pp != module_world_size: + raise ValueError( + f"expt_tp*ep*expt_dp*pp ({expt_tp * ep * expt_dp * pp}) must equal " + f"module_world_size ({module_world_size})" + ) + + grid = HyperCommGrid( + shape=[tp, cp, dp, pp], + dim_names=["tp", "cp", "dp", "pp"], + rank_offset=offset, + backend="nccl", + ) + grid.register_layout( + "expert", + [expt_tp, ep, expt_dp, pp], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, + ) + + for dims in ( + ["tp"], + ["cp"], + ["pp"], + ["dp"], + ["dp", "cp"], + ["tp", "cp"], + ["ep"], + ["expt_tp"], + ["expt_dp"], + ["tp", "pp"], + ["tp", "cp", "dp"], + ["tp", "cp", "pp", "dp"], + "tp_ep", + "tp_ep_pp", + ): + grid.create_pg(dims) + + _ACTIVE_GRIDS.append(grid) + return grid + + +def destroy_runtime_process_groups() -> None: + """Destroy process groups created by this script.""" + destroyed_embedding_pgs = set() + for pos_embd_pg, embd_pg in _EMBEDDING_PG_CACHE.values(): + for pg in (pos_embd_pg, embd_pg): + if id(pg) in destroyed_embedding_pgs: + continue + destroy_process_group_if_member(pg) + destroyed_embedding_pgs.add(id(pg)) + _EMBEDDING_PG_CACHE.clear() + + for grid in _ACTIVE_GRIDS: + grid.destroy() + _ACTIVE_GRIDS.clear() + BridgeCommunicator.destroy_broadcast_pgs() + + +def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: + """Build a ProcessGroupCollection from a populated HyperCommGrid.""" + return ProcessGroupCollection.from_hyper_comm_grid( + grid, + required_pgs=[ + "tp", + "cp", + "pp", + "dp", + "dp_cp", + "tp_cp", + "mp", + "tp_dp_cp", + "ep", + "expt_tp", + "expt_dp", + "tp_ep", + "tp_ep_pp", + "intra_dist_opt", + ], + ) + + +def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: + """Create PP-derived embedding groups in a consistent global order.""" + pp_rank_sets: list[tuple[int, ...]] = [] + seen_pp_rank_sets = set() + for grid in sorted(grids, key=lambda candidate: (candidate.rank_offset, candidate.size)): + for pp_ranks in grid.get_rank_enum("pp"): + pp_rank_tuple = tuple(pp_ranks) + if pp_rank_tuple in seen_pp_rank_sets: + continue + pp_rank_sets.append(pp_rank_tuple) + seen_pp_rank_sets.add(pp_rank_tuple) + + for pp_ranks in pp_rank_sets: + if pp_ranks not in _EMBEDDING_PG_CACHE: + pos_embd_ranks = [pp_ranks[0]] + embd_ranks = [pp_ranks[0]] + if pp_ranks[-1] != pp_ranks[0]: + embd_ranks.append(pp_ranks[-1]) + _EMBEDDING_PG_CACHE[pp_ranks] = ( + dist.new_group(ranks=pos_embd_ranks), + dist.new_group(ranks=embd_ranks), + ) + + +def add_embedding_groups( + pg_collection: ProcessGroupCollection, is_language_model: bool = False +) -> ProcessGroupCollection: + """Attach cached embedding process groups to a ProcessGroupCollection.""" + if not is_process_group_member(getattr(pg_collection, "pp", None)): + return pg_collection + + pp_ranks = tuple(dist.get_process_group_ranks(pg_collection.pp)) + pos_embd_pg, embd_pg = _EMBEDDING_PG_CACHE[pp_ranks] + + pg_collection.pos_embd = pos_embd_pg if is_pp_first_stage(pg_collection.pp) else None + if is_language_model: + pg_collection.embd = ( + embd_pg + if is_pp_last_stage(pg_collection.pp) or is_pp_first_stage(pg_collection.pp) + else None + ) + else: + pg_collection.embd = None + + return pg_collection + + +def get_pg_collection_with_embedding_groups( + grid: HyperCommGrid, is_language_model: bool = False +) -> ProcessGroupCollection: + """Build a ProcessGroupCollection and add PP-derived embedding groups.""" + return add_embedding_groups(get_pg_collection(grid), is_language_model=is_language_model) + + +def is_rank_in_grid(grid: HyperCommGrid) -> bool: + """Return whether this global rank is inside a grid's rank span.""" + rank = dist.get_rank() + return grid.rank_offset <= rank < grid.rank_offset + grid.size + + +def get_grid_dim_size(grid: HyperCommGrid, dim: str) -> int: + """Return a base-layout dimension size.""" + return grid.shape[grid.dim_names.index(dim)] + + +def get_group_size_or(pg: Optional[dist.ProcessGroup], fallback: int) -> int: + """Return pg size on member ranks, otherwise fallback.""" + return pg.size() if is_process_group_member(pg) else fallback + + +def get_group_rank_or(pg: Optional[dist.ProcessGroup], fallback: int = 0) -> int: + """Return rank inside pg on member ranks, otherwise fallback.""" + return dist.get_rank(pg) if is_process_group_member(pg) else fallback + + +def get_grid_coordinate(grid: HyperCommGrid, dim: str) -> int: + """Return this rank's coordinate for a base-layout dimension.""" + if not is_rank_in_grid(grid): + return 0 + + local_rank = dist.get_rank() - grid.rank_offset + coordinates = {} + for dim_name, dim_size in zip(grid.dim_names, grid.shape): + coordinates[dim_name] = local_rank % dim_size + local_rank //= dim_size + return coordinates[dim] + + +def get_mock_data_seed( + args: argparse.Namespace, grid: HyperCommGrid, module_seed_offset: int +) -> int: + """Seed mock data by data-parallel lane so PP/TP stages see coherent batches.""" + dp_lane = get_grid_coordinate(grid, "dp") if "dp" in grid.dim_names else 0 + return args.seed + module_seed_offset + dp_lane + + +def build_no_sync_func(mimo_model: MimoModel): + """Build a no_sync context spanning all active MIMO submodules.""" + + @contextmanager + def no_sync_func(): + with ExitStack() as stack: + if mimo_model.language_model is not None: + stack.enter_context(mimo_model.language_model.no_sync()) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + stack.enter_context(submodule.no_sync()) + yield + + return no_sync_func + + +def projection_layer_spec() -> ModuleSpec: + """Return the TE-backed projection MLP spec.""" + if TEColumnParallelLinear is None or TERowParallelLinear is None: + raise RuntimeError("TEColumnParallelLinear and TERowParallelLinear are required") + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules(linear_fc1=TEColumnParallelLinear, linear_fc2=TERowParallelLinear), + ) + + +def language_model_spec( + args: argparse.Namespace, + pg_collection: Optional[ProcessGroupCollection], + llm_grid: HyperCommGrid, +) -> ModuleSpec: + """Create the GPT language ModuleSpec for the local language grid.""" + pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None + tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None + ep_pg = getattr(pg_collection, "ep", None) if pg_collection is not None else None + expt_tp_pg = getattr(pg_collection, "expt_tp", None) if pg_collection is not None else None + + fallback_tp_size = get_grid_dim_size(llm_grid, "tp") + pp_rank = get_group_rank_or(pp_pg) + pp_size = get_group_size_or(pp_pg, get_grid_dim_size(llm_grid, "pp")) + tp_size = get_group_size_or(tp_pg, fallback_tp_size) + ep_size = get_group_size_or(ep_pg, args.llm_ep) + expt_tp_size = get_group_size_or(expt_tp_pg, args.llm_expt_tp or fallback_tp_size) + num_moe_experts = args.num_moe_experts if args.num_moe_experts > 0 else None + bf16 = not args.fp32 + + moe_kwargs = {} + if num_moe_experts is not None: + moe_kwargs = { + "num_moe_experts": num_moe_experts, + "moe_router_topk": args.moe_router_topk, + "moe_router_pre_softmax": args.moe_router_topk == 1, + "expert_model_parallel_size": ep_size, + "expert_tensor_parallel_size": expt_tp_size, + "moe_grouped_gemm": args.moe_grouped_gemm, + } + + config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + use_cpu_initialization=True, + variable_seq_lengths=True, + moe_token_dispatcher_type="alltoall", + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, + bf16=bf16, + calculate_per_token_loss=True, + cross_entropy_loss_fusion=True, + cross_entropy_fusion_impl="te", + **moe_kwargs, + ) + return ModuleSpec( + module=GPTModel, + params={ + "config": config, + "transformer_layer_spec": get_gpt_layer_with_transformer_engine_spec( + num_experts=num_moe_experts, moe_grouped_gemm=args.moe_grouped_gemm + ), + "vocab_size": args.vocab_size, + "max_sequence_length": args.seq_length, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + "pg_collection": pg_collection, + }, + ) + + +def vision_submodules_spec( + args: argparse.Namespace, + pg_collection: Optional[ProcessGroupCollection], + encoder_grid: HyperCommGrid, +) -> ModuleSpec: + """Create the mock vision ModuleSpec for the local encoder grid.""" + from megatron.core.transformer.transformer_block import TransformerBlock + + pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None + tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None + tp_size = get_group_size_or(tp_pg, get_grid_dim_size(encoder_grid, "tp")) + pp_size = get_group_size_or(pp_pg, get_grid_dim_size(encoder_grid, "pp")) + pp_rank = get_group_rank_or(pp_pg) + bf16 = not args.fp32 + + vision_config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + use_cpu_initialization=True, + variable_seq_lengths=True, + moe_token_dispatcher_type="alltoall", + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, + bf16=bf16, + calculate_per_token_loss=True, + ) + vision_encoder_spec = ModuleSpec( + module=TransformerBlock, + params={ + "config": vision_config, + "spec": get_gpt_layer_with_transformer_engine_spec(), + "pg_collection": pg_collection, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + }, + ) + + projection_config = TransformerConfig( + num_layers=1, hidden_size=args.hidden_size, num_attention_heads=1 + ) + projection_config.ffn_hidden_size = args.hidden_size + projection_config.activation_func = torch.nn.functional.gelu + + vision_projection_spec = ModuleSpec( + module=MultimodalProjector, + params={ + "config": projection_config, + "submodules": projection_layer_spec().submodules, + "projector_type": "mlp", + "input_size": vision_config.hidden_size, + "tp_group": tp_pg if is_process_group_member(tp_pg) else None, + }, + ) + + return ModuleSpec( + module=VisionModalitySubmodules, + params={"pg_collection": pg_collection}, + submodules={ + "encoders": {"clip_encoder": vision_encoder_spec}, + "input_projections": [vision_projection_spec], + }, + ) + + +def build_mimo_model( + args: argparse.Namespace, + encoder_grid: HyperCommGrid, + llm_grid: HyperCommGrid, + encoder_name: str, +): + """Build the MIMO model and wrap active modules in MCore DDP.""" + language_pg = get_pg_collection_with_embedding_groups(llm_grid, is_language_model=True) + vision_pg = get_pg_collection_with_embedding_groups(encoder_grid, is_language_model=False) + rank_in_language_grid = is_rank_in_grid(llm_grid) + rank_in_encoder_grid = is_rank_in_grid(encoder_grid) + debug_rank( + "building model specs " + f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" + ) + + module_to_grid_map = {encoder_name: encoder_grid, MIMO_LANGUAGE_MODULE_KEY: llm_grid} + mimo_config = MimoModelConfig( + language_model_spec=language_model_spec( + args, language_pg if rank_in_language_grid else None, llm_grid + ), + modality_submodules_spec={ + encoder_name: vision_submodules_spec( + args, vision_pg if rank_in_encoder_grid else None, encoder_grid + ) + }, + special_token_ids={encoder_name: args.image_token_id}, + module_to_grid_map=module_to_grid_map, + ) + + debug_rank("constructing MimoModel") + mimo_model = MimoModel(mimo_config) + debug_rank("moving MimoModel to cuda") + mimo_model.to(torch.device("cuda")) + if not args.fp32: + mimo_model.to(torch.bfloat16) + debug_rank("MimoModel moved to target dtype/device") + + ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=True, bucket_size=10000, use_distributed_optimizer=True + ) + if mimo_model.language_model is not None: + debug_rank("wrapping language model in DDP") + mimo_model.language_model = DistributedDataParallel( + config=mimo_model.language_model.config, + ddp_config=ddp_config, + module=mimo_model.language_model, + pg_collection=language_pg, + ) + debug_rank("language model DDP ready") + + if encoder_name in mimo_model.modality_submodules: + submodule = mimo_model.modality_submodules[encoder_name] + if submodule is not None: + debug_rank("wrapping vision submodule in DDP") + mimo_model.modality_submodules[encoder_name] = DistributedDataParallel( + config=submodule.encoders["clip_encoder"].config, + ddp_config=ddp_config, + module=submodule, + pg_collection=vision_pg, + ) + debug_rank("vision submodule DDP ready") + + return mimo_model, module_to_grid_map, language_pg, vision_pg + + +class MockVLMIterator: + """Infinite iterator yielding synthetic VLM-like microbatches.""" + + def __init__( + self, args: argparse.Namespace, micro_batch_size: int, encoder_name: str, seed: int + ) -> None: + self.args = args + self.micro_batch_size = micro_batch_size + self.encoder_name = encoder_name + self.image_seq_length = args.image_seq_length or args.seq_length // 2 + self.dtype = torch.float32 if args.fp32 else torch.bfloat16 + self.generator = torch.Generator(device="cuda") + self.generator.manual_seed(seed) + if self.image_seq_length >= args.seq_length: + raise ValueError("--image-seq-length must be smaller than --seq-length") + + def __iter__(self): + return self + + def __next__(self): + args = self.args + image_tokens = torch.full( + (self.micro_batch_size, self.image_seq_length), + args.image_token_id, + dtype=torch.long, + device="cuda", + ) + text_tokens = torch.randint( + 1, + args.vocab_size, + (self.micro_batch_size, args.seq_length - self.image_seq_length), + device="cuda", + generator=self.generator, + ) + input_ids = torch.cat([image_tokens, text_tokens], dim=1) + + labels = input_ids.clone() + labels[input_ids == args.image_token_id] = -100 + + loss_mask = torch.ones( + self.micro_batch_size, args.seq_length, dtype=torch.float32, device="cuda" + ) + loss_mask[input_ids == args.image_token_id] = 0.0 + + encoder_hidden_states = torch.randn( + self.image_seq_length, + self.micro_batch_size, + args.hidden_size, + device="cuda", + dtype=self.dtype, + generator=self.generator, + ) + num_image_placeholders = (input_ids == args.image_token_id).sum().item() + expected_image_placeholders = self.image_seq_length * self.micro_batch_size + if num_image_placeholders != expected_image_placeholders: + raise RuntimeError( + f"mock batch has {num_image_placeholders} image placeholders, " + f"expected {expected_image_placeholders}" + ) + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "position_ids": torch.arange(args.seq_length, device="cuda") + .unsqueeze(0) + .expand(self.micro_batch_size, -1) + .clone(), + "modality_inputs": { + self.encoder_name: { + "clip_encoder": {"hidden_states": encoder_hidden_states, "attention_mask": None} + } + }, + } + + +def wire_training_hooks( + mimo_model: MimoModel, language_pg: ProcessGroupCollection, vision_pg: ProcessGroupCollection +) -> None: + """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" + + def is_token_source_rank() -> bool: + return ( + is_process_group_member(getattr(language_pg, "pp", None)) + and is_process_group_member(getattr(language_pg, "tp", None)) + and is_pp_last_stage(language_pg.pp) + and language_pg.tp.rank() == 0 + ) + + def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): + if num_tokens is None: + raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") + + token_count = num_tokens.to(device="cuda", dtype=torch.float32) + if not is_token_source_rank(): + token_count.zero_() + dist.all_reduce(token_count, op=dist.ReduceOp.SUM) + global_num_tokens = token_count.item() + + if mimo_model.language_model is not None: + finalize_model_grads( + [mimo_model.language_model], + num_tokens=None, + pg_collection=language_pg, + force_all_reduce=force_all_reduce, + ) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + finalize_model_grads( + [submodule], + num_tokens=None, + pg_collection=vision_pg, + force_all_reduce=force_all_reduce, + ) + + if global_num_tokens > 0: + scale = 1.0 / global_num_tokens + if mimo_model.language_model is not None: + mimo_model.language_model.scale_gradients(scale) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + submodule.scale_gradients(scale) + + mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) + mimo_model.config.finalize_model_grads_func = finalize_grads_func + mimo_model.config.grad_scale_func = lambda loss: ( + torch.tensor(loss, dtype=torch.float32, device="cuda", requires_grad=True) + if isinstance(loss, (int, float)) + else loss + ) + + +def select_data_iterator( + args: argparse.Namespace, + encoder_grid: HyperCommGrid, + llm_grid: HyperCommGrid, + encoder_name: str, +) -> Optional[MockVLMIterator]: + """Create the per-role data iterator needed by local ranks.""" + llm_mbs = args.micro_batch_size + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") + encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp + + encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( + encoder_grid.get_pg("pp") + ) + llm_needs_data = is_rank_in_grid(llm_grid) and ( + is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) + ) + + if encoder_needs_data and not llm_needs_data: + return MockVLMIterator( + args, + encoder_mbs, + encoder_name, + get_mock_data_seed(args, encoder_grid, module_seed_offset=0), + ) + if llm_needs_data and not encoder_needs_data: + return MockVLMIterator( + args, + llm_mbs, + encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + if encoder_needs_data and llm_needs_data: + return MockVLMIterator( + args, + llm_mbs, + encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + return None + + +def build_schedule_pg_collection( + encoder_name: str, + encoder_grid: HyperCommGrid, + llm_grid: HyperCommGrid, + vision_pg: ProcessGroupCollection, + language_pg: ProcessGroupCollection, +) -> MultiModuleProcessGroupCollection: + """Build the schedule-facing process group collection for this rank.""" + module_pgs = {} + language_model_module_name = None + if is_rank_in_grid(encoder_grid): + module_pgs[encoder_name] = vision_pg + if is_rank_in_grid(llm_grid): + module_pgs[MIMO_LANGUAGE_MODULE_KEY] = language_pg + language_model_module_name = MIMO_LANGUAGE_MODULE_KEY + + return MultiModuleProcessGroupCollection( + module_pgs=module_pgs, language_model_module_name=language_model_module_name + ) + + +def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): + """Return raw loss sum, local token count, and logging tensors.""" + if output_tensor is None: + zero = torch.tensor(0.0, device="cuda", requires_grad=True) + zero_count = torch.tensor(0, device="cuda", dtype=torch.int) + return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + + if isinstance(output_tensor, dict): + output = output_tensor.get( + MIMO_LANGUAGE_MODULE_KEY, next(iter(output_tensor.values()), None) + ) + else: + output = output_tensor + + if output is None: + zero = torch.tensor(0.0, device="cuda", requires_grad=True) + zero_count = torch.tensor(0, device="cuda", dtype=torch.int) + return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + + output = output.float() + if loss_mask is not None and output.shape == loss_mask.shape: + masked = output * loss_mask.float() + num_tokens = loss_mask.float().sum().to(torch.int) + loss_sum = masked.sum() + else: + loss_sum = output.sum() + num_tokens = torch.tensor(output.numel(), device="cuda", dtype=torch.int) + return ( + loss_sum, + num_tokens, + {"lm loss sum": loss_sum.detach(), "lm tokens": num_tokens.detach()}, + ) + + +def forward_step(data_iterator, model): + """Forward step consumed by the MCore pipeline schedule.""" + batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} + output_tensor, loss_mask = model(**batch) + return output_tensor, partial(loss_func, loss_mask) + + +def run_train_loop(args: argparse.Namespace) -> None: + """Run mock-data heterogeneous MIMO training.""" + world_size = dist.get_world_size() + encoder_size, llm_size = validate_args(args, world_size) + + encoder_name = "images" + debug_rank("creating encoder grid") + encoder_grid = create_hypercomm_grid( + offset=args.encoder_offset, + tp=args.encoder_tp, + cp=args.encoder_cp, + pp=args.encoder_pp, + dp=args.encoder_dp, + ep=args.encoder_ep, + expt_tp=args.encoder_expt_tp, + expt_dp=args.encoder_expt_dp, + ) + debug_rank("creating language grid") + llm_grid = create_hypercomm_grid( + offset=args.llm_offset, + tp=args.llm_tp, + cp=args.llm_cp, + pp=args.llm_pp, + dp=args.llm_dp, + ep=args.llm_ep, + expt_tp=args.llm_expt_tp, + expt_dp=args.llm_expt_dp, + ) + debug_rank("creating embedding groups") + create_all_embedding_groups([encoder_grid, llm_grid]) + debug_rank("embedding groups ready") + + torch.manual_seed(args.seed + dist.get_rank()) + debug_rank("building MIMO model") + mimo_model, module_to_grid_map, language_pg, vision_pg = build_mimo_model( + args, encoder_grid, llm_grid, encoder_name + ) + debug_rank("wiring training hooks") + wire_training_hooks(mimo_model, language_pg, vision_pg) + + debug_rank("building MIMO optimizer") + optimizer = get_mimo_optimizer( + mimo_model, + OptimizerConfig( + optimizer="adam", + lr=args.lr, + weight_decay=args.weight_decay, + clip_grad=args.clip_grad, + bf16=not args.fp32, + use_distributed_optimizer=True, + ), + ) + debug_rank("MIMO optimizer ready") + debug_rank("building pipeline communicator") + communicator = MultiModulePipelineCommunicator( + module_to_grid_map, + {encoder_name: [MIMO_LANGUAGE_MODULE_KEY], MIMO_LANGUAGE_MODULE_KEY: []}, + mimo_model.config, + dim_mapping={"s": 0, "h": 2, "b": 1}, + module_output_ndim={encoder_name: 2}, + ) + debug_rank("building schedule process groups") + schedule_pg_collection = build_schedule_pg_collection( + encoder_name, encoder_grid, llm_grid, vision_pg, language_pg + ) + debug_rank("selecting data iterator") + data_iterator = select_data_iterator(args, encoder_grid, llm_grid, encoder_name) + debug_rank("training setup ready") + + print_rank_0( + "Starting hetero MIMO mock training: " + f"world_size={world_size}, encoder_size={encoder_size}, llm_size={llm_size}, " + f"train_iters={args.train_iters}" + ) + + try: + for iteration in range(1, args.train_iters + 1): + optimizer.zero_grad() + losses = schedule.forward_backward_pipelining_without_interleaving( + forward_step_func=forward_step, + data_iterator=data_iterator, + model=[mimo_model], + num_microbatches=args.num_microbatches, + seq_length=args.seq_length, + micro_batch_size=args.micro_batch_size, + forward_only=False, + p2p_communicator=communicator, + pg_collection=schedule_pg_collection, + ) + success, grad_norm, _ = optimizer.step() + if not success: + raise RuntimeError(f"optimizer step failed at iteration {iteration}") + + if iteration % args.log_interval == 0: + loss_acc = torch.zeros(2, dtype=torch.float32, device="cuda") + if ( + losses + and is_process_group_member(getattr(language_pg, "pp", None)) + and is_process_group_member(getattr(language_pg, "tp", None)) + ): + is_log_source = is_pp_last_stage(language_pg.pp) and language_pg.tp.rank() == 0 + if is_log_source: + for loss_dict in losses: + loss_sum = loss_dict.get("lm loss sum") + num_tokens = loss_dict.get("lm tokens") + if isinstance(loss_sum, torch.Tensor): + loss_acc[0] += loss_sum.float() + elif loss_sum is not None: + loss_acc[0] += float(loss_sum) + if isinstance(num_tokens, torch.Tensor): + loss_acc[1] += num_tokens.float() + elif num_tokens is not None: + loss_acc[1] += float(num_tokens) + dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM) + loss_value = loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None + if dist.get_rank() == 0: + print_rank_0(f"iteration {iteration}: loss={loss_value}, grad_norm={grad_norm}") + finally: + mimo_model.destroy() + + +def main() -> None: + """Program entrypoint.""" + args = parse_args() + initialize_distributed() + try: + run_train_loop(args) + dist.barrier() + print_rank_0("Heterogeneous MIMO mock training completed") + finally: + try: + torch.cuda.synchronize() + dist.barrier() + except Exception: + pass + destroy_runtime_process_groups() + if dist.is_initialized(): + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py index 4f9a8d7251e..e27d4fce0ea 100644 --- a/megatron/core/hyper_comm_grid.py +++ b/megatron/core/hyper_comm_grid.py @@ -31,6 +31,13 @@ HAVE_ABSL = False +def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Return whether pg is a real process group for this rank.""" + group_member = getattr(dist, "GroupMember", None) + non_member = getattr(group_member, "NON_GROUP_MEMBER", None) + return pg is not None and pg != non_member + + @dataclass class _GridLayout: """Rank layout owned by a HyperCommGrid. @@ -270,7 +277,7 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG def destroy(self) -> None: """Destroy all process groups created by this grid.""" for pg in self._pgs.values(): - if pg is not None: + if _is_process_group_member(pg): dist.destroy_process_group(pg) self._pgs.clear() diff --git a/megatron/core/pipeline_parallel/bridge_communicator.py b/megatron/core/pipeline_parallel/bridge_communicator.py index 515ddf1743a..79141073779 100644 --- a/megatron/core/pipeline_parallel/bridge_communicator.py +++ b/megatron/core/pipeline_parallel/bridge_communicator.py @@ -11,6 +11,13 @@ from megatron.core.hyper_comm_grid import HyperCommGrid +def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Return whether pg is a real process group for this rank.""" + group_member = getattr(dist, "GroupMember", None) + non_member = getattr(group_member, "NON_GROUP_MEMBER", None) + return pg is not None and pg != non_member + + class CommRole(Enum): """Communication role for ranks in bridge communication. @@ -53,7 +60,7 @@ class BridgeCommunicator: def destroy_broadcast_pgs(cls): """Destroy all cached broadcast process groups.""" for pg in cls._broadcast_pg_cache.values(): - if pg is not None: + if _is_process_group_member(pg): dist.destroy_process_group(pg) cls._broadcast_pg_cache.clear() From 772202048e74ee9e232fd2b09be3eed0fb09fd5f Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sun, 10 May 2026 16:17:31 +0000 Subject: [PATCH 03/44] NMFW-464 add Nemotron 20L hetero mock workflow --- .../mimo/scripts/run_hetero_mock_train.sh | 3 +- .../run_hetero_nemotron_20l_mock_train.sh | 75 ++ examples/mimo/train_hetero.py | 704 ++++++++++++++++-- .../core/distributed/finalize_model_grads.py | 21 +- .../common/language_module/language_module.py | 4 +- .../models/hybrid/hybrid_layer_allocation.py | 8 + megatron/core/models/hybrid/hybrid_model.py | 2 + megatron/core/models/mimo/model/base.py | 61 +- megatron/core/models/mimo/optimizer.py | 30 +- megatron/core/models/mimo/partition/utils.py | 12 +- megatron/core/models/vision/radio.py | 2 + megatron/core/optimizer/__init__.py | 23 +- .../core/tensor_parallel/cross_entropy.py | 43 +- .../vision/libraries/multimodal_tokenizer.py | 22 +- megatron/core/transformer/moe/moe_layer.py | 6 +- megatron/core/transformer/moe/moe_utils.py | 14 +- .../core/transformer/moe/shared_experts.py | 10 +- .../unit_tests/models/test_mimo_partition.py | 27 +- 18 files changed, 944 insertions(+), 123 deletions(-) create mode 100755 examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh diff --git a/examples/mimo/scripts/run_hetero_mock_train.sh b/examples/mimo/scripts/run_hetero_mock_train.sh index 62a9a5d1e1f..c0926d7cea2 100755 --- a/examples/mimo/scripts/run_hetero_mock_train.sh +++ b/examples/mimo/scripts/run_hetero_mock_train.sh @@ -9,8 +9,9 @@ export CUDA_DEVICE_MAX_CONNECTIONS=1 GPUS_PER_NODE=${GPUS_PER_NODE:-8} TRAIN_ITERS=${TRAIN_ITERS:-2} +PYTHON_BIN=${PYTHON_BIN:-python} -uv run python -m torch.distributed.run \ +"${PYTHON_BIN}" -m torch.distributed.run \ --standalone \ --nproc-per-node "${GPUS_PER_NODE}" \ examples/mimo/train_hetero.py \ diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh new file mode 100755 index 00000000000..f113d988a66 --- /dev/null +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Run a heterogeneous mock-data loop with the Nemotron6-MoE VLM 20L architecture. + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" + +GPUS_PER_NODE=8 +TRAIN_ITERS=${TRAIN_ITERS:-1} +NUM_MICROBATCHES=${NUM_MICROBATCHES:-4} +NUM_IMAGE_TILES=${NUM_IMAGE_TILES:-12} +TRAINING_STAGE=${TRAINING_STAGE:-stage2} +MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} +LLM_DP=2 +GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP))} +LR_WARMUP_ITERS=${LR_WARMUP_ITERS:-0} +TOKENIZER_MODEL=${TOKENIZER_MODEL:-} +IMAGE_TOKEN_ID=${IMAGE_TOKEN_ID:-511} +PYTHON_BIN=${PYTHON_BIN:-python} + +TOKENIZER_ARGS=() +if [[ -n "${TOKENIZER_MODEL}" ]]; then + TOKENIZER_ARGS+=(--tokenizer-model "${TOKENIZER_MODEL}") +else + TOKENIZER_ARGS+=(--image-token-id "${IMAGE_TOKEN_ID}") +fi + +case "${TRAINING_STAGE}" in + stage1|stage2|stage3) + ;; + *) + echo "ERROR: Unknown TRAINING_STAGE='${TRAINING_STAGE}'. Use stage1, stage2, or stage3." >&2 + exit 1 + ;; +esac + +"${PYTHON_BIN}" -m torch.distributed.run \ + --standalone \ + --nproc-per-node "${GPUS_PER_NODE}" \ + examples/mimo/train_hetero.py \ + --model-preset nemotron-moe-vlm-20l \ + --training-stage "${TRAINING_STAGE}" \ + --encoder-tp 2 \ + --encoder-pp 1 \ + --encoder-dp 2 \ + --llm-offset 4 \ + --llm-tp 2 \ + --llm-pp 1 \ + --llm-dp "${LLM_DP}" \ + --llm-ep 4 \ + --llm-expt-tp 1 \ + --llm-expt-dp 1 \ + --vocab-size 131072 \ + --num-image-tiles "${NUM_IMAGE_TILES}" \ + "${TOKENIZER_ARGS[@]}" \ + --tokenizer-prompt-format nemotron6-moe \ + --image-token "" \ + --micro-batch-size "${MICRO_BATCH_SIZE}" \ + --global-batch-size "${GLOBAL_BATCH_SIZE}" \ + --num-microbatches "${NUM_MICROBATCHES}" \ + --lr 2e-4 \ + --min-lr 2e-6 \ + --lr-decay-style cosine \ + --lr-warmup-iters "${LR_WARMUP_ITERS}" \ + --lr-decay-iters 10 \ + --weight-decay 0.05 \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --clip-grad 1.0 \ + --no-overlap-grad-reduce \ + --ddp-bucket-size 0 \ + --train-iters "${TRAIN_ITERS}" \ + "$@" diff --git a/examples/mimo/train_hetero.py b/examples/mimo/train_hetero.py index 583a44bca50..743e16933cc 100644 --- a/examples/mimo/train_hetero.py +++ b/examples/mimo/train_hetero.py @@ -11,26 +11,38 @@ import argparse import os import sys -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from functools import partial from typing import Optional +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + import torch import torch.distributed as dist import megatron.core.pipeline_parallel.schedules as schedule +from megatron.core import parallel_state +from megatron.core.activations import fast_gelu, squared_relu from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig from megatron.core.distributed.finalize_model_grads import finalize_model_grads from megatron.core.hyper_comm_grid import HyperCommGrid from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec +from megatron.core.models.mamba.mamba_model import MambaModel from megatron.core.models.mimo.config.base_configs import MimoModelConfig from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY from megatron.core.models.mimo.model.base import MimoModel from megatron.core.models.mimo.optimizer import get_mimo_optimizer from megatron.core.models.mimo.submodules.vision import VisionModalitySubmodules +from megatron.core.models.multimodal.llava_model import pixel_shuffle from megatron.core.models.vision.multimodal_projector import MultimodalProjector +from megatron.core.models.vision.radio import RADIOViTModel +from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec from megatron.core.optimizer.optimizer_config import OptimizerConfig +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage @@ -38,20 +50,32 @@ MultiModuleProcessGroupCollection, ProcessGroupCollection, ) +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.enums import AttnBackend from megatron.core.transformer.mlp import MLP, MLPSubmodules from megatron.core.transformer.spec_utils import ModuleSpec from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.utils import sharded_state_dict_default try: from megatron.core.extensions.transformer_engine import ( TEColumnParallelLinear, + TELayerNormColumnParallelLinear, TERowParallelLinear, ) except ImportError: TEColumnParallelLinear = None + TELayerNormColumnParallelLinear = None TERowParallelLinear = None +MOCK_MODEL_PRESET = "mock" +NEMOTRON_20L_MODEL_PRESET = "nemotron-moe-vlm-20l" +NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" +NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 +NEMOTRON_20L_MAX_NUM_TILES = 12 +NEMOTRON_20L_DEFAULT_STAGE = "stage2" + _ACTIVE_GRIDS: list[HyperCommGrid] = [] _EMBEDDING_PG_CACHE: dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] = {} @@ -79,27 +103,74 @@ def parse_args() -> argparse.Namespace: grid.add_argument("--llm-expt-dp", type=int, default=1) model = parser.add_argument_group("model") + model.add_argument( + "--model-preset", + choices=[MOCK_MODEL_PRESET, NEMOTRON_20L_MODEL_PRESET], + default=MOCK_MODEL_PRESET, + help="Model config preset. The Nemotron preset matches the 20L reference script.", + ) model.add_argument("--hidden-size", type=int, default=128) model.add_argument("--num-layers", type=int, default=2) model.add_argument("--num-attention-heads", type=int, default=8) model.add_argument("--vocab-size", type=int, default=512) model.add_argument("--seq-length", type=int, default=32) model.add_argument("--image-seq-length", type=int, default=None) - model.add_argument("--image-token-id", type=int, default=50257) + model.add_argument("--image-token-id", type=int, default=511) + model.add_argument("--pad-token-id", type=int, default=0) + model.add_argument("--image-token", type=str, default="") + model.add_argument("--tokenizer-model", type=str, default=None) + model.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") + model.add_argument("--image-tag-type", type=str, default="") + model.add_argument("--force-system-message", action="store_true") model.add_argument("--num-moe-experts", type=int, default=4) model.add_argument("--moe-router-topk", type=int, default=1) model.add_argument("--moe-grouped-gemm", action="store_true") + model.add_argument("--img-h", type=int, default=512) + model.add_argument("--img-w", type=int, default=512) + model.add_argument("--patch-dim", type=int, default=16) + model.add_argument("--class-token-len", type=int, default=8) + model.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) + model.add_argument("--freeze-lm", action="store_true", help="Freeze language model params") + model.add_argument("--freeze-vit", action="store_true", help="Freeze vision encoder params") + model.add_argument( + "--freeze-projection", action="store_true", help="Freeze vision projection params" + ) + model.add_argument( + "--training-stage", + choices=["stage1", "stage2", "stage3"], + default=None, + help="Nemotron VLM freeze stage. Defaults to stage2 for the 20L preset.", + ) model.add_argument( "--fp32", action="store_true", help="Build and train in fp32 instead of bf16" ) train = parser.add_argument_group("training") train.add_argument("--micro-batch-size", type=int, default=2) + train.add_argument("--global-batch-size", type=int, default=None) train.add_argument("--num-microbatches", type=int, default=2) train.add_argument("--train-iters", type=int, default=2) train.add_argument("--lr", type=float, default=1.0e-4) + train.add_argument("--min-lr", type=float, default=None) + train.add_argument("--lr-decay-style", type=str, default="constant") + train.add_argument("--lr-warmup-iters", type=int, default=0) + train.add_argument("--lr-decay-iters", type=int, default=None) train.add_argument("--weight-decay", type=float, default=0.01) + train.add_argument("--adam-beta1", type=float, default=0.9) + train.add_argument("--adam-beta2", type=float, default=0.999) train.add_argument("--clip-grad", type=float, default=1.0) + train.add_argument( + "--overlap-grad-reduce", + action=argparse.BooleanOptionalAction, + default=True, + help="Enable DDP gradient-reduce overlap. Disable for parity with the 20L reference script.", + ) + train.add_argument( + "--ddp-bucket-size", + type=int, + default=10000, + help="DDP bucket size. Use 0 for a single unbounded bucket.", + ) train.add_argument("--seed", type=int, default=12345) train.add_argument("--log-interval", type=int, default=1) @@ -122,6 +193,10 @@ def initialize_distributed() -> None: torch.cuda.set_device(local_rank) if not dist.is_initialized(): dist.init_process_group(backend="nccl") + try: + parallel_state.get_global_memory_buffer() + except AssertionError: + parallel_state._set_global_memory_buffer() dist.barrier() @@ -140,6 +215,70 @@ def debug_rank(message: str) -> None: sys.stderr.flush() +def is_nemotron_20l(args: argparse.Namespace) -> bool: + """Return whether the run should use the Nemotron6-MoE VLM 20L architecture.""" + return args.model_preset == NEMOTRON_20L_MODEL_PRESET + + +def apply_model_preset(args: argparse.Namespace) -> None: + """Apply architecture defaults for the selected model preset.""" + if not is_nemotron_20l(args): + return + + args.num_layers = 20 + args.hidden_size = 2688 + args.num_attention_heads = 32 + args.num_moe_experts = 128 + args.moe_router_topk = 6 + args.moe_grouped_gemm = True + args.seq_length = 8192 + args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles + + +def apply_training_stage(args: argparse.Namespace) -> None: + """Apply the reference Nemotron VLM freeze stage defaults.""" + if not is_nemotron_20l(args): + return + + stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE + if stage == "stage1": + args.freeze_vit = True + args.freeze_lm = True + elif stage == "stage2": + args.freeze_vit = True + elif stage != "stage3": + raise ValueError(f"unsupported Nemotron VLM training stage: {stage}") + args.training_stage = stage + + +def resolve_image_token_id(args: argparse.Namespace) -> None: + """Resolve the image token id from the reference MultimodalTokenizer when provided.""" + if not is_nemotron_20l(args) or not args.tokenizer_model: + return + + from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( + MegatronMultimodalTokenizer, + ) + + tokenizer = MegatronMultimodalTokenizer( + path=args.tokenizer_model, + prompt_format=args.tokenizer_prompt_format, + special_tokens=[args.image_token], + image_tag_type=args.image_tag_type, + force_system_message=args.force_system_message, + ) + image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) + if image_token_id is None: + raise RuntimeError( + f"tokenizer at {args.tokenizer_model} did not produce an id for {args.image_token}" + ) + args.image_token_id = int(image_token_id) + if tokenizer.pad is not None: + args.pad_token_id = int(tokenizer.pad) + if tokenizer.vocab_size is not None: + args.vocab_size = int(tokenizer.vocab_size) + + def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: """Validate the Phase 2 non-colocated 1F1B mock-training layout.""" if args.encoder_cp != 1 or args.llm_cp != 1: @@ -150,10 +289,16 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: raise ValueError("--num-moe-experts must be divisible by --llm-ep") if args.log_interval < 1: raise ValueError("--log-interval must be >= 1") + if not 0 <= args.image_token_id < args.vocab_size: + raise ValueError("--image-token-id must be within --vocab-size") + if not 0 <= args.pad_token_id < args.vocab_size: + raise ValueError("--pad-token-id must be within --vocab-size") image_seq_length = args.image_seq_length or args.seq_length // 2 if image_seq_length >= args.seq_length: raise ValueError("--image-seq-length must be smaller than --seq-length") + if args.seq_length - image_seq_length < 2: + raise ValueError("mock next-token training needs at least two text tokens") if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") @@ -367,6 +512,62 @@ def get_group_rank_or(pg: Optional[dist.ProcessGroup], fallback: int = 0) -> int return dist.get_rank(pg) if is_process_group_member(pg) else fallback +def set_module_requires_grad(module: Optional[torch.nn.Module], requires_grad: bool) -> None: + """Set requires_grad for every parameter in a module when the module exists.""" + if module is None: + return + for param in module.parameters(): + param.requires_grad = requires_grad + + +def set_model_init_seed( + args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_offset: int +): + """Seed CPU model init consistently across TP/DP peers for one module role.""" + pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) + torch.manual_seed(args.seed + role_offset + (100 * pp_rank)) + + +def initialize_model_parallel_rng(args: argparse.Namespace, pg_collection: ProcessGroupCollection): + """Initialize CUDA RNG tracker using the active module's hetero process groups.""" + pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) + tp_rank = get_group_rank_or(getattr(pg_collection, "tp", None)) + ep_rank = get_group_rank_or(getattr(pg_collection, "ep", None)) + expt_tp_rank = get_group_rank_or(getattr(pg_collection, "expt_tp", None)) + model_parallel_cuda_manual_seed( + args.seed + (100 * pp_rank), + tp_rank=tp_rank, + ep_rank=ep_rank, + etp_rank=expt_tp_rank, + force_reset_rng=True, + ) + + +def active_ddp_modules(mimo_model: MimoModel) -> list[DistributedDataParallel]: + """Return active DDP-wrapped submodules owned by this rank.""" + modules = [] + if isinstance(mimo_model.language_model, DistributedDataParallel): + modules.append(mimo_model.language_model) + modules.extend( + submodule + for submodule in mimo_model.modality_submodules.values() + if isinstance(submodule, DistributedDataParallel) + ) + return modules + + +def broadcast_active_params(mimo_model: MimoModel) -> None: + """Synchronize initial parameters across each module's DP groups.""" + for module in active_ddp_modules(mimo_model): + module.broadcast_params() + + +def zero_active_grad_buffers(mimo_model: MimoModel) -> None: + """Clear MCore DDP grad buffers before each training iteration.""" + for module in active_ddp_modules(mimo_model): + module.zero_grad_buffer() + + def get_grid_coordinate(grid: HyperCommGrid, dim: str) -> int: """Return this rank's coordinate for a base-layout dimension.""" if not is_rank_in_grid(grid): @@ -404,6 +605,82 @@ def no_sync_func(): return no_sync_func +class RADIOEncoderWrapper(torch.nn.Module): + """RADIO encoder wrapper matching the Nemotron6-MoE VLM provider.""" + + def __init__( + self, + transformer_config: TransformerConfig, + transformer_layer_spec: ModuleSpec, + pg_collection: Optional[ProcessGroupCollection], + img_h: int, + img_w: int, + patch_dim: int, + class_token_len: int, + drop_class_token: bool = True, + apply_pixel_shuffle: bool = True, + force_eval_mode: bool = False, + ) -> None: + super().__init__() + self.class_token_len = class_token_len + self.drop_class_token = drop_class_token + self.apply_pixel_shuffle = apply_pixel_shuffle + self.force_eval_mode = force_eval_mode + self.radio_model = RADIOViTModel( + transformer_config=transformer_config, + transformer_layer_spec=transformer_layer_spec, + patch_dim=patch_dim, + img_h=img_h, + img_w=img_w, + class_token_len=class_token_len, + add_class_token=True, + max_img_h=2048, + max_img_w=2048, + has_cpe=True, + embedder_bias=False, + pg_collection=pg_collection, + ) + if self.force_eval_mode: + self.radio_model.eval() + + def train(self, mode: bool = True): + """Keep frozen RADIO in eval mode while allowing the projection to train.""" + super().train(mode) + if self.force_eval_mode: + self.radio_model.eval() + return self + + @property + def config(self): + """Expose the underlying RADIO config for DDP wrapping.""" + return self.radio_model.config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run RADIO, drop class tokens, and apply pixel shuffle.""" + context = torch.no_grad() if self.force_eval_mode else nullcontext() + debug_rank(f"RADIO forward start: input_shape={tuple(x.shape)}") + with context: + x = x.to(dtype=self.radio_model.embedder.weight.dtype) + embeddings = self.radio_model(x) + debug_rank(f"RADIO forward done: output_shape={tuple(embeddings.shape)}") + if self.drop_class_token: + embeddings = embeddings[:, self.class_token_len :, :] + debug_rank(f"RADIO class tokens dropped: output_shape={tuple(embeddings.shape)}") + if self.apply_pixel_shuffle: + embeddings = pixel_shuffle(embeddings, scale_factor=0.5) + debug_rank(f"RADIO pixel shuffle done: output_shape={tuple(embeddings.shape)}") + return embeddings + + def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): + """Delegate checkpoint sharding to the wrapped RADIO model.""" + sharded_sd = {} + for name, child in self.named_children(): + sharded_sd.update( + sharded_state_dict_default(child, f"{prefix}{name}.", sharded_offsets, metadata) + ) + return sharded_sd + + def projection_layer_spec() -> ModuleSpec: """Return the TE-backed projection MLP spec.""" if TEColumnParallelLinear is None or TERowParallelLinear is None: @@ -414,12 +691,151 @@ def projection_layer_spec() -> ModuleSpec: ) +def nemotron_projection_layer_spec() -> ModuleSpec: + """Return the Nemotron VLM RADIO-to-language projector layer spec.""" + if TELayerNormColumnParallelLinear is None or TERowParallelLinear is None: + raise RuntimeError("TELayerNormColumnParallelLinear and TERowParallelLinear are required") + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear + ), + ) + + +def nemotron_language_config( + args: argparse.Namespace, tp_size: int, pp_size: int, ep_size: int, expt_tp_size: int +) -> TransformerConfig: + """Build the exact Nemotron6-MoE 20L language TransformerConfig.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=20, + hidden_size=2688, + num_attention_heads=32, + attention_backend=AttnBackend.flash, + num_query_groups=2, + ffn_hidden_size=1856, + kv_channels=128, + activation_func=squared_relu, + gated_linear_unit=False, + attention_dropout=0.0, + hidden_dropout=0.0, + normalization="RMSNorm", + add_bias_linear=False, + init_method_std=0.0173, + use_cpu_initialization=True, + variable_seq_lengths=True, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + expert_model_parallel_size=ep_size, + expert_tensor_parallel_size=expt_tp_size, + sequence_parallel=tp_size > 1, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + calculate_per_token_loss=True, + bias_activation_fusion=False, + masked_softmax_fusion=True, + persist_layer_norm=True, + bias_dropout_fusion=False, + recompute_granularity="selective", + recompute_modules=["core_attn"], + moe_ffn_hidden_size=1856, + num_moe_experts=128, + moe_router_topk=6, + moe_grouped_gemm=True, + moe_router_score_function="sigmoid", + moe_router_topk_scaling_factor=2.5, + moe_router_enable_expert_bias=True, + moe_router_dtype="fp32", + moe_router_load_balancing_type="seq_aux_loss", + moe_aux_loss_coeff=0.0001, + moe_shared_expert_intermediate_size=3712, + moe_shared_expert_overlap=True, + moe_token_dispatcher_type="alltoall", + moe_permute_fusion=True, + use_fused_weighted_squared_relu=True, + is_hybrid_model=True, + mamba_num_heads=64, + mamba_head_dim=64, + ) + config.position_embedding_type = "none" + config.seq_length = 8192 + config.max_position_embeddings = 8192 + return config + + +def require_per_token_loss(config: TransformerConfig) -> None: + """The hetero MIMO loop scales both language and vision grads by real LM tokens.""" + if not config.calculate_per_token_loss: + raise ValueError("train_hetero.py requires calculate_per_token_loss=True") + + +def radio_vision_config(args: argparse.Namespace, tp_size: int, pp_size: int) -> TransformerConfig: + """Build the exact RADIO vision TransformerConfig from the 20L reference provider.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=32, + hidden_size=1280, + num_attention_heads=16, + use_cpu_initialization=True, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + ) + config.kv_channels = 80 + config.num_query_groups = 16 + config.ffn_hidden_size = 5120 + config.gated_linear_unit = False + config.activation_func = fast_gelu + config.add_bias_linear = True + config.add_qkv_bias = True + config.normalization = "LayerNorm" + config.layernorm_epsilon = 1.0e-6 + config.layernorm_zero_centered_gamma = False + config.apply_rope_fusion = False + config.qk_layernorm = False + config.bias_activation_fusion = False + config.bias_dropout_fusion = False + config.attention_softmax_in_fp32 = True + config.attention_dropout = 0.0 + config.hidden_dropout = 0.0 + return config + + +def nemotron_projection_config(args: argparse.Namespace, tp_size: int) -> TransformerConfig: + """Build the exact RADIO-to-Nemotron projection config.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=1, + hidden_size=2688, + num_attention_heads=1, + use_cpu_initialization=True, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + ) + config.tensor_model_parallel_size = tp_size + config.ffn_hidden_size = 4 * 5120 + config.bias_activation_fusion = False + config.bias_dropout_fusion = False + config.add_bias_linear = False + config.activation_func = squared_relu + config.normalization = "RMSNorm" + return config + + def language_model_spec( args: argparse.Namespace, pg_collection: Optional[ProcessGroupCollection], llm_grid: HyperCommGrid, ) -> ModuleSpec: - """Create the GPT language ModuleSpec for the local language grid.""" + """Create the language ModuleSpec for the local language grid.""" pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None ep_pg = getattr(pg_collection, "ep", None) if pg_collection is not None else None @@ -431,6 +847,25 @@ def language_model_spec( tp_size = get_group_size_or(tp_pg, fallback_tp_size) ep_size = get_group_size_or(ep_pg, args.llm_ep) expt_tp_size = get_group_size_or(expt_tp_pg, args.llm_expt_tp or fallback_tp_size) + if is_nemotron_20l(args): + config = nemotron_language_config(args, tp_size, pp_size, ep_size, expt_tp_size) + require_per_token_loss(config) + return ModuleSpec( + module=MambaModel, + params={ + "config": config, + "mamba_stack_spec": mamba_stack_spec, + "vocab_size": args.vocab_size, + "max_sequence_length": args.seq_length, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + "hybrid_override_pattern": NEMOTRON_20L_HYBRID_PATTERN, + "position_embedding_type": "none", + "scatter_embedding_sequence_parallel": False, + "pg_collection": pg_collection, + }, + ) + num_moe_experts = args.num_moe_experts if args.num_moe_experts > 0 else None bf16 = not args.fp32 @@ -461,6 +896,7 @@ def language_model_spec( cross_entropy_fusion_impl="te", **moe_kwargs, ) + require_per_token_loss(config) return ModuleSpec( module=GPTModel, params={ @@ -482,7 +918,7 @@ def vision_submodules_spec( pg_collection: Optional[ProcessGroupCollection], encoder_grid: HyperCommGrid, ) -> ModuleSpec: - """Create the mock vision ModuleSpec for the local encoder grid.""" + """Create the vision ModuleSpec for the local encoder grid.""" from megatron.core.transformer.transformer_block import TransformerBlock pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None @@ -492,6 +928,42 @@ def vision_submodules_spec( pp_rank = get_group_rank_or(pp_pg) bf16 = not args.fp32 + if is_nemotron_20l(args): + vision_config = radio_vision_config(args, tp_size, pp_size) + vision_encoder_spec = ModuleSpec( + module=RADIOEncoderWrapper, + params={ + "transformer_config": vision_config, + "transformer_layer_spec": get_vit_layer_with_transformer_engine_spec(), + "pg_collection": pg_collection, + "img_h": args.img_h, + "img_w": args.img_w, + "patch_dim": args.patch_dim, + "class_token_len": args.class_token_len, + "drop_class_token": True, + "apply_pixel_shuffle": True, + "force_eval_mode": args.freeze_vit, + }, + ) + vision_projection_spec = ModuleSpec( + module=MultimodalProjector, + params={ + "config": nemotron_projection_config(args, tp_size), + "submodules": nemotron_projection_layer_spec().submodules, + "projector_type": "mlp", + "input_size": 5120, + "tp_group": tp_pg if is_process_group_member(tp_pg) else None, + }, + ) + return ModuleSpec( + module=VisionModalitySubmodules, + params={"pg_collection": pg_collection}, + submodules={ + "encoders": {"radio_encoder": vision_encoder_spec}, + "input_projections": [vision_projection_spec], + }, + ) + vision_config = TransformerConfig( num_layers=args.num_layers, hidden_size=args.hidden_size, @@ -558,6 +1030,12 @@ def build_mimo_model( "building model specs " f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" ) + if rank_in_language_grid: + set_model_init_seed(args, language_pg, role_offset=20_000) + initialize_model_parallel_rng(args, language_pg) + elif rank_in_encoder_grid: + set_model_init_seed(args, vision_pg, role_offset=10_000) + initialize_model_parallel_rng(args, vision_pg) module_to_grid_map = {encoder_name: encoder_grid, MIMO_LANGUAGE_MODULE_KEY: llm_grid} mimo_config = MimoModelConfig( @@ -574,7 +1052,11 @@ def build_mimo_model( ) debug_rank("constructing MimoModel") - mimo_model = MimoModel(mimo_config) + mimo_model = MimoModel( + mimo_config, + cp_group=language_pg.cp if rank_in_language_grid else None, + tp_group=language_pg.tp if rank_in_language_grid else None, + ) debug_rank("moving MimoModel to cuda") mimo_model.to(torch.device("cuda")) if not args.fp32: @@ -582,9 +1064,13 @@ def build_mimo_model( debug_rank("MimoModel moved to target dtype/device") ddp_config = DistributedDataParallelConfig( - overlap_grad_reduce=True, bucket_size=10000, use_distributed_optimizer=True + overlap_grad_reduce=args.overlap_grad_reduce, + bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, + use_distributed_optimizer=True, ) if mimo_model.language_model is not None: + if args.freeze_lm: + set_module_requires_grad(mimo_model.language_model, False) debug_rank("wrapping language model in DDP") mimo_model.language_model = DistributedDataParallel( config=mimo_model.language_model.config, @@ -597,15 +1083,22 @@ def build_mimo_model( if encoder_name in mimo_model.modality_submodules: submodule = mimo_model.modality_submodules[encoder_name] if submodule is not None: + encoder_module_name = "radio_encoder" if is_nemotron_20l(args) else "clip_encoder" + if args.freeze_vit: + set_module_requires_grad(submodule.encoders[encoder_module_name], False) + if args.freeze_projection: + for projection in submodule.input_projections: + set_module_requires_grad(projection, False) debug_rank("wrapping vision submodule in DDP") mimo_model.modality_submodules[encoder_name] = DistributedDataParallel( - config=submodule.encoders["clip_encoder"].config, + config=submodule.encoders[encoder_module_name].config, ddp_config=ddp_config, module=submodule, pg_collection=vision_pg, ) debug_rank("vision submodule DDP ready") + broadcast_active_params(mimo_model) return mimo_model, module_to_grid_map, language_pg, vision_pg @@ -630,6 +1123,10 @@ def __iter__(self): def __next__(self): args = self.args + debug_rank( + f"mock batch start: micro_batch_size={self.micro_batch_size}, " + f"image_seq_length={self.image_seq_length}" + ) image_tokens = torch.full( (self.micro_batch_size, self.image_seq_length), args.image_token_id, @@ -643,24 +1140,55 @@ def __next__(self): device="cuda", generator=self.generator, ) + special_token_ids = {args.image_token_id, args.pad_token_id} + replacement_token_id = next( + ( + token_id + for token_id in range(1, args.vocab_size) + if token_id not in special_token_ids + ), + None, + ) + if replacement_token_id is None: + raise RuntimeError("mock data needs at least one non-special token id") + if 1 <= args.image_token_id < args.vocab_size: + text_tokens[text_tokens == args.image_token_id] = replacement_token_id + if 1 <= args.pad_token_id < args.vocab_size: + text_tokens[text_tokens == args.pad_token_id] = replacement_token_id input_ids = torch.cat([image_tokens, text_tokens], dim=1) - labels = input_ids.clone() - labels[input_ids == args.image_token_id] = -100 - - loss_mask = torch.ones( - self.micro_batch_size, args.seq_length, dtype=torch.float32, device="cuda" - ) - loss_mask[input_ids == args.image_token_id] = 0.0 + labels = torch.full_like(input_ids, -100) + labels[:, :-1] = input_ids[:, 1:] + labels[(labels == args.image_token_id) | (labels == args.pad_token_id)] = -100 + loss_mask = (labels != -100).to(dtype=torch.float32) + + if is_nemotron_20l(args): + encoder_inputs = { + "radio_encoder": { + "x": torch.randn( + self.micro_batch_size * args.num_image_tiles, + 3, + args.img_h, + args.img_w, + device="cuda", + dtype=self.dtype, + generator=self.generator, + ) + } + } + else: + encoder_hidden_states = torch.randn( + self.image_seq_length, + self.micro_batch_size, + args.hidden_size, + device="cuda", + dtype=self.dtype, + generator=self.generator, + ) + encoder_inputs = { + "clip_encoder": {"hidden_states": encoder_hidden_states, "attention_mask": None} + } - encoder_hidden_states = torch.randn( - self.image_seq_length, - self.micro_batch_size, - args.hidden_size, - device="cuda", - dtype=self.dtype, - generator=self.generator, - ) num_image_placeholders = (input_ids == args.image_token_id).sum().item() expected_image_placeholders = self.image_seq_length * self.micro_batch_size if num_image_placeholders != expected_image_placeholders: @@ -669,6 +1197,7 @@ def __next__(self): f"expected {expected_image_placeholders}" ) + debug_rank("mock batch ready") return { "input_ids": input_ids, "labels": labels, @@ -677,16 +1206,15 @@ def __next__(self): .unsqueeze(0) .expand(self.micro_batch_size, -1) .clone(), - "modality_inputs": { - self.encoder_name: { - "clip_encoder": {"hidden_states": encoder_hidden_states, "attention_mask": None} - } - }, + "modality_inputs": {self.encoder_name: {**encoder_inputs}}, } def wire_training_hooks( - mimo_model: MimoModel, language_pg: ProcessGroupCollection, vision_pg: ProcessGroupCollection + mimo_model: MimoModel, + language_pg: ProcessGroupCollection, + vision_pg: ProcessGroupCollection, + token_count_group: dist.ProcessGroup, ) -> None: """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" @@ -702,34 +1230,40 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar if num_tokens is None: raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") - token_count = num_tokens.to(device="cuda", dtype=torch.float32) - if not is_token_source_rank(): - token_count.zero_() - dist.all_reduce(token_count, op=dist.ReduceOp.SUM) + token_count = torch.zeros(1, dtype=torch.float32, device="cuda") + if is_token_source_rank(): + token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() + dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=token_count_group) global_num_tokens = token_count.item() if mimo_model.language_model is not None: + debug_rank("finalizing language grads") finalize_model_grads( [mimo_model.language_model], num_tokens=None, pg_collection=language_pg, force_all_reduce=force_all_reduce, ) + debug_rank("language grads finalized") for submodule in mimo_model.modality_submodules.values(): if submodule is not None: + debug_rank("finalizing vision grads") finalize_model_grads( [submodule], num_tokens=None, pg_collection=vision_pg, force_all_reduce=force_all_reduce, ) + debug_rank("vision grads finalized") if global_num_tokens > 0: scale = 1.0 / global_num_tokens if mimo_model.language_model is not None: + debug_rank("scaling language grads") mimo_model.language_model.scale_gradients(scale) for submodule in mimo_model.modality_submodules.values(): if submodule is not None: + debug_rank("scaling vision grads") submodule.scale_gradients(scale) mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) @@ -825,13 +1359,17 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} output = output.float() - if loss_mask is not None and output.shape == loss_mask.shape: - masked = output * loss_mask.float() - num_tokens = loss_mask.float().sum().to(torch.int) - loss_sum = masked.sum() - else: - loss_sum = output.sum() - num_tokens = torch.tensor(output.numel(), device="cuda", dtype=torch.int) + if loss_mask is None: + raise RuntimeError("train_hetero.py requires a loss_mask for per-token loss") + if output.shape != loss_mask.shape: + raise RuntimeError( + f"loss output shape {tuple(output.shape)} does not match loss_mask shape " + f"{tuple(loss_mask.shape)}; per-token loss cannot be scaled correctly" + ) + + masked = output * loss_mask.float() + num_tokens = loss_mask.float().sum().to(torch.int) + loss_sum = masked.sum() return ( loss_sum, num_tokens, @@ -842,12 +1380,53 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): def forward_step(data_iterator, model): """Forward step consumed by the MCore pipeline schedule.""" batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} + debug_rank("forward_step batch prepared") + debug_rank("forward_step model call start") output_tensor, loss_mask = model(**batch) + debug_rank("forward_step model call done") return output_tensor, partial(loss_func, loss_mask) +def get_global_batch_size(args: argparse.Namespace) -> int: + """Return the language-side global batch size for scheduler accounting.""" + derived_global_batch_size = args.micro_batch_size * args.num_microbatches * args.llm_dp + if args.global_batch_size is None: + return derived_global_batch_size + if args.global_batch_size != derived_global_batch_size: + raise ValueError( + "--global-batch-size must equal " + "--micro-batch-size * --num-microbatches * --llm-dp in this hetero loop " + f"({derived_global_batch_size}); got {args.global_batch_size}" + ) + return args.global_batch_size + + +def build_optimizer_param_scheduler(args: argparse.Namespace, optimizer) -> OptimizerParamScheduler: + """Build the MCore optimizer parameter scheduler using Megatron train-iters semantics.""" + global_batch_size = get_global_batch_size(args) + lr_decay_iters = args.lr_decay_iters if args.lr_decay_iters is not None else args.train_iters + return OptimizerParamScheduler( + optimizer, + init_lr=0.0, + max_lr=args.lr, + min_lr=args.min_lr if args.min_lr is not None else 0.0, + lr_warmup_steps=args.lr_warmup_iters * global_batch_size, + lr_decay_steps=lr_decay_iters * global_batch_size, + lr_decay_style=args.lr_decay_style, + start_wd=args.weight_decay, + end_wd=args.weight_decay, + wd_incr_steps=args.train_iters * global_batch_size, + wd_incr_style="constant", + use_checkpoint_opt_param_scheduler=False, + override_opt_param_scheduler=True, + ) + + def run_train_loop(args: argparse.Namespace) -> None: """Run mock-data heterogeneous MIMO training.""" + apply_model_preset(args) + apply_training_stage(args) + resolve_image_token_id(args) world_size = dist.get_world_size() encoder_size, llm_size = validate_args(args, world_size) @@ -877,14 +1456,17 @@ def run_train_loop(args: argparse.Namespace) -> None: debug_rank("creating embedding groups") create_all_embedding_groups([encoder_grid, llm_grid]) debug_rank("embedding groups ready") + debug_rank("creating MIMO optimizer stats group") + mimo_optimizer_stats_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") + debug_rank("MIMO optimizer stats group ready") - torch.manual_seed(args.seed + dist.get_rank()) + torch.manual_seed(args.seed) debug_rank("building MIMO model") mimo_model, module_to_grid_map, language_pg, vision_pg = build_mimo_model( args, encoder_grid, llm_grid, encoder_name ) debug_rank("wiring training hooks") - wire_training_hooks(mimo_model, language_pg, vision_pg) + wire_training_hooks(mimo_model, language_pg, vision_pg, mimo_optimizer_stats_group) debug_rank("building MIMO optimizer") optimizer = get_mimo_optimizer( @@ -892,12 +1474,17 @@ def run_train_loop(args: argparse.Namespace) -> None: OptimizerConfig( optimizer="adam", lr=args.lr, + min_lr=args.min_lr, weight_decay=args.weight_decay, + adam_beta1=args.adam_beta1, + adam_beta2=args.adam_beta2, clip_grad=args.clip_grad, bf16=not args.fp32, use_distributed_optimizer=True, ), + stats_group=mimo_optimizer_stats_group, ) + opt_param_scheduler = build_optimizer_param_scheduler(args, optimizer) debug_rank("MIMO optimizer ready") debug_rank("building pipeline communicator") communicator = MultiModulePipelineCommunicator( @@ -923,7 +1510,9 @@ def run_train_loop(args: argparse.Namespace) -> None: try: for iteration in range(1, args.train_iters + 1): + zero_active_grad_buffers(mimo_model) optimizer.zero_grad() + debug_rank(f"iteration {iteration}: starting forward/backward schedule") losses = schedule.forward_backward_pipelining_without_interleaving( forward_step_func=forward_step, data_iterator=data_iterator, @@ -935,19 +1524,21 @@ def run_train_loop(args: argparse.Namespace) -> None: p2p_communicator=communicator, pg_collection=schedule_pg_collection, ) + debug_rank(f"iteration {iteration}: schedule complete") + debug_rank(f"iteration {iteration}: optimizer step starting") success, grad_norm, _ = optimizer.step() + debug_rank(f"iteration {iteration}: optimizer step complete") if not success: raise RuntimeError(f"optimizer step failed at iteration {iteration}") + opt_param_scheduler.step(increment=get_global_batch_size(args)) if iteration % args.log_interval == 0: loss_acc = torch.zeros(2, dtype=torch.float32, device="cuda") - if ( - losses - and is_process_group_member(getattr(language_pg, "pp", None)) - and is_process_group_member(getattr(language_pg, "tp", None)) - ): - is_log_source = is_pp_last_stage(language_pg.pp) and language_pg.tp.rank() == 0 - if is_log_source: + is_log_stage = is_process_group_member( + getattr(language_pg, "tp_dp_cp", None) + ) and is_pp_last_stage(language_pg.pp) + if is_log_stage: + if losses and language_pg.tp.rank() == 0: for loss_dict in losses: loss_sum = loss_dict.get("lm loss sum") num_tokens = loss_dict.get("lm tokens") @@ -959,12 +1550,19 @@ def run_train_loop(args: argparse.Namespace) -> None: loss_acc[1] += num_tokens.float() elif num_tokens is not None: loss_acc[1] += float(num_tokens) - dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM) - loss_value = loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None - if dist.get_rank() == 0: - print_rank_0(f"iteration {iteration}: loss={loss_value}, grad_norm={grad_norm}") + dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.tp_dp_cp) + loss_value = ( + loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None + ) + language_group_ranks = dist.get_process_group_ranks(language_pg.tp_dp_cp) + if dist.get_rank() == min(language_group_ranks): + sys.stdout.write( + f"iteration {iteration}: loss={loss_value}, grad_norm={grad_norm}\n" + ) + sys.stdout.flush() finally: mimo_model.destroy() + destroy_process_group_if_member(mimo_optimizer_stats_group) def main() -> None: @@ -978,10 +1576,10 @@ def main() -> None: finally: try: torch.cuda.synchronize() - dist.barrier() except Exception: pass destroy_runtime_process_groups() + parallel_state.destroy_global_memory_buffer() if dist.is_initialized(): dist.destroy_process_group() diff --git a/megatron/core/distributed/finalize_model_grads.py b/megatron/core/distributed/finalize_model_grads.py index ca6bdd354ce..2857cb5c99a 100644 --- a/megatron/core/distributed/finalize_model_grads.py +++ b/megatron/core/distributed/finalize_model_grads.py @@ -290,7 +290,11 @@ def reset_model_temporary_tensors(config: TransformerConfig, model: List[torch.n module.reset_global_aux_loss_tracker() -def _update_router_expert_bias(model: List[torch.nn.Module], config: TransformerConfig): +def _update_router_expert_bias( + model: List[torch.nn.Module], + config: TransformerConfig, + tp_dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, +): """ Update the expert bias of the router for a global batch. This requires all-reduce of local_tokens_per_expert across TPxCPxDP ranks @@ -312,7 +316,10 @@ def _update_router_expert_bias(model: List[torch.nn.Module], config: Transformer stacked_tokens_per_expert = torch.stack(tokens_per_expert_list, dim=0) stacked_expert_bias = torch.stack(expert_bias_list, dim=0) stacked_updated_expert_bias = get_updated_expert_bias( - stacked_tokens_per_expert, stacked_expert_bias, config.moe_router_bias_update_rate + stacked_tokens_per_expert, + stacked_expert_bias, + config.moe_router_bias_update_rate, + tp_dp_cp_group=tp_dp_cp_group, ) for expert_bias, updated_expert_bias in zip(expert_bias_list, stacked_updated_expert_bias): @@ -433,12 +440,16 @@ def finalize_model_grads( embd_group = pg_collection.embd pos_emb_group = pg_collection.pos_embd dp_cp_group = pg_collection.dp_cp + tp_dp_cp_group = getattr(pg_collection, 'tp_dp_cp', None) else: tp_group = parallel_state.get_tensor_model_parallel_group() pp_group = parallel_state.get_pipeline_model_parallel_group() embd_group = parallel_state.get_embedding_group(check_initialized=False) pos_emb_group = parallel_state.get_position_embedding_group(check_initialized=False) dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True) + tp_dp_cp_group = parallel_state.get_tensor_and_data_parallel_group( + with_context_parallel=True + ) # All-reduce / reduce-scatter across DP replicas. if config.timers is not None: @@ -478,7 +489,11 @@ def finalize_model_grads( config.timers('embedding-grads-all-reduce').stop() if config.moe_router_enable_expert_bias: - _update_router_expert_bias(model, config) + if tp_dp_cp_group is None: + raise RuntimeError( + "pg_collection.tp_dp_cp is required when moe_router_enable_expert_bias is enabled" + ) + _update_router_expert_bias(model, config, tp_dp_cp_group=tp_dp_cp_group) reset_model_temporary_tensors(config, model) diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py index 85870726269..16289ecba32 100644 --- a/megatron/core/models/common/language_module/language_module.py +++ b/megatron/core/models/common/language_module/language_module.py @@ -180,7 +180,9 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor: elif self.config.cross_entropy_fusion_impl == 'native': loss = fused_vocab_parallel_cross_entropy(logits, labels, self.pg_collection.tp) else: - loss = tensor_parallel.vocab_parallel_cross_entropy(logits, labels) + loss = tensor_parallel.vocab_parallel_cross_entropy( + logits, labels, tp_group=self.tp_group + ) # [s b] => [b, s] loss = loss.transpose(0, 1).contiguous() diff --git a/megatron/core/models/hybrid/hybrid_layer_allocation.py b/megatron/core/models/hybrid/hybrid_layer_allocation.py index f1ba94ef7fa..df5a91bf6b1 100644 --- a/megatron/core/models/hybrid/hybrid_layer_allocation.py +++ b/megatron/core/models/hybrid/hybrid_layer_allocation.py @@ -333,6 +333,8 @@ def select_pipeline_segment( vp_stage: Optional[int], first_stage_layers: Optional[int] = None, last_stage_layers: Optional[int] = None, + tp_group: Optional[torch.distributed.ProcessGroup] = None, + dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, ) -> Tuple[List[str], int]: """Select and validate the pipeline segment for the given PP rank and VP stage. @@ -352,6 +354,8 @@ def select_pipeline_segment( uneven PP. Only valid when the pattern has no pipe separators. last_stage_layers: Number of layers on the last pipeline stage for uneven PP. Only valid when the pattern has no pipe separators. + tp_group: Tensor parallel process group used for rank-local logging. + dp_cp_group: Data/context parallel process group used for rank-local logging. Returns: Tuple of (layer_type_list, layer_offset) where layer_type_list is @@ -445,6 +449,8 @@ def select_pipeline_segment( f"HybridModel: pp_rank={pp_rank}/{pp_size}, vp_stage={vp_stage}, " f"layers='{''.join(selected)}' ({len(selected)} layers), " f"layer_offset={offset} (auto-split)", + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) return selected, offset @@ -479,6 +485,8 @@ def select_pipeline_segment( f"segment_index={segment_index}/{len(segments)}, " f"layers='{my_segment}' ({len(layer_type_list)} layers), " f"layer_offset={layer_offset}", + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) return layer_type_list, layer_offset diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index 4399c6984a7..00c3545d698 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -192,6 +192,8 @@ def __init__( vp_stage, first_stage_layers=self.config.num_layers_in_first_pipeline_stage, last_stage_layers=self.config.num_layers_in_last_pipeline_stage, + tp_group=getattr(self.pg_collection, 'tp', None), + dp_cp_group=getattr(self.pg_collection, 'dp_cp', None), ) # Determine if MTP is needed (based on pattern parsing) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index bdfe4289dd0..66bc273255e 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -78,7 +78,9 @@ def __init__(self, mimo_config: MimoModelConfig, cp_group=None, tp_group=None) - self.partition_adapter: Optional[PartitionAdapter] = None # Create partition adapter only if parallelism is enabled - if language_config.context_parallel_size > 1 or language_config.sequence_parallel: + if self.role.has_language_module and ( + language_config.context_parallel_size > 1 or language_config.sequence_parallel + ): partition_config = PartitionConfig.from_mp_config( mp=language_config, max_seq_len=max_seq_len, @@ -377,11 +379,14 @@ def forward( return self._forward_encoders(modality_inputs, input_tensors), loss_mask if self.role.has_language_module: - return ( - self._forward_language_module( - input_ids, position_ids, attention_mask, labels, input_tensors - ), + return self._forward_language_module( + input_ids, + position_ids, + attention_mask, loss_mask, + labels, + input_tensors, + packing_kwargs, ) raise RuntimeError(f"Rank has no modules assigned in role: {self.role}") @@ -424,9 +429,11 @@ def _forward_language_module( input_ids: torch.Tensor, position_ids: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], labels: Optional[torch.Tensor], input_tensors: Optional[Dict[str, torch.Tensor]], - ) -> torch.Tensor: + packing_kwargs: Optional[dict] = None, + ): """Forward pass for language module on this rank. Args: @@ -439,6 +446,15 @@ def _forward_language_module( Returns: Language model output (hidden states, logits, or loss depending on stage) """ + packed_seq_params = None + if packing_kwargs is not None: + for key in packing_kwargs: + if 'cu_seqlens' in key and packing_kwargs[key] is not None: + packing_kwargs[key] = packing_kwargs[key].to(dtype=torch.int32) + packed_seq_params = PackedSeqParams(**packing_kwargs) + packed_seq_params.qkv_format = 'thd' + logger.debug(f"Packed sequence parameters: {packed_seq_params}") + lang_name = MIMO_LANGUAGE_MODULE_KEY if self.role.is_first_stage(lang_name): @@ -463,12 +479,27 @@ def _forward_language_module( special_token_ids=self.special_token_ids, ) + if self.partition_adapter is not None: + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + combined_embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( + self.partition_adapter.shard( + embeddings=combined_embeddings, + labels=labels, + loss_mask=loss_mask, + attention_mask=attention_mask, + packed_seq_params=packed_seq_params, + ) + ) + if combined_embeddings is not None: + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + lm_output = self.language_model( input_ids=None, position_ids=None, decoder_input=combined_embeddings, labels=labels, - attention_mask=attention_mask, + attention_mask=None, + packed_seq_params=packed_seq_params, ) else: # Non-first stage: receive hidden states from previous LM stage @@ -480,19 +511,31 @@ def _forward_language_module( if hasattr(underlying_lm, 'set_input_tensor'): underlying_lm.set_input_tensor(hidden_states) + if self.partition_adapter is not None: + _, labels, loss_mask, attention_mask, packed_seq_params = ( + self.partition_adapter.shard( + embeddings=None, + labels=labels, + loss_mask=loss_mask, + attention_mask=attention_mask, + packed_seq_params=packed_seq_params, + ) + ) + lm_output = self.language_model( input_ids=None, position_ids=None, decoder_input=None, labels=labels, attention_mask=attention_mask, + packed_seq_params=packed_seq_params, ) # Key output for non-last stages so schedule can route to next LM stage if not self.role.is_last_stage(lang_name): - return {lang_name: lm_output} + return {lang_name: lm_output}, loss_mask - return lm_output + return lm_output, loss_mask def _build_colocated_communicators(self): grid_map = self.mimo_config.module_to_grid_map diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 456199c61cd..c330b1e9a36 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -38,9 +38,15 @@ class MimoOptimizer(MegatronOptimizer): across all modules via all_reduce MAX. """ - def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: OptimizerConfig): + def __init__( + self, + module_infos: Dict[str, ModuleOptimizerInfo], + config: OptimizerConfig, + stats_group: Optional[torch.distributed.ProcessGroup] = None, + ): self.module_infos = module_infos self.config = config + self.stats_group = stats_group self._active_optimizers: List[MegatronOptimizer] = [ info.optimizer for info in module_infos.values() @@ -53,8 +59,10 @@ def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: Optimiz def prepare_grads(self) -> bool: """Prepare gradients for all active module optimizers.""" found_inf = False - for opt in self._active_optimizers: - found_inf |= opt.prepare_grads() + for name, info in sorted(self.module_infos.items()): + if not (info.is_active and info.optimizer is not None): + continue + found_inf |= info.optimizer.prepare_grads() return found_inf @torch.no_grad() @@ -68,7 +76,9 @@ def get_grad_norm(self) -> float: module_norm = info.optimizer.get_grad_norm() or 0.0 norm_sq[i] = module_norm**2 - torch.distributed.all_reduce(norm_sq, op=torch.distributed.ReduceOp.MAX) + torch.distributed.all_reduce( + norm_sq, op=torch.distributed.ReduceOp.MAX, group=self.stats_group + ) return torch.sqrt(norm_sq.sum()).item() @torch.no_grad() @@ -79,7 +89,9 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: # if encoder ranks detect inf but LLM ranks don't, the early return # would skip the all_reduce in get_grad_norm(), causing a hang. found_inf_tensor = torch.tensor([found_inf], dtype=torch.float32, device="cuda") - torch.distributed.all_reduce(found_inf_tensor, op=torch.distributed.ReduceOp.MAX) + torch.distributed.all_reduce( + found_inf_tensor, op=torch.distributed.ReduceOp.MAX, group=self.stats_group + ) found_inf = found_inf_tensor.item() > 0 if found_inf: return False, None, None @@ -356,7 +368,11 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: return pg -def get_mimo_optimizer(mimo_model: "MimoModel", config: OptimizerConfig) -> MimoOptimizer: +def get_mimo_optimizer( + mimo_model: "MimoModel", + config: OptimizerConfig, + stats_group: Optional[torch.distributed.ProcessGroup] = None, +) -> MimoOptimizer: """Create optimizer for MimoModel with heterogeneous parallelism.""" from megatron.core.optimizer import get_megatron_optimizer @@ -401,4 +417,4 @@ def get_mimo_optimizer(mimo_model: "MimoModel", config: OptimizerConfig) -> Mimo optimizer=optimizer, grid=grid, pg_collection=pg_collection, is_active=is_active ) - return MimoOptimizer(module_infos, config) + return MimoOptimizer(module_infos, config, stats_group=stats_group) diff --git a/megatron/core/models/mimo/partition/utils.py b/megatron/core/models/mimo/partition/utils.py index 0b43e5548ff..cbb6d4cba5d 100644 --- a/megatron/core/models/mimo/partition/utils.py +++ b/megatron/core/models/mimo/partition/utils.py @@ -145,6 +145,8 @@ def shard( shard_factor = None seq_dim = None # which dimension holds the token sequence + # MimoModel.forward() passes embeddings in batch-first layout + # [B, S, H], so the token sequence dimension is always 1 here. if self.cfg.use_cp and self.cfg.seq_parallel: shard_factor = get_pg_size(self.cfg.tp_group) * get_pg_size(self.cfg.cp_group) * 2 seq_dim = 1 # embeddings shape: [B, S, H] @@ -153,7 +155,7 @@ def shard( seq_dim = 1 elif self.cfg.seq_parallel: shard_factor = get_pg_size(self.cfg.tp_group) - seq_dim = 0 # embeddings shape: [S, B, H] + seq_dim = 1 if shard_factor is not None and ( packed_seq_params is None @@ -178,7 +180,13 @@ def shard( ) if self.cfg.seq_parallel and embeddings is not None: - embeddings = tensor_parallel.scatter_to_sequence_parallel_region(embeddings) + # GPT/Hybrid output layers gather sequence-parallel hidden states + # before per-token loss, so labels/loss_mask remain full sequence. + embeddings = embeddings.transpose(0, 1).contiguous() + embeddings = tensor_parallel.scatter_to_sequence_parallel_region( + embeddings, group=self.cfg.tp_group + ) + embeddings = embeddings.transpose(0, 1).contiguous() return embeddings, labels, loss_mask, attention_mask, packed_seq_params diff --git a/megatron/core/models/vision/radio.py b/megatron/core/models/vision/radio.py index 5e9525adfee..b7d245da5c1 100644 --- a/megatron/core/models/vision/radio.py +++ b/megatron/core/models/vision/radio.py @@ -126,6 +126,7 @@ def __init__( self.has_cpe = has_cpe # Using non-TE version so we can force gather_output + tp_group = getattr(pg_collection, "tp", None) if pg_collection is not None else None self.embedder = ColumnParallelLinear( input_size=3 * self.patch_dim * self.patch_dim, output_size=self.visual_hidden_size, @@ -133,6 +134,7 @@ def __init__( config=transformer_config, gather_output=True, init_method=lambda tensor: torch.nn.init.normal_(tensor, mean=0.0, std=1.0), + tp_group=tp_group, ) self.model_type = ModelType.encoder_or_decoder diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py index c6d3e41aed5..0eefccfb70a 100644 --- a/megatron/core/optimizer/__init__.py +++ b/megatron/core/optimizer/__init__.py @@ -300,6 +300,7 @@ def _get_param_groups( model_chunks: List[MegatronModule], config: OptimizerConfig, config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], + param_group_sync_group: Optional[torch.distributed.ProcessGroup] = None, ) -> List[Dict]: """Create parameter groups for optimizer. @@ -360,8 +361,12 @@ def _get_param_groups( # so we need to align the param groups across ranks, otherwise we may have # runtime error when loading the checkpoint or numerical error when resuming training. params_key = list(params_map.keys()) - gathered_params_key = [None for _ in range(torch.distributed.get_world_size())] - torch.distributed.all_gather_object(gathered_params_key, params_key) + gathered_params_key = [ + None for _ in range(torch.distributed.get_world_size(group=param_group_sync_group)) + ] + torch.distributed.all_gather_object( + gathered_params_key, params_key, group=param_group_sync_group + ) for keys in gathered_params_key: for key in keys: if key not in params_key: @@ -419,6 +424,7 @@ def _get_param_groups_and_buffers( config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], filter_fn: Callable, buffer_name: str, + param_group_sync_group: Optional[torch.distributed.ProcessGroup] = None, ) -> Tuple[List[Dict], Dict[int, List[_ParamAndGradBuffer]]]: """Returns parameter groups and buffer for optimizer. @@ -437,7 +443,9 @@ def _get_param_groups_and_buffers( Returns: List of parameter groups and dictionary of model chunk IDs to buffers. """ - param_groups = _get_param_groups(model_chunks, config, config_overrides) + param_groups = _get_param_groups( + model_chunks, config, config_overrides, param_group_sync_group=param_group_sync_group + ) param_groups = list(filter(filter_fn, param_groups)) buffers = {} for model_chunk_idx, model_chunk in enumerate(model_chunks): @@ -783,7 +791,10 @@ def _get_megatron_emerging_optimizer( # Build param groups and bucket by (optimizer_name, is_expert_parallel). # Layer-wise distributed optimizer handles expert params internally so we skip that split. - all_param_groups = _get_param_groups(model_chunks, config, config_overrides) + param_group_sync_group = getattr(pg_collection, 'intra_dist_opt', None) + all_param_groups = _get_param_groups( + model_chunks, config, config_overrides, param_group_sync_group=param_group_sync_group + ) grouped_param_groups = defaultdict(list) for group in all_param_groups: opt_name = group.get('optimizer', eopt_name) @@ -926,6 +937,7 @@ def get_megatron_optimizer( intra_dp_cp_group_gloo = process_groups_dict['intra_dp_cp_group_gloo'] intra_expt_dp_group_gloo = process_groups_dict['intra_expt_dp_group_gloo'] intra_dist_opt_group = process_groups_dict['intra_dist_opt_group'] + param_group_sync_group = intra_dist_opt_group model_parallel_rank = get_pg_rank(mp_group) @@ -949,6 +961,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: True, buffer_name='buffers', + param_group_sync_group=param_group_sync_group, ) optimizer_part = _get_megatron_optimizer_based_on_param_groups( @@ -999,6 +1012,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: not g['is_expert_parallel'], buffer_name='buffers', + param_group_sync_group=param_group_sync_group, ) for model_chunk in dense_model_chunks: model_chunk.overlap_param_gather_with_optimizer_step = ( @@ -1036,6 +1050,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: g['is_expert_parallel'], buffer_name='expert_parallel_buffers', + param_group_sync_group=param_group_sync_group, ) if dump_param_to_param_group_map is not None: for param_group in moe_param_groups: diff --git a/megatron/core/tensor_parallel/cross_entropy.py b/megatron/core/tensor_parallel/cross_entropy.py index 27c8f063440..0b43f1ad1b7 100644 --- a/megatron/core/tensor_parallel/cross_entropy.py +++ b/megatron/core/tensor_parallel/cross_entropy.py @@ -1,14 +1,10 @@ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -from typing import Tuple +from typing import Optional, Tuple import torch -from megatron.core.parallel_state import ( - get_tensor_model_parallel_group, - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) +from megatron.core.parallel_state import get_tensor_model_parallel_group from .utils import VocabUtility @@ -121,21 +117,20 @@ def calculate_gradients( class _VocabParallelCrossEntropy(torch.autograd.Function): @staticmethod - def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0): + def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0, tp_group=None): """Vocab parallel cross entropy forward function.""" vocab_parallel_logits, logits_max = VocabParallelCrossEntropy.calculate_logits_max( vocab_parallel_logits ) - torch.distributed.all_reduce( - logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group() - ) + tp_group = get_tensor_model_parallel_group() if tp_group is None else tp_group + torch.distributed.all_reduce(logits_max, op=torch.distributed.ReduceOp.MAX, group=tp_group) # Get the partition's vocab indices get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size partition_vocab_size = vocab_parallel_logits.size()[-1] - rank = get_tensor_model_parallel_rank() - world_size = get_tensor_model_parallel_world_size() + rank = torch.distributed.get_rank(tp_group) + world_size = torch.distributed.get_world_size(tp_group) vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size) (target_mask, masked_target_1d, predicted_logits, sum_exp_logits, exp_logits) = ( @@ -146,15 +141,11 @@ def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0): # All reduce is needed to get the chunks from other GPUs. torch.distributed.all_reduce( - predicted_logits, - op=torch.distributed.ReduceOp.SUM, - group=get_tensor_model_parallel_group(), + predicted_logits, op=torch.distributed.ReduceOp.SUM, group=tp_group ) torch.distributed.all_reduce( - sum_exp_logits, - op=torch.distributed.ReduceOp.SUM, - group=get_tensor_model_parallel_group(), + sum_exp_logits, op=torch.distributed.ReduceOp.SUM, group=tp_group ) exp_logits, loss = VocabParallelCrossEntropy.calculate_cross_entropy_loss( @@ -213,10 +204,15 @@ def backward(ctx, grad_output): grad_2d, arange_1d, masked_target_1d, softmax_update, grad_input, grad_output ) - return grad_input, None, None + return grad_input, None, None, None -def vocab_parallel_cross_entropy(vocab_parallel_logits, target, label_smoothing=0.0): +def vocab_parallel_cross_entropy( + vocab_parallel_logits, + target, + label_smoothing=0.0, + tp_group: Optional[torch.distributed.ProcessGroup] = None, +): """ Performs cross entropy loss when logits are split across tensor parallel ranks @@ -228,5 +224,10 @@ def vocab_parallel_cross_entropy(vocab_parallel_logits, target, label_smoothing= label_smoothing: smoothing factor, must be in range [0.0, 1.0) default is no smoothing (=0.0) + + tp_group: tensor-parallel process group. Defaults to Megatron's global tensor-parallel + group for backward compatibility. """ - return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing) + return _VocabParallelCrossEntropy.apply( + vocab_parallel_logits, target, label_smoothing, tp_group + ) diff --git a/megatron/core/tokenizers/vision/libraries/multimodal_tokenizer.py b/megatron/core/tokenizers/vision/libraries/multimodal_tokenizer.py index 80712351095..c71a503553b 100644 --- a/megatron/core/tokenizers/vision/libraries/multimodal_tokenizer.py +++ b/megatron/core/tokenizers/vision/libraries/multimodal_tokenizer.py @@ -85,12 +85,16 @@ def __init__( pretrained_model_name_or_path=path, **kwargs ) + # Some tokenizers, including the Nemotron6-MoE tokenizer used by the + # VLM recipe, already contain . Re-adding such a token returns + # 0, so validate that each requested token resolves instead. + tokenizer.add_tokens(special_tokens, special_tokens=True) self._vocab_size = len(tokenizer) - - num_added_tokens = tokenizer.add_tokens(special_tokens, special_tokens=True) - assert num_added_tokens == len( - special_tokens - ), f"failed to add {len(special_tokens)} special tokens; only added {num_added_tokens}" + for token in special_tokens: + token_id = tokenizer.convert_tokens_to_ids(token) + assert ( + token_id is not None and token_id != tokenizer.unk_token_id + ), f"special token {token!r} could not be resolved (got id={token_id})" self.tokenizer = tokenizer @@ -181,6 +185,14 @@ def __init__( has_bos=True, has_system_role=True, ) + elif prompt_format == "nemotron6-moe": + self._prompt_config = PromptConfig( + assistant_prefix_len=None, + pad_token_id=tokenizer.convert_tokens_to_ids(""), + custom_chat_template=None, + has_bos=False, + has_system_role=True, + ) else: raise NotImplementedError("unknown multimodal tokenizer type", prompt_format) diff --git a/megatron/core/transformer/moe/moe_layer.py b/megatron/core/transformer/moe/moe_layer.py index a64afee719f..8a01fc031b5 100644 --- a/megatron/core/transformer/moe/moe_layer.py +++ b/megatron/core/transformer/moe/moe_layer.py @@ -8,7 +8,7 @@ import torch -from megatron.core import parallel_state, tensor_parallel, utils +from megatron.core import tensor_parallel, utils from megatron.core.extensions.transformer_engine import HAVE_TE from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.module import MegatronModule @@ -468,7 +468,7 @@ def shared_experts_compute(self, hidden_states: torch.Tensor): apply_module(self.shared_experts), False, tensor_parallel.random.get_cuda_rng_tracker, - parallel_state.get_tensor_model_parallel_group(), + self.tp_group, hidden_states, ) else: @@ -621,7 +621,7 @@ def custom_forward(hidden_states, intermediate_tensors=None, padding_mask=None): custom_forward, False, tensor_parallel.random.get_cuda_rng_tracker, - parallel_state.get_tensor_model_parallel_group(), + self.tp_group, hidden_states, intermediate_tensors, padding_mask, diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index f258f3474ae..acdbb04d916 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1161,7 +1161,10 @@ def track_moe_metrics( def get_updated_expert_bias( - tokens_per_expert: torch.Tensor, expert_bias: torch.Tensor, expert_bias_update_rate: float + tokens_per_expert: torch.Tensor, + expert_bias: torch.Tensor, + expert_bias_update_rate: float, + tp_dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, ) -> torch.Tensor: """Update expert bias for biased expert routing. See https://arxiv.org/abs/2408.15664v1# @@ -1175,11 +1178,12 @@ def get_updated_expert_bias( """ with torch.no_grad(): # All Reduce Across TPxCPxDP group - torch.distributed.all_reduce( - tokens_per_expert, + if tp_dp_cp_group is None: # TODO(Hepteract): delete the usage of the global parallel_state. - group=parallel_state.get_tensor_and_data_parallel_group(with_context_parallel=True), - ) + tp_dp_cp_group = parallel_state.get_tensor_and_data_parallel_group( + with_context_parallel=True + ) + torch.distributed.all_reduce(tokens_per_expert, group=tp_dp_cp_group) average_tokens = tokens_per_expert.sum(dim=-1, keepdim=True) / tokens_per_expert.shape[-1] offset = average_tokens - tokens_per_expert updated_expert_bias = expert_bias + torch.sign(offset) * expert_bias_update_rate diff --git a/megatron/core/transformer/moe/shared_experts.py b/megatron/core/transformer/moe/shared_experts.py index 61ea47955b8..a565e2ec718 100644 --- a/megatron/core/transformer/moe/shared_experts.py +++ b/megatron/core/transformer/moe/shared_experts.py @@ -229,10 +229,12 @@ def pre_forward_comm(self, input, wait_current_stream=True): self.gate_score = torch.nn.functional.sigmoid(logits) if self.config.sequence_parallel: self.cached_fc1_input = gather_from_sequence_parallel_region( - input, tensor_parallel_output_grad=True + input, tensor_parallel_output_grad=True, group=self.tp_group ) else: - self.cached_fc1_input = copy_to_tensor_model_parallel_region(input) + self.cached_fc1_input = copy_to_tensor_model_parallel_region( + input, group=self.tp_group + ) set_tensor_grad_fn_sequence_sr(self.cached_fc1_input, torch.iinfo(torch.int).max) @overlap_state_check( @@ -321,11 +323,11 @@ def post_forward_comm(self): with torch.cuda.stream(self.stream): if self.config.sequence_parallel: self.cached_output = reduce_scatter_to_sequence_parallel_region( - self.cached_fc2_output + self.cached_fc2_output, group=self.tp_group ) else: self.cached_output = reduce_from_tensor_model_parallel_region( - self.cached_fc2_output + self.cached_fc2_output, group=self.tp_group ) self.cached_fc2_output = None set_tensor_grad_fn_sequence_sr(self.cached_output, torch.iinfo(torch.int).max) diff --git a/tests/unit_tests/models/test_mimo_partition.py b/tests/unit_tests/models/test_mimo_partition.py index 1527fb92935..23fb2ef7ed3 100644 --- a/tests/unit_tests/models/test_mimo_partition.py +++ b/tests/unit_tests/models/test_mimo_partition.py @@ -195,8 +195,7 @@ def test_sp_only_scatters(self): mock_tp_group = MagicMock() cfg = self._make_cfg(seq_parallel=True, max_seq_len=8, tp_group=mock_tp_group) adapter = PartitionAdapter(cfg) - # SP uses seq_dim=0: embeddings shape [S, B, H] - embeddings = torch.rand(8, 2, 16) + embeddings = torch.rand(2, 8, 16) labels = torch.randint(0, 100, (2, 8)) loss_mask = torch.ones(2, 8) attention_mask = torch.ones(2, 8) @@ -209,7 +208,23 @@ def test_sp_only_scatters(self): ), ): out = adapter.shard(embeddings, labels, loss_mask, attention_mask) - assert out[0].shape == (4, 2, 16) + assert out[0].shape == (2, 4, 16) + assert out[1] is labels + assert out[2] is loss_mask + + def test_sp_only_leaves_labels_and_loss_mask_without_embeddings(self): + mock_tp_group = MagicMock() + cfg = self._make_cfg(seq_parallel=True, max_seq_len=8, tp_group=mock_tp_group) + adapter = PartitionAdapter(cfg) + labels = torch.arange(16).view(2, 8) + loss_mask = torch.arange(16, dtype=torch.float32).view(2, 8) + attention_mask = torch.ones(2, 8) + with patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2): + out = adapter.shard(None, labels, loss_mask, attention_mask) + assert out[0] is None + assert out[1] is labels + assert out[2] is loss_mask + assert out[3] is attention_mask def test_cp_and_sp_combined(self): mock_cp_group = MagicMock() @@ -233,7 +248,7 @@ def test_cp_and_sp_combined(self): 'loss_mask': loss_mask[:, :8], 'attention_mask': attention_mask[:, :8], } - scattered = torch.rand(2, 4, 16) + scattered = torch.rand(4, 2, 16) with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), @@ -248,6 +263,8 @@ def test_cp_and_sp_combined(self): ): out = adapter.shard(embeddings, labels, loss_mask, attention_mask) assert out[0].shape == (2, 4, 16) + torch.testing.assert_close(out[1], labels[:, :8]) + torch.testing.assert_close(out[2], loss_mask[:, :8]) def test_seq_not_divisible_raises(self): mock_cp_group = MagicMock() @@ -270,7 +287,7 @@ def test_tp_comm_overlap_seq_len_assertion(self): ) adapter = PartitionAdapter(cfg) # S=8 but max_seq_len=16 → assertion fires - embeddings = torch.rand(8, 2, 16) # [S, B, H] for SP + embeddings = torch.rand(2, 8, 16) labels = torch.randint(0, 100, (2, 8)) loss_mask = torch.ones(2, 8) attention_mask = torch.ones(2, 8) From 7cfb2da851f1adb7c25df0a6ab457c581b686fec Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sun, 10 May 2026 17:48:21 +0000 Subject: [PATCH 04/44] NMFW-464 modularize hetero MIMO training loop --- examples/mimo/data/hetero_mock.py | 127 ++ examples/mimo/model_providers/hetero_vlm.py | 476 ++++++ examples/mimo/train_hetero.py | 1572 +----------------- examples/mimo/training/__init__.py | 2 + examples/mimo/training/hetero/__init__.py | 1 + examples/mimo/training/hetero/args.py | 232 +++ examples/mimo/training/hetero/distributed.py | 54 + examples/mimo/training/hetero/logging.py | 157 ++ examples/mimo/training/hetero/loop.py | 153 ++ examples/mimo/training/hetero/runtime.py | 201 +++ examples/mimo/training/hetero/scheduler.py | 44 + examples/mimo/training/hetero/step.py | 207 +++ examples/mimo/training/hetero/topology.py | 329 ++++ examples/mimo/utils/hetero.py | 57 + 14 files changed, 2050 insertions(+), 1562 deletions(-) create mode 100644 examples/mimo/data/hetero_mock.py create mode 100644 examples/mimo/model_providers/hetero_vlm.py create mode 100644 examples/mimo/training/__init__.py create mode 100644 examples/mimo/training/hetero/__init__.py create mode 100644 examples/mimo/training/hetero/args.py create mode 100644 examples/mimo/training/hetero/distributed.py create mode 100644 examples/mimo/training/hetero/logging.py create mode 100644 examples/mimo/training/hetero/loop.py create mode 100644 examples/mimo/training/hetero/runtime.py create mode 100644 examples/mimo/training/hetero/scheduler.py create mode 100644 examples/mimo/training/hetero/step.py create mode 100644 examples/mimo/training/hetero/topology.py create mode 100644 examples/mimo/utils/hetero.py diff --git a/examples/mimo/data/hetero_mock.py b/examples/mimo/data/hetero_mock.py new file mode 100644 index 00000000000..997310bd231 --- /dev/null +++ b/examples/mimo/data/hetero_mock.py @@ -0,0 +1,127 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Mock VLM data provider for heterogeneous MIMO training examples.""" + +from __future__ import annotations + +import argparse + +import torch + +from examples.mimo.utils.hetero import ( + MOCK_VISION_ENCODER_KEY, + NEMOTRON_VISION_ENCODER_KEY, + debug_rank, + is_nemotron_20l, +) + + +class MockVLMIterator: + """Infinite iterator yielding synthetic VLM-like microbatches.""" + + def __init__( + self, args: argparse.Namespace, micro_batch_size: int, encoder_name: str, seed: int + ) -> None: + self.args = args + self.micro_batch_size = micro_batch_size + self.encoder_name = encoder_name + self.image_seq_length = args.image_seq_length or args.seq_length // 2 + self.dtype = torch.float32 if args.fp32 else torch.bfloat16 + self.generator = torch.Generator(device="cuda") + self.generator.manual_seed(seed) + if self.image_seq_length >= args.seq_length: + raise ValueError("--image-seq-length must be smaller than --seq-length") + + def __iter__(self): + return self + + def __next__(self): + args = self.args + debug_rank( + f"mock batch start: micro_batch_size={self.micro_batch_size}, " + f"image_seq_length={self.image_seq_length}" + ) + image_tokens = torch.full( + (self.micro_batch_size, self.image_seq_length), + args.image_token_id, + dtype=torch.long, + device="cuda", + ) + text_tokens = torch.randint( + 1, + args.vocab_size, + (self.micro_batch_size, args.seq_length - self.image_seq_length), + device="cuda", + generator=self.generator, + ) + special_token_ids = {args.image_token_id, args.pad_token_id} + replacement_token_id = next( + ( + token_id + for token_id in range(1, args.vocab_size) + if token_id not in special_token_ids + ), + None, + ) + if replacement_token_id is None: + raise RuntimeError("mock data needs at least one non-special token id") + if 1 <= args.image_token_id < args.vocab_size: + text_tokens[text_tokens == args.image_token_id] = replacement_token_id + if 1 <= args.pad_token_id < args.vocab_size: + text_tokens[text_tokens == args.pad_token_id] = replacement_token_id + input_ids = torch.cat([image_tokens, text_tokens], dim=1) + + labels = torch.full_like(input_ids, -100) + labels[:, :-1] = input_ids[:, 1:] + labels[(labels == args.image_token_id) | (labels == args.pad_token_id)] = -100 + loss_mask = (labels != -100).to(dtype=torch.float32) + + if is_nemotron_20l(args): + encoder_inputs = { + NEMOTRON_VISION_ENCODER_KEY: { + "x": torch.randn( + self.micro_batch_size * args.num_image_tiles, + 3, + args.img_h, + args.img_w, + device="cuda", + dtype=self.dtype, + generator=self.generator, + ) + } + } + else: + encoder_hidden_states = torch.randn( + self.image_seq_length, + self.micro_batch_size, + args.hidden_size, + device="cuda", + dtype=self.dtype, + generator=self.generator, + ) + encoder_inputs = { + MOCK_VISION_ENCODER_KEY: { + "hidden_states": encoder_hidden_states, + "attention_mask": None, + } + } + + num_image_placeholders = (input_ids == args.image_token_id).sum().item() + expected_image_placeholders = self.image_seq_length * self.micro_batch_size + if num_image_placeholders != expected_image_placeholders: + raise RuntimeError( + f"mock batch has {num_image_placeholders} image placeholders, " + f"expected {expected_image_placeholders}" + ) + + debug_rank("mock batch ready") + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "position_ids": torch.arange(args.seq_length, device="cuda") + .unsqueeze(0) + .expand(self.micro_batch_size, -1) + .clone(), + "modality_inputs": {self.encoder_name: {**encoder_inputs}}, + } diff --git a/examples/mimo/model_providers/hetero_vlm.py b/examples/mimo/model_providers/hetero_vlm.py new file mode 100644 index 00000000000..9c4704ab9c3 --- /dev/null +++ b/examples/mimo/model_providers/hetero_vlm.py @@ -0,0 +1,476 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Model provider helpers for heterogeneous MIMO VLM examples.""" + +from __future__ import annotations + +import argparse +from contextlib import nullcontext +from typing import Optional + +import torch + +from megatron.core.activations import fast_gelu, squared_relu +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec +from megatron.core.models.mamba.mamba_model import MambaModel +from megatron.core.models.mimo.submodules.vision import VisionModalitySubmodules +from megatron.core.models.multimodal.llava_model import pixel_shuffle +from megatron.core.models.vision.multimodal_projector import MultimodalProjector +from megatron.core.models.vision.radio import RADIOViTModel +from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.enums import AttnBackend +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.utils import sharded_state_dict_default + +from examples.mimo.utils.hetero import ( + MOCK_VISION_ENCODER_KEY, + NEMOTRON_20L_HYBRID_PATTERN, + NEMOTRON_VISION_ENCODER_KEY, + debug_rank, + get_grid_dim_size, + get_group_rank_or, + get_group_size_or, + is_nemotron_20l, + is_process_group_member, +) + +try: + from megatron.core.extensions.transformer_engine import ( + TEColumnParallelLinear, + TELayerNormColumnParallelLinear, + TERowParallelLinear, + ) +except ImportError: + TEColumnParallelLinear = None + TELayerNormColumnParallelLinear = None + TERowParallelLinear = None + + +class RADIOEncoderWrapper(torch.nn.Module): + """RADIO encoder wrapper matching the Nemotron6-MoE VLM provider.""" + + def __init__( + self, + transformer_config: TransformerConfig, + transformer_layer_spec: ModuleSpec, + pg_collection: Optional[ProcessGroupCollection], + img_h: int, + img_w: int, + patch_dim: int, + class_token_len: int, + drop_class_token: bool = True, + apply_pixel_shuffle: bool = True, + force_eval_mode: bool = False, + ) -> None: + super().__init__() + self.class_token_len = class_token_len + self.drop_class_token = drop_class_token + self.apply_pixel_shuffle = apply_pixel_shuffle + self.force_eval_mode = force_eval_mode + self.radio_model = RADIOViTModel( + transformer_config=transformer_config, + transformer_layer_spec=transformer_layer_spec, + patch_dim=patch_dim, + img_h=img_h, + img_w=img_w, + class_token_len=class_token_len, + add_class_token=True, + max_img_h=2048, + max_img_w=2048, + has_cpe=True, + embedder_bias=False, + pg_collection=pg_collection, + ) + if self.force_eval_mode: + self.radio_model.eval() + + def train(self, mode: bool = True): + """Keep frozen RADIO in eval mode while allowing the projection to train.""" + super().train(mode) + if self.force_eval_mode: + self.radio_model.eval() + return self + + @property + def config(self): + """Expose the underlying RADIO config for DDP wrapping.""" + return self.radio_model.config + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Run RADIO, drop class tokens, and apply pixel shuffle.""" + context = torch.no_grad() if self.force_eval_mode else nullcontext() + debug_rank(f"RADIO forward start: input_shape={tuple(x.shape)}") + with context: + x = x.to(dtype=self.radio_model.embedder.weight.dtype) + embeddings = self.radio_model(x) + debug_rank(f"RADIO forward done: output_shape={tuple(embeddings.shape)}") + if self.drop_class_token: + embeddings = embeddings[:, self.class_token_len :, :] + debug_rank(f"RADIO class tokens dropped: output_shape={tuple(embeddings.shape)}") + if self.apply_pixel_shuffle: + embeddings = pixel_shuffle(embeddings, scale_factor=0.5) + debug_rank(f"RADIO pixel shuffle done: output_shape={tuple(embeddings.shape)}") + return embeddings + + def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): + """Delegate checkpoint sharding to the wrapped RADIO model.""" + sharded_sd = {} + for name, child in self.named_children(): + sharded_sd.update( + sharded_state_dict_default(child, f"{prefix}{name}.", sharded_offsets, metadata) + ) + return sharded_sd + + +def get_encoder_module_name(args: argparse.Namespace) -> str: + """Return the concrete encoder key for the active vision provider.""" + return NEMOTRON_VISION_ENCODER_KEY if is_nemotron_20l(args) else MOCK_VISION_ENCODER_KEY + + +def get_vision_encoder_module(args: argparse.Namespace, vision_submodule): + """Return the provider-owned encoder module used for DDP config and freezing.""" + return vision_submodule.encoders[get_encoder_module_name(args)] + + +def iter_vision_projection_modules(vision_submodule): + """Return the provider-owned projection modules used for freeze-stage policy.""" + return iter(vision_submodule.input_projections) + + +def projection_layer_spec() -> ModuleSpec: + """Return the TE-backed projection MLP spec.""" + if TEColumnParallelLinear is None or TERowParallelLinear is None: + raise RuntimeError("TEColumnParallelLinear and TERowParallelLinear are required") + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules(linear_fc1=TEColumnParallelLinear, linear_fc2=TERowParallelLinear), + ) + + +def nemotron_projection_layer_spec() -> ModuleSpec: + """Return the Nemotron VLM RADIO-to-language projector layer spec.""" + if TELayerNormColumnParallelLinear is None or TERowParallelLinear is None: + raise RuntimeError("TELayerNormColumnParallelLinear and TERowParallelLinear are required") + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear + ), + ) + + +def nemotron_language_config( + args: argparse.Namespace, tp_size: int, pp_size: int, ep_size: int, expt_tp_size: int +) -> TransformerConfig: + """Build the exact Nemotron6-MoE 20L language TransformerConfig.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=20, + hidden_size=2688, + num_attention_heads=32, + attention_backend=AttnBackend.flash, + num_query_groups=2, + ffn_hidden_size=1856, + kv_channels=128, + activation_func=squared_relu, + gated_linear_unit=False, + attention_dropout=0.0, + hidden_dropout=0.0, + normalization="RMSNorm", + add_bias_linear=False, + init_method_std=0.0173, + use_cpu_initialization=True, + variable_seq_lengths=True, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + expert_model_parallel_size=ep_size, + expert_tensor_parallel_size=expt_tp_size, + sequence_parallel=tp_size > 1, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + calculate_per_token_loss=True, + bias_activation_fusion=False, + masked_softmax_fusion=True, + persist_layer_norm=True, + bias_dropout_fusion=False, + recompute_granularity="selective", + recompute_modules=["core_attn"], + moe_ffn_hidden_size=1856, + num_moe_experts=128, + moe_router_topk=6, + moe_grouped_gemm=True, + moe_router_score_function="sigmoid", + moe_router_topk_scaling_factor=2.5, + moe_router_enable_expert_bias=True, + moe_router_dtype="fp32", + moe_router_load_balancing_type="seq_aux_loss", + moe_aux_loss_coeff=0.0001, + moe_shared_expert_intermediate_size=3712, + moe_shared_expert_overlap=True, + moe_token_dispatcher_type="alltoall", + moe_permute_fusion=True, + use_fused_weighted_squared_relu=True, + is_hybrid_model=True, + mamba_num_heads=64, + mamba_head_dim=64, + ) + config.position_embedding_type = "none" + config.seq_length = 8192 + config.max_position_embeddings = 8192 + return config + + +def require_per_token_loss(config: TransformerConfig) -> None: + """The hetero MIMO loop scales both language and vision grads by real LM tokens.""" + if not config.calculate_per_token_loss: + raise ValueError("train_hetero.py requires calculate_per_token_loss=True") + + +def radio_vision_config(args: argparse.Namespace, tp_size: int, pp_size: int) -> TransformerConfig: + """Build the exact RADIO vision TransformerConfig from the 20L reference provider.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=32, + hidden_size=1280, + num_attention_heads=16, + use_cpu_initialization=True, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + ) + config.kv_channels = 80 + config.num_query_groups = 16 + config.ffn_hidden_size = 5120 + config.gated_linear_unit = False + config.activation_func = fast_gelu + config.add_bias_linear = True + config.add_qkv_bias = True + config.normalization = "LayerNorm" + config.layernorm_epsilon = 1.0e-6 + config.layernorm_zero_centered_gamma = False + config.apply_rope_fusion = False + config.qk_layernorm = False + config.bias_activation_fusion = False + config.bias_dropout_fusion = False + config.attention_softmax_in_fp32 = True + config.attention_dropout = 0.0 + config.hidden_dropout = 0.0 + return config + + +def nemotron_projection_config(args: argparse.Namespace, tp_size: int) -> TransformerConfig: + """Build the exact RADIO-to-Nemotron projection config.""" + bf16 = not args.fp32 + dtype = torch.bfloat16 if bf16 else torch.float32 + config = TransformerConfig( + num_layers=1, + hidden_size=2688, + num_attention_heads=1, + use_cpu_initialization=True, + params_dtype=dtype, + pipeline_dtype=dtype, + bf16=bf16, + ) + config.tensor_model_parallel_size = tp_size + config.ffn_hidden_size = 4 * 5120 + config.bias_activation_fusion = False + config.bias_dropout_fusion = False + config.add_bias_linear = False + config.activation_func = squared_relu + config.normalization = "RMSNorm" + return config + + +def language_model_spec( + args: argparse.Namespace, + pg_collection: Optional[ProcessGroupCollection], + llm_grid: HyperCommGrid, +) -> ModuleSpec: + """Create the language ModuleSpec for the local language grid.""" + pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None + tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None + ep_pg = getattr(pg_collection, "ep", None) if pg_collection is not None else None + expt_tp_pg = getattr(pg_collection, "expt_tp", None) if pg_collection is not None else None + + fallback_tp_size = get_grid_dim_size(llm_grid, "tp") + pp_rank = get_group_rank_or(pp_pg) + pp_size = get_group_size_or(pp_pg, get_grid_dim_size(llm_grid, "pp")) + tp_size = get_group_size_or(tp_pg, fallback_tp_size) + ep_size = get_group_size_or(ep_pg, args.llm_ep) + expt_tp_size = get_group_size_or(expt_tp_pg, args.llm_expt_tp or fallback_tp_size) + if is_nemotron_20l(args): + config = nemotron_language_config(args, tp_size, pp_size, ep_size, expt_tp_size) + require_per_token_loss(config) + return ModuleSpec( + module=MambaModel, + params={ + "config": config, + "mamba_stack_spec": mamba_stack_spec, + "vocab_size": args.vocab_size, + "max_sequence_length": args.seq_length, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + "hybrid_override_pattern": NEMOTRON_20L_HYBRID_PATTERN, + "position_embedding_type": "none", + "scatter_embedding_sequence_parallel": False, + "pg_collection": pg_collection, + }, + ) + + num_moe_experts = args.num_moe_experts if args.num_moe_experts > 0 else None + bf16 = not args.fp32 + moe_kwargs = {} + if num_moe_experts is not None: + moe_kwargs = { + "num_moe_experts": num_moe_experts, + "moe_router_topk": args.moe_router_topk, + "moe_router_pre_softmax": args.moe_router_topk == 1, + "expert_model_parallel_size": ep_size, + "expert_tensor_parallel_size": expt_tp_size, + "moe_grouped_gemm": args.moe_grouped_gemm, + } + + config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + use_cpu_initialization=True, + variable_seq_lengths=True, + moe_token_dispatcher_type="alltoall", + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, + bf16=bf16, + calculate_per_token_loss=True, + cross_entropy_loss_fusion=True, + cross_entropy_fusion_impl="te", + **moe_kwargs, + ) + require_per_token_loss(config) + return ModuleSpec( + module=GPTModel, + params={ + "config": config, + "transformer_layer_spec": get_gpt_layer_with_transformer_engine_spec( + num_experts=num_moe_experts, moe_grouped_gemm=args.moe_grouped_gemm + ), + "vocab_size": args.vocab_size, + "max_sequence_length": args.seq_length, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + "pg_collection": pg_collection, + }, + ) + + +def vision_submodules_spec( + args: argparse.Namespace, + pg_collection: Optional[ProcessGroupCollection], + encoder_grid: HyperCommGrid, +) -> ModuleSpec: + """Create the vision ModuleSpec for the local encoder grid.""" + from megatron.core.transformer.transformer_block import TransformerBlock + + pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None + tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None + tp_size = get_group_size_or(tp_pg, get_grid_dim_size(encoder_grid, "tp")) + pp_size = get_group_size_or(pp_pg, get_grid_dim_size(encoder_grid, "pp")) + pp_rank = get_group_rank_or(pp_pg) + bf16 = not args.fp32 + + if is_nemotron_20l(args): + vision_config = radio_vision_config(args, tp_size, pp_size) + vision_encoder_spec = ModuleSpec( + module=RADIOEncoderWrapper, + params={ + "transformer_config": vision_config, + "transformer_layer_spec": get_vit_layer_with_transformer_engine_spec(), + "pg_collection": pg_collection, + "img_h": args.img_h, + "img_w": args.img_w, + "patch_dim": args.patch_dim, + "class_token_len": args.class_token_len, + "drop_class_token": True, + "apply_pixel_shuffle": True, + "force_eval_mode": args.freeze_vit, + }, + ) + vision_projection_spec = ModuleSpec( + module=MultimodalProjector, + params={ + "config": nemotron_projection_config(args, tp_size), + "submodules": nemotron_projection_layer_spec().submodules, + "projector_type": "mlp", + "input_size": 5120, + "tp_group": tp_pg if is_process_group_member(tp_pg) else None, + }, + ) + return ModuleSpec( + module=VisionModalitySubmodules, + params={"pg_collection": pg_collection}, + submodules={ + "encoders": {NEMOTRON_VISION_ENCODER_KEY: vision_encoder_spec}, + "input_projections": [vision_projection_spec], + }, + ) + + vision_config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + num_attention_heads=args.num_attention_heads, + use_cpu_initialization=True, + variable_seq_lengths=True, + moe_token_dispatcher_type="alltoall", + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, + bf16=bf16, + calculate_per_token_loss=True, + ) + vision_encoder_spec = ModuleSpec( + module=TransformerBlock, + params={ + "config": vision_config, + "spec": get_gpt_layer_with_transformer_engine_spec(), + "pg_collection": pg_collection, + "pre_process": pp_rank == 0, + "post_process": pp_rank == pp_size - 1, + }, + ) + + projection_config = TransformerConfig( + num_layers=1, hidden_size=args.hidden_size, num_attention_heads=1 + ) + projection_config.ffn_hidden_size = args.hidden_size + projection_config.activation_func = torch.nn.functional.gelu + + vision_projection_spec = ModuleSpec( + module=MultimodalProjector, + params={ + "config": projection_config, + "submodules": projection_layer_spec().submodules, + "projector_type": "mlp", + "input_size": vision_config.hidden_size, + "tp_group": tp_pg if is_process_group_member(tp_pg) else None, + }, + ) + + return ModuleSpec( + module=VisionModalitySubmodules, + params={"pg_collection": pg_collection}, + submodules={ + "encoders": {MOCK_VISION_ENCODER_KEY: vision_encoder_spec}, + "input_projections": [vision_projection_spec], + }, + ) diff --git a/examples/mimo/train_hetero.py b/examples/mimo/train_hetero.py index 743e16933cc..7eb4c81da41 100644 --- a/examples/mimo/train_hetero.py +++ b/examples/mimo/train_hetero.py @@ -1,1568 +1,23 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Standalone heterogeneous MIMO mock training loop. +"""Standalone heterogeneous MIMO mock training entrypoint.""" -This entrypoint is intentionally separate from examples/mimo/train.py. The -standard Megatron pretrain path owns a single homogeneous model-parallel -topology, while this loop owns one HyperCommGrid per MIMO module and wires the -multi-module pipeline schedule directly. -""" - -import argparse import os import sys -from contextlib import ExitStack, contextmanager, nullcontext -from functools import partial -from typing import Optional + +import torch.distributed as dist _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) -import torch -import torch.distributed as dist - -import megatron.core.pipeline_parallel.schedules as schedule -from megatron.core import parallel_state -from megatron.core.activations import fast_gelu, squared_relu -from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig -from megatron.core.distributed.finalize_model_grads import finalize_model_grads -from megatron.core.hyper_comm_grid import HyperCommGrid -from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec -from megatron.core.models.gpt.gpt_model import GPTModel -from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec -from megatron.core.models.mamba.mamba_model import MambaModel -from megatron.core.models.mimo.config.base_configs import MimoModelConfig -from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY -from megatron.core.models.mimo.model.base import MimoModel -from megatron.core.models.mimo.optimizer import get_mimo_optimizer -from megatron.core.models.mimo.submodules.vision import VisionModalitySubmodules -from megatron.core.models.multimodal.llava_model import pixel_shuffle -from megatron.core.models.vision.multimodal_projector import MultimodalProjector -from megatron.core.models.vision.radio import RADIOViTModel -from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec -from megatron.core.optimizer.optimizer_config import OptimizerConfig -from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler -from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator -from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator -from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage -from megatron.core.process_groups_config import ( - MultiModuleProcessGroupCollection, - ProcessGroupCollection, +from examples.mimo.training.hetero.args import parse_args +from examples.mimo.training.hetero.distributed import ( + initialize_distributed, + print_rank_0, + shutdown_distributed, ) -from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed -from megatron.core.transformer.enums import AttnBackend -from megatron.core.transformer.mlp import MLP, MLPSubmodules -from megatron.core.transformer.spec_utils import ModuleSpec -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.transformer.utils import sharded_state_dict_default - -try: - from megatron.core.extensions.transformer_engine import ( - TEColumnParallelLinear, - TELayerNormColumnParallelLinear, - TERowParallelLinear, - ) -except ImportError: - TEColumnParallelLinear = None - TELayerNormColumnParallelLinear = None - TERowParallelLinear = None - - -MOCK_MODEL_PRESET = "mock" -NEMOTRON_20L_MODEL_PRESET = "nemotron-moe-vlm-20l" -NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" -NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 -NEMOTRON_20L_MAX_NUM_TILES = 12 -NEMOTRON_20L_DEFAULT_STAGE = "stage2" - -_ACTIVE_GRIDS: list[HyperCommGrid] = [] -_EMBEDDING_PG_CACHE: dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] = {} - - -def parse_args() -> argparse.Namespace: - """Parse standalone hetero MIMO loop arguments.""" - parser = argparse.ArgumentParser(description=__doc__) - - grid = parser.add_argument_group("module grids") - grid.add_argument("--encoder-offset", type=int, default=0) - grid.add_argument("--encoder-tp", type=int, default=2) - grid.add_argument("--encoder-cp", type=int, default=1) - grid.add_argument("--encoder-pp", type=int, default=2) - grid.add_argument("--encoder-dp", type=int, default=1) - grid.add_argument("--encoder-ep", type=int, default=1) - grid.add_argument("--encoder-expt-tp", type=int, default=None) - grid.add_argument("--encoder-expt-dp", type=int, default=None) - grid.add_argument("--llm-offset", type=int, default=4) - grid.add_argument("--llm-tp", type=int, default=1) - grid.add_argument("--llm-cp", type=int, default=1) - grid.add_argument("--llm-pp", type=int, default=2) - grid.add_argument("--llm-dp", type=int, default=2) - grid.add_argument("--llm-ep", type=int, default=2) - grid.add_argument("--llm-expt-tp", type=int, default=1) - grid.add_argument("--llm-expt-dp", type=int, default=1) - - model = parser.add_argument_group("model") - model.add_argument( - "--model-preset", - choices=[MOCK_MODEL_PRESET, NEMOTRON_20L_MODEL_PRESET], - default=MOCK_MODEL_PRESET, - help="Model config preset. The Nemotron preset matches the 20L reference script.", - ) - model.add_argument("--hidden-size", type=int, default=128) - model.add_argument("--num-layers", type=int, default=2) - model.add_argument("--num-attention-heads", type=int, default=8) - model.add_argument("--vocab-size", type=int, default=512) - model.add_argument("--seq-length", type=int, default=32) - model.add_argument("--image-seq-length", type=int, default=None) - model.add_argument("--image-token-id", type=int, default=511) - model.add_argument("--pad-token-id", type=int, default=0) - model.add_argument("--image-token", type=str, default="") - model.add_argument("--tokenizer-model", type=str, default=None) - model.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") - model.add_argument("--image-tag-type", type=str, default="") - model.add_argument("--force-system-message", action="store_true") - model.add_argument("--num-moe-experts", type=int, default=4) - model.add_argument("--moe-router-topk", type=int, default=1) - model.add_argument("--moe-grouped-gemm", action="store_true") - model.add_argument("--img-h", type=int, default=512) - model.add_argument("--img-w", type=int, default=512) - model.add_argument("--patch-dim", type=int, default=16) - model.add_argument("--class-token-len", type=int, default=8) - model.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) - model.add_argument("--freeze-lm", action="store_true", help="Freeze language model params") - model.add_argument("--freeze-vit", action="store_true", help="Freeze vision encoder params") - model.add_argument( - "--freeze-projection", action="store_true", help="Freeze vision projection params" - ) - model.add_argument( - "--training-stage", - choices=["stage1", "stage2", "stage3"], - default=None, - help="Nemotron VLM freeze stage. Defaults to stage2 for the 20L preset.", - ) - model.add_argument( - "--fp32", action="store_true", help="Build and train in fp32 instead of bf16" - ) - - train = parser.add_argument_group("training") - train.add_argument("--micro-batch-size", type=int, default=2) - train.add_argument("--global-batch-size", type=int, default=None) - train.add_argument("--num-microbatches", type=int, default=2) - train.add_argument("--train-iters", type=int, default=2) - train.add_argument("--lr", type=float, default=1.0e-4) - train.add_argument("--min-lr", type=float, default=None) - train.add_argument("--lr-decay-style", type=str, default="constant") - train.add_argument("--lr-warmup-iters", type=int, default=0) - train.add_argument("--lr-decay-iters", type=int, default=None) - train.add_argument("--weight-decay", type=float, default=0.01) - train.add_argument("--adam-beta1", type=float, default=0.9) - train.add_argument("--adam-beta2", type=float, default=0.999) - train.add_argument("--clip-grad", type=float, default=1.0) - train.add_argument( - "--overlap-grad-reduce", - action=argparse.BooleanOptionalAction, - default=True, - help="Enable DDP gradient-reduce overlap. Disable for parity with the 20L reference script.", - ) - train.add_argument( - "--ddp-bucket-size", - type=int, - default=10000, - help="DDP bucket size. Use 0 for a single unbounded bucket.", - ) - train.add_argument("--seed", type=int, default=12345) - train.add_argument("--log-interval", type=int, default=1) - - return parser.parse_args() - - -def clear_transformer_engine_env() -> None: - """Clear attention backend overrides that can conflict with GPTModel construction.""" - os.environ.pop("NVTE_FLASH_ATTN", None) - os.environ.pop("NVTE_FUSED_ATTN", None) - os.environ.pop("NVTE_UNFUSED_ATTN", None) - - -def initialize_distributed() -> None: - """Initialize torch.distributed for torchrun.""" - clear_transformer_engine_env() - os.environ.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") - - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - torch.cuda.set_device(local_rank) - if not dist.is_initialized(): - dist.init_process_group(backend="nccl") - try: - parallel_state.get_global_memory_buffer() - except AssertionError: - parallel_state._set_global_memory_buffer() - dist.barrier() - - -def print_rank_0(message: str) -> None: - """Print only on global rank zero.""" - if not dist.is_initialized() or dist.get_rank() == 0: - sys.stdout.write(f"{message}\n") - sys.stdout.flush() - - -def debug_rank(message: str) -> None: - """Emit per-rank startup checkpoints when MIMO_HETERO_DEBUG is set.""" - if os.environ.get("MIMO_HETERO_DEBUG"): - rank = dist.get_rank() if dist.is_initialized() else 0 - sys.stderr.write(f"[rank {rank}] {message}\n") - sys.stderr.flush() - - -def is_nemotron_20l(args: argparse.Namespace) -> bool: - """Return whether the run should use the Nemotron6-MoE VLM 20L architecture.""" - return args.model_preset == NEMOTRON_20L_MODEL_PRESET - - -def apply_model_preset(args: argparse.Namespace) -> None: - """Apply architecture defaults for the selected model preset.""" - if not is_nemotron_20l(args): - return - - args.num_layers = 20 - args.hidden_size = 2688 - args.num_attention_heads = 32 - args.num_moe_experts = 128 - args.moe_router_topk = 6 - args.moe_grouped_gemm = True - args.seq_length = 8192 - args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles - - -def apply_training_stage(args: argparse.Namespace) -> None: - """Apply the reference Nemotron VLM freeze stage defaults.""" - if not is_nemotron_20l(args): - return - - stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE - if stage == "stage1": - args.freeze_vit = True - args.freeze_lm = True - elif stage == "stage2": - args.freeze_vit = True - elif stage != "stage3": - raise ValueError(f"unsupported Nemotron VLM training stage: {stage}") - args.training_stage = stage - - -def resolve_image_token_id(args: argparse.Namespace) -> None: - """Resolve the image token id from the reference MultimodalTokenizer when provided.""" - if not is_nemotron_20l(args) or not args.tokenizer_model: - return - - from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( - MegatronMultimodalTokenizer, - ) - - tokenizer = MegatronMultimodalTokenizer( - path=args.tokenizer_model, - prompt_format=args.tokenizer_prompt_format, - special_tokens=[args.image_token], - image_tag_type=args.image_tag_type, - force_system_message=args.force_system_message, - ) - image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) - if image_token_id is None: - raise RuntimeError( - f"tokenizer at {args.tokenizer_model} did not produce an id for {args.image_token}" - ) - args.image_token_id = int(image_token_id) - if tokenizer.pad is not None: - args.pad_token_id = int(tokenizer.pad) - if tokenizer.vocab_size is not None: - args.vocab_size = int(tokenizer.vocab_size) - - -def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: - """Validate the Phase 2 non-colocated 1F1B mock-training layout.""" - if args.encoder_cp != 1 or args.llm_cp != 1: - raise ValueError("Phase 2 mock training currently supports CP=1 only") - if args.hidden_size % args.num_attention_heads != 0: - raise ValueError("--hidden-size must be divisible by --num-attention-heads") - if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: - raise ValueError("--num-moe-experts must be divisible by --llm-ep") - if args.log_interval < 1: - raise ValueError("--log-interval must be >= 1") - if not 0 <= args.image_token_id < args.vocab_size: - raise ValueError("--image-token-id must be within --vocab-size") - if not 0 <= args.pad_token_id < args.vocab_size: - raise ValueError("--pad-token-id must be within --vocab-size") - - image_seq_length = args.image_seq_length or args.seq_length // 2 - if image_seq_length >= args.seq_length: - raise ValueError("--image-seq-length must be smaller than --seq-length") - if args.seq_length - image_seq_length < 2: - raise ValueError("mock next-token training needs at least two text tokens") - if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: - raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") - - encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp - llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp - encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) - llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) - all_ranks = set(range(world_size)) - - if not encoder_ranks.isdisjoint(llm_ranks): - raise ValueError( - "Phase 2 train_hetero.py supports non-colocated 1F1B only; " - f"module rank spans overlap at {sorted(encoder_ranks & llm_ranks)}" - ) - if encoder_ranks | llm_ranks != all_ranks: - raise ValueError( - "The non-colocated module grids must cover every torchrun rank exactly once; " - f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" - ) - - return encoder_size, llm_size - - -def is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: - """Return whether pg is a real process group for this rank.""" - group_member = getattr(dist, "GroupMember", None) - non_member = getattr(group_member, "NON_GROUP_MEMBER", None) - return pg is not None and pg != non_member - - -def destroy_process_group_if_member(pg: Optional[dist.ProcessGroup]) -> None: - """Destroy pg when this rank owns a process-group handle.""" - if is_process_group_member(pg): - dist.destroy_process_group(pg) - - -def create_hypercomm_grid( - offset: int, - tp: int, - cp: int, - pp: int, - dp: int, - ep: int, - expt_tp: Optional[int], - expt_dp: Optional[int], -) -> HyperCommGrid: - """Create a dense grid plus expert layout and required process groups.""" - expt_tp = tp if expt_tp is None else expt_tp - module_world_size = tp * cp * pp * dp - expert_model_size = expt_tp * ep * pp - if module_world_size % expert_model_size != 0: - raise ValueError( - f"module_world_size ({module_world_size}) must be divisible by " - f"expt_tp*ep*pp ({expert_model_size})" - ) - if expt_dp is None: - expt_dp = module_world_size // expert_model_size - if expt_tp * ep * expt_dp * pp != module_world_size: - raise ValueError( - f"expt_tp*ep*expt_dp*pp ({expt_tp * ep * expt_dp * pp}) must equal " - f"module_world_size ({module_world_size})" - ) - - grid = HyperCommGrid( - shape=[tp, cp, dp, pp], - dim_names=["tp", "cp", "dp", "pp"], - rank_offset=offset, - backend="nccl", - ) - grid.register_layout( - "expert", - [expt_tp, ep, expt_dp, pp], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, - ) - - for dims in ( - ["tp"], - ["cp"], - ["pp"], - ["dp"], - ["dp", "cp"], - ["tp", "cp"], - ["ep"], - ["expt_tp"], - ["expt_dp"], - ["tp", "pp"], - ["tp", "cp", "dp"], - ["tp", "cp", "pp", "dp"], - "tp_ep", - "tp_ep_pp", - ): - grid.create_pg(dims) - - _ACTIVE_GRIDS.append(grid) - return grid - - -def destroy_runtime_process_groups() -> None: - """Destroy process groups created by this script.""" - destroyed_embedding_pgs = set() - for pos_embd_pg, embd_pg in _EMBEDDING_PG_CACHE.values(): - for pg in (pos_embd_pg, embd_pg): - if id(pg) in destroyed_embedding_pgs: - continue - destroy_process_group_if_member(pg) - destroyed_embedding_pgs.add(id(pg)) - _EMBEDDING_PG_CACHE.clear() - - for grid in _ACTIVE_GRIDS: - grid.destroy() - _ACTIVE_GRIDS.clear() - BridgeCommunicator.destroy_broadcast_pgs() - - -def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: - """Build a ProcessGroupCollection from a populated HyperCommGrid.""" - return ProcessGroupCollection.from_hyper_comm_grid( - grid, - required_pgs=[ - "tp", - "cp", - "pp", - "dp", - "dp_cp", - "tp_cp", - "mp", - "tp_dp_cp", - "ep", - "expt_tp", - "expt_dp", - "tp_ep", - "tp_ep_pp", - "intra_dist_opt", - ], - ) - - -def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: - """Create PP-derived embedding groups in a consistent global order.""" - pp_rank_sets: list[tuple[int, ...]] = [] - seen_pp_rank_sets = set() - for grid in sorted(grids, key=lambda candidate: (candidate.rank_offset, candidate.size)): - for pp_ranks in grid.get_rank_enum("pp"): - pp_rank_tuple = tuple(pp_ranks) - if pp_rank_tuple in seen_pp_rank_sets: - continue - pp_rank_sets.append(pp_rank_tuple) - seen_pp_rank_sets.add(pp_rank_tuple) - - for pp_ranks in pp_rank_sets: - if pp_ranks not in _EMBEDDING_PG_CACHE: - pos_embd_ranks = [pp_ranks[0]] - embd_ranks = [pp_ranks[0]] - if pp_ranks[-1] != pp_ranks[0]: - embd_ranks.append(pp_ranks[-1]) - _EMBEDDING_PG_CACHE[pp_ranks] = ( - dist.new_group(ranks=pos_embd_ranks), - dist.new_group(ranks=embd_ranks), - ) - - -def add_embedding_groups( - pg_collection: ProcessGroupCollection, is_language_model: bool = False -) -> ProcessGroupCollection: - """Attach cached embedding process groups to a ProcessGroupCollection.""" - if not is_process_group_member(getattr(pg_collection, "pp", None)): - return pg_collection - - pp_ranks = tuple(dist.get_process_group_ranks(pg_collection.pp)) - pos_embd_pg, embd_pg = _EMBEDDING_PG_CACHE[pp_ranks] - - pg_collection.pos_embd = pos_embd_pg if is_pp_first_stage(pg_collection.pp) else None - if is_language_model: - pg_collection.embd = ( - embd_pg - if is_pp_last_stage(pg_collection.pp) or is_pp_first_stage(pg_collection.pp) - else None - ) - else: - pg_collection.embd = None - - return pg_collection - - -def get_pg_collection_with_embedding_groups( - grid: HyperCommGrid, is_language_model: bool = False -) -> ProcessGroupCollection: - """Build a ProcessGroupCollection and add PP-derived embedding groups.""" - return add_embedding_groups(get_pg_collection(grid), is_language_model=is_language_model) - - -def is_rank_in_grid(grid: HyperCommGrid) -> bool: - """Return whether this global rank is inside a grid's rank span.""" - rank = dist.get_rank() - return grid.rank_offset <= rank < grid.rank_offset + grid.size - - -def get_grid_dim_size(grid: HyperCommGrid, dim: str) -> int: - """Return a base-layout dimension size.""" - return grid.shape[grid.dim_names.index(dim)] - - -def get_group_size_or(pg: Optional[dist.ProcessGroup], fallback: int) -> int: - """Return pg size on member ranks, otherwise fallback.""" - return pg.size() if is_process_group_member(pg) else fallback - - -def get_group_rank_or(pg: Optional[dist.ProcessGroup], fallback: int = 0) -> int: - """Return rank inside pg on member ranks, otherwise fallback.""" - return dist.get_rank(pg) if is_process_group_member(pg) else fallback - - -def set_module_requires_grad(module: Optional[torch.nn.Module], requires_grad: bool) -> None: - """Set requires_grad for every parameter in a module when the module exists.""" - if module is None: - return - for param in module.parameters(): - param.requires_grad = requires_grad - - -def set_model_init_seed( - args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_offset: int -): - """Seed CPU model init consistently across TP/DP peers for one module role.""" - pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) - torch.manual_seed(args.seed + role_offset + (100 * pp_rank)) - - -def initialize_model_parallel_rng(args: argparse.Namespace, pg_collection: ProcessGroupCollection): - """Initialize CUDA RNG tracker using the active module's hetero process groups.""" - pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) - tp_rank = get_group_rank_or(getattr(pg_collection, "tp", None)) - ep_rank = get_group_rank_or(getattr(pg_collection, "ep", None)) - expt_tp_rank = get_group_rank_or(getattr(pg_collection, "expt_tp", None)) - model_parallel_cuda_manual_seed( - args.seed + (100 * pp_rank), - tp_rank=tp_rank, - ep_rank=ep_rank, - etp_rank=expt_tp_rank, - force_reset_rng=True, - ) - - -def active_ddp_modules(mimo_model: MimoModel) -> list[DistributedDataParallel]: - """Return active DDP-wrapped submodules owned by this rank.""" - modules = [] - if isinstance(mimo_model.language_model, DistributedDataParallel): - modules.append(mimo_model.language_model) - modules.extend( - submodule - for submodule in mimo_model.modality_submodules.values() - if isinstance(submodule, DistributedDataParallel) - ) - return modules - - -def broadcast_active_params(mimo_model: MimoModel) -> None: - """Synchronize initial parameters across each module's DP groups.""" - for module in active_ddp_modules(mimo_model): - module.broadcast_params() - - -def zero_active_grad_buffers(mimo_model: MimoModel) -> None: - """Clear MCore DDP grad buffers before each training iteration.""" - for module in active_ddp_modules(mimo_model): - module.zero_grad_buffer() - - -def get_grid_coordinate(grid: HyperCommGrid, dim: str) -> int: - """Return this rank's coordinate for a base-layout dimension.""" - if not is_rank_in_grid(grid): - return 0 - - local_rank = dist.get_rank() - grid.rank_offset - coordinates = {} - for dim_name, dim_size in zip(grid.dim_names, grid.shape): - coordinates[dim_name] = local_rank % dim_size - local_rank //= dim_size - return coordinates[dim] - - -def get_mock_data_seed( - args: argparse.Namespace, grid: HyperCommGrid, module_seed_offset: int -) -> int: - """Seed mock data by data-parallel lane so PP/TP stages see coherent batches.""" - dp_lane = get_grid_coordinate(grid, "dp") if "dp" in grid.dim_names else 0 - return args.seed + module_seed_offset + dp_lane - - -def build_no_sync_func(mimo_model: MimoModel): - """Build a no_sync context spanning all active MIMO submodules.""" - - @contextmanager - def no_sync_func(): - with ExitStack() as stack: - if mimo_model.language_model is not None: - stack.enter_context(mimo_model.language_model.no_sync()) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - stack.enter_context(submodule.no_sync()) - yield - - return no_sync_func - - -class RADIOEncoderWrapper(torch.nn.Module): - """RADIO encoder wrapper matching the Nemotron6-MoE VLM provider.""" - - def __init__( - self, - transformer_config: TransformerConfig, - transformer_layer_spec: ModuleSpec, - pg_collection: Optional[ProcessGroupCollection], - img_h: int, - img_w: int, - patch_dim: int, - class_token_len: int, - drop_class_token: bool = True, - apply_pixel_shuffle: bool = True, - force_eval_mode: bool = False, - ) -> None: - super().__init__() - self.class_token_len = class_token_len - self.drop_class_token = drop_class_token - self.apply_pixel_shuffle = apply_pixel_shuffle - self.force_eval_mode = force_eval_mode - self.radio_model = RADIOViTModel( - transformer_config=transformer_config, - transformer_layer_spec=transformer_layer_spec, - patch_dim=patch_dim, - img_h=img_h, - img_w=img_w, - class_token_len=class_token_len, - add_class_token=True, - max_img_h=2048, - max_img_w=2048, - has_cpe=True, - embedder_bias=False, - pg_collection=pg_collection, - ) - if self.force_eval_mode: - self.radio_model.eval() - - def train(self, mode: bool = True): - """Keep frozen RADIO in eval mode while allowing the projection to train.""" - super().train(mode) - if self.force_eval_mode: - self.radio_model.eval() - return self - - @property - def config(self): - """Expose the underlying RADIO config for DDP wrapping.""" - return self.radio_model.config - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Run RADIO, drop class tokens, and apply pixel shuffle.""" - context = torch.no_grad() if self.force_eval_mode else nullcontext() - debug_rank(f"RADIO forward start: input_shape={tuple(x.shape)}") - with context: - x = x.to(dtype=self.radio_model.embedder.weight.dtype) - embeddings = self.radio_model(x) - debug_rank(f"RADIO forward done: output_shape={tuple(embeddings.shape)}") - if self.drop_class_token: - embeddings = embeddings[:, self.class_token_len :, :] - debug_rank(f"RADIO class tokens dropped: output_shape={tuple(embeddings.shape)}") - if self.apply_pixel_shuffle: - embeddings = pixel_shuffle(embeddings, scale_factor=0.5) - debug_rank(f"RADIO pixel shuffle done: output_shape={tuple(embeddings.shape)}") - return embeddings - - def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): - """Delegate checkpoint sharding to the wrapped RADIO model.""" - sharded_sd = {} - for name, child in self.named_children(): - sharded_sd.update( - sharded_state_dict_default(child, f"{prefix}{name}.", sharded_offsets, metadata) - ) - return sharded_sd - - -def projection_layer_spec() -> ModuleSpec: - """Return the TE-backed projection MLP spec.""" - if TEColumnParallelLinear is None or TERowParallelLinear is None: - raise RuntimeError("TEColumnParallelLinear and TERowParallelLinear are required") - return ModuleSpec( - module=MLP, - submodules=MLPSubmodules(linear_fc1=TEColumnParallelLinear, linear_fc2=TERowParallelLinear), - ) - - -def nemotron_projection_layer_spec() -> ModuleSpec: - """Return the Nemotron VLM RADIO-to-language projector layer spec.""" - if TELayerNormColumnParallelLinear is None or TERowParallelLinear is None: - raise RuntimeError("TELayerNormColumnParallelLinear and TERowParallelLinear are required") - return ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear - ), - ) - - -def nemotron_language_config( - args: argparse.Namespace, tp_size: int, pp_size: int, ep_size: int, expt_tp_size: int -) -> TransformerConfig: - """Build the exact Nemotron6-MoE 20L language TransformerConfig.""" - bf16 = not args.fp32 - dtype = torch.bfloat16 if bf16 else torch.float32 - config = TransformerConfig( - num_layers=20, - hidden_size=2688, - num_attention_heads=32, - attention_backend=AttnBackend.flash, - num_query_groups=2, - ffn_hidden_size=1856, - kv_channels=128, - activation_func=squared_relu, - gated_linear_unit=False, - attention_dropout=0.0, - hidden_dropout=0.0, - normalization="RMSNorm", - add_bias_linear=False, - init_method_std=0.0173, - use_cpu_initialization=True, - variable_seq_lengths=True, - tensor_model_parallel_size=tp_size, - pipeline_model_parallel_size=pp_size, - expert_model_parallel_size=ep_size, - expert_tensor_parallel_size=expt_tp_size, - sequence_parallel=tp_size > 1, - params_dtype=dtype, - pipeline_dtype=dtype, - bf16=bf16, - calculate_per_token_loss=True, - bias_activation_fusion=False, - masked_softmax_fusion=True, - persist_layer_norm=True, - bias_dropout_fusion=False, - recompute_granularity="selective", - recompute_modules=["core_attn"], - moe_ffn_hidden_size=1856, - num_moe_experts=128, - moe_router_topk=6, - moe_grouped_gemm=True, - moe_router_score_function="sigmoid", - moe_router_topk_scaling_factor=2.5, - moe_router_enable_expert_bias=True, - moe_router_dtype="fp32", - moe_router_load_balancing_type="seq_aux_loss", - moe_aux_loss_coeff=0.0001, - moe_shared_expert_intermediate_size=3712, - moe_shared_expert_overlap=True, - moe_token_dispatcher_type="alltoall", - moe_permute_fusion=True, - use_fused_weighted_squared_relu=True, - is_hybrid_model=True, - mamba_num_heads=64, - mamba_head_dim=64, - ) - config.position_embedding_type = "none" - config.seq_length = 8192 - config.max_position_embeddings = 8192 - return config - - -def require_per_token_loss(config: TransformerConfig) -> None: - """The hetero MIMO loop scales both language and vision grads by real LM tokens.""" - if not config.calculate_per_token_loss: - raise ValueError("train_hetero.py requires calculate_per_token_loss=True") - - -def radio_vision_config(args: argparse.Namespace, tp_size: int, pp_size: int) -> TransformerConfig: - """Build the exact RADIO vision TransformerConfig from the 20L reference provider.""" - bf16 = not args.fp32 - dtype = torch.bfloat16 if bf16 else torch.float32 - config = TransformerConfig( - num_layers=32, - hidden_size=1280, - num_attention_heads=16, - use_cpu_initialization=True, - tensor_model_parallel_size=tp_size, - pipeline_model_parallel_size=pp_size, - params_dtype=dtype, - pipeline_dtype=dtype, - bf16=bf16, - ) - config.kv_channels = 80 - config.num_query_groups = 16 - config.ffn_hidden_size = 5120 - config.gated_linear_unit = False - config.activation_func = fast_gelu - config.add_bias_linear = True - config.add_qkv_bias = True - config.normalization = "LayerNorm" - config.layernorm_epsilon = 1.0e-6 - config.layernorm_zero_centered_gamma = False - config.apply_rope_fusion = False - config.qk_layernorm = False - config.bias_activation_fusion = False - config.bias_dropout_fusion = False - config.attention_softmax_in_fp32 = True - config.attention_dropout = 0.0 - config.hidden_dropout = 0.0 - return config - - -def nemotron_projection_config(args: argparse.Namespace, tp_size: int) -> TransformerConfig: - """Build the exact RADIO-to-Nemotron projection config.""" - bf16 = not args.fp32 - dtype = torch.bfloat16 if bf16 else torch.float32 - config = TransformerConfig( - num_layers=1, - hidden_size=2688, - num_attention_heads=1, - use_cpu_initialization=True, - params_dtype=dtype, - pipeline_dtype=dtype, - bf16=bf16, - ) - config.tensor_model_parallel_size = tp_size - config.ffn_hidden_size = 4 * 5120 - config.bias_activation_fusion = False - config.bias_dropout_fusion = False - config.add_bias_linear = False - config.activation_func = squared_relu - config.normalization = "RMSNorm" - return config - - -def language_model_spec( - args: argparse.Namespace, - pg_collection: Optional[ProcessGroupCollection], - llm_grid: HyperCommGrid, -) -> ModuleSpec: - """Create the language ModuleSpec for the local language grid.""" - pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None - tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None - ep_pg = getattr(pg_collection, "ep", None) if pg_collection is not None else None - expt_tp_pg = getattr(pg_collection, "expt_tp", None) if pg_collection is not None else None - - fallback_tp_size = get_grid_dim_size(llm_grid, "tp") - pp_rank = get_group_rank_or(pp_pg) - pp_size = get_group_size_or(pp_pg, get_grid_dim_size(llm_grid, "pp")) - tp_size = get_group_size_or(tp_pg, fallback_tp_size) - ep_size = get_group_size_or(ep_pg, args.llm_ep) - expt_tp_size = get_group_size_or(expt_tp_pg, args.llm_expt_tp or fallback_tp_size) - if is_nemotron_20l(args): - config = nemotron_language_config(args, tp_size, pp_size, ep_size, expt_tp_size) - require_per_token_loss(config) - return ModuleSpec( - module=MambaModel, - params={ - "config": config, - "mamba_stack_spec": mamba_stack_spec, - "vocab_size": args.vocab_size, - "max_sequence_length": args.seq_length, - "pre_process": pp_rank == 0, - "post_process": pp_rank == pp_size - 1, - "hybrid_override_pattern": NEMOTRON_20L_HYBRID_PATTERN, - "position_embedding_type": "none", - "scatter_embedding_sequence_parallel": False, - "pg_collection": pg_collection, - }, - ) - - num_moe_experts = args.num_moe_experts if args.num_moe_experts > 0 else None - bf16 = not args.fp32 - - moe_kwargs = {} - if num_moe_experts is not None: - moe_kwargs = { - "num_moe_experts": num_moe_experts, - "moe_router_topk": args.moe_router_topk, - "moe_router_pre_softmax": args.moe_router_topk == 1, - "expert_model_parallel_size": ep_size, - "expert_tensor_parallel_size": expt_tp_size, - "moe_grouped_gemm": args.moe_grouped_gemm, - } - - config = TransformerConfig( - num_layers=args.num_layers, - hidden_size=args.hidden_size, - num_attention_heads=args.num_attention_heads, - use_cpu_initialization=True, - variable_seq_lengths=True, - moe_token_dispatcher_type="alltoall", - tensor_model_parallel_size=tp_size, - pipeline_model_parallel_size=pp_size, - pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, - bf16=bf16, - calculate_per_token_loss=True, - cross_entropy_loss_fusion=True, - cross_entropy_fusion_impl="te", - **moe_kwargs, - ) - require_per_token_loss(config) - return ModuleSpec( - module=GPTModel, - params={ - "config": config, - "transformer_layer_spec": get_gpt_layer_with_transformer_engine_spec( - num_experts=num_moe_experts, moe_grouped_gemm=args.moe_grouped_gemm - ), - "vocab_size": args.vocab_size, - "max_sequence_length": args.seq_length, - "pre_process": pp_rank == 0, - "post_process": pp_rank == pp_size - 1, - "pg_collection": pg_collection, - }, - ) - - -def vision_submodules_spec( - args: argparse.Namespace, - pg_collection: Optional[ProcessGroupCollection], - encoder_grid: HyperCommGrid, -) -> ModuleSpec: - """Create the vision ModuleSpec for the local encoder grid.""" - from megatron.core.transformer.transformer_block import TransformerBlock - - pp_pg = getattr(pg_collection, "pp", None) if pg_collection is not None else None - tp_pg = getattr(pg_collection, "tp", None) if pg_collection is not None else None - tp_size = get_group_size_or(tp_pg, get_grid_dim_size(encoder_grid, "tp")) - pp_size = get_group_size_or(pp_pg, get_grid_dim_size(encoder_grid, "pp")) - pp_rank = get_group_rank_or(pp_pg) - bf16 = not args.fp32 - - if is_nemotron_20l(args): - vision_config = radio_vision_config(args, tp_size, pp_size) - vision_encoder_spec = ModuleSpec( - module=RADIOEncoderWrapper, - params={ - "transformer_config": vision_config, - "transformer_layer_spec": get_vit_layer_with_transformer_engine_spec(), - "pg_collection": pg_collection, - "img_h": args.img_h, - "img_w": args.img_w, - "patch_dim": args.patch_dim, - "class_token_len": args.class_token_len, - "drop_class_token": True, - "apply_pixel_shuffle": True, - "force_eval_mode": args.freeze_vit, - }, - ) - vision_projection_spec = ModuleSpec( - module=MultimodalProjector, - params={ - "config": nemotron_projection_config(args, tp_size), - "submodules": nemotron_projection_layer_spec().submodules, - "projector_type": "mlp", - "input_size": 5120, - "tp_group": tp_pg if is_process_group_member(tp_pg) else None, - }, - ) - return ModuleSpec( - module=VisionModalitySubmodules, - params={"pg_collection": pg_collection}, - submodules={ - "encoders": {"radio_encoder": vision_encoder_spec}, - "input_projections": [vision_projection_spec], - }, - ) - - vision_config = TransformerConfig( - num_layers=args.num_layers, - hidden_size=args.hidden_size, - num_attention_heads=args.num_attention_heads, - use_cpu_initialization=True, - variable_seq_lengths=True, - moe_token_dispatcher_type="alltoall", - tensor_model_parallel_size=tp_size, - pipeline_model_parallel_size=pp_size, - pipeline_dtype=torch.bfloat16 if bf16 else torch.float32, - bf16=bf16, - calculate_per_token_loss=True, - ) - vision_encoder_spec = ModuleSpec( - module=TransformerBlock, - params={ - "config": vision_config, - "spec": get_gpt_layer_with_transformer_engine_spec(), - "pg_collection": pg_collection, - "pre_process": pp_rank == 0, - "post_process": pp_rank == pp_size - 1, - }, - ) - - projection_config = TransformerConfig( - num_layers=1, hidden_size=args.hidden_size, num_attention_heads=1 - ) - projection_config.ffn_hidden_size = args.hidden_size - projection_config.activation_func = torch.nn.functional.gelu - - vision_projection_spec = ModuleSpec( - module=MultimodalProjector, - params={ - "config": projection_config, - "submodules": projection_layer_spec().submodules, - "projector_type": "mlp", - "input_size": vision_config.hidden_size, - "tp_group": tp_pg if is_process_group_member(tp_pg) else None, - }, - ) - - return ModuleSpec( - module=VisionModalitySubmodules, - params={"pg_collection": pg_collection}, - submodules={ - "encoders": {"clip_encoder": vision_encoder_spec}, - "input_projections": [vision_projection_spec], - }, - ) - - -def build_mimo_model( - args: argparse.Namespace, - encoder_grid: HyperCommGrid, - llm_grid: HyperCommGrid, - encoder_name: str, -): - """Build the MIMO model and wrap active modules in MCore DDP.""" - language_pg = get_pg_collection_with_embedding_groups(llm_grid, is_language_model=True) - vision_pg = get_pg_collection_with_embedding_groups(encoder_grid, is_language_model=False) - rank_in_language_grid = is_rank_in_grid(llm_grid) - rank_in_encoder_grid = is_rank_in_grid(encoder_grid) - debug_rank( - "building model specs " - f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" - ) - if rank_in_language_grid: - set_model_init_seed(args, language_pg, role_offset=20_000) - initialize_model_parallel_rng(args, language_pg) - elif rank_in_encoder_grid: - set_model_init_seed(args, vision_pg, role_offset=10_000) - initialize_model_parallel_rng(args, vision_pg) - - module_to_grid_map = {encoder_name: encoder_grid, MIMO_LANGUAGE_MODULE_KEY: llm_grid} - mimo_config = MimoModelConfig( - language_model_spec=language_model_spec( - args, language_pg if rank_in_language_grid else None, llm_grid - ), - modality_submodules_spec={ - encoder_name: vision_submodules_spec( - args, vision_pg if rank_in_encoder_grid else None, encoder_grid - ) - }, - special_token_ids={encoder_name: args.image_token_id}, - module_to_grid_map=module_to_grid_map, - ) - - debug_rank("constructing MimoModel") - mimo_model = MimoModel( - mimo_config, - cp_group=language_pg.cp if rank_in_language_grid else None, - tp_group=language_pg.tp if rank_in_language_grid else None, - ) - debug_rank("moving MimoModel to cuda") - mimo_model.to(torch.device("cuda")) - if not args.fp32: - mimo_model.to(torch.bfloat16) - debug_rank("MimoModel moved to target dtype/device") - - ddp_config = DistributedDataParallelConfig( - overlap_grad_reduce=args.overlap_grad_reduce, - bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, - use_distributed_optimizer=True, - ) - if mimo_model.language_model is not None: - if args.freeze_lm: - set_module_requires_grad(mimo_model.language_model, False) - debug_rank("wrapping language model in DDP") - mimo_model.language_model = DistributedDataParallel( - config=mimo_model.language_model.config, - ddp_config=ddp_config, - module=mimo_model.language_model, - pg_collection=language_pg, - ) - debug_rank("language model DDP ready") - - if encoder_name in mimo_model.modality_submodules: - submodule = mimo_model.modality_submodules[encoder_name] - if submodule is not None: - encoder_module_name = "radio_encoder" if is_nemotron_20l(args) else "clip_encoder" - if args.freeze_vit: - set_module_requires_grad(submodule.encoders[encoder_module_name], False) - if args.freeze_projection: - for projection in submodule.input_projections: - set_module_requires_grad(projection, False) - debug_rank("wrapping vision submodule in DDP") - mimo_model.modality_submodules[encoder_name] = DistributedDataParallel( - config=submodule.encoders[encoder_module_name].config, - ddp_config=ddp_config, - module=submodule, - pg_collection=vision_pg, - ) - debug_rank("vision submodule DDP ready") - - broadcast_active_params(mimo_model) - return mimo_model, module_to_grid_map, language_pg, vision_pg - - -class MockVLMIterator: - """Infinite iterator yielding synthetic VLM-like microbatches.""" - - def __init__( - self, args: argparse.Namespace, micro_batch_size: int, encoder_name: str, seed: int - ) -> None: - self.args = args - self.micro_batch_size = micro_batch_size - self.encoder_name = encoder_name - self.image_seq_length = args.image_seq_length or args.seq_length // 2 - self.dtype = torch.float32 if args.fp32 else torch.bfloat16 - self.generator = torch.Generator(device="cuda") - self.generator.manual_seed(seed) - if self.image_seq_length >= args.seq_length: - raise ValueError("--image-seq-length must be smaller than --seq-length") - - def __iter__(self): - return self - - def __next__(self): - args = self.args - debug_rank( - f"mock batch start: micro_batch_size={self.micro_batch_size}, " - f"image_seq_length={self.image_seq_length}" - ) - image_tokens = torch.full( - (self.micro_batch_size, self.image_seq_length), - args.image_token_id, - dtype=torch.long, - device="cuda", - ) - text_tokens = torch.randint( - 1, - args.vocab_size, - (self.micro_batch_size, args.seq_length - self.image_seq_length), - device="cuda", - generator=self.generator, - ) - special_token_ids = {args.image_token_id, args.pad_token_id} - replacement_token_id = next( - ( - token_id - for token_id in range(1, args.vocab_size) - if token_id not in special_token_ids - ), - None, - ) - if replacement_token_id is None: - raise RuntimeError("mock data needs at least one non-special token id") - if 1 <= args.image_token_id < args.vocab_size: - text_tokens[text_tokens == args.image_token_id] = replacement_token_id - if 1 <= args.pad_token_id < args.vocab_size: - text_tokens[text_tokens == args.pad_token_id] = replacement_token_id - input_ids = torch.cat([image_tokens, text_tokens], dim=1) - - labels = torch.full_like(input_ids, -100) - labels[:, :-1] = input_ids[:, 1:] - labels[(labels == args.image_token_id) | (labels == args.pad_token_id)] = -100 - loss_mask = (labels != -100).to(dtype=torch.float32) - - if is_nemotron_20l(args): - encoder_inputs = { - "radio_encoder": { - "x": torch.randn( - self.micro_batch_size * args.num_image_tiles, - 3, - args.img_h, - args.img_w, - device="cuda", - dtype=self.dtype, - generator=self.generator, - ) - } - } - else: - encoder_hidden_states = torch.randn( - self.image_seq_length, - self.micro_batch_size, - args.hidden_size, - device="cuda", - dtype=self.dtype, - generator=self.generator, - ) - encoder_inputs = { - "clip_encoder": {"hidden_states": encoder_hidden_states, "attention_mask": None} - } - - num_image_placeholders = (input_ids == args.image_token_id).sum().item() - expected_image_placeholders = self.image_seq_length * self.micro_batch_size - if num_image_placeholders != expected_image_placeholders: - raise RuntimeError( - f"mock batch has {num_image_placeholders} image placeholders, " - f"expected {expected_image_placeholders}" - ) - - debug_rank("mock batch ready") - return { - "input_ids": input_ids, - "labels": labels, - "loss_mask": loss_mask, - "position_ids": torch.arange(args.seq_length, device="cuda") - .unsqueeze(0) - .expand(self.micro_batch_size, -1) - .clone(), - "modality_inputs": {self.encoder_name: {**encoder_inputs}}, - } - - -def wire_training_hooks( - mimo_model: MimoModel, - language_pg: ProcessGroupCollection, - vision_pg: ProcessGroupCollection, - token_count_group: dist.ProcessGroup, -) -> None: - """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" - - def is_token_source_rank() -> bool: - return ( - is_process_group_member(getattr(language_pg, "pp", None)) - and is_process_group_member(getattr(language_pg, "tp", None)) - and is_pp_last_stage(language_pg.pp) - and language_pg.tp.rank() == 0 - ) - - def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): - if num_tokens is None: - raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") - - token_count = torch.zeros(1, dtype=torch.float32, device="cuda") - if is_token_source_rank(): - token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() - dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=token_count_group) - global_num_tokens = token_count.item() - - if mimo_model.language_model is not None: - debug_rank("finalizing language grads") - finalize_model_grads( - [mimo_model.language_model], - num_tokens=None, - pg_collection=language_pg, - force_all_reduce=force_all_reduce, - ) - debug_rank("language grads finalized") - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("finalizing vision grads") - finalize_model_grads( - [submodule], - num_tokens=None, - pg_collection=vision_pg, - force_all_reduce=force_all_reduce, - ) - debug_rank("vision grads finalized") - - if global_num_tokens > 0: - scale = 1.0 / global_num_tokens - if mimo_model.language_model is not None: - debug_rank("scaling language grads") - mimo_model.language_model.scale_gradients(scale) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("scaling vision grads") - submodule.scale_gradients(scale) - - mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) - mimo_model.config.finalize_model_grads_func = finalize_grads_func - mimo_model.config.grad_scale_func = lambda loss: ( - torch.tensor(loss, dtype=torch.float32, device="cuda", requires_grad=True) - if isinstance(loss, (int, float)) - else loss - ) - - -def select_data_iterator( - args: argparse.Namespace, - encoder_grid: HyperCommGrid, - llm_grid: HyperCommGrid, - encoder_name: str, -) -> Optional[MockVLMIterator]: - """Create the per-role data iterator needed by local ranks.""" - llm_mbs = args.micro_batch_size - if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: - raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") - encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp - - encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( - encoder_grid.get_pg("pp") - ) - llm_needs_data = is_rank_in_grid(llm_grid) and ( - is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) - ) - - if encoder_needs_data and not llm_needs_data: - return MockVLMIterator( - args, - encoder_mbs, - encoder_name, - get_mock_data_seed(args, encoder_grid, module_seed_offset=0), - ) - if llm_needs_data and not encoder_needs_data: - return MockVLMIterator( - args, - llm_mbs, - encoder_name, - get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), - ) - if encoder_needs_data and llm_needs_data: - return MockVLMIterator( - args, - llm_mbs, - encoder_name, - get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), - ) - return None - - -def build_schedule_pg_collection( - encoder_name: str, - encoder_grid: HyperCommGrid, - llm_grid: HyperCommGrid, - vision_pg: ProcessGroupCollection, - language_pg: ProcessGroupCollection, -) -> MultiModuleProcessGroupCollection: - """Build the schedule-facing process group collection for this rank.""" - module_pgs = {} - language_model_module_name = None - if is_rank_in_grid(encoder_grid): - module_pgs[encoder_name] = vision_pg - if is_rank_in_grid(llm_grid): - module_pgs[MIMO_LANGUAGE_MODULE_KEY] = language_pg - language_model_module_name = MIMO_LANGUAGE_MODULE_KEY - - return MultiModuleProcessGroupCollection( - module_pgs=module_pgs, language_model_module_name=language_model_module_name - ) - - -def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): - """Return raw loss sum, local token count, and logging tensors.""" - if output_tensor is None: - zero = torch.tensor(0.0, device="cuda", requires_grad=True) - zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} - - if isinstance(output_tensor, dict): - output = output_tensor.get( - MIMO_LANGUAGE_MODULE_KEY, next(iter(output_tensor.values()), None) - ) - else: - output = output_tensor - - if output is None: - zero = torch.tensor(0.0, device="cuda", requires_grad=True) - zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} - - output = output.float() - if loss_mask is None: - raise RuntimeError("train_hetero.py requires a loss_mask for per-token loss") - if output.shape != loss_mask.shape: - raise RuntimeError( - f"loss output shape {tuple(output.shape)} does not match loss_mask shape " - f"{tuple(loss_mask.shape)}; per-token loss cannot be scaled correctly" - ) - - masked = output * loss_mask.float() - num_tokens = loss_mask.float().sum().to(torch.int) - loss_sum = masked.sum() - return ( - loss_sum, - num_tokens, - {"lm loss sum": loss_sum.detach(), "lm tokens": num_tokens.detach()}, - ) - - -def forward_step(data_iterator, model): - """Forward step consumed by the MCore pipeline schedule.""" - batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} - debug_rank("forward_step batch prepared") - debug_rank("forward_step model call start") - output_tensor, loss_mask = model(**batch) - debug_rank("forward_step model call done") - return output_tensor, partial(loss_func, loss_mask) - - -def get_global_batch_size(args: argparse.Namespace) -> int: - """Return the language-side global batch size for scheduler accounting.""" - derived_global_batch_size = args.micro_batch_size * args.num_microbatches * args.llm_dp - if args.global_batch_size is None: - return derived_global_batch_size - if args.global_batch_size != derived_global_batch_size: - raise ValueError( - "--global-batch-size must equal " - "--micro-batch-size * --num-microbatches * --llm-dp in this hetero loop " - f"({derived_global_batch_size}); got {args.global_batch_size}" - ) - return args.global_batch_size - - -def build_optimizer_param_scheduler(args: argparse.Namespace, optimizer) -> OptimizerParamScheduler: - """Build the MCore optimizer parameter scheduler using Megatron train-iters semantics.""" - global_batch_size = get_global_batch_size(args) - lr_decay_iters = args.lr_decay_iters if args.lr_decay_iters is not None else args.train_iters - return OptimizerParamScheduler( - optimizer, - init_lr=0.0, - max_lr=args.lr, - min_lr=args.min_lr if args.min_lr is not None else 0.0, - lr_warmup_steps=args.lr_warmup_iters * global_batch_size, - lr_decay_steps=lr_decay_iters * global_batch_size, - lr_decay_style=args.lr_decay_style, - start_wd=args.weight_decay, - end_wd=args.weight_decay, - wd_incr_steps=args.train_iters * global_batch_size, - wd_incr_style="constant", - use_checkpoint_opt_param_scheduler=False, - override_opt_param_scheduler=True, - ) - - -def run_train_loop(args: argparse.Namespace) -> None: - """Run mock-data heterogeneous MIMO training.""" - apply_model_preset(args) - apply_training_stage(args) - resolve_image_token_id(args) - world_size = dist.get_world_size() - encoder_size, llm_size = validate_args(args, world_size) - - encoder_name = "images" - debug_rank("creating encoder grid") - encoder_grid = create_hypercomm_grid( - offset=args.encoder_offset, - tp=args.encoder_tp, - cp=args.encoder_cp, - pp=args.encoder_pp, - dp=args.encoder_dp, - ep=args.encoder_ep, - expt_tp=args.encoder_expt_tp, - expt_dp=args.encoder_expt_dp, - ) - debug_rank("creating language grid") - llm_grid = create_hypercomm_grid( - offset=args.llm_offset, - tp=args.llm_tp, - cp=args.llm_cp, - pp=args.llm_pp, - dp=args.llm_dp, - ep=args.llm_ep, - expt_tp=args.llm_expt_tp, - expt_dp=args.llm_expt_dp, - ) - debug_rank("creating embedding groups") - create_all_embedding_groups([encoder_grid, llm_grid]) - debug_rank("embedding groups ready") - debug_rank("creating MIMO optimizer stats group") - mimo_optimizer_stats_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") - debug_rank("MIMO optimizer stats group ready") - - torch.manual_seed(args.seed) - debug_rank("building MIMO model") - mimo_model, module_to_grid_map, language_pg, vision_pg = build_mimo_model( - args, encoder_grid, llm_grid, encoder_name - ) - debug_rank("wiring training hooks") - wire_training_hooks(mimo_model, language_pg, vision_pg, mimo_optimizer_stats_group) - - debug_rank("building MIMO optimizer") - optimizer = get_mimo_optimizer( - mimo_model, - OptimizerConfig( - optimizer="adam", - lr=args.lr, - min_lr=args.min_lr, - weight_decay=args.weight_decay, - adam_beta1=args.adam_beta1, - adam_beta2=args.adam_beta2, - clip_grad=args.clip_grad, - bf16=not args.fp32, - use_distributed_optimizer=True, - ), - stats_group=mimo_optimizer_stats_group, - ) - opt_param_scheduler = build_optimizer_param_scheduler(args, optimizer) - debug_rank("MIMO optimizer ready") - debug_rank("building pipeline communicator") - communicator = MultiModulePipelineCommunicator( - module_to_grid_map, - {encoder_name: [MIMO_LANGUAGE_MODULE_KEY], MIMO_LANGUAGE_MODULE_KEY: []}, - mimo_model.config, - dim_mapping={"s": 0, "h": 2, "b": 1}, - module_output_ndim={encoder_name: 2}, - ) - debug_rank("building schedule process groups") - schedule_pg_collection = build_schedule_pg_collection( - encoder_name, encoder_grid, llm_grid, vision_pg, language_pg - ) - debug_rank("selecting data iterator") - data_iterator = select_data_iterator(args, encoder_grid, llm_grid, encoder_name) - debug_rank("training setup ready") - - print_rank_0( - "Starting hetero MIMO mock training: " - f"world_size={world_size}, encoder_size={encoder_size}, llm_size={llm_size}, " - f"train_iters={args.train_iters}" - ) - - try: - for iteration in range(1, args.train_iters + 1): - zero_active_grad_buffers(mimo_model) - optimizer.zero_grad() - debug_rank(f"iteration {iteration}: starting forward/backward schedule") - losses = schedule.forward_backward_pipelining_without_interleaving( - forward_step_func=forward_step, - data_iterator=data_iterator, - model=[mimo_model], - num_microbatches=args.num_microbatches, - seq_length=args.seq_length, - micro_batch_size=args.micro_batch_size, - forward_only=False, - p2p_communicator=communicator, - pg_collection=schedule_pg_collection, - ) - debug_rank(f"iteration {iteration}: schedule complete") - debug_rank(f"iteration {iteration}: optimizer step starting") - success, grad_norm, _ = optimizer.step() - debug_rank(f"iteration {iteration}: optimizer step complete") - if not success: - raise RuntimeError(f"optimizer step failed at iteration {iteration}") - opt_param_scheduler.step(increment=get_global_batch_size(args)) - - if iteration % args.log_interval == 0: - loss_acc = torch.zeros(2, dtype=torch.float32, device="cuda") - is_log_stage = is_process_group_member( - getattr(language_pg, "tp_dp_cp", None) - ) and is_pp_last_stage(language_pg.pp) - if is_log_stage: - if losses and language_pg.tp.rank() == 0: - for loss_dict in losses: - loss_sum = loss_dict.get("lm loss sum") - num_tokens = loss_dict.get("lm tokens") - if isinstance(loss_sum, torch.Tensor): - loss_acc[0] += loss_sum.float() - elif loss_sum is not None: - loss_acc[0] += float(loss_sum) - if isinstance(num_tokens, torch.Tensor): - loss_acc[1] += num_tokens.float() - elif num_tokens is not None: - loss_acc[1] += float(num_tokens) - dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.tp_dp_cp) - loss_value = ( - loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None - ) - language_group_ranks = dist.get_process_group_ranks(language_pg.tp_dp_cp) - if dist.get_rank() == min(language_group_ranks): - sys.stdout.write( - f"iteration {iteration}: loss={loss_value}, grad_norm={grad_norm}\n" - ) - sys.stdout.flush() - finally: - mimo_model.destroy() - destroy_process_group_if_member(mimo_optimizer_stats_group) +from examples.mimo.training.hetero.loop import run_train_loop def main() -> None: @@ -1574,14 +29,7 @@ def main() -> None: dist.barrier() print_rank_0("Heterogeneous MIMO mock training completed") finally: - try: - torch.cuda.synchronize() - except Exception: - pass - destroy_runtime_process_groups() - parallel_state.destroy_global_memory_buffer() - if dist.is_initialized(): - dist.destroy_process_group() + shutdown_distributed() if __name__ == "__main__": diff --git a/examples/mimo/training/__init__.py b/examples/mimo/training/__init__.py new file mode 100644 index 00000000000..3f37e3ee40d --- /dev/null +++ b/examples/mimo/training/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + diff --git a/examples/mimo/training/hetero/__init__.py b/examples/mimo/training/hetero/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/mimo/training/hetero/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py new file mode 100644 index 00000000000..3b4446b62b9 --- /dev/null +++ b/examples/mimo/training/hetero/args.py @@ -0,0 +1,232 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Argument handling for the standalone heterogeneous MIMO training loop.""" + +from __future__ import annotations + +import argparse + +from examples.mimo.utils.hetero import ( + MOCK_MODEL_PRESET, + NEMOTRON_20L_DEFAULT_STAGE, + NEMOTRON_20L_IMAGE_SEQ_PER_TILE, + NEMOTRON_20L_MAX_NUM_TILES, + NEMOTRON_20L_MODEL_PRESET, + is_nemotron_20l, +) + + +def parse_args() -> argparse.Namespace: + """Parse standalone hetero MIMO loop arguments.""" + parser = argparse.ArgumentParser( + description=( + "Standalone heterogeneous MIMO mock training loop. " + "This entrypoint owns one HyperCommGrid per MIMO module." + ) + ) + + grid = parser.add_argument_group("module grids") + grid.add_argument("--encoder-offset", type=int, default=0) + grid.add_argument("--encoder-tp", type=int, default=2) + grid.add_argument("--encoder-cp", type=int, default=1) + grid.add_argument("--encoder-pp", type=int, default=2) + grid.add_argument("--encoder-dp", type=int, default=1) + grid.add_argument("--encoder-ep", type=int, default=1) + grid.add_argument("--encoder-expt-tp", type=int, default=None) + grid.add_argument("--encoder-expt-dp", type=int, default=None) + grid.add_argument("--llm-offset", type=int, default=4) + grid.add_argument("--llm-tp", type=int, default=1) + grid.add_argument("--llm-cp", type=int, default=1) + grid.add_argument("--llm-pp", type=int, default=2) + grid.add_argument("--llm-dp", type=int, default=2) + grid.add_argument("--llm-ep", type=int, default=2) + grid.add_argument("--llm-expt-tp", type=int, default=1) + grid.add_argument("--llm-expt-dp", type=int, default=1) + + model = parser.add_argument_group("model") + model.add_argument( + "--model-preset", + choices=[MOCK_MODEL_PRESET, NEMOTRON_20L_MODEL_PRESET], + default=MOCK_MODEL_PRESET, + help="Model config preset. The Nemotron preset matches the 20L reference script.", + ) + model.add_argument("--hidden-size", type=int, default=128) + model.add_argument("--num-layers", type=int, default=2) + model.add_argument("--num-attention-heads", type=int, default=8) + model.add_argument("--vocab-size", type=int, default=512) + model.add_argument("--seq-length", type=int, default=32) + model.add_argument("--image-seq-length", type=int, default=None) + model.add_argument("--image-token-id", type=int, default=511) + model.add_argument("--pad-token-id", type=int, default=0) + model.add_argument("--image-token", type=str, default="") + model.add_argument("--tokenizer-model", type=str, default=None) + model.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") + model.add_argument("--image-tag-type", type=str, default="") + model.add_argument("--force-system-message", action="store_true") + model.add_argument("--num-moe-experts", type=int, default=4) + model.add_argument("--moe-router-topk", type=int, default=1) + model.add_argument("--moe-grouped-gemm", action="store_true") + model.add_argument("--img-h", type=int, default=512) + model.add_argument("--img-w", type=int, default=512) + model.add_argument("--patch-dim", type=int, default=16) + model.add_argument("--class-token-len", type=int, default=8) + model.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) + model.add_argument("--freeze-lm", action="store_true", help="Freeze language model params") + model.add_argument("--freeze-vit", action="store_true", help="Freeze vision encoder params") + model.add_argument( + "--freeze-projection", action="store_true", help="Freeze vision projection params" + ) + model.add_argument( + "--training-stage", + choices=["stage1", "stage2", "stage3"], + default=None, + help="Nemotron VLM freeze stage. Defaults to stage2 for the 20L preset.", + ) + model.add_argument( + "--fp32", action="store_true", help="Build and train in fp32 instead of bf16" + ) + + train = parser.add_argument_group("training") + train.add_argument("--micro-batch-size", type=int, default=2) + train.add_argument("--global-batch-size", type=int, default=None) + train.add_argument("--num-microbatches", type=int, default=2) + train.add_argument("--train-iters", type=int, default=2) + train.add_argument("--lr", type=float, default=1.0e-4) + train.add_argument("--min-lr", type=float, default=None) + train.add_argument("--lr-decay-style", type=str, default="constant") + train.add_argument("--lr-warmup-iters", type=int, default=0) + train.add_argument("--lr-decay-iters", type=int, default=None) + train.add_argument("--weight-decay", type=float, default=0.01) + train.add_argument("--adam-beta1", type=float, default=0.9) + train.add_argument("--adam-beta2", type=float, default=0.999) + train.add_argument("--clip-grad", type=float, default=1.0) + train.add_argument("--log-num-zeros-in-grad", action="store_true") + train.add_argument( + "--overlap-grad-reduce", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Enable DDP gradient-reduce overlap. Disable for parity with the 20L " + "reference script." + ), + ) + train.add_argument( + "--ddp-bucket-size", + type=int, + default=10000, + help="DDP bucket size. Use 0 for a single unbounded bucket.", + ) + train.add_argument("--seed", type=int, default=12345) + train.add_argument("--log-interval", type=int, default=1) + + return parser.parse_args() + + +def apply_model_preset(args: argparse.Namespace) -> None: + """Apply architecture defaults for the selected model preset.""" + if not is_nemotron_20l(args): + return + + args.num_layers = 20 + args.hidden_size = 2688 + args.num_attention_heads = 32 + args.num_moe_experts = 128 + args.moe_router_topk = 6 + args.moe_grouped_gemm = True + args.seq_length = 8192 + args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles + + +def apply_training_stage(args: argparse.Namespace) -> None: + """Apply the reference Nemotron VLM freeze stage defaults.""" + if not is_nemotron_20l(args): + return + + stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE + if stage == "stage1": + args.freeze_vit = True + args.freeze_lm = True + elif stage == "stage2": + args.freeze_vit = True + elif stage != "stage3": + raise ValueError(f"unsupported Nemotron VLM training stage: {stage}") + args.training_stage = stage + + +def resolve_image_token_id(args: argparse.Namespace) -> None: + """Resolve the image token id from the reference MultimodalTokenizer when provided.""" + if not is_nemotron_20l(args) or not args.tokenizer_model: + return + + from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( + MegatronMultimodalTokenizer, + ) + + tokenizer = MegatronMultimodalTokenizer( + path=args.tokenizer_model, + prompt_format=args.tokenizer_prompt_format, + special_tokens=[args.image_token], + image_tag_type=args.image_tag_type, + force_system_message=args.force_system_message, + ) + image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) + if image_token_id is None: + raise RuntimeError( + f"tokenizer at {args.tokenizer_model} did not produce an id for {args.image_token}" + ) + args.image_token_id = int(image_token_id) + if tokenizer.pad is not None: + args.pad_token_id = int(tokenizer.pad) + if tokenizer.vocab_size is not None: + args.vocab_size = int(tokenizer.vocab_size) + + +def prepare_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: + """Apply presets, resolve runtime args, and validate the hetero layout.""" + apply_model_preset(args) + apply_training_stage(args) + resolve_image_token_id(args) + return validate_args(args, world_size) + + +def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: + """Validate the Phase 2 non-colocated 1F1B mock-training layout.""" + if args.encoder_cp != 1 or args.llm_cp != 1: + raise ValueError("Phase 2 mock training currently supports CP=1 only") + if args.hidden_size % args.num_attention_heads != 0: + raise ValueError("--hidden-size must be divisible by --num-attention-heads") + if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: + raise ValueError("--num-moe-experts must be divisible by --llm-ep") + if args.log_interval < 1: + raise ValueError("--log-interval must be >= 1") + if not 0 <= args.image_token_id < args.vocab_size: + raise ValueError("--image-token-id must be within --vocab-size") + if not 0 <= args.pad_token_id < args.vocab_size: + raise ValueError("--pad-token-id must be within --vocab-size") + + image_seq_length = args.image_seq_length or args.seq_length // 2 + if image_seq_length >= args.seq_length: + raise ValueError("--image-seq-length must be smaller than --seq-length") + if args.seq_length - image_seq_length < 2: + raise ValueError("mock next-token training needs at least two text tokens") + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") + + encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp + llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp + encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + + if not encoder_ranks.isdisjoint(llm_ranks): + raise ValueError( + "Phase 2 train_hetero.py supports non-colocated 1F1B only; " + f"module rank spans overlap at {sorted(encoder_ranks & llm_ranks)}" + ) + if encoder_ranks | llm_ranks != all_ranks: + raise ValueError( + "The non-colocated module grids must cover every torchrun rank exactly once; " + f"covered={sorted(encoder_ranks | llm_ranks)}, world={sorted(all_ranks)}" + ) + + return encoder_size, llm_size diff --git a/examples/mimo/training/hetero/distributed.py b/examples/mimo/training/hetero/distributed.py new file mode 100644 index 00000000000..c5f4e115061 --- /dev/null +++ b/examples/mimo/training/hetero/distributed.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Distributed setup helpers for heterogeneous MIMO examples.""" + +from __future__ import annotations + +import os +import sys + +import torch +import torch.distributed as dist + +from megatron.core import parallel_state + + +def clear_transformer_engine_env() -> None: + """Clear attention backend overrides that can conflict with model construction.""" + os.environ.pop("NVTE_FLASH_ATTN", None) + os.environ.pop("NVTE_FUSED_ATTN", None) + os.environ.pop("NVTE_UNFUSED_ATTN", None) + + +def initialize_distributed() -> None: + """Initialize torch.distributed for torchrun.""" + clear_transformer_engine_env() + os.environ.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + try: + parallel_state.get_global_memory_buffer() + except AssertionError: + parallel_state._set_global_memory_buffer() + dist.barrier() + + +def print_rank_0(message: str) -> None: + """Print only on global rank zero.""" + if not dist.is_initialized() or dist.get_rank() == 0: + sys.stdout.write(f"{message}\n") + sys.stdout.flush() + + +def shutdown_distributed() -> None: + """Tear down process-global Megatron and torch.distributed state.""" + try: + torch.cuda.synchronize() + except Exception: + pass + parallel_state.destroy_global_memory_buffer() + if dist.is_initialized(): + dist.destroy_process_group() diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py new file mode 100644 index 00000000000..a9ce28ea4c7 --- /dev/null +++ b/examples/mimo/training/hetero/logging.py @@ -0,0 +1,157 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Megatron-shaped interval logging for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +import math +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional + +import torch +import torch.distributed as dist + +from megatron.core.optimizer_param_scheduler import get_canonical_lr_for_logging +from megatron.core.pipeline_parallel.utils import is_pp_last_stage + +from examples.mimo.training.hetero.scheduler import get_global_batch_size +from examples.mimo.training.hetero.step import TrainStepResult +from examples.mimo.training.hetero.topology import HeteroTopology +from examples.mimo.utils.hetero import is_process_group_member + + +@dataclass +class HeteroTrainingLogger: + """Accumulate and print interval training metrics.""" + + args: argparse.Namespace + topology: HeteroTopology + consumed_train_samples: int = 0 + advanced_iterations: int = 0 + skipped_iterations: int = 0 + nan_iterations: int = 0 + loss_total: float = 0.0 + loss_count: int = 0 + interval_start: float = field(default_factory=time.time) + + def record_step(self, result: TrainStepResult) -> Optional[float]: + """Update interval state from one train step and return this iteration's loss.""" + self.consumed_train_samples += get_global_batch_size(self.args) + loss_value = reduce_language_loss(result.losses, self.topology) + + if result.skipped_iter: + self.skipped_iterations += result.skipped_iter + if loss_value is not None and not math.isfinite(loss_value): + self.nan_iterations += 1 + return loss_value + + self.advanced_iterations += 1 + if loss_value is not None: + if math.isfinite(loss_value): + self.loss_total += loss_value + self.loss_count += 1 + else: + self.nan_iterations += 1 + return loss_value + + def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: + """Print Megatron-like interval metrics on the language logging rank.""" + if iteration % self.args.log_interval != 0: + return + + elapsed = time.time() - self.interval_start + interval_iters = max(1, self.advanced_iterations + self.skipped_iterations) + elapsed_ms = (elapsed / interval_iters) * 1000.0 + loss_value = self.loss_total / self.loss_count if self.loss_count else None + learning_rate = get_canonical_lr_for_logging(optimizer.param_groups) + learning_rate = reduce_max_optional_float( + learning_rate, self.topology.optimizer_stats_group + ) + grad_norm = reduce_max_optional_float(result.grad_norm, self.topology.optimizer_stats_group) + num_zeros_in_grad = reduce_max_optional_float( + result.num_zeros_in_grad, self.topology.optimizer_stats_group + ) + loss_scale = optimizer.get_loss_scale().item() + + if is_language_log_rank(self.topology): + log_string = f" [{datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')}]" + log_string += " iteration {:8d}/{:8d} |".format(iteration, self.args.train_iters) + log_string += " consumed samples: {:12d} |".format(self.consumed_train_samples) + log_string += " elapsed time per iteration (ms): {:.1f} |".format(elapsed_ms) + if learning_rate is not None: + log_string += f" learning rate: {learning_rate:.6E} |" + log_string += f" global batch size: {get_global_batch_size(self.args):5d} |" + if loss_value is not None: + log_string += f" lm loss: {loss_value:.6E} |" + log_string += f" loss scale: {loss_scale:.1f} |" + if grad_norm is not None: + log_string += f" grad norm: {grad_norm:.3f} |" + if num_zeros_in_grad is not None: + log_string += f" num zeros: {int(num_zeros_in_grad)} |" + log_string += " number of skipped iterations: {:3d} |".format(self.skipped_iterations) + log_string += " number of nan iterations: {:3d} |".format(self.nan_iterations) + sys.stdout.write(f"{log_string}\n") + sys.stdout.flush() + self.reset_interval() + + def reset_interval(self) -> None: + """Reset interval accumulators after a log event.""" + self.advanced_iterations = 0 + self.skipped_iterations = 0 + self.nan_iterations = 0 + self.loss_total = 0.0 + self.loss_count = 0 + self.interval_start = time.time() + + +def reduce_language_loss(losses: list[dict], topology: HeteroTopology) -> Optional[float]: + """Reduce raw loss/token vectors over the language DP/TP/CP logging group.""" + language_pg = topology.language_pg + loss_acc = torch.zeros(2, dtype=torch.float32, device="cuda") + is_log_stage = is_process_group_member(getattr(language_pg, "tp_dp_cp", None)) and ( + is_pp_last_stage(language_pg.pp) + ) + if not is_log_stage: + return None + + if losses and language_pg.tp.rank() == 0: + for loss_dict in losses: + loss_sum = loss_dict.get("lm loss sum") + num_tokens = loss_dict.get("lm tokens") + if isinstance(loss_sum, torch.Tensor): + loss_acc[0] += loss_sum.float() + elif loss_sum is not None: + loss_acc[0] += float(loss_sum) + if isinstance(num_tokens, torch.Tensor): + loss_acc[1] += num_tokens.float() + elif num_tokens is not None: + loss_acc[1] += float(num_tokens) + + dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.tp_dp_cp) + return loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None + + +def is_language_log_rank(topology: HeteroTopology) -> bool: + """Return whether this rank should print language-side training metrics.""" + language_pg = topology.language_pg + if not ( + is_process_group_member(getattr(language_pg, "tp_dp_cp", None)) + and is_pp_last_stage(language_pg.pp) + ): + return False + language_group_ranks = dist.get_process_group_ranks(language_pg.tp_dp_cp) + return dist.get_rank() == min(language_group_ranks) + + +def reduce_max_optional_float(value, group: dist.ProcessGroup) -> Optional[float]: + """Reduce optional scalar stats so the language log rank can see non-local optimizers.""" + if isinstance(value, torch.Tensor): + value = value.item() + local_value = -1.0 if value is None else float(value) + stat = torch.tensor([local_value], dtype=torch.float32, device="cuda") + dist.all_reduce(stat, op=dist.ReduceOp.MAX, group=group) + return None if stat.item() == -1.0 else stat.item() diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py new file mode 100644 index 00000000000..1cb250f3e27 --- /dev/null +++ b/examples/mimo/training/hetero/loop.py @@ -0,0 +1,153 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Top-level orchestration for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +from typing import Optional + +import torch + +from megatron.core.models.mimo.optimizer import get_mimo_optimizer +from megatron.core.optimizer.optimizer_config import OptimizerConfig +from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage + +from examples.mimo.data.hetero_mock import MockVLMIterator +from examples.mimo.training.hetero.args import prepare_args +from examples.mimo.training.hetero.distributed import print_rank_0 +from examples.mimo.training.hetero.logging import HeteroTrainingLogger +from examples.mimo.training.hetero.runtime import HeteroRuntime, build_mimo_runtime +from examples.mimo.training.hetero.scheduler import build_optimizer_param_scheduler +from examples.mimo.training.hetero.step import train_step, wire_training_hooks +from examples.mimo.training.hetero.topology import ( + HeteroTopology, + create_topology, + get_grid_coordinate, + is_rank_in_grid, +) +from examples.mimo.utils.hetero import debug_rank + + +def run_train_loop(args: argparse.Namespace) -> None: + """Run mock-data heterogeneous MIMO training.""" + world_size = torch.distributed.get_world_size() + encoder_size, llm_size = prepare_args(args, world_size) + + topology: Optional[HeteroTopology] = None + runtime: Optional[HeteroRuntime] = None + try: + topology = create_topology(args, encoder_size, llm_size) + + torch.manual_seed(args.seed) + debug_rank("building MIMO model") + runtime = build_mimo_runtime(args, topology) + debug_rank("wiring training hooks") + wire_training_hooks(runtime, topology) + + debug_rank("building MIMO optimizer") + optimizer = build_optimizer(args, runtime, topology) + opt_param_scheduler = build_optimizer_param_scheduler(args, optimizer) + debug_rank("MIMO optimizer ready") + + debug_rank("building pipeline communicator") + communicator = MultiModulePipelineCommunicator( + topology.module_to_grid_map, + topology.module_dependency_map, + runtime.model.config, + dim_mapping={"s": 0, "h": 2, "b": 1}, + module_output_ndim={topology.encoder_name: 2}, + ) + debug_rank("selecting data iterator") + data_iterator = select_data_iterator(args, topology) + logger = HeteroTrainingLogger(args=args, topology=topology) + debug_rank("training setup ready") + + print_rank_0( + "Starting hetero MIMO mock training: " + f"world_size={world_size}, encoder_size={topology.encoder_size}, " + f"llm_size={topology.llm_size}, train_iters={args.train_iters}" + ) + + for iteration in range(1, args.train_iters + 1): + debug_rank(f"iteration {iteration}: train step start") + result = train_step( + args, runtime, topology, optimizer, opt_param_scheduler, communicator, data_iterator + ) + logger.record_step(result) + logger.maybe_log(iteration, optimizer, result) + debug_rank(f"iteration {iteration}: train step complete") + finally: + if runtime is not None: + runtime.destroy() + if topology is not None: + topology.destroy() + + +def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime, topology: HeteroTopology): + """Build the MIMO optimizer for active hetero module optimizers.""" + return get_mimo_optimizer( + runtime.model, + OptimizerConfig( + optimizer="adam", + lr=args.lr, + min_lr=args.min_lr, + weight_decay=args.weight_decay, + adam_beta1=args.adam_beta1, + adam_beta2=args.adam_beta2, + clip_grad=args.clip_grad, + bf16=not args.fp32, + use_distributed_optimizer=True, + log_num_zeros_in_grad=args.log_num_zeros_in_grad, + ), + stats_group=topology.optimizer_stats_group, + ) + + +def select_data_iterator( + args: argparse.Namespace, topology: HeteroTopology +) -> Optional[MockVLMIterator]: + """Create the per-role mock-data iterator needed by local ranks.""" + llm_mbs = args.micro_batch_size + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") + encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp + + encoder_grid = topology.encoder_grid + llm_grid = topology.llm_grid + encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( + encoder_grid.get_pg("pp") + ) + llm_needs_data = is_rank_in_grid(llm_grid) and ( + is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) + ) + + if encoder_needs_data and not llm_needs_data: + return MockVLMIterator( + args, + encoder_mbs, + topology.encoder_name, + get_mock_data_seed(args, encoder_grid, module_seed_offset=0), + ) + if llm_needs_data and not encoder_needs_data: + return MockVLMIterator( + args, + llm_mbs, + topology.encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + if encoder_needs_data and llm_needs_data: + return MockVLMIterator( + args, + llm_mbs, + topology.encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + return None + + +def get_mock_data_seed(args: argparse.Namespace, grid, module_seed_offset: int) -> int: + """Seed mock data by data-parallel lane so PP/TP stages see coherent batches.""" + dp_lane = get_grid_coordinate(grid, "dp") if "dp" in grid.dim_names else 0 + return args.seed + module_seed_offset + dp_lane diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py new file mode 100644 index 00000000000..4848f5a85a4 --- /dev/null +++ b/examples/mimo/training/hetero/runtime.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Model runtime construction for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Optional + +import torch + +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.models.mimo.config.base_configs import MimoModelConfig +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + +from examples.mimo.model_providers.hetero_vlm import ( + get_vision_encoder_module, + iter_vision_projection_modules, + language_model_spec, + vision_submodules_spec, +) +from examples.mimo.training.hetero.topology import HeteroTopology, is_rank_in_grid +from examples.mimo.utils.hetero import debug_rank, get_group_rank_or + + +@dataclass +class HeteroRuntime: + """Runtime-owned model state for a hetero MIMO training run.""" + + model: MimoModel + + def destroy(self) -> None: + """Destroy runtime-owned model communication state.""" + self.model.destroy() + + +def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> HeteroRuntime: + """Build the MIMO model and wrap active modules in MCore DDP.""" + language_pg = topology.language_pg + vision_pg = topology.vision_pg + rank_in_language_grid = is_rank_in_grid(topology.llm_grid) + rank_in_encoder_grid = is_rank_in_grid(topology.encoder_grid) + debug_rank( + "building model specs " + f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" + ) + if rank_in_language_grid: + set_model_init_seed(args, language_pg, role_offset=20_000) + initialize_model_parallel_rng(args, language_pg) + elif rank_in_encoder_grid: + set_model_init_seed(args, vision_pg, role_offset=10_000) + initialize_model_parallel_rng(args, vision_pg) + + mimo_config = MimoModelConfig( + language_model_spec=language_model_spec( + args, language_pg if rank_in_language_grid else None, topology.llm_grid + ), + modality_submodules_spec={ + topology.encoder_name: vision_submodules_spec( + args, vision_pg if rank_in_encoder_grid else None, topology.encoder_grid + ) + }, + special_token_ids={topology.encoder_name: args.image_token_id}, + module_to_grid_map=topology.module_to_grid_map, + ) + + debug_rank("constructing MimoModel") + mimo_model = MimoModel( + mimo_config, + cp_group=language_pg.cp if rank_in_language_grid else None, + tp_group=language_pg.tp if rank_in_language_grid else None, + ) + debug_rank("moving MimoModel to cuda") + mimo_model.to(torch.device("cuda")) + if not args.fp32: + mimo_model.to(torch.bfloat16) + debug_rank("MimoModel moved to target dtype/device") + + wrap_active_modules(args, mimo_model, topology) + broadcast_active_params(mimo_model) + return HeteroRuntime(model=mimo_model) + + +def wrap_active_modules( + args: argparse.Namespace, mimo_model: MimoModel, topology: HeteroTopology +) -> None: + """Freeze and DDP-wrap active local MIMO modules.""" + ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=args.overlap_grad_reduce, + bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, + use_distributed_optimizer=True, + ) + if mimo_model.language_model is not None: + if args.freeze_lm: + set_module_requires_grad(mimo_model.language_model, False) + debug_rank("wrapping language model in DDP") + mimo_model.language_model = DistributedDataParallel( + config=mimo_model.language_model.config, + ddp_config=ddp_config, + module=mimo_model.language_model, + pg_collection=topology.language_pg, + ) + debug_rank("language model DDP ready") + + if topology.encoder_name in mimo_model.modality_submodules: + submodule = mimo_model.modality_submodules[topology.encoder_name] + if submodule is None: + return + + encoder_module = get_vision_encoder_module(args, submodule) + if args.freeze_vit: + set_module_requires_grad(encoder_module, False) + if args.freeze_projection: + for projection in iter_vision_projection_modules(submodule): + set_module_requires_grad(projection, False) + debug_rank("wrapping vision submodule in DDP") + mimo_model.modality_submodules[topology.encoder_name] = DistributedDataParallel( + config=encoder_module.config, + ddp_config=ddp_config, + module=submodule, + pg_collection=topology.vision_pg, + ) + debug_rank("vision submodule DDP ready") + + +def set_module_requires_grad(module: Optional[torch.nn.Module], requires_grad: bool) -> None: + """Set requires_grad for every parameter in a module when the module exists.""" + if module is None: + return + for param in module.parameters(): + param.requires_grad = requires_grad + + +def set_model_init_seed( + args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_offset: int +) -> None: + """Seed CPU model init consistently across TP/DP peers for one module role.""" + pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) + torch.manual_seed(args.seed + role_offset + (100 * pp_rank)) + + +def initialize_model_parallel_rng( + args: argparse.Namespace, pg_collection: ProcessGroupCollection +) -> None: + """Initialize CUDA RNG tracker using the active module's hetero process groups.""" + pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) + tp_rank = get_group_rank_or(getattr(pg_collection, "tp", None)) + ep_rank = get_group_rank_or(getattr(pg_collection, "ep", None)) + expt_tp_rank = get_group_rank_or(getattr(pg_collection, "expt_tp", None)) + model_parallel_cuda_manual_seed( + args.seed + (100 * pp_rank), + tp_rank=tp_rank, + ep_rank=ep_rank, + etp_rank=expt_tp_rank, + force_reset_rng=True, + ) + + +def active_ddp_modules(mimo_model: MimoModel) -> list[DistributedDataParallel]: + """Return active DDP-wrapped submodules owned by this rank.""" + modules = [] + if isinstance(mimo_model.language_model, DistributedDataParallel): + modules.append(mimo_model.language_model) + modules.extend( + submodule + for submodule in mimo_model.modality_submodules.values() + if isinstance(submodule, DistributedDataParallel) + ) + return modules + + +def broadcast_active_params(mimo_model: MimoModel) -> None: + """Synchronize initial parameters across each module's DP groups.""" + for module in active_ddp_modules(mimo_model): + module.broadcast_params() + + +def zero_active_grad_buffers(mimo_model: MimoModel) -> None: + """Clear MCore DDP grad buffers before each training iteration.""" + for module in active_ddp_modules(mimo_model): + module.zero_grad_buffer() + + +def build_no_sync_func(mimo_model: MimoModel): + """Build a no_sync context spanning all active MIMO submodules.""" + + @contextmanager + def no_sync_func(): + with ExitStack() as stack: + if mimo_model.language_model is not None: + stack.enter_context(mimo_model.language_model.no_sync()) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + stack.enter_context(submodule.no_sync()) + yield + + return no_sync_func diff --git a/examples/mimo/training/hetero/scheduler.py b/examples/mimo/training/hetero/scheduler.py new file mode 100644 index 00000000000..80f7e0b2793 --- /dev/null +++ b/examples/mimo/training/hetero/scheduler.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Optimizer scheduler helpers for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse + +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + + +def get_global_batch_size(args: argparse.Namespace) -> int: + """Return the language-side global batch size for scheduler accounting.""" + derived_global_batch_size = args.micro_batch_size * args.num_microbatches * args.llm_dp + if args.global_batch_size is None: + return derived_global_batch_size + if args.global_batch_size != derived_global_batch_size: + raise ValueError( + "--global-batch-size must equal " + "--micro-batch-size * --num-microbatches * --llm-dp in this hetero loop " + f"({derived_global_batch_size}); got {args.global_batch_size}" + ) + return args.global_batch_size + + +def build_optimizer_param_scheduler(args: argparse.Namespace, optimizer) -> OptimizerParamScheduler: + """Build the MCore optimizer parameter scheduler using Megatron train-iters semantics.""" + global_batch_size = get_global_batch_size(args) + lr_decay_iters = args.lr_decay_iters if args.lr_decay_iters is not None else args.train_iters + return OptimizerParamScheduler( + optimizer, + init_lr=0.0, + max_lr=args.lr, + min_lr=args.min_lr if args.min_lr is not None else 0.0, + lr_warmup_steps=args.lr_warmup_iters * global_batch_size, + lr_decay_steps=lr_decay_iters * global_batch_size, + lr_decay_style=args.lr_decay_style, + start_wd=args.weight_decay, + end_wd=args.weight_decay, + wd_incr_steps=args.train_iters * global_batch_size, + wd_incr_style="constant", + use_checkpoint_opt_param_scheduler=False, + override_opt_param_scheduler=True, + ) diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py new file mode 100644 index 00000000000..42a7d5321b9 --- /dev/null +++ b/examples/mimo/training/hetero/step.py @@ -0,0 +1,207 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Forward/backward step behavior for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from functools import partial +from typing import Any, Optional + +import torch +import torch.distributed as dist + +import megatron.core.pipeline_parallel.schedules as schedule +from megatron.core.distributed.finalize_model_grads import finalize_model_grads +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY +from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.utils import is_pp_last_stage + +from examples.mimo.training.hetero.runtime import ( + HeteroRuntime, + build_no_sync_func, + zero_active_grad_buffers, +) +from examples.mimo.training.hetero.scheduler import get_global_batch_size +from examples.mimo.training.hetero.topology import HeteroTopology +from examples.mimo.utils.hetero import debug_rank, is_process_group_member + + +@dataclass +class TrainStepResult: + """Megatron-style result returned by one hetero training step.""" + + losses: list[dict[str, Any]] + skipped_iter: int + update_successful: bool + grad_norm: Optional[float] + num_zeros_in_grad: Optional[int] + + +def wire_training_hooks(runtime: HeteroRuntime, topology: HeteroTopology) -> None: + """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" + mimo_model = runtime.model + language_pg = topology.language_pg + vision_pg = topology.vision_pg + token_count_group = topology.optimizer_stats_group + + def is_token_source_rank() -> bool: + return ( + is_process_group_member(getattr(language_pg, "pp", None)) + and is_process_group_member(getattr(language_pg, "tp", None)) + and is_pp_last_stage(language_pg.pp) + and language_pg.tp.rank() == 0 + ) + + def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): + if num_tokens is None: + raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") + + token_count = torch.zeros(1, dtype=torch.float32, device="cuda") + if is_token_source_rank(): + token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() + dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=token_count_group) + global_num_tokens = token_count.item() + + if mimo_model.language_model is not None: + debug_rank("finalizing language grads") + finalize_model_grads( + [mimo_model.language_model], + num_tokens=None, + pg_collection=language_pg, + force_all_reduce=force_all_reduce, + ) + debug_rank("language grads finalized") + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + debug_rank("finalizing vision grads") + finalize_model_grads( + [submodule], + num_tokens=None, + pg_collection=vision_pg, + force_all_reduce=force_all_reduce, + ) + debug_rank("vision grads finalized") + + if global_num_tokens > 0: + scale = 1.0 / global_num_tokens + if mimo_model.language_model is not None: + debug_rank("scaling language grads") + mimo_model.language_model.scale_gradients(scale) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + debug_rank("scaling vision grads") + submodule.scale_gradients(scale) + + mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) + mimo_model.config.finalize_model_grads_func = finalize_grads_func + mimo_model.config.grad_scale_func = lambda loss: ( + torch.tensor(loss, dtype=torch.float32, device="cuda", requires_grad=True) + if isinstance(loss, (int, float)) + else loss + ) + + +def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): + """Return raw loss sum, local token count, and logging tensors.""" + if output_tensor is None: + zero = torch.tensor(0.0, device="cuda", requires_grad=True) + zero_count = torch.tensor(0, device="cuda", dtype=torch.int) + return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + + if isinstance(output_tensor, dict): + output = output_tensor.get( + MIMO_LANGUAGE_MODULE_KEY, next(iter(output_tensor.values()), None) + ) + else: + output = output_tensor + + if output is None: + zero = torch.tensor(0.0, device="cuda", requires_grad=True) + zero_count = torch.tensor(0, device="cuda", dtype=torch.int) + return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + + output = output.float() + if loss_mask is None: + raise RuntimeError("train_hetero.py requires a loss_mask for per-token loss") + if output.shape != loss_mask.shape: + raise RuntimeError( + f"loss output shape {tuple(output.shape)} does not match loss_mask shape " + f"{tuple(loss_mask.shape)}; per-token loss cannot be scaled correctly" + ) + + masked = output * loss_mask.float() + num_tokens = loss_mask.float().sum().to(torch.int) + loss_sum = masked.sum() + return ( + loss_sum, + num_tokens, + {"lm loss sum": loss_sum.detach(), "lm tokens": num_tokens.detach()}, + ) + + +def forward_step(data_iterator, model): + """Forward step consumed by the MCore pipeline schedule.""" + batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} + debug_rank("forward_step batch prepared") + debug_rank("forward_step model call start") + output_tensor, loss_mask = model(**batch) + debug_rank("forward_step model call done") + return output_tensor, partial(loss_func, loss_mask) + + +def train_step( + args: argparse.Namespace, + runtime: HeteroRuntime, + topology: HeteroTopology, + optimizer, + opt_param_scheduler, + communicator: MultiModulePipelineCommunicator, + data_iterator, +) -> TrainStepResult: + """Run one Megatron-shaped hetero training step.""" + zero_active_grad_buffers(runtime.model) + optimizer.zero_grad() + + debug_rank("starting forward/backward schedule") + losses = schedule.forward_backward_pipelining_without_interleaving( + forward_step_func=forward_step, + data_iterator=data_iterator, + model=[runtime.model], + num_microbatches=args.num_microbatches, + seq_length=args.seq_length, + micro_batch_size=args.micro_batch_size, + forward_only=False, + p2p_communicator=communicator, + pg_collection=topology.schedule_pg_collection, + ) + debug_rank("schedule complete") + + debug_rank("optimizer step starting") + update_successful, grad_norm, num_zeros_in_grad = optimizer.step() + update_successful = reduce_update_success(update_successful, topology.optimizer_stats_group) + debug_rank("optimizer step complete") + + if update_successful: + opt_param_scheduler.step(increment=get_global_batch_size(args)) + skipped_iter = 0 + else: + # Match Megatron train_step semantics: failed updates skip LR advancement but + # do not abort the run. + skipped_iter = 1 + + return TrainStepResult( + losses=losses, + skipped_iter=skipped_iter, + update_successful=update_successful, + grad_norm=grad_norm, + num_zeros_in_grad=num_zeros_in_grad, + ) + + +def reduce_update_success(update_successful: bool, group: dist.ProcessGroup) -> bool: + """Match Megatron's cross-rank success agreement for hetero process groups.""" + value = torch.tensor([1 if update_successful else 0], dtype=torch.int, device="cuda") + dist.all_reduce(value, op=dist.ReduceOp.MIN, group=group) + return bool(value.item()) diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py new file mode 100644 index 00000000000..21e9a4a8fa4 --- /dev/null +++ b/examples/mimo/training/hetero/topology.py @@ -0,0 +1,329 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""HyperCommGrid and process-group ownership for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from typing import Optional + +import torch.distributed as dist + +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY +from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator +from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage +from megatron.core.process_groups_config import ( + MultiModuleProcessGroupCollection, + ProcessGroupCollection, +) + +from examples.mimo.utils.hetero import debug_rank, is_process_group_member + +ENCODER_MODULE_NAME = "images" + +_EMBEDDING_PG_CACHE: dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] = {} + + +@dataclass +class HeteroTopology: + """Process groups and rank topology for one hetero MIMO run.""" + + encoder_grid: HyperCommGrid + llm_grid: HyperCommGrid + language_pg: ProcessGroupCollection + vision_pg: ProcessGroupCollection + schedule_pg_collection: MultiModuleProcessGroupCollection + optimizer_stats_group: dist.ProcessGroup + encoder_size: int + llm_size: int + encoder_name: str = ENCODER_MODULE_NAME + + @property + def module_to_grid_map(self) -> dict[str, HyperCommGrid]: + """Return the MIMO module-to-grid mapping consumed by schedules and models.""" + return {self.encoder_name: self.encoder_grid, MIMO_LANGUAGE_MODULE_KEY: self.llm_grid} + + @property + def module_dependency_map(self) -> dict[str, list[str]]: + """Return the static encoder-to-language MIMO dependency graph.""" + return {self.encoder_name: [MIMO_LANGUAGE_MODULE_KEY], MIMO_LANGUAGE_MODULE_KEY: []} + + def destroy(self) -> None: + """Destroy all process groups owned by this topology.""" + destroy_process_group_if_member(self.optimizer_stats_group) + destroy_embedding_groups() + self.encoder_grid.destroy() + self.llm_grid.destroy() + BridgeCommunicator.destroy_broadcast_pgs() + + +def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) -> HeteroTopology: + """Create all rank-global process groups in one deterministic order.""" + world_size = dist.get_world_size() + encoder_grid = None + llm_grid = None + optimizer_stats_group = None + try: + debug_rank("creating encoder grid") + encoder_grid = create_hypercomm_grid( + offset=args.encoder_offset, + tp=args.encoder_tp, + cp=args.encoder_cp, + pp=args.encoder_pp, + dp=args.encoder_dp, + ep=args.encoder_ep, + expt_tp=args.encoder_expt_tp, + expt_dp=args.encoder_expt_dp, + ) + debug_rank("creating language grid") + llm_grid = create_hypercomm_grid( + offset=args.llm_offset, + tp=args.llm_tp, + cp=args.llm_cp, + pp=args.llm_pp, + dp=args.llm_dp, + ep=args.llm_ep, + expt_tp=args.llm_expt_tp, + expt_dp=args.llm_expt_dp, + ) + debug_rank("creating embedding groups") + create_all_embedding_groups([encoder_grid, llm_grid]) + debug_rank("embedding groups ready") + + language_pg = get_pg_collection_with_embedding_groups(llm_grid, is_language_model=True) + vision_pg = get_pg_collection_with_embedding_groups(encoder_grid, is_language_model=False) + schedule_pg_collection = build_schedule_pg_collection( + ENCODER_MODULE_NAME, encoder_grid, llm_grid, vision_pg, language_pg + ) + + debug_rank("creating MIMO optimizer stats group") + optimizer_stats_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") + debug_rank("MIMO optimizer stats group ready") + + return HeteroTopology( + encoder_grid=encoder_grid, + llm_grid=llm_grid, + language_pg=language_pg, + vision_pg=vision_pg, + schedule_pg_collection=schedule_pg_collection, + optimizer_stats_group=optimizer_stats_group, + encoder_size=encoder_size, + llm_size=llm_size, + ) + except Exception: + destroy_process_group_if_member(optimizer_stats_group) + destroy_embedding_groups() + if encoder_grid is not None: + encoder_grid.destroy() + if llm_grid is not None: + llm_grid.destroy() + BridgeCommunicator.destroy_broadcast_pgs() + raise + + +def create_hypercomm_grid( + offset: int, + tp: int, + cp: int, + pp: int, + dp: int, + ep: int, + expt_tp: Optional[int], + expt_dp: Optional[int], +) -> HyperCommGrid: + """Create a dense grid plus expert layout and required process groups.""" + expt_tp = tp if expt_tp is None else expt_tp + module_world_size = tp * cp * pp * dp + expert_model_size = expt_tp * ep * pp + if module_world_size % expert_model_size != 0: + raise ValueError( + f"module_world_size ({module_world_size}) must be divisible by " + f"expt_tp*ep*pp ({expert_model_size})" + ) + if expt_dp is None: + expt_dp = module_world_size // expert_model_size + if expt_tp * ep * expt_dp * pp != module_world_size: + raise ValueError( + f"expt_tp*ep*expt_dp*pp ({expt_tp * ep * expt_dp * pp}) must equal " + f"module_world_size ({module_world_size})" + ) + + grid = HyperCommGrid( + shape=[tp, cp, dp, pp], + dim_names=["tp", "cp", "dp", "pp"], + rank_offset=offset, + backend="nccl", + ) + grid.register_layout( + "expert", + [expt_tp, ep, expt_dp, pp], + ["expt_tp", "ep", "expt_dp", "pp"], + aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, + ) + + try: + for dims in ( + ["tp"], + ["cp"], + ["pp"], + ["dp"], + ["dp", "cp"], + ["tp", "cp"], + ["ep"], + ["expt_tp"], + ["expt_dp"], + ["tp", "pp"], + ["tp", "cp", "dp"], + ["tp", "cp", "pp", "dp"], + "tp_ep", + "tp_ep_pp", + ): + grid.create_pg(dims) + except Exception: + grid.destroy() + raise + + return grid + + +def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: + """Build a ProcessGroupCollection from a populated HyperCommGrid.""" + return ProcessGroupCollection.from_hyper_comm_grid( + grid, + required_pgs=[ + "tp", + "cp", + "pp", + "dp", + "dp_cp", + "tp_cp", + "mp", + "tp_dp_cp", + "ep", + "expt_tp", + "expt_dp", + "tp_ep", + "tp_ep_pp", + "intra_dist_opt", + ], + ) + + +def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: + """Create PP-derived embedding groups in a consistent global order.""" + pp_rank_sets: list[tuple[int, ...]] = [] + seen_pp_rank_sets = set() + for grid in sorted(grids, key=lambda candidate: (candidate.rank_offset, candidate.size)): + for pp_ranks in grid.get_rank_enum("pp"): + pp_rank_tuple = tuple(pp_ranks) + if pp_rank_tuple in seen_pp_rank_sets: + continue + pp_rank_sets.append(pp_rank_tuple) + seen_pp_rank_sets.add(pp_rank_tuple) + + for pp_ranks in pp_rank_sets: + if pp_ranks not in _EMBEDDING_PG_CACHE: + pos_embd_ranks = [pp_ranks[0]] + embd_ranks = [pp_ranks[0]] + if pp_ranks[-1] != pp_ranks[0]: + embd_ranks.append(pp_ranks[-1]) + pos_embd_pg = None + embd_pg = None + try: + pos_embd_pg = dist.new_group(ranks=pos_embd_ranks) + embd_pg = dist.new_group(ranks=embd_ranks) + _EMBEDDING_PG_CACHE[pp_ranks] = (pos_embd_pg, embd_pg) + except Exception: + destroy_process_group_if_member(pos_embd_pg) + destroy_process_group_if_member(embd_pg) + raise + + +def destroy_embedding_groups() -> None: + """Destroy cached embedding process groups created by this module.""" + destroyed_embedding_pgs = set() + for pos_embd_pg, embd_pg in _EMBEDDING_PG_CACHE.values(): + for pg in (pos_embd_pg, embd_pg): + if id(pg) in destroyed_embedding_pgs: + continue + destroy_process_group_if_member(pg) + destroyed_embedding_pgs.add(id(pg)) + _EMBEDDING_PG_CACHE.clear() + + +def add_embedding_groups( + pg_collection: ProcessGroupCollection, is_language_model: bool = False +) -> ProcessGroupCollection: + """Attach cached embedding process groups to a ProcessGroupCollection.""" + if not is_process_group_member(getattr(pg_collection, "pp", None)): + return pg_collection + + pp_ranks = tuple(dist.get_process_group_ranks(pg_collection.pp)) + pos_embd_pg, embd_pg = _EMBEDDING_PG_CACHE[pp_ranks] + + pg_collection.pos_embd = pos_embd_pg if is_pp_first_stage(pg_collection.pp) else None + if is_language_model: + pg_collection.embd = ( + embd_pg + if is_pp_last_stage(pg_collection.pp) or is_pp_first_stage(pg_collection.pp) + else None + ) + else: + pg_collection.embd = None + + return pg_collection + + +def get_pg_collection_with_embedding_groups( + grid: HyperCommGrid, is_language_model: bool = False +) -> ProcessGroupCollection: + """Build a ProcessGroupCollection and add PP-derived embedding groups.""" + return add_embedding_groups(get_pg_collection(grid), is_language_model=is_language_model) + + +def build_schedule_pg_collection( + encoder_name: str, + encoder_grid: HyperCommGrid, + llm_grid: HyperCommGrid, + vision_pg: ProcessGroupCollection, + language_pg: ProcessGroupCollection, +) -> MultiModuleProcessGroupCollection: + """Build the schedule-facing process group collection for this rank.""" + module_pgs = {} + language_model_module_name = None + if is_rank_in_grid(encoder_grid): + module_pgs[encoder_name] = vision_pg + if is_rank_in_grid(llm_grid): + module_pgs[MIMO_LANGUAGE_MODULE_KEY] = language_pg + language_model_module_name = MIMO_LANGUAGE_MODULE_KEY + + return MultiModuleProcessGroupCollection( + module_pgs=module_pgs, language_model_module_name=language_model_module_name + ) + + +def destroy_process_group_if_member(pg: Optional[dist.ProcessGroup]) -> None: + """Destroy pg when this rank owns a process-group handle.""" + if is_process_group_member(pg): + dist.destroy_process_group(pg) + + +def is_rank_in_grid(grid: HyperCommGrid) -> bool: + """Return whether this global rank is inside a grid's rank span.""" + rank = dist.get_rank() + return grid.rank_offset <= rank < grid.rank_offset + grid.size + + +def get_grid_coordinate(grid: HyperCommGrid, dim: str) -> int: + """Return this rank's coordinate for a base-layout dimension.""" + if not is_rank_in_grid(grid): + return 0 + + local_rank = dist.get_rank() - grid.rank_offset + coordinates = {} + for dim_name, dim_size in zip(grid.dim_names, grid.shape): + coordinates[dim_name] = local_rank % dim_size + local_rank //= dim_size + return coordinates[dim] diff --git a/examples/mimo/utils/hetero.py b/examples/mimo/utils/hetero.py new file mode 100644 index 00000000000..ea16f44f40c --- /dev/null +++ b/examples/mimo/utils/hetero.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Shared helpers for heterogeneous MIMO examples.""" + +from __future__ import annotations + +import os +import sys +from typing import Optional + +import torch.distributed as dist + +from megatron.core.hyper_comm_grid import HyperCommGrid + +MOCK_MODEL_PRESET = "mock" +NEMOTRON_20L_MODEL_PRESET = "nemotron-moe-vlm-20l" +NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" +NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 +NEMOTRON_20L_MAX_NUM_TILES = 12 +NEMOTRON_20L_DEFAULT_STAGE = "stage2" +MOCK_VISION_ENCODER_KEY = "clip_encoder" +NEMOTRON_VISION_ENCODER_KEY = "radio_encoder" + + +def is_nemotron_20l(args) -> bool: + """Return whether the run should use the Nemotron6-MoE VLM 20L architecture.""" + return args.model_preset == NEMOTRON_20L_MODEL_PRESET + + +def debug_rank(message: str) -> None: + """Emit per-rank startup checkpoints when MIMO_HETERO_DEBUG is set.""" + if os.environ.get("MIMO_HETERO_DEBUG"): + rank = dist.get_rank() if dist.is_initialized() else 0 + sys.stderr.write(f"[rank {rank}] {message}\n") + sys.stderr.flush() + + +def is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Return whether pg is a real process group for this rank.""" + group_member = getattr(dist, "GroupMember", None) + non_member = getattr(group_member, "NON_GROUP_MEMBER", None) + return pg is not None and pg != non_member + + +def get_grid_dim_size(grid: HyperCommGrid, dim: str) -> int: + """Return a base-layout dimension size.""" + return grid.shape[grid.dim_names.index(dim)] + + +def get_group_size_or(pg: Optional[dist.ProcessGroup], fallback: int) -> int: + """Return pg size on member ranks, otherwise fallback.""" + return pg.size() if is_process_group_member(pg) else fallback + + +def get_group_rank_or(pg: Optional[dist.ProcessGroup], fallback: int = 0) -> int: + """Return rank inside pg on member ranks, otherwise fallback.""" + return dist.get_rank(pg) if is_process_group_member(pg) else fallback From c61976207a6cbb20804f2d405e1440a11b4f53dc Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sun, 10 May 2026 22:05:10 +0000 Subject: [PATCH 05/44] NMFW-464 address hetero MIMO PR cleanup --- examples/mimo/data/hetero_mock.py | 24 +-- .../{hetero_vlm.py => nemotron_moe_vlm.py} | 129 ++++++++++++++- .../run_hetero_nemotron_20l_mock_train.sh | 2 +- examples/mimo/training/hetero/args.py | 147 ++---------------- examples/mimo/training/hetero/distributed.py | 11 +- examples/mimo/training/hetero/logging.py | 31 ++-- examples/mimo/training/hetero/loop.py | 5 +- examples/mimo/training/hetero/runtime.py | 2 +- examples/mimo/training/hetero/step.py | 9 +- examples/mimo/training/hetero/topology.py | 21 +-- examples/mimo/utils/hetero.py | 14 -- megatron/core/hyper_comm_grid.py | 79 +--------- .../common/language_module/language_module.py | 4 +- .../models/hybrid/hybrid_layer_allocation.py | 4 +- megatron/core/models/hybrid/hybrid_model.py | 10 +- megatron/core/models/mimo/model/base.py | 54 ++----- megatron/core/models/mimo/optimizer.py | 43 ++--- megatron/core/optimizer/__init__.py | 23 +-- megatron/core/process_groups_config.py | 18 +-- .../core/tensor_parallel/cross_entropy.py | 42 ++--- .../models/test_mimo_1f1b_schedule.py | 11 +- .../ssm/test_hybrid_layer_allocation.py | 11 ++ tests/unit_tests/test_hyper_comm_grid.py | 68 ++------ .../unit_tests/test_process_groups_config.py | 26 ++-- 24 files changed, 286 insertions(+), 502 deletions(-) rename examples/mimo/model_providers/{hetero_vlm.py => nemotron_moe_vlm.py} (77%) diff --git a/examples/mimo/data/hetero_mock.py b/examples/mimo/data/hetero_mock.py index 997310bd231..137028a68e1 100644 --- a/examples/mimo/data/hetero_mock.py +++ b/examples/mimo/data/hetero_mock.py @@ -8,12 +8,16 @@ import torch -from examples.mimo.utils.hetero import ( - MOCK_VISION_ENCODER_KEY, - NEMOTRON_VISION_ENCODER_KEY, - debug_rank, - is_nemotron_20l, -) +from examples.mimo.utils.hetero import debug_rank + + +def validate_mock_data_args(args: argparse.Namespace) -> None: + """Validate synthetic next-token VLM data constraints.""" + image_seq_length = args.image_seq_length or args.seq_length // 2 + if image_seq_length >= args.seq_length: + raise ValueError("--image-seq-length must be smaller than --seq-length") + if args.seq_length - image_seq_length < 2: + raise ValueError("mock next-token training needs at least two text tokens") class MockVLMIterator: @@ -26,6 +30,8 @@ def __init__( self.micro_batch_size = micro_batch_size self.encoder_name = encoder_name self.image_seq_length = args.image_seq_length or args.seq_length // 2 + self.vision_encoder_key = getattr(args, "vision_encoder_key", "clip_encoder") + self.vision_input_mode = getattr(args, "vision_input_mode", "hidden_states") self.dtype = torch.float32 if args.fp32 else torch.bfloat16 self.generator = torch.Generator(device="cuda") self.generator.manual_seed(seed) @@ -76,9 +82,9 @@ def __next__(self): labels[(labels == args.image_token_id) | (labels == args.pad_token_id)] = -100 loss_mask = (labels != -100).to(dtype=torch.float32) - if is_nemotron_20l(args): + if self.vision_input_mode == "pixels": encoder_inputs = { - NEMOTRON_VISION_ENCODER_KEY: { + self.vision_encoder_key: { "x": torch.randn( self.micro_batch_size * args.num_image_tiles, 3, @@ -100,7 +106,7 @@ def __next__(self): generator=self.generator, ) encoder_inputs = { - MOCK_VISION_ENCODER_KEY: { + self.vision_encoder_key: { "hidden_states": encoder_hidden_states, "attention_mask": None, } diff --git a/examples/mimo/model_providers/hetero_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py similarity index 77% rename from examples/mimo/model_providers/hetero_vlm.py rename to examples/mimo/model_providers/nemotron_moe_vlm.py index 9c4704ab9c3..f2c941b60bb 100644 --- a/examples/mimo/model_providers/hetero_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -1,6 +1,6 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Model provider helpers for heterogeneous MIMO VLM examples.""" +"""Model providers and configs for MIMO VLM examples.""" from __future__ import annotations @@ -29,14 +29,10 @@ from megatron.core.transformer.utils import sharded_state_dict_default from examples.mimo.utils.hetero import ( - MOCK_VISION_ENCODER_KEY, - NEMOTRON_20L_HYBRID_PATTERN, - NEMOTRON_VISION_ENCODER_KEY, debug_rank, get_grid_dim_size, get_group_rank_or, get_group_size_or, - is_nemotron_20l, is_process_group_member, ) @@ -51,6 +47,127 @@ TELayerNormColumnParallelLinear = None TERowParallelLinear = None +MOCK_MODEL_PROVIDER = "mock" +NEMOTRON_20L_MODEL_PROVIDER = "nemotron-moe-vlm-20l" +NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" +NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 +NEMOTRON_20L_MAX_NUM_TILES = 12 +NEMOTRON_20L_DEFAULT_STAGE = "stage2" +MOCK_VISION_ENCODER_KEY = "clip_encoder" +NEMOTRON_VISION_ENCODER_KEY = "radio_encoder" + + +def is_nemotron_20l(args: argparse.Namespace) -> bool: + return args.model_provider == NEMOTRON_20L_MODEL_PROVIDER + + +def add_model_provider_args(parser: argparse.ArgumentParser) -> None: + provider = parser.add_argument_group("model provider") + provider.add_argument( + "--model-provider", + choices=[MOCK_MODEL_PROVIDER, NEMOTRON_20L_MODEL_PROVIDER], + default=MOCK_MODEL_PROVIDER, + ) + provider.add_argument("--hidden-size", type=int, default=128) + provider.add_argument("--num-layers", type=int, default=2) + provider.add_argument("--num-attention-heads", type=int, default=8) + provider.add_argument("--vocab-size", type=int, default=512) + provider.add_argument("--seq-length", type=int, default=32) + provider.add_argument("--image-seq-length", type=int, default=None) + provider.add_argument("--image-token-id", type=int, default=511) + provider.add_argument("--pad-token-id", type=int, default=0) + provider.add_argument("--image-token", type=str, default="") + provider.add_argument("--tokenizer-model", type=str, default=None) + provider.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") + provider.add_argument("--image-tag-type", type=str, default="") + provider.add_argument("--force-system-message", action="store_true") + provider.add_argument("--num-moe-experts", type=int, default=4) + provider.add_argument("--moe-router-topk", type=int, default=1) + provider.add_argument("--moe-grouped-gemm", action="store_true") + provider.add_argument("--img-h", type=int, default=512) + provider.add_argument("--img-w", type=int, default=512) + provider.add_argument("--patch-dim", type=int, default=16) + provider.add_argument("--class-token-len", type=int, default=8) + provider.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) + provider.add_argument("--freeze-lm", action="store_true") + provider.add_argument("--freeze-vit", action="store_true") + provider.add_argument("--freeze-projection", action="store_true") + provider.add_argument("--training-stage", choices=["stage1", "stage2", "stage3"], default=None) + provider.add_argument("--fp32", action="store_true") + + +def prepare_model_provider_args(args: argparse.Namespace) -> None: + apply_model_provider_defaults(args) + apply_training_stage(args) + resolve_image_token_id(args) + args.vision_encoder_key = get_encoder_module_name(args) + args.vision_input_mode = "pixels" if is_nemotron_20l(args) else "hidden_states" + + +def apply_model_provider_defaults(args: argparse.Namespace) -> None: + if not is_nemotron_20l(args): + return + + args.num_layers = 20 + args.hidden_size = 2688 + args.num_attention_heads = 32 + args.num_moe_experts = 128 + args.moe_router_topk = 6 + args.moe_grouped_gemm = True + args.seq_length = 8192 + args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles + + +def apply_training_stage(args: argparse.Namespace) -> None: + if not is_nemotron_20l(args): + return + + stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE + if stage == "stage1": + args.freeze_vit = True + args.freeze_lm = True + elif stage == "stage2": + args.freeze_vit = True + elif stage != "stage3": + raise ValueError(f"unsupported Nemotron VLM training stage: {stage}") + args.training_stage = stage + + +def resolve_image_token_id(args: argparse.Namespace) -> None: + if not is_nemotron_20l(args) or not args.tokenizer_model: + return + + from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( + MegatronMultimodalTokenizer, + ) + + tokenizer = MegatronMultimodalTokenizer( + path=args.tokenizer_model, + prompt_format=args.tokenizer_prompt_format, + special_tokens=[args.image_token], + image_tag_type=args.image_tag_type, + force_system_message=args.force_system_message, + ) + image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) + if image_token_id is None: + raise RuntimeError( + f"tokenizer at {args.tokenizer_model} did not produce an id for {args.image_token}" + ) + args.image_token_id = int(image_token_id) + if tokenizer.pad is not None: + args.pad_token_id = int(tokenizer.pad) + if tokenizer.vocab_size is not None: + args.vocab_size = int(tokenizer.vocab_size) + + +def validate_model_provider_args(args: argparse.Namespace) -> None: + if args.hidden_size % args.num_attention_heads != 0: + raise ValueError("--hidden-size must be divisible by --num-attention-heads") + if not 0 <= args.image_token_id < args.vocab_size: + raise ValueError("--image-token-id must be within --vocab-size") + if not 0 <= args.pad_token_id < args.vocab_size: + raise ValueError("--pad-token-id must be within --vocab-size") + class RADIOEncoderWrapper(torch.nn.Module): """RADIO encoder wrapper matching the Nemotron6-MoE VLM provider.""" @@ -197,6 +314,8 @@ def nemotron_language_config( pipeline_dtype=dtype, bf16=bf16, calculate_per_token_loss=True, + cross_entropy_loss_fusion=True, + cross_entropy_fusion_impl="te", bias_activation_fusion=False, masked_softmax_fusion=True, persist_layer_norm=True, diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh index f113d988a66..f16136cf36d 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_mock_train.sh @@ -40,7 +40,7 @@ esac --standalone \ --nproc-per-node "${GPUS_PER_NODE}" \ examples/mimo/train_hetero.py \ - --model-preset nemotron-moe-vlm-20l \ + --model-provider nemotron-moe-vlm-20l \ --training-stage "${TRAINING_STAGE}" \ --encoder-tp 2 \ --encoder-pp 1 \ diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 3b4446b62b9..f1dffc5d3c9 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -6,13 +6,11 @@ import argparse -from examples.mimo.utils.hetero import ( - MOCK_MODEL_PRESET, - NEMOTRON_20L_DEFAULT_STAGE, - NEMOTRON_20L_IMAGE_SEQ_PER_TILE, - NEMOTRON_20L_MAX_NUM_TILES, - NEMOTRON_20L_MODEL_PRESET, - is_nemotron_20l, +from examples.mimo.data.hetero_mock import validate_mock_data_args +from examples.mimo.model_providers.nemotron_moe_vlm import ( + add_model_provider_args, + prepare_model_provider_args, + validate_model_provider_args, ) @@ -43,48 +41,7 @@ def parse_args() -> argparse.Namespace: grid.add_argument("--llm-expt-tp", type=int, default=1) grid.add_argument("--llm-expt-dp", type=int, default=1) - model = parser.add_argument_group("model") - model.add_argument( - "--model-preset", - choices=[MOCK_MODEL_PRESET, NEMOTRON_20L_MODEL_PRESET], - default=MOCK_MODEL_PRESET, - help="Model config preset. The Nemotron preset matches the 20L reference script.", - ) - model.add_argument("--hidden-size", type=int, default=128) - model.add_argument("--num-layers", type=int, default=2) - model.add_argument("--num-attention-heads", type=int, default=8) - model.add_argument("--vocab-size", type=int, default=512) - model.add_argument("--seq-length", type=int, default=32) - model.add_argument("--image-seq-length", type=int, default=None) - model.add_argument("--image-token-id", type=int, default=511) - model.add_argument("--pad-token-id", type=int, default=0) - model.add_argument("--image-token", type=str, default="") - model.add_argument("--tokenizer-model", type=str, default=None) - model.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") - model.add_argument("--image-tag-type", type=str, default="") - model.add_argument("--force-system-message", action="store_true") - model.add_argument("--num-moe-experts", type=int, default=4) - model.add_argument("--moe-router-topk", type=int, default=1) - model.add_argument("--moe-grouped-gemm", action="store_true") - model.add_argument("--img-h", type=int, default=512) - model.add_argument("--img-w", type=int, default=512) - model.add_argument("--patch-dim", type=int, default=16) - model.add_argument("--class-token-len", type=int, default=8) - model.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) - model.add_argument("--freeze-lm", action="store_true", help="Freeze language model params") - model.add_argument("--freeze-vit", action="store_true", help="Freeze vision encoder params") - model.add_argument( - "--freeze-projection", action="store_true", help="Freeze vision projection params" - ) - model.add_argument( - "--training-stage", - choices=["stage1", "stage2", "stage3"], - default=None, - help="Nemotron VLM freeze stage. Defaults to stage2 for the 20L preset.", - ) - model.add_argument( - "--fp32", action="store_true", help="Build and train in fp32 instead of bf16" - ) + add_model_provider_args(parser) train = parser.add_argument_group("training") train.add_argument("--micro-batch-size", type=int, default=2) @@ -104,10 +61,10 @@ def parse_args() -> argparse.Namespace: train.add_argument( "--overlap-grad-reduce", action=argparse.BooleanOptionalAction, - default=True, + default=False, help=( - "Enable DDP gradient-reduce overlap. Disable for parity with the 20L " - "reference script." + "Enable DDP gradient-reduce overlap. The hetero example defaults this off " + "to match Megatron's conservative DDP default and the 20L reference script." ), ) train.add_argument( @@ -122,93 +79,23 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def apply_model_preset(args: argparse.Namespace) -> None: - """Apply architecture defaults for the selected model preset.""" - if not is_nemotron_20l(args): - return - - args.num_layers = 20 - args.hidden_size = 2688 - args.num_attention_heads = 32 - args.num_moe_experts = 128 - args.moe_router_topk = 6 - args.moe_grouped_gemm = True - args.seq_length = 8192 - args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles - - -def apply_training_stage(args: argparse.Namespace) -> None: - """Apply the reference Nemotron VLM freeze stage defaults.""" - if not is_nemotron_20l(args): - return - - stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE - if stage == "stage1": - args.freeze_vit = True - args.freeze_lm = True - elif stage == "stage2": - args.freeze_vit = True - elif stage != "stage3": - raise ValueError(f"unsupported Nemotron VLM training stage: {stage}") - args.training_stage = stage - - -def resolve_image_token_id(args: argparse.Namespace) -> None: - """Resolve the image token id from the reference MultimodalTokenizer when provided.""" - if not is_nemotron_20l(args) or not args.tokenizer_model: - return - - from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( - MegatronMultimodalTokenizer, - ) - - tokenizer = MegatronMultimodalTokenizer( - path=args.tokenizer_model, - prompt_format=args.tokenizer_prompt_format, - special_tokens=[args.image_token], - image_tag_type=args.image_tag_type, - force_system_message=args.force_system_message, - ) - image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) - if image_token_id is None: - raise RuntimeError( - f"tokenizer at {args.tokenizer_model} did not produce an id for {args.image_token}" - ) - args.image_token_id = int(image_token_id) - if tokenizer.pad is not None: - args.pad_token_id = int(tokenizer.pad) - if tokenizer.vocab_size is not None: - args.vocab_size = int(tokenizer.vocab_size) - - def prepare_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: """Apply presets, resolve runtime args, and validate the hetero layout.""" - apply_model_preset(args) - apply_training_stage(args) - resolve_image_token_id(args) + prepare_model_provider_args(args) return validate_args(args, world_size) def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: - """Validate the Phase 2 non-colocated 1F1B mock-training layout.""" + """Validate the current disjoint-grid mock-training layout.""" if args.encoder_cp != 1 or args.llm_cp != 1: raise ValueError("Phase 2 mock training currently supports CP=1 only") - if args.hidden_size % args.num_attention_heads != 0: - raise ValueError("--hidden-size must be divisible by --num-attention-heads") - if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: - raise ValueError("--num-moe-experts must be divisible by --llm-ep") if args.log_interval < 1: raise ValueError("--log-interval must be >= 1") - if not 0 <= args.image_token_id < args.vocab_size: - raise ValueError("--image-token-id must be within --vocab-size") - if not 0 <= args.pad_token_id < args.vocab_size: - raise ValueError("--pad-token-id must be within --vocab-size") - - image_seq_length = args.image_seq_length or args.seq_length // 2 - if image_seq_length >= args.seq_length: - raise ValueError("--image-seq-length must be smaller than --seq-length") - if args.seq_length - image_seq_length < 2: - raise ValueError("mock next-token training needs at least two text tokens") + + validate_model_provider_args(args) + validate_mock_data_args(args) + if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: + raise ValueError("--num-moe-experts must be divisible by --llm-ep") if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") @@ -220,7 +107,7 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: if not encoder_ranks.isdisjoint(llm_ranks): raise ValueError( - "Phase 2 train_hetero.py supports non-colocated 1F1B only; " + "train_hetero.py currently expects disjoint module rank spans; " f"module rank spans overlap at {sorted(encoder_ranks & llm_ranks)}" ) if encoder_ranks | llm_ranks != all_ranks: diff --git a/examples/mimo/training/hetero/distributed.py b/examples/mimo/training/hetero/distributed.py index c5f4e115061..22dd2003ebd 100644 --- a/examples/mimo/training/hetero/distributed.py +++ b/examples/mimo/training/hetero/distributed.py @@ -4,7 +4,6 @@ from __future__ import annotations -import os import sys import torch @@ -13,17 +12,9 @@ from megatron.core import parallel_state -def clear_transformer_engine_env() -> None: - """Clear attention backend overrides that can conflict with model construction.""" - os.environ.pop("NVTE_FLASH_ATTN", None) - os.environ.pop("NVTE_FUSED_ATTN", None) - os.environ.pop("NVTE_UNFUSED_ATTN", None) - - def initialize_distributed() -> None: """Initialize torch.distributed for torchrun.""" - clear_transformer_engine_env() - os.environ.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") + import os local_rank = int(os.environ.get("LOCAL_RANK", "0")) torch.cuda.set_device(local_rank) diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py index a9ce28ea4c7..6654442b288 100644 --- a/examples/mimo/training/hetero/logging.py +++ b/examples/mimo/training/hetero/logging.py @@ -68,13 +68,9 @@ def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: elapsed_ms = (elapsed / interval_iters) * 1000.0 loss_value = self.loss_total / self.loss_count if self.loss_count else None learning_rate = get_canonical_lr_for_logging(optimizer.param_groups) - learning_rate = reduce_max_optional_float( - learning_rate, self.topology.optimizer_stats_group - ) - grad_norm = reduce_max_optional_float(result.grad_norm, self.topology.optimizer_stats_group) - num_zeros_in_grad = reduce_max_optional_float( - result.num_zeros_in_grad, self.topology.optimizer_stats_group - ) + learning_rate = reduce_max_optional_float(learning_rate) + grad_norm = reduce_max_optional_float(result.grad_norm) + num_zeros_in_grad = reduce_max_optional_float(result.num_zeros_in_grad) loss_scale = optimizer.get_loss_scale().item() if is_language_log_rank(self.topology): @@ -109,16 +105,18 @@ def reset_interval(self) -> None: def reduce_language_loss(losses: list[dict], topology: HeteroTopology) -> Optional[float]: - """Reduce raw loss/token vectors over the language DP/TP/CP logging group.""" + """Reduce raw loss/token vectors over the language DP/CP logging group.""" language_pg = topology.language_pg loss_acc = torch.zeros(2, dtype=torch.float32, device="cuda") - is_log_stage = is_process_group_member(getattr(language_pg, "tp_dp_cp", None)) and ( - is_pp_last_stage(language_pg.pp) + is_log_stage = ( + is_process_group_member(getattr(language_pg, "dp_cp", None)) + and (is_pp_last_stage(language_pg.pp)) + and language_pg.tp.rank() == 0 ) if not is_log_stage: return None - if losses and language_pg.tp.rank() == 0: + if losses: for loss_dict in losses: loss_sum = loss_dict.get("lm loss sum") num_tokens = loss_dict.get("lm tokens") @@ -131,7 +129,7 @@ def reduce_language_loss(losses: list[dict], topology: HeteroTopology) -> Option elif num_tokens is not None: loss_acc[1] += float(num_tokens) - dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.tp_dp_cp) + dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.dp_cp) return loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None @@ -139,19 +137,20 @@ def is_language_log_rank(topology: HeteroTopology) -> bool: """Return whether this rank should print language-side training metrics.""" language_pg = topology.language_pg if not ( - is_process_group_member(getattr(language_pg, "tp_dp_cp", None)) + is_process_group_member(getattr(language_pg, "dp_cp", None)) and is_pp_last_stage(language_pg.pp) + and language_pg.tp.rank() == 0 ): return False - language_group_ranks = dist.get_process_group_ranks(language_pg.tp_dp_cp) + language_group_ranks = dist.get_process_group_ranks(language_pg.dp_cp) return dist.get_rank() == min(language_group_ranks) -def reduce_max_optional_float(value, group: dist.ProcessGroup) -> Optional[float]: +def reduce_max_optional_float(value) -> Optional[float]: """Reduce optional scalar stats so the language log rank can see non-local optimizers.""" if isinstance(value, torch.Tensor): value = value.item() local_value = -1.0 if value is None else float(value) stat = torch.tensor([local_value], dtype=torch.float32, device="cuda") - dist.all_reduce(stat, op=dist.ReduceOp.MAX, group=group) + dist.all_reduce(stat, op=dist.ReduceOp.MAX) return None if stat.item() == -1.0 else stat.item() diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 1cb250f3e27..4e9b920b6a5 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -47,7 +47,7 @@ def run_train_loop(args: argparse.Namespace) -> None: wire_training_hooks(runtime, topology) debug_rank("building MIMO optimizer") - optimizer = build_optimizer(args, runtime, topology) + optimizer = build_optimizer(args, runtime) opt_param_scheduler = build_optimizer_param_scheduler(args, optimizer) debug_rank("MIMO optimizer ready") @@ -85,7 +85,7 @@ def run_train_loop(args: argparse.Namespace) -> None: topology.destroy() -def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime, topology: HeteroTopology): +def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime): """Build the MIMO optimizer for active hetero module optimizers.""" return get_mimo_optimizer( runtime.model, @@ -101,7 +101,6 @@ def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime, topology: use_distributed_optimizer=True, log_num_zeros_in_grad=args.log_num_zeros_in_grad, ), - stats_group=topology.optimizer_stats_group, ) diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 4848f5a85a4..fe389dc7c0a 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -17,7 +17,7 @@ from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed -from examples.mimo.model_providers.hetero_vlm import ( +from examples.mimo.model_providers.nemotron_moe_vlm import ( get_vision_encoder_module, iter_vision_projection_modules, language_model_spec, diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 42a7d5321b9..83425d33d6f 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -44,7 +44,6 @@ def wire_training_hooks(runtime: HeteroRuntime, topology: HeteroTopology) -> Non mimo_model = runtime.model language_pg = topology.language_pg vision_pg = topology.vision_pg - token_count_group = topology.optimizer_stats_group def is_token_source_rank() -> bool: return ( @@ -61,7 +60,7 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar token_count = torch.zeros(1, dtype=torch.float32, device="cuda") if is_token_source_rank(): token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() - dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=token_count_group) + dist.all_reduce(token_count, op=dist.ReduceOp.SUM) global_num_tokens = token_count.item() if mimo_model.language_model is not None: @@ -180,7 +179,7 @@ def train_step( debug_rank("optimizer step starting") update_successful, grad_norm, num_zeros_in_grad = optimizer.step() - update_successful = reduce_update_success(update_successful, topology.optimizer_stats_group) + update_successful = reduce_update_success(update_successful) debug_rank("optimizer step complete") if update_successful: @@ -200,8 +199,8 @@ def train_step( ) -def reduce_update_success(update_successful: bool, group: dist.ProcessGroup) -> bool: +def reduce_update_success(update_successful: bool) -> bool: """Match Megatron's cross-rank success agreement for hetero process groups.""" value = torch.tensor([1 if update_successful else 0], dtype=torch.int, device="cuda") - dist.all_reduce(value, op=dist.ReduceOp.MIN, group=group) + dist.all_reduce(value, op=dist.ReduceOp.MIN) return bool(value.item()) diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py index 21e9a4a8fa4..6f5da1ee3e1 100644 --- a/examples/mimo/training/hetero/topology.py +++ b/examples/mimo/training/hetero/topology.py @@ -35,7 +35,6 @@ class HeteroTopology: language_pg: ProcessGroupCollection vision_pg: ProcessGroupCollection schedule_pg_collection: MultiModuleProcessGroupCollection - optimizer_stats_group: dist.ProcessGroup encoder_size: int llm_size: int encoder_name: str = ENCODER_MODULE_NAME @@ -52,7 +51,6 @@ def module_dependency_map(self) -> dict[str, list[str]]: def destroy(self) -> None: """Destroy all process groups owned by this topology.""" - destroy_process_group_if_member(self.optimizer_stats_group) destroy_embedding_groups() self.encoder_grid.destroy() self.llm_grid.destroy() @@ -61,10 +59,8 @@ def destroy(self) -> None: def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) -> HeteroTopology: """Create all rank-global process groups in one deterministic order.""" - world_size = dist.get_world_size() encoder_grid = None llm_grid = None - optimizer_stats_group = None try: debug_rank("creating encoder grid") encoder_grid = create_hypercomm_grid( @@ -98,22 +94,16 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) ENCODER_MODULE_NAME, encoder_grid, llm_grid, vision_pg, language_pg ) - debug_rank("creating MIMO optimizer stats group") - optimizer_stats_group = dist.new_group(ranks=list(range(world_size)), backend="nccl") - debug_rank("MIMO optimizer stats group ready") - return HeteroTopology( encoder_grid=encoder_grid, llm_grid=llm_grid, language_pg=language_pg, vision_pg=vision_pg, schedule_pg_collection=schedule_pg_collection, - optimizer_stats_group=optimizer_stats_group, encoder_size=encoder_size, llm_size=llm_size, ) except Exception: - destroy_process_group_if_member(optimizer_stats_group) destroy_embedding_groups() if encoder_grid is not None: encoder_grid.destroy() @@ -156,12 +146,7 @@ def create_hypercomm_grid( rank_offset=offset, backend="nccl", ) - grid.register_layout( - "expert", - [expt_tp, ep, expt_dp, pp], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, - ) + grid.register_layout("expert", [expt_tp, ep, expt_dp, pp], ["expt_tp", "ep", "expt_dp", "pp"]) try: for dims in ( @@ -177,8 +162,8 @@ def create_hypercomm_grid( ["tp", "pp"], ["tp", "cp", "dp"], ["tp", "cp", "pp", "dp"], - "tp_ep", - "tp_ep_pp", + ["expt_tp", "ep"], + ["expt_tp", "ep", "pp"], ): grid.create_pg(dims) except Exception: diff --git a/examples/mimo/utils/hetero.py b/examples/mimo/utils/hetero.py index ea16f44f40c..3d3ea8ee409 100644 --- a/examples/mimo/utils/hetero.py +++ b/examples/mimo/utils/hetero.py @@ -12,20 +12,6 @@ from megatron.core.hyper_comm_grid import HyperCommGrid -MOCK_MODEL_PRESET = "mock" -NEMOTRON_20L_MODEL_PRESET = "nemotron-moe-vlm-20l" -NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" -NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 -NEMOTRON_20L_MAX_NUM_TILES = 12 -NEMOTRON_20L_DEFAULT_STAGE = "stage2" -MOCK_VISION_ENCODER_KEY = "clip_encoder" -NEMOTRON_VISION_ENCODER_KEY = "radio_encoder" - - -def is_nemotron_20l(args) -> bool: - """Return whether the run should use the Nemotron6-MoE VLM 20L architecture.""" - return args.model_preset == NEMOTRON_20L_MODEL_PRESET - def debug_rank(message: str) -> None: """Emit per-rank startup checkpoints when MIMO_HETERO_DEBUG is set.""" diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py index e27d4fce0ea..74813304132 100644 --- a/megatron/core/hyper_comm_grid.py +++ b/megatron/core/hyper_comm_grid.py @@ -1,7 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import os -from dataclasses import dataclass, field +from dataclasses import dataclass from operator import itemgetter from typing import Any, Optional, Tuple, Union @@ -40,15 +40,8 @@ def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: @dataclass class _GridLayout: - """Rank layout owned by a HyperCommGrid. - - The base layout is the original Cartesian grid. Registered layouts are - alternate factorizations over the same rank span. - """ - shape: list[int] dim_names: list[str] - aliases: dict[str, list[str]] = field(default_factory=dict) class HyperCommGrid: @@ -138,15 +131,8 @@ def __init__( self.backend = backend self._pgs: dict[str, dist.ProcessGroup] = {} self._layouts: dict[str, _GridLayout] = {"base": _GridLayout(self.shape, self.dim_names)} - self._aliases: dict[str, tuple[str, list[str]]] = {} - def register_layout( - self, - name: str, - shape: list[int], - dim_names: list[str], - aliases: Optional[dict[str, list[str]]] = None, - ) -> None: + def register_layout(self, name: str, shape: list[int], dim_names: list[str]) -> None: """Register an alternate rank layout over this grid's rank span. Registered layouts are useful when the same module rank universe has @@ -159,8 +145,6 @@ def register_layout( shape: Shape of the alternate layout. Its product must equal the base grid size. dim_names: Dimension names for the alternate layout. - aliases: Optional names for composite groups in this layout. - For example, ``{"tp_ep": ["expt_tp", "ep"]}``. """ if name == "base": raise ValueError("'base' is reserved for the default HyperCommGrid layout") @@ -187,45 +171,12 @@ def register_layout( "but has different rank enumeration" ) - aliases = aliases or {} - for alias_name, alias_dims in aliases.items(): - if alias_name in self._aliases: - raise ValueError(f"Alias {alias_name!r} is already registered") - if alias_name in self.dim_names or alias_name in dim_names: - raise ValueError(f"Alias {alias_name!r} conflicts with an existing dimension name") - if "-" in alias_name: - raise ValueError( - f"Alias {alias_name!r} cannot contain '-' because process group keys use '-'" - ) - if len(set(alias_dims)) != len(alias_dims): - raise ValueError(f"Alias {alias_name!r} has duplicate dimensions: {alias_dims}") - missing_dims = [dim for dim in alias_dims if dim not in dim_names] - if missing_dims: - raise ValueError( - f"Alias {alias_name!r} references dimensions not in layout {name!r}: " - f"{missing_dims}" - ) - layout.aliases[alias_name] = alias_dims[:] - self._layouts[name] = layout - for alias_name, alias_dims in layout.aliases.items(): - self._aliases[alias_name] = (name, alias_dims[:]) def has_layout(self, name: str) -> bool: """Return whether a named layout is registered.""" return name in self._layouts - def has_alias(self, name: str) -> bool: - """Return whether an alias is registered.""" - return name in self._aliases - - def get_alias_dims(self, name: str) -> list[str]: - """Return a copy of the dimensions referenced by an alias.""" - if name not in self._aliases: - raise KeyError(f"Alias {name!r} is not registered") - _, alias_dims = self._aliases[name] - return alias_dims[:] - def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessGroup | None: r"""Create a process group based on a list of dimension names @@ -308,7 +259,7 @@ def get_rank_enum( Args: dims: Dimension name or list of dimension names. layout_name: Optional registered layout name. When unset, the - owning layout is inferred from dims or aliases. + owning layout is inferred from dims. Returns: List of rank lists (one per subgroup). @@ -316,7 +267,6 @@ def get_rank_enum( if layout_name is None: layout_name, ordered_dims, _ = self._resolve_dims(dims) else: - dims = self._expand_alias(dims, layout_name) ordered_dims, _ = self._order_dims_for_layout(dims, layout_name) return self._gen_rank_enum_for_layout(ordered_dims, layout_name) @@ -405,11 +355,6 @@ def _order_dims_for_layout( return ordered_dims, unique_group_key def _resolve_dims(self, dims: Union[str, list[str]]) -> Tuple[str, list[str], str]: - if isinstance(dims, str) and dims in self._aliases: - layout_name, alias_dims = self._aliases[dims] - ordered_dims, _ = self._order_dims_for_layout(alias_dims, layout_name) - return layout_name, ordered_dims, dims - raw_dims = [dims] if isinstance(dims, str) else dims if all(dim in self.dim_names for dim in raw_dims): @@ -431,27 +376,9 @@ def _resolve_dims(self, dims: Union[str, list[str]]) -> Tuple[str, list[str], st ) layout_name = candidate_layouts[0] - if len(raw_dims) > 1: - aliases = sorted(self._layouts[layout_name].aliases) - raise ValueError( - f"Composite dimensions {raw_dims} from registered layout {layout_name!r} " - f"must use an explicit alias. Available aliases: {aliases}" - ) - ordered_dims, unique_group_key = self._order_dims_for_layout(raw_dims, layout_name) return layout_name, ordered_dims, unique_group_key - def _expand_alias(self, dims: Union[str, list[str]], layout_name: str) -> Union[str, list[str]]: - if not isinstance(dims, str) or dims not in self._aliases: - return dims - - alias_layout_name, alias_dims = self._aliases[dims] - if alias_layout_name != layout_name: - raise ValueError( - f"Alias {dims!r} belongs to layout {alias_layout_name!r}, not {layout_name!r}" - ) - return alias_dims[:] - def is_current_rank_in_grid(self) -> bool: """Check if the current rank belongs to this grid. diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py index 16289ecba32..85870726269 100644 --- a/megatron/core/models/common/language_module/language_module.py +++ b/megatron/core/models/common/language_module/language_module.py @@ -180,9 +180,7 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor: elif self.config.cross_entropy_fusion_impl == 'native': loss = fused_vocab_parallel_cross_entropy(logits, labels, self.pg_collection.tp) else: - loss = tensor_parallel.vocab_parallel_cross_entropy( - logits, labels, tp_group=self.tp_group - ) + loss = tensor_parallel.vocab_parallel_cross_entropy(logits, labels) # [s b] => [b, s] loss = loss.transpose(0, 1).contiguous() diff --git a/megatron/core/models/hybrid/hybrid_layer_allocation.py b/megatron/core/models/hybrid/hybrid_layer_allocation.py index df5a91bf6b1..67103fe67f1 100644 --- a/megatron/core/models/hybrid/hybrid_layer_allocation.py +++ b/megatron/core/models/hybrid/hybrid_layer_allocation.py @@ -354,8 +354,8 @@ def select_pipeline_segment( uneven PP. Only valid when the pattern has no pipe separators. last_stage_layers: Number of layers on the last pipeline stage for uneven PP. Only valid when the pattern has no pipe separators. - tp_group: Tensor parallel process group used for rank-local logging. - dp_cp_group: Data/context parallel process group used for rank-local logging. + tp_group: Optional tensor-parallel process group used for per-stage logging. + dp_cp_group: Optional data/context-parallel process group used for per-stage logging. Returns: Tuple of (layer_type_list, layer_offset) where layer_type_list is diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index 00c3545d698..ba33352533c 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -186,14 +186,20 @@ def __init__( self.mtp_pattern = parsed.mtp_pattern self.mtp_num_depths = parsed.mtp_num_depths + logging_tp_group = getattr(self.pg_collection, 'tp', None) + logging_dp_cp_group = getattr(self.pg_collection, 'dp_cp', None) + if logging_tp_group is None or logging_dp_cp_group is None: + logging_tp_group = None + logging_dp_cp_group = None + layer_type_list, layer_offset = select_pipeline_segment( parsed.main_pattern or '', self.pg_collection.pp, vp_stage, first_stage_layers=self.config.num_layers_in_first_pipeline_stage, last_stage_layers=self.config.num_layers_in_last_pipeline_stage, - tp_group=getattr(self.pg_collection, 'tp', None), - dp_cp_group=getattr(self.pg_collection, 'dp_cp', None), + tp_group=logging_tp_group, + dp_cp_group=logging_dp_cp_group, ) # Determine if MTP is needed (based on pattern parsing) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index 66bc273255e..d0e48a19b65 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -379,14 +379,11 @@ def forward( return self._forward_encoders(modality_inputs, input_tensors), loss_mask if self.role.has_language_module: - return self._forward_language_module( - input_ids, - position_ids, - attention_mask, + return ( + self._forward_language_module( + input_ids, position_ids, attention_mask, labels, input_tensors + ), loss_mask, - labels, - input_tensors, - packing_kwargs, ) raise RuntimeError(f"Rank has no modules assigned in role: {self.role}") @@ -429,11 +426,9 @@ def _forward_language_module( input_ids: torch.Tensor, position_ids: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], - loss_mask: Optional[torch.Tensor], labels: Optional[torch.Tensor], input_tensors: Optional[Dict[str, torch.Tensor]], - packing_kwargs: Optional[dict] = None, - ): + ) -> torch.Tensor: """Forward pass for language module on this rank. Args: @@ -446,15 +441,6 @@ def _forward_language_module( Returns: Language model output (hidden states, logits, or loss depending on stage) """ - packed_seq_params = None - if packing_kwargs is not None: - for key in packing_kwargs: - if 'cu_seqlens' in key and packing_kwargs[key] is not None: - packing_kwargs[key] = packing_kwargs[key].to(dtype=torch.int32) - packed_seq_params = PackedSeqParams(**packing_kwargs) - packed_seq_params.qkv_format = 'thd' - logger.debug(f"Packed sequence parameters: {packed_seq_params}") - lang_name = MIMO_LANGUAGE_MODULE_KEY if self.role.is_first_stage(lang_name): @@ -481,14 +467,11 @@ def _forward_language_module( if self.partition_adapter is not None: combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() - combined_embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=combined_embeddings, - labels=labels, - loss_mask=loss_mask, - attention_mask=attention_mask, - packed_seq_params=packed_seq_params, - ) + combined_embeddings, labels, _, attention_mask, _ = self.partition_adapter.shard( + embeddings=combined_embeddings, + labels=labels, + loss_mask=None, + attention_mask=attention_mask, ) if combined_embeddings is not None: combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() @@ -499,7 +482,6 @@ def _forward_language_module( decoder_input=combined_embeddings, labels=labels, attention_mask=None, - packed_seq_params=packed_seq_params, ) else: # Non-first stage: receive hidden states from previous LM stage @@ -511,31 +493,19 @@ def _forward_language_module( if hasattr(underlying_lm, 'set_input_tensor'): underlying_lm.set_input_tensor(hidden_states) - if self.partition_adapter is not None: - _, labels, loss_mask, attention_mask, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=None, - labels=labels, - loss_mask=loss_mask, - attention_mask=attention_mask, - packed_seq_params=packed_seq_params, - ) - ) - lm_output = self.language_model( input_ids=None, position_ids=None, decoder_input=None, labels=labels, attention_mask=attention_mask, - packed_seq_params=packed_seq_params, ) # Key output for non-last stages so schedule can route to next LM stage if not self.role.is_last_stage(lang_name): - return {lang_name: lm_output}, loss_mask + return {lang_name: lm_output} - return lm_output, loss_mask + return lm_output def _build_colocated_communicators(self): grid_map = self.mimo_config.module_to_grid_map diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index c330b1e9a36..ca5a31eb887 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -38,15 +38,9 @@ class MimoOptimizer(MegatronOptimizer): across all modules via all_reduce MAX. """ - def __init__( - self, - module_infos: Dict[str, ModuleOptimizerInfo], - config: OptimizerConfig, - stats_group: Optional[torch.distributed.ProcessGroup] = None, - ): + def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: OptimizerConfig): self.module_infos = module_infos self.config = config - self.stats_group = stats_group self._active_optimizers: List[MegatronOptimizer] = [ info.optimizer for info in module_infos.values() @@ -76,9 +70,7 @@ def get_grad_norm(self) -> float: module_norm = info.optimizer.get_grad_norm() or 0.0 norm_sq[i] = module_norm**2 - torch.distributed.all_reduce( - norm_sq, op=torch.distributed.ReduceOp.MAX, group=self.stats_group - ) + torch.distributed.all_reduce(norm_sq, op=torch.distributed.ReduceOp.MAX) return torch.sqrt(norm_sq.sum()).item() @torch.no_grad() @@ -89,9 +81,7 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: # if encoder ranks detect inf but LLM ranks don't, the early return # would skip the all_reduce in get_grad_norm(), causing a hang. found_inf_tensor = torch.tensor([found_inf], dtype=torch.float32, device="cuda") - torch.distributed.all_reduce( - found_inf_tensor, op=torch.distributed.ReduceOp.MAX, group=self.stats_group - ) + torch.distributed.all_reduce(found_inf_tensor, op=torch.distributed.ReduceOp.MAX) found_inf = found_inf_tensor.item() > 0 if found_inf: return False, None, None @@ -118,25 +108,25 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: @torch.no_grad() def step_with_ready_grads(self) -> bool: - """Step active module optimizers after gradients have been prepared.""" + """Step each active optimizer using already-ready gradients.""" success = True for opt in self._active_optimizers: success &= opt.step_with_ready_grads() return success def zero_grad(self, set_to_none: bool = True): - """Clear gradients on all active module optimizers.""" + """Clear gradients on each active optimizer.""" for opt in self._active_optimizers: opt.zero_grad(set_to_none) def get_loss_scale(self) -> torch.Tensor: - """Return the loss scale from the first active optimizer, or one for stubs.""" + """Return the active optimizer loss scale.""" if self._active_optimizers: return self._active_optimizers[0].get_loss_scale() return torch.tensor([1.0], dtype=torch.float32, device="cuda") def count_zeros(self) -> int: - """Count zero gradients across all active module optimizers.""" + """Count zero gradients across active optimizers.""" return sum(opt.count_zeros() for opt in self._active_optimizers) @property @@ -306,14 +296,14 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: Only fetches process groups required by the optimizer. Assumes all groups are pre-created in the grid via grid.create_pg() - does not create any new groups. - For extended HyperCommGrid instances with registered expert layouts, the following - groups must be pre-created in the grid before calling this function: + For HyperCommGrid instances with registered expert dimensions, the following + groups must be pre-created before calling this function: grid.create_pg(["dp"]) grid.create_pg(["dp", "cp"]) grid.create_pg(["tp"]) grid.create_pg(["pp"]) grid.create_pg(["tp", "pp"]) - grid.create_pg("tp_ep_pp") + grid.create_pg(["expt_tp", "ep", "pp"]) grid.create_pg("expt_dp") grid.create_pg(["tp", "cp", "dp", "pp"]) @@ -336,10 +326,7 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: required_pgs=['dp', 'dp_cp', 'tp', 'pp', 'mp', 'tp_ep_pp', 'expt_dp', 'intra_dist_opt'], ) except (KeyError, ValueError) as exc: - has_registered_expert_aliases = hasattr(grid, 'has_alias') and ( - grid.has_alias('tp_ep') or grid.has_alias('tp_ep_pp') - ) - if has_registered_expert_aliases: + if hasattr(grid, 'has_layout') and grid.has_layout('expert'): raise exc # Backward-compatible fallback for older tests/grids that encoded EP # directly in the base Cartesian layout. @@ -368,11 +355,7 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: return pg -def get_mimo_optimizer( - mimo_model: "MimoModel", - config: OptimizerConfig, - stats_group: Optional[torch.distributed.ProcessGroup] = None, -) -> MimoOptimizer: +def get_mimo_optimizer(mimo_model: "MimoModel", config: OptimizerConfig) -> MimoOptimizer: """Create optimizer for MimoModel with heterogeneous parallelism.""" from megatron.core.optimizer import get_megatron_optimizer @@ -417,4 +400,4 @@ def get_mimo_optimizer( optimizer=optimizer, grid=grid, pg_collection=pg_collection, is_active=is_active ) - return MimoOptimizer(module_infos, config, stats_group=stats_group) + return MimoOptimizer(module_infos, config) diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py index 0eefccfb70a..c6d3e41aed5 100644 --- a/megatron/core/optimizer/__init__.py +++ b/megatron/core/optimizer/__init__.py @@ -300,7 +300,6 @@ def _get_param_groups( model_chunks: List[MegatronModule], config: OptimizerConfig, config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], - param_group_sync_group: Optional[torch.distributed.ProcessGroup] = None, ) -> List[Dict]: """Create parameter groups for optimizer. @@ -361,12 +360,8 @@ def _get_param_groups( # so we need to align the param groups across ranks, otherwise we may have # runtime error when loading the checkpoint or numerical error when resuming training. params_key = list(params_map.keys()) - gathered_params_key = [ - None for _ in range(torch.distributed.get_world_size(group=param_group_sync_group)) - ] - torch.distributed.all_gather_object( - gathered_params_key, params_key, group=param_group_sync_group - ) + gathered_params_key = [None for _ in range(torch.distributed.get_world_size())] + torch.distributed.all_gather_object(gathered_params_key, params_key) for keys in gathered_params_key: for key in keys: if key not in params_key: @@ -424,7 +419,6 @@ def _get_param_groups_and_buffers( config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], filter_fn: Callable, buffer_name: str, - param_group_sync_group: Optional[torch.distributed.ProcessGroup] = None, ) -> Tuple[List[Dict], Dict[int, List[_ParamAndGradBuffer]]]: """Returns parameter groups and buffer for optimizer. @@ -443,9 +437,7 @@ def _get_param_groups_and_buffers( Returns: List of parameter groups and dictionary of model chunk IDs to buffers. """ - param_groups = _get_param_groups( - model_chunks, config, config_overrides, param_group_sync_group=param_group_sync_group - ) + param_groups = _get_param_groups(model_chunks, config, config_overrides) param_groups = list(filter(filter_fn, param_groups)) buffers = {} for model_chunk_idx, model_chunk in enumerate(model_chunks): @@ -791,10 +783,7 @@ def _get_megatron_emerging_optimizer( # Build param groups and bucket by (optimizer_name, is_expert_parallel). # Layer-wise distributed optimizer handles expert params internally so we skip that split. - param_group_sync_group = getattr(pg_collection, 'intra_dist_opt', None) - all_param_groups = _get_param_groups( - model_chunks, config, config_overrides, param_group_sync_group=param_group_sync_group - ) + all_param_groups = _get_param_groups(model_chunks, config, config_overrides) grouped_param_groups = defaultdict(list) for group in all_param_groups: opt_name = group.get('optimizer', eopt_name) @@ -937,7 +926,6 @@ def get_megatron_optimizer( intra_dp_cp_group_gloo = process_groups_dict['intra_dp_cp_group_gloo'] intra_expt_dp_group_gloo = process_groups_dict['intra_expt_dp_group_gloo'] intra_dist_opt_group = process_groups_dict['intra_dist_opt_group'] - param_group_sync_group = intra_dist_opt_group model_parallel_rank = get_pg_rank(mp_group) @@ -961,7 +949,6 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: True, buffer_name='buffers', - param_group_sync_group=param_group_sync_group, ) optimizer_part = _get_megatron_optimizer_based_on_param_groups( @@ -1012,7 +999,6 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: not g['is_expert_parallel'], buffer_name='buffers', - param_group_sync_group=param_group_sync_group, ) for model_chunk in dense_model_chunks: model_chunk.overlap_param_gather_with_optimizer_step = ( @@ -1050,7 +1036,6 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: g['is_expert_parallel'], buffer_name='expert_parallel_buffers', - param_group_sync_group=param_group_sync_group, ) if dump_param_to_param_group_map is not None: for param_group in moe_param_groups: diff --git a/megatron/core/process_groups_config.py b/megatron/core/process_groups_config.py index 58bae2dd1b7..914722ade29 100644 --- a/megatron/core/process_groups_config.py +++ b/megatron/core/process_groups_config.py @@ -267,14 +267,7 @@ def from_hyper_comm_grid( required_pgs: Optional[List[str]] = None, num_distributed_optimizer_instances: int = 1, ): - """Build a ProcessGroupCollection from an extended HyperCommGrid. - - The grid must expose expert groups via registered layout dimensions - such as ``expt_tp``/``expt_dp`` and aliases such as ``tp_ep_pp``. - When ``create`` is True, the helper owns group creation in a - deterministic order. Otherwise it only reads groups that must already - exist on the grid. - """ + """Build a ProcessGroupCollection from a HyperCommGrid with expert dimensions.""" if num_distributed_optimizer_instances != 1: raise ValueError( "ProcessGroupCollection.from_hyper_comm_grid only supports " @@ -293,8 +286,8 @@ def from_hyper_comm_grid( 'ep': 'ep', 'expt_tp': 'expt_tp', 'expt_dp': 'expt_dp', - 'tp_ep': 'tp_ep', - 'tp_ep_pp': 'tp_ep_pp', + 'tp_ep': ['expt_tp', 'ep'], + 'tp_ep_pp': ['expt_tp', 'ep', 'pp'], 'intra_dist_opt': grid.dim_names, } if required_pgs is None: @@ -304,11 +297,6 @@ def from_hyper_comm_grid( if invalid_pgs: raise ValueError(f"Invalid process groups requested: {invalid_pgs}") - if 'tp_ep_pp' in required_pgs and hasattr(grid, 'get_alias_dims'): - alias_dims = grid.get_alias_dims('tp_ep_pp') - if 'pp' not in alias_dims: - raise ValueError("tp_ep_pp alias must include the shared pipeline dimension 'pp'") - def get_or_create(dims): return grid.create_pg(dims) if create else grid.get_pg(dims) diff --git a/megatron/core/tensor_parallel/cross_entropy.py b/megatron/core/tensor_parallel/cross_entropy.py index 0b43f1ad1b7..8abfad8d3e2 100644 --- a/megatron/core/tensor_parallel/cross_entropy.py +++ b/megatron/core/tensor_parallel/cross_entropy.py @@ -1,10 +1,14 @@ # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -from typing import Optional, Tuple +from typing import Tuple import torch -from megatron.core.parallel_state import get_tensor_model_parallel_group +from megatron.core.parallel_state import ( + get_tensor_model_parallel_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) from .utils import VocabUtility @@ -117,20 +121,21 @@ def calculate_gradients( class _VocabParallelCrossEntropy(torch.autograd.Function): @staticmethod - def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0, tp_group=None): + def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0): """Vocab parallel cross entropy forward function.""" vocab_parallel_logits, logits_max = VocabParallelCrossEntropy.calculate_logits_max( vocab_parallel_logits ) - tp_group = get_tensor_model_parallel_group() if tp_group is None else tp_group - torch.distributed.all_reduce(logits_max, op=torch.distributed.ReduceOp.MAX, group=tp_group) + torch.distributed.all_reduce( + logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group() + ) # Get the partition's vocab indices get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size partition_vocab_size = vocab_parallel_logits.size()[-1] - rank = torch.distributed.get_rank(tp_group) - world_size = torch.distributed.get_world_size(tp_group) + rank = get_tensor_model_parallel_rank() + world_size = get_tensor_model_parallel_world_size() vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size) (target_mask, masked_target_1d, predicted_logits, sum_exp_logits, exp_logits) = ( @@ -141,11 +146,15 @@ def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0, tp_group=No # All reduce is needed to get the chunks from other GPUs. torch.distributed.all_reduce( - predicted_logits, op=torch.distributed.ReduceOp.SUM, group=tp_group + predicted_logits, + op=torch.distributed.ReduceOp.SUM, + group=get_tensor_model_parallel_group(), ) torch.distributed.all_reduce( - sum_exp_logits, op=torch.distributed.ReduceOp.SUM, group=tp_group + sum_exp_logits, + op=torch.distributed.ReduceOp.SUM, + group=get_tensor_model_parallel_group(), ) exp_logits, loss = VocabParallelCrossEntropy.calculate_cross_entropy_loss( @@ -204,15 +213,10 @@ def backward(ctx, grad_output): grad_2d, arange_1d, masked_target_1d, softmax_update, grad_input, grad_output ) - return grad_input, None, None, None + return grad_input, None, None -def vocab_parallel_cross_entropy( - vocab_parallel_logits, - target, - label_smoothing=0.0, - tp_group: Optional[torch.distributed.ProcessGroup] = None, -): +def vocab_parallel_cross_entropy(vocab_parallel_logits, target, label_smoothing=0.0): """ Performs cross entropy loss when logits are split across tensor parallel ranks @@ -225,9 +229,5 @@ def vocab_parallel_cross_entropy( label_smoothing: smoothing factor, must be in range [0.0, 1.0) default is no smoothing (=0.0) - tp_group: tensor-parallel process group. Defaults to Megatron's global tensor-parallel - group for backward compatibility. """ - return _VocabParallelCrossEntropy.apply( - vocab_parallel_logits, target, label_smoothing, tp_group - ) + return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing) diff --git a/tests/unit_tests/models/test_mimo_1f1b_schedule.py b/tests/unit_tests/models/test_mimo_1f1b_schedule.py index 004ec52697f..09595b94131 100644 --- a/tests/unit_tests/models/test_mimo_1f1b_schedule.py +++ b/tests/unit_tests/models/test_mimo_1f1b_schedule.py @@ -99,12 +99,7 @@ def create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1, ep=1, expt_tp=None, rank_offset=offset, backend="nccl", ) - grid.register_layout( - "expert", - [expt_tp, ep, expt_dp, pp], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, - ) + grid.register_layout("expert", [expt_tp, ep, expt_dp, pp], ["expt_tp", "ep", "expt_dp", "pp"]) grid.create_pg(["tp"]) grid.create_pg(["cp"]) grid.create_pg(["pp"]) @@ -117,8 +112,8 @@ def create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1, ep=1, expt_tp=None, grid.create_pg(["tp", "pp"]) grid.create_pg(["tp", "cp", "dp"]) grid.create_pg(["tp", "cp", "pp", "dp"]) - grid.create_pg("tp_ep") - grid.create_pg("tp_ep_pp") + grid.create_pg(["expt_tp", "ep"]) + grid.create_pg(["expt_tp", "ep", "pp"]) _active_grids.append(grid) return grid diff --git a/tests/unit_tests/ssm/test_hybrid_layer_allocation.py b/tests/unit_tests/ssm/test_hybrid_layer_allocation.py index fe0d7c2dc1e..130f781792d 100644 --- a/tests/unit_tests/ssm/test_hybrid_layer_allocation.py +++ b/tests/unit_tests/ssm/test_hybrid_layer_allocation.py @@ -466,6 +466,17 @@ def test_logging_is_called(self, mock_log): select_pipeline_segment("M*M*", pp_group=None, vp_stage=None) mock_log.assert_called_once() + @patch('megatron.core.models.hybrid.hybrid_layer_allocation.log_on_each_pipeline_stage') + def test_logging_receives_explicit_groups(self, mock_log): + """Explicit rank groups are forwarded to the per-stage logging helper.""" + tp_group = object() + dp_cp_group = object() + select_pipeline_segment( + "M*M*", pp_group=None, vp_stage=None, tp_group=tp_group, dp_cp_group=dp_cp_group + ) + assert mock_log.call_args.kwargs["tp_group"] is tp_group + assert mock_log.call_args.kwargs["dp_cp_group"] is dp_cp_group + @patch('megatron.core.models.hybrid.hybrid_layer_allocation.log_on_each_pipeline_stage') def test_mutual_exclusivity_pipes_with_first_stage(self, mock_log): """Pipe separators + first_stage_layers should raise ValueError.""" diff --git a/tests/unit_tests/test_hyper_comm_grid.py b/tests/unit_tests/test_hyper_comm_grid.py index 0e89cf896c3..a4c307425a6 100644 --- a/tests/unit_tests/test_hyper_comm_grid.py +++ b/tests/unit_tests/test_hyper_comm_grid.py @@ -314,12 +314,7 @@ def test_register_layout_for_expert_groups(self, monkeypatch): monkeypatch.setenv("WORLD_SIZE", "16") grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) - grid.register_layout( - "expert", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, - ) + grid.register_layout("expert", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) assert grid.has_layout("base") assert grid.has_layout("expert") @@ -340,47 +335,42 @@ def test_register_layout_for_expert_groups(self, monkeypatch): [10, 14], [11, 15], ] - assert grid.get_rank_enum("tp_ep") == [ + assert grid.get_rank_enum(["expt_tp", "ep"]) == [ [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15], ] - assert grid.get_rank_enum("tp_ep_pp") == [ + assert grid.get_rank_enum(["expt_tp", "ep", "pp"]) == [ [0, 1, 2, 3, 8, 9, 10, 11], [4, 5, 6, 7, 12, 13, 14, 15], ] @patch('torch.distributed.get_rank', return_value=0) @patch('torch.distributed.new_subgroups_by_enumeration') - def test_create_pg_from_registered_layout_and_alias( + def test_create_pg_from_registered_layout_composite( self, mock_new_subgroups, _mock_get_rank, monkeypatch ): - """Test create/get with registered expert dims and aliases.""" + """Test create/get with registered expert dimensions.""" monkeypatch.setenv("WORLD_SIZE", "16") mock_ep_pg = MagicMock(spec=dist.ProcessGroup) mock_tp_ep_pg = MagicMock(spec=dist.ProcessGroup) mock_new_subgroups.side_effect = [(mock_ep_pg, None), (mock_tp_ep_pg, None)] grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) - grid.register_layout( - "expert", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"]}, - ) + grid.register_layout("expert", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) assert grid.create_pg("ep") == mock_ep_pg - assert grid.create_pg("tp_ep") == mock_tp_ep_pg + assert grid.create_pg(["expt_tp", "ep"]) == mock_tp_ep_pg assert grid.get_pg("ep") == mock_ep_pg - assert grid.get_pg("tp_ep") == mock_tp_ep_pg + assert grid.get_pg(["expt_tp", "ep"]) == mock_tp_ep_pg assert "ep" in grid._pgs - assert "tp_ep" in grid._pgs + assert "ep-expt_tp" in grid._pgs first_call_enum = mock_new_subgroups.call_args_list[0].args[0] second_call_enum = mock_new_subgroups.call_args_list[1].args[0] assert first_call_enum == grid.get_rank_enum("ep") - assert second_call_enum == grid.get_rank_enum("tp_ep") + assert second_call_enum == grid.get_rank_enum(["expt_tp", "ep"]) def test_registered_layout_rejects_invalid_shapes_and_collisions(self, monkeypatch): """Test validation for registered layouts.""" @@ -396,43 +386,11 @@ def test_registered_layout_rejects_invalid_shapes_and_collisions(self, monkeypat with pytest.raises(ValueError, match="collides.*different rank enumeration"): grid.register_layout("bad_tp", [1, 4, 2, 2], ["expt_tp", "tp", "expt_dp", "pp"]) - with pytest.raises(ValueError, match="conflicts with an existing dimension"): - grid.register_layout( - "bad_alias", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp": ["expt_tp", "ep"]}, - ) - - with pytest.raises(ValueError, match="cannot contain '-'"): - grid.register_layout( - "bad_alias_key", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"dp-cp": ["expt_tp", "ep"]}, - ) - - with pytest.raises(ValueError, match="duplicate dimensions"): - grid.register_layout( - "bad_alias_dims", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"bad_dims": ["ep", "ep"]}, - ) - - def test_registered_layout_rejects_implicit_cross_layout_composites(self, monkeypatch): - """Composite expert groups must use registered aliases.""" + def test_registered_layout_rejects_cross_layout_composites(self, monkeypatch): + """Composite groups must come from one layout.""" monkeypatch.setenv("WORLD_SIZE", "16") grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) - grid.register_layout( - "expert", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"]}, - ) - - with pytest.raises(ValueError, match="must use an explicit alias"): - grid.get_rank_enum(["expt_tp", "ep"]) + grid.register_layout("expert", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) with pytest.raises(ValueError, match="single registered layout"): grid.get_rank_enum(["tp", "ep"]) diff --git a/tests/unit_tests/test_process_groups_config.py b/tests/unit_tests/test_process_groups_config.py index 546eecc77bd..fa1c883cd35 100644 --- a/tests/unit_tests/test_process_groups_config.py +++ b/tests/unit_tests/test_process_groups_config.py @@ -118,7 +118,6 @@ def pg_for(dims): return pgs[key] grid.get_pg.side_effect = pg_for - grid.get_alias_dims.return_value = ["expt_tp", "ep", "pp"] collection = ProcessGroupCollection.from_hyper_comm_grid( grid, @@ -131,7 +130,7 @@ def pg_for(dims): assert collection.dp_cp is pgs[('dp', 'cp')] assert collection.mp is pgs[('tp', 'pp')] assert collection.expt_dp is pgs['expt_dp'] - assert collection.tp_ep_pp is pgs['tp_ep_pp'] + assert collection.tp_ep_pp is pgs[('expt_tp', 'ep', 'pp')] assert collection.intra_dist_opt is pgs[('tp', 'cp', 'dp', 'pp')] assert collection.intra_dp_cp is collection.dp_cp assert collection.intra_expt_dp is collection.expt_dp @@ -144,7 +143,7 @@ def test_from_hyper_comm_grid_rejects_multi_instance_distopt(self, mocker): ProcessGroupCollection.from_hyper_comm_grid(grid, num_distributed_optimizer_instances=2) def test_from_hyper_comm_grid_creates_from_real_extended_grid(self, mocker, monkeypatch): - """Test helper against real HyperCommGrid alias resolution without distributed init.""" + """Test helper against a real HyperCommGrid expert layout without distributed init.""" monkeypatch.setenv("WORLD_SIZE", "16") mocker.patch('torch.distributed.get_rank', return_value=0) mock_new_subgroups = mocker.patch('torch.distributed.new_subgroups_by_enumeration') @@ -160,12 +159,7 @@ def make_pg(rank_enum, **_kwargs): mock_new_subgroups.side_effect = make_pg grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) - grid.register_layout( - "expert", - [1, 4, 2, 2], - ["expt_tp", "ep", "expt_dp", "pp"], - aliases={"tp_ep": ["expt_tp", "ep"], "tp_ep_pp": ["expt_tp", "ep", "pp"]}, - ) + grid.register_layout("expert", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) collection = ProcessGroupCollection.from_hyper_comm_grid( grid, @@ -191,22 +185,20 @@ def make_pg(rank_enum, **_kwargs): assert collection.ep is grid.get_pg("ep") assert collection.expt_tp is grid.get_pg("expt_tp") assert collection.expt_dp is grid.get_pg("expt_dp") - assert collection.tp_ep is grid.get_pg("tp_ep") - assert collection.tp_ep_pp is grid.get_pg("tp_ep_pp") + assert collection.tp_ep is grid.get_pg(["expt_tp", "ep"]) + assert collection.tp_ep_pp is grid.get_pg(["expt_tp", "ep", "pp"]) assert collection.intra_dist_opt is grid.get_pg(["tp", "cp", "dp", "pp"]) assert collection.intra_dp_cp is collection.dp_cp assert collection.intra_expt_dp is collection.expt_dp assert collection.inter_dist_opt is None - def test_from_hyper_comm_grid_rejects_tp_ep_pp_without_shared_pp(self, monkeypatch): - """tp_ep_pp must include the same pp dimension used by the base layout.""" + def test_from_hyper_comm_grid_rejects_missing_expert_pp(self, monkeypatch): + """tp_ep_pp requires the registered expert layout to include pp.""" monkeypatch.setenv("WORLD_SIZE", "4") grid = HyperCommGrid([2, 2], ["tp", "pp"]) - grid.register_layout( - "expert", [2, 2], ["ep", "expert_pp"], aliases={"tp_ep_pp": ["ep", "expert_pp"]} - ) + grid.register_layout("expert", [2, 2], ["ep", "expert_pp"]) - with pytest.raises(ValueError, match="shared pipeline dimension 'pp'"): + with pytest.raises(ValueError, match="Dimensions .*pp"): ProcessGroupCollection.from_hyper_comm_grid( grid, create=True, required_pgs=['tp_ep_pp'] ) From 3cf88ab5a5c7bb888f72a0275ab8148c5fc464c0 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 15:06:59 +0000 Subject: [PATCH 06/44] NMFW-464 add hetero Energon training path --- examples/mimo/data/__init__.py | 14 +- examples/mimo/data/hetero_energon.py | 533 ++++++++++++++++++ .../mimo/model_providers/nemotron_moe_vlm.py | 24 +- examples/mimo/scripts/install_energon.sh | 24 + .../run_hetero_nemotron_20l_energon_train.sh | 99 ++++ examples/mimo/train_hetero.py | 4 +- examples/mimo/training/hetero/args.py | 49 +- examples/mimo/training/hetero/loop.py | 31 +- examples/mimo/training/hetero/step.py | 14 + megatron/core/models/mimo/model/base.py | 95 +++- 10 files changed, 852 insertions(+), 35 deletions(-) create mode 100644 examples/mimo/data/hetero_energon.py create mode 100755 examples/mimo/scripts/install_energon.sh create mode 100755 examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh diff --git a/examples/mimo/data/__init__.py b/examples/mimo/data/__init__.py index df73bc4abd5..be521ff65cd 100644 --- a/examples/mimo/data/__init__.py +++ b/examples/mimo/data/__init__.py @@ -1,5 +1,11 @@ -from .energon_avlm_task_encoder import VisionAudioQASample +"""MIMO data providers and task encoders.""" -all = [ - VisionAudioQASample, -] +__all__ = ["VisionAudioQASample"] + + +def __getattr__(name): + if name == "VisionAudioQASample": + from .energon_avlm_task_encoder import VisionAudioQASample + + return VisionAudioQASample + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py new file mode 100644 index 00000000000..2dcca22b197 --- /dev/null +++ b/examples/mimo/data/hetero_energon.py @@ -0,0 +1,533 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Energon multimodal iterator for heterogeneous MIMO training.""" + +from __future__ import annotations + +import warnings +from typing import NamedTuple, Optional + +import torch +import torch.distributed as dist + +from examples.mimo.model_providers.nemotron_moe_vlm import NEMOTRON_VISION_ENCODER_KEY +from examples.mimo.training.hetero.topology import get_grid_coordinate, is_rank_in_grid +from examples.mimo.utils.hetero import debug_rank, is_process_group_member + + +class ExpandedSample(NamedTuple): + """Token row and expanded-coordinate segment boundaries.""" + + tokens: torch.Tensor + labels: torch.Tensor + cu_lengths: torch.Tensor + kept_tiles: int + + +def build_energon_iterator(args, topology): + """Build an Energon iterator for the current rank, or return None if unused.""" + from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage + + encoder_grid = topology.encoder_grid + llm_grid = topology.llm_grid + encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( + encoder_grid.get_pg("pp") + ) + llm_needs_data = is_rank_in_grid(llm_grid) and ( + is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) + ) + + if encoder_needs_data: + return _build_iterator_for_grid(args, encoder_grid) + if llm_needs_data: + return _build_iterator_for_grid(args, llm_grid) + return None + + +def validate_energon_data_alignment(data_iterator, topology) -> None: + """Check the first actual-data batch aligns across non-colocated module grids.""" + if not dist.is_initialized(): + return + + signature = data_iterator.peek_signature() if data_iterator is not None else None + local = (get_current_dp_lane(topology), signature) + gathered = [None for _ in range(dist.get_world_size())] + dist.all_gather_object(gathered, local) + + signatures_by_lane = {} + for lane, candidate in gathered: + if lane < 0 or candidate is None: + continue + signatures_by_lane.setdefault(lane, set()).add(candidate) + mismatched = {lane: values for lane, values in signatures_by_lane.items() if len(values) > 1} + if mismatched: + raise RuntimeError(f"hetero Energon data loaders diverged across grids: {mismatched}") + + +def get_current_dp_lane(topology) -> int: + """Return the active module DP lane for this rank, or -1 for inactive ranks.""" + if is_rank_in_grid(topology.encoder_grid): + return get_grid_coordinate(topology.encoder_grid, "dp") + if is_rank_in_grid(topology.llm_grid): + return get_grid_coordinate(topology.llm_grid, "dp") + return -1 + + +def _build_iterator_for_grid(args, grid): + """Build a deterministic per-DP-lane loader for one module grid.""" + tp_group = grid.get_pg("tp") + if get_grid_coordinate(grid, "tp") != 0: + return EnergonIterator(None, tp_group=tp_group, source_rank=False) + + from megatron.energon import NoCachePool, WorkerConfig, get_loader, get_train_dataset + + tokenizer = _build_tokenizer(args) + encoder = MimoMultiModalPackingEncoder.from_args(args, tokenizer) + dp_rank = get_grid_coordinate(grid, "dp") + worker_config = WorkerConfig( + rank=dp_rank, world_size=args.llm_dp, num_workers=args.num_workers, data_parallel_group=None + ) + debug_rank( + "building energon dataloader " + f"dp_rank={dp_rank} dp_world={args.llm_dp} batch_size={args.micro_batch_size}" + ) + dataset = get_train_dataset( + args.data_path, + batch_size=args.micro_batch_size, + task_encoder=encoder, + worker_config=worker_config, + packing_buffer_size=args.packing_buffer_size, + shuffle_buffer_size=args.shuffle_buffer_size, + max_samples_per_sequence=args.max_samples_per_sequence, + ) + return EnergonIterator( + get_loader(dataset, cache_pool=NoCachePool()), tp_group=tp_group, source_rank=True + ) + + +def _build_tokenizer(args): + from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( + MegatronMultimodalTokenizer, + ) + + return MegatronMultimodalTokenizer( + path=args.tokenizer_model, + prompt_format=args.tokenizer_prompt_format, + special_tokens=[args.image_token], + image_tag_type=args.image_tag_type, + force_system_message=args.force_system_message, + ) + + +class TokenizerAdapter: + """Wrap Megatron's multimodal tokenizer for Energon's tokenizer protocol.""" + + def __init__(self, megatron_tokenizer) -> None: + self._tok = megatron_tokenizer + self._hf = megatron_tokenizer.tokenizer + + @property + def pad_token_id(self) -> int: + """Return the tokenizer pad id.""" + return self._tok.pad + + @property + def eos_token_id(self) -> int: + """Return the tokenizer EOS id.""" + return self._tok.eod + + def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: + """Encode text with the wrapped HuggingFace tokenizer.""" + return self._hf.encode(text, add_special_tokens=add_special_tokens) + + def decode(self, token_ids, skip_special_tokens: bool = False) -> str: + """Decode token ids with the wrapped HuggingFace tokenizer.""" + return self._hf.decode(token_ids, skip_special_tokens=skip_special_tokens) + + def convert_tokens_to_ids(self, tokens): + """Convert special tokens to ids.""" + return self._tok.convert_tokens_to_ids(tokens) + + +def _get_multimodal_encoder_base(): + from megatron.energon.task_encoder.multimodal import MultiModalPackingEncoder + + return MultiModalPackingEncoder + + +def _get_pretraining_conversation_cooker(): + from megatron.energon.task_encoder.cooking import Cooker + + return Cooker( + cook_pretraining_conversation, has_subflavors={"cook": "pretraining_conversation"} + ) + + +_MULTIMODAL_PACKING_ENCODER_BASE = _get_multimodal_encoder_base() + + +def _get_energon_cooker_decorators(): + from megatron.energon.task_encoder.base import stateless + from megatron.energon.task_encoder.cooking import cooker + + return stateless, cooker + + +_stateless, _cooker = _get_energon_cooker_decorators() + + +@_stateless +@_cooker(need_cache=True) +def cook_pretraining_conversation(sample: dict, cache, media_source=None): + """Cook fragment-style pretraining conversation samples into multimodal text.""" + from megatron.energon.task_encoder.cooking import basic_sample_keys + from megatron.energon.task_encoder.multimodal.cookers._image_util import to_pil_image + from megatron.energon.task_encoder.multimodal.sample_types import ImageRef, MultiModalSample + + text_parts = [] + image_refs = [] + for turn in sample["json"]["conversation"]: + sender = turn.get("sender", "user") + content_parts = [] + for fragment in turn.get("fragments", []): + fragment_type = fragment.get("t") + value = fragment.get("value", "") + if fragment_type == "image": + content_parts.append("") + ext = value.rsplit(".", 1)[-1] if "." in value else "" + if ext in sample: + image = to_pil_image(sample[ext]) + else: + image = to_pil_image(cache.get(media_source, value, sample=sample)) + width, height = image.size + image_refs.append(ImageRef(data=image, width=width, height=height)) + elif fragment_type == "text": + content_parts.append(value) + content = "".join(content_parts).strip() + if content: + text_parts.append(f"{sender}: {content}") + + return MultiModalSample( + **basic_sample_keys(sample), text="\n".join(text_parts), images=image_refs + ) + + +class MimoMultiModalPackingEncoder(_MULTIMODAL_PACKING_ENCODER_BASE): + """Adapt Energon multimodal packing batches to MIMO's forward signature.""" + + cookers = list(_MULTIMODAL_PACKING_ENCODER_BASE.cookers) + [ + _get_pretraining_conversation_cooker() + ] + + @classmethod + def from_args(cls, args, tokenizer): + """Construct the encoder from hetero training args.""" + from megatron.energon.task_encoder.multimodal import PackingConfig, VisionConfig + + vision_config = VisionConfig( + img_h=args.img_h, + img_w=args.img_w, + patch_dim=args.patch_dim, + vision_model_type=getattr(args, "vision_model_type", "radio"), + disable_vision_class_token=getattr(args, "disable_vision_class_token", True), + pixel_shuffle=getattr(args, "pixel_shuffle", True), + max_num_tiles=args.num_image_tiles, + use_tiling=getattr(args, "use_tiling", True), + use_thumbnail=getattr(args, "use_thumbnail", True), + class_token_len=args.class_token_len, + conv_merging=getattr(args, "conv_merging", False), + use_tile_tags=getattr(args, "use_tile_tags", False), + use_image_break_token=getattr(args, "image_break_token", None) is not None, + use_area_weighted_aspect_ratio=getattr(args, "use_area_weighted_aspect_ratio", False), + dynamic_resolution=getattr(args, "dynamic_resolution", False), + ) + packing_config = PackingConfig( + seq_length=args.seq_length, pad_id=args.pad_token_id, image_token_id=args.image_token_id + ) + return cls( + vision_config=vision_config, + packing_config=packing_config, + tokenizer=TokenizerAdapter(tokenizer), + encoder_name=NEMOTRON_VISION_ENCODER_KEY, + encoder_input_key="x", + target_seq_length=args.seq_length, + ) + + def __init__( + self, + vision_config, + packing_config, + tokenizer, + encoder_name: str, + encoder_input_key: str, + target_seq_length: Optional[int], + ) -> None: + super().__init__(vision_config, packing_config, tokenizer) + from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings + + self.encoder_name = encoder_name + self.encoder_input_key = encoder_input_key + self._target_seq_length = target_seq_length + self._embeddings_per_tile = get_num_image_embeddings( + img_h=vision_config.img_h, + img_w=vision_config.img_w, + patch_dim=vision_config.patch_dim, + class_token_len=vision_config.class_token_len, + disable_vision_class_token=vision_config.disable_vision_class_token, + pixel_shuffle=vision_config.pixel_shuffle, + conv_merging=vision_config.conv_merging, + use_tile_tags=vision_config.use_tile_tags, + max_num_tiles=vision_config.max_num_tiles, + use_image_break_token=vision_config.use_image_break_token, + ) + + def batch(self, samples: list) -> dict: + """Expand image placeholders and return a MIMO-compatible batch.""" + image_token_id = self.packing_config.image_token_id + ignore_index = self.packing_config.ignore_index + pad_id = self.packing_config.pad_id + + expanded_samples = [] + all_images = [] + for sample in samples: + expanded = self._expand_sample(sample, image_token_id, ignore_index) + expanded_samples.append(expanded) + all_images.extend(sample.images[: expanded.kept_tiles]) + token_rows = [sample.tokens for sample in expanded_samples] + max_len = self._target_seq_length or max(len(row) for row in token_rows) + input_ids = torch.full((len(samples), max_len), pad_id, dtype=torch.long) + labels = torch.full((len(samples), max_len), ignore_index, dtype=torch.long) + for row_idx, expanded in enumerate(expanded_samples): + tokens = expanded.tokens + input_ids[row_idx, : len(tokens)] = tokens + labels[row_idx, : expanded.labels.numel()] = expanded.labels + + loss_mask = (labels != ignore_index).float() + loss_mask[labels == image_token_id] = 0.0 + position_ids = torch.arange(max_len).unsqueeze(0).expand(len(samples), -1).contiguous() + result = { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "position_ids": position_ids, + } + image_tensor = None + if all_images: + image_tensor = self.tiling_strategy.stack(all_images)[0] + result["modality_inputs"] = { + "images": {self.encoder_name: {self.encoder_input_key: image_tensor}} + } + packing_kwargs = self._build_packing_kwargs( + [sample.cu_lengths for sample in expanded_samples], max_len + ) + if packing_kwargs is not None: + result["packing_kwargs"] = packing_kwargs + return result + + def _expand_sample(self, sample, image_token_id: int, ignore_index: int) -> ExpandedSample: + """Expand one image token into one placeholder per image embedding.""" + if not hasattr(sample, "labels"): + raise RuntimeError("Energon multimodal samples must provide labels") + + tokens = [] + labels = [] + image_idx = 0 + kept_tiles = 0 + budget = self._target_seq_length + sample_labels = sample.labels.reshape(-1).tolist() + if len(sample_labels) < sample.tokens.numel(): + raise RuntimeError( + "Energon multimodal sample labels must be at least as long as tokens" + ) + + for idx, token in enumerate(sample.tokens.tolist()): + if token != image_token_id: + if budget is not None and len(tokens) + 1 > budget: + break + tokens.append(token) + labels.append(int(sample_labels[idx])) + continue + + num_tiles = sample.num_tiles[image_idx] if image_idx < len(sample.num_tiles) else 1 + num_placeholders = num_tiles * self._embeddings_per_tile + if budget is not None and len(tokens) + num_placeholders > budget: + warnings.warn( + "Energon sample truncated at an image boundary to fit " + f"target sequence length {budget}.", + stacklevel=2, + ) + break + tokens.extend([image_token_id] * num_placeholders) + labels.extend([ignore_index] * num_placeholders) + kept_tiles += num_tiles + image_idx += 1 + + return ExpandedSample( + tokens=torch.tensor(tokens, dtype=torch.long), + labels=torch.tensor(labels, dtype=torch.long), + cu_lengths=torch.tensor( + self._expanded_boundaries(sample.cu_lengths, len(tokens)), dtype=torch.long + ), + kept_tiles=kept_tiles, + ) + + @staticmethod + def _expanded_boundaries(cu_lengths: torch.Tensor, expanded_token_count: int) -> list[int]: + """Normalize Energon cumulative lengths in expanded sequence coordinates.""" + boundaries = cu_lengths.to(dtype=torch.long).flatten().tolist() + boundaries = [min(max(int(value), 0), expanded_token_count) for value in boundaries] + if not boundaries or boundaries[0] != 0: + boundaries.insert(0, 0) + if boundaries[-1] != expanded_token_count: + boundaries.append(expanded_token_count) + + deduped = [] + for boundary in boundaries: + if not deduped or boundary != deduped[-1]: + deduped.append(boundary) + return deduped + + @staticmethod + def _build_packing_kwargs( + cu_lengths_by_sample: list[torch.Tensor], max_len: int + ) -> Optional[dict]: + """Build packed-sequence metadata when Energon selected packed samples.""" + is_packed = any(cu_lengths.numel() > 2 for cu_lengths in cu_lengths_by_sample) + if not is_packed: + return None + if len(cu_lengths_by_sample) != 1: + raise RuntimeError("Energon packing requires micro_batch_size=1") + + cu_seqlens = cu_lengths_by_sample[0].to(dtype=torch.int32).clamp(max=max_len) + if cu_seqlens[0] != 0: + cu_seqlens = torch.cat([torch.tensor([0], dtype=torch.int32), cu_seqlens]) + if cu_seqlens[-1] != max_len: + cu_seqlens = torch.cat([cu_seqlens, torch.tensor([max_len], dtype=torch.int32)]) + segment_lens = cu_seqlens[1:] - cu_seqlens[:-1] + max_seqlen = segment_lens.max() + return { + "cu_seqlens_q": cu_seqlens, + "cu_seqlens_kv": cu_seqlens, + "cu_seqlens_q_padded": cu_seqlens, + "cu_seqlens_kv_padded": cu_seqlens, + "max_seqlen_q": max_seqlen, + "max_seqlen_kv": max_seqlen, + "total_tokens": max_len, + } + + +class EnergonIterator: + """Endless wrapper around an Energon dataloader with TP-rank-0 ownership.""" + + def __init__(self, dataloader, tp_group=None, source_rank: bool = True) -> None: + self._dataloader = dataloader + self._iterator = iter(dataloader) if dataloader is not None else None + self._tp_group = tp_group + self._source_rank = source_rank + self._prefetched = None + + def __iter__(self): + return self + + def __next__(self): + if self._prefetched is not None: + batch = self._prefetched + self._prefetched = None + return batch + + batch = self._next_local_batch() if self._source_rank else None + if is_process_group_member(self._tp_group) and self._tp_group.size() > 1: + obj = [batch] + dist.broadcast_object_list(obj, src=self._tp_source_rank(), group=self._tp_group) + batch = obj[0] + return batch + + def peek_signature(self): + """Read and retain the next batch, returning a compact deterministic signature.""" + if self._prefetched is None: + self._prefetched = next(self) + return self._batch_signature(self._prefetched) + + def _next_local_batch(self): + """Read the next local Energon batch on the TP source rank.""" + try: + return next(self._iterator) + except StopIteration: + self._iterator = iter(self._dataloader) + return next(self._iterator) + + def _tp_source_rank(self) -> int: + """Return the global source rank for the local TP batch broadcast.""" + if hasattr(dist, "get_global_rank"): + return dist.get_global_rank(self._tp_group, 0) + return dist.get_process_group_ranks(self._tp_group)[0] + + @classmethod + def _batch_signature(cls, batch: dict) -> tuple[int, ...]: + """Return a compact signature for cross-grid data-alignment checks.""" + image_tensor = cls._nested_get(batch, ("modality_inputs", "images")) + if isinstance(image_tensor, dict): + image_tensor = cls._first_tensor(image_tensor) + packing_kwargs = batch.get("packing_kwargs") + return ( + cls._checksum_tensor(batch.get("input_ids")), + cls._checksum_tensor(batch.get("labels")), + int(batch.get("loss_mask", torch.zeros(1)).sum().item()), + 0 if image_tensor is None else int(image_tensor.shape[0]), + cls._checksum_tensor(image_tensor), + cls._checksum_packing_kwargs(packing_kwargs), + ) + + @staticmethod + def _nested_get(value: dict, keys: tuple[str, ...]): + """Return a nested dict value if every key exists.""" + current = value + for key in keys: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + @classmethod + def _first_tensor(cls, value): + """Return the first tensor inside a nested mapping.""" + if isinstance(value, torch.Tensor): + return value + if isinstance(value, dict): + for item in value.values(): + tensor = cls._first_tensor(item) + if tensor is not None: + return tensor + return None + + @classmethod + def _checksum_packing_kwargs(cls, packing_kwargs: Optional[dict]) -> int: + """Checksum packed-sequence metadata used by the language model.""" + if packing_kwargs is None: + return 0 + checksum = 0 + for key in sorted(packing_kwargs): + value = packing_kwargs[key] + if isinstance(value, torch.Tensor): + value_checksum = cls._checksum_tensor(value) + elif value is None: + value_checksum = 0 + else: + value_checksum = int(value) + checksum = (checksum * 131 + value_checksum) % 2_147_483_647 + return checksum + + @staticmethod + def _checksum_tensor(tensor: Optional[torch.Tensor]) -> int: + """Return a stable, bounded checksum for a CPU tensor-like batch field.""" + if tensor is None or tensor.numel() == 0: + return 0 + values = tensor.detach().reshape(-1) + stride = max(values.numel() // 4096, 1) + values = values[::stride] + if values.is_floating_point(): + values = (values.float() * 1024).to(dtype=torch.long) + else: + values = values.to(dtype=torch.long) + positions = torch.arange(1, values.numel() + 1, dtype=torch.long, device=values.device) + return int(((values * positions).sum() % 2_147_483_647).item()) diff --git a/examples/mimo/model_providers/nemotron_moe_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py index f2c941b60bb..981a09438c9 100644 --- a/examples/mimo/model_providers/nemotron_moe_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -58,10 +58,12 @@ def is_nemotron_20l(args: argparse.Namespace) -> bool: + """Return whether the Nemotron6-MoE VLM 20L provider is active.""" return args.model_provider == NEMOTRON_20L_MODEL_PROVIDER def add_model_provider_args(parser: argparse.ArgumentParser) -> None: + """Register model-provider arguments for hetero MIMO examples.""" provider = parser.add_argument_group("model provider") provider.add_argument( "--model-provider", @@ -88,7 +90,18 @@ def add_model_provider_args(parser: argparse.ArgumentParser) -> None: provider.add_argument("--img-w", type=int, default=512) provider.add_argument("--patch-dim", type=int, default=16) provider.add_argument("--class-token-len", type=int, default=8) - provider.add_argument("--num-image-tiles", type=int, default=NEMOTRON_20L_MAX_NUM_TILES) + provider.add_argument( + "--num-image-tiles", + "--max-num-tiles", + dest="num_image_tiles", + type=int, + default=NEMOTRON_20L_MAX_NUM_TILES, + ) + provider.add_argument("--vision-model-type", type=str, default="radio") + provider.add_argument("--pixel-shuffle", action="store_true") + provider.add_argument("--disable-vision-class-token", action="store_true") + provider.add_argument("--use-tiling", action="store_true") + provider.add_argument("--use-thumbnail", action="store_true") provider.add_argument("--freeze-lm", action="store_true") provider.add_argument("--freeze-vit", action="store_true") provider.add_argument("--freeze-projection", action="store_true") @@ -97,6 +110,7 @@ def add_model_provider_args(parser: argparse.ArgumentParser) -> None: def prepare_model_provider_args(args: argparse.Namespace) -> None: + """Apply provider defaults and derived tokenizer/vision settings.""" apply_model_provider_defaults(args) apply_training_stage(args) resolve_image_token_id(args) @@ -105,6 +119,7 @@ def prepare_model_provider_args(args: argparse.Namespace) -> None: def apply_model_provider_defaults(args: argparse.Namespace) -> None: + """Apply the exact Nemotron6-MoE VLM 20L model defaults.""" if not is_nemotron_20l(args): return @@ -116,9 +131,14 @@ def apply_model_provider_defaults(args: argparse.Namespace) -> None: args.moe_grouped_gemm = True args.seq_length = 8192 args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles + args.pixel_shuffle = True + args.disable_vision_class_token = True + args.use_tiling = True + args.use_thumbnail = True def apply_training_stage(args: argparse.Namespace) -> None: + """Apply stage-specific freeze flags for the Nemotron VLM recipe.""" if not is_nemotron_20l(args): return @@ -134,6 +154,7 @@ def apply_training_stage(args: argparse.Namespace) -> None: def resolve_image_token_id(args: argparse.Namespace) -> None: + """Resolve image, pad, and vocab ids from the configured tokenizer.""" if not is_nemotron_20l(args) or not args.tokenizer_model: return @@ -161,6 +182,7 @@ def resolve_image_token_id(args: argparse.Namespace) -> None: def validate_model_provider_args(args: argparse.Namespace) -> None: + """Validate derived model-provider arguments.""" if args.hidden_size % args.num_attention_heads != 0: raise ValueError("--hidden-size must be divisible by --num-attention-heads") if not 0 <= args.image_token_id < args.vocab_size: diff --git a/examples/mimo/scripts/install_energon.sh b/examples/mimo/scripts/install_energon.sh new file mode 100755 index 00000000000..9b95e7b47a7 --- /dev/null +++ b/examples/mimo/scripts/install_energon.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Install the custom Megatron-Energon build used by the MIMO multimodal data path. + +set -euo pipefail + +DEFAULT_PATH="/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_genai/users/ykarnati/public/Megatron-Energon-sasatheesh" +ENERGON_PATH="${1:-$DEFAULT_PATH}" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +echo "Installing Megatron-Energon from: ${ENERGON_PATH}" +if [[ ! -d "${ENERGON_PATH}" ]]; then + echo "ERROR: Directory not found: ${ENERGON_PATH}" >&2 + exit 1 +fi + +"${PYTHON_BIN}" -m pip install -e "${ENERGON_PATH}[multimodal]" + +"${PYTHON_BIN}" - <<'PY' +from megatron.energon.task_encoder.multimodal import MultiModalPackingEncoder +import megatron.energon + +print(f"Megatron-Energon installed from: {megatron.energon.__file__}") +print(f"MultiModalPackingEncoder OK: {MultiModalPackingEncoder.__name__}") +PY diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh new file mode 100755 index 00000000000..e6197aa44d4 --- /dev/null +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Run non-colocated heterogeneous MIMO Nemotron6-MoE VLM 20L training on Energon data. + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" + +TRAINING_STAGE="${TRAINING_STAGE:-stage2}" +case "${TRAINING_STAGE}" in + stage1|stage2|stage3) + ;; + *) + echo "ERROR: Unknown TRAINING_STAGE='${TRAINING_STAGE}'. Use stage1, stage2, or stage3." >&2 + exit 1 + ;; +esac + +if [[ "${INSTALL_ENERGON:-0}" == "1" ]]; then + if [[ -n "${ENERGON_PATH:-}" ]]; then + bash examples/mimo/scripts/install_energon.sh "${ENERGON_PATH}" + else + bash examples/mimo/scripts/install_energon.sh + fi +fi + +GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +TRAIN_ITERS="${TRAIN_ITERS:-100}" +NUM_MICROBATCHES="${NUM_MICROBATCHES:-4}" +MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +LLM_DP=2 +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP))}" +LR_WARMUP_ITERS="${LR_WARMUP_ITERS:-2}" +LR_DECAY_ITERS="${LR_DECAY_ITERS:-10}" +PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" +NUM_WORKERS="${NUM_WORKERS:-2}" +SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" +MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +DATA_PATH="${DATA_PATH:-/lustre/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/kshih/workspace/blends/eagle_recipe_online_packing/final_recipe/pretrain_base_non_sft_cw_dfw.yaml}" +TOKENIZER_MODEL="${TOKENIZER_MODEL:-/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_genai/users/ykarnati/checkpoints/models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16-multimodal-pretraining/snapshots/7344a79074e20d9ab548e14c25b0492345394f67}" + +echo "=== Hetero MIMO Nemotron6-MoE VLM 20L Energon training ===" +echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} gbs=${GLOBAL_BATCH_SIZE}" +echo "data=${DATA_PATH}" +echo "tokenizer=${TOKENIZER_MODEL}" +echo "===========================================================" + +DATA_LOADER_ARGS=( + --num-workers "${NUM_WORKERS}" + --shuffle-buffer-size "${SHUFFLE_BUFFER_SIZE}" + --max-samples-per-sequence "${MAX_SAMPLES_PER_SEQUENCE}" +) +if [[ "${PACKING_BUFFER_SIZE}" != "0" ]]; then + DATA_LOADER_ARGS+=(--packing-buffer-size "${PACKING_BUFFER_SIZE}") +fi + +"${PYTHON_BIN}" -m torch.distributed.run \ + --standalone \ + --nproc-per-node "${GPUS_PER_NODE}" \ + examples/mimo/train_hetero.py \ + --model-provider nemotron-moe-vlm-20l \ + --dataset-provider energon_multimodal \ + --training-stage "${TRAINING_STAGE}" \ + --encoder-tp 2 \ + --encoder-pp 1 \ + --encoder-dp 2 \ + --llm-offset 4 \ + --llm-tp 2 \ + --llm-pp 1 \ + --llm-dp "${LLM_DP}" \ + --llm-ep 4 \ + --llm-expt-tp 1 \ + --llm-expt-dp 1 \ + --vocab-size 131072 \ + --max-num-tiles 12 \ + --data-path "${DATA_PATH}" \ + "${DATA_LOADER_ARGS[@]}" \ + --tokenizer-model "${TOKENIZER_MODEL}" \ + --tokenizer-prompt-format nemotron6-moe \ + --image-token "" \ + --micro-batch-size "${MICRO_BATCH_SIZE}" \ + --global-batch-size "${GLOBAL_BATCH_SIZE}" \ + --num-microbatches "${NUM_MICROBATCHES}" \ + --lr 2e-4 \ + --min-lr 2e-6 \ + --lr-decay-style cosine \ + --lr-warmup-iters "${LR_WARMUP_ITERS}" \ + --lr-decay-iters "${LR_DECAY_ITERS}" \ + --weight-decay 0.05 \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --clip-grad 1.0 \ + --no-overlap-grad-reduce \ + --ddp-bucket-size 0 \ + --log-interval 1 \ + --train-iters "${TRAIN_ITERS}" \ + "$@" diff --git a/examples/mimo/train_hetero.py b/examples/mimo/train_hetero.py index 7eb4c81da41..963f439ecb3 100644 --- a/examples/mimo/train_hetero.py +++ b/examples/mimo/train_hetero.py @@ -1,6 +1,6 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Standalone heterogeneous MIMO mock training entrypoint.""" +"""Standalone heterogeneous MIMO training entrypoint.""" import os import sys @@ -27,7 +27,7 @@ def main() -> None: try: run_train_loop(args) dist.barrier() - print_rank_0("Heterogeneous MIMO mock training completed") + print_rank_0("Heterogeneous MIMO training completed") finally: shutdown_distributed() diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index f1dffc5d3c9..238e8b2a57c 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -18,7 +18,7 @@ def parse_args() -> argparse.Namespace: """Parse standalone hetero MIMO loop arguments.""" parser = argparse.ArgumentParser( description=( - "Standalone heterogeneous MIMO mock training loop. " + "Standalone heterogeneous MIMO training loop. " "This entrypoint owns one HyperCommGrid per MIMO module." ) ) @@ -43,6 +43,14 @@ def parse_args() -> argparse.Namespace: add_model_provider_args(parser) + data = parser.add_argument_group("data") + data.add_argument("--dataset-provider", choices=["mock", "energon_multimodal"], default="mock") + data.add_argument("--data-path", type=str, default=None) + data.add_argument("--num-workers", type=int, default=2) + data.add_argument("--packing-buffer-size", type=int, default=None) + data.add_argument("--shuffle-buffer-size", type=int, default=100) + data.add_argument("--max-samples-per-sequence", type=int, default=100) + train = parser.add_argument_group("training") train.add_argument("--micro-batch-size", type=int, default=2) train.add_argument("--global-batch-size", type=int, default=None) @@ -86,14 +94,17 @@ def prepare_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: - """Validate the current disjoint-grid mock-training layout.""" + """Validate the current disjoint-grid training layout.""" if args.encoder_cp != 1 or args.llm_cp != 1: raise ValueError("Phase 2 mock training currently supports CP=1 only") if args.log_interval < 1: raise ValueError("--log-interval must be >= 1") validate_model_provider_args(args) - validate_mock_data_args(args) + if args.dataset_provider == "mock": + validate_mock_data_args(args) + else: + validate_energon_data_args(args) if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: raise ValueError("--num-moe-experts must be divisible by --llm-ep") if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: @@ -117,3 +128,35 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: ) return encoder_size, llm_size + + +def validate_energon_data_args(args: argparse.Namespace) -> None: + """Validate the actual-data non-colocated path.""" + if not args.data_path: + raise ValueError("--data-path is required for --dataset-provider energon_multimodal") + if not args.tokenizer_model: + raise ValueError("--tokenizer-model is required for --dataset-provider energon_multimodal") + if args.model_provider != "nemotron-moe-vlm-20l": + raise ValueError("energon_multimodal is currently wired for the Nemotron 20L VLM provider") + if args.encoder_pp != 1 or args.llm_pp != 1: + raise ValueError("energon_multimodal currently supports encoder and LLM PP size 1") + if args.encoder_dp != args.llm_dp: + raise ValueError( + "energon_multimodal currently requires --encoder-dp == --llm-dp so the " + "encoder and LLM grids consume matching DP-lane samples" + ) + if args.overlap_grad_reduce: + raise ValueError( + "energon_multimodal currently requires --no-overlap-grad-reduce because " + "the blend can yield text-only batches on vision ranks" + ) + if args.packing_buffer_size is not None and args.packing_buffer_size > 0: + if args.micro_batch_size != 1: + raise ValueError( + "Energon packed multimodal batches currently require --micro-batch-size 1" + ) + encoder_micro_batch_size = args.micro_batch_size * args.llm_dp // args.encoder_dp + if encoder_micro_batch_size != args.micro_batch_size: + raise ValueError( + "energon_multimodal currently requires equal encoder and LLM microbatch sizes" + ) diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 4e9b920b6a5..55d29695718 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -31,7 +31,7 @@ def run_train_loop(args: argparse.Namespace) -> None: - """Run mock-data heterogeneous MIMO training.""" + """Run heterogeneous MIMO training.""" world_size = torch.distributed.get_world_size() encoder_size, llm_size = prepare_args(args, world_size) @@ -61,13 +61,15 @@ def run_train_loop(args: argparse.Namespace) -> None: ) debug_rank("selecting data iterator") data_iterator = select_data_iterator(args, topology) + validate_data_iterator(args, data_iterator, topology) logger = HeteroTrainingLogger(args=args, topology=topology) debug_rank("training setup ready") print_rank_0( - "Starting hetero MIMO mock training: " + "Starting hetero MIMO training: " f"world_size={world_size}, encoder_size={topology.encoder_size}, " - f"llm_size={topology.llm_size}, train_iters={args.train_iters}" + f"llm_size={topology.llm_size}, train_iters={args.train_iters}, " + f"dataset_provider={args.dataset_provider}" ) for iteration in range(1, args.train_iters + 1): @@ -104,7 +106,28 @@ def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime): ) -def select_data_iterator( +def select_data_iterator(args: argparse.Namespace, topology: HeteroTopology) -> Optional[object]: + """Create the per-role data iterator needed by local ranks.""" + if args.dataset_provider == "mock": + return select_mock_data_iterator(args, topology) + if args.dataset_provider == "energon_multimodal": + from examples.mimo.data.hetero_energon import build_energon_iterator + + return build_energon_iterator(args, topology) + raise ValueError(f"unsupported dataset provider: {args.dataset_provider}") + + +def validate_data_iterator( + args: argparse.Namespace, data_iterator, topology: HeteroTopology +) -> None: + """Run data-provider checks that must happen outside the pipeline schedule.""" + if args.dataset_provider == "energon_multimodal": + from examples.mimo.data.hetero_energon import validate_energon_data_alignment + + validate_energon_data_alignment(data_iterator, topology) + + +def select_mock_data_iterator( args: argparse.Namespace, topology: HeteroTopology ) -> Optional[MockVLMIterator]: """Create the per-role mock-data iterator needed by local ranks.""" diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 83425d33d6f..82665e656ad 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -143,6 +143,7 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): def forward_step(data_iterator, model): """Forward step consumed by the MCore pipeline schedule.""" batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} + batch = move_batch_to_cuda(batch) debug_rank("forward_step batch prepared") debug_rank("forward_step model call start") output_tensor, loss_mask = model(**batch) @@ -150,6 +151,19 @@ def forward_step(data_iterator, model): return output_tensor, partial(loss_func, loss_mask) +def move_batch_to_cuda(value): + """Move tensors in nested batch structures to the current CUDA device.""" + if isinstance(value, torch.Tensor): + return value.cuda(non_blocking=True) + if isinstance(value, dict): + return {key: move_batch_to_cuda(item) for key, item in value.items()} + if isinstance(value, list): + return [move_batch_to_cuda(item) for item in value] + if isinstance(value, tuple): + return tuple(move_batch_to_cuda(item) for item in value) + return value + + def train_step( args: argparse.Namespace, runtime: HeteroRuntime, diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index d0e48a19b65..3edd3e10010 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -376,12 +376,17 @@ def forward( if self.role.mode == ModuleLayout.NON_COLOCATED: if self.role.has_modality_modules: - return self._forward_encoders(modality_inputs, input_tensors), loss_mask + return self._forward_encoders(input_ids, modality_inputs, input_tensors), loss_mask if self.role.has_language_module: return ( self._forward_language_module( - input_ids, position_ids, attention_mask, labels, input_tensors + input_ids, + position_ids, + attention_mask, + labels, + input_tensors, + packing_kwargs, ), loss_mask, ) @@ -392,6 +397,7 @@ def forward( def _forward_encoders( self, + input_ids: Optional[torch.Tensor], modality_inputs: Optional[Dict[str, Dict[str, Any]]], input_tensors: Optional[Dict[str, torch.Tensor]], ) -> Dict[str, torch.Tensor]: @@ -411,16 +417,47 @@ def _forward_encoders( continue submodule = self.modality_submodules[encoder_name] - output = submodule.forward( - encoder_inputs=modality_inputs.get(encoder_name) if modality_inputs else None, - hidden_states=input_tensors.get(encoder_name) if input_tensors else None, - ) + encoder_inputs = modality_inputs.get(encoder_name) if modality_inputs else None + hidden_states = input_tensors.get(encoder_name) if input_tensors else None + output = submodule.forward(encoder_inputs=encoder_inputs, hidden_states=hidden_states) + if output is None and encoder_inputs is None and hidden_states is None: + if self._has_encoder_tokens(input_ids, encoder_name): + raise RuntimeError( + f"{encoder_name} inputs are missing, but matching special tokens exist" + ) + output = self._empty_modality_output(submodule) if output is not None: outputs[encoder_name] = output return outputs + def _has_encoder_tokens(self, input_ids: Optional[torch.Tensor], encoder_name: str) -> bool: + """Return whether the batch contains tokens for an encoder module.""" + if input_ids is None or encoder_name not in self.special_token_ids: + return False + return bool((input_ids == self.special_token_ids[encoder_name]).any().item()) + + @staticmethod + def _empty_modality_output(submodule: torch.nn.Module) -> torch.Tensor: + """Return an empty projected activation for text-only non-colocated batches.""" + unwrapped = unwrap_model(submodule) + projections = getattr(unwrapped, "input_projections", None) + if not projections: + raise RuntimeError("cannot build empty modality output without input projections") + + projection = projections[0] + hidden_size = getattr(getattr(projection, "config", None), "hidden_size", None) + if hidden_size is None: + hidden_size = getattr(projection, "out_features", None) + if hidden_size is None: + raise RuntimeError("cannot infer hidden size for empty modality output") + + param = next(projection.parameters(), None) + device = param.device if param is not None else torch.device("cuda") + dtype = param.dtype if param is not None else torch.float32 + return torch.empty((0, hidden_size), device=device, dtype=dtype, requires_grad=True) + def _forward_language_module( self, input_ids: torch.Tensor, @@ -428,6 +465,7 @@ def _forward_language_module( attention_mask: Optional[torch.Tensor], labels: Optional[torch.Tensor], input_tensors: Optional[Dict[str, torch.Tensor]], + packing_kwargs: Optional[dict] = None, ) -> torch.Tensor: """Forward pass for language module on this rank. @@ -442,6 +480,7 @@ def _forward_language_module( Language model output (hidden states, logits, or loss depending on stage) """ lang_name = MIMO_LANGUAGE_MODULE_KEY + packed_seq_params = self._build_packed_seq_params(packing_kwargs) if self.role.is_first_stage(lang_name): # First stage: receive encoder embeddings, combine with text, pass to LM @@ -467,11 +506,14 @@ def _forward_language_module( if self.partition_adapter is not None: combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() - combined_embeddings, labels, _, attention_mask, _ = self.partition_adapter.shard( - embeddings=combined_embeddings, - labels=labels, - loss_mask=None, - attention_mask=attention_mask, + combined_embeddings, labels, _, attention_mask, packed_seq_params = ( + self.partition_adapter.shard( + embeddings=combined_embeddings, + labels=labels, + loss_mask=None, + attention_mask=attention_mask, + packed_seq_params=packed_seq_params, + ) ) if combined_embeddings is not None: combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() @@ -482,6 +524,7 @@ def _forward_language_module( decoder_input=combined_embeddings, labels=labels, attention_mask=None, + packed_seq_params=packed_seq_params, ) else: # Non-first stage: receive hidden states from previous LM stage @@ -499,6 +542,7 @@ def _forward_language_module( decoder_input=None, labels=labels, attention_mask=attention_mask, + packed_seq_params=packed_seq_params, ) # Key output for non-last stages so schedule can route to next LM stage @@ -507,6 +551,24 @@ def _forward_language_module( return lm_output + @staticmethod + def _build_packed_seq_params(packing_kwargs: Optional[dict]) -> Optional[PackedSeqParams]: + """Build packed-sequence params from dataloader kwargs.""" + if packing_kwargs is None: + return None + converted_kwargs = {} + for key, value in packing_kwargs.items(): + if 'cu_seqlens' in key and value is not None: + converted_kwargs[key] = value.to(dtype=torch.int32) + elif key == 'total_tokens' and isinstance(value, torch.Tensor): + converted_kwargs[key] = int(value.item()) + else: + converted_kwargs[key] = value + packed_seq_params = PackedSeqParams(**converted_kwargs) + packed_seq_params.qkv_format = 'thd' + logger.debug(f"Packed sequence parameters: {packed_seq_params}") + return packed_seq_params + def _build_colocated_communicators(self): grid_map = self.mimo_config.module_to_grid_map if any( @@ -562,16 +624,7 @@ def _forward_all_modules( This is the original behavior, preserved for backward compatibility. """ - # If packing_kwargs is provided, construct PackedSeqParams - packed_seq_params = None - if packing_kwargs is not None: - # Ensure correct dtype for seqlens tensors - for key in packing_kwargs: - if 'cu_seqlens' in key and packing_kwargs[key] is not None: - packing_kwargs[key] = packing_kwargs[key].to(dtype=torch.int32) - packed_seq_params = PackedSeqParams(**packing_kwargs) - packed_seq_params.qkv_format = 'thd' - logger.debug(f"Packed sequence parameters: {packed_seq_params}") + packed_seq_params = self._build_packed_seq_params(packing_kwargs) # 1. Process each modality to get embeddings modality_embeddings = {} From 76b0d7f347e03ffc5fbf1f0451ce09b83fa04774 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 15:17:16 +0000 Subject: [PATCH 07/44] NMFW-464 document e2e parity plan --- .../mimo/docs/e2e_training_parity_plan.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 examples/mimo/docs/e2e_training_parity_plan.md diff --git a/examples/mimo/docs/e2e_training_parity_plan.md b/examples/mimo/docs/e2e_training_parity_plan.md new file mode 100644 index 00000000000..b2f84090a08 --- /dev/null +++ b/examples/mimo/docs/e2e_training_parity_plan.md @@ -0,0 +1,72 @@ +# E2E Training Parity Plan + +This note tracks the plan for checking end-to-end training parity between the +previous `examples/mimo/train.py` flow from `feat/nemotron-moe-vlm-mimo` and the +new heterogenous `examples/mimo/train_hetero.py` flow. + +## Goal + +Verify that the new heterogenous MIMO training loop matches the previous +Megatron `pretrain()`-based flow for the Nemotron 20L VLM workflow. The strongest +parity signal is matching behavior on a frozen batch stream before comparing live +Energon training runs. + +## Plan + +1. Compare resolved training configuration. + - Dump the final args used by old `train.py`. + - Dump the final args used by new `train_hetero.py`. + - Compare behavior-relevant fields: model config, vision config, MoE config, + TP/PP/EP/ETP/EDP, batch sizes, optimizer, scheduler, seeds, loss scaling, + per-token loss, and dataloader settings. + +2. Start both runs from the same initial weights. + - Prefer a canonical initialized checkpoint or state dict over relying only on + seed-based initialization. + - Compare parameter hashes by logical module: vision encoder, LLM backbone, + MoE experts, router parameters, and projector/MIMO bridge. + +3. Validate data parity before training. + - First use a recorded frozen batch stream, not live Energon. + - Dump exact batch tensors and metadata from the old path: tokens, labels, + loss mask, position ids, modality inputs, packed sequence params, and sample + signatures if available. + - Feed the same frozen batches to the new heterogenous loop and compare batch + hashes before forward. + +4. Run forward-only parity. + - Use the same initialized weights and same frozen batch. + - Disable optimizer updates. + - Compare logits checksums where practical, unreduced loss numerator, token + denominator, normalized loss, and auxiliary/router losses. + +5. Run single-step training parity. + - Use the same frozen batch. + - Run forward, backward, optimizer step, and LR scheduler step. + - Compare loss before step, grad norm, skipped/nan flags, LR, selected + parameter deltas, and post-step parameter hashes. + +6. Run short frozen-stream loss-curve parity. + - Use a fixed stream of 10 to 20 frozen batches. + - Compare per-iteration loss, grad norm, LR, loss scale, skipped/nan counts, + consumed samples, and token counts. + +7. Run actual Energon parity. + - Run the old `train.py` flow and the new `train_hetero.py` flow against the + real Nemotron 20L Energon setup. + - Log sample signatures per global step in both paths. + - First verify that both paths consume the same samples in the same order. + - Compare loss curves only after sample order parity is established. + +## Expected Limits + +Bitwise parity may not be realistic between the old colocated Megatron +`pretrain()` path and the new non-colocated heterogenous grids because collective +ordering, parameter partitioning, and optimizer sharding can differ. The first +strict gates should therefore be configuration parity, initial-weight parity, +frozen-batch forward parity, token-count parity, LR schedule parity, and a short +frozen-batch training curve within a tight tolerance. + +The known parity gap is the old `--use-loss-scaling` path. The new heterogenous +loop uses per-token global loss normalization, but it does not yet implement the +old optional sqrt-weighted scaled loss behavior. From 74f06cd0eaf6a09aefedc6d0c315333fb95cbc56 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 17:30:58 +0000 Subject: [PATCH 08/44] NMFW-464 simplify hetero Energon dataloader --- .../mimo/data/energon_multimodal_provider.py | 294 ++++++++++++++ examples/mimo/data/hetero_energon.py | 365 ++---------------- .../compare_energon_dataloader_parity.py | 355 +++++++++++++++++ examples/mimo/scripts/install_energon.sh | 24 -- .../run_hetero_nemotron_20l_energon_train.sh | 20 +- examples/mimo/scripts/verify_energon.sh | 66 ++++ ...on-7.3.3.dev30+gd456cbd4a-py3-none-any.whl | Bin 0 -> 325361 bytes .../vendor/old_energon_multimodal_provider.py | 248 ++++++++++++ pyproject.toml | 5 +- tests/unit_tests/test_hetero_energon.py | 37 ++ uv.lock | 289 ++++++++++---- 11 files changed, 1259 insertions(+), 444 deletions(-) create mode 100644 examples/mimo/data/energon_multimodal_provider.py create mode 100644 examples/mimo/scripts/compare_energon_dataloader_parity.py delete mode 100755 examples/mimo/scripts/install_energon.sh create mode 100755 examples/mimo/scripts/verify_energon.sh create mode 100644 examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl create mode 100644 examples/mimo/vendor/old_energon_multimodal_provider.py create mode 100644 tests/unit_tests/test_hetero_energon.py diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py new file mode 100644 index 00000000000..653e0017ecd --- /dev/null +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Energon multimodal data provider for MIMO. + +This module intentionally mirrors the provider used by the previous +``feat/nemotron-moe-vlm-mimo`` branch. Energon's ``MultiModalPackingEncoder`` +owns sample cooking, preencoding, and packing; the MIMO-specific adapter only +expands each single ```` placeholder into one placeholder per image +embedding and remaps the batch to MIMO's forward signature. +""" + +from __future__ import annotations + +import warnings +from typing import Optional + +import torch + +from megatron.energon.task_encoder.multimodal import ( + MultiModalPackingEncoder, + PackingConfig, + VisionConfig, +) +from megatron.energon.task_encoder.multimodal.sample_types import PackedSample +from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings + + +class TokenizerAdapter: + """Wrap Megatron tokenizers for Energon's tokenizer protocol.""" + + def __init__(self, megatron_tokenizer) -> None: + self._tok = megatron_tokenizer + inner = megatron_tokenizer + if hasattr(inner, "_tokenizer"): + inner = inner._tokenizer + if hasattr(inner, "tokenizer"): + inner = inner.tokenizer + self._hf = inner + + @property + def pad_token_id(self) -> int: + """Return the tokenizer pad id.""" + return self._tok.pad + + @property + def eos_token_id(self) -> int: + """Return the tokenizer EOS id.""" + return self._tok.eod + + def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: + """Encode text with the wrapped HuggingFace tokenizer.""" + return self._hf.encode(text, add_special_tokens=add_special_tokens) + + def decode(self, token_ids, skip_special_tokens: bool = False) -> str: + """Decode token ids with the wrapped HuggingFace tokenizer.""" + return self._hf.decode(token_ids, skip_special_tokens=skip_special_tokens) + + def convert_tokens_to_ids(self, tokens): + """Convert tokens to ids with the wrapped Megatron tokenizer.""" + return self._tok.convert_tokens_to_ids(tokens) + + +class MimoMultiModalPackingEncoder(MultiModalPackingEncoder): + """Remap Energon multimodal packed samples to MIMO batch inputs.""" + + def __init__( + self, + vision_config: VisionConfig, + packing_config: PackingConfig, + tokenizer, + encoder_name: str = "radio_encoder", + encoder_input_key: str = "x", + target_seq_length: Optional[int] = None, + ) -> None: + super().__init__(vision_config, packing_config, tokenizer) + self.encoder_name = encoder_name + self.encoder_input_key = encoder_input_key + self._target_seq_length = target_seq_length + self._embeddings_per_tile = get_num_image_embeddings( + img_h=vision_config.img_h, + img_w=vision_config.img_w, + patch_dim=vision_config.patch_dim, + class_token_len=vision_config.class_token_len, + disable_vision_class_token=vision_config.disable_vision_class_token, + pixel_shuffle=vision_config.pixel_shuffle, + conv_merging=vision_config.conv_merging, + use_tile_tags=vision_config.use_tile_tags, + max_num_tiles=vision_config.max_num_tiles, + use_image_break_token=vision_config.use_image_break_token, + ) + + def batch(self, samples: list[PackedSample]) -> dict: + """Expand image placeholders and return a MIMO-compatible batch.""" + image_token_id = self.packing_config.image_token_id + ignore_index = self.packing_config.ignore_index + pad_id = self.packing_config.pad_id + emb_per_tile = self._embeddings_per_tile + + expanded_tokens_list = [] + expanded_labels_list = [] + all_images = [] + + for sample in samples: + tokens = sample.tokens + labels = sample.labels + num_tiles = sample.num_tiles + budget = self._target_seq_length + new_tokens = [] + new_labels = [] + img_idx = 0 + truncated = False + truncated_padding_only = False + kept_tile_count = 0 + + for idx, token in enumerate(tokens.tolist()): + if token == image_token_id: + n_tiles = num_tiles[img_idx] if img_idx < len(num_tiles) else 1 + n_tokens = n_tiles * emb_per_tile + if budget is not None and len(new_tokens) + n_tokens > budget: + truncated = True + break + new_tokens.extend([image_token_id] * n_tokens) + new_labels.extend([ignore_index] * n_tokens) + kept_tile_count += n_tiles + img_idx += 1 + else: + if budget is not None and len(new_tokens) + 1 > budget: + truncated = True + truncated_padding_only = _remaining_tokens_are_padding( + tokens=tokens, + labels=labels, + start=idx, + pad_id=pad_id, + ignore_index=ignore_index, + ) + break + new_tokens.append(token) + new_labels.append(labels[idx].item()) + + if truncated and len(sample.cu_lengths) > 2 and not truncated_padding_only: + raise RuntimeError( + "Packed Energon sample exceeds target sequence length after MIMO image-token " + "expansion. Refusing to clamp packed cu_seqlens because that can create " + "zero-length packed segments. Increase --total-seq-length or lower image " + "tiling/packing settings." + ) + + if truncated and not truncated_padding_only: + warnings.warn( + f"Sample truncated to fit target_seq_length ({self._target_seq_length}): " + f"kept {len(new_tokens)} of ~{len(tokens)} original tokens, " + f"{img_idx}/{len(num_tiles)} images ({kept_tile_count} tiles). " + "Consider increasing --total-seq-length or reducing --max-num-tiles.", + stacklevel=2, + ) + + all_images.extend(sample.images[:kept_tile_count]) + expanded_tokens_list.append(torch.tensor(new_tokens, dtype=torch.long)) + expanded_labels_list.append(torch.tensor(new_labels, dtype=torch.long)) + + max_len = max(len(tokens) for tokens in expanded_tokens_list) + if self._target_seq_length is not None: + max_len = self._target_seq_length + + batch_size = len(samples) + tokens_batch = torch.full((batch_size, max_len), pad_id, dtype=torch.long) + labels_batch = torch.full((batch_size, max_len), ignore_index, dtype=torch.long) + + for idx, (tokens, labels) in enumerate(zip(expanded_tokens_list, expanded_labels_list)): + tokens_batch[idx, : len(tokens)] = tokens + labels_batch[idx, : len(labels)] = labels + + loss_mask = (labels_batch != ignore_index).float() + loss_mask[labels_batch == image_token_id] = 0.0 + position_ids = torch.arange(max_len).unsqueeze(0).expand(batch_size, -1).contiguous() + + result = { + "input_ids": tokens_batch, + "labels": labels_batch, + "loss_mask": loss_mask, + "position_ids": position_ids, + } + + if all_images: + images = self.tiling_strategy.stack(all_images)[0] + result["modality_inputs"] = { + "images": {self.encoder_name: {self.encoder_input_key: images}} + } + + is_packed = any(len(sample.cu_lengths) > 2 for sample in samples) + if is_packed: + if batch_size != 1: + raise RuntimeError(f"Packing requires micro_batch_size=1, got {batch_size}") + result["packing_kwargs"] = _build_packing_kwargs(samples[0], max_len) + + return result + + +def _remaining_tokens_are_padding( + tokens: torch.Tensor, labels: torch.Tensor, start: int, pad_id: int, ignore_index: int +) -> bool: + """Return whether truncation only drops right-padding tokens.""" + remaining_tokens = tokens[start:] + remaining_labels = labels[start:] + return bool( + remaining_tokens.numel() > 0 + and torch.all(remaining_tokens == pad_id).item() + and torch.all(remaining_labels == ignore_index).item() + ) + + +def _build_packing_kwargs(sample: PackedSample, max_len: int) -> dict[str, torch.Tensor]: + """Build validated packed-sequence metadata for the MIMO language model.""" + cu_seqlens = sample.cu_lengths.to(dtype=torch.int32) + if cu_seqlens.numel() < 2: + raise RuntimeError(f"Packed sample must have at least two cu_lengths, got {cu_seqlens}") + if torch.any(cu_seqlens[1:] < cu_seqlens[:-1]): + raise RuntimeError(f"Packed cu_lengths must be monotonic, got {cu_seqlens.tolist()}") + + if cu_seqlens[0] != 0: + cu_seqlens = torch.cat([torch.tensor([0], dtype=torch.int32), cu_seqlens]) + if cu_seqlens[-1] > max_len: + raise RuntimeError( + f"Packed cu_lengths end at {int(cu_seqlens[-1])}, beyond sequence length {max_len}" + ) + if cu_seqlens[-1] != max_len: + cu_seqlens = torch.cat([cu_seqlens, torch.tensor([max_len], dtype=torch.int32)]) + + segment_lens = cu_seqlens[1:] - cu_seqlens[:-1] + if torch.any(segment_lens <= 0): + raise RuntimeError( + "Packed cu_lengths must be strictly increasing after MIMO expansion, " + f"got {cu_seqlens.tolist()}" + ) + max_seqlen = segment_lens.max() + return { + "cu_seqlens_q": cu_seqlens, + "cu_seqlens_kv": cu_seqlens, + "cu_seqlens_q_padded": cu_seqlens, + "cu_seqlens_kv_padded": cu_seqlens, + "max_seqlen_q": max_seqlen, + "max_seqlen_kv": max_seqlen, + "total_tokens": torch.tensor(max_len, dtype=torch.int32), + } + + +def build_multimodal_encoder( + args, tokenizer, encoder_name: str = "radio_encoder", encoder_input_key: str = "x" +) -> MimoMultiModalPackingEncoder: + """Build the MIMO Energon encoder from train args.""" + target_seq_length = _resolve_target_seq_length(args) + image_token_id = getattr(args, "image_token_id", None) + if image_token_id is None: + image_token_id = tokenizer.convert_tokens_to_ids(getattr(args, "image_token", "")) + pad_id = getattr(args, "pad_token_id", tokenizer.pad) + + vision_config = VisionConfig( + img_h=args.img_h, + img_w=args.img_w, + patch_dim=args.patch_dim, + vision_model_type=getattr(args, "vision_model_type", "radio"), + disable_vision_class_token=getattr(args, "disable_vision_class_token", False), + pixel_shuffle=getattr(args, "pixel_shuffle", False), + max_num_tiles=getattr(args, "max_num_tiles", getattr(args, "num_image_tiles", 1)), + use_tiling=getattr(args, "use_tiling", False), + use_thumbnail=getattr(args, "use_thumbnail", False), + class_token_len=getattr(args, "class_token_len", None) or 1, + conv_merging=getattr(args, "conv_merging", False), + use_tile_tags=getattr(args, "use_tile_tags", False), + use_image_break_token=getattr(args, "image_break_token", None) is not None, + use_area_weighted_aspect_ratio=getattr(args, "use_area_weighted_aspect_ratio", False), + dynamic_resolution=getattr(args, "dynamic_resolution", False), + ) + packing_config = PackingConfig( + seq_length=target_seq_length, pad_id=pad_id, image_token_id=image_token_id + ) + return MimoMultiModalPackingEncoder( + vision_config=vision_config, + packing_config=packing_config, + tokenizer=TokenizerAdapter(tokenizer), + encoder_name=encoder_name, + encoder_input_key=encoder_input_key, + target_seq_length=target_seq_length, + ) + + +def _resolve_target_seq_length(args) -> int: + """Return the sequence length used by Energon and MIMO expansion.""" + target_seq_length = getattr(args, "total_seq_length", None) + if target_seq_length is None: + target_seq_length = getattr(args, "seq_length", None) + if target_seq_length is None: + raise AttributeError("Energon multimodal provider requires total_seq_length or seq_length") + return target_seq_length diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 2dcca22b197..0c3a7125534 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -1,29 +1,20 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Energon multimodal iterator for heterogeneous MIMO training.""" +"""Heterogeneous-rank wrapper for the MIMO Energon multimodal provider.""" from __future__ import annotations -import warnings -from typing import NamedTuple, Optional +import hashlib +import random +from typing import Optional import torch import torch.distributed as dist -from examples.mimo.model_providers.nemotron_moe_vlm import NEMOTRON_VISION_ENCODER_KEY from examples.mimo.training.hetero.topology import get_grid_coordinate, is_rank_in_grid from examples.mimo.utils.hetero import debug_rank, is_process_group_member -class ExpandedSample(NamedTuple): - """Token row and expanded-coordinate segment boundaries.""" - - tokens: torch.Tensor - labels: torch.Tensor - cu_lengths: torch.Tensor - kept_tiles: int - - def build_energon_iterator(args, topology): """Build an Energon iterator for the current rank, or return None if unused.""" from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage @@ -79,10 +70,16 @@ def _build_iterator_for_grid(args, grid): if get_grid_coordinate(grid, "tp") != 0: return EnergonIterator(None, tp_group=tp_group, source_rank=False) - from megatron.energon import NoCachePool, WorkerConfig, get_loader, get_train_dataset + from examples.mimo.data.energon_multimodal_provider import build_multimodal_encoder + from megatron.energon import WorkerConfig, get_loader, get_train_dataset tokenizer = _build_tokenizer(args) - encoder = MimoMultiModalPackingEncoder.from_args(args, tokenizer) + encoder = build_multimodal_encoder( + args, + tokenizer, + encoder_name=getattr(args, "vision_encoder_key", "radio_encoder"), + encoder_input_key="x", + ) dp_rank = get_grid_coordinate(grid, "dp") worker_config = WorkerConfig( rank=dp_rank, world_size=args.llm_dp, num_workers=args.num_workers, data_parallel_group=None @@ -101,7 +98,7 @@ def _build_iterator_for_grid(args, grid): max_samples_per_sequence=args.max_samples_per_sequence, ) return EnergonIterator( - get_loader(dataset, cache_pool=NoCachePool()), tp_group=tp_group, source_rank=True + get_loader(dataset), tp_group=tp_group, source_rank=True, random_seed=args.seed + dp_rank ) @@ -119,312 +116,21 @@ def _build_tokenizer(args): ) -class TokenizerAdapter: - """Wrap Megatron's multimodal tokenizer for Energon's tokenizer protocol.""" - - def __init__(self, megatron_tokenizer) -> None: - self._tok = megatron_tokenizer - self._hf = megatron_tokenizer.tokenizer - - @property - def pad_token_id(self) -> int: - """Return the tokenizer pad id.""" - return self._tok.pad - - @property - def eos_token_id(self) -> int: - """Return the tokenizer EOS id.""" - return self._tok.eod - - def encode(self, text: str, add_special_tokens: bool = True) -> list[int]: - """Encode text with the wrapped HuggingFace tokenizer.""" - return self._hf.encode(text, add_special_tokens=add_special_tokens) - - def decode(self, token_ids, skip_special_tokens: bool = False) -> str: - """Decode token ids with the wrapped HuggingFace tokenizer.""" - return self._hf.decode(token_ids, skip_special_tokens=skip_special_tokens) - - def convert_tokens_to_ids(self, tokens): - """Convert special tokens to ids.""" - return self._tok.convert_tokens_to_ids(tokens) - - -def _get_multimodal_encoder_base(): - from megatron.energon.task_encoder.multimodal import MultiModalPackingEncoder - - return MultiModalPackingEncoder - - -def _get_pretraining_conversation_cooker(): - from megatron.energon.task_encoder.cooking import Cooker - - return Cooker( - cook_pretraining_conversation, has_subflavors={"cook": "pretraining_conversation"} - ) - - -_MULTIMODAL_PACKING_ENCODER_BASE = _get_multimodal_encoder_base() - - -def _get_energon_cooker_decorators(): - from megatron.energon.task_encoder.base import stateless - from megatron.energon.task_encoder.cooking import cooker - - return stateless, cooker - - -_stateless, _cooker = _get_energon_cooker_decorators() - - -@_stateless -@_cooker(need_cache=True) -def cook_pretraining_conversation(sample: dict, cache, media_source=None): - """Cook fragment-style pretraining conversation samples into multimodal text.""" - from megatron.energon.task_encoder.cooking import basic_sample_keys - from megatron.energon.task_encoder.multimodal.cookers._image_util import to_pil_image - from megatron.energon.task_encoder.multimodal.sample_types import ImageRef, MultiModalSample - - text_parts = [] - image_refs = [] - for turn in sample["json"]["conversation"]: - sender = turn.get("sender", "user") - content_parts = [] - for fragment in turn.get("fragments", []): - fragment_type = fragment.get("t") - value = fragment.get("value", "") - if fragment_type == "image": - content_parts.append("") - ext = value.rsplit(".", 1)[-1] if "." in value else "" - if ext in sample: - image = to_pil_image(sample[ext]) - else: - image = to_pil_image(cache.get(media_source, value, sample=sample)) - width, height = image.size - image_refs.append(ImageRef(data=image, width=width, height=height)) - elif fragment_type == "text": - content_parts.append(value) - content = "".join(content_parts).strip() - if content: - text_parts.append(f"{sender}: {content}") - - return MultiModalSample( - **basic_sample_keys(sample), text="\n".join(text_parts), images=image_refs - ) - - -class MimoMultiModalPackingEncoder(_MULTIMODAL_PACKING_ENCODER_BASE): - """Adapt Energon multimodal packing batches to MIMO's forward signature.""" - - cookers = list(_MULTIMODAL_PACKING_ENCODER_BASE.cookers) + [ - _get_pretraining_conversation_cooker() - ] - - @classmethod - def from_args(cls, args, tokenizer): - """Construct the encoder from hetero training args.""" - from megatron.energon.task_encoder.multimodal import PackingConfig, VisionConfig - - vision_config = VisionConfig( - img_h=args.img_h, - img_w=args.img_w, - patch_dim=args.patch_dim, - vision_model_type=getattr(args, "vision_model_type", "radio"), - disable_vision_class_token=getattr(args, "disable_vision_class_token", True), - pixel_shuffle=getattr(args, "pixel_shuffle", True), - max_num_tiles=args.num_image_tiles, - use_tiling=getattr(args, "use_tiling", True), - use_thumbnail=getattr(args, "use_thumbnail", True), - class_token_len=args.class_token_len, - conv_merging=getattr(args, "conv_merging", False), - use_tile_tags=getattr(args, "use_tile_tags", False), - use_image_break_token=getattr(args, "image_break_token", None) is not None, - use_area_weighted_aspect_ratio=getattr(args, "use_area_weighted_aspect_ratio", False), - dynamic_resolution=getattr(args, "dynamic_resolution", False), - ) - packing_config = PackingConfig( - seq_length=args.seq_length, pad_id=args.pad_token_id, image_token_id=args.image_token_id - ) - return cls( - vision_config=vision_config, - packing_config=packing_config, - tokenizer=TokenizerAdapter(tokenizer), - encoder_name=NEMOTRON_VISION_ENCODER_KEY, - encoder_input_key="x", - target_seq_length=args.seq_length, - ) - - def __init__( - self, - vision_config, - packing_config, - tokenizer, - encoder_name: str, - encoder_input_key: str, - target_seq_length: Optional[int], - ) -> None: - super().__init__(vision_config, packing_config, tokenizer) - from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings - - self.encoder_name = encoder_name - self.encoder_input_key = encoder_input_key - self._target_seq_length = target_seq_length - self._embeddings_per_tile = get_num_image_embeddings( - img_h=vision_config.img_h, - img_w=vision_config.img_w, - patch_dim=vision_config.patch_dim, - class_token_len=vision_config.class_token_len, - disable_vision_class_token=vision_config.disable_vision_class_token, - pixel_shuffle=vision_config.pixel_shuffle, - conv_merging=vision_config.conv_merging, - use_tile_tags=vision_config.use_tile_tags, - max_num_tiles=vision_config.max_num_tiles, - use_image_break_token=vision_config.use_image_break_token, - ) - - def batch(self, samples: list) -> dict: - """Expand image placeholders and return a MIMO-compatible batch.""" - image_token_id = self.packing_config.image_token_id - ignore_index = self.packing_config.ignore_index - pad_id = self.packing_config.pad_id - - expanded_samples = [] - all_images = [] - for sample in samples: - expanded = self._expand_sample(sample, image_token_id, ignore_index) - expanded_samples.append(expanded) - all_images.extend(sample.images[: expanded.kept_tiles]) - token_rows = [sample.tokens for sample in expanded_samples] - max_len = self._target_seq_length or max(len(row) for row in token_rows) - input_ids = torch.full((len(samples), max_len), pad_id, dtype=torch.long) - labels = torch.full((len(samples), max_len), ignore_index, dtype=torch.long) - for row_idx, expanded in enumerate(expanded_samples): - tokens = expanded.tokens - input_ids[row_idx, : len(tokens)] = tokens - labels[row_idx, : expanded.labels.numel()] = expanded.labels - - loss_mask = (labels != ignore_index).float() - loss_mask[labels == image_token_id] = 0.0 - position_ids = torch.arange(max_len).unsqueeze(0).expand(len(samples), -1).contiguous() - result = { - "input_ids": input_ids, - "labels": labels, - "loss_mask": loss_mask, - "position_ids": position_ids, - } - image_tensor = None - if all_images: - image_tensor = self.tiling_strategy.stack(all_images)[0] - result["modality_inputs"] = { - "images": {self.encoder_name: {self.encoder_input_key: image_tensor}} - } - packing_kwargs = self._build_packing_kwargs( - [sample.cu_lengths for sample in expanded_samples], max_len - ) - if packing_kwargs is not None: - result["packing_kwargs"] = packing_kwargs - return result - - def _expand_sample(self, sample, image_token_id: int, ignore_index: int) -> ExpandedSample: - """Expand one image token into one placeholder per image embedding.""" - if not hasattr(sample, "labels"): - raise RuntimeError("Energon multimodal samples must provide labels") - - tokens = [] - labels = [] - image_idx = 0 - kept_tiles = 0 - budget = self._target_seq_length - sample_labels = sample.labels.reshape(-1).tolist() - if len(sample_labels) < sample.tokens.numel(): - raise RuntimeError( - "Energon multimodal sample labels must be at least as long as tokens" - ) - - for idx, token in enumerate(sample.tokens.tolist()): - if token != image_token_id: - if budget is not None and len(tokens) + 1 > budget: - break - tokens.append(token) - labels.append(int(sample_labels[idx])) - continue - - num_tiles = sample.num_tiles[image_idx] if image_idx < len(sample.num_tiles) else 1 - num_placeholders = num_tiles * self._embeddings_per_tile - if budget is not None and len(tokens) + num_placeholders > budget: - warnings.warn( - "Energon sample truncated at an image boundary to fit " - f"target sequence length {budget}.", - stacklevel=2, - ) - break - tokens.extend([image_token_id] * num_placeholders) - labels.extend([ignore_index] * num_placeholders) - kept_tiles += num_tiles - image_idx += 1 - - return ExpandedSample( - tokens=torch.tensor(tokens, dtype=torch.long), - labels=torch.tensor(labels, dtype=torch.long), - cu_lengths=torch.tensor( - self._expanded_boundaries(sample.cu_lengths, len(tokens)), dtype=torch.long - ), - kept_tiles=kept_tiles, - ) - - @staticmethod - def _expanded_boundaries(cu_lengths: torch.Tensor, expanded_token_count: int) -> list[int]: - """Normalize Energon cumulative lengths in expanded sequence coordinates.""" - boundaries = cu_lengths.to(dtype=torch.long).flatten().tolist() - boundaries = [min(max(int(value), 0), expanded_token_count) for value in boundaries] - if not boundaries or boundaries[0] != 0: - boundaries.insert(0, 0) - if boundaries[-1] != expanded_token_count: - boundaries.append(expanded_token_count) - - deduped = [] - for boundary in boundaries: - if not deduped or boundary != deduped[-1]: - deduped.append(boundary) - return deduped - - @staticmethod - def _build_packing_kwargs( - cu_lengths_by_sample: list[torch.Tensor], max_len: int - ) -> Optional[dict]: - """Build packed-sequence metadata when Energon selected packed samples.""" - is_packed = any(cu_lengths.numel() > 2 for cu_lengths in cu_lengths_by_sample) - if not is_packed: - return None - if len(cu_lengths_by_sample) != 1: - raise RuntimeError("Energon packing requires micro_batch_size=1") - - cu_seqlens = cu_lengths_by_sample[0].to(dtype=torch.int32).clamp(max=max_len) - if cu_seqlens[0] != 0: - cu_seqlens = torch.cat([torch.tensor([0], dtype=torch.int32), cu_seqlens]) - if cu_seqlens[-1] != max_len: - cu_seqlens = torch.cat([cu_seqlens, torch.tensor([max_len], dtype=torch.int32)]) - segment_lens = cu_seqlens[1:] - cu_seqlens[:-1] - max_seqlen = segment_lens.max() - return { - "cu_seqlens_q": cu_seqlens, - "cu_seqlens_kv": cu_seqlens, - "cu_seqlens_q_padded": cu_seqlens, - "cu_seqlens_kv_padded": cu_seqlens, - "max_seqlen_q": max_seqlen, - "max_seqlen_kv": max_seqlen, - "total_tokens": max_len, - } - - class EnergonIterator: """Endless wrapper around an Energon dataloader with TP-rank-0 ownership.""" - def __init__(self, dataloader, tp_group=None, source_rank: bool = True) -> None: + def __init__( + self, dataloader, tp_group=None, source_rank: bool = True, random_seed: Optional[int] = None + ) -> None: self._dataloader = dataloader self._iterator = iter(dataloader) if dataloader is not None else None self._tp_group = tp_group self._source_rank = source_rank self._prefetched = None + self._python_random_state = None + if random_seed is not None: + rng = random.Random(random_seed) + self._python_random_state = rng.getstate() def __iter__(self): return self @@ -450,6 +156,20 @@ def peek_signature(self): def _next_local_batch(self): """Read the next local Energon batch on the TP source rank.""" + if self._python_random_state is None: + return self._read_next_local_batch() + + global_random_state = random.getstate() + try: + random.setstate(self._python_random_state) + batch = self._read_next_local_batch() + self._python_random_state = random.getstate() + return batch + finally: + random.setstate(global_random_state) + + def _read_next_local_batch(self): + """Read from the underlying dataloader, cycling at epoch boundaries.""" try: return next(self._iterator) except StopIteration: @@ -519,15 +239,12 @@ def _checksum_packing_kwargs(cls, packing_kwargs: Optional[dict]) -> int: @staticmethod def _checksum_tensor(tensor: Optional[torch.Tensor]) -> int: - """Return a stable, bounded checksum for a CPU tensor-like batch field.""" + """Return a stable full-tensor checksum for a CPU tensor-like batch field.""" if tensor is None or tensor.numel() == 0: return 0 - values = tensor.detach().reshape(-1) - stride = max(values.numel() // 4096, 1) - values = values[::stride] - if values.is_floating_point(): - values = (values.float() * 1024).to(dtype=torch.long) - else: - values = values.to(dtype=torch.long) - positions = torch.arange(1, values.numel() + 1, dtype=torch.long, device=values.device) - return int(((values * positions).sum() % 2_147_483_647).item()) + tensor = tensor.detach().cpu().contiguous() + digest = hashlib.blake2b(digest_size=8) + digest.update(str(tuple(tensor.shape)).encode("ascii")) + digest.update(str(tensor.dtype).encode("ascii")) + digest.update(memoryview(tensor.numpy()).cast("B")) + return int.from_bytes(digest.digest(), byteorder="big", signed=False) diff --git a/examples/mimo/scripts/compare_energon_dataloader_parity.py b/examples/mimo/scripts/compare_energon_dataloader_parity.py new file mode 100644 index 00000000000..875d11fb644 --- /dev/null +++ b/examples/mimo/scripts/compare_energon_dataloader_parity.py @@ -0,0 +1,355 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# pylint: disable=bad-builtin + +"""Compare current MIMO Energon batches with the previous branch provider. + +This is a dataloader-only parity check. It instantiates the previous branch's +``MimoMultiModalPackingEncoder`` and the current branch's encoder in the same +process, feeds both through Megatron-Energon with identical loader settings, and +requires exact equality for the emitted batch tensors and packed-sequence +metadata. Defaults favor deterministic sample identity; override workers and +shuffle settings when intentionally stress-testing training-like loader behavior. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import os +import random +import subprocess +import sys +import tempfile +from pathlib import Path +from types import ModuleType +from typing import Any, Optional + +import torch + +REPO_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO_ROOT)) +os.chdir(REPO_ROOT) + +from examples.mimo.data import energon_multimodal_provider as current_provider + +OLD_PROVIDER_REPO_PATH = "examples/mimo/data/energon_multimodal_provider.py" +OLD_PROVIDER_BUNDLED_PATH = REPO_ROOT / "examples/mimo/vendor/old_energon_multimodal_provider.py" + + +def parse_args() -> argparse.Namespace: + """Parse dataloader parity options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-path", required=True) + parser.add_argument("--tokenizer-model", required=True) + parser.add_argument("--old-provider-path", type=str, default=None) + parser.add_argument( + "--old-provider-ref", + type=str, + default="origin/feat/nemotron-moe-vlm-mimo", + help="Git ref used when --old-provider-path is not supplied.", + ) + parser.add_argument("--image-token", type=str, default="") + parser.add_argument("--tokenizer-prompt-format", type=str, default="nemotron6-moe") + parser.add_argument("--image-tag-type", type=str, default="") + parser.add_argument("--force-system-message", action="store_true") + parser.add_argument("--seq-length", type=int, default=8192) + parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--num-batches", type=int, default=8) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--dp-rank", type=int, default=0) + parser.add_argument("--dp-world-size", type=int, default=1) + parser.add_argument("--seed", type=int, default=12345) + parser.add_argument("--seed-offset", type=int, default=0) + parser.add_argument("--packing-buffer-size", type=int, default=128) + parser.add_argument("--shuffle-buffer-size", type=int, default=0) + parser.add_argument("--max-samples-per-sequence", type=int, default=100) + parser.add_argument("--img-h", type=int, default=512) + parser.add_argument("--img-w", type=int, default=512) + parser.add_argument("--patch-dim", type=int, default=16) + parser.add_argument("--class-token-len", type=int, default=8) + parser.add_argument("--max-num-tiles", type=int, default=12) + parser.add_argument("--vision-model-type", type=str, default="radio") + parser.add_argument("--pixel-shuffle", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument( + "--disable-vision-class-token", action=argparse.BooleanOptionalAction, default=True + ) + parser.add_argument("--use-tiling", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--use-thumbnail", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--encoder-name", type=str, default="radio_encoder") + parser.add_argument("--encoder-input-key", type=str, default="x") + return parser.parse_args() + + +def main() -> None: + """Run the parity comparison.""" + args = parse_args() + set_seed(args.seed) + + old_provider = load_old_provider(args) + tokenizer = build_tokenizer(args) + image_token_id = tokenizer.convert_tokens_to_ids(args.image_token) + if image_token_id is None: + raise RuntimeError(f"Tokenizer did not produce an id for {args.image_token!r}") + pad_id = int(tokenizer.pad) + + old_loader = build_loader(old_provider, tokenizer, image_token_id, pad_id, args) + set_seed(args.seed) + current_loader = build_loader(current_provider, tokenizer, image_token_id, pad_id, args) + + for batch_idx in range(args.num_batches): + batch_seed = args.seed + batch_idx + set_seed(batch_seed) + old_batch = next(old_loader) + set_seed(batch_seed) + current_batch = next(current_loader) + mismatches = compare_values("batch", old_batch, current_batch) + if mismatches: + print(f"Batch {batch_idx} mismatch") + for mismatch in mismatches[:20]: + print(f" - {mismatch}") + print(f"old: {batch_summary(old_batch)}") + print(f"current: {batch_summary(current_batch)}") + raise SystemExit(1) + print(f"batch {batch_idx}: OK {batch_summary(current_batch)}") + + print(f"Parity OK for {args.num_batches} batches") + + +def set_seed(seed: int = 12345) -> None: + """Set process-local RNG state before loader construction.""" + random.seed(seed) + torch.manual_seed(seed) + + +def load_old_provider(args: argparse.Namespace) -> ModuleType: + """Load the previous branch provider from a path or git ref.""" + if args.old_provider_path is not None: + provider_path = Path(args.old_provider_path) + else: + try: + provider_path = materialize_old_provider_from_git(args.old_provider_ref) + except RuntimeError: + if not OLD_PROVIDER_BUNDLED_PATH.is_file(): + raise + provider_path = OLD_PROVIDER_BUNDLED_PATH + return import_module_from_path("old_energon_multimodal_provider", provider_path) + + +def materialize_old_provider_from_git(ref: str) -> Path: + """Write the provider from a git ref to a temporary importable file.""" + provider_source = git_show(ref, OLD_PROVIDER_REPO_PATH) + temp_dir = Path(tempfile.mkdtemp(prefix="old_energon_provider_")) + provider_path = temp_dir / "energon_multimodal_provider.py" + provider_path.write_text(provider_source) + return provider_path + + +def git_show(ref: str, repo_path: str) -> str: + """Return file content from a git ref, with a local-branch fallback.""" + refs_to_try = [ref] + if ref.startswith("origin/"): + refs_to_try.append(ref.removeprefix("origin/")) + + errors = [] + for candidate in refs_to_try: + command = ["git", "show", f"{candidate}:{repo_path}"] + result = subprocess.run(command, check=False, text=True, capture_output=True) + if result.returncode == 0: + return result.stdout + errors.append(result.stderr.strip()) + raise RuntimeError("Unable to load old provider from git:\n" + "\n".join(errors)) + + +def import_module_from_path(name: str, path: Path) -> ModuleType: + """Import a Python module from an explicit path.""" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import module from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def build_tokenizer(args: argparse.Namespace): + """Build the Megatron multimodal tokenizer used by both providers.""" + from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( + MegatronMultimodalTokenizer, + ) + + return MegatronMultimodalTokenizer( + path=args.tokenizer_model, + prompt_format=args.tokenizer_prompt_format, + special_tokens=[args.image_token], + image_tag_type=args.image_tag_type, + force_system_message=args.force_system_message, + ) + + +def build_loader( + provider: ModuleType, tokenizer, image_token_id: int, pad_id: int, args: argparse.Namespace +): + """Build one Energon dataloader using a provider module's encoder.""" + from megatron.energon import WorkerConfig, get_loader, get_train_dataset + + encoder = build_encoder(provider, tokenizer, image_token_id, pad_id, args) + worker_config = WorkerConfig( + rank=args.dp_rank, + world_size=args.dp_world_size, + num_workers=args.num_workers, + seed_offset=args.seed_offset, + data_parallel_group=None, + ) + dataset = get_train_dataset( + args.data_path, + batch_size=args.batch_size, + task_encoder=encoder, + worker_config=worker_config, + packing_buffer_size=args.packing_buffer_size, + shuffle_buffer_size=args.shuffle_buffer_size, + max_samples_per_sequence=args.max_samples_per_sequence, + ) + return iter(get_loader(dataset)) + + +def build_encoder( + provider: ModuleType, tokenizer, image_token_id: int, pad_id: int, args: argparse.Namespace +): + """Build a provider-specific MIMO multimodal packing encoder.""" + if provider is current_provider: + return provider.build_multimodal_encoder( + args, + tokenizer, + encoder_name=args.encoder_name, + encoder_input_key=args.encoder_input_key, + ) + + vision_config = provider.VisionConfig( + img_h=args.img_h, + img_w=args.img_w, + patch_dim=args.patch_dim, + vision_model_type=args.vision_model_type, + disable_vision_class_token=args.disable_vision_class_token, + pixel_shuffle=args.pixel_shuffle, + max_num_tiles=args.max_num_tiles, + use_tiling=args.use_tiling, + use_thumbnail=args.use_thumbnail, + class_token_len=args.class_token_len, + conv_merging=False, + use_tile_tags=False, + use_image_break_token=False, + use_area_weighted_aspect_ratio=False, + dynamic_resolution=False, + ) + packing_config = provider.PackingConfig( + seq_length=args.seq_length, pad_id=pad_id, image_token_id=image_token_id + ) + adapter_cls = getattr(provider, "TokenizerAdapter", None) + if adapter_cls is None: + adapter_cls = getattr(provider, "_TokenizerAdapter") + return provider.MimoMultiModalPackingEncoder( + vision_config=vision_config, + packing_config=packing_config, + tokenizer=adapter_cls(tokenizer), + encoder_name=args.encoder_name, + encoder_input_key=args.encoder_input_key, + target_seq_length=args.seq_length, + ) + + +def compare_values(path: str, old_value: Any, current_value: Any) -> list[str]: + """Return exact mismatches between nested batch values.""" + if isinstance(old_value, dict) or isinstance(current_value, dict): + if not isinstance(old_value, dict) or not isinstance(current_value, dict): + old_type = type(old_value).__name__ + current_type = type(current_value).__name__ + return [f"{path}: type mismatch {old_type} != {current_type}"] + mismatches = [] + old_keys = set(old_value) + current_keys = set(current_value) + if old_keys != current_keys: + mismatches.append( + f"{path}: keys differ old={sorted(old_keys)} current={sorted(current_keys)}" + ) + for key in sorted(old_keys & current_keys): + mismatches.extend(compare_values(f"{path}.{key}", old_value[key], current_value[key])) + return mismatches + + if isinstance(old_value, torch.Tensor) or isinstance(current_value, torch.Tensor): + if not isinstance(old_value, torch.Tensor) or not isinstance(current_value, torch.Tensor): + return [f"{path}: tensor/type mismatch"] + if old_value.shape != current_value.shape: + return [ + f"{path}: shape mismatch {tuple(old_value.shape)} != {tuple(current_value.shape)}" + ] + if old_value.dtype != current_value.dtype: + return [f"{path}: dtype mismatch {old_value.dtype} != {current_value.dtype}"] + if torch.equal(old_value, current_value): + return [] + detail = f"checksum {tensor_checksum(old_value)} != {tensor_checksum(current_value)}" + if old_value.is_floating_point(): + max_abs = (old_value - current_value).abs().max().item() + detail += f", max_abs={max_abs}" + return [f"{path}: tensor mismatch ({detail})"] + + if old_value != current_value: + return [f"{path}: value mismatch {old_value!r} != {current_value!r}"] + return [] + + +def batch_summary(batch: dict) -> str: + """Return a compact human-readable batch summary.""" + image_tensor = first_tensor(nested_get(batch, ("modality_inputs", "images"))) + packing_kwargs = batch.get("packing_kwargs") + cu_seqlens = None + if packing_kwargs is not None: + cu_seqlens = packing_kwargs["cu_seqlens_q"] + return ( + f"input={tuple(batch['input_ids'].shape)}:{tensor_checksum(batch['input_ids'])} " + f"labels={tensor_checksum(batch['labels'])} " + f"loss_tokens={int(batch['loss_mask'].sum().item())} " + f"images={None if image_tensor is None else tuple(image_tensor.shape)}:" + f"{tensor_checksum(image_tensor)} " + f"cu={None if cu_seqlens is None else cu_seqlens.tolist()[:8]}" + ) + + +def nested_get(value: dict, keys: tuple[str, ...]): + """Return a nested value if every key exists.""" + current = value + for key in keys: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +def first_tensor(value): + """Return the first tensor in a nested mapping.""" + if isinstance(value, torch.Tensor): + return value + if isinstance(value, dict): + for item in value.values(): + tensor = first_tensor(item) + if tensor is not None: + return tensor + return None + + +def tensor_checksum(tensor: Optional[torch.Tensor]) -> int: + """Return a deterministic bounded checksum for a tensor.""" + if tensor is None or tensor.numel() == 0: + return 0 + values = tensor.detach().reshape(-1) + stride = max(values.numel() // 4096, 1) + values = values[::stride] + if values.is_floating_point(): + values = (values.float() * 1024).to(dtype=torch.long) + else: + values = values.to(dtype=torch.long) + positions = torch.arange(1, values.numel() + 1, dtype=torch.long, device=values.device) + return int(((values * positions).sum() % 2_147_483_647).item()) + + +if __name__ == "__main__": + main() diff --git a/examples/mimo/scripts/install_energon.sh b/examples/mimo/scripts/install_energon.sh deleted file mode 100755 index 9b95e7b47a7..00000000000 --- a/examples/mimo/scripts/install_energon.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Install the custom Megatron-Energon build used by the MIMO multimodal data path. - -set -euo pipefail - -DEFAULT_PATH="/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_genai/users/ykarnati/public/Megatron-Energon-sasatheesh" -ENERGON_PATH="${1:-$DEFAULT_PATH}" -PYTHON_BIN="${PYTHON_BIN:-python3}" - -echo "Installing Megatron-Energon from: ${ENERGON_PATH}" -if [[ ! -d "${ENERGON_PATH}" ]]; then - echo "ERROR: Directory not found: ${ENERGON_PATH}" >&2 - exit 1 -fi - -"${PYTHON_BIN}" -m pip install -e "${ENERGON_PATH}[multimodal]" - -"${PYTHON_BIN}" - <<'PY' -from megatron.energon.task_encoder.multimodal import MultiModalPackingEncoder -import megatron.energon - -print(f"Megatron-Energon installed from: {megatron.energon.__file__}") -print(f"MultiModalPackingEncoder OK: {MultiModalPackingEncoder.__name__}") -PY diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh index e6197aa44d4..43f6d6d1f82 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh @@ -16,14 +16,6 @@ case "${TRAINING_STAGE}" in ;; esac -if [[ "${INSTALL_ENERGON:-0}" == "1" ]]; then - if [[ -n "${ENERGON_PATH:-}" ]]; then - bash examples/mimo/scripts/install_energon.sh "${ENERGON_PATH}" - else - bash examples/mimo/scripts/install_energon.sh - fi -fi - GPUS_PER_NODE="${GPUS_PER_NODE:-8}" TRAIN_ITERS="${TRAIN_ITERS:-100}" NUM_MICROBATCHES="${NUM_MICROBATCHES:-4}" @@ -36,11 +28,21 @@ PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" NUM_WORKERS="${NUM_WORKERS:-2}" SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" -PYTHON_BIN="${PYTHON_BIN:-python3}" +if [[ -z "${PYTHON_BIN:-}" ]]; then + if command -v python >/dev/null 2>&1; then + PYTHON_BIN=python + else + PYTHON_BIN=python3 + fi +fi DATA_PATH="${DATA_PATH:-/lustre/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/kshih/workspace/blends/eagle_recipe_online_packing/final_recipe/pretrain_base_non_sft_cw_dfw.yaml}" TOKENIZER_MODEL="${TOKENIZER_MODEL:-/lustre/fs1/portfolios/coreai/projects/coreai_dlalgo_genai/users/ykarnati/checkpoints/models--nvidia--NVIDIA-Nemotron-3-Nano-30B-A3B-BF16-multimodal-pretraining/snapshots/7344a79074e20d9ab548e14c25b0492345394f67}" +if [[ "${VERIFY_ENERGON:-1}" == "1" ]]; then + PYTHON_BIN="${PYTHON_BIN}" bash examples/mimo/scripts/verify_energon.sh +fi + echo "=== Hetero MIMO Nemotron6-MoE VLM 20L Energon training ===" echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} gbs=${GLOBAL_BATCH_SIZE}" echo "data=${DATA_PATH}" diff --git a/examples/mimo/scripts/verify_energon.sh b/examples/mimo/scripts/verify_energon.sh new file mode 100755 index 00000000000..91c11ef4cb9 --- /dev/null +++ b/examples/mimo/scripts/verify_energon.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Verify the custom Megatron-Energon build used by the MIMO multimodal data path. + +set -euo pipefail + +if [[ -z "${PYTHON_BIN:-}" ]]; then + if command -v python >/dev/null 2>&1; then + PYTHON_BIN=python + else + PYTHON_BIN=python3 + fi +fi + +"${PYTHON_BIN}" - <<'PY' +import json +from importlib import metadata + +try: + import megatron.energon + import torchvision +except ModuleNotFoundError as exc: + raise SystemExit( + "ERROR: missing Energon multimodal runtime dependency. " + "Run through a PyTorch base image/Cog synced venv that already provides torch and " + "torchvision, then install repo deps with `uv sync --locked --extra dev --extra mlm`. " + "For a non-container local env, install torch/torchvision separately with versions that " + "match your CUDA stack before syncing this project. " + f"Original error: {exc}" + ) from exc + +from megatron.energon.task_encoder.multimodal import MultiModalPackingEncoder, PackingConfig, VisionConfig +from megatron.energon.task_encoder.multimodal.sample_types import PackedSample +from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings +from packaging.version import InvalidVersion, Version + +EXPECTED_COMMIT = "d456cbd4a9a8a760b20be51194a0209c9a945b0a" +EXPECTED_LOCAL = f"g{EXPECTED_COMMIT[:9]}" + +dist = metadata.distribution("megatron-energon") +version = dist.version +direct_url = dist.read_text("direct_url.json") +commit = None +if direct_url: + commit = json.loads(direct_url).get("vcs_info", {}).get("commit_id") + +try: + local_version = Version(version).local +except InvalidVersion: + local_version = None + +if commit != EXPECTED_COMMIT and local_version != EXPECTED_LOCAL: + raise SystemExit( + "ERROR: megatron-energon is not the pinned MIMO fork " + f"({EXPECTED_COMMIT}); found version={version!r}, commit={commit!r}" + ) + +print(f"Megatron-Energon path: {megatron.energon.__file__}") +print(f"Megatron-Energon version: {version}") +print(f"Megatron-Energon commit: {commit or 'version-local-tag'}") +print(f"torchvision OK: {torchvision.__version__}") +print(f"MultiModalPackingEncoder OK: {MultiModalPackingEncoder.__name__}") +print(f"PackingConfig OK: {PackingConfig.__name__}") +print(f"VisionConfig OK: {VisionConfig.__name__}") +print(f"PackedSample OK: {PackedSample.__name__}") +print(f"get_num_image_embeddings OK: {get_num_image_embeddings.__name__}") +PY diff --git a/examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl b/examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..443224314b832785b99394e0cdcb6389d9c6f2db GIT binary patch literal 325361 zcmZsCQ*fx^5@c-Kwr$(CZQHhO+fGhwJ14enJK1wzw(9QQhrhn6|8Zt|rmLq%K^hnY z1poj5;@>BvaDHj&6$1(YAdU(Efb;KtEqgyIS`v=D9~$d3YOY)KRw}9W~G!G-?yDt5ZBF zwZe?sfXJ*=C1%AHtXKlzu?w1C@%_>60Z^e+%89AyK9OEnk$Bkod-MIWkvRx9vOJMCt0wQcqtT}+~ARw zQ!U~Bfmh;v;YZTN9Amy`@4fMiNF^oBXPp})hTS`)WLZbWU}5598)v@|*cmtOep#P0xEHWH*`CU0D(tj85_#haDx3MYul?1uCTV=#Sd&6D6}RwX7y z;6YBb?LsW~Ds>O*)C>Ie+%{|3T_Fd#l*~#qjZw#HQ9DcY5?+AbhXM8I;nNZhrh?|5z7ZR5Dl z@V>=BxtFFl{Lv}@>?M%&Y~Wr;4VUP_PPp{QRR%2o1+G*^eF&R2;yzx|uSUriBO9I&( zZ?pX>SIzb6IcjOn8!exg2g%0H+ODPQc3jCuemXdDMMEzXkkxIT?!8xE{;7Y zaO}SGTSf`9G`cY;U`vp^eK?BnO2H7KLyY$K! zRT!enr6onpx7?-^FMocgKRuy(4Bg{(8lvqJd)b5-qfTFpYva1F5P`gJ^&Xj}0P|%C zI0%s1sTqUo0gu6m-}B3QoS|>odD=Ty>u_(BiRWe9H7BjT&JmzTM_b~SJNk318yCLY zDa`bq%*!8&VVuv82Lb!zI&W@E)yDE}40H(G%YvBb^-th}K4?q;m!O_89N5fDK)<$C zg=Rz3z4=x))jeA0=n&H8nl{tR7Bf^^ZVTE|JsTLj%2_+^E9nyRM_kJi>kXJ^ymXM! z;a%YQeZSxM#%)yMH}^Zf!hdz~C&D)`dT0OuT>=0A!vEF9jhqaPO^xksYz!ToP5-s> z9?fmL&2iM;Rb`oT?7xW?l5#IW6@SLLx@s2)G>65M1yC`fa5P)q)Zs9-8DXBP;1!-A zUnRX}Gxr_BQhK0UDgw6^h922ZeNN58$gi4e;9R}BtTV8^Z8-INGn?T3JG|VV({%m5 zo*u{K?d|LM{O-ufuRc!Q{d0TdjrAb-x}-c7N;Xqg0YCRYwy%8LGDU0mk|nd$O}87e z?lEwhAE(f({-zE)b4U6s``C7^{VriBuA8Bl|Z zil*+(dNDcToXYc5d(5=i77p7_Wkr^WDav-7ihj@zqSH_zjpA2jK?m})+h&mp90kKD zD71KLI4dYP^V2<2&9rNC`%CL_?wBwEYHL=MII4qG^cy$dmikh+xPnSwQfBrfRW~r0 zS}x50u3Wk@7~X|n-0facNn@{Ed6?56q{6It;F_AyWDH-jn5n9;50aPtS@(58=ns^> z;I^WHcsdqN0q$Jl)&u}rc9!hG$B*1qA-4zk>yvQV-^M~2 zfp+wiHFbM4s-P`w_i>gLnH z<-wS*~3tS{a`b zH?nbB2o$ZXhJ$O?>9r}lgk;^-q~ivYGPy2pp!7#IB_CMJhFLrc1V>P|yd`Hu;I)C4 zqfn~g9!<$V$oZl|cRs)kBTM!Kip>>(B+EE>M#Ev_lVW8rWpPGvtkhH$CXv7Z-vj(< zJoJRQ{neK-_MoMwja9JG!_XeLq84*HsF40=D6T~o8Cv@5lbZ*LX8Pe-5elYsK_@^bVE=0SY~ptZojYR;xZYn|(Zv*cs< zx05XnkxK3Y4#4qFcSafMkWYdS73ai4JwK+#zet@_Z2G9=fp9dJA*;+fC(r6Y{^fy% zVEW8vUxc=BfS+f6cX4d+izvf}C1rey=Y-)<40`M28aSvmd-{&URF^oTus}VK$3-?JSlEFCBI-86 zrIJZa>gd6B(K9@>T6S44p6R39T~#FpYj7}tjs6(w#1)GT$LmH$dqXs9cSY*i3)n+-A1m_sucZL&ASqMj}WlAYT@k$qpE_%P4fhy>Jt9)9bv;AG|UR3n{idApqo& zdmyjcS{+#2lgWAE+)--LtXjQ+{;S2;gKl25Ekh&D*v#rL1`2cFtv>D7b89)K763Qk z>yLy~!ocJ6%YHptNr_(2PUq|&Z&*Xyr3FuPolvejw^9K(B?AhLlk!(S&BWSj_2s@z zgK@7_A%%|4o9R5vCWB%xeG5<0G>m_2*AFs)Pc^cLx6}dJn#SSgysQ~t&}e$R_&k7H z@qb|+x%@2}*kE!OS7W%lqT!{Ib*XPn-dm{P3!vc}rFLXgftz*QRd)3oNKMQ42Msj{ z(w#U;H$dZj=*tnzxI`u}MTruJr=;x9!0f|x{JbfpfW)B{JekaXyUU&%(QXLh6Gq>bypr>c3dE?)~f#LPd z2uQfS<}18;%-fu8MhY8B%*~UO>MK^Ll1np2M@*yZLf6=AGzb4zx5&Mj(&zVK!i;8= zOX6zv&^n6o9NcC`A^K&MG6E^XwbYd_;W38G*N*YbK)db*A=hX-&s#x8ML=DkVFev! zxFSaiE-i1l{SERoto@cqAr_$+sEjZy`9UyJ_2|2x!mjFRfr~CMi@XqkWiJ*O0D`{k zBPD6n3UE*^#xK1vd>Kh8M7gO#_njg5c#t&+92N0ajUAj9J@+r4eC#8ud*4$lW+Vsn%@s+Up-n*kwS_Cs`DoU^ePX1XW|JLaxpktwv|?Hzj2x2JfY^ z(X(LPWM+;C5`70DTkRV0lEW|}YzT+JcIHy?07RaDrO(58?xI_$v@w_f2OOq4er7kb zC2`=HK0^Q6k+{Wohn1bHUv>l(i8>cGd3^U)kYlfkKsOZ zV4o-zEAZgt>$+2)Uk^OkJDWr~Q^b@ih7Bsa4cBoU=BPf_2gf!Bf=J_4%vw1)%0JmF zRf#^Y83k+w*`*L59d`NVYnb3!91;P1GHVn7jI##7&F~iN070d9t%#ihWD-^^%RgWl zQNLd4+bdpVehbB*RETLAqh+sri_YV0k{M0zSc+Jd`I83)>VX2^?Ow)+9)|>E?{T#l z&xnfHAT}cyjXgH3;4@%%aRUJ_ZqaLdhl_l|QdTeN!-?EAe^CKS-KB4lyT$K1&;a|Rv5Dv8@uxJY5hvmFe};$z#+sQ%2JSC*UajzeV|yL9n4<~1KZV)V~}6V;z#zxUI=r8|nFk;ik& z*tqg?i?%GX{QO-nM-25OF##dTB+>1!vL`@s-4uyefHy*dhA(U8(w9m+a_VuN6E)JW zEfaA20DSKjl}7)Qs3d|#ZlyRZ`sq85Lr5IM2*JVIIP_`AAsjDv3@$9BRi4dREGqnq zGiaubn9p3r3uQ9DqWo!dKh( zYl-_@`1n8`Z z?rESF5p2T|ov;`kf&jnXQS1K@lJeGX*XbR=l#{%n>WC%UpnAC)Xr?`&E}ztrX8Um870>T7{^w&CSG__23omO{DjX};5 zW3L}@pQ3E8a=g}~T5!)@&e<&yimeDK?}$`)QpJYb>P85KFnhrjjhUCh2$9yvb2eG> z&2|@xxB%@Zsa&#K-h#{$NeOYMSyUt24xhc}4`upOnkCZx-^IHp4R|Ue7X-c|qyt!^hbi>yuh$R#3V5mO zvU(Jr>Ap8*eRh?a2S3aacvc<;n=CRiCZ^WD-e&cTIz6oh7&Z8lA_H3vv%lYo)1OwR z&Fd<^^#VJEX=G<3Fh!7xg6OIyOJG26sBj9`lJw%dJ;`?p#8cCa^Bs`o@ zwz#%P5EpL2T7=nn;wFH)0T)O;AWTHjwBZ#9cNjGzhVMskNIa5+|4Pb{W(0;k^Xvu> z@T@XPuS}{u6*zC8MoA?GI@MqqJI+?EGb=#rx#C>s=u@`tJK#mkOEagnWEm~kAZI04 z05w&8Nbyt9NYTSy$jc@t%fRxuBAwHvQHpni8{D2ph2{=l?m)c4`yrhsbAc5>q6BrLW^+t*)Q*RG z1okvHd2uuB=;~#Vc4I{qV|iiI979#zMNsQbCmx_(U4U zSf~l+IxdF*I3v?};;#_H14JK?XI5v9JCt+Q7q&6i$(}X+tOzbhiU2}_ERZZ@x8Ng| zE>E^Z-L7S~))Yc2kxI6xz9t9x1}V9L*Hy{12RJ6cp2-j{wOSdu<){ZF2OvubC-|uY zVTQF2HAAMA4>Oc-IZeW#P*b>|rlBy)l!cw5#Q;Qzk`YC|%G$glt&XY$ye(WkxZ(!c z_-poHeXipP;&2gL032yLF$w$PlGvkf#=clR>9fV=;vF}}*qED!eb~%a%`we#^McMA z+m57#-XT)o$ zWPs9bcmRqpZlK~LKoQUGmU|sHKJJ!th+gdvB3Rhh^#}N#{uG$U{-Yu7lL5QjMHhZM zeYl8Yc<_az`c>@N(C);3Xx}!%kiK^xHp32>cyDa#lzi;psflHemrWbK-nuyK7trl} zax8Qk8WAXW1)J$d+LkcuatlFCWmr8ulQ&tYEb8{M#RKekPCr`8Pm{kbEA9HgZYfI= z65H;2cMAHF8BD(3?CMzU!`(}VXV1d%rpvT^NTjj_&nZN((9fWdg*I7B zsaB{~mi2PG8QG+<*S|4_>xD_pyOdRlP}B9XT^40(cQ|PD#vaz1Ujz5eD0KkC&0Pz3kSTuETGBq}I zv9!1QHxo@2WUU7U5PELwF`kklAW8p5D^#$e0F6YXZD?z-A$Mh5g{{k$M*Q}iOpS7t z@AqV~-_Bs9b$$_c{5I`Dpa+HTIYL#(Q^cS(#IRp_G<(D!q|S{t=!WO-WTRBTcqv9r zPRbC??3-eHG23T8P7)Y5QLt)qI4LPz!%j({C#Xtk)H(z6%67BFG@>+{u}C(Opllgk zAUG8-qr@B^BpkAz$Om$pJ5=f|*JoeuS({Q1>P~H6eYe`VU){)+Z#fAe|H;bPuoqu? z-deL=pkCAH+apb4-fv-7&X9o{>%^{ksS5{eUZ_laNEW#VNHl*TgFvPyJNW8eue(u0j;v3T(JEY(4l_;#rt0YHT@6KFt#ytcK#R7 z*aUs+K?anO-A78g52Hfbp+(j>OIw?huLqDQ*!^d3 za~4A*`BKELn1l4L1b^#(_lB-mv&4)urqiR}ox0;rniE_NTQhpW>50Tl86%KXNR$H2 z0OG4H9q?sH=xLbJuyTb5Bgc}oqtQhC=EF8>PoPr43tzCmh{!2gKD-@;!>F^Zl9^B5 z>>(h_ga{o#b)f79atGX8Jm;I0l5y)~1NnS%*t8~^^15qpLEJFZ(e{tf)ImxLu_%$z zjB;;74vu0tu64A5c#0_HF|lCQ&+wId!WtawJP0fISMcgycuvUrSIlT?*$wu3<;s+<6Uv|+atb=2}aJA zF9rqn5xygIL4yUPlPjV~ph;E-GbeqwF#04)N`Flha}?GXMQ361GtJG$dx_AyIirmH z6-Xq_kIEPGe^|-!`RaG-W+z6*s^#(PXW6Uc@zEBjn|_{N{v^ALPFhXKsbN;DCZfyG zO!TeV%lq+lP*_rHq8taQlN`4ixmVG+l~iESrC?WD$iB*CXkM(LJnzhK)W9Mk? zJ`OmHqR>jE@sg$gAu`=8n(aXDS`ntk$V}!17W_`3*hpYDw?eW%&_ofnlq@B=%r4Tj zPMtW`oy1bT^J??jaSGweTg0-J?uKYMk8UTCPVSM*WS&Xk5cJPhbM6ep=QS-P|H zd@_~uM4u)?#YjwUKRzlkB+jrLmDQg`5+Cd5v|H~<&;f?W=oJT;Jef{rN7WAwB%KWi z7jalfE4njtK6=*}+&~teezf!B6Oq3&Y+^uqayorojlKdtP_MG^w-cyQ9?Y(>*F%&I z1M-ikW>OY)lkt(jB}_=tw&G{bz7;oxkA1zrzCGQpJ-vTd`&Y=4?$8Pnuc2>Bl&aSC z7Ti+QjwdiR?(PrFNOr0aUXbngdw~@=2_3LF(^g z-*em_*O$2RmZz}r{+VJt6GTyQluq>O)6wHRcktTc3B5t8 z9392B*&TN}K!dGio|O82kQR*%mc)7G-M5zRBR*BTa1vhN&Gk5^m!J1yL_2riLAzAQ z?rs=@5&8F?f`qDnJ`IY^thrDc%B{({OEFM;se=G&S~KWS<>=aO9X03jGA`ITS!q(J z>au8m&#=11L zBETWnu7?D05H&+{f$4pCw4HV*de7_Zh#s9dW9yO%8P6(Y6gwNQz{8-y>Qca0D!_e> zK{o!e4I`?jH1gn$xxxnldkV?xvq54}j@OgV>oj%#Q+wEY@PFcd_y z%dx0Ra7E|fD_RF64o(iSGx_;4^JTiA{`oW{@+5J}BR7OF2uqJOPra?)sKg zjnx%Ztd>g&y1XW}!!X687iYeF?ZLb}3dTxX#JFXS%ew#tX;`0)GRg}H^EMefKu_xc z!e6bR6q|eGELmacfd=RR%I0$hr2~vA4z!x-AVdyYjK%Rt##K%ZmTOD!E=#bf9r|?l zP^=}ek_!jFUHkB4*L;>7ZmTQ`l}$)exfsui& z$0fWuu;@|lA8wStc?^XN?{;3HfcK&8u!cm&IbQ3xQP^M;9}tO7~9vR-gZwy2yVI=1A1B}6D zTv%ut_3eW=44GuBd&^7jy5 zO)F0eUbg$QXxob5eZfXs=m(0hI7^Z04zp(Nu+pdwLi!|_y7B0qL)$+q5Kz-C5bh(7 zdwb80vF0CQ;W1Uu%Am9WKboZj$T%&qyZE7q@xU4{a-4=L0&M^USiYx`i0?S@Vy2p-6-dx&|JC+pvMSpaO>W4f@{%CT4{gx)K-wfDsY^0P}wm7*9i6 zn}7TzS5?t|lMTUdwVs0lJ{dq8o9k*s0K8oi+++NJz%d$DlyLsW~j zaVbXJ%3*e14HOT(UAH&A#dCW)x8B6=t-&P{`^)h4epv&{pUWdwkf)c~m(Md!&06}V z3z%Da`d@r~DWFu*kE)TA6H-N*{n?w*@9~e1VlHUK5^dhLI6h5UqNxkC?SAyjaADu2 z7v^_jRE8hRUSPK=BwdImA8nC9RU^3$V%0FHS-5?#gUhj7VIyZZ*Q~xGiYi8+v1q0{ z(QOl)9)o7l1_^^-=5gH-0eKv8^b;r`UyWbaNG*Z-_N(d&6JP^lX#Hz2Yf+WW+zDtK zbrpq3D}{jS(4aUp(cO!s>+5@Y`3T<1#rlDZCh>$owV7uCU4pBat|2fO#*I9t>QLkY z8V6+|$?g!VJ=KwEJAQ+Mrp87;wABd#NVH1dQr`5sxx<)?v5UeqJ8)@U@l=sBY;v4p zt_=HQ9&zChWU3c)jh>VU=vTksMlPcZaDg;-**qFoSJUk>v}ZSEHQKhuA#XJsY5H4w@#>0APIM?1Q zZWfDWWVEsqk*r3!3Z)FCt7r*3u#H4=xxwBcO4YYH?nYfb_y7wmz@#WDye-LIILf-c z!)^1Ekd+xs&3Aa3PP{p@XGJz)cunhn0oMi!1jOKb(!x))wa zbg8^g)Wz#YnJf1Q)pQq!nInFq@PGHCnKMB+7YT#CT96xHbL9r91TPG7{vtQ%zq!nd z`g4$yJEAoMPX!|yYk>D9tEbqD7%iRxR>&UG`B6ID#@$p_a)Hg@P}*qGMVF&F`fD}% zuVgSMxI{vPD=alpO70X;XNfjF{q{s*ee-+-PyGz@QC_$RAx9%CF*{`oiIx^ZM^hbA z#wEUGtn=hKw7hw0@!{j zcPjlov;S{=+xxKU#DM?+U}FFP;QlXso7p>A>s#2{TmQrOwTAXTjHCPq;|_WyLTF)d z-i!%`dO<*7D`gVFwgLzcEu!loriuiWUOSz?y~0ZDQcZ^?+Z(p51{&yj@w*+qB))*^ z-5d~YI}~^7fZbmw$L@sP@1Bl58AHo6*PGd!@7o6_vPZi8U4K^lCakEh$0tmT{7wzRD_|8Hh&eTh>18nz7*SdD_j?W+s-|o6V@{c@9HvJ=g-EBn zi4FjGS}FL@2~zjoxy}e*0*r=y%@}Sp>jp<3KR?4~Wl=;k81fdu-wgcmX#kOPGmd3O z8tE<5)ELjbs`;UD1|&xP%FkFtm8`=+;1|veVv&78SldJRe{9|pNf-^n^euv_Xw8i= z2+NG64}+j0K^i~I8iqfFhSf=uSpv>q&Gy>UA4vfJn!z4kb8W(*Sg#A6$xmIt$zHHn zLMWazlu)9xxp*`#N0K#+QBxHl;3QKMgWK(m5QTCp{+%~ky7Nvmg@_vox*OLes-ewk zvlH60Abyci;;mFJ?I$tdCp+{sIS&XIegG1#q>>4NG1NS;Z6gPdRIX4u3bVze>@)+NuneQy{^0 zZ?8-~ZTD#_e|ye9SDbg_Y?EBxqla)q|LI=4d0r;w^^U#{O=eFuJHpbqX|aOE%`W(y8^|OfQ$3S1dshy2ga0bG{5eU2V&&JgxlID-V3LdChk71#9({ZT0XcTlG zyougvEN>pFQRq~{%HA%M)9>1>QGX@V@x^wZz^|z=UIfwQyMb#Ov!huyrrEZgNVunv zGhW)f#Ze7cF3H8{8@jf@#Sx)#+=0W=j)m0Yn#^YUMO+=@$H#|*JkN~_NFfzOy^?}qV~*$Irr#h z{JE2)leBoT$OTHrPnil>M9Z>R;?+!KEk(?PWxm7Lr4%*FT?O+NsEz|OV0E$eH1!J3 zoh=Jm@EGGf%vp-gIs$Gh6&2podjN*SCrxbaz?(B>UFqI}IISBnh81j4rNxYFsERs72qx^IjMyLyvCUa04g`&DEtjBtU$V1W0RtA-cZSSOyWheP7!QvgIGe{Ds3#@ zBU{NHk4(ATD#X(ovAqc8fTwk}icuJZq}mO#B_dN9m#BtfA+#m{MbocnF>sQa_2Z<` zmG~+Vh#xytydP7i;L=)>TnCF}8x|b18)jBl1q|KsFk@Re`sCg=Z-N`qL z8Gh$T<0Hn+$da%-I5AHcr!h}IN{yHEw!S%zfKc7T<5umAnamoVup!h$VZ1@0bTcFu zU`_f85*CXhAVfm)=WM5Zu(!axPj`QR7GkWY{Z6gKQ%WVTPEa*B;V%+J2kvRrP~q|% zVF_cRWL`8K;y$tQC0t{2d2|;hvkRlLK^5il=>7Em$CPaiKrW1oIV}JzmOo zj^G@}G&aew;)_U5Va@a?t$T-JZRRmNm>a-Ar7)SyV0R`Do=+~k*{QL>OVCsi^Qs41 zMPTqr4X`5RGn_2pB8PA{3miCc%CX?ABi^AZ$u?wU5!E$RYqm0u#y3Olwkx_#oUwY^PMD`iBKF=sbxrjY zLhX%U!jU!x+MlwDwITXf$=_z|?l;UFJnuK466SOGf!x)B`vvanEmkpM>-+MGrYZNm^0O{tFFfB0Yta(#b zpwJu&#Ow-uv6J2G-$m&jlrsCgxxl_oZ?Uhp-eN2+kx(euSpVg&s?W3T$dXMzw`u*! zwd<>2d^Z9{PA7v3FXMyJ_K>R7NHtS&!AM@gfiCdE3RP!Mlx2LGW`yZ|Tjl`x7-Y~v zF_3a01zaU40Svzm1_fp}AYovz0A+Fw%AuZ)DM3Z%q+p3fN*hGv!UEEmgeQ>PD=M=p z?+Jio4zndM65W*BXjNSpWTRU}ec&C1eC;%KG)$bVCB+PehPrD-Yh$a*P=(U;d)O-NYZN~53ExOy*YYOoG%4QflE6M5#YG7Clo~*}^>=OyF@=F6*R(@{1W91`Fkb?s)FoS1D`!X}#xd zcfTz!eANi_m%@Sy-O)$$@KagPl);CoS7bADatF@ed#T!FiL4&33(qPuWVV#V2n^PiuWMzHzD1tVo-jx28WP!?EcY3Fxns8D%^)8k>WKWSx^e)P4y;Q~N zgS?DGW&*Chmwlic%~id9o9h=-(YY$lvYr{aF)G~45B&!G@5*&p%FL|(CquCRlOg`6 zjosGX#MQ=B|Njp8P_=(@1P6@YTs^@SI=I7D7w!#dKkOOnCPIOAw4FK(1dw>+xk?5N zX3~}}!&P5=3l6tj&!Q)i>XlmL=)Q@PmU7_ks?x21T5BVo z-4kD+c45jIRY@w+EH89+zt9O^VEBe~aj4WDzic#I(O~Y~${7ml2JY|5U>`*XZ$Ynj z$?Jqk5;Kkui)iuPY(x-+Y(O4G)P{_L_9GSa3GoAt(7Vlg5@KL{OYT7qK`9FbeaB#1_iCjtXVo5ZE%e9@wDC9jr} z*0pw2-RU?v?;9i1vGt*`U*=1heUeOfY8I9&&bq~<*SgdAV0^P@S~d{mNLr<+;xMq7 z`M9jAwAt=^7;;H$DO<#y|8y~72nQG-9QGTP3Qo~D8%Gn$x!?vWaFf%MABk?8Fg>rD z<_q?*!AsgQKIB?Ge*ykd@HJmU{?iL~m$R{=P}Uqh2FOjPNrZ>cu7)oX%LNxj1@+%( z7e7U2O|WV|zA;!cvW9=xx}%L#7xor(-JdqMGMGz-7N8Y~ zJp96F;~$x*kor*6Jgo>V6@&&NX`R-`Jh8mW5_96`@e`46SPA#E!R3s{dHS;RaW}ON z(KQGXu9x5@)6E#}DqZTwGIaCBb4D?KN|sCaH6l+M--r+D5zkJb0j833PV!n-dm8Z^ z-0SQ{{SKtimJDy>R|i<5%NQ#kb&&G_J_pf{Y-(;Q`!|f25;`}$3B(ySO za=Q$R0mjxkwZCH^3H|v)s-njZFhZJgRm0#1)J*(~gHY!Pd7)N!xvit43!jSWs#_&X zd_67V%~|<@XOff9V_bYQjU>fI%&K+AoZEOISmbzPI_{%^+Lh~xW(!mrE7ftUo$iJ9 zjDP*3yR%UpE9Xw|@4q1b8|i{Idubg1kpA`$=>-2vkZ>|}aq`r6c5yN_wEZ`wQdJeL zHyIHA0iEM7dN?Q9_1X{&Ws4~bf`7c5m1qGz4)&{}f3 zNe6H=oOByJcv2qJ8=(<_j#>AppHZL77wKt(#)6!S5h8{0CIqW}Xe7|Y$i1)$y5+JR zuxp62jpb?q4Y)d!_-xu=gsGEA99WDl7uaY}jH%eYwP6TTnBvjOV-=C8t6u{Ja`Tel zCIj|>eaT$4hun_;6)ZXVZ;Et6`A z2|DJs61FWF3G#=Yfj|c zI2#DR>2rzsw9f)>s1Z}kBq4IwjM=(UlqT3~oL*hrD4R|nl+C(Y&$RfM%1c8|nk>{N zR@_g&5Qf4kSeejwwtHN))p8Vd2%;$zEBA0A4_#vUdxt`?8@|lno#h!{8${whh<6?X@WONHwaU8>bb?n}Sz1Zo2iTfHwRj=CPN~nlqwdV5UR=*9XKUT)YL5j@k`nEnYMpjd(t zKe*vRNix1QY5wgwP0C4Uc>$xP!e}z{@Z60T&TG7UP@wME#6}w?^me_!pl5b_y?Hr# ziPG)s@c%sdvGV*pq07o?4FALa%l9ieS~P#~#fBJ~Sczga`}5=A{B!;8I!aAThMb8n zWxAGk=}Y!!ruaZq=_FYtm6Q2d5RLmpy)MdY)Gujcdads*P|fUWW?;N&Z#{-LLZ4c3 z*l0BES+M)C3ROYnz0eL%rt;u>k5H;#X~Nc2gh{6&X4)miVTfEVU+1b_(G)RKr~>!9 ze1zjri(SgiK5xC~op{%*sPNaEjE@(?uHZFT+tX}UxsnfR*~Z&Gzix^swQIs;0xfoV zz=ac*JULQ%GI1NdtDhPE6gh0ph3i|<1pLm~#j7c#yR4=nR*@yfEXH76<~nrS)Fkbk9LX1EPN>A5 z-&L~MgUk4DLO8V*fkl91aE zyPJu`n0}cY2&Nn&Q7UH9h5id|i76Cpq6^GuYD@y|yxE~1*gj>P(xIvIVDg&vjbhZtOhKd z0Kc&y$0gV3;AB7qLz38$gD*AJPCAAlcBaomlY*u;?<-F_hFmEZYK+W`2%^SD0xjB= z69#4dn&{^PONRLsR-u5taQ&=E-2GT(VFQz?XKLYC8fPd(U<&kqyL2au@-tn=gJN6QR3$uP&?p?q_HKg%#r`hP zF2B!51;3mU+o9-c9rX_BbWCZJ-gXSNDPj^@FHanuI5Yasu(i5b`c>oglQ!$8I*<4o zPz6W-DQpQ^ zN0=Th*#VzUytyuZh9J4khqtm`hW?XP-mja2d*N8JBw(Reo^T#bpb;*-rGWhK7A~i{ zc51dck2tfxnAMzoA6gK(Vwkkk=??mydo^*jI#OzVa zH1=3jZa<1~MPx+tRN8gUT-0p>HK^Vxs~73M&2J;g>kfw)mc&NQeHo7bL)SS4X&Q7} zx@_CFZFbqV@s(}cc6Hgd)n(i6vTbYnpP7h>b7IcL8<`P#lM(OD%C+~ic2NuaM@|Ne zu!H@zT`iPQEfj>`2aF#2&zDAK>fdU7`*aA>sS0r70+i7v_wsxbrkptri*q#M5_*uTn;(iW;iK!UN2j;U^@318Pd zL#xGWyP{&od+$FAF};#ln1&)QUYSth;Qm8~SgTOmWz@IGd&N#pI15WF5|p8JR-1nv z{}f-d@@S&-yzRlpDmz9-c}tl7D8iU`xOh#&R*LSQFGk=XJ%!IcUOmG0POY1NTqvAR zvRxAXPUWKVH@rUc6HfmRc9bjUY!QE~^j^8)*FqITrdz zq;ofc@1hR}{?S6d6CJL1luFPyJ1P6+im+1(i%pdM$}5o9G8$@yn7rR zW&%%t+&e+98t%@=oV|&qTD~l7Ga+kCiN>MFNFSq~L%0jm;^i>AXb#;gms@yWcTzV8 zY#pG=LP8A|ISAg0pN`FCEggH@F64jJP(x0{4Oj!t-ADu*2pWZ=YwASA$<=@xnLwZ%+>&y@-Ct#4=>Ke;G9kUz1pwDg3{mzIL4)Fz0m} z^=BQkd#MQ=hcn0~RgrSXVnU-RX@4byYt?)9Xzwvf9ncxG1fX{l#5bu@HReZ!A*Uvd z8%NMmsAh^cwxLR2)yWMlFr&uTR&2WTBCz%c>NQ<>!KY78nhQy|`#V^?mFWJbe zwde9T#=1PMGSl3T?S_f!Rz|8COMiDpJWj^g=mO3RaJ-=bE zB)CWs8C%q;kEibj@@c*D1dxFP_A_KK&*WlKmg&ITS{JkU{?{8m6rv!CvK;CghsV zU$h@Z_8ny2n3%*Qzgr#l021R#aK+oZcf+4 zIKUjV0%4jZNkdc7U@GASi%FN;lV_^LT4Lzp>?c5K5FRSHUCSiXr1%|zBkp>p(6EH< z(puhao>Yg@TEC@S7fKV6Yy7KEbE>$QmMZ%6FO}YSz$bVZrmPY{L#QxZO>CBL6++fA z&7_>%)`MEZ%2QD7y`IMn@OgdBu4224O#QgBwSgi-Bf)oQ&n;KH)IhP_yBAjar0cuv zF{g_Q@bWToA$eB|N#8y^{8Pr&(Vav zOSW@yLB^NOv3kVo!pMQbs;MUY>cZqYgs`F7YS>m-b(1Sj?~l-2$Zj6BLT1pbW&E!B znGK=4Nu%$3&Kh?Pid=9MO%V4jR$sp2eMVKw1zV^3*B}Ysi(B3`7vI|oA*j=y2IX8` z9$6izfwzd>md|~Nc~fyUa5^$$5wgWrt;S|~#ptHHm92SQI91;1sp+{n?unPa$|g6r>zy;)BJ=YErIYXaw=9lqn7bLOd@uwJ0>oX0V_ zqDQw5MA}jvNb}G#C=foK$-U<&4dckRKfAyTr4<&aktHT(x*et1UE3SFOTBKp*6oVw zMemq#jg(`XR+>M5wG8kF5>160^MB3BVMbX!!;wAVTf3W-(>C3k?-j-Ix0M(FhE&d( z`#o*O>mHpmlsy6Pud+7wDz$V#+ob!YtHceZ_Xr5Hu({}_rm(*Trk3PT< z{)t0tiT+9K zWh3SqkfQ9$eGlEb*hx@BxRf_*)z7`lI+iZAd^vZ+n%CZ!7nn-y{d~LTbCf}A(P(^q z3KPH{nbEe>p=c}A6hjV+Sy=)7f>3WKY@4N)_Pz9hJ`M+8s~|~eVG6LT@{OyN-^TqE z*PY6<)f)yFs_K0|1S0OXU4m}Bg+z7U2~5Tk5^vYA|B$hW)8QG ziVu<GL#v3U`xbj3S{!A4x5fOyz zU_#h7ItaBm@FOh4BKH>LYy-o=a4IARxmSU;@Ti^#R@o%Ao@!YD@?PLQp~uOv=QhOwzuK}U=&P8 zZfzcF_0I!}ZN7Pbr;Z*-m#G5f&nZ!MG9-Yfmm+AwLHlw>1h-R>T9bP*P0H}&?xaYk zcQWU&*!tkPWEHHVTr4f%VG)PdCA^aFr7UU{$d*!6@uX$TF!sGrd*Ny>^qsce*Gda$ z)xN6@m1*RFzz2${IgC?H@t_p*MlDXT-fT_i*O{JT=U={fzBO1mZl<&L`VH;y`N@(k=*tKc_S^4`_9 z1NH;c0p^SOBjTunWc!KOMkWX0K?RN&YuVUk_cM2K7pZn0a_{1$XO#vkoTm+bB-B79 zjZ6bY_A~JfDKj2D-kMIRA@YpuqjP6r`4*^lkdL}w@A_IN zpo1=2U#JwWU!P5(!MEpnhGlyJr8<5v*6`AhBt_s$wAl+u5OG0F;|U@18Rp*f%Z7!Z zU$4d^mOjFsx$35TaBmxSwfDbyMh+$6uG0S?B(r}NMcn_NXXI?;W@K#hpHLy+1Uc(q zA*8UI58ALp6ycKM$0QNULbQ;i5KU=&Kv9vMmfkwH$fCgxJH<3)v-5iV`b*BsIm3@rE|ppE+YENsZT1#5bKTJ^hZzB6T>Cp;i`xJ;tdwVfqW4z z21rne8p**sWd&C1m|!1AkGY#0)3Vkk(k2rUE^A7+mO?iO>vX1SvgEqV;`Ac2swolLbgE{Fz`VU%=!g4*Uej?xx32rp${J$Kenvb<=B>wtCuv3>a9`c6z-?Rxe%T{w;n(>>dUgtJ0UNuSJ=Gd=2Jb6uJ0}!#J?lM}SkCpnqw`G9^nb-0 zpFfPrKM=v#Gy4JkSIPSPx$;xU00dNg@jvk9|MwUCzq;6!?}an2MAC`-7dpSk+*m=Rl!N^T1+?{rd%I%fNL|H?`h%>AQ(oTbj^tzNt0x(HZxep=yC!GaJ=d29zEzh8PncxiqqxNXL_a5b1eUUW%1~#bTfGe&HUTyKZ*PORGRJnx$0hsBFB9~XwN%;s7H_L zYtYIvAAeC>UF9?0yA6A)z>fKOVbi2vDd=upQ1xmJt~D|%o+-U-*0K%o%u=)7+U(5O z;S!_(SKUa_$dX69hRkdJ3wUonn*i`q-!o|B_=EcU8=Qn%jQp*rD(Q2tIn8yPNo}9$ zDy^kh`>}p?&z-donfSr+{{%~sCu11uM1H`lC~Pd(n7j5M2!zi*U@E6OmLr)o&e&O& zBFM=;-z4_kNgeGx&P?Xj^rNs{=R>>~5+| zv<#{oZ_jClNFc@Y^rG!X^a6pU8^Rgvqp4x3mT0fc-}IZlwK&w8T)4P!`wo2|*v;HF z!)uU+;i6J|6{HZd)79LWXI_`9>Eu~Gn0d%+ao`~Ai3P_B?ltCvd{7_#$AO(DR?V$)mO}JIg3$=rbVBp6+GRO>CeNG=<5?msASY-emaW@&ne>C#ousggJ!&Q zcvmYJO+V1=6_}3ZCSscR*FSoO5}nI3O&5&D>QBr~(h?;H9L0aQyMVN%$G!dBj83VV zlwYbkdG12=SDZ>R@J`&4x`RF8qJdCbs`DkE9FuakQfw*6`VUBl^-)PR>x&o8A- z>l4tml!7kL)4HHhQk4Y+LoZ28D%v5WBF2ep2(Z;)OG-^yW!?UfcJFM~O43xGpfS#& zd9SbvfhgJ!fmqTk+EEiw{D;`|@W1h!{L8e7NxwyE0@_NEV^2BchEu60 zv;mn5ra#{xtk{x&ujr14A6c(Us9(8N*CJJN`|^>Ae_Ge&wvYA7Hh2;|A&Z!054dTH znT9;mjWTU?7}85~JfR7D&x?(p{b;HJuH>MV z*;Yi|NnY-i66R|3zb8)gJR#r0i6=K2&CccWlw@TUwFb&a1oWDmb>a|b-dl})A-GnJ zmY7`qIbjNTS)$CIval8;HTGvV5*Y4uD@hkaaCl?PnwsK3N3hMWJ%prjA}o~d{3OWc zZ_3uIEH>~NEVK-+mS!xrog9IBH)fqna7|0tE*gD+{Q)f2EC=XoWo;6T;bw7WusP_>f`NQMPS#NjAa=;L_i8;TO&KkjXCQy}zdgL}Yy{c& z=?pixX3lx%*#N|5jkHudWkziXyD)>$fxF4k_XR8iT7S9=npomSglF>ijMhtU_Bg36 z|1y>uD3$W3p>xO17tCD^;=QP~be(FLn(CXEgjetVVq|56@O4Y1bM>-cy<-9~IaoKK zA_mpxq?S+lup)ae({#rl%)r%5t8o$3^Q7yK>8?_%F|>umZzbw@-iWn7f8Ge5m+wRjkhMf zk<$War2T?>{Z{$fCu=@Udp8E->+FX<-Sn(cfy9UY6y_$3xL)=R=!VRIh*fn6F=P(j zN9&7T?>w1ha8Bls5y3-!LYoZAr?r|B2timNmSx#zimUboXK1G8(B$&k|D~4FPat=q zAn-my6^Kvx9$6HvbnN_ zM#m^>j%d&Xl1M56Ma~^Qc@|gYC*|eNwD&t)nW(H$E*O4=!03XHa4AEMg4AR@=IL0q z;xZ8_$Dp!L-~BX9-_k*Pg@2S_0Ogt!r7>;#i&^-{{j!y@t{~?N>4l6`tzW~THqHxs z*)w!E$Ur#2Z&PMK(8VBEZ2ep=T~14T*wGjysIhby5c=CXYK;WtZi+isVnL7t-oBBF z!Uf9w!`-Ckj}QzoawabpElU`&jIgs|R{AN&xH>?smV-85p$&a(esg2Ldj(SwWNCd1 zZsM*OO}#8CX_AR;C&yVCrmQm4^v3>t8l~4fy%Vlh=(&Ab-_$@~em^$o+&op)M90j( z%CS^QJ$0>Khj>LYD>Wcc%r!JrTB-HnPd~j(m!5&s*76h$S^0U<=cqb&b1=>c2p!g| zX8u$71LdgFGHoBg)T=(-vxmk^)rPi>tMDTx`6pP4AUsvBv64yAL6tpexfwdtG}Y7V=oMVGBSdvUqj<8vWTd`iTQjMJ zJqzY5`j6OddxDvhq4}hjpLGqf7o+d}wfCzNv0jT=0JkvPDE3!L)8(e(T(1`ghR>*( zORzBIIWjqa|qoHb^bg5ZCVxZlopTIC|N>bUCnIJq_zr;KH>}%-qf;zC)gB7GO_dnO4s#j15}p z7QY($JN^Q!AImVQ?f|9)>hHiVfNrBG6zMmt&x%w=C7EtoXKeCd_y~pT6GevmEn^1* z7Ef`+vX^fX+ASW|H!H5p6s+o3B6b8hkae{UgdiW#!B{76A*Zks0QKNJmBcI9wVOAk z+L`^{KGJ5+5M&2qc&L^=D2^Lnnpr_8dJ zhET9v`wu`8;dltYmA*>@-UllLJ*gQHI{mUD+g1}#cHW}WNf~>@kdg#?z(~u!^Rbpt;6)!}&ar9_RxV1T46`WQ6!>gpeNZeJ7vDJ1 zXh7nAg!2*c9o6vrIYC2WPrf_~+f?Dw>%f^Q_1CY6aFKV&<&T_X`~e5nE$TMGj}IXu z;&(ZhLFkg4hkZ_DnFqjDqw2!Npud|%TqSB78EO`suvW_R&&^!!FSD)CDZ`Nr_iTUZ5i6a1m3VSR$uiQQ=-xxYIeF z@WAK$MQ*)a|6>W&NOI;oZ#bc`)fes!ZT}lu?90(gPI>4C5Iu;H&4tg$w6Wta+0;gA z*l&p1PRTM8)%gw*Na3dCmlGT%sbl)8F>y_bj6`DIN4^zWr<`l2@2*~BfJOpj{au#NJK2bL2tfY1-Uf%vv z5iXfyi7N<2(3E?@_q2IzQo#ORg~|;=@ra;^oHWg-p*$ zk$G-aY%qb8cPTq6iA1i*1=J2d44l_KzkP^0pNMYv^Kf~1X!CN`FAu>BD(XuG0~#J` z%T8^66-Z;9o6L#CW>g>GaJWcSXz?|!-Yj16_NgDXrgNF;-cOLPKb=D^aK8Bx$s$~4 zj((RICXDNNkw?O?<`p@@dtI8LQn0N|q`Y z^-y>9VoydQ3cK($*(8xjLBgruy?}|z)B+;aP(i*U9)q{?pc#GkN|Ot3oY*NlTV<60 z_l(PJ{TiAhemLFIkz<&o{eI(DPJLXlC5e=Tfe`Ik?$cc?6T~ltLMtUiiPi(RYduVNYs&iy;21fSeWqyBg{z z5If;kAgVa|T2WEw0Jl@UgGY)Xmo(KHu}LOoA}|Peu$LX?p{RxgMZJHD+$caiye4_zIMZg#-oEumrQxeLTMP?Q$Od59!|ivZFZEudIpZ6=AK1qf$j= zZ6|k?k+V(n30(~50A<~gm`5^~j<0QPUTTLWJESRf&Efty1u2g-n&(c6APgs z`Z|nr`wS*)t2k?`#i*Qe5;IE)@(-hndcA01ng>FHl8VLUWHe-BnJPc znE}kNXgZ_|qA=fEGNM&6kF)`vSrP^K$1KGeuP&77mV+kq^!x`@Z4DJY_jZlM7M4yg=4OfjCGYHHOloWciR+I6S&qSE5d2 zOqgW~5*SMGyG1y&oe_S?Z>$n9wG>d*B&K5ieK4t!7TlL8zQY|S_`?$=i`JVS9(>cYP>FM&jjYTm3$aNDBgJnF(t`=7h4VfJceeRtuw3BQY-GRI`PY@IQbxj zR*$pK(jI^?mCLg=%vE$jN>WaJVbf0p4fA3UowGhKmPLF$7MCmFr4T^+4bbCoYF<;B z#;m$7ALaLHo^oF|{z9)PDIp3dfqO|$3(@kyCDLvGmM4rdKh*-QG$pW^&CYMhp!AZC ziGx302!$Yj&3GgPu-7XY{>3i8tGh*O;2`+kh#F|sw4jWbVKjABkwh`n|0tKPxc$vG zF#M;2b)X#L{gXz4%H~Mvl)1*y-LqrJPF8hN98J=3dG1q6vyWL8^TH79sdB3tS)ofT9(b?wHu4v&EEUJ=-?=<{ zQ2zX&?%TaW1G!(KMAp_B9P&nFiFjomOU{fFiAROK#W+3L4#AP|o%fZxXgKlHWcmLhP_D#da)RJ_bYwOS;zsdslnRFwuZe@VpJi)u zuFOiZ5+?P02%KQvSebMh+s5(V6PHcpL!66-QPF954Dkxx&Sn@B*Ll0Q>du!Qv>6f_ zw`e%@cz{pYo=Wk>OV9YEJRSl@FQ}DyVQIKmo<-muHMEDt6U@Tl$N;6i6Z zuLnkfmiFdfk5ERn;CmARILPAxrV&6S!NG);145iq1^H5<)-~ANYCWQwndQR=eMe{Q zM?yLEPVQT(vr2w#Beu93GhSgzpVAlh7TEr();D6z;s$6D;70=yQEF-)0se{T(PR`I zz42OAs$tI-8m$u<;Ob9d^3aDli}hOhR|Snop<_>Rv{9W@!^qqV zgh7NfWLh%TQ;JauO6vBs!C{gFG{L@wMFYjlMd<=&2xWTG$Zidv?c^9>F(z@P4KI91 zeNxZ1P@ATgfkj71xD_x{)MkR?DaAK0`DlYLuw|nt+^<(zve##equ3eZ7N`|6En|ms z2;V37!J+39qeaEoV$fGs#rMUfLU~-pHiY52tJ==7o+^RMmTC30-grXzfh`l;v}g+0 z)ngLLhvO`nea%qEHm%CHFrA`+_8Mu6SvGRq+z*;uRdASbU70u{GJ2rck|Sk0+eqQk zMxN*bSB-|+6HX_+jQRKss}O)`Wz|B0`6Xz8Xo( z)N{@=tL9qUHT1QV+r2dSIgh>@jUG4l8ZkNcy6}z)s5;kNr&H%<6spL2<+3@3Bl24E zE^jWs{Gerl3%VSGjMq}kCLPsv%Vm>5iic)@0}6fH%BacLdcK#Xs9^bvfj$Ak#R(TH znLoMX_dBY=thvRnz$?)(P_~DxTUN0USgm>nHWZHjW+*tb6LL%4w_{Zg75ojL_g(Ca_fFhZw5jts>EQzy z+D!(X35VqR{PM*T!WhQn@tv9zJ=VR*=xEuk32H`Y zvv-#2!HTl-Ddskh7%qq7+hqe6>ghbIl2hd*swpeZ6B{A#k9*27ukP)Efvc#H|Mj%Yc-t`TNBnXtuYYafV#emPs|OlG zxYP$DfFRLYei3KW%66rF$>F{s1&)oK%)&o?)*f4k-ciN49@E-Tb?e0rBiM#aPV(b57ufWv))s$6 zHNerUB`<8+S;c!PyL9aGSUTp>g^)Jp1H$_J3t`4UaXPxV8nFQ%4*Tip&)$rGO2jBC z3gI#yn@nkiS=hR!xV(tv86Kl~x%Opb0iNd`j=*gd?#=_7 z+BpfM?9VMP#^>0W0_G1V&fA#?nD=uOT>Bx1;h!EoAAjp#94^L>MYkhTP$k{;D)s9~?#q-)OO+D>>T2N?Fsb+=OONs8jxe>&^S;>SI@QeS&s5D8gx+J8R) z>$}fvX3|{klDLr>l)0C2kw0&XJ$M=;Y_9bu$-O~(P`~vFDBq{T(TJTCGpc$7`_VeM z1HJUrXDJnbpHz=Jx%9O3>M_FZT{09L_Vr;Y-MLZP)o*Fx+&lY{|6Mj~K#J`ynQWN( z<7o8`mQ?O*hlX31AQ7Pi?m`bpfIA+(&t z{e0p?66WH45ys*!!_5~@ab!tmXDt)W4TPp8b!pm%48OVx2~2X68SmQ)(Vz9k7(=OZ zMH`FH!c60A!SXZ_zk;xS(q_!Ew|#Jh;}mIZOWKbBv&na__7HS4ZeMmDHYn5wu=^JQ zkKoyST&!|$1lGj^G~uISI(XAa)i|H@v4&8*J+AXvK!J3OSe-ITEojo4QGt;cX0MJB ziA3U0%Mq#}$0TEIX5lHR;Q7!Q83R#DjH22){j5Jo?wfhRV8Zw(C(;!A^|{Sm{$F(; zU=)l4W;DA-gFICEi(oO8FEjITdr*CvvS%(ulwCWl5(0BHY@jN;j+7H-rVY7p(mmoH zi~{midvv75e*pg}*24yO6k3ziQ=0-HO>;0N=QYJFl8mM}n*5G3?41@$j}zEOzy5nI zBxX%HCoo^4;AyU`BIg1}F`n5KEY#wa=tHPEif%=~1%gCv!DEsb%eDt8MzCIMIF7>j z;%}^oMHH~<`J9|8 zDFvjKANIw5+s|Rr zWk|l>@_E>J8BEG(Rs9JPWbBSMTOhhVzJ~c8{r&EYJ$?dMBg_JMM8egb-30sG6&b%D z5(173XbKGms1H==5jZnW0elVmVMPYy(nb;2Q?Bj6)Mq*{AjsD0pQm?)@d*;~2~?Jr ziAgAN>WifG1eD|m2?&tV;4(OmrSNYz0nr3YL{FX4Z1?!KqYLU$W|*D+RNxosIz<^IPFCrp8hr2T%X*+A=}4j`oD@z2VSeC6S{9H&J>BC#Z+FAirL(PI z;^TIxA=Gk57~2X-xh9?}6i{`L!{y7KygPR!-N}?m^-Uw=i0BA6Ft&wC_tfzeXn;{3 zz0v6wqd#NX%GPr#A*6O)hqt!-T-r>=goyamy*h1eLRLm9@wH!69LY~^XFWalc>l51 zw*XVy3a&O);OV_yfeu_PPof0+V(_`TCw>KFusk_ZtwSERSE=vYk2wwaDk!B z0p6kJUGu>9ySiPdx1pR6-{fiPG43MST{7zY;b=|J{f+YOKhY)XY?ne7b)&Lk0$9g6 z5|_5M2M9{AH+}rGtPaS;K6O$%BY&MePx}cVK~r(TOtNh3G4U)wY(fI}W-0_+LjW-w z8TXyHe;yAUo)I4IKrdtLK{Spn=1Qmn#vU#W?k*!;JtY12MX2A6i@-R9Itf)y(%X#o zm=4%R3hW?$syn9kYs!jF#X+}T!i81u!+JnUN{hq%-$x93Y)M3sM06dik)>WD4^12v zwP9_EX;MFeQUTt^M3$yO4tpA1mkT=j3Z7C8=zp$3L z4i=8D&5pJf(e`Y6GoDipLEG={>OD#zR5nl8;eF2inN-M=NGJ04u2(6Wr^-LZf@Wlk zOz$|eBQ^AQ0x;q({PHWMhoj;urDeh4I8}+k)GIs zkOR9B3@z#Eb$g3VFw4vIh3V*Mx_%R|aluy8{8Ax&k7dywTn}=ik2l@l0>|pX?gK!^ zJa%!BhPJ9kq0%4e8YPD`Eeo5(ee}=?hdPA@{fbHgVHc`Y)*2dgC2n+>43x$?S0z&Z z*WfVU8(8grq-zRAnH4{9nRr2BJoNq$*gQblhamScvTc-pRu1fzS;$7>R>G^LfGXN& z&fNa8`3hv)8!s~z?jf9*HIyERU7j1D>>s5}lgT;G_Jek`G6$Tt2J7_*)J)5_jf4%% z6H*aE4*I4vq_iVB6voU6$^0}vK!0tXXO0Gidw8;g?+DQ2%0F3_bd*m{K({%KaT|-sLu}?j+`-;U5yBf^Qy+V4bj{f*WX!xv`X?<*VQ$u|gu2uvBFXg~QS| z1SeHyOEk~QM2fZ$D+kC9*t#WQuu-2rb9KrL{S^3G*eJf(2c0V(loO9tn9jTI#_@|j zon9X%nyx>5p01wUk(CTznGr$Z@KQUd*Vu4Qy2mz~Qn>lg=9G~i8;YkJtfd-(LPnKQ zZ5msT(~av=nzp|)OMRtT9hD7+Br$BdhPf2OB+!kwpuof$u34}(Tj$8}d*>tDCp{u` zi7-fWcH8JFw|)N{nr>LuSI&>IfVc=rGDgdVBh6*G2G)p}57ln(CVC5OsV|An+Un1r zEnMj4nWu9I5WtgyI>?$QLd-um(MwQ4dHk?a;)wY~E%?!I#5K-!7|nK#z5GNx_>R~SK-Lgsm?2gx8h9sDQdyPrFh%jm14#5R+_kTq$MpEdGh1IUFx z)1RQQ(6imE3(L->%ZR{gP%Qgcn{*5S4;RPv&;X( zzCDP9}f?xPmm{V?U*`Un58Vc*l=b-(g7b60p*rZBtA0Am|2>BMdx6f zn4fI|?zaPt4%*p}=7c{OyXW-S5h-T-Suufy#^N5u>|<=K!yc>*eG9Jptr9oWEv%=4 zMK5bANc^FyJd3OuxOsz_*mO4IET}NJwr=J(Oe>zlX11Lw;kbRTbP(S{Ti%QWY`I8? zgs9uW+m2@WF100B2N64B4@F(nK?9U5;Eg5NgB=PN0}2w*K6IKCe-<;A!`YMrO%0ix@| zLgCXJ+6VM0+zfSvsOEk8;?Ab=pi(qv3_^LJ)Z{3-lIv4x>V;lWQTSff)NKCr$44Ly zjA&wOl#&0M@y%}&F>BgYIX?S0qnMZ#VXX4>MHxY|@Rt?td4{9eXvBuY&y6?G)hQd~ zv>A4l+251{MLo1LnMJ)?S|~S4m(JT%l8#wxe6=N8!{Fuut02S?c*6OZFE!XlAqq~W z6_O@$)5`t1gfZ{xrzI{^w4?UppSO~RpeU;^G_%S#ViU}LWDsHGgvFAo;f{lSA)Y0C z$0|ke=y8C8d+LZ9*%0aBXsfs!Ts>`-)rRrAe1+8(6l7|d^RXf^uVhlcf@Z+*k{Y~e-|Uv{Q6W|m$KbpQAm>BE2wO)yqdeJ z5^+5ld^#Q$dU@6VdK%Jh+>B4@f5MP6qlaXL9?+dXYsv;?kD*SJ>Oi6%6~~4-vP)Up zo4au24oIe+yphe-JlTeY#w=YWeuUIws33|@dU5-dF5dMQo1b@6c4P+uOAZgM?D#Zw zySFFTuW1xi9@>B4Vvdfdv`T-%dkv9emmA@K)`P|IDud_i++wgP(Q^RHBDee81Beh@ z2-Fmq|2VCHbak9C+ByWySrG|aI&HpSeFf|F01TdLYX2sFJM*P*%zI>JsncE2`WmIjOVRLsW%__o^*N%+Cg+jL#Q4%qw>)1|)h?Zw}D>?hw) zx1vSyd}=s6Q|fd=*QNS4J$q)}wDWAJC{|H72s$Uz{gH0-lpGq;idg3$CD^`-`ldhn z=9LjDX6k*asFYxG5rFS zz26simwZJZizB|Kr|}0ugo4mE!(cuLzNAw_>+Bn^?esiLSWMe~s&KT40(jL0isuS2 z>fB51-Lg~Hn;_|azH0gxx?h+#cvd@?`gHl+7 zC_QA8Ujm@_AJ77VRMqaS5iw5m{|NcKGlSkq?L&y~eV0$gC$scICs_E*5W;**kn?%X z5OS@?|2}~H6 zK)vQM_l(!~AVr|NcpCdz)^Ow&Ub9=zd(A3+pJR6(HWv#0c>TSld3dE|_bie7rEqM(09* z(s>Up-{5>p|M_HG^w#UQE#3s3_TS>0W_ntJVNzcoze~Y(0Nx;gwkJFXafYCQ@?RQpX8 z3sLXI1#W;ufG8_&2-(65J&1gQzfoiNwu}o@RZfU`W@lXxKu}_Wk7hf~#UzbH$+O60 z=cNSjE((UnHA}WmEuTQVz%D&ew4V--!L;gcTON90ndEqhg1O5K%1}tU4telaWjwMp zqrE5rVVLU26cyJOftqAL|F`+Y9kX#NRsR5k7s&r%bd0mTtCI=f-za)B`+uN8x2oR1 zG#E7Bm%4T`MmH(hv$f~~fYxSspM}g~tT2;!OD!t11WZyRhl!xyd*i{7rRGchpP{4~ zex8?H6U063>b4~MG@N~w!03nQc+&O{j_wTk-S_Xiy>G{Rw(ee;BH}3myPo*odX$cV zyA+buYGW$aSH3Q6e=LJl(;}uQ8ftZ+ND6>l$}e!677XQ^R=!{Mt1*n3gU~y1yR_y( zkU{7S|1x08#*`Q?e=<)2)45qun~+U$g{LZ$xLE@RV~lx+5EA)T|Kh>F@06kR?YC$m z@vzK{NoBI{rA?84?-(-G!*-Btlmp$8o;L?P#iNCjdP z6{d$0gJ}(3LK2Vj{N0}nto4;IrjxJOb#SY@_CVLXOl^|AU)8;3>G+b>O%NRn@C$J+ zDVZO0-6r2R4W*w=l7z{WEZ2-hIBJD~o##=w77(FC2Q@w ztAu+Tx%Be1%sd_uh%&29!SI^^XT=!m;d&_Gy2!=~%dv%ysiP}yvHw-kNZ{E^w;g3A z>+HwHFL9u@d{~x1YMD8vzy)Larogy-olhHJ3#b zCA>cjQAkt`%JkQPlIz-VCR5-VrPIW6UbCvVB6jwD6EazzWY+zU!GyGLb2|9BNAEwc zWBMik&AaP7oii3c8}CZ4a9eVlY?*R?%%3)r!H!+Ut0&qJp_jKiBVen?N=mxs`PV7P zi-EC+q0H|9{0p7yY?pPnt{bOVzHABy@Dm}fE6kbMhujojZf9~CrX6hu!lYXzXX_?= z_jhRtBuc66$f2&Kke7#@Czs;GGN*Or4b5ykI|b$0nwcD>pH5PZP*I)DeTVFJcUB?# zK#8CK2GeMQn{qqJF1qHBbyvbW+QbC$c6wr$%sZ`-zQ+qQMvwr$(C-92~S?tk&m zY-&}jjEc(0%yZ5+d=c1P8>#)9a{~Q>=xLzPErsmQj$vGOSy;VJRI~U_RYRp)p?=V- z>RPOKw2ZY^&X3siq2u@r%>W<*Hrc?D(o0J4LeZOEH+pWFf>-kv%sYPa9Q>WQ+$E!q z{Nku%CK}tjW4og9gBD8k^G!=jezl4d2@Z5a$uD==FovA__<-@ddR8@)$zD%(*~Ayn zUjZhs*Z+^;Cm`@nkFjZYZaJ8jYN8mMnRj^Z-w#~ul?r^~&v5vbtJn#7e;B^1K(d?n zP$?DXL0TSTgYG?U+DJ2dv2HD5cUSRd$!Sowe4nh!8l9F$T=Q-iN|PgH=`F%j8kiW~ zTdlP`H>iSeLogXoEt!aebjH~EOYEh1Y6w$zb5&ozNsDHCH@Ws(II`qj9f)x0| z-FM&=-#$U61O=cc@I#24WV9>ypR%pdyO=N<5#(VAOgVX!VXMWpgQ{;1l35+#Q0# z`^~AdDyB$lI><&*jCjhd;88?CY-s5m=s?FB))Xep&*I!jjIU2hDr%Y&5E*!S5al@W z1Y6v7q(2ocj38+!LiG*2K z`ozWS?XI2FCkO=WH&NBd z4As8PC2Sk}nifS@?%v?LxPNM3NG~1KD-hG#k)V|sij|{bSP4CQ8a#u4+*0!d(;PXM zxJnb)m7K8zGyO?8(>(Ll!;oEJEKikS&L1q(x>WX1d0T=zeU1BgV6*}j+tt)ax* z9lAW3C+6<+Xlt=>B6PsYdIZGU;ZPLT(&^3ji%Db3pRZ4`pOKUiRu;y}w<{cn%t2mC zi|nTXoyQC=F}#Nkg6>AV;&?+=^gRO@uQa>ER!qC4n2+Q0Ap(&d17G%OYWe`n`$P%F zL!zPIV$V%g54pyd8dq~z0l?Bgq>Qyd#FvW86`bG8@RWx(*ACEIWbiDc0zst^L7{=Q zqk$OL&D62rw>(gqY&vATUUm*}G;vm>3^i-D7CLn3&vICMJImb+h7a(%RY~|lGov+W z`+!*r`Z>hXA)856RSEJkgRYy;3;!*XmNU1whzrI|87z6ekpTg!^kH#jvm<4zdB90F`2g38 zkxj@2c`{{B!(sg!B3Qjv7lWQ!pwo$%$V&f{BDa3{`*tHpA{0c+kj{mQcmo`{RHv%8 zj9iZ{f`}GL-f&zgVYP3E^AdNWA+my~`5?rnId)pFv7*kQy^)!$#lf*51*3jt`gih* zrK&Nm3o9vGb2G0<^`#&K^;Xv(x{e@wr+}b~xrGP$`*WMafI*Adz25-TD{;lzM==k@ zve7@MlvN@gllB9()vr`}Qy=>F`dxJKy!;(=4*gX5DEsvz(3}TCvYnJW&Fj=lx&N~p z`i)f^Z5Ux_K}3J*<<%y7=G8B~F3P8p#}?X;RVhr7$ao^0d=%0KFb*`x|3e(7ym8N; zz^km}N{2=$G+qK*P@0kRN^!%moLCRV#mH-#$ubVaK)x2Khjsg!mzX3JHzGV^V+r~r zXu)to0ENIET{=mgAvIKZ`r%m&htS?V55$Zet->WSG7CEuc12o4!ip3p1F^WiIs-{c z+j4EQv>@Iy4O15`Ps?kB;hsKXwIIZ8vP00|I87vVrNE>a`_sJT){)I}sY~TfXtO92 z-PKlvdaB;4;aCxxaIqY4`GmRf@-G_>cBQyUx_GlYjK;Xcq-XWut0#Nnh(7#}X#&~? zo*s&}$f~3cdl}_g30e=SD!@Fzz>vyZ9R(@ohE>CZ{ zaUR*sLs6?Ah~XqmJ!U2`abq$s;NYzj21@)_|F9L)I+_QoaMVc#(Zbt%uIwGO)?0$K$2_nwq;0KO%Oah<}Y-iF@v|#h2{E-Q9bxi3b-h z>m*2>Q0Jx+`?bx3oqJ9{f*Ru4idYU{Vpli(Wo@hB1`osK@g+Z?zjp{A9PfSn3=IKP zE6nfO?%ReR1}E2_nYFS^Joe23UmU!9D_lgxH*jIm|KlSuG$0ZOW!4pEn(vz}^Mrp< ze=IV$qSg?B=A)^WD6)m8JLwwK9oiblnCU`Eu&=6S)^)Y4pBc0%B<<*onx(b_B@U{C zr|f3h1$u3BOA7bi>NbOjNZG}wSG+t911~#I^p@#8g!JEpm8{y4D}*FhmiB=CJd6M``U>m@ss15+o<{fYnN577D?T*$*bL7NLdTBH_m|@S?<5&S&|-q)2}}N*Op@%KO}Z1 z!U#w2p$qgxUdaTXYkUd*tTTiSuqX6(!b0W}b{MYQRpvo88EDBZM7fcMYUbe0+Y z&paeN?g7J4swyhqbqQVOs^F|R?H}G27+c1naT~0IqZXL@wmVZzouq0NZO~Wsr6SHC zHy}6DIIR;B!rFV=;TF(#?oPUMd$U!$&A)D-5%+b0_(!!Dz=*v4p033vODHlUTEl@w zwtxrrr)UhcSKzwMxv_oXUHP_mql!PVh2Xf+Wi2?cJuyIO&)~AQ|#=&ME_GATm4$63HH|1>+nDT<4 zs63jti%a4wxNVjZxrcq~*Uf)w*svXt$+SQJjUzQx)gLRHNi3E-vR;qtqQ=eZC4@kW zJ@djId?UvA54!afO^wnrC0j7ZifU<;G&cZGjCiI1viYW<6h6*>tsYlGfQ_L0382QQ z!<`cuBqC{FG;m4DK6XQh9wlpYZBR4eDZ99xOGU0fR6eH3a16Hj&OtnNIyeFq-CA1U zcv%Pb2e%g@h7^k2r~!6x=iWu22j1jsD6hMsXMWCt8H4BTvRK;ahIUdg}CI_V6Hy5hsU1*ASZxoujD zej0>U51*)x4IgJ}!6ac8`F+8X_e7>RpRtZ{n!66MF84BdpWYk}Xk(FH`NX=Sh==xr z|8(+lU3o>&Nmd}#27%u6o`-h!fVE7OabXgZp=H$7&UE*ERP+4gj-2KclP{EC7XINuNG5UFGxsk47>u}@YQKPW z_sQdV+XPnS!Kz9I?AV~yofE1TK;WtXU?G-MkR$DoF0O_2P<{xth%{915%J>Ai0L?V zb0$Ql(AnG6AVRs6(^xnZOUf=hphBpw*`(3sCHEW|6U|Ns~~`oVqL03SFwUWr_l&WKZM% z(5Q%Gc)HEey?r&Jr&)A!UTo(mlg*mM(A0W*WJGk|tRp(Gnv@3a<%c-_ha_g zy0G?+H#tbAWw4s1_(G|Dsg16>Zpbpvs6 zEiMP3kK8`g&jfnw$Jalkmv;dO^#lzGmx7tNa1S1njyZy?Ss;5mkrins^{QXomB*mD zb;^3oEQu0|UfVF1+*LN8T{H_4yL6m)Zt#kx8^HpkN=k1{7B-E1&G9-H>@p^;LPP-d z3X74kNYAcMDDQ&v+e+7FWsE$12CSG!Pse?co1tnDa3ytz?`T+aM}t{xsm+?=&<&ZR zHPAps7>kIV90==bRa=jFxhbkHw&Q6CGSYy0=SmGd=rZzu42x`ze@3@#j?!* z-m>`HS7Cz@0RUz1lnJA&q=kr;^cGw+%eKNuhnb^C)W%D88_PMk#~EZ3=tLI1l=u7%KJp-GyvCzQKRne)o%Fy-**Q@ZBpl5CVVi9`m?+x3E|Sx$Er7Vu~` z($T?cCZSzoW8%>K3p>J4WBV612b=zCy@DxiaP){PJUj_bQ1MK16v0Fd4^h=F@N@1{ z#6rE^0{9W$2oe{vJ@x&}D#5Ut^R*JVVOU%@bv^KPa+<)uoh@9dcbLui*8PW(Ix72Qpld8`-C+R;Cab!w z8X|uZW^nQ8{@=Pmsal1Ml)3NoiHA32UnP$LI1Ox?IvxI6->H(h83FR-aiQTPU=9~j z<5Lcm%h3sbjANK#5*$iPcA|l_I5Ikt_g6F8^%(pVs~R$ zgDiq>_PyKD`FYkHS1gm>?jXn1JYQ0hdx2Om8hkuBDB;Ld&j01c`8jJf|=>Ngf_A2 z%d_p1tP=IkkYb~sli>9b@F?^3e+!6hVHylDM?3bEuOX(NcOa%5HhU5Ma7@vALr>aY zzd3E&FnsCf!_GNXH{=kxx&rTRG+HdFxb0~tSRU86{-NUb<)U3zTs_7;yR6G;%a3+) zuf;Ll&m&RhE&cnyvASor*?k69J?#9N8ISe#bd(dRp`{a4P`IX#8u5C1L{&zCrS~o5 zEYmS+>HRoAfD(qew(wL^CVA$?=Hw>@j49(J0;yhCob>%4=8xvtv*Cn-Rjk+Y<2NvB zm=}Cq6YA97;_;rQeOGRV6-Eb5GMTB6PTxoAI;Se1N9#^E+>iryZp)JH;d+=8zslKo zpN7flK04n~<9}OYMz+X!b3kgVCPgf|aq+@Ba%YVYH_l4lcg#SFb~e`g3CjP_(tKE1 zfjhqeQxwdbc-SPx6X{*fw{7}ke!F1QD~ztR>JaM>f{}qG@Q`A`d=FTJLuVVeYu1cS zGb&tNWePOKj5EDZi?Sei*U^~wiVaQ@n#|soyC6J7-!u9s@g$`Jh(%SxmRvuuxSH3) z;jSI#kHkFhv26hMvTmB^t_MCgJA$c9jNTlj-q%HYf{d9vfXm*$JCXKmWn>M__Rkl| z->6wftUQ00_6kVx4;oA7;nKd6bFTcpbDL^;vTvRr=Ewe8 zyF6B=msMq06qzo2rY!$MxJ)>$A@Q|qhKlrB{VJjHX&K8q3sf$8;AwCN$j=ss)3Wt>N}&ZYJPjK0-M#`KAbj{2P9s4-jl{aM_d60uuFX$FU$4K6h-A6tN9=v2M_x+;~q`px26Z3%& z9+hA8jK7PgHWGH+jfXklM!ar#GQlYS9=FMP+^_nHm!1qdu$@Tq2aVY9rL2IYeB5{X zi8<~J<12Vm_?f_uGYOEWyPQmUxT?T}1%JOWx2Bu0{Ow(u@0khrS(;<{(l;yb+v2kX znUi;_$MyeZ3JgPB5i?1J3QFd7ewBmVQ(fJ zh(*DESiV_?xFJvLXf}gEwEr&nye~PLORcttxNowQ!$-yTe!!f9`-dt5MDm^Sj4jLrISFt-DKJCo2LT~U zJ|kqobg|-D&?;;j0 z^tW}>JyfF1L%JV&iOXO-R-6v&<2g=+1sUPFs^YQ70G=$D$mQuOHH>IpiCen%()7tW zjf4-yyV^SMCX(dp-)&Wuf^Ss{-SZSwYc>ZRhj3VDOvk62AW`f^46S?v#|1|oI%Ii% zIi{7tartKDC;i?z<@6=BTG2fMrBx&{_WL@10=wwXf?98z2Ragz7J+A^>Rg2L69XwT zL=~#{-)n(DlNj@T2>KF7FmK!JtuJM+u(!UiEhhZTf*{6O$F`!s5za;Qh0SvLrVr(_ z5zc;yv~T?}_KtxPGoX%m(eNDbqe1-ukH7%53Hr@+s9NTm&F~(GKikqF2kuHd5->NSGR%2vhv8?=_{1LiO#jvOj zw2SHS^`AKxW1sV%0R(RO2NyBoq?+N-2>sGg{&4-}5laz1YT zf?1B8#A*w^eV2nA_T;MNg@_*bjD4jpLN}WCbXxRJbC4{w@lrzM8g8r%ve@z`!{=OD z_llf6Zzn(d@+xL3KhZW!Np!h^=t z%P~?y|9;yhoD7=6h00&4y#H>?+)fF!^gebPHVNU?$1`xTlqEe_AKlN1EgI=x;dsp)AL5 zq8>{#mlSnMdj&OH8L=)^T$(G7NHUGIaUL?y{EWa@n77kmybi6eSK96dH!JLzZldml z{c9M9Y+dm;0f#%*=6CxM6kgWT>H2$n;}A&w`m+qW_=Kdaf5R;HPhv2g11>aKAQemc$6liV{0|yh?^KiPxSZ+zgzJ%#H2-1Bk%-zih>qZ9PE__=MJ$lhB^FX@Cslj;vv4 zru5<*y1;F6-&SVKu`?S`W$nm4 zlx>a?=7m|U9!P3B8cbsFwpaZ}qVrNeIXR0&1=T^blAr+|b2ln8k|sNjRVAZ1+jmy4 z+%`0%^f|W8tKTrpw3}e{Txcf`M6At|h9BrB6VcL@SxwiepP&EudW$D;iyxt5*s(`n znP=G|!D07JH;l7#GD?dtUY^4k_?5hRZ`w%qPq=gvs>)Fy3aIx>G=CtkN!ybPm}w!r$`$Jbvi;e?G_2lP5|3Q6L;c*XFKTnS>13b>>R> zm0!-?>olgbO=^J(j{gV0kyMMuX+-i!!$l@I!f-FxRr6W{Lavrshwe!HT@GbNpp5jh zWhB&~hPd9al-9}()PFHg=n`j zm)+xEWfCKk1d>!8@T35)5awJ^1j5orO+$f$x6x{;ai}fcR*mum79ufRY}Rb128;_9 z&mpO{s2pK{_N_H7Ut{b%{imyJ8^c27xQ%QN^5iAv0VjMYBxu@mi=M+_Jy@1z1V@fQ z=H+h*OoYdrluW!!Y?VaPv<^gVAB<^I)#)VhP@G@!8P&{2i@;^i^dkoB4XpI8&Ta}P zoeyFub#{7mp)2TxD>K<{krRPN9!{*TtQ7dR<$@QYUKl%zrrD8ljx7+AEGcBO+OOXr ze=%y+0sIxX2LZkw1EGMq%jHZGexBhhCvfbk{!Y-lx7XA@ASX^~4dQFgxd-q19j-!$bFxRKTS;|uu&lUcboQ67NK#JHYuetpP zD%QYR-EvH`tsE7b<_($_q}m)Z>=~;LZBPaXF9UrDqq*_iUFC0iU;RESMX>Kh%&Rew zeK$R?yHz21>LxElTnxZb_p>@$1d6F)NJX=&Az)lQb%cY5L|s#)0&m-)!qs#aNWST4 zLIGURfl#!ce*iy%G@E&CYVhEGPsmBZ_!7sc%uZasBQC4S^ahAdktS;<=OySke{-ZO zL=qmnsMazA2`YF5nl~&qVjJMyLA(mbE7mcAMt#Qn7B6ttVdA4()4l{%X2ip9xeqPz zd1$KDN}7&mST)PLsP(-&3T}cM!IKNH_81FA1k*%|DrZuCd>?X{1W~l<3;Me1G_4g9 z@2cMIA3jliyuJlwd{4P0B=T~J#xjJFg^t1ubYQTDev1V!sc|}=X^!GSBO#n+XUax^ zHUpd0Wbms6+I4({h1)_`30kP@A4pK>ROY={%t*~x&@kXJg&C!;{GhH0XFi~Ist@3N zqB@;10Eo`o{(di4w}^2#^>ZqBxqQSV0fnT>F$I3b^HT`_eeG!O-tEvJrXr+Zky{R9 zU$L;k0)e`=5qk_jaS=?UwdYtY_%CGlNSOq(b+kA+Rh&E=-R9eKxa&Rb&(1u7CpO~f z@m}r#L2wS?@yON%`P8}#AxA%N`Otsf=BOsD?5;??8L@sw@uQO4B3uV8sOW&F_hR|! z!c&A%jJ?udZy{YgqAzIRly>C`Q!(#vyakmMVqdncdKB^B$2q00=Y)1H7$+D8rISuMNSw0;EiC0^bkO;auxBmDkWbTolX_vyG z0p|grG>Z62Au#m?l!Kl5O_UCiAmMjv@;}Xoh=OPhTPUGMF3y$#<1Q#4>~Gs-*EaXJ z%)GV08}bpX(g8wnxK&d+Xd7&>IFp+i)n@H*m!xSbCChVkfnbwl&Oh1z&aewyMY)Gy zphTVpzasqhml;;eF(QJMd-fu*GJto=>w@Y9rj>)VTy>NJ&SgB&>s0ZL9znHWbIj-G z3V&)ma)XteA(Q@F!9ep$G6E|Y_!5W3H9x}gruvLyN5XxHF!9ZE%PpzTiN)tUnxwZK z_?ulI2&}4r{oP!cC$OG`K0kt`M9j%KISPm7Z>o2WlwO!1YEvy~pO;s2)tYiF=<|GP zq5X9BG;(UD%`RiZ6E^G$xentI9!0LB9doIWWiQ#}J8yK)uQ;rTs-fc{n!gv%t-=`~ zA0x0zt@ln!P$NpRaOeB3N#@bz(NKw7OhAaor%ocn+e(zd?@GZd8|7sVxu`r0k>XU< z5^7Ot=8GS*bSa-IkUQuaqFS0>)n^HlFL>FLKVEL9k3hV=C5*Mu5w^uS7G@_=4}TQl zdbPKiOYIW;zEg~eU@>ycVl+)#XV@ONiAz8Q8nPhO3-cFHgP-xn>NBha_h{8s?}p7O zN4r7E!@uIglYG)g;HSf1IvQPnlB9=0S~5W`_t|i5BB0s^0j}VJ04UN5hTU)iS~B;J znA&iX9j39ZJJ*RLKW^w{#A^1wxO+{N_=7hJ*}kE|4Fm|yBOGz9$FN-(c%_eF2?#;8 z)s^T&;c*h66;|uk0im^UI|=f<~3yD4nLo`@?U zh2#%^XZbOb-ftQCqo~%RR2<4!lq?7uqthNXS}j z2OMg|F7At|j54T^T2&fM5q`tGT(6Q8Jyn%iS1+jw?u+eYGzO}jMx zR5ms6XSgEa%8@Jgs{q38^k`-bfiVtzaGz@C?e5hcTJW&Iz$GbM7Sv;V+j~w~$ECn2U zwqk&vc(3lvRzd~%kHvF~LET1@T}R)q=>&Z3g^s~3blj?cD_7HCl&zF-e4Hi^{O2JV z#Vj03SJFbU$BUXWGZ5vBsJ*Or+T0Q&c~-E|^$qq)@38l31ITEEbo| zcr)`EkMyF3b`yVI`K<~kCq+slhidF|OWyVJR0gmb-1`}9AflV&_Rb5Kx=)t=Pi$$I zaS%Pi`^TM)6mLw$MB+Oz&2m`xB7t3tK2@O`toF|CHf|D6cI{nj%?)lk_Eo*(g_Ba~ zuDGgm^xqDU3q*U>+(-tA(LrdWAqtGPr)9o;f@qMJSkTcZ?CaS21>9ih!@sPR>0e}L zSZ~Owb{gkcr1dqY>`utaaz_;^XHg7LMt#F3ZkksP7SmG0q2_EzQE@WVYPz^WAPuC{ z+^@V`VgJH7bZ@+4*fb8W6UfZb=(2ADW#)Q&0eYCa#Pv~F{8P2#4mASpFi~}Yz4|9b z`}zPDI5a~rrqs=)-3x~gn~psPv-j?Eqm@Es<}aTk_@tB0_Bo&F*YhX#oKf4|HQj22 zZ4@sqAx>~ywq#+knh#Ak(A!C^c!IU$U_)QCQ8PrC`JC#QG)QswRaVj-DY@j@dI$Q<+Ju3 z5MYM2Ni5Rl=NDJcXU%$Z@{n8>uWN1|4Gd$oM2DCgnaekc8H0qpzT`3o>9T^@CxPvc zhM^xmsE-Jh89=K2hNcf7B^p#*(j0aO0Ah8qqozsvTQ*rF@Ajx9ou>f#z2#e70@x?} z06AYfw!OfCgnxNC;2Fie%Y#h~d7kMpvW?=fjbn$KI4vFb!Q%4AZr9k{;8)Kf-j)2rpOR6^&t{fCa3U1FdkKI#8wBDly4Z9M=S0KgnG z007qiI;`0_TKz|qxs7dOx7G6A=?7Y+AJjCFxoy*De+KJv?aUBQCT8f2%_E#Dzl4jP zQyiH%t3LbdGy51{DABzpw*jkhE9_04F#E}!p?=j={nD&qhjmg5^s+p!?gzELi9a(t zBfoau`HFtKlc$~A>xGTmWr$BV^W*y_R_#vszV9&+C1xDB3jJ~&);P$IlY1gb{oWt2 zre?BTCriZM-gfQ#Bb_Sx>a%$gQ-0w%T!J$EK~ObGE1g89c^r|f&D1@^qeNXaV*So#anTU&-v5t*t3z4^dmGL*jE`=T zF-vpeqO%fu-=#hJHA%F4srNN{>?TR) zVJ|?P0qgG2w|D4)_uXAS$(>N>8j$IkMLbdQT#S0LX5el33`X(AKu+__?)}FcZq${ zidvqu4KTwKQ}PypkGBh=pVwf}njOGdhS<+Hn|>sKE#ZZWq!SK{C)q-}yAGX89ca>^ zYNx4L)oTgbw+Yqo8c&+IgJ7;NqSkJ%0hx?BM@O7^Ez%f}R;R)E0DAo`zTRNdK)wNE zyM@4%P|w>%%OT&~d#pVO6l(>iHhtSB?O^;Qs;5gUFr$3*rCVG*g;Qf;r-ooZ4Yahi zrk~jW$cuLUg0>dCO{dYFX9dwMBxdgkF`*>E-b446qb0j3g%2x`8_G|6S+iW2tcEbvIOkU|ipfhr26_3vdv4Utj0Lo0nLIyAEX zUoi)=4u#?$RCL`StsYwF8}s0yJi?y*fb9eR&OwkC5;>n_{ zctBmyJkqLCP_EaF{C<``Ta?ebm-Kg{1H?;#;`%}(4;+yz-rssu)^zMK;TF^od<2NQ zum=E37AV;y>%`Ea452l1jNM2u_f?2>I0#Wz0^J~M4`agi?MC=4NS9>xQ?{_ z-%n)D&M*`gc_4MSQfevxwAqC%`N`s$L5JZ2oB&W!c^boj?uiIzP|Ri<>xHp$b)qap zJlm(XM$Pn&IX0c626FrGEhgGJT zMvl-XG;l}+C2BOEM`ll`&Q2L)akbE2s#YVk$BOZom{4jgVgO$)I*l?%TNDOoEBI_C z&~lIuR!9P8ycmPtPvJVZ1p~L4YDnlKIU}whtEG@u(asu#kU|L&+?gBf$C=0R433ke z^)i}4`8fU>9diS@Mo|V5!=O|*pxT9=ww#pNmAK`|#b;l{u4Cf%Xz!JU*pB+f%b>Ed zQ70wnzc7Jo0Cfr$udVmmFQU+>E(#Fn7IOWNvZHm|`!`~-`F&|Z0dK<9rP}>XPSBlx_X1s!ki8J$vJHNNzqqqU3EZR za;S*hhnhg3VSkzH9GCC&FxNU{6a+u~Mt0hwTRg6&6wC+|i@$oLp<6O6Q30i;f+n<= z{-(PQ$6Ko5*N_-U4DAG2pLkAKf}z&ICDD@tGO-)Ktwq!;v$U1`J{t!VT&G!@(@0y{ zcLnVKsld>SzV34#kxQ`T&=t76Dy^xL zyy9Jmvp}rHe|Qz6^^|1|=UysK!Lpr|;l~3Y$v30-;;o_jdgGa?N%?4;ie|-fvg`N0 z1mn7+VF%!k0r^Dr*>q9|bEU6S5cIm^MBK0`H-J6zN4{v9pu2mUIIK zBY9Oa*kiStY2v-XREi#{t35J0a)Nmk^(ELh4l4mE`Q0o-_GTrF?$QX=sY5vjl?7#( z1c&0rku?#7=IW2_^;%IYV+Z;aPxpP!(^>8miIl+x@XJW1IFI;;8t6Kt465)8#*&JD zsRoP|%a4dch{xn2A9L5CwhgNdF0or#nCHyb#`i8lhv z?w?p&##4FEMEmlrLe*#JzcBtXt|GdApvuu-ncaRr{H#xrO{Rb9w1jYP{uRH6ENe7u zKAtX>i7{1Tm9%%#p1raUg@?R~+oxx?AzzrQ*s}Fl>18Pa&QrRNRA7Q8%@lD=gUEDz zVgv=+8dBmd`*%#*3xZ(1JxJm@P1Ja3($Y6O^EL_%iH7B%7(UClu}sJqa10j${!}?0 zXd+L9mNW0K@w2AR0$if^z`~h5W(cMDh&^G%<5$)HB-Wp-?|s@!b=on&Jab$@C)3c9 z8~$6t!!U+QF9q5*X58#-*(G9T5riNn_jANzRta!>5aHw*c)xE>wZ$71^lge_4TE#3 zGKrz$C$_t;+&TnRMCA0smz5NH1+&TX>28C7Y8K%)ELSO&!yFBlrLr^a0TxM#?PB?t ze9faB%{3-*Vhl)<9JedpM)q50;S8-Bf0>lZXPyuRa7qLgRk?XxI@qdz0mGXo7`i%1 z&gqSlAY@K*S>$j&ms3!2uMT>N)O)CB@W?}+|7`8Ulp+9I1**xM4i0*Y5yc}u*9E&O z9iRikq3~YdW;eY0Tx{MX$1xGG_(n&m>~LX*m+ZQ-k(%J*Q zDFZLqw_-6npr3S10v8sW6u!|Te&`V&*x>qLotu55*cv=rMYa1)-DL@E*fy_$UI}-- z&uM6BPWsJ+XD!cog}O?NiG;eAn&dpbTr@VD{kgAOcj{l!nR;kEH5LNcX{Y=dT(Ee? zHfI2$MDWdr;&@`%*i$pYO&bulPPuK2b;GaXrq3%F>(u%EK3sl!*&(@*ca%_zg-}{N z=iwrNr(>Ee@&gz_ha#Ih%=EI)yM1d^`PX0WnZvCij43@IcNor^1L`jCbzD90Zcv{r ziJcfVa4$jf?Pe~a0woQJ8d{ZYh_uHiGtL~x6OhC^ULp#2H*#=()#+7tb;KeTLnQ}C zj@kF#=S^~CDpQHH6!o?HYmQRO%ZZF}_KGnk+Y5(vRb`|F)0q_w^gq%HnQ(Uo$*X3% z2~N7KuKDdYMa7I|Vfkr+J}#0$^?0x=`!)s|CG@c3j!8J58LtdOpBQ`^40zq)iy_#% zR#{hfv$_ZZ$nBvYEAV1Q#C35H^oc{QpP|yb`mmUM`JVRY+Xprp=5?!}^3UCBFIuk7 zzd1iof&%cwGLs=`QTGQh+X`*EF!lT#hA#@RJB8{In@FBM!t1n|R*bxpBOT3+HKpwH z>I0ZH?Ahr9fO{!dJKO{O)ywBgZS9;P&gmg#(dIgD=4v{XS+`}?=GzVIF#I;%UU@swKDAptM*R!{5(Xs3PwH>@CCG{qfC+R(!VT1 ziEze+^>=+`M6Vyc2(#M0%73i_J;}l8o}#}5L_jd?z(00oyza%wEn=gV$z8=du{>AX z{Smq#KtQ4AKK%a6)mLukLb&&DUM&0v^uqt&=0*codcA)xTMK7By?-u5ANgJBK>--w z-d-YWdZ_F82>L+yW*^EiF1C!oF^86O%r<;dH`K4&?uwkC3c~oK&qVrh94Y=T=JQnu zvujQo`c!qyYF8IkbOp97mqyB)OVefqr?tOTqhGOqoB!QKoXWPiAl8ukOPjJhGCnogJf$z2w(GOS0)XQ z4%J>N4>rF2S8;s1U+Q`P6=vX{5gG6QQ5*wTJ!1oBgMYQDiD$GLWIzaU`;H8vfm|ys zcD`3x)rBF1lY|ICh>V|Y5l9U#ZaUgkS4-TOqYJR)Ja7Bl@yZnZ`!syl;>2JEt-(jD zN2Gy2Q8-0V+mdCObvNJhcx_R}Z~;epFTr>sGEpPLfU z&y>=bpv@&8>+Oh@NZNuSZC{+W5ECb z_;d#VApHLn-o(hx_&-aHW8O{2&5?gj(@4l~`!Nh5WAf9GX6G83O-fbHmdOVPSLrH4 z0!NAi2#`8}^2z&;-Bzy~b%f*`jw};WG{~u^rzacF4(uTDqvFcjtS`&$gnL7wTs+=y zFQFZ89v=r!8M&Pvzt@-V`>@4UyM=%6U2RAGa(L&^o3AenE|SuAdZ#R!O6BHtb9~+3 z6_2&h@^Xx^yDU4C)E>2NR#QGcRPt)DE{9fUTF)P=t|K{ornL3hxxKPBrM%{znyhT) z^~jE0b`{C0ksR5?#N2OUe&5p4qO%bGQtT@URbp?TsJ5tZByY<4$Ty@WH+L!2(t4nvnbgor(Im>)-{Thl5?G#en;DBj5r! zQ$#RNd^deI@QxnP zw&AuVAg^FTelz$!i6W}q6!{YH0_WLSk^w_d4nefzSlK<5Rnwh!b>-ue>Jw1cWXC%` zwq(iBKEq^h+GHz951!TqzL*jros%eHb)sLJUS|QFdW@Rm1x{Wk7y@QAlGaQFcm^RX zoe<}`ZP6c^#gqlHprhN$)v7B|KDWq;Q=5tsD6tSMm^9(;06;6px3$|l&vwaZeeXR} z;j}a~*C0dIrlzM0UPs4+fI_iTJhE|{`C?Y+Z>Y(P3cj-f+@j~XKVx-`~Kax zWD-G@n-*obDRg=}My(NM(V~GI3i|iGO#%Rqlr*|yEjNlmr!;`lp6mzM0bq+27UKu& zu-=$n>z&isK8?P=W{Jl1HQ8MbsK`TR*Om2*?MFZ7ea)Lf<=doHZY=Y_7dV=lML;s1 z8k|V0UzbE&Ji92^pJj8W(Stji`{Y14m>!OLaH;6Ipkjk?rsZ1)26wp|cCsl^J8A+- zB0Rs6vK2g)tZW8}@m`H%+7N^TuKa8W&?3u42oDfZ%b;5Jn81cqd+YNGjHf?wuVtv| z+_SbwOhB0q*Vf9TpI1mXkYq}gK8w}X6U1oM7Jh3%)xQi~4R zvHeW+B;`xMg#)c%G$$1%0CplO{NEK5;N9u@S&+P3x^G=9pTXJ&_H83z>jS9yp&$up@T3kaH-r2NVqhw4IrOv1 z@B?h-ueajtghzepuQx0+Uj(Q#fcJhRhPMj$+^d`4BZF@rq6zrTK{RXOqgl4KOGAFN z-piL`dS980rO*$8HMJ|g!L0l`jy>@fiF;}O%V1aq4ktHKyAT`)=*Y*zmner|Y1MyN zi9U0!y7UYLo@nSkwsKl~Mfw{96YY{B-Kc4?jWL8yX)j1~*{|~ReIoWQc zD}s)Cyk%FYXHp}wO?h%X=F%ZOmMc`$(V^62O)nSJT-iK!3IETOSH+EB${qt?lcheREvzgN}vpCB6-Z?x1JOZ)&jO2D;KR@ zJyBY|TQ})fQOIt`@dL%_=ff19xWRZ3yst74Fvbb`-1f05BUbk=0S$&_roOq1e3B4~ z|AkZYp7bF6rqgt1clW4OA!sNSJ$LeVyQP-24GD-_Bn0GjT@2S|FfHuxrUgEd_4{H> z40ZR%Z}aovTd%;v(~g~~%3pX%fH`cp0`(r6lMS!O4Kgl@JW8(>gCcTTvECTyc-`T> zbA)M~D9f>H4V=FCVQridv|KJhhgvaw#w+7!@p6~&p7^==Q8yDd1Bn8;;s)4dSvcG! z#1Ck`7+fqme9FQGq1d|`HCG06c^c-@>ivJG(BcR}-uc$%(hAwIS=G|y{MHTdG;C|C zp~PC(PT$Th7TD?Ic3vK#7T9cQFvSe^*TQA{CsGI>EL4rK%qBzdeBwE!{Np$`t25Fj zDlO{Y?~IX91M;{)8W8Z%x{(})R0<&ILIp%UV*(@q;i#t9=-mWu4u!Fd_{f@HL!N;+E(bUCEJ;paOrPHvuepdj;)E&40ddHOqniLZJEj4r6TOkD?I!&E(4^`O z*zgh>=(Myany?1^^+(kKP{5V0^x%rAL#3d9tOG z{U|fmq6&+Ou>>+b6Yd;pC5RRq7wBk28|2QTPCkXp=u^)Vabp_g_m2W5rPJo@D|`H= zq|-7}?vb?j8c}bor-^4r{r7dlP1`3iwfZhe1iE~fbvjFv&X7pqxD|Eu=v9VH1|?MP zqJR}u0>&nigTXrO$|V;}kT53Ys4$7sB}gSCN-91^t{dl9MsF~Lkcq@8dW@Vgvvwko z+|_)=hYwIWJ%GpwJp8g@;AU=LOzfr8*_k=Ote!`AZ*{quIV#iB`Vg}*`Q2H544R~4 zw7>}^-2wbdM}|LMxgt{Fh3HoaIJk|-qnVwZutFzmXA2Q9^`JKe3Me)WF_E=ct|Z_i zc!zX6y8mKe0;a(wwYFG4`qazq)VdiD=XR!h@Zx+Gy6Iyk5Muu-T?l&lYJGIz@FpJE za(?!SbSaUJRey}46*9?&Qscw$JHDpf;Vbgsu8FFg%&29Cl0SA)qNRYiQ@j$K>;f8J zatLTX%!x+7sH8DqPx^JXPR;;wUxr*{)y9xlmZG1|i5fq^*JC{z1ctLg)B!G5babS> z4L1>{4AEfcjD=&s&~#CGd|E%HFW7)hGb8*BhhjzZmmI1=A~41v0kqD_;Kz&-R`?Wc zAB>@W4+DhFLBjk|q8u!?g64HW9QaA{H6G2-cQdjoMEWnXHtbqU9$IMhGG=OUDhskc z1bEPdKoB*MVn;h|LbSAb9^okly0YG1oy@ojGDj_=QJ}5iWBSf_r3U+bIfF6Xd9(_) z6UQXTN+kd|*z-qMG*dgkq*O!}`0JMdmvcr=$?xtsJTb6K$a2iZ3+R|K&Jq7RMFmbr z?AXz@7!*%&zSj19c8?}htscQSHZ_4)eQz^X?#NoT%3H2K`j>Z0R5M`+qNwW3`DnmB zkEsgQ8xU6Xzt{*g=ix`<`H)iyc{+gcU;B2$Vm5m)nQ-EO0;u!`&Z%8b(lAB}48N(L zxYk#UOw@apiUI;W+F-+fXZaI1a|^4-xohx!sM!eW-h)}gROUuCL6wVj7>;g=`N>Zv z$Ni-qpM>QIgbIx(IiA>2)OCuB17lI4joAeV+y^IIZ=X2oA6A9PbaDt=)DXsyjk?c} z5g!KYVi$N(?Y^PR;13o5)s^y2m8(v@&F>o2NCpiuEoi`+P?UKT*lGoHQqOI!0 z%=UhQq)UAlyyH^223IqMHjR^hC}#uDo}PL*6xay4 zP~72JI%*fZbKGwp)Y|(*XGw!1RE2;cdk57ucj=~7e|CyVHFSHr6_G2@g}uXiq|O2d zEbO;HO{s$zjCvdc8KL1{xup^b^F)k<-Vek{~5e3Chp>asA1*=&O(@Ie)ARoueYg4 zC^Uwr3%a;qK?D5?sQSeBfe9%+JU!1~t!{UK(lXy_ZDv7fase5L# zN)C0M+q-n_C6;LJ4AQ_cP46Yw9Ec+-pl4KEfju^V zmc$`HUtHa!BYYXP4Y4F{_w$-obhPNNMjBS5l$Xy)-;1K3hjNiJ zO@{!=8F^JSv3nbi$|DuECLN~ie7pt(<>1R*5A96$m+@Tab7|-XIvEJi&RlC-8~*y% zMPY0>5sVaJWTj-P^&CY_7Z5IlpqP$xk4TWZ6c9EGzWpfB1y?)tZ3(i%3$xB8G^{MK z+&dC!fcDcpEuN-9eedI-21L~)+ax)XaEZwL2CRsQ*{`u0p9wEJB+86n0i`+ra9H$( z9*lf`kB+bP3x5>LDlCl2-Gr|@^+-iu^lg6%(q!aH)jE;udw9wq)gHNbVCgd117q8B z89;`|$<>9?o_StiNqd9?yv2d?Bf$n0(ZQ9I4$8+ChYf+0^O1`>U-s?X01FKf%sdDC z?2-9I{%R{u!+w6q(3!Gze7-2fW+yD0>>`E9sd9LM-##{(bwVd3Fk0YvThK&Z3!J$S zXRDda41iu5A~(OU4V&huUr@E~E}QzTs9>-E+0TPU4Q9YPt14@sv~@3>CGD|y5*`q% zV?)VJBKbJ-(4+s9Mm4Y?LG20MZE6NW;|1L*LYVYuXDk#G1LBVwkBV(wGL0*x5q0Z8 zua4WF0sde>Mw)F^sa8IUwEkXwuDXzZl@|Eu^K?D=j0ya%Wg#~?q@>%BM#TC76Gtu8 z|DwEUYN~GCgX$)r{%HP7_)*#L%AIemt!#||Y1~n=j7Xn>k4AmB1#W2y>Vd7sg82lm z#uUmLD2Qe*goEk62%fZ-_qt}VLbp5wbIc6g$T+Lfx5N-4qDwbp@uD)N$aHY_P_(AE z;ASj?uG)laoWq5X`rz=Ad-H5>DZ|irpq%xpEyIk9$GF;o_S;lxNjxf;<=42n{@kuY zULIUorT#BH*H%q@*T)?Voevx5S=&Qiw_S=PT<&i-7xmIHfDO$krO)K3X_*GN8GBjD>B`*PzVmp63r@h(FLYAEtTtuweK^jX`k}_YBO<*nywV{RQ}I z3I|2y`((tKkxQj5mvtuU-meZRUi_H)a*|du{jQYUjDJ1HSw}I?tc?E>!PWKDs~K+L z)b@sUA!ft7+5+?)C26D{zN0j@Y+33`p4&LSJ#MZs6iI&Cc)ZfDFUEq2T*LRpv5P2< z9U9OKGR%i9#|BbW6{-AT;lFbgKU&h=3+Q_ zrwkSNoVLv{P-*%VmmRsdsZN>>dGmDofiQ%geP(KSrq22Dr(wGtKu@XicRF-?i{cX^ zrw%63LaN^bNvOkYQS!T?ZBh-Pe z<@mrg=4k0148Zhdq0Rx(wU8H%2m{GIZ~YcOI%-eyGnu*b zT(l9@T_sl=3V<8}M}pfZ1%%FeM(5HVP2Ru`QX_cbZ6Ly0cuvv3pvYZ_ynaycd}O$y z-ZkPSZUT&{xX^=ikT0DET7;h)T_tnYuOZWu-IAz`6~H;N$cwiywmm%u9JF)IWD>V; z$Zvzzt9@sLACn+YezUyyht7Fki}Y=Kv;j^vYNmel9-DjN@JVCf9Q%HVc7LrRrQ)+vc^Nsx;U`1f7uEMh0Is!o+tUb4Msw(z0MdBB?(pg{SUCVi}2tgNBR(t7P~)zf8i&r*ls(qIqho zl7Idy#*^@8ZolOnHmMMqmVy@9phP!dgpx@2qBK$>A-f^G8HdX!(r*~rGDk+iyfioK z;uRLp=gaiOJ&j+>Ps7YI3lJLFSrOKM$atB_teY~_-ERw>mYn>j+y|KzJ2xZYC#YP= z6u*IPaH2@)44(oe!bQuE_2O4eS0UmY_nW?lO|oHXr^p(pzfLWs40Id0DT8Mb#&0}_ z-Jd74J5%Hijqfg!(V53iAf$sx!^upnn`qy#iGE|&4#}r+h>_INoAu4T5|lS!aJg;+ zvs$YcH}&?hDu@2H{4;4!+iU}#QE4N|;9gx}9#V9q@)zfh$A??&lKdFE|Ja<9cZ_J| z(Kiy&GwMFH|rXfE*!TaD++%bV=1EJ?);eOmxi z{itQYyc}~M3G*Vitf-3g@;3!&X4i4aHCjAKGSGJN!k^p+{q@GffymNidC5~@N&ST{ z!tO=97h$vKexS_(d)8!F<#;O9VJUThU#T3&%beTC*k#gt0BC_G0e&h+6`jP1x*8t* zuIaI1*?Z7W`aG8s6DCof?0Ch*x7mBKU2X>Ak6RH&Pa&`TTAp*wSKo^`-Y zEXtDf`~fLE_1bH?Vs=}0ADeD5U$EObiE_`Cw=kFAp#KUv5LPc~8vfusoS^?(tmhvj z><8X`HN`<$|kuA9e0 zCy{mSemfpvA48R{;*R&N0)jb2u(h-8(Hq$J@a50#b$)xT_0Fn1o)AsY>0221oH%Vs zHRp*vhLV{r7Q-6YTKGKOFxQM4HcCw~Zb5*P%~YG!GNbGr#HluLvF9jjy465CWtjG_ z3En#01!n9FS4DRkafKPX)hU2xkQ-qplCrS|$9S2%B`)DQ=m|u{jg79uy9jI+Tg;oy z-$N@B%Z(qYG<>-$i(dxGhgEzwA(VB$rc_dQ>|JNN%Ufh!SyJH$o|7A$8H;RlkooNz z=OLAR@|IXAC=eai1&7LmLpCFtx>+IVMS)Fh1mD~2&kK1HFA#4N9r-l;h(LuXU_sBu z$|OzcW6iA`EOZx543=M0I)!YEes=IWy7#qoAW%}8bh@aFsWg{ooy~I zan*tsN(~}wZ%2(pR0>3pMO6xto!A4BHZwTBu2mjWurG|f)nPt5wfv`7U$vscEo(Uq z;Si>PJ6_{Yt`x+f!bqJb$!8eRjm->xjZy12!?xc?dio?irU8QNYkL2>Iu*eaZ2wuA zm5Bd^7V`(Db#b<^cKTVHOKLWD8*B())p`z2e3RG4Qx~)JU@SCJOauZ6C}n9j3@||n z6q%VKmLw_D?@!#_>dD5O=4vnlfh4J89S2)Z^xn>ZY6X11nm4R|n=-q1xqp&{c8qLX zY)G}X`?5U9`=jK8fn)03^2Ql!8V2}MaY z8E#IMkOV9PjTIt`{pjg5;0Y}5ne!13VQix+my6osd=vw zp*};*pTDn{>Cd3+#4o!CF{Fbp)jhPCB_yav4Q@#1)irIP17DeWeysAx=kz0Gnmvhd zB*oW+k1#S}(jb7EY1aNr-P2#(9Ulv7-L_T9DVjJwskDa&*Ox;2)57ODXGsq9h05`MENh=p9 z^w@v<5ZdfdA-NWyqOPvPL=B$5z~~dS7dt>lO!xn(h~(%DRl0Y5|)7E`wf8zVTLZ; zTte3HXMxH_w5Xc16nPjLrH<8Xa7Ge`__Wj~KDW>jMr#pXH0yW5JbL=Hrj9C1*T6K> z8`SYgek(HHs%jkq?2LFPZk{L7dGFGf@6J-d^+?A#l+orAwEA9Gd_q0XpzbV6Xq@aV zJ?rHd^OI;8DL9lB4$!t_FZd=|4|h70Rih=99X9*bsZd0;{!3C|YM9a8Uo>{&0`?*8 zJIj2DHBl&9xv;To;eBBuNd+(3Cm3&sl1i~sj3P=JpnCVYWfr4TBe7&!myj5B#hqKJ zR>7aTSBST$%VGj2$AQNpNH5>aLTgA0E~=&fXcmdu!Zqe*RZb`Wf&I$$rv(W#N!|K+ zw952%USr`S6xvv(>SF6c8V&&~zDlwC3#}fP-5n))bOBzIT6>?#)_P|9AZSzrMyS;7 zQtOn403sx)Gs-R7zKi01mugb9Y#8Fu0K>k$8fRwas1qo2N0>#UF^QwS&0xR@-QaGuK_XbZF|#RqU4c*=m7OUutBx-P?Y`beot%KmBaG zgcJ)|Kr?Pf#gR=7U!-@{bV5x$ZKk}mjmeMBrq4?1hArSkquO-Lrd}wJc6N$H;_F7C z+E|#KkHf4G+t6AMX1Te8Ez_ts7vtE!&`&Jh%;Pbg;9X>E*j&#TjPX37bwYMCb7uEA z=TF-l4Tr89#An@W%{YT9TG-iR-T;TwT!cz%F^ZY+ zu2Z{zJ#v2+oB8(sa1Shh2p|;y1qZ>@z{%Ok#Kh`9Qe<3Wq@ep55Q1;sP^G*S%tcY? z1Hsl zR9l`wt>`oY3Jy~uXf(yJmxoE}iN|fl?~7!`-=9A2s=7TYQ3*AM?|CNl!MK5@G{{Ui z=|B1GR?QadEM33braXW(v(xSI`MfrC{8aIAiRkfi{E<76 z`8dKJkc6vdT?1L8McDM-#9F%Ab73@*N&Wq@ zQgvLAzqFR_C4V2;mq;K@O>5v2yYo?kzxxhaT^DuVWzKR0n?zMpmMWy9)YDk3SXbr8 zrMn|pXLM%xFk01VI-)qAaaV%kyTrfQ0L#P8&Ap+fOnEOWC1&N_$VGgcK4t2Q+c$-z z3?K}-6(Nf(Y3tqxqF|~97d10keWj`*RB@GBDZzFyWG!f_8)8$Ta-Yc#+F>-t%$lhs z7^ix;-<###AGOJIGz67AcNbafB#44B0#(nARKqsff;~`?DXd%S0N1Ff68zkWWdRj-+SUyNg4R*pQh-{YO z0!mGY*16}|F@;hw9`dra@2fXCcJD=L$Ye+O3}OsFJ?^2R@0Tli{O9nsk^s5Wj;eeX&pR!l}Zc zQ^-ATn8nhSFGe+5$!GlS=U1Q(L}dZj%rB&{c8$Qa!y=kk_d!L@y3m-Hk+=c8gWtQo z!$sHUz?!EbLw!4^i!#e#a|(zqDbFsOM8*YJC?Dtv@-#qa_1PKW97v&_D<{HSwOVBa0E8YSn7VB^~283q$7=b-&#Dlu8*0@pb%(Mw1Xu{DoRY0ylT${{3H<2 zzoNhjhy>Dch~+D&UE|)E?f9^Zsd|kYtgbsAcOYaJa31e0X2&X(UXUgSg|%+;?5nao zy!O?+06^fK4=G`ILG&_&^)_{2TgT9NVTrp}w$~CbFMtZ$<+VlWR22)*F4tvN(>`pT zU@5EMRRy>co+ye>91%t^!w~g*nUFFXi2?rX`rTb9T)t)2Ae5wV*J~jKU7@BE#?jZ6 zmfX6<{WPXIW){ybZO~^M`YjO;WP_cw@0CvIcqkAFm1uGmY>v7fxd7{DZvS6s7KV+& zB)}=z>{rK!dvW9aNu{!Y4=hQekZtaZML}nkUrt)Xk7)A>FN5{ySxo7JqX+;rxAm1;t|3nr zuB1RxJg5(T5p)z034 zJP0eafE7&(B!DlA1Y;!Zp%xv;uwHBY1uH}CSF;{~7rZl^1=C`67^mO~0Sf}CgzcWa zjioUqlVUFz0FKpgjq5j3krwA^U1>~N9ixjFTod)om+J6~QJ69vanrl(POF(PbQt`Z z%=0HrlM-@p0q9jlCWu$_VPs2sb4ci3SDFO~Pa5VNfT3YDda;DPV(MOv)+IaOm{>E@ z8$X_H&kBPIbA4PVT%WA@Tn9NYd~XQw78ujf5J#A0BO+OO2Xd00GI(T%cyUr4LtSiQ z86zb5p7~xyZ0ig{ey59thwKY{Re;hoL0L4^RW_1gHdRjqvz)M-CygIJ^bGFuk$i(D zi8m~aT&OIDHe7^1PggOcQTAMJ^#g|YQs*k}CE+5%0WBzkw_PO6;ULO@i2=%igf56c zOla7d6-dkz@{0%cc|VzyzIWn6^vU=}e5x~3(T6}u7o z%YrJvTlS;R(!kfBEG|Z1q2Y5$+jAhn`R)NAQ=THH@IXgo6!3sHG%Hvrwm)~50RxVK zwn{9P_vOtjlNfvvr0V{@OyZK?LmN2s`M=uM^FJ{Ms=o1>fRTbiX>){@Pl}BrAnR%$8 z7%V~45H;RPk-A!aUwXZa$%v?5ol84#P-D|&S$#2}N&8Ap9J)~vP?2w%f6$G` zUHQhx+Dzz=$u1m>r?s%77Yn9*5+W3Dw)JU{668lp@@^ofIB7-Z3MjI?P} zuu7SpvWf!rl@Z90-#Z|9&A8ed-1011O~Kvb_q))QzyjeN zWmIg`!H1{2r+H*ISq(De#A?orTo%mKoF4E3t?)sZ?U|SlMR`r7gnbAAk~>HqxGa7b zO2lFS3&eBW)`1gW9tokz#0%#`oGfnc7{!!?>m^vu(GQe_FlffBGcHhf4yL34 zSiTDD`L0=VsG=rV?BD}3L28h*+f~;dRc&89hmRUsR-WGQ^k%#*_kPn$}xj=av3Td>vU= zcgS=g2kDdS`<yLf0jTZ-w1!T;Y>Nqb$`T7IS1XiolgE8GTk!3Xw1hWbJ4c5NHE+NX#M%~+{ zUxO}*9G*omQ8I5LUmZ}5stQ`|-rC#PA0)2BP0D|?%|6eYr5;O)NFPdw9-DlXU+B2J z+Xpl`Nh2@C#Hg%y4rZ(gKA0=^V_4d}d%rpG8ovFM^6czc0)?E0+|+IHMZ011d`~<0 zYG=*go-K2oWYcAH-@w|yYf^Z~;YGOqj^pFB@1uZ20QFYjg%W)20sSxo?nH=!TUX#Q z#VXPWt{+n7@bG#|a|<~98L_dW_w-$(|3&g^u%=D0LCFVj#ph}n0R(UXkQfh)WtLDZ z5@j&7W?l~BmIBOdn)pRpR_(4ksR0%5g@v~(w>Ua=ZqpcDTnj%Z=?nN@+uHRp`A5^A z#y<1s#gFFy)ZAN{c$hjG*qHoxuc_>KL92cNgpi*GAHIvioFY{hC`7X>7!5UL{(>xh zaLAbbQg%_m&zrrX>a59YEqi2|{bmXQTBk(}4Sbja*XWtSD?|SYk^X=a?c&O`EBof) zi3%ok)h0D}!7V+po+XA^k;X}atpCE6(3!Gcw8N*B?l>>62fGkLv8ci`FeK&8$*)dU>VwAF=Q?Ypb6v;!jki<<| zxQXz>)j^c2mH8+}U&H`W);7elJa}@|$g(-%eW;#jqM#gCoqAaPk-%H>$Q zHk6dhwIxouxmr;3ao-jDKF;8<(5sL?!H*4@Xu57blFZMTBYEEYs%`pO3+{ca*wEWU z%x~Sa=lz>z1!ucu9{U#dU-Jd%MHaIT1^}@0%6f6AAmiNSwog==X2yo90nzSMBI z;F%UiF1xHoF``Q;8s9RI2+pHoNA?$!jvo_6m5Y!%bbP(w>WWG_(&_A&qEt!atWqo{aA8>Iqlru87*J$)tVziImOi>O)I+P&NaDd&U* zF&f?Fa_n0YS`)F4nwHd+lzQ3KiEwgzlqNp&nNrrB5Mtn98*9DTn0tPF|L3$Xd0ao| zX5A4lR7gx}cZE(%E^5rkdstY5xMjwRixZd#heecE{Rk~?1OuuRs9{Q!A9V*cr%S<` z(1FtsZKfy^WVBi*C9Q1ECmPBC>QX`=#AC$5rmO&Gc5ld^rrxdSL1aR}L9i?qq$_Mw zkC>omBBY~{1E(Gr^AGxU9$!th@@PR?=Zw?Sh^ZVsbC#s*C3$JqkHWy7#jXl9Xg_$A@qv zr{^vK7l)EZ*<(qbEO%rd0G~==q25Q~*0no~VlL`@cl8RNGmc_NxigVBPCMarSM7Pg z$GD3aF4dErrjs4F=}gZQ(L7h*Gie2_7^cHwkTRv%>&y&#DVlr9Kket&vqI<`ABB-L zmR8fS2eop-O#E!V#OHcHT|w>&?5 zs*u13fp`?kO{Sk7%1rx8-2XZganh28c}^Kr=Ef!qKvJV^y{Jr^cQhzbMd+6yOR}KX zZ!Q{e_PBG?;!{$*+EOGE)V%@rCepfvGV7rgoTLy>QFA#!a+y4?x6)ttCBsF$*dCl| z5UvQ^flahTW#w|PTjLQ2?(n$yH_A^-N%m`I=5k%>K^fJ>&a?yUXQ>Y*;timRwe)#$ z+xKKYBfF^)q6)GRxT1@Bx>y>7bKt~V85L9?L+?u1OdA0e`HMF27XG3}HEx-&qsff< ziNQ~YeDPr90PAssse>iufmEMO?nQ$%*ws6Qu_ClSp{cRByi!!ZiT|9a;Szn`c~u#h7`YG8AV%D} zRcmY{stCv3 zOe@hL+TslMO7dHumN3%3;I@0mxnpV3Fq)gq)6feNQcYDZ#|H?$I>-5w&@sHr95*gu zI_biEXxR-?LUYm*pI5Pa?`<)sS9umMCo7fckG{RtRn?S0H@}w_|66dj7&Vl6@Iw|U2ICkXgffPfPOMX# zDF{J@Rz-|aL$`8R^dTHyjCg2lAf*NIeFX?!CYu_b_Op<#{=0@7mk)P0!Y|}Ez0S7t z1itDu>e!KQy@?MFx$l-vJ9DBx;755MD#kOSeCOJz^>@YX!Ion)+jaDR$K%#UDOJJF z2bcRzg(;xlp`q44E|vW(I8D#veoK@BBhZL5CraJ)u! zRNG0UjOW$PEFDB%w-TbMq%IG|M%MRt%GB4XkXnEGsmfB}f1Y;#5k%TA*PY~fQ+4OL zl8gFugWQ=ONQ)#yn0oihJDU^vvZ(>QhVpk$cf>g?%Lnpf4T~_v#9VM1;IIg)8g!q_ z*Z5}Tw||xZ-f2Z`Ttm*P#pm_usKC1kdz=eXCwAX%RolPDHkR{ba2vNg88^V|cwFdU ze8gQ>vV_a#1;@)9vPE_7c%N)0Y9pi}ugDrtu(Tz9>30O^VLe~+PqhVb&mp#97{c4L zVd$jC>D7y@pVO5+Y7(ZB%qTAi+2injL*L17mj110!;QX7I6mT*oCPQM`W{J(@kS_H zu>OQ>3YZok2WH8iMe<0OU+hoaC-vFDl(DG>@)8I0e%8_#psY8?7a2~lO+bW$67g{Z z-#X+&QlweS1D;5-_!SnA===N|kwZI8&qd`O{a>r2D2FSo@=sM%{P}+7zx>bEd_U;W z|9G3D|1s!uHn8~d>~s2$ua^J+?FE7UlbFu?0@ytDr`>}3DdYbyKWX!i%b=6hKMK1p zWf`jj286D6HJIxmu#sdsFUX?V5);S{8zE##LXg7o12N_1Xc9Z@)Xy94#48SomxE?5 z44-JzZ7r9%_TLW4%F96hgDy`Kk{rmmn1Zv%#3_Dva6Hw(KksZp7wbL@R} zOdJ_U>KY0Vi$<-w+tGY6C5fzHK|&Qst%*o&sO3xqGs?iccU#hHr^>b;dg5EIvaz`5F-gRLXuARxRv> zT?7!V#>2gheJQ3NF8R^70waL4qfEpptQug*;X)Tg{Up!gzKmcBh=46*VkmS}1`RY! z6j31Zv2l#U4DG>QDh8QbF+vu*$wQzw>{|$+w~a(u4Cw-^2hedroAynLaiIeDkK9SH zc=wtV{jXYF1%nI$sj8|klLN8%+0Bp7)?vTz#QVO|px_>JMG)w0Yx;VYHiNz^DuHbz zzxJq}Aj_`YSER8gB~6={YG|l}NdUcoz2>K{t@gU8qIK}O92zj)gl@oShC9;B;g)0j zy9NI0j$c=G-bX6wdl9?b;Ryg~8FN}PYn*{?n0ue7D+MXNX;r1j>a*RD!Ik9>_rl=t z#GCdL@NnsTdMF3G;lj;i{zZcl19fsoc7;9-+y(z43FaHa^cIBf6ttr`c!{==={ODo z#ECf=(cF)6l4-qjqX_Z}4f+;8Y_v}AvVjErRY{b~6)MGn&ynX0)E3mJ>ca9%{7D|K zZQx}xf)eL>tUI45`R=SvW8q|x`N}h?zI~V~4sM6XQ6}EMN2Flz4p6zD#)mb&XMT?Z-Usy={Pc^?|(A%#PmvsEB?7z7y|6oN-GA;E6AD;-+r*k;U%s`q@fj3u$TMea$dlU0aq~R2~i*HuAN*+wh#K)>CoNcqr45(RLY+tQe>|G|ZG;@WG)MiFKL6!dB# zJQK4gx?uAOymG|}i0jiwm^PULxel*Yn|DYw&od34_oUCG_5~~TmVh!J(m)H@F5_Gc zt%?;J`I|b{0^X&kL+xCg&^2r^D<$cu6fjP2B<2zcI(=fa zb7Jgh(r^$={`v}2%zqer^$;c=9q`^Iq0?UJG}}-L^_&WbTH zO(H*pUQI2U2{!c0x~p{x^-pX!HNRjUes#JlTX(=Rme$~R3M9?=i^1%ZvjggmV&eB2 z##&{H7wGDbD#M@jnA@QcdV=^ao#wuP<*qNIpBRzR09t z?DoK^t7Bi-IW3G}t=j`1q??u+100WOqOo73cOuP*iG&g`Z2et z*d%CO!4S}dz!kh@3aF=&;GEfa5?Y`NEV{)L9QnANhu{ZC-sG9eL1o8)1}1^=I0}}n zgk0GgU5-zz3X$5#aNESXcFUxSptagbEED~~zwbfI@jP7WOW(p9U!7wM*M+D>I@ERb zyO!EhP5H1!KxK=ji1;a4G{h$9Cbtk_`IyihEwfu1lRPxd5usVE`cO01I?qRjCw7*h zatD)#(!6izLB|Iy-phO;{8|4gz&V-#-nl(w*>%#5@qh}j}RqekFBfs%i__0dU;xCDK@Y~hx%Ye^>2YygoYP~307 zZVnuD>5o{p%{j8QSSN#Arsev*(HfuR%F4;Yuib^kq2u%6#mez#i3g!y-tQmIbiFVk zh9G*CmQhqr7R|n2p57bZtaKe@)kI^!Fjb*N%OR7YOBK^{^KilKZK+QocObY{E3%K+ zrq=&fS%{PwjrGWRu%HuqmPHYoATd6eQuN+mpU?@%Y|%tT^v&7(PS(mf-2AO5&pY2> zOsF`vS98SCpzRu3Xy$tx9z?!X5u_9HJl`ZL9AHf zkx(w$ZLk2#>|Hkr+JTq}F$nW(JG1KcK%I&SZG_pN7&VFm4>Q4xkZ!|_fnDFLhq(ht zLhYb-Z?}ng7W++MHH8iagLG7&9#zluUKJ$;_9&7f60FHq9@nb-T8(E*-2kry=kO7; zmrv@}!$i=cKoLM&8PWqf30mc{idBz}FnTH0UzIz}pMo4$2&#{5%D(Ai-WXC?yjtMM zWk5oa3d^@36todB3S$%D!7)_-R0RUUhb!5?FQzjg{yw(-q zpY`?pSz&8@CA40jYA8Y3-V5TnK7f|rjN5GM=3?nbJ}w!WePR-XR;GJ`#P>&)RV+B1 zf>kbpJqJE-f$=(y+@<01GYnNLW?33s`wK1nxbiHBO4>+DYKE8b%3+T4;sxqM4jl<; zmn3#*i>ws*8f~bDmnM^~!!vxhv1i2&ak^9hgC^YQJTJH0&|T9q$9> z_{keCo`}zs&4mg-eZ*Sg?p2oH(k}kZKM^e*bbUgcX-j|4BH;#bqLqSs(5`}w2hYui zTV?14J(n{Hlrt^uyyK1v6emtkJf&b2JY~z=$6!GwL(`*DmDles*=@BRrr_Vk`E!ge zRV{ij`=roc#*x~$T|h`XvBw}VoGOMk>?kl2J_!kFA&7mnJ<-C$UVywviHc1EkgMQu z#m~)gn|s=XylqC+Mi_pOtK)rOR#M^@F&Jvin>N-d`iKB2>nc7u0<#q=hb3spQz>~r z!sth|m%qnT(5P?by}-kZMB9q0+sv(3TM1;EW*_JsByqj36yaMkC{#Zc`FU>X>;ej1 zy{EhA*S{Dz;FSu#72&4oAre5vJ|NChei5vL_3SxZBk88<`xr!7Bef$z61tp zQ%xs>|8lu4O5N|S&R;U6rSf)>oxKX#Ca_?Dv{){Ls!*d*$n%~Y&8^@|{af+ky+_Q!PTXj<8 z9{$>LiT~`zr25U^1?Zr!mddVT6Au>;_a?ak(w7uVf0H@Qux&rK6`F9oALtD~tc0g6 z{cRoHuZ+cH9w^>gqbi#6CHkogN>`n%yB)1hS>J)X>{nU6t>PRN%#`SR%F?r^eduhj9wZZC4M2XWa1V2lx8_*Yfj)y}+Y_a_mj zsz^w#jTL_#?mg!lwkPSW=}yrTalV1?H^GO1W@@#i&569Jp}U?UiOBXk(<%H4$8dJ+xx}Db5+v}B70Y1sy(&@ z`qaD9Q2MET4+>;+gi;zAF=j+K7jaFUE~=gl!W>y&5(P}o?UAfPQ>szdY z4=doLJ*-+%Q=`-DBa^4bOFG^%8l=>~LHWX*d2(efe{eQAeJIGH46j?CEd)TKT_@|+ zE4Yk(IP(5#TVzgb=$w6pyF+`!-d3!{&du$NPgg*d@4Zq--FI`kvn{cEC#zsborl~^ zqgJXGB9D>RV^4T*bI2n8JNCa`T}cA^jhIn>+GS|~0LuTmcQUoGHqraHDz9g6XJ`FS zmFKZa+F(oEIZ@L;f-k{=Wp;3_X>KfC^xz#!tw>xoksDHDz7J@w|7`}MoVxa{z(LONBZd)kB-{oC8k_wUu&1d&Y&59e4*N4f1F znMx*+UPf*T(~j%gc}e`ST6U^N6KjvM(#cWN+tTQjZT3dpR7tAEF^u^J54Xn3l4@nS zJ;4r!=F!-t_vM3j=vURftkrHC3b*!;i9Rmw z_bs{j5pUC_$sTXC+JJ`v;ZuHz(yk!Awj~yn{TR^cY-z8kXqg&@)nqJ@U6U=Ok@p%> zh7TKplYlx%VArL|Sy5_&RvzBHTGEl>wsE6L<52hu`;gxw>Ei$v<;Ly$^NawMq!eM&K!oT zrY7)_TcLRGn0~mFOtZ|KKTj+DyYTW{LCN87cB_7QN3z*@P(ku|0dPjd-!(w5>|NbJ*-km>dcZxUrK=`;>k}^$%Bp5~XS$0M^PxlPiX$i);G?rx#fqcW zmL#CmK&Y)?sRkn}dkz&f$#}sX*7nM~wuGit?r{tNYMw?oJVDR>m=oBSH{Keb`EL z(BSwAkOdBM6bM5@C00RgF|DsR=*Bl~*D{B8ig6n!Dmt0Y!Ni5UE6s zf;j*J6>B^@Ff!Cma%+NHos>cs5m0Oa5A%C&JIHTY9h}1(tf2Hi@h2pw!Ba>s9|v=Q zdaYZ&(y{TPT`)+42Y0k4F#$uLORi?FNW0Jq$9 z)-UvE(4UdM?SrGn2Ct0xY4iEP*AJ!#<9Z5JwjYSE9z@fKKpC})+vA$?fMPDHu$pz! z)dF?`3<|_*dGE02n7bs{XRGpwn}GIaPjIq^yCebu#ww$Y)@})ikr+)NxNa;6asC&C z&~eXS$U5*F#tO84!WY3H73*MZX#t$mYLr+fdDKE6F;yac1~-5Sl1azw)0VS1Na>^) z7+nQ2)07m5-xsTI^yjK{-C6K&Y1ZajK;e^7X-nH&iS=B=aX5_)#7Np##$)paXKctx zo}B3gt~FsqSD5;y0@}(JXjP@;K@Mo~V8`=ip ziOTFzQSB+>gx-}QKV`62P@^^1v{oFpymSoTN<$cJaa}D9%{1KS___d0X%7j^F%{5Y z8F}Y))-cK|YDDimEWL35Z;B7VO=(gdt5eYAbZEAl24E-`)V`*)j$++Govrk;zWq5i zm9|jAvFjYI;rvibdd9}%V7j1!4z6!)J&b~u8RZ$iZAsQGyedl@qK ztX&S~Z|NDoo&*XxSKHWxiyPYD4os}k^idBlp)GO|1UEFLN_OR$HS+xqqz~d4MTnGC_fxhRcZ638_ zvR-(9OB;iCVYeDe;K+?pOM*&KVi|1ux&jyIhC19%syn^m^+bd3Si+#pJM=3TiHM0l zV(m-oUdw1RaI}#WG|R>A_}CG!EYC#J&C;WWyxi4;U@lG*Hx<^$6elBH0Xeo*J;<6N z-ksId2*P(;D+&OZ)lwo8Q>>)>lpxX-b z*et+nN^s3x{x{m*DM*wkOcN~Iw(Y80wr$(CZQH(O+qP}n_AR@%dNy`qy0>TcX&*8o zAM<47cg`RAiKmrmmcK0)DeI{BQf4bKS#5o5NsNx%r6boL+4PjNy@71>9^MUPeyO|# z%NQ_yYa9;yhBTcD#`3Z< z!_0!~$S?Rcy)HC9N`z=||2i^K7AD)!T&p^A_ke`w#{`tYf&PHC`oUR4KEC|kYAtKc z&bXXK(6`2JL6ccbK~Z86|NnFQp#2Pthus+KBJRcK2N|OPl@{3-WLUp+2FQ*Xw(A zPJcHmm(C_ z)i)POBO@h}g`Sr|Ny+;Miev_-6WTC*BrTmgp(e`C`~o<8R{AX$6n10r2ihjaqh{qc zbFGQ|C}rLATZ9Gf+sXXtJI`<{Oj7Uj&QUpByZS7}t5HLB)nm$|MO2cCJ(=gLdaiM> z?QyFR3!2!oB7-_aoi_^HJ~FLlLVi6$<<2A*<`^z7gCzI~Z9;gQF%Ii*&fXN#;9-Ks zBjEaPJF&KN0X?^YjD7v3z%4`L8PvLH|J{8R=4?+^TDYlezI$@^2?(QXJ@T8KQJRo& zX>ofkREwa1C(bmH5>Indi4HKS6nO={2*=P;WNyDQw3LukO!SzSdrD9GgR)!bm_6Xn z>#YpJ*!xN}8T>QBExxwPUSsqy<26O~Fw$JS=bUmt2>rpryf4IA;o#?=^+vl7>8o`^za6P-epomQ8ZGZOLB z&Wh^NB2ewNSrCUArzNuWq8Ub^0h_i*dric?)r4ySL7Y?c5C;_HzB)TT9VlYdL0 z3@7jJh}^Ljr5q%P0@T>vYQ$o`0p}Ux2|cU?j&bNY{+P+-f;hrP2AZsJPNQPoby(K2 zfn11A;GQy}ScSzFkIWkFOtCG0|oBjtWD1c?OX!%>VepAr*oz&0NL@E_xc_#z0wz$4Z8(1 z;+ymL;9B*v)1iOCBV*Q$FOjZCRqoBXz&*FKhcCbF0hl<~)ezzHQnn|!MU}b98_tNo z1CYuyMQiJ642fVZ>N$%;g99i?tc%pA~7i7oWxrxshl}uu+vf81D@jt2+xyO zbr)2iLv-aKjF}5q!+X~UHx#R;l6vW@#|70fq|!uuIbyl~;b%w9a_I9KF;z%Q zg|)CICpN7kQy6`^`7Y7Zh+$F%uX&5RJar1{qv_rPY4CJ zQ9$b%*l4JI!y(=Yz{Zlnzg>2oITK z@9^?r)hT>&lL+~P(~iVTBHC~+cedlbm$CGnz-KDD#|#78g8fp}N>qj(9C5>c%9$b{ zDwT%|ksXRdpHiq}gEB;^CPt-m>qO=ef*unFL@=4u5TGW5h09-Jmh4PEH{x*UnfFRA z?(QpSJZN6gP5C-a5xnybjw<`;CZ>wKaulPl0>p4v;#Z2+k_f>$6g0 zpUFq!x}6V&XO4!aH4s`>cTt~UL_xvltqniAA&u}SJ~hKAlm2f9j@0t*IiJvdK4Z_G zG_HjWSR(1|p)v7y8G2`F6#0^;A&|7ojAKut26^D zyG87bwbPlm`}hDzg^GvVgw3+0#z+*^xTo`&mb30*;g)*AFkQr~iY>43+^3^3H~QUhPY3w?0ZU{vjvZ-$ z{6O)TZHix`;Z$F%Nz_LqknAf|+zEj|3ik8%<}$vpaxL0R4;c_b(bGW>F&Fo~D`8?$ z*m>{dIE_ECFjds}Ud$=ntV>F=`RBm|M$!l%$$M?j(i2kbW9^MQWQ!{|^;5tJ& zb}1fl%3p(n#=A}QDKqgf03o^nOtlgarm47I{g$VQj8r3$XyBKdTLwm+frL7`HC2SU z5V|O5u;K^rjXZhudr)jHD~Ya=@yV!!#pK)yt#e$_*HTcd^JFi~y9scXZ~=8Sf9qvp z(g){U?wkg5SNlJdhPeB*O&CA#m0pqD(P&p1_p5Ztwj+fKfdcmSGJ|%kpeObSLsyvg zqZV8)oG!Gvsv^Zj*LF{{%REtVC*cE9i(1T1HJf=^4>4FYGx}+2*EqhR9QJcrp$8lM z<*qh{Iz6Y1%%QF6S`~AIgNWqFfYH1}eUN80WCp>(mroY>0OnYUR`>~JZ={$S(VRs|{{D=|F7tl`4#a9S*zoCnF6VRsHuOWtwt@c^XHSdp z!R4?!Zq8Q5LJHJ3kJ@Oo^DF9;t9Oh9%h%j_GX15x;W2c}?+n{u`sof)BjECNqTMe# z;{+Lt=vl_GYwUf?H-MojW5Sf#F!gzV8F$9AI|`%eO`V|isG39|*cfvF$YF2GyZvXM z4#emNx|VN#&%(+`hU`|L^LG^+|8ghOK6}N}u~yAb{Dy@+&W2}qSi4azJ$kmZCEMR7 zf0KGD%iSwTN8?i{!63^(N0epogXS41zUQeCe)nBtG&onQj=1V$ltUKg0g?wlhss^8 z{Q3*GZ{DYHIWX|LIN6!U0clN_uNk*@&sn)ST|N7NW13SG3q;aM6UkXaVg3yof-)sx zKMs=dwR>hfREv-3Y8CqJrA{zUy9HKy(O@QqtDzMH>K1Ly6i586{bv1!U!TM+xxp%_ z`JHH{^v~m9pxT+O(j0s{dapPiSc1&%6G+x-t|B;Lfnh^MCA{#4fB>B~^4nTw*oPN? zYJUt*O>M;LEyeoHOElGH26`Fkx|Jgy_;%ul27w~9+J>i79Q8Y#r40tOe%-oCV`q(B zN2Q@B;5pez9)6wHMLPQmm0m+Kg1OXU*hXjftFX6Tg(|wCGj)g8i@%{;sD6ix-8)zl z13KrIxJgA}WWB4*N6&lurXdB4DB&Q0Mf^&~o0f>lA+?JqS)B6E`$9OU`;b)-#IOQ1(yq%*9@z^LT%&!C0 zXZ}_>*Fq~HkAiIN4AO97c=UzY5UFp4a(-RwSK*mUADsDN`yBcg&bSOvfUT2DW-eZ! z9=@Bd!bB5e>|In_gz>$ijGr!~zzQNRo$A(XO?=wvb007v(j3UYZ$DcS*V%%U zQKS@YRk?eKPnH#WAjgI)jznY|{XY{2(*knZjNO`0MDQB2BDh}h7v1{@I8v%KeFpxx zs-OjePOloFp+h3CjFAhdQIoZZn^0@gG4a+K%km98q_$g#-a-=&a?E<%n}3T0Dy-FM z@_EfZXxzJ(gT0Ks388tQIX>0Wo`~Uai%$UmC2s<0O*YUzqO8s4;^NEsM$`H7=IqQ7 zC2w3jf16)tCNL9gd^kHgcy#x~+#lON@Q@IZnx%GWX|$p$@t|a-S83DEYonkx%oM>Q zPdT!Bx|3euoj8o!ax7VWn`lyx#u(+EgVtQQGzTLJ79^0RKp7Eg8MA1BUC)Mq+xqR!|68@RL!U@m zL^B55McUNQmhS;XRY0RihoLZn!jbPVRRm*j%pW1HE=e>Ok*2_`O#GTaBNg->amo?u zK>K0ZLz&TKL)K;0b3uE(u5wjQDe6$@{q-fjHGUG)e9BU`vu1n+lb!Jt%f8CJCw-^d z(ocVt1L-d?o(j@0#C*6GC?r-s0&R8oG6!$BJExC|I zx1K(x_-g!!OSxJZy-m?!6>q=E4g^n2z-mk8xhQVoSU9x3vIt7OO60c0^b=-jng{Y8 zD=ekO!-DkKZywb+DvayOztjEk$-keTe30i@Seyu(dH%wd90q0Y+6t*2jRtiv9&l_q4xgp*H7WV>22hE+Nduhk;$Q{pA~SqGF{{5`N*mKJ$^>| zbJt4%g~AMb$kh?y*zG?3C$X=dL!j_@_M*8iO?Q5hMF+8hTdh5U7^&`v@!u`+}oW zKXVYd9WWC$DxM+7O4O0tDONOqj3G>``_yLOUh|zaj4v0eMB7zK><*#0|4Q;ONNI3X zfq4%sYI+Qijz=k42&;op9kQt@URuleHhZs21gFruO$HcJe#2lI3Cfy>F0Z-PIQEri zOvnq+t91uRGKQUBwk@%1m}NNX1LG#`K2e;MV#(F1CD|;b{^{2-Pmco8E;(p_ZVvQ& z!rJce384P*v-Fxos8UPzCA+a?$xo!puSyA)l!ViQRd2O+MH60U;xqK>7f(C2h$Yb5dJvE z;w&Wn3#AaAUmBd)v4e6o%ff)gUnOHVW<|Tn4bH8cM#y<1)4nN1JV+)h{0~6t(H?@@ zpcZ1umGD$YMy?dU#)_2KeAQp-Qp{RqZ2Wi_)1j3(*Ed~PHvc{pj{rz+Kz7%QZlt5H zfje^I8uq(Kz2!78i;S*4`p)32^FFijEyCup-Fc_dXS1~VNYU^x((D0*i!W^GQs=xs zKWSHnoL?0POIp0OFu`%HERikKPSgP7I~I{>4Og~w?Sntsx#*o|tfEo z?`GffO`<_rYc5_BVR(l|!3*#_VZ>lxxF3p{a#^tE&G^P+b6h^(Ut5D)owSb*y*U{`%!i0w$^I7FZPw_>dxzi6V&pyTo!5~>{l9%&1&!R>72V| zSdyb4suUqg2=Tu}hIOfRuM4Lk%~Iwq1K;g8p9=bgOsujOLAtZ8GiI6nX4GBzEkQ~> zx4?He5tp#+iJqbXUlD40(jv#dI*+*CfF`S5MmjfcHA$RlEZZXrDN`*pt!M$CxPmG7-V@*;m+)fsq4W^72$wi9Cm1oPE8 z!DTV>$qAP4KIjQTkr~u>8}7Ah2%wy^Hwz+H^C58A_2uj!4ZQy4s=?Wm{NX6&Sdk6& zYp!tBlj+PQ#HQWES?|xbwe`$vxxTN`e6?iYIufRFHvyHejH;#=RlzTCeNxSqJ&2@J zTa)q+`zGGvJW`L?){Lc&Jkv%63G?H?*;W;5!J&v}aT2+fTFd?y!} z1o~hbP-8Zjc|erIdxBkwbo?ih>d$O6y8EB$%XbCgx#wTm7yhq0>Hi@8vbFoa$;)5f zGFwH*YLfxQ=cTr-h@O?bF9Q$fr)RZ z!TXly6z^?IyH9e-Okz?pC>PY;#p8)G_gv?IxI^eKmPs7CM4(E1s|9$}NFseHnY0fjrYVnqnCh z+jU1=;eh0iG$p-oTBDN|b|W&-_=7Xbfd}M}j?_0z258Qy4`u-Ko)s12VrrS4-}5-3 zy1RbOy7kG8gb1t%sJ3~$I0hH66RaZaY%=4kha#3EjeKAY0*${QEXm~D((1r9nsR0t zQZv4=k4LMx%|HQPNH^~?%!>s3#kD;{wl(M|*`R*AB$NhXAdy&7U7gvwC@8}ZjKrb{ z^wTs#12c`&=v>nSOz^2H%O*DSL;!N@@s@HdC*hR{bX{2jv5iW-J8)vR=U7}|uiFpJ}oNXKHx**Q07EMV|OC7aGJGU@$F z!_yg(8h;(Fy?np$!{-z)W>M>@R@5($e;>Lm-~2X$(sD!YvicAG z%=8d^mlH6pWvweq6z!VPrO8GO0L90D@tERjJH{xrAs|5{-?7`1-0YJcW;(BypwcVH z7M!Cwnfi-Dmaty*eAs6OxwX22F=INkhubZk4$)Sju(^z2<}{0Bx8OEnsQ&!jiCO=G zW}Y5~O|&W7V=(y4|GOu$`IXK8A~;3)ev`2vj z#cTi-dqfhUeF9@8gVqyj&LVE0S51_0G`eBn2wrbxx@Sl9dz-)1Tg(|5McFj8eyBac7YTh7>vW`u$mJ>XzZjCk?(0eGas@XeZhkTM@TW=D zG1Kw&vVvVcpcbig98b^DT-r%DO;Qx>fd5(+t%&^ zW?U`B5Q9yNa=ii-BWqxp`dMsQ6v~>)M6kV|1J9>eg}d_jD1Hq(Fp zL{CYnNBy3ziz#Q%kf`ks8lD+eq z`*;ye!X6;g<~qIe)+sq>MRDtfN`4v<)4?=Ec6w$~DMW)bStnRV9;%k$4%2;85W@3+ zanB+zy1`o%t2AP%&fR#eKqYC3v&)xgL^Ht4VI!P?sD(cKi^k+Bp%dek zWTsVsSCQsj4D9ektF&1a$73+XuY6VwYxmj>M+#8vhj5&tlv>=x_Kx5yys+_cl#KS@ zUU{qJ-@8waP(0|_Wi97qsVEhYy&|wPP&2rSJ$NUG(w!gfaNKMc_d&FLkochS|2_)8 z4hfPU{Hz^<4n>^H9LB;NI<e6p6vf~q5tFN zcVB4MymZ=RMf$1f1v0q-cFCvI<9ZQUEINIk|0j*)Wlg?_9XF9LE^jJ>$O+I;W$Nws zaeW|wL}Ff=HiB=B6z5~`(i&B)mtfSg{^00BE|D%X{t~wP^XHc~%FfNgPdJRKoQ$e0 z{QD3E|L+L3y@&hlX-_9(UUM$!A#0_9Msnix==3rv2p=xC(wX|qSh0pJT56!&2gyxK zLp3R|k!8K;zJ@AoNLpurS|a2fdhm6$*LL!p6#3YwQ}Bk(q`Lvym+mt8c9x##9=e)R zj`R@GLrv1Xipa4(fwc19c_^=>woBu-GBXjr6-FF{S6-K)#STR8j%ou6KZ(wK6_tQW zvLkInglSJnrLv>y=$JLApHs&^vs6M1szus8XZ<0Hxl~WC9K@$N5-^&>$VGv*atCKO zZ}+>iXQH;QcJ5E6wysWAb}yFmfM{0?gG6KU+1x*zi zDQf1ESk(*qQY$h_%85v%di2B^k*c#@2E~PL3{J3+hPlKB4NIuENxw%dI(1Ydk z%$b!B5;+s9GL%MKoy^mq!NEcG67Fh_LAf+ ziFU+1@h|xk6Rg@k)lC!ah7896;KLB`4Op|xzN#vxcZ}U}m*Q+R_38k694PJ@7MDv1 z%}ceT5-%K>8QpAnvqY1s$Z-eFq$Z?jngp2kZvh3C6*ZCxEI~9P76GaU-7PowGh~77 zhnyJB*do^SVy8N+Tlq73>LHv%F4owBFX2j10sce~A4Z9R5h0N3$TE*VIhI4Jb&k+5 z#05qq;T;IAa~~zYMapjgVyt%KC`BkHw8old%vVS;O;WO(&8p#R^_apoZhyHrwya>+ zH%&srE}`f3G{a3%t3h0jy<~G^1w1v&jTSrnVY6)m2YF(cc)6WY;YCCddty0+82KV)cbA_}w~@qVwRe2AeUxfL|lI1M_kG6fxtbIY(eq+-I?3 zRdoy$7-C3?3!a#?%vFK=*s)rv4et^8C4(ph-102&zjW_|CwsETs;MG#VmXZ@DDB!E6Bt#ays+j)Bs&qq- zSLVMgEi1NZiVAm6r#lX%^3MMa@S*pN;lcgx zHWz*wRW517?Q!-f3)mb4eUlyTmC7z9^L83<}HY0Z$t6sMQ84A5EzB)pcReih}hY{rUF|m=&s9D``jL$MD+oo^P5vOztnoZA)jHTkARiP> z)_-849}&bN=CslG56W-g{7Fq@KhVR*y!;gAK;D45`vgb}@mF+&^jijtt(>TLHnrVO;xxCK26jQ)%~EeR3h%@MpEs0B_plvxTDEvd~awnCIQ{Ne8ha zvjlhxvnRuEVy6H@4|@eKijll9X?n7UT5cB;-hS$<5tk=Es;{daO0GpTaM6Mi$BORN z%L}6>17^h}^T&=qM?5Zn4a1m!m`R5_LYhn_PT+>Tv(ZNdI$p)J zXSap!AnH9uB2x=3J_6!}r*#aQwV&9t2!RBgwoz+KVIoC^%H+>*Wx;tu$XxEn7(mev z;@A}yGbXWV!UXImYeAMl4OM~mC(HVbcSu)fYKHU%cd|@S+>W3zE+N3)Dxe`^B390B zs_}aqD0|fGX)L@N%~m=QCJpFCo!bFjl*ya7eDW|SFO{eg$;p-1P`0{}ZI`O-F_pza zHP3S2|1GLJX$7P`Pc z=Zo!i{$<0-Fxp}kMS#NULSQ9}r8Ik<8ljX%j zKw6u^hmBCbVUr95V4ss_>amN`eV{yt3I6Om!rWZ(I1N&a%ck zf^_t;lR*sdM!1ITw2Ne*AtpM!9H5=%A1)S=_Os=gVf2 zol6!%E%ffc0#A)30O>70dzcIj?m6jXH!Z@NyJ_z^XZ6G;LpD2x{=92bK3#e`MPYau&-q-WZ$3h!Ip)W!`H694!b>E!LZ<^=rf@#2(b6xiCAIgZTIdO9+3kM8`d?s+s~yWO zkVG@0Iw2c~xcXHrWxnAJh;gM1ZVK=0)fMyUbT)HG!*bNJJC}AV5?L zXbPAt;+-|~*S@Z~KINt7CATI|kfI(;*&pyYvY@%qKV%zh3LP{}U#Icyc4Y4^CLQa$ zMjoEFO?}+HJlb`S&wzIA(DX|2H6+>#7EMIjO<4->p?-XN)4w^gTaWT61g%@q6%Q5f zGHqUjsi;0!^569;ILucHG$V0$khhzzu%Am{y*Yz!o}=wPo2)H^( zHKB5pWRF)qYkx!7rIIhcVyPkW-TKpnrfqWAv?p(%3|54kGf z6REq9aQ0YE1P?WF+*5>{kM42Fg3NRuKg9+o7PqS4g(J7LCw@b^ih1xXKqlk?7$`jy zTliaPwA3lK9Gshs-QQOegpu#&ZJYw^GL-FIu{C|!#@SB+Stja?(lapl#g<7zPfh9J znVG35;MJrT6rtDKm30&mT2D=tdx!QD%^VH+)`l4DR0s$xjAt{bELRqoq0PDXfnDRi zd>OXYHQrIIWq9KZ=mhfWuYn9?apTRtGz|=E!u*kfmxrA!&ptZb^`=%UH;+FK2HzK_ z&jOk>8EjtVm0u#41b_8J;1d`c;jtud8a5wGSb%}kd0NN5wK#B!nZgk`DiJy{NYCk_ zp;HJTojEB+OPG2G-`qHUoI2lf8qR}9;Z*kBWBJQ%sHF8Je)LZ02lvU2kCw&ll=RI{ zZwTY>GjCgb)uYsP1qF@As$TZIcPit7f8Y=X>N3gcy* zyimSAtQhc)txqGqMH&E8GCT$+8W<&a`-EGyepems|K`N9GP~dp)Rf1n9IpEq%Q$hj z;1<73cbu5&(i8Mg1o_08ysQbz7ReR{>jbe-0yGuSQww(+Ju|henZw!8$=Gk z%rB(OwVL8|Bq)wCXG95UwG&40{A!I>1UhaEZB=t%L^F0fr!n8*|MNajlOSZ6_4^mh z`F9`qZz*w8YXeui{~R2VqS_z3@tetTN?o-rl>nfk^`ux1&T3s*H6e&DOs%3~j5OFG zX?=kK<)_EhAB)RHV&CukaI*fJROAFN*unRw%j@U|TiP$SXCEX1D5d}fF}4+>UT)8U1J?&51d|(u0-`1Qgg}gI43pGI7-2Kw z#|A7e8TAmBMm|fB?wh^uEr+Sz9$@HJfuGRCTobdmFIdSHFSP}f>`!vkIZQ4?c^&E0 z3#Q^B@s{y~3kD3QYeFB-XtI8Tk{1Mv<2*||zPZ!7LmCZNkvR241X@zk4kvpUS2gl@ z!~!LG>-mrbSd<`+3Xlo6>w;AG98tQ*Q~Kp_=VCOZ7>&?fxFW6$?0LFnnVT`Tw7sV# z3Idse=Y7g7ip@UZp|7ACf*+J5lUMOQb1tc`NwcfsYGnJ}$T&8t8ZGa4h~;%g@F0?| zUvAmey7?{0LNMjF$7{vVP+hBFfK!%0=o`m#mQzYu@k!h1j%-`;5Ig5NZv&DMY6Q*; z+;R9~+sp%2sf*{Ngdd^M@LZb=VJJyV%md<0Yt|&6Ow+7kL#hkAvQhFJO8tnl%Z?tL zsOzDpgr1L{c1=>x#P)ts$Yb*D+nXa-4o2bB8RtxF4kekXjk=^t#qX@5eT3M+8i$ob zn2{=gFvK|3zz&w%Uz_NtR(xe!W|q<|)7W+(6)@%4q}KDc{o z(~24J4fkI&-(2&o-+X_qX7=L$n<(S|=j;9tANl_%^>#A(kFJMnZVRVP(T3eOYSVLg zFdgOt_Rf!hr{qW)uFeYdQ97yd9;y^593UY?Y6N|I@q>qpUORjzpga<@hgfA=YShE@ z^Yfc?FDy?o*^2F+nOfHB@P!QbuZySLlY`M*x!oMRs3iDYUf)+|Z{PR*QTKc4EFbR= zJ$k<}B^B0-V~dgrR=v{--EcNDJ3Zfr`=WTALb4?9&dLi_MC$s7ATz5iC2fbYrbieN$O>yQddZYrhL5RZb~)KjYcn|Ap!NRV$`>z3JM- zWIz4Lb~6F|RvxhLktPMo(5Puv$+l(gamRGC(Pd^#G-TkjQHd$*2{DVY^f^FsSsl0m zaGnhNqf$lAZCY7l{5ydsD4^N&@G!SsTqu)zwAWmLcsfJL!A)D3T!zywZDaDXGrYD`)T-Es{>0qK zRybzs$&j5&5VzJ1ABIPLrMXwGr5?R_yf7 z-rhS39580f#toz0g2HbsFU#cQ<48FSFHoe`J&mW9&<%G`Dl z7}BXW$gn{>@P@sxukEVTpl{8+9WK*{+bbVGeU3%^IV(qG0BZm zv{hLt80nV*#9D_?7r8?|2na~2?I-gGwKU172#?RQ!sPh=S#{Hj#9m#i21ir%6*c`lWto_UI@S1Iy7cBf0 znOeZy4!Dxna zkCDQcRKXqmzWY9S4{Fi-vxCcacOLZJ#}rXwi9_IMqwZ`7*w4NBVf<4d!yP_g0O);; zXJ!(x7Hn4hgc5!uWAFqjFkJ9IG0}IW_m z_fMM3@q%+_<%N;n{eXP_4B)QV`L%~#Q(6H^M#10|>lk>sj?Oqiev(vUF853a4uAai z@YC%5Tfj^pec!TDeFWvvpaDmGi6|n0YB`6knGhb3jy@5W5*KmRk&>&Z z8J4ORM*1l!EU@7WTyqx1aBn;=OS30`umEA+LYkJOk7D z9;Z4XA!T}kowV6zy#N#k)okMPwDbnIpByJZC^7S~IAx?4Gmni9M8p;8(Qq{eAUEv^ zikXjY$Gx@anjaxDY8~sXqt9A;^=O`r+O)|iHuzr|DQw7Mq4 z>lW+&lYpz~U1=(F$Kmga`o22@KBY1y=B%4DiSD$BIZp{bv`FY*^PRsc&hCOUp^_v# z6CT>QMi60{M-~_tL_U za}vA@GF%W8sF95>vKc`aqp}Qoys33Vu2?&Kg^7NR@k}UkCsmjrKtUzghiKx06AyNFR0f`zSNM^Rx2^l&kGHXT4>?)6Jj5Cq~tX!Kr!6! z;A=z8%}Ko}fe(5)!?rF9c^FI4_JmK~@qAmC@bujSQUaF^7ThHjNf)w2>w8La^Cp?{ z5tZIqYKsOj?+S9npBwo^BY*i(*0(mZ%3jJ)QOqQBX2oA3v-?c68B6)_LXE?dVtEYS zpOto*;>ek(=?oIc%Tz2&Vw4kitAvFR;;j=v_aiBIN=wvOF1nwnkkIVW(@TP+g!mE~ z7z&T0BBlE(9W?qQO1Oqdo=p_&?Jc1>qjSI1Ti%E-FDq+nAe{Yl1Ep*0SA3HYlt4`M zG0)+((bKHfcvgbH1_G4pD!>Z4pu`9BJM`llS@IZ_*=lORfFI4{>&=Evf+o*5B*YC~ zm5YPek65nn27THzp(Ggv+(8AW+}#aWYVf1QU)Ogi(*V9zWv$eretGYt1+<+X=(^$L3+1+kH z`%`p9GedID*#%Yu>qCsT=>q%|$y$wt^tinN)~G8jvq334d8Zqy-eFV+`PJPqU6nM2M!ekDU}*21PKcBg1$51 zl~ooE+W}dcb0@#oXIWSjB!HE4BQr)FC7a5=et(5`IQ&$9m7>gB7wX>%eU?RM%NJAC zgy0w5Ekl{a7vZQeHsa(1IHvqy#}YOTt@D7jwF|b|?4qU7hBKGu_?i20Y)wEIO>5Xm zbrNMj=^)V~y+31{g~D!+>GBgfjoSA08J6uOj+W9WPZcC6Xrvtq$ADElgR)`c{qp`; zTbCCKwe?dpP?#zMgM%gySFzq%ui#=Ey67<6u$KIsAEdkhZPW}2nSP?2(ln{AEVw=f zM^}QVy7L-CPT*pz4imPWK>Lufv;N~8E;Z{HiU55wfSFu0df_}bVUxiW8Q`6J{G#cl zrx;QJHWoDxxTv}|Z#}Mze|gmHOyN~UyiIc!1>o?c%x}z7@|2KnH0*ijoU8w= z5SGH4$^57m8We==Tog(psC;2vW3#8OyQ2abJ0XpMt3O+z(|ulU#@+e$Zt{#UEw0bo zw66~>R%CHWI~wyL_znlCl{GNt)s8L>=*x$GA_Vq zkig)18X*>6GvU;t=4x=^f#|XF6e1&4cz8;rEF+|a=HruNU#i5Ki_d~Qc;sxhBSUbF z4F<9o0f>ZwcAFDyS(w?ov{_h0o9=T}70ct~QB!WA`d39Hfow;qna@qzwsE_GK6#6! z-}C!k*yFdoQ0hM`H zEb_n=kKYNclo?$axW}>}lQilb)gOZvfU|d1H^99nOYhq*AB#Yf$eA6$i|t$oyo0~A z>`Y)iY$D>^neB|YGwMzZ?EcooTH5Z59Nnvc9|Tm#zEBk68qTRDk>mT(K@lKvx=Jf~|{03fO!y%V=x&!pQHe7`FT+m zRpFvB6wF5cmC;1#HjYdW(6Izt{G0vrh0^Q`a=osXuLY( zQD*hkVqaQ3GCF=EmH`NB1KtTj?sctj+F$Ha%}rL`aeLrN&%f!6KNx_hK1&q?`<}@5fdj_1bs@-9~DXQ zF0GP~Y#v@^NCB5d;8W$mZ#6ZQ9jk&!SJfKx&eQhUROX=jT|pybEG>xO&LA;_C+dN` zssOSVoDVPhD|c3Dqp^7QSY}a825kWh{ix3x73F5lD?K*TdEjU>zA^-Z7ERYBCRiWD zWvk#ZrfZUiym1oD4Uj~lF@pkLxUYzxc`O%f+X~aphSFMve0O=jP6(s=%Tdk8s`o)W zK;&K;n)MabR!Y)kPuMZ1w`cLL1Nwx(%-%u9xq3E;CP=0-hgm-W%5dGgEzWbyy7I3f zSI;{8V-ubQ9MbXRiSJ&hl-)c_vc--8U}tGO6AvDw5ZUlEnm+^km|UuCn^sm{{}*BJ z7+!gtZ3)M=S+Q-~wr$&XDmE*&?TT4(#kOsn_)YhnJMZm&W_muI^ZEa*-`;DljiqFe zKxB$4xrA7M9AAm?pu|4vtWS!LQ>Qa?0M12B%aPcN#T0*n$Q-k{La463^kFa4(300A zxR7U2%YcIxL=5J4U{<}8kH404``h2w8W9(k?p-&lW%=5hB=zwX0hxLTj)%SWDZ&l9 zPM3;#vJj5L?Ho(l$c6pGdX1$2E4QKMxm!1T_FS(29{Wu$KAo^P|KW5PP@K<%z{5x+ zK?b^Eqrs8`^AXPu|M`GVx7Y)bqFA_opWqw7gyr)>N-Tafh3C$fc=7T}L2va)hwaF8 z>j=98PMRVcxs)!g2P!b{bhO`=EN4P_3v5^He$j|S1nXoj>C->qA@Q#EOT~|u#jBLz z0MGAThwbZF?f$tqs8@%MW+7Gf8T!i#G&Zq^>FfYqhU=}j-P6K&uj8#c0)2YR__*;o zarUqTp@{z20Ei(Q^edt8liib(?O3q=2?Y8zret$}nKcV%&u!kwPE&+T7q9%f!l4(r zPLJjZkK}S)`-*VPohY8aDSr-TC2u6a^0Pgr7hFoj52&;n#G_WNZ-Iq&J&|QLo(tuIon!!7Ix%96$2)*tP^(k5_R=- z%iJpfp+t+f-c3ID=Swrij-M?_pHN=?5mct6-XtwV7|)qXjT?_k2V-fe)+YRu~E zbeHwW&VHnvTrBocD7rG|h;XzSKiKwXvDG_LZSCjm5PB>8xrPPE;NVXKFNbrMr2+^Z zB#gLGqM3tbcJo+j=SA)VY{$%?yx_Gxlg%-L)pOB28p%}MA}2JtWNawh7Uud#wd-1w zjyP)9_unm4TQ`-(OWBbNkRhqsDnil{WVH>XB-E)j(%0#A5e>hYVsetc3`n~oQnF*d zynM5$gKFC7uO8yBk~Caba!~K^m6o>>iRpc#yQNq=t|IX9Tt>Q-=s8*r)~a zrRmxSVV2Bcl-^2j?ZlkazXbHEEAwk)74}q)lFd zZ8t}F{C)TwTh*->kj?@GX2ock&$KGJOlpx`;C&;@U%;=|qC)Kz3rvVQgNbooEnG%D z1q2csod*w6Py;JmKo=!9_?AE~%T=Xbh_Xd`7Hd+FF`Bz{@YqxDcYnZ{=Hp#m=SCe* zmnevRzB5+^M1dwm#T$ZwV3UYS5imImV&aRdp@$|=`qim1o2P>ro`HC*a$zKpO39O? z2%PT;iXWaSbqx%0F4tUsk}6+l4|#@)7}0^WNCB%d2s3N|y*+MuLsBt4t<#_kO2)$< zeAd(>*BBZ5WYZnKD`v{OlZPWPrdqcdt~#E1)huzS{p3+~c`q6hw*%4+eU&Q-vIfLT zyCfH*oa%1E^LhEIX>AW?eYR`dl28ehYC>&7?7^y9?~LW;H_&+<6(1b#^~mNYS*hSE zWf@mCQujMMNyp`sK<-d`1O{m>2W~RXQhyry$PtMCAq%FDRaymh&WcX1(;^Y4J7-tr zUVWX=B}(cZ%6yKdH)(G;0xh1LTH1RD(h>FYtRe+SA71^PmV2B*)(KBJyo5p6zs(<; z$~xVAlIt&2_yL?dG@cTSKKiegm6?6{Thr=R_DX$YUm;+(r-~zNrRsP*@}3ArBvt|x z_`e^me{tH#vDhPRWt88@+mt{aJK1NE-Aa;`i*zqMm^fYHyHjSNcLP8}2lFXjP#z7j zzB;*U6blth8BZspNj_f5V~1>x+r7l|DvaiuiV&%;x(0U`{`Gw@suG`&eZP<4?@;_7 z--nf*k;OmXhq7MuIwO+rbDd@q=3Bkrp|k;9n0|6vc$!RmsD1&RaKmz31s%53G_vrg zk4EGDtnS==bo1P_wT1al#-u6n)Zy{KJP60kS4`XJ(h;E`u zi+zIGHfFH`reCI%vYF}SC;(0Z35O!3VV&6A<5V}2RA3Ud-*;WQV>s5d?Mli+l#1xh z7Ausp9aJTO{9lbnx?T@M}t z)eh;;kM6g{3Du?EQJ0WL1l}ih*Pw?;Qh~8k&C+f#3HGiXVGJ6EDL|96c~upe*3JeJ zVelTVXcd6jm4Q+S0`2H3ZerJOcRhPAUMFGtLLU)OSoyf}Vk*^}q@^3#`KSQXTgN_1 zQIDEqJhG|2L#nQlI<|n~;0UlxfZ5b&KDVuZ9}LlZF|-kIDGA0;!HLTxni1roYm9@H zDuTa@S(xtob?(aDn*ELv(sTw3UHpZUT3`(?6;1g>`QMBk4kRiX)xR;8{B8aH ze>^7t-!b@~7&9=jHF9yWGPg1@`bX~n4#}x1>(;nTNPgROSQqJT4IE@846tFP?lN*R zcqsE|bA$(*S~{vU&{FPUoE(}_pfne=ciS~^Syo}*PivU~Xgwtr3# z?9T!~!jsRd#vX@Vc{^wu={Iy^{SkViVNwVC+rbSPad3%oFU8`s>yW&~{GK-HVliVeaxSu_MOApr$f6eKp4GF4YP^4M({e zNjz=CmkRPiaL=$*=2i`1em#X#(4r(S(mEQG?IIW__^)(V;ceo3G<}fS`2FH5Hzf}w z)vmEm?jB!Z8l(%{6OHkZk;Pzt=<@a%X~#lI)xG%?RO%)J#mnZU-gcmbi|ACLn-M;5 z@)?oQ(bB~P+d@x4geaRHP7!#qw^uqGk0jqngEEjUK$}LnV z$hT2`MIZ6qvlI%hGv2#jmHFX{akE=~AcqW#T*i6U<;~g!k3;5?KSEpdwLRf_{kdrt ztY-?3EpT_E*c`}Ghhb&)>Q~Ot{Uo!3hR|2XHLK1`;0D47m@xQEF1P^Od7+@CcUlBJ zc)ebgp??VVoJ>hCMVy$P9+ZinQQnD-9I9?>I3_4c0|x;Mw^;Mck|E^Ijhcyd1e zjgn@LDelO9W0dh5qZI#%Cs^6LnmOB=8M&L8{*x`N+pjai`8{hOjYX@hT+J9UKtNQ> zuoG&|qRF`fHwr4+t|6Ptb2;&Ap}=oHaRcGkRwQAoE9nmGnDi6gCs1mYV=Le}p;=`uE#}4NB-;kI^!&EGtWjwx>`^jkb$E= z2(IDC`2Yf=64lxD+uL7Wq5}2reMnB~Vme1kx-*a$lzH~bZGZ0u$Y$;wd7DmiLR0Aw zPD08KRYyj*Mj-pXJ%S`n*2jB1IZ?~Hg3}R3tsU_4_=C9LcBZCQ)Re)|17S-?3cudXO>0c982^$KXD()3F6I#o1DphhLu@HXj>N9;)x^`a9F zZNg$S`WOq{--w*itRy2VuB`7P!ypx{nEUM*+!2Fi<2|^-1^AL0Kvwa65~hE15E-s_ zpA(~iC)K%2V$YHZ^A)PbxhyA^?=H?mi4>i;M(pYSm|j4)?!~-Z*Z)|zy8gG!43re@ z8UKyQKi`O?`A>+nbF+1|GO={9GBGo7clw9JOqi;k!aufi|JJoz8_ZF`M_4f=lE=1W zsmCfopvxMMGT+MC&^55EQ>jLL_1I9@6{PH@1NL+Geg1aH6NvxT$L2ENjJOBJ!~fwY zx%1%6oQpST>Fv+XCy*;wT4uHIGrH-vZZ~pLNefRNjG6*Al04)u=Kk*6Zuvp%FePp9DWY#!0dTdm1#7ZccY`a)=A2G%C?I`)ZP-qH54ojYoJd$@f( zU7JC%4LfqDBN;Y9N7gQ%smG)X)KP^B3NTi%??OOa+Lp|RcgIdXP8C^;OUiY!J=2*nkb=!IE z<(~OkXg?QFf?ZV_U;Vf~pBb@sw&n2GQMXWhgU&zN`sBlY;)D{aKS{s%$J*G39RMEP zf2*>l$wq87SsYMITz7j;)KP-Khh@m{O&W$9CHp+=mY!@tuD3o4eloY=w-E3kQf%P? zlpcP(*92MLK+O%XGXs0i3DM80OE_!()J?Dj@x2c>UpdPCQ18Zn%6#gfpPgaIuLy~x z*~Id^(GvJkY|T|Ya~mRG>pEPO^ve=ag#b?7TMGKrTXPRZ2BmPWkgD69pJXIhDyAFk zOF+tVAf?;0>%6|wZ*HE2BgZSr z(YpLMC+_p#A}u>|dnEezm@W?V_rm#q!nK2m^EZ}j)Rn%mjO72fjx#kKqr}K$pUik( zv(w5lg4oamrHYuNqZuO-f7QOg`29KOlHIDH59wTHZ)Mx#+0P=c^Sc>8g-)OV*F2AK z(X+Y}G>#*fVy8Je?U zmD#|isRGFZ*55z*Q60~fg;5$`F{!AxaBrX`Ej|@O2xulAg2j7~_iJtqaNPqsS+;I@ zUr7TcN{1m`Bs2prMlFp%LQjZxrfp~ABo^FB&|%6}Kx!{9WoKe1hoR;{b_5PLe+Wf3 z>BJmpL!#$KI9M9)$)4@d>5(?S84;uoGtxC|C3Pljo1FfWKZRpST1MQ&V@2wq7&3`( zsXF=~U8f|dFKNE^hp6kTsVx9?fH(9UiJZXC9wCu4seLOVy{ZCX7Zgt>sA@K+Q{m^~ z0JvZ0H3{dC_9XZpY1Ov3W#gY$p}I*=yN)V95Nkxzny)V%_S)JwdO{Jz!z( z9FFwE*{iZu7$lwA6E1fcT3?YP)2fwDDh%pwv@0*rVesDrsG3b4V^NLiTdR1UuGpDd-`nM8Bf?cVjqeCQHI8(W=4v+9h0rL8 zYl_X77}O1=8(@bL7ZAO~_vB)QpIcC<&_Br#ww@|rbQeYxP^tzBU4QTY2}(Yqw%-xkre|KD>r%iyF8W0D)Q4VZ(5dQC!_Rx+rqLK>mQ27ioO z52T#e6koSJ$hZimG>$QK?)Z-Nb9`Jo9V^aCFzyG087>E3-tOZ;+dr9p#4_yM9bG*g zT^V!tbi@_W%i2ryhHDUi%O0H5irR^=3(&rqG2sN^>0}OY;!3q@;O<(mT#!nO(mL%Z zBM3XCt-@0J;!2%%sq)bnio4l5HKs8rkJ($;OE@Z5gfzb-YRPmiP29o0{bhLRI+$oS z%#X2(k6I~*X??2=4IHeTz!KAa3z{P>HV9oFATD7zcK?|%B(g92#53+on@MEV!e5VH zzc_M}6elR7_#N;`@f?Y%m7M7=oN4LKP+mZq5_&Q9KS)8mzZs2Y=QQDgbjS)}`J}8P zqntJnhv9AH5QaW9zXVI57Am`{$>7$^pBv>#GcoZ9za*>vwx@hYRTy*0>~MA}g&Xl8 z!=*40F_6fz4C)wO84e<1Ta}P;5#CQb#nKa!fx4mfv#l)3+$5@0{}pma^dBQ{0(2WI z%an`1&^(i&!AGLqGe#sxZFIJ;)9-=ucC@8PHO~VzoXyVq|1UqqUHrn$eks1>pDRJpW z8PTlP?oL9OOToE@ssFCgXPOuf$<6<{kQNr-Qr=sbTG6fC!fm_!9GddByhbzW$Dh=r z8YvG(%_me2=jsndOKZsOXC?^D;*`o@1ND5qLWJ+=uwrSqusC^K64*%g$klUJSPXD_ zwIhZU++R7*32Rbzh!M;6!ffx~_i5k~zSD7cUnzr8Cc`YZ^5tNY3awX-4Y$yEoY11h z9E710uo;d{@1m}7gUtAC8X`?sPN;?`qiZN7tV(SoO_5U-!L-da^3-1z9ZFYf>ah1C z&OwFj=At;-LtDy5YB{MPT?FzwKlxmqh-Q0B)siQU&bp(n8w6yQif^9(Id5IA5Lef~=)^XCVW$_FgxH(mFkxZr~8;jn(a&><^!6 zTPM?(3++wdHbZX2ut)9F^ik+$_d8dgM6Kh&Y88I^#{`syrF!p&zB)2XVmZbI*jZx! z{7V?!m!n^YXmBAW z55Ifc$2c_fb?BZLY%4DYi858QgO)sljFf$}iU?3*YJ@M+-tn@(C1m#m{=wygdCRn; zu(WoR9;h6nhgYn z+c8)FSnbhPp-BO+?cJG9zBv;fl8Q0z{aM;eJha({@PG{P zMY>z8{IsW}KfT>HZg^fI^OybDRtVQ)RS)L?w$Qr81y!j{M#(3!`VHK*BV>Q78tu$8 z;m30A=cele+`npRt$a9ue|SS8erwId{|PwmR;C93P1L_z8vi8f!ITpN**RyN$DIf`oWOt2{Ia z#2xP$^QI^m^)@!?ugE7}_Q>I09RqmYxO&W)U}E>Zf(PnT3Zi!IKWyLqc@LXsu)r#k zurvIohafc5WXyJcS+N9**=L}oKnnTU7Du(Q5e~w$Nk<`pSH^zKH3}*GFvpqzTg09K za|vT|JAgNpNF|)&BDV<4P!utdZ?Sj$lkAd^{fo}x{H!dD6ren3b~Wod2=f_x;=v~OT{ttCtwP$bVj8QH3jv7aGE%@-RW zWlS8O5#b|0i&)!rAq6b!w$|#g(&>fO*%8kiVY8=eH?jTU*2N!)W!Lam^ zF25X=Sk8}6b8HEY$DpX6=n~=hfn$1R;&obHuz|M%GU;V{8tB^q%PtN(Y@Nd2(>4 zaz_Nztu)dd%KH~#!5~_9#_95c!g5C;@KwVsrH; zvhLP&X_HIFWL@3S@^SR)JM%ZDUD!UjdUl|tL-w_z=I&yO;&$$*T)#1mT+NCFmZ{FQ zV%L8N zuMwERJb}_5v$~I9pdW%ZSqpBk#Ne}%PP5h(JPKBo!oHh?9fRL3)cI$#z6%Ke;TSSO zvVz8<4#P7nT2g+!c3($JRM0Af74A2ma!-qm<1W(6V{*%!Beb0)-BS5*g_ z-q$moR%vd}z-R5qt%Z|5{ytGo5^Us*A`ERDtEZU!Oj`~reLPrYGMQ7DdYj}|M0rkw z4~O&f5a0%g0Ggs^-oCWm=!EowdgcfYfgc|Zziaow z@BB<;-vSS}H$ErFN%q1{vPLrs8{BEk4i6D7$`;H%smMEA!qNFIi~jF8tFJx2Ua64J zj{4Gihia}h0BWO;$QjIas@vXb6EgN^!>IBc`QsBniwh0!BdMF@Da^T5qx`f@5o3XW zKs^=5z4q+SyrTL(9vzQeD0CYwCaVbp<9C#zO))NRVu&D*mu>M=B6BZ1F!m-LzUX)oy$f#2#-N~?$% zZe-(gyKuTg@UKC7V*u*q{u|CezRB``6#Bm5Y+z(>YVcoWXsYV=f0dyIPSys>DdCZ^ z>6Td|YjPqYbf93#+-MzRB`s+ynU~F~F<*Wb6n3C=`G;hmPTL+H+b^}Spjx$Pc~kHJ zDa5!fJbXyT-~9NwxqXJdPW(Nb9o+cw!ga9e$AftvEV=nJD*|##Sh-HB$i6*zPuowB zTve>z`lY78=4#GNp!zAw*qr-oV2t$e>A{D&%XqjINnW6ojw3Ni#G*mg;$LVoVbDzp zY*R^8$aR02{2(HzSZK#Gmuxnk4!9)8CV7wNy&J;)xXPwjB4{mOD_l9va7<+CW~-gj zt0`T4!P|#UQCE;y)DO)gB4w2}ZLfM8LqoN_a-SwJscC1o`f#^!@lPGXh?Eu86~W zXi(HP!`_;reNQ!}Xi+7CB?YSPju{pk0cHt4yCRl#%-vAuA>7s0b`vxMvi>g9++8GO zPG7F)Ve`uu+_v9y`ow+NVB@BFqxEs)*o&Gbag97{oM!{9K81QwffNV)=IZ+PtrJKc zm=-ic0ffe4q~a%P*T;%Y#mP-W(R;k6J54BUCJ*zu|FV!?H74oPkM0{g4nSS=Sn(!G zn;gqfgZtaclQ3R#e5=}rzVhT~9IXA`%5sJU@QM-Aq6uMONXRy8fcftN*9zWTiO-_} zf34#!Zx_{W$Q_*4G~k(HS9IxfTS+FngpvmwgVBI*o^J z=>dm?1;c2UaDyLDu~kr4HLknKYqR?G!5D~dDBrZ0`;wBxJF1muLlN2BW_`WdRju&O zXSej?a9`Tz>)$R3-FmiYtbNNiRZ#yIi?jc3;uu@mTbX(?D2oY+%8Ak2nWn0*+v9Q| z`5kM(?fsOlrN1qyq!C1nTO@>`#VnsM6vMetp7=gLMc+Zx(W(j{Qg{h0G{|5$O_0&E zoe;MXWOsKznw-qcBq8XdsrZfB5X$7SQ5f=VS?PAe<_W3O*wisXNLUpnk z$oDoVVqb87Ag8{=f=?FjnI9I(T;-=ejV27VB|H^tmfG!La30KQE>FSO(}`LlG4<-G ztpo|X^jm-j>tN~q#Z)ZZw@KgrTb2%{V#(H~OxV@^d3uJ@j^Y^I)PhQFLAv(C1QPNl z`q0JHwvh5gq!)nj1MKw1gqCvq&1)a8_3nT|voFaBUoD?N=$ z2R{$5Cwk!X@6{y>!aPUD4y>{8>UU^v5b*{XZT{qZ2qjDDGhuWuAsUv1y47|Hq0;<` zPAMu>(d+;DN5YjVSd(^U-Y_8=ofZ&l0b2s>S0zXtG*SBEl1x`TIJp?knIUab4t9+b zIpKB0Iu&E>VTrbr1UDS@`a1+xPhA^1-Lb78Ga_$(-`nYGMXzmr)0gtZf*K0Gqd2)u zy{gblJ|w7+($dbVWC89mDZ}tYjYb<8@s@i@lnApF1IuwQYQzJ$F31O942pZ)HH>dE zeLi64O&($C%`W$_nn$-F)08h{6|%+G=Q-cfc7gWW#6}9-YNQGL>4PvCsY|CkFQb?* ze`hQ22V+>B!6&(+KzQJ)tu{?FZ^ z%$FDN%xh5GT!CGa!K~&UN&Av=M?clZmsM(u8>0xPb<-_RQX&K!XWnc}qVfeNLF`!* z0$x1n2S4^$kr1T3#N9zV=yvS4mkVf!($$x*zN*x9|ZHCTD&{9m16W}SXuF%%@!a@UbCmR|aT#~l_ckXARavs)o8 z^i0OkIg4bQuEHjX*lXfGTZx3M7EoqmG_WjkC|IJUkXA{Pvx!kDQgek3ZSy-eM1Cvd zKHhIcVd5`=>?D4w-ma?^1Ord3*0^bF8Z+g~=lRa7!Bt!^BM69fH}0OElG>emt(2J` zlyx6RuW!(7lzK@3w>O5*_~w%UuuI!k2ItDKNwTOi>P-)T&_>4%^Oq!jZjI?%&GI(i zhmWTcf#RYWfE|H0x@GhtY`X|VXQ&*K7-pjB@p76-aA)o-+UOaA@AUoO`W4TzzaxHs zS3Nl2rSpI6SNyL4VeqX%S-Bb*d{@wZaYEmh)v&^^-_QxXsT84$WM+i$`w3a8i6gLN z1nd|&GHK4XDHfHy8UF72d3yPc%04$Xc5r2+s6<%Y5(mDIQpyn0xUU#nz;`Wo(VFRh~F|0j&!~ z<>OAUv$${!>`PF!R_8ly(~h3QeuLqMAJ*PT#lo`C_O#i6>{9m8ozkdG5l5UHpR{T0U()VR{!fZ?*8(696r~W_~ z+b(!(!+1juOBk$hL79}AYFaAN7JZJ}Zzl$pqi9G&8rtRr|lQ zq`8H824C;zaGRekzqR!W4>v#FI^QM`t2);WA7J>pdS89~bB2ozO?#vz`+uEFbc_Quuud)77 zAT#U7U_l|v7$o5EO-2sh*2&NgZlq}eApFUHWlgq01V~*-F-(cir#G95rSYEkMxbOP zSvS{gnGaKY^`s{=j7D1JuwH3K^@H-Oz9fjittZ+(Vl4V!u8c-e|3c?A_K{7FI7IS& zx@695CK{LV&>4?>29(pH)hhSAtuR)m+KfF-)w({y*wl2iN>&F5W*s2vD)(`(l;@3XsAD?ZcowMZ_5_D2SuD=beVkdWSETO}V6&(-S znK{s}CwbSNmMbjpgG3SgwjM=0&R{B@ISc{d^(`Y*vrp?ohvm>8c=G7_&{Mk^r` zV-n1Yh6zjzu3QbrZbJ7`H<&JpYz>3|iBw2fc?R5;Hwq1BC_a%lhmAxc zF{i6OuScIJK)i#@_>Ei6DDMz8A*nLv*>?6JoNb{H{^BFn1};oo*jcZTS#tD8_tCM8^!-eVw|3XOg*YE<8Auz^UO_S2*d|L}QXM z&S7nW`h(hrO;ni1K^#~au`$1e04dCj!W`NirBCouI=gx}L7%WeFyKS$2hj~UqZG99 zgf=5eupYF~gq$B#XAPpx4)we_ekwY02mKfph~eoaU@yGiBP6d$#19FQfNpI8g=k`? z#kYtEI>-n;ayW=KvqMPm5AEOCG)rmur0FYKaS9+GVc=qwhg$kQa#m}-Auk_cXuRba z5-u(!XCWA}JV67L?^ER(bd>vIW9TzV-1_66H)6E%U>^@O+zT<$#3|p#^!FqBTrgnX zL{_HQzasjZJEsE;H)GiOgCf~yG8FRqL}(SgO!Ar`3;SkR)>|83>2}Vup0Ixkf=!CZ z(1noUgvQ<#nWXiHHjc#LG}5}2iRp^TA`1>@=Ef9c$CUZcDz%Rm{0;kr=7^Zk&Sp(L zO(wTyuO_uH)m*V@3+hLEV2Gq*{lW*XrYo`@Pg}mTJEv73U7iHXO4C&@m&}2{5Kd7f zAAT+KyAEu2)MLZ5x#SGc2rr`Q%+Y2K%;M1=5s`|rgKXOuvk#&sTy^_jr z;^X7>wd)SqohC+sFJ)wR_5InbxexcrWbIh#k3)w<8;T1sD;6ZIq+I z>SYPxHA(6JSP>}%T&kkL<{rQ5K3yW@uB8QKTDl`~-oBgSPPcm{;jAlrACb}23r`N~ z6K(U`wcxL6d*GJZg~pw`8mNSbonH#$Tt3vvx4ABnOAiI&AQRQV^bi+K`&?r$l_JI~ zR#Bw!NRlc*$w_aqXw#f8<31a5%^c%PxW43cmus#|M*Wd;Qp$?dn2`I!u*Y}-s)eU6 z;!_;Pa$`{*c=k99sq12NoAkFVQZ;&iA+k;Z*n$?i9~1Xz)f~Y0Xk1RY@{R}@5Ao%f ze9lgrvzctAvhrZHEF@(R>4FU3Avp!zi2C6AJ`7tAbEV-03*F1g1mMQ!sI$``9d?`c zA=rE+*I}?uhtAG5#C&a-+gMKZFjXccrAIZ+EpDZ_6>S~9Eg~E~qtolAZTXSb1xb+F zcJ+qvJA=!6vkZ}e;ny*&tJAILkn5k)Eojf`$s&`JzmmuRiAk`lzt;YnGu+QDIC}!f zwM*gZy8bWFh=S+gtOz`w=Sx2s`Yyk8=iS}ZS zc6m!4Zc@JFMlvlz?HLRdoUYZ>pk2O-&n@^>*}+X@el6o`g;`6Q1}- z>q>dliB-4GIK6ebc_lWi2r#-BszBI4eVgtphimY@wi8Y%o~9h-?t9jKia_`J7w`1}6yzQxFSh$E zrrZB?M1R>ti(ZoDRnMV$N#G*f1dq=%H~m1A%JnYoHu>QyI0Djne#756fq1jcisH*c^S%CJJp; zF(qwmY6wplrmU}pG*TQ)__Yaz_gb7)2! z#;4zkEbl;gme3VhPY91(xJj5Z0PUgvX#;4-3qJ5z1PEPTzh9S2}d(KAUz+AE}8Wlt1>np91toYEU_3ZCc^ZZcD4ygpehiOE6D!!qtW{%wn>&vqHGoij`46>K)4lW0^qFhcZ zi`pjo!Tb{w0zp*BOR+OLfkDF%_Kq{_NpMG%C>yRTnHA6zk4#7c!>TVcr5Ha%lJepp zrL6$kxv~6%fcy5>@e*0O4u@)5`vJI=J;)bgz+=(cixC$6Q<@1d>lsV4L)T|)RU;!h zgl1)KXI&4kFLT0zA})I*56Z+kd)mjQ=7p7(pCZ0`rkD+Yi7uK$(XHCyXRtea47?c* zvV%si)U9X=My)|>^t4RQdY(g3uZv*gic{2Zbg7qqQ)@^`1p-2%sB~jZh6QMOxG`Ee zyy*$rhs{?X69Y96%j3X6op9YMmN%*^@%dnzvEs!$8zI5^oWsn-Om%;f(_hJ{$~=q1 z`c@EGf&2*IIu1!jbbD|q1+y7gPG`d$H|}_%^9#nWh$;Vf9&v--i=15pN;=zydc6=n zsCC!54P$Rf87-%8YN>4u&_Qi-9FGn^%FAUe^ibGd-=3$iy^m{D8pIoD%l zBT+nvYACGrXC#!uWxQSkkS#)7L2%(^;ZHc*Yv8onM7vx&_HE|5Bgb_!;AU-kj&=5F z8d%MVM!lG#3Z>Yneh8Q)g(5Y-7jsSAZVA9(gD=I>QoWwd;3O!U%sl>bj>_4Vnnlh6 zjCZH*CXKFpe7%N%pR5HJOUGhew+*oh?5man54(t;5^8g*DvRJIU-oN=Tv_>kMH58e z0%46f)7)xdd8hri?Tz2-={49068s1YAO-@}J_si0RGeR{!8HdSl^ct3rcs?6O~XIhwFzcD>hVo>WxU{z?h(8^!4 zpUwr8%=6aL19hjG9E_5ybVXig5kxWksaz9|9BjnR&nMD`gC(CiX`qZ|=#pReWS6uQ zmDfsX*5}MGYUk?}7Qx!mRE zlRp&aom<#9bg-+q{bJI69PB^AHp9%v!p~YHbyy&{<;LOdA9J?KFn?XYwI)I5I&5ZD zSYu{$X)Hl38~8XOm9%DCcFwi?Td3oEJRf~@q+e-&1RTh3w%q&ZS9Cr(UQ2K4=E?CC z+H3A!;f9=_Xl$F~&RjiR?*mxGbwl%j@-nx?@_;isui|$vzwq?h!!%q_rV`a%zG5+H z7K8ln+{SzVZD0)#W<0w0yPBu}7B>EmcP0N@`LuR%u($o67vsa?`e6nb;li)|q8J%q z>_>;9cfN;dA~}_ig^E+?vQlgDttbFb(_R=ZQaf*V_lSfIQ_-q!&y152ND6zaSnkIu zWipbDgZwj zjPxzO+Rn3sT17?KYan6)(Zb%RGa;3@GJ4Puo1_Fr_q*@W%goPd`un-gv}RQw4pu`M zNXj-uhQQ086aX{G?a3Vl%d*WC+<&D!59@4)JKy3j`M1*?+kX-_{_8WC{9~WW;Gh5h zhrm(uzpdv8vww~U8QXf#@PjD^v7YXECXz>+&SEga1*K8yeBXHzqt^Fm`08fqk}Rf| zk6J=(Ae)YLc`+Z&PQ>jIP=7I}kxG~xeZKDbA|ie6rL)IeJ~-OF-5=b4EE^vD@O2i? z<=^^ra!RN8J}zXO*r4Bs#bj79(k$H<8F|WgF)i;-wO!y^f0-Zhi`8VOxVYP5?_zgUIM zqM7&iF$g>#H?*HSy?^hTr8$gXQUBg%sdtii-W>W&y^1wUhE}JBinAS?=8)a0@DRdF zcm1>H$(|{j6bw_KqFxeh0AEIJc6#t4echJq7>A0*J6XGM@5;T1FmD{uO1O_MMJ-N7 z3`6!i|Kcfyf}fXfV>=P3cT`WGnTZ4&C_FrmCKe$GZgqe z0VFgrG(G941ZYzl)F8k-G8=a`=HwyFY^O4#x+qZus5jvkGo#l4dZPY9BpHgKYJtA==K}-cS=z*P7A%RC>b!`B;mWzJ526x?_Xvth^ zo@NxG8TJVXJN6&$NdR*7=)lpK(n1ffL%^%|`ZGmJo2apl1@=X9FA&xGy(vxYFaHLx ze69X120asMOZsX?K(a^+JmbcFQVTE7>u|fukRoe?VZc(g?23)AFnX)VA-Zqg58Gs? zQlvhk#wo`RFeI(8aVeU-)rC+UOnW;+HO0a=&?7o?-7-$u*Zt$)YZ2=DxPmbRiMl2F zG&WaCtp&QKH;N*Vkl9pmL6`H}?VlfeGn-#iTpAv4+9lLWiIE3da$yk)Rjrqdbrbmo^oN_O7 z_E{C=0~dD40<&=sECThwfEXlw8No6?A=K883VU|T9q^T(L5dJoqBhwZU>n|W#8626 z)eV8Rz#!66)1!wI0sM z(y=fq)_59gy=LIVB>1|@(O4BWR2#h6JjnO>I)SGIZ4KZ13Jap11~56STO9~9dz+`Y z>4PlqvdD7 z=|q}9F@0_r)osLm&l4dZ4noLyMj&?tU3V*(e~HLw-4vdM9vnO2&5_|oX+(e5tI0?* zn;_g$`VdNb6RO}OJ&}l3wd&wTdMJB*9VeXua8zp%yc=cb)D+37OpglKDm|afN$qpQ zN>y!5j?N2IZ(W}!G3p#lbI>~$t85p}N6bd5wgC*bQQ>QsAvGBML- zdhRRXvqng>_m81yAHxtxNh`v}^=|hRrwQJ%6+KEYmKc!iQZ@x{cuyuS4vLf7t*P!S~6tG)dJFx+ak4YGV4|mIn}wkUS2*X za$bP6%kzfAE=o=~n>bO`X4MVkj}KoN{hpoZ2Ys{Ip(n*!QCcRT*yM4cMp2R*^inzc+}4mR`dyhfc)fWx68_AZ~HyJ{F*+#>_s!w`Nx*wXQ%%| zdPphh#?d9`^n>w-<6=jf3ONs7+wyKsi(oXCQphb+<&K8!qL)X`{@U~?BwiU6)3V>x zDHlV(mgtgCwm$zZCMjoKG@1@A96!(xneMihR*y~x z>epz-KA`#Ub-NgaTowwT4!8>i`ZfTfDmVHzVNU}rm$>k$8u(|j0IMhm7L1-kv8kK+5@C*7OW>*H5Z1$x6cAI0`AC*-QIHY4((uLCL&J>ES z8yP3tUgAW5rRh|IVTikHd!d=Y&(8S9j@_=d#z9N`TsUq^Kc3FjzOYUuW73er+`m=B z>h{9CCyehC)sfU$ws%z!+3rNCh766o)~jo`yym${oMXNefYs(1$`a6HgE@0+e-Skm zVF*~5D#1V(feY)=J$%7Tnv3wt`HC-yuKzMb?m9ud-9t;ulkpE+J>%IOnr#uw37iFK zsV@8huxZkNv*JDPC|<|;WP`}f(-CV$@j~J!nGt z|E5e<|71(w>d+#Njl+KXZD#=ZUtsNuDnM5!B@)=<)68tXNtg*ZQn!}z26&9HJJW$ak=ZgemC!G7-_hd*^= zSz~2YV`a-6`(uBEu48l!x<6jtK^3%!R$n+x!uDLIhsa(&UoT%K?m4(>NY5Vu)_O2v z#VBf(-Y!bD5o@(x4TMi)&t3V(Jh`L?bP8(1{)XVFs;R_8$t`xW3b@vC&++@H@^>q?G(|}P?U;(Cm``P+tV~ z#v8!K5DK>&|Jr!ZMk)?F^6?S&U4rUOY?xOhHm}!jHcm+`922VNgOuyVj)%R&q%9g5 z1y70E`tgVqRbf&MA6O0%R;{w|Bz*i{`pxGI9DuCz_xAE;x*07^N`uv6fsK71BMJ<> zk}mDl%MIHxG~RNx!=>@>Vka{C2K!oQln982f<7UWFIXQlcC_nM4J**>Rd;6`5FAWQ zq}xAWYwn50I1eQ65j>%x=1@$k=Im@N3&n-5q)DPscoU#5Dw7bS4QWb5wWR|-vlgs=$Jxya#mhBw%)uGr z{h22xUZ;7J%}geb2wB>f#8{fi(`P%`8ec1Gp&^9h908|CPMHW=RgJ&j`?5N;t=<_G zL1l0ejtTl)z|CM->l6vECN{m`V5cmZ3bgGopY5W%K1Hd?9qi<9TI8LT^?BXMZoPZ(3}z?i^C2^&pPFJ&Z&3V zLbMD9e)WxX5DRb<;g}Ta@~&SvxuRE~lHd9iuAnGU+ak~5+7-Fkoxqfr&58F@s?*Vg z2HsmE>i66n)}}J+Ud^;5Nc9wA#utx_BECPr=B1tu14G<^B`j05Dg_K_^%v}$hs1CM z&mCzsJsA9hj0-}?ORCpn!NVcmlt1JdAe+KQ*BuMs0ZNCS0hR~I$4f9+)IJzo#o!(R zE;=ih@}>np`2HBD8EMbqiLED)`}o5`HPUma6I>S6xNF1XfG|QBgUr!z%#w?$Sa<;U z>f;((!%<+unQG!Mm>@`%P9_Q~oeM89WEO=tT=&v zg!fmaqx-43VbI!AU{VI~ zUrvvf=C$(^-RoC$!v*qHKxJkdb?y5|?Yh0^Bk84{mzj8;a6u#jRPxp@A6+B*h~w8+ zV?hF=HN{IQ&g5m;RL5Iu;TfM5Ggq0aRRkp z;{-RIgWpX4;=!Y`u`s6DfRf>5a3OKMXU@&yftcL(!AwUaT{6;c3Au-(XWCs(n6Okfw9=ZE;$px_s^!`9c{Wgtb(BzHw5_|#LwEk&2r6=RF$zb zsK7AcQ+f>;%y67|9qjG9@=;s{WH><3grPT01Y%%o3_o~`03b-tg|>Q8tmVM(FKY&= z@8WX!?V5+2EnZ*iZ{b4saHe_lZk-n03{LTV-6&_R%(>mGvk8@3)TYnQ(V%f7rao@K zq9>8&G`rTYOuYrnKo-zDjh%sDWY6xv7_iPrmkyy=A;; zjO-Xmgn+L*D|oh5v=y-CA!G4(U6F3GdwCnBu9#6ZBS+{|8#IJ2ORsG8L8qj`=J0H< zAD4*7?U#0aWyXOHSftCo0N5X&CjNefUVyDbU#=i9Qp!CY+-x88O^-Fj7))-l+rh^! z=B!o{u3RoF^;Qnu-gH8nPrk3?yP+|VWQ5nHbM^6)$B0uqzjzij+Jx9DI&ZFO$-Vb# z-NDsnj%|#Q%fr)lgcM4tvi(uUXXHR*X&`vdd<^_Y1buaa!eT_-90fNY#hwRjn($dV zz(aj5QtpQOxFrm^|3J*)ebHj!II7)UTK|zS?(UxJw`Y%T`Pu}QEo{7NH~$<%jzO57 z%VWka^M?xoc<7R&TL4+5+`PuDUr_Utw4Jl-{Oh>x7MxHUKnd)2H-psgD=ST!olHq}jAU1eK zC@9lm&!V<4>*WXb*ThZaVL}Bj{zi@aeaNlnz5_fthHiHBi!*HNEW*mte%?E8%lNqF zc<>k#6Eoa*2^4d%Ehv;{P*H>xn>YkxH%j(bAaNA#T8HsjG@WHPAf%^({A*(*vYMKGs0z;)oX^{h_l|f1(0+2>vF<8V{2^B zdxQCrYTHT+G5qBr0{mU3YgM+LEDRXINkz0oR?~C$-PYxpGpwPVAM4!^s~`td(p}yM z0&cetqIkiI<%%&=lA~^2*}dgXVC-ns&b8+X#^!XC`SrVxcI$23J3RdbEta|DH%@O2 zO}Tw@A=O{Bptd1ltUSlhq8Gpq@bOrogwWOy7vU>#hDJYLI z31=8%YUO8U8rE1HUvD3CB?#U7afV{#WtPo#9Vm_umJZJg^@0dO0le;!w-HCx*t^k{OmdUW&!RGGiO2UslS{81j=Cz) zesb?pszNMAcJ2_eeFSZ!T{$s2i*W7t3wNVUhXejF`xsOvSi)ei79ZBkR)|7834zj9 z4}e-Gt&RmAc9s8Xy_$#19vHQD1!wh7pz8=5R^&x1E<$Qmaa=zc($b`b$j?D%SFLdQ z+kr87Md7rshbG3MAzp@(Ac@4x8-oLovNooLN&>lzF6rvGz@r#%%Wp{E+Qo%H$%L1^ zK+|Cb*?`{Wh?p%QLlO9U6ovr~DnpTg9KqMeyR_Q{LToU#B(0tXGH)C-6=CLkKeLyB`?%vS({J_vkPaHeUh+f%UC z@g68E`s-1L)v;JB_2DlnE!u_rOC-iuqBgrS;WXPCD(vsi=y06qZx4$*K6Pb*zd7F1 z(2B6}b>iqkH$DWu?AB>sU<4nUmO&>%sd1iiJ+G*ql{TK{(7@K zQfv^oa!W3G=oR|k|4<#5(D;<+)XaE8AAoFeFLRo$rK8uZD~)v`(e0y6f>Q|A54)gW z>sFi%IL!QKMZx6L0}u0gicYmO?^$3$mU@B04YsYY|7@E=Xzx|jW>jD)>_FMT?*j)g z8r5S(krCf;EaA1jaPUcs!2aWK1E5LL^BqM%N@ax8t4NTkD<({W*05*(Eo3w_88$Em^D!+;pa;;E%*st{_~*UH zhDkl^!=$lWBCF4-L(O&ru!>R3AVh=IwOdZ^`=-E?;J^&c7v>$M-95}^TI@7Z7ZywF zDSL;zl*=(M4i^BoK!QRtV|gBj5uDNsCHl;LGE#6aN}!BVw|#YZpgy~vtpV@Qy4-0N zV*JlX2eb%uJb+_mN0AV_C~K(gxJNshFdgwqhqux^DSM~>#8Yo&AW~4iqIUZ zgq3R2D3LogdVBc<-)Ao=$f$!c={6+@#=x_d5>-armAgeu5gfEAw$HfwY;C4X5f9|w z3)g|YDG)xH%~l6SrU>Ocm@s<=7XnMJ&-~yc+%$5B9S^gMV~rg&SS8c#6}41KtXMm0 zNn_;;O++wY?+nM~DQ!_)(i6rGS7uT_H6e=G!dGjY`Ba>WolgF4bSu?JTzuoTaeRa6 zr{5kg1WSn(bDt9>arVCv#Y<5xwA~CEy1l%HU=&C%UHAzblI>7M`ut&M zDR&#{OP~h=jd4WvUNlFnZu%ny>>$`eW^x)k9<&7Lu8@xAMvc$F&jZ3d?I4w5MW`SB za`pa}eR4~Q`w}Z+^i!@gfq(j~^V47EZCp{{_-BN1MKtA zib}hrX5l%`V@*qbP-vMM+Z5?FkuWyQY`8?|A%!O~=@^fZUKK&Ky1ftLg>;rFmfLk` zA}uW>evh#FEP}L0+jbH`Zd<&_qdpkH$FU+k2*37T`10jkxF9U7by9E{rM3Ie6gbke zRZlGr`knB>%`NP0kws{hsZ}qy?aFU$trPaBo2eDAyy6EhN&VMI9d@lk6zd;pgY=Ih z#ebtb{;!#qz2i?}vg7}7L@QSPU*&N{&p}n!coCxm!HYRA-=YDYAdo_G%T*K^tI4@x zO|eL5-i3eg+v{3#LkY!Wi$_3%#HF|Ay0dU*cb*NG7E=Zh!0eyOsQbkh%y~_zo(S94 z)!y0N-kBzEyL~W*-PPUwX5>r{yNDo>h|-*z{EIhq#jaLdFoq#ngIg8_z&Y%xU|JvN z8LzUuI1AOQ;)w8W*V+_K_)$V!IOvfAR1MpW9hyoDczJokwQL%|#cf@X3%VZiy_c6M z{4+)rj%=k3WfwiYX3{?+G3IB{C|F;e^kQ8#A;?{*$}qUU%lVq^)+-${`1$@T_2n=q znZPE2Nw>+~7bu5uR_{-vHOE51@LH5~RziQz8(nsZDAlByLGkRDx23uyhE#sA6B5d% zCXK)@0c2Ti7N#x+`&%*fpj0@z{TG0`%l^bHWG*KuT=vfJVPA4y;`owjulFYpR{z0C z=1PKs4<{btc@a|F=%bsE5VXX?)B9VwfCjiXtV0p%9YHfgg_l&svVRb&45KRZRt&HpBt22(FQwpIa-&&Sn|<5L*8_Zx zvL9R={2M0_J`vLf>veK>;NvDO)5($!z76JPkebIUL0eKmhwhVtX{JH0v;BZbS_EUl zs!9!g%UlrYcFf80u%9!0y+{i~duqfw6$Z|pKL0tsOP!2pYrJrt^jh6+G#q`rqaU6r z_w&aqvNF>+*YZ~6)$*bdk3Iw3r>IBuzLP=;xwpt0UIi^i%LdOaI^$jffK;y(%Te1s zZt2M%VZ7kYHRs+yp$xi>_!~SrOx|3ud-E6&k2{^2f-j64Xu3@`TP|7(0fjN9H^k0i z68QxOBoAC^uatCr)~%g7IZvVd{bxLw@GjEnE+;3PpNQT=^`5>6f_$&u`W3iU(XJ7R zG>7$urJpy*>^8v&qRlr$EK$B`0+inr7>IN4Wgakjr2}Scd7_f9x&GI|OxOJNJ;EjM zoa$thuQ304zbdnJEol2QmEZeO>i>UkTsIR#V*}@(KrH9~dpuvPtYa7WqrN^?XL~Dv zjvF#zlzGbA!$^^f#%eVNWWZo9hbSpkz<&FBqZbM-j)ETMIC;$+KD5M^@X3S2`;y_b ziL!Oz@}HRZ)YE%Kl|(K%*5lM(4s9;JjRG`thjCqqrTgmm!^j?~!z9=o==QhMLXA`X zOI`z6{}K@?-HZl9{fetLj-`rhk2CM84nyJ(l36kPRO3k54`pW1#w zs*FgadnubU+mmLPpQ!ZQP1K>PS8rJ$VVYHV(lF6oauHQn30<**@IY{xvFJ&%Xte-| zG>NE$3PRGRst3pe$XO@CCn~=jaROHEKx@Gr2HTn9$iCIU5sSB3kM4M&4IH4(++#nh z|JY{_=V8W8NdmzyQb(xVsJ2ih{31ZM!Qe(;Vv`z73nm^uf%ZXv>%Lomc;KPYvHD?RIqKQl*;)S_z19D_Gg;9S=0a>_2AuF*45j^PKji?U zO{jui2NouponJd5k-?D~a`}GSihmQ9ZdgEnjI>Gr=}Vq{+$8ksCJOkZ>$fa8fYsR61_m^q|%P;t)+D?83-|CHG2PVZ57;ZBT03xkQe<4usUqZ(Ne3}_q zSB)UBanv$oenARC!+U}8{XT`gLbDuk?st75108{yh^UqKGXJf-)|t$$^_=osw}-3t|E9H)9i$Ypw6L$;yp`!)SA?NnTXs0BI}&!e-h1tE_6j_X+C=O!WO+<77WDy7(aTmy{G}&c!4*nu;d*i@OioEdk6c*xfHO zMKllWy72cDYNlC2*ErUXZLXjucZia|80aBZ34vT)oL@YA!Ur|Q7XKcbeUex$# zLl|r!{f!xy5EiKM4sFQ&fYMip!)sJPU<$~r)m~`*OqA@q{qy0xAn>IM3ujxi$dg01 zE6$ZMVa)t-Qh|CDO6Zq8T@v51mV!(FB#uZ^8BGb%@9@XvlmYW-@wP5}P$9PQf8|0S?%VO3K8LLVr3{mzYv2;$5$i5P(%7uUEimVQ9TLQeg!A zo7tQUa+O&Eu&P@xMN3&{?Vx;|S_VtW@R>&DTU@}EdS%!82zxx{NjwK4WIw}t%iXTY zUA+|p;;Ih;A^oa+mM`3~O#5S)ee`#AUZ7Ynv$)^f{+|-R*P7Kvt8DMnuO80W*z6a> zCI)A`-&zN9)E2#u`RBf`o*z88dawn}HzNF6W7V4O!FPe}HBVL**qH%6Bz6muS-t8W zh7q}Ql}EqJq--56-^y#zE0`0bCg^3YfgASHUrXC0tKJ%J*S%JrJ3)vXCU%kzW~5)Ck-M*Sts98<*f>xxtR~# z#4djOD;Z?(nx52~ER7D_H|PkMspGPGVwKVJ4tO?!TWEZZM;SC?%xi$8p*_XdJFW?xRLedz(BYlg6XrJgQk_lQ(w~|My4p>g)o;i9H(bV`b1>rEZi!O2f98w=feWeY*J~{3 zUP+`}2)VudEH2WcxC{N38?bS(9+SyG$xL=!lY}$CjV&J+U4j)kW6zK~ZE}@WbLwtU z2GUK{qdYBF>KAlM$mM(O_^YPkk-$t;T6_!Tgmf#i^WRkn_QiCXc_=9zkJYiOBxjv? zq?tHIDXaoH3weW6L(5Y!`bi8~VU|~mQ#H#1)Djt{O5u<<6Oi+l@4;VtdV7TU< zh;{6dZLy_x?`YB4>B@)JWl_JZvT(WTkAHK;lX5xuQ@SRwsBh24Zzyl+n2?jABHz39 zpsTlkcC9i7PCeS<46Q2Vt-l4a)9u65u24$(9#Sj5=AqlUp~oh`QrPgEGn@I|=6mYK z@Y;A!+99(HsQx4zYo$HGZb^31OAJ)Ri@*INt4(!XZJfiu4**N4t*1S3<x*G7T8=^?kh0wQ+siI2<~}=KXg3e4Kb(yF5&#HMv&Zz1jHZG9UL6=UcX~T}<3?94 zgiu9G*QTK^pxl*PzeWFa+Vt2r;iWQo1 z@h>g(_ZMwfY@c|zUn+WWm{T4QQU+(@5?xUhCsDh0gBXDz(76npJ_&}v=*MPM4hM1*O zw$NI8@4f1!rs!SV?iheGbj}<+Fyn;4V*A;JtD5_%Ey|4xiA$GM3F=D=^pkL!bFd@gfk?>GFGij@_QBHCw^lUOAtj zEy_%Wnpo&ud-z;>KabU`=_0_g&+5t0oOD@xaOO#u4gAX}GmeA~OCpRwJ|lA0b^(z9 zqdCQ`-4!-JKQwH~UVrq!mo2Z=Qx(rJ4g~KrAAfzAMXvCT@!+!7hs@5^H_&EYXuFoh9zH@CScJe0 z6p#iRX6XQ>3aQUG z1A?ZRaxRvoCX+7*yu2SSvFF8nFhjJt?1yAa&0m+T9^tqXV&kqXZz&;O!%(b)>qC2C zOkZ@t39Gb25t0f*dVqY-SNH4CL+%5aHRqS>dx?jP&qt!CN(%&~N(c{=O;ryeQ3JZW=y~s+2KTH$r7@|;AvPbpo|;6wX@{w*bUS~`@sfjyiQEc?7>R%aF+>#>%{n{o$Mn1KMkPiC71=Il2`-rqP;r@^iYmHv*TcVcga@k zF-PKG0nM>Zwww^=`1^pxSJhxWl>J3~!8o$EQs_gCL&Tp`@lGgWcRaVZV7?(jqhp|7 zXIfD`brAA2__tF^HV5nwS~;p(*~@1(d#R+LFdDgRXE?eX=rmO6WuG)=&~@bfM7U?u zy;McdFP^R1SD)S)PfR}Tb+*_0j>Yp50oz3bQSS!#c-zm6Uv0tpkPuB=|d|h8u;ez)>bhN+W7)bTYRKdps_;JRN@Qlq|xrJK-Rb8Mv zAX<$VV}NI<=U#_f1InP`pTeKIR_+}80LXJ}IQKLq3pMtkB(Vi2oFIZ*`_P-^9Gi0` z$teJ&R(%_y3}P#qq^1+MgZ&BmQ~F~>MdWvV^T26^#713pWp9%+U7-W7Bs|&Zp|c5I zl6}Ld#b1``UqqeN2zxS+<9Ho#_V;B$E7&7&f2Y&KsjW!$M z&q%=6H?Yfh!2sE6niV3iD~5wg&gw2%!M~PLo){V;G%HfYke5*O!411fT8`#xTad{o z=De)nu&>cKudjOAu#!?+eI4Y?EMo$`c&@}oy(T@JrsJ_LU*;-n8COqnFZ#7Cj}X#N!GtA0E-3+I9re1xB$ zZ%j&EM#U?SM3m`~0bKUoi`%(DTB9vp&lHpg_lU>bQwG+TUTd@e7q-2WEuAMMf3Yfl zt5y;Z8IG`v#(pBSoZ|v~iTbdgy&DyRi&bl^qpky@UURzUH*2J_LGiqfIfoX)&|%%& zmgt{vd3VGnO1Gv=7a7cAFSC9$U(_cu=CPdLH~@k0o_yOwS*y@Hjz&yjXBirJumyKW zFh}^OP?TX1!RLacr*hkh{Q!e@;zWEq{p!+J@@PWc@RfrKBl(TW`%~quh`lAnUvHR` zXE;4plfY?!%&3ec@#!!GI8;WRuDC9D=2@#-AmMwPjalBt3>+*T0u;PmDetVz*7d(i zElxvE;P1FG69TBM9hW)yrTVmLOF)-(S|6ad*-KJka%dO?SnA76T{#umLb8|kOv*}J z#J8^b^>gq-$Hupo<-Z@g10dz{sMpEiP6qb;Ew_|X2jOnLwmZoVhX#+p29f9Z_UWMFdJ^4i%n+U+c46pBfKIjTW~f0_||?FwO?>3 zmv?V6jYmN5&mn+*G>2U-Gg;$b6>!!(h$Vh2V6|e)+|Chz6$C6Tg{~Md$q9kr$02ww$vtesg}wY%JM;7xR9#79kVtpT6lSSK9B#Dei4yX+)Y~ICafAaRpD96 zrSx?8_a2JJ5iRkwz(SQzlGBew=(YaE}jd<(|kC?>);xvBde~rSPw76<&;tR zzb36gNs9}=%t9$T_WJ7wVhjzYGZb$14d!wkTqXG`=7K2tL`x2u#etej(fw0Q#Tb@U z7g#x9x=ux1-<;!%M7W#xPyQQF$e@e}4d^+$RtBjWxb;}a+Q`6B81zaD;JlOu>%b?C z#lZAIFeW6%viWFcYGgFTkD=8h@G|hrUF}IOvdsFXKy0S-WrYy zZ$)<=9@}Wg{ zvDS5ZJGon>k@9G0nNT_h%ntBq;zF^*W3bctZp`#{17`_h2I@Z$CXO$fqg_j$!7=hi z$$wGn14zfO=f%w<+_#Le`?9}RGe5PSwC5>nkA>b9DsUS1zwhn&T+Sh-s`3K+r+?! zFNx7gKC_z7d9nJ?(KBK)u4H3n79aGTv#&z>F9V1oOSpQJ?(4Os&|uvsU?Jz?1p7dm z+*Ou-{IvR`~})M)-4U+4ZyN|C?7|Ms~KQ7XKpz%Vt*2E>H*w zV)y+k0^SoyD`mI@GqEH;a16(o-rPvkS=7X4WIzPd+Hx=U`pt#ukt5w>+t^gIO;1e= zoG=%jBNgLn+lA29;YFJNY)>F^n9h+hDFm$9p6|)j6-b8I%3wiu6he(i1yOJ$$H(sZ z-dPzoIIm24NxyiaV?6&K3q6=Rh~OCkAVmzkohiAVX{waql#7=Z5ce9L9M_&sm*p{H ziHlLuo3|HYK2Iw(jTqz9*zH6YEx`c)>JOencsq;#%A=3;YIJ=Mfy2UIoPRv?;t%nK zT_qETXH4Nc#w!VmFZO{l8}0`jTk^Hs8pfI_U+xBO!x48m=6`E2IRGmxR3qI`BjbT$ zcSY4k(qa*hOGR+b|CKl>@%sSrKi_m?y*}a8kJsDU5BEX%zxk$(O-v1b4E6PlO^ob* zwAyw#2S9GEfbMpE1!8@lAsz2XrG<#4_!kG$4#Tqfg?13Lv7dV@bh9 z4`m%gE}*cMv-xAliRNtz>hw-uzKw2u-j~v9`t@hEDsM)lVhy;JdPyZer#Nk{V%A}s zEasga^2AGDeDQDdqt8`Ajd?)`9}45kb5_LF1js%bYJ7pSN5SFJ$HF5;OiH9nQx~bd z-XAf!MZY{N0J$djDy^D%QW}SOPHTAPgZ*jRBc-j42ods(3d{K8p|~2>kDwY&g?hzL zYl`vnz4qq*M2XJ@=UGNrLzypK_U!tuQ#^-emN!_9fGfTjiLLpFo2fe}LzEXaojTiU zoU0_R9t0g&m7F&(1v+Enl~qQO3JHaaCfXuv7JntDMyW5 z%lOZqQD&KWiVV#ZFA5xjt6VJ$RdHQG3E1R@p-`IR23T-^5~S&JuUbVtyrif9rJ_;g zV_IZg4eIb~wM2#%`GK89L&a3byZ5Q($#W%2G2tXnx6OT#4DLfroodUp`I9eF}=U(a7kh9z1Lh3$OpX|UKQ)m*y{9z4&SY)9=+$k=uU4aA;&!QaK z)5?HQw=)HPJ>&a^p^%fyd1(+@JN)YL$nwBT@CC&*aoMx})(FZ4yScjTd?w!X1=KDd z+dDkJE^Hm&zdvr-9CQavl`aXItk;xO@1&sni=H9zA+E!2An5;9fgq3r3eYLlT$|bW91hp915!uEzHEc% zmR6&+`2_Cc>|h;aS6i|DC!oJ?XVpz2rE|A%u9(;`nijn=_4>xq)6uD`;~Qf)W5=Ku z8`M6&kVXcLULbb9A%dhjeb#2@#otlpk+_KQ5^>0W$hH+Hn!I49k6^NXz&0(Dpkzx~ z7PT^dh#$+4!V|VLg@E8d)3eVz*fHLw)zje-&eL0(HOxp&f7nP5a3C?u z2Wul!QyQWxp6#&BA7N@%R96@2azeoBriIajTlDBcv~+;=k%7gUR*7Fa9zd`zy4dW#6{T6-GJ&RG8+@pIiGAe7LA)Qx>M(Wsz zQY2oNqT#}}Stb*Y9&biGGiP>eot6;oqt+jXD)#IGUCrg-x^A|9#_HJTS>8dMOtaIG z)On%GNgQwp%Zh2^d<$XGWeGDR9h1ai_R{D}a^M(<4^A=^n1zRVH^`)-4bd<}9@kuh z#sIA2DH12_|JNA1@UMl^F=c8qDVENvHNqk9+#5EtSNK@iBO{ZQNvnd05KG%e$vo!# zKnB@i;kn~jp_J4(u!fUQrn$a|6tDkI$}aC~L#@aSK1QPqohl0j23UKiC}MP(i9U@k z0roik85P)5xCxV^MpComS;Xe-6?w^y+vsnHi)Z|X^anoSrD9saD})wj4#u6!2!S;H zG(;t`#ptRE&wj&S>xWIw-C|x5s4Rv!zunmcg3}jSf#F+T(Z^x+h@ zEatRlnVk~z)OxO*g9zmpEqalLbd^L$>#cu-Db~&wS9yGX8KL-u&=FHYR`HIvDeO4X z(^mCA=;^a%tFFw<94$J`sy}{b%$R{v&mby(ba=<;U~|f|TdF_O!t|BlZUYYD)j$Km z*tC|ez`kKNuT`C_e22Ai7pE4~JvC+VqFtuo7UlhQME1c{4NoPiN+AD1?v%F4@K@w0 z3M0I&njw3-QTkY8_8?f}0=i`8xDW!vpyyPQWa@513tDB!C?T4u6eOaKA*rSJ)q6!8 zy131TCDsY@U1Hw-InSp|ecm1DM2#=uVpyF@JjuflR;Pz`O{5^ro2tvHXL#sK$d}}e zMRQ#218>;aEvzSihHu>xgHl`KPSK6VV>uXiE&g7d543ls!{83gQK95kVTjTGf*evc z>6Nt{{nM%iZX%z}-$Xiw8)5$jXTgy4K;rbe)9fK2 z8AdSdqWQNivVoh&DWw%RU=Om%*sud4;@_#4TYFMIi-B@6aZ|hfHy+gP#c%WLwJqO1 zi>(`wlI2vv+}{UXa?-Pkvn!N`Ivx(SC8T zFAlNS`i}g)0nh)1ElRjH*%AMPmI{6r&wmp&nb_F>gi8GH8M~f|qodvbaJeC$_@BOh z{{P;6hyLiIWdzR+WpX11Da?h{7`=}N$gQO*)Izd+|2qTBC zzpqg~O6;RtT+9ddtD%W?Hv~g$lkzC&f8o#pe_dIlUcmk*JjBQ*RFC-K{j~i^v{?U} z4{B;*Vr{HvU}R+C2c&Gc~X&`D+i-2DobXO`oq(W-It?>uF)k(m`L5F7~UN2s6#Vg zZBdCnPba%f4zAYN$nZsHOd6b^X+X@>xfn&I4D+elr8sl?j6rLOyyqp!$fjz}A#*t2 z)QWVX`mk%ha&mILx=V8jU>m3}{&&J~d9j#{HQHX@8MTvECd&Cmz|;3W@=uv{2<8XB zALUxke4-IR>>0Zf?GGD2{HQ2xBE!SRAGw&%ke!ntg|tO-9? z6wozE6|i%$6K-@jP{F;f%zRI5HPF(5Eqzp5rEPQ;3#~hv6%#d&kEb%2HZH11SW!{5 zP@yEba22Jz3NvmIHJzDX!$;gS9)ZRI0eeJm(?5mMA0pS)L*ZG^>O-H;i4P^)taq9c z6k@$0slD8edkU2(2J-BEpkJycaCL4?c4c$!rfGjoVjUm&EP2$v)v5G8SqkkqPDkdFY2^flICcMbGvD=(EnC~EdGL2+W z@OMF9CQZ>HhCl zx8Jn;Mm%4XA~6B-@A#Zo!vs8*3mSRhZ~C1C&|_bh;rgF5Hr-u9^S9+EHjpDjt>6;b z8;U8p<7K!LSlgPi@BFIZzK%Oyj;^hJg$8)P3&sL#|8r{(pwIDY{vg>l_5VII$imjx z#QlFoO<(eTI&E+?^uE^$C3`R`NmsgVog071*Y2-MSLE2d8ox4RM@dM;$4#+{(SV^# zq`vw2b;8Dh3C$)5&TnUO%TZvA4S~FOYtTIg;+I6KV(91`SK&LaKJ)H;j~SKv;o`v~ ztp2li=D@>`f*f49T84G?b$#%C=Ynx3d3ho_kRq5O$+>LgJeJt-+W1r7i{@=bo546w zXSQdAcj#T)HQ8tqXE)X0JdNS&i!Y&ELXh2G>>XE&q3Ec(>4!@0+kykf{}e;XcYn`z zzv@8d7z>Z-kwoEpQGe@*>=dg*X5UR-Pd4#ts7e%`@X_2Mc&zT5Q!Is9uk%mOiulu# zoP8B=;U~>H*L}D!JLA)TZ=3WEV0LDDYK-^ux|%WIHuFu%EoNf<1HO;yqP4y59r(es z@5>dw&=*UYkUQjUQ0f3n85l=XV8CLN9`Shn*=~KsHX9*s^;&sa#0GWJ&l~I@t1^nq2~&YSkC3!Qj4p z+5Up6rq()~Wc-+k}50R}rD2GcNCZh>u?aK;1c^~Ue2 zu)}a-t@uyP}7q7AcKwI;9KZOq0H^=g}x1cqP?#_gp1VqM5n} z(Z6Ukw$RU^|06j2`wb0=&?ICMwDXGj?5nH4)o|8cI#Xwy%y2Q!TA%GY@Tok=izO1~W!2qv!$sIm!Tta=no(e#zSY5X`7 z^+PD|BuH5n#c%rhAF@v%nk?HGC2nKB!1z>RkVX+5P;7e8ro~#Gp8S|P$Ys?6+{|te zY}d55Q`B+6i$j<+0k4Cn3xx}uG7Y0{T`Q7{5zL%YGFK>k*r4;oU37A*(6NS2x;}#) z@J-FebU?{7$$wXs|oU6njyX+o11X)xA7Buw$ zOA;n3EwJ~)Y7%4`DYaTXQ^*jxg>F=!$+U7?Z8WluP3Y~7dD{%cXe~F?0`#=}$5Ill zNvc&C^GSoZ{1qcGCswKNi}E28Scnx_DXpVQ(*{+qZEKH}3*R!wh3&GEHc854DhXsl znjpGG7oF)s>Yy>vt6}41jpH^tB=dSxc)8B3X##~J)odG`a3C4(Nwi6 zgwkT1E0<)VjgmApZA?BPAQ{`%X>rJFd4yGGYBkhJ$9WS3$*-oy#(TkP1eVX`SVtIq z>aA?p^gEzxPQEH<>I`<;_n!V=L)RG@o>lAvG9^tMC`7xl6I8SfbRy9i#VEMYrP#HJ zeT8#3aD01;-~6~;CZEqi_qW`U12dN4hDz;SYRBPaC4Hhdio2YBVm{kYTQ9p)4tk=c z3jM2;)ncBL;k$&nl1Uf77f-tkDB zc*~(vBTZdiD|KYJBw`7~uYMP4yqM6E2HX6~I=8|`lfA!cXOSSztWbD~bc)}i7zL=k zN&m-pl^CZ24AOuqZ3e|x>)%ph7)YzTWUA6s_8RdjhX(N*8FX8p27(VQ_JG$gKdtVcy-{Mh zSzlbEgd=KjGXH2mY(xkACc>V|O`XZrs)_@0acK!_kvg?$%er4jD|S}SvxLzD`XEiO zqs?F8sHc+6)c{`C?V)2k$Pvs>r{%9Pqb0%8;^oWlaI0Io4aT0^8DPC6v5zYtrGQGh zaT75turQ!^s#vijy{S*1I3hB*T=(IYIhUJXzjF06(~8y81#<^WhSDep*By+y0hRqs z6_$hA?4`lfGy6IT6Zc6{NrzQ<;fd@GDbyA-a&|@B#91|K;GYOj15NY#`(LX_nKpzj z8&oJewaVoKGY~CGjfuxP@*$lBG*d~{Qi2m%Q}b-3oK~Kp7HLnD4pblywBu~X;1(Dh=jp8ckS?I`!zjVwQ%1g7dZiPUR`asl9Nrl4zEL6w64r}?yaHSo zGFwfkrwQLVV~#Ticij=K@`oJRM%^4^!K(P--FM?3$~401=X2!m(P?K25s)(atiURG zgk8M&F>s-&9&Mz|?5k__>Bm^R2dLG_Tnljz_$2;BrEU-6m6xt-`ViRCBE@>DfTF*} zRjz-~Wu&dKB_xnB2r`FRPkEUhwnY4k_Rq1oUi@K!-8-gD__y&m7+HJt|1>~o=QOkU zL3A1SH0cY*M7ObmnlT@a+E8F*{lRDmkmW(`rV17{?HL7Rm2ggkFojmlv@D9tU+Gq$ zWVPfvrBbimEV}3}XgDynA)`Al8Zj6N-fQF_GZ+%8kc+xXaKqLpD;4@tz4+;a{L%;T z4DoT%-S7I9DBf{#R2A-2RiE*h3nY57$vhYhN>2v*X&bIUns*cC1^U5DSB~3=?e8PZ za};iyX>4qTK?@0RMR}{9@Z!hm1We`ek4b^Ul0sE`bS0G2w!nDvQ_PJM_7(r3K1K+<#?}G0y`21e{X@=mbafy^tPZJb^ z&?=H{Tk3Y<*f5%uG#Nv?!lu)`Ier|F7Vp|)jbGR%vSt>noB2n)$&&J|Wdj&NwL1mp z1yIDz%*9qz&f;iAIDRk=T1`LW&oxgLWgO&1N|N`4eC3iW*-;On_5uTV^K7iWf7=tQ zqg(X5x*)Fvlc-aAaT+H6h7 zpnLx)#^#&yGZMy1XPW=*Q`6(b?#bu$^mGkhAOCG_flxV(1!a+?h~SQ1&~H};S4>~~ z1zR6$4>|~`3U(JBgb|Ynl|MDE{`I@l$_DrukDlzoPvLyNrgdAO<>8Wf_sdw2{g8`? zW`Zh|H#j-GqmM%FyKuFAr9N||-Bu5;9q!ioc@(2hJ$vzqJ(B*GhrTE`9GB6Ki-!o{ zO4^`H-v19{=hP%hw5{2~F59+k+qP}nwr!icY}>YN8@sHo^K`qI)+Ly^8_n%np zL`oShLaW__ih5D0QF_1Az#KiNX~zDu>Rl6iin~PCS}J6l#54mVOO55(i@Sc@a^HUm zdRha=r12n(*(!#-$SxW2SyHZGE+zsZvOk0??ueV+b{4%7ZHJOz5g5`;6?X>xW~D92c@wR4fcvABhq!* zlZDV~phg@$7@deq6#XzX#?^TVub!<}bBqTIVcY&_B6HEZiDac}?#px?&n7kYX#E0t zLDxKp8Z&Cnx_xb}rupF354y9>N~><9_$R$in92#;paW_UaHY}a&=(XDH1k?pZ5@4bAO#Q|{xP%Oy*DCwz zB?BOi-k%YjHU>)11>mj@Hu$Ot0M4=&ZEX32U2a2i!>JzT<(iY-#IiguvS?oK$LaLJ zL3TN&9NZ7L$8Np(CrV|Ny}+!NpJM8!VEPTSNIQK*WNsZYh6bTV>+vcTO0#Mw8M-rV z}ct=w?N2e(%mB~^G{%<%1D-p_n*3ZK~|17X8OAMacyo`xw* zC59??wjBQ|Eb&}AY)e3Fn4Yf9X}j@NPPk0_>&k5sJ4mJ-t5z7xH(!K0{4g!E92pz{ zpT+PxQn%S>h}3p(vis;!MH?|j-4@rYdnq+->iooW{E4jUXu$nQAiR|#PsNrrlz2pE z+?OGOxSfYfY2yn$pMF*ghsj4T{`{3ydb36FcV9b*5vTVbD+gKtfF@eF2kc9cJb++2 z5~@Q`tNg;`VY>mv`ar;;P2sy2mLp-k%%SxxWTBm_L~%xf!$MXPygs!f_RNPkXb6W- z!Y;I!0{!C8^}iFgsC*(g-=g+S9}O;J zLfva|d&-2vgtExA2BpfAf1Ea!jbZ0;y9J?i_-KA+Vr}+CfQeWVRR~()=e_!|kH{fz zz$!adNP~U_9veDU5rY4tZZoAr6EKhPUH7ADP`pDpT=tP7XHC zi0@tkVs@<*hcPYhg=04M_czF?Bsjx+hu~#IE$KKQQBKxA_py3FTYO-Ks>vZ7`l_?0 zF3SGXbxGkh{5-e&`5N@RNOSS);yPY)a7aO9kEx#7=ZHIWm4T{DgCx7I`!T#;Z6md; z+$eWLgszAWr7F-9zVsV0Rh>I;XhcqShO6sJU9yxhVjj=wy1pWn7szs78;&hGM=Ubc z>S0A*m@oGhm%eo76ZvfE?!M&E+Da_WGn%WDanqrXH7B(A)Iym*-%>Hymbi{+7yy9Q-|fBsuQTX>S{QT|&IbR<7N-BhNMn8Y zWu#q#*p|c;{vz+S^|QUEZCy73x=GWFH-Rkl=TL1L6G+qw6Wc_7KBl255|WL{`gy(x zVoN-n*y$cnKH}|KLbPj;p-u?AxI8~j!+m}R*U#4>KJ0xx9DF?-eO$kNxZQ_qXl4C4 zKV@e`R3ir)g^eX?i^)f>zLwATI|tz?Y9$%a$DT;(kh4xR5aTFQ8s8<#BQ0w#^dixq zG9vp6$5y3n*b0qLrd|3^4M$GhR_~fT=o(F6?sQGN)^c;iu9Rm~hI$Y($lE5O!gK7B zrF78##ax4HKL!RP8v7ZuYcD}6qEXJ zm5HY24fC#Zgjl5=4F=6SxF6qhZZNJq!%hVeJzkX@p>NWIYVIDIYUVM5sIfB~?A%?* zrsU0(0zow-h%7A3VaM1r;xchYAf6jeXzRYnR~w!pM_OkUK>HS?L8Mzo zwrT;)Kla(%Y8bA27glbBFw!{0OXLMVpIM4}?IN)-facGr+`TLNBS#w$kAG)tpq09I ziZ%Q9m@~o^`Ywv8X^_OCaGL-DiZy3Ky5gLfT%;phoFkgv2&c`Kyh|9J-%cn-ISZfw zUmu=do(~)gp3xIVxi@xxVzEJ!`p9VS4{lVb;*e6^cOpDZ3c3oU;PY|X)mm7pIWQda zZa6hMYNG+W{XcC;k`#5e3Pd0b3ga-n{Z zBau4v1-NG}g3tbk$n3(sb;YGb4BZ&qdPECIOtD*HR=r?dRLUq_5Iw`f98udvfhT; zP~m>6i$7Jl^J8_(xBx!Ma}LYDJ?PY|WStr|08>k^WEre=KJhr3zVNws=GpBT7&zq)`*#4^Bs`)K!=Vwq;M5?=WRhZ`CtDjvVFPXTmCDV{4AHynnfUVm1imr`LB}1_6*tMFaZ%8v7UDl7^JQfgm?51(dR{62#6nrn zcJCNd21Eebk+jR`4pb)mBopVtkFP-K49i8+`_Q?3y0CLq$`GhpB_&kxpWlTOVxdBp zq`fmnXNQUP@R>Y`5+T4WWUU-^>_(aZM^sD?c5-gk+s-9T8g*L)n+N1oSYi_2kw8#H^P3SU3O)NK&) zvs#IcEHN`Shbu>Ufp6Sdos9{|Wn@K!ABQ%cn?uNE0DURL-)l5?n1Jlu1<*PzVp8yGT6pwHk19?CCC%3;j!N7N*S)}qS8yC@ZN`+JD0 z1tj_S8C$#OW|ZbB8-nshqYj=^3KHqM4#jIPB00mK=mg}JHM!xwN+temIuerMNs?AM zsJs=0>^oMZ908Wvg$a1sGFvEY9a@E5`r$myIHh+6S8h^_{ykK|f?kR9?lX2z7 zPmwyOOENQk1RLw4N^_Q9S00?1qB#F;JO@6jqnRu?y?gh7_n;3vF<}n=cBcgthesr8 zQtmdy%`OM%80CWtY!c*&85q8z5I`ZiXs)K7c9n8ltA>rl)CexqdWNoM{cUxG^|4BnwZEY( zhjZo9v3-}QZKB*Fjn?atyMu7*4K$+Aqf59S^bHC4w&C3|>m2R^_W&9gE1}9@ni=l+ ze(MbYVx|YU)Uv}#{H*0ot)DZ1jBLRo(e;fMVy+~3{EBu_K2#LuW zyLT4U&Hy!=;Lbq-Y;`nzgWq*Y*R4W$ln2~FEmTq=#h#5z(|zn^!1}3N>-aPuBZP_T zD*;zxyoe#P=HnaZJt-M93n=bbd@t1z|1c8Q>-V8HV#RzKgMv`jXcDQY^ zXnYC&byk`<>~2o4uhcF}ckq@~$9z9<$`LSDu0n%3R=dAclmD?T)T>Ywm~CN5ng#MD zaF${Y-)^MoArEAa6)JYASE=MSx;^N{-uq+OV`w*_qFXYKmHjdSZ6*wli60sSVJ)7n z5^2JbgA$N_j_PyXIXTPl5nK*ziTR15Fp)l33HdO_p$HEl$}X2A}SWFk$^^3Zjx^^p) zx=s>+SfEhr3O`raQa)B{vNvxR1e2e@pte*oH1{t&l*ekM@GKJ~XB&tCx(2w&oZ6+w ziRxg0ObkS|2ck*L6-_5B4X%oHDNqWR6(t_$p1lzv{Z06S7J3+!?I4NIbE?{iKrL~& zn6-I0p9;mkQPA;l@Je=(>z*Df{uChw6l=*6sSC{!8Iv}DW!t<(kYL|P@VLQpBR28x zvP_!6UmRk0XXP9}1rIth;d&ko{JnlnZt}{!RLd8vKcSHQL8^G+%$u?0_SjF8CVkK`b#s5 zP+PFp#7!qJ_QJEWHI1M`hz2zV#7@8vmO^gN6!_;Ou57^GW8gNet*2Id@J#|Sj7PS4 zR03cbA9Hz}t#zCMk+Zn*_d@382q$Lp|RXH4V|Q z&i3{Dm+*HAK1#!DfD(DHrIxeAq3Q{=uqvZqa9r~AZF|aL@@57;wS=sk^@m;vr^Jg< zCzI}YOJiS0YBT(wG7Z1qOQa~GMHtHk>*=*}>dVyV%MzQe0TG(@sOWBcP#r((SL`aZ z2p74FEU6Wc_&15p4AR6ldWU+I2EF{rtyRCents!qQ>l`veM(UAJ>D9L!qL!lh`QM)l z7!Y&HIw6?9h_sH{8sIuWFeJ486<*~^UUQ!TPPL9ilfN?HcQ1WZNZCxJ)@Vm@xuNb_ z!~x`o)|pBk_Fnh?a6@n{n;YD#-YGQ)k-gJGSEn)+)fk`vI@{8wGnP4LiOl91Z|zC~ zKt|K@1KW!Rv!YjJ&Ix<5&fTipRL{Ius8seDWlujnA z--f_i7kCHIU$?*;wF{WQL;xZk+^IiTIP=XC(FMjvx{Q+dEx*pBjd0~l=#Av=>T1!Z=sv!zs_*N=^h+Nu_6^HDUQE?Uvhqm10jD6P~Dh;2}vaM>bFQ&d;>f=TKI_ zhp5TxM*}mj7CrKX;r-UHkB9RIr=y4KvvEa-1Sz>{C(O|_eo0tVyc)GGP}L~icLQO( zjlN=T$(l=>u$8Oj*0H$&L}m1x^b`+Df-sv5y)F1c9*QCY#AZSG1X z3Il$wNUzHLteD8%Dfje(c6TVJgr;0W-^Z(r>&UO|Vt3d$(fG_|6VEkX!s{S+7 z{a?UAZk?n}*2LRKD*FeaF_^H7tK}Z)YQ@vFq~sRtO)IYTYDJlPa>Psn=>*Nh%zE3E zAD>PvfP0CUBlh+V<&%C?iTru&!-*h&g z&Uy4w-r`@p#RKdwCo+x>jvp)iOznQY4exI#KWy(0E{+%RYW@g*eOP7zB0N?lmA!0s zU^bBM+Iynl@i{Lie+CKc_cp6E_PN=)>psvL*!_Uyx8Gm(}(njqXY5e=L>EJI75Q}a9u^o9#O;JS)jUbdZsNxF?1TvotN&( zB*C|L^H4jUrvIU>^@6?fP+juag}OknAxh+?fRtK);_M-2oH1p3|FK}2Al2$GPWNxH z&4XU zfIU@{S@{u%V22DlKt>pOXPUbM`zV}YcNM#SQC6FHI)e@>TCLodH%&yC;qxg2IRgkF z>M&3X2u3#>pX4bKykADWwL=u6zLW`A+8;85%9iR{kuXWXE0pSaX<7r&@iKD}DYKL~ zz?&*UUxY{6w@j0K!#-4#gcBm&+NAiig+ECY8Q2g;9@f>#W=vsD$|F z%!BnZV!#P+cpZfmG_z3K89{D z=|h;CbFWWpX9Tps;Q`Aq`|(4a#%ymHqMBRtY|7A#_?NCU#@GBfe3~7QC9>nggJTaw ztiQ31HDyNeQ*`eS!otvH6uJU@RbK}#tf~-QlV*l1bAZ0C9`7XsW5GdLw+Ep6r>W_z zy2crR)XG%t%@0_9ehtE2nToLo+R}OGIWMP1vpFY>=;K>$BF|Y1GlE|sSR*tR+IrF6 zEr2H+;;-UGARLANqpfx8ii`Gr{i|8>D%bhWfkHI2FM2b2CoVGGzZ!*Cz~E7{rsBAw z>ORQa9;|WoKjp=f%vfWq69KDw_K!{qwF2OLWSXf?~ zt3pDtg`mQ$XpqXFFsy1IEQX|)lBt0jm@(v4@bG}iVJUYbLiibLDPDifWA-W3?@=xT zFIc;Y0-gZB+_9mouWX@mPDPD<&-7-Z(zMfb`l5=A@NdX=Lr+`3^q-Me9^J-b2=rSbthvk6f1vJ05ITan~XZ2;Y&qz$$}zg zVrXztI*B@B^9YIf1^=`Nq-OQ>efDG6(pZ6HRwwpa373Gu589yVerA|?$};yDl{yYl z`v8i(T5@q`L0SyvaSr9~;x6dcZ1E^g>&>14jb~8v;xb|BVgUN4t1xWoE}Uo!%`LS( z-rTaQz693h5*Z30_rwh$drNk#)Y7i%Gtm#Rnp4X>m$z=Kda^>2-s5AUa)wXf>$PhN z9QO#`TD1?rr-~USnQ-)qe(jF%Bep1#L1#c0ljLSZ56z?Bz0}1kBQu*) z(%=k8ce$_8`60373oKrA3OLLebV1J6XL3~G%xW6h%#GxGA<1^pWWm^yINOU$!7K)x zW;gT!K-&dX^Xzf)bN!1VAtjcV?)iY#r(~vzjeps_a0QXI@xB^TZyRe`QUg@<;tUZ#>THjMQ+-;$=q_)fN?c@fL;TkGwp7$23U@$KIhYy~C6t{GyW>FTYK zJbdYP@^ILHfdwCKiF;*G&ADi`eG`42?Ubu)Ec5aCwAlWV(2EDHz1aZ7?Z(OD)uig{ zbG~;_2BYxgGYD=Rh?5`y5K%nJcHR^A+LoxOFmqEfBgIPzP|;Zu3!5y(+1E*x1X(kT zF>p#30v8KXD%I)ZQ)nl+bXrZwDcgl9X9=+<;zB?qIP`|`ZaZJwN~zYSL5Mer3KHmT zo}*e>uX`gS{K%uf769}?wibb6|3Dbo02=SJ zI7zQs)uKqd+GjrbV26r5oHe3Zmz5P3ZBa$d`3pEX>yIF~y^ zIIQ8={8dExAb%iLtn?hPi=X&s)`RjcNwnzA{&vbgUYCQG{Q^ti<@i*nRH!I)v;!GV z8Zt#j?vSWr{T58>nTy^JUHAmz-~oRo3&sN&;!?FfwPsOasQutP(WFxv=$a}i1H-}4 zQsS;^%)}7Te+E7cuoRO{_B|A8#%_yUqmZP*7^aAt_bmD!tSJo`Z~F~*TDTx5cbpJv z5!3g`*GP+I`xKtdl^j{R!-E|cc_UdMyW6GiG5+pd`-@W$|4hE2O02`yrh#JRUWFc( z0(rcggi23@m=cvxx!6-50OuvYnz_SBMZkLkk-qBlT?-b!p?(G}=+qXuB$Dx5u1}_J zAG@ebY@0iFkSkFcANRAG;+gr58FJ0E@SCPbPd2ni=^iv8UeILanab#pQ%b@ci8lI0 z4^uY|!4eq40Yj36gK!h2zV7Vf14j3le@2K6|m(W{>iH zjq5MlH-+uvig>Lr%8L1tr#krTz8qx;oeQQDIYeBB1ja zpy)wfF}+4Y2U;`aEON%LDXws&x$z=nuf+_gjf z&kC(jW9dfk<`(myvE?Wh(rNErzmiS;NjTzt^0r`~o(`sVUeb8wdp8XU@NIvklrnd$ zn?XG-%Sp{6<+_?xO6Yi3qJ|0c>!o<6CqEc>EBM|?^~EqeNU&w^7~-wCM&*+{J%RL5 zrhzqbMvd_ZJ6Pgb+BlSh;Elfd*g`LxHUc<9<#ceUvomDUi6M_$o{((Bj8oNQodpEY zPKT{YSxvv0WNwaf$@q`c<+!c+{nnOe=8Q(+&7}80rHprquFzv?%M*69i#ye>1@_K7 z-lEePWd1PKpSZwPI6{b@15lmEVE_gu^3sUnbXQYunxL=vSkdoER*S0C0= zSsC>vm6U$;kp&!0R;RO3cOf`V8YR(T5_|!dR6VOcWR3Vq>m#${AZETfE02Argw~0c zCkDgjcHCiBdC#GCj*i{%tD#pbLP`#0`m^B|!Q@9%I;dMj<2Iho+CDOX)yVDE?GGW# z2Zejw$qU2oD|OdqoVCq@ZCX%TN@DFMVG6a;UjAAM!Z2NhK+pR9LU}Ds@F{onP2s}8 zC-@aH@UalQBcjW68JY?_%JAf`ZN#1Owvp=D5QkGZNsm!Hq`@MOIw6}^)H7^^-KTm` z!@p<;{A}|kJ6*`Mi?TpXEleM|NfbfgV-7Oa(D(q~yndf|B)M4c^F5+fq zF^Wh966af?(RWK2urdBfGZJq@bd2v9S-My-&0E#=uvZ0nA-V*pN{}CvGh+3e-!Nx>1 z;CGBPNe2u8JRfi_O(a^^3tZFxR%xAF8G1yU%4^qbZKy38T`II)M6@<)nq(!)R4t|a ztVOhpTX?iCQyItV)pn&s)$`ISm>30vRr}wgDTE@bFSD&Ey)6vIkDj*O^PVZa@E-W! zXhJX87+}=pfR$c~At-%2%^(2*--#D7Vy_=Atu(i*#kS^4>b(y@NOZ$2ouj3Wo5|U5 zt4$({iO6LuNcE8cV(L`FOB+G&@iY=B!hkzVJ6{Uf)yxYz=ToS#L>{Y=oL&$&JCr?tsYp^wqRrf5AnEmv&_ zEE`>qlB(@zVWL-n{hL_VoNEN8(JK{wBG-{rd3yrp9ECZGmkMGW1hIp}(>LCdQJD-d z(tCmdFrjxoK)+ijW0wlYUC{39tI&&llz}JsoV^=sY+Z54Fcr?ne9^gx#37}t4_Fri z)UU^{6vy8IM$}@=$_aSxjJLST^A;4uqkCFX5t`qMwMiqcC-CYg&+DIp_vuqFeyA^4 zA7ubdIe*^Y(L@vb<1b!tMI$?@^n29O{T~j+p~-zy*dMNU_4>w|*rJG?g2V@`qAx$G zSj|h+)@^p;MnsSma+_LTeq6|4FA*zv2W7|<>@0{-$KW>(z zPSxxfyF!kg-EU~9z9X?DHPYyfg{Wvr5nA2^<_ZWmRY|0g>zJe2I$=u3+@1BFm1EO8 zRx^e?!KRZd*l*0!&3ECH`jY#KXQ6vqvA${^^3$O*$O1 zDQyB2j(bgDFuV#51*E{n0YkDaGdKc*Vr2D)>iHRz`hn?kenUj{mzDZlndvZjqHz^$ zvWc!I2EB`Bo)}@Ce>Tq7G0D^Y0DZSE5xLS)*9p*LGtp%flB1sGe?}I;X#v z*sKuf!zpQMMU`l~Dy}7Je*&hfZ1V#ePGW;pU42ha@bXq7e58TxAVYH=Z<~IA&SYN2 zGf6U`v+Y%rHBj$-b~8)((?3N59WQc8ekzJZ5a?+2ZkLj7$4FIeOR`vy@pyPYt*Zd$ zI(GXW*6IOii=EBT!bebDJKE%r?t&+@vkMMrKvPhtN(b_Y#(kc7r4H>!w4~ou0EH&j zAv^*j611?f!qdo@&GR{F7U~F!OrBu0UK+U={Hbs~m+r{a5SR^M^pzb9i7apw@^>)$ zM&|h3HDsusj;~>Ji5a~$yX;SqB^@2&-(sKGK&!tNUye!{egIgy;fYEXtfNZSX}&nk zF$~)`PX==#tBewtrwzv)djg*5IOb7dqlFs3!>X)hcuC-M#Fz@gE6GakXoQeMW1Hrs zG%?%s%$hqad{642SdupufV%xltO zV@$w_o2&IWe}wlW@)22egPtLj#D5db1N0A-5EG!L7d{{}3YVfTv)qbi- zmx3Av%ct%&-Q4`RwljH-%>X|lYun2kD(B=RjeOH{c8!fnO!T#k-UCQWUm*ln&CC|X zY8G_vRl8hHSmVF)0vsT+eIPMD|f z8Dd>TsF~A)@y+$DNSQAP9)npb?lH~V^~}R%ThO${6j%jv6YNc(Ku|t@$`@xpc1Bhq z40L+P#c@8U;@Q_Mx8lr|i}Vj>SR?6;F)OUT;cnMH{&NQ@nhEUf1Zv!SSuoEn0h%r3 zO}Yob*w!DgtvKPfqgXC|KFomjcM_N2jgeKeAJ`8NVz<0vtkSK7XldSyQhBFAlyr>0 zd8Vtf4*1_OnQB=}P6Nv(S3vv~d zYA?i3s6Kgh5i_Offdvy`0wvp~sA_vTk}?>QE3&x|uyv^wZ_jrR^JWQauwWknu;FXe zm_8U4EXx~xT^NY9XTT$W4HNFVV;anfy#$3|M z8UP^aS9r+&zr6jmw6m}^G5&w?5!ISDc3Z57-#xv7@qEIJD}oD!Z*}a)GxdwkQ9}^c ztsH$iW!e!9%jP61bh@3P|LONgNFbA}ws(dY)RNEGzfRp{5S#7a_LFc9e$uP_{aWi_lgf4z>dwz`>Q?p>L_3G6>*@6}sKL&`Pr&VqPz z&J>OLeAYcGlq7n)BdAt+6w@tae7h^RULZWp@a};3YaxtFE}i^vyt`J@9S_rUj8~`a z3SqV(tMO1mX^$(?8QcvQV-w7JH_D8NDlHq5HdA;wA(d+F`01Rz<6dMNf3;Cq{!U#b zjP?Nb!$eAKopS~X%9%*xdjJnrCD*yrDQD=|YXwigAl=-PQdi2WLuARs_CAm5aO$;_i&-* zLOIl6Sf;{xn0-rf0^T^(CMMAH#&4K|)dP6(9;z0EKzDS=HpZrc#nPOrMylPU#VlN* zbt=J7WUr^J_tD=ww9O-CO@&GyB)7-wWAfwf;P2t%;@#uh`xWA!sEuCKn6Y1B?e==-<4fL*U?tX>qb?^d z6wpbRf;Qlr93*c7r$ooX)l&jXdhCRM(}-Bjsco6yHxoM@qL}S{OC4z+l7Q zp{QrcZuAzJ-39y+(rUOv$;T>ktRIU=k};aR3o~mfq8i*#)4Eg+-h9I`vzs037k=268dJynGnru3?Y}Pe%MU)H zHU7P-F%i1QslLXeZ&L8;Z&mKh+PU@xv@r*v(sYORK&^zac=0=6aZ6Ua{5$#baCZ?{ zG`O*0v$Q#}fIefJkY&hF$x+N`HNhw#d=T=A{@RQ)gaqnsLney&M?;A$vG9c6vkqLf zRTsN9lr$>FG3;G4yv+^+Efx$$L1h4M*V$(ar$Jx1Ma%<;2QeDM36S285%^aRFdmS= z_Be^Qn?eL_^J(5hs8_-RliLENC}k%=Sox~$33D|y-;HpAnH*_4tO z#)jgubQ9PkXcC0_l&yOm#f^#F23!Y!4dJBdzz7!n)5(XSQQ%nC2I%dyU!?hGy>Q$t zV3f%mNpJk)`!a!083vIhh@>e!f+U-gw9}cVeniPsRL$z7%6TBYO|+y7KqXLom#OXC zw9a7q@yQHe_Dg&21T1jTxc~*kfn+6r&L;u|4jG@L+&ffA*oxxJLatDYiR;PXE_e$0t7??lO;_YsaG+2=xYkt zbDlzUV@D{GK*^Dup09}G%Mht^z7x_3>hskKCaB`(FkXd{fRS|NUzm;x-M-6YlQ{-b zjuixtw8Tc^@KW}7mpZ{*bXXBGU&j?kNGXkoVL{xi0{bI$|IOS z>x6adDOD?BmEN+Dk&yvmsrLI#**V@ zP2a~~V|c`%K=`C|%Hm1acq}aU1}lN&Qf{CsS>@p#V!r-PNwu|;5zm^>af+|{t-*3o}eHKqj^e1_XGf)G>{ z(t+c}%Q6D*<5}Ivaki{C4J#@F)E#|VAEF8Y1cX#wx~7Z@++J>~WKYWxaJN;N(4UMt zC;HB~RbxUJwvGAxur`lXW%*%VDZHo(VAtt1&dNu*(M9mjL^p5b`{G!XW8zY5qs0Ps zkzESLMul1VHbf**zA&i2e2(r7#PK5`D{|@0LfIM&ui)5g2_7IYjy+U4Y=@h~b!KCFE`FB zfXqbLnEyoe;olu*e_k0x(iwWn`2}V9g|2FnV!J7-9ukXu$?6DC8Ysp#u`LhZSXND0 z=2=Jl#W}M}#ClKAOITQwQfTt5Xfin{eubb-WITiF=H3PSJaW}zt^sjOqJ~98F4`8k z;>S2Tcf;Qtvh5N2x8X&T8h4Ur6l5v31`ppjzah3f+izL?6^Z<@3{yQHyDrNq@i3@0Y(D_wNGh=_T?_r9$9UmuZD@0ak z+tIJo2~y?FlEE#26iY?U%@@E4&nn(d|FLo1EA~$i+4|xV17VSVW|1_;cxk`CVVV*x zz(zf+mrR*Mc^Tm1t85xP5kJAgA8SGL$~|6?ZnH)(WJr~uj(Hj+LjLvC=&{4U#Id?I zsHxD>D1*ielxsIVEi{91R|r70efklPcZTD-^6njTN2m>+bmW`T8V&EhhdhkCFF*J6 ze6I*zU$3D)l@~>~YWbV>^%+OzRt)ZVhT{Ump=jpE z_*-%9hH7v;3vpQ%I)f~0yNv`aIv8JOwp-TV5i}GuAViSOar=dYR&E{ue&#Y-xAXW* z$&Wc@z>0(ibyUui)HZ@0V12Skqu$Q-5p2z=))v_}7jY{f8NPyrUS?2;zh_-+LlAiB z6xh;avxF?sfEZC$37d-8TVAT2BE52O@X&qw6pzvf7Wa$epTs`W7TEqP zStIRXM2r~OkCefbyfNvA#a;GVF|D-kCNt_vAl0f|*J?MTpo`o|Lz-f)cVD*oufP5J zzrOi~7L6iW?n|Wm&I(G()c{fPauVG>!D^+WcWjB#z#`Dkv@S2}o6dcPgG*oaj-f)S z*qG*wcFdYYGi%~7H!UOVBiqtWW^@~k=+5V9CN`GX!>W%ckSr^MFy6F?~09R zH=Vy5T3$;PqPX25nqtQ6LS2Y*j-0EQX_~JC;g{w9zP_(;f#&*Yfh}AP40XihEUrAx zmU#1Kc^h%%FIdlsvYVmR@^^c<2rm##j_FR>*Ka;Fm)2wXTst_|FY$Q%!fMUjidfp# z%kFm(GNu^25#v}z0u|($SO@XPdZdv>^O)V~Cm;mEWf>9Qo*Ac48Ox%JQ;a4Sq z;cNmxVD50Boh^o~At7sW==s6yedTj9*ezrf!qhp|8_0}LeB3oSsVDQvUgm`-8Gxm5 zhoAOkEYM&gka0_5iT} zeiAtPmw7a^^!srzDblO$q-W}q>w7$5<B^<=!U|9S$(TD@jett7hE$ZmIfKDxz9^F{{m z{k64sceHn5PYy3F_Oj#Gr|%}$gNK=s{#j?wNjH!beP(;7!_4|}b&40|qs{>L?19C7BU;D!rH8hI_5Np9M8gh@)mc-v09X;pC}+62+n%L$V699PcnOswS(iH z1Nk5-gF58WXrR=e!oNoU9h^_Ie_l1_#tTV195?4|!+e6AKX!R_aOJty{NZfNWL~KS zHInbng%)cq(0z(~iLq^Q#6#^BobZ^T3YTYIN<@H+ZKWjM9adZt%B?2Kjczol+5Xlp zcdMRmp-4KgCzlsth5%d#n;@z7{sxa|k7&!^a=_LhAnGmxH-`P-rWXCt4r0rM3k`bH zXU$CFbzoB#4FRipHviX;m<8XA+{i2dh$>vM${YRna34)uyDpl9?;mBU^|K2ro=$=B zxFY4V;6jX6@$;fSBW_{137o;<^?YPb9VNwKX+$pqV0jjj}fR$VxZr&>u(yn*9@&5%hMAw|=#*O^(! zB%ekN@-`%zII4frCATakm^P)hz!p`9;kCbqXZ3oc64j4IFNW^)53D(469)<5PohPx z0szL3g-BlZm%tWh)s2Sb}0f|X?eRQTrFGdPrazB9A;AMvWbjAV8&Srf-T zwE^sv=#$6HVXMI1h$)me_&Cr?|7=Oz`l}`}*#hOeykf!JcY&}H#!o02fpi#VQ19w& zHC;=UDS)L)V-M0mS%9?$(`5Pr|0~G+Ie$mhUSMLG)q2NB97w~(8eHP=;Re1_u8S<9 z`uK-oHcY9x(1V(END|zY(A%XZ0U0eg_kZXU+{(iH%ic{@&z6C-l8P#dK1trY2(|1d z%AZSi@W*@}i-h*-4F>0n@UD>$~Zbwju01kr;kl~CJ~2^p3pNRu2&yjcR0U; zr`~jLA?V@xk=v*xH6!`LX5wzvi)h^Jr zb7j0lEHGI!@VL7xbCN}}p6A=vVit4WeEsY<3G5VXxx3vk%~!gWN31Jz_W7yEJyNrV zR4o{A12rX?YTLI*S3I=n2vm5lH72w=7Q4DgQrFU#Ju)A#QVb7{lyBTu&_OPym)h1p z$>%g0MFD=QT4?8sT$O(|18yu}i^xl)LW;%%{|QahQeXX#3h{F_j@++ z7{l|kfnz(`B#Y!IKG&QX`DII*#{WzDm z;Ujo8E@SKnKe&yHR6Fnn4qXcrhDtHs@e^0)HzLAmXC$ewL%K(C)U0)5FP258@cw< zMf&%bd|M$ZL#4*h%5nu8Cc>{Fnh&Y(quhhIp}D5lUFPt%8S=L;0hu|jN<#a<{iR2R z@PUN1Rw0fbHBgS*9u0t%7zFG0M_oEbGgKFSVq^F`tC&Fb>9hkadTGb$?1n9svjgkA zB*fxleivVwWqhAEaaAFr$-E@dpEXLX<^3>|*V^{D%v1=VaRFT~nZ&qWtpR7DfYqfk zzo}}v)2Ey)pZJDeqiX;Ghc2xGRu#deNs(dzi!on=ur7^k%007$m)=Qc|BmOs&s$)3_3CD6Ti1npQ+h+5 z?$5u+XrJ%v*X?_(=jX}RTD_ib4xaA^4qo?u*`A({kNr`@{>oH5+|#a$NR$37qUXu+ z&cR;n9-2m(NjTWZI=ZL}#co!Ol-0`bm`c6t7Q1be z=?3zT#Ocy8xKx>LLyr}cC`K!-9Y&4xW{>(a;d{=@nhC?2$ew{p^0OAJK7OA#Safc? zitA|KuIUNJtTgSOWtB7(6uY5IE&y8JFR-vU4YkR@!x`@n^`?f6G#9lc71hXABwD^a z+m(v2#p}$U)bY?>Re3L0`fCXey9SU`H>>=__oivp;{f!rNIg>>B{c>@Ck-`AaSPAE zEL8pyi#9`Pdo&hkmu<&o>2A2DeV%R=;4hQ1h{2$jObA~cZ1l?#;z=b;i?)Y`Dcg(E z|3le1wO0ad+cvgs+qSi0+gP#fs@P5{tT+|hwrxA9*miE+``z=f-}ZT(e_(ub^gddP zb$lgow(`cdOiM%5tjKah>@OkdIu-sITEqSXufDwV1l?0wVZH}5b$jf2{hgEZ@jP;{ z@O&D65PvP)^ZM|^0^{@T{Fc9FcOdOSSgG3&$^UyKONlu9I(M5ue2JNOr|UcQtme|a zPEo6KVqR)orGqJ&sj42cL}}p@psCK=b5AwMk1}VP=>9>W7g+m+z0>W8N4ZP=9WwpG z%JtC|Xf$=e7lK>G$3KM!KKaGsbn&9s*LG+?H!)wJyZkR}?yzV$@yQTs8_)c}A(>pg z_NRRQgoqTRc%aY0)|tHKOE)7ig}b5t_{gU*Nq3Y7q zQh+*9H*i_E<5}c+$;yWdi6W^kj|_u*jG(Y&%YgBpI1-VjI%5iy&GX0*sW9KtfLf=i zSp7YaG;5z3bu4TLZ+IkNdC%A(y@F`O{TS3@57+P3rz}ji8piD9Om6F~T7K-0AQ8Y^ z5#2fx^<&UkKmd;woy-xa#>yc=F%(M2ct@_Oo}He$NY!<;K@OE<@uSeY#CE!cQ0e?H z+Tg_ND%i$Q31O-KROXe;32$j7C{mr#0YbrIUwtlAT_fCcZ)$NrHEmxo4EUght(u~Y zf7Bh}OH4nb5VD%=)OFtvy-HEpLuuRK9HvWDrUT2_5neYkka(C(<>IFT*P}-PbgVvH zJS0p1+~bDs3f!!PGf7MdgBJmDw%h^NJY{MLReksPLsPlRnvTkl$a%vSsN^g>lmOU9 zh$LG6zw#Nv$LhzaQf3&wjhzP8RT0FxsMJzC+Qy8ijUTJV?(*%sn~EzX@{v4)CIxxqx7Rab9{ItB z>8SM4Wy6(`klniQE;tf>yJbvB7_d!`HLy#Lu(P<#gicQS?B`?$eFOx3>uc!;`~CTe zgie$OIs0w6go2)T6<%oN#d>JK(L^7T4#_E=8;oebe3}|JXTr&e3(GjE)lo3U zU24@GM$&$dklERM?yVn2x7`xltx(5N0!Oc_xkOk8fDeM?|3$!|EBQhjWN=OyuWy~M zXqgjY_2p!IANNDb#Fi0^+?HSeOvanV^1$`b&b#>HW+z*8{qFSiVRU3Z9f7@qgFx40|G}-uB47!HqKeXmEC29|&C{bewOAQ;CuK1~$>LJ!- z6LEt?F$=O43v*T@$|3buJ2||ywc2k6--IG;ZA7&S*zB3^%uHRQwb9F!(#?}RYwT6> zhz79c*RbVLe*dl|t{NAascjaxn=SUSPTe6nRa+uikLK?RJ$5>RbAHzt^;pgF) zasYJ(DsT3C(WP(+@4gLSQ2C^4g9xW!P>|9YrO$3W{uTTA;S)e3)uj!iwuro?87()< zd%2X1opSwWS4tjrbq{(n_Lfti2w6hh2)`E!;>5L0p;OUR17ipHmx^W0L?gsTBTL>Q z`6a$RLBHO(ly4zF!;!3$<~$Vu6+YVm$pB^sBH^nBr=?|uOTK;4lqT>!y@~zjBtC&ROFXfcs^+*`nQT0_kTq-yOCvau6~)lKfm;+E-w2HlFxgvu z=f4r9DL`V$2}7S>Dv;7|%%d(StBIQxS69bZnwjv3V2Mf-jvv9xQoQl{&>P zY_-u)4yp4(p-j}Yp8u!3Vu7rOzhE`pLl#%>wo-J*c($Z&{3-=~^v+6Pid%%e9CA4sy-Od}UVAzg>n7g)C% z0Qnr^I#dOZ$}Hox~- zK-CES_8&HG==W-gAha^VLjkf?psMsqKuGjwnO@io)adC20Js%Dad; zp{F&6o>Zp_tK+; zYWNRGl2*QV_R~BgoY&ObPOl>-v{sKZxLH_z5-8xkMNz@Fni4bN(V<@o%y2L(xvPxv(>L>^4+$WmkU^## z>VrNOfrdDAg*#lT%ryDha1#(Ctv1fV=y>)VNElz(2m(kR#_zJh8ohVts;-<=;r6w3 z>zQQQREXVk>k+Q$w=biZKDGKao531yn1cG|Q51+7jkkGQaJf)L;KXgvm)tqJiO5_k zR{dXGGvxN!VYJ13b-ZAD7hmR`CHtE8g@2}~IQ~jI)&p|ug83(A4>*Oh9_nUc@n{E3 zjfJr0^G3saqu`S7vqehyZDDNZ5FC4a{1lY!shX{+W#o-|ge( zCb(wS&Qk?*5Yo7P#Rav69<6YoFoL^lo%8i<#hza}Hp8u5^K1dO`1i0I`U7R>Ir+wf zZSxBf-FNaT%5Je(4J-SwmV>sb!U`pc+9c%<3t#VXMCgouy0may%PDQr#cb&rEKqz^ zHSz5>N(HBs1_RM!{eEh zOYU1NC`Kz%O`D|Y<%I(5P=O?VeuHh)x-s6l0{e0~3r#L3t4}pLpng~y#1y7*YaNDI zA`|FA;j|~z@o%4wfN-fIF;oH>3j#vJV+t)kiUogMe+eM<^TT+yY4?43!GW7lR9`3V z5~_cI*)APH;=&Cfa1p+SkYFuMyjhrK&$H=7ktWkDGB&qQ+}qSV(yl#9uBx@vHNvYq zPS+o(Zz_r8TrAqLwr!_2#9>TiB}9)n2urGVl9%*V5*E~I9fgW*ql;Bnw}sW?s9PB5 zydd|8KflS%3{TJ2$L=>$ce_1oGs_dybwV&hlB=?mdF6VvDLJ@X%Gs8gfIsCWHUQ8l z1aV!!__y1trkh}6cMG+*Diisi7L7pQJaF|^Ggk!coJ@f(=WWEZG!lM;S!Y}v`r1qR zafT67?1Ht8mvcgjck`zn=7XfF?pS?3;dt_AT~x_$Al& zDY!IoHRBy| z(>j>2yB4JRN(MZB4r1hJ&88Mps^({25tq)K%M-p!7!t~(RH#*O`v>LT%S}>BI*eRE z(699-;_<;=HY+{tExfQ{5kxh3r9JBiKff@Qic@i4~qwFjQ2tiNy3vg8Te8mIYi-vI)Z&e`Gb! z?m+5C^GvXh!{Pje-Z~413qWk_WR9Z2O8&M1G$N~ zpf8m}spVougCp5UcP~-)>K-W4vX@Umc;JQ*Eb)8(eL4gRcpx<-Afc2(@4zBtwxZyu zmXejM=*+vwa--niIp1T5f-C6?cYsYUV$svW=d^VgANONw~BP zHod;?#I32-r-&_Fu3q~s{Q<>9wEVn%qnIxzVOl@Q^!1z188yb}4i2r6u8mz8+HB48 zhJQ9M)PbYE6`=}wh{Qoq>PbKwCM@v_YFj@LmRza#>$$?U^X!R%6P;ZeGH-5s9vXU_tZM>**oPeU@ap<-`O9Nc10|oFK96xZixWijb zdqS|$XNNE@3hkoF^WF(TaJHaK-R-72(U}$m2Q3e~G17R$XV6gB&jSXRs6b88?2H3S zSndmkWNeNjyzaen7^}-{HNYlP30E&HPmc{}Qv}M{GCZ?Po}x*8s=KVGPMA~bA!7OJ z-I2@Wiv?~w`aPO#?&v%4EEWzK}iPsMQZ6E zDUZqC-So#D3QQ6tSzCHd8&PJDW#@o~f7YLBLs?jFd0ZT?h(C2>3EDqB@5K0%P00xW z)|8CjM0FwLCYgsd=<9Cv>}aMrC$U>2(bA-Pb1ck{ArNm`V2>e82 zcCRY|(ZT=N>1tv{*+N$GWWGr4+@bOZOYVjosJuZ7Z@LD{d6~(`(@Ai8QU)!|k#pTG zHhCd6bJKfA8pK9J*zlDPr37k}E~qbC)%fXWbB8$bgn=KW=MGj{9u{uyObDVB?aV6< z0bs;;ie74Jq&YY|fJ+Vkja9`R}XF%9tK7dl&LUOVNdhw<%3OUb1xQ6YELi4zobW)rl@@JY?Ny}T-cSIMK+ za>ky~jkHTkn87tgD66ooHA;8 zxuduQY#mXfMBC7&56Z;NZZ194-ki+g&Kew1LI7v;7Q<|(uEV&?c^r9Xe&Vdq%2kwT zxnJ>9hs2NIBt97$xAoO7dU151d|O{h;zpW}<_Y*>4>AXr^Dqm|P`Tj@9GC>pOh` zhs)pqZ+wENvlLu_{h@!TF>Mv(q*(Bf+%xA8*MwyH2C2E8N83jSI!hh&im5));SMnM|Rk)R};ur4qRE;?Q!RA(`zdu` zg9C0O&DDjRfsTvsBAXZz)bHXeoRPr@4^=NwLi&GKQ_V5jvOjqFKqVY=TJnyIR+gBE zXo&`2Et=xq=&8sZwOf5lI`x&u)n<3t*Sj_T{VB-X?q(bEcM&T9zsrlo-^B{+DduWi z2%&D8pY#-k?#|G-OU^D*Z??7dhC8wP@Lkdxd<)~D5nwiby9DIL;J4vD#9F}ch}wn< z97bJ9yR>hpz-55;g=cgo{MOgNjaEn=)Zv>yKa+)sb)<7)kqK<7UGc*&!WLdR(%hur zPf^gQ*MKF*5=tV&cs4N;V0BZ)6%l;rTOoJ+Bu=cOm5(F1(5F<$eQVF+>sw<>5(a~r zI9l~#3>>JYN>`5&l((jYRGwkq#_KF+Y5GOfB5Hdi;1Yq!&sX~1ESL9!77JDE*$*7` zH7PMC`%$=y$kQ1rg!@D=0*3wlR;3VkX1(2zSSR=b7Mc#0EZOa8y$khK864#VmTT|? zU!2Fbz>BETmMW~T`rUj^K0m+OAs*O^K;P+|y+cg_1lq2yenf)RMHm!VZKMk604yVA z)p=ZY;k~@A>G14PxSNcgi*p+fXyJ68*DPhU?Z9HvO$Lx-B=ZnPyWdICx(&2hcwW zE2?>Dsu(XEzx4x4*UY!d){QH2!UV3cWV@9SBSo|)`;mGKpMr)!y%CndDM8@26Vf`6;Bp3_LM=JeEv2H}WK+`*^tFWK(-X1kyHTyfXpk8y zb-MsAj<;NzO%P33>kJ~-r@ zc8#H=S!)h6b=5DEk0tiWcTq-U8_B(G%GtKpbkZ^-fswJU`Qr)CQKzci0C)Y{(^QDd zKL_=r?z+{zz9z;;F=4bj9WPg~pEBrj7_uess0r?q=lsFUrE|;1;H>3mD+)t)vd8mGWu#C&+*LJ3?p0O=15f{O1XE>b(u% zFyG>DSgnD9v&{6kxWu%z%4`W3*y<1zU5n$`&4*!@O2R|`k7v!Jn}B7SL}ZH>DD zQ>h!<7FPkN8YK3Y6fwpzU1lRELuc_7)C5XztzR@IU4{R2ntBa${jN2*M?=i_qcJZ> zV}xq_w7)gd^6(G37VMb;N(LrdV2@f+d)?f`te)_k^l+9J2!yB>#a=`f4GkUEAnn1U zo2sQU{(0}b^;7dfaH3yUA~0^v!qyp>y}~ZKE3^HAxid4P(K@6qa+GT{HYhxxV7U}; zY!$Mqe|;T}@RXh&`*K!8OFkm(in_WQhjb|naq zDM7>Tld7uRpx|}sZe3{8GQwO+py=RAeph6&)4#?zO8hK-tITW}JtrtNj-qZ-TZ!f$ zHy<%;(b?|cL|?D@MCUaJ+th|RViMPp>6ax%4A;=i#TV z6XI8<3ukK1tPHh|Vcf*#n=&D5TRi2!eFg??T2^!=MFbkL})EcChQ7Yw5;!C1NBSLDS-_>WQbTltt9$PAi4 z1M`2XI$8(@`VW?I<$5e})fwf)vSKu-P!j%mf53cXFB4$75Kdz6a8OH_BOkTgaMnm5 zsEGD?eu#vTa4Z6BhE|RIaU5FG8BnA(#nM8Y#6!WL1BY>1tHDS3p6RQcaS?jz0pq}Z zHc0!4&HyC@@bw&2TT+K$mM4X|c5$gIj7EAlD5F2LgW1{~_!5pNTO-y`gb%?Jhz$EM zr^$d0`Y*|=n=6jm61WH<(o=#P1Z8wOjIuFr3X~b`MZVd7uaHG(P;WJ>7*8v958Z>l zLl1v%8K<)Omfk7CU;P->Xecv{sAzk$tQ9fNotf0#>*WZyXANf;=DUue;KR+thDkAh z*m%yuG7S?7spmkN^SlTzpgx0 z$A52fzk@oMZ6mHKb=1X0^w+vJ;14ltJInG?i+y}YI%wVDj@|J`Zx?!z%eQ6H6N;H^ zMxR`gtGa}oTLkF!5#lZTt`FhpoN~>67;-n?t2wUU9Kt#;_$5a9rZh%#ciPyXKKu^g z>^&Ax$(TRFA5yG`lg}zacI3~(Q;p2G4-)JjgaBufXRu3nQvBco0+3-qg(lYj7>ejR z$K6GP!`6PsEJ1QtEs&`j*bp32Ur*DymKw$P_p-zv`yU)M7IZuM<7Rm0P^=bFjrp+# zcuGVbGo%=NX4iD-ZITZS%y|Y`3uEymTjweVhuU72>GN_Nyh8<6;hh$F{%VHAQ~VT8 zY8%>xb*iXIEu5q{AzIAZozQZW8&gZKzazFXliejlKzO} z{r!pfpRYp~98LZ?f&Yz`OZ~0njb_w;G`MLXpVwFW!62Zj^Vu6R3wNZqiK~#cxxx$} z+4UxJ)r zr^y(kkn-0o^=ubo`uX%!FOVtKCm7u2atq|%CK%W+<9=K_^7`@eCSC+sG^Y?&y(p-r zNfMFn;D3AzlqKt=s{cCc@?H?FA5GdQs4O@y+Gpje_Nt2LnRz*$;43phNgK0w$-d<- z=p!=J`@zU7X`P+oJeJJMjJa}OQPi2o9o|rO=E7d_SlO+_oHED z1vB+g@VpOtO5lv#DIhrRdr?$cfq^09$NQAG8d(=3lmKdOiK9et4+m2pIe;n{|Jnm> zgRco)9~(fi!eT0Gazd1F3aR47lodbSwf@?U3s-4tDshUW&@4E*OVnm(3SRZ;V!>}tY0mFBP( z@UW@FFfD70<-)5}4i(#&y}UBzqvdv4kslW;MI;8eX27HADsu zYOlspr@yMxtz#f=o6f>ot(_pX`^!9OCR(F8*^fTwR)jwJ6%xhZ)Y z6MB~DO6d=9V-~wfmUV7d$xs{QF%e*ucu7RvP*~{ndIsQ{4H}d3&x}FIF?r`h=Yttz z^=CxgY0i+W##skpGu>C5!lvukI<`E8>-Gud?VrrHj9U^pR z@JAA1I=gN5|s?Se9`OOUSuj3VQU=fwzgN2y(`PyksFQHaOq$i`ZurAX^`dvN+I~)Vz z1ME93p8JxAC@-wdm*H!G0tF+@QXv=Z6+YXsvV;;Dhl!AML50Es!2IE={aa8t{O?$oJpxKtMRb>CRfORL-GXK(696m;n)FJJi#shqPA8vFLNE4TwP zlS3Huh1Bb>i0tc}=kvj|luA;+kc{a^`_t2$$8)^VZmU>FsugT`El+llbaj_5-xOL5 zJ|j`xk~bh)dLx99sRTOPT_jSHm-})p24V@y2Qe%$?ssZ_d{s{vwWfBg{B;;>#?N&` z!5<)(f{ylVxoiQ`&+WKtv>io{J)}f#c6rLETUI{i+%79-`O0XdBA~xtkK5%(TPcCy zk`^F}+Nm^ydDMAo5x5z~Q2u3qDBG|orQ%YtYIBsMA@H#`1Xvm&;DLtALZW6UqK3wJ z+cO8Ot-zI$G~=I&#My9#%Y5Q3fM_>ykEbLl_IgW;JoNN-!xe}yyC(DX5JR_u!IcGd z(d^*NGmV6vrs$t}TM+n)Yg$`(HqNZKNVCHB{#?DP$6Dc`nou(p3TyQ}hOydW@gZk_ zQsp~>C4)zN{*0}`^x6L1nr6R601rB#Hm5rkz)?gL@CqK5x-an#l_HHfIw{lN2aoVo zJ9=8_#q+GF8NQ!&qV>in z3q{Hi;#V{KQ)~IiS^Ge?y>H1)wW!fDwX%b}iobj^^1llquh~g6p2p0Q=;eg2&sU4( zR)!fmOF!)TbjFGS$}jN4ocpR$`o8rzVK<6&8cg8}Md+Eio!Ctb7#kGg#trBu3eYQGSB6 zkO>L-e*wJuZG+L)TAbS?9vLfz5vt4s5f<_=L8dyGZhl4~!qg9(u``-)-8)qxnSeF@ zO-V8LR*Hn=Z5@;%$w=_)v#;^iw3{*(1CL6~k6_QZjniy z{*1otsGh!pR`A4u-<5;>r$_`p|5>h56W!Fvymt>D6U#zpp3Un^EP!9nXMl^UJqETJ zfdgehot8V_6jfAzLZ^vTj<&tnt(RKaFUnubzrBNOec4%N^*iP^Nznc{zrq~U)N|sA zq^L>TG*hm`VnapH^@g1?Yb?VcEjN=a=!*Y|gs<2#z;{U$f9UiX$wMsHrZ!n9*7#a5 ztX^MybSp@sUEd;QKp87ixnlbPMpO|^)4sH`(to@mttfi%g(G^^Vr7LR0{k|u2?LbA zL&|xsFY*N25KF_e7j~_4yXsY&!w@K``t#lI#Jn=F!}9M!3Io5}O5vIXUMb%#ZCS9J zL+{Iz;=8j88*R}P2Ld=X)LiTE?9UUwBqA0M{@)_+WeO}D(OAMRd>&@%jMbhmT5##i zpP?c+mF8A6V76v0yMC@L^=w_+v#)^`9ldULqfvpJTH_r3(h5t6uzF7r$ib5QTpzS; zl39u_U0W22qsDPfp2rePwnOMJWqM3%E=@WzE=sRx#k(!(MC$KI+_SprP2@QQ7Qc?f zY){y0rb*j#Xu?iJl!&$Tshd(bh=`TBd&8BfacI~pkn`M-%F<0WqZ{Y}zcBSnPn-XU z@{@)!Y_B`m2oo{0zQts!QDzfNBkk{%T#M9r*o)d-4eL0g19#olUNQihM0~H<^WOYD z=fCIkh+j<>DJF7JS3dYb968ZU~pwejP1$B$7bi*lPPVCj>w21Xbh^6MYu6%uD!mExQ53 z*XAFh;x`v7$UnXr#CWy(#uvRjw61o4xiGMDG+em#i;1P; zF@G*?BNDo*o&5@`CLX*R8M5kgp|k)Ps|t^x2uP2l>HW9z)JS&=#qiJzrRMd5j>*=w z-E)UMf9v%UCCBB^(5CW8UD}gTne94{n0hX(X0}-DCT6VN8MZgD+QOq#+Fg#e?Uni~9Alx|^?X{7NDb@j>PJ&TuURd|yv+-%-+0ttiGq zX`(15k{dZb(($ShR4qlbRJ#2EsO}MU(&dm8T8leVONrg@9IY$IY>gt2>NUee*g`v= z^9?t}I{GphJtB~5m35VdJ(aghJQc|EnD%M$`G7WT4<@RtMc%$@mlfKZPUVYqkm*S4 zYB1;4!@8?b8l+|-#>`^eNk0*~khWZIf4*=XyLYmQI5O<9O}f;k5in=%by zUzkUdZeK||7*P8?v%F|7G%U=VjF@vF1-on&*m(OID=^LcBwm%-#ab`OBB{X;zv>_m zQI{((AM;!KdIydg&M%yR-+LBJl4qH(2xjp6Nc@939qRv~#$*%JuYJXOK@ zhs%|@EBNLj{)GW5jZfSk)@=l;E8Aa1ei5Rp+7Imt7fC{Pvx+5X(m=2F87c03wu1kL zNUlBi3V+?e^QKdaUn^p!-+`w}>+9LTpv_{1(D#XVZNW^TmEGxU$L<+{ni(PavmeUI zSn{sTaxQWPygcdU0sAh#TG0me8G}0VHW)5N;`M~=&q?k9Dz}sK?(*f0Z@NH{tV{@U zsXCWie5oc{GC+gjbA$d0R>87uwyVu&eEWdREBZJm)0N6DOTF?fW&alcd3KF_{Q2dB z?{{c*=I1)0q~jiJY_nphQqH4Y0IDE|ZRrx4Gf7%Ru>ax`8||^7&zlQK_%|Q1;NZYb z-})pM#xo{jkMnvY6e02Ls_yv$`2B1m(_ha&g$6WZ^arR4&_Yjv${hPSHE3r6GGH_SO7 zxAEB(Ly6qaRHqv&dhh2u1##5$qYo^|$`@DPy7I!e^xj>P`MHc5^H+Z|8A(Te>DC8R zs&=h8c3+S@TxQewT#p^aeGz*f=CMbPwl~ZS8u4IrhA43ig8fl$+T`mHhhB|k;gfTT z6^XTtCrV2*^weawtI9Dv`-Ev7N>`Ll%9>HkqI8w%dWi-LXvImSTVM?n}qI&p`^HM z1^?4X1?EkoJB!bz4#q$+{q_V zPg@hu*p53f(BX%7;aXZFJE1~yJpA1-+DYLR3Fc~dH4|DRN^vvfx=vg;mGvA{Ce!r- z3gn9YUda}`uauh%1spo;ITN_A4(L?pE-!kiwJ}H5M~N+20`lm1-iJw_CB~660(af3 z5;srR5K>U!kjWUg762ORF{vuN2F|Gl4qWnP&dwS#pDYW%PV1NI zyIb?Rkne#cZVq^+B#reTrz{KMhDwMp89#odWQ|rL1gIYpylFH6jNo83Kkt4YE`qC@U%+Q^W;k?htdC`U@8!RK-Nbx+lzPYJ3t|N@A4Om$&a_CfRKKv?EN7?jNkLh zg?)#LpRtm^6%2rg?qHpBB6!_SP=p>c=dmxSHH^9lOdn)NH{6grT#8_s=5AU_iS&KO zrzD*CBP(+yFX7|pMp>(&h-2SbiiK~Loq#_dn=T|Oqb6RkS6GWAsP~JC!-s-?I-$5U zF-c~n-thJZ`}x0V5Mzc+avFNk!Fyfs`;w{#C`7{x0)_~{o1fGMoF<2Hv{H8uOuE&R zk}NwRL^g@ZQV}p;d%4+OLB6WR9@z9;)wCGu_J4Tm0)L#+V9&lkREfyW@(+iXro1}bPH;m!k$YL>Fmw#xTfwgI<>q#UeGfJoPvo}sx!tY-h zw(Ldrr_s;&owJmj<>f&d4L1Mq_QD4?!|Oq<_9TbhH<v#@DF4W#@Ks)HXr+xXL7 zts?$$@(O2xwBlQZ*}fQWN$OffAi}i%kJrn+7L+*?wJdo18ju#4#bKFH{=mE$d{{v>~`)A_RMj*78|A`7XHGtI2}33-D?7^$*T&X zAZIDl0Zo8pg3x$0-3UmqD~+-(_J^WgUOU$uweTiI%2pb1pVcY`%{7c>2xO5;nvx0RLatuQR9_`xxOvr{BPBL535HXCA9M%LpD|Gt(1%s?0`lYx80 zeWqTfWuETd9b8 znr(%e=yjZ2Nsx{A5#~c-w~yUx;NQzSx#i7XBw*JO47od`rNO)FCZ3czp@0T#t^IGl&_4~qS^5??m+Sl6G#HZGK%`@}!_j>NN(PhtC&o(!P|If`&8xMf97*1ez>XzcW!R1yQtow@6o(;UG<6 z#vlf{ZXgH34n_*-TTILGf;Q2>-<3Vu@DmEFpPIw^!`P(Gwq zNLW{D1mSxs zob9UdXJ#z1k*X|c-T<)zG?rfH5D^`iJWO(;+zaeM+JM6BNa59tyu&M#Y< zUOcr(u;e@mO+95!kYkUt)!<=-&=xs(#Io@m`GUdubjz+=3Bx~edLGM-E|Qg0C=wzk z7|{>NqtY#8j{WLXh;X?}52XeS)>k_q!OaPEqCS`*6_^vo*jx&nE`9r|SLN%BJ%m|r zs`%)}^q2~SzG8T0R%6;(1v>c@YQamaggP|4dJuyF_Q)JFDL7@_eD}GL$&#+j@l{Jw zOn2C$WspQSHUccS9|QgDEKR?}$T}5=L;-NH4xO}jvP@t^FnTpLZ zACs$4fu_>t+!$yI8OK3cfgwLj@c#nkr<=0d&W#7a#*}$Lb9Ut~3+JkMD3pc|^+e?vrOA-;NOSPn zxbW`Hm8-{HgW(WA7g%49kW_t*+Vf~0ZFQGyRBcUX5h82kG^U52wxG^wR%DL6jAdW= zQ-AhVY=Lsg2#!Lo0F-p`%6SerD`mK426S1NmQ@f@aR=Z97%Pz87=4@Ah~5ol26AD5 zBhAf;facFL7%9Yis+o?@;PV%$)$;WlJbVtThW+IT2%7-6J3Dz&OX(n!=o(+35t=o> zx{Y#ZIf(RDSLKXMVWKDJWYdcGY9zEaJq~@dyU&8q#$SA-U%V)q00&S*LOJysP+u?H zXxbf8*Ij}hKNel!=@ z)K?fudU%+XVBCT^0i3}Q?H}n56j5-xl&Z86@X%0XHQ)vDaOhI(T4`J+8#wV6UeePpW#76V}zj9PPYb4`){)o6)NuZRaMW1L~f!M+{ChGNhb z^7Fy9w2Xh)6am-M-HQd~p?V&vqUuu9TJH`pLVp0+I^OwABk}Incy9pzd8tG zesl5I)Aj1CpD@LpF;+LHRx}Rc1(dPv$kvnh8D~*QKsuHczIl<~f9;&w*H22bwC0-_ zT3hW5)NB^TG>t^2myjr9;Wth+hEB~49*-YCEs__)g!?oDE9gzMETjtM=7aiBR@^+= zVk89!UD4;#nY{5x#X};`74XiljOkU~l9}{9G%JF6#;(nRg_Lj`gg)Cicbc{Qs*s8% z%hivys84$mjW{P8c1GdvDgu3$G#(aOKPx+|7pf(XH2?m!gMUmEo#T8-_V_!}x)r|_=a765?mCfb>`cr;NcR%Au))1+|s z`A41{dr%Z~2y@>r=*Ffg@*mGrS+4Fe%-4t&@2ncvr59+`e7eFll4cP{IUTswl9+31 zS_Zei9@FQs(W$Ihvyq}%R&>hlu#%`YOj|!MyE`43J2bt1LDSmL1pIY-U0cIJUy83- zx`*DrX7#9QyaGdZOpg~{Q0qxho|?uBP0yG(3fir*l7aK(a^~V_=q3uSg|x)181I)9 zeb^G52QG};RdJpyy7E&;{k4Cp`_M=n^;^2GTU3opc(H~}|Fvo3+CHtFC36}KCupbs zNIb*YGK*E$Nc9?BVQt=FcP!v zq;9N!T)DQ~(z+SC_WZc=crrkH2lS{~u6Oemp7S7$Ja*4ehFetVHfMb_HJ*Pmig?I9 ze64!@^7`ZrJH9&gW~YU?kQF-ZyVb5D8Yh16?mRM`5k{)dK> z+J?*LRdX<(i>&@@4v-_ZY*r-Gi^OJ@cz40kR+VW_^;DT>=-gcfM@( z{X$7ZdL>AWt)u7Vlu(@JwYJGGq2h`uo5ytg7C!w!t7Lb-7R7vJ-(tw~i=jdyp7k2I zv6w*UiT0#Z7tl;U%efx1mmTZLr_&{ibFZ2eThE_%A%QqLd9;|~+NA<`#-xa=VIwoz zibw(#?4?6Fp1UsNWYQkMDtXW{@N-|G}&dX~sgxJmwk5uX!Qo{3I5YIfj)rn};rm8}zeC?m4ImKQ?fw`9ei6 znHt~XI%sb85lNA}avf643$S39qfw}ept}~QV4>296gL*ezwmt_7GoC1)iLCmEBH0F z5h^(Oo0<3$oHM`A6|IFlNLTRQ-TKFVLVpwe3UP@Ea3x{C5KD91Gw};k#n3E`GpS?2 z$ADXkk-?ouBl8`Gqt0fyPkq#TL6=m#_{@m}inX0gu)Z}EQ^d#4~x!zfwzFWWY{Y#Uv+ZL`a^ZQHhO+qP}Hs&`M! z*>TQH>^ZTozWetgzZGj`KDleyp#ROEhLy$5Mh>s_c2)wz8yiV#07rtTQzK-~rTkn&rQT_8Zf=nKvxEd;zU-Ytm%r;if7 zeZL8{MW9$68*6*+W)2%74Z*Npn3ySt$Xu-=l-f;7=|LpmbZnLt z8;L1qJng7{kI_h7VC#?DRGFR^_JkqW05vPOQ_MX3S)Q`Ow)P1;-KdJW>2}Qh=xv_I z*5>3`YQrsSlI&3X;fr7JKPYHJXve^c!5dj^^*Q^wqSTkXvB91N#4n#FZf_5w~RVYl+rQ{Irod0WRO zXNAIG==k#8AkgQ%qVtkYj%@q}p4_Vf_n2;mcm9k8wkQx$Op;T0+wmbPxX@V{=y5L` z4Y{g3yePQ)_eTO#Uwg%8amuBM0e%$QqjwYi1|!;puUFK;^WXa5GzJahBTC44)C>1K zjT+w&|5X)r9rPv-!2p1~e-^O+TQ-WLlY_I7ljDDQiq#bUQweK1%IIDGP}@XwV@lM~ zXoLcEkeB+|@*u4?3apwN%Hoxge@*)CZYv~-Y+7`{!v(An-}Z3(ylyMFgPB!mUs}~S z&1Jvt*}1(b5zzf@!cJ=Kms#+mAXHr77 zXd*M#{Tx`0D|=?P+MF(u|2Qfos>zt?0`McOAdEn0TG2qb*Ha(9-@QQX|3m?|*w%(q zRfGn?2o|+!m@H>7J7QKCVG&h^7vjqrkXanWT@Kd(K?N2W`-}vTs|eOT#13i40I!pV zQU@y*C2KN1x&^oTI63qH51>pZWzp&Hu`?76{*=a+F25;z&4(u;@#=0B9v`wJJ35U*B zw%SO6a2zk(k;CuJ?mXxi*atA<6=W>Vi;Qw}3Z@u9Lq?p6osT6*$@_{?4C$ws7*0@P zSOc6Yd<^cYxIQt#96)F?)LoafM)CP|o(!-Ouhoz_rmWnaU1VsJtV^IY(%sF9snm6< zVqI6x2$G6+QC0LKR!rMmfi6W=y;N4hK$SX>yDe6aV-3r;c1jL?ZC7@O4f!@TEgXCzcX=sWSd*&ye zS$N@{a%f?~WLj#&JmLZ8%pT<5o2#HIj;~5Y&2+)MLP2^46{sB!>UqCuURcyS<~*2{ zIP4w5iP2>ke4;e^?-&EOa?cTrreW|l-nTjU=GYEUAbDEmTkCOclKNMO*(W*TyZY+tWwucNnjuO|F*t zzW5Nlg6&Y9*eL}r-pf+7fo=DOzkr1u&?F0Q332(Y%&63HFM(oZ5fD(V5`i3rE-(aB*kP^Z*WQI#9MYbr1W3d7%DU!VF z?YH9b^fAuR==(Y>mrTZ8`%=(@cf?D(G`m-O_>T3qA=KsT1s9U zJg+_9$Av5|n9eBAb}^IO=X9cq>QQ1x2~YC2E&@2Y7Lcu2G)DHQFTbNbZX(~YYUf*K zUb`p9(Txfcdl~fRbT)9s&{*27LvjxPQNa5fC3KwUry@*c;CM5`UpY&l8Y#N57yeS0fQfTo*SBAoqTC4{ z0upb%<2of&%=Xo2@*sf>-F@~_j{S@~^0Th3d3vo6?@uo+uU&cj|3+-U;AXdzPb=j1 zneUeHngafF?;MQ8Elfd~n1WJ1;R3?)QIE856Jjr^X&xV~b#Y|vRQZdEP^&>|-TR67 z-L-cU+!JOIu08yG8EKf0IpRx1E36tjom{r>f8*I_Ndls1Z~%ZntN;5p)Bh+lwKH%s zGXGDnFt3H{rf9ptEKjaJR0n~wsM?Lp&r|Bpay_xtAfW1`W^)zkLo zqU)#`d8uu{47}&xm-pE3)%&1Qu-jZ|cg>N;v{Rv%r|-v~!JkGFO|-7XaEwVB`gIa1 z@=Bgi)&75|NeA7sjA1IuLz5gCH-(||EDl9M@nm1z= z)Alo_=n3-#VG)CkivoNQ<-~IHwy23n68t}YcPh$Bm!!lbler@nNwspao>%TXl%7}4 z+I_RHU#Dnt=kCMNCrCnK#coXs?R)}QM?X!IJHj7;C`hw@UsPsgzk9qOYJ%c2^S zOqgBblUe>2D4kpyg!Ye~hp5Mvr^%xRR}6=?8CyY&4~5d?lxob(isg#W8d>mM2^WFr6>z-zsh<#!Pz)G(Zsyc?eqNj-;P;^kO=J;Z-e<w1L`#s)!89b0Z#L)|oN)8$5 z2K3x=f)7bDwLP9t%z-Mg^2_S<&Tk6>@DT6dByIUY&~=7-@6O|RJNFW$&}aQlK*314 zC6Pnsz{VxE{lZ55;3$HeX1aq)4W)P4AHgw|UGQVq)WodxEomy^!d$C>hed_akhe_- zniLJOj=>OM(xOB8L-JMwog|I(&I6xdq+QmO^hbVv`iUM#jCsAlTZl7>N8N4Zc5}bZ zCPTQdylnr@+uiQx|0AVf5AYF0t@SqcFQfdi;f@lYD8Zs%@Fb;Hl~NpK+z^#tAfgqH zMKcCM?g12zk3b^{@ApF@}RwDj>spZe{v#3)7=*T@7<5D%ZMr}287Uy3HPf>?*s zf_&|F{t4XcO7eb$^E?2#Y>PXLw>5AEW^(TbTEOMNg@w0+3A2n9c2IGskXEP@FkBy{ z(69$*5_Zt0c^@)G$GH_lvspKo#I*zbDbx=~E0opg4Q~Wj0g~o+VBIAS^}Eg*5|2rk zb>Ashu-{{6^gtQ56UhIEw;uda2C13_M0`Bh?Pg=Y$(m&S%LyE^U(KUz-)p(twJG?4 z!=v<7Y>$d~>XQj>e*z|3hiR6XN@TXN%1Cb{EG`)-oquSDY60D!Mv6I;?!gx-}pcY5J51>jJD~I8eU+$1z@iNEeA}%)pC`>gAq606$DHzngylJ z?3j?xVEmiFB;n5GoX3c&c9~* z%yw~!s8|O9ji-?&?JwZGY{SJI0mu#(ekW27lwiA(qw!@^#BZ_No86%i!h|q)&)Vuq zvQU3$lov6|{1M=z7Iyy_+TvDE>@a}Qh|k*5XS3p3} z4QBKo#t_4Mi&@!-CBr2HH)-?L9W&tO^J5OE3vyYoR#OQd0Bsr##xJ@@J!8k5haW=+ z<%C@bErD1JCj&Mj;5+VGDt!-mND)~(4wkH8&suVVmyY&9@d6E-@96rvM}9X<*HG=$ zcA`&^`8{C#P6MV=e}v_64V$5>6L!m++Vp%XLKu2tT%k-s0!^I4(Y{9Tvb%F&Nlv6l zuM!?Rn$S&8R&Z#P3ki^tbO&jJKU=)KOFoX!G{RtRRjaB@qLVQuqaop zB@ZDtRFDv`y+i?Gs$0s>tAa3f3!4PvQAuic_DU?)CWL4>hN;$bS`hte>O_cHO zaHaN)kQ%L>z%8h?Ymr1YP^K?&-mlzLetIZot!_Bb->Yy1P&9iEUlM~j2tNnrA8+iL z!8UkEkmB;VEzX{U2t^m&2iT*}^W^L?9MWN3`oeaU{i+OnEz6uEa6r5BishRt`Ul@qb zZHK;J65JQNbT3E?4GdCr(adhK1#3-Fm?=H^;|- zc;QkFeC?$!W^7E;D2}T7$?{V6Mj3rwL_kREmDmfn?LjQg3TpF9wQ$g;wMw9ZF4+MY zfK^$&Ib-qbz@j)bDg+7I1WPCUZEcW7MkbOZ4R`%4NA1fINurzgC*D6JU@u87*?`;? zZ{J{Lr1J<6Qoygbc9v=!hk4OuIM(5t!DQ|rO#2h>qQgz6exWV80|$-=LcmhZL9Xy{ zP8?DO*pg4#i072S%be*iR~;r8BU0HFo<&$oRL3*O@y!DjWwrbCWAmI%NkgFMDBqkw z8_ALr#OyUdeiIO}w?GxyG9tK;3}EQMJ-?Ipt`!Tr@k?u|TIm>Tzvg{D^}f?$EO=TH zCr)c3#}{h0{IO;Rb5f{h*@y7)v$g|+1;t^23#a|CibXfDtqbp3TF_1663Lg~mcbct zjD9$VTdIVyH91&T9zaf(K@p_t;wreiu&YJVG9ZV4y;ut%5n#6@8PVUnUalSzbBF<> zS&ujj`Q3~criVzIDt~Ih`SOi!nAR+`Ux%?&Hmgcso)LA+_}~;gS%iYzvP;smVF}f^ zmp{j&@>*6p_$`_W6)JdJ$&zsm0W6kIT|UJ(d<-VMN%>b8s1P`O9-^h-3$z*O*q?CF zo{xU&0#VZpoUBaD-!(6jLbyH|_B>yOCHsJ$H4c>S0Ttbg1cE6vnk`7$Fp82|6=f?_ zh~9y=8gw$x`iGK^1~o>&kHQ3^9_RO%F?4HQ*PuyPQX%znR0!QaE4f5>fl|I=iK5!x z#!P}aoeq3hSduNp+Ba+cr%_nq(5u?Pr@sTZjIUL2#Vlog1Ds)MMF-mAH_@@06rX^@ z*JvhzdP1=yOGf7#gnEJ0L}krF!Xa5o6rIH%va=67h-GR)Yg^wuyX?q-{y{S1d>u3` za}DObCJJ5|B}T)P(MNd(9qq8C3=$V3{XFEAz6#|MJkq7zB%C?D(KMcfcgo6_-2qDN1Al;C}*m4hA_K<)={(F;9A`;Joc~Y2@tQXWt>`5W=yZG;o_?m=Jx9A z`J1hlSsooQ*K^0z`W^nyo1^DlTi&cIm&LGv(4t5n?r5L3K%!0xVDXQ#`&_a#h! z-!eF)<^Jq4SK!ncKfzf16FyxFD)Haws*d?kz>Ey}kpinoH)vfY+Xk6FF3g5;UE$4R ze#Y_GQQ>$!yo9j`JG*BP;#_s8M=07Ke^{1<5^)#gT}tal*aDJ!3ck5zAEl|b;gzrH znNGTXW(>2t)CsQ(+%xc8{NYS^vnfLSSL|brTaT*qs8R6OoCoY@nh?_{50;u2_@zE! z;e4!T4HV~-;H6MLaTLV`pviI{%E*<7v%Gn`kWU2pM$4jVKfBW{6pHfrJPpW zCZ8pdn^G%T&ZbaG!^vXgazqIA`3S2bVDv95v?lSQRs?O(BYG(13g~_=Y?VYu_o4$* zd{8UAvZ>YNN3m!?kO7V7R;o)axF;%mU1x6rUe6_T?4?9$3)L16T1jM2&8jHR zrM!H9vAM4@nI#(9LJnus5Eko9Z3S@7Ud3d zq5(rl`ZT|4i{?J}!M7$c2R#^wtNd~?D8iX(6bsJj{BeZaf&_D`Mn9f>fB6CjDwv{C&C{vq)$1D)f z4hWupv}y*;RyXC`HRd8Yizp@8^L19nL5@A^>yVsn%LclJ+K{*8#a>w=Mdb9S%D0KQ z{AjSZ@;syByo92Aes2&?WgVJ^qJLgrXDeOS4?ogp*ZB1%9@(@l4<`!6&;G_d{XiIQ zfNChTFjg>(^HWX)MbaoVvHVSrXc%eVQW|g!%d1C={^wyj`Y=&8ulV?b|1IGSkALX; zXgM^X(>#iw5juP$L2E>>0T}sewRYeV>Ngq`8qyl)x&a=-aGMFZU0L9~R)sa#T^VDN zW&Z$NeO@?_?S#dN?8p?2Ty2H))}TRV>jcFYl3K$Ju103lS~l?6W^kA9P_1|nq;lul z*yVb^mC0eEPkZEPj7@*JteuwNgCk5bqUgMuQ*XsJk8&`zfoAyA^TZ1k;Vz?M{UOTNYNjz%R+FvCOYzmBv3C%-aRy&mJXYUmj$$`Er*h_(nNN?{8~y2|m(vV;Ug?F5Gf_l3(--eKK<7jYPz)YX0@Jim;fW{tc#X+#>~XUjD-dh>jY^5gn*ac*_o0e7or~* zneU!$T7i)%iejrN!zS1tbXp;q0RctH>G7vJMnfJMeKf@!(|UjO(I3wqox3+qRy9Fj zRz;kOlknyMPVhQiq)XmnXqw(<+Sq(Xdj&GOH}T8c<=GF04#Jx8ZrT->HJj#OCkJI( z+QRQmWy&&#z_-VEL?$?}ffmxM9!ad~F_q&mf<60CP_qR?@z3o3hYbq6>OXPO@ph*b zBJx!uFpyb8vcDX{{XkX_$`2w_VfW~X*n&5>TKLz5yWIVwFAxks_Z)xulHhjPATXLs zAf032V^gz2Om5zbIn4d`dd&#*xQ`OzR)3xlfp@XOm7(LlhAblkGQk$~+hQW~q0qOK z4brIcsWBDtVNPx?{v8)D9I2maH3mnC_e|6}ZoF##nmtmtD`Ji}1j)+xN|Y;Er|7}k z1$F)IgD6v~3D&$QjX%sRKMPlel+*u*O6n){9f2zYpq;*w#BIu-Mj!D?f~8xz236qbg5z}@XDo=cmq|c_^59;I{>JZ7QotLRMWw2pUP0; zA)jCp+z+sORt+J2R;^rnjPbD(yUQSYJZolcuv!u`&G@@%*8o{oO zxqCi8n`=Pws{`uf0go@>KhI=QY}UUjjEY*^I>xk&2vour{$Q2NV>C34w%AC?R4Wg8 znM1R`6TXUEeo~OB_AD=O%Dyb)9m1%Dmpo>(K*g5iI^3yNfA6ns4^+&Q#o$gQsMs4& z-@i0|JRi3Knxo{}&&ly*xL=ccO7Jr@9-utX8^eq82d%3wT|a-FkqIIup^HO^i^>T$ zzP5Z-a1~BYd~CHZx>Qb&ULzDxLsKqxAaz#DRUe*)BwjkSD0I|RmT}`M@46xHs7T0| zZBUN~t_=BEVPOnM{PAsFZ(TnjArXnvJ6>(&j)dHWt_@$<2INUyf0$^A3-D(i1eL4~o$Vn9A5v>v@O-5wW>gV;ODZN%dt~IV-VZj1Q z#V!NpvkPVsTzQCO70jGdm(Lyu!WD8B-pC?<@D7c%wOu4yW{6G*Iw$@T{_qa1=Z-qH zygUzYjje=*Wjv3GhZw8pzpGN_+QC3LOZV#?w`PnfcnJ(=k*yJzvM|;3nB)70Mens= za!kOmZeN0>)$K8We6sMy0@`L$xcft6cNvtmc`5?f`6TyPgX<3SU|lo!0-+-eBmq;I(TUc3liZ3bHod)K!-G@)7ETz(La=QTn>V9w)ie?}#(Uxq zA;8eZvkIrW#2!7O_F|xS>?7hRJ(*)OUm`5ebj6!oPSJY=eN`au3u`_?cXi|na}Q$= zVYX(Oe5&Y`TDWWY(Roi*77fk=lDdolXJftsTMl@5@Xep&mvJr&inYrSvDu!uI8VoW zn^buA`kaN>Aq<8htjThOnPP_7AEl^_-&y4NGDK85fxd82T(rtHzpK&KK@m`bq{wT{ zBWvit=_LPX9SiWuHPo+m%fVDG_zEK2kBfkxoqZ3^x{O|iETZIzxhan=MQWV4u+F#} ztmB_Dg!B0CP30}h@{4dv!djPPS=fB-)_gxanN3Be2mHa>GAe&Qcx-l*-h1TS-U0*! z@uKNfBS_M0y8v)u5tL27DhC5z%!Wpq2a*8t>Y+#l(dkyw-V<` z{otFoPw84CW}Rkmto&9|zia$rgVZfvD<`d4wZ!Yq%7QDj0@1k##W~5>;MAei#R-)Zt(@koTV|PJgA<${q!LPc& z<;>)#)}f5z(NO!Kbp=M4YN0(-%h~0N&ddGbW0(Mc$@WN~92Xao&Ax^c^)58x~>~W_!e{hZvyO1B3+>S^KPobTFc)}W?o#U0=ht!s-d25GW%zSglmfM@gell zvRN{vFoIS4MTx-T;)N~vYt=bI6{#HS>l6(wSPvmln?$SA)OCO;&Z~&cx%t9YYvZQy zs@A&A#l{c?tEr>=$?Iv#vZBCr*4y7fjKNEZg95=YIxluu4E?l0dA?EU^O~8z|BV9I zlIKZR@z1IO@s9%ce`{cKGIuaBFxIoOH8QZ$bF{aza5DL4+Za~Y`e)lf^;xYYOv12` z2dx#+<1x;>tO%~((ytVWEReTIk7<@HkW`fE((4_JB)Ldn(O{pVKn&+ez1@oS2uNw- zYPdQfj7$AETfbV51nBf=b!*bjJy?2sH|q{nR!+`w3S?fY-O9~2CM9KM)wmR)uJiq} ziqW;v&F16RIcYl?vTrCTkS3hSBoCRyqs?IdVVwna-|30~`Q?_H%ziDPifA@#?mtu? z;AlL9am@HTFYwVdjWZ^3LJXDq#WCZ+>lei!yYUU-TYF9A4_#ad3Iq+}jH3t4EUJAT zsd*@vs7835fPID(=LIroa7+T?_I|+xl*gpjelEXnP0Yb>spk?jZQN1$uKV~}(lNaR z^Xl-043aZgcN=?Y5-Rq)?36pusws{D-!DI`b)qb$3af;xuHwRj-)V-qfhvZQer$Eb zQw*&nqPKqbVlQ}8Ra3$VZdpg3lMy{$aXBD;n;?_|f%gp8JTw+XS+J+;dd3(4lN9_p zphy3{1n4GGLaPgcnytO^Z5nIHz04Np`D|Ix|K1jqjrOrl4 z{-p;r32TtDk|vZ1a=skTgHqbsP=&IgYNBTvIMg4N1e1#Txyy5`jW^I%b%o7%ZN+Vr zIw{SB@i5+jNYRJEJnlUmGjrIqfR(|S=3t{Mf4cg1={ACvL*MBTzVU7ieOR9?z0$1} zX!)aFwJ5<|w9%P$qe17!xSegq3|AY!pC+ETr%0!N`<;@%0-H-R3&==piMOBaH}GGO za0je2?b(Rnh8aq-A=%Oa38&}<{eb3tz{6Zwrcmr7<<~^d1KzPpCjE;^pQ_jO1qa>EBEWGR41(|=T0SZ zEIG@RX%$AIw8(%8iXcm2I{7+C#KJClUQq=8=YPICsY-&U=M9fCd;sR7nW3y_x zsr=PJTta}8X+G{+39C4zcl(Lq8FZxIRy^v$j$`d=Bs&I7Q}sJze6-&R@KJZ#v}x}e z9ORonvwygGz*94sd@7e9nrYjfV-be~6a@DF5xs2$Dh0Zg8Kn(HC( zTE$Is+-gR)(VVa+!uyzNzmIqJDgfgG^v`Z45zybn^ylv&ucx^NMvLJ=UZXgFC6*Fp z&o>jXBvCXD>h7BT;h;+*mW#v=lor*rSTlLJGfc0B&`<0x$VX9~%Z~(@rLx_Qf-*tx z+Yj`62K5^UJlX@u;mFieB+YKoR5iwkom#ega44w7Nf&cxEHT>x)(+iTI`m6~v)Hph zd;mqv&gjn|v}$UmIi3`jnrK0W`q}oumvxiCjxFDlP9Kcg-S!rSNqZK$2${#OT0-IK z9PjiBcV?`&5owYYG|?r)BU1-(y9Cg}0k8-sada$}jw+`0J z+BuT@)mQUhJ&UjgH%&57001670PyR7eg+daCkF!~C)@uZ)KzO}I3Bhk{rltnJ_{{Y zWMaQuKN3)ptVVDzj2LxU$2VhFQlbe9qPU6!O$A&l#Qe>%(e65GEP%iP#fiSDmGgr* zQdaKFFw+tlFqVw@ae6w%YVXp^u8tW$U!Hzmp1$9{xM0e_joZ9z)PtvY8;~4_SEHmx z+qF}j9Us45-wz#J%wkKBAs^M3!0B_%v@K~4da;sEW7CPL_u>C=?R{D2g`bqJMg#o!zoActt7niw3&Gm+LY z=V*YN+5||HAx|8yHopkU-nW4P8905GW$Yth%ybXx0aUX^tzTxHMl82hlth?x%2#-l z&@yMsL4E}g+TB8ONJ%CUQc3|>9@db|;?^=)V*B`*kr^n)Lu)v~7K;8F!Ma14c;OM*d9lt9`a%FD90)m4z>Dp-g&zT72D4$E_2f zABC#Qek~_SiQZW3q-`O->{x=(e`4o3PFEey%r$G4vYYOjD-*I|mosQr1t39ljz$QG)o-D=Z}T$h*=$gE72LQ^6ZK1t=Edxx!k=y zIGf&O8dijQ-pe{kdkRzu=bFo6c&g)UXH7@FpznYnSu~zh%mia#lPLj91uOqOpLVkM zNwa%4HYjcmHhfiRG+lbhvpTMjkdW>Luf-wDV9=0Qt(E64;~WyfGulbJY7p+9aaI6^ z$jdYvliCnaRkC{-1V07DZz2gnp=lR};`OHhsL?6&h^wzaG+ro$2K{p(hjn#v?u?g( zgE?SW@w8rT-#BQL_2@O^WQ};NbdE?Se0i(__91#-(1&%XQ&~P*-0&fn@JZkfRgzoA z`tF7CkaP|eHewOppe85@*Fl7!R+}fFL~|h+onwJ{pK!4of2JpwAg)udHZ|K&r4hHj zPjMw4&CxoYVIbdmir`!pc>5NvmN6VB^{icCbWUA(m#zY#vReVm3@(vL^faPgB+?r( zlvZ1b1y&SGwRxO@s~MHg!kL1|uy(qMo}jvB$=K`#&a!RjJ0M?g(E(!vzx7w-tVK!4 zI)wBvFK0=0rPga?Hh)X9M5pIx5x3A7RgaL>Za$E=`q~4$Y*F))Fqg7ubVc!1@a#gE zQQ-zexW!6++2yr!%biWl;Q)q+sCh!zhH$_dj!fmx0UWAzBDoCyB~vDrNn2B!kiWHz zm$q5MEjxTMH+{v;NF3v1hO<+Iw6T%(3eIbWb^fNI!oLPU?^;{N-o}cvIp=PNv z(B~S|My2yd_`ao3jY)=`ty`Z$e@qIWQ)c^pIkMaOcz-SZ2d|(M%8L(C`VLN(vp6s! zwV)Yt!ASg{LT4FQ!hkSUwX;8q7J9lk+4!xx`Zwn@Gu%P8!}H<{SNU7!TPIiX(_~SP z{%!IhL{ECipmM$ax~KguYJnL0N|Ci}NwS{5kutOJS`+iN_?aaJn9kKA;1MeOc2GCv zGIZ_MT(<4X`uXUazc#)-HrtgRq%z@7wgfrShbv=8`RR@HP;7|u8GV62S~}uDi;t`CNV>NzQ$JCUVQ}?4nHx5idgI?&p=!U zWp-pS_!nH>_CobG^UXt-LtWkqaU!OqzQhT*6wy^D9+!uaCw;uU9i1DZY}Nj12Ly@v zast=Yzg^kWP7h+B3^Fl7?BB1ib?vBSjG#f>{)@Uo%*3h(`J`onbYJE8_vcTQ(6b@7 zx$HO#C$$yUC3j5930vZoUt0 z=?pxzG?(8bYjrN#W#)>7=E#vgNuNNMdP=`^X7HSS5_oPlLWsIPqqsOmt1Q)mEN$3p z2{_3T3>s+vxO+B}N1kS@v1P*j1rIQ))ThBqZ>NBO?}84bNc%wF$yloa?!_tX$Y&M% z6!RFzSN+z0V#+=Iq)QuVe>Gxe5M&?H^PgAKE%`W^UijNT=I6xk!eo ztW#JjH~Yv~+j_L8u*!ddk^?X+y}C_g|JIn~DZgy6k!!}mX`!mJ*W5TcT_$l#Q<(!91XrV1FUK|1H5ld9%I&(lTi*hs zJ^~*XWPk_W8DiYqd^zSWS8VZokZsZk*Sg~0MLn)O-U`bMG$;6JkR}0yxe8oCIUvE} z4;$D{knP`KS1RHf$g3d*>*B@|J^ z*s9~O4Q7HzGN5ZrnS05S5^K11Es4@Dg0*N^ho`Pv)cKh@u3v5TYB%RPQyxNnzEPP& zTBC~0i^GRB#bb+!5Y(mmnFqqX7oZK5asOW0=v$A*5O^>fH zw4hET7cS*YA^E6v<5ThLoHy#cB&Z_j!$(8|wR%6N%E(+^3xmix*D_|%ZAS~5t|aja zyk5ghnh%=YM1+jujE_X5V9$v@jnMbNO#BX)K5i;Uacm`gHIbmwMgEcmZ_|H#uUo0c zW4pVev6={9Cy~kQ&vVa85IGqn?rd7?jr zi>nuC7vgsRh1!aU@nQ2#Rn_Oq-AzvW6$P$81`@bXAs^^$n9$pj6YI$hO* z@}8Hq!<#tMCE6H7MHPPkNe5E^aV30{ODUW>p53Ivhl>sS^CtfED<1MrS-BbZwvX_; z-7)}n!|+^S2bmBNf4}P@gBBP4Ss&Xju1GlFY)n$7O#v1|eblNyuIe>=@eN+bu|=rj zj|>^+kaSX-Mk?{?BC557g&Qno=3^Oh9)4OSe=vy!XqA|m@=5cqUak)Pe1!#;1z=r3 z!71ETD?n0$Q2)lqPZ-E!5&H_)nHHt(WB6hSdI9Zbg$YVQ^EVa{Z-rR}LE`(pSNul- zTdv0qu(bpVYxaUZ;(!*Z`sywQh#=jKFWB|psp^_x8e{IGLWw71SVN>k=1D9SFk@+8 zyNmaCDVTDh3ZhVXI53{+ZB*VqxEBxK$D9fLjzv#KnV-NAxw~5kv z^%Wj0sWBLp4n(>4?yI~?OwB#bb>AJ{6?}QkqyWacM#yJ@q-PbIo&|;Vqx)PI5}$vi zW6N~kRki!TQmhgThi`V7cVhYcXx%LZCzEqntCu~@HU21Uln%Rv!*CxsI#9BEijPKU z3m#?rM2l7sAsf?n5GJSV%QRB~QAA6;EKtv_wDb6waECyRu;%5qP2Adl){&WZcTkhw z7u&yO6-LWvPRU>&QGb^)OL7nTOIF@_F)r`8c$MO3UQ(eL4 zJ-|$%rfcWd88oRG1GDwxrNoU{NRzn8PrRBIInOyTKg?1uhCAcc{nB8jm~u|TCv@!# zVZB4)S^jmi(Bo5%|GBApchZZC3D7{G>m=nzH)K-n;I#=aoQFnmfLK~ zTQ4#}4Jf=Z*(xr)f^X&)ZZ_J}q4!XJ_FoW+eYB5#7@Rlk@tc)*13@=Tloa8lS?xvV z$fRZ3@X8kX<>irLAu$VcJb+(3xDYWjDvR533$n~xB?dzQli`X`j{)?R+c-ip%EjjE zE5bw)hUdsqkk~jncBVuZ^?^tpd+ly=1f>#VaHDSmx7;p>nfFJC-1E-Wp_vhI*+C_T z#@56V<$y=LrgD&1Z!G7?bFhx(a_G1m)B(9?LjMbSo-M8h~U7l)5E zJq{&ktP0Y`lQ_hzn$5&6%y)sM4cC$tZ%cb5MV=okiLI;1%fcbrso?e>>fiBodFNuC zg3_k%OFyqn%CH3`E-6?#06L~OU|VVTRgh)fX1ghYf6b;!m+iTgZZd?`TFm;mbepxU za_~b@ef>$1@Jwrz{u8ka`wQJ`Q|s7GKBK4G6B)N`S4~^^)^xYmAd$-Mm9N!3ysozv zsLS&=)SKIt-YE9__we4-x2dwJ$ohHB0;tgcu%Cbd>drcos6PLULzrW z$SS9XbfY?$Pmg%z>XkXt&D1S(9FrKkbhKrCGPJMMtLGjaD51p@ zx>lNpNPu*e1ugYWC|MS@lO`Uq*U3pGOLG~g7x0vzRaym1c$RONQ=I1~YR`HTFT95* zWvFwLKOyO&qR|qOv=>ewYcx!ph}CbXooqigI5hS`EGQ=^Qz+O^S{(~nH>7xG9rbD$ z53jIUnRBE1e_L?PwxNyS&mDgDXyhAb49shRs;G_NnmUnPm$Yueqix=TPVO<&GqV;) z=QJ~D^>e;m^|w~OKVNMpBi#j9*D1IgbrT6`#cK$JaHiEEJA?|tFJwpT`CnT``3n1}=b^-}Bkbe6;of!{bPhblSaqFo=$9W;Ojwy!0j ztUTI6JEO@h+_YBU zx=Yj=voSPcP5XMkf$CIIct&3~ZLU$ek)j8Y@b(5JVH!Tz>@dgk3$6rIx7(73b0%%Q z!kL7FP;M84D)NMl$rq4AM&-TfvaIf-Lw%^|Hn0jgSK{wP+HWVm5@2@Z(hM z_uTYhiHLKjs8_h?DGTSsopDMGV+``|E&iz~_0_f3j(D?2caH$wm(fA}9x3VIDQ#P} z#e@li`%ljdDw`}=8!;AxDgFHjr9?B6h<-Ijf;a4!L4zWXCc}`_vUr!X7=rD@ji-em zLZi}&BOhb4Yq%Tv-u)j=b_&<*5_-X-BX#~fU58L?cjg`mI3uQ`Kjjv;Z2ged+!tcV zc{h=FZ!Aa0gaV5`O9-)$-T(dIxOBOz`}1$2uL1s7g8P4-?Eh1+98H}5zf0%jc!fWK z42YwP@y}mcOos-DK zi(he7vgkJto{$6s z#rP|R)gq&kjZrD8f=CNNmc&&s&@>ewsWqGWNgp=CfT4;kVKu^LogN{~+O|wz$$jE^ z%pq|E#>ijIxE;!A6)7E@R8wFtV1<%1Yr>48J21O+J)3UEv%c(or~j&nCG!4-$HUhc z!#pjh;S9xi)1Rvzmg$ALg)IfaPxE95;p=>NwX{)hPg`~ExE{y)Vk!>YS~4ml8gX7w12^|B6vI(ctM_l!1+ApC8V zGP=8i6$1<>%gd~>By7g$K6`h^9uO#77I|p0@R2Q8cFz(|aHI(I`yUJSKQX{`Is9XL zy}uSiKWi(GFR;-sM?;?vpHr7VeAqq!nW4w6=Xfv{Wamz3^Z$#nbLtW-Y?gG{wry9J zZQDkdZQHhO+qP}nHoH7^*38AsHy7vZ|FARH8yOLgbbXovV`%2I!~3XHe4OCIJOvsz zsPmLY3wX+jHbvV^3@?pFnOB=(1XRMmx*62N^qa55zoi4kPI5_m>rp1kSND8G5#kjs zgtG=KAX|mSIrD869t2>^HGc(*s_v_eArg?E6Y6we+r0bk{cN-1*yu9W*v2gSL3OZv zd=usE?YU#KanEWtW~HeVf+`@@g4a>FK5@em6h$oD1|)=sQ;gka7$MDZN3j80s5WE- z8_$?x%WSo7CC@5EGujt0RPcC8xX9+YG4);c4w54>278Zx8%SBOyc-tu7Y-5b-_2!C zNW<^ls>n$Y%39z^L4i1(nX>SbeLR@iusjC}pJk|WCppr|rL7%_mb&H(T|7bnZrnwq zmFg=Sj5y30GwsT*O;>-4!w!RN-mpvjPq@9}H=D#!28W%RnHd>Gu>N_5J2?Bq$d12N zWkQM@j2%|T ztw|ZC_dV76!^>t0>p(gGVC7#&WtON!1qQtHKX7xfA`^(C-6wgIXY5Y_ZnkSLq~%RT z!#2U9fOjEji`y#p!$8lsiY8a$eiO+OmE3l1PL&zakRkHUB3!aB?c$Qh@e zqd7vnF?K-OSePj}S8J-5pRlwKh0d1-iM*Q#+(uLn!Efc9Zpf`7u96v+POR=#*#Lx< z7L;rt##n{xRK!X{PhXVRYl+tbv&V3A00K`T$td%uz-o`Tn4j9`j7USo%F7qON1}hV zR2tMy;|hyhFAIMZS0QXbzV{MPci$Ig=}>c~q2ohtRHgI7o^7KV=E09cvG@Tw25~ZQ zKFsqV4X!1jJgY{~b~~JvmaINi%{!DxHJ}ds8&NF>z*kcv>&KhZYWije$mb*L<1)T# zo5A}Y6Y7YttTY3X0f;#3XGpVW@$VeC$)fGeg)?sIEOTE_LwpAy(|3GYh* zWO@W&L|m6(XY%ULb!Di^u>t(kEeP*ngX z${V9MUigQSYWd$fxVPz-9QwnX*dl$m02lnmn>nQu_ZHFG`RfzwFu1eb`#{ci?X9in zGDRTyXjNuo%xZeeoPdT~w_L;Gi!ANNbwEec<}k}e)xA$Q=S`|g25D-e6g+C)cE_+; zt3>Y`|Fa{Z-SCP&&Say`@y~x;5IsEX1c%W8004h!PfY*aNVl*xx6%9mX3{mcrQ`n& zaG#vP19hVuj2iRVMP?R~Zv2&nuO%6kI9H}Yz>ri=1PJ^E1Qq}EZ1da#@{eo<=`CS{ zo*Hz<@A4YY`K)0-fzVm{;Hrq^1alt_v-prHpbl4-(VzyLNT{XCDDPhlp)Xs%Bie9! z(5TH@>n71b?ud}`jY0cB3?_GZ;18?lidXD)&U(c+b-h!{G37XQamTI*2lm?t4{&Ys zL_J!idu*j=j=;j|oSc25 zibc|OlaD#uXUYrPd1MAgqA5)4lunr2sdh`jISpSR@Qg~opk#r>Cuc#;J0Ga*0(jqj8plFAQuv zKQfr0D;Y&%gvVv&Fg;x?Jnskf9N)>!#$#eymrp>HSDk~_T^1oT0l?F5l%y6Eod%$Q zgGgmlh`*}+N2{04rjR1u<7Nwy)+KSWDg+f)ct{_DeVFx2 zfx_N`F`lOl5f~1*kI23X-(|n05j~L;et1J-PP_Si>OoxwS&48Dy0}~T3?^`@og#t? zg2aRuicD99aaNHY0PBqgGnG4vb&Dl+T{Yv3yc2;|_6QMdiiKoEN2eb0rVp|H)uMq^`;$d)x z8dQlrw?&?_c#^kla&B`E?T%*DJ^StlcA5{AbB7nlhzI)ZB_3Q>*NoR$reEks}G z+1k>4(z9!$jqCSES86Z2i+b5FqE@Ixz}ACFd0LW|>A1wg*G!2(jt;y$o&3zQxoP71 z`8@iq6cMD*J{W#AmdV&yPQ8NaL1P>X4_-A;us@4y8<0K0wIe+eHE^yC-v;lKB8|bM z5L2witk1cmgfBy@3N6j#yGI_U+?{p|NDM|4AXx9lp@1&JEAGb=we@FegSwd{+a;ed ze&ub#8o_)Mw=oo>=Cwq4r+>ws2DKx<-$u{ZlalkTmt|$+-)aAljMh_Zn+X=)tSrL; z_FFnQh!;wcUJUI$*VuP92e5U7pFhsMe8VwnYGV?O3u$T8#Ue(NqwDbYImAn9bR7u=>1L)qn_h=gkUT{+{?t+rfo1=Gud@{WIb4Lcc$lDD@eA@WdZ->Go z!V0A52t^jvWvs3`X%@)(pA2NF=wnosfQfq~`D%O%7 z7QBBvoJc){AjSx>Q}_pHW>*wIVen!#at)h8=~f`##5gei5}yJiDH{x5r7&&d;@>L# z@Syo|3il29t0H{HnY+DV2ZelA|7olh0;6;nElBb#5>x7Kjajgg!`4!k0R))jUxUSA zh1xh1n1+}FUU=f07{y<49~cEaI;f&sxEE1ywBAvun=~7*q;<=PS|T;TP`*9I1X;Ut zi5Cn@tKkS)kulu;abtb6<@!pW8og-*Xtx~$TR_Q(hWvS^)05bz*SB=zG z2LnoM%DR$ZJxN`PlO4v0R{t9n7S?l4^2APo(s|%U z-xV*rMZjhRUYaXQ7mx~^q!{&{$Gx;_q$*4XQ5KRAEo z=F;C|>t^;xknqgI3s{wd;Q`vbJ-sm#oG<&=#}IdQ1IZN~gY?wiPbu0=us0S+dP^L3 z@KhE{N0}>Y)F%m*Tk?f=)YOrLu`1HflKKXtNKGmOF*UZm`?R4do`;H758wj>mkRJRvSq)@A3i?8TcrAp%7 z{tk_a$aO4E=!*-rciAZ6li&pC7SY8r(Hmt}Kvn)qQ{ee7M3)az9p{~RA)+-c*xG~V z*jG`;)@Y{LbRI5$Xw9Zh=rPvo!MD%r_g&V{AsvI~DO=daY5AoJU{%{MsbrObopF3uN*RaZ=Oc%sBigX>Q4%P#gFr)Tt;MvxF`*%XD zs0^@(Xv|l3hXiBQZ-$(S!agMB$@kP?yo$ca=XiTTKkJNuAQ8R72qc=GfV>DQnezw& z9aa7!o1es-3GDyNB`oxY56Umwi}MkQ6EUR#V|T7BAiUw3Dq}ysE`->CCIe2gzg)*< zX<02TeKo2S&HJISX=2vd>GS&so<@m;I6$#h)|Q#DD$(OxTGFrr(7!xWzMj_YfLF8F zfP%PoTuJ?sQ0<1Mj+iWg;MfnD=eOtAyVU`1`#Myh7rYrzS5y=gwnV8k`RC8Jr9A=* zX@3X5ej&iUP##X^fWsab^4Wzpp`F1{3aFIY%3r)xTYsc;mD*o-R}M0G$`!~jnz(=f zQiH0U1bi2zJRD0Me+f+mD;LoJoGJM|n@6ndJ9Terz$oAErQuEcNwqcl;Xk52GVm~n$tunl;HXq8RD;I4{#MCL>)X!P|R6|O<2KuciSui z2G*%F8ByI>VW0R-SmE#`_)@W&T5Ja_$H<9hJJ(F$-xMki{T2ECs~X#;wApaPB2g8# zA(wmeUDmuB)*5IJf6D?S#)_=bf8-QjOC}&pA3wy(CzQeT85?GJdo_1wv541dfATW_ zy%em*fP63f>7U-Fb6eNQG@c^_a_cSxJ~p;_WV@c;v`SH-vJ3D)p8LqO*cbzd!DoVa zYTCo8fJhz50n|hfrY!|3m|yx4oyx!SHJ>gYXTs%;OGdRyy61Yw;Oe1VW+Ht?&d2HRttdXGsh#2{otOI+~3HML$o%aD8 zv|n`^tDC$X9(bI3e_vBzg8D{8QZj2q&%^d%UxijC0aV7XN=Ld9LkY`5Ox4yf{Iibfx+np zg;u}G%t>WpT3ny3|3VU>T)1mODsEV}_Em3%&b{7qMm*M7f=L;6a3n9Y+4l3%RfC?3T6L{ zIONEz4yicGiJjG!Imnm zI4VXLuBGlJhI{6&+_FK~sZ6~np5*7kQi=W`@4q}YWtU)GyY<5M>;Rd9F9qY+Gk%z< zXzi0P#EJ#>C)>O>6x;Y@Rh#4qhX^^&lxChLbjC{mR|)#*WG5gl;Wd&GrLb=6_|r0^ zkvZ|3*d9ygHO-%hsi6rPnaK&Sh``p?7mqXzDzmF6^eS5#Rdx6X#S*dtgw(os@8|Vf zD#r^0SiVadI(NAMym0~82#N;Nmr1_T{z8-T@q|8S}>8 zM;K}o63I>kQvwdhn4hKg*#p&O_K4@`vX#;&_Yi^RNr`>F!M$y^)*Sk-eu5JO35i~V zW*LMh^kS(i#My6dZ>Mw@tqGz;Vmpib4#*+BwTN^Wz)(hcGbh@fIBEz+7^0`;M>o;_ zCp|?d3^fUEt)gslL6USDdd#qosK|a|I6zy$6iY|Y*cX3b*c&R@ELLW%<#e+rG3Fu` zI?xqo3bh6Oia3F+f6>jLT5%J9j<%Qcm6L?HHr!@nfrcMhz39M6B4MXQ0=HTd%flSd z8ZP7dVvofaJHn3RaAHf|S*YYr$opTaXR^Aph(r(2tsVSG@M(Xco5dMM#HdTiBZI2| zHo_47u%^V|%$}=xNUF8c^l77zO51zTlvuoQnHQM5_(0hLRYCBA6hL8y&+WWZ<8aT$z}I;1buUpAEgrJvZr@rK!Q~qAeoz?Jm4lz8ipjMenQaR#(OQ zmK4S3y!EZ!8{1(z@HY>@CbGnU{5VcobjGFaWVbMpwU4g3?Xz`t+}aWEU%c0kE+$tDxP~8p*#lOh!pDrTaXYg!mX?HlS5Or8-H-@ttKYkWho<7>pe;`*U5%fgAcHM~9p&n(R?J^;1E`hX&-~)scON9( zCwb6;qXuvB1Fo5jG zU6tEZ;Lopu&*oa?Q3a%=R2NdX6*3BBpQWy_)%iltD$FY*m#Vv*_=8$A&`FQzuA z(>W3(DC$S8MPhqr0sgNGUZ7&|9%(wUYechZCvQptHkuKJ&mBejxs|PbJiR&k&K78> znpBXyGo2RLaDvKtOk-=kadTEb567vUWo;=@BrLW+Uy&~hfu6cUq})OGsN*<8%$3zX zfaQiMN0(2p5R6W%cvd$!5sRoUdGNXz1HP?N)R{-Rc^zihLW;hC_-(1US5INthXpZ? zc&g2Y1-Q^?I9tRyg|xkM1=uH;!0=6^u&3{x!q~5C`|uq!Lm{Z*#RQat>_pQ9O4c00 zi1G>!F~Y$A6m8t1_>dD}Aj1L$9}YN<$=~gxi{iW16;tW1horycBkqX{(ee|jN2YUd^}SD* zz?4tobm_tKfGZ(O>u)wwNZ>Lku+A?;=bQYQFOQ{a4-wWd)U63#Jl&dJVpzQHJ~e7u z3*Amp{)H$PK5{ca<~Q8koESU5t(#U#)zKR}j$BMCt&g~=kwv8%+KkNTfLqDz*Nc3_ zbd~2{9j2Ieo)Fw$$WAUFZ+L%Lh+gX+H+HK=rgp;63Gr}88mQTaUh@@@pp}|0c2PqD z`+6hMb0-nKSk<`OPp7{H=Ox&yy^SFwLm23*>Ln{y&Ody*O$KD!u_lhF6w*3@TZ+@(!uG_g{g zM>*fze3{s!6%a=B?33++ctfHj7*1e*o%I}|F9hI5u)5glD7Xna=Tw9HikC%uUd`Al zbJ6JMGeq3(?4cM$4~KqKlmlF!%fCfG5H=&wtX7mm(>++C)_PDrGcz zUO#7=2t?!SW4|42ne9V80Qvq{%l?||^n#$}WaUceYMcAm3h_*{ViIcID2uKnQVk~u zk!?c5-n>==SJgW4u@2wp!ZP^e3~BAxIh;~~Hq=_#1}Y1Yd`20Mu{#G!KHBONtB-$x z{@){ABI@U8EieE;(=Qr;?7vgR|F7em)PI>5doI-wb|O*6b_^nQVcPtVx%|+=ajUvx z4eAlh%#Z5w#r=yCoFl(qAM#CUj2CrR(4`*#vF}dViQ3Mg(#r^Qy6h9Q> zkfojv!{?6F-wv@e$yE3p3(n%{&*!vB2^=_Sw|6a}F5St6f|yqbRS7LM?}zY!HP*Lu zMCJ>_>Jsd(A5hCCyU zg-Wkf0X_B<|B*I7skzcCM#`s_HS0XmGLxaB!^^2-pO}N5dwNAQakS)2myXgVit-X7 zDbp|yHl-C02_uoFnnJpR_EeQbLS+YdoNWFn*i`i?tRRN-M4E<4T|)JlmsqJwCT?U1 z-FQS#spoUIBuON&`Uh6+JI4(vU#w3yLgvk>4*pj+%b3cVLG!VTkxP^lHG111uhO*j z^2?!li{AC}Yj!e*dOnh4o^Tq9F1ZI(7*Tp$Z!gva#w?WW?-)M8x4XB-JzVI+gHP-B z7K{a5p-j{J!Z~SXmF_59!^PT9+Bib|K9H^$+vPTX=hu5n^lw8pp!YyiP&0z|zeGzx zft;cM9F^jDihdY-QMoe!ak&SSzuu)2w?RlWveD~SNXq%*Zu>!2NcCbeWYxHcs%N%| z0lv(Br=%!S;Wq9Mb8s-d-Y^-R)<$EHLQhU|37$nDCJ6?hNXb|J_vE;*`*PyU;KR=t zV=Pn+;7v?@(go^%M^A_brrWIbAQ{avDUVG>2uxXnIO%UoW?8#&Wcdy@F4#>(2NTR< z;4|iOS^ARq{&3}6Nek9_PbpEfp@;y5vyv$kJR+TBF1i=p8!MvWGg76cMjB^X;EnT5 zFeD`0N-ho#HB06k=&IqE%1h;#%5Ws6C`Fk$Jqn(!d!9hSy3Q#(A(fW84NEPzTbJ!k z(;d!~zf7~Y_4$_3E0!3;-4~ROu<7X7LqN+%YG#`siq6WduM>kR;+eXNh!(^bnyaNV zl2$?9%L28&j$~bkFLzRc8!xthHszbe%JE;l_)dpCKl_=(1sIubUarrtE3vLCkslR} z#ZX+;XtN=iIp`aI>(NP-TLjp~{Bh^hIr* z&_kh9K|8A3pD{xX1ZwP`4IGhVFwMC>k^{szs%t)foDz||qEsqYNXNV4?u1v4(4 zW@Bg`Qu|?^r$WxutwryW@7I=;KR3-D@FQZydB*5@|N2H@plO%aUst_N6Ovo>HBL0% z+6yqT)VCD$V>V&(Cx_9x)i9?#aKd9b7q$bZ6#B`o{_A#XCN zQ%z;iHchHo*oG`e@EVGRwKkKun#W|42S6UC+OizLhIGLX%3}!%1C?#H4$uITkVge| ziewG5CRn+&P^=KYYG=i__BX2L@@@Ju^Agi@_8ltz!^fDj2cO>Lb+Wect+e;nymp}% zhJSy_;PdUpqNOWyd*rxn2Zgd{0SdHej)2${LDjo-7u<={^B_hO;2|A z=QBAsO*%)Tm$RVDvO71*5;t7(&`=h~;G;=5AmA!5>kzL3qr-$6+!h{0HzBI?U@bu} zcB=GmthjnXtyUB$or&UqfWXjyBGq#?NSAYCZoR(BvsKm=*8l7)8blDXG5L!zifQ)Q z?08sOw6n+f+N^_>YjAnYg}HxZl&)T@1@Tu~Fd0}@8+Y0;Hsb1zFoRTDd<*SHJ`u0@ z9whMMsIU4)VPrh60R>sKNP7Wd<0du6q&Vm-&onv>_ADY)6F1Q|!aPdy5%$RWc1}s1 zgJeiY*8vU?BtB&Vd-GOm>TGv-Ra`t@+H%>jvR1*Y59`g^cSe{$W(QcVQ&6k?(acVI z?+5|89m-ClEl*Lsq-G`H4IwiBS3@>bH-9ir9XscYEf=GiXPFva9WQNR9Jz<;oUZB{JlOy^_wVCzM&WxRy)Ck7GP zGHPWE4M!VJWX2&Zcyz%H*YvofNmF0-!>B@q)ie6lA{W=ci@$Mp|6`}{1Av(yoc&2N z8xY%RE`<6b)`ohHLT|sGaWpe6{0ij@#4FPJDJuZ(`=gawE-Vj_<6NipxYxuS{cM|x zlO_Bbqyup@LXw@wpOpE<)NF;-1ll(S;*d?%LdCjXW;|X9j@!IG5_l$3rO?Ez6ZBUr zyul0RlhC+H(wUz(hSYaS#}T`gD8XC+=oblvM9L zCNow&Xp@~}563e!G)-9xamYFhB`0}lhEkeFiTgVTETM9kWRi@%^^H8^BO z&41XZX`_^|dAmN8s2izz5TF|qfoSJPWMTU)_X2)r9QAD$j0NO}~$9_u7QG1=& zh$9|H;^%e)<6i@gWeH)TRkJ&oU{BDBWbh5t|ezQ_Quo8DfEA?cCV zuoG!y9D0}LJm0zWpAu!_mQ9W!#XsFMLj`~Tox|~lHPOx+thM&##58#_`ht)87#A6^&D?{WCeYE5X- z-sF^a{Sjal`*8IPXXz^39+lYp2VUiovJ4I)Z{1ISa;=S4uFetDctklwT8R=1n{fcM z{fGe7o&i+t&Hmj_W8$G?;+0}>hd`U=|K?>e#hbP6n-MLr=`nZKaPHuD%~FkR+oUq z>s*yvJ(dtIDFD*g?k+=P=E?;+NHT_zs{u zc;4k2qA5hRZE)`H-Re!9ci^FfngaKDXIN`wMX962EYs5c)%LV`x7fc^6BiMsWQl-6 z>;xg_+8JaR)TL%M+;5pNZEL!!_U&}hbTed6mzo(hdhY~MhZkR;WyX29sbr`v9c>0h zjuM7UD;HIre=Zwxx1FIi84R>zq$Q=^NIQzAZWxu2#pRiTs@^b zNCpq5)PH!%h1rj|k74j&xao$y-nbPTj)ts&*-g+?%=~FhaNMepgTReCC*J@nGzdM# z6iwEmvFzih$$F2;;))3cPd+zy7O|48VXp{kGROK4&u|;5qEBC4`rOL3+|ZcBn|Rzwm^$%=0TWVxxojg zDn=WzxO3~Wk+|8NDX}8fg2tW?l=AI6cVf}YAYHNMnAJVSOQ*o@B;26_(vz9XoiE1U zOR_!eydiLih!W%u_P?)C#V%wBh1tFOuJ|dFtlQ!u)7$PF1`eK;j55}X$Ra)bjgvPy zLK{5a+i|y70BM)Y879IgF#oGoybD&B6_|8<@*MlN>o9?8#s=SSUc~(ylm}1Xii0Jk z%YLIsOg1N5=K10lzO;V0bJh zE#N|%bw|_ST~z0?B(^Y-)D5-mg(sCJGbm`vM*ULQP4=}B4PWpXuPg_}wnKxc`&w@_(WozwX}O?Q*TY{a@HD-&Z{W3i=kp zN_-(b7&G;ZA}v2M%8V*p1dIS01x5vp1#!+AwI|=*cob5_B&$}lR^BD}aN+G&Twm8C zNKWcXTvB-In2~W19^d!R;Mkg7ogBTKg2v2NTbrGn9i1I*@Iqv#kCV^Zb<2dkfcb-QjD{JtI+zCK20atwicH#F{;z-=G_m+n69KfkD>a~&Q-=3d>xrncyuusy ziblpDtW06=_kS=;GlSUZvMP@^r;)zl#OF;QT&lH)Fc)KzRvqpkC?l}B{gO=W1Ddn$ zA^K}mUjB@Vc&L=Dv8g4;NKKdbj1Ma-g*jl!fK9EXJPBF;^L%qDxiDsapB^ilaH4&r zonaXM#lb1&ieG9cpYnh#sfFu|*;Y+J##W>~xGPNrnE4Aj>7c12W%m(}Iu5Z)B@+u{ zP(^x${&Ab>KQjtVaDa^%OYrN{$qTZ4=V`#kGC>H>E|Ct7Zmm04%cPmKSLPLzv{NCY zMyqDyAm4*vMHT1(Ib<3)w(a#k8FRZLO?1+tK`@Q^Ljgy(1R}Q#u5(wy*4?Z>DP3jh#FjEyf`XFdcy>GXziWCCx^7C_z7_@Hy%+KYQ#PZ z?92u%!^T(l<%EUn%NM=y}gRIZMVsZRxCA4yx zt)--m0hV3`GCv@UI6=oW5rdS3C=lfHXh5>0;@)aS;j_0cpHIlcBqBRoW>Rpe3a&g37m>;q|4e z1Dg@!JzL!~TVipH&>6I6<{M}pUAC`=E3{UXPP)l8xXTVk+0lR4O}4at1E|HSei>*P zH^wp0At{I0HT_S;I;#t-n{dkrrxh8p%cAns$jl$hxM?pfC$j=l3+^Z$=R570Rd)ef%3sN2D8`T)ufdnZf7%M0! z&6%>#_554s0l)IM`AL#+3?S#P*6Y6|4r*Li{}J0tqxICg=}7Rk2Gvf$f}2+2kd?Fa zt*W&#d$yQ<wW{~Vr70QmuX;w+TVv^$0CHte>e4Q z@Gpn6oF$wxdwGzD_(#FPJ)1blr%#|60bh_bDUFsa3o;iq1q`j<)YtiO({FQ+9~*U$ zwaTyiSH?Q~&An{Z;+}8wBzBH<-2}Ism zsm^;@ef7T!-Oq>UI2fu0vpW2=(NeQD(9@BX!NI&~T8!VKmW9vnj{3s6(7aOL_$IImXHc4>j_948?s2RflL6+FB}cN;wEGPN>EvA>RL+pH9EQ_1EuU$P+0r%IoUo(uyfLTIC6FUYtp3SIb2W)Ev2)%0aCUsoCibiN zKDpQ}@NscogI_(F*|Bx^wc#kSR9P+J;k>x{tx5!V*`)4&EUdY*R(ax)6hfz-#!FWjf;7 z?nb--Lc0jqy%S@dbU-Vnet8@o_d(jV2YZYFD~X0jcULt@q&dH$c{EB=3dF;pj@xbVjE0SINbmXF zI`_tD^dHR`kz|q?yU<|&GMj2la9RC`bs%)_!CNyh$TS<-_b*hD5fAA-W4)`S?c;%|H>kZ^XG2^Sd0>tVL%>>hB8w9$_oCI&${tYYF;b#~ zX3|5?`%I-13ISr%_csr5;WXBT`!?+|FF-K)#D%wXvlw9g? zWRwj9Qss<=4r;LsqRm3x*pX%d<)};dxO9pmlYs36_80o0nWZ_M^8ROlMQov6)SX1Q z@mt(4HER+}Wj1X_6Tm3{dA8)PFrZ0Qih$byl*iaTX~*sg-HAk#CILekF?iI9l|?^~R`eDfJ-&LJ)sqaKH< zXN~W3AAEjY3#WCN@J^5s50hiWov?`3ZD{~n5GA>bS0K5x4?-Y;wj8W1A0=NZjZMT- z!MT19dCH}P28gRpdIDpU>@$ZPv>HTV{6L9JHK|i z2c6Vch(=c7+aK5nGvbiR!*9z0hbLZEALQQ;up3{+&x8b!g#-&3Ymyu&&<<`5`o&^g zlm|zGDS3~;7bMBtQMaJhkteE?uV&`7&i6N{Z~kI|p;tj+xZZ$yLs28YxkYsj7R2$@ ztRBW4@q)mBJ%2Dj2@NSF`X?qQ*N+b&9bep@k3uziXs`|oT0>viHN&T_*Juzz5_pfF zfw0q7X;p_u5o-)4;9PNQP;b*xyjFirR}fL&5POQD3dFD*1o4or&2pX8g#ZEe9h9;( zf{SRFpQP+l^&fqjvow&6+m6E=FwXv95ru9OSJf&q)QO@2g01@yRxuE69)(@ZMzyUb z?;mvTT)*2$n>MLLAMiwxEObLSN8Z^R_xh~q12MrZbD{&kuix%-f3Ag}6|OI7M5dEt z;8V&b0S+LBR3Z*5;C*Tk7BSqDIUpTy-N67>K(NOzijA+`EXW<@M(i_2r1IdmXqfV&2{~gW2U{Yd`6v`Do-K| z|6ikOWE_Dybi&+}^-#BeC{XX9NTz8A_B6x&)Fya%uAlHYuvIGBI!nCeh(C!03>nX6 zQ*B47!U_a{X_P%I7)9P?X6YJRW3zc_nx-e{gYzF0bo-hoxa|yqpVF9-%BD#STV&;f-~22xfN9%_v8)iw&Ip0ESf ztbQmA;3Z1>zY~m}qYN@}#hsw3Qyd@q_puW)WyA)Yd_g7hmPd-lv^)w65h>#yh#L{B zM*mE!*%TrI5gSxsm!^gy>d=nagwx_t4MVPogXLL(?<9CrTze#^yo$ld3mMp4uF`7k zr9#xb&fcpumg`U6=(wmNjO|!bG-$YuEAeL2{J6v2h`)cz8eH(l}Y z&Rlf4Y#v21oXNHA;(@a9Vh$sD1oZB_NJ3%7IbW=0)|fp$MD)dV9Gwq`l9fGI!7_M1zE%m4cBO zjS>exlLolx&;*7gr_G>6LV~&#I)D(f3?t@InWrGptBpvfN0S%#xhBkcwjj`hTCIlgcHb8+_X?BcA!_n4)TNLhmyYdt@zItZc44(?J)pyugqk$FgGmqgy# zu>!4M-pH!V#H!>E%l@CgR??lyp;wB1Trp&>@Cv-cNS`=qSh0VTjwOTkhvbvxv7|bx zz9#>YJDsXcBTr@2fAu@dYN}?b5sV{Z;ATwb%o{V2B={;lmE@^d53MggC_uAiK*C*4 zEY|acBGHLQZQz}I?EXkVsqG>6^y=i~X1ncvGP7gj{9GuZ1}3Kf+NGPvt{*2-g7b)Z zaC{>W~$ ziBawb^`yy{nLMOEjGn=`40ZX%eP&R;{*TO;WZTZ0iv^ywpRoo(yiyxq-Ry zGM0aS1Q$A?k2Sb+>DBjc&OO!G$!YeHrF0&jcPyWGS{GYBZ@pM=&6cC}UA^qAj$H~5 zF^hT?Ogs^z~q zxIX-kZ`w#rSzv&Q*bIo`of#pxUu#ZVIAc_43m(OkLs-^DJ*dP?=!Zv3L%5_PUn2Gz z--e49s3!I%iZ2CqTzO(Z#3r69xpf^WvN;muA&LX!Y!+kTdmk&Xd2+^na!4rZSa;ou zRPq3b-aZ+!OZw0o3^!-Z(L1r*+HFHW<9eV}MLa9N^(1FjC10GsQ${h0hO5T0Rb;1|zbu-r{pT#BvUz@KLMpxDT%BS` zsS0Z!J7r>w6!2nO-yyVJ_B<=SW38hKKUCCn(n)k>{`{yt*?|yetX~R!{AORYh%oVo zNJ(VFB6Fef*j&;-K4ntT(}|sq%Se(=Uam8LmMBeHP4|M;GwXdOAJ2m7(N>*5y&R@> z4g*ZusK#z)fS|hJw#pMMd`E0#fwMBCIJ92aq`{3&^AG9hd%V|tNm|3HZ3kpcG#cU4 z2L*q+Dk`Je`QMs?4kYX!OtIod*;o5=JI?$PYZnU(x?T|ZP{i7WMGrwSRL2;+5zpjb zYlkt5@_A8Ra6|qQ#FD}OfTFQ+AA6rt%tflaQ?|oy9o6RV?fd~ zHrhoVX`zm(;OmS9@>GG z!RR8D2lQFqjOr{3`D|bpJPCZM>~oQGL#j+Gsn9%M*9no{OqfxBBI&G+Zbag01Rcf5mF*7W=@!p^Zdv}j$kak69E zwryv}wr$(CZQHhO+qUhb_x;kupMa~Z6%5?LWw5%?q zSJ8T_A84TC(Rl`%+(7$&$}g6JZn6^>n*9AB57~d*{9Ww{M{QzI!{JTVv%q*5MFchC z(mg^a8@OK<@&?4U@a$EfJ|!xz=IJUw*^$$)eKSK&7I2H%3U6!CQ2JLfASG{pXVbF8 z?7=R^y5i2EDT>LnZBp)*(7Nb&{_OVV*p2&>bC>}Y8vBV3viVPn$3+S1pTuw)Z;<<~2A8MliOx-760(yd?^XuBeM z(LOhZr`1_NR`6C_LNytFX_)o?KG>ugffx%Z&x;|On~VTycy(m0rPgU0Z*PUm%LGLG zc>!CZljnQHt>facT;wyPkMmA=<=lZc1RgU_iiE$ih<6vC_pl_ux)DMrs45qAG7IgX z6V&MoHY4$o`gjx?8I!pf6j%2+7J=3x?3>Z?3snGP7Eo`>^sS$j1eRV!+#s^bJhJ%^ zeu~!SFduwP`40}t2t(D${GFM@WlWx;Bu3M|OJb-f2UJU3935FUeaM@bBz4B$-wcB< z%Mx_iDY%TGQ_GUOUuwlRZccC41w8Gi-RD}cH~+=S+Vz1CiaXFfLn0o_A)9j zJcc5yr|6%jl5$w@!)p;?(0oHCH-kk>fiLP2x8d!YEWnXbD1k*m%!u5#L8s*syhD(I z;Tky4*$e&wOK{KD1fz1#0h@8puvWJ_IX2cd(a z87B?Z_MvRztTwGL8aQ#D9%6Pll2rHcYkk!W>(J^)d$SvjDVVWukQ~jQVZN89$9L*& zQi<%{mYH91nZA{9r~s-*uBhjM0*BCyy8cSVc8$(+oV~77&2^5_mZ#nQkN}x}QfF7wbMr==v zD}hVgp!@Sj*VIqlTw-JA1wt-0hf#1NFFv2B z@dIn#OxeE;TD2*c<1Rf~Q3br~RVKHCs{H~O=rf9PDp#OqWQ^NWA#j7j4Tb*X8N65%@igSfHO#H&~$(WU*Aid)ZRP_6(g*q8|Z6}H*@A>c9{W&B>r(8;qhTfe96f7zCTcNmPGue~0- zYHuy<&Ns0|6gy?fHW%T{EGp<|2{byoEUiNA_6`$p+_MQccO-=WGT2wuTv6o43JMZA z<6=0gmp5v)<8znfPWElekm8+E1h8`59W!Pit}2LX+rut7dSrBW5}dpH=TG1^yDK)W zz`e|KDEZG?Mt0j0u3;T7U7xzo#c>EkV5wocFQ&*9OTh?D)#i>{7O>Wqj^c(m1zdg@n?@e8c#uv%JxVK!z$7Lv3t|aeYj}I(NdX>~7XQ6I7T- zs!F}jMDWG+NY@Yk#ioJ%pV+5KX5I4E!9ob$(svgr-I)YxNt5(FUE4+X>D?ZbcJjmy z^6K34EZ;9hh3yQ^-Qwh`$L|uENbd2s)!;X6qK9^B=XEP3(4i@>PlkBuuQ(y&9}O>5 zV-@1a`}w~}NOI}B8#oceshA^xl;=6QWX26W03NvZ{mp75&gw@X>@!viv`8JLb;4&ep#gjWtVZWu)f@i1u-JtE9M8|Q&1IJ$GlTSDk!i5#YUf9E10{QV^N3D+!xW<#U{tG7Fhvo3^F=$Y%RfD^ zsd-2=;X#Ah`YA+~jTK@#;ZLLYg7L&4iPaaGA`Zx!r-%?%BScC!ge7<(DC3W~9+r_8 zDgdOoCn?E;(MUtQkulgxWaU+KSe>jzQex8D(K0NqvQH#7t1Yp^MM024+olMLTVaw@ z6jZ4dJIX!916*VF4`tAh`mwRZ)9P;ojyK5YN$mINbOSleHPN>Rh3AorZ7F~Kr7qWk z&JHUF*0L2Y7G42GM6eC~7|dRi;YrYC0M_cSyfphW6hDOJc7KxJ_A%I~%DjBqob{eugoQrA zWLU3i=B3vRXA{@s?LfaLy&B+jI)Ml)9Ou_d< z{bc`ECNGv4%JIKJ?5?KF9imDxX`Qq76@KYL@nBDFf(K>8=vreKxPw%1`r$S4b(+B^ zGl2TnB+%fe%#S|RUoZn;YDK7dN0g|8hYWpXa|N+f1gOi%)WL+n^t;4;ulPpd8T~WBSG;RPo|tn;|Ai z{(&zZ#MjPEdTo*raAiSuGLy>2LxAQ%i)CQIacEmnX=;$O`IEA(0n}vVRJ(QlfwtlS z7ElMo9()K$mHY|^tsC>LSC0+xLlRyDJ##~WaK%*q%L0F~;3Ga*%kP(2Gsybh1@{35 z?5X-3678~B7bcPtu1T5&7UPO|M*b4aku&iFjd6p!VpsKZep3_P8`FjXQi6qMcY<~? z3A3mNgo2|N``IIIdjT^!jjX(OHc(Gs@sBr)B_L0$mf+`E{&gdu=Y;Pr*cYd=WcMvg zK#`Ju@@5RT+tQtBFz{i^7R{tt6L^oeIkcp~|3HX}i?VVkiKBibsHdE^-!DG&IF9lCrLKO1B6A8|voUsVx;$SRyZdmabhYzm0UbdNNgisk08l)7(hEfqrcZEG zffNPcX!LqyCyc~`ZwcZEwV>2L+DuSkk%?ABgMp@U12|zt4V5S`$S6Zdd%&lpOTA#K zOU)KwI#@MRDP3B?3c+g`k$6(RREDS0a}0pJLkJ4OAc2MqF$y_7tXKADJ|Y>v*uhYBHn&x^Mw`v9odPFV9ArQ7iV) zan1lt9q?nrY;__($KWOv^Ef}oDhc`x(^J7k0433G97TxBBP)QRPoxb+ax-|J;i2Hf;3E&meMBk`71nobZxb0phpnyQO%)^UIG z@M#})*v~-TOx)L&?f_aW5cXCU<8gY`PdugzV+Q5nAB^ZJ^M!zR1CX~QUAG{?ijwW6 zzS!q0YHyJ*(bdD6>|Ee63roZ^sO4y}G4gTyY)gTS8128u<}id3MnG?Fh0}Jj+oJ7g zJ%jVfER8c%5zMSD&S)rIYhGfql1sw`;w-5OT`|9iIq4#2&jV1&i4EFjQvX~7DT>q) znT_ZwK63wN(V|i$GH%Aq<8PwNO@gf!=m}~>z_)RzfgC0G6;%^{T`>~(4fU~*K45Ok z?UF&s7etH*vLRi=^`;rHU%EyEKoKje6QIx;rjL2js2bU&XI2c~XRnRiC#fvd^oe(< z%>Afa{SjJGnkY+b=?zzK9qO zOJ$0y27ob(Ek)1@)(uBoVi+|y;ude7^e^LYO$tcfE+mPif_(#gV;D$=B40Ixrqi-y z%%PMsQhlsI5v|Hdp9t8)hlABjRB82-@uus$NuW}sToAhqPe5&JE1+lcgJ+-#Vg~p? zGwG)@4~j+Dj}UJZSYP$~kj$CKHIrFe|CNEjTp#rz1uFA_uk ze8Py3zFRA*hm_zTn8(BKWBg!gpS!Z8>CB4+EzAEZsg+>x8ra~*pXJg)DNNUrr)CM- z$EWOQg66pIjuOEo`}imXW3CBMlygDA;cM+QzMhTQFcZ^cgYXt7IHd?H8VgKDz*jh= zesqr3@MvtJVO*-2Ks@ z(K`C07A6>)tr9LaEUM6wR=%ZNc35_a2(DPN)h-O{k0r+&S$wC?MGfPsL$S(pLwD0w ztJ&9x_A4t@z5_Y`+}E%sRCE37gfHg>f~n~{5Ap27w$kagwmtLr2Yfm2;=r(wj6#(X zRn)K4Lx7r}(ReW1ao!{-dp1o#jzGU8K~i?vBn$dTCDb-KrLLrcxTMP_fCFL6xjobQ zp_z1=WOZ^o9WB@Tl!aAXe^kTH;zYLQg};$RfFCZ1Yqvgi;?#UaP6?Xah5|yZNWr6z~T9S8EGceYib2eTJTtxNhO_;5zascWzvW8!2rTj9f*+wVf~?uVXE*Qq9m!E zT7ss0g4i5%z-_e|7F0b6$r%|`(N*lV;X`mejPT@gnBJFSdcG>ZH`bBq4?g47)QK&O zkq{E1zn^oW1)|eDOWHdipTtq*VKiyJr+sLq*Z8aiM5*9E_hU*6)^~A-CbX^XP|=0H zI>R;i`iuM83-?56Bmjog-PnnX z)%w}w7dW{9Gwl-Iq|)@D-nQ5tuA{Z)@Y6FQ zPK`Z`j_Dma=Qzi)G~ht0<}3^rB;vvC_p6G+pK2h=L(b?@f;OCR*a-OBtMV8^GZA^pF|z3%Lf- zPJ{T@!xmLs;!W&#IE{_&5GThZC=Fg^_mg33X;KVK;6(el$P{ot(* zx)j}rR5k0^P`^}R8(sP7TPg8>3zViUCg8Cey@PpUXgWnUirzVojwmM?v5Yy%*WNvGR4;4zBSM6!Q+2GGhjC3{ zcJ0m&l#iw_mNK%0(T58&$Brw)s4Z{PQD%h8>!*r<$My=(0#z~W7UTJ$^pjJmHAR;` zx4HozhNn`a*p6{MLFs4>Niv;EVePS>*1FSA9ywMPpNnx86b-;VLAhUYDW#!@PxQAn0 zzW7VotK|T03791%5NvEBa3j=k%$;jS*)s)9dI&0|iM9|1zEY%4ZGOLNE}DWy>c6K! zTFfy?(mFJeQKWT2BeodQR%@6%80I%9y^gy8y}&PC{8%(KX`VMxS{7ZxW{n5CP5j(& z?E{}%8O0ld5@S@c1Vn{EPG8{j7z$@$M(+xH(C&iy+XVIip2Kz`@?%Sx=)cmP)G8Ti z_{aKe)AA|T-Ol^PUG0Y>295vY(U0fT&v{t8qj&+Q(Nv)wEew^^EN^(+^A<=RPfHdk zz|S$dw}Qo#dB!&LD<;sofhk46ZUmRhv!aLcqxms1L8Y48nZ*x#k3slX=RK7NVS7-X6q@js)3w4ZBW zCIEeIgKndpDg~TqF85(bXU(9^IGMYWfbg|Q2TwPQ>@=b6;N*0qU>&&BGs{{&z)n_W zJCKo_cuG#T+xCHpS<}WDm}zp!K^!D4C4&rO9;1Z_G<&uPI$KNn?mcfSOfi-|V&S~E zApzgK};g`p2~cl-0QWI+ZAU zmbp0X!dhdzSJURrYG;gd)4ln}5AsXslo6zO`PO%p8EN@4mfV>Vw;>dJNTXqrOLlaD zxXmcq*h~Fj7HR(s&(qMR8})s08XIuBXtkn37886{n1~u%Mtl4S<2)3chVN~bF{gug z*WWZTY?7oeLd=@;K8B|4FGmGEz3M^*tzND82P|?U8F4@a@Nn#C3UkE8l|_XAiH#o#8{w{?|6S<)MhQq4a#;1GOcZ-| zp2<)m_;yB!oLJy{8JU#}Dk-k9JO_dlP&Erd4yk3R01Swl0EKLKV7fRa$oA#wg&^$K z9_MOPI(FpNzJ%p2Cj?gx0!;}0D*(}OG&baG4mS}LE2I|b3DeUV$EZb4(SBH1sTe#o z#9EvmY*&?ui|~~8BTREefR%O9pc~=Md)*hrxfx<3jiFCjTia=ph})LCq54vrpCK|l z%Fb*;egJao#IwapwljS3e80kHz7K9rDMLN6*XcQymZH)p!GrF*RhG}}51c}G%0lQP z=8R5(p}lLwvQhA&kn}%5nqeTt$!Q8OUEpv#R!i4PdcR!LA?RT0^`gFbo!~)&8fT9v z5|+JAVGy52EFKi?mr4!vy2(*1{`sNFi%-lA^;DOF3!iXG%E5`rw~me%D(55Il)H>I z!rqdRvf39h<3cZN9h_x*X(XLStth=G-S)$s=2BHTHaKWs2$onu8=#Qj4L==wv!=0;{>n}U<3Ba9{AqvTm_ zWaStV8Y)OvVufGUN~I+yEiZTL7$aJ7V-d!P2nmcbJy}q}e8ge{qNNWO!2veJD?5gJ zSr-s{?9&shG5zFVbEe*QF@}+oGItBgq2{=xLAn#nrBU(yx9Q<^7x~#5e9Oj;a#WyJ z+!ER-Zf)j-K;B$VN9fDa7O}1>Q_%ssIJ{hF#B5XT548}62b+LraIO){dJaau(>hZJ zjwDjfd4b!_5NA|N_Fd0i@tQ&JP07YEjytct52)Uy_t2#g65S&bTXkM#Hc7mA=Q`+aworN4oMsAjVV^fW5SD>=>}-Hrb6AiQe3 zcGJ)iF+Qh_zHv(ULy?$zyYp05l?CWXJnUV*gdb z`2rqFy615cVe6d8X7zD?M=Nzkz8TX?591PBNT|eYnJEBXs9u^jZg?)+j5V|_k4fF8 z_M@8(+?Cjijl{FGN2P^h5DGY%XWVMhiAc)6J-)0S^aPX5IGEE)`4?ei!`jfE#cKW` z^ZlcdC=tf^N7w1^d+cw_gCJc?fo|n;g)P|xsKca&(z2Ukr_0rzXF@Wex>d10UdzKr zNzmSf-YNAiXZ?@gxKFwdg2j<|WM-}Zdggl0eE~?TjL~MYvD9|OB5mRO-q2yBSO!jP zE7H-#S|TRs+xa1or#M|5gB2gsEaiGhS5%N7! zUmaanDlHA@n(s><#)J<;%S1?db1D910oTzy<*KdNbMzjdV03OE#StzN-?pXIcONDH~a zZ=o0>kW|a3s;$4x=Htv(%6Odng);avV9~TD`%n%_6pc7*WP)xQj1wWthcRLGs?{{u z{v)np&>GOorJh&qZw&V-r4YHz8*fBI)Box~VTIbZ8%tRtw!J2Jdz&1QU+4q4zgKP3 zuUHhSmy^5MYMjSwd)t9iM`{N#eM=YqokKY=7RPwjr;`M1;xf|>Hl$2$irZ{%!v^7! zAxWCf8*Z=>RDK;ep)P!=WH+t68=ZKv*(1>?y;MLcElXdBo6@6Q1MS!CncvhvC?61} zspMk1GG(W!Z@?6{a!|KeC|;|-w_*S*r#G`JR8_h&KUNuF`C^x8uaU5lq-M~?aw$rS z-VGWUnD>fzEW|k2Pysb`BDa!K3L5gQ6sFk$2bXxHJz|MNiwW59myf#WHcV#t((w4; zw(vd7;0Q7RJ-~BhssLM@?jUm7E$carKH|!}y3@vjnYSflnXz z?|)64FN^cUnEX@Fa{sIYjdl&4qD>ClegqPUMcQW4KER23u|_pt2z;cg2u)$EXBG`b zBM8yp#sJ|-zHsgH8CsnTT`dzsyT4p8Vh1cH3m^H%e|b{}s>pVor&?Bw*yWgG0W@6h zr&jcZSFVxOOGL%jZF>(2(6@>kXP*&^^VsKxSRR<=Eqo0ETrGXL_^s9?4bCDVTDs9= zNBi6;B?10!Gs$w@S{jE+1BuU$QC-+psTeZc)5P3EFj;o`(6h`v z$az%JKHp;%MGVr9%M8Q=(xu4sM`aqUwkTOF`$*(xfgxM(OsMb``EVa=*ZZ^lHLT%DL=_i(?+%Eq zlUm!EBNJ_9%@uHbcl5jzIf*b%esB3tJwUWW-h3U})qA+p!f3c-RM>{S2@jUL1si4- z2}g@0*bj7~KYezBqHh(=yhX+QXE2S;B^(5{RC)I!ZP&z1f^Ems&MiWOyBz<3`LX*U zyo{nOj*s2vJ+VCab2Xom zxR>&70tz6Bu6jIZ&+HfcSdsxotv-EeHP!We>OHPBug~v&cK0FdE%z$U@KpxyHWmbR zF3Kf_HpulFt+J8njmuDitqo98SNr_^5zoPxO5Iyr9HR|;X?Z|{O1BWLKQKq>o!V)w zOu^Z<&z~yB_{f%73paN$L`#NI%^A)1DtX~Slx?W3dDZf7S4=IK6t5pfq+AhEHngPk zs{QL>bBYrdwaB*j982IMM7#q_-c}osntg6_N+FS8UQ*?W!vprugI!yjJ%6LsOe~wW zi?EzdaP#B?RkOK-{`bVR^De!PQz_8!th5MNK@3t0P^GGA&769smrO`uwE+eyykQbN znG=JHp?;}cRdoj7a+B}CPoF!Blhqtk`ti(WO2&wo?$kRd=E9!Ev>=6bBPq8a&*0NN z4{coM`0I^)ON$zRuqpT$INzgNq=y~d;?;TG91vVFy291f9@H1=i3(?>dT_E2sN5(g z*>F{e54&f`IWt2BZ}WY<>P5A8H7~Aghi`UL#KYx|oK`f4M~}z+B6xgs#zMDPOY&y% za$3vxF;3|f2C+Zy5~XomHkBjY!B=gwKj`lPPIHhck#b{n{9Z3MydXBO9_Xu(;MBNX z#38wXQ!RVJs*w|asxvS!v~N~GzFwXlo({H+zZN@RcYhX*=lG7)*JpP}XZOApdWQEC z&_XnQU~C^wx3~!;4=5T)u)g!=;A_Q+dHgYEncPC+4AeNz-Ed1r`CGV~L9zMv0l$PH zuBX}{BC1uMJjD=~PWOr;VbyBMYOmNMv-FCAXVpt3;pL($sOK}Ui+EH139tz+`gTjP zK)Wv}Pq*z(Z7_WhvL})xOkhzI54iYY&MNp##)CWHBGh7FiJpo5bp9f(J0rP{^$(wi zKHlyO=k8?pc6#}~|9r-xjg`T9BFo@;j5oWZIKCpRXrg$6BUI&>lpuV)pJg=A^ z&<^6xSmF;}%ES%Ddq!*paxKxgyzTch^>aYI{zSk@Bta^+0uRTDpisME!|2{(;u1RN z_0fsG0IMhe+9toz8B5(gc73d|O2+MOGmJv9=zL~A+cVL;5+!VFRV3;+RmzWd0yZ~WX)bg8h?QNGKhaP>HFy0Cmlz;WB> z9N4vg_xcD!&rZfSB>jm)X?9ROnim1|!hBmLD!wj+;YA%;8^lU;{1lE+N)P?1(^+zI z@0jGfhT`?I(pG;3E!bTd{cF#abG#KOr0NMe(e$>U=hQ#ED|Ze~oSbh9Z}uQxwHH(X zcVDG**8OHt2(@i#%MtgB{*86QHI_xQgHStje33VogQ44Pw88ZKop)6W9A44WYz%Zu zsp^y?dV#OHp)8Dgur~2>MWQg5%o8@s zfK7kzHA$e}A#>Ry7)r+n0QiZ9=L6V(!3SRci(M3xh3`D+bXD)gZ*$7yvugh)*vAj= z9;$n2FS|!j}#BQ%6xk{yKNDN*j`--Y7d5&j?kYNHSSaZm;e}PP)<}+>x@?oo$G{ zeMgeyl-1}l9xIwqj6xr2pB_woVJ%mF#5rR|Xy@jpJ8YgHZVhV?^^b}d0X;jCKWNC{ z9+3|;jo0g3D%-MvV!b1j(D1{QD##})an}%jq!($P9Rg_vw z<0fq$MY?U93*u?{Xu0_qB3m#rc7xvybhl}6TLhK?zqa$K6^aSmj^ZBaIIS@fo0MpU1Ql0y- zf!#$eKTbWPt7a6h;yMoxrLV6+tqQwZYaWSF5+OE3ne2!Kq}vhrNW}pETqocIG@J)J zmu3B4qfv14Xb0~4hBl7&`C>nSwm8ge;r-3S6j*sXCnUR5{9wqwG$5>dw>{Hs(o_W= zy8AqBg1dtzql@ch`t~-~$?~<(f!Ewp1FhN#OR0C!p+;8eBtzW+<|&E;9hYR3XT?VO z^;J8Dlh^kw-R34=VgT0Vk06?_W(c2k1BcK$e%b_JIo=~S)*BkJnhN0+f%aVVtTHR+ zftmEUJZS0Ke!w$;{D z%EQVob_uvgMz4zWg>{KRb|t4{(oEsd^BQW4rJPE*4?RU@@ffe>rjHUWZapH5^He$O zjt7+b82-WU%HLWf$$hqD$@B9>0=s7edU!O9^@FgYwS2AHnZn^CsB%VC1!-Z78(1TI z$-YAb^pVHg1QZYMRGRJH-Dzqg#pG4@=y_(JVI(0em|#vKL-Z9vY(rYI($7v;xOjGA znbq)@Wcv=ZS<%@r{-i9J!I z7MhEQlx;Uqn}@mUn=Ovlx=8IUPZ7@ag4+~?TcC`ePmzxGnNI5=^K$Uojdr~w2DStz zRz_~#6JFi}6U1K`@z0P1)2`|I%Yp`8>!rQz=ZkD_wKOF~OkVB==BCSq3Gd=-m@J%D zM(rTvF#m-W{J=HNZTuMUMRLiNMiE5sbkQANYa%!QOfqd=D4ghPU2*fK8n$!}ih_K^ zL%Ca-qumvGu--l_C_JW6XS)be6TdigXgCcRvT9C&=_9b8Zw@Lao1)T_ZVU;9o47v_ zSm!F=h(>y%3AV5~aaP1UU+l6Lr%uEKPt&5b#<`A+4MnHsr})5YxgHxZd><`Tct?&h z6pTvF8&z_gxkO6O#0Zq(csaoap(SIvlBkhM+g3OkfW7TI9rHz?#cg{Uluv%&+TiR!H33Lq#hmtKD;w)I1w{w|Ygu8~iyqKmLQ=ZDx}3 zQvYYXjQ(f5{9oHV|37xu(AL({+{W}D+c~VNX?eto?DJa7F0FI5=Qf$r16`EKOdy$O zl|rlyIk+cpR>KH(w5U-8_Raem#w>6 z#ljsi)FQtjI8IGvVFJyzEx$Z5K&hNrwqU_k;%7)Is}C0)S9$Z0vCQIG;|`hIex!j+ zvBjoBK7N6m0}Jyz;sn_%$t^B$-ByD_MMP&lsXApr!gQdu-&nCOW1h&vH0Y>c*>;W0 z(_pRm$c7$m;oR>?K()NsS;)m5UQoKlZqs%q6(QTvn24OCOf?Briytg_orWs45lOcF zKpbH+YSqf)efrKl*}$CE%KvcI;M#bE5}}Aya|m&#I*#EbpKz5YhDl=gBZ)Wg{^b%s zaBToaey`18TJHIGX0l6fIAeB>RJm|XQw58+!@6mW(VM-;SM}CX#g#*3Vdw3422mB) zQOxsFJ;qwHaCk~PD=Ta3PSD&W8LSe(*v9XjQk0+%+Qy_ra{n16 zL2qaH%@BW3uyFKPuTQ6OWGGo`3zVcjfdhv%ZH&ojs>sfzWqnceVFBSiV8pU-jNa0j z8$xoQc(&X0eNQ0A&=fj%oX;3G2j*(O59+M?@=eIQrp?=2P%4?zda`B`t!Ad|>4Whj zYOO)nvGc@3i{F4euLmpurE(|JRdDZu(|vGiZ}ZpejD4%SJQaATvzOgh>l6HMF5Jko zs5a=b9!p}0g2(05y!AMNpS4wxXG%fcABgbni9Y78zCX%N!tB!QJNdQ)2A&>$1W~#k z!rCS`XMa3h#4%GLeq5^SJlkJ17-$0#raAFVb#$Ru<;g8eP&Ex=PlhvI?r& zQy0@6>mX447t*ph|2F*THi*B6HY1-~4Hba%<{Xd*L#zc$1*uPfrUE9iHrcnr@ck>$ z=>7M4s`yVQ-?iQSA~Y^M>GDejc^d;8F3BG}wa@Qpu2rzyvX(zUWeH@kg)8`8;>VkF z8S()MVY3j8o>}eQ-WFuMrZT(;n%E=V$fc*!7`O*y+{nFbnl=d_i@c#MK^Dj92~#jK zAk+$Ido?!O*^=&mW<%|_jqHL^>h{48N2X$yX=*fg9;_{?XQ(x+y0tA$9s~}}W)xUE z_vc`vaLuMko5MQFWGU~st42>@(@BNIpOGE{JlH0sx z*rMJIf?nx3#5x?o^Ks5#U$^V)?Vr#@%%O24*X^2OS6~wi?^*L|^qT5v5@*0&x=L7* zBs*liS;wxCISSvjA#0&xuXNe8ZC<{)2C>F2?2zxu*)U6;M(u`kd*CTu-lr~mh8!t{ z5yKG0az%Sy$9G>Pj$&a%Q}n;^DG!F^;#KJ+cEm?i(<`SJGRNix>VDF!K2uzy*!B z6@fDhF|F*C{X}$tK1IvLKA4@bpMGceuip%Kv(~$N#M1wC%#CjZI~4Z*2X`+8LsXmp zuV{Ju--!P2(F#i&eLF{eL(6~j*;?t=a+4nZ`=nTXRqh$&3O`38``t_1NG|OFt3)bj;~JN;^v3GH~8yp zE!}Lb>-@(%g%hpb^x8Ia2Tz4rJ^u9!qUNOdY_zQ50H zBM!B1wn~#J`jV6`kphZ%k>m;ui#~D9%o^ShHl4FPCaq3>sO8=aq-xgbym|7~6y0Ft zX;X6Nd9T4PW=ML8~jCn)VNY0wN_z=es+xw$e;Uv8Cp<}*^9yl zWgV0SsEdS^BPkgq#9c?FgG;6eufosXc~U<6 zaw5YN+%qKoK!CyGG#V&P*GvjP7!YmaszQ*{AF!x@;Dw3 zSLo=CsN3vn_Zt?Lu0^l8I%=Bk^_R?CstOc#X(~t@(KBYjSP2NB;CM&vvSl}Nj#fUH z9@UphV-$4+_sZQN_z1@pnE8)w$VzQt<=!6vfX;tEIoW^Bm;7JZ{x#;N&F0v%wl0Gl zE@eA&rp-JWphjPPWAOYoaoBQ^pWYae1Nn5uK^>D2$HjF$-kj`J>Qh#G9Ckv(F}`}9 z$dg_?(<$qzkEo9Tt%!S^SL3o#g4DPxUGJy2@QznoPe%`as_kyC=jV@s{r8JBw}*hu zj`sKamsn1nVEyA|(!Weg);NZ{v!nA*=i_9SN<_41Ix?4nLihB0#+^1r5$WyTvdx*$ zcx1G0FF7p2dpuRNFH0hG?AWG)qx{}t0i`3DZfJsc9wVVIrQRUJT0P9N0x|tyU=1l}5FvtNAy7dt~(R#=kw?D0sI&iXLXkc*U}*l^)-KuP73Jc8b>&oHDAyAE*P zX!pOcf3A09Ad+UNx7rGcRf+r;h&65`a6)R4(D8=NM zL^8)}rDxrwsPxrpR1x>QqQg(Ods;Mvmz)zR4*uNFCf`&;3V7#kpn43=d4dE_4=8je zc4|RZQ)>xWwzq^@`8eMCeshu_E3;(9Qesw?glag4SFEP#r=(pIIQszI8h3K0QAdKv z7#Xty66;QA*EY0&%RS-3OQHH1IoybRmRIAPZPfB^Sb+MU(Zl+89jkUib!B4G6qat& z2<8L!6=A4?)+j4}dKwD4uCyT3a`o-SE}h4vq;(cDrLZ2NF1sMe7k;lj4pfLp&3+6B zq3i@2s}pz@BBBRS%0iV!)fpf=EOS6@`yyWVJ$D%w!sB(ovq61#fr%_qE6i19PD)UT zHo+~(g+k+WyeKc0Sg`KG7ZGnrgH%d$reg-q5O`7(!iJuQYkBkH;7Mhs22{dPYJh#* zbnTu(EEQYb!%Rti0M-!hbgN?%9+$pMpoKD7FRDTVWu$NalL`6a&y1ttP}J^^VT7%~ zl5RUHjbbfmc}LN#c*%^@wEEP^RZr5avAf?gmbd#eb5T92Tu(O+_1*&BYp#-lC42N@fA2t)}jd# z@7MI*??Wr_SuTi~%LS{7g&l%t_+-#04eZr{sKBEumDqKTDpG=mGUKmD$Bs$4;oN9& zZdEpf;Mcsn@Jxu5&#NGunf70ed)nfMo-T5U=S@&Mj3g=qN-e3>$N149Lr_t0aXJA{ zxa7S)ASA(|I|?GrDTysU0)nFk*b#;i>v05q;`0v8EHWZzuVJ_a2tKHCUh}nv%&B=f zz__4Kw;XzO+&$U)U_#|nA$B}BA^nv%ezU@Wcj0?+XnHNr91 zrkzWW@j!elnP5bbzpwoTa!O^9__5>N=(SQ#bF_O%p0MEBtA{WoJ$r92s25FnSAv& z^DLx)TbMAy-Jbj&ug&2I1%e{mNY|R7QtuEQ27#(6W?QL7{GxCQ`zF=}k^Z)HXG_S} zI8;2zDXJ|T;U=|fsGi9R6$O;YS_8s{yd3VK+KBO0UwPH4kN`FGD&_s?@A0H|1V{}@ z03HaMZ{->2#vW-o`506Ypud}bp=l8 z=oCr?mx&iQh7CyS3L*tgV6Zyq8YB>us2Zr&s58Ns!UtJHq1YP=>KYLHJFaH$XEu(= zR3GlCl67h|>oCT#SLCIgu0E0KBxJ1s5&8!ybqEHe|D%6_;!fhO0s3%ZY>Ymkns3eK zIQQ=2$_AsNUwjb^J0$fVz0q8WV?LAR61|66;Eyi`J{l_)G6GPFS0ADT=|dYri>)x@-fZ|rnzJZxk`36|2!?}{t03|}vD@!@SKN-*mPXk(m-YSs$*BF;9s zcg$ZM(72RTsz3UOIyMGwW&~!$Ld;G-g`a7jg7=ZanNn$+6&y9+qEr%rU&eD2g9=^= z>ipzU=E#C9Tspf{3(Fg=vF|R=tW%P~G&F<#cPvGuY{-qd5-H60kY9gLkzHu`s1%Ye z8Zymlg&IH5o5SqQ)VJQ0vFm<7+0c+{6Rat)@_MQcsJahf@g%C}Y-&s-!$I@y!d}(c zp|CwXkT@{)nYOa06?UF@B^bDMnY_kQ<7bkz54q9Ppp@dPEj&0kvZQHhO+}O5_8{4*R+qRwDWYW|9VYRMI5px&y3 zz0Y&l+zy7;lHQj~{GNwhcT_A66I@l(+2iINP}9Nf=abHjd~Jxi7zs||?Ac9poVoBX zObc_J*HSD$fq1?Eiw-=nTAFD^KT_KL!v762FR!RndOWUe*H9mHJ?;R&p|rjCX+2ro z=E;R{%9P2K)aO9OvF8_fx&+uCLJgn z`3c&qHt6|1NB8ViF3FHv7|7i6>6d(9PW#va`uiR+Jo!_ZXli6+ z_syq8veZg-KdB1&rRi?@&2GL>*XTPs^h)H=aGZaLrbm^?Dq>J1=0S!6g%UC#*&_cA1{W=w2>yJ=@<U_0x=x6tjPS&cY>Cf7srJ*OhLezC!;6Iqhr!1Ye4Ib-tB*Qf0%tRnuCw?+B0S7P^+0#%?L2I1M3x;j#z7?v8vE) zM;HOCPM20}-3xt9&qwbbKC~93M_%y(D3&3u9ORlWW}{;_gO2}j8DP*6&vsT!awj-K zT1^s~ifV=@AM2VZ-s_<7ya31%ZRxcGV`Eyq;F4^rb9!cp+POfjnt@Eq6Sq}x7Gno( zMK%K^k<_RfO_=_X;5@oYJw=+{6>-o#783LrwYFqSOgT4<3#s4FYRa-L+7zx|tm!?J zN~hm~1jE*0^77a1r;0G<_+$%&mnqhUdf>>%A!8I#L>UN1${WRxk5G}+b(ux}tf)f0 zX&9U6ZmpT*HIb=jF;}|4U3B3VLA%IbFl|iw>(OsrGQky0FI2%s7F(np$MfPnsG^IO z(6W8jYdb~;gHPjh)G)KItJ(U}0(#hjB@zLu=rT!S-(}};fE*J>fU1mx{kas`{X6eE zu-*(75gX7inu#Eki0cLa}!i9HA1?zL^u#`ug1dAAf)B(>%ZZJvGa_NC(lYro>zBhh7&ej2Bdbt9x1& zA01b!V)ElPR}+V1eh=Jv$C-qV+{wkQ`}-t z>pGqY_38lBGCi{#k!O@Knw6}AGiR?cxUMTq&QbiVa$@WLB+`_Ro^1;vBc6KjmfcG= z^+UZA65o!cYg#wX_{gY+x?Xi%@RuvSQ>(m1VCB(4*Hzq3<~QRVj*;$eeu!N2l`eL{ z;g=0YS;H!UYI$6r(p?8XG`wnLwn~tHrc;$-OmMlRAu+aZyO{`;2@teBHXu%g1c{+-uuQI05G$Q3jj7ay6)8rwGuuM)m-rzUJB{iJ6!VI$Hu(5ck>pyhOF}Hz9Zu@ z??dO(uVNQ_%ANlF{yJnAE6{lbBP>4X(+iCUmhUALvvsKk><*8NNjdFKG>ttkkK`Da zdu1kOmA@EN)?w@sy+Zug{4;*{*0W zbv+Qgvt1syF*gQAzpKcMZuj+~_x2|TjbWm;ZMeKb$DO*hYW}u19ut|m)oIH5k@rgE z{Yjr9v6AP^#~pl~W_eGQL8--MLKB_uQp=ShZU>s(22_A1o)@yCQ5c8*I=Vl~Z#bhR zm+b(kA+_|=_z&P28^3msn zpU=m6+DCz&G^Hkm$-;0Jz#h{+&Dc{FXM!NLd%_n+y!HJ@nv=l8%56l`TH@^*gk_63 zV`jK+YG~=9Yq9}~Xr?+6+B!hs%|1U&%L2o0RtOJwjz3{K;dS{#P7qXCmvo7~zPmfo z#nzj^N`h-Zf5rdb4-2=+Jxj{(9tS-H005@{&0#UNw==W+4}~m8{nBoe4dwf`l*8d? z&HtKI6AT56i$z+EMj#8sb_JFKECkIm0)o_v=y>RT*=;SiPQKwbKJiK?+XfmzfNb`C z=HGz2i%q3#b>r3|1@7;fn0|OlQ+)avemvdW+`qq0zq7^R@!Q2$F4bb? zMdfm-+BB_Y=a2Kn@u$!8dc+vzpneAqo|a=Jb6MK17E;N5QLA9*iTQ<^B>B)DXGk(~B<_Yo)< zu*nUySCU^rw|N&w6sotzCBrrCwpqQI0hy~-gUL^KX=QWipwiC3AwQ8t{k&NNBc03r z^NqoOvAA2UxJTJ_l>dTnh*CAk$%4U2uoWbom`B9DQGZN2| zMF6<{r+Xa2Ew^eW&1jNw`SUWM3H(Ej9 zgrfE8#!!2-1ZKqLh=&$MV5hnkmE-_3i;C@n|CA+5;iS%TB*=Hdl0$?*PMiO+vjc~% z!XGnPOS*}wX{bE{Xu?KTf(c*N(=A!lM13(_O7;91SILwf)scCL=>|+(ydFGpGhGu& zzEh3cp|<&jQ+k1|tuXh!IjJ|6FjBV)(;a>o+x^C#?4BRHpE^(|lZd6^Bm)cF35Vwj zgv3x(6NIAJKz_vqL4Nmov4~&N3hxbi^TXiY+)hYT`V1j9QO8njr{aPX>0>OonGi2NKRq=4<#~8Egg3_(R zt0gBzSqiIc__MT9IvXWGP~8=R6A9@A5*D#Y$KLu}8oM&VIZ}^Y*Vq;_LX=j-rCLnL z1G`iT;2&tRB{d|7zcCBoj+Id*;a4j6V+L|d2om#*4sgyyc7%3G4U~bAHxUx|PaZI$ zg$xD2aO5OlHldn}T(G*goKt-2wA#!0(Kw;4w;v?#G&$S8;kmz1htb!qc5WQ)vvfw1 z_;=ZQiV!xJq`}e{V@t!vvS`n$U!ZNB(p z8(uczXI1HmXHqcV{8x@=iI2XAu-0fIIqOb`&67QI9E*x>{$te%b*3$^*O&UAjopq2 zF&#%(>nLPsA-We&BK6*$bDXp>sQp{(m*i2s6%2{)^~Ccgbz4!eR&6qGOqTTcW&_aN z`ympwhJ!)Hb-bjCQdby$Y#99)-&Sii-sUTVS?cPlCK71rMip}I+wC0(r81-gY^1I+ zP(ip1XBUxw>Cc+O!7fjd@ndRhfbi_uw|HDLsDF0xx{iuB%Hs2z_6Am{~s>{~e zaxWt$ToZ;~lI(2lHv0CPK`+;tN!jtU*{-F_PsjN-Dy#$mift8M5KmJV7eMJ%|zb^uXKspdwONEAG?|Z7Lr_-Y0Y+u zZ>7ahCCQPIznr)q)-hzt%d=UVpdz<#ijd|K##!HWYvjGC4X{u#X_guThaI|vE-Tl! z#<=2C7Fe}Bvh0v|Bww@y<=))d&hZbk%XP8~wBG@gsfI8?CLjz|qtAppbJr~6Y6r0Q4-8@oKHlPhcvC8l$GozUf%9`|jv= ztZLa$0@}QT|MwtQXA+_mhy(!eDFpz){J#lu|HX3qjdIgGn|6m{ska}IkzWW%sVLm* zS3z@DHu7$X=NT@e7T7_W5I|W((oC%$ij5Mj+d+UYmV5?v`swr;&KCn6N-f;?2B0L2 zQKfP@T(8a*>BtWaoLkMq-yOt?Xi4_7J^y^gcD|3FD_7~qCiQo8zaDOemoN2q=*Y;A z9uAiEPx;X?t1)wWXDqi-I|Ey!AHpWpA8$|SC!5S&m@?!_?LQi*cUU~M^orKQ3U+=J z%w^@YA1;5s@j3eth()*OTZ+zMrzI7Oy*xMOH{rst8>sCp`RFaXZ)+e6@x8RnlWED4 zT4Br2_^vlf^h{)>B^NIOYGk9ZR){!x=di{mq0&J=$b+tdtlUE+-%CgzVu> z6Hsy6DU}}50{f)_If)a^p3_%2B_&tA^IY#SU*>d6aSi=r`)}5dq`KQIvTDXNkgPb|GBJm9S6<1G@HF^1vTZ`l5LAU{_0V z>4`Pry1h`?hF}|frhDNWJPeBP%yxUq>02b$F9Z}VO@Ejsun_8lg*w;}c#jo=P+H)u zp*X#)%SWM$I@zJ{KYs62sYvhA?;Wql(wZYE8k3$s*s1k26WSuZ{~ky$vY_rrqHn0f zp=s)(f$y`@s4D((9P*^o>(`WD00r;heazL|RlC0*Q~P79)l;{NzuQer0C{{B2o-k} zuy{vm?6LN=AJEeUh<1nTS}MgL{mV$7X2F#{qv zI2lgRu=nqN5oV8Y32zY?#LV7NItu;c+=5+`fmdP^#--&bfh4-2mt(bt`fXua3kZQ>%)2}1(Xm5peK$Sl-p zgGdkN4W4fSJRnUeM>NE(3}C`QiZR|x`NB%pqN5^XjJ+^R>Y=!cxn3qAa*L3F?jv_2CXylno^TU}MHhL&_9wK~ocWQ6K!gshhFGLaY7HHZp#ZG(&3d zeD_cjAp>`M<~hBPcd3d1!J@*KHKq27RfZwF#}>O7W&;HW@4(Ly76-ie*tuEC5At6@-J;R0h#0uK9EBL4uk z;njly89@tyj$&Y-yIcor9tYO&>H}Lb>sZ(qdDNBFq= zl@w0LO%Rg(SfS#sdo#Vv=An`vFjxWSBQDyPP=l_qQiA&F1NlL(TSV!K10gK}hwNxe za$tB~OwX9X|C$J1ExZm_j5`-F4xn^J;zZ~knqOnb_0UodD`K?y$2;uJrTXwwY4rhv z$`VYpo>jL%e|=Juz_g>7^w1Oh`4%XPLQ>N^FTp=&-xrO@JwWkjX^iUW>DkG*N3mma z<#aLz6r?8K-a5{_h0n_K*D-h7tYJg?eb|d8MW1Zeyz>U2yI5SutyQ~+-`>9l7S)_A zH`e)-B%u|5EnVG^eu>Ij7{4#&Gt8cTZ1U7WNmx6_bMn{>uqRm=5WjV&#FGW0BlNKd zi4vbPc9rHY$e3VH6=)Hu85NEgx*5;9DfJ|dK@=70tu@IAn&7O0aO^RWR}4N*PH7en z#RemZY~p_=O%%f2vW$wFb(4m7XsB?|vb`;|hFg%8z%q*rCZ$u9h|>}Y)u5Ho$nnCU z#X@i3)igugE_1b@5^tWKV_88c&-+@=6R?y9%^Ctzx0AM%T=x4l6%7XGOdjAhEC937 z5Zeh35KgLxrbgtD3GiEQ^ur7ro8$*Yf<#rO_)}zg+EZRmD#A-~D-+66*N4PpA5fir z5a4l)x6Bf*qfrs|)g?=ss4|~QrGoH8t!`m`P;sGmiF7xhB8;7lsZ%2vWrd{0 zNmZy7N!fFjid!IA&$i(~Jm6L*_B$EKP+ZWNq$eu$03)ZHquIv#-S_=!Tabk7PT*`P z@eKzWSpHk8619F4>olo(%c&=v1g9PFW;D#fb(05n3kJMNMBFqEB_qS6?gEgR*z?*V z7S18ylD@7w4?*zYCKS@p!Z2BI3jD7Wj2WL{JrvHE+5*M3QQLh3*SLs7W|p)5kNmxu zC)>#E#e(?IfAi-kOYb49;@y{DTvH1iZzW_c?Segy*837qhgtwupmlh%endFA z`UDz!8k@)mf``KGrYHhH8~Bwy1puWer000JbQcRqwY1HwETvoy_5byGB4bkd5WkRm zUPv!BROX1iWoN(F?7Vi6y{1_`jAU_y>q^qYK-F4M^nE#OHa8XI$=~Ub0#Lp=h3~gw zMhd+}hoEtei8&kJujO!TdH02OVcIcRL{0qB5ilJ7B&%cfP+_Oa0333KNoY*SL)Xar z3`Z9guRJRwXT@Kgyert5*_oeuYTPPV&=r@>awM9&J)&S`x?ZF4&0EDY)EzrK&*7zb zZdtH9ZU?gTWq539uzNH_+u5;tlLqEdU4Ey1bIuiV{+{f?QTr>rX%DTL8+jwIXW^t} zl8^x)89_3B9t4w8O3gZ7)A!)aF{JaUYg0uu+{?ZJnxBQ#fm$X5>kN-xF2($}8ii=H zeH#@;h+cmLU<0{d$}xB-pfsLXk{4RbH86FpDzL1J$N+ASa!%)5KC2!1(2%r5p*$T`ZF@yf{-Eyh|=tHs2kxKr(aY-MOh@AEk{K$g>C<@W3 zlttq0CFpn2O9o0P;H_5(PUWD!B2w>m|ch$9$c#sDFD?h8r>T#yVZuLMFq3_W2(qvl~dhKTX#zXeF zf89Ha>ID}!_|jBN2S*6|zAxuu^D@eU=RxG{Q^a}0u2r>F5=)4l!!YZ{78byc8;B7_ zMxTkK>KU!CEAQ!g(cr7@QP+C6JI8VTRB2(Z1s(;p)A+HkaTg}MeW|iJ;ekGCiJP*p zo%Ayy^4c}oj#H@Fl17R<-$lQ~eG$SlI{%4#xD3;7X|W3H&iDSb%p}|0Z1)0Buko@CWTuN3u&6@Qr3~?^YWVdzF6sWQd<9K_rau!Zvf^b>EMO zq)b~Zao9r#1Kep3!j+T(4r%n-pwsdi9)1Sxym8~6Q>zcW*Lr#!x4hVpMngf6V#I)Em9Okj1@;NA23)TM;omGFJRP0#1beN_VyfH$?< z%OZPTuRhc) zY0YEPZ4Sq1nNtl8>=Y|jD!Qc|ek`UPr~DA(dG|lN4Cymli-)?<#qciUp@bYS!wq|i z^~TW$0M`nh+-~vAuJr^V!Xbcc&7(u-tt>KP${v23(un%edjBr`{(57`Rb=NWp1~jX zyXm<{_NMK{{i?PRVj~+kGoR%&OZBZXFM=tiDuw;vLpcNG+NQV27AdLra4QnXi-k#|a>4};gg(hb920KN2oqP20Zfe%r9TMAQ>XJzh!;57eEoSzKe#KrMo9)E`9 z(RK4@7P(j_a5;hSCn=NjWVoeQ)%7D}!DU<)XE$=9@!*Pc?J0oAjykw=J5t$h`WfJl zixSxbK4|Ap98^aIV}&>iHw{DyyxG&CylHO>v%BcQSuUTTi}3u#l&JtEetM`s)L7R4 z#zaK|y>zGbk+nIi7XL;h@Dj$EKp5WaNqim%@cSX=`wqRZRzEnMVYO10nmMfv?;>9O zZBFs&ycK~l`EDlAhPOSidZ&#{2Rv-*DxEQ&W8&I%%@DFT(INhg@2aifws|_4F2!1t zS&X#VB%5wbc7^;Ap|z)|>>WAbq+r-lk+T-66m}!0Ty9Nupyv8oyIJ#eLY?t~SXt+0c@7Fb%SwoD6xFV}&w(3epyjmJ-%xT7BzoOG2(z@APBcU zwe*{r5OH`y%4O}h6C~-6bxT&-O3~5pI?%k0gD}z`<@)ac%R_zD6kNY)!eOOI=H++1 z!J01|{CxL!67lB}0P={T5o>FjxNpz4_k_l_(Za2X?|<8TXSBe^5q(91G`s=pwGPoa zLn4BS&t{YLnY$#m?Wnt(`YN)s5=AJby7U=#bXa*|WtHQ86SOCW)jXRM$0>zrU@%_J z*7FTDz_TnyQ=db%w(CjhV4Ir@_J_@f^ZN@k5HHNp)$Yt!0KN%2wryFiIMXPN6FX8x zrbBg6+8gavsfa-j7<3sDp~cxAZn4Rp%a44v1Gj(eSMEdJKH_a|Vq4co>aZz1qi}u;ZL)`c_w27 z3+5ld=D1nZT+4|;y^+mwL6F`|5~!jwyAn-VYY{cYrW8-ghk|N?dtnp-LnZcs2*{{5 z8DhTqbQr>MCT>FjmY(KKvWswdKPYR%^)q?-1i<&t+l=H0+hynNoelpZGvO7?F)#Rb z7!r=y+BjA$6TK7n@D)6ia}rZfJDb}$RkXtLt9M~hMYNl#t30O!Qa~lJt4!_j(VdWI zs-Sj#Gd8KJt%S%#?%3Q^&XfbylBWxHIrl)?%6oG0e2zPzP6{H! zYSTgZ14w}Q`CR4E(TeO>dFZb_UpUGIJ}sUAxHkXk=X^o>&dYs%9q$GB?=7vaSA zYybe%5&!^>|4mEF($>)2RNuwY=0EKs|G}=`lt}z#*L(762sl}2{>dJNQmo4$VKyhA zA6mZx3eBHWkv<3}Y9>KlXJ+sF7ycRYDqT~7e??9?kzh{N`0D^|)YSdIm#VIfuXwKc zEr#ZgdZqGUL1C1mEFpY#9bf#Z^lJ6<^ z!vj^pWFk#tS9D_&fuZ0+RS|MS?&y4WFsEx)oBR%)<;UTu`aY=v#3H2RL3j!1l2$Qt zW|EWK_~jyeL%Iwz1-YY?UJjq%`^gn4oA=Aja4(#mqkqsMr0k?S)}psW55JeA!_S@v zQVrC``PKDx4&4AIscH7mc{GYCUaAY)B2eAQk+crLI=E!|j2cAq6eYkArT+_ej@ukH zfmNnCbb|(|Y1bH;AybKEGGNB=J_*gBV`2z00Dp9S#y!(4krs-EpNH&{ap;X2eAaH50`f@d=b+z_b(W+FtJ_DN;g)WZzZ z)F3Er0LX*i&tq?&15@EK$CSBfE4ayDRUe!`5IXh^@KuimUzw_bg3*3pEhZ*d`hw*5 zbOf3c9HO69M2Fd$uZM`0x&Wr6(GV(r&d-^ugpEA8@i&%GF2(rzfTgv_8;9X~I;HzW zu{(=I(rRnG@MfW}(&?l7qy*ZbrYzt$G=ex7=9~qq8dH~?9KUsX+n|wTVIo6rqL<0Q zrNOBvX++DL47u$c5HEODYnmjQ4`2GwK8^HXaIk3v5d|U;)komWg98#51zA=0W4phH zkTOQd?A`6L70}_BIA4KFAT!KS>tX~tNADOB&FRA}PKj*G8N)mbJuy+_U!NZyf3M_+ zG7!WJb7Z)n0xVKPMS&!s7ODe;<7XsJGCV%;^5FmogBm0Ffg&PeU0Q)A#{zm&L__MXS7j=TtEq{iv7}36nI2 z#P?Wcc45S2CnU0#e8MyYpiWM&%ig!>`$idaj0_TtwUuOJUR%ZTAkT2tAW!@KH*|e3a4teXk)-;;l3&cL5~7g3qL|{|YLW9G z9e_?y%4F?3jF_dhfB(HbOi#iex|RNTzFU2L95=F?h|GZhOB$_{>Yoh?yE4DK8wln7 zybBD%!~cObjE$3jn+=MZSwD}fmD}Z%z?Axq163wu7RddK&*tId@^E;FUwq!1KRguA zUg$qOEbK4*6r&YA92|v@`+4zKe!Dy4qZx=>-;KlL@q9WvJ1cnnUXNOAJ;e8k#(?G;);OJr^0#0|NB4h2 zC(HIyM52EUEwXkbB`^!~@-DeTy_ZpO5F;jFdo(Rhr=`_DK)Un@CMKvs8SBV%JhXuZ z!u%3U$bkWis;-=_3*|NgbQBX?$*MJDPM=z2D9~mKY34Zj!YL%J2W>_qf1R9vaW%WH z_}Wt~aBuu^gc$InXIdDFR43W5I8=jhn*I#gm&k8;!04Ds| zVbR6vYDU>dg(pcGRvkE%O?jGpu2WB`2mAoycLFFXbN}gmUiWUvGuw9Wp!_AnWaK*> zwCx4Lucae7cgsjM`HYmr1#pM_{fQC?-cogeF$w|YFJJxHM5+9~Q-+Ruh|fCB=&ReL z{t(j=1{ob`JE&~%U}1@)T&WVS#NRWa{=Cfr-Auuizt3O zN&VC3KeO0#o0hj>?BLvXIiqFbO5v&gYd?S*)AseWRn%gYx$%(736rzu3xu zg_Kl7U@(h^oE6w*okC1ey>m-2@B}51wRH^~(|b%;90ki1cjVAmC4!)=fWro>9By|` zl0}k&s4YCvf}1_#6UybkuiG@&yuF=6UUW}G)P=DjrjbyNks@1@FkU}<f%1d|*QOLAw9@Y-9WEUj@}O*9XyCKE04Z*gCi$oC*&uf=3m(RzrM-E z`WA3zi=X(tyO;YI29yg@-%LbR$@P#?UI8JGyiurGSN+oHWDL4M1-tNcG(0v}&l+1} z5mV2-{663CZ@X1zbB~0F%H5Yd9>hdyO;HO>3!va4W-5?zYKj!p=_lTRct8I4cg%f? z)yuihn_8WUSWVR-PI##l97(Do9^U`_#WxktpDI(*Wak{SM~>ZfT6;dsi%LE?CT*Ry zy^b_?+{@qgF}}?4RF(uhg@dxc*K+EK(^x8F<*@uOTYV+u%7K<){(xQ<;#9mvUaL`w zvN}B%Sf*b#fst|Y#-%ouUj8*12cZq1l1H$0kf0z@n|^OK7R;DBrgeUCSXyB@0q|!1 zlQQFh&Z^D`pQdwK)_E9}%kbB{E!SZSfU%zt%dv`J|1~bFDQcXwwa5+o#eWNr3rom) zL8N-KxDyq9c!&6^!%Cr?rw30$SZZO!95Q35uIS*4;^U}#O>U{>9_IKJyA8wDZs?i| zN9b;Pc)ffN%58aBQLgmnK@*XO>w#ZI%R!}FXd zR*Y$8VNEx-Q-6MGbbE4V*j8@jkZ?bTz-STMt)n|@i@Rk1*2;V(0^cFjm*;}Z)X3B6 zct8DxieaT3UJ78N#L{ns{oi`kX{>UEI}l#U+T2KXn2Zfp5NVT^P6-2{o6WO;vgegT zYAqC3O(ss_B#m?vC4T8jvb_>`a|=?S20}A8(4i$)h0j^07Iok5^|S7cy9bk#RF$g= zvym9;Xcxm)Q#tShWkNz2^#@gXoVA}0q1d6^w{1U$F}TA>H>foXDSUN>7q%L{R`BvI z?OI8|kcQddp)^0DtEA+XGae6?eZ17AdFxzCnwQe6*;@_y&z> zfYR_LyvBua^AwuE`4Icp>mC?)@8q2e*j`C)TO78StKtM$CWu%iGLi+f8FJ32*@3bEwSUoENJs!~gFk-K!34Lc<^WL|%65WOH> z4la5%GLW-7a+}L@^9x>v_KY9Ynf1!i!XweU^+<~1-nnl-QJJ>L$ zDK%iDmLxVLUJH4&{{(#U8rRn!h@}jX~=u#_>$A-p2E`iDSYUIo{~9F-RhHW6v>J87MD|={8`I!4LooIP>A>0GepK94rEJ_Hd~rs?>TX5eKR)HY)hQg zy*KqF0K8VdCWfmyalCkrXvJifRxIik{#aXXV^b7+$Fk$Pt^ScJY3)y06fQH+zj8vm zueND!-D&RA4>qNMPYKjg7Py9L7Z`GSfR;UyL<;SGno$YnDw=$EN2^vuv-9^z!A77S zuF`1m2U^@?8kkDA@|kBD?3daG0FOkHDO^-K&I|Y^FprY~Z`K?mfl+)P!OST41)RHn znfUv_6Xuq=PehM_vs{fh5Cw05yriU2K7`a~%^5?3Q%$CMW1+ZhjpP~UU`2#2^bM4V z4at{4QUI-Z&E%!v3|99_ktnh8f>M*d<rnKU9YuJCW^NZKP`j~0H9=FYeBn4Vgo7S4Ja`|+9t|$G2X|7kp2`AZ}Wc2 zD-Gs0Q6%k{F}*e2xdu%h4@38i%~|(tF<*>P?X;v{WElZHH2tzUZM|5JG4DurSMUnu zTee)-3UsD>^O)$4Zl3mImeq<;96c#>%68hXdouY{^WnzY5*0~2b+qxGYp>UtR}1Sc zuy~#dKT17LTOSRBIU$|VNYKObe zpR0#)oBkWtsoM2Hq2LlU7EMJqwJHt<5QciJS)i(!Yc+p0D{qz8C+yA?DhNZdh)J&s zZAeeq0LY4jzgjaX{j^Ub0EFjWIOFBXiW1V)`~Gm4BT8hG`5#scJz`7}-JWn=J8bid zAnT()E%sM-5Q13zIaKazJD368pW%zOrjS#pF`#bAwnvuhT77EINB*?4=^tg0 zT0I#Q&}?GttSa*(xnO1GrQ7v)ZVT7++`fFnu0|TLIwSgi7OGm^YulCKV2@E;2FvUr z{keP*7+rj*r7yZzyjJCDlHF{&bkbm$Z)&2JXMS)1Ww*TtN;zUR%yZ7Y_@4JZlPkQi zn{iT)CP$?@En+>}&H!X30F(L<`Gpl{&mJ>g{nc0R?YYP1d1rO~hGArKbZrEWpVjU^ zy!w;5ofcW1J#1b*sZE$G!_ajH5p}cgfMj=n8;V`uUDyt-b-S={mI_~hv_-i_)WAG- z1s0j{wiG<9h?3RvG<7NiwOfiCKMhRsdh*RSgG0bma=w?P1*L<$J>v$gf53+9vn_JHLje9wF~%LoqF~Fa>zY*HxIRH% zLc*`tmfpeYI06aeRL?Z8m}yT$Lze3RGoLiw(*Aj_7Td^=`4|rcSXQ%S%kA%8{o`_c ze5q8|UU(v*?oHElSC$qQX2;MsTrdv%Bre=hyUnqAWcG9MBDRIO^O6!B`b8vfm*DH4 z23zyNr(|b)$en;C9j%^ncPgJ^pq0j^#hk5WQq4J}T5nK+J9s*~RfQ12FA$c!q}e`& zVwxKRXRI*|G_(t%#rZWdGp0fmM@epV^z?-wqpUqA`J1ujVj_t>VQ|-HGO$y6vdVT! z&L4KHhJM)`>+-*{dG7y_iMk(e5!ra_n@D*$PCqD~Mw(Sl2*E-KGM$^4y;mwEc39hR zPI*8&u~9kapfF(SK+dBRo?%WO}Qj!Rtjr0k!NFvbU`63*5ewwhn2=7^}UT3d?; zjB{TmSNgVIRCwYM^FURkcu=KY&CX}`p94IOgBCZs7b<_2B7rUu)^{X5f%WL}!7&4f zP!3WAOHWclB9mnA9(sjokp7MwGWGX^+Mk1XkqDFL;Xz*9B;IA2Lwv91gZGRPF*n>& z95}B@l(JT7&n2IvDeG8L%8)j%8`fPF)oosRFhLv5wN<>g`1Krjrw~kv=sZb`Z^PQ? zIBu%zGD=nPP=-FHR;_1gH5`oPbW}OSywZZ{^mbgwX!tGFPzF`oB9*w~{=4x9;2{vm z0=^2@#wuNJn2Rz8#fE=L6}t8SI6Zwf2h_3gK=W@AI5_jKI+^O~BiXx2u{G2I)>SX4 z4mymA@tu5-s&!~;58T5ym+37SL4BZB7rquPf=1EPf8BeH)X`L6`QOCpz4^VQ8kAlUbW|9zBCK1LalzlIkc#|n_iO!{7t#5Mda%x zQp42_2(4WVNY^Gu%uyU(KsTkwS`?2!iJd{Q=IJTzjcKb0EHHg6P zVdLBxv7SCEo8Dd{l@$DeC#h8$%l7?Y!#HM1EV=T8Pt}x%wRt%XO~&Fv(;`lL3U}3fDz*BMi<0QezJ| z=MUvyjDBVwd3q1>0L0|8SmxL;bE9XGpF@D|deaV$>`8rD9kDWv=ttQiE6U;Dtb!vH-vNXP-{a7O|0UYwq+M~8gfo&vkRi^S>oT+>oJ3!Ap=jiU1HJ`EbLUubcJ z0su@yqX?-K4spVv%y>s#A=A!U7$1@NJ(}BCaOk?UvN!x!U&z_(%-Hc5=EL|qD@MOt zZ((SNLa%p8-LN^-$L5i0m7k#mV#whD`{;i^zZPs_QnAKvh5ppjkoXgIg$JIM=+wF`XuW;ZuA~riGu1K{=lB!IvTy9 zFPmvha$C@~GzaSg_f_dKCTy1e$8hC-j(5N>dNM*XoIOP7)=c?(XvFPRmvO@Z0ZNZ{ z0IFHJc=(+{?1w!kp}=uq!n7=G*8u}cPj>$qO12qp8(R`f#y8TyrO`OxrLaZnzyq+% z**ezlfgU}L6PABdmii<20~X`&kX4a?PqmxnmytEi>vIaMsM%qeJhrpfdDyYTpQ8u( zmeMv4BwHW582!eTziNvoF4+IHC6>ubUFTrbORE1MwsM`647dS{THS2f`V~Zw^0;|f z{^yNGeEYKSb+mn5d~L)oy(#U*fU+yR0$;HEwtcv{t84ux@_5s-s^tydv1x`ZXWP~t z$*;Wqz4vK-wWdF!@U`_wM0YQI2s*6VUNp{{W!n5Vo$b$-uiK>6VVjM$R&gHYqV=ZD zx`{{e#Zn}eHg)%k%K+G=)IJhig{#i4Je5GAo;Gx*Yqq`>{cp8^dpg>-bi%{aNiJRf$#wDH2cNk2TJ*HP`1a214eirZ$n;EJuI>|M5jK_ZQ@PPH($*VmI1v`an z#tPX}_xed~%4_G^tM1k3YmCosZq#?YPi7>NkIO2jT(Pd7aO&sq+8Xq-TBppSACREC zNXp#@lqAomKym_my7EiDOpWG>GgQGx$O${win2GIZWHU1&#_h3*KEnvd{&79mt+Fj z^a6P`6wO&y38r1rM4PQv4kII$^A*ZxPxI{V))BLR?roKKby%Zz?J&9jzm9ze@B!Tg zmI|w$tblHdic>yjPI~%N*=22w&9?zEbC@3L&a29OBXFYn$LDH>Gy;j*?O)o!l&JE~ zzueE^0^3^>1_%wUvmc$m@3uxk_Zd6d43@qIrzQhJ(>p6303V$c(8g+Js z53AwEnSqnX$~wy)b79lft_QOtE2!&=y+K|{`>)ax8uJ?KrW%{%l{07eT0z{Pv%@H| z=DK?SODI@~13~omTE{>Mo!-XTV+8_kDIIk3_w6zOUy6PDbQO4U=j$K`qzHfSf*nZtF@W ze47B;qF~vFWfmV`7jZ*G#ph@#NT<2ubBP^6bSlJN_+@n%l|xqx^Z{uY&g_;bDjAP7 z-!;FoCgFK2>OmV&F8+~X>Cjf$Gap$Mo#oO(N42^K+STapV}C-ZGhjq7hs(OyBRx^< z^idu~a_F_74qv-7)b6UtTbIB``}KA^VJcBT^Ny6^Syz+Hhm(83uGto)#S~b>#M&g- z@ZeTxw(lK_5sG&NccQZO=8p|+3$LV3fYe^2M1C(5=o-{NZcbQ7ty{kG|=K95}a(Pt}ZJxw|nb>_9`v5FDSMz63=4E zX7;vk*j%#iiF<50Yx!nX^qE?1&f9{uiH9f8>b83<3eY8fT(L3u1l$pe_RRVDAO7>y zn}J@8-+^wxU-;nvO007>v~{rgRqH$)Or3vC=*?>Vc86@gQ+d>2Sy9m&Lh*UL7~2Kq z#x^lzEeROcK!OH~zq8%>5~@Xs?($w2;3)qF{w2=DC)12t6NC+g4W)(-X1wn3Jb-|8 ztfd-t*CUkTa&Ys0)DPrW60Q*BxOadTQJELAh+e{`QGS8BvTK16-HB2s|BX{ zyj{OExb0v|sA1xX;`4d8nR*a%uG#j(R=Gg_86>PNLT#T(4-PDb7pl%j08JCRCy6ZC zx5nSg1gfQEIC_7X?PZ{SS19H>)chnoS@(*MSW*I1Y@%|7!)&BUnZlqEJ_@0`zEMQ4 zbI*7es7k(oWK0@>s!oqCxa+%M+I|~5KdXSlKDL=2l;hNA*0xG<7@lPPXU|qK=q4Xa zqbd1PHc7VAiYglJeS6K9ZkzWEOfRV3Qbyzb`SG`qeA%NW{9bkJhlHu~JV;^X-P~I_qKNb!(!R?8fUCt0HcjZ{hOf;{6a{KVs z7*=#q_eY*e>)j;R9xZ?*vq~eP!&X2^;|L$`%*eE*e7)}UNBL!5n?TGx(NUT~sYjr) zu=3a9;e+l~dJ9eQUy~L~3O6LHKh|`|>`jH5B%t6(?-5$JrnI((V~!kH0BDM`AyNu; z4h-x!Gfl1>t(YYvYEq{G&@-ULU>968mOb#O?| za_KpYqj$wP!~D2pK2lD;aV&%+-AY{><5)%^KTl8#rtA)dx|{)#ZfoCelppi7Z|DZf zPFT7)Ij%0gH$^vNy;Ix2$llTl$?Dp7CzkX=OKCZ zrz;6N$o2hzh;1N>IBMNLq;KO0IE-;Rh<-i>jNZ3Y?H<~Dn-~utu<7t$ZBDz1Iu$k0 zdDC#L{;~NJTJJ-XeIi2wAx}Hz_K3r))s|u06qN_pa>@D8ww_W*V%l)Jcr3}b9j>p| zXWng+Z)$75)&vDa2Z>BvP z+V%6M&eUoeP;|gB*p7Rzbf3%js$iCLILX?vwWE4w`1q0@5f%!P#YuC|c(b(qw~ZNa zBWzPl6o`?aMvdV`673rsT~_o*rC}B!kP6J6_KR zNczAG$@Jn_xCD3~EN86<8u>WR?NDHo^XL~fOHT?y56-y@?-L*XwbDBJk@`9Dz59Ig z%~|9A-x-Z?vvjt!x6^mAw>Gu=M{bMMCjODz&NFI=k%7(Vq*Yl)2qmJk+<-tfW)%0C z3_)qqaE%DIpQ8dlF3J!U?xCL~c?$jLxFZ8-7@2A7F^zV3`R_ z__YKcRgDcDfYRydsWVm~o*j%iW9js*o!(51pvEfVWYVGjPAeTpmfOSOiczRvDo=*eG*hKk8pjTU=Qij=HbOmg^!87?Sk z76bKX1I??MVIB$^#oy{ajceQbJ&xNkLjXsDDe_H_DVAYP57`b2t6?=wc#x&t9SD8w zQQQr_=K(zN4mh~D2gLBppuJP5AiF1na`whNGSR*jB6#9ZH>6qj;qvNb3sUt04Ye_x zE>BPZrZx;fZkZ1%2u%5TcuW!Ken$-Bz8r^j)%Q zZjj(ApN;t#l_P^iZcagch{Hk?eW4(yZU8!zCABf*0dQ#VEQ(-~XbF${i`uPRaka#f z%i&-J-KU4oHDZ#S%*9}Eap6lz3`MeDMT>xJpgMa#s&1XHapMv64T6R%SiMuM!LvRT zDO?>=v3nfN#Vm_wYu0A8e95Ne8MLW5n5){m+r}xW_xf!rv}1@R|{+e2GlNH;N*$1r|N;U$z?7=!e*imj_Gzz?6M zg{*l^3U<-O>D%BZrfs-E@47KWZ?#1E)8g;P_O2@p+q@Z znOE60cl@Bf6S?!X?d-CU6YtZ>0SE3$rZ0vT`62AW{BNO9tgID?12Bq(Z?|(-l{vkt~IywKZtOR8>he^g#x>1=aD#huaiz-Yk zOe_nE*RXdkH!dm;kBBhWh)!@f_fK{NBbo6ouApaWnN}b-SZ1|?5(tj`N3;aiK`#+|>T))S@zI820WyWfa3C(v& zT_*t(*QV(VE2r}uc`5BiPF^#|5qv-ly~usE z>3vxPU`qJ2;L}-+gwBkIPv9%H{p#(PoYQUL#G199xqgs%76rHC=JiP0u?vn0DDoT$ zIY2jm#gen1nvFdvEsiz}frAKM17G+H)>-Hb5sF4z44(pmGiGha&8i?74S9==$U=tH zn<8ZetCJu#M!u}($Q<@ChdN`2tafUIEI9XW6+Em|>FF-@x=!Y6tx$-^Cpo(5gbb2&oAEEU zx@e42QQ1-W=~0#oavu3z_Or4E3+a1Xo=&b*J$*bTZ4-rB;pCtCj;AJ^8umhNVcvq6 z1&U5->msVsOC}zVM@#lwfIZLah$riCv$9p0-zwIt!Xfu2VDOlacYqclwRW@IQr&=> zReG|7i0~!+7VuVmJ>W&;K_x|(NlM?Lz3qfIFlPhMu^>tgoLr&c)OY2Xwg)v@G$(p- z9kY^e7C~b@*Jl-uZ;E#jfC)opig_ZHeRbZggB)IJtd@EmLjqcd!h!J0W&jBD+99<& zY^XwG*sCdT+=?bjI&9 zWr#x7Eq_7Cs!5soTt-E+;tvad-k3f;nBw)f^H(fude@S&Qs%rY;>@{L0HqdR8|Iwt zdJ!sav98%jw#m6`-xk3+xVyqKww~RB4Dp!gr+u@i@bx;olCv?P*N7jSZ$6LD3Tl6a z9G=<3=`TMIpSE8)u$+TgcFUC|*@=pwvqiYzVG6eQUdbL|dj^1-qtZdYcENVzM-QjZ z?71IreBtll|8@Lv#N}qX@SAah^&RJV|C1A8WND{wY2xvH8ln?=?ADmz09Rfxy)Db3 zwrJFD&~bUTPSDjHVytkW5xoj@gUCkG5CeZ72n-ObCDs#F8rR({7#0RlPQD6;^-(pJ z8%cF9uyCU!HywWv;zS=_gXqxmvDK>$ymr785G#qTY3 zqQka!xn%8~<;<0R`163Li(%Jn=-543q<}#02UxhL=kSeu`MF&w_B8s*1x(z3@TSWs zlEsA#<1RAOD_?(&O266SxIHO&l7KqgnJs7z@{W_q+%>J9BzSu=k(v7x{BVORk-vbfvlcpu!-o%|R zt*nH@8}CWJJ}Q^;5oZ6K^s}Z$s6kT^cw$!7R{o%bowE77#u5(q_!lht4`TX|mOJyJiPMx;;4Q-rf8@Rf59GtV8NP8#u0BV8` zvOKr8j+TI}1H>%1rLIQnH-yXxYmXD;T$e`}(38hC0TVZ5{Ty8#glJFldaiXztcBeR zo*#j}E=oU7o%IA{FcHu@U)$F`bcS|@1JK;YZ&o-=Ep#qCBRbdR;OoExYdO%aSx>M_ z>K(|Ys~yGWlaDbgvnZxy$+aLXApn-RhIVMIp@%Jsof}mh4wFkpT$tg=`DynbP|gG& zx|AKb?QI|?Lf&%BD7<+JC<@xm5diY@wdlTJSaZJsC>EpTc49T{G|Wjrn%*yoD2D0R zwYAgfrY2w`6Q816ZYMao z&T~#_lKx}az8NfL%Y6>&D!jgjo~v~yWn^@ouhmO)A8SO6=7--rgG;EA~ zT$tILlY@~HSuf^RfT61Tj+HKxCT29DPI99%v>)C_JmXX6E#T+-rB-qIS^{MUa*vZv}v!`bszE9FakecD8_(t}gtEA%)i$kBm z%51op^x4ndp!;{D`#DPHTc^;y)dN~5{NLjKX$0v=B2@}W=+Mf(P4nvzSQp^3+iyN?b*%hqUkat38 zQmUnjnKMOFC)dkq{rgX{SK-|9)n=@r;0eg3?MSsaA8xtNgzo4!>D_oDtUvy$FKF&w zMwT|BL~bzjr7P2Cx%p9gsOIVJJDeWyalXSYMqaNJ*h5^OC)){kVKIvHcs~mPMkGR^ z9nYv7?X7n_&&~uAGNVJ@@9O{k!4Spi$2a+l~%G2hzpen$1^^uR(I)kHh z*9tdmQ%QiyUY=eE#Ah#c&Xw+vMOb3xvSXTx=`@zWB)>_ypqkR(#So#y!JKUOy%L$f z;Cgp-aFOn`mDrZVqSRZ1gYfHNk04Rac6pL->*$UQigOw`tf$=+2Pb!yV$e2@ zVtW3Ei@|9fnTG?7`x8pj-%B?0r?phP@`ifZ9)G6j?QJNEM7nQ4Yq)nA=y~AG+v+uv z6GH)jGMd$3D|%ww8zoD*A}L18k6vMEDK+m1f7vX<ax}#eqYmYILC7x7-8q?vxJ;-6`i9|RASdR5xuNyl7C?#T8u-u6c>@A~}6a!Dw! zKBqmS5){m(cn_NHBP)8YYl}0)aE?NVZ-Fhk@X$eX@{2n(m2u@`Ror<0`<%P?r^ z0G2pb7u)f~*C%CY|J%68vq$l9=VJfwll4`-Npvp}RNuAAFLWuYkP!Gu=i)SBc%KQ3 z;^W*gv3`MyS3xO_iHDJp%i_Fh9iOrS1<&r&(%E}W!TrvaP4e_jTS;y z0uhPFv)@A7%8!%AL;PXE5IS1eZw?-<{xBC>k@Q5Mu*VH$t1ce( z+qe1l0K~cqxLf=JjiDm|emXS#v&rQoQ6pxf=Y#t(5mx$S**su(Z;yG9_o&z^s`Qxp zY71Qz5dX`77GNSUl-!%uD)hS`ttb->@jdEeH>4^tNK||eqvV>h+({TNrQqo^y{OgKCvoOy_$ZmotP!u&E>}jV9-V-v)L{!_ob`Qtv z9&6rAn`oKS;N7@3M2tsKc)QhDc@uF%x3qIHb+R!vbTc*iC$asX1~O`;Eh@pJ=@q3jgxqEy?usZiRXk# z^nr5mwQ&iwaW&&)M<*CSGEBvG-uvxf!$fx&#`IoOODu%tFvNOoJt4q0PGf;($sH+- zYO63M!N`RblwYO3_Fe_R2LBOs3*){{n7FR!DGJmvi%iHoA%luZK{o@OgVCoELs_V) zfyeObH=t!nb`gRJ=7hxzoXQanu9gajD;dB^Cde2ab8RuE5(IB&Qmdh904YaQu)V(> z2?!d-nL0-#?Xf`U)Q+Rk5@9*O3rA$T#)FZ5RZQxNjL?)89s;C=Lf)1}xh zWQgc6Lw9~y1Il+sBt_jjy0oUDjb5DZDd`{sgUhfWrHB(Ma{r;1wS@u5nK;%H6_ zh!_~D`WBHw?ZAwthDxv6mG}Hi2k4n^py$3`aq);cj?6A$4VR;?jTDPFC5+zeHjvgW zMosw~1u{tQ?Up)f`zGj>(NS69q#NO2(Ol(_nIk*BOXB4x{NUi$p>B}?mcT4A%!5%;?b15hG9r{6+)1avg6@{<<#JzDHmR-xQVEuOP z!0+ZYSGK^AQt(fxN{F0p311y3vGcb89EeP;C=eZ5iob>gqlzy@;gSD!jD%tTrrO?( zi%zfk57xe-@Lq?9w+lAMn{_URcdFt;WyOgKx*5QddN;6#RX@Jm9jpw$RF8_nuPh!I z=<>{>wuTHd)gSF@Wov<5Ta=JX5^Gj&^pqyMN`SDesT(uiY(Ay?F8&^$4zbRP4KJkk zXqR17N08qhbJl_F;nAr8T3dxTY?%3S|8Q5+9ayvi(lU$CF3ISpjIbWls6sj%Gxrk3#WS!+g>EbOb z95qqtn7H*Pd`WIKsNP%unzLY?;4e*UeLk5icttLougo?I=h&Z3UwKab+KcM6+R>(` z7KbIp)qxr*#RcJ-r9W@dp)cbR15LDhtpX*o&(qC5_hsvJ$H~!m67t|nPbq(DPgwZo z<%dq}td?}r-HeYvm!k=~Tzx4!)rv&%{nqV;aGgj_79I-Pdc`*Vm!a^;PYt zAi*MbvX9L8tL>&gW%0&I1?cr=vT0ID_x^h~|1T%S4U)Fg=u z2N7XOzl6ZRl>MD|VQB-_t3wIHwubvIb`B;1t`((kPHL#}tIa5+pt$`lnOIX}1?yVN zuK68uw}F#oL?ph}vv6!)X=1%Cl(oo~43d}4WHNDuJvM%tc1s2>ed$l#k^Vl#(B zHOuPC{ZX0*0Afnz1mfgL$mdcx?p8 zCCN1_e@wr+&rHCT5TyLJ=*?|&E`mT_F57}V;WvYH+LDBf%-dZ(_c*BoMJ34dLYG5dJ_iBxP^wRaGf8N12 z$&33DZ<~Z0q$_ijo1z@G_Rwm&05g@8KtmUlW>fL>ZQk?4mlty07O^N83qk;5MtfS- z)>%WKjTXsnR-E7(uWy#*+%*41-$oN=LMpri8>R^Y5%CG53Acx8GbpM;fIjGo|Ki5_ ziNane_8K{B)4p*eR@thf|9afHP9}@j<*Vm}#mmAI`QN}oE1S9|`tLj->-*{;@_%p? zoK4MbP3>I%X=Wy={=d!4Q8BGyt7(#v5JL}|103|? z^078!g2oGR>5U11G&ZQouBKF6y5d~Txy5EE!UnV*6g3&d>j4xK=&8Jw1}r&|;4Rry zmmxNpIkxXwM3B%Vt{O3Lt1}J(Az>IJ_4tK6ybQ5};x(n7jRt(zmX5_Myjf_jx(s@t z0X)Wa`{0K3>$#0dyN4v<8qDX%8zB`_3OKNh5C6Uh8o2ySd(w<=AbosytQHE9$#^IX{Cm)<#AoNbXe4KE_!8M{7Mz zirT@XC7BQ&*xguSbe3pM(7wMRyMayp>{NqWr=3ux6VliYSRU+E((M~p{dO{_GPj#3uSaDIjz5_6i(?%Q1nVr+7E)t*<*QV3Cz!^m zj@!ETm;U=F)6%i$nVu_2X!Ux*Q+pbIm&K$gzr>R3d_wCO|1M(jI%ybEWaBW%ELJH^ zMVFm9!X?VvQ*DNtWXQ4e>WqpIq{G4p?_+7|?5amjKia4g)Mh zDPZ2D^Pi1Q0_l|z8m_Z-715GE?T&d*S`)lb=RO)Q2c&NM8!7qEs|fuM!{_}MkHsg$`&bSHt1chE(#}xDR0HA`D+aX#ot0XY}hd@ zzjNjp4gpEX-|l(uL%wGMqzTFTX5N09{k)QDKp7GI^{@8hWczt*;n@_ym0hF3vw% zxjS)VDvj(B?W*LzPIa%*q(eIl0Y!((F}Fx^cJkf=PY~SNH1~X4PmscN3cV^nvCzp0 zp)~1L?U{h|tbTLXN{k0!+5iMBd=}%vT{MxhBYs?+xAdyaeWLQ4$gqa40-?)@h^0oL ztHy*8OgHFlpOSy(i>7H*_XqX(f4#%z*h#3zITs1Z28iY?igO?7wuKs$tyK|9hI#*|DwU_O(No7!AMPI2M?TJZck=It6Nr-dk5)FC6!C*aD^a~;&%Uv}ySn|$XS~<#FS9xyQNss* zmuz2$zd(n{hxS6`D(p?MN3jlWTaP$?QvDN4hFZR?P{BzVnq?Js)DCRp1E@P*Ws z3*WtbLe$bk6Bu<+zvJzBm+;L7*S7L84P|KqNf?3KkIkNy{BY+UmW5x+QH{@weWn@3 zS=6F)%-d<2XT_2d>6okwwZ(c%ff*JNX1&2zVOs{a^Ej)G@=kPX(MU1dxz2!O6zP(! z`B#8|+GYd^`L%4(gY@2Y4xN5|AmyEOwz+}xeSewtnGx`WS*53P{E>@Iy=n=a+(01* zf9%=sWhlv2S8KN!Jf#L=p6qlVmwSt;>&LPRRjoNU_XX7+* z_QhrBO?Sc6F2J9uEP6UrHYwLJOC9MAdudr*q1(UVcN_|!hofa%QqftvA2HR*<+@;J z1XHpx(0vrd82jekVm?N3VFkGjC;+zTQHkGZ0~vRT(L{E7^2p=R<>JxkU$|Sp&k|Nc z8p{H^GeJIZi1K?!?%VARX5KIFk9O7oO|$bcrMj9vay7FKW`r(59r#H|nO#9TnqAPIbucwlSnJ`Bf| zrqr&d`3prpUv5kyE5uGQS%pRgcD9J&Zdg%0ww1P3OurHS*NX_+wbhTT@BX3+G7yl= zf0BJUIGMUQ8Cu$XBX+)J@4xgKi!{DD(!F@g$1gPdDnp-zV`GS!WCsZD&ynPBpoa=E1kl@Z2mnI&FQb!$k; z_}+4X_SZSE%`b?i~_2dJ=O~QnQkTAgu;ldqS~5U+Z8>gd-O ziO7^RHA!-*7PmvVYvX(y&kO%DWyuvl9Y1V;-O>`INEH5($tp7qEME!Lo+92kq1lz>#_}ZUi(x5dE8-@vogy-i-8qnqVTnJu9mGC|( zxHS+9NyJ$|>D}~Ryw~b^FnMt4uZQLYr0K14=Ffl%)FTI#+OWH$=r_9|hD@ zJ|p+Bx2NC1HFk{hd~oz7s0$r~3mv7JJ*ix&y3Gzs z4sy;Dtq#C%wWMG2x45FpF3PQ71NV$Qe_;jkgIr$z-zjO$<_xr$F7PJuIPRofvdrm{ zj0^>p-$QrZn_c{Pg-%}6*baPujob#il%fem^VG665tDziyl|sY~)zwgtG226SpdQmzUyWD}LfsGIy>Mf0kG=Ky4RzQ3}6wny+4g z?vTUOjKgYWI=C^=NZ$o%h9M$SQ?iv(mQpjVfZgKAC0Po94{tnaF()yTUa;y&h__ex z`N}^;k!5SCVi)VHV@BpZ8adW#gwPNApX2<3GO-UU9EhPVaV%{DxJjbX?mD!&TsYKc zlZ!{%%KLO(HG!VZWPeAG^SQNvuDmUf(oQMMpOKLjKL{v@t0zA_C|UUT`1Gx01(>df z=na3Lq)xp%30OPv#d?OvfD%V1u-paR-4JY4Zvamv+}8m#lt)F+$?8P>`evU^2J9+u(R1?2c`@%hdR_+kbH`2^41=scjV zR$z3Ue&69F97&*Zy& z59}#J5cZ@i%DYLLZ(UVoV>e~C<$JDoR?$xOPQ;j&Do^@a=$E`ap}bb{_BOMTC6dgX zp?PP?BK)>449c(SsT>q1+YR6%aKhSi0bb2e%RoPJt5zkK?TMca2>O7-s`z`qCvJIU zY1%q8&tKWKff1a%+(c;)<4<*2x6+#GHxZ0sV$r!#Hwz`{_!=lNfoohZDPEFSIvHYw zSzal#o!7AAe&~4man$^(!V0gb9h(XF#-pf(9lPz_cS%`|z&-i*9s2FBYtgb?nwzBT zave7HHHlj5Llkk`mjttR$sTh3)O1JemXC9FIj5)m8)4G=4XZq{aozdfCc9ae`G4CG z{;xBVB>$gb(B9tW|MDaxX6EF2_P5nlsNDabfA)WWCqp|!8&5A&eG^k7*MB^UI`}@E z*Enjs&Z-2{Je6ybuoW9kwUrODG72l~StKbYb*~0XcBWBG?!-t`6Cx@+-n+Uszy*QG zey&-1W%g6t)M(H>;s)f@q^y{wP^vGHpH2C4;KU$6o?i$|>1Acrrd;v2`#*7oiRBMaFjT3Jn1Vy|i;1j&G1k>bCH_7Fef3Eim%e$V4eP}M zpy!yA`{-6SKYJyj6?yeVBt3gHQ76)=>Jv8ME`09U^D+bV@TXh+gSF+|@j{_jL+t{)%NOfz6oh%7xS3FYzV&L8 zc5t3dPOaV39&A)6HQG7E=dae#kD(DB%{PUMPr&lhg;`4s8}`hvhoKc} z;{#Wb`*n@=dG#b0lZXF^C%m@<4Et1r-xIvG1J#WerocOhW8oN8BmoI>4gJm^aMg$E zF|_qY2ZvmK4YY|c^rM@BZ`E94$Vu4Bs35o|6}+D3bfV}5mYJk@bQ3eJ+1o@-ed>A43!jbAk94RKIKLJHW(Bv1b z2hal$xEbQTF{F`TzcZTAjUr+SJDdn}d3-7xk|=`%SFfSS2zxW$HKBi)lii!(;|(-1 zCIP9eK_+pUM=%>tAJ5+6s5?vhfAEj_T1f@`HtLIbwf>B6q|AllCVD|F6=;tiBL@g~EqdImeQ@5g4 zx|&jyjtz|bx9tw8{>5~}=uNjjkTyXI_*gQXUIkWs0TPR(FKz&=Ffw*HvMHl8i@TNaMJG-4e2lt%t?fs_WB|>C^5{jmR2F|rq5H;X_EIOu* z2h#DTRv%9!B#=udJGrdv`d|ufWh)H%jO0^ECbC4lg6q>m!l;p%i(z3Gk#CjOdNz*U z?!)F(OFMw})V-wIx6G5&_X9D;;fHSBg&t|nTrB}gg$S#C{S{czI#El(km&cgxq_HP zmuKO+)1!F;EB3T4>X*R(SQ#}Dd!ZpuXNs_Uc>_DV0QBL^fBK0r461{xYixt`M}`)5 zh!Si5gxRw=WzU(*n0nN!ZqgiO&vIm6csl_BOSEON7#WSilB*CDfc5|)I^>4T#!Ton zvT%bJ;OMrX!uHFhbWPVs_d-iA9-Nv+8wq-F_r9@JdUmt!&Bp1)J=5=s>tuUC_>6?emm}e~nvkWU=5^4l?R$`d{eWu-!g@m^svP&WMAt$;z ztbH5Bq#ZRgymNec~!2Dkv(lkG_E#ki>*Js2lj?Y6yK# zRC1^G*$hc)tK6T?@_kl$XuA?ERM3L_Ue0qJHed)f%s|u^!)u7oKCd8V^ca-#%xDN0@8&LS@B3cSVVeZG-`H zhP~MBm^!ZpBh=E&2fS)@7{|cmHpJu%wek9mWJV@^a@+&x02&e&{;FIe!iTV27#~6! z;!(VSv*<2g;{f`f8#Pt&dFRprmnlE1e;W7+$^fj(9Y@07+3^>|;~02ZJwiq;mg-wM z#e4k=L{XtGyx^@j<$Ij7y&Q3>-1MI}gY@mHh|-gzci*krAS9H=yR}W#kteI;YGhy@ z23+I^Rrh7W{irm35dh6}#mCj~%P2CUzkMUzZ@uI94=IJCp@7^Tn9WkCOn%EJLY-fX zoLm?}RynWh8#x3*oI|{;%d9R8B`rh;7ES&VnlCNAqIO#;6if?aCjFWfh1aEpwc%1h zK4RCmE#BmQC1cT07>9|HnuU74<&HtER*w3M#MLIb{eVIec}_v!4G;N^jRngCquJ^9 z221Xx|D^DM!B-kmfuP$0TB6NjA{LcAJS$EA_4$hCx)KOfpLzEKu7aq0Wz6YUH5q{< zGQIjuk^v1YYL7Yui@6{ZF$feq8x4ds4a0B447{KjBtd13Lyy41;&%9kb3xSgiv_oy zr^OW#J@SLeLUavITmjlIGA19-EYt13!MKpd)I?JU(jeB$CH%otH={!XiF@GZfhL<%N7fgP0zY~&yefywHHwu1G} zg|3(smW`B|DOFl(P^XG{Xr+28`=LMg9be0k`tqOh*yvp=RZ7Qm@*>ixn}wZW~bJ$jGaV=Mx!AroQkRtzeFR?(xHSXZ+y@gIwU9a zK{|tEChQ!d`mvgL3eQ-W~ z>FsBXZE;m5+9-<&^T+L0YgQex*k2{zB;Bm&&_Hi+R?lV*~V zYRZt1;jhk-N{6-DUi`f*E=FSh)~tM)H}t!N$=~DBb(sr!;wijRV+JH&K`7$&9n=y$ z(kUx~RP5HrD>lXEA&B91#`aB!dNwYJ2RR0%mRyg_FJq=z6GG2sq}b4a^EAJseXFNlRU1D%LdaOE;E<0;8B)^|r2x$b=U0-lbQA$mj1J(1Bg5lgeAUUswR1yac-n zq5fJmi|X`^Va5)9g%TA8X>z{iLU;7=OQ8za$vPqX==C2HN0S@nGt1UFV~JQqNNXJN zN`zQP{Jz+-L>#qd6FCwq)e_GqmrM0BhjWALsf6`lB8wCEM8XT+B{_R0Vt$z+>^Fa9 z&_p?WkS(FJl)T!lYBn)U$WrdcZX8iE523q^ul0pR=%YyjS_S7-^Hr57$ zeq60CKInNYb4agR8<)~apBpl);&EWoYqmnc_;qyb81PSmmu_aPb|z_N*K9GJsHJ$A zYkSKwlQ3m*7c3G;k@eMhqK3klUJE;@6gBP2ROetH$1j*4I(iuxTJbHk;>xTKgPo!tC0`O%WADq7sK+)cB^agavJ9^nTC58V1GO7qA|JPY#kESrSsHg;a`Dbt^ij^X3nK?Nm;{mYwFki zNT)i6PVf_UpY1I}yez1N2IknK=4x-B=2Mr^rF!KE+YW*I*?iBpQF$|te{iO|Kaa$= zAR6#1JLh^UbB?V6_a!-FjLUtb(&?ed%Fhr$dzxl{H~cs`ag@eIqv7Q16Ch_90I4;6 zk6N9yjt342Tn{8=Qyhd=lBl_V_Y*~WTl|uNXB(o?)=GW{l$3(j$ZRpE7|DV~60tk1 z84O$8MrHTaCVmKomU|(a?_oP^EC?g!xlGD`iD|9K@(EY>3DSj9b4C=Kc+S z_2pu3XK#^#J!&({y;Sx=B}@+#$EZH+v-2+E~N)?j(_~#b1{&&OAttjwaoM1TKz;QnW zZ*VETskg)u-%^mOhZh+5lL>8SkIuer;iDKsed6;*{Ba6*IV{S$#86f1L6PIZ^0D49 znt)^ZB6+(8aAzrgx0b(biQi!AlWBO)5wW#g*|1vo$bf%%hFp9WZU#$)lRx!XDD2VG zi?Ez)&<1aqqLxQ}dO|IIoIVHohUDaz%J5rlwEP17m(FfhhsIMbVuC>9p{F^My}KGf ztk+XbC;=DSsjD@hdeW0(KU>us*F8pr5T_mYm8a8I7|^R$oj)0nb~=%yVpEPq>2k~# z)l$G+?MCh!>DG0mX_-~t%If_`HRVxTmZd=T~ z;r)lNt*EU7ndw?6#2wM6=Q}oQc)oJ39DjlGt+Ni9^F9BmYh}T+>Yz~e`~|d~T*}A- zSbK=4rbJaCeacY2d2S#(!+q)dOlf>u9*oUA&sC7i9tEQtTfB8w-!+rleB5%cuF1N3 zwZN!XgA5Iq-B77_0OkUFnr1YOMPOG9Rq}-!@{rwigEUps4UoWZ8%MK*RrY;D0}N|^ zwn$k5rZIRLI4{9`&MP_nj!&eWmHGSr#9~l z*NXO&ODk3W<`6rq2XPl4`7M`*q&b!hHjH_mVPX}+i@9<}R?0&Zot98qAl^$yED2Q? zKB0hVkt+Q;q$ASe@zq9Dc2#pe!9SbPXQcgUG@aD@$1VnzO1fj@#0BNB3mqGr)$;#!atz`YkVoF{?b4x+C+@*CApY^`_j!3i%Rhyon*nkfPw`@!Y4p1- z*N%-+L5h28pRLnVK@862F~(pHqKj5s-^*OxGTGRKE@PMAykj|aL0TD~XD=4P``wT@T-S+AK)J zNNXRdCYtmIcb>J7KO9$cL)$jn$^?c_e^*bPV&C)+ywU0N11%Q8NNkHY5e?6~_hh(` z#wxEsM8-b(kwmHtkheHUPis6}=WCk27}caFbU4#d(Dtd3tF8p3Z?cckB?@t=PIu07e53sflGNJ}4p=10pEK_5K@Q=M*GLv~Ah4ZQDF$+tw-D zwr$(CZQHhO+pg-<-CyrU-1nQGnGrj7&b7vv^p9`H1H1=^YL4Yn+m;hL(!(5}tz+d! z_%#=@S^lCJuS7|0<4j$+@X!y7xL;0L50*2cHlc0t?=f-qTqi^Tb#Rqw@CNfO3Y0$= zVH?-|_MIVfZ3?aF?#uW*zL-R;xM-AUQeDz3y@3@yE@$7K_*W>}8D;nj$>dUGEz=}- zCxk}oymp(+lmr(VIRNHahKMfnuc&Z0(JTQ9%fRmC=0YZl_6K!5@I_`ubUH5PB#R%Y-^12r+Ukl?F? zi!7@W*G?Bp!qFqvl}7|@9ydDL&<1-`$@zT+j5aLmSaERc6Yd%xoXzlMR|I`jgx ze&|w#5UuK65Fi{9{fi1(G<3r6-Tx^IN^Kd+xBu71{=otOK=R*XS3@&nLrXi`e+!TQ z^37>YUB}H4MBmfWw!TG3y~MC2X79PAY|e!2GH2`PDw1q{_B3KUlv*G_G;j?A=AWDG z?ask%z`QFbvh0g~r2fwB?(fe{o;OUcTGvp~9UBd_Q^?Pcqf6V;t82Mb^i=xw_s+{o z*HwC3y4`LW^4Yb|%1GatoiVMyl2dfh3e66*B90?Hp1us9CN4S;-AIYyTQ?Ua#?fh2 znI4lHgR&~EQ{3Lf%3Q{`F4a}kIxdlqiYKlwLYi#vCIzmx!?d=U9g-rX8Jb_+hu0-Z z)KYiLs9wAcD);bS$`Msn?u1yKhuuCN{hB`^J#M=XXh%6H2GdkKH`qERP%0*_2MCo% zRs(zF5kw!~9=gXA+~VVVZC2WBF35v2ZYv(sD-9^9yHLrpW}DNZR3$FOR;ZS20pk{} zw&@nwTD_cg7QNcxq6|8yRTY)#znD+mwn0Rk*CDMo_QYyBwyt-bdRl<{vY1m!9)|$lT<-0A&1x#f3vM zD8Gtb;sumS;|peW2-RboFd(y7fRzsNHejNbB1*$$DODi&XXekI?*f0Ln7$?|V)Kjl zX)*}RW%|wcpxJxPjAmez6%%+-5f81lF(H>&j9;+{avmE%Shgo!3{i)v&?-|b3*(`) zNt9e7Z-50ggMoW7ujqxwggB12x4&QI+lZEM0b0Pfwps0v{rEG3&d`EuTNz2 z67nvlXsv!$K*)%2X|9;dP)A3Fd9BHHZAGfeD6!c)%}{GuHVt$6IMKz4S-D>OqhyJO zqrG5ePOq5{>?keu3rKu>E-@n1>A)%gO zMyXs`@4kA;FTigxIScloyYwJ9!J+d9Ty$ zBGgTHSl^4!P_;|J8!(Vm^rSsR1U@Jn@5AqmHsp`RJzz7t{By%73fxPUYUt@-3s!^3 z9yyYxQ(QlD$Fu^p%@Egg#d`5d$-nLL3LC^9xP2pvaoJ4{#>x=PDW!(=Z{KrJ66r%_ z!q}IC;>?u0*TTv}cLb0s1s0?2FOSFbTMWYq z&=}AtR2<>p?+P$xH*F4raO=E6mv6>y9^CaVRk7zzd$^Fwd5GU|6>@S=hRgiRl$}+5 z77b^6rPO?JEW?9vN}<))P5!6=B^=^rVFQOAOB}A&KF87$6xDfEHR+$9~iN05Ec}2sB5?wB_}H z<%DKLLZ4-$Bcl>SoV8XlKv0NwTjzfHNlv5J0t(MtY-!KM1HP#;JX#J#{Aq5C=WRMT zV+M@$Z!FMv_EX01t8SM~b)o3r5ln^+yV$QMdYnXd*Jcw5_1EM;_NcY$;;ulsvHtn1sU@w$fhWfDdh!ql$v5W zOv!~o+^k47p9X1qiwboTvTfJSY@uNpL`8t>=J?=XMmX%^d0m3V0pjTzR}9e@O!V&%ZxvrPjT%;wRnCI9m_4=7Jb zZkcLRqnZ6q0V}jY8Pqw(#x;@WALonkpF@>DU?oWsL;Z2$N48p<59&SilS}x4%Ah#R zZv=6B*iZ_d%;kAIi@|h5_2{4vk$bgm9uaJL*CI^HH_6U?SLz*&DtOYZqjZ__s{K|P z@LA{)DR(}jW)x^-b*>*p5%KGDWL0_;Oua^0Js^=O7xetLVB}&k42=Imqj}0=4uzqG zj)VToL{^Xsw7o%pB2WsWetRCo)(3@H4rXfc*O*G9}uwD(9g(!z50>o5?2dM&& zMIh2XbM+(-CP<7`BMiSrJJ{Bm#{Ci|Y$-akypBhSixeXdvY+5(PI0nG;SHvYFpfUG z0KsCCI3~4m8C%}l(NYt=0Mdfco1j!uIUL?C0cclAlR7ZWYXe1z2*gTr*R~Kxbw8ig zAYDCqC+#Kb(PNjAcP~5{RR&-22-$8CyIm}Uf_uVV6Idp0mxa8#c5>*uB^<_9B0?u- zaB~pwM;On5l6b6p<{242QTSze&ky6^EzweYD{&OHVq!O*A806&aku%4ek=ms@MjbI zz|!SC`1TOH9Or{+w%b<*5oCJb4Var!j0gyKz zDof;I<9^NExZCMlFX!d00|}B&g>vphhdeZ2%#eP1J}PFjN>%c#exvX@$-inNV=h%F~KZ%T0h9{>Stlr!nrg+P3nu z_pxOW&(#p7=a6nsM`EnPg48{UU5{ zQL>xawd>p`IM!9-wJPohlvMDJoc=tAkVP0)IIoEN(2ZEs34<`~lyjrCNM6sxC(fV8 zp0h9^5>a-5Rn%PWr!~}o^$d^Z-=6rrSFG^^0RV54%MOs#C}3@wdKe%@M%|oqsS`>^ zE?5-|SLGVaCoYu=$1F>%Ob`=Z>R1$<}Z)Xt2p%mQ*-DT)(DAY2S zFoFBid+{8;N7+|<&JI+iO4kCTwhPCRz#PArLnnT2`w?(yDLRoFA%U~<^}Xf$JDj*b zHO@g}45#qtR>pAG8&LV~C3&e|Bn-5$a}uALz;@7hdVSSmSFVA}DgMuQl-TxH{-Q#T z2ReM;P9jXwl0x|G;tFW}DrWRQlC=p%?%mr!8c~S!v&3f=gj$4BnYQkZoudMoE;nHt zMB4Dg*DWB z9J0I8Hp@AH+JP!o#HycYondAda3%ASg112!mPQ`!CufvAXI&jfkqO}CsX@fCe3v@} z1M%cBL@JDKQGU&IG2aImfl zL|nno%?S;;^%G~|$haWSPdB63MIO%&C*Z3wyp4fy+bNxwAdefNwfQ^pTG)3uzp$bA zc$6)c7w`=T!6M`_c->L^JF|Bf=WNW9J=&;QJXx^2p&of5t7Iy)p<`?;1SORvGr(FPunblQil9xJOkkw~1@s}rWh1+JYoUmAZN`{3bS@wU-F~(T# zli@PIw4T=^R$Ccz^uTHsriG90A#m1pAUy&}bqh~FOZr`5>1r^p^*7$}6~^jv5W&0? zMgTDDalJq$#Ph@!(c1hh0WHG`)#`G_?p&XE+WSO zeQ>4I3q^D$WLsV9NJ{AQ5uA-7ikDdCDKOKYH=JO^9&7YZPU`aijxT_*`QxsI^cuWf z>3}uty;4LYWWu|HZd~l6a^Vk{(6kr?r8T=Y1t*)D)A!31UUEs!3C}NU#Ey+8WIWf` z-q|`Xi=AA@IBwKtY0T{Ih{_1?D!bCp>sz@R$M@wmmNS{UX5I5rTG0a4vBAr+Uz@Mq z2Am>rId%JIH>6zWvn;J0i_|s7W}ULSftzhvN+tBE3iit{@)*ut+~D1HD_6o3e{t#~ zV@oo!cHZ3%hheD7B@0zqY{$%xAR3}I~q)E8h!O-eeEjj&~zYg;B)nV|XpuGxGc8c!}qNaUIeISFJq%It%R|+MuxV zg07}v)=H5i;W9z{PH zp{HZ^SSd&1(*2o15xB*+Z&>K#k@}9}XFU$E9 zIsHE0*PG$Q0+LH;Lz=c)@-Affd(*)h_`I|T@Gy3=3xO5Qs*4u96pgiGH}8DRU;3l& zTTUmMk}Wd0z{_cV=Kp#plO%_2W9KC*3;d5t;>&S&b7N2Q@Su;x=LvoM14xz>JHE~U z5DUKp#sZ7N#|qE80fZ2oLz<3j#EXXw1U`P-QvOfNUBo!$n*)3a62t~d0hgrz^^0N= z1Kf{64xWz3eAYSnPLyGN=lrw|(;JyW<~x2ND&Cc1U$@_v3hVA$f+9YXOHNL^r)l)&=Z^a}Zq;O2J?YGxhvzIM;S>yJdHOH(1~lf*$&3A1)`ACs;QY<5`>W_p#Y#IfcXkta)4 z2v6aq#~UEr`j3UwF^2M)_fVB$dC<_=*!hY&P25)~Oh}~+kfb~GzlwR!D5lp|6<7KL zmbqoq2ILw|*@wr~Dc{=>dkv+OC!Xc}a^439Kqec+MX*a_$x~JDJ1^l+(Q^+f6JybP zV-g*Q&9(@5?3U%DMdDN3oZ15*v{s#U21?Ya)Mi=nQ3Q-=8JOtp()d&+9;p0L3fLQi%Gn9iDNy2_Y}QWWw2vq zPm(@C>qOcW`>?X|SeVb6K}Wo-HOo&CIoz68r{$ZZM&awENbU`)-LBtzLP$<=&+4Mh z&3tVK?c3iLO7lA3;mw|`zg?&UOM?7(7y%XUxBQ`&4=UWKCQb*Mt3Iv)5&zWaz#E@j zxag%=y5e~muUBk(5v9n4&U3^7SclCwY1S>r#gnmh*kjWha&wgmh>h1xVZ3VoeI7{P_^S(zvfhuAaj~BT%H9Qmum4Lhz#_aC#}zSL6iKp93=AO!D(MD4DHQ-H8a&&VVeQ-I1^wi&y^JK5My-UfCW&KBL~J zWuy5_-&E=gK#Dk3lhe1w{@gLi29~^9?^?d;vRtSZvTnf!P+SfTk?dx(p2POcuJ5({ zqMia$QQp!ilWA_5#r*by>`5v(Tb6okfs_V*bnzgao z<-Q+bEF8arN&8eV6(Qs~Xkif|JDj8J}|IT6?pRhtE~~-3>1D ze!9UW!Ig4Uc8$OPR~H8lB+BqH?<{G%cf!b%gf&R~&p3a!Vh!nhs8C!|EEf1Kj&|-5 z)3IMOu{Vx*&G$FN{ed}n3vb@eQ27NkB)ZL%-H+#^z-uc`u4>IV|q7X>nghMjISZb64Xg0kj1nl zn1cZCw$4;DETYh=?DVDw{l6Qpv!;#eY9nO zV3)CH@Va}BS$ur`TaTV3#FMY~>yykt$On>^`5`CxYl69lgl+i+DM`}U47>?EN7|3QKu)ny1eVm} z@?0Bo)@|!aDccjd>~d|#=<~wEqmrTm9XGv-6X(SIMP`VWN3 zi%4faJ)G>HA!$Itl42>;1!PT)gFk3*d@!b5F@7)|Ly;AUHfV$<8e*H*Ia%+9h{kkh zmV&l7n|i-d>k>LRg_0jG%yYUD>Cc)WW$M*N(b+;I6xXm7zhX$S4h7x6uX zZ~8Y~M)L?WbfsiVf0tv`4pWMJ@v^#I!TIL1Op~18xbrWD*ZDrK@c@w!mf%KT*Su1# zO%KynuLqq=8*Xo516xDubTDf1gDzl(gZ16l!nn>i#Kv{UM_ViD=pGW-iwk^1T&8MN z(-RAn)Vj@3Y2)quTz>051T_b-V9?eFC`G}k?Fj^aypG(I@MvC^P4`lT<$KzyU8^aR z9is(kSI^P@>XEJ(@c4_lM{6hfrv~Y1d$26B|BO*-3~fPYc%J}6Ho62K`9(*uBrkZ; z@zM1?E0EgpF5Mv{W}l04o(5&cHWhn{=_711u;Ioz=U@8$CEj{>h~mEIpi^TBLw5*+ zoRshh?rO227s%6xsJ72BN_UqtYy=%D(i*j87>+N0r?>>s^oV~gg|AO*LFX``xw!3q z8h(8bm$1#6#GeYbWOF6P!*8fE56DM6x}r0bI9CFAWq4$Z+_+8LlCJK={f-pfxwdzS zLF92Q`^JkQ1w!_5AFs;O1+8)q09j%Yx}|b{*VleaSHGGTgVrIx;E~xA3yVz8IQ#;sXpb^m%Hev)wL6x8@8i28r&egxl)Wql8il zN@2@Ubq4rqwRqho3rA?c;Fo3QkBs&$CDyhgX2s32S*5+!NGy$zsP^9H~I@U zo~eJ*Fm*+&E~+S8O;c$nG1-9D_~n8y{qrAZwSc2&+ARzKKnXJd0RDf!SzFsW|Nl84 zwx#1{+kK}GXiUzzMD-9wyAfw0=FSrOB*!dAi`WB8_qZEIL=Z_1WdLa(d&TANjy8Z- zpa;9T=NEz0kU{<0E6^qKLkj68vsN|e`6uv$#n;7joYwdE!PP-hS9i{~)~u{8eNFAv z4rKan*0-S(<_GhMlM~{cTF_)%3G&0@m(|nYne~xz){DnJnSMDz)ah>9ov>|s18lxI z7E^()oG&L27kMR?q>4zr9>3Kc=U z%z#_+U{@gnP`wkx9#l)-bgbZSPpOIZ*#t6Wq;-&PzwYki>XJ?fqBDndDaQc36QbL| z;oq#mHJnh+KXW6zB@V>hue~}uI&Ju}rm91PXg}R;9;Y*MyK>vQCB5zFsx?7v?O7eI zrA)712M(OA9^E^C^LcXRBC@@Lx)-=Gkk4Hfi0>#QQwaQxGHX45F1&*(gZ zNuhq73CoI@xX5y!8N|SgUIbjatoMrkk-WU=;*)GCw&h?-6dKR!4M|&z*Kv1ux=!Lj zsgjc0e*oYz###&GBGE*?1+ZCo_mS!o)54Lo9c$dlc*&2jn*X}JqC6!Abd+cjL@U}$ zU^vY%*FGVdwF`y!>sTtCqvw)YmoAmSyaalhU&P19EJ1USHo|0Gg-CjR%X(C6uH~#i zKJ3}g8QTd^!j1`1teC|rc8;0O{mF~1Gq%ZNO$P$`w--63>a`FS0un?mK`KrqplCfQ zKA88M67Z4s3VIV_AYE4e9e1^sm1HnzQsbw?v07s{o*m}Z0n+;@L;u*e#)$y`Vl zsMutOo_=Z5ARZ*HQS-G>R(=NWl2OnP)w_`v<El6<^=so;IpoS@C4X9 z?+!`Eod=W>ygXmN;x9V}p43??lYyM#X96ztz^}i_91qpK2DQg_3 z?5{3?G3bz&8|VJMu#WHqZ#KW+gr0!-iQB4SMSMDO{KsQ2<3Tw$vI&9`II(tc%yD># zWAY$xa6Oy9q)tD8YQBuyKe72W9q4gWb4}=Atq6zFXi$&TX%!i2iaWK7o(ZBr5o|33 zW=`NNl5;*TW{UrhIZOo*Rf9s4J9G*pE?G{Z{+bX#SFap z2-Y-A16d($k^Y=mvoEDXrWc-=kC+>W`*=l9fY=!g~i2fEe%n&EU-ppOMWUL!UUfqFBSZFBq}5FbOx=ecs=<`8Q_JMQPiJw$3eMO zS4vIF5&JZMGJOn}rxJv#jO| z2Y&1lz6$Cm0#y8XI`gPQGTDJRf!tq#| zLnGBImGrJQ>rkEqsfIJJbhn87NpK+s)4qsZzg(@Yl!bH%C%(`M{C*DnQ_Nd7+_Lyn z^!lB$;z|?wCPtzUZ@6*as)K-)-eBt5*|c|ejZ5JD_~QckeHTCiZ}v7HgVq_Kiq{qa ziy;69p%V&PGq@1OY|uC=%1r`utpjHydU-Pg|wPY!X}o~+RGW1)`UTKXyI-y%FD>4XN;|>)8o;OA zZ!)?G3B5{Z5)=4=*!`KM+y@!A>Sw`@Ut!#r<`RHhFMGJe(oFkj^YZKZ`107r@8mM&G5L!te}Ko5R-zl8gD zGL-}`H3GBt?T4wv_V430H5uGwN@cR4SW73YISNAMp%WkeH3r=Sr{Plb{a*ehXT=a5 zB2GL$jsg%jlltgmdY~kzSIpEd{7Gwj8xZNz==Xz_Cf-3NfIV9~FBgd8`OVWmpK)il z9@_+@J9d)d!|K)5AkD9jha*!rCsQ^$UY7O`z3h69Gm49Fu}2@Ink8I^xw{N2NzFT6 z&86MpuXf^Xz6R=Xb9o0fNsu4tvfI_74{H2Zv)A(iqO$MoX-c46MVx{~58D)l?$G)Ig$pWJS5vd=d zVei$Y$S{wr8+zi+@$4l4j!(7sQ~?@W)%ts*L2%}^8Kj`C*M&Zms2^xSA6d}1>LM*< z#^??btwaoA&G?4udzuQ`qONqHwkX!r*2aky*iL)JLU{fR;l1*Swn`a8^JKR9IBUTY zbqXLOp;%Ebxs-9x6-$D0H1|dM`6XyZN-i(Yj^I*X!pxw@9=%}$+%mZcS%a>ZP+JX) zy-Wq%eqcuFdj7{A^p0FoRoP{NMGf6%o97=gu3GC{Yu3GhPmB&6DI&k=a{+p7V{CDC z`L(FI(DsDdVF)Yvz zO$bH5B=McSfAeZ5yrVhUVlFqC!l8N-OjeD!;M3h&MXS7q4ppUm_oBue17)AD3O35) zwStrQS^Wrehnz=#hssHuOf5tDGvJKuQp-^sNtOgwDk4wt&@yUqq}own zCh8xe1IDsbx4<|@#InAFQ20{nI;~UWyh00x}_s>5iNn9RQ$6832Ir zzyB-!le5_AI~e~zD%{fXuq}S~o7(OPT%i+XmC4gHSt;3N7IQOP>9L|dv70tFLP#oZ zm_>we!J<TsE4$$+r z^)m9(@#SQ&G!~yXdsoNT4L>KVAu|^Jm&+q_%1z~d{yEFkxk22G!^F3%9pjtbBXQ(1 zA~%bql8D}tHTKdiWkh+HdO68Rx!<(8ACahLbaPR;cDMV;e`5Ynwkt>tGZD= z`)Hk(`C@cVnTKzd70@&DuX7(S${K?#uuCVMBAl;XREB;ES)-V9W4P*hI6&(GyuJ2` z`WTZA@7^v^O{%#=tx4M##4C?Cy7?CiJxjw6WZ-42{jl%x>sUm@s_K2!%%lu|>`u}p z$1k)ULHC*GMifdk+dWh`2j-0!q1VFUfO-xs(saJ->FRA^CA=*?9iPbU?5;K*yy@^I zhVSN)b_8Goz)?NFQ_E4~sCZ0tf>pej$T&3u#{Czd#&qfSN`~GuV#mQF3EaD*HqtD! z`9l(6rUeH{oC@D1M0O3N;Ae}U0n%-DKgt7bd z&M7AP-942#(3=tgnh+>qxt`{-J?r5d)*Q&734*LbkV4bM>1C1_Vy1MOYcCjCKSk}i zsWkuhZ%@o3jZS~q%NMk*rNCD3=1Q2hkhXtOkp5c!K%i)CD2QVrJQuM!BXBdT=8%VS zlPXbtm%{WuT}?au2UWJJA*-B89+_UXWt)V6DGcsqndo?NBY((n2yMdODpFX~JOxpV zI<>hF;Bwr*xmS9$uq?siFfMo96dp&J6c0dPeEOq)c8-nZ8oQIzkhDUc&w7=E88z)T z(uq;PRM36$EY_Kju8E939gjAAT3&Z0fQF?}e1#4tj!5|on-=ltf#P*8l1{Whe*5>F z)yZpE8GK%@-j0snKXR-ULF=!5#eU=Yh9~v6!&n*wp&nWNIvA7$e{LPg+G~fPum*M= z*F-(`vRRs4Cu~yv$ZK#tBlOc(oqHlKjnGQZ!JH7z!Z6b9di8aS2IMXFrOrTc_4;}l zI$>(70V;0fjc*?uTCDi2cU^@yLYI*Zq$41Fqj)@Feq5nAi$aFG zbcW&eB>W-JO3Tt&``*=gN0(;=<{F`lDtO8P2q?PtS>~B1Cy|PRyrUOc0E)8XOQz`G zcUL9GPY3ZFqFDsBzE>VioN*6BM(w~~9hwM^Zx3%7F0>D(y>m(ry_+Bt1)AbS^&&7( zfJinDy090F-w+N$){ZU{Bo+lU8NrrEeCXeR%T7)+hS#&ca{RVyEw83?9<1br=x%`&+oSFK}q#HSq_O7uVk8ymHMuha3*ByXw>y=g>2YQjUFhs-v^ z$B$2CBP{g@o5=x5NPMIdh!V{c!S6#+;foSguel{rGbbZ!JfPT6cGiLue6a+A`r|&D%&lKUf}bt#dff33xZbph z)Dyb|ZPJ%N9}}PGa-+V84kRhFp5L`R;pnz1M&k}kevV-K6l!0p7#KX9)X&u?IlGG=g@m* zx~ls*$>&7fM$J>ql?+`#G~EdFI^E>5+2lFigZNU%OyJP99+V1u^? z-vjT#86>DC)80}O@Z3H4et^w!9{J~1%I;`CgzV6|`@Mh>$9wSom3ydTJLJc5gbB1% z0&W78)l6%qsEvPF0?aHyob36_r-W`vKTt}t?-AMMkCh--c2uY@aP-A8MFPC6 zOi^?gp|ZT;YFzukYz=^bp(!}w>ZWB2VMD-;=DPv*D8hG*l&*E?QG{U}MJwC>Kw@ht zgaBxWure9?w!h!yyr8IAOYEHXuSop~dnEV^79I2Mm!5~ANzioXk{iz}k5Yyt?*o`% z*$B#+nlCIdghI7Bk5m-*-F!guJEf5s7cT-?nLDwksC1}I1l6_OX8i(u($(z zmKpXW6ZA2XA*S{Vkqy?EO_L;xDf*fTA;Tau^wSSwed9CmaUEJ5owN>XrMR?4@(4-e zW<=wbDpxcM!%G+mlw6f@FK#JeY1Y_qlTXo)C|>tSX~CPlJ}hGxz`99UO|DudG3K3g z0(T^4l3{*Zll-@$sYV=RdjRa)&F9lpt;P#>vtrC2H|dCgE4iR+@sA_~jCMZ;Oplx@&b;}vDp)4}r zV>Eguk*dxECmz9;-dHq7)RDw}HTJr}FA>W~=6sW^zgk%}LB7@es@f^WIv=v+a76N~ zkxV3j$}&AbwFquB%hJ>^aS)Jyg=5GQ$gz=E>?#s};RS4{cAU;kP3I=rSCKQd1B@?H zBP~A&LDxLSNz@-D%cdkwL4pd5vd>n($zlRi@3xPx--q9?-=~kS-RK#sje_e<1zreHU(>7c{duEiqsy_|%O|WO0HOEHN&5Mt2OY1nVelwhexAl-@~%`O z1npqcoN=vG=El0A2)(vs;E`2wlvoFOOzC~)aH=d7=16?teJmpcBm2EC|Il%e;`W-m z8c}X#K5CeoQaKm@pyUD!Gkf7wc)Z8}*B&TVbv+Miw_-tZZq~_~txvxmLV_}pt#-f> z$Bov0s#V+dYB8efvoguUPdDDsCGss=WhSPBCnZx^)%jOh8{SoGaSb<{l zS85(@{00a(4N4B;qz8*R>q4dw$Af&Yz&|gObW{pYg`3V)N>>mqIH^Q6Z$ehP96h?TBb#ku2;QR-~?7?jK3>r%z16SY^R+AcINiSj%i{D02DYT%#%R(gAH2>!Hy23HKIY5h`Rs`=Z0d zbEgWl2^cySJ#^tDPJ5jfoCU{SY@xUxNvKCANJl1{3lOpdi^LI;q=;HwHRcFL{8K8N z{>vohbnfJzjJ^I^4mc)Z72o@~gfPF@`QVjgIV#UQH#4-+OU#K}^!d7&bn ziCmAsJm4K(k~1P-D5|iC!XubS=Ge=TqLt$j0W8Ts&ccv2lSOx(4b%-2iao%#?9F}R zp%BTa&V=9ZhJ%V4J}1Ahmf#euFl!vJfbn^sE)tRXwzW)q8brltK-d2=!Sr&&dq%AZ zlj*Bz=Tpxnm^!X=2D5(zpBMbzWr=D5B)UXvcMjZk=ZZYw$Oy}@=`9`~h-^7g_NN6* zBfAy)^gPj{{t7={s*+;$uw1@r^4>z(XWNw|eZV<$$vuzUKQUTe=$s50c%!x$e-q7P zY1tU{A^%7K@f@w_Sktq{F>PjCn;5@;#1>%Iy3 zoB2V`8i~mIM4sDJe9vsv*-*o~OZ=4d>d4p}{Ve(j*FbTjt}qZx7j}fe}yU zmk5)vi9ER|YqR@5>o!@;lg#4cjG64!W}kHG8ng&r>3mTmX}1K`AR?DBK7$_WG5Nb- zhum`_r$zZuP_d*R+2bkyQn{b#5kDiYFq^|3S|$jj$~>yaAyu6L-=D#6!Qj9&H~*&; z*l=b4W^Ukh*0(DlPB&(4?h3noZ0$WF13t|!RDQ4?Ty8dPPQ9EyAwav>p(}vIC4$Ea zI*TefnIKn1+^DYDXcw6r@BoeH}?Py*9<@Na1 zNX&yd2hHVLjQz|X4E!mrmNgf15<{yc4*3WdMr(`dTmB<9diZLmq1=(Di71c05Gw!-b~ zSIy#C*m4>o{eNe`=FK$LFT!KOmWhRRd;ZDVKB!}!Wd#|=-LLyy%y(eA5vlS^cyj+n z=-dmRJ%jZ<0YK_&p>St@_K2lx6nlA}WrRU#3i_m+#uAhWsV%M`UfH=BRIMu$B?s-z zev9H%yS64l;YDjJG<&Zb{sWpi`BfRk^b@>Xkguf+)D53A0WsXtLQ`g#Q~S1o_O8fb z#5{F|njGSSRlitd6)Uu}F-XtVjG#|kO6s@Vhk_v3=j5L2fWM2~XM&YE8P>3()?tN_ zpq;(nIJj2bRHwr%zBo25B3wQfvO^t#8oqQgfU-O#3Z_mh6%)AW7C{6}BqdY}BtKlI z7J+?DN~qZTAUb1?(!2u7lREo*9Z+SwL$EQWYWX))!#n{o{@o1rj5DG&Ct)&a(30OH zMy=VxEjod!OQijAH*(BkA)HB<9FSVZ>5MczsEb%07m=X7=HS@*X&#M6Hp>xeli$aL zwJET2jdN7ZydcQw=ZuTBA#X`2mQX<*Ry!cOk=bQpw?^E?i$H>#lLF5cQMdmFrO_BA z81QEd!{_;KazMfgb>|yaND`SYsLXb|$JxwOvLsA9(lOUie5T&w`@5nw?9QR)6$sGj ze8pYhczC-5mjtBhimz0qVi-{bIf!2C+6bJ`GkFc?eJy12a)rGtcYp*c6+B|c#WobU z$|siwETijYtY-I|o!^5r^+zKOV~{9v?BHo7DM8lABq%^KL(I2Csyymorzkh$!iu#3 zi^iSJ11y59YrA?nMM~e}aqn7GA_@5;fK%rzKx^gI;v$(vv2LY7xlwH)FA}>!j>1?1 z;hed=v9|R1H4p8uwra!dpttAI`JNNC#gE98SP+k)+K}~kD;K^mJ`29YSoIp7F*egy zE)A*4NyT7GCU3JfvYtZ&t^@#Rdj?1by32&>n9hJsLgC+W-eA05?`Hu@rUgmP~~{Bb1I6&DK%%5&3N!FYeRF(CYVI8t~e$ zxt?k?tu9|2J=Mf&xz}6a*KbUqxJs`o(pI{GZg5nsE**+IYdS_E5R(^WZl9#Mttc3{ z=HQ*k0P1)`kyD(L%x#I!fp+VQ-2jc`@Zj`Y#zuC`#em#Hz+@>9z+^NXMj?`kSrc~A z$YJX;e4*o96PKMI`>DTJz3=DMF{(Q>nyK=4;yRt;(BZX1(cN(tbwTdVuA|aW$C^s{ ztM)2u1g`teXntpaa&P8#ZwgUfjn zBW+S8CQm-v2|VhHe}4&OwnX%!%TnJRnlN$46~lD8ji>4haCJuwT52oOgXGPOETv;+Zcrd)Bdh<4mM-3E-_*e$W+I zA+*_Wk&|2^gPvr520$KRzErl}u)%nAB3(X%!@)UQdVlT7yIu}LgnA|}cJ^N472IEV zS?7CSM`le!v$y4%x;gVU?!65S2oRhNtN9fKv*&%F`GKzmrV6V1!WP~VWxiPJCam$@ z4t(_utz`7ak8H#yj92;6TUd~J9sU3$c)QGW1x^1nfw^m!M|e?#kM;(TAPzoux+5El__17Gghx#CTBolVld7gB>*>W-P}e>{_0WT|1Y@P$I6=$YY?BXwZ; zim}+cnud-q&4c#~{1693cRFIgcE#93Y@l1ks^P>bGQxVBday0+?{hf#ipezU?kp`= z8M2vK>y8UL^G&7IfLvEA>>y)xo+{QNDRxKDE%!xWKKJLRvR~Vng^)ODlYCyvHMy~i zqG~tw27<}tWV=(c?jTrx)Q_#WzUd_})wq8Y)IUB^?)?2gar4Kwsvnx_k7PS5FQXeB$_B1v>oteFqEz$gGEhW3yzD!p{#42$v^t9`Wz9D9 z|1kCrOrk~Il3>}kZPzW^wr$(C?Yd>#wr#s^*|t6PW+r;#>*($`e<9C_y>n%*oq1P4 z)EZ&5>Q@Q*^DOVM=QFb%zM3}-&2!Cy(X^^xx0n*ZmJ4>l1`A#LvCeQ8QeEf-Q4>D* zZHvnJ8{dMt;S&pF3{7azRN`qbCc@kDS;m~`AJ6!s0H#E z3K^Qd9GskdflA2~P{8y&sqXCzE?1!m%2LU$b#QVhh}xq?AA5-D%XoGO(iEUk<8MZ? z%fD+7hC!?sjGyy~{GTd&48|2L4hR6iJoNuDf%X4Y+4XEpj4cfQYw2HAChoUmx%)#6 zVWMBPUU38Es^8z=b_GtKU|nZObRh>s5=+9GM3JD}Hk#z8dk}v7ezhg)Twi0~{&+f* zO{km5tzbh+bvc9PKECVw?5URR@cY2Dxt-Jv9v=^%ck`MR{^jlxb)C27JAAtyr!CN2 z31GF#m>~6V+7w*38Dw8Er zg%m21JD4cc$7r?p}p~A;hArz}m@6R{|1{#P zza6Us>(zQb&~BnV8V+1UOYFM|uR0;Xi7u;pN2@#&hmke`63Zf_b`UM$g>E#)l(y~` z6e@Bf;+zI~p(`j$DoD&*ic=UGem&emlV9kWAD4Z$-vYhaIF%TiC0Is*J9uB4ob7Et z!OEb6y#8PX{@rx^)GYuN(8GOkO!YSenwRrF!l-?m!0ci) z+~Eu?%!#|qTH+5O)(f0SvL_+UI0Y^=7p{KkIxowU1`&`%Pl6ZCCJUF?IP*aJbhtMI zbZTrZf%_w^^mx8i#EqufSqWoso@bTbpVt_N2WzC-rU&BbD*(drH~9$0SfJ6Rr)0aD z5_by|=_yZG+#JF62!9OO(JEZ@d?f*oIw`uTv)9G<>2p`NPMw<+V&hWwF{}F9AxvGk z!t^|pXp)X^Bm)N=_h7s}inu{7KGBilrbB4+C0Gl8wGP>6q=BP^_V%2#8lDaVGtr~{ zsTu8&t>KVR(>ao~v@wTX3I1h@Z-bR=-3l~03-YC+gc)wmxocNjq>P9l_Spx<)f|DK zY%^J+CJy=F*S#MmGJ%&wY%_4R_S1IJb!|=sBTbaEdVsjIt5b`}#b51?dAz7Qb8;m7}iGzAaMa3<1znRN^yj*bL&JwAF(gSL0iD@91#XrT7MP_?cJh&12 zXZZg-VD@Qo1IWPv0Lp)3p8wA_o~wn4+wakmt?p&_d$j!Qe2J*i6j9#I{AE$KBDI|DAu2uj7UzSXN_aNy&Zdd z{^6NC)2ts4M*kb1VXVB!Tq?<^(yLh6eC+X6L@KkMRq~P)f&x0f5t%Brk0v*1Lrgi% zfv~JCSW+2bLb;Y`NjVYzSDH>lgK1~3U_wLOmYJ4`s&Qu>r&&STuYeL_Ew)iCmO?_L zeK&@xJuls>IV?^^xz>@lmV>Tl&jbF7s1#nX!hG=*#ijM*>G7WG?eQ)3A&P!{`K%Vb zHgzyk(k|&A1X*!>jA)y)ts0%ZL7Z~aUlwgC1f!ix!W{^YUgDoE?raS?88S>Q{wAd2 z;vfD8mP`YAp#dr3ndrU`e`&vgo%P_pVf0{XT`CC82^uN$L0liuF-S~x&MpuFRz?yM zvg1|_PJ&g%sWa}KKwal?US{gL0K%AwXmpa;aY#V#HaANVYF96J@b`p`!uYJ=t{ zA!oPPa; zVA5z^MIK=kN}TJ#$7Z1K{{oQ-W3BE%OkTg?!w5Ni@ttuE<(r|dZc6Sd9cJvY|HMel zNbUf2P?skX{k1Ai2cis{mzYiT8lhrnGA#d9V(8up9I?Dll!^E{XnPw=V2PErV00g7 z&o)|+emz0i{gOD4;Sm#C%|8iV6i%fkYNj!m(D6uAD;?7P%HaYaNa{r#79j_UJzF$mL%j(_N;C;+f*MwZerGbKt>4fq2&m!K^Xa(jh zlVriisaAu6{QkzM29&YjKe=OkS9;LL`}(h(;rk z^Ofq4bqxU!_YdGXDFr6|8Aw=*qU8xxfo7DK_ zlJ=XQTL-_jt8Y!R!f{CMXg)()3+u~(2ZH>VviS9nxS1_b=Uy{%*MvDv+!GS8Rp3Lw}@)dfvCZ2rPJp48E6o;JoldgGm50id#K@LvClUZBHUO81{qT==w8vfu(G?ngO+q@#YB6Ksf` zb_$wDV=neZIQH-V6*#leNDkEW%-ErCI=Vb}o`pd@?TnUq&225l8U;W5P-A0^(}Ic! zEocP?)Yt~b22hB^5#{gw`xLrQ9e&Nh%=nR%_Qr@k8&Dx9-N_RsNWGm7X|lw=%v{l^ zluod_*xUHphu%6!W6}nFy}87}s|P=Y?Ew3Locs34%;`6o-WvrDetzX+{0l6;azDD% zvnyZ^kMSFOqR+}qzpE3MoTcsXXkpJcxdb`9?Sa`C;yKgg-t5LL#lDFBU>HwHHOBkz zDQd23b-BWKQ{y+LB{w5J50RtOEcK@^ibuaY;6@w>2y*gPha_{SP4 z1`j1)5Fq+7+PRFH>;ntkA6AmSd9D`V3KXZ0?7bh+#G5a$L8Z_jnkFSEPzqw~8nq$s zAjMti5rXNIiC4}b+de1M^a6OEN?w@#d^}GDz}*nhL(d{=<^IF}spIH8`*Mm55C8!C zx1gTwukpZ@Ku7A}#iM`U|Ke0j&!9vWi1pr2|VPjUy$zoCvKi%|Bk&s(FSLIvw z7Mzy`D}G3^w$`( z@`9$&?Xeg-4Hn6Cr@e7f)zjvEW(BXoZPItK^tF>~qzMd`BTL8b(R0PM{ZY-HFF!$V zocuot4GbC$`PwhU?7wi6{x?t?+1c3G+5VCml~_5eAby15n+M9!!vciV#04n=@IpMa zsz}LGm%>U7`=7c_yw7JA3+uD2?N4{6n=8h7(|W@$FPIL5IJ?m}U^KOi1&pMInh8!@ z3k`%dBc)odR=qzg$|V!!cI1-d0r9gj zp41sENLxAHmLcT;thUc(p=oH>^7#dggXPACjb%LN?hWr^|0gs$Qd;TPzmRDC|AEHE z$=S~47aE;dIjO<_g64xV^l>3jqiDalHBB3+h0+F5V5p`u$=Kj0cs*jy&Mv$q+zB&{ z?dEG6Xo6jP%#~+Qu~y%w*S9NZOG;-MG#|6_&q1&IcvjU&JZ%*Wcw>dg zX7^0}2;SptDNP>YB1{|xN8&}W35Cb~5=99H30RTMh745^dz}1inSp@xwK=u&sWF@B5E$D8q-nqUx%DnYs!X4)d;FSD@x>0v+g^{R}YGGFp-* z5`RzKvk88_Ohb!=6%+9-oHl^WP&6`eW?;-hABaAJw+@GUq*v*B!89%O5Z@}fi*({o> zUx!5+DDM^PWJnCZcs6XWJv4NxCcFm=K0uWYc6{BC`)(wY=LN0X!XJu*5kH8=S6I|r z;4$)QD^?XS>+n5BCXEs6auX#%E=33vxuX{Rw``)42XlP@f4-@M=Anav3}*`>O+xUH zJlexiIX>T6%k@J_AZbjNsp#2IYdSjP{jqyuh11Ol8|p%PVb$Bl*fLGOwz6~D+uGC# zZmRI!IH;g>J)69m(=CAc9!Ak&cvX66(Y@gcb`LyXlr}e$Utt?*J~{Mkm3T9C<{~rF z>wo>66z}2V2NJ0v?`suq9nh`0NY;7qGE+b61sVngx&r2E1=V>lX8m1?_O+Dspa~^? zq-nOl;ld5_|5n2x*<(On@J6iv9wSnC*7m$v-?E%CPNgVlh<=1({ee92p6xCL)G9b6 zyUj%Nc?;R;lL-Ut%mI81Vq{e`@=E=fCKR32CP-Cik$q$QrlY5~ySE{8=TI+t+h5gD zbcg_ep*L=A-<2Lh#L$~s_XC?3Qe=Fs9ZQm~GwGQW?7Kzm#MJ}0mi7wr*dNc7-Mb*F zB{}f*XHrQl&u#RV0bhXtWWH@o_e)8pVn~uJoiGTa5<97Ei1bkQ{X-^(D-Abm_D8o^ zV(%-!5j%RO;O9B6pw?wcZU+)31K2+2%y-)zdC~wB!)*d2dCeZ&TiH;T^rnqNa?%_h zg|&pN>kT(>)_Q7y!MbakNU*oOC+u{cI z0W24!XEKr8cuYKb3uZfXZYBZ(lRLg7YPqgMatf4^>>7}>V+XNp0O!-Cgn?LphJaZV z$yc4Oa?l5uC`9wH>aB;wWa@)A^)6fa8cUcla&^06Pcq*)+Baatq(NdQ&YCbhR7(n; zY+sK!Z*$@WDeTF3WHHWA+=oFDBz6p@*FZ%eJ%f}c5^ajvSx&$0*q|DAcyJa< z@ET-e^<5&$46{tZYRte5l6ZAOEwMXB^ZQqasKBqLDQ8XdHt}tCoamr zlrpoxP}N3y5C^ykA38&I922P)F5zq%w<1}_4Et*Dv-uJbiaoc+sL@D##}r8n=ZDZy zc`$wQ(R^AL1hQmO%;0_N{_*y?ga?}#YH20M$!WtQ(eOTj-0~N46U;Um3nK>E`s74W zy2Vc1wD=L>$cX8Hy(zPuG8dK%^6YScMgJIY^()L*a~gozl^mRjabyT)mtlq>N%S8c zC(L6TTq$NI(-hk7x|twmVR1g6Aa#ZPa&4Un`>~1$N2n{3OmkK+$o+UnD?^6h8Jd5$ zZ`xFgKS2*5qs%l}GT#7UY&&CHGPVq(}0YrhPA<|4Pit9jYa2F*#P z>0O`Qp$4w_yVp2l;4Nh_=3f z%^6uD2Br7me|Aiz;JJJZp0p^rl~~o#@O$hYFmnC(9?-zpB5)ea{;bBA1X2RS4JZ4? zhd37{H;dn<5yA`Eq#y_zK4W5!3-R8G2JBGUes-M+60c(b$dMZ=$g)7LW%>_1g&?B( z`WxTzYJNgFuLYXvLsPy19h9v|mT1l=l|q4#`%UPuC^M$dN#!uIFK(F`Y%Vl&8-KkI zj_pLvur=}|fllbj84%XMe*#iR_wRe&z2Rb>(9QJoB@}Ww48p;iOXwPL5WLK;yYPI3 zS~A%n%oxXQ|0thXm5xKfpoMqR)Vk%5Qf z&BhALuvf)Ak~GTHu*Fk@*F6pe+gT!};m&?4kAQ1S&YFRDW-xM1VU#~fDy@S7O3GA+I^HCLE$&unQ&VHHclHJ@(3!lMTIMOUP2f&+kvR-mP)^^`- zX1y?SR>gQ6IaxDm?V&cHLa4_|MzgmqR9?0j?oLi1mLIvg+7AJsHPi~?y}mRe z)kDWo`_D$GRL)?kj5O3lSV}lgL1zyw5~LwuUTv_Z1ZB&f`1P<`)5x`?Z?)QmH_yZ@ z2Q>`4Oh};6Xtco2*hx_*@l?wt9xU0<`CxNmj`69jmfJ-lM(f^ZEi#?o*A!rH*#+C2 zx$$;{d>1!6aJ5@5fE(Tl<&tWS1t5#HCF+!CV)BHqzx?1Ys2nP+-psNnI=4i(=FFeo zPB7NG%MP$cgc@HC5H?kQQ!Z7bQ zKqkMN&%*S_eCc+k4B`#f?s=p|Hph0)Jps+@47PJnwNH7jHX<&L>2LO*>RHb#K>V+o ze;hw!vj@fJ0R-R~SNJXtFQaG%6|{vgj_Z=xBjrO(bgM zlg=H@hxW{V&c<_?=Tn*L12CK$Q|=2?mEUmz+b7t_+7K6o#B|lN*yXg z0}1k})uxJik|kr0AS-bSj|6xLW+@KRWR#ulqdG)GPQE%(*5-iE%2RGw8FtkOPAcF!s7!y`?ml1Z*h(_ zF2C>ifL;b9C-HkY|H$Q{tM|W?(w}{kdwA_qW z>xhbUu{{Fl3e0)!NOnsl-K!3$Ex=&RrikTZxw!HyDGyWfXjp|WOtRoqLe7n$Pwzz^ zKLw<~?c>ttg9*~#>_e@oy>pApzh$)ahp1KSXXygM75hs9PY~zWuyqOl+|p5E(E%y_ z>2cyI(#Rk}_P67%OjlCtsm5TW6?I;P^sB26ljD5!3);sbUPVx+^a&sxN>%qo!r6lvtyv=_3d*)DajKu@!R|mH!sod+~St=j{ zb-PC$?C97IR_l2d#3k7(lM2okmIKDea!%x#A?*nKzIeBo&dBt~G$*!~x%8avn*HTy zt*r0LY&b4Q$E`JF?q^Q{`#csQhx&ejC!0Vh{rExob7^!W+kfo5i9?H{XK|@XdSqy0 z$JY|GZCQvd(ELKz60!!y6;hR{bfX&BtlFiQ0{pJAgL#u>F0+eI@OUsy1(csf>&u_> zh(Yyc?RU6izg&}$4GX)uGrod(Kfv>Ju{@K*zhZIw>7?s}ZPtKy1;mQ)Q7#BzrwudT z{ZZ5bI61Reg{^)6c^}Yvzu-+?&;j8t#hp}1sQwUyk??LZB2P!Txg?H&+x1kv8f0wrY|LGt$CXQzR z_q!ntEjw&BG~d-)cIw_Kl)?zw8?+F4&09#FFfd#AIU9F~1wa z6G=D|O6)u^W2r2s9;aS;-k^)FwMO@D#NA-J=-JVq``zb*%jrv~nj5_98@!t1#!f9i zd$70{yKgQ}^f2ZFXXYq#vV9flbr4?A0$9dhHzGyOV44=6*up>?eNrm=K?H%4^4 zsTWY*n%D*#&C3A$vB>vR-{+a!Vq=*L;DkndrM>aiS;n}BDlpV&WADF>$x>+^aWi(1 zpUbS!pg9Re$2lf}4McL`+`hSDOF=}kfhaq71f(I#y$z(rRzw&;3(*bDa5%Q@{E1MH znm!q;adC}O0?Bt7y?frZGvGl2X$UiWJbxCB^9!RN8>hs&TOnP@f%|gNTnlh|80J!d zMbPF^K?Jgfi6C%flm4!rFoV!gL*9qkAx=1afeqN;6RMB0rws_=f=+7?+~+Z`Bhu|c zLWDQQewqNgg7qo_8swDx+27I!94AsAmH?V1Sy$YyJ5^R{n`W}~!+?0GA85?dps=cj zb<#pF*9So@xIM~@O-=q&nL#iDG!~MW2dr$&l8~KO=n>%-FSt&qnS~ATBg6OI9Wm)PT%Jv$V1l<;wP7FE@D9 zjOo|4vpx8O1Lqf$OWD$n2611?XC6+>Jkv(N?iP_pz>4xgY?O@dfi`ZTWx2;D{{V|6 ziIg)4=0x1j{+IpLEku+71=%IIOhc$W4y&mrrRSh{EuIs$-fV_bZMLfM+`~;Wj3kI0 zs*OaLyQQ9Hwa{YhuOP9X$+nhgaoz zJb{H#D`0DA>EtaXgt^ZqwBu``oLNE(Pfe58Drl+hk;fAWBF`KrdIB zUYKpBhx>g*U72Kyj#w?qOSF4fc?K%)meR>WkT{Xd-2>Hl#Oqz|bwJ;Bm+KdmANI4T zHVF5(w3PDz#yZ!`%#M6JX93oCDLWE&D>33l1-0DDKZmjGSvHUhBjy|$cq2rb$v_50 zI!q~AKutBg%=zNZE>ZYR0IR9cVK$@%#`MSX1u8O33p9=+@)vml+^0@`uSteKo_cmp zI*(e1ibDCF@n}HbLN8RfwC&YM&#dT;P2%Q}le{tKQ_JK~-e=HONG;z9DtFQj!_|(; zqURC8eiPE6o?F-8rP47ZBf~y1NIBFmFMaqziF1*oU=Nm>-CEH+81&$#UzuCqyY_0# z203}}Bf4c>8K?6s`aJe?Qg~&W&a|*1LN^Ybx4Y*tlami`&B z@gZp>28mARO?a8n^%&?UcviUT~b_=CE_X7=CJAhSISLFm9ct# zhoru1JQDHg)_MJXym$9~?8*D#-tgo5{_$z1it;?^SemgIeKIvRGEo=_6i$nLp~=R1 zigNrLpOrj$Fn#dp{)20Z!lq9b`eke1(d4PaBe1CyO^YbvTiSI2|H>($-8P&`Ii0e+ z!^dhpzrfY#y2qG%(7gYtwbWo{aWbHJwjTMac`;XGXL%DZvD({l3YMJ@M?)I1=_w2s zH6H?wE~QMbby3}h>E{DFeWc0)`iSpB^6#`@hwppqV9!yP)5v}Kk}(R}`tBd#v*&Nc z+(-{e!TQPEgc?IilwtR7d5}`F4Q%rE0!OsW?%2v4=@(zvR>p9GHo`@e&c}3t9%~lA zBF^b-WW+6h*|ExGay!lNe&xD{lwvc5b}ankZQ?Pa(UJJI4M<&0`nV4`_L@$|@kuxh z={ms)L5|evLMPT7Z9E5V&s%4KYMT{1Jep?ubs*V8=QA(&Rlvt&klD4rx5t>{wf0N2 zww#~R8Z%*dy5t8gp;Abb_Iz1Ar0R{HFG)6%N{xCeVPd%)8OLsR*JSom7DeBL3P`Pb z?auXIP2;Cs+)h4b*zFIP(p9MQ1iPj`cvlX zGyCcOb)e?!Eir-_1vAHBRgevhb$q?Qp{f9g8VE&*nLy%(ZF{W)z z<5}1FIN`d%rF0JZx~27YbY3lC0omPj$*rIyk&q(42qQ`ONDdLJyU!f@Hbl#(AWhzb zMhQp*k$_5-kPs4$tfXRG46N7 zXFkC0hf*QNSd=ne0A)bc0hCZ2E0i)>p}Cv%$(NI(KX4m*p4DEA0lhWmr(XQ~M_Q#b zK41P02c;5eFiefskcOINVli#-O}&A&!-*jFDxJj7H`6l^ssTA5;bsd*%91QTIQ8;y zjPp=zvgl}>Nwu*|s#?slrf?~A=3}xw5ZEvNB zYJFS__|8zy@5PSN{p>Z!{6e+8RqY|Qr<#g#S?PvsWmzu1K@<7CuA0!|fO^$xi}%&! z`#*(T4G|IWRQz(Vu3rvD_`edh|HoH1G;sPa0yg|hz&2P>elrOPlEXC0iJWnt4f})^ zjcI{kWaizk00t3^BU&QHk_i?fx(R-KPU4e=C7Y5AfsezMN@k9@UWA^awf`>Xn_k3_ z9MF*5Z1;Sy5asr6Y42z!R&Lbl+4kzr*xK@q7BV@$9KF{@@u`cp=b^K493@s`RFfYL zZaodtqpOK@AUiW_E+3^7O-9$*sH`Mv`?f{yw#K_;jMM(j{IkleF;2QQ8Ayw*=+q$N z(NWS|k{A|w=RzHgPjnyQs9D-jHLaE+aws}5-;jscUQI&np($+KQhBTIS~j&~M_Knd zzE>a6UVNdvLK>$qh4rn2dM2b1<(>|eaabPgXI$vTO?3OFKKGR^O&YT_+vyx7IrRb_ zY+t?psF86JAIcnblLZ^FzOGe$fVXW}Dw5&GQu?xJ#!sud^7^vA{; z54@28XBr0XWHSpC@%5XqT4UnKS@p(TEGL=k%&7lC$vWDr9pi-|MpPpKN`TjYr)Skd zW&R(#!{xiPt(9KWUar0BmxCO6yX$O=p_=;EMC?_#`FN^ z#NsR(%6;X9VbfMkRCQ=rCJX?mDBv6uF7X;#jVt%ivs#kL7*T|zPDd#&CrMo#<}Ba^ ziZjq7t9HVpiX*|zJOd?Op#5Bsatjf{s|Sotc17+`NonWe>A2xY2M8gYQRqQYHIe`! zvLxZv@si|}(yQ~KP>CzbJYaf4c+gMX<5`DRD;2dsaxEyZUngh$|N zD~_xxo4U?H)&*twJ;tq(!D9gqB3@Ky?-D&e{f|9f zb&Yfz47uf``;fSZ_3csDF7?ZvG^YgvELh{dg~)Aq1D#cf8M4j0X$eKPJPOZ|My0%J zs1SUOoWQoi2V{!W-r85gf*VR!U1`XV2RsC$2!HIgy@n0Q1MOeGA~C?Y^PtxIjCCh; zWj*qMZ1_OqOQFk4*@+CRjivz>8^Ksr%8clUxA>+n6%kw1q9V5T;+E>5Kg0<~dEC+J zl%{iz5ZhP`-ta!H?xx6Z(0WuS9fN}*izdO3F~U-^NqC~G^K#bQkbBj+$Pr@hbxxT! zd!gaIIxh%lpNdio0w91*C^}|Ju1f;&E8h)%z);(E++<30-Z}x2%0HX#~x6sAK&6o8;SQlJg6F+N?f4X3NO7#l+G;vm5ar7iHv^8{ha0;{8f z2iWe_IGXswk3-nqV&|%}8ot@006N9qYIZS)tbA@JA#34cO(3xPDY@)e#EJ&GQ`Hqc zs8X=SwX6v^s@-}nRul1))bdf$0@=>UKy~SI=qq-63+Tv&i8@0(yYq5=%DY4K7T4J# zgV8Y?xprW4v?7YYyzwkpX1B`5UHWlMwzhRd7omSHSZT30&`258wJvf8C?twXrak(| zI~Q_0bX}Wc*7+OqW=Nc_FSn?@OV9?gFYR^FK66UM?NaH#0q&X)*)wt5=Fs*PNOsaH zWf`CxpB0t|{-KoZBEE_sCHt}x37xWzjiR}d>aBf*{p-gasGBL@_{atkCViA>rGH?0 zwr%AA%mMM|=?R*Rxh}O~!K?yZwAbf1vtGLKago)>TI_dh{o~aN{H5>icxNG-D5A&o-JnDo1lOlPh$;MCmgk z&(|Hj0~n){yTu2NYe+B51#awPALOWy64?N};35asPv^PxL&f=8F4-(7=SQGyawIxo z)5se|Zyn`F5WmXIlK=TbY~j&)x4s0W1Y+;EqsBbg)YvR$4D0K+z&$Ils*xtVCI3%M z_l$w#YH#@D&SLAU3@@2k!pKVNl<3 z*K2&w9v7iiTH03HDOtAs02=W$EW#v94Zyd~s@!z=@K!PyUq`a0E&LL^iNVx@bUd4e z`pEvVSnsN#RjTyg=Ms!eqZS}(s;6LeI{@qW6E+hcJ9XS91)qiu*I~$qn85=nNUikl&LcFZG zWw+Ue=(p1wxRuvuE33N6^=J|R8{g;^5@6-X-kAy%ntvdfoEoG0FETl)`r_BZekDvq zYHo4r2|MrCh+rDr^ZqmyKSt%SdJaq3u_e_wl)AqE`_QACq2Equgy;Kt+&gho zL&x7PZB#qFFMaUK{v2~sg?4pRsMEYpY;+ar>E-5oyFRQc$kuPf+_}nBMbeGpnLMhp z&csa!>wz1M$$fSqV%4%hJ z(>sHl#O~UB(t>tX&1N(uWepEf&wbgW=|S;M_}iP!xb2@X6BpXB(1u{ zpZGqL6bqbwvAPQBceaDrWrESrV;2};d|0rYBZ|=vPZBcZL`g6Q;wIpJ3cfkXI!1#U z$Mpw)!78hv5i7zCkbYz`7tMYtb_n=zzn!I=M}g1i(pKNUuG%q(nVAPZd`JP7+rDJ&3hhF-mLud`M}PoP z=ICoPeXyhOp?m|MnpcEVNr1^g{Yb3!#qEDQBJWwH{G_f?>Z6ua(JP>OCv;Ij3Z4!Y zS1-pT2e9OUjCC$p)@8}1-Tf4PV9r-^8N{_-y*5-FPh%~Q&wm1 zbLs=VB~10XS9ETHtpUCWX!YOA-gB8osTUDCKysdbDk+ zkKS7o#L-12d{^9Wn73^6(45GIfP4iejhacad=*f9{#2`;eP;0&%E$j<^TXrOqZE*7p6Ces_D_4y2xs(L}h&^j524~1s zopEbuhrjze45sh7PyikD6a-_gbH%p8+RN#{RQ`q<$;wwhf0b>9r|>Fp4&*y9t{uA$ zF%>Ixz3Gs+0LXStv`$r4B#!N@?O*=40E9Z~qc*heDj5_C#7pfhb;+Yl_?fmXu}1Ro z!r{!W(`rlI-Abc^)})GZ9Mq})#!=yn;8$m~Mz|0>umyZuJw&VYmn-*7N$UqCdbfkk zo!%=GJ#qtvh31Uco_P}j!8lk3-{VZIU{Ph+~{470Po=nBcclAMD4F5MnWPwG5qAVxrv1?BUJV_GQ!fRX%)1|a>SJ&YA`P~?J>EhSo!yMRf#{% zKeFQEU1|I@OaLtKHAZf4l)LvMKHR6<{;R%&GW6VQ(%o5JdVrv3OnRi;14eYnxRawF zi{#HEL?-3K7XmZ8nl`7nnr9316XBso!Kd}8Hq}>eT+RIloFQdRUO<*R4|aC(z`djoZAcKTQY|zQC8*eLZPwLH+7ZM$}lcWYisQM z4$WF@U6Dw5yqwjL@q00=c+$8XR&~mJj2c&Wu(@$ho=vuxt)l8YTQTX7D7N3X8Dv=} zSI94h3GMGTYO)vBbpfYr`tDR0B;7=2O(W%+qLt_T*Xom#z!kwK+@40@A^s}X;95W~ zMI@B&VFN2OO!_IsIb+b%Taeso1jdS5WdwXMZevo(4 zJqv7c$4k1|;|I+^3E3TP1p0f}&CZLeZ@fQa;?sTA?w^%STP}YOIq#ry8udw+NVMHK zkyN&$PRZG*4U`{*MF;k}R%EmHRKHBz32jMi{1@55xi z1V8*3*#Yd;DA%q;?UISX%`Eum@qrW7seK;R`&oo~KjwC=v`$hjGxf?$@K%8yjklk^ z7NK2h=og!1ovU`^5>_w{`@_ymR|md+TskSgcYPO!YI?YuNg;@=oC@IkZh-&dD}$uS zfU6({3jNSW@SAp3e?qBo+kTElf5Sk_YR%ZWqS&`9;%_T-iewINgEt(9Pd%tW%>A61 z7YlG^xnFzo>0>$vF)FTTc1#J%9+zMI?t}Y$d0M}Jl=)oJh*C&DkBPDt@imYhUQ@Xs z?q!({@kdQR4Iu}It@V#94V8P`2tAu@lGiLlcVRsG6B_LVYK49e`ZJUjr?Au1tvn7= zI&a;OGuZT{z4!O(1TeIwQWx^W+kv3dDXzW*&H0FLf|g)j?%*SNL3Y4F>lxZ*(1!VX zsn3}8{!enu4s;SF$gcpmfcW3CG;0&vUt01%SlY1qPFyBCg6~N!Mm@cJy?O0&x3FZf zO^cgmxn<>+ikBh^A$S%DsdEc@t__s50611zm zx82*Ny+VEUo?vaf1k8mq#Sq@a$GRxc%0Y6sWJ(37HBzbsE4{C?Vfe=DyZM0V1d`i% zG*Y*e_-Du%A+4Dr6F+(-<0d^ZM!bF3G44iZxQC@drnTxwqaURwdFwI9$7myR=dlp7 zu3$`4^?)q>khbDz9o^Scv>qqsXLF2RdpqKbsUXgCJ5`s(P>r{fzu_POyYi}o?e)Wn z&wvIl5Zs8KhL%dfd~|>vBCyNQ2HK8_4yutN<3eMDG`#w)IRYJU7dbTFmk(zGjmCR~ zdoA%F+Ifv4DW*c=iTxZ@%V`FW8aT>U>JB@D`{5ArqnBk9X9EjYKiXdcIv|$f@0`~p zmAK2LSfHSQ_ps&0xh9Q|9$eYBb@RP#xa8$z&6|qQ{GUbUt+tK^fhkA(tx3L8Zouks z>w#RjF}Go8qlFfW3SY7d%ml(PG&ryK_7=4qgG1%;Yor4EITSJk&ZzO>f4V(EG{njc zGz-L!OYFxHOGO3fyhiBEuP{3C#TrC4Ay zWpTMS`SJ1E$=cs^_;YLEgk_L0Au8Sa@7BSDT>-_`t=C(sBvtNNpMuH6pD|q1o+d>5 zY7#tKb;mR;j1qF9=A%igY~tq@0AYd%HF-pSYf?y>GZMcSA@a9-CykhkWwdPB-Z1F&20uh^LuyANNH!h9 z2fuNUy}Cn$xRGA{>MD+~-2un$#zf}op-QJrH4XbpXmZd_2|$0$Kjd+{o4VOM>^Hy2 zN#zSdPS0t{ln=X)s5skD)H(EndnmLbL`2Y(4O!AZ<>R$o6!XBXu(`{P_RX~~6(d>i zwJ*)^rN2Xg|M_Z-j7I;#1GhpNjXd7>)~`;mD#U2qjD(yo>lxBNuJvCPbHR$m0E1(t z9-rUBxJ50|5Ht;kg|g@NrL-B5)8C*pqr|lu0>3~t4RI_DyDXpogD~ySDq2=Bu(WYS z!ZfK4Bq(--VM<(`Eg*06uxR70-#H5ip5jl&ypv) zqWCO~2rl3`0odG#r!ke7<+hoO1*9PSXb^`y+Q>2jLv0g#5#az+qPHm>HZl9$mLc^@ z*d>Z@UFMgF3qRv7?Uq_TkkXPc-&X*%q1JPG;+*aMRVNEiKmfis3X`UoD+YrrRR{{Sh#?}foKBRsJWAoQscc8Ax9|^%84v~H5dWY z%aZeR0akPQ6yQ&@*eLvrWGk!^?6sbec#cZ%dC`!hPWC&+xC^ipU{RT_Jn>}Ik{)_z zz4y+Nc57hOKpCC06nlvl-|Oz3Klty;k|76h_RFG2cnsEYKFgGhcx+!Z?xquS<7NB1 z-GnA%L6uO0Py{XvqTK>WbF!C$w`TFe=iKg9q$#l_{lB9wD(uCtE3X*r`(QorY}C{F zF&&eTEhK%i;M#9#*F3W*xX(YC{eh9_{xdfj$=uT}FlyNSxysJZ{7IX4$q&T37s^hT zmGCV}eB-3Re@57p3@(lptgp``cris>B;Wkvd2>bOoQvI$GqBgG_3ijrgZ}89Tk@VQ z`!0ND-Jq9?oobcW3*}vWW7R@HZhdE8_c4?2@H_}fw(SLevz{KLywB9)R)SP|q>%R9 z4nb(Q<^(W0^N!TGSM}=R^R&#L%$$^ONl)E+iZdH-tH)--Q~R~)pbi*iR$%gX&ULQJ z!D#vNnO(>Imb!ud=fux;jn#4T+ckuU3IIU-f1dbVOifK3e@Fddj1Aij*2gWqeoFX8 zFcAvQOA-IDqXsz4$fB}FSAR2z1`!Tn6UpK-lA060ojn;6(sc<&9ug#v>z4y2XO(3x zQC4^ynN>p1kK-$P#Evg#Cm*MPnrmL)whwm)KPNYAxtY1o!Ka;*n{li~u-u))G)F~* zDI{l(Uf!R#&x)ykU~*)jl`Y{nmV#{I))$e0N05Ohf!9QDjxsCS-CqHerkJ&94-zZq z2cey6Hxh*o*OszW8Gp~MVtZTCYU@F6%t}Q<)7uWn0_#?>jQ*5KyFwIb7Kzm_7 z=_fAh4Q09}u(FOZ!~8ZOerkqhopKy-@;#o@swKUv0b=JoE7s7bV4lt1TvX-*tz!x6 zs%q+wZExxJz8Zh?;)$C2xosL~%hLXA`diq)gWixOdHF<|ie6ag&j8JNG|)AcFfPm< zy_smCq{)O?qk%CA9@}7I&N`<~G9-k;2JQk?lh7)oitDf8Y~#0(q}a3Iw6WI^78es*aQYr#oRLFRl$Bdun2k7bv7x+Urd{*E8o`a) zY+D)3_0Y=~0S_>o6ATW37;Ew@Se%pG?nSyK#k_)-_p86$Y8z2{!3C$q zsMG(%@KAg_#qe@N*bnd@LO3Q>HN^4|WX{ZI!#kv?pRu7cR$x4YfG1=sNT3(p!K)Vs z&5(;JC(UqtT*^n%EX`6J!s|d{$8%~E;q*wPyDGAo9&P*JbU+<|6vG<3!!m?Whk|)X zQJ_si#o&AB>676rB|bc9q+EkmszjfIOp&tBhsZ()VrIobt(}8qqL-IgEOZ=#%h?4^sxh{SFp05-VqI!O_&~Q0Uo|Y+i6w!<k((5>We7Vo_Cf@_%1Are=0V>@4Ye0v@?1F@uHsb7paV!1C^FY8 zmZAk~O8EstW^8^;D;0zDY`o|^$b1(2ClA6B@LH$JxED?l_lV z-2X+`J2qz)a9g{vZJQn2wr$(F)9Ki@ZQHhO+Z{XU*m?8RsXAxxeX92RY5jpU*P0mP z8d~d5G{^t63$%pDBzaqt;zATptrfFW)2uQ6%1GHu5mZveR;2Mp@Fv^#)<$STjGoeD z#JoD{_Ut(KGy^#=aSgsBV^_n?Ju9>QvP6E^5_DA4pX*Ji-2~oiI$;GsMW|)6sX|_@ zGBZX4`OjyooQ{yHYKiw2;V-d9JyaOy7@C^LRJ)WF93cQhQG+KrvHJE|ZZkt%#2*tn zkwFIU4+9I`q6lwHJ;4n#gU}4x4FiI_boRpUcd;uCff>^V2%7f!s3jg?@LCvEOxPr~ zN45z}*K_kPBEH7vxoIZt+ngOlng(p@(F-V}9HHNm{Po<(xg>SUuP8bGxEvsTH`=v< z2ZduuHF%T01-fsgZfC|W^RDWNM;IF?<4Q#+(RpiAqAmN?E-VM*S8Oicm$7Rf81dCN z-t*b~o1=FGy4$Sp^*fukQJjtCSjoRyreb>(y|iAM)X5$C1LlQ&Aj*TY=^wHg+aAxN z(J5VARgguK{|RD`@vk1dKm>s=>zk!Shc2cXBqx&hM6^smqU`rEy3N7!lz2)?LSm-o|@!+Kn|H*?uT__G+D&uU%t;xlUF|JYp3p+}`#)=L$*?pJgp27GAFs z9225?JznjGoHh#-U+Ix>c8C|+j~HtPFTd}jn{ACIXmCw|75+8E;+-^mpD0u|eZBG1 zvW0NqkPznu2llj&3$`1Jat?f&?Bu$OVPu)E(^ZXkL< zfPi{`<_?<(us1Ps`Y%I=x9W||AtTb~POov9!A9XGYV3PMUBLA~y+c{qslR0S zl@ol)xPHPpUC#Hr+(wCjZg+#shUrdbDn}w-pR%2627rL>XWWnomDok)?9{4Vp{(QW z?c(g^ZP(AuE$#=_)vNNE_PN6=?qBZ^v~nL;n*IA=&Gyc&1EY=$7oc1Tb7?<$1mV`K zwlNUCMP9exF1H6My~L|vXs5(d=>)RcNw>qeO-!@JSeY!Vw-+*^TxU@%_tpnz4&KPB zso*u7vmV%|#*+y|6ERh1O~bfo8HV3Tx#cbVQSg9jt>C=rMVCzbdd%WyzDehz%zK@* z1K13f7e~@{B{BTVff?*W9~DS62F?;vmCDtK|7(M?;~wj-%(L?b{}S`&z#Zt}hfE!{ z8oTB9T%HK$o(Mz!39s5*w>5rH196y($YHn)aYcHbWGTL=W=G3wIB4J~)drRn>G}6C zNEIUuNyKf2OiZM*R#HdmK}Gq}FQcdmJV|L>`Kb zNc*2am>$9!XKR6xGcTqq+wUkiYftmKQN23XrGoHIz|Hfod+xt5XCWgF*YxKrxT}@D zHWXx)jFTndA~(3e*Iv$@58t&Xr)}NQGBV8GX3s$>e;$QM3Q?Rgu=5ZY(oLdKLnM1X zb!YhJGKorLl;}v5>Vtq5%k!2uA?I>e4_HM=Z*AgZGP(qQoVVEDG-9AdITmE8mi8rX ziv$1;ga8pu06Z6s%#!8HbOht=1k$ZDG2^6>VR-;2Gb=b9@nq7V7REU`8(8&ZTdtPD z0$m#d1Y8j*VTc&$9_?zH93L1iNn|7w`x56o(cHg6jDTPkiu6a*Q7>u;;GBMqUs_cM z?zwVMIuxSL8h|%Z34+@Dy(*nuVp+{I2Z)4qRuV(2@wUldCE&>Y+K?p#jY=S^G#hUF z%UElSWDVBbzl^WQ2UFCIVoLkO)*OCyut5kWOY}S0<)OCyf=*SzKVqMGG95h3Z-mI# z*+3B?OUQOd##i^jBImPzlm2Hq+pTrzPgBpv+xqW`V{}X+fhFp6s#LJGe98&m{IVWe z!g6Ov`2iWvXUTYhPOJn+ag?VPPm!Tx`H#GeecT^EWZl;6VXJ9J*<9xtFjN%_5dUb= z{y;-YCfMChbqdBJn^SFYCUP-msq=o5sp6KENQ(uhi>VT5{;9v+1|S=8Rk3$e$9&k; z;NEnoUEc2j@Hu(g7=ISFbndEt4M6d`{p#@)65$4Jx46NL>X%_crCC`PQ*SiWT!hq~ zjU_QSM0pA(eRoI^ez_6;5|}}r5UPS0@^)Ct1ssl2h*4MK9$^#W1D6sEcLgmN0_{ZH=9TiH69IsTV#X0^J~|Ff~wh&qL>g5rL3VvzJ)U{_Lzzjmp#ui>MPD8rOB#%Id1 z>xgc7-v!u4^%KXxDH5R6S@fj89^($;YO-HR)qM&HsSe#frMC8xJA7MpYwz~o{XGZ` zjdjg@9Q4NN@*hR4^u_RP2ym3byhv~y{v|F3aYLXHX@EJI;gkR&M4mxfrZV|LuvTvu z6u{#nZ76k~!Mn`rk4~cs;;SeRPJ^OJ^DE3+6a=x_7@pD@BvVC309p4f^B@A8Z-Gy# z;BWcU)`9Pjfg(EY3^ksWA$&l9ZODtvz`04S*8_a79r;_Csm+UBVKPL+)^}PcuBDj$ zaPJx6M&KEjT{qxwtexCySJ7^OVWP}ERI7-Q5@%KLZzM1=xuT%wDs=>sj6!=z!o$&> zE4*0yKbRK4d*~s5kNe2ry4IF}eQm_ak*K)K>&7O6f-4S!O9)j#lh7dRxKQ0oX_=&J*w$1{z4bCl;Ic|&YF*yDzmb!5n4C!Lx6SQ{1KW{1z>F7< zrD`ZJ=r{PToFR_@1+p&wecd$sJZ(`p51lWLl!GFEB*Y0Biy-%yvmNpzJE}svE0(DU*c&Q zA+p{PR)>E_>{mY7^7HXRI;s_XYi3JuA!84v|Vf zD~9=;rNn%B8df-D!`|U@8RhRDF}nR9gTwrfnYsE$JNofM3;R)(0NI&Y7&$ot>>14L zf807h&;M6DYGLwU>b_W0>xYQ&pHyq2tWfD{jkS`~Nf7F~b!TNzE0oI)E=4pLx@c1u zGi4%bKvv6lHw&4wmWy_tPaU}<=LrYoXV7-pTb@{ZGwp=&)5G0aEbPMn7RBVuFmT^<%5! zy#j4k2ofGoh(2oBcu{jKamluQ2K5=<-Flv+SO@*)^L&Gj|HN_un?CS__5F((fNeRJ z38$%G_3DP@T-4-4*+Mal~+TH#mJq*Z%f3RbH^~$(v%USe#_bYRE(2 zHOqm%^|s6cN098}H#7BoeI-6FoOFUIv*Rsm zP+IwThd|xaJ9JgGKT?;v4hw$^o^p6}S|gNp{UPJUX7W-V4^#e!3aLn5`CJe6HCOde zLsk-Z?BxE{g9eTUCtWL_j74Wzm1`4d(oHQtAQSZ9t`CMgyK(PWyAni37P}0tp?#ik;VjU1hu*E-eJhGUlhUZ)NeCJPhr2BnoK2Frx>c+u{>@H z@9ZAb6UYI39JvhKG7g)l4V&2sr-7Z{!c-t(MBl#JC|0d69{gZ)5J3@_h>Ls`SAU;6 z*u^}F^Hix;POBWgcCDA#@OQ#MW8Ad8Uztp6NC%_XTdJ0FfZXyiBARUhlHv)o_o=}t zDGax9nra{zK_VElQ4niN;U-BllvzJ!q?@$w-Bi)QP!y|t&~J#GW8qxNONc>Hc{{5! zA|t>gd@Dx!7?bN06=O{XtgIn@dL^%c(Kk!J(R`&aff7nQh9LI?ia^2-Ac_cnw~TM7 zu`xhCnB}iKLHWyNx=~*5U{EfSurmcGrH)9#*Mh2g*JGV{>QlM!=cz0edUMRk@wt+6 z^&-5J=`B~_MnflisqQfLpZr@XIP?{HDSHxtyKRC3R|$r1UsjRsdz2ls#vN{C&Z@%@ zc}Gpu;B%34-uULo9(JZ3(n zm{X6iU~Z$DT19T)L)ByUxOuqY^T2JA`G>57Y?@na$zp8pVCXNZ%aki2Ld{|4b#Bzk zW-BD9HKh%*dlHuOlbz2cIb9%DjIce)85lRyP4Qz*XdQjp#XdT=U+E^nRY2~Ca9N6F z=n>&j8$zeNcK>3t_f=?;g{9;yN|QgmT=BqgqY;O8<9;rz>6->?p9WPI z0jf=5`}NajY$;7fl^9f;%e^TL$%7+!ZUpqe&=qI;mALy^f2nqBSO3+C@x&D@gTQnn z1#vc6Mrb!LEr^m;JXe7{pJK1EI2hARq2d|UG+f=L@z+ZXXb8+M03vzn_ET8v0coWA z5|w2)wA{g$g~ypFDX8z~0+T7bE%#?CJ_ybX8aNB-I*A0qzBW~rUY4{I$rVI@Vw9AK z@+hp&(#nv@iq$m{FtTn5zU&WrYdkxSix7LpfY`4rRBBV0%6ZuQVv?0h&kU#)XZfwbs)RZ@k}lOG zz}iHTgx$+^N_z^G6r37HQRuRvMkrjG*9DJ$7}V*#+j>PH>!;0lkCEsy4 z#sdWJ|7rVx5z9=49fIl4?S+r}#z_?Jd#ZvuJwUvvmAG$+hU`2=?_*q>97N!OV_RtE zTQx9(cdbXC+({$*wQjL)5CaTTL^z715&P=C9k4y_XE94hpoZK8i@$uf_Vqf48KECw znVyf3sFXmDv0YrLsI)aRTx2&}faO3ei#psJO@83?*tq(l&LcGaQE06TYiF5qB@>ru={$j&EPJ*fwj2ewUH(! zLoXOVdZ?laOzV+ z-3FV83(7PPoQ!fTyEVV#n3FJ0DY8{qBzCpjMKH$gK7!OL=*DxsK!#!d$*CFR65n z9H12@-0^t+kGuc~==<_?c6WSb$jZ-6$EJSsehRZFvmZYEqBN6%l|HBD z?rrD&JUAPMaD`8oJ$2DorYkUOB-P@fKDUnCD8;AKkfaYr1cvwO0xXB<=Cp4&Z%61_z=bo|Ld{3P%(u2&uQ~&DP*%0{}o#1#Wx8E zO`#w>&sj#jt-6xg=%jjW7q69bbFV}RE{(%P=e(nW?ZseyQB1Anus(VoC-pU?h|S8+ zI00r3nRggschI_F2wrMEAKprj+GitfG!vsWp&r+3cF2)haG2*Mf!(dM5uuw@AYU-1&J=V64!h3OsUBzMqq zmg_pcm57u^6|MU$Hf+9BQm$&t4I9xd& zj367H8$p_D|3I*M(Stn7=Am(&VFv2Po36(8byj|23Dpo2>wM0Z30W?t2@X3F>bNoz zSgXwj(b2|2fs)Qa19gSRCQ%h;rSN##y4X>TLswDwd_=s0W0RM#Tr)XUCAFwTm_vA& zmrVeBk#4gM*hz{SEQanXQkh8*?h>Nvkd5>cVi!&t2>yp zLKL|$lr6ZB5ebM{3q=R$1F9vs5JTB!m5q3ybgdI^%1~_?4GUVTL3-{T(bp#^7t8zQ zrBvc2e17zqkl-(+a2}FqM1VP(BjLcdHLNcaCi;+5KTA=&Xt}y}`=0$qqNElp z9Pa+Xpeju3dD}goxQ+g;`B-2txoq63-CDQC=PRISsEj@PXhx{8&fMD)Vv)Y66Q4*e zeZA@Sgv^#0t#!yyoQl1Ey&%NAZ|!7^ryM?nSPgA^nmHzVmiV7d%V(!6gftpWhIpqo zvY>hi@OLnJUPa(oXs8x(y5(G>$8uV9O!L+qQR{^^{vi=qJ=KjLqTP`SLqvleElL9t zSb@lbCyJ9Al+r&an zyGTA(t!6z(GWc~Psy!)zUN$5-VsO<-Uy%Umx!r%L6-;(vf!W?AhfJAY%v4B#>r;8K z9D_6+OENKRG0=1Q)nGoL1k-wSJ9NF6?!R3f@ownbec_ z^qFbqK+`#gfOmY5A@~`<88!%z2Af|;jS-IIQU5&8{ zL-@C)$(bo)UDr6;U0I1o%T(_?(aP%sEEI{;xr=4&8?5&brYd4zSmt;-;K46LtRQwO zbSTqZ>iYQ)B~`@lqpAtxphg4${0Zf0rezlKlWUiqj=EktWmv0IRXm6mharVarrCbQ zWrx^U+>=ebX;^sHFLY**A-!riwm0SA?SVOqHkTs9OT>k=0AOpqW&)wU0UTA^7V_LY z26QLas?Aw3iJ3T&Vp;cOiwV}auqq78^2jvLt}{tg@t#HJclsmXv5TGgPMjqajQ)@O zp3JN0Ed)cP#?Le^axF+0Y8`}jLE2oUF6z0wZaeOmnm~zfH1%3zbFopujy4VW&J(E)N__C778^jK4P)5%fZ#pP8)yO;!hk!u2UfJkh*N5x@@BCR^R?dM)^C z7CdfDXlBd?Gl#c4M5=Msgn~`RjEy8_A>+i?zgi0hvRp@Z2g{@Bkcrb<5CCEFBBi7La zYW;wyMAU-(*GzX%Evk|z9?SfkFzI9i)TdF1u{#IbolFT z-H5Xj*WB)@VL)JH5XT~y3%I;R(V23Yjh%1#G{5q#kXOgT0*#gq$37kxk5f%W)BTKxRdnG zKrXz75xmYU_xzi%ycF2(v?3ZHm-4k!-95F_!k{{IhCx1Q@LYrwlA?M(680JcE=*tH z8foa941}`dgO~DgrIZ$4M+6Ui)tfS$A~zo~FCgah>|m=q4}TxZrAGD!PMdEAgPYf& zja^thUvB!iDumcrap@9c43RHqp1f3RgYEm&s#-TyG?W{7INQQ@d(`fCR@+CO;bO^r zX1Be3W7sM5Zkl@}BrYR*Eht2Y-{ax;Y<2n8OB~;nNJ5h+G+%TWB@q8$503`4qPHpR zA|9OGU{n=aNM4Umrv#aiV$(YlQ%kbxkJ&#$Ae2=NHZtvVTz~!U;Zh=tUD^;}A5O9w ziM<`$TfPE%d2ptN1phDxo*-~QrZmH zIUti7`Vf)l^H|imsfP(To^1s)Y*5|3)Y&iMl3Bc3l%L>}Qep2j%<_DWzaS^dZVl*c!=8Y%p zHyVFBnWSHW3$gjR7wsJ(3KJULs+UtKqM%a4pCl&v&j2~!;^ak~pXdJ4&1l!@Hrbv* zXmobFb5p{}*tmRQhH*TLDjm*(lq0deLn^1eoQ$1g)KN^JkO#P4)}&zwh+hkc+!yeyn0DME|!- zsMUWq`ai7G|FYey*0TGLTkPi&8ldTW`YU;P3;omB5ccp=-)JubtlpE_5Q|&8z$M(W zn2vmSr{wj<`$hhI*v0=o^ildhtWyGhmcyL*$tO1yuhyIK63MYxfbWNl^v<`3xASL= zuFuQEzrj-1)>XnCi%J&9`+w&KC%v};Z1_CA^jeGe5iGxjd^*~>zwcko#nvi=5>MtxIxi+~zL;z83JPO#rdmG1yOs?NAZI7(M{5 z_C@LDU(4j}oR~`u>!Po9K`K4+l~Q%IciU6#b#^|_bCrftIju(BIs|de8ZpcVRkqKF`AguDJLSzLZNTYjUt-c8 z>CNe1tkbUndDsX-oqxg5mFJBf0Y88jIys~VH`5Vxo*PQ9?<&YFT#kw0ixN`>>hqvW6RA-IPT15elBYK4RMP$I@=X@a+2>{;cp|B$MYF^S*fyp{^m9<6iIqC z&XtSDs~p!M^FFY^y~g7z(j+ucbyuS0IW!czKEyL{yN(KUpv9c{8q6pHNgd>Bg2ar3 zEY3<9vw(yUKKWv1_|pi(&ZHoYxC^J9Y?fef3`kaPWvHEd5Qk8$jl^BffJDMmrrM0% zL(T&Si9VDoY&a*Juj;w|4K+9^!oxvR{;5IKv#xPVDZ^~&%euFub%*MuRKx4w&2d*( z7vGMuT~gG<)L+2Q(hAcV*lb<#*h#t4W7Sj|=EavHn=2*~cf$Q21OEoHH{M(mw6BrA zRUHu@Jb2u~Aa9~|R+Q-beUM2E5IrlCRzd4(TlqB^O@oq`ix)C9P*Cs{d$lCqR;>!i zWz^&GZrpNDGH932we*Uw#|;mRMN#zmC5y{D+wi9z(jmQ%WxwIGFeNCroum?qC;dSJx*HVyqMql z)H3Ncak%`nBsZ5R6d+_V7A!#xG8iPEL z7J~h9ly^f8&7oB$4EU$@Ox$+yobn+n`GImMLuqA?Wlyt$=@|-W&xzom9jAz>)fCImODsXRF_p1&$F{F>I0sTo~w@a8x4qtGw zh`+XbND^%(gH7#$2Stv2%_fK5<{J`5trK>+v!Cz!$M7`J6-7y>t~&)Ub4?N9DuQUM zQ!SD7X?HpD?2IGFQgJ|J-r3=u*L;0m?JU5cHH;u5ED}m965kp-O+oeVfS5Wr?!Na| zpKjC&K0A!2&0KhyyVfCT3{1h=FYlc!+3XZ$7CdFQ@Vh7vxvxsNOq_DOew~|8;BP3@ zjgI2^zL3?NA-m`(*K16k-QRAFr{CMqgDyup3Eu&$+)&3vNL>5zJ|!0hvI#vQhG=-t zR9u8jfzxlrR)pgRIhdYuRw)rMNS#$(e|H2(9hSq3BJ7;`%IcGQ8!XctybaH_t>o`Y z&uaeaDvlNR-o{->qsf!Uo?f)mlMNGr$#1ooIYeQ%%7^g@Kdz9D8(MxjJ9s5dpKW)$ zP86d5QG?+4<$4`PkX54^Sj>NO5ON5|hVkoRpkU2$Rk0I3(xnU>w9~ohkG{AHnH>q) zKH01yOI~}L;*)Ju4zx7IBkPg**I1|0NTO+h)Q^;rHLQ$6Y|t(9w?iTlQtmUJP>0xG zNU)zaMfqQRO16D*tZ>7g9`m&Pas*Ia`RD{3u{8>V$6#>3{cf5dB@2d^<1vFedm{@k z=K|(P@@^zE!g?eMz?|_VN^bDw9z{hH!bhBl!CKtEPs<%LOu&8|5fwUEM&U7!vs4TB zu83jQb8*7hBO^UL{Myul2OfZ|$As!lFlA6tuI(sfB)vZs`vx(*ecomcz2#EZrA^P; z;|$xu*tmz?d%$jlH<$!Q-8sjRb1YaD(^mgkcUf+YY2yK%+)C3%(YHR9g#^_E=a!l!{PtT3n_FZaLO6 z;Zhd$O?G@H!-wU$_?AG~{Y1=}&Om`!qSC{`%D@+OKnRH#R2U9I7*B2;cn|z93DLcX zPH#^~0>x;ShCJmjwuYl;;LL_qTbySLO3?~^t>*U!$m)gqrN3#d!HklRr>@KT@nU5Tfj0xXLqk* zq8=i%cix&`|L16KL1(;Q{7H6gTgPqArU0QB?6EjF)w%kYI*I!?;@z8#1z#3>CVE>z zp*I7FV3BDxYZa#7Avp^Lhvdk6ZW@fUNglZXRVD$VO*@{ zvRhnK5okaIi=$^_0;GPZEc?R0<7qVD-D4)EZbrX(3+~p}%+O#TiK9LPaJj9Vt;(ox zO?WrjDG#&UCvODlTD9^^-$=En`=i;r+0w~LaGah{3D3OQoJ-0V+}$2qkek-U%(Nu} z%c(T?x16eNK)@mm(6dU;8-sxh3niI1B}~9#;2j!sw`Oa^QtDnF$_LP2A9f}V3T~vQ z74HX@hSZ$baEVqY1_e3s>`cA+m6^}pYb}tNTMIhZo`xOS%nBl$y9B@nUK&d>!3*Jk z@5;vK=gnMy^l9Bl|2J!xt(Dymp6UOyhFxh~*=(_6eP8zqb4IOK9++dlJ3fF|4U7m$ zNjo+LqC`2SFcff)Cmx!%(tp1bXeHRK_n6l)JsE`@(4IYa^`8Di?m&9fU$_bsAEyv2 zr7RUNKRq8jcXW=SuL0~B>}J(!tZKMB1=;T!d^vu^*h6U?x~ZYmSc!$}=C!t_jtbxz zFm6$yh$W-(6g{&Y>4>^E$Khn(dxy}UiCdgn41^bD!d0`zYr>}IKlV@%);T5dGz@FQ z(5lM2M2f=HxZm4tC+8_DDn+MKwQi0zI~3-3%CCPPps@GPdzK>qzZi7 z?q-SH0R0DlDCfaTaB(1Gso}J$_y}OPQ6QilBWU`i4PCbho44!-qQsW4dxq3T7dj$T z^c1kzu889@$)z3sdQt!iA10KH%?WTpsGiFtfAe9Q5Z{*MkJ5LQ%>xo)M8%zg$y}Z! z+OIX}CF0&geY=$FJIlnx3dV_U_5g>NS?A~r%IBE5{6iU5MY^q&sVm<4}3ikpF;uPxn(X;lv~4sHwDDR7y%O9X)i5a5O2 z0+DQ_#uVJnmYu0ooahoiWyPyXDF=DvJiSgZ-{K^t_(TDjr%xt?<5BkIfCOGMH3Jh_a0@#`TuHve<-W&E!0$DDRdb&*JrE?Zi+k?)|fdXW`=(`))i^ z9C&>9WdmDw2(s^JPjGy>y(Kl{B!8u`t-Ls6`yey`uM%^m~@+v_vf`L}=$)n1 z>M8pT>2hx@v-_Dv%B5jhp3!*x*k8~zH5cki$bWSDwCT~@BR<(#YjEA84_I++y+4H6 z_SO1i>HJHc7&5)$NkLc3xd{H|)tlj+MF-A@okEtZE@3(gpUSpBDsB~7bHpVf^9yVn zI;9Yu!7k>@Ji$u;*wQY&MEwkJ+u`|_V-pw0CGIludi?T|HTemBL5KzU=}&AH&F7dC=G%5{ zinMx{4~^hriBbLu{WVLMF92SzcRZwNI$tbQIHHp;i(E<>#u~0e>9Knk*8j|`#tsw0 zEu=M+;A!V24jnpxaK!D;9)z<4j2iAxP8xq9i-qy*6BVmDdIKbc+m(E?Yo>_pej?{3 zo^?@zn8J$-&o73ydL5HT(6?~=WK%@`>$0b+)6-q!O6pY3p9=-dmIsoFmI@KQ9m)#b zY?0$Mwjj-q@A^dR0@#a0xa}|{Zt12i*Qj9PT@91(c;R-_O34a%g;PPsV= zg~D|f+lkx8>94tHyVJ0D;k@c(=-hx1a3Bht(&rIPkM@8txbF3YsMk0GQ^&B+xz zxn8?hweB~>svDt##>VvL!andfxU&V}h@@W8io{;~`gcA!K8G4D?V@k^G0KXtO`S1T zV!t@RLP~!bDY@r0n1A(H)1>VZSgu97)?#v<%-MFDQN@`!z!{{q4i@5hoe0%@bY;=}8X%6MyqB6Z!ioAQcIujG?OrgW@YOTeUYO{Q%O^s|QcNHag(zOk9N2i0*J=u>dBgAZ~I29Rw z*e0JX{(6K(t;%n(&(g`c$fP;z>DZCcht)po+_VRzZxcLSB3hSyk@`_-4Z&6~#@LnF zTE~;@!hZoWI;_0Ozr$zgt%=tu3zt(O3XHV|wUK6-yP?`c(o@hsp}@{S?X=Vtp}&jJ ze~Sq0_b@yz2F?9A+dDRSE`KaMa0jCuuYEt;eFbO8h!iiD*1Y_8K9-6#XMw>RfyYSw zXx!p~WU95&Alo#!v!*sSh4Bp>F6|AA$u{5YjJmYRdeF zD`YrnmI8mLt)2{uh$>X6bj;mzBlVpFg#FTs;}Zf3G4;ft-E3R6Fe!@Ypb_ynRc&Af zuU|}JK~n~l_EtMP1Mv%9yBmhy8rz}@B9t!m79z-{JqQn0^T$XqL^Jei4AiT^>j`q!?9? zr2R@8-9ub-#@=(f}yp^2>An>_|qPX+!9i*`-Zz_!$klq&ytf{1t&Az&U(NJH_=> zN(Rok9xO3olr9uXxcdFNSnZ+95S~W1tn^S^;uQOf+AUJ&UozD3mnEV79pcN1(2m&e zjlw@hJm|4U8=aASPzuq*t0NCzlfUDD*I$&vGUJz6*q-^%h8_g@-H#cdm!&k1Feb(t ztMYhS=u;7deO#74_R)m8JjyS^3HbAouQu7XRUbjCopx=SaaV?6eHoA>)&vXK8sC&e zR3ASIVRN7mW-kT9a8(=gJ5F>>WQ-#wEl7cUNEH9kRhc2cqc{Y;u`4 zlpT%jxd|!E7;>jE1vgGX+VCK|Wj2@u`ruAiqIyorkC{~uImC0=LN1no5R(R8oPj~e z3)(!zuL7NR-RN9BjvcWh30l$1twj;I%7Hp2&G&;1^?=37-@1mfh{nI_tUzO)h5jBZ z=n8JvG_DaNfGquntzBGnn%LZNZ1E6{Iprjt7hV2=X+iK04MN-j!9!@fOAznCG+)r~ zmdM0L)+Q9e%LjYw2Ne#*Mzv>g-(R3vYx)@!_CJKz^o__fUHb|B6&APcSoPCTo5&lL zfFKlo*^Dfm;U`{$ipR<1+boq0BU<*ue6_m5wVb#r6N(!~Jal%5x6d3X1yfOZ@ZdU@ zF)zmpoc+W7v>5%A(fk^T6dCVGSAu_8)btKQT z355ZJZ(4+fL6!fnTo9x|Z>4EN6@6@U`U6b4T z#AJ7(!p$4kasTlL9JIj)Dug~-3Kc_2t-p)(fvTFVb5wYdmeJqmSykX& z{ikP|$%ocQL(a3p_OaVlUJS$1Obs*U#BOnBC#u>%xSl@`YC0!S5WX|L(aT=cvGb9s zmkFoSq=5U9@x&OX1P;fjo3XGCao%mo9GtW7GD^#Q5%}nH9$-5<6nv8W6=EV!vD{8N zB@Z2+%`YIg_@KYupY0(|YN7}# zZ7>+cut+=jmv1u0iHd%+wUTi&zvUT*IuV{HhoYu8x&90Ah4Hz|`PMo1HZ*lw=YW8v zN{Rk~hslX5$rtgpH^7kC;~yg*EAsl@u|Ai0@_wM>*@SsMt6##AIpEdC2kh!4_UBGG zdR$h;n1UnhlHc>V6Vr15pI&RQ#oU3!iQSChK=O0ld_*QD5oS#M=Pk;YUD_TYcx(;7 z!|H4Ptr@%VeTy)W>txurL)hA1C9>6x=&V+JqPX#)AeUJ2wyt6xuGpme8uri|qmGmO zQP9JB3i3d(h6TTm+>GEYleMlpffxLqbf@0q!REYuQpeIAXoB=8YI7Ygg4rKB3o)NW zq`m`N<#jChNX{zGADEf&wr+sWZod**%ywVoeU|F|AzfJ$`?MgqlgrHWkE>eD@_7r()J&hk!n8!YnHK2WN%`urIgh6@&>=ZB8o5l`Z{xas zn^N@DXX{koE$0sd9Bw_)RjcU5VI1mIkfxLyVQa_5Ft;7{#VM629Zd-+0`m;XwKAHD z-BnIhgFPH8SU5tX7)Tu7U`u>(klZK|p+^dc~7X@UaRDZ-ARRi7nr&<0<3tg?7W4y@6PRyv$%12U; zUxEHt!6~`Ff^samOzVzNM-mlq;`dx3*=OGk&H$a-1;KNkg>7(R5xM5anBLE2C2FcW zJgRLaoEU&#4s`Ix}o-onNKXe=0t;sBwjFMl^|72QtpdZ~F zn4}*p+{589cW+X6*Z%#cF3&quRkk$JF)n58lJ2JDA!d(kmC^N;I_GV%^rh10VW=t5 zgG@FqZaIfTOOChyY2)NjYzEB+69-y0)`p^bsX+b>r<(szo*)SV?q%Xcfb9hWoc$)JDxXzBkI0*!~k?mKM4@e2 zykcNNW$D6fo9Sa-5O1`Wj-8k|6Ec>TRNS@JU(uK+6WQ1w>|Qs@n}jamw!g4u{o>EL zOS(2Pku@x;Eo<4Ve(co|RsfE(2{2mLb@KoTJVYGp6&%(3<%h>Z34#a;ic5!~KHVX- z#a`XiCb#NkK}- z=WJi2o{g13Rp~&A< z6uR(P{w5jwx1U z!fGA`wmnpB)5b!#6NvWkW`k)v$nrA5e(Q`I{Ce(rnJl5WT-I(0-)X{qLmB2QWGK>S zz$k;s;oJ4@ojRgg**C0ZxN$TqGg7UsV)jtOe1J_qzd#vTotVi&@v6z2=Dwc zF_*E$tQyZ88EQ9dFH_A{drAo!GTnwYqBEoxp^kdQSm|_`I^l=-&ax3no{v%25!BCt z*|Y!BQ1q3OnO=E9`N#P(ewXonRe6SyRfyV0r)h7?gz&=1dO-(jMou7m@KBG6yGAFu zzS4f|-7J(>9v{3vA9_)WAQ&KijGdM|V_$Pn)jUFrxf~&U)eD5^J>0-f3f3Z|hX@k^ z%wh}=LTI5MafkB7CxkM3aSMDH5=?CtvV1GwA&p`71Ha2HNLYN3yLP=M_7d;o{FnG4 zz)k9P=-kFQ^xT3JTb(EUnjT;yjnRrmj>VF?i<=KnKG>?ya{lxemxS6ER_k+Xm{<;bI{@q@#PeD38zCYJjALy&<_Wp5X=7)ow z>OUO*$>dEoXK$)flXSCA8#LpS@PB81ocuJ0e5J{gebm)u#gi&8Pfu#c*zeQ3Y^<|f zlB+A+N;X$LJ;jISY=m|nivH2B;io$)4rK@3m#cd&-1UCTNpB44E?dNV4&c{v`JT#d zM{(L;(EXig+m&Da$oj^DF3^|4(3CkwmRpG19@bS(MJ(zbyeX^5Qd@W>=F@emMKi%p zw-j83wKbIOI4%I>(M0qQnzul1b;%vU^>HOT!QYo3yP@VAsfb>5*&6B8(mu+A0t8HO zx4|peEws9}Bwqa-%L_=Q>4lcH*zYI{GUL1<{rpOVmyil{D%r1;IXeMue@Jq5h~%3D z+Yh1O*iN^xU}{IR7?--lzsa+n)K+K{YbPzu@NbKZ-g3fxBqR~J{~j4^(A|3L z_@_u3#E^fbZZQIzz4mQ@EMKt$_mIzU#|vGiE9!LGBd zxX-kcB)gIdLm~}$fuVv;%bXjBWLtGeVSq-)IE*s~`_Y`;c9mvsuOA5*_0|Bc$OB`f zk+gkViO>gV6LwPtWmk4Sa-gN6`=-TFe1o=t1bYHfk~j=VhDi?4XF8GQD~$|Im>)fO z(5%hSeO5cFDp2iDD3v#$pz+GVM$TNg&cb{P1Nt$9kC0ao)7xPEQg`8zZmVh5lt+5g z7OSfM1(clvB>Zkci<7EgZ3qZ9!IUXr^y3R&F*bxFJeA!XPXa^)GTI#o6MR2OjfM6% zBwNYfOlRtTYM`))u1oHctGtD*!jlwf71|nO2(w*SJc_BkB;z5*-2oGQ2iO?JcplK( zBy4<)%L2t5m0g4!_C-o(XC0k^zO|6j>-SN`EVuFbjh1)Y3|Hwxk8u4s;tWw&#A(76 z<{{X8*#-`JF9?$|f|dP~=8Ps5CgAKS;{saEm>B~9g#4-`xkH&U=Q=#v1vDbeco`Z! zjb}bx3*lNe@bJ)YkWp7eYyo0!o4!zYnSg1|M#||iEU!TqEp(M5UFTf zsHHgL!7=5a^?;*_M6 zq;$wV#S8}@(a{07aoI?9Q+JX)>uNd|>n#eWM+M)DlraGLz6*l2re&E#+`%P(M^7Vr zBE!o%#-PQmHZFQInmk@GH-(f@y%4`F$Yk|v4nx-}P z0eZ*&OwC(BVn>KwVR<6ATFptUy5He3r@4jyV@<5Dz0S~kF%4QFaS16G5Qgltz9clp z84rG(m9O(cK2f-*Jv%XZ%T{jkZJV{m6Z1Fbgyb=b>_cOm&-|VxQnzhO?2+6bUcvUp z6RYhktcuZL=RK5di7P~;9B?%0PwwZB7Gh(CtkbO;#{cPFW`3xD6GW4# zQ)V`7H@`0OlHf^1%ru%7xcD|G%8}MI3tDFrvO-?37Q9)otLl-E6z=RX^cyIom{*S+ zqs|)kov)X{{qmML@a=gie|7G;a4)%Z$F>vKsE{BMy$4eoXK)m2Hp6Qmsn$eJg#Xt? zf1A%@pBRov%WdV$b`XMs6UR>n>oe27l!>fh2+Go%=RhDkU2k0QdTPBfb(}b{qqNS? zn`c!&`7Y1$UusHRLqU{U9CErVhu_Oyh~4H5fGOwHZjhow3w~G z$)lqoni$liEh|+!%>xj;sHUh@?&j_H{@S(6f#Lzyo1_cmRcTzSco7xC>$3zKVGzi1Hk>s^v zRn9FjL$(|87V0t>NZ*t;o!yLb)8COHzNt)Yny`yado!h-ro~3iS|mQPc|IO7a4{t{ zTxJyJ@WUWX`XljBX9)o^D{T>Z#AM1!;d8qgT#zukgKQC|N9HAx2z8H?06BAA-W28< zPy=|v&zv0o)|QaJwk-k%wDQSuh7mx3k+GVJ<2nP27?1P^QOBpO!EUFPOMU0;t;7ai z!J&$kUPjFBd!UxtA&4l?0OTTE3rz1+)l!-QWY%!QW)zYqXXbhx4|Z7!LSAoe_2(g6Pg3T8iRcP zvbw5Dcb;SS(XyBb5FlCW`Vz&C2l94XrX7hIW(FaE>zn1eYYhHDR;j)cG^dg%xZH+A zM*KEtVp!VwTfej8m{_W%_nZK1EoVz*t=Dfuw}MZ*+>LU9>yO=;Igyo9p#|-Sb_LBx z(h5jA+Cc*~?%e{2rf4QaI)n610x08w)nLBAYkH-SOy4HrsHd5TdweX>kSTDnH&(Ev zp^ua6)?3~g$*5p&R2L6d1O5|29};<+uc_|pSl^($2g=5rwEW0=@^548ETeN34IA#K z7zu;`&)gY&+wn(6`IlkOpRxi?0 zmOi9Bl&56=X`|B~?b2Y8mO9)SQ9{&|EvC57gXXZJ#etsL2jRrC(tpl* zJfItYPIye)?0Fd3wcV4Z9Eb_oQOHMpz8IZ7{YwN)HYC1)ajsotd`wv4GwNUH3 zqoJep$eu|DX4pNm^*7a{c6z`W0D~R{I7>G7h!5^gc3CwN0X6vG0mil!zk~b$RpGE^ zpz)_rRa{iHNiz|;9zO$CYko5HDotMv@ubJ#5k?xR=_SqN^zS@M&BwY+Re9Pj?l3=W zC_u2`t$N!})LLv+3d-QI_CNqLob3ze`{PWeOBrOLz3RR%?rdCWm$pjX624{h2knT@ zkwkNzqNT&g>Ml~8k}(pqKjWitprl zRfa*LB_UP$u1#2X()Mk5m@`;!drdYFCd1oL`9#i;6OsN~G`>!4?QPvpo}}%fHr{h5 zVMtW?q=hPrOjRjpf<=Q%Toj)UX8r6Qu?~al_rfs->DFw@(&%&1F1&Rd4?CJ^Zz<%- zE~qQgj# z^K69sz*MVZf_A+Xn+?y`J_7c8^wCzh{bZ;V#^v$*I`h?cdy8vWMbQZwTiYEL%iRz7`+o*@);Gzp<<*|EL_r{-zw>v}ury8{aFdxVAiV zj49*GyOZ;gsk0f?#v+=2Bx$Yw`bXLPl7kWAg^`|G`Cl$ttv8P?PV*<=O%P8&tLES zYD)OfYZ5X(n$e)mw?5~e$sc!`JxD-lj_wD4g$LcPYuT0V2hW~+MG8-`0e+Fknyd5 zTErpH&Im=<5Wwg=aJ4q*XzP#7dR8A%kYS5FvA5d=is+!=P6qrS^yeGbDhFHJAn_1W z*;FlMv=L=E(mu_BXL&rs^G`R$U&pI(H5$MDSma9tYCWZ#3K5@PpUpFIJ@9~u3Qg83i*rvW~Jwz2~2NCcgY%I zHWSUB4f_6e_oI)0>_B#K2hh>UC!wT~b-l^?LZibZGAa+oV>ZRYPN?(|K_1mAp z#x;Wgmw|m8D8%#0$EO++tFnXr>_9x8g((<+?^Erm{}pUxJNaBWL(kn1FAKn=77-m| zU7p=h#n}POw|Yf_y!leQHVh*g@v(Lz5KXupF0hRfbnaj=KDqJLxqY>ia;#*fER!y` zHw1*wWq|BV(=)7@uXdd}Mg!Y=9d*orBa1QpLrKE-^CxnDoq1KP$K%PG3c+#1AQm_r z3XB6~?d_IoYYH?Ydetc>kT_lf&o{T2xUOEcH>1&(5_?Y$YBeS>N!q$>tnv^i)1-Ya z{vB0F!ND_lPSNq>C^ECGq>8>}gO9H1B~DEa-QBQDs_8zYH1u*NO7QNHBVXwI&71_hbeli^L*nQ4lqqlG6^w6lR_BQ@fpnzZeq! zj0GZU%W*b!NI9ljPQbmYvH9%@>sZoB=^Fj<_i1)_%zCRPrWN8ST@H65A?WSw_UJjP zF3vInyJdtkCFvg^voFtm+0oZdk?yjxa|@$MLwRVkMHEz4G^8Dbu>BK~q{gD~dSFWX zd3c>muHo^Gq9MD;qjy_rQisZfF>aAdXg;O{i@6GFNl{Fd65ZlD1TiMUP=71}H9EC{ z$^UkFfLRe2?;xV-7UIcx;!y;9sj-fp-_0`|XDKUAHvwo*AT`h5JZ~YD)F{FUR7yiu zNbLujN_I>x4uD13b2$Ys!C&zKDQ*>pF^s2`L!@ zmaXKhcFYyw=4ja*g$6DX0i>Hci8_Y{BUgxMQ~VJPh@v@Pn;8~+77}c7FB_7r>Oi8t zE}8&}q+6&OtGKlwJ7g_x#Af}R-QOpjalBH`)?ITCwubD*7Q?NsP;N1cldTpp4c~_zEOhfB$ z#JaF)rg=r&Z){6 ze$l|ooy+zy*#Xwgn5?y{X0q!T8-sGTL@!U)z)@(X+p=8}+)qyFiCpatXm}5I(uhGW z9vTuM%Rv}EhP^~DPWos|ElAcu2ly|Q!(0`n%Y$~$M%=y7w;wy@Emu+477G>;v4QM1 z#$$QD{Bz!G%l00r$v;Z+0FkEPZJe4?;)$s)3W%Cy;*agLqMhe={WD;le&sK6*SfQ< zFQOr37=LTXL-ijGR0q<})?wB`?xYjLO5@eK9y~um4tTiApW5?bz`Qa+HnktFt}B5* z-wn)AEl``pj(jx!aJ*i!UuLc@FV9@j#s9NRTO|GzFZJ75iVOt+!2G{z%FY(Qp!?sp zQltNucmLm{d``w2EX-x;fj= z8*c8I@J#@3O+B2=m%GlLt#!TtNkuzSeO?L(G(sF)Jm1LOOE0$E+`hxl`~A=RLn}8w zG;eT*-i&Vy-vekA)HpL~lqho?Q=syPv(pEQq(<4$y!3G+SSxL(dY~RU=!#tt8{&-n z(5PK%s1zT>ewjj($^=9m+X)=YwKR`ui*MaqAq{S89Y#`UMC)eMSk0L{D9z!Q>H(K= z6xd2fW^?&VzER8W_b|Zq{tn7R;tA<`FX6}a%g%*lA(Ky8(D4Z*+77XslyCd<*q21t zjcl5W&y7FFHnJ?aS+ujAQq;`hj)U5}?ACs#p4&v`s-O?xN%l?EF9!H?w#8DKb^h+Q zl^fHY40i#1J1;f5brOw2L0$rNZxNCM%tfd_p^y%RQ#3V@vXr2b@ji{iXi@xzTG%pS zLDvpTEtX~`N>l?rnuy4OH(7VYx88iL^jeOQm?f0xI;EBdYA`BOjem02C6aQp&KX>a zY-6?-%CA>D^fMYHbBHQKX)4mzB1Os&FjVlF{{gKjkOec?88-+eUie&236bAwfD6K$ z_irUgq%BAxRyKK{3C0g@(bD1%oMOyC1x8wzrGq&w9&eux-2gf>OLYLJDr$QYpA*8{ zLU2MwPpw(FzTFR(pnPW-jMd=DI5cexlPTJk1oZs0nKw2br{UrHrj(`qJ*K#umu;E>ev}xc!e8~J9`UMPSqWkr8geYU_0$V*MS&4sP)jX{Yn+B zAP&FN1070=JmNhbejr%riFcU2RCE6zU~|IID6hpjUIySJZT*OTM}92=s(scym;{pM zN>Ei?p)B^cS{veRzB_;$#b1OBuWkP6+vRUqy>Xzs9F46QEvQ^sIZatpqsX@`* zc_#sBuDD6yh`W)AWiRFZ$IY$b*7f}PRIoh#&o$9ce+lBW-LduLq;)V@9v% zcwzKp7IqR;+c-wz`y?gnkadg&jD+x(24@XiPHf96Y|G0vyyzf-dIcl__}($3bF4!;HzyfNKSzkcFk zFFeDd=%Hd!rP0~TaDhceQfF#HT`SUGkYu6HSJ*i3viRULkcApk?xiQY1PhEMACGOp z=VVfe!iuf3TgU2(>X6WaY)85J5(hSk(QxMCyn9*wl4+lP(Td%)Oe2qfszKj~h?h;o zVrKATN9LJEF@bvtAig2~0aX!23w*;vBs0bP4X&UI1a1*Urmw;O=UZXea(rL@E;v=c zB&7eH6z%?B{3Da!e)j+WFiy9cwEY1aLeGUdOpdUv02-Qh6F4wkLhR;%Kq3v2u`&S! zh**)$025`xqgU+J_Y0hU5uW3Mvg_c`I4yr7Yn?=Zy z15pxlkE$f;FyLc*Km4ut>SxYC6e?FJKV^9mDrGfmst9!EfNpW-Zi$?sv)w5=5puX= zfp$^pZl>b;{(liqg(@#YDYDW<4)Wp1@W#~dl@?RAy8#@CI@VH)o-XG7^^wq>lox~^ zzO|mD9lAYuwnzZvkDD~CGlB+kAN&Ox2;DZ!F-*NhqES`>vM@SQ-5TM;(C44u7=?X;QStCo;6}|68-;4|5wxlp7tSrcXVjMN|~I+9&%w=ET|Tu$tz4!tQ628hm`R7y`PP%G)`!ZS zBpLXpaM5?N*h*k1mWQZlp~z97Jh$)Lu7rZZ?Y9cZbucj_)pE)Sofh>DQvZZK^rONczuK02L+Idg1X%^2)U^+6 z>_UHxMdOSMv}mR^k5KFOBXw=Vxhm3ojXUT|2&Fav&S`7V1||pxWSy6W?%?)kj8#QA zA!6XbNdFIAX~N=mPtA%Xhj?FSY;Fkf8lC{&l1E|anlLQ~RDtT_*@w01L-AFy$?=iD3j=Ou2fq{X_bHAPIHF5DRrh!vx%s*t#0ZYp=Hx04<|4!C?vXYu{|cRiZtrfNlT~pU2DO6S=u3mMme>M) z0GZMj25EINE&G*G?Ud{Tr2MZT5f8U#ht*r-q8?H$7SWwvc9!xJ$*VjTRCN_C ztau=&`c4kUw##3aIj284zueqXylWRc%eU_64`VjT|LMom45Xy8_1it4iU9z?_22Ro z{r?Y_O3O~i{Ck<0x-m0Qo13t(vKbqhuo%*rSUS7VTH2Y})60sg2#N@*2-aw9Ds4(& z^{gsCCPBj)j~g&|s_ISA0hf#@gJ>8e)kw(-N?UF6N?4V4n%=79S-BzJ#S3-yqK8m3BBTw%X@qL#X zt?Yc+C0+aftdhvRcsl4Fajrf9dMi8IVWp{OrW)2N^`_i$m? z;j7X~@gd@}HMg(?+qn~6HANMR9BndSRIMFo%a#vlq8`}T8 z&JLLc;hHd$o-Xo7XUdS03x?ymai(U{NIz?3nNYECQ;dEu5@|Y+AuD1a=JBOzVcguo zuaX<`^UQdd4k7njX9AWr&flHAJy`}wBgB`^0lO(eP1n$nU3%G7kF5ikuiB`T6rsGT zu8}$gFwXPLM()L+Cqrodols?XHU#B--`f^Isw;|zEK^6%4^BIH-mw%mo ziePaY)E`G8#FNr5ELV{vkAI!-VeD>rQo@CM&)Y2_rN<*iI&E{gotafHFtCLf2>5dR zi+%rUi71-b1d_@$JqMI3WZtmenmOT_Rg-6>H8?Evc>h2cvx-( zM^xU!Fs!ghC~xERZOVejNI$I501tU#y`uLw56W0~7-PF=oNn3G%!m>xrJeTyvv@B4 zvT&D_6Yg1$>S!9VGMU?R;6E4rdTJBNF<*d^A}Lw$;)#A+&|c*!ZbG#zf|?}Vh~mU~ z{vK*Ylj>E~;@QJ7r_!TGqHQ@UXsX@Jn)lMKoU<56vL=2kVo_-6T3Pn^qy%S{V2VGHnS;}L&EN>%}NHXnk6bY+e90o+n6Pp6?)-hp_fs= zz*%{&Oj9*dDN-8>a+-$Fw_|Hq)|*zXp}DM|WcF@rm}b3{R;s!eN{RF!`VG2)4#Ha3 z#RI>Az|U+0Mm|8Im4-&L$GCv|9L{dxX#ggS-nr^qi5Eq{@hXiGcBzr-OiQjj*^*?t zvU%mZW9XM@G>WdCj;=1wUL5^E(bmKB=Dv2xLaZfs4r<_dW@zYvf#aZNz=~t1p=yVN z=kiAAQtC9~K}J6808CUphaC=R48Hiayf|wg3chvLV}9)$mAQO@FX0{6hGjOIH>o4Bl7!eH9a%5Tuh%}ri>anM*ipS@U>n+w>PDK#jS_iqtQ z#~wPY&<#|4o@oP)$jyaQ$moGnx=bTY)CK!@&+epbg_%M_`bGrd=C1PWYCQNmTy7Oq z!RWo?@YT>9c42Pcvc96DO0x$_WQJ1wo?3VC_Lex^T10ikWY2y z3((?jhBlX}KX&T)a4yEWf+{1F?A(M~9(H!pMw59hXjg1DkVFz7EtdQeFz_`B0N3~? zjzaib)sU141M4|?z5kAiQn)1NNOJ^`X`ApGa;2oCltwv(O|(#qdx_{^%?=RVeT}n{ zjNatB66S6wl8pb1Q;vZ!_bw*<22;%_?3F`+F^Gq&G`It7*f1n0yHjo&a2%Qm>8e; z7B}|^I(zw2+%_bMUr?RmC1n9M8>?l#=sac0li(j>IbTx2d9pRQ7UG=^yxY^K=EEb| zx0r#0QdFUmv7$0pk_NiiQkNpQgI$d&+$xo@x-PNvLkaf?frBXp3sK|Rzh5WE8-`*8 z&C)%B~mZN_d8+eQch|(^SYD9iCuC>OK_j2Sdiy*`-dG4r+t2wP7I7*&1p~Pi>Kj} zsNS^~C`fI>6-p186qeYgtWZ;tvPf%acsNFm!0k^S(NUQh#x=A7<6xHYu0T}MX*Gq= zwH<`bncH(sTijkPs*tKS?-2NP^lx~!v!`sW*I^NeiST<<>T=<)swfcT%{j{dj3n%Zx&r2aeD|4&Jix`e2x4D_so z?DRaWG%eN4)NG>?!xHnZll-(atrX2TeS^}N1dZg(*!Yw*_;2GK&{5F+gR~28v@5Vw zw30JZay5!nl$3J25Yl1|3KUhW>%;wnk}L89RXf0c{uY|1A-J&qhRFRpg#J&Vn%cQI zdFngZTiUre)46!KM9a<0QqRmv#?s5tP0&zK#vMuhSIo1s%5|z^va<3tloPXy!BA(U zCno1dWM;>jfe>{6BdCOBwN;JZYDK@v|1~HZOJh?zXH#c-8A)MLIc3o+9c!n9vBYov z^0ACq4QXVAr_-|*?F$DCqC&JX$|UFnT56oQJR${>dLTe7_=@%3OK%VOI`lQ*%MraH z^7SdI002paeQTV17St^yNdQgFzx2KDRMA_B0fMI=d&%f9{VzC(LqM-4``$!hBp}xx z47WZ$$zZSCVGNj&r{ja2K@5o3q{dGUOtD>i6oJGi7<4mq+g5Y$CTude=eAXgEswS9 zcFP!zAs54vhfj+`(7?Na(3Bf@KoNs1P9dKqH=sOOMasV>ZM);rVhCdQ_8DY0I33%4 z&K#I?0pdVpL|ERCUO>qdK94#+i7_5Qy%Vg?1AR}Yg&Fh0#)$<7C#)Inkk;&DzK%lZ zH_`_nM2QZMi7@wmuM+7EP|N^Upf;?q^xlkY1l#d!QTryj$G-hYxcT13h|6ok4bU1Y z6|hd|bxw9sc3&`h0hKtJj>6=)(FA**;JHSdlX&5b@*;IEa)O9iAn zZwR_DuNn#LlfppVbasihH>8a1fpOH{hA77U%BRp+lytBXh;6SG1%iQbF^(`nR@o$A z=)68CbeR~ttjk!A91?kd78@Bs?nF3PB5pb3!_-6u$rXSp<4jRCUPhYw?eyJmTqXg0 zw(2)dd*=hHJ}R?LYruN!U1=XGS8cV~KF-LuMo#>ms68d^w}S7&<4o7snrIp9xn+@U z3mv8HU1bMX`fQf7yTjJs3jnhWYTVGXj%(J3H#=-$JJ`jBjpus8vA)X*=nyVpyA zG`qLjTc_aJb=xf2vBN}r(ybxY!MG~)Sz!}M4Qa8=Xmo4$)YvlG^x0L! z%`x$@Ov8&+*(I>sHaMAYj4(c@S}wwtQ9a9zGI=*lEQz0{dTC5=nLmK7zQCP zg1~`6W<8AyK|h)&POFo_2tjV4N3fe?!=nE(kY0klbmFw)T8!gt+>CD{9!iJYTmDD9 zzz(?fF*jATgxJHW2giS6-!JD8@N_D6SWF+}hXspyF-69)D;h#RpFhB`56D$XXaE7J zJQE=lqj;RAk&)yvkd8GRJy$i zTQg}8;T*b6Rt(0`R5*N{WC$eUsLC;Q4uS8b7V~k4#Ilz<| zut58mtevDVPyr9TA!+juU);Lfo->&u(43*lD#1tmX+k<|gz1##fS=4DJ2`DUt*wXI zFl22a{%+l%ZFIFYq(s6~-|O5KzM<6?yyaO^H|&f-+knY;PBB`(EkEe(fQi$H1%f8M zOa+w72K?8&+_S6^W_yJrtR2=MBn#`z8`L`I9T|`*Xo2B{Y>OB*Np;L_A0P-f6BIt_NNb8nkphVsR2dvY9bMm)gUaIgM+;&9LryYpT7Vp(bi{=DK%5 z@tEkOwy?$tH_3KVH;bUr!W>wt*~qhp&Pnt6Mem=?@$%C9Nxm61OsB|}m5ANAmZ~ti zVpK29T4PXuNs0zd->C6S6kTJJ|lz ziPlLs0eT#EJnV7AK%BI4q4jMn~*o4F5u&k)! zW{wNuXGUw%tuVke(3OiJlripxCJX_z_G~1fD815SZCf6iQVg-F23=v37~7n{6IHj> zltA-ePXtWSwHYl4)rc!`XifGg7akz=KTA|xQcR4aJ`x&hkMD^6{}e}U@D@8X zKWF{>-#%V?e;FEna9@;b1A-D^WH+M)`l{QXZf*~*B>%>|1I)16h)0-O9Y&-6N3NL}-?n*X*o7}|^Z@`(}i!oq`7(u(iT=T`@9ZEj-C+kI4m{<(GS-Zgnk69_A zW=KF`0s^xM35B+45Vl<$UQM*%z$&XO2^%aTobKhphC$w!LZqXhbx9g@A?R)3CRsDW zC~pZZxtWX>HW5d6(IBs_CK@>)h1DamZDu@bH5@nY9#WneV zPi_%s|Dl_xB;h&}v7*9oMCT%=H=?7OiFY0wAe66fXljyTIAz3q9MgYMEHK5wVY-#v znvgD$Nr!eJVM?XcK=XJ2GmkEmr!LBi33RTXWjjv`v?$eIKg1!#GYm(PGZO^rh$F#^6dVS>A821ZqPaNeQrzbFxeXiq1@Mu%>?l#I%?vN zgu`Q`G0Vb_W!Lrj8>G-WXW@_{Kf0li|aiG-Y+!W+k1svyDPmuXiL^lRRC<%+0p$Hdi9sx+J0rM;vz>->H?THzph#l>-?TBVr48j){k%-+FJhDLB15oA`ggC#VT z(l{6k)bz&0uGF_8D@FQWWz+?|e=E}@o=i>Uxw=h{@f-i9(;#|dVz*SIJ`MF>vNu;{ zzc1s`)5Q4a%lYSx@=qQV0kdwXO$fMdMJ6~dXv^9Ugmj=7P~1|fFb5TB%o1Vx$@pdz zy8-3}Ge+N#4ho1;cn~4=Z#DMleYKFrDMuA2aN`WbBVUvL&`cQHfE+{SI^)w77D+Xzyjdr`f1iE0bCB5&{>6hMJe3HPx-#QP z$=tBS8#IRf!zthQ%5?wEq4fC1_<1j%7H>@rrAQ9tD9e~5G}PCi=S@{@M!hEC< zYnr1_8Dcf$nl%P(T+xEwu}MRKPT7yu39sznLC0ZazIQ8aHqEL=x;kLt5hW&H-=>_^ zIceF6>Z+>+(=8bO$}#ZG6;ZqyRa{x?jVw;-nT|U&Pdhx%I%8T*8z)2y*3E|;mv>_V_bI!vt|Pp2NYW!)>mHoq|S zFZen?nQ+z3L)0S9MIPw~?B|aDA~>FZC>ea%r$fZ$?~UFk44?g|ZNKn0l1ncJ*vNXx z6Id5MbHD;Bj9bnzVWpVidH4~4e~0OlK~HNLfA=Ii0Keski6bwjYgf93juY~gQPiZ-jHAkUqY zaQcpa60nLFr*^|}*8PH6)!Vsf?QUx}=(DbXH;3bWMGlAdeu(Hco}4N^;bat3%f+P` zk&DGV$nfeRM7;i$2EJ+k^CL##9kX0Bf}Z5UsHvGb-G{*O4D(-4j(d5?o@*w|D|Q_+ z<1Qf2zJG($=og3|_9VRjU!NM^qh#b*$OiKXkS>K0w;c9swKs8Y+$*xQ)P9Wx;=6?`^8q*Pd zjsSChxBnG+kqR)Y7EK~4RE0)cB*o*vEZf`&B>>hm{t7k1FMHEndKCF%ugCue} zGfpxjS>9xrRk)Q{Ch;Ji{Ir2bKGG(Uqq zaJ299)>Jr_PgI=G-rS9?WH`hd3&=fbB>UmkgS@~yQH~0ktxl-H@Hn8C;Fb?s>(20+ zQhWK$t>@2jqBL^F{k&ZdjOeS@Ou0ii_uwA*LAV5H5v(MPS`4r==qxVBm%k6>rEu*T z;W|_Peg#~IM&Fn18JG&H{`u!X5~{wLU==aQT!Ykl9%p>eOFs_m$<#PMJ#^7E$9)rq?v?ycrGVVn#)&Mi`U`;S`iR_eCQgZq5?_) zD%`sSp*I5Ive@r-`76uVwSTcRD_~k1g|=&K9CQt*xFd)I#7n@TwGX)tNKNc@z~2G{ zmu~9zMw7k^aOHe>8Ya;-=z_3nd!kMoo0#{ayioZKEN^h6Lt`bJEn6>Y9H8_B7LUB^>g&a z1$ksk=*gRzeG6@I3eZ$Q#yRRusPee7 zRfK)n?A9}!-EUi-?KT~TKri4h8eUFCuR4d1m8b8)wAE&%b%p3R7e5fwQ1aV*IU=8r z2fUcBK$jP}&CzQg9WH=F3=VJVOmSgEPd{6BNBcffuaKFnJ}}WyZE3@yI^Ep0c$n8C zhY~iRuHbmV-@;AZb1Gp!+#EZ4V7KmJGh!7-Q(dI!uq-Umc14B&es=aDHf)%{^oUxszr^3vBe+f)hyp#rXJ1^_*l!Wz6RXv zyV|YWkYT(xdAz8FlV%*)q?h|hlm1DqygeF5+|O|g)HIt&)~*kt`?b_L1cxH)=iQxu zih4P`?_$;=s>y>Qknl&nr@V}FToNy?ifB4g+%RTAVeWM)?e;`_Z%ydESv8=+h;PCC z9FltwH4}7i^u1s+xe`4IkNd~%>dYg>l{k>5%?7{SUmLyx->QSXY~Lp%{0vGJBait2 z%OsgDF0T?CsJF6o#@FgENt*lO=I;%8)Jb7|rqA!@)g@a!wFRsdJ&RV8Nt;T6l`4WJER% zcNadY0q^eINo1#bE7`pMDv1n&r9<*Xi9Q_6u?9BYh!gW+w%QFF$+Qh=L_3UzP5?#;JcNV|>6V>Bewe#3YNqJusN=># zQ&{@Kxpwmf&1eHBjkKYKMh_BNXUr#(T(%D)?j!5qFILjFB;_{w;xo(_kVa2je!7YF z(cyNW^bZ^RD(w`#E}CU&zJfKq7~6ux%_m)qWAE$}>G!!lH+|O};XDJ)fs?UusU|Wd#Ah^1QLCq&6oD|>7DcNVbkFu0qPPxn{a z$Va=5FY(q^+|y>Pdk+AjD2i(JI3vy)aoU6ESq0j0sW^oYny&dS*{3qlA4cW3|EIGv zkB74R0ys%_B1$48q_QOxNh!)+C`y^e7!1ZRW8X)#5lNBk8QF;x$zHOQELpS5Qr2wQ z`Cawq{f!<|Z%-ee>96lO_uO;Ox%b@Xxpyo@#NA(1?d#UYJ;w{H?>6{|a@<0Rou5v< z%6I{MuzRymw8ln6-QGjYM!xV{_QU7j+5<-^`)E0{JK1pX=+@l&nLrxDYvPDAcQa12 zP2K(aQ6*z$Sc-K`Q0p9M@13B2anK2D|Yx;SKtQ+#&D@bLtcbaQN!=aHC1 zFCRN5+c4)h^dhpyA!M@9niez1U%btY*EmNCW%sx;bc@~#hNg$)N&J#&`i_FfOa-Uh zuHUXrF>{;$e9(3=l{Y=4g!f|3;Vlnn)Fb#euv+_T6cv^7G(FSjt-SC7Ztz5kK@r~M zH{7u&C*QK8QL~tNGG5w8za&rFY2a@6vp~(@;>hO5CPfv+aoO>O_aqo!r}{228q{~_ zE^ktw+KTNtE%Yw;OsIEYdm*hjC*DYJhwC|VIz+ly#^{G!;jmVpjRmKdMO&9Bp0_TH zGH3-M2c43Ql}ilx0}=(;{Lh6nGx6UOSa#T;e8-GX7gCebdbWbI`IZvu;rdMdyX{jG$77^!Uz*=K+AwZGm4{Qv0xwN# zOwcsXHiX2Tozwq>(X0$HqFM4}TuMngoglSm275AMd*{p4CJy9?eL8n?zc5AAR}Ji} zvJP4yhq_IizP;D8rX<W%~#CebbgbsZc)FYXR4lMtl_De$mUv%d^@7g%bHma?&;oT{RcYgI8{ZyRIvbi(wCc+=1q~R0FU%t6GZaeOqB6Hz% znYwv(bxPcU0s|f1o1;uKZ&0Ptw$+oL_6N-r8)|=Ee8&6Ivih+pAt&^agn1E0!K1Ny zhk+bxk6Hj%r+uhhuzE+6qmJCEcl;@SnQ7>>$uu-esOV2FH!EF3J}vj;-V%w~;y>M2l@gt;UH;Bxt}%!E%6;qy=FsbiEq)7aE|(?w zqiKz(rN`#(+qoiqYoAbaHcr3N{J?QqbC0IFRnS9UOPpczgru~!=A}CZ1_yf&sxPt( zNi?AJ>vt`6`8__j`F-S+15Z!wgod2zpyPQ5SHqogd<4(2u?Im}+V3Pa4^h=i!uZfh zexD7pa!=L09JRaZoHo$hlkY@(rycg5;ZTHb6s63S1D78*gwH!ka*Hux$EC>^tG*3f z?U|l*Fw+1ZT`tU7)!xu-no~LvoMrsD>~&rDhWLG$4w~muKlh@?ykHx>A;BA!*ZGxUh9g)e*{4Jg57!*9*~C#`u)36${hVk()4Z^Y?iv zrs+&$p3%Dz6gSUbml~2L&pSIjA05l#Qd)XS+xKi|$emL&VUb03G3VK29N#(FSby5u z7WSlET267{slvyp;m+yN5M?{wUXhKx6Ta;?4)i5%zAuC;W8U^w@`i}zTf>Qj^!Gdt z$I|nz7I8iFKVx}RKr%A9>};lf_*PZ83+hJZ`}ldXVuyhtvyrQNKbk)9L(t$<>d_IW zUFy~Has1rMuV#;W3>zrUp&ly6Z_TK<@iLSC>g6XPZZ{GS8!=U18=*z<@MB;Xue@M9 zXn(9cZ&q>pri7PqHit5qPUX4G#q@OEy+B~fH=u`6a>A}gEgWrgRM{SWqgYzA!`wEX z4gv@;*%h4p7`_L6S&c)k;mD|dl8^=4w$^g_4rM#7 z&_Zj|^z!l?%z;fuWENjd*TbXA)XHiaVDqIZbh`5ioWm3wMi~fh9!s|?j&zJBo@>8{ z$5iZ(%zAUQfox#ObnKMRM}yjPb0G;v+tTDY;^aG*tEmcNgW2#0bj%LWxr8^<>g|h8 zx`hZ95xHmDogng6O{_XmR=kzGnaeHjg9|HHKjuZ4o(p^Rhr?!^#drni>NS-3 ztN5cAsYLb%n0oj(HsBPRBJR2h&NH|(K3*bsM%nb_Z9&aVxi%f>6%xMqBI)wpk3;M} zg_VaR<0#aI+bn0fmge!QJ#(X${EER+@5x5oCZ5c&p&}$Zcm#(c_}^T}h)Q03I*s|{ z^R$;jfydp!uQsivdhFH{C;3{(o23UzJISRsvj_z+j+3{)`;5bCv+u}m)9xA9YCGxM zY5AdahI?n<@Pu=lns4g~9N%UGQJDZP$Kn3UGkD|i5UF<5{$knK*wd-qj2o3ku9b>h zmna@R_^C{zPObCcFiWMXGL?Xx(ZxsHl(-6-`(*wS{97!nK2}Jm4AFEjNZJ<<=<~PJ z1?i;4N+h1+NBSQ(d~P&<(ffT_G*x@K;enl_c#)psj}srsJZjuEOU$@3bxXWuZ{$o^i(Dg?j(o=vCbpJxZ^YCTw%>r17Ei z=B}$a8uNT+zHO)N?zwO0#@4`6^h1oj-`AU6S)E|hI#cbvoJ`V|8)d97Gc#q3Rb6sx zKX7f_R!5a^x<wb$w_LZ=@O#swD)T0PR*ik}p(6CQu!`da z7`YNIJ;3p~l97u=iAM9x>+xZo>|_f2>D+#fi%(jQPQ4v;VG(-uoPS?6_mQc0dxxy^ z&EIT1oSmc=h;R?Pv+yOLz0A%0I=MA^yLp4zgm^Jh;-UQ+uheonS0&>j+UZbJ)+`gX zhFy-wc+F1HE^^hw+?}}3mvdaX(i`#|*7J0SLkWVb=XKi@)!WEQYaI;I?`5LVD)!Yzj^o+}gKE{memvm)JgzIK3sWk4b|1SKJ07tUqJ*@H?*z z`74hEKDD^CEIm!Jbb8kHgnVjrZij){7jN;H8mgjBE|rjc|=JI%&yw#hQf+Id++y>_hzJ#=`z@=&saKc@dU$ zF4)~z=16yql*ajVorr4`8(MM&FTleqWqupFm29>OIhM|-u4Em=W51E zt`v{KdnEUdof9~@MUkoM;&J62Whc9XMn2Z3)!+gi_en*2E^l+Y9DHuT!YOTgyJL$N zlfc$OSk9xY!>J7=yNWg!P2Ql&8`)SDpDliPw}LA@;_LF8?#j~3deV#RXj%79pDhFt z^^&?buNB}q4Sw7@^T}k z_30*31KNUnoDtT~U4wL(OJ3Sp8lSA7)z#kaaWM@SD`Nl-Lk(R_{cg$u#!WV}<(16tp8-7cfss`g=DBaYWua@@_9v z(JsLNFTO>V08dBbXx>f@CD_v?mtDvrEaisIz^ zH>NYLBu3*ILTiw?3g3E|Rh4l70)?4-Ct z>sZ)%S@L7fxo!367dsB!#l~*V(GQH(WGQs7O>p7274A4R>T67W=e_KOMnBdBv?2!6 zwTz!0%vfOkDrOD8c!?{_BxBc6mvgH0Q@wSFV_`B?+hmxF40hAN?nvu4%d$CBap&Z_ z6&PW{X|jr6v~Opp*-q!_~zOTHy=Y*YWw&YVYW<60$A6kjZBhKNO{>H|-_}ngxyQBT-N?-3STRu)- zm(3b_%Z5C3riy8>Sak2#2YxAq?GdMSWp!sg0-du=9)Fv^YhP98a<{bm3CHY9g{@bc zgp5%*Pwf&M{gn(B`%Bo?482z|RMZ_W$!51K220heab6AKqRu~{XBlO@z>=ujF0a4; z-l5muSmE)0&*H{LN^jP*Z|q}uF&!8lZFBRK?~B;uLQI^Le03MMq+M%lPewSnMQIjG zxGug~(Y;EW6!LBQ3 zcYP}5>d%X=ci(eu&s^P{SKz3Mj%M=F5)SXun8KjFn$iQz97&E&H{gN&mS4xOjeHVj zH;apzznORcY{#&DLxE(o-0P&iVftF8v*%O$CqG*?1}fgw%@dSXd{vgI2}{b5J@&di zJU)^Wu05}uGjiFP&ugJM?8-f5i^wt+{ruyL0UG^5uTjHF#WWY+mv7k*U0ypvrAXx4b_8_`Lte59Yv=M zSU!i%?sXZoT_ol$fSLgLrK1MrkO;R}^!Me#h;BqFM z62_)_P(iLXcua>fP~?k2$1H_0M#f|`^{r;mF5wSy{kv@P8y^XipL;{$9Wcl4#22m^ zv2!;;>EPV~Kk5VKMl)qu8gp;aq61B{;-BVjUqA9_HakCK^Hed%8T`qoDFduu$vHn$ z4zr9P(GSPRY5n!Lc197Ai`X zWPCsJCY-;o-n&2SSwQ2RozzDHEWIP*t1dn;-^rROXfGB`rDM+BI2xa}0r^3!%km<5 z(w*Y;Ss`cFpn;uNpJs@n_C$o_>>fBj%evqEMcSjfdG56Oq0bYMoR>3tOVBL^aiv@$ zshxVpg|JGkT_TxH3gn@+w}QR->8r$iAL_T#mprql87D(HYbP&T^`a)e$cSC}q_9=T zIF!#mfKbD1?_at3*o{*y&DcqTvJ$uTeXSj z;`GP19@#6e!@wLF&sJ2g+FvKmm`4mV9!~ zllkd}9IJ}(dB*}#4bP1QA62Ja&ie(4j+GdE(z(Zw{UoAj`vVlU0S!&(0_Ny!$Ag(S zT=hGg4=q$JgFAc3H?UHGGwaE~$A)AIXTA)goP5EdHK*9f$iR;3>O;V<{1=Thf!km) zmZJZBg(wPxp)A4mfiUnj_FB>;?~Icu_~`jE&yUZ!f1(KZH-P^O$q)-iAQ2b~3%C^y z34ya7udQy}L`GJFX|Z~t*}T0+y;fQgn%Vl2-Tng>=l6RB!Y!)hkzkhv_n|HaX1K8^zgLAP0$%| zV15oFmf{X578Z-aLUDW>_~;VAhv(wp4mF^}>O;V9+6~2VgrhAW)WNt&Uh)GsN$vnv z;aY{P*u75>0%MH9n!`*n7;^|z2UB{|5I{`cIs&+iNGw2Y& z%>Wz^5zZdGnD9#zvB;HgIB+&tBpeN)+``UFZom2v@WbG{kzZ++P>${s z>2YZsK#qZ4=UGLpbl})Y2*^K|>Dk(#ETFu~D)zdq3h;DeV1M9_s}BLss{u)~gBw~v zjpuI*4a<+fMHz&_wd0wfxjtxp%fG1d(@p|PO@li)h{y9aHz*#)7K=cF=w*z7FuWe6 z(P12Dn+^CQ9y^wJD3}f02C9uLu+t?^fR&gs$;kLtO|@crc25Y5jiVLPXeDCVA+a`) zo^RMic!*;lBRjZnZFHuIfB^nHQ39fcC%*}&lQDvD3J$>`9*fVDph#d?LuwR{vE}Uq zO?w1^p8~_!Aek2~)bT6S{aJ%oPWk58rSt_^sAc$G>4r>Rt zSMK^Okb$6M4sUAxMMI$0;a_6TbltkRmD~uSnif@nNaSSF&t-uLn6&dbwlhy)Q13YKGG}xwN%FT-%x+zAzJ;h{LG{= z&@&WRapg4spAP}w`}sFibEKm&7LG=)Gg`@=1#W)?Q-cmLR0V!w!7l-Sf8aM{5O|^c zTk((`gAQP5fxne=qJBOE{L``DFu?$}fuk&uSR5%My9`p81_4vh0>z01=#Brm0QAaq z3}+69u=jT75XM+wZzGV7#D1y|ehGN_sXrHBg~b>`g|_g)&C0xhUIv=5i->+;8iJ01 zBTSK`%n@;s%q0?pZbJ|*iHCmR%(^^7IOJ4uH@+Al!3tWgMn)z?q?;uf`LAj7hi=9w zkk`Ne7kpERftm@)F!oDPfJC|j=m&D?>O;VP1{29&vhn|6aW)t%lo7%8a@F5}0l7iL ziH#`CwJy;T^FvLDF8_b0(S7_?;-TM z$6wU@80b?5{yG-xqHvW7N7U!jSSao$7!t2IzuBZ5KOFif#!eeilf$9nuKGpk#M_ z9Q*;a6hOfYaP=YJi}grA(Y6*gD1<2ne47okvxdqQQq0*m*?{+RfMpBuEbgH`2`~l$ zwLH666(zq6w7d%9#(`BGSBBD%0SOomg+^Jxu_zlyFe@iz1#71x@8&`3u>*d;!=A1g0eRqU~VH_e;w889hKJ!{C-iFzEgnb9kov5p;GE$YsRo@hqretP3>+vsoht zQ7y&eC)E{2&_-+PJ=1DAH*$ahIY8GCJ15rq_gE5zBnuG{hEQOD7O+kM>-*J*fVai| z9{fXX5|OWqg_b=6;O~LiBXJynIsGwy-FYM@ZG}$(P+0_6g*dOMdXk`<;V_mKq_V(d zi)3#n=s*E58Cf|;;^#xaYhC><7qJpyVc%ttFbw&FliCU(#l{2A+w+qTehGMQ-`|Qr zeWydjunq}}p;|E3n!wsjl!(d@{D)N7y4IG|k72O@uB`&%7%_Z*#Bbq0>;cMrR&dDK znrhpqyd~(yZV-2f;l0r$;Pyzve~OW$`g4%g(M1k;p&T%a6p>k?V*U&cH!;DklxH#6 z^(L^f517=qf_cbWFzShAh$sD-jPH|(e;WkRy*gbRc5?$G<$}MJ<1T+b1bpc8Ka;WU zRO)H^74Aw944Z-X9RJA$zXUvG_Mf4Rkj8M33t>h-miaJ}Eup;GU!4Sk__Hl*#fiWC zZ{kc=s^usIX>r`m?{We_YaW3=8KTz77X6txB-#q(&Yy*EM%utvTnNH*Oeg{zY(aQhLjAbm z_-88!_@&K%246q6dRLz?qydWC0>>dvX9Yrk#{Q>n{m;;bP+wX*>s>3TxOsrG&cUC$ zgI@xEhuD7-v2OAI+>?73F`%Xzp|I9K$_jrd!4!^#u16(b7jMi5iDwDKZ{qN<@$4V6 zZNQWe;7NOs0T%Z56Nq|Iz*3+(zxojH9AL4u?&R?QJP7P&*doA@RKM;BhN+PG?g#aq zs}^g=nDV7RV}r2^1H}PQOM)81&!e105`peiz=?=$RAT>!eAAVw1=wU**)hUdTc9Az z`a?;zZ%Tn>9)ldWYt=$4o>b}zAqR~#LcvMpetP&wraG9haDllT9}(|8*yj4h=0A8~ zH3N1`LAHl5baL<;+)9@80frzhwO+chE^ws)hI7Q(Aknb($7u+C$4z_ShSxwSBW}7o zSeUM(Dv6jVeK4K>K-KTM|1);3{nO?`AW<81kmy-(^@BXy8j#M&-A(O zAa`=IQYZo8l^lc-38K!TdH5UlN(5ic&+7^ZWLqdr0|`14jC27a0o>`o5wI@vln2NA zATTn5L3AQslFPC{7)W(u69v33M{V7>k_W4b&yk_pbg>1|GM2zX9i%H-2n(| z9Bor<3ImaW02~lZv{xSj{&Xt@+Xjv^2ZuU;-*+PA)3^7u2nPY)+v2q~)P3#0!>&^& zefashb7ZQvEe3%rfg6u(-5NK*vZ;)kB1sWI@SMT6no2FS$ik74*7)!Y)i{Q>TB zhA?YGT1-+W2$C-W6D&wq9|GQI^Y5_hj*DYE7u6?$p02g_Ub& z(8uJ!%ZX>+c)kB1<)8WG${sz`+@@mj!BQpY>M`J=hgY3*Wlk`9`9DapL*YO<9cF_u z2ghhanC$;|c#aT>*<$z~#H@_o-|SC5sP0Gwxq(a=c$hqqq@ur)^rM$nHnJg_Rc0zldXpsz<85|=LRBiGXg231T?`q)Ido7HG z64zY^nsf;`#c?8O_LhGk?S~2f*>Zts9H-Q&h$xVCQo+)mxXTi;e-2 zBNPlRMWVLF+5d$wOFJ~E`omCYll9wYX7Myj7_=`OnDYdY$PA~y7l}ifput(XP^0aA z?odu35UBEtL65org+L725(Rb&u~xQF!dSnW$%O;g)B&?7HKNvudHsbjFoy>V1W>cL zH2LS1qIKQ$|AWddBFWO=g1cX4LpET0oa7GWwy^iD5v*in-WqFD_1pD%a7(y_qZ8Po zL>k&cZ6*dke;sWNGTj^~59}sVGBs*lCBZ@wVgCJyWQgJNdeeL!7I^O*hy`HfwfYe7 z@bq%4+8mR9Rv1z7CAu$Z#eWr2hRwIg*Lr}SVHtgjw}X#?>N(#za+S!j$cB+QHD ztDqowcaU&y?WSpe`MR+6*MGO!hlM%8h^PUJ6XIEzde`r8uqwM=E4ht+IIr* z8Q#@ySjnBKy`-RaD5O0EZXkT3WtST;=pUGXp%lnzjeQ06VUAt z0P_>+=srRkiol@JkS`5iR@Dy~0VuG;x(4}h^fyRb@J0#L3!}^$cPbb_dGre?e+Uq1 z`SvSmBpPmI1u_|gIb~0sol^t76U(u7;BhYfhKa;N?z|f)adbZgjrIc3j@XuolpBAI zH}=>cx{+EPz23zmmk5;11xoS~H9U%n6mp%FQS{jIO#_f+#(@9qCBofgB8>yJsh>-H z2wQg01qy8ht*!+zo|r4Tog{ZFdl*lKcH`LkLCZ8%8(f zgX~@alqFty$MBN^f(a3LC4z*e9$(vh2SEVg1r{J)aV72{1x8w75T>Y=`~Zs6(2II5 z`*`$IFy@&+Y$DdV6U-WZNn@*S1qxr_bpsOMnn#VdQ-E_VfcOL)V)Y^5Tg6F%O%NoY zLD<~b2SC#*U^?%X-{W7<;O;~&5$-i=r-T50i z-|s(JD`f3mCI1Q;{3{`AZ!TFYZ|yZC|H|_S_?t$py_96FytT)N{wwd$qrZ{&<1o=P Ws+1skf}J$*@1it_YXu-^lKl^CROikB literal 0 HcmV?d00001 diff --git a/examples/mimo/vendor/old_energon_multimodal_provider.py b/examples/mimo/vendor/old_energon_multimodal_provider.py new file mode 100644 index 00000000000..a0ca5aaf083 --- /dev/null +++ b/examples/mimo/vendor/old_energon_multimodal_provider.py @@ -0,0 +1,248 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +"""Previous-branch MIMO Energon encoder fixture for dataloader parity checks. + +Bridges energon's ``MultiModalPackingEncoder`` (which produces 1 image-placeholder +per ```` tag) to MIMO's contract (which requires N placeholders per image, +where N = num_tiles * embeddings_per_tile, for 1:1 ``masked_scatter_`` alignment). +""" + +import warnings +from typing import List, Optional + +import torch + +from megatron.energon.task_encoder.multimodal import ( + MultiModalPackingEncoder, + PackingConfig, + VisionConfig, +) +from megatron.energon.task_encoder.multimodal.sample_types import PackedSample +from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings + + +# --------------------------------------------------------------------------- +# Tokenizer adapter: Megatron tokenizer → energon TokenizerProtocol +# --------------------------------------------------------------------------- +class _TokenizerAdapter: + """Wraps a Megatron tokenizer to satisfy energon's ``TokenizerProtocol``. + + Handles both HuggingFaceTokenizer (single wrapper) and MultimodalTokenizer + (double wrapper) by walking the ``_tokenizer`` / ``tokenizer`` chain. + """ + + def __init__(self, megatron_tokenizer): + self._tok = megatron_tokenizer + # Walk the wrapper chain to reach the HF PreTrainedTokenizerFast. + # Chain: DefaultTokenizerVision._tokenizer → MegatronMultimodalTokenizer + # MegatronMultimodalTokenizer.tokenizer → HF AutoTokenizer + # IMPORTANT: Do NOT drill into PreTrainedTokenizerFast.tokenizer — that's + # the raw Rust tokenizer whose encode() returns tokenizers.Encoding, not list[int]. + inner = megatron_tokenizer + # Unwrap DefaultTokenizerVision → MegatronMultimodalTokenizer + if hasattr(inner, '_tokenizer'): + inner = inner._tokenizer + # Unwrap MegatronMultimodalTokenizer → HF AutoTokenizer + if hasattr(inner, 'tokenizer'): + inner = inner.tokenizer + self._hf = inner + + @property + def pad_token_id(self) -> int: + return self._tok.pad + + @property + def eos_token_id(self) -> int: + return self._tok.eod + + def encode(self, text: str, add_special_tokens: bool = True) -> list: + return self._hf.encode(text, add_special_tokens=add_special_tokens) + + def decode(self, token_ids, skip_special_tokens: bool = False) -> str: + return self._hf.decode(token_ids, skip_special_tokens=skip_special_tokens) + + def convert_tokens_to_ids(self, tokens): + return self._tok.convert_tokens_to_ids(tokens) + + +# --------------------------------------------------------------------------- +# MIMO-specific MultiModalPackingEncoder subclass +# --------------------------------------------------------------------------- +class MimoMultiModalPackingEncoder(MultiModalPackingEncoder): + """Subclass that remaps energon batch output to MIMO's forward() signature. + + Key transformation: expand each single ``image_token_id`` placeholder in the + token stream into ``num_tiles * embeddings_per_tile`` copies so that MIMO's + ``align_embeddings_by_token_positions`` can do a strict 1:1 scatter. + """ + + def __init__( + self, + vision_config: VisionConfig, + packing_config: PackingConfig, + tokenizer, + encoder_name: str = "radio_encoder", + encoder_input_key: str = "x", + target_seq_length: Optional[int] = None, + ): + super().__init__(vision_config, packing_config, tokenizer) + self.encoder_name = encoder_name + self.encoder_input_key = encoder_input_key + self._target_seq_length = target_seq_length + + # Compute embeddings per tile using the standalone math function. + self._embeddings_per_tile = get_num_image_embeddings( + img_h=vision_config.img_h, + img_w=vision_config.img_w, + patch_dim=vision_config.patch_dim, + class_token_len=vision_config.class_token_len, + disable_vision_class_token=vision_config.disable_vision_class_token, + pixel_shuffle=vision_config.pixel_shuffle, + conv_merging=vision_config.conv_merging, + use_tile_tags=vision_config.use_tile_tags, + max_num_tiles=vision_config.max_num_tiles, + use_image_break_token=vision_config.use_image_break_token, + ) + + def batch(self, samples: List[PackedSample]) -> dict: + """Override to expand image placeholders, build packing_kwargs, and remap to MIMO format. + + Energon's token stream has 1 placeholder per image. + MIMO needs ``num_tiles * embeddings_per_tile`` placeholders per image + for 1:1 ``masked_scatter_`` alignment. + + The base class pipeline (preencode → pack_selected_samples) already + computes ``cu_lengths`` using expanded ``total_len`` values, so the + cumulative lengths are correct for the MIMO-expanded token stream. + + When ``target_seq_length`` is set, samples whose expanded length would + exceed the limit are **right-truncated** at image boundaries. + + Returns dict with keys: input_ids, labels, loss_mask, position_ids, + modality_inputs, and optionally packing_kwargs. + """ + image_token_id = self.packing_config.image_token_id + ignore_index = self.packing_config.ignore_index + pad_id = self.packing_config.pad_id + emb_per_tile = self._embeddings_per_tile + + expanded_tokens_list = [] + expanded_labels_list = [] + all_images = [] + + for sample in samples: + tokens = sample.tokens + labels = sample.labels + num_tiles = sample.num_tiles # e.g. [5, 3, 1] for 3 images + + budget = self._target_seq_length # None means unlimited + + # Expand each single image placeholder → N copies, respecting budget. + new_tokens = [] + new_labels = [] + img_idx = 0 + truncated = False + kept_tile_count = 0 + + for i, tok in enumerate(tokens.tolist()): + if tok == image_token_id: + n_tiles = num_tiles[img_idx] if img_idx < len(num_tiles) else 1 + n_tokens = n_tiles * emb_per_tile + if budget is not None and len(new_tokens) + n_tokens > budget: + truncated = True + break + new_tokens.extend([image_token_id] * n_tokens) + new_labels.extend([ignore_index] * n_tokens) + kept_tile_count += n_tiles + img_idx += 1 + else: + if budget is not None and len(new_tokens) + 1 > budget: + truncated = True + break + new_tokens.append(tok) + new_labels.append(labels[i].item()) + + if truncated: + warnings.warn( + f"Sample truncated to fit target_seq_length " + f"({self._target_seq_length}): kept {len(new_tokens)} of " + f"~{len(tokens)} original tokens, {img_idx}/{len(num_tiles)} " + f"images ({kept_tile_count} tiles). " + f"Consider increasing --total-seq-length or reducing " + f"--max-num-tiles.", + stacklevel=2, + ) + + all_images.extend(sample.images[:kept_tile_count]) + expanded_tokens_list.append(torch.tensor(new_tokens, dtype=torch.long)) + expanded_labels_list.append(torch.tensor(new_labels, dtype=torch.long)) + + # Pad to target length or max length in batch + max_len = max(len(t) for t in expanded_tokens_list) + if self._target_seq_length is not None: + max_len = self._target_seq_length + + B = len(samples) + tokens_batch = torch.full((B, max_len), pad_id, dtype=torch.long) + labels_batch = torch.full((B, max_len), ignore_index, dtype=torch.long) + + for i, (t, l) in enumerate(zip(expanded_tokens_list, expanded_labels_list)): + tokens_batch[i, : len(t)] = t + labels_batch[i, : len(l)] = l + + loss_mask = (labels_batch != ignore_index).float() + # Don't train the model to predict tokens — they are special + # placeholders replaced by vision embeddings, never naturally generated. + loss_mask[labels_batch == image_token_id] = 0.0 + position_ids = torch.arange(max_len).unsqueeze(0).expand(B, -1).contiguous() + + result = { + "input_ids": tokens_batch, + "labels": labels_batch, + "loss_mask": loss_mask, + "position_ids": position_ids, + } + + # Only include modality_inputs when there are actual images. + if all_images: + imgs = self.tiling_strategy.stack(all_images)[0] # (total_tiles, C, H, W) + result["modality_inputs"] = { + "images": {self.encoder_name: {self.encoder_input_key: imgs}} + } + + # Build packing_kwargs from base class cu_lengths when packing is active. + # The base class pipeline computes cu_lengths using expanded total_len, + # so they match our MIMO-expanded token stream. + is_packed = any(len(s.cu_lengths) > 2 for s in samples) + if is_packed: + # Build per-sample cu_seqlens from PackedSample.cu_lengths. + # With micro_batch_size=1 (required for packing), B==1. + assert B == 1, f"Packing requires micro_batch_size=1, got B={B}" + sample = samples[0] + cu_seqlens = sample.cu_lengths.to(dtype=torch.int32) + + # Clamp to actual sequence length (cu_lengths are based on expanded + # total_len which should match, but clamp for safety). + cu_seqlens = cu_seqlens.clamp(max=max_len) + + # Ensure starts at 0 and ends at max_len. + if cu_seqlens[0] != 0: + cu_seqlens = torch.cat([torch.tensor([0], dtype=torch.int32), cu_seqlens]) + if cu_seqlens[-1] != max_len: + cu_seqlens = torch.cat([cu_seqlens, torch.tensor([max_len], dtype=torch.int32)]) + + # Compute per-segment lengths and max segment length. + segment_lens = cu_seqlens[1:] - cu_seqlens[:-1] + max_seqlen = segment_lens.max() + + result["packing_kwargs"] = { + "cu_seqlens_q": cu_seqlens, + "cu_seqlens_kv": cu_seqlens, + "cu_seqlens_q_padded": cu_seqlens, + "cu_seqlens_kv_padded": cu_seqlens, + "max_seqlen_q": max_seqlen, + "max_seqlen_kv": max_seqlen, + "total_tokens": torch.tensor(max_len, dtype=torch.int32), + } + + return result diff --git a/pyproject.toml b/pyproject.toml index f7611078b9e..e40fb704f7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ dev = [ "mamba-ssm~=2.2", "causal-conv1d~=1.5", "flash-linear-attention~=0.4.0", - "megatron-energon[av_decode]~=6.0", + "megatron-energon[av_decode,multimodal]==7.3.3.dev30+gd456cbd4a", "av", "flashinfer-python~=0.5.0", "wget", @@ -121,7 +121,7 @@ lts = [ "opentelemetry-api~=1.33.1", "mamba-ssm~=2.2", "causal-conv1d~=1.5", - "megatron-energon[av_decode]~=6.0", + "megatron-energon[av_decode,multimodal]==7.3.3.dev30+gd456cbd4a", "av", "flashinfer-python~=0.5.0", "wget", @@ -204,6 +204,7 @@ transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "17ae86b64d7f75653351664f5d8c9e466faede00" } emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.2.0" } nvidia-resiliency-ext = { git = "https://github.com/NVIDIA/nvidia-resiliency-ext.git", rev = "b2bb3d728a18795807d9f76c535e005a609a1b01" } +megatron-energon = { path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" } [tool.isort] profile = "black" # black-compatible diff --git a/tests/unit_tests/test_hetero_energon.py b/tests/unit_tests/test_hetero_energon.py new file mode 100644 index 00000000000..32264232f6c --- /dev/null +++ b/tests/unit_tests/test_hetero_energon.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +import random + +from examples.mimo.data.hetero_energon import EnergonIterator + + +class RandomLoader: + """Small dataloader stub that consumes Python's global random module.""" + + def __iter__(self): + return self + + def __next__(self): + return {"value": random.randrange(1_000_000)} + + +def test_energon_iterator_uses_isolated_python_random_state(): + """Same DP-lane iterators should match without perturbing caller RNG state.""" + first = EnergonIterator(RandomLoader(), random_seed=12345) + second = EnergonIterator(RandomLoader(), random_seed=12345) + + first_values = [] + second_values = [] + for _ in range(8): + random.seed(111) + caller_state = random.getstate() + first_values.append(next(first)["value"]) + assert random.getstate() == caller_state + + random.seed(222) + caller_state = random.getstate() + second_values.append(next(second)["value"]) + assert random.getstate() == caller_state + + assert first_values == second_values + assert len(set(first_values)) > 1 diff --git a/uv.lock b/uv.lock index 16d960dcc2b..ced46cb032f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and platform_machine != 's390x' and sys_platform == 'win32' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", @@ -249,7 +249,7 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ @@ -305,7 +305,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "sniffio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" } wheels = [ @@ -644,7 +644,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -774,7 +774,7 @@ name = "click" version = "8.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } wheels = [ @@ -909,7 +909,7 @@ name = "cryptography" version = "47.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } wheels = [ @@ -962,7 +962,7 @@ name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, @@ -1009,37 +1009,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-curand", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] [[package]] @@ -1179,6 +1179,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging", version = "25.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "packaging", version = "26.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, +] + [[package]] name = "dill" version = "0.4.1" @@ -1272,7 +1285,7 @@ version = "0.2.0" source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=v0.2.0#1effa026ff096b7fa1063ca2fba19d98be6e6cdf" } dependencies = [ { name = "absl-py" }, - { name = "torch", marker = "sys_platform == 'never'" }, + { name = "torch", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] [[package]] @@ -1968,7 +1981,7 @@ dependencies = [ { name = "filelock" }, { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or sys_platform != 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "fsspec", version = "2026.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform == 'win32') or (python_full_version < '3.14' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging", version = "25.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "packaging", version = "26.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, { name = "pyyaml" }, @@ -2651,7 +2664,7 @@ dev = [ { name = "flashinfer-python" }, { name = "hypercorn" }, { name = "mamba-ssm" }, - { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "megatron-energon", extra = ["av-decode", "multimodal"], marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "multi-storage-client" }, { name = "nvidia-modelopt", marker = "(sys_platform != 'darwin' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "nvidia-resiliency-ext" }, @@ -2677,7 +2690,7 @@ lts = [ { name = "fastapi" }, { name = "flashinfer-python" }, { name = "mamba-ssm" }, - { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-lts'" }, + { name = "megatron-energon", extra = ["av-decode", "multimodal"], marker = "extra == 'extra-13-megatron-core-lts'" }, { name = "multi-storage-client" }, { name = "onnxscript", version = "0.7.0", source = { registry = "https://pypi.org/simple" } }, { name = "opentelemetry-api", version = "1.33.1", source = { registry = "https://pypi.org/simple" } }, @@ -2779,8 +2792,8 @@ requires-dist = [ { name = "hypercorn", marker = "extra == 'dev'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, - { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, - { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'lts'", specifier = "~=6.0" }, + { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'dev'", path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" }, + { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'lts'", path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" }, { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy" }, @@ -2866,25 +2879,27 @@ test = [ [[package]] name = "megatron-energon" -version = "6.0.1" -source = { registry = "https://pypi.org/simple" } +version = "7.3.3.dev30+gd456cbd4a" +source = { path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" } dependencies = [ { name = "braceexpand" }, { name = "click" }, + { name = "filetype" }, + { name = "mfusepy" }, { name = "multi-storage-client" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, { name = "pillow" }, { name = "pyyaml" }, + { name = "rapidyaml" }, { name = "s3fs", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.14' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev') or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "s3fs", version = "2026.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.14' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "torch", marker = "sys_platform == 'never'" }, { name = "tqdm" }, { name = "webdataset" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/19/7cbb748913db83662c9e4a82164e2a008482048809d3aa163440aa3824bd/megatron_energon-6.0.1.tar.gz", hash = "sha256:39dddd2c91ddf2938ad5440a061363930b09a0c09ee1b459764df149cac34f21", size = 141410, upload-time = "2025-03-17T12:11:22.452Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/d8/d67bac7beaba18d595b3a7d038661b6f27d69fa2099b2fabe85a63343054/megatron_energon-6.0.1-py3-none-any.whl", hash = "sha256:2214250bdc7956791556f3a48b221601fd63d36844644cff9110c312b1cd47a5", size = 202240, upload-time = "2025-03-17T12:11:20.657Z" }, + { filename = "megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl", hash = "sha256:34c309d2baa623eabedf0cc6e94d430fc5b0abb09d9f2b5d20da2464b54cce54" }, ] [package.optional-dependencies] @@ -2894,7 +2909,61 @@ av-decode = [ { name = "ebmlite" }, { name = "filetype" }, { name = "sortedcontainers" }, - { name = "soundfile" }, +] +multimodal = [ + { name = "einops" }, + { name = "torchvision", marker = "sys_platform == 'never'" }, + { name = "transformers" }, +] + +[package.metadata] +requires-dist = [ + { name = "av", marker = "extra == 'av-decode'", specifier = ">=14.4.0" }, + { name = "bitstring", marker = "extra == 'av-decode'", specifier = ">=4.2.3" }, + { name = "braceexpand" }, + { name = "click" }, + { name = "ebmlite", marker = "extra == 'av-decode'", specifier = ">=3.3.1" }, + { name = "einops", marker = "extra == 'multimodal'" }, + { name = "filetype", specifier = ">=1.0.0" }, + { name = "filetype", marker = "extra == 'av-decode'", specifier = ">=1.2.0" }, + { name = "mfusepy" }, + { name = "multi-storage-client", specifier = ">=0.33.0" }, + { name = "multi-storage-client", extras = ["aistore"], marker = "extra == 'aistore'" }, + { name = "multi-storage-client", extras = ["azure-storage-blob"], marker = "extra == 'azure-storage-blob'" }, + { name = "multi-storage-client", extras = ["boto3"], marker = "extra == 's3'" }, + { name = "multi-storage-client", extras = ["google-cloud-storage"], marker = "extra == 'google-cloud-storage'" }, + { name = "multi-storage-client", extras = ["huggingface"], marker = "extra == 'huggingface'" }, + { name = "multi-storage-client", extras = ["oci"], marker = "extra == 'oci'" }, + { name = "myst-parser", marker = "extra == 'dev'" }, + { name = "numba", marker = "extra == 'tar-patcher'" }, + { name = "numpy" }, + { name = "pillow", specifier = ">=10.0.1" }, + { name = "pyyaml" }, + { name = "rapidyaml", specifier = ">=0.10.0" }, + { name = "ruff", marker = "extra == 'dev'" }, + { name = "s3fs" }, + { name = "sortedcontainers", marker = "extra == 'av-decode'", specifier = ">=2.4.0" }, + { name = "soundfile", marker = "extra == 'dev'" }, + { name = "sphinx", marker = "extra == 'dev'" }, + { name = "sphinx-click", marker = "extra == 'dev'" }, + { name = "sphinx-rtd-theme", marker = "extra == 'dev'" }, + { name = "sphinxcontrib-napoleon", marker = "extra == 'dev'" }, + { name = "torch" }, + { name = "torchvision", marker = "extra == 'multimodal'" }, + { name = "torchvision", marker = "extra == 'transforms'" }, + { name = "tqdm" }, + { name = "transformers", marker = "extra == 'multimodal'" }, + { name = "webdataset" }, +] +provides-extras = ["aistore", "av-decode", "azure-storage-blob", "dev", "google-cloud-storage", "huggingface", "multimodal", "oci", "s3", "tar-patcher", "transforms"] + +[[package]] +name = "mfusepy" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/47/746287c8962274f73ee25edb3840d80899464bfffbe2c435424c2d60a071/mfusepy-3.1.1.tar.gz", hash = "sha256:338ece54513d7d1a5e9492837679a0c7432ecf96a03490a2683a1ce1d19570e1", size = 34549, upload-time = "2026-03-13T00:36:52.636Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/d5/56bc9326bf75f7ea0f4fe1178f5c9f20d1a1e15e288ecf66088513538ccd/mfusepy-3.1.1-py3-none-any.whl", hash = "sha256:69fb70cfc7f7cce595e6ff586f8451d8298f01f18286a157f24f564b24ec5a37", size = 28264, upload-time = "2026-03-13T00:36:51.843Z" }, ] [[package]] @@ -3505,7 +3574,7 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, @@ -3534,7 +3603,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3566,9 +3635,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparse", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvjitlink", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3581,7 +3650,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -3821,11 +3890,11 @@ resolution-markers = [ "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, - { name = "protobuf" }, - { name = "typing-extensions" }, + { name = "protobuf", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c5/93/942d2a0f6a70538eea042ce0445c8aefd46559ad153469986f29a743c01c/onnx-1.21.0.tar.gz", hash = "sha256:4d8b67d0aaec5864c87633188b91cc520877477ec0254eda122bef8be43cd764", size = 12074608, upload-time = "2026-03-27T21:33:36.118Z" } wheels = [ @@ -3901,12 +3970,12 @@ resolution-markers = [ "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, - { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" } }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "sympy", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/35/e6/672fefb2f108d077f58181a7babf4c0f8d1182a30353ffc9c79c63afc5ee/onnx_ir-0.2.1.tar.gz", hash = "sha256:8b8b10a93f43e65962104de6070c43c5dacb0e3cdfefc7c8059dd83c9db64f35", size = 144279, upload-time = "2026-04-20T20:21:47.735Z" } wheels = [ @@ -3969,14 +4038,14 @@ resolution-markers = [ "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, - { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" } }, - { name = "onnx-ir", version = "0.2.1", source = { registry = "https://pypi.org/simple" } }, + { name = "onnx", version = "1.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnx-ir", version = "0.2.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging", version = "25.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging", version = "26.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9b/99/fd948eba63ba65b52265a4cd09a14f96bb9f5b730fcef58876c4358bf406/onnxscript-0.7.0.tar.gz", hash = "sha256:c95ed7b339b02cface56ee27689565c46612e1fc542c562298dddfdad5268dc5", size = 612032, upload-time = "2026-04-20T17:09:19.775Z" } wheels = [ @@ -5446,6 +5515,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/e9/cc28f21f52913adf333f653b9e0a3bf9cb223f5083a26422968ba73edd8d/quart-0.20.0-py3-none-any.whl", hash = "sha256:003c08f551746710acb757de49d9b768986fd431517d0eb127380b656b98b8f1", size = 77960, upload-time = "2024-12-23T13:53:02.842Z" }, ] +[[package]] +name = "rapidyaml" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/a2/b733d539de5c6e78c767671dba148b3c33b0d1dcdda4af2c14feed0c41ec/rapidyaml-0.12.1.tar.gz", hash = "sha256:53ea4f2277d0b35f0409f0939a95424af6a0b48e003a20ece8b6ae74f5c7f0d0", size = 483550, upload-time = "2026-05-07T19:44:12.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/f7/d71685b58392f93df3247ba6b3519f1c0433f61e606716920535dd92db59/rapidyaml-0.12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bea636ed2bc7b99d2dd5243010806ac673c70fee7d316e828bd27e3593851321", size = 5124112, upload-time = "2026-05-07T19:43:18.461Z" }, + { url = "https://files.pythonhosted.org/packages/57/5a/cf91f2d2258a11f52f2f62bf9c65739f49ea808cebaeedc648e9f78a393e/rapidyaml-0.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7ca18d04a2d9244d253d99d17a80671be7f4cd7a9daf5ff9dd850737e9b0a8c4", size = 5147555, upload-time = "2026-05-07T19:43:20.363Z" }, + { url = "https://files.pythonhosted.org/packages/90/d6/42d441eeceac1d8fc7d2a7480654db661282dd1f94deef344c5421ba0c3c/rapidyaml-0.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6ddd43a1f6a12e706477b85cf51d881c2362d74f4c0433d6092f6b40bae20567", size = 5124110, upload-time = "2026-05-07T19:43:22.166Z" }, + { url = "https://files.pythonhosted.org/packages/42/70/ec54f6b47b47adfdcce6695817ceaa102b64cd41d5fc5cb2b512353481f9/rapidyaml-0.12.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:edb8133c0f0952e9a5e1efa805f32aec5dd83315cf63710a3cd3a36a7b742615", size = 292548, upload-time = "2026-05-07T19:43:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/9a/57/27ec9e7f491f8329bfc7eb082050cca6957166407fd24d81d98a0939499e/rapidyaml-0.12.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29c97a8d7022d851c3875818a4dad232f4d921f98fc578aad5bd37bc6dd6946e", size = 272421, upload-time = "2026-05-07T19:43:24.87Z" }, + { url = "https://files.pythonhosted.org/packages/af/cb/87da89ae11d0de562928a7e8ca40d5daee55049408019d5d1b918af6800b/rapidyaml-0.12.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:489f1cba591f3e6a6325f3b08edf64abb293b5353906686ec04949ccc3a44855", size = 282080, upload-time = "2026-05-07T19:43:26.18Z" }, + { url = "https://files.pythonhosted.org/packages/d1/f7/6f0878b1c7e67b4d805cf0720a32aa2d85adca356f12e75d8fe4e99107b8/rapidyaml-0.12.1-cp312-cp312-win32.whl", hash = "sha256:e2375a375946afbc29aa456f4403207885b610f800e5d61491933dd511d975d1", size = 262785, upload-time = "2026-05-07T19:43:27.26Z" }, + { url = "https://files.pythonhosted.org/packages/3f/9f/71728292cc78a2d69ddf83a59d5248776174458fb4db13630eb066566800/rapidyaml-0.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:dbcb37383a184db48c52182f236d5b2ac2d0fcb8ed768d076e967cc6d107e094", size = 315347, upload-time = "2026-05-07T19:43:28.239Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ff/0165060d60d1c9c0bbaa5492bbffaffe821afdfee4300b7fa035da7dadcb/rapidyaml-0.12.1-cp312-cp312-win_arm64.whl", hash = "sha256:aaf84d355b3dc0c8b027cc27418eff2c37888d77d5e92bfd7b54d6124f06daf6", size = 312000, upload-time = "2026-05-07T19:43:29.52Z" }, + { url = "https://files.pythonhosted.org/packages/20/fe/a24ceda80e0ead701b997af6be525ed1f12405e192c5080cee2e7607c087/rapidyaml-0.12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:26df56dd872ae679ad7eb78ec704f75514ca6ea5c9229c2e336d7df2f90360dd", size = 5069159, upload-time = "2026-05-07T19:43:30.874Z" }, + { url = "https://files.pythonhosted.org/packages/52/0c/3702d9125258abcdc9049651e34158ae2cf3557221dde637edee66765869/rapidyaml-0.12.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b9f3900d2ae4bf72358b01e823d97cd4eea541c77fbf83198ee5189c088ec34a", size = 5092680, upload-time = "2026-05-07T19:43:32.197Z" }, + { url = "https://files.pythonhosted.org/packages/44/d7/5215d653c8617b90ca121d04f7d1ef508dea36dc07c43646c9361d6080eb/rapidyaml-0.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ff4002c071ebd731257c25713b57bb9df3525e7aea1347ede824e209e9b5638f", size = 5069159, upload-time = "2026-05-07T19:43:33.519Z" }, + { url = "https://files.pythonhosted.org/packages/52/70/ed49d7c6a3786c087dda8753549ecc00a7be56fadaea8b787e2ebb70364b/rapidyaml-0.12.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b32a72128bcbfd399330e5d4c3f6fd7366cecbb369ecdaa9b32fd79934cc8d1f", size = 292423, upload-time = "2026-05-07T19:43:35.216Z" }, + { url = "https://files.pythonhosted.org/packages/a5/59/371171d64e109964bab0a527bf6bd4696adb2ff41b3a851edae812270e30/rapidyaml-0.12.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09132b2eb4e8b1d3fe285b38609d9fc58dccdd43d6df3798ee38acbfd7f2bf15", size = 272203, upload-time = "2026-05-07T19:43:36.197Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/03a3c30aa899a40a068fcf7c28e1afa08cc528f55298d53ddeeb872f5c66/rapidyaml-0.12.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f3fe91b67ec3afbb290d370e7acce4f02399f1d13a4b157df6dac395a107d1b", size = 281894, upload-time = "2026-05-07T19:43:37.23Z" }, + { url = "https://files.pythonhosted.org/packages/61/8b/85799d7e330f44d5352037e19d867d0a373d46428027aa669b9de261cb9e/rapidyaml-0.12.1-cp313-cp313-win32.whl", hash = "sha256:43a96e347c6f1aaca0378519d6bb512e1312e83005fbecdaeb5ea6ae8a9d8a13", size = 263066, upload-time = "2026-05-07T19:43:38.383Z" }, + { url = "https://files.pythonhosted.org/packages/da/34/007cdf7aa9dbbd7a3e742a7af9235eb329b210946cdcfd48bf5966e5a809/rapidyaml-0.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:72c8966ca52425eb875645bed1d02b227774d13d90479b0d671f3ec29a3b4101", size = 315204, upload-time = "2026-05-07T19:43:39.397Z" }, + { url = "https://files.pythonhosted.org/packages/67/2d/932fd4ab7bbce7eeaab195adaf711f609f4bf94e3b9dbc26b30f7fdb18a7/rapidyaml-0.12.1-cp313-cp313-win_arm64.whl", hash = "sha256:5d0d2d0d85493b7239810073893ff301004226b85b35425d46dbced9f34d2fb4", size = 311874, upload-time = "2026-05-07T19:43:40.394Z" }, + { url = "https://files.pythonhosted.org/packages/d3/82/c126035a3470e0cdc0f9446de842acf38af2a7f1e5c0b6dfb072895ca67e/rapidyaml-0.12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9dae5e7cfc7ebed8d32d4378ea77c72b9de2ff7db2ea3ae066d7623731420db1", size = 5649856, upload-time = "2026-05-07T19:43:41.747Z" }, + { url = "https://files.pythonhosted.org/packages/69/f7/09c6dc1ff4327d1bb9e47b8068bb4a5dced83d1190568303a0b4a0fb8972/rapidyaml-0.12.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5e5dd676cd5071592c36dc5a8fe01e8eacf036b8df47fbe5bdedb540856a24a1", size = 5673400, upload-time = "2026-05-07T19:43:43.132Z" }, + { url = "https://files.pythonhosted.org/packages/cf/fa/df944aa94d29544c87f96d508b19ed5c00de02b42de6d3541055329e80b8/rapidyaml-0.12.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:eb2ed5a365f39041f9da353851951846f7bbec8aa7e0f7a8c531707e2ee99211", size = 5649853, upload-time = "2026-05-07T19:43:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/1e/19/322e915eb0a4bfa9eca6d3be9f724ad026f9c3c8e3281c543d2980c799e4/rapidyaml-0.12.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:30f4270c78e4613cd6cbfbd68d23065e44dd783e5bbfccfa301a066aa743b360", size = 292289, upload-time = "2026-05-07T19:43:46.077Z" }, + { url = "https://files.pythonhosted.org/packages/4d/e2/ab47a96c90dd32c039d983bc04f11836f6d50ae080d739696c98d3871689/rapidyaml-0.12.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b64974ba546d6d21935da84e85829ee8ab5c78b7db0d607aa3399a21f31355e", size = 272462, upload-time = "2026-05-07T19:43:47.262Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d2/15a25bb2947a5decf149ef36547d3ad367f032654910f25e6bc7d6f871c8/rapidyaml-0.12.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d57e96874d66588bf2af8b96e6b178cae7f33a791e1cccd78bef1656c0b10fc6", size = 281860, upload-time = "2026-05-07T19:43:48.35Z" }, + { url = "https://files.pythonhosted.org/packages/33/07/a64ab8cc6c34565d3a5e687d3e7898fe59c1a4b76608b56b0782f40007e2/rapidyaml-0.12.1-cp314-cp314-win32.whl", hash = "sha256:96b303eb6537c4dd9a4ed1c6d0296ec7c7340b637f1580dce60d99ccf1869448", size = 270639, upload-time = "2026-05-07T19:43:49.394Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/e80303b4c14d1dc41c39d71f4cb46e6f3b6683921f99bd039fec98397808/rapidyaml-0.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:ce932c0df3237c4a92428bf0407c59af217739c23c99751151eee46871c7690a", size = 325210, upload-time = "2026-05-07T19:43:50.521Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3a/9dc97f8f98394381e054b21a6f2d288ecf0e266f225a70356dd2f4394d56/rapidyaml-0.12.1-cp314-cp314-win_arm64.whl", hash = "sha256:e7492e635020edb2813393ece03bede28d6f90f30daa3464217263015c3c06b0", size = 321624, upload-time = "2026-05-07T19:43:51.896Z" }, +] + [[package]] name = "ray" version = "2.55.1" @@ -5501,7 +5608,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -6100,26 +6207,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] -[[package]] -name = "soundfile" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, - { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, - { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, - { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, - { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, -] - [[package]] name = "soupsieve" version = "2.8.3" @@ -6319,7 +6406,7 @@ version = "0.52.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/68/79977123bb7be889ad680d79a40f339082c1978b5cfcf62c2d8d196873ac/starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933", size = 2653702, upload-time = "2026-01-18T13:34:11.062Z" } wheels = [ @@ -6331,7 +6418,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-lts') or (python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and sys_platform == 'win32' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -6449,7 +6536,7 @@ resolution-markers = [ "python_full_version < '3.13' and platform_machine == 's390x' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" } }, + { name = "ml-dtypes", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts'" }, ] @@ -6610,21 +6697,20 @@ name = "torch" version = "2.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-bindings", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "filelock" }, - { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14' or sys_platform != 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "fsspec", version = "2026.3.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform == 'win32') or (python_full_version < '3.14' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'never'" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "fsspec", version = "2026.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "jinja2", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "networkx", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "setuptools", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "sympy", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "triton", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "(sys_platform != 'emscripten' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, @@ -6649,6 +6735,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991, upload-time = "2026-03-23T18:08:19.216Z" }, ] +[[package]] +name = "torchvision" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-dev') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform != 'emscripten' and sys_platform != 'win32' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'emscripten' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'win32' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "pillow", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "torch", marker = "sys_platform == 'never'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a9/c272623a0f735c35f0f6cd6dc74784d4f970e800cf063bb76687895a2ab9/torchvision-0.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:7993c01648e7c61d191b018e84d38fe0825c8fcb2720cd0f37caf7ba14404aa1", size = 4255155, upload-time = "2026-03-23T18:12:32.652Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/0762f77f53605d10c9477be39bb47722cc8e383bbbc2531471ce0e396c07/torchvision-0.26.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5d63dd43162691258b1b3529b9041bac7d54caa37eae0925f997108268cbf7c4", size = 1860809, upload-time = "2026-03-23T18:12:47.629Z" }, + { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, + { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/f1bc86a918c5f6feab1eeff11982e2060f4704332e96185463d27855bdf5/torchvision-0.26.0-cp313-cp313-win_amd64.whl", hash = "sha256:4280c35ec8cba1fcc8294fb87e136924708726864c379e4c54494797d86bc474", size = 4319880, upload-time = "2026-03-23T18:12:38.168Z" }, + { url = "https://files.pythonhosted.org/packages/66/28/b4ad0a723ed95b003454caffcc41894b34bd8379df340848cae2c33871de/torchvision-0.26.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:358fc4726d0c08615b6d83b3149854f11efb2a564ed1acb6fce882e151412d23", size = 1951973, upload-time = "2026-03-23T18:12:48.781Z" }, + { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, + { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a4/f1155e943ae5b32400d7000adc81c79bb0392b16ceb33bcf13e02e48cced/torchvision-0.26.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ebc043cc5a4f0bf22e7680806dbba37ffb19e70f6953bbb44ed1a90aeb5c9bea", size = 4248202, upload-time = "2026-03-23T18:12:41.423Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c8/9bffa9c7f7bdf95b2a0a2dc535c290b9f1cc580c3fb3033ab1246ffffdeb/torchvision-0.26.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:eb61804eb9dbe88c5a2a6c4da8dec1d80d2d0a6f18c999c524e32266cb1ebcd3", size = 1860813, upload-time = "2026-03-23T18:12:39.636Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ac/48f28ffd227991f2e14f4392dde7e8dc14352bb9428c1ef4a4bbf5f7ed85/torchvision-0.26.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9a904f2131cbfadab4df828088a9f66291ad33f49ff853872aed1f86848ef776", size = 7727777, upload-time = "2026-03-23T18:12:22.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/21/a2266f7f1b0e58e624ff15fd6f01041f59182c49551ece0db9a183071329/torchvision-0.26.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f3e572efe62ad645017ea847e0b5e4f2f638d4e39f05bc011d1eb9ac68d4806", size = 7522174, upload-time = "2026-03-23T18:12:29.565Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ba/1666f90bc0bdd77aaa11dcc42bb9f621a9c3668819c32430452e3d404730/torchvision-0.26.0-cp314-cp314-win_amd64.whl", hash = "sha256:114bec0c0e98aa4ba446f63e2fe7a2cbca37b39ac933987ee4804f65de121800", size = 4348469, upload-time = "2026-03-23T18:12:24.44Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/1f0402ac55c2ae15651ff831957d083fe70b2d12282e72612a30ba601512/torchvision-0.26.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:b7d3e295624a28b3b1769228ce1345d94cf4d390dd31136766f76f2d20f718da", size = 1860826, upload-time = "2026-03-23T18:12:34.1Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6a/18a582fe3c5ee26f49b5c9fb21ad8016b4d1c06d10178894a58653946fda/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7058c5878262937e876f20c25867b33724586aa4499e2853b2d52b99a5e51953", size = 7729089, upload-time = "2026-03-23T18:12:31.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9b/f7e119b59499edc00c55c03adc9ec3bd96144d9b81c46852c431f9c64a9a/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8008474855623c6ba52876589dc52df0aa66e518c25eca841445348e5f79844c", size = 7522704, upload-time = "2026-03-23T18:12:20.301Z" }, + { url = "https://files.pythonhosted.org/packages/d0/6a/09f3844c10643f6c0de5d95abc863420cfaf194c88c7dffd0ac523e2015f/torchvision-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e9d0e022c19a78552fb055d0414d47fecb4a649309b9968573daea160ba6869c", size = 4454275, upload-time = "2026-03-23T18:12:27.487Z" }, +] + [[package]] name = "torchx" version = "0.7.0" @@ -6675,7 +6794,7 @@ name = "tqdm" version = "4.67.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } wheels = [ From 7312481cf0e2a2ded4e02c0c1025bd5ac122a512 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 17:38:45 +0000 Subject: [PATCH 09/44] NMFW-464 address core cleanup comments --- megatron/core/models/mimo/optimizer.py | 14 ++------------ .../core/pipeline_parallel/bridge_communicator.py | 9 +-------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index ca5a31eb887..c0d46179927 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -51,12 +51,9 @@ def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: Optimiz @torch.no_grad() def prepare_grads(self) -> bool: - """Prepare gradients for all active module optimizers.""" found_inf = False - for name, info in sorted(self.module_infos.items()): - if not (info.is_active and info.optimizer is not None): - continue - found_inf |= info.optimizer.prepare_grads() + for opt in self._active_optimizers: + found_inf |= opt.prepare_grads() return found_inf @torch.no_grad() @@ -75,7 +72,6 @@ def get_grad_norm(self) -> float: @torch.no_grad() def step(self) -> Tuple[bool, Optional[float], Optional[int]]: - """Run a synchronized optimizer step across active module optimizers.""" found_inf = self.prepare_grads() # Synchronize found_inf across all ranks to prevent deadlock: # if encoder ranks detect inf but LLM ranks don't, the early return @@ -108,25 +104,21 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: @torch.no_grad() def step_with_ready_grads(self) -> bool: - """Step each active optimizer using already-ready gradients.""" success = True for opt in self._active_optimizers: success &= opt.step_with_ready_grads() return success def zero_grad(self, set_to_none: bool = True): - """Clear gradients on each active optimizer.""" for opt in self._active_optimizers: opt.zero_grad(set_to_none) def get_loss_scale(self) -> torch.Tensor: - """Return the active optimizer loss scale.""" if self._active_optimizers: return self._active_optimizers[0].get_loss_scale() return torch.tensor([1.0], dtype=torch.float32, device="cuda") def count_zeros(self) -> int: - """Count zero gradients across active optimizers.""" return sum(opt.count_zeros() for opt in self._active_optimizers) @property @@ -140,7 +132,6 @@ def param_groups(self) -> List[dict]: # Checkpointing def state_dict(self): - """Return per-module optimizer state dicts.""" return { name: info.optimizer.state_dict() if info.is_active and info.optimizer else None for name, info in self.module_infos.items() @@ -192,7 +183,6 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, return sharded_state def reload_model_params(self, state_dict=None): - """Reload model parameters for all active module optimizers.""" for opt in self._active_optimizers: opt.reload_model_params(state_dict) diff --git a/megatron/core/pipeline_parallel/bridge_communicator.py b/megatron/core/pipeline_parallel/bridge_communicator.py index 79141073779..515ddf1743a 100644 --- a/megatron/core/pipeline_parallel/bridge_communicator.py +++ b/megatron/core/pipeline_parallel/bridge_communicator.py @@ -11,13 +11,6 @@ from megatron.core.hyper_comm_grid import HyperCommGrid -def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: - """Return whether pg is a real process group for this rank.""" - group_member = getattr(dist, "GroupMember", None) - non_member = getattr(group_member, "NON_GROUP_MEMBER", None) - return pg is not None and pg != non_member - - class CommRole(Enum): """Communication role for ranks in bridge communication. @@ -60,7 +53,7 @@ class BridgeCommunicator: def destroy_broadcast_pgs(cls): """Destroy all cached broadcast process groups.""" for pg in cls._broadcast_pg_cache.values(): - if _is_process_group_member(pg): + if pg is not None: dist.destroy_process_group(pg) cls._broadcast_pg_cache.clear() From 434ee4e174d0e8c1571fc9e35f6bc704219ebf4a Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 18:26:57 +0000 Subject: [PATCH 10/44] NMFW-464 remove vendored Energon artifacts --- .../compare_energon_dataloader_parity.py | 8 +- ...on-7.3.3.dev30+gd456cbd4a-py3-none-any.whl | Bin 325361 -> 0 bytes .../vendor/old_energon_multimodal_provider.py | 248 ------------------ .../core/distributed/finalize_model_grads.py | 4 - pyproject.toml | 2 +- uv.lock | 50 +--- 6 files changed, 5 insertions(+), 307 deletions(-) delete mode 100644 examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl delete mode 100644 examples/mimo/vendor/old_energon_multimodal_provider.py diff --git a/examples/mimo/scripts/compare_energon_dataloader_parity.py b/examples/mimo/scripts/compare_energon_dataloader_parity.py index 875d11fb644..2d39c678484 100644 --- a/examples/mimo/scripts/compare_energon_dataloader_parity.py +++ b/examples/mimo/scripts/compare_energon_dataloader_parity.py @@ -33,7 +33,6 @@ from examples.mimo.data import energon_multimodal_provider as current_provider OLD_PROVIDER_REPO_PATH = "examples/mimo/data/energon_multimodal_provider.py" -OLD_PROVIDER_BUNDLED_PATH = REPO_ROOT / "examples/mimo/vendor/old_energon_multimodal_provider.py" def parse_args() -> argparse.Namespace: @@ -126,12 +125,7 @@ def load_old_provider(args: argparse.Namespace) -> ModuleType: if args.old_provider_path is not None: provider_path = Path(args.old_provider_path) else: - try: - provider_path = materialize_old_provider_from_git(args.old_provider_ref) - except RuntimeError: - if not OLD_PROVIDER_BUNDLED_PATH.is_file(): - raise - provider_path = OLD_PROVIDER_BUNDLED_PATH + provider_path = materialize_old_provider_from_git(args.old_provider_ref) return import_module_from_path("old_energon_multimodal_provider", provider_path) diff --git a/examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl b/examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl deleted file mode 100644 index 443224314b832785b99394e0cdcb6389d9c6f2db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 325361 zcmZsCQ*fx^5@c-Kwr$(CZQHhO+fGhwJ14enJK1wzw(9QQhrhn6|8Zt|rmLq%K^hnY z1poj5;@>BvaDHj&6$1(YAdU(Efb;KtEqgyIS`v=D9~$d3YOY)KRw}9W~G!G-?yDt5ZBF zwZe?sfXJ*=C1%AHtXKlzu?w1C@%_>60Z^e+%89AyK9OEnk$Bkod-MIWkvRx9vOJMCt0wQcqtT}+~ARw zQ!U~Bfmh;v;YZTN9Amy`@4fMiNF^oBXPp})hTS`)WLZbWU}5598)v@|*cmtOep#P0xEHWH*`CU0D(tj85_#haDx3MYul?1uCTV=#Sd&6D6}RwX7y z;6YBb?LsW~Ds>O*)C>Ie+%{|3T_Fd#l*~#qjZw#HQ9DcY5?+AbhXM8I;nNZhrh?|5z7ZR5Dl z@V>=BxtFFl{Lv}@>?M%&Y~Wr;4VUP_PPp{QRR%2o1+G*^eF&R2;yzx|uSUriBO9I&( zZ?pX>SIzb6IcjOn8!exg2g%0H+ODPQc3jCuemXdDMMEzXkkxIT?!8xE{;7Y zaO}SGTSf`9G`cY;U`vp^eK?BnO2H7KLyY$K! zRT!enr6onpx7?-^FMocgKRuy(4Bg{(8lvqJd)b5-qfTFpYva1F5P`gJ^&Xj}0P|%C zI0%s1sTqUo0gu6m-}B3QoS|>odD=Ty>u_(BiRWe9H7BjT&JmzTM_b~SJNk318yCLY zDa`bq%*!8&VVuv82Lb!zI&W@E)yDE}40H(G%YvBb^-th}K4?q;m!O_89N5fDK)<$C zg=Rz3z4=x))jeA0=n&H8nl{tR7Bf^^ZVTE|JsTLj%2_+^E9nyRM_kJi>kXJ^ymXM! z;a%YQeZSxM#%)yMH}^Zf!hdz~C&D)`dT0OuT>=0A!vEF9jhqaPO^xksYz!ToP5-s> z9?fmL&2iM;Rb`oT?7xW?l5#IW6@SLLx@s2)G>65M1yC`fa5P)q)Zs9-8DXBP;1!-A zUnRX}Gxr_BQhK0UDgw6^h922ZeNN58$gi4e;9R}BtTV8^Z8-INGn?T3JG|VV({%m5 zo*u{K?d|LM{O-ufuRc!Q{d0TdjrAb-x}-c7N;Xqg0YCRYwy%8LGDU0mk|nd$O}87e z?lEwhAE(f({-zE)b4U6s``C7^{VriBuA8Bl|Z zil*+(dNDcToXYc5d(5=i77p7_Wkr^WDav-7ihj@zqSH_zjpA2jK?m})+h&mp90kKD zD71KLI4dYP^V2<2&9rNC`%CL_?wBwEYHL=MII4qG^cy$dmikh+xPnSwQfBrfRW~r0 zS}x50u3Wk@7~X|n-0facNn@{Ed6?56q{6It;F_AyWDH-jn5n9;50aPtS@(58=ns^> z;I^WHcsdqN0q$Jl)&u}rc9!hG$B*1qA-4zk>yvQV-^M~2 zfp+wiHFbM4s-P`w_i>gLnH z<-wS*~3tS{a`b zH?nbB2o$ZXhJ$O?>9r}lgk;^-q~ivYGPy2pp!7#IB_CMJhFLrc1V>P|yd`Hu;I)C4 zqfn~g9!<$V$oZl|cRs)kBTM!Kip>>(B+EE>M#Ev_lVW8rWpPGvtkhH$CXv7Z-vj(< zJoJRQ{neK-_MoMwja9JG!_XeLq84*HsF40=D6T~o8Cv@5lbZ*LX8Pe-5elYsK_@^bVE=0SY~ptZojYR;xZYn|(Zv*cs< zx05XnkxK3Y4#4qFcSafMkWYdS73ai4JwK+#zet@_Z2G9=fp9dJA*;+fC(r6Y{^fy% zVEW8vUxc=BfS+f6cX4d+izvf}C1rey=Y-)<40`M28aSvmd-{&URF^oTus}VK$3-?JSlEFCBI-86 zrIJZa>gd6B(K9@>T6S44p6R39T~#FpYj7}tjs6(w#1)GT$LmH$dqXs9cSY*i3)n+-A1m_sucZL&ASqMj}WlAYT@k$qpE_%P4fhy>Jt9)9bv;AG|UR3n{idApqo& zdmyjcS{+#2lgWAE+)--LtXjQ+{;S2;gKl25Ekh&D*v#rL1`2cFtv>D7b89)K763Qk z>yLy~!ocJ6%YHptNr_(2PUq|&Z&*Xyr3FuPolvejw^9K(B?AhLlk!(S&BWSj_2s@z zgK@7_A%%|4o9R5vCWB%xeG5<0G>m_2*AFs)Pc^cLx6}dJn#SSgysQ~t&}e$R_&k7H z@qb|+x%@2}*kE!OS7W%lqT!{Ib*XPn-dm{P3!vc}rFLXgftz*QRd)3oNKMQ42Msj{ z(w#U;H$dZj=*tnzxI`u}MTruJr=;x9!0f|x{JbfpfW)B{JekaXyUU&%(QXLh6Gq>bypr>c3dE?)~f#LPd z2uQfS<}18;%-fu8MhY8B%*~UO>MK^Ll1np2M@*yZLf6=AGzb4zx5&Mj(&zVK!i;8= zOX6zv&^n6o9NcC`A^K&MG6E^XwbYd_;W38G*N*YbK)db*A=hX-&s#x8ML=DkVFev! zxFSaiE-i1l{SERoto@cqAr_$+sEjZy`9UyJ_2|2x!mjFRfr~CMi@XqkWiJ*O0D`{k zBPD6n3UE*^#xK1vd>Kh8M7gO#_njg5c#t&+92N0ajUAj9J@+r4eC#8ud*4$lW+Vsn%@s+Up-n*kwS_Cs`DoU^ePX1XW|JLaxpktwv|?Hzj2x2JfY^ z(X(LPWM+;C5`70DTkRV0lEW|}YzT+JcIHy?07RaDrO(58?xI_$v@w_f2OOq4er7kb zC2`=HK0^Q6k+{Wohn1bHUv>l(i8>cGd3^U)kYlfkKsOZ zV4o-zEAZgt>$+2)Uk^OkJDWr~Q^b@ih7Bsa4cBoU=BPf_2gf!Bf=J_4%vw1)%0JmF zRf#^Y83k+w*`*L59d`NVYnb3!91;P1GHVn7jI##7&F~iN070d9t%#ihWD-^^%RgWl zQNLd4+bdpVehbB*RETLAqh+sri_YV0k{M0zSc+Jd`I83)>VX2^?Ow)+9)|>E?{T#l z&xnfHAT}cyjXgH3;4@%%aRUJ_ZqaLdhl_l|QdTeN!-?EAe^CKS-KB4lyT$K1&;a|Rv5Dv8@uxJY5hvmFe};$z#+sQ%2JSC*UajzeV|yL9n4<~1KZV)V~}6V;z#zxUI=r8|nFk;ik& z*tqg?i?%GX{QO-nM-25OF##dTB+>1!vL`@s-4uyefHy*dhA(U8(w9m+a_VuN6E)JW zEfaA20DSKjl}7)Qs3d|#ZlyRZ`sq85Lr5IM2*JVIIP_`AAsjDv3@$9BRi4dREGqnq zGiaubn9p3r3uQ9DqWo!dKh( zYl-_@`1n8`Z z?rESF5p2T|ov;`kf&jnXQS1K@lJeGX*XbR=l#{%n>WC%UpnAC)Xr?`&E}ztrX8Um870>T7{^w&CSG__23omO{DjX};5 zW3L}@pQ3E8a=g}~T5!)@&e<&yimeDK?}$`)QpJYb>P85KFnhrjjhUCh2$9yvb2eG> z&2|@xxB%@Zsa&#K-h#{$NeOYMSyUt24xhc}4`upOnkCZx-^IHp4R|Ue7X-c|qyt!^hbi>yuh$R#3V5mO zvU(Jr>Ap8*eRh?a2S3aacvc<;n=CRiCZ^WD-e&cTIz6oh7&Z8lA_H3vv%lYo)1OwR z&Fd<^^#VJEX=G<3Fh!7xg6OIyOJG26sBj9`lJw%dJ;`?p#8cCa^Bs`o@ zwz#%P5EpL2T7=nn;wFH)0T)O;AWTHjwBZ#9cNjGzhVMskNIa5+|4Pb{W(0;k^Xvu> z@T@XPuS}{u6*zC8MoA?GI@MqqJI+?EGb=#rx#C>s=u@`tJK#mkOEagnWEm~kAZI04 z05w&8Nbyt9NYTSy$jc@t%fRxuBAwHvQHpni8{D2ph2{=l?m)c4`yrhsbAc5>q6BrLW^+t*)Q*RG z1okvHd2uuB=;~#Vc4I{qV|iiI979#zMNsQbCmx_(U4U zSf~l+IxdF*I3v?};;#_H14JK?XI5v9JCt+Q7q&6i$(}X+tOzbhiU2}_ERZZ@x8Ng| zE>E^Z-L7S~))Yc2kxI6xz9t9x1}V9L*Hy{12RJ6cp2-j{wOSdu<){ZF2OvubC-|uY zVTQF2HAAMA4>Oc-IZeW#P*b>|rlBy)l!cw5#Q;Qzk`YC|%G$glt&XY$ye(WkxZ(!c z_-poHeXipP;&2gL032yLF$w$PlGvkf#=clR>9fV=;vF}}*qED!eb~%a%`we#^McMA z+m57#-XT)o$ zWPs9bcmRqpZlK~LKoQUGmU|sHKJJ!th+gdvB3Rhh^#}N#{uG$U{-Yu7lL5QjMHhZM zeYl8Yc<_az`c>@N(C);3Xx}!%kiK^xHp32>cyDa#lzi;psflHemrWbK-nuyK7trl} zax8Qk8WAXW1)J$d+LkcuatlFCWmr8ulQ&tYEb8{M#RKekPCr`8Pm{kbEA9HgZYfI= z65H;2cMAHF8BD(3?CMzU!`(}VXV1d%rpvT^NTjj_&nZN((9fWdg*I7B zsaB{~mi2PG8QG+<*S|4_>xD_pyOdRlP}B9XT^40(cQ|PD#vaz1Ujz5eD0KkC&0Pz3kSTuETGBq}I zv9!1QHxo@2WUU7U5PELwF`kklAW8p5D^#$e0F6YXZD?z-A$Mh5g{{k$M*Q}iOpS7t z@AqV~-_Bs9b$$_c{5I`Dpa+HTIYL#(Q^cS(#IRp_G<(D!q|S{t=!WO-WTRBTcqv9r zPRbC??3-eHG23T8P7)Y5QLt)qI4LPz!%j({C#Xtk)H(z6%67BFG@>+{u}C(Opllgk zAUG8-qr@B^BpkAz$Om$pJ5=f|*JoeuS({Q1>P~H6eYe`VU){)+Z#fAe|H;bPuoqu? z-deL=pkCAH+apb4-fv-7&X9o{>%^{ksS5{eUZ_laNEW#VNHl*TgFvPyJNW8eue(u0j;v3T(JEY(4l_;#rt0YHT@6KFt#ytcK#R7 z*aUs+K?anO-A78g52Hfbp+(j>OIw?huLqDQ*!^d3 za~4A*`BKELn1l4L1b^#(_lB-mv&4)urqiR}ox0;rniE_NTQhpW>50Tl86%KXNR$H2 z0OG4H9q?sH=xLbJuyTb5Bgc}oqtQhC=EF8>PoPr43tzCmh{!2gKD-@;!>F^Zl9^B5 z>>(h_ga{o#b)f79atGX8Jm;I0l5y)~1NnS%*t8~^^15qpLEJFZ(e{tf)ImxLu_%$z zjB;;74vu0tu64A5c#0_HF|lCQ&+wId!WtawJP0fISMcgycuvUrSIlT?*$wu3<;s+<6Uv|+atb=2}aJA zF9rqn5xygIL4yUPlPjV~ph;E-GbeqwF#04)N`Flha}?GXMQ361GtJG$dx_AyIirmH z6-Xq_kIEPGe^|-!`RaG-W+z6*s^#(PXW6Uc@zEBjn|_{N{v^ALPFhXKsbN;DCZfyG zO!TeV%lq+lP*_rHq8taQlN`4ixmVG+l~iESrC?WD$iB*CXkM(LJnzhK)W9Mk? zJ`OmHqR>jE@sg$gAu`=8n(aXDS`ntk$V}!17W_`3*hpYDw?eW%&_ofnlq@B=%r4Tj zPMtW`oy1bT^J??jaSGweTg0-J?uKYMk8UTCPVSM*WS&Xk5cJPhbM6ep=QS-P|H zd@_~uM4u)?#YjwUKRzlkB+jrLmDQg`5+Cd5v|H~<&;f?W=oJT;Jef{rN7WAwB%KWi z7jalfE4njtK6=*}+&~teezf!B6Oq3&Y+^uqayorojlKdtP_MG^w-cyQ9?Y(>*F%&I z1M-ikW>OY)lkt(jB}_=tw&G{bz7;oxkA1zrzCGQpJ-vTd`&Y=4?$8Pnuc2>Bl&aSC z7Ti+QjwdiR?(PrFNOr0aUXbngdw~@=2_3LF(^g z-*em_*O$2RmZz}r{+VJt6GTyQluq>O)6wHRcktTc3B5t8 z9392B*&TN}K!dGio|O82kQR*%mc)7G-M5zRBR*BTa1vhN&Gk5^m!J1yL_2riLAzAQ z?rs=@5&8F?f`qDnJ`IY^thrDc%B{({OEFM;se=G&S~KWS<>=aO9X03jGA`ITS!q(J z>au8m&#=11L zBETWnu7?D05H&+{f$4pCw4HV*de7_Zh#s9dW9yO%8P6(Y6gwNQz{8-y>Qca0D!_e> zK{o!e4I`?jH1gn$xxxnldkV?xvq54}j@OgV>oj%#Q+wEY@PFcd_y z%dx0Ra7E|fD_RF64o(iSGx_;4^JTiA{`oW{@+5J}BR7OF2uqJOPra?)sKg zjnx%Ztd>g&y1XW}!!X687iYeF?ZLb}3dTxX#JFXS%ew#tX;`0)GRg}H^EMefKu_xc z!e6bR6q|eGELmacfd=RR%I0$hr2~vA4z!x-AVdyYjK%Rt##K%ZmTOD!E=#bf9r|?l zP^=}ek_!jFUHkB4*L;>7ZmTQ`l}$)exfsui& z$0fWuu;@|lA8wStc?^XN?{;3HfcK&8u!cm&IbQ3xQP^M;9}tO7~9vR-gZwy2yVI=1A1B}6D zTv%ut_3eW=44GuBd&^7jy5 zO)F0eUbg$QXxob5eZfXs=m(0hI7^Z04zp(Nu+pdwLi!|_y7B0qL)$+q5Kz-C5bh(7 zdwb80vF0CQ;W1Uu%Am9WKboZj$T%&qyZE7q@xU4{a-4=L0&M^USiYx`i0?S@Vy2p-6-dx&|JC+pvMSpaO>W4f@{%CT4{gx)K-wfDsY^0P}wm7*9i6 zn}7TzS5?t|lMTUdwVs0lJ{dq8o9k*s0K8oi+++NJz%d$DlyLsW~j zaVbXJ%3*e14HOT(UAH&A#dCW)x8B6=t-&P{`^)h4epv&{pUWdwkf)c~m(Md!&06}V z3z%Da`d@r~DWFu*kE)TA6H-N*{n?w*@9~e1VlHUK5^dhLI6h5UqNxkC?SAyjaADu2 z7v^_jRE8hRUSPK=BwdImA8nC9RU^3$V%0FHS-5?#gUhj7VIyZZ*Q~xGiYi8+v1q0{ z(QOl)9)o7l1_^^-=5gH-0eKv8^b;r`UyWbaNG*Z-_N(d&6JP^lX#Hz2Yf+WW+zDtK zbrpq3D}{jS(4aUp(cO!s>+5@Y`3T<1#rlDZCh>$owV7uCU4pBat|2fO#*I9t>QLkY z8V6+|$?g!VJ=KwEJAQ+Mrp87;wABd#NVH1dQr`5sxx<)?v5UeqJ8)@U@l=sBY;v4p zt_=HQ9&zChWU3c)jh>VU=vTksMlPcZaDg;-**qFoSJUk>v}ZSEHQKhuA#XJsY5H4w@#>0APIM?1Q zZWfDWWVEsqk*r3!3Z)FCt7r*3u#H4=xxwBcO4YYH?nYfb_y7wmz@#WDye-LIILf-c z!)^1Ekd+xs&3Aa3PP{p@XGJz)cunhn0oMi!1jOKb(!x))wa zbg8^g)Wz#YnJf1Q)pQq!nInFq@PGHCnKMB+7YT#CT96xHbL9r91TPG7{vtQ%zq!nd z`g4$yJEAoMPX!|yYk>D9tEbqD7%iRxR>&UG`B6ID#@$p_a)Hg@P}*qGMVF&F`fD}% zuVgSMxI{vPD=alpO70X;XNfjF{q{s*ee-+-PyGz@QC_$RAx9%CF*{`oiIx^ZM^hbA z#wEUGtn=hKw7hw0@!{j zcPjlov;S{=+xxKU#DM?+U}FFP;QlXso7p>A>s#2{TmQrOwTAXTjHCPq;|_WyLTF)d z-i!%`dO<*7D`gVFwgLzcEu!loriuiWUOSz?y~0ZDQcZ^?+Z(p51{&yj@w*+qB))*^ z-5d~YI}~^7fZbmw$L@sP@1Bl58AHo6*PGd!@7o6_vPZi8U4K^lCakEh$0tmT{7wzRD_|8Hh&eTh>18nz7*SdD_j?W+s-|o6V@{c@9HvJ=g-EBn zi4FjGS}FL@2~zjoxy}e*0*r=y%@}Sp>jp<3KR?4~Wl=;k81fdu-wgcmX#kOPGmd3O z8tE<5)ELjbs`;UD1|&xP%FkFtm8`=+;1|veVv&78SldJRe{9|pNf-^n^euv_Xw8i= z2+NG64}+j0K^i~I8iqfFhSf=uSpv>q&Gy>UA4vfJn!z4kb8W(*Sg#A6$xmIt$zHHn zLMWazlu)9xxp*`#N0K#+QBxHl;3QKMgWK(m5QTCp{+%~ky7Nvmg@_vox*OLes-ewk zvlH60Abyci;;mFJ?I$tdCp+{sIS&XIegG1#q>>4NG1NS;Z6gPdRIX4u3bVze>@)+NuneQy{^0 zZ?8-~ZTD#_e|ye9SDbg_Y?EBxqla)q|LI=4d0r;w^^U#{O=eFuJHpbqX|aOE%`W(y8^|OfQ$3S1dshy2ga0bG{5eU2V&&JgxlID-V3LdChk71#9({ZT0XcTlG zyougvEN>pFQRq~{%HA%M)9>1>QGX@V@x^wZz^|z=UIfwQyMb#Ov!huyrrEZgNVunv zGhW)f#Ze7cF3H8{8@jf@#Sx)#+=0W=j)m0Yn#^YUMO+=@$H#|*JkN~_NFfzOy^?}qV~*$Irr#h z{JE2)leBoT$OTHrPnil>M9Z>R;?+!KEk(?PWxm7Lr4%*FT?O+NsEz|OV0E$eH1!J3 zoh=Jm@EGGf%vp-gIs$Gh6&2podjN*SCrxbaz?(B>UFqI}IISBnh81j4rNxYFsERs72qx^IjMyLyvCUa04g`&DEtjBtU$V1W0RtA-cZSSOyWheP7!QvgIGe{Ds3#@ zBU{NHk4(ATD#X(ovAqc8fTwk}icuJZq}mO#B_dN9m#BtfA+#m{MbocnF>sQa_2Z<` zmG~+Vh#xytydP7i;L=)>TnCF}8x|b18)jBl1q|KsFk@Re`sCg=Z-N`qL z8Gh$T<0Hn+$da%-I5AHcr!h}IN{yHEw!S%zfKc7T<5umAnamoVup!h$VZ1@0bTcFu zU`_f85*CXhAVfm)=WM5Zu(!axPj`QR7GkWY{Z6gKQ%WVTPEa*B;V%+J2kvRrP~q|% zVF_cRWL`8K;y$tQC0t{2d2|;hvkRlLK^5il=>7Em$CPaiKrW1oIV}JzmOo zj^G@}G&aew;)_U5Va@a?t$T-JZRRmNm>a-Ar7)SyV0R`Do=+~k*{QL>OVCsi^Qs41 zMPTqr4X`5RGn_2pB8PA{3miCc%CX?ABi^AZ$u?wU5!E$RYqm0u#y3Olwkx_#oUwY^PMD`iBKF=sbxrjY zLhX%U!jU!x+MlwDwITXf$=_z|?l;UFJnuK466SOGf!x)B`vvanEmkpM>-+MGrYZNm^0O{tFFfB0Yta(#b zpwJu&#Ow-uv6J2G-$m&jlrsCgxxl_oZ?Uhp-eN2+kx(euSpVg&s?W3T$dXMzw`u*! zwd<>2d^Z9{PA7v3FXMyJ_K>R7NHtS&!AM@gfiCdE3RP!Mlx2LGW`yZ|Tjl`x7-Y~v zF_3a01zaU40Svzm1_fp}AYovz0A+Fw%AuZ)DM3Z%q+p3fN*hGv!UEEmgeQ>PD=M=p z?+Jio4zndM65W*BXjNSpWTRU}ec&C1eC;%KG)$bVCB+PehPrD-Yh$a*P=(U;d)O-NYZN~53ExOy*YYOoG%4QflE6M5#YG7Clo~*}^>=OyF@=F6*R(@{1W91`Fkb?s)FoS1D`!X}#xd zcfTz!eANi_m%@Sy-O)$$@KagPl);CoS7bADatF@ed#T!FiL4&33(qPuWVV#V2n^PiuWMzHzD1tVo-jx28WP!?EcY3Fxns8D%^)8k>WKWSx^e)P4y;Q~N zgS?DGW&*Chmwlic%~id9o9h=-(YY$lvYr{aF)G~45B&!G@5*&p%FL|(CquCRlOg`6 zjosGX#MQ=B|Njp8P_=(@1P6@YTs^@SI=I7D7w!#dKkOOnCPIOAw4FK(1dw>+xk?5N zX3~}}!&P5=3l6tj&!Q)i>XlmL=)Q@PmU7_ks?x21T5BVo z-4kD+c45jIRY@w+EH89+zt9O^VEBe~aj4WDzic#I(O~Y~${7ml2JY|5U>`*XZ$Ynj z$?Jqk5;Kkui)iuPY(x-+Y(O4G)P{_L_9GSa3GoAt(7Vlg5@KL{OYT7qK`9FbeaB#1_iCjtXVo5ZE%e9@wDC9jr} z*0pw2-RU?v?;9i1vGt*`U*=1heUeOfY8I9&&bq~<*SgdAV0^P@S~d{mNLr<+;xMq7 z`M9jAwAt=^7;;H$DO<#y|8y~72nQG-9QGTP3Qo~D8%Gn$x!?vWaFf%MABk?8Fg>rD z<_q?*!AsgQKIB?Ge*ykd@HJmU{?iL~m$R{=P}Uqh2FOjPNrZ>cu7)oX%LNxj1@+%( z7e7U2O|WV|zA;!cvW9=xx}%L#7xor(-JdqMGMGz-7N8Y~ zJp96F;~$x*kor*6Jgo>V6@&&NX`R-`Jh8mW5_96`@e`46SPA#E!R3s{dHS;RaW}ON z(KQGXu9x5@)6E#}DqZTwGIaCBb4D?KN|sCaH6l+M--r+D5zkJb0j833PV!n-dm8Z^ z-0SQ{{SKtimJDy>R|i<5%NQ#kb&&G_J_pf{Y-(;Q`!|f25;`}$3B(ySO za=Q$R0mjxkwZCH^3H|v)s-njZFhZJgRm0#1)J*(~gHY!Pd7)N!xvit43!jSWs#_&X zd_67V%~|<@XOff9V_bYQjU>fI%&K+AoZEOISmbzPI_{%^+Lh~xW(!mrE7ftUo$iJ9 zjDP*3yR%UpE9Xw|@4q1b8|i{Idubg1kpA`$=>-2vkZ>|}aq`r6c5yN_wEZ`wQdJeL zHyIHA0iEM7dN?Q9_1X{&Ws4~bf`7c5m1qGz4)&{}f3 zNe6H=oOByJcv2qJ8=(<_j#>AppHZL77wKt(#)6!S5h8{0CIqW}Xe7|Y$i1)$y5+JR zuxp62jpb?q4Y)d!_-xu=gsGEA99WDl7uaY}jH%eYwP6TTnBvjOV-=C8t6u{Ja`Tel zCIj|>eaT$4hun_;6)ZXVZ;Et6`A z2|DJs61FWF3G#=Yfj|c zI2#DR>2rzsw9f)>s1Z}kBq4IwjM=(UlqT3~oL*hrD4R|nl+C(Y&$RfM%1c8|nk>{N zR@_g&5Qf4kSeejwwtHN))p8Vd2%;$zEBA0A4_#vUdxt`?8@|lno#h!{8${whh<6?X@WONHwaU8>bb?n}Sz1Zo2iTfHwRj=CPN~nlqwdV5UR=*9XKUT)YL5j@k`nEnYMpjd(t zKe*vRNix1QY5wgwP0C4Uc>$xP!e}z{@Z60T&TG7UP@wME#6}w?^me_!pl5b_y?Hr# ziPG)s@c%sdvGV*pq07o?4FALa%l9ieS~P#~#fBJ~Sczga`}5=A{B!;8I!aAThMb8n zWxAGk=}Y!!ruaZq=_FYtm6Q2d5RLmpy)MdY)Gujcdads*P|fUWW?;N&Z#{-LLZ4c3 z*l0BES+M)C3ROYnz0eL%rt;u>k5H;#X~Nc2gh{6&X4)miVTfEVU+1b_(G)RKr~>!9 ze1zjri(SgiK5xC~op{%*sPNaEjE@(?uHZFT+tX}UxsnfR*~Z&Gzix^swQIs;0xfoV zz=ac*JULQ%GI1NdtDhPE6gh0ph3i|<1pLm~#j7c#yR4=nR*@yfEXH76<~nrS)Fkbk9LX1EPN>A5 z-&L~MgUk4DLO8V*fkl91aE zyPJu`n0}cY2&Nn&Q7UH9h5id|i76Cpq6^GuYD@y|yxE~1*gj>P(xIvIVDg&vjbhZtOhKd z0Kc&y$0gV3;AB7qLz38$gD*AJPCAAlcBaomlY*u;?<-F_hFmEZYK+W`2%^SD0xjB= z69#4dn&{^PONRLsR-u5taQ&=E-2GT(VFQz?XKLYC8fPd(U<&kqyL2au@-tn=gJN6QR3$uP&?p?q_HKg%#r`hP zF2B!51;3mU+o9-c9rX_BbWCZJ-gXSNDPj^@FHanuI5Yasu(i5b`c>oglQ!$8I*<4o zPz6W-DQpQ^ zN0=Th*#VzUytyuZh9J4khqtm`hW?XP-mja2d*N8JBw(Reo^T#bpb;*-rGWhK7A~i{ zc51dck2tfxnAMzoA6gK(Vwkkk=??mydo^*jI#OzVa zH1=3jZa<1~MPx+tRN8gUT-0p>HK^Vxs~73M&2J;g>kfw)mc&NQeHo7bL)SS4X&Q7} zx@_CFZFbqV@s(}cc6Hgd)n(i6vTbYnpP7h>b7IcL8<`P#lM(OD%C+~ic2NuaM@|Ne zu!H@zT`iPQEfj>`2aF#2&zDAK>fdU7`*aA>sS0r70+i7v_wsxbrkptri*q#M5_*uTn;(iW;iK!UN2j;U^@318Pd zL#xGWyP{&od+$FAF};#ln1&)QUYSth;Qm8~SgTOmWz@IGd&N#pI15WF5|p8JR-1nv z{}f-d@@S&-yzRlpDmz9-c}tl7D8iU`xOh#&R*LSQFGk=XJ%!IcUOmG0POY1NTqvAR zvRxAXPUWKVH@rUc6HfmRc9bjUY!QE~^j^8)*FqITrdz zq;ofc@1hR}{?S6d6CJL1luFPyJ1P6+im+1(i%pdM$}5o9G8$@yn7rR zW&%%t+&e+98t%@=oV|&qTD~l7Ga+kCiN>MFNFSq~L%0jm;^i>AXb#;gms@yWcTzV8 zY#pG=LP8A|ISAg0pN`FCEggH@F64jJP(x0{4Oj!t-ADu*2pWZ=YwASA$<=@xnLwZ%+>&y@-Ct#4=>Ke;G9kUz1pwDg3{mzIL4)Fz0m} z^=BQkd#MQ=hcn0~RgrSXVnU-RX@4byYt?)9Xzwvf9ncxG1fX{l#5bu@HReZ!A*Uvd z8%NMmsAh^cwxLR2)yWMlFr&uTR&2WTBCz%c>NQ<>!KY78nhQy|`#V^?mFWJbe zwde9T#=1PMGSl3T?S_f!Rz|8COMiDpJWj^g=mO3RaJ-=bE zB)CWs8C%q;kEibj@@c*D1dxFP_A_KK&*WlKmg&ITS{JkU{?{8m6rv!CvK;CghsV zU$h@Z_8ny2n3%*Qzgr#l021R#aK+oZcf+4 zIKUjV0%4jZNkdc7U@GASi%FN;lV_^LT4Lzp>?c5K5FRSHUCSiXr1%|zBkp>p(6EH< z(puhao>Yg@TEC@S7fKV6Yy7KEbE>$QmMZ%6FO}YSz$bVZrmPY{L#QxZO>CBL6++fA z&7_>%)`MEZ%2QD7y`IMn@OgdBu4224O#QgBwSgi-Bf)oQ&n;KH)IhP_yBAjar0cuv zF{g_Q@bWToA$eB|N#8y^{8Pr&(Vav zOSW@yLB^NOv3kVo!pMQbs;MUY>cZqYgs`F7YS>m-b(1Sj?~l-2$Zj6BLT1pbW&E!B znGK=4Nu%$3&Kh?Pid=9MO%V4jR$sp2eMVKw1zV^3*B}Ysi(B3`7vI|oA*j=y2IX8` z9$6izfwzd>md|~Nc~fyUa5^$$5wgWrt;S|~#ptHHm92SQI91;1sp+{n?unPa$|g6r>zy;)BJ=YErIYXaw=9lqn7bLOd@uwJ0>oX0V_ zqDQw5MA}jvNb}G#C=foK$-U<&4dckRKfAyTr4<&aktHT(x*et1UE3SFOTBKp*6oVw zMemq#jg(`XR+>M5wG8kF5>160^MB3BVMbX!!;wAVTf3W-(>C3k?-j-Ix0M(FhE&d( z`#o*O>mHpmlsy6Pud+7wDz$V#+ob!YtHceZ_Xr5Hu({}_rm(*Trk3PT< z{)t0tiT+9K zWh3SqkfQ9$eGlEb*hx@BxRf_*)z7`lI+iZAd^vZ+n%CZ!7nn-y{d~LTbCf}A(P(^q z3KPH{nbEe>p=c}A6hjV+Sy=)7f>3WKY@4N)_Pz9hJ`M+8s~|~eVG6LT@{OyN-^TqE z*PY6<)f)yFs_K0|1S0OXU4m}Bg+z7U2~5Tk5^vYA|B$hW)8QG ziVu<GL#v3U`xbj3S{!A4x5fOyz zU_#h7ItaBm@FOh4BKH>LYy-o=a4IARxmSU;@Ti^#R@o%Ao@!YD@?PLQp~uOv=QhOwzuK}U=&P8 zZfzcF_0I!}ZN7Pbr;Z*-m#G5f&nZ!MG9-Yfmm+AwLHlw>1h-R>T9bP*P0H}&?xaYk zcQWU&*!tkPWEHHVTr4f%VG)PdCA^aFr7UU{$d*!6@uX$TF!sGrd*Ny>^qsce*Gda$ z)xN6@m1*RFzz2${IgC?H@t_p*MlDXT-fT_i*O{JT=U={fzBO1mZl<&L`VH;y`N@(k=*tKc_S^4`_9 z1NH;c0p^SOBjTunWc!KOMkWX0K?RN&YuVUk_cM2K7pZn0a_{1$XO#vkoTm+bB-B79 zjZ6bY_A~JfDKj2D-kMIRA@YpuqjP6r`4*^lkdL}w@A_IN zpo1=2U#JwWU!P5(!MEpnhGlyJr8<5v*6`AhBt_s$wAl+u5OG0F;|U@18Rp*f%Z7!Z zU$4d^mOjFsx$35TaBmxSwfDbyMh+$6uG0S?B(r}NMcn_NXXI?;W@K#hpHLy+1Uc(q zA*8UI58ALp6ycKM$0QNULbQ;i5KU=&Kv9vMmfkwH$fCgxJH<3)v-5iV`b*BsIm3@rE|ppE+YENsZT1#5bKTJ^hZzB6T>Cp;i`xJ;tdwVfqW4z z21rne8p**sWd&C1m|!1AkGY#0)3Vkk(k2rUE^A7+mO?iO>vX1SvgEqV;`Ac2swolLbgE{Fz`VU%=!g4*Uej?xx32rp${J$Kenvb<=B>wtCuv3>a9`c6z-?Rxe%T{w;n(>>dUgtJ0UNuSJ=Gd=2Jb6uJ0}!#J?lM}SkCpnqw`G9^nb-0 zpFfPrKM=v#Gy4JkSIPSPx$;xU00dNg@jvk9|MwUCzq;6!?}an2MAC`-7dpSk+*m=Rl!N^T1+?{rd%I%fNL|H?`h%>AQ(oTbj^tzNt0x(HZxep=yC!GaJ=d29zEzh8PncxiqqxNXL_a5b1eUUW%1~#bTfGe&HUTyKZ*PORGRJnx$0hsBFB9~XwN%;s7H_L zYtYIvAAeC>UF9?0yA6A)z>fKOVbi2vDd=upQ1xmJt~D|%o+-U-*0K%o%u=)7+U(5O z;S!_(SKUa_$dX69hRkdJ3wUonn*i`q-!o|B_=EcU8=Qn%jQp*rD(Q2tIn8yPNo}9$ zDy^kh`>}p?&z-donfSr+{{%~sCu11uM1H`lC~Pd(n7j5M2!zi*U@E6OmLr)o&e&O& zBFM=;-z4_kNgeGx&P?Xj^rNs{=R>>~5+| zv<#{oZ_jClNFc@Y^rG!X^a6pU8^Rgvqp4x3mT0fc-}IZlwK&w8T)4P!`wo2|*v;HF z!)uU+;i6J|6{HZd)79LWXI_`9>Eu~Gn0d%+ao`~Ai3P_B?ltCvd{7_#$AO(DR?V$)mO}JIg3$=rbVBp6+GRO>CeNG=<5?msASY-emaW@&ne>C#ousggJ!&Q zcvmYJO+V1=6_}3ZCSscR*FSoO5}nI3O&5&D>QBr~(h?;H9L0aQyMVN%$G!dBj83VV zlwYbkdG12=SDZ>R@J`&4x`RF8qJdCbs`DkE9FuakQfw*6`VUBl^-)PR>x&o8A- z>l4tml!7kL)4HHhQk4Y+LoZ28D%v5WBF2ep2(Z;)OG-^yW!?UfcJFM~O43xGpfS#& zd9SbvfhgJ!fmqTk+EEiw{D;`|@W1h!{L8e7NxwyE0@_NEV^2BchEu60 zv;mn5ra#{xtk{x&ujr14A6c(Us9(8N*CJJN`|^>Ae_Ge&wvYA7Hh2;|A&Z!054dTH znT9;mjWTU?7}85~JfR7D&x?(p{b;HJuH>MV z*;Yi|NnY-i66R|3zb8)gJR#r0i6=K2&CccWlw@TUwFb&a1oWDmb>a|b-dl})A-GnJ zmY7`qIbjNTS)$CIval8;HTGvV5*Y4uD@hkaaCl?PnwsK3N3hMWJ%prjA}o~d{3OWc zZ_3uIEH>~NEVK-+mS!xrog9IBH)fqna7|0tE*gD+{Q)f2EC=XoWo;6T;bw7WusP_>f`NQMPS#NjAa=;L_i8;TO&KkjXCQy}zdgL}Yy{c& z=?pixX3lx%*#N|5jkHudWkziXyD)>$fxF4k_XR8iT7S9=npomSglF>ijMhtU_Bg36 z|1y>uD3$W3p>xO17tCD^;=QP~be(FLn(CXEgjetVVq|56@O4Y1bM>-cy<-9~IaoKK zA_mpxq?S+lup)ae({#rl%)r%5t8o$3^Q7yK>8?_%F|>umZzbw@-iWn7f8Ge5m+wRjkhMf zk<$War2T?>{Z{$fCu=@Udp8E->+FX<-Sn(cfy9UY6y_$3xL)=R=!VRIh*fn6F=P(j zN9&7T?>w1ha8Bls5y3-!LYoZAr?r|B2timNmSx#zimUboXK1G8(B$&k|D~4FPat=q zAn-my6^Kvx9$6HvbnN_ zM#m^>j%d&Xl1M56Ma~^Qc@|gYC*|eNwD&t)nW(H$E*O4=!03XHa4AEMg4AR@=IL0q z;xZ8_$Dp!L-~BX9-_k*Pg@2S_0Ogt!r7>;#i&^-{{j!y@t{~?N>4l6`tzW~THqHxs z*)w!E$Ur#2Z&PMK(8VBEZ2ep=T~14T*wGjysIhby5c=CXYK;WtZi+isVnL7t-oBBF z!Uf9w!`-Ckj}QzoawabpElU`&jIgs|R{AN&xH>?smV-85p$&a(esg2Ldj(SwWNCd1 zZsM*OO}#8CX_AR;C&yVCrmQm4^v3>t8l~4fy%Vlh=(&Ab-_$@~em^$o+&op)M90j( z%CS^QJ$0>Khj>LYD>Wcc%r!JrTB-HnPd~j(m!5&s*76h$S^0U<=cqb&b1=>c2p!g| zX8u$71LdgFGHoBg)T=(-vxmk^)rPi>tMDTx`6pP4AUsvBv64yAL6tpexfwdtG}Y7V=oMVGBSdvUqj<8vWTd`iTQjMJ zJqzY5`j6OddxDvhq4}hjpLGqf7o+d}wfCzNv0jT=0JkvPDE3!L)8(e(T(1`ghR>*( zORzBIIWjqa|qoHb^bg5ZCVxZlopTIC|N>bUCnIJq_zr;KH>}%-qf;zC)gB7GO_dnO4s#j15}p z7QY($JN^Q!AImVQ?f|9)>hHiVfNrBG6zMmt&x%w=C7EtoXKeCd_y~pT6GevmEn^1* z7Ef`+vX^fX+ASW|H!H5p6s+o3B6b8hkae{UgdiW#!B{76A*Zks0QKNJmBcI9wVOAk z+L`^{KGJ5+5M&2qc&L^=D2^Lnnpr_8dJ zhET9v`wu`8;dltYmA*>@-UllLJ*gQHI{mUD+g1}#cHW}WNf~>@kdg#?z(~u!^Rbpt;6)!}&ar9_RxV1T46`WQ6!>gpeNZeJ7vDJ1 zXh7nAg!2*c9o6vrIYC2WPrf_~+f?Dw>%f^Q_1CY6aFKV&<&T_X`~e5nE$TMGj}IXu z;&(ZhLFkg4hkZ_DnFqjDqw2!Npud|%TqSB78EO`suvW_R&&^!!FSD)CDZ`Nr_iTUZ5i6a1m3VSR$uiQQ=-xxYIeF z@WAK$MQ*)a|6>W&NOI;oZ#bc`)fes!ZT}lu?90(gPI>4C5Iu;H&4tg$w6Wta+0;gA z*l&p1PRTM8)%gw*Na3dCmlGT%sbl)8F>y_bj6`DIN4^zWr<`l2@2*~BfJOpj{au#NJK2bL2tfY1-Uf%vv z5iXfyi7N<2(3E?@_q2IzQo#ORg~|;=@ra;^oHWg-p*$ zk$G-aY%qb8cPTq6iA1i*1=J2d44l_KzkP^0pNMYv^Kf~1X!CN`FAu>BD(XuG0~#J` z%T8^66-Z;9o6L#CW>g>GaJWcSXz?|!-Yj16_NgDXrgNF;-cOLPKb=D^aK8Bx$s$~4 zj((RICXDNNkw?O?<`p@@dtI8LQn0N|q`Y z^-y>9VoydQ3cK($*(8xjLBgruy?}|z)B+;aP(i*U9)q{?pc#GkN|Ot3oY*NlTV<60 z_l(PJ{TiAhemLFIkz<&o{eI(DPJLXlC5e=Tfe`Ik?$cc?6T~ltLMtUiiPi(RYduVNYs&iy;21fSeWqyBg{z z5If;kAgVa|T2WEw0Jl@UgGY)Xmo(KHu}LOoA}|Peu$LX?p{RxgMZJHD+$caiye4_zIMZg#-oEumrQxeLTMP?Q$Od59!|ivZFZEudIpZ6=AK1qf$j= zZ6|k?k+V(n30(~50A<~gm`5^~j<0QPUTTLWJESRf&Efty1u2g-n&(c6APgs z`Z|nr`wS*)t2k?`#i*Qe5;IE)@(-hndcA01ng>FHl8VLUWHe-BnJPc znE}kNXgZ_|qA=fEGNM&6kF)`vSrP^K$1KGeuP&77mV+kq^!x`@Z4DJY_jZlM7M4yg=4OfjCGYHHOloWciR+I6S&qSE5d2 zOqgW~5*SMGyG1y&oe_S?Z>$n9wG>d*B&K5ieK4t!7TlL8zQY|S_`?$=i`JVS9(>cYP>FM&jjYTm3$aNDBgJnF(t`=7h4VfJceeRtuw3BQY-GRI`PY@IQbxj zR*$pK(jI^?mCLg=%vE$jN>WaJVbf0p4fA3UowGhKmPLF$7MCmFr4T^+4bbCoYF<;B z#;m$7ALaLHo^oF|{z9)PDIp3dfqO|$3(@kyCDLvGmM4rdKh*-QG$pW^&CYMhp!AZC ziGx302!$Yj&3GgPu-7XY{>3i8tGh*O;2`+kh#F|sw4jWbVKjABkwh`n|0tKPxc$vG zF#M;2b)X#L{gXz4%H~Mvl)1*y-LqrJPF8hN98J=3dG1q6vyWL8^TH79sdB3tS)ofT9(b?wHu4v&EEUJ=-?=<{ zQ2zX&?%TaW1G!(KMAp_B9P&nFiFjomOU{fFiAROK#W+3L4#AP|o%fZxXgKlHWcmLhP_D#da)RJ_bYwOS;zsdslnRFwuZe@VpJi)u zuFOiZ5+?P02%KQvSebMh+s5(V6PHcpL!66-QPF954Dkxx&Sn@B*Ll0Q>du!Qv>6f_ zw`e%@cz{pYo=Wk>OV9YEJRSl@FQ}DyVQIKmo<-muHMEDt6U@Tl$N;6i6Z zuLnkfmiFdfk5ERn;CmARILPAxrV&6S!NG);145iq1^H5<)-~ANYCWQwndQR=eMe{Q zM?yLEPVQT(vr2w#Beu93GhSgzpVAlh7TEr();D6z;s$6D;70=yQEF-)0se{T(PR`I zz42OAs$tI-8m$u<;Ob9d^3aDli}hOhR|Snop<_>Rv{9W@!^qqV zgh7NfWLh%TQ;JauO6vBs!C{gFG{L@wMFYjlMd<=&2xWTG$Zidv?c^9>F(z@P4KI91 zeNxZ1P@ATgfkj71xD_x{)MkR?DaAK0`DlYLuw|nt+^<(zve##equ3eZ7N`|6En|ms z2;V37!J+39qeaEoV$fGs#rMUfLU~-pHiY52tJ==7o+^RMmTC30-grXzfh`l;v}g+0 z)ngLLhvO`nea%qEHm%CHFrA`+_8Mu6SvGRq+z*;uRdASbU70u{GJ2rck|Sk0+eqQk zMxN*bSB-|+6HX_+jQRKss}O)`Wz|B0`6Xz8Xo( z)N{@=tL9qUHT1QV+r2dSIgh>@jUG4l8ZkNcy6}z)s5;kNr&H%<6spL2<+3@3Bl24E zE^jWs{Gerl3%VSGjMq}kCLPsv%Vm>5iic)@0}6fH%BacLdcK#Xs9^bvfj$Ak#R(TH znLoMX_dBY=thvRnz$?)(P_~DxTUN0USgm>nHWZHjW+*tb6LL%4w_{Zg75ojL_g(Ca_fFhZw5jts>EQzy z+D!(X35VqR{PM*T!WhQn@tv9zJ=VR*=xEuk32H`Y zvv-#2!HTl-Ddskh7%qq7+hqe6>ghbIl2hd*swpeZ6B{A#k9*27ukP)Efvc#H|Mj%Yc-t`TNBnXtuYYafV#emPs|OlG zxYP$DfFRLYei3KW%66rF$>F{s1&)oK%)&o?)*f4k-ciN49@E-Tb?e0rBiM#aPV(b57ufWv))s$6 zHNerUB`<8+S;c!PyL9aGSUTp>g^)Jp1H$_J3t`4UaXPxV8nFQ%4*Tip&)$rGO2jBC z3gI#yn@nkiS=hR!xV(tv86Kl~x%Opb0iNd`j=*gd?#=_7 z+BpfM?9VMP#^>0W0_G1V&fA#?nD=uOT>Bx1;h!EoAAjp#94^L>MYkhTP$k{;D)s9~?#q-)OO+D>>T2N?Fsb+=OONs8jxe>&^S;>SI@QeS&s5D8gx+J8R) z>$}fvX3|{klDLr>l)0C2kw0&XJ$M=;Y_9bu$-O~(P`~vFDBq{T(TJTCGpc$7`_VeM z1HJUrXDJnbpHz=Jx%9O3>M_FZT{09L_Vr;Y-MLZP)o*Fx+&lY{|6Mj~K#J`ynQWN( z<7o8`mQ?O*hlX31AQ7Pi?m`bpfIA+(&t z{e0p?66WH45ys*!!_5~@ab!tmXDt)W4TPp8b!pm%48OVx2~2X68SmQ)(Vz9k7(=OZ zMH`FH!c60A!SXZ_zk;xS(q_!Ew|#Jh;}mIZOWKbBv&na__7HS4ZeMmDHYn5wu=^JQ zkKoyST&!|$1lGj^G~uISI(XAa)i|H@v4&8*J+AXvK!J3OSe-ITEojo4QGt;cX0MJB ziA3U0%Mq#}$0TEIX5lHR;Q7!Q83R#DjH22){j5Jo?wfhRV8Zw(C(;!A^|{Sm{$F(; zU=)l4W;DA-gFICEi(oO8FEjITdr*CvvS%(ulwCWl5(0BHY@jN;j+7H-rVY7p(mmoH zi~{midvv75e*pg}*24yO6k3ziQ=0-HO>;0N=QYJFl8mM}n*5G3?41@$j}zEOzy5nI zBxX%HCoo^4;AyU`BIg1}F`n5KEY#wa=tHPEif%=~1%gCv!DEsb%eDt8MzCIMIF7>j z;%}^oMHH~<`J9|8 zDFvjKANIw5+s|Rr zWk|l>@_E>J8BEG(Rs9JPWbBSMTOhhVzJ~c8{r&EYJ$?dMBg_JMM8egb-30sG6&b%D z5(173XbKGms1H==5jZnW0elVmVMPYy(nb;2Q?Bj6)Mq*{AjsD0pQm?)@d*;~2~?Jr ziAgAN>WifG1eD|m2?&tV;4(OmrSNYz0nr3YL{FX4Z1?!KqYLU$W|*D+RNxosIz<^IPFCrp8hr2T%X*+A=}4j`oD@z2VSeC6S{9H&J>BC#Z+FAirL(PI z;^TIxA=Gk57~2X-xh9?}6i{`L!{y7KygPR!-N}?m^-Uw=i0BA6Ft&wC_tfzeXn;{3 zz0v6wqd#NX%GPr#A*6O)hqt!-T-r>=goyamy*h1eLRLm9@wH!69LY~^XFWalc>l51 zw*XVy3a&O);OV_yfeu_PPof0+V(_`TCw>KFusk_ZtwSERSE=vYk2wwaDk!B z0p6kJUGu>9ySiPdx1pR6-{fiPG43MST{7zY;b=|J{f+YOKhY)XY?ne7b)&Lk0$9g6 z5|_5M2M9{AH+}rGtPaS;K6O$%BY&MePx}cVK~r(TOtNh3G4U)wY(fI}W-0_+LjW-w z8TXyHe;yAUo)I4IKrdtLK{Spn=1Qmn#vU#W?k*!;JtY12MX2A6i@-R9Itf)y(%X#o zm=4%R3hW?$syn9kYs!jF#X+}T!i81u!+JnUN{hq%-$x93Y)M3sM06dik)>WD4^12v zwP9_EX;MFeQUTt^M3$yO4tpA1mkT=j3Z7C8=zp$3L z4i=8D&5pJf(e`Y6GoDipLEG={>OD#zR5nl8;eF2inN-M=NGJ04u2(6Wr^-LZf@Wlk zOz$|eBQ^AQ0x;q({PHWMhoj;urDeh4I8}+k)GIs zkOR9B3@z#Eb$g3VFw4vIh3V*Mx_%R|aluy8{8Ax&k7dywTn}=ik2l@l0>|pX?gK!^ zJa%!BhPJ9kq0%4e8YPD`Eeo5(ee}=?hdPA@{fbHgVHc`Y)*2dgC2n+>43x$?S0z&Z z*WfVU8(8grq-zRAnH4{9nRr2BJoNq$*gQblhamScvTc-pRu1fzS;$7>R>G^LfGXN& z&fNa8`3hv)8!s~z?jf9*HIyERU7j1D>>s5}lgT;G_Jek`G6$Tt2J7_*)J)5_jf4%% z6H*aE4*I4vq_iVB6voU6$^0}vK!0tXXO0Gidw8;g?+DQ2%0F3_bd*m{K({%KaT|-sLu}?j+`-;U5yBf^Qy+V4bj{f*WX!xv`X?<*VQ$u|gu2uvBFXg~QS| z1SeHyOEk~QM2fZ$D+kC9*t#WQuu-2rb9KrL{S^3G*eJf(2c0V(loO9tn9jTI#_@|j zon9X%nyx>5p01wUk(CTznGr$Z@KQUd*Vu4Qy2mz~Qn>lg=9G~i8;YkJtfd-(LPnKQ zZ5msT(~av=nzp|)OMRtT9hD7+Br$BdhPf2OB+!kwpuof$u34}(Tj$8}d*>tDCp{u` zi7-fWcH8JFw|)N{nr>LuSI&>IfVc=rGDgdVBh6*G2G)p}57ln(CVC5OsV|An+Un1r zEnMj4nWu9I5WtgyI>?$QLd-um(MwQ4dHk?a;)wY~E%?!I#5K-!7|nK#z5GNx_>R~SK-Lgsm?2gx8h9sDQdyPrFh%jm14#5R+_kTq$MpEdGh1IUFx z)1RQQ(6imE3(L->%ZR{gP%Qgcn{*5S4;RPv&;X( zzCDP9}f?xPmm{V?U*`Un58Vc*l=b-(g7b60p*rZBtA0Am|2>BMdx6f zn4fI|?zaPt4%*p}=7c{OyXW-S5h-T-Suufy#^N5u>|<=K!yc>*eG9Jptr9oWEv%=4 zMK5bANc^FyJd3OuxOsz_*mO4IET}NJwr=J(Oe>zlX11Lw;kbRTbP(S{Ti%QWY`I8? zgs9uW+m2@WF100B2N64B4@F(nK?9U5;Eg5NgB=PN0}2w*K6IKCe-<;A!`YMrO%0ix@| zLgCXJ+6VM0+zfSvsOEk8;?Ab=pi(qv3_^LJ)Z{3-lIv4x>V;lWQTSff)NKCr$44Ly zjA&wOl#&0M@y%}&F>BgYIX?S0qnMZ#VXX4>MHxY|@Rt?td4{9eXvBuY&y6?G)hQd~ zv>A4l+251{MLo1LnMJ)?S|~S4m(JT%l8#wxe6=N8!{Fuut02S?c*6OZFE!XlAqq~W z6_O@$)5`t1gfZ{xrzI{^w4?UppSO~RpeU;^G_%S#ViU}LWDsHGgvFAo;f{lSA)Y0C z$0|ke=y8C8d+LZ9*%0aBXsfs!Ts>`-)rRrAe1+8(6l7|d^RXf^uVhlcf@Z+*k{Y~e-|Uv{Q6W|m$KbpQAm>BE2wO)yqdeJ z5^+5ld^#Q$dU@6VdK%Jh+>B4@f5MP6qlaXL9?+dXYsv;?kD*SJ>Oi6%6~~4-vP)Up zo4au24oIe+yphe-JlTeY#w=YWeuUIws33|@dU5-dF5dMQo1b@6c4P+uOAZgM?D#Zw zySFFTuW1xi9@>B4Vvdfdv`T-%dkv9emmA@K)`P|IDud_i++wgP(Q^RHBDee81Beh@ z2-Fmq|2VCHbak9C+ByWySrG|aI&HpSeFf|F01TdLYX2sFJM*P*%zI>JsncE2`WmIjOVRLsW%__o^*N%+Cg+jL#Q4%qw>)1|)h?Zw}D>?hw) zx1vSyd}=s6Q|fd=*QNS4J$q)}wDWAJC{|H72s$Uz{gH0-lpGq;idg3$CD^`-`ldhn z=9LjDX6k*asFYxG5rFS zz26simwZJZizB|Kr|}0ugo4mE!(cuLzNAw_>+Bn^?esiLSWMe~s&KT40(jL0isuS2 z>fB51-Lg~Hn;_|azH0gxx?h+#cvd@?`gHl+7 zC_QA8Ujm@_AJ77VRMqaS5iw5m{|NcKGlSkq?L&y~eV0$gC$scICs_E*5W;**kn?%X z5OS@?|2}~H6 zK)vQM_l(!~AVr|NcpCdz)^Ow&Ub9=zd(A3+pJR6(HWv#0c>TSld3dE|_bie7rEqM(09* z(s>Up-{5>p|M_HG^w#UQE#3s3_TS>0W_ntJVNzcoze~Y(0Nx;gwkJFXafYCQ@?RQpX8 z3sLXI1#W;ufG8_&2-(65J&1gQzfoiNwu}o@RZfU`W@lXxKu}_Wk7hf~#UzbH$+O60 z=cNSjE((UnHA}WmEuTQVz%D&ew4V--!L;gcTON90ndEqhg1O5K%1}tU4telaWjwMp zqrE5rVVLU26cyJOftqAL|F`+Y9kX#NRsR5k7s&r%bd0mTtCI=f-za)B`+uN8x2oR1 zG#E7Bm%4T`MmH(hv$f~~fYxSspM}g~tT2;!OD!t11WZyRhl!xyd*i{7rRGchpP{4~ zex8?H6U063>b4~MG@N~w!03nQc+&O{j_wTk-S_Xiy>G{Rw(ee;BH}3myPo*odX$cV zyA+buYGW$aSH3Q6e=LJl(;}uQ8ftZ+ND6>l$}e!677XQ^R=!{Mt1*n3gU~y1yR_y( zkU{7S|1x08#*`Q?e=<)2)45qun~+U$g{LZ$xLE@RV~lx+5EA)T|Kh>F@06kR?YC$m z@vzK{NoBI{rA?84?-(-G!*-Btlmp$8o;L?P#iNCjdP z6{d$0gJ}(3LK2Vj{N0}nto4;IrjxJOb#SY@_CVLXOl^|AU)8;3>G+b>O%NRn@C$J+ zDVZO0-6r2R4W*w=l7z{WEZ2-hIBJD~o##=w77(FC2Q@w ztAu+Tx%Be1%sd_uh%&29!SI^^XT=!m;d&_Gy2!=~%dv%ysiP}yvHw-kNZ{E^w;g3A z>+HwHFL9u@d{~x1YMD8vzy)Larogy-olhHJ3#b zCA>cjQAkt`%JkQPlIz-VCR5-VrPIW6UbCvVB6jwD6EazzWY+zU!GyGLb2|9BNAEwc zWBMik&AaP7oii3c8}CZ4a9eVlY?*R?%%3)r!H!+Ut0&qJp_jKiBVen?N=mxs`PV7P zi-EC+q0H|9{0p7yY?pPnt{bOVzHABy@Dm}fE6kbMhujojZf9~CrX6hu!lYXzXX_?= z_jhRtBuc66$f2&Kke7#@Czs;GGN*Or4b5ykI|b$0nwcD>pH5PZP*I)DeTVFJcUB?# zK#8CK2GeMQn{qqJF1qHBbyvbW+QbC$c6wr$%sZ`-zQ+qQMvwr$(C-92~S?tk&m zY-&}jjEc(0%yZ5+d=c1P8>#)9a{~Q>=xLzPErsmQj$vGOSy;VJRI~U_RYRp)p?=V- z>RPOKw2ZY^&X3siq2u@r%>W<*Hrc?D(o0J4LeZOEH+pWFf>-kv%sYPa9Q>WQ+$E!q z{Nku%CK}tjW4og9gBD8k^G!=jezl4d2@Z5a$uD==FovA__<-@ddR8@)$zD%(*~Ayn zUjZhs*Z+^;Cm`@nkFjZYZaJ8jYN8mMnRj^Z-w#~ul?r^~&v5vbtJn#7e;B^1K(d?n zP$?DXL0TSTgYG?U+DJ2dv2HD5cUSRd$!Sowe4nh!8l9F$T=Q-iN|PgH=`F%j8kiW~ zTdlP`H>iSeLogXoEt!aebjH~EOYEh1Y6w$zb5&ozNsDHCH@Ws(II`qj9f)x0| z-FM&=-#$U61O=cc@I#24WV9>ypR%pdyO=N<5#(VAOgVX!VXMWpgQ{;1l35+#Q0# z`^~AdDyB$lI><&*jCjhd;88?CY-s5m=s?FB))Xep&*I!jjIU2hDr%Y&5E*!S5al@W z1Y6v7q(2ocj38+!LiG*2K z`ozWS?XI2FCkO=WH&NBd z4As8PC2Sk}nifS@?%v?LxPNM3NG~1KD-hG#k)V|sij|{bSP4CQ8a#u4+*0!d(;PXM zxJnb)m7K8zGyO?8(>(Ll!;oEJEKikS&L1q(x>WX1d0T=zeU1BgV6*}j+tt)ax* z9lAW3C+6<+Xlt=>B6PsYdIZGU;ZPLT(&^3ji%Db3pRZ4`pOKUiRu;y}w<{cn%t2mC zi|nTXoyQC=F}#Nkg6>AV;&?+=^gRO@uQa>ER!qC4n2+Q0Ap(&d17G%OYWe`n`$P%F zL!zPIV$V%g54pyd8dq~z0l?Bgq>Qyd#FvW86`bG8@RWx(*ACEIWbiDc0zst^L7{=Q zqk$OL&D62rw>(gqY&vATUUm*}G;vm>3^i-D7CLn3&vICMJImb+h7a(%RY~|lGov+W z`+!*r`Z>hXA)856RSEJkgRYy;3;!*XmNU1whzrI|87z6ekpTg!^kH#jvm<4zdB90F`2g38 zkxj@2c`{{B!(sg!B3Qjv7lWQ!pwo$%$V&f{BDa3{`*tHpA{0c+kj{mQcmo`{RHv%8 zj9iZ{f`}GL-f&zgVYP3E^AdNWA+my~`5?rnId)pFv7*kQy^)!$#lf*51*3jt`gih* zrK&Nm3o9vGb2G0<^`#&K^;Xv(x{e@wr+}b~xrGP$`*WMafI*Adz25-TD{;lzM==k@ zve7@MlvN@gllB9()vr`}Qy=>F`dxJKy!;(=4*gX5DEsvz(3}TCvYnJW&Fj=lx&N~p z`i)f^Z5Ux_K}3J*<<%y7=G8B~F3P8p#}?X;RVhr7$ao^0d=%0KFb*`x|3e(7ym8N; zz^km}N{2=$G+qK*P@0kRN^!%moLCRV#mH-#$ubVaK)x2Khjsg!mzX3JHzGV^V+r~r zXu)to0ENIET{=mgAvIKZ`r%m&htS?V55$Zet->WSG7CEuc12o4!ip3p1F^WiIs-{c z+j4EQv>@Iy4O15`Ps?kB;hsKXwIIZ8vP00|I87vVrNE>a`_sJT){)I}sY~TfXtO92 z-PKlvdaB;4;aCxxaIqY4`GmRf@-G_>cBQyUx_GlYjK;Xcq-XWut0#Nnh(7#}X#&~? zo*s&}$f~3cdl}_g30e=SD!@Fzz>vyZ9R(@ohE>CZ{ zaUR*sLs6?Ah~XqmJ!U2`abq$s;NYzj21@)_|F9L)I+_QoaMVc#(Zbt%uIwGO)?0$K$2_nwq;0KO%Oah<}Y-iF@v|#h2{E-Q9bxi3b-h z>m*2>Q0Jx+`?bx3oqJ9{f*Ru4idYU{Vpli(Wo@hB1`osK@g+Z?zjp{A9PfSn3=IKP zE6nfO?%ReR1}E2_nYFS^Joe23UmU!9D_lgxH*jIm|KlSuG$0ZOW!4pEn(vz}^Mrp< ze=IV$qSg?B=A)^WD6)m8JLwwK9oiblnCU`Eu&=6S)^)Y4pBc0%B<<*onx(b_B@U{C zr|f3h1$u3BOA7bi>NbOjNZG}wSG+t911~#I^p@#8g!JEpm8{y4D}*FhmiB=CJd6M``U>m@ss15+o<{fYnN577D?T*$*bL7NLdTBH_m|@S?<5&S&|-q)2}}N*Op@%KO}Z1 z!U#w2p$qgxUdaTXYkUd*tTTiSuqX6(!b0W}b{MYQRpvo88EDBZM7fcMYUbe0+Y z&paeN?g7J4swyhqbqQVOs^F|R?H}G27+c1naT~0IqZXL@wmVZzouq0NZO~Wsr6SHC zHy}6DIIR;B!rFV=;TF(#?oPUMd$U!$&A)D-5%+b0_(!!Dz=*v4p033vODHlUTEl@w zwtxrrr)UhcSKzwMxv_oXUHP_mql!PVh2Xf+Wi2?cJuyIO&)~AQ|#=&ME_GATm4$63HH|1>+nDT<4 zs63jti%a4wxNVjZxrcq~*Uf)w*svXt$+SQJjUzQx)gLRHNi3E-vR;qtqQ=eZC4@kW zJ@djId?UvA54!afO^wnrC0j7ZifU<;G&cZGjCiI1viYW<6h6*>tsYlGfQ_L0382QQ z!<`cuBqC{FG;m4DK6XQh9wlpYZBR4eDZ99xOGU0fR6eH3a16Hj&OtnNIyeFq-CA1U zcv%Pb2e%g@h7^k2r~!6x=iWu22j1jsD6hMsXMWCt8H4BTvRK;ahIUdg}CI_V6Hy5hsU1*ASZxoujD zej0>U51*)x4IgJ}!6ac8`F+8X_e7>RpRtZ{n!66MF84BdpWYk}Xk(FH`NX=Sh==xr z|8(+lU3o>&Nmd}#27%u6o`-h!fVE7OabXgZp=H$7&UE*ERP+4gj-2KclP{EC7XINuNG5UFGxsk47>u}@YQKPW z_sQdV+XPnS!Kz9I?AV~yofE1TK;WtXU?G-MkR$DoF0O_2P<{xth%{915%J>Ai0L?V zb0$Ql(AnG6AVRs6(^xnZOUf=hphBpw*`(3sCHEW|6U|Ns~~`oVqL03SFwUWr_l&WKZM% z(5Q%Gc)HEey?r&Jr&)A!UTo(mlg*mM(A0W*WJGk|tRp(Gnv@3a<%c-_ha_g zy0G?+H#tbAWw4s1_(G|Dsg16>Zpbpvs6 zEiMP3kK8`g&jfnw$Jalkmv;dO^#lzGmx7tNa1S1njyZy?Ss;5mkrins^{QXomB*mD zb;^3oEQu0|UfVF1+*LN8T{H_4yL6m)Zt#kx8^HpkN=k1{7B-E1&G9-H>@p^;LPP-d z3X74kNYAcMDDQ&v+e+7FWsE$12CSG!Pse?co1tnDa3ytz?`T+aM}t{xsm+?=&<&ZR zHPAps7>kIV90==bRa=jFxhbkHw&Q6CGSYy0=SmGd=rZzu42x`ze@3@#j?!* z-m>`HS7Cz@0RUz1lnJA&q=kr;^cGw+%eKNuhnb^C)W%D88_PMk#~EZ3=tLI1l=u7%KJp-GyvCzQKRne)o%Fy-**Q@ZBpl5CVVi9`m?+x3E|Sx$Er7Vu~` z($T?cCZSzoW8%>K3p>J4WBV612b=zCy@DxiaP){PJUj_bQ1MK16v0Fd4^h=F@N@1{ z#6rE^0{9W$2oe{vJ@x&}D#5Ut^R*JVVOU%@bv^KPa+<)uoh@9dcbLui*8PW(Ix72Qpld8`-C+R;Cab!w z8X|uZW^nQ8{@=Pmsal1Ml)3NoiHA32UnP$LI1Ox?IvxI6->H(h83FR-aiQTPU=9~j z<5Lcm%h3sbjANK#5*$iPcA|l_I5Ikt_g6F8^%(pVs~R$ zgDiq>_PyKD`FYkHS1gm>?jXn1JYQ0hdx2Om8hkuBDB;Ld&j01c`8jJf|=>Ngf_A2 z%d_p1tP=IkkYb~sli>9b@F?^3e+!6hVHylDM?3bEuOX(NcOa%5HhU5Ma7@vALr>aY zzd3E&FnsCf!_GNXH{=kxx&rTRG+HdFxb0~tSRU86{-NUb<)U3zTs_7;yR6G;%a3+) zuf;Ll&m&RhE&cnyvASor*?k69J?#9N8ISe#bd(dRp`{a4P`IX#8u5C1L{&zCrS~o5 zEYmS+>HRoAfD(qew(wL^CVA$?=Hw>@j49(J0;yhCob>%4=8xvtv*Cn-Rjk+Y<2NvB zm=}Cq6YA97;_;rQeOGRV6-Eb5GMTB6PTxoAI;Se1N9#^E+>iryZp)JH;d+=8zslKo zpN7flK04n~<9}OYMz+X!b3kgVCPgf|aq+@Ba%YVYH_l4lcg#SFb~e`g3CjP_(tKE1 zfjhqeQxwdbc-SPx6X{*fw{7}ke!F1QD~ztR>JaM>f{}qG@Q`A`d=FTJLuVVeYu1cS zGb&tNWePOKj5EDZi?Sei*U^~wiVaQ@n#|soyC6J7-!u9s@g$`Jh(%SxmRvuuxSH3) z;jSI#kHkFhv26hMvTmB^t_MCgJA$c9jNTlj-q%HYf{d9vfXm*$JCXKmWn>M__Rkl| z->6wftUQ00_6kVx4;oA7;nKd6bFTcpbDL^;vTvRr=Ewe8 zyF6B=msMq06qzo2rY!$MxJ)>$A@Q|qhKlrB{VJjHX&K8q3sf$8;AwCN$j=ss)3Wt>N}&ZYJPjK0-M#`KAbj{2P9s4-jl{aM_d60uuFX$FU$4K6h-A6tN9=v2M_x+;~q`px26Z3%& z9+hA8jK7PgHWGH+jfXklM!ar#GQlYS9=FMP+^_nHm!1qdu$@Tq2aVY9rL2IYeB5{X zi8<~J<12Vm_?f_uGYOEWyPQmUxT?T}1%JOWx2Bu0{Ow(u@0khrS(;<{(l;yb+v2kX znUi;_$MyeZ3JgPB5i?1J3QFd7ewBmVQ(fJ zh(*DESiV_?xFJvLXf}gEwEr&nye~PLORcttxNowQ!$-yTe!!f9`-dt5MDm^Sj4jLrISFt-DKJCo2LT~U zJ|kqobg|-D&?;;j0 z^tW}>JyfF1L%JV&iOXO-R-6v&<2g=+1sUPFs^YQ70G=$D$mQuOHH>IpiCen%()7tW zjf4-yyV^SMCX(dp-)&Wuf^Ss{-SZSwYc>ZRhj3VDOvk62AW`f^46S?v#|1|oI%Ii% zIi{7tartKDC;i?z<@6=BTG2fMrBx&{_WL@10=wwXf?98z2Ragz7J+A^>Rg2L69XwT zL=~#{-)n(DlNj@T2>KF7FmK!JtuJM+u(!UiEhhZTf*{6O$F`!s5za;Qh0SvLrVr(_ z5zc;yv~T?}_KtxPGoX%m(eNDbqe1-ukH7%53Hr@+s9NTm&F~(GKikqF2kuHd5->NSGR%2vhv8?=_{1LiO#jvOj zw2SHS^`AKxW1sV%0R(RO2NyBoq?+N-2>sGg{&4-}5laz1YT zf?1B8#A*w^eV2nA_T;MNg@_*bjD4jpLN}WCbXxRJbC4{w@lrzM8g8r%ve@z`!{=OD z_llf6Zzn(d@+xL3KhZW!Np!h^=t z%P~?y|9;yhoD7=6h00&4y#H>?+)fF!^gebPHVNU?$1`xTlqEe_AKlN1EgI=x;dsp)AL5 zq8>{#mlSnMdj&OH8L=)^T$(G7NHUGIaUL?y{EWa@n77kmybi6eSK96dH!JLzZldml z{c9M9Y+dm;0f#%*=6CxM6kgWT>H2$n;}A&w`m+qW_=Kdaf5R;HPhv2g11>aKAQemc$6liV{0|yh?^KiPxSZ+zgzJ%#H2-1Bk%-zih>qZ9PE__=MJ$lhB^FX@Cslj;vv4 zru5<*y1;F6-&SVKu`?S`W$nm4 zlx>a?=7m|U9!P3B8cbsFwpaZ}qVrNeIXR0&1=T^blAr+|b2ln8k|sNjRVAZ1+jmy4 z+%`0%^f|W8tKTrpw3}e{Txcf`M6At|h9BrB6VcL@SxwiepP&EudW$D;iyxt5*s(`n znP=G|!D07JH;l7#GD?dtUY^4k_?5hRZ`w%qPq=gvs>)Fy3aIx>G=CtkN!ybPm}w!r$`$Jbvi;e?G_2lP5|3Q6L;c*XFKTnS>13b>>R> zm0!-?>olgbO=^J(j{gV0kyMMuX+-i!!$l@I!f-FxRr6W{Lavrshwe!HT@GbNpp5jh zWhB&~hPd9al-9}()PFHg=n`j zm)+xEWfCKk1d>!8@T35)5awJ^1j5orO+$f$x6x{;ai}fcR*mum79ufRY}Rb128;_9 z&mpO{s2pK{_N_H7Ut{b%{imyJ8^c27xQ%QN^5iAv0VjMYBxu@mi=M+_Jy@1z1V@fQ z=H+h*OoYdrluW!!Y?VaPv<^gVAB<^I)#)VhP@G@!8P&{2i@;^i^dkoB4XpI8&Ta}P zoeyFub#{7mp)2TxD>K<{krRPN9!{*TtQ7dR<$@QYUKl%zrrD8ljx7+AEGcBO+OOXr ze=%y+0sIxX2LZkw1EGMq%jHZGexBhhCvfbk{!Y-lx7XA@ASX^~4dQFgxd-q19j-!$bFxRKTS;|uu&lUcboQ67NK#JHYuetpP zD%QYR-EvH`tsE7b<_($_q}m)Z>=~;LZBPaXF9UrDqq*_iUFC0iU;RESMX>Kh%&Rew zeK$R?yHz21>LxElTnxZb_p>@$1d6F)NJX=&Az)lQb%cY5L|s#)0&m-)!qs#aNWST4 zLIGURfl#!ce*iy%G@E&CYVhEGPsmBZ_!7sc%uZasBQC4S^ahAdktS;<=OySke{-ZO zL=qmnsMazA2`YF5nl~&qVjJMyLA(mbE7mcAMt#Qn7B6ttVdA4()4l{%X2ip9xeqPz zd1$KDN}7&mST)PLsP(-&3T}cM!IKNH_81FA1k*%|DrZuCd>?X{1W~l<3;Me1G_4g9 z@2cMIA3jliyuJlwd{4P0B=T~J#xjJFg^t1ubYQTDev1V!sc|}=X^!GSBO#n+XUax^ zHUpd0Wbms6+I4({h1)_`30kP@A4pK>ROY={%t*~x&@kXJg&C!;{GhH0XFi~Ist@3N zqB@;10Eo`o{(di4w}^2#^>ZqBxqQSV0fnT>F$I3b^HT`_eeG!O-tEvJrXr+Zky{R9 zU$L;k0)e`=5qk_jaS=?UwdYtY_%CGlNSOq(b+kA+Rh&E=-R9eKxa&Rb&(1u7CpO~f z@m}r#L2wS?@yON%`P8}#AxA%N`Otsf=BOsD?5;??8L@sw@uQO4B3uV8sOW&F_hR|! z!c&A%jJ?udZy{YgqAzIRly>C`Q!(#vyakmMVqdncdKB^B$2q00=Y)1H7$+D8rISuMNSw0;EiC0^bkO;auxBmDkWbTolX_vyG z0p|grG>Z62Au#m?l!Kl5O_UCiAmMjv@;}Xoh=OPhTPUGMF3y$#<1Q#4>~Gs-*EaXJ z%)GV08}bpX(g8wnxK&d+Xd7&>IFp+i)n@H*m!xSbCChVkfnbwl&Oh1z&aewyMY)Gy zphTVpzasqhml;;eF(QJMd-fu*GJto=>w@Y9rj>)VTy>NJ&SgB&>s0ZL9znHWbIj-G z3V&)ma)XteA(Q@F!9ep$G6E|Y_!5W3H9x}gruvLyN5XxHF!9ZE%PpzTiN)tUnxwZK z_?ulI2&}4r{oP!cC$OG`K0kt`M9j%KISPm7Z>o2WlwO!1YEvy~pO;s2)tYiF=<|GP zq5X9BG;(UD%`RiZ6E^G$xentI9!0LB9doIWWiQ#}J8yK)uQ;rTs-fc{n!gv%t-=`~ zA0x0zt@ln!P$NpRaOeB3N#@bz(NKw7OhAaor%ocn+e(zd?@GZd8|7sVxu`r0k>XU< z5^7Ot=8GS*bSa-IkUQuaqFS0>)n^HlFL>FLKVEL9k3hV=C5*Mu5w^uS7G@_=4}TQl zdbPKiOYIW;zEg~eU@>ycVl+)#XV@ONiAz8Q8nPhO3-cFHgP-xn>NBha_h{8s?}p7O zN4r7E!@uIglYG)g;HSf1IvQPnlB9=0S~5W`_t|i5BB0s^0j}VJ04UN5hTU)iS~B;J znA&iX9j39ZJJ*RLKW^w{#A^1wxO+{N_=7hJ*}kE|4Fm|yBOGz9$FN-(c%_eF2?#;8 z)s^T&;c*h66;|uk0im^UI|=f<~3yD4nLo`@?U zh2#%^XZbOb-ftQCqo~%RR2<4!lq?7uqthNXS}j z2OMg|F7At|j54T^T2&fM5q`tGT(6Q8Jyn%iS1+jw?u+eYGzO}jMx zR5ms6XSgEa%8@Jgs{q38^k`-bfiVtzaGz@C?e5hcTJW&Iz$GbM7Sv;V+j~w~$ECn2U zwqk&vc(3lvRzd~%kHvF~LET1@T}R)q=>&Z3g^s~3blj?cD_7HCl&zF-e4Hi^{O2JV z#Vj03SJFbU$BUXWGZ5vBsJ*Or+T0Q&c~-E|^$qq)@38l31ITEEbo| zcr)`EkMyF3b`yVI`K<~kCq+slhidF|OWyVJR0gmb-1`}9AflV&_Rb5Kx=)t=Pi$$I zaS%Pi`^TM)6mLw$MB+Oz&2m`xB7t3tK2@O`toF|CHf|D6cI{nj%?)lk_Eo*(g_Ba~ zuDGgm^xqDU3q*U>+(-tA(LrdWAqtGPr)9o;f@qMJSkTcZ?CaS21>9ih!@sPR>0e}L zSZ~Owb{gkcr1dqY>`utaaz_;^XHg7LMt#F3ZkksP7SmG0q2_EzQE@WVYPz^WAPuC{ z+^@V`VgJH7bZ@+4*fb8W6UfZb=(2ADW#)Q&0eYCa#Pv~F{8P2#4mASpFi~}Yz4|9b z`}zPDI5a~rrqs=)-3x~gn~psPv-j?Eqm@Es<}aTk_@tB0_Bo&F*YhX#oKf4|HQj22 zZ4@sqAx>~ywq#+knh#Ak(A!C^c!IU$U_)QCQ8PrC`JC#QG)QswRaVj-DY@j@dI$Q<+Ju3 z5MYM2Ni5Rl=NDJcXU%$Z@{n8>uWN1|4Gd$oM2DCgnaekc8H0qpzT`3o>9T^@CxPvc zhM^xmsE-Jh89=K2hNcf7B^p#*(j0aO0Ah8qqozsvTQ*rF@Ajx9ou>f#z2#e70@x?} z06AYfw!OfCgnxNC;2Fie%Y#h~d7kMpvW?=fjbn$KI4vFb!Q%4AZr9k{;8)Kf-j)2rpOR6^&t{fCa3U1FdkKI#8wBDly4Z9M=S0KgnG z007qiI;`0_TKz|qxs7dOx7G6A=?7Y+AJjCFxoy*De+KJv?aUBQCT8f2%_E#Dzl4jP zQyiH%t3LbdGy51{DABzpw*jkhE9_04F#E}!p?=j={nD&qhjmg5^s+p!?gzELi9a(t zBfoau`HFtKlc$~A>xGTmWr$BV^W*y_R_#vszV9&+C1xDB3jJ~&);P$IlY1gb{oWt2 zre?BTCriZM-gfQ#Bb_Sx>a%$gQ-0w%T!J$EK~ObGE1g89c^r|f&D1@^qeNXaV*So#anTU&-v5t*t3z4^dmGL*jE`=T zF-vpeqO%fu-=#hJHA%F4srNN{>?TR) zVJ|?P0qgG2w|D4)_uXAS$(>N>8j$IkMLbdQT#S0LX5el33`X(AKu+__?)}FcZq${ zidvqu4KTwKQ}PypkGBh=pVwf}njOGdhS<+Hn|>sKE#ZZWq!SK{C)q-}yAGX89ca>^ zYNx4L)oTgbw+Yqo8c&+IgJ7;NqSkJ%0hx?BM@O7^Ez%f}R;R)E0DAo`zTRNdK)wNE zyM@4%P|w>%%OT&~d#pVO6l(>iHhtSB?O^;Qs;5gUFr$3*rCVG*g;Qf;r-ooZ4Yahi zrk~jW$cuLUg0>dCO{dYFX9dwMBxdgkF`*>E-b446qb0j3g%2x`8_G|6S+iW2tcEbvIOkU|ipfhr26_3vdv4Utj0Lo0nLIyAEX zUoi)=4u#?$RCL`StsYwF8}s0yJi?y*fb9eR&OwkC5;>n_{ zctBmyJkqLCP_EaF{C<``Ta?ebm-Kg{1H?;#;`%}(4;+yz-rssu)^zMK;TF^od<2NQ zum=E37AV;y>%`Ea452l1jNM2u_f?2>I0#Wz0^J~M4`agi?MC=4NS9>xQ?{_ z-%n)D&M*`gc_4MSQfevxwAqC%`N`s$L5JZ2oB&W!c^boj?uiIzP|Ri<>xHp$b)qap zJlm(XM$Pn&IX0c626FrGEhgGJT zMvl-XG;l}+C2BOEM`ll`&Q2L)akbE2s#YVk$BOZom{4jgVgO$)I*l?%TNDOoEBI_C z&~lIuR!9P8ycmPtPvJVZ1p~L4YDnlKIU}whtEG@u(asu#kU|L&+?gBf$C=0R433ke z^)i}4`8fU>9diS@Mo|V5!=O|*pxT9=ww#pNmAK`|#b;l{u4Cf%Xz!JU*pB+f%b>Ed zQ70wnzc7Jo0Cfr$udVmmFQU+>E(#Fn7IOWNvZHm|`!`~-`F&|Z0dK<9rP}>XPSBlx_X1s!ki8J$vJHNNzqqqU3EZR za;S*hhnhg3VSkzH9GCC&FxNU{6a+u~Mt0hwTRg6&6wC+|i@$oLp<6O6Q30i;f+n<= z{-(PQ$6Ko5*N_-U4DAG2pLkAKf}z&ICDD@tGO-)Ktwq!;v$U1`J{t!VT&G!@(@0y{ zcLnVKsld>SzV34#kxQ`T&=t76Dy^xL zyy9Jmvp}rHe|Qz6^^|1|=UysK!Lpr|;l~3Y$v30-;;o_jdgGa?N%?4;ie|-fvg`N0 z1mn7+VF%!k0r^Dr*>q9|bEU6S5cIm^MBK0`H-J6zN4{v9pu2mUIIK zBY9Oa*kiStY2v-XREi#{t35J0a)Nmk^(ELh4l4mE`Q0o-_GTrF?$QX=sY5vjl?7#( z1c&0rku?#7=IW2_^;%IYV+Z;aPxpP!(^>8miIl+x@XJW1IFI;;8t6Kt465)8#*&JD zsRoP|%a4dch{xn2A9L5CwhgNdF0or#nCHyb#`i8lhv z?w?p&##4FEMEmlrLe*#JzcBtXt|GdApvuu-ncaRr{H#xrO{Rb9w1jYP{uRH6ENe7u zKAtX>i7{1Tm9%%#p1raUg@?R~+oxx?AzzrQ*s}Fl>18Pa&QrRNRA7Q8%@lD=gUEDz zVgv=+8dBmd`*%#*3xZ(1JxJm@P1Ja3($Y6O^EL_%iH7B%7(UClu}sJqa10j${!}?0 zXd+L9mNW0K@w2AR0$if^z`~h5W(cMDh&^G%<5$)HB-Wp-?|s@!b=on&Jab$@C)3c9 z8~$6t!!U+QF9q5*X58#-*(G9T5riNn_jANzRta!>5aHw*c)xE>wZ$71^lge_4TE#3 zGKrz$C$_t;+&TnRMCA0smz5NH1+&TX>28C7Y8K%)ELSO&!yFBlrLr^a0TxM#?PB?t ze9faB%{3-*Vhl)<9JedpM)q50;S8-Bf0>lZXPyuRa7qLgRk?XxI@qdz0mGXo7`i%1 z&gqSlAY@K*S>$j&ms3!2uMT>N)O)CB@W?}+|7`8Ulp+9I1**xM4i0*Y5yc}u*9E&O z9iRikq3~YdW;eY0Tx{MX$1xGG_(n&m>~LX*m+ZQ-k(%J*Q zDFZLqw_-6npr3S10v8sW6u!|Te&`V&*x>qLotu55*cv=rMYa1)-DL@E*fy_$UI}-- z&uM6BPWsJ+XD!cog}O?NiG;eAn&dpbTr@VD{kgAOcj{l!nR;kEH5LNcX{Y=dT(Ee? zHfI2$MDWdr;&@`%*i$pYO&bulPPuK2b;GaXrq3%F>(u%EK3sl!*&(@*ca%_zg-}{N z=iwrNr(>Ee@&gz_ha#Ih%=EI)yM1d^`PX0WnZvCij43@IcNor^1L`jCbzD90Zcv{r ziJcfVa4$jf?Pe~a0woQJ8d{ZYh_uHiGtL~x6OhC^ULp#2H*#=()#+7tb;KeTLnQ}C zj@kF#=S^~CDpQHH6!o?HYmQRO%ZZF}_KGnk+Y5(vRb`|F)0q_w^gq%HnQ(Uo$*X3% z2~N7KuKDdYMa7I|Vfkr+J}#0$^?0x=`!)s|CG@c3j!8J58LtdOpBQ`^40zq)iy_#% zR#{hfv$_ZZ$nBvYEAV1Q#C35H^oc{QpP|yb`mmUM`JVRY+Xprp=5?!}^3UCBFIuk7 zzd1iof&%cwGLs=`QTGQh+X`*EF!lT#hA#@RJB8{In@FBM!t1n|R*bxpBOT3+HKpwH z>I0ZH?Ahr9fO{!dJKO{O)ywBgZS9;P&gmg#(dIgD=4v{XS+`}?=GzVIF#I;%UU@swKDAptM*R!{5(Xs3PwH>@CCG{qfC+R(!VT1 ziEze+^>=+`M6Vyc2(#M0%73i_J;}l8o}#}5L_jd?z(00oyza%wEn=gV$z8=du{>AX z{Smq#KtQ4AKK%a6)mLukLb&&DUM&0v^uqt&=0*codcA)xTMK7By?-u5ANgJBK>--w z-d-YWdZ_F82>L+yW*^EiF1C!oF^86O%r<;dH`K4&?uwkC3c~oK&qVrh94Y=T=JQnu zvujQo`c!qyYF8IkbOp97mqyB)OVefqr?tOTqhGOqoB!QKoXWPiAl8ukOPjJhGCnogJf$z2w(GOS0)XQ z4%J>N4>rF2S8;s1U+Q`P6=vX{5gG6QQ5*wTJ!1oBgMYQDiD$GLWIzaU`;H8vfm|ys zcD`3x)rBF1lY|ICh>V|Y5l9U#ZaUgkS4-TOqYJR)Ja7Bl@yZnZ`!syl;>2JEt-(jD zN2Gy2Q8-0V+mdCObvNJhcx_R}Z~;epFTr>sGEpPLfU z&y>=bpv@&8>+Oh@NZNuSZC{+W5ECb z_;d#VApHLn-o(hx_&-aHW8O{2&5?gj(@4l~`!Nh5WAf9GX6G83O-fbHmdOVPSLrH4 z0!NAi2#`8}^2z&;-Bzy~b%f*`jw};WG{~u^rzacF4(uTDqvFcjtS`&$gnL7wTs+=y zFQFZ89v=r!8M&Pvzt@-V`>@4UyM=%6U2RAGa(L&^o3AenE|SuAdZ#R!O6BHtb9~+3 z6_2&h@^Xx^yDU4C)E>2NR#QGcRPt)DE{9fUTF)P=t|K{ornL3hxxKPBrM%{znyhT) z^~jE0b`{C0ksR5?#N2OUe&5p4qO%bGQtT@URbp?TsJ5tZByY<4$Ty@WH+L!2(t4nvnbgor(Im>)-{Thl5?G#en;DBj5r! zQ$#RNd^deI@QxnP zw&AuVAg^FTelz$!i6W}q6!{YH0_WLSk^w_d4nefzSlK<5Rnwh!b>-ue>Jw1cWXC%` zwq(iBKEq^h+GHz951!TqzL*jros%eHb)sLJUS|QFdW@Rm1x{Wk7y@QAlGaQFcm^RX zoe<}`ZP6c^#gqlHprhN$)v7B|KDWq;Q=5tsD6tSMm^9(;06;6px3$|l&vwaZeeXR} z;j}a~*C0dIrlzM0UPs4+fI_iTJhE|{`C?Y+Z>Y(P3cj-f+@j~XKVx-`~Kax zWD-G@n-*obDRg=}My(NM(V~GI3i|iGO#%Rqlr*|yEjNlmr!;`lp6mzM0bq+27UKu& zu-=$n>z&isK8?P=W{Jl1HQ8MbsK`TR*Om2*?MFZ7ea)Lf<=doHZY=Y_7dV=lML;s1 z8k|V0UzbE&Ji92^pJj8W(Stji`{Y14m>!OLaH;6Ipkjk?rsZ1)26wp|cCsl^J8A+- zB0Rs6vK2g)tZW8}@m`H%+7N^TuKa8W&?3u42oDfZ%b;5Jn81cqd+YNGjHf?wuVtv| z+_SbwOhB0q*Vf9TpI1mXkYq}gK8w}X6U1oM7Jh3%)xQi~4R zvHeW+B;`xMg#)c%G$$1%0CplO{NEK5;N9u@S&+P3x^G=9pTXJ&_H83z>jS9yp&$up@T3kaH-r2NVqhw4IrOv1 z@B?h-ueajtghzepuQx0+Uj(Q#fcJhRhPMj$+^d`4BZF@rq6zrTK{RXOqgl4KOGAFN z-piL`dS980rO*$8HMJ|g!L0l`jy>@fiF;}O%V1aq4ktHKyAT`)=*Y*zmner|Y1MyN zi9U0!y7UYLo@nSkwsKl~Mfw{96YY{B-Kc4?jWL8yX)j1~*{|~ReIoWQc zD}s)Cyk%FYXHp}wO?h%X=F%ZOmMc`$(V^62O)nSJT-iK!3IETOSH+EB${qt?lcheREvzgN}vpCB6-Z?x1JOZ)&jO2D;KR@ zJyBY|TQ})fQOIt`@dL%_=ff19xWRZ3yst74Fvbb`-1f05BUbk=0S$&_roOq1e3B4~ z|AkZYp7bF6rqgt1clW4OA!sNSJ$LeVyQP-24GD-_Bn0GjT@2S|FfHuxrUgEd_4{H> z40ZR%Z}aovTd%;v(~g~~%3pX%fH`cp0`(r6lMS!O4Kgl@JW8(>gCcTTvECTyc-`T> zbA)M~D9f>H4V=FCVQridv|KJhhgvaw#w+7!@p6~&p7^==Q8yDd1Bn8;;s)4dSvcG! z#1Ck`7+fqme9FQGq1d|`HCG06c^c-@>ivJG(BcR}-uc$%(hAwIS=G|y{MHTdG;C|C zp~PC(PT$Th7TD?Ic3vK#7T9cQFvSe^*TQA{CsGI>EL4rK%qBzdeBwE!{Np$`t25Fj zDlO{Y?~IX91M;{)8W8Z%x{(})R0<&ILIp%UV*(@q;i#t9=-mWu4u!Fd_{f@HL!N;+E(bUCEJ;paOrPHvuepdj;)E&40ddHOqniLZJEj4r6TOkD?I!&E(4^`O z*zgh>=(Myany?1^^+(kKP{5V0^x%rAL#3d9tOG z{U|fmq6&+Ou>>+b6Yd;pC5RRq7wBk28|2QTPCkXp=u^)Vabp_g_m2W5rPJo@D|`H= zq|-7}?vb?j8c}bor-^4r{r7dlP1`3iwfZhe1iE~fbvjFv&X7pqxD|Eu=v9VH1|?MP zqJR}u0>&nigTXrO$|V;}kT53Ys4$7sB}gSCN-91^t{dl9MsF~Lkcq@8dW@Vgvvwko z+|_)=hYwIWJ%GpwJp8g@;AU=LOzfr8*_k=Ote!`AZ*{quIV#iB`Vg}*`Q2H544R~4 zw7>}^-2wbdM}|LMxgt{Fh3HoaIJk|-qnVwZutFzmXA2Q9^`JKe3Me)WF_E=ct|Z_i zc!zX6y8mKe0;a(wwYFG4`qazq)VdiD=XR!h@Zx+Gy6Iyk5Muu-T?l&lYJGIz@FpJE za(?!SbSaUJRey}46*9?&Qscw$JHDpf;Vbgsu8FFg%&29Cl0SA)qNRYiQ@j$K>;f8J zatLTX%!x+7sH8DqPx^JXPR;;wUxr*{)y9xlmZG1|i5fq^*JC{z1ctLg)B!G5babS> z4L1>{4AEfcjD=&s&~#CGd|E%HFW7)hGb8*BhhjzZmmI1=A~41v0kqD_;Kz&-R`?Wc zAB>@W4+DhFLBjk|q8u!?g64HW9QaA{H6G2-cQdjoMEWnXHtbqU9$IMhGG=OUDhskc z1bEPdKoB*MVn;h|LbSAb9^okly0YG1oy@ojGDj_=QJ}5iWBSf_r3U+bIfF6Xd9(_) z6UQXTN+kd|*z-qMG*dgkq*O!}`0JMdmvcr=$?xtsJTb6K$a2iZ3+R|K&Jq7RMFmbr z?AXz@7!*%&zSj19c8?}htscQSHZ_4)eQz^X?#NoT%3H2K`j>Z0R5M`+qNwW3`DnmB zkEsgQ8xU6Xzt{*g=ix`<`H)iyc{+gcU;B2$Vm5m)nQ-EO0;u!`&Z%8b(lAB}48N(L zxYk#UOw@apiUI;W+F-+fXZaI1a|^4-xohx!sM!eW-h)}gROUuCL6wVj7>;g=`N>Zv z$Ni-qpM>QIgbIx(IiA>2)OCuB17lI4joAeV+y^IIZ=X2oA6A9PbaDt=)DXsyjk?c} z5g!KYVi$N(?Y^PR;13o5)s^y2m8(v@&F>o2NCpiuEoi`+P?UKT*lGoHQqOI!0 z%=UhQq)UAlyyH^223IqMHjR^hC}#uDo}PL*6xay4 zP~72JI%*fZbKGwp)Y|(*XGw!1RE2;cdk57ucj=~7e|CyVHFSHr6_G2@g}uXiq|O2d zEbO;HO{s$zjCvdc8KL1{xup^b^F)k<-Vek{~5e3Chp>asA1*=&O(@Ie)ARoueYg4 zC^Uwr3%a;qK?D5?sQSeBfe9%+JU!1~t!{UK(lXy_ZDv7fase5L# zN)C0M+q-n_C6;LJ4AQ_cP46Yw9Ec+-pl4KEfju^V zmc$`HUtHa!BYYXP4Y4F{_w$-obhPNNMjBS5l$Xy)-;1K3hjNiJ zO@{!=8F^JSv3nbi$|DuECLN~ie7pt(<>1R*5A96$m+@Tab7|-XIvEJi&RlC-8~*y% zMPY0>5sVaJWTj-P^&CY_7Z5IlpqP$xk4TWZ6c9EGzWpfB1y?)tZ3(i%3$xB8G^{MK z+&dC!fcDcpEuN-9eedI-21L~)+ax)XaEZwL2CRsQ*{`u0p9wEJB+86n0i`+ra9H$( z9*lf`kB+bP3x5>LDlCl2-Gr|@^+-iu^lg6%(q!aH)jE;udw9wq)gHNbVCgd117q8B z89;`|$<>9?o_StiNqd9?yv2d?Bf$n0(ZQ9I4$8+ChYf+0^O1`>U-s?X01FKf%sdDC z?2-9I{%R{u!+w6q(3!Gze7-2fW+yD0>>`E9sd9LM-##{(bwVd3Fk0YvThK&Z3!J$S zXRDda41iu5A~(OU4V&huUr@E~E}QzTs9>-E+0TPU4Q9YPt14@sv~@3>CGD|y5*`q% zV?)VJBKbJ-(4+s9Mm4Y?LG20MZE6NW;|1L*LYVYuXDk#G1LBVwkBV(wGL0*x5q0Z8 zua4WF0sde>Mw)F^sa8IUwEkXwuDXzZl@|Eu^K?D=j0ya%Wg#~?q@>%BM#TC76Gtu8 z|DwEUYN~GCgX$)r{%HP7_)*#L%AIemt!#||Y1~n=j7Xn>k4AmB1#W2y>Vd7sg82lm z#uUmLD2Qe*goEk62%fZ-_qt}VLbp5wbIc6g$T+Lfx5N-4qDwbp@uD)N$aHY_P_(AE z;ASj?uG)laoWq5X`rz=Ad-H5>DZ|irpq%xpEyIk9$GF;o_S;lxNjxf;<=42n{@kuY zULIUorT#BH*H%q@*T)?Voevx5S=&Qiw_S=PT<&i-7xmIHfDO$krO)K3X_*GN8GBjD>B`*PzVmp63r@h(FLYAEtTtuweK^jX`k}_YBO<*nywV{RQ}I z3I|2y`((tKkxQj5mvtuU-meZRUi_H)a*|du{jQYUjDJ1HSw}I?tc?E>!PWKDs~K+L z)b@sUA!ft7+5+?)C26D{zN0j@Y+33`p4&LSJ#MZs6iI&Cc)ZfDFUEq2T*LRpv5P2< z9U9OKGR%i9#|BbW6{-AT;lFbgKU&h=3+Q_ zrwkSNoVLv{P-*%VmmRsdsZN>>dGmDofiQ%geP(KSrq22Dr(wGtKu@XicRF-?i{cX^ zrw%63LaN^bNvOkYQS!T?ZBh-Pe z<@mrg=4k0148Zhdq0Rx(wU8H%2m{GIZ~YcOI%-eyGnu*b zT(l9@T_sl=3V<8}M}pfZ1%%FeM(5HVP2Ru`QX_cbZ6Ly0cuvv3pvYZ_ynaycd}O$y z-ZkPSZUT&{xX^=ikT0DET7;h)T_tnYuOZWu-IAz`6~H;N$cwiywmm%u9JF)IWD>V; z$Zvzzt9@sLACn+YezUyyht7Fki}Y=Kv;j^vYNmel9-DjN@JVCf9Q%HVc7LrRrQ)+vc^Nsx;U`1f7uEMh0Is!o+tUb4Msw(z0MdBB?(pg{SUCVi}2tgNBR(t7P~)zf8i&r*ls(qIqho zl7Idy#*^@8ZolOnHmMMqmVy@9phP!dgpx@2qBK$>A-f^G8HdX!(r*~rGDk+iyfioK z;uRLp=gaiOJ&j+>Ps7YI3lJLFSrOKM$atB_teY~_-ERw>mYn>j+y|KzJ2xZYC#YP= z6u*IPaH2@)44(oe!bQuE_2O4eS0UmY_nW?lO|oHXr^p(pzfLWs40Id0DT8Mb#&0}_ z-Jd74J5%Hijqfg!(V53iAf$sx!^upnn`qy#iGE|&4#}r+h>_INoAu4T5|lS!aJg;+ zvs$YcH}&?hDu@2H{4;4!+iU}#QE4N|;9gx}9#V9q@)zfh$A??&lKdFE|Ja<9cZ_J| z(Kiy&GwMFH|rXfE*!TaD++%bV=1EJ?);eOmxi z{itQYyc}~M3G*Vitf-3g@;3!&X4i4aHCjAKGSGJN!k^p+{q@GffymNidC5~@N&ST{ z!tO=97h$vKexS_(d)8!F<#;O9VJUThU#T3&%beTC*k#gt0BC_G0e&h+6`jP1x*8t* zuIaI1*?Z7W`aG8s6DCof?0Ch*x7mBKU2X>Ak6RH&Pa&`TTAp*wSKo^`-Y zEXtDf`~fLE_1bH?Vs=}0ADeD5U$EObiE_`Cw=kFAp#KUv5LPc~8vfusoS^?(tmhvj z><8X`HN`<$|kuA9e0 zCy{mSemfpvA48R{;*R&N0)jb2u(h-8(Hq$J@a50#b$)xT_0Fn1o)AsY>0221oH%Vs zHRp*vhLV{r7Q-6YTKGKOFxQM4HcCw~Zb5*P%~YG!GNbGr#HluLvF9jjy465CWtjG_ z3En#01!n9FS4DRkafKPX)hU2xkQ-qplCrS|$9S2%B`)DQ=m|u{jg79uy9jI+Tg;oy z-$N@B%Z(qYG<>-$i(dxGhgEzwA(VB$rc_dQ>|JNN%Ufh!SyJH$o|7A$8H;RlkooNz z=OLAR@|IXAC=eai1&7LmLpCFtx>+IVMS)Fh1mD~2&kK1HFA#4N9r-l;h(LuXU_sBu z$|OzcW6iA`EOZx543=M0I)!YEes=IWy7#qoAW%}8bh@aFsWg{ooy~I zan*tsN(~}wZ%2(pR0>3pMO6xto!A4BHZwTBu2mjWurG|f)nPt5wfv`7U$vscEo(Uq z;Si>PJ6_{Yt`x+f!bqJb$!8eRjm->xjZy12!?xc?dio?irU8QNYkL2>Iu*eaZ2wuA zm5Bd^7V`(Db#b<^cKTVHOKLWD8*B())p`z2e3RG4Qx~)JU@SCJOauZ6C}n9j3@||n z6q%VKmLw_D?@!#_>dD5O=4vnlfh4J89S2)Z^xn>ZY6X11nm4R|n=-q1xqp&{c8qLX zY)G}X`?5U9`=jK8fn)03^2Ql!8V2}MaY z8E#IMkOV9PjTIt`{pjg5;0Y}5ne!13VQix+my6osd=vw zp*};*pTDn{>Cd3+#4o!CF{Fbp)jhPCB_yav4Q@#1)irIP17DeWeysAx=kz0Gnmvhd zB*oW+k1#S}(jb7EY1aNr-P2#(9Ulv7-L_T9DVjJwskDa&*Ox;2)57ODXGsq9h05`MENh=p9 z^w@v<5ZdfdA-NWyqOPvPL=B$5z~~dS7dt>lO!xn(h~(%DRl0Y5|)7E`wf8zVTLZ; zTte3HXMxH_w5Xc16nPjLrH<8Xa7Ge`__Wj~KDW>jMr#pXH0yW5JbL=Hrj9C1*T6K> z8`SYgek(HHs%jkq?2LFPZk{L7dGFGf@6J-d^+?A#l+orAwEA9Gd_q0XpzbV6Xq@aV zJ?rHd^OI;8DL9lB4$!t_FZd=|4|h70Rih=99X9*bsZd0;{!3C|YM9a8Uo>{&0`?*8 zJIj2DHBl&9xv;To;eBBuNd+(3Cm3&sl1i~sj3P=JpnCVYWfr4TBe7&!myj5B#hqKJ zR>7aTSBST$%VGj2$AQNpNH5>aLTgA0E~=&fXcmdu!Zqe*RZb`Wf&I$$rv(W#N!|K+ zw952%USr`S6xvv(>SF6c8V&&~zDlwC3#}fP-5n))bOBzIT6>?#)_P|9AZSzrMyS;7 zQtOn403sx)Gs-R7zKi01mugb9Y#8Fu0K>k$8fRwas1qo2N0>#UF^QwS&0xR@-QaGuK_XbZF|#RqU4c*=m7OUutBx-P?Y`beot%KmBaG zgcJ)|Kr?Pf#gR=7U!-@{bV5x$ZKk}mjmeMBrq4?1hArSkquO-Lrd}wJc6N$H;_F7C z+E|#KkHf4G+t6AMX1Te8Ez_ts7vtE!&`&Jh%;Pbg;9X>E*j&#TjPX37bwYMCb7uEA z=TF-l4Tr89#An@W%{YT9TG-iR-T;TwT!cz%F^ZY+ zu2Z{zJ#v2+oB8(sa1Shh2p|;y1qZ>@z{%Ok#Kh`9Qe<3Wq@ep55Q1;sP^G*S%tcY? z1Hsl zR9l`wt>`oY3Jy~uXf(yJmxoE}iN|fl?~7!`-=9A2s=7TYQ3*AM?|CNl!MK5@G{{Ui z=|B1GR?QadEM33braXW(v(xSI`MfrC{8aIAiRkfi{E<76 z`8dKJkc6vdT?1L8McDM-#9F%Ab73@*N&Wq@ zQgvLAzqFR_C4V2;mq;K@O>5v2yYo?kzxxhaT^DuVWzKR0n?zMpmMWy9)YDk3SXbr8 zrMn|pXLM%xFk01VI-)qAaaV%kyTrfQ0L#P8&Ap+fOnEOWC1&N_$VGgcK4t2Q+c$-z z3?K}-6(Nf(Y3tqxqF|~97d10keWj`*RB@GBDZzFyWG!f_8)8$Ta-Yc#+F>-t%$lhs z7^ix;-<###AGOJIGz67AcNbafB#44B0#(nARKqsff;~`?DXd%S0N1Ff68zkWWdRj-+SUyNg4R*pQh-{YO z0!mGY*16}|F@;hw9`dra@2fXCcJD=L$Ye+O3}OsFJ?^2R@0Tli{O9nsk^s5Wj;eeX&pR!l}Zc zQ^-ATn8nhSFGe+5$!GlS=U1Q(L}dZj%rB&{c8$Qa!y=kk_d!L@y3m-Hk+=c8gWtQo z!$sHUz?!EbLw!4^i!#e#a|(zqDbFsOM8*YJC?Dtv@-#qa_1PKW97v&_D<{HSwOVBa0E8YSn7VB^~283q$7=b-&#Dlu8*0@pb%(Mw1Xu{DoRY0ylT${{3H<2 zzoNhjhy>Dch~+D&UE|)E?f9^Zsd|kYtgbsAcOYaJa31e0X2&X(UXUgSg|%+;?5nao zy!O?+06^fK4=G`ILG&_&^)_{2TgT9NVTrp}w$~CbFMtZ$<+VlWR22)*F4tvN(>`pT zU@5EMRRy>co+ye>91%t^!w~g*nUFFXi2?rX`rTb9T)t)2Ae5wV*J~jKU7@BE#?jZ6 zmfX6<{WPXIW){ybZO~^M`YjO;WP_cw@0CvIcqkAFm1uGmY>v7fxd7{DZvS6s7KV+& zB)}=z>{rK!dvW9aNu{!Y4=hQekZtaZML}nkUrt)Xk7)A>FN5{ySxo7JqX+;rxAm1;t|3nr zuB1RxJg5(T5p)z034 zJP0eafE7&(B!DlA1Y;!Zp%xv;uwHBY1uH}CSF;{~7rZl^1=C`67^mO~0Sf}CgzcWa zjioUqlVUFz0FKpgjq5j3krwA^U1>~N9ixjFTod)om+J6~QJ69vanrl(POF(PbQt`Z z%=0HrlM-@p0q9jlCWu$_VPs2sb4ci3SDFO~Pa5VNfT3YDda;DPV(MOv)+IaOm{>E@ z8$X_H&kBPIbA4PVT%WA@Tn9NYd~XQw78ujf5J#A0BO+OO2Xd00GI(T%cyUr4LtSiQ z86zb5p7~xyZ0ig{ey59thwKY{Re;hoL0L4^RW_1gHdRjqvz)M-CygIJ^bGFuk$i(D zi8m~aT&OIDHe7^1PggOcQTAMJ^#g|YQs*k}CE+5%0WBzkw_PO6;ULO@i2=%igf56c zOla7d6-dkz@{0%cc|VzyzIWn6^vU=}e5x~3(T6}u7o z%YrJvTlS;R(!kfBEG|Z1q2Y5$+jAhn`R)NAQ=THH@IXgo6!3sHG%Hvrwm)~50RxVK zwn{9P_vOtjlNfvvr0V{@OyZK?LmN2s`M=uM^FJ{Ms=o1>fRTbiX>){@Pl}BrAnR%$8 z7%V~45H;RPk-A!aUwXZa$%v?5ol84#P-D|&S$#2}N&8Ap9J)~vP?2w%f6$G` zUHQhx+Dzz=$u1m>r?s%77Yn9*5+W3Dw)JU{668lp@@^ofIB7-Z3MjI?P} zuu7SpvWf!rl@Z90-#Z|9&A8ed-1011O~Kvb_q))QzyjeN zWmIg`!H1{2r+H*ISq(De#A?orTo%mKoF4E3t?)sZ?U|SlMR`r7gnbAAk~>HqxGa7b zO2lFS3&eBW)`1gW9tokz#0%#`oGfnc7{!!?>m^vu(GQe_FlffBGcHhf4yL34 zSiTDD`L0=VsG=rV?BD}3L28h*+f~;dRc&89hmRUsR-WGQ^k%#*_kPn$}xj=av3Td>vU= zcgS=g2kDdS`<yLf0jTZ-w1!T;Y>Nqb$`T7IS1XiolgE8GTk!3Xw1hWbJ4c5NHE+NX#M%~+{ zUxO}*9G*omQ8I5LUmZ}5stQ`|-rC#PA0)2BP0D|?%|6eYr5;O)NFPdw9-DlXU+B2J z+Xpl`Nh2@C#Hg%y4rZ(gKA0=^V_4d}d%rpG8ovFM^6czc0)?E0+|+IHMZ011d`~<0 zYG=*go-K2oWYcAH-@w|yYf^Z~;YGOqj^pFB@1uZ20QFYjg%W)20sSxo?nH=!TUX#Q z#VXPWt{+n7@bG#|a|<~98L_dW_w-$(|3&g^u%=D0LCFVj#ph}n0R(UXkQfh)WtLDZ z5@j&7W?l~BmIBOdn)pRpR_(4ksR0%5g@v~(w>Ua=ZqpcDTnj%Z=?nN@+uHRp`A5^A z#y<1s#gFFy)ZAN{c$hjG*qHoxuc_>KL92cNgpi*GAHIvioFY{hC`7X>7!5UL{(>xh zaLAbbQg%_m&zrrX>a59YEqi2|{bmXQTBk(}4Sbja*XWtSD?|SYk^X=a?c&O`EBof) zi3%ok)h0D}!7V+po+XA^k;X}atpCE6(3!Gcw8N*B?l>>62fGkLv8ci`FeK&8$*)dU>VwAF=Q?Ypb6v;!jki<<| zxQXz>)j^c2mH8+}U&H`W);7elJa}@|$g(-%eW;#jqM#gCoqAaPk-%H>$Q zHk6dhwIxouxmr;3ao-jDKF;8<(5sL?!H*4@Xu57blFZMTBYEEYs%`pO3+{ca*wEWU z%x~Sa=lz>z1!ucu9{U#dU-Jd%MHaIT1^}@0%6f6AAmiNSwog==X2yo90nzSMBI z;F%UiF1xHoF``Q;8s9RI2+pHoNA?$!jvo_6m5Y!%bbP(w>WWG_(&_A&qEt!atWqo{aA8>Iqlru87*J$)tVziImOi>O)I+P&NaDd&U* zF&f?Fa_n0YS`)F4nwHd+lzQ3KiEwgzlqNp&nNrrB5Mtn98*9DTn0tPF|L3$Xd0ao| zX5A4lR7gx}cZE(%E^5rkdstY5xMjwRixZd#heecE{Rk~?1OuuRs9{Q!A9V*cr%S<` z(1FtsZKfy^WVBi*C9Q1ECmPBC>QX`=#AC$5rmO&Gc5ld^rrxdSL1aR}L9i?qq$_Mw zkC>omBBY~{1E(Gr^AGxU9$!th@@PR?=Zw?Sh^ZVsbC#s*C3$JqkHWy7#jXl9Xg_$A@qv zr{^vK7l)EZ*<(qbEO%rd0G~==q25Q~*0no~VlL`@cl8RNGmc_NxigVBPCMarSM7Pg z$GD3aF4dErrjs4F=}gZQ(L7h*Gie2_7^cHwkTRv%>&y&#DVlr9Kket&vqI<`ABB-L zmR8fS2eop-O#E!V#OHcHT|w>&?5 zs*u13fp`?kO{Sk7%1rx8-2XZganh28c}^Kr=Ef!qKvJV^y{Jr^cQhzbMd+6yOR}KX zZ!Q{e_PBG?;!{$*+EOGE)V%@rCepfvGV7rgoTLy>QFA#!a+y4?x6)ttCBsF$*dCl| z5UvQ^flahTW#w|PTjLQ2?(n$yH_A^-N%m`I=5k%>K^fJ>&a?yUXQ>Y*;timRwe)#$ z+xKKYBfF^)q6)GRxT1@Bx>y>7bKt~V85L9?L+?u1OdA0e`HMF27XG3}HEx-&qsff< ziNQ~YeDPr90PAssse>iufmEMO?nQ$%*ws6Qu_ClSp{cRByi!!ZiT|9a;Szn`c~u#h7`YG8AV%D} zRcmY{stCv3 zOe@hL+TslMO7dHumN3%3;I@0mxnpV3Fq)gq)6feNQcYDZ#|H?$I>-5w&@sHr95*gu zI_biEXxR-?LUYm*pI5Pa?`<)sS9umMCo7fckG{RtRn?S0H@}w_|66dj7&Vl6@Iw|U2ICkXgffPfPOMX# zDF{J@Rz-|aL$`8R^dTHyjCg2lAf*NIeFX?!CYu_b_Op<#{=0@7mk)P0!Y|}Ez0S7t z1itDu>e!KQy@?MFx$l-vJ9DBx;755MD#kOSeCOJz^>@YX!Ion)+jaDR$K%#UDOJJF z2bcRzg(;xlp`q44E|vW(I8D#veoK@BBhZL5CraJ)u! zRNG0UjOW$PEFDB%w-TbMq%IG|M%MRt%GB4XkXnEGsmfB}f1Y;#5k%TA*PY~fQ+4OL zl8gFugWQ=ONQ)#yn0oihJDU^vvZ(>QhVpk$cf>g?%Lnpf4T~_v#9VM1;IIg)8g!q_ z*Z5}Tw||xZ-f2Z`Ttm*P#pm_usKC1kdz=eXCwAX%RolPDHkR{ba2vNg88^V|cwFdU ze8gQ>vV_a#1;@)9vPE_7c%N)0Y9pi}ugDrtu(Tz9>30O^VLe~+PqhVb&mp#97{c4L zVd$jC>D7y@pVO5+Y7(ZB%qTAi+2injL*L17mj110!;QX7I6mT*oCPQM`W{J(@kS_H zu>OQ>3YZok2WH8iMe<0OU+hoaC-vFDl(DG>@)8I0e%8_#psY8?7a2~lO+bW$67g{Z z-#X+&QlweS1D;5-_!SnA===N|kwZI8&qd`O{a>r2D2FSo@=sM%{P}+7zx>bEd_U;W z|9G3D|1s!uHn8~d>~s2$ua^J+?FE7UlbFu?0@ytDr`>}3DdYbyKWX!i%b=6hKMK1p zWf`jj286D6HJIxmu#sdsFUX?V5);S{8zE##LXg7o12N_1Xc9Z@)Xy94#48SomxE?5 z44-JzZ7r9%_TLW4%F96hgDy`Kk{rmmn1Zv%#3_Dva6Hw(KksZp7wbL@R} zOdJ_U>KY0Vi$<-w+tGY6C5fzHK|&Qst%*o&sO3xqGs?iccU#hHr^>b;dg5EIvaz`5F-gRLXuARxRv> zT?7!V#>2gheJQ3NF8R^70waL4qfEpptQug*;X)Tg{Up!gzKmcBh=46*VkmS}1`RY! z6j31Zv2l#U4DG>QDh8QbF+vu*$wQzw>{|$+w~a(u4Cw-^2hedroAynLaiIeDkK9SH zc=wtV{jXYF1%nI$sj8|klLN8%+0Bp7)?vTz#QVO|px_>JMG)w0Yx;VYHiNz^DuHbz zzxJq}Aj_`YSER8gB~6={YG|l}NdUcoz2>K{t@gU8qIK}O92zj)gl@oShC9;B;g)0j zy9NI0j$c=G-bX6wdl9?b;Ryg~8FN}PYn*{?n0ue7D+MXNX;r1j>a*RD!Ik9>_rl=t z#GCdL@NnsTdMF3G;lj;i{zZcl19fsoc7;9-+y(z43FaHa^cIBf6ttr`c!{==={ODo z#ECf=(cF)6l4-qjqX_Z}4f+;8Y_v}AvVjErRY{b~6)MGn&ynX0)E3mJ>ca9%{7D|K zZQx}xf)eL>tUI45`R=SvW8q|x`N}h?zI~V~4sM6XQ6}EMN2Flz4p6zD#)mb&XMT?Z-Usy={Pc^?|(A%#PmvsEB?7z7y|6oN-GA;E6AD;-+r*k;U%s`q@fj3u$TMea$dlU0aq~R2~i*HuAN*+wh#K)>CoNcqr45(RLY+tQe>|G|ZG;@WG)MiFKL6!dB# zJQK4gx?uAOymG|}i0jiwm^PULxel*Yn|DYw&od34_oUCG_5~~TmVh!J(m)H@F5_Gc zt%?;J`I|b{0^X&kL+xCg&^2r^D<$cu6fjP2B<2zcI(=fa zb7Jgh(r^$={`v}2%zqer^$;c=9q`^Iq0?UJG}}-L^_&WbTH zO(H*pUQI2U2{!c0x~p{x^-pX!HNRjUes#JlTX(=Rme$~R3M9?=i^1%ZvjggmV&eB2 z##&{H7wGDbD#M@jnA@QcdV=^ao#wuP<*qNIpBRzR09t z?DoK^t7Bi-IW3G}t=j`1q??u+100WOqOo73cOuP*iG&g`Z2et z*d%CO!4S}dz!kh@3aF=&;GEfa5?Y`NEV{)L9QnANhu{ZC-sG9eL1o8)1}1^=I0}}n zgk0GgU5-zz3X$5#aNESXcFUxSptagbEED~~zwbfI@jP7WOW(p9U!7wM*M+D>I@ERb zyO!EhP5H1!KxK=ji1;a4G{h$9Cbtk_`IyihEwfu1lRPxd5usVE`cO01I?qRjCw7*h zatD)#(!6izLB|Iy-phO;{8|4gz&V-#-nl(w*>%#5@qh}j}RqekFBfs%i__0dU;xCDK@Y~hx%Ye^>2YygoYP~307 zZVnuD>5o{p%{j8QSSN#Arsev*(HfuR%F4;Yuib^kq2u%6#mez#i3g!y-tQmIbiFVk zh9G*CmQhqr7R|n2p57bZtaKe@)kI^!Fjb*N%OR7YOBK^{^KilKZK+QocObY{E3%K+ zrq=&fS%{PwjrGWRu%HuqmPHYoATd6eQuN+mpU?@%Y|%tT^v&7(PS(mf-2AO5&pY2> zOsF`vS98SCpzRu3Xy$tx9z?!X5u_9HJl`ZL9AHf zkx(w$ZLk2#>|Hkr+JTq}F$nW(JG1KcK%I&SZG_pN7&VFm4>Q4xkZ!|_fnDFLhq(ht zLhYb-Z?}ng7W++MHH8iagLG7&9#zluUKJ$;_9&7f60FHq9@nb-T8(E*-2kry=kO7; zmrv@}!$i=cKoLM&8PWqf30mc{idBz}FnTH0UzIz}pMo4$2&#{5%D(Ai-WXC?yjtMM zWk5oa3d^@36todB3S$%D!7)_-R0RUUhb!5?FQzjg{yw(-q zpY`?pSz&8@CA40jYA8Y3-V5TnK7f|rjN5GM=3?nbJ}w!WePR-XR;GJ`#P>&)RV+B1 zf>kbpJqJE-f$=(y+@<01GYnNLW?33s`wK1nxbiHBO4>+DYKE8b%3+T4;sxqM4jl<; zmn3#*i>ws*8f~bDmnM^~!!vxhv1i2&ak^9hgC^YQJTJH0&|T9q$9> z_{keCo`}zs&4mg-eZ*Sg?p2oH(k}kZKM^e*bbUgcX-j|4BH;#bqLqSs(5`}w2hYui zTV?14J(n{Hlrt^uyyK1v6emtkJf&b2JY~z=$6!GwL(`*DmDles*=@BRrr_Vk`E!ge zRV{ij`=roc#*x~$T|h`XvBw}VoGOMk>?kl2J_!kFA&7mnJ<-C$UVywviHc1EkgMQu z#m~)gn|s=XylqC+Mi_pOtK)rOR#M^@F&Jvin>N-d`iKB2>nc7u0<#q=hb3spQz>~r z!sth|m%qnT(5P?by}-kZMB9q0+sv(3TM1;EW*_JsByqj36yaMkC{#Zc`FU>X>;ej1 zy{EhA*S{Dz;FSu#72&4oAre5vJ|NChei5vL_3SxZBk88<`xr!7Bef$z61tp zQ%xs>|8lu4O5N|S&R;U6rSf)>oxKX#Ca_?Dv{){Ls!*d*$n%~Y&8^@|{af+ky+_Q!PTXj<8 z9{$>LiT~`zr25U^1?Zr!mddVT6Au>;_a?ak(w7uVf0H@Qux&rK6`F9oALtD~tc0g6 z{cRoHuZ+cH9w^>gqbi#6CHkogN>`n%yB)1hS>J)X>{nU6t>PRN%#`SR%F?r^eduhj9wZZC4M2XWa1V2lx8_*Yfj)y}+Y_a_mj zsz^w#jTL_#?mg!lwkPSW=}yrTalV1?H^GO1W@@#i&569Jp}U?UiOBXk(<%H4$8dJ+xx}Db5+v}B70Y1sy(&@ z`qaD9Q2MET4+>;+gi;zAF=j+K7jaFUE~=gl!W>y&5(P}o?UAfPQ>szdY z4=doLJ*-+%Q=`-DBa^4bOFG^%8l=>~LHWX*d2(efe{eQAeJIGH46j?CEd)TKT_@|+ zE4Yk(IP(5#TVzgb=$w6pyF+`!-d3!{&du$NPgg*d@4Zq--FI`kvn{cEC#zsborl~^ zqgJXGB9D>RV^4T*bI2n8JNCa`T}cA^jhIn>+GS|~0LuTmcQUoGHqraHDz9g6XJ`FS zmFKZa+F(oEIZ@L;f-k{=Wp;3_X>KfC^xz#!tw>xoksDHDz7J@w|7`}MoVxa{z(LONBZd)kB-{oC8k_wUu&1d&Y&59e4*N4f1F znMx*+UPf*T(~j%gc}e`ST6U^N6KjvM(#cWN+tTQjZT3dpR7tAEF^u^J54Xn3l4@nS zJ;4r!=F!-t_vM3j=vURftkrHC3b*!;i9Rmw z_bs{j5pUC_$sTXC+JJ`v;ZuHz(yk!Awj~yn{TR^cY-z8kXqg&@)nqJ@U6U=Ok@p%> zh7TKplYlx%VArL|Sy5_&RvzBHTGEl>wsE6L<52hu`;gxw>Ei$v<;Ly$^NawMq!eM&K!oT zrY7)_TcLRGn0~mFOtZ|KKTj+DyYTW{LCN87cB_7QN3z*@P(ku|0dPjd-!(w5>|NbJ*-km>dcZxUrK=`;>k}^$%Bp5~XS$0M^PxlPiX$i);G?rx#fqcW zmL#CmK&Y)?sRkn}dkz&f$#}sX*7nM~wuGit?r{tNYMw?oJVDR>m=oBSH{Keb`EL z(BSwAkOdBM6bM5@C00RgF|DsR=*Bl~*D{B8ig6n!Dmt0Y!Ni5UE6s zf;j*J6>B^@Ff!Cma%+NHos>cs5m0Oa5A%C&JIHTY9h}1(tf2Hi@h2pw!Ba>s9|v=Q zdaYZ&(y{TPT`)+42Y0k4F#$uLORi?FNW0Jq$9 z)-UvE(4UdM?SrGn2Ct0xY4iEP*AJ!#<9Z5JwjYSE9z@fKKpC})+vA$?fMPDHu$pz! z)dF?`3<|_*dGE02n7bs{XRGpwn}GIaPjIq^yCebu#ww$Y)@})ikr+)NxNa;6asC&C z&~eXS$U5*F#tO84!WY3H73*MZX#t$mYLr+fdDKE6F;yac1~-5Sl1azw)0VS1Na>^) z7+nQ2)07m5-xsTI^yjK{-C6K&Y1ZajK;e^7X-nH&iS=B=aX5_)#7Np##$)paXKctx zo}B3gt~FsqSD5;y0@}(JXjP@;K@Mo~V8`=ip ziOTFzQSB+>gx-}QKV`62P@^^1v{oFpymSoTN<$cJaa}D9%{1KS___d0X%7j^F%{5Y z8F}Y))-cK|YDDimEWL35Z;B7VO=(gdt5eYAbZEAl24E-`)V`*)j$++Govrk;zWq5i zm9|jAvFjYI;rvibdd9}%V7j1!4z6!)J&b~u8RZ$iZAsQGyedl@qK ztX&S~Z|NDoo&*XxSKHWxiyPYD4os}k^idBlp)GO|1UEFLN_OR$HS+xqqz~d4MTnGC_fxhRcZ638_ zvR-(9OB;iCVYeDe;K+?pOM*&KVi|1ux&jyIhC19%syn^m^+bd3Si+#pJM=3TiHM0l zV(m-oUdw1RaI}#WG|R>A_}CG!EYC#J&C;WWyxi4;U@lG*Hx<^$6elBH0Xeo*J;<6N z-ksId2*P(;D+&OZ)lwo8Q>>)>lpxX-b z*et+nN^s3x{x{m*DM*wkOcN~Iw(Y80wr$(CZQH(O+qP}n_AR@%dNy`qy0>TcX&*8o zAM<47cg`RAiKmrmmcK0)DeI{BQf4bKS#5o5NsNx%r6boL+4PjNy@71>9^MUPeyO|# z%NQ_yYa9;yhBTcD#`3Z< z!_0!~$S?Rcy)HC9N`z=||2i^K7AD)!T&p^A_ke`w#{`tYf&PHC`oUR4KEC|kYAtKc z&bXXK(6`2JL6ccbK~Z86|NnFQp#2Pthus+KBJRcK2N|OPl@{3-WLUp+2FQ*Xw(A zPJcHmm(C_ z)i)POBO@h}g`Sr|Ny+;Miev_-6WTC*BrTmgp(e`C`~o<8R{AX$6n10r2ihjaqh{qc zbFGQ|C}rLATZ9Gf+sXXtJI`<{Oj7Uj&QUpByZS7}t5HLB)nm$|MO2cCJ(=gLdaiM> z?QyFR3!2!oB7-_aoi_^HJ~FLlLVi6$<<2A*<`^z7gCzI~Z9;gQF%Ii*&fXN#;9-Ks zBjEaPJF&KN0X?^YjD7v3z%4`L8PvLH|J{8R=4?+^TDYlezI$@^2?(QXJ@T8KQJRo& zX>ofkREwa1C(bmH5>Indi4HKS6nO={2*=P;WNyDQw3LukO!SzSdrD9GgR)!bm_6Xn z>#YpJ*!xN}8T>QBExxwPUSsqy<26O~Fw$JS=bUmt2>rpryf4IA;o#?=^+vl7>8o`^za6P-epomQ8ZGZOLB z&Wh^NB2ewNSrCUArzNuWq8Ub^0h_i*dric?)r4ySL7Y?c5C;_HzB)TT9VlYdL0 z3@7jJh}^Ljr5q%P0@T>vYQ$o`0p}Ux2|cU?j&bNY{+P+-f;hrP2AZsJPNQPoby(K2 zfn11A;GQy}ScSzFkIWkFOtCG0|oBjtWD1c?OX!%>VepAr*oz&0NL@E_xc_#z0wz$4Z8(1 z;+ymL;9B*v)1iOCBV*Q$FOjZCRqoBXz&*FKhcCbF0hl<~)ezzHQnn|!MU}b98_tNo z1CYuyMQiJ642fVZ>N$%;g99i?tc%pA~7i7oWxrxshl}uu+vf81D@jt2+xyO zbr)2iLv-aKjF}5q!+X~UHx#R;l6vW@#|70fq|!uuIbyl~;b%w9a_I9KF;z%Q zg|)CICpN7kQy6`^`7Y7Zh+$F%uX&5RJar1{qv_rPY4CJ zQ9$b%*l4JI!y(=Yz{Zlnzg>2oITK z@9^?r)hT>&lL+~P(~iVTBHC~+cedlbm$CGnz-KDD#|#78g8fp}N>qj(9C5>c%9$b{ zDwT%|ksXRdpHiq}gEB;^CPt-m>qO=ef*unFL@=4u5TGW5h09-Jmh4PEH{x*UnfFRA z?(QpSJZN6gP5C-a5xnybjw<`;CZ>wKaulPl0>p4v;#Z2+k_f>$6g0 zpUFq!x}6V&XO4!aH4s`>cTt~UL_xvltqniAA&u}SJ~hKAlm2f9j@0t*IiJvdK4Z_G zG_HjWSR(1|p)v7y8G2`F6#0^;A&|7ojAKut26^D zyG87bwbPlm`}hDzg^GvVgw3+0#z+*^xTo`&mb30*;g)*AFkQr~iY>43+^3^3H~QUhPY3w?0ZU{vjvZ-$ z{6O)TZHix`;Z$F%Nz_LqknAf|+zEj|3ik8%<}$vpaxL0R4;c_b(bGW>F&Fo~D`8?$ z*m>{dIE_ECFjds}Ud$=ntV>F=`RBm|M$!l%$$M?j(i2kbW9^MQWQ!{|^;5tJ& zb}1fl%3p(n#=A}QDKqgf03o^nOtlgarm47I{g$VQj8r3$XyBKdTLwm+frL7`HC2SU z5V|O5u;K^rjXZhudr)jHD~Ya=@yV!!#pK)yt#e$_*HTcd^JFi~y9scXZ~=8Sf9qvp z(g){U?wkg5SNlJdhPeB*O&CA#m0pqD(P&p1_p5Ztwj+fKfdcmSGJ|%kpeObSLsyvg zqZV8)oG!Gvsv^Zj*LF{{%REtVC*cE9i(1T1HJf=^4>4FYGx}+2*EqhR9QJcrp$8lM z<*qh{Iz6Y1%%QF6S`~AIgNWqFfYH1}eUN80WCp>(mroY>0OnYUR`>~JZ={$S(VRs|{{D=|F7tl`4#a9S*zoCnF6VRsHuOWtwt@c^XHSdp z!R4?!Zq8Q5LJHJ3kJ@Oo^DF9;t9Oh9%h%j_GX15x;W2c}?+n{u`sof)BjECNqTMe# z;{+Lt=vl_GYwUf?H-MojW5Sf#F!gzV8F$9AI|`%eO`V|isG39|*cfvF$YF2GyZvXM z4#emNx|VN#&%(+`hU`|L^LG^+|8ghOK6}N}u~yAb{Dy@+&W2}qSi4azJ$kmZCEMR7 zf0KGD%iSwTN8?i{!63^(N0epogXS41zUQeCe)nBtG&onQj=1V$ltUKg0g?wlhss^8 z{Q3*GZ{DYHIWX|LIN6!U0clN_uNk*@&sn)ST|N7NW13SG3q;aM6UkXaVg3yof-)sx zKMs=dwR>hfREv-3Y8CqJrA{zUy9HKy(O@QqtDzMH>K1Ly6i586{bv1!U!TM+xxp%_ z`JHH{^v~m9pxT+O(j0s{dapPiSc1&%6G+x-t|B;Lfnh^MCA{#4fB>B~^4nTw*oPN? zYJUt*O>M;LEyeoHOElGH26`Fkx|Jgy_;%ul27w~9+J>i79Q8Y#r40tOe%-oCV`q(B zN2Q@B;5pez9)6wHMLPQmm0m+Kg1OXU*hXjftFX6Tg(|wCGj)g8i@%{;sD6ix-8)zl z13KrIxJgA}WWB4*N6&lurXdB4DB&Q0Mf^&~o0f>lA+?JqS)B6E`$9OU`;b)-#IOQ1(yq%*9@z^LT%&!C0 zXZ}_>*Fq~HkAiIN4AO97c=UzY5UFp4a(-RwSK*mUADsDN`yBcg&bSOvfUT2DW-eZ! z9=@Bd!bB5e>|In_gz>$ijGr!~zzQNRo$A(XO?=wvb007v(j3UYZ$DcS*V%%U zQKS@YRk?eKPnH#WAjgI)jznY|{XY{2(*knZjNO`0MDQB2BDh}h7v1{@I8v%KeFpxx zs-OjePOloFp+h3CjFAhdQIoZZn^0@gG4a+K%km98q_$g#-a-=&a?E<%n}3T0Dy-FM z@_EfZXxzJ(gT0Ks388tQIX>0Wo`~Uai%$UmC2s<0O*YUzqO8s4;^NEsM$`H7=IqQ7 zC2w3jf16)tCNL9gd^kHgcy#x~+#lON@Q@IZnx%GWX|$p$@t|a-S83DEYonkx%oM>Q zPdT!Bx|3euoj8o!ax7VWn`lyx#u(+EgVtQQGzTLJ79^0RKp7Eg8MA1BUC)Mq+xqR!|68@RL!U@m zL^B55McUNQmhS;XRY0RihoLZn!jbPVRRm*j%pW1HE=e>Ok*2_`O#GTaBNg->amo?u zK>K0ZLz&TKL)K;0b3uE(u5wjQDe6$@{q-fjHGUG)e9BU`vu1n+lb!Jt%f8CJCw-^d z(ocVt1L-d?o(j@0#C*6GC?r-s0&R8oG6!$BJExC|I zx1K(x_-g!!OSxJZy-m?!6>q=E4g^n2z-mk8xhQVoSU9x3vIt7OO60c0^b=-jng{Y8 zD=ekO!-DkKZywb+DvayOztjEk$-keTe30i@Seyu(dH%wd90q0Y+6t*2jRtiv9&l_q4xgp*H7WV>22hE+Nduhk;$Q{pA~SqGF{{5`N*mKJ$^>| zbJt4%g~AMb$kh?y*zG?3C$X=dL!j_@_M*8iO?Q5hMF+8hTdh5U7^&`v@!u`+}oW zKXVYd9WWC$DxM+7O4O0tDONOqj3G>``_yLOUh|zaj4v0eMB7zK><*#0|4Q;ONNI3X zfq4%sYI+Qijz=k42&;op9kQt@URuleHhZs21gFruO$HcJe#2lI3Cfy>F0Z-PIQEri zOvnq+t91uRGKQUBwk@%1m}NNX1LG#`K2e;MV#(F1CD|;b{^{2-Pmco8E;(p_ZVvQ& z!rJce384P*v-Fxos8UPzCA+a?$xo!puSyA)l!ViQRd2O+MH60U;xqK>7f(C2h$Yb5dJvE z;w&Wn3#AaAUmBd)v4e6o%ff)gUnOHVW<|Tn4bH8cM#y<1)4nN1JV+)h{0~6t(H?@@ zpcZ1umGD$YMy?dU#)_2KeAQp-Qp{RqZ2Wi_)1j3(*Ed~PHvc{pj{rz+Kz7%QZlt5H zfje^I8uq(Kz2!78i;S*4`p)32^FFijEyCup-Fc_dXS1~VNYU^x((D0*i!W^GQs=xs zKWSHnoL?0POIp0OFu`%HERikKPSgP7I~I{>4Og~w?Sntsx#*o|tfEo z?`GffO`<_rYc5_BVR(l|!3*#_VZ>lxxF3p{a#^tE&G^P+b6h^(Ut5D)owSb*y*U{`%!i0w$^I7FZPw_>dxzi6V&pyTo!5~>{l9%&1&!R>72V| zSdyb4suUqg2=Tu}hIOfRuM4Lk%~Iwq1K;g8p9=bgOsujOLAtZ8GiI6nX4GBzEkQ~> zx4?He5tp#+iJqbXUlD40(jv#dI*+*CfF`S5MmjfcHA$RlEZZXrDN`*pt!M$CxPmG7-V@*;m+)fsq4W^72$wi9Cm1oPE8 z!DTV>$qAP4KIjQTkr~u>8}7Ah2%wy^Hwz+H^C58A_2uj!4ZQy4s=?Wm{NX6&Sdk6& zYp!tBlj+PQ#HQWES?|xbwe`$vxxTN`e6?iYIufRFHvyHejH;#=RlzTCeNxSqJ&2@J zTa)q+`zGGvJW`L?){Lc&Jkv%63G?H?*;W;5!J&v}aT2+fTFd?y!} z1o~hbP-8Zjc|erIdxBkwbo?ih>d$O6y8EB$%XbCgx#wTm7yhq0>Hi@8vbFoa$;)5f zGFwH*YLfxQ=cTr-h@O?bF9Q$fr)RZ z!TXly6z^?IyH9e-Okz?pC>PY;#p8)G_gv?IxI^eKmPs7CM4(E1s|9$}NFseHnY0fjrYVnqnCh z+jU1=;eh0iG$p-oTBDN|b|W&-_=7Xbfd}M}j?_0z258Qy4`u-Ko)s12VrrS4-}5-3 zy1RbOy7kG8gb1t%sJ3~$I0hH66RaZaY%=4kha#3EjeKAY0*${QEXm~D((1r9nsR0t zQZv4=k4LMx%|HQPNH^~?%!>s3#kD;{wl(M|*`R*AB$NhXAdy&7U7gvwC@8}ZjKrb{ z^wTs#12c`&=v>nSOz^2H%O*DSL;!N@@s@HdC*hR{bX{2jv5iW-J8)vR=U7}|uiFpJ}oNXKHx**Q07EMV|OC7aGJGU@$F z!_yg(8h;(Fy?np$!{-z)W>M>@R@5($e;>Lm-~2X$(sD!YvicAG z%=8d^mlH6pWvweq6z!VPrO8GO0L90D@tERjJH{xrAs|5{-?7`1-0YJcW;(BypwcVH z7M!Cwnfi-Dmaty*eAs6OxwX22F=INkhubZk4$)Sju(^z2<}{0Bx8OEnsQ&!jiCO=G zW}Y5~O|&W7V=(y4|GOu$`IXK8A~;3)ev`2vj z#cTi-dqfhUeF9@8gVqyj&LVE0S51_0G`eBn2wrbxx@Sl9dz-)1Tg(|5McFj8eyBac7YTh7>vW`u$mJ>XzZjCk?(0eGas@XeZhkTM@TW=D zG1Kw&vVvVcpcbig98b^DT-r%DO;Qx>fd5(+t%&^ zW?U`B5Q9yNa=ii-BWqxp`dMsQ6v~>)M6kV|1J9>eg}d_jD1Hq(Fp zL{CYnNBy3ziz#Q%kf`ks8lD+eq z`*;ye!X6;g<~qIe)+sq>MRDtfN`4v<)4?=Ec6w$~DMW)bStnRV9;%k$4%2;85W@3+ zanB+zy1`o%t2AP%&fR#eKqYC3v&)xgL^Ht4VI!P?sD(cKi^k+Bp%dek zWTsVsSCQsj4D9ektF&1a$73+XuY6VwYxmj>M+#8vhj5&tlv>=x_Kx5yys+_cl#KS@ zUU{qJ-@8waP(0|_Wi97qsVEhYy&|wPP&2rSJ$NUG(w!gfaNKMc_d&FLkochS|2_)8 z4hfPU{Hz^<4n>^H9LB;NI<e6p6vf~q5tFN zcVB4MymZ=RMf$1f1v0q-cFCvI<9ZQUEINIk|0j*)Wlg?_9XF9LE^jJ>$O+I;W$Nws zaeW|wL}Ff=HiB=B6z5~`(i&B)mtfSg{^00BE|D%X{t~wP^XHc~%FfNgPdJRKoQ$e0 z{QD3E|L+L3y@&hlX-_9(UUM$!A#0_9Msnix==3rv2p=xC(wX|qSh0pJT56!&2gyxK zLp3R|k!8K;zJ@AoNLpurS|a2fdhm6$*LL!p6#3YwQ}Bk(q`Lvym+mt8c9x##9=e)R zj`R@GLrv1Xipa4(fwc19c_^=>woBu-GBXjr6-FF{S6-K)#STR8j%ou6KZ(wK6_tQW zvLkInglSJnrLv>y=$JLApHs&^vs6M1szus8XZ<0Hxl~WC9K@$N5-^&>$VGv*atCKO zZ}+>iXQH;QcJ5E6wysWAb}yFmfM{0?gG6KU+1x*zi zDQf1ESk(*qQY$h_%85v%di2B^k*c#@2E~PL3{J3+hPlKB4NIuENxw%dI(1Ydk z%$b!B5;+s9GL%MKoy^mq!NEcG67Fh_LAf+ ziFU+1@h|xk6Rg@k)lC!ah7896;KLB`4Op|xzN#vxcZ}U}m*Q+R_38k694PJ@7MDv1 z%}ceT5-%K>8QpAnvqY1s$Z-eFq$Z?jngp2kZvh3C6*ZCxEI~9P76GaU-7PowGh~77 zhnyJB*do^SVy8N+Tlq73>LHv%F4owBFX2j10sce~A4Z9R5h0N3$TE*VIhI4Jb&k+5 z#05qq;T;IAa~~zYMapjgVyt%KC`BkHw8old%vVS;O;WO(&8p#R^_apoZhyHrwya>+ zH%&srE}`f3G{a3%t3h0jy<~G^1w1v&jTSrnVY6)m2YF(cc)6WY;YCCddty0+82KV)cbA_}w~@qVwRe2AeUxfL|lI1M_kG6fxtbIY(eq+-I?3 zRdoy$7-C3?3!a#?%vFK=*s)rv4et^8C4(ph-102&zjW_|CwsETs;MG#VmXZ@DDB!E6Bt#ays+j)Bs&qq- zSLVMgEi1NZiVAm6r#lX%^3MMa@S*pN;lcgx zHWz*wRW517?Q!-f3)mb4eUlyTmC7z9^L83<}HY0Z$t6sMQ84A5EzB)pcReih}hY{rUF|m=&s9D``jL$MD+oo^P5vOztnoZA)jHTkARiP> z)_-849}&bN=CslG56W-g{7Fq@KhVR*y!;gAK;D45`vgb}@mF+&^jijtt(>TLHnrVO;xxCK26jQ)%~EeR3h%@MpEs0B_plvxTDEvd~awnCIQ{Ne8ha zvjlhxvnRuEVy6H@4|@eKijll9X?n7UT5cB;-hS$<5tk=Es;{daO0GpTaM6Mi$BORN z%L}6>17^h}^T&=qM?5Zn4a1m!m`R5_LYhn_PT+>Tv(ZNdI$p)J zXSap!AnH9uB2x=3J_6!}r*#aQwV&9t2!RBgwoz+KVIoC^%H+>*Wx;tu$XxEn7(mev z;@A}yGbXWV!UXImYeAMl4OM~mC(HVbcSu)fYKHU%cd|@S+>W3zE+N3)Dxe`^B390B zs_}aqD0|fGX)L@N%~m=QCJpFCo!bFjl*ya7eDW|SFO{eg$;p-1P`0{}ZI`O-F_pza zHP3S2|1GLJX$7P`Pc z=Zo!i{$<0-Fxp}kMS#NULSQ9}r8Ik<8ljX%j zKw6u^hmBCbVUr95V4ss_>amN`eV{yt3I6Om!rWZ(I1N&a%ck zf^_t;lR*sdM!1ITw2Ne*AtpM!9H5=%A1)S=_Os=gVf2 zol6!%E%ffc0#A)30O>70dzcIj?m6jXH!Z@NyJ_z^XZ6G;LpD2x{=92bK3#e`MPYau&-q-WZ$3h!Ip)W!`H694!b>E!LZ<^=rf@#2(b6xiCAIgZTIdO9+3kM8`d?s+s~yWO zkVG@0Iw2c~xcXHrWxnAJh;gM1ZVK=0)fMyUbT)HG!*bNJJC}AV5?L zXbPAt;+-|~*S@Z~KINt7CATI|kfI(;*&pyYvY@%qKV%zh3LP{}U#Icyc4Y4^CLQa$ zMjoEFO?}+HJlb`S&wzIA(DX|2H6+>#7EMIjO<4->p?-XN)4w^gTaWT61g%@q6%Q5f zGHqUjsi;0!^569;ILucHG$V0$khhzzu%Am{y*Yz!o}=wPo2)H^( zHKB5pWRF)qYkx!7rIIhcVyPkW-TKpnrfqWAv?p(%3|54kGf z6REq9aQ0YE1P?WF+*5>{kM42Fg3NRuKg9+o7PqS4g(J7LCw@b^ih1xXKqlk?7$`jy zTliaPwA3lK9Gshs-QQOegpu#&ZJYw^GL-FIu{C|!#@SB+Stja?(lapl#g<7zPfh9J znVG35;MJrT6rtDKm30&mT2D=tdx!QD%^VH+)`l4DR0s$xjAt{bELRqoq0PDXfnDRi zd>OXYHQrIIWq9KZ=mhfWuYn9?apTRtGz|=E!u*kfmxrA!&ptZb^`=%UH;+FK2HzK_ z&jOk>8EjtVm0u#41b_8J;1d`c;jtud8a5wGSb%}kd0NN5wK#B!nZgk`DiJy{NYCk_ zp;HJTojEB+OPG2G-`qHUoI2lf8qR}9;Z*kBWBJQ%sHF8Je)LZ02lvU2kCw&ll=RI{ zZwTY>GjCgb)uYsP1qF@As$TZIcPit7f8Y=X>N3gcy* zyimSAtQhc)txqGqMH&E8GCT$+8W<&a`-EGyepems|K`N9GP~dp)Rf1n9IpEq%Q$hj z;1<73cbu5&(i8Mg1o_08ysQbz7ReR{>jbe-0yGuSQww(+Ju|henZw!8$=Gk z%rB(OwVL8|Bq)wCXG95UwG&40{A!I>1UhaEZB=t%L^F0fr!n8*|MNajlOSZ6_4^mh z`F9`qZz*w8YXeui{~R2VqS_z3@tetTN?o-rl>nfk^`ux1&T3s*H6e&DOs%3~j5OFG zX?=kK<)_EhAB)RHV&CukaI*fJROAFN*unRw%j@U|TiP$SXCEX1D5d}fF}4+>UT)8U1J?&51d|(u0-`1Qgg}gI43pGI7-2Kw z#|A7e8TAmBMm|fB?wh^uEr+Sz9$@HJfuGRCTobdmFIdSHFSP}f>`!vkIZQ4?c^&E0 z3#Q^B@s{y~3kD3QYeFB-XtI8Tk{1Mv<2*||zPZ!7LmCZNkvR241X@zk4kvpUS2gl@ z!~!LG>-mrbSd<`+3Xlo6>w;AG98tQ*Q~Kp_=VCOZ7>&?fxFW6$?0LFnnVT`Tw7sV# z3Idse=Y7g7ip@UZp|7ACf*+J5lUMOQb1tc`NwcfsYGnJ}$T&8t8ZGa4h~;%g@F0?| zUvAmey7?{0LNMjF$7{vVP+hBFfK!%0=o`m#mQzYu@k!h1j%-`;5Ig5NZv&DMY6Q*; z+;R9~+sp%2sf*{Ngdd^M@LZb=VJJyV%md<0Yt|&6Ow+7kL#hkAvQhFJO8tnl%Z?tL zsOzDpgr1L{c1=>x#P)ts$Yb*D+nXa-4o2bB8RtxF4kekXjk=^t#qX@5eT3M+8i$ob zn2{=gFvK|3zz&w%Uz_NtR(xe!W|q<|)7W+(6)@%4q}KDc{o z(~24J4fkI&-(2&o-+X_qX7=L$n<(S|=j;9tANl_%^>#A(kFJMnZVRVP(T3eOYSVLg zFdgOt_Rf!hr{qW)uFeYdQ97yd9;y^593UY?Y6N|I@q>qpUORjzpga<@hgfA=YShE@ z^Yfc?FDy?o*^2F+nOfHB@P!QbuZySLlY`M*x!oMRs3iDYUf)+|Z{PR*QTKc4EFbR= zJ$k<}B^B0-V~dgrR=v{--EcNDJ3Zfr`=WTALb4?9&dLi_MC$s7ATz5iC2fbYrbieN$O>yQddZYrhL5RZb~)KjYcn|Ap!NRV$`>z3JM- zWIz4Lb~6F|RvxhLktPMo(5Puv$+l(gamRGC(Pd^#G-TkjQHd$*2{DVY^f^FsSsl0m zaGnhNqf$lAZCY7l{5ydsD4^N&@G!SsTqu)zwAWmLcsfJL!A)D3T!zywZDaDXGrYD`)T-Es{>0qK zRybzs$&j5&5VzJ1ABIPLrMXwGr5?R_yf7 z-rhS39580f#toz0g2HbsFU#cQ<48FSFHoe`J&mW9&<%G`Dl z7}BXW$gn{>@P@sxukEVTpl{8+9WK*{+bbVGeU3%^IV(qG0BZm zv{hLt80nV*#9D_?7r8?|2na~2?I-gGwKU172#?RQ!sPh=S#{Hj#9m#i21ir%6*c`lWto_UI@S1Iy7cBf0 znOeZy4!Dxna zkCDQcRKXqmzWY9S4{Fi-vxCcacOLZJ#}rXwi9_IMqwZ`7*w4NBVf<4d!yP_g0O);; zXJ!(x7Hn4hgc5!uWAFqjFkJ9IG0}IW_m z_fMM3@q%+_<%N;n{eXP_4B)QV`L%~#Q(6H^M#10|>lk>sj?Oqiev(vUF853a4uAai z@YC%5Tfj^pec!TDeFWvvpaDmGi6|n0YB`6knGhb3jy@5W5*KmRk&>&Z z8J4ORM*1l!EU@7WTyqx1aBn;=OS30`umEA+LYkJOk7D z9;Z4XA!T}kowV6zy#N#k)okMPwDbnIpByJZC^7S~IAx?4Gmni9M8p;8(Qq{eAUEv^ zikXjY$Gx@anjaxDY8~sXqt9A;^=O`r+O)|iHuzr|DQw7Mq4 z>lW+&lYpz~U1=(F$Kmga`o22@KBY1y=B%4DiSD$BIZp{bv`FY*^PRsc&hCOUp^_v# z6CT>QMi60{M-~_tL_U za}vA@GF%W8sF95>vKc`aqp}Qoys33Vu2?&Kg^7NR@k}UkCsmjrKtUzghiKx06AyNFR0f`zSNM^Rx2^l&kGHXT4>?)6Jj5Cq~tX!Kr!6! z;A=z8%}Ko}fe(5)!?rF9c^FI4_JmK~@qAmC@bujSQUaF^7ThHjNf)w2>w8La^Cp?{ z5tZIqYKsOj?+S9npBwo^BY*i(*0(mZ%3jJ)QOqQBX2oA3v-?c68B6)_LXE?dVtEYS zpOto*;>ek(=?oIc%Tz2&Vw4kitAvFR;;j=v_aiBIN=wvOF1nwnkkIVW(@TP+g!mE~ z7z&T0BBlE(9W?qQO1Oqdo=p_&?Jc1>qjSI1Ti%E-FDq+nAe{Yl1Ep*0SA3HYlt4`M zG0)+((bKHfcvgbH1_G4pD!>Z4pu`9BJM`llS@IZ_*=lORfFI4{>&=Evf+o*5B*YC~ zm5YPek65nn27THzp(Ggv+(8AW+}#aWYVf1QU)Ogi(*V9zWv$eretGYt1+<+X=(^$L3+1+kH z`%`p9GedID*#%Yu>qCsT=>q%|$y$wt^tinN)~G8jvq334d8Zqy-eFV+`PJPqU6nM2M!ekDU}*21PKcBg1$51 zl~ooE+W}dcb0@#oXIWSjB!HE4BQr)FC7a5=et(5`IQ&$9m7>gB7wX>%eU?RM%NJAC zgy0w5Ekl{a7vZQeHsa(1IHvqy#}YOTt@D7jwF|b|?4qU7hBKGu_?i20Y)wEIO>5Xm zbrNMj=^)V~y+31{g~D!+>GBgfjoSA08J6uOj+W9WPZcC6Xrvtq$ADElgR)`c{qp`; zTbCCKwe?dpP?#zMgM%gySFzq%ui#=Ey67<6u$KIsAEdkhZPW}2nSP?2(ln{AEVw=f zM^}QVy7L-CPT*pz4imPWK>Lufv;N~8E;Z{HiU55wfSFu0df_}bVUxiW8Q`6J{G#cl zrx;QJHWoDxxTv}|Z#}Mze|gmHOyN~UyiIc!1>o?c%x}z7@|2KnH0*ijoU8w= z5SGH4$^57m8We==Tog(psC;2vW3#8OyQ2abJ0XpMt3O+z(|ulU#@+e$Zt{#UEw0bo zw66~>R%CHWI~wyL_znlCl{GNt)s8L>=*x$GA_Vq zkig)18X*>6GvU;t=4x=^f#|XF6e1&4cz8;rEF+|a=HruNU#i5Ki_d~Qc;sxhBSUbF z4F<9o0f>ZwcAFDyS(w?ov{_h0o9=T}70ct~QB!WA`d39Hfow;qna@qzwsE_GK6#6! z-}C!k*yFdoQ0hM`H zEb_n=kKYNclo?$axW}>}lQilb)gOZvfU|d1H^99nOYhq*AB#Yf$eA6$i|t$oyo0~A z>`Y)iY$D>^neB|YGwMzZ?EcooTH5Z59Nnvc9|Tm#zEBk68qTRDk>mT(K@lKvx=Jf~|{03fO!y%V=x&!pQHe7`FT+m zRpFvB6wF5cmC;1#HjYdW(6Izt{G0vrh0^Q`a=osXuLY( zQD*hkVqaQ3GCF=EmH`NB1KtTj?sctj+F$Ha%}rL`aeLrN&%f!6KNx_hK1&q?`<}@5fdj_1bs@-9~DXQ zF0GP~Y#v@^NCB5d;8W$mZ#6ZQ9jk&!SJfKx&eQhUROX=jT|pybEG>xO&LA;_C+dN` zssOSVoDVPhD|c3Dqp^7QSY}a825kWh{ix3x73F5lD?K*TdEjU>zA^-Z7ERYBCRiWD zWvk#ZrfZUiym1oD4Uj~lF@pkLxUYzxc`O%f+X~aphSFMve0O=jP6(s=%Tdk8s`o)W zK;&K;n)MabR!Y)kPuMZ1w`cLL1Nwx(%-%u9xq3E;CP=0-hgm-W%5dGgEzWbyy7I3f zSI;{8V-ubQ9MbXRiSJ&hl-)c_vc--8U}tGO6AvDw5ZUlEnm+^km|UuCn^sm{{}*BJ z7+!gtZ3)M=S+Q-~wr$&XDmE*&?TT4(#kOsn_)YhnJMZm&W_muI^ZEa*-`;DljiqFe zKxB$4xrA7M9AAm?pu|4vtWS!LQ>Qa?0M12B%aPcN#T0*n$Q-k{La463^kFa4(300A zxR7U2%YcIxL=5J4U{<}8kH404``h2w8W9(k?p-&lW%=5hB=zwX0hxLTj)%SWDZ&l9 zPM3;#vJj5L?Ho(l$c6pGdX1$2E4QKMxm!1T_FS(29{Wu$KAo^P|KW5PP@K<%z{5x+ zK?b^Eqrs8`^AXPu|M`GVx7Y)bqFA_opWqw7gyr)>N-Tafh3C$fc=7T}L2va)hwaF8 z>j=98PMRVcxs)!g2P!b{bhO`=EN4P_3v5^He$j|S1nXoj>C->qA@Q#EOT~|u#jBLz z0MGAThwbZF?f$tqs8@%MW+7Gf8T!i#G&Zq^>FfYqhU=}j-P6K&uj8#c0)2YR__*;o zarUqTp@{z20Ei(Q^edt8liib(?O3q=2?Y8zret$}nKcV%&u!kwPE&+T7q9%f!l4(r zPLJjZkK}S)`-*VPohY8aDSr-TC2u6a^0Pgr7hFoj52&;n#G_WNZ-Iq&J&|QLo(tuIon!!7Ix%96$2)*tP^(k5_R=- z%iJpfp+t+f-c3ID=Swrij-M?_pHN=?5mct6-XtwV7|)qXjT?_k2V-fe)+YRu~E zbeHwW&VHnvTrBocD7rG|h;XzSKiKwXvDG_LZSCjm5PB>8xrPPE;NVXKFNbrMr2+^Z zB#gLGqM3tbcJo+j=SA)VY{$%?yx_Gxlg%-L)pOB28p%}MA}2JtWNawh7Uud#wd-1w zjyP)9_unm4TQ`-(OWBbNkRhqsDnil{WVH>XB-E)j(%0#A5e>hYVsetc3`n~oQnF*d zynM5$gKFC7uO8yBk~Caba!~K^m6o>>iRpc#yQNq=t|IX9Tt>Q-=s8*r)~a zrRmxSVV2Bcl-^2j?ZlkazXbHEEAwk)74}q)lFd zZ8t}F{C)TwTh*->kj?@GX2ock&$KGJOlpx`;C&;@U%;=|qC)Kz3rvVQgNbooEnG%D z1q2csod*w6Py;JmKo=!9_?AE~%T=Xbh_Xd`7Hd+FF`Bz{@YqxDcYnZ{=Hp#m=SCe* zmnevRzB5+^M1dwm#T$ZwV3UYS5imImV&aRdp@$|=`qim1o2P>ro`HC*a$zKpO39O? z2%PT;iXWaSbqx%0F4tUsk}6+l4|#@)7}0^WNCB%d2s3N|y*+MuLsBt4t<#_kO2)$< zeAd(>*BBZ5WYZnKD`v{OlZPWPrdqcdt~#E1)huzS{p3+~c`q6hw*%4+eU&Q-vIfLT zyCfH*oa%1E^LhEIX>AW?eYR`dl28ehYC>&7?7^y9?~LW;H_&+<6(1b#^~mNYS*hSE zWf@mCQujMMNyp`sK<-d`1O{m>2W~RXQhyry$PtMCAq%FDRaymh&WcX1(;^Y4J7-tr zUVWX=B}(cZ%6yKdH)(G;0xh1LTH1RD(h>FYtRe+SA71^PmV2B*)(KBJyo5p6zs(<; z$~xVAlIt&2_yL?dG@cTSKKiegm6?6{Thr=R_DX$YUm;+(r-~zNrRsP*@}3ArBvt|x z_`e^me{tH#vDhPRWt88@+mt{aJK1NE-Aa;`i*zqMm^fYHyHjSNcLP8}2lFXjP#z7j zzB;*U6blth8BZspNj_f5V~1>x+r7l|DvaiuiV&%;x(0U`{`Gw@suG`&eZP<4?@;_7 z--nf*k;OmXhq7MuIwO+rbDd@q=3Bkrp|k;9n0|6vc$!RmsD1&RaKmz31s%53G_vrg zk4EGDtnS==bo1P_wT1al#-u6n)Zy{KJP60kS4`XJ(h;E`u zi+zIGHfFH`reCI%vYF}SC;(0Z35O!3VV&6A<5V}2RA3Ud-*;WQV>s5d?Mli+l#1xh z7Ausp9aJTO{9lbnx?T@M}t z)eh;;kM6g{3Du?EQJ0WL1l}ih*Pw?;Qh~8k&C+f#3HGiXVGJ6EDL|96c~upe*3JeJ zVelTVXcd6jm4Q+S0`2H3ZerJOcRhPAUMFGtLLU)OSoyf}Vk*^}q@^3#`KSQXTgN_1 zQIDEqJhG|2L#nQlI<|n~;0UlxfZ5b&KDVuZ9}LlZF|-kIDGA0;!HLTxni1roYm9@H zDuTa@S(xtob?(aDn*ELv(sTw3UHpZUT3`(?6;1g>`QMBk4kRiX)xR;8{B8aH ze>^7t-!b@~7&9=jHF9yWGPg1@`bX~n4#}x1>(;nTNPgROSQqJT4IE@846tFP?lN*R zcqsE|bA$(*S~{vU&{FPUoE(}_pfne=ciS~^Syo}*PivU~Xgwtr3# z?9T!~!jsRd#vX@Vc{^wu={Iy^{SkViVNwVC+rbSPad3%oFU8`s>yW&~{GK-HVliVeaxSu_MOApr$f6eKp4GF4YP^4M({e zNjz=CmkRPiaL=$*=2i`1em#X#(4r(S(mEQG?IIW__^)(V;ceo3G<}fS`2FH5Hzf}w z)vmEm?jB!Z8l(%{6OHkZk;Pzt=<@a%X~#lI)xG%?RO%)J#mnZU-gcmbi|ACLn-M;5 z@)?oQ(bB~P+d@x4geaRHP7!#qw^uqGk0jqngEEjUK$}LnV z$hT2`MIZ6qvlI%hGv2#jmHFX{akE=~AcqW#T*i6U<;~g!k3;5?KSEpdwLRf_{kdrt ztY-?3EpT_E*c`}Ghhb&)>Q~Ot{Uo!3hR|2XHLK1`;0D47m@xQEF1P^Od7+@CcUlBJ zc)ebgp??VVoJ>hCMVy$P9+ZinQQnD-9I9?>I3_4c0|x;Mw^;Mck|E^Ijhcyd1e zjgn@LDelO9W0dh5qZI#%Cs^6LnmOB=8M&L8{*x`N+pjai`8{hOjYX@hT+J9UKtNQ> zuoG&|qRF`fHwr4+t|6Ptb2;&Ap}=oHaRcGkRwQAoE9nmGnDi6gCs1mYV=Le}p;=`uE#}4NB-;kI^!&EGtWjwx>`^jkb$E= z2(IDC`2Yf=64lxD+uL7Wq5}2reMnB~Vme1kx-*a$lzH~bZGZ0u$Y$;wd7DmiLR0Aw zPD08KRYyj*Mj-pXJ%S`n*2jB1IZ?~Hg3}R3tsU_4_=C9LcBZCQ)Re)|17S-?3cudXO>0c982^$KXD()3F6I#o1DphhLu@HXj>N9;)x^`a9F zZNg$S`WOq{--w*itRy2VuB`7P!ypx{nEUM*+!2Fi<2|^-1^AL0Kvwa65~hE15E-s_ zpA(~iC)K%2V$YHZ^A)PbxhyA^?=H?mi4>i;M(pYSm|j4)?!~-Z*Z)|zy8gG!43re@ z8UKyQKi`O?`A>+nbF+1|GO={9GBGo7clw9JOqi;k!aufi|JJoz8_ZF`M_4f=lE=1W zsmCfopvxMMGT+MC&^55EQ>jLL_1I9@6{PH@1NL+Geg1aH6NvxT$L2ENjJOBJ!~fwY zx%1%6oQpST>Fv+XCy*;wT4uHIGrH-vZZ~pLNefRNjG6*Al04)u=Kk*6Zuvp%FePp9DWY#!0dTdm1#7ZccY`a)=A2G%C?I`)ZP-qH54ojYoJd$@f( zU7JC%4LfqDBN;Y9N7gQ%smG)X)KP^B3NTi%??OOa+Lp|RcgIdXP8C^;OUiY!J=2*nkb=!IE z<(~OkXg?QFf?ZV_U;Vf~pBb@sw&n2GQMXWhgU&zN`sBlY;)D{aKS{s%$J*G39RMEP zf2*>l$wq87SsYMITz7j;)KP-Khh@m{O&W$9CHp+=mY!@tuD3o4eloY=w-E3kQf%P? zlpcP(*92MLK+O%XGXs0i3DM80OE_!()J?Dj@x2c>UpdPCQ18Zn%6#gfpPgaIuLy~x z*~Id^(GvJkY|T|Ya~mRG>pEPO^ve=ag#b?7TMGKrTXPRZ2BmPWkgD69pJXIhDyAFk zOF+tVAf?;0>%6|wZ*HE2BgZSr z(YpLMC+_p#A}u>|dnEezm@W?V_rm#q!nK2m^EZ}j)Rn%mjO72fjx#kKqr}K$pUik( zv(w5lg4oamrHYuNqZuO-f7QOg`29KOlHIDH59wTHZ)Mx#+0P=c^Sc>8g-)OV*F2AK z(X+Y}G>#*fVy8Je?U zmD#|isRGFZ*55z*Q60~fg;5$`F{!AxaBrX`Ej|@O2xulAg2j7~_iJtqaNPqsS+;I@ zUr7TcN{1m`Bs2prMlFp%LQjZxrfp~ABo^FB&|%6}Kx!{9WoKe1hoR;{b_5PLe+Wf3 z>BJmpL!#$KI9M9)$)4@d>5(?S84;uoGtxC|C3Pljo1FfWKZRpST1MQ&V@2wq7&3`( zsXF=~U8f|dFKNE^hp6kTsVx9?fH(9UiJZXC9wCu4seLOVy{ZCX7Zgt>sA@K+Q{m^~ z0JvZ0H3{dC_9XZpY1Ov3W#gY$p}I*=yN)V95Nkxzny)V%_S)JwdO{Jz!z( z9FFwE*{iZu7$lwA6E1fcT3?YP)2fwDDh%pwv@0*rVesDrsG3b4V^NLiTdR1UuGpDd-`nM8Bf?cVjqeCQHI8(W=4v+9h0rL8 zYl_X77}O1=8(@bL7ZAO~_vB)QpIcC<&_Br#ww@|rbQeYxP^tzBU4QTY2}(Yqw%-xkre|KD>r%iyF8W0D)Q4VZ(5dQC!_Rx+rqLK>mQ27ioO z52T#e6koSJ$hZimG>$QK?)Z-Nb9`Jo9V^aCFzyG087>E3-tOZ;+dr9p#4_yM9bG*g zT^V!tbi@_W%i2ryhHDUi%O0H5irR^=3(&rqG2sN^>0}OY;!3q@;O<(mT#!nO(mL%Z zBM3XCt-@0J;!2%%sq)bnio4l5HKs8rkJ($;OE@Z5gfzb-YRPmiP29o0{bhLRI+$oS z%#X2(k6I~*X??2=4IHeTz!KAa3z{P>HV9oFATD7zcK?|%B(g92#53+on@MEV!e5VH zzc_M}6elR7_#N;`@f?Y%m7M7=oN4LKP+mZq5_&Q9KS)8mzZs2Y=QQDgbjS)}`J}8P zqntJnhv9AH5QaW9zXVI57Am`{$>7$^pBv>#GcoZ9za*>vwx@hYRTy*0>~MA}g&Xl8 z!=*40F_6fz4C)wO84e<1Ta}P;5#CQb#nKa!fx4mfv#l)3+$5@0{}pma^dBQ{0(2WI z%an`1&^(i&!AGLqGe#sxZFIJ;)9-=ucC@8PHO~VzoXyVq|1UqqUHrn$eks1>pDRJpW z8PTlP?oL9OOToE@ssFCgXPOuf$<6<{kQNr-Qr=sbTG6fC!fm_!9GddByhbzW$Dh=r z8YvG(%_me2=jsndOKZsOXC?^D;*`o@1ND5qLWJ+=uwrSqusC^K64*%g$klUJSPXD_ zwIhZU++R7*32Rbzh!M;6!ffx~_i5k~zSD7cUnzr8Cc`YZ^5tNY3awX-4Y$yEoY11h z9E710uo;d{@1m}7gUtAC8X`?sPN;?`qiZN7tV(SoO_5U-!L-da^3-1z9ZFYf>ah1C z&OwFj=At;-LtDy5YB{MPT?FzwKlxmqh-Q0B)siQU&bp(n8w6yQif^9(Id5IA5Lef~=)^XCVW$_FgxH(mFkxZr~8;jn(a&><^!6 zTPM?(3++wdHbZX2ut)9F^ik+$_d8dgM6Kh&Y88I^#{`syrF!p&zB)2XVmZbI*jZx! z{7V?!m!n^YXmBAW z55Ifc$2c_fb?BZLY%4DYi858QgO)sljFf$}iU?3*YJ@M+-tn@(C1m#m{=wygdCRn; zu(WoR9;h6nhgYn z+c8)FSnbhPp-BO+?cJG9zBv;fl8Q0z{aM;eJha({@PG{P zMY>z8{IsW}KfT>HZg^fI^OybDRtVQ)RS)L?w$Qr81y!j{M#(3!`VHK*BV>Q78tu$8 z;m30A=cele+`npRt$a9ue|SS8erwId{|PwmR;C93P1L_z8vi8f!ITpN**RyN$DIf`oWOt2{Ia z#2xP$^QI^m^)@!?ugE7}_Q>I09RqmYxO&W)U}E>Zf(PnT3Zi!IKWyLqc@LXsu)r#k zurvIohafc5WXyJcS+N9**=L}oKnnTU7Du(Q5e~w$Nk<`pSH^zKH3}*GFvpqzTg09K za|vT|JAgNpNF|)&BDV<4P!utdZ?Sj$lkAd^{fo}x{H!dD6ren3b~Wod2=f_x;=v~OT{ttCtwP$bVj8QH3jv7aGE%@-RW zWlS8O5#b|0i&)!rAq6b!w$|#g(&>fO*%8kiVY8=eH?jTU*2N!)W!Lam^ zF25X=Sk8}6b8HEY$DpX6=n~=hfn$1R;&obHuz|M%GU;V{8tB^q%PtN(Y@Nd2(>4 zaz_Nztu)dd%KH~#!5~_9#_95c!g5C;@KwVsrH; zvhLP&X_HIFWL@3S@^SR)JM%ZDUD!UjdUl|tL-w_z=I&yO;&$$*T)#1mT+NCFmZ{FQ zV%L8N zuMwERJb}_5v$~I9pdW%ZSqpBk#Ne}%PP5h(JPKBo!oHh?9fRL3)cI$#z6%Ke;TSSO zvVz8<4#P7nT2g+!c3($JRM0Af74A2ma!-qm<1W(6V{*%!Beb0)-BS5*g_ z-q$moR%vd}z-R5qt%Z|5{ytGo5^Us*A`ERDtEZU!Oj`~reLPrYGMQ7DdYj}|M0rkw z4~O&f5a0%g0Ggs^-oCWm=!EowdgcfYfgc|Zziaow z@BB<;-vSS}H$ErFN%q1{vPLrs8{BEk4i6D7$`;H%smMEA!qNFIi~jF8tFJx2Ua64J zj{4Gihia}h0BWO;$QjIas@vXb6EgN^!>IBc`QsBniwh0!BdMF@Da^T5qx`f@5o3XW zKs^=5z4q+SyrTL(9vzQeD0CYwCaVbp<9C#zO))NRVu&D*mu>M=B6BZ1F!m-LzUX)oy$f#2#-N~?$% zZe-(gyKuTg@UKC7V*u*q{u|CezRB``6#Bm5Y+z(>YVcoWXsYV=f0dyIPSys>DdCZ^ z>6Td|YjPqYbf93#+-MzRB`s+ynU~F~F<*Wb6n3C=`G;hmPTL+H+b^}Spjx$Pc~kHJ zDa5!fJbXyT-~9NwxqXJdPW(Nb9o+cw!ga9e$AftvEV=nJD*|##Sh-HB$i6*zPuowB zTve>z`lY78=4#GNp!zAw*qr-oV2t$e>A{D&%XqjINnW6ojw3Ni#G*mg;$LVoVbDzp zY*R^8$aR02{2(HzSZK#Gmuxnk4!9)8CV7wNy&J;)xXPwjB4{mOD_l9va7<+CW~-gj zt0`T4!P|#UQCE;y)DO)gB4w2}ZLfM8LqoN_a-SwJscC1o`f#^!@lPGXh?Eu86~W zXi(HP!`_;reNQ!}Xi+7CB?YSPju{pk0cHt4yCRl#%-vAuA>7s0b`vxMvi>g9++8GO zPG7F)Ve`uu+_v9y`ow+NVB@BFqxEs)*o&Gbag97{oM!{9K81QwffNV)=IZ+PtrJKc zm=-ic0ffe4q~a%P*T;%Y#mP-W(R;k6J54BUCJ*zu|FV!?H74oPkM0{g4nSS=Sn(!G zn;gqfgZtaclQ3R#e5=}rzVhT~9IXA`%5sJU@QM-Aq6uMONXRy8fcftN*9zWTiO-_} zf34#!Zx_{W$Q_*4G~k(HS9IxfTS+FngpvmwgVBI*o^J z=>dm?1;c2UaDyLDu~kr4HLknKYqR?G!5D~dDBrZ0`;wBxJF1muLlN2BW_`WdRju&O zXSej?a9`Tz>)$R3-FmiYtbNNiRZ#yIi?jc3;uu@mTbX(?D2oY+%8Ak2nWn0*+v9Q| z`5kM(?fsOlrN1qyq!C1nTO@>`#VnsM6vMetp7=gLMc+Zx(W(j{Qg{h0G{|5$O_0&E zoe;MXWOsKznw-qcBq8XdsrZfB5X$7SQ5f=VS?PAe<_W3O*wisXNLUpnk z$oDoVVqb87Ag8{=f=?FjnI9I(T;-=ejV27VB|H^tmfG!La30KQE>FSO(}`LlG4<-G ztpo|X^jm-j>tN~q#Z)ZZw@KgrTb2%{V#(H~OxV@^d3uJ@j^Y^I)PhQFLAv(C1QPNl z`q0JHwvh5gq!)nj1MKw1gqCvq&1)a8_3nT|voFaBUoD?N=$ z2R{$5Cwk!X@6{y>!aPUD4y>{8>UU^v5b*{XZT{qZ2qjDDGhuWuAsUv1y47|Hq0;<` zPAMu>(d+;DN5YjVSd(^U-Y_8=ofZ&l0b2s>S0zXtG*SBEl1x`TIJp?knIUab4t9+b zIpKB0Iu&E>VTrbr1UDS@`a1+xPhA^1-Lb78Ga_$(-`nYGMXzmr)0gtZf*K0Gqd2)u zy{gblJ|w7+($dbVWC89mDZ}tYjYb<8@s@i@lnApF1IuwQYQzJ$F31O942pZ)HH>dE zeLi64O&($C%`W$_nn$-F)08h{6|%+G=Q-cfc7gWW#6}9-YNQGL>4PvCsY|CkFQb?* ze`hQ22V+>B!6&(+KzQJ)tu{?FZ^ z%$FDN%xh5GT!CGa!K~&UN&Av=M?clZmsM(u8>0xPb<-_RQX&K!XWnc}qVfeNLF`!* z0$x1n2S4^$kr1T3#N9zV=yvS4mkVf!($$x*zN*x9|ZHCTD&{9m16W}SXuF%%@!a@UbCmR|aT#~l_ckXARavs)o8 z^i0OkIg4bQuEHjX*lXfGTZx3M7EoqmG_WjkC|IJUkXA{Pvx!kDQgek3ZSy-eM1Cvd zKHhIcVd5`=>?D4w-ma?^1Ord3*0^bF8Z+g~=lRa7!Bt!^BM69fH}0OElG>emt(2J` zlyx6RuW!(7lzK@3w>O5*_~w%UuuI!k2ItDKNwTOi>P-)T&_>4%^Oq!jZjI?%&GI(i zhmWTcf#RYWfE|H0x@GhtY`X|VXQ&*K7-pjB@p76-aA)o-+UOaA@AUoO`W4TzzaxHs zS3Nl2rSpI6SNyL4VeqX%S-Bb*d{@wZaYEmh)v&^^-_QxXsT84$WM+i$`w3a8i6gLN z1nd|&GHK4XDHfHy8UF72d3yPc%04$Xc5r2+s6<%Y5(mDIQpyn0xUU#nz;`Wo(VFRh~F|0j&!~ z<>OAUv$${!>`PF!R_8ly(~h3QeuLqMAJ*PT#lo`C_O#i6>{9m8ozkdG5l5UHpR{T0U()VR{!fZ?*8(696r~W_~ z+b(!(!+1juOBk$hL79}AYFaAN7JZJ}Zzl$pqi9G&8rtRr|lQ zq`8H824C;zaGRekzqR!W4>v#FI^QM`t2);WA7J>pdS89~bB2ozO?#vz`+uEFbc_Quuud)77 zAT#U7U_l|v7$o5EO-2sh*2&NgZlq}eApFUHWlgq01V~*-F-(cir#G95rSYEkMxbOP zSvS{gnGaKY^`s{=j7D1JuwH3K^@H-Oz9fjittZ+(Vl4V!u8c-e|3c?A_K{7FI7IS& zx@695CK{LV&>4?>29(pH)hhSAtuR)m+KfF-)w({y*wl2iN>&F5W*s2vD)(`(l;@3XsAD?ZcowMZ_5_D2SuD=beVkdWSETO}V6&(-S znK{s}CwbSNmMbjpgG3SgwjM=0&R{B@ISc{d^(`Y*vrp?ohvm>8c=G7_&{Mk^r` zV-n1Yh6zjzu3QbrZbJ7`H<&JpYz>3|iBw2fc?R5;Hwq1BC_a%lhmAxc zF{i6OuScIJK)i#@_>Ei6DDMz8A*nLv*>?6JoNb{H{^BFn1};oo*jcZTS#tD8_tCM8^!-eVw|3XOg*YE<8Auz^UO_S2*d|L}QXM z&S7nW`h(hrO;ni1K^#~au`$1e04dCj!W`NirBCouI=gx}L7%WeFyKS$2hj~UqZG99 zgf=5eupYF~gq$B#XAPpx4)we_ekwY02mKfph~eoaU@yGiBP6d$#19FQfNpI8g=k`? z#kYtEI>-n;ayW=KvqMPm5AEOCG)rmur0FYKaS9+GVc=qwhg$kQa#m}-Auk_cXuRba z5-u(!XCWA}JV67L?^ER(bd>vIW9TzV-1_66H)6E%U>^@O+zT<$#3|p#^!FqBTrgnX zL{_HQzasjZJEsE;H)GiOgCf~yG8FRqL}(SgO!Ar`3;SkR)>|83>2}Vup0Ixkf=!CZ z(1noUgvQ<#nWXiHHjc#LG}5}2iRp^TA`1>@=Ef9c$CUZcDz%Rm{0;kr=7^Zk&Sp(L zO(wTyuO_uH)m*V@3+hLEV2Gq*{lW*XrYo`@Pg}mTJEv73U7iHXO4C&@m&}2{5Kd7f zAAT+KyAEu2)MLZ5x#SGc2rr`Q%+Y2K%;M1=5s`|rgKXOuvk#&sTy^_jr z;^X7>wd)SqohC+sFJ)wR_5InbxexcrWbIh#k3)w<8;T1sD;6ZIq+I z>SYPxHA(6JSP>}%T&kkL<{rQ5K3yW@uB8QKTDl`~-oBgSPPcm{;jAlrACb}23r`N~ z6K(U`wcxL6d*GJZg~pw`8mNSbonH#$Tt3vvx4ABnOAiI&AQRQV^bi+K`&?r$l_JI~ zR#Bw!NRlc*$w_aqXw#f8<31a5%^c%PxW43cmus#|M*Wd;Qp$?dn2`I!u*Y}-s)eU6 z;!_;Pa$`{*c=k99sq12NoAkFVQZ;&iA+k;Z*n$?i9~1Xz)f~Y0Xk1RY@{R}@5Ao%f ze9lgrvzctAvhrZHEF@(R>4FU3Avp!zi2C6AJ`7tAbEV-03*F1g1mMQ!sI$``9d?`c zA=rE+*I}?uhtAG5#C&a-+gMKZFjXccrAIZ+EpDZ_6>S~9Eg~E~qtolAZTXSb1xb+F zcJ+qvJA=!6vkZ}e;ny*&tJAILkn5k)Eojf`$s&`JzmmuRiAk`lzt;YnGu+QDIC}!f zwM*gZy8bWFh=S+gtOz`w=Sx2s`Yyk8=iS}ZS zc6m!4Zc@JFMlvlz?HLRdoUYZ>pk2O-&n@^>*}+X@el6o`g;`6Q1}- z>q>dliB-4GIK6ebc_lWi2r#-BszBI4eVgtphimY@wi8Y%o~9h-?t9jKia_`J7w`1}6yzQxFSh$E zrrZB?M1R>ti(ZoDRnMV$N#G*f1dq=%H~m1A%JnYoHu>QyI0Djne#756fq1jcisH*c^S%CJJp; zF(qwmY6wplrmU}pG*TQ)__Yaz_gb7)2! z#;4zkEbl;gme3VhPY91(xJj5Z0PUgvX#;4-3qJ5z1PEPTzh9S2}d(KAUz+AE}8Wlt1>np91toYEU_3ZCc^ZZcD4ygpehiOE6D!!qtW{%wn>&vqHGoij`46>K)4lW0^qFhcZ zi`pjo!Tb{w0zp*BOR+OLfkDF%_Kq{_NpMG%C>yRTnHA6zk4#7c!>TVcr5Ha%lJepp zrL6$kxv~6%fcy5>@e*0O4u@)5`vJI=J;)bgz+=(cixC$6Q<@1d>lsV4L)T|)RU;!h zgl1)KXI&4kFLT0zA})I*56Z+kd)mjQ=7p7(pCZ0`rkD+Yi7uK$(XHCyXRtea47?c* zvV%si)U9X=My)|>^t4RQdY(g3uZv*gic{2Zbg7qqQ)@^`1p-2%sB~jZh6QMOxG`Ee zyy*$rhs{?X69Y96%j3X6op9YMmN%*^@%dnzvEs!$8zI5^oWsn-Om%;f(_hJ{$~=q1 z`c@EGf&2*IIu1!jbbD|q1+y7gPG`d$H|}_%^9#nWh$;Vf9&v--i=15pN;=zydc6=n zsCC!54P$Rf87-%8YN>4u&_Qi-9FGn^%FAUe^ibGd-=3$iy^m{D8pIoD%l zBT+nvYACGrXC#!uWxQSkkS#)7L2%(^;ZHc*Yv8onM7vx&_HE|5Bgb_!;AU-kj&=5F z8d%MVM!lG#3Z>Yneh8Q)g(5Y-7jsSAZVA9(gD=I>QoWwd;3O!U%sl>bj>_4Vnnlh6 zjCZH*CXKFpe7%N%pR5HJOUGhew+*oh?5man54(t;5^8g*DvRJIU-oN=Tv_>kMH58e z0%46f)7)xdd8hri?Tz2-={49068s1YAO-@}J_si0RGeR{!8HdSl^ct3rcs?6O~XIhwFzcD>hVo>WxU{z?h(8^!4 zpUwr8%=6aL19hjG9E_5ybVXig5kxWksaz9|9BjnR&nMD`gC(CiX`qZ|=#pReWS6uQ zmDfsX*5}MGYUk?}7Qx!mRE zlRp&aom<#9bg-+q{bJI69PB^AHp9%v!p~YHbyy&{<;LOdA9J?KFn?XYwI)I5I&5ZD zSYu{$X)Hl38~8XOm9%DCcFwi?Td3oEJRf~@q+e-&1RTh3w%q&ZS9Cr(UQ2K4=E?CC z+H3A!;f9=_Xl$F~&RjiR?*mxGbwl%j@-nx?@_;isui|$vzwq?h!!%q_rV`a%zG5+H z7K8ln+{SzVZD0)#W<0w0yPBu}7B>EmcP0N@`LuR%u($o67vsa?`e6nb;li)|q8J%q z>_>;9cfN;dA~}_ig^E+?vQlgDttbFb(_R=ZQaf*V_lSfIQ_-q!&y152ND6zaSnkIu zWipbDgZwj zjPxzO+Rn3sT17?KYan6)(Zb%RGa;3@GJ4Puo1_Fr_q*@W%goPd`un-gv}RQw4pu`M zNXj-uhQQ086aX{G?a3Vl%d*WC+<&D!59@4)JKy3j`M1*?+kX-_{_8WC{9~WW;Gh5h zhrm(uzpdv8vww~U8QXf#@PjD^v7YXECXz>+&SEga1*K8yeBXHzqt^Fm`08fqk}Rf| zk6J=(Ae)YLc`+Z&PQ>jIP=7I}kxG~xeZKDbA|ie6rL)IeJ~-OF-5=b4EE^vD@O2i? z<=^^ra!RN8J}zXO*r4Bs#bj79(k$H<8F|WgF)i;-wO!y^f0-Zhi`8VOxVYP5?_zgUIM zqM7&iF$g>#H?*HSy?^hTr8$gXQUBg%sdtii-W>W&y^1wUhE}JBinAS?=8)a0@DRdF zcm1>H$(|{j6bw_KqFxeh0AEIJc6#t4echJq7>A0*J6XGM@5;T1FmD{uO1O_MMJ-N7 z3`6!i|Kcfyf}fXfV>=P3cT`WGnTZ4&C_FrmCKe$GZgqe z0VFgrG(G941ZYzl)F8k-G8=a`=HwyFY^O4#x+qZus5jvkGo#l4dZPY9BpHgKYJtA==K}-cS=z*P7A%RC>b!`B;mWzJ526x?_Xvth^ zo@NxG8TJVXJN6&$NdR*7=)lpK(n1ffL%^%|`ZGmJo2apl1@=X9FA&xGy(vxYFaHLx ze69X120asMOZsX?K(a^+JmbcFQVTE7>u|fukRoe?VZc(g?23)AFnX)VA-Zqg58Gs? zQlvhk#wo`RFeI(8aVeU-)rC+UOnW;+HO0a=&?7o?-7-$u*Zt$)YZ2=DxPmbRiMl2F zG&WaCtp&QKH;N*Vkl9pmL6`H}?VlfeGn-#iTpAv4+9lLWiIE3da$yk)Rjrqdbrbmo^oN_O7 z_E{C=0~dD40<&=sECThwfEXlw8No6?A=K883VU|T9q^T(L5dJoqBhwZU>n|W#8626 z)eV8Rz#!66)1!wI0sM z(y=fq)_59gy=LIVB>1|@(O4BWR2#h6JjnO>I)SGIZ4KZ13Jap11~56STO9~9dz+`Y z>4PlqvdD7 z=|q}9F@0_r)osLm&l4dZ4noLyMj&?tU3V*(e~HLw-4vdM9vnO2&5_|oX+(e5tI0?* zn;_g$`VdNb6RO}OJ&}l3wd&wTdMJB*9VeXua8zp%yc=cb)D+37OpglKDm|afN$qpQ zN>y!5j?N2IZ(W}!G3p#lbI>~$t85p}N6bd5wgC*bQQ>QsAvGBML- zdhRRXvqng>_m81yAHxtxNh`v}^=|hRrwQJ%6+KEYmKc!iQZ@x{cuyuS4vLf7t*P!S~6tG)dJFx+ak4YGV4|mIn}wkUS2*X za$bP6%kzfAE=o=~n>bO`X4MVkj}KoN{hpoZ2Ys{Ip(n*!QCcRT*yM4cMp2R*^inzc+}4mR`dyhfc)fWx68_AZ~HyJ{F*+#>_s!w`Nx*wXQ%%| zdPphh#?d9`^n>w-<6=jf3ONs7+wyKsi(oXCQphb+<&K8!qL)X`{@U~?BwiU6)3V>x zDHlV(mgtgCwm$zZCMjoKG@1@A96!(xneMihR*y~x z>epz-KA`#Ub-NgaTowwT4!8>i`ZfTfDmVHzVNU}rm$>k$8u(|j0IMhm7L1-kv8kK+5@C*7OW>*H5Z1$x6cAI0`AC*-QIHY4((uLCL&J>ES z8yP3tUgAW5rRh|IVTikHd!d=Y&(8S9j@_=d#z9N`TsUq^Kc3FjzOYUuW73er+`m=B z>h{9CCyehC)sfU$ws%z!+3rNCh766o)~jo`yym${oMXNefYs(1$`a6HgE@0+e-Skm zVF*~5D#1V(feY)=J$%7Tnv3wt`HC-yuKzMb?m9ud-9t;ulkpE+J>%IOnr#uw37iFK zsV@8huxZkNv*JDPC|<|;WP`}f(-CV$@j~J!nGt z|E5e<|71(w>d+#Njl+KXZD#=ZUtsNuDnM5!B@)=<)68tXNtg*ZQn!}z26&9HJJW$ak=ZgemC!G7-_hd*^= zSz~2YV`a-6`(uBEu48l!x<6jtK^3%!R$n+x!uDLIhsa(&UoT%K?m4(>NY5Vu)_O2v z#VBf(-Y!bD5o@(x4TMi)&t3V(Jh`L?bP8(1{)XVFs;R_8$t`xW3b@vC&++@H@^>q?G(|}P?U;(Cm``P+tV~ z#v8!K5DK>&|Jr!ZMk)?F^6?S&U4rUOY?xOhHm}!jHcm+`922VNgOuyVj)%R&q%9g5 z1y70E`tgVqRbf&MA6O0%R;{w|Bz*i{`pxGI9DuCz_xAE;x*07^N`uv6fsK71BMJ<> zk}mDl%MIHxG~RNx!=>@>Vka{C2K!oQln982f<7UWFIXQlcC_nM4J**>Rd;6`5FAWQ zq}xAWYwn50I1eQ65j>%x=1@$k=Im@N3&n-5q)DPscoU#5Dw7bS4QWb5wWR|-vlgs=$Jxya#mhBw%)uGr z{h22xUZ;7J%}geb2wB>f#8{fi(`P%`8ec1Gp&^9h908|CPMHW=RgJ&j`?5N;t=<_G zL1l0ejtTl)z|CM->l6vECN{m`V5cmZ3bgGopY5W%K1Hd?9qi<9TI8LT^?BXMZoPZ(3}z?i^C2^&pPFJ&Z&3V zLbMD9e)WxX5DRb<;g}Ta@~&SvxuRE~lHd9iuAnGU+ak~5+7-Fkoxqfr&58F@s?*Vg z2HsmE>i66n)}}J+Ud^;5Nc9wA#utx_BECPr=B1tu14G<^B`j05Dg_K_^%v}$hs1CM z&mCzsJsA9hj0-}?ORCpn!NVcmlt1JdAe+KQ*BuMs0ZNCS0hR~I$4f9+)IJzo#o!(R zE;=ih@}>np`2HBD8EMbqiLED)`}o5`HPUma6I>S6xNF1XfG|QBgUr!z%#w?$Sa<;U z>f;((!%<+unQG!Mm>@`%P9_Q~oeM89WEO=tT=&v zg!fmaqx-43VbI!AU{VI~ zUrvvf=C$(^-RoC$!v*qHKxJkdb?y5|?Yh0^Bk84{mzj8;a6u#jRPxp@A6+B*h~w8+ zV?hF=HN{IQ&g5m;RL5Iu;TfM5Ggq0aRRkp z;{-RIgWpX4;=!Y`u`s6DfRf>5a3OKMXU@&yftcL(!AwUaT{6;c3Au-(XWCs(n6Okfw9=ZE;$px_s^!`9c{Wgtb(BzHw5_|#LwEk&2r6=RF$zb zsK7AcQ+f>;%y67|9qjG9@=;s{WH><3grPT01Y%%o3_o~`03b-tg|>Q8tmVM(FKY&= z@8WX!?V5+2EnZ*iZ{b4saHe_lZk-n03{LTV-6&_R%(>mGvk8@3)TYnQ(V%f7rao@K zq9>8&G`rTYOuYrnKo-zDjh%sDWY6xv7_iPrmkyy=A;; zjO-Xmgn+L*D|oh5v=y-CA!G4(U6F3GdwCnBu9#6ZBS+{|8#IJ2ORsG8L8qj`=J0H< zAD4*7?U#0aWyXOHSftCo0N5X&CjNefUVyDbU#=i9Qp!CY+-x88O^-Fj7))-l+rh^! z=B!o{u3RoF^;Qnu-gH8nPrk3?yP+|VWQ5nHbM^6)$B0uqzjzij+Jx9DI&ZFO$-Vb# z-NDsnj%|#Q%fr)lgcM4tvi(uUXXHR*X&`vdd<^_Y1buaa!eT_-90fNY#hwRjn($dV zz(aj5QtpQOxFrm^|3J*)ebHj!II7)UTK|zS?(UxJw`Y%T`Pu}QEo{7NH~$<%jzO57 z%VWka^M?xoc<7R&TL4+5+`PuDUr_Utw4Jl-{Oh>x7MxHUKnd)2H-psgD=ST!olHq}jAU1eK zC@9lm&!V<4>*WXb*ThZaVL}Bj{zi@aeaNlnz5_fthHiHBi!*HNEW*mte%?E8%lNqF zc<>k#6Eoa*2^4d%Ehv;{P*H>xn>YkxH%j(bAaNA#T8HsjG@WHPAf%^({A*(*vYMKGs0z;)oX^{h_l|f1(0+2>vF<8V{2^B zdxQCrYTHT+G5qBr0{mU3YgM+LEDRXINkz0oR?~C$-PYxpGpwPVAM4!^s~`td(p}yM z0&cetqIkiI<%%&=lA~^2*}dgXVC-ns&b8+X#^!XC`SrVxcI$23J3RdbEta|DH%@O2 zO}Tw@A=O{Bptd1ltUSlhq8Gpq@bOrogwWOy7vU>#hDJYLI z31=8%YUO8U8rE1HUvD3CB?#U7afV{#WtPo#9Vm_umJZJg^@0dO0le;!w-HCx*t^k{OmdUW&!RGGiO2UslS{81j=Cz) zesb?pszNMAcJ2_eeFSZ!T{$s2i*W7t3wNVUhXejF`xsOvSi)ei79ZBkR)|7834zj9 z4}e-Gt&RmAc9s8Xy_$#19vHQD1!wh7pz8=5R^&x1E<$Qmaa=zc($b`b$j?D%SFLdQ z+kr87Md7rshbG3MAzp@(Ac@4x8-oLovNooLN&>lzF6rvGz@r#%%Wp{E+Qo%H$%L1^ zK+|Cb*?`{Wh?p%QLlO9U6ovr~DnpTg9KqMeyR_Q{LToU#B(0tXGH)C-6=CLkKeLyB`?%vS({J_vkPaHeUh+f%UC z@g68E`s-1L)v;JB_2DlnE!u_rOC-iuqBgrS;WXPCD(vsi=y06qZx4$*K6Pb*zd7F1 z(2B6}b>iqkH$DWu?AB>sU<4nUmO&>%sd1iiJ+G*ql{TK{(7@K zQfv^oa!W3G=oR|k|4<#5(D;<+)XaE8AAoFeFLRo$rK8uZD~)v`(e0y6f>Q|A54)gW z>sFi%IL!QKMZx6L0}u0gicYmO?^$3$mU@B04YsYY|7@E=Xzx|jW>jD)>_FMT?*j)g z8r5S(krCf;EaA1jaPUcs!2aWK1E5LL^BqM%N@ax8t4NTkD<({W*05*(Eo3w_88$Em^D!+;pa;;E%*st{_~*UH zhDkl^!=$lWBCF4-L(O&ru!>R3AVh=IwOdZ^`=-E?;J^&c7v>$M-95}^TI@7Z7ZywF zDSL;zl*=(M4i^BoK!QRtV|gBj5uDNsCHl;LGE#6aN}!BVw|#YZpgy~vtpV@Qy4-0N zV*JlX2eb%uJb+_mN0AV_C~K(gxJNshFdgwqhqux^DSM~>#8Yo&AW~4iqIUZ zgq3R2D3LogdVBc<-)Ao=$f$!c={6+@#=x_d5>-armAgeu5gfEAw$HfwY;C4X5f9|w z3)g|YDG)xH%~l6SrU>Ocm@s<=7XnMJ&-~yc+%$5B9S^gMV~rg&SS8c#6}41KtXMm0 zNn_;;O++wY?+nM~DQ!_)(i6rGS7uT_H6e=G!dGjY`Ba>WolgF4bSu?JTzuoTaeRa6 zr{5kg1WSn(bDt9>arVCv#Y<5xwA~CEy1l%HU=&C%UHAzblI>7M`ut&M zDR&#{OP~h=jd4WvUNlFnZu%ny>>$`eW^x)k9<&7Lu8@xAMvc$F&jZ3d?I4w5MW`SB za`pa}eR4~Q`w}Z+^i!@gfq(j~^V47EZCp{{_-BN1MKtA zib}hrX5l%`V@*qbP-vMM+Z5?FkuWyQY`8?|A%!O~=@^fZUKK&Ky1ftLg>;rFmfLk` zA}uW>evh#FEP}L0+jbH`Zd<&_qdpkH$FU+k2*37T`10jkxF9U7by9E{rM3Ie6gbke zRZlGr`knB>%`NP0kws{hsZ}qy?aFU$trPaBo2eDAyy6EhN&VMI9d@lk6zd;pgY=Ih z#ebtb{;!#qz2i?}vg7}7L@QSPU*&N{&p}n!coCxm!HYRA-=YDYAdo_G%T*K^tI4@x zO|eL5-i3eg+v{3#LkY!Wi$_3%#HF|Ay0dU*cb*NG7E=Zh!0eyOsQbkh%y~_zo(S94 z)!y0N-kBzEyL~W*-PPUwX5>r{yNDo>h|-*z{EIhq#jaLdFoq#ngIg8_z&Y%xU|JvN z8LzUuI1AOQ;)w8W*V+_K_)$V!IOvfAR1MpW9hyoDczJokwQL%|#cf@X3%VZiy_c6M z{4+)rj%=k3WfwiYX3{?+G3IB{C|F;e^kQ8#A;?{*$}qUU%lVq^)+-${`1$@T_2n=q znZPE2Nw>+~7bu5uR_{-vHOE51@LH5~RziQz8(nsZDAlByLGkRDx23uyhE#sA6B5d% zCXK)@0c2Ti7N#x+`&%*fpj0@z{TG0`%l^bHWG*KuT=vfJVPA4y;`owjulFYpR{z0C z=1PKs4<{btc@a|F=%bsE5VXX?)B9VwfCjiXtV0p%9YHfgg_l&svVRb&45KRZRt&HpBt22(FQwpIa-&&Sn|<5L*8_Zx zvL9R={2M0_J`vLf>veK>;NvDO)5($!z76JPkebIUL0eKmhwhVtX{JH0v;BZbS_EUl zs!9!g%UlrYcFf80u%9!0y+{i~duqfw6$Z|pKL0tsOP!2pYrJrt^jh6+G#q`rqaU6r z_w&aqvNF>+*YZ~6)$*bdk3Iw3r>IBuzLP=;xwpt0UIi^i%LdOaI^$jffK;y(%Te1s zZt2M%VZ7kYHRs+yp$xi>_!~SrOx|3ud-E6&k2{^2f-j64Xu3@`TP|7(0fjN9H^k0i z68QxOBoAC^uatCr)~%g7IZvVd{bxLw@GjEnE+;3PpNQT=^`5>6f_$&u`W3iU(XJ7R zG>7$urJpy*>^8v&qRlr$EK$B`0+inr7>IN4Wgakjr2}Scd7_f9x&GI|OxOJNJ;EjM zoa$thuQ304zbdnJEol2QmEZeO>i>UkTsIR#V*}@(KrH9~dpuvPtYa7WqrN^?XL~Dv zjvF#zlzGbA!$^^f#%eVNWWZo9hbSpkz<&FBqZbM-j)ETMIC;$+KD5M^@X3S2`;y_b ziL!Oz@}HRZ)YE%Kl|(K%*5lM(4s9;JjRG`thjCqqrTgmm!^j?~!z9=o==QhMLXA`X zOI`z6{}K@?-HZl9{fetLj-`rhk2CM84nyJ(l36kPRO3k54`pW1#w zs*FgadnubU+mmLPpQ!ZQP1K>PS8rJ$VVYHV(lF6oauHQn30<**@IY{xvFJ&%Xte-| zG>NE$3PRGRst3pe$XO@CCn~=jaROHEKx@Gr2HTn9$iCIU5sSB3kM4M&4IH4(++#nh z|JY{_=V8W8NdmzyQb(xVsJ2ih{31ZM!Qe(;Vv`z73nm^uf%ZXv>%Lomc;KPYvHD?RIqKQl*;)S_z19D_Gg;9S=0a>_2AuF*45j^PKji?U zO{jui2NouponJd5k-?D~a`}GSihmQ9ZdgEnjI>Gr=}Vq{+$8ksCJOkZ>$fa8fYsR61_m^q|%P;t)+D?83-|CHG2PVZ57;ZBT03xkQe<4usUqZ(Ne3}_q zSB)UBanv$oenARC!+U}8{XT`gLbDuk?st75108{yh^UqKGXJf-)|t$$^_=osw}-3t|E9H)9i$Ypw6L$;yp`!)SA?NnTXs0BI}&!e-h1tE_6j_X+C=O!WO+<77WDy7(aTmy{G}&c!4*nu;d*i@OioEdk6c*xfHO zMKllWy72cDYNlC2*ErUXZLXjucZia|80aBZ34vT)oL@YA!Ur|Q7XKcbeUex$# zLl|r!{f!xy5EiKM4sFQ&fYMip!)sJPU<$~r)m~`*OqA@q{qy0xAn>IM3ujxi$dg01 zE6$ZMVa)t-Qh|CDO6Zq8T@v51mV!(FB#uZ^8BGb%@9@XvlmYW-@wP5}P$9PQf8|0S?%VO3K8LLVr3{mzYv2;$5$i5P(%7uUEimVQ9TLQeg!A zo7tQUa+O&Eu&P@xMN3&{?Vx;|S_VtW@R>&DTU@}EdS%!82zxx{NjwK4WIw}t%iXTY zUA+|p;;Ih;A^oa+mM`3~O#5S)ee`#AUZ7Ynv$)^f{+|-R*P7Kvt8DMnuO80W*z6a> zCI)A`-&zN9)E2#u`RBf`o*z88dawn}HzNF6W7V4O!FPe}HBVL**qH%6Bz6muS-t8W zh7q}Ql}EqJq--56-^y#zE0`0bCg^3YfgASHUrXC0tKJ%J*S%JrJ3)vXCU%kzW~5)Ck-M*Sts98<*f>xxtR~# z#4djOD;Z?(nx52~ER7D_H|PkMspGPGVwKVJ4tO?!TWEZZM;SC?%xi$8p*_XdJFW?xRLedz(BYlg6XrJgQk_lQ(w~|My4p>g)o;i9H(bV`b1>rEZi!O2f98w=feWeY*J~{3 zUP+`}2)VudEH2WcxC{N38?bS(9+SyG$xL=!lY}$CjV&J+U4j)kW6zK~ZE}@WbLwtU z2GUK{qdYBF>KAlM$mM(O_^YPkk-$t;T6_!Tgmf#i^WRkn_QiCXc_=9zkJYiOBxjv? zq?tHIDXaoH3weW6L(5Y!`bi8~VU|~mQ#H#1)Djt{O5u<<6Oi+l@4;VtdV7TU< zh;{6dZLy_x?`YB4>B@)JWl_JZvT(WTkAHK;lX5xuQ@SRwsBh24Zzyl+n2?jABHz39 zpsTlkcC9i7PCeS<46Q2Vt-l4a)9u65u24$(9#Sj5=AqlUp~oh`QrPgEGn@I|=6mYK z@Y;A!+99(HsQx4zYo$HGZb^31OAJ)Ri@*INt4(!XZJfiu4**N4t*1S3<x*G7T8=^?kh0wQ+siI2<~}=KXg3e4Kb(yF5&#HMv&Zz1jHZG9UL6=UcX~T}<3?94 zgiu9G*QTK^pxl*PzeWFa+Vt2r;iWQo1 z@h>g(_ZMwfY@c|zUn+WWm{T4QQU+(@5?xUhCsDh0gBXDz(76npJ_&}v=*MPM4hM1*O zw$NI8@4f1!rs!SV?iheGbj}<+Fyn;4V*A;JtD5_%Ey|4xiA$GM3F=D=^pkL!bFd@gfk?>GFGij@_QBHCw^lUOAtj zEy_%Wnpo&ud-z;>KabU`=_0_g&+5t0oOD@xaOO#u4gAX}GmeA~OCpRwJ|lA0b^(z9 zqdCQ`-4!-JKQwH~UVrq!mo2Z=Qx(rJ4g~KrAAfzAMXvCT@!+!7hs@5^H_&EYXuFoh9zH@CScJe0 z6p#iRX6XQ>3aQUG z1A?ZRaxRvoCX+7*yu2SSvFF8nFhjJt?1yAa&0m+T9^tqXV&kqXZz&;O!%(b)>qC2C zOkZ@t39Gb25t0f*dVqY-SNH4CL+%5aHRqS>dx?jP&qt!CN(%&~N(c{=O;ryeQ3JZW=y~s+2KTH$r7@|;AvPbpo|;6wX@{w*bUS~`@sfjyiQEc?7>R%aF+>#>%{n{o$Mn1KMkPiC71=Il2`-rqP;r@^iYmHv*TcVcga@k zF-PKG0nM>Zwww^=`1^pxSJhxWl>J3~!8o$EQs_gCL&Tp`@lGgWcRaVZV7?(jqhp|7 zXIfD`brAA2__tF^HV5nwS~;p(*~@1(d#R+LFdDgRXE?eX=rmO6WuG)=&~@bfM7U?u zy;McdFP^R1SD)S)PfR}Tb+*_0j>Yp50oz3bQSS!#c-zm6Uv0tpkPuB=|d|h8u;ez)>bhN+W7)bTYRKdps_;JRN@Qlq|xrJK-Rb8Mv zAX<$VV}NI<=U#_f1InP`pTeKIR_+}80LXJ}IQKLq3pMtkB(Vi2oFIZ*`_P-^9Gi0` z$teJ&R(%_y3}P#qq^1+MgZ&BmQ~F~>MdWvV^T26^#713pWp9%+U7-W7Bs|&Zp|c5I zl6}Ld#b1``UqqeN2zxS+<9Ho#_V;B$E7&7&f2Y&KsjW!$M z&q%=6H?Yfh!2sE6niV3iD~5wg&gw2%!M~PLo){V;G%HfYke5*O!411fT8`#xTad{o z=De)nu&>cKudjOAu#!?+eI4Y?EMo$`c&@}oy(T@JrsJ_LU*;-n8COqnFZ#7Cj}X#N!GtA0E-3+I9re1xB$ zZ%j&EM#U?SM3m`~0bKUoi`%(DTB9vp&lHpg_lU>bQwG+TUTd@e7q-2WEuAMMf3Yfl zt5y;Z8IG`v#(pBSoZ|v~iTbdgy&DyRi&bl^qpky@UURzUH*2J_LGiqfIfoX)&|%%& zmgt{vd3VGnO1Gv=7a7cAFSC9$U(_cu=CPdLH~@k0o_yOwS*y@Hjz&yjXBirJumyKW zFh}^OP?TX1!RLacr*hkh{Q!e@;zWEq{p!+J@@PWc@RfrKBl(TW`%~quh`lAnUvHR` zXE;4plfY?!%&3ec@#!!GI8;WRuDC9D=2@#-AmMwPjalBt3>+*T0u;PmDetVz*7d(i zElxvE;P1FG69TBM9hW)yrTVmLOF)-(S|6ad*-KJka%dO?SnA76T{#umLb8|kOv*}J z#J8^b^>gq-$Hupo<-Z@g10dz{sMpEiP6qb;Ew_|X2jOnLwmZoVhX#+p29f9Z_UWMFdJ^4i%n+U+c46pBfKIjTW~f0_||?FwO?>3 zmv?V6jYmN5&mn+*G>2U-Gg;$b6>!!(h$Vh2V6|e)+|Chz6$C6Tg{~Md$q9kr$02ww$vtesg}wY%JM;7xR9#79kVtpT6lSSK9B#Dei4yX+)Y~ICafAaRpD96 zrSx?8_a2JJ5iRkwz(SQzlGBew=(YaE}jd<(|kC?>);xvBde~rSPw76<&;tR zzb36gNs9}=%t9$T_WJ7wVhjzYGZb$14d!wkTqXG`=7K2tL`x2u#etej(fw0Q#Tb@U z7g#x9x=ux1-<;!%M7W#xPyQQF$e@e}4d^+$RtBjWxb;}a+Q`6B81zaD;JlOu>%b?C z#lZAIFeW6%viWFcYGgFTkD=8h@G|hrUF}IOvdsFXKy0S-WrYy zZ$)<=9@}Wg{ zvDS5ZJGon>k@9G0nNT_h%ntBq;zF^*W3bctZp`#{17`_h2I@Z$CXO$fqg_j$!7=hi z$$wGn14zfO=f%w<+_#Le`?9}RGe5PSwC5>nkA>b9DsUS1zwhn&T+Sh-s`3K+r+?! zFNx7gKC_z7d9nJ?(KBK)u4H3n79aGTv#&z>F9V1oOSpQJ?(4Os&|uvsU?Jz?1p7dm z+*Ou-{IvR`~})M)-4U+4ZyN|C?7|Ms~KQ7XKpz%Vt*2E>H*w zV)y+k0^SoyD`mI@GqEH;a16(o-rPvkS=7X4WIzPd+Hx=U`pt#ukt5w>+t^gIO;1e= zoG=%jBNgLn+lA29;YFJNY)>F^n9h+hDFm$9p6|)j6-b8I%3wiu6he(i1yOJ$$H(sZ z-dPzoIIm24NxyiaV?6&K3q6=Rh~OCkAVmzkohiAVX{waql#7=Z5ce9L9M_&sm*p{H ziHlLuo3|HYK2Iw(jTqz9*zH6YEx`c)>JOencsq;#%A=3;YIJ=Mfy2UIoPRv?;t%nK zT_qETXH4Nc#w!VmFZO{l8}0`jTk^Hs8pfI_U+xBO!x48m=6`E2IRGmxR3qI`BjbT$ zcSY4k(qa*hOGR+b|CKl>@%sSrKi_m?y*}a8kJsDU5BEX%zxk$(O-v1b4E6PlO^ob* zwAyw#2S9GEfbMpE1!8@lAsz2XrG<#4_!kG$4#Tqfg?13Lv7dV@bh9 z4`m%gE}*cMv-xAliRNtz>hw-uzKw2u-j~v9`t@hEDsM)lVhy;JdPyZer#Nk{V%A}s zEasga^2AGDeDQDdqt8`Ajd?)`9}45kb5_LF1js%bYJ7pSN5SFJ$HF5;OiH9nQx~bd z-XAf!MZY{N0J$djDy^D%QW}SOPHTAPgZ*jRBc-j42ods(3d{K8p|~2>kDwY&g?hzL zYl`vnz4qq*M2XJ@=UGNrLzypK_U!tuQ#^-emN!_9fGfTjiLLpFo2fe}LzEXaojTiU zoU0_R9t0g&m7F&(1v+Enl~qQO3JHaaCfXuv7JntDMyW5 z%lOZqQD&KWiVV#ZFA5xjt6VJ$RdHQG3E1R@p-`IR23T-^5~S&JuUbVtyrif9rJ_;g zV_IZg4eIb~wM2#%`GK89L&a3byZ5Q($#W%2G2tXnx6OT#4DLfroodUp`I9eF}=U(a7kh9z1Lh3$OpX|UKQ)m*y{9z4&SY)9=+$k=uU4aA;&!QaK z)5?HQw=)HPJ>&a^p^%fyd1(+@JN)YL$nwBT@CC&*aoMx})(FZ4yScjTd?w!X1=KDd z+dDkJE^Hm&zdvr-9CQavl`aXItk;xO@1&sni=H9zA+E!2An5;9fgq3r3eYLlT$|bW91hp915!uEzHEc% zmR6&+`2_Cc>|h;aS6i|DC!oJ?XVpz2rE|A%u9(;`nijn=_4>xq)6uD`;~Qf)W5=Ku z8`M6&kVXcLULbb9A%dhjeb#2@#otlpk+_KQ5^>0W$hH+Hn!I49k6^NXz&0(Dpkzx~ z7PT^dh#$+4!V|VLg@E8d)3eVz*fHLw)zje-&eL0(HOxp&f7nP5a3C?u z2Wul!QyQWxp6#&BA7N@%R96@2azeoBriIajTlDBcv~+;=k%7gUR*7Fa9zd`zy4dW#6{T6-GJ&RG8+@pIiGAe7LA)Qx>M(Wsz zQY2oNqT#}}Stb*Y9&biGGiP>eot6;oqt+jXD)#IGUCrg-x^A|9#_HJTS>8dMOtaIG z)On%GNgQwp%Zh2^d<$XGWeGDR9h1ai_R{D}a^M(<4^A=^n1zRVH^`)-4bd<}9@kuh z#sIA2DH12_|JNA1@UMl^F=c8qDVENvHNqk9+#5EtSNK@iBO{ZQNvnd05KG%e$vo!# zKnB@i;kn~jp_J4(u!fUQrn$a|6tDkI$}aC~L#@aSK1QPqohl0j23UKiC}MP(i9U@k z0roik85P)5xCxV^MpComS;Xe-6?w^y+vsnHi)Z|X^anoSrD9saD})wj4#u6!2!S;H zG(;t`#ptRE&wj&S>xWIw-C|x5s4Rv!zunmcg3}jSf#F+T(Z^x+h@ zEatRlnVk~z)OxO*g9zmpEqalLbd^L$>#cu-Db~&wS9yGX8KL-u&=FHYR`HIvDeO4X z(^mCA=;^a%tFFw<94$J`sy}{b%$R{v&mby(ba=<;U~|f|TdF_O!t|BlZUYYD)j$Km z*tC|ez`kKNuT`C_e22Ai7pE4~JvC+VqFtuo7UlhQME1c{4NoPiN+AD1?v%F4@K@w0 z3M0I&njw3-QTkY8_8?f}0=i`8xDW!vpyyPQWa@513tDB!C?T4u6eOaKA*rSJ)q6!8 zy131TCDsY@U1Hw-InSp|ecm1DM2#=uVpyF@JjuflR;Pz`O{5^ro2tvHXL#sK$d}}e zMRQ#218>;aEvzSihHu>xgHl`KPSK6VV>uXiE&g7d543ls!{83gQK95kVTjTGf*evc z>6Nt{{nM%iZX%z}-$Xiw8)5$jXTgy4K;rbe)9fK2 z8AdSdqWQNivVoh&DWw%RU=Om%*sud4;@_#4TYFMIi-B@6aZ|hfHy+gP#c%WLwJqO1 zi>(`wlI2vv+}{UXa?-Pkvn!N`Ivx(SC8T zFAlNS`i}g)0nh)1ElRjH*%AMPmI{6r&wmp&nb_F>gi8GH8M~f|qodvbaJeC$_@BOh z{{P;6hyLiIWdzR+WpX11Da?h{7`=}N$gQO*)Izd+|2qTBC zzpqg~O6;RtT+9ddtD%W?Hv~g$lkzC&f8o#pe_dIlUcmk*JjBQ*RFC-K{j~i^v{?U} z4{B;*Vr{HvU}R+C2c&Gc~X&`D+i-2DobXO`oq(W-It?>uF)k(m`L5F7~UN2s6#Vg zZBdCnPba%f4zAYN$nZsHOd6b^X+X@>xfn&I4D+elr8sl?j6rLOyyqp!$fjz}A#*t2 z)QWVX`mk%ha&mILx=V8jU>m3}{&&J~d9j#{HQHX@8MTvECd&Cmz|;3W@=uv{2<8XB zALUxke4-IR>>0Zf?GGD2{HQ2xBE!SRAGw&%ke!ntg|tO-9? z6wozE6|i%$6K-@jP{F;f%zRI5HPF(5Eqzp5rEPQ;3#~hv6%#d&kEb%2HZH11SW!{5 zP@yEba22Jz3NvmIHJzDX!$;gS9)ZRI0eeJm(?5mMA0pS)L*ZG^>O-H;i4P^)taq9c z6k@$0slD8edkU2(2J-BEpkJycaCL4?c4c$!rfGjoVjUm&EP2$v)v5G8SqkkqPDkdFY2^flICcMbGvD=(EnC~EdGL2+W z@OMF9CQZ>HhCl zx8Jn;Mm%4XA~6B-@A#Zo!vs8*3mSRhZ~C1C&|_bh;rgF5Hr-u9^S9+EHjpDjt>6;b z8;U8p<7K!LSlgPi@BFIZzK%Oyj;^hJg$8)P3&sL#|8r{(pwIDY{vg>l_5VII$imjx z#QlFoO<(eTI&E+?^uE^$C3`R`NmsgVog071*Y2-MSLE2d8ox4RM@dM;$4#+{(SV^# zq`vw2b;8Dh3C$)5&TnUO%TZvA4S~FOYtTIg;+I6KV(91`SK&LaKJ)H;j~SKv;o`v~ ztp2li=D@>`f*f49T84G?b$#%C=Ynx3d3ho_kRq5O$+>LgJeJt-+W1r7i{@=bo546w zXSQdAcj#T)HQ8tqXE)X0JdNS&i!Y&ELXh2G>>XE&q3Ec(>4!@0+kykf{}e;XcYn`z zzv@8d7z>Z-kwoEpQGe@*>=dg*X5UR-Pd4#ts7e%`@X_2Mc&zT5Q!Is9uk%mOiulu# zoP8B=;U~>H*L}D!JLA)TZ=3WEV0LDDYK-^ux|%WIHuFu%EoNf<1HO;yqP4y59r(es z@5>dw&=*UYkUQjUQ0f3n85l=XV8CLN9`Shn*=~KsHX9*s^;&sa#0GWJ&l~I@t1^nq2~&YSkC3!Qj4p z+5Up6rq()~Wc-+k}50R}rD2GcNCZh>u?aK;1c^~Ue2 zu)}a-t@uyP}7q7AcKwI;9KZOq0H^=g}x1cqP?#_gp1VqM5n} z(Z6Ukw$RU^|06j2`wb0=&?ICMwDXGj?5nH4)o|8cI#Xwy%y2Q!TA%GY@Tok=izO1~W!2qv!$sIm!Tta=no(e#zSY5X`7 z^+PD|BuH5n#c%rhAF@v%nk?HGC2nKB!1z>RkVX+5P;7e8ro~#Gp8S|P$Ys?6+{|te zY}d55Q`B+6i$j<+0k4Cn3xx}uG7Y0{T`Q7{5zL%YGFK>k*r4;oU37A*(6NS2x;}#) z@J-FebU?{7$$wXs|oU6njyX+o11X)xA7Buw$ zOA;n3EwJ~)Y7%4`DYaTXQ^*jxg>F=!$+U7?Z8WluP3Y~7dD{%cXe~F?0`#=}$5Ill zNvc&C^GSoZ{1qcGCswKNi}E28Scnx_DXpVQ(*{+qZEKH}3*R!wh3&GEHc854DhXsl znjpGG7oF)s>Yy>vt6}41jpH^tB=dSxc)8B3X##~J)odG`a3C4(Nwi6 zgwkT1E0<)VjgmApZA?BPAQ{`%X>rJFd4yGGYBkhJ$9WS3$*-oy#(TkP1eVX`SVtIq z>aA?p^gEzxPQEH<>I`<;_n!V=L)RG@o>lAvG9^tMC`7xl6I8SfbRy9i#VEMYrP#HJ zeT8#3aD01;-~6~;CZEqi_qW`U12dN4hDz;SYRBPaC4Hhdio2YBVm{kYTQ9p)4tk=c z3jM2;)ncBL;k$&nl1Uf77f-tkDB zc*~(vBTZdiD|KYJBw`7~uYMP4yqM6E2HX6~I=8|`lfA!cXOSSztWbD~bc)}i7zL=k zN&m-pl^CZ24AOuqZ3e|x>)%ph7)YzTWUA6s_8RdjhX(N*8FX8p27(VQ_JG$gKdtVcy-{Mh zSzlbEgd=KjGXH2mY(xkACc>V|O`XZrs)_@0acK!_kvg?$%er4jD|S}SvxLzD`XEiO zqs?F8sHc+6)c{`C?V)2k$Pvs>r{%9Pqb0%8;^oWlaI0Io4aT0^8DPC6v5zYtrGQGh zaT75turQ!^s#vijy{S*1I3hB*T=(IYIhUJXzjF06(~8y81#<^WhSDep*By+y0hRqs z6_$hA?4`lfGy6IT6Zc6{NrzQ<;fd@GDbyA-a&|@B#91|K;GYOj15NY#`(LX_nKpzj z8&oJewaVoKGY~CGjfuxP@*$lBG*d~{Qi2m%Q}b-3oK~Kp7HLnD4pblywBu~X;1(Dh=jp8ckS?I`!zjVwQ%1g7dZiPUR`asl9Nrl4zEL6w64r}?yaHSo zGFwfkrwQLVV~#Ticij=K@`oJRM%^4^!K(P--FM?3$~401=X2!m(P?K25s)(atiURG zgk8M&F>s-&9&Mz|?5k__>Bm^R2dLG_Tnljz_$2;BrEU-6m6xt-`ViRCBE@>DfTF*} zRjz-~Wu&dKB_xnB2r`FRPkEUhwnY4k_Rq1oUi@K!-8-gD__y&m7+HJt|1>~o=QOkU zL3A1SH0cY*M7ObmnlT@a+E8F*{lRDmkmW(`rV17{?HL7Rm2ggkFojmlv@D9tU+Gq$ zWVPfvrBbimEV}3}XgDynA)`Al8Zj6N-fQF_GZ+%8kc+xXaKqLpD;4@tz4+;a{L%;T z4DoT%-S7I9DBf{#R2A-2RiE*h3nY57$vhYhN>2v*X&bIUns*cC1^U5DSB~3=?e8PZ za};iyX>4qTK?@0RMR}{9@Z!hm1We`ek4b^Ul0sE`bS0G2w!nDvQ_PJM_7(r3K1K+<#?}G0y`21e{X@=mbafy^tPZJb^ z&?=H{Tk3Y<*f5%uG#Nv?!lu)`Ier|F7Vp|)jbGR%vSt>noB2n)$&&J|Wdj&NwL1mp z1yIDz%*9qz&f;iAIDRk=T1`LW&oxgLWgO&1N|N`4eC3iW*-;On_5uTV^K7iWf7=tQ zqg(X5x*)Fvlc-aAaT+H6h7 zpnLx)#^#&yGZMy1XPW=*Q`6(b?#bu$^mGkhAOCG_flxV(1!a+?h~SQ1&~H};S4>~~ z1zR6$4>|~`3U(JBgb|Ynl|MDE{`I@l$_DrukDlzoPvLyNrgdAO<>8Wf_sdw2{g8`? zW`Zh|H#j-GqmM%FyKuFAr9N||-Bu5;9q!ioc@(2hJ$vzqJ(B*GhrTE`9GB6Ki-!o{ zO4^`H-v19{=hP%hw5{2~F59+k+qP}nwr!icY}>YN8@sHo^K`qI)+Ly^8_n%np zL`oShLaW__ih5D0QF_1Az#KiNX~zDu>Rl6iin~PCS}J6l#54mVOO55(i@Sc@a^HUm zdRha=r12n(*(!#-$SxW2SyHZGE+zsZvOk0??ueV+b{4%7ZHJOz5g5`;6?X>xW~D92c@wR4fcvABhq!* zlZDV~phg@$7@deq6#XzX#?^TVub!<}bBqTIVcY&_B6HEZiDac}?#px?&n7kYX#E0t zLDxKp8Z&Cnx_xb}rupF354y9>N~><9_$R$in92#;paW_UaHY}a&=(XDH1k?pZ5@4bAO#Q|{xP%Oy*DCwz zB?BOi-k%YjHU>)11>mj@Hu$Ot0M4=&ZEX32U2a2i!>JzT<(iY-#IiguvS?oK$LaLJ zL3TN&9NZ7L$8Np(CrV|Ny}+!NpJM8!VEPTSNIQK*WNsZYh6bTV>+vcTO0#Mw8M-rV z}ct=w?N2e(%mB~^G{%<%1D-p_n*3ZK~|17X8OAMacyo`xw* zC59??wjBQ|Eb&}AY)e3Fn4Yf9X}j@NPPk0_>&k5sJ4mJ-t5z7xH(!K0{4g!E92pz{ zpT+PxQn%S>h}3p(vis;!MH?|j-4@rYdnq+->iooW{E4jUXu$nQAiR|#PsNrrlz2pE z+?OGOxSfYfY2yn$pMF*ghsj4T{`{3ydb36FcV9b*5vTVbD+gKtfF@eF2kc9cJb++2 z5~@Q`tNg;`VY>mv`ar;;P2sy2mLp-k%%SxxWTBm_L~%xf!$MXPygs!f_RNPkXb6W- z!Y;I!0{!C8^}iFgsC*(g-=g+S9}O;J zLfva|d&-2vgtExA2BpfAf1Ea!jbZ0;y9J?i_-KA+Vr}+CfQeWVRR~()=e_!|kH{fz zz$!adNP~U_9veDU5rY4tZZoAr6EKhPUH7ADP`pDpT=tP7XHC zi0@tkVs@<*hcPYhg=04M_czF?Bsjx+hu~#IE$KKQQBKxA_py3FTYO-Ks>vZ7`l_?0 zF3SGXbxGkh{5-e&`5N@RNOSS);yPY)a7aO9kEx#7=ZHIWm4T{DgCx7I`!T#;Z6md; z+$eWLgszAWr7F-9zVsV0Rh>I;XhcqShO6sJU9yxhVjj=wy1pWn7szs78;&hGM=Ubc z>S0A*m@oGhm%eo76ZvfE?!M&E+Da_WGn%WDanqrXH7B(A)Iym*-%>Hymbi{+7yy9Q-|fBsuQTX>S{QT|&IbR<7N-BhNMn8Y zWu#q#*p|c;{vz+S^|QUEZCy73x=GWFH-Rkl=TL1L6G+qw6Wc_7KBl255|WL{`gy(x zVoN-n*y$cnKH}|KLbPj;p-u?AxI8~j!+m}R*U#4>KJ0xx9DF?-eO$kNxZQ_qXl4C4 zKV@e`R3ir)g^eX?i^)f>zLwATI|tz?Y9$%a$DT;(kh4xR5aTFQ8s8<#BQ0w#^dixq zG9vp6$5y3n*b0qLrd|3^4M$GhR_~fT=o(F6?sQGN)^c;iu9Rm~hI$Y($lE5O!gK7B zrF78##ax4HKL!RP8v7ZuYcD}6qEXJ zm5HY24fC#Zgjl5=4F=6SxF6qhZZNJq!%hVeJzkX@p>NWIYVIDIYUVM5sIfB~?A%?* zrsU0(0zow-h%7A3VaM1r;xchYAf6jeXzRYnR~w!pM_OkUK>HS?L8Mzo zwrT;)Kla(%Y8bA27glbBFw!{0OXLMVpIM4}?IN)-facGr+`TLNBS#w$kAG)tpq09I ziZ%Q9m@~o^`Ywv8X^_OCaGL-DiZy3Ky5gLfT%;phoFkgv2&c`Kyh|9J-%cn-ISZfw zUmu=do(~)gp3xIVxi@xxVzEJ!`p9VS4{lVb;*e6^cOpDZ3c3oU;PY|X)mm7pIWQda zZa6hMYNG+W{XcC;k`#5e3Pd0b3ga-n{Z zBau4v1-NG}g3tbk$n3(sb;YGb4BZ&qdPECIOtD*HR=r?dRLUq_5Iw`f98udvfhT; zP~m>6i$7Jl^J8_(xBx!Ma}LYDJ?PY|WStr|08>k^WEre=KJhr3zVNws=GpBT7&zq)`*#4^Bs`)K!=Vwq;M5?=WRhZ`CtDjvVFPXTmCDV{4AHynnfUVm1imr`LB}1_6*tMFaZ%8v7UDl7^JQfgm?51(dR{62#6nrn zcJCNd21Eebk+jR`4pb)mBopVtkFP-K49i8+`_Q?3y0CLq$`GhpB_&kxpWlTOVxdBp zq`fmnXNQUP@R>Y`5+T4WWUU-^>_(aZM^sD?c5-gk+s-9T8g*L)n+N1oSYi_2kw8#H^P3SU3O)NK&) zvs#IcEHN`Shbu>Ufp6Sdos9{|Wn@K!ABQ%cn?uNE0DURL-)l5?n1Jlu1<*PzVp8yGT6pwHk19?CCC%3;j!N7N*S)}qS8yC@ZN`+JD0 z1tj_S8C$#OW|ZbB8-nshqYj=^3KHqM4#jIPB00mK=mg}JHM!xwN+temIuerMNs?AM zsJs=0>^oMZ908Wvg$a1sGFvEY9a@E5`r$myIHh+6S8h^_{ykK|f?kR9?lX2z7 zPmwyOOENQk1RLw4N^_Q9S00?1qB#F;JO@6jqnRu?y?gh7_n;3vF<}n=cBcgthesr8 zQtmdy%`OM%80CWtY!c*&85q8z5I`ZiXs)K7c9n8ltA>rl)CexqdWNoM{cUxG^|4BnwZEY( zhjZo9v3-}QZKB*Fjn?atyMu7*4K$+Aqf59S^bHC4w&C3|>m2R^_W&9gE1}9@ni=l+ ze(MbYVx|YU)Uv}#{H*0ot)DZ1jBLRo(e;fMVy+~3{EBu_K2#LuW zyLT4U&Hy!=;Lbq-Y;`nzgWq*Y*R4W$ln2~FEmTq=#h#5z(|zn^!1}3N>-aPuBZP_T zD*;zxyoe#P=HnaZJt-M93n=bbd@t1z|1c8Q>-V8HV#RzKgMv`jXcDQY^ zXnYC&byk`<>~2o4uhcF}ckq@~$9z9<$`LSDu0n%3R=dAclmD?T)T>Ywm~CN5ng#MD zaF${Y-)^MoArEAa6)JYASE=MSx;^N{-uq+OV`w*_qFXYKmHjdSZ6*wli60sSVJ)7n z5^2JbgA$N_j_PyXIXTPl5nK*ziTR15Fp)l33HdO_p$HEl$}X2A}SWFk$^^3Zjx^^p) zx=s>+SfEhr3O`raQa)B{vNvxR1e2e@pte*oH1{t&l*ekM@GKJ~XB&tCx(2w&oZ6+w ziRxg0ObkS|2ck*L6-_5B4X%oHDNqWR6(t_$p1lzv{Z06S7J3+!?I4NIbE?{iKrL~& zn6-I0p9;mkQPA;l@Je=(>z*Df{uChw6l=*6sSC{!8Iv}DW!t<(kYL|P@VLQpBR28x zvP_!6UmRk0XXP9}1rIth;d&ko{JnlnZt}{!RLd8vKcSHQL8^G+%$u?0_SjF8CVkK`b#s5 zP+PFp#7!qJ_QJEWHI1M`hz2zV#7@8vmO^gN6!_;Ou57^GW8gNet*2Id@J#|Sj7PS4 zR03cbA9Hz}t#zCMk+Zn*_d@382q$Lp|RXH4V|Q z&i3{Dm+*HAK1#!DfD(DHrIxeAq3Q{=uqvZqa9r~AZF|aL@@57;wS=sk^@m;vr^Jg< zCzI}YOJiS0YBT(wG7Z1qOQa~GMHtHk>*=*}>dVyV%MzQe0TG(@sOWBcP#r((SL`aZ z2p74FEU6Wc_&15p4AR6ldWU+I2EF{rtyRCents!qQ>l`veM(UAJ>D9L!qL!lh`QM)l z7!Y&HIw6?9h_sH{8sIuWFeJ486<*~^UUQ!TPPL9ilfN?HcQ1WZNZCxJ)@Vm@xuNb_ z!~x`o)|pBk_Fnh?a6@n{n;YD#-YGQ)k-gJGSEn)+)fk`vI@{8wGnP4LiOl91Z|zC~ zKt|K@1KW!Rv!YjJ&Ix<5&fTipRL{Ius8seDWlujnA z--f_i7kCHIU$?*;wF{WQL;xZk+^IiTIP=XC(FMjvx{Q+dEx*pBjd0~l=#Av=>T1!Z=sv!zs_*N=^h+Nu_6^HDUQE?Uvhqm10jD6P~Dh;2}vaM>bFQ&d;>f=TKI_ zhp5TxM*}mj7CrKX;r-UHkB9RIr=y4KvvEa-1Sz>{C(O|_eo0tVyc)GGP}L~icLQO( zjlN=T$(l=>u$8Oj*0H$&L}m1x^b`+Df-sv5y)F1c9*QCY#AZSG1X z3Il$wNUzHLteD8%Dfje(c6TVJgr;0W-^Z(r>&UO|Vt3d$(fG_|6VEkX!s{S+7 z{a?UAZk?n}*2LRKD*FeaF_^H7tK}Z)YQ@vFq~sRtO)IYTYDJlPa>Psn=>*Nh%zE3E zAD>PvfP0CUBlh+V<&%C?iTru&!-*h&g z&Uy4w-r`@p#RKdwCo+x>jvp)iOznQY4exI#KWy(0E{+%RYW@g*eOP7zB0N?lmA!0s zU^bBM+Iynl@i{Lie+CKc_cp6E_PN=)>psvL*!_Uyx8Gm(}(njqXY5e=L>EJI75Q}a9u^o9#O;JS)jUbdZsNxF?1TvotN&( zB*C|L^H4jUrvIU>^@6?fP+juag}OknAxh+?fRtK);_M-2oH1p3|FK}2Al2$GPWNxH z&4XU zfIU@{S@{u%V22DlKt>pOXPUbM`zV}YcNM#SQC6FHI)e@>TCLodH%&yC;qxg2IRgkF z>M&3X2u3#>pX4bKykADWwL=u6zLW`A+8;85%9iR{kuXWXE0pSaX<7r&@iKD}DYKL~ zz?&*UUxY{6w@j0K!#-4#gcBm&+NAiig+ECY8Q2g;9@f>#W=vsD$|F z%!BnZV!#P+cpZfmG_z3K89{D z=|h;CbFWWpX9Tps;Q`Aq`|(4a#%ymHqMBRtY|7A#_?NCU#@GBfe3~7QC9>nggJTaw ztiQ31HDyNeQ*`eS!otvH6uJU@RbK}#tf~-QlV*l1bAZ0C9`7XsW5GdLw+Ep6r>W_z zy2crR)XG%t%@0_9ehtE2nToLo+R}OGIWMP1vpFY>=;K>$BF|Y1GlE|sSR*tR+IrF6 zEr2H+;;-UGARLANqpfx8ii`Gr{i|8>D%bhWfkHI2FM2b2CoVGGzZ!*Cz~E7{rsBAw z>ORQa9;|WoKjp=f%vfWq69KDw_K!{qwF2OLWSXf?~ zt3pDtg`mQ$XpqXFFsy1IEQX|)lBt0jm@(v4@bG}iVJUYbLiibLDPDifWA-W3?@=xT zFIc;Y0-gZB+_9mouWX@mPDPD<&-7-Z(zMfb`l5=A@NdX=Lr+`3^q-Me9^J-b2=rSbthvk6f1vJ05ITan~XZ2;Y&qz$$}zg zVrXztI*B@B^9YIf1^=`Nq-OQ>efDG6(pZ6HRwwpa373Gu589yVerA|?$};yDl{yYl z`v8i(T5@q`L0SyvaSr9~;x6dcZ1E^g>&>14jb~8v;xb|BVgUN4t1xWoE}Uo!%`LS( z-rTaQz693h5*Z30_rwh$drNk#)Y7i%Gtm#Rnp4X>m$z=Kda^>2-s5AUa)wXf>$PhN z9QO#`TD1?rr-~USnQ-)qe(jF%Bep1#L1#c0ljLSZ56z?Bz0}1kBQu*) z(%=k8ce$_8`60373oKrA3OLLebV1J6XL3~G%xW6h%#GxGA<1^pWWm^yINOU$!7K)x zW;gT!K-&dX^Xzf)bN!1VAtjcV?)iY#r(~vzjeps_a0QXI@xB^TZyRe`QUg@<;tUZ#>THjMQ+-;$=q_)fN?c@fL;TkGwp7$23U@$KIhYy~C6t{GyW>FTYK zJbdYP@^ILHfdwCKiF;*G&ADi`eG`42?Ubu)Ec5aCwAlWV(2EDHz1aZ7?Z(OD)uig{ zbG~;_2BYxgGYD=Rh?5`y5K%nJcHR^A+LoxOFmqEfBgIPzP|;Zu3!5y(+1E*x1X(kT zF>p#30v8KXD%I)ZQ)nl+bXrZwDcgl9X9=+<;zB?qIP`|`ZaZJwN~zYSL5Mer3KHmT zo}*e>uX`gS{K%uf769}?wibb6|3Dbo02=SJ zI7zQs)uKqd+GjrbV26r5oHe3Zmz5P3ZBa$d`3pEX>yIF~y^ zIIQ8={8dExAb%iLtn?hPi=X&s)`RjcNwnzA{&vbgUYCQG{Q^ti<@i*nRH!I)v;!GV z8Zt#j?vSWr{T58>nTy^JUHAmz-~oRo3&sN&;!?FfwPsOasQutP(WFxv=$a}i1H-}4 zQsS;^%)}7Te+E7cuoRO{_B|A8#%_yUqmZP*7^aAt_bmD!tSJo`Z~F~*TDTx5cbpJv z5!3g`*GP+I`xKtdl^j{R!-E|cc_UdMyW6GiG5+pd`-@W$|4hE2O02`yrh#JRUWFc( z0(rcggi23@m=cvxx!6-50OuvYnz_SBMZkLkk-qBlT?-b!p?(G}=+qXuB$Dx5u1}_J zAG@ebY@0iFkSkFcANRAG;+gr58FJ0E@SCPbPd2ni=^iv8UeILanab#pQ%b@ci8lI0 z4^uY|!4eq40Yj36gK!h2zV7Vf14j3le@2K6|m(W{>iH zjq5MlH-+uvig>Lr%8L1tr#krTz8qx;oeQQDIYeBB1ja zpy)wfF}+4Y2U;`aEON%LDXws&x$z=nuf+_gjf z&kC(jW9dfk<`(myvE?Wh(rNErzmiS;NjTzt^0r`~o(`sVUeb8wdp8XU@NIvklrnd$ zn?XG-%Sp{6<+_?xO6Yi3qJ|0c>!o<6CqEc>EBM|?^~EqeNU&w^7~-wCM&*+{J%RL5 zrhzqbMvd_ZJ6Pgb+BlSh;Elfd*g`LxHUc<9<#ceUvomDUi6M_$o{((Bj8oNQodpEY zPKT{YSxvv0WNwaf$@q`c<+!c+{nnOe=8Q(+&7}80rHprquFzv?%M*69i#ye>1@_K7 z-lEePWd1PKpSZwPI6{b@15lmEVE_gu^3sUnbXQYunxL=vSkdoER*S0C0= zSsC>vm6U$;kp&!0R;RO3cOf`V8YR(T5_|!dR6VOcWR3Vq>m#${AZETfE02Argw~0c zCkDgjcHCiBdC#GCj*i{%tD#pbLP`#0`m^B|!Q@9%I;dMj<2Iho+CDOX)yVDE?GGW# z2Zejw$qU2oD|OdqoVCq@ZCX%TN@DFMVG6a;UjAAM!Z2NhK+pR9LU}Ds@F{onP2s}8 zC-@aH@UalQBcjW68JY?_%JAf`ZN#1Owvp=D5QkGZNsm!Hq`@MOIw6}^)H7^^-KTm` z!@p<;{A}|kJ6*`Mi?TpXEleM|NfbfgV-7Oa(D(q~yndf|B)M4c^F5+fq zF^Wh966af?(RWK2urdBfGZJq@bd2v9S-My-&0E#=uvZ0nA-V*pN{}CvGh+3e-!Nx>1 z;CGBPNe2u8JRfi_O(a^^3tZFxR%xAF8G1yU%4^qbZKy38T`II)M6@<)nq(!)R4t|a ztVOhpTX?iCQyItV)pn&s)$`ISm>30vRr}wgDTE@bFSD&Ey)6vIkDj*O^PVZa@E-W! zXhJX87+}=pfR$c~At-%2%^(2*--#D7Vy_=Atu(i*#kS^4>b(y@NOZ$2ouj3Wo5|U5 zt4$({iO6LuNcE8cV(L`FOB+G&@iY=B!hkzVJ6{Uf)yxYz=ToS#L>{Y=oL&$&JCr?tsYp^wqRrf5AnEmv&_ zEE`>qlB(@zVWL-n{hL_VoNEN8(JK{wBG-{rd3yrp9ECZGmkMGW1hIp}(>LCdQJD-d z(tCmdFrjxoK)+ijW0wlYUC{39tI&&llz}JsoV^=sY+Z54Fcr?ne9^gx#37}t4_Fri z)UU^{6vy8IM$}@=$_aSxjJLST^A;4uqkCFX5t`qMwMiqcC-CYg&+DIp_vuqFeyA^4 zA7ubdIe*^Y(L@vb<1b!tMI$?@^n29O{T~j+p~-zy*dMNU_4>w|*rJG?g2V@`qAx$G zSj|h+)@^p;MnsSma+_LTeq6|4FA*zv2W7|<>@0{-$KW>(z zPSxxfyF!kg-EU~9z9X?DHPYyfg{Wvr5nA2^<_ZWmRY|0g>zJe2I$=u3+@1BFm1EO8 zRx^e?!KRZd*l*0!&3ECH`jY#KXQ6vqvA${^^3$O*$O1 zDQyB2j(bgDFuV#51*E{n0YkDaGdKc*Vr2D)>iHRz`hn?kenUj{mzDZlndvZjqHz^$ zvWc!I2EB`Bo)}@Ce>Tq7G0D^Y0DZSE5xLS)*9p*LGtp%flB1sGe?}I;X#v z*sKuf!zpQMMU`l~Dy}7Je*&hfZ1V#ePGW;pU42ha@bXq7e58TxAVYH=Z<~IA&SYN2 zGf6U`v+Y%rHBj$-b~8)((?3N59WQc8ekzJZ5a?+2ZkLj7$4FIeOR`vy@pyPYt*Zd$ zI(GXW*6IOii=EBT!bebDJKE%r?t&+@vkMMrKvPhtN(b_Y#(kc7r4H>!w4~ou0EH&j zAv^*j611?f!qdo@&GR{F7U~F!OrBu0UK+U={Hbs~m+r{a5SR^M^pzb9i7apw@^>)$ zM&|h3HDsusj;~>Ji5a~$yX;SqB^@2&-(sKGK&!tNUye!{egIgy;fYEXtfNZSX}&nk zF$~)`PX==#tBewtrwzv)djg*5IOb7dqlFs3!>X)hcuC-M#Fz@gE6GakXoQeMW1Hrs zG%?%s%$hqad{642SdupufV%xltO zV@$w_o2&IWe}wlW@)22egPtLj#D5db1N0A-5EG!L7d{{}3YVfTv)qbi- zmx3Av%ct%&-Q4`RwljH-%>X|lYun2kD(B=RjeOH{c8!fnO!T#k-UCQWUm*ln&CC|X zY8G_vRl8hHSmVF)0vsT+eIPMD|f z8Dd>TsF~A)@y+$DNSQAP9)npb?lH~V^~}R%ThO${6j%jv6YNc(Ku|t@$`@xpc1Bhq z40L+P#c@8U;@Q_Mx8lr|i}Vj>SR?6;F)OUT;cnMH{&NQ@nhEUf1Zv!SSuoEn0h%r3 zO}Yob*w!DgtvKPfqgXC|KFomjcM_N2jgeKeAJ`8NVz<0vtkSK7XldSyQhBFAlyr>0 zd8Vtf4*1_OnQB=}P6Nv(S3vv~d zYA?i3s6Kgh5i_Offdvy`0wvp~sA_vTk}?>QE3&x|uyv^wZ_jrR^JWQauwWknu;FXe zm_8U4EXx~xT^NY9XTT$W4HNFVV;anfy#$3|M z8UP^aS9r+&zr6jmw6m}^G5&w?5!ISDc3Z57-#xv7@qEIJD}oD!Z*}a)GxdwkQ9}^c ztsH$iW!e!9%jP61bh@3P|LONgNFbA}ws(dY)RNEGzfRp{5S#7a_LFc9e$uP_{aWi_lgf4z>dwz`>Q?p>L_3G6>*@6}sKL&`Pr&VqPz z&J>OLeAYcGlq7n)BdAt+6w@tae7h^RULZWp@a};3YaxtFE}i^vyt`J@9S_rUj8~`a z3SqV(tMO1mX^$(?8QcvQV-w7JH_D8NDlHq5HdA;wA(d+F`01Rz<6dMNf3;Cq{!U#b zjP?Nb!$eAKopS~X%9%*xdjJnrCD*yrDQD=|YXwigAl=-PQdi2WLuARs_CAm5aO$;_i&-* zLOIl6Sf;{xn0-rf0^T^(CMMAH#&4K|)dP6(9;z0EKzDS=HpZrc#nPOrMylPU#VlN* zbt=J7WUr^J_tD=ww9O-CO@&GyB)7-wWAfwf;P2t%;@#uh`xWA!sEuCKn6Y1B?e==-<4fL*U?tX>qb?^d z6wpbRf;Qlr93*c7r$ooX)l&jXdhCRM(}-Bjsco6yHxoM@qL}S{OC4z+l7Q zp{QrcZuAzJ-39y+(rUOv$;T>ktRIU=k};aR3o~mfq8i*#)4Eg+-h9I`vzs037k=268dJynGnru3?Y}Pe%MU)H zHU7P-F%i1QslLXeZ&L8;Z&mKh+PU@xv@r*v(sYORK&^zac=0=6aZ6Ua{5$#baCZ?{ zG`O*0v$Q#}fIefJkY&hF$x+N`HNhw#d=T=A{@RQ)gaqnsLney&M?;A$vG9c6vkqLf zRTsN9lr$>FG3;G4yv+^+Efx$$L1h4M*V$(ar$Jx1Ma%<;2QeDM36S285%^aRFdmS= z_Be^Qn?eL_^J(5hs8_-RliLENC}k%=Sox~$33D|y-;HpAnH*_4tO z#)jgubQ9PkXcC0_l&yOm#f^#F23!Y!4dJBdzz7!n)5(XSQQ%nC2I%dyU!?hGy>Q$t zV3f%mNpJk)`!a!083vIhh@>e!f+U-gw9}cVeniPsRL$z7%6TBYO|+y7KqXLom#OXC zw9a7q@yQHe_Dg&21T1jTxc~*kfn+6r&L;u|4jG@L+&ffA*oxxJLatDYiR;PXE_e$0t7??lO;_YsaG+2=xYkt zbDlzUV@D{GK*^Dup09}G%Mht^z7x_3>hskKCaB`(FkXd{fRS|NUzm;x-M-6YlQ{-b zjuixtw8Tc^@KW}7mpZ{*bXXBGU&j?kNGXkoVL{xi0{bI$|IOS z>x6adDOD?BmEN+Dk&yvmsrLI#**V@ zP2a~~V|c`%K=`C|%Hm1acq}aU1}lN&Qf{CsS>@p#V!r-PNwu|;5zm^>af+|{t-*3o}eHKqj^e1_XGf)G>{ z(t+c}%Q6D*<5}Ivaki{C4J#@F)E#|VAEF8Y1cX#wx~7Z@++J>~WKYWxaJN;N(4UMt zC;HB~RbxUJwvGAxur`lXW%*%VDZHo(VAtt1&dNu*(M9mjL^p5b`{G!XW8zY5qs0Ps zkzESLMul1VHbf**zA&i2e2(r7#PK5`D{|@0LfIM&ui)5g2_7IYjy+U4Y=@h~b!KCFE`FB zfXqbLnEyoe;olu*e_k0x(iwWn`2}V9g|2FnV!J7-9ukXu$?6DC8Ysp#u`LhZSXND0 z=2=Jl#W}M}#ClKAOITQwQfTt5Xfin{eubb-WITiF=H3PSJaW}zt^sjOqJ~98F4`8k z;>S2Tcf;Qtvh5N2x8X&T8h4Ur6l5v31`ppjzah3f+izL?6^Z<@3{yQHyDrNq@i3@0Y(D_wNGh=_T?_r9$9UmuZD@0ak z+tIJo2~y?FlEE#26iY?U%@@E4&nn(d|FLo1EA~$i+4|xV17VSVW|1_;cxk`CVVV*x zz(zf+mrR*Mc^Tm1t85xP5kJAgA8SGL$~|6?ZnH)(WJr~uj(Hj+LjLvC=&{4U#Id?I zsHxD>D1*ielxsIVEi{91R|r70efklPcZTD-^6njTN2m>+bmW`T8V&EhhdhkCFF*J6 ze6I*zU$3D)l@~>~YWbV>^%+OzRt)ZVhT{Ump=jpE z_*-%9hH7v;3vpQ%I)f~0yNv`aIv8JOwp-TV5i}GuAViSOar=dYR&E{ue&#Y-xAXW* z$&Wc@z>0(ibyUui)HZ@0V12Skqu$Q-5p2z=))v_}7jY{f8NPyrUS?2;zh_-+LlAiB z6xh;avxF?sfEZC$37d-8TVAT2BE52O@X&qw6pzvf7Wa$epTs`W7TEqP zStIRXM2r~OkCefbyfNvA#a;GVF|D-kCNt_vAl0f|*J?MTpo`o|Lz-f)cVD*oufP5J zzrOi~7L6iW?n|Wm&I(G()c{fPauVG>!D^+WcWjB#z#`Dkv@S2}o6dcPgG*oaj-f)S z*qG*wcFdYYGi%~7H!UOVBiqtWW^@~k=+5V9CN`GX!>W%ckSr^MFy6F?~09R zH=Vy5T3$;PqPX25nqtQ6LS2Y*j-0EQX_~JC;g{w9zP_(;f#&*Yfh}AP40XihEUrAx zmU#1Kc^h%%FIdlsvYVmR@^^c<2rm##j_FR>*Ka;Fm)2wXTst_|FY$Q%!fMUjidfp# z%kFm(GNu^25#v}z0u|($SO@XPdZdv>^O)V~Cm;mEWf>9Qo*Ac48Ox%JQ;a4Sq z;cNmxVD50Boh^o~At7sW==s6yedTj9*ezrf!qhp|8_0}LeB3oSsVDQvUgm`-8Gxm5 zhoAOkEYM&gka0_5iT} zeiAtPmw7a^^!srzDblO$q-W}q>w7$5<B^<=!U|9S$(TD@jett7hE$ZmIfKDxz9^F{{m z{k64sceHn5PYy3F_Oj#Gr|%}$gNK=s{#j?wNjH!beP(;7!_4|}b&40|qs{>L?19C7BU;D!rH8hI_5Np9M8gh@)mc-v09X;pC}+62+n%L$V699PcnOswS(iH z1Nk5-gF58WXrR=e!oNoU9h^_Ie_l1_#tTV195?4|!+e6AKX!R_aOJty{NZfNWL~KS zHInbng%)cq(0z(~iLq^Q#6#^BobZ^T3YTYIN<@H+ZKWjM9adZt%B?2Kjczol+5Xlp zcdMRmp-4KgCzlsth5%d#n;@z7{sxa|k7&!^a=_LhAnGmxH-`P-rWXCt4r0rM3k`bH zXU$CFbzoB#4FRipHviX;m<8XA+{i2dh$>vM${YRna34)uyDpl9?;mBU^|K2ro=$=B zxFY4V;6jX6@$;fSBW_{137o;<^?YPb9VNwKX+$pqV0jjj}fR$VxZr&>u(yn*9@&5%hMAw|=#*O^(! zB%ekN@-`%zII4frCATakm^P)hz!p`9;kCbqXZ3oc64j4IFNW^)53D(469)<5PohPx z0szL3g-BlZm%tWh)s2Sb}0f|X?eRQTrFGdPrazB9A;AMvWbjAV8&Srf-T zwE^sv=#$6HVXMI1h$)me_&Cr?|7=Oz`l}`}*#hOeykf!JcY&}H#!o02fpi#VQ19w& zHC;=UDS)L)V-M0mS%9?$(`5Pr|0~G+Ie$mhUSMLG)q2NB97w~(8eHP=;Re1_u8S<9 z`uK-oHcY9x(1V(END|zY(A%XZ0U0eg_kZXU+{(iH%ic{@&z6C-l8P#dK1trY2(|1d z%AZSi@W*@}i-h*-4F>0n@UD>$~Zbwju01kr;kl~CJ~2^p3pNRu2&yjcR0U; zr`~jLA?V@xk=v*xH6!`LX5wzvi)h^Jr zb7j0lEHGI!@VL7xbCN}}p6A=vVit4WeEsY<3G5VXxx3vk%~!gWN31Jz_W7yEJyNrV zR4o{A12rX?YTLI*S3I=n2vm5lH72w=7Q4DgQrFU#Ju)A#QVb7{lyBTu&_OPym)h1p z$>%g0MFD=QT4?8sT$O(|18yu}i^xl)LW;%%{|QahQeXX#3h{F_j@++ z7{l|kfnz(`B#Y!IKG&QX`DII*#{WzDm z;Ujo8E@SKnKe&yHR6Fnn4qXcrhDtHs@e^0)HzLAmXC$ewL%K(C)U0)5FP258@cw< zMf&%bd|M$ZL#4*h%5nu8Cc>{Fnh&Y(quhhIp}D5lUFPt%8S=L;0hu|jN<#a<{iR2R z@PUN1Rw0fbHBgS*9u0t%7zFG0M_oEbGgKFSVq^F`tC&Fb>9hkadTGb$?1n9svjgkA zB*fxleivVwWqhAEaaAFr$-E@dpEXLX<^3>|*V^{D%v1=VaRFT~nZ&qWtpR7DfYqfk zzo}}v)2Ey)pZJDeqiX;Ghc2xGRu#deNs(dzi!on=ur7^k%007$m)=Qc|BmOs&s$)3_3CD6Ti1npQ+h+5 z?$5u+XrJ%v*X?_(=jX}RTD_ib4xaA^4qo?u*`A({kNr`@{>oH5+|#a$NR$37qUXu+ z&cR;n9-2m(NjTWZI=ZL}#co!Ol-0`bm`c6t7Q1be z=?3zT#Ocy8xKx>LLyr}cC`K!-9Y&4xW{>(a;d{=@nhC?2$ew{p^0OAJK7OA#Safc? zitA|KuIUNJtTgSOWtB7(6uY5IE&y8JFR-vU4YkR@!x`@n^`?f6G#9lc71hXABwD^a z+m(v2#p}$U)bY?>Re3L0`fCXey9SU`H>>=__oivp;{f!rNIg>>B{c>@Ck-`AaSPAE zEL8pyi#9`Pdo&hkmu<&o>2A2DeV%R=;4hQ1h{2$jObA~cZ1l?#;z=b;i?)Y`Dcg(E z|3le1wO0ad+cvgs+qSi0+gP#fs@P5{tT+|hwrxA9*miE+``z=f-}ZT(e_(ub^gddP zb$lgow(`cdOiM%5tjKah>@OkdIu-sITEqSXufDwV1l?0wVZH}5b$jf2{hgEZ@jP;{ z@O&D65PvP)^ZM|^0^{@T{Fc9FcOdOSSgG3&$^UyKONlu9I(M5ue2JNOr|UcQtme|a zPEo6KVqR)orGqJ&sj42cL}}p@psCK=b5AwMk1}VP=>9>W7g+m+z0>W8N4ZP=9WwpG z%JtC|Xf$=e7lK>G$3KM!KKaGsbn&9s*LG+?H!)wJyZkR}?yzV$@yQTs8_)c}A(>pg z_NRRQgoqTRc%aY0)|tHKOE)7ig}b5t_{gU*Nq3Y7q zQh+*9H*i_E<5}c+$;yWdi6W^kj|_u*jG(Y&%YgBpI1-VjI%5iy&GX0*sW9KtfLf=i zSp7YaG;5z3bu4TLZ+IkNdC%A(y@F`O{TS3@57+P3rz}ji8piD9Om6F~T7K-0AQ8Y^ z5#2fx^<&UkKmd;woy-xa#>yc=F%(M2ct@_Oo}He$NY!<;K@OE<@uSeY#CE!cQ0e?H z+Tg_ND%i$Q31O-KROXe;32$j7C{mr#0YbrIUwtlAT_fCcZ)$NrHEmxo4EUght(u~Y zf7Bh}OH4nb5VD%=)OFtvy-HEpLuuRK9HvWDrUT2_5neYkka(C(<>IFT*P}-PbgVvH zJS0p1+~bDs3f!!PGf7MdgBJmDw%h^NJY{MLReksPLsPlRnvTkl$a%vSsN^g>lmOU9 zh$LG6zw#Nv$LhzaQf3&wjhzP8RT0FxsMJzC+Qy8ijUTJV?(*%sn~EzX@{v4)CIxxqx7Rab9{ItB z>8SM4Wy6(`klniQE;tf>yJbvB7_d!`HLy#Lu(P<#gicQS?B`?$eFOx3>uc!;`~CTe zgie$OIs0w6go2)T6<%oN#d>JK(L^7T4#_E=8;oebe3}|JXTr&e3(GjE)lo3U zU24@GM$&$dklERM?yVn2x7`xltx(5N0!Oc_xkOk8fDeM?|3$!|EBQhjWN=OyuWy~M zXqgjY_2p!IANNDb#Fi0^+?HSeOvanV^1$`b&b#>HW+z*8{qFSiVRU3Z9f7@qgFx40|G}-uB47!HqKeXmEC29|&C{bewOAQ;CuK1~$>LJ!- z6LEt?F$=O43v*T@$|3buJ2||ywc2k6--IG;ZA7&S*zB3^%uHRQwb9F!(#?}RYwT6> zhz79c*RbVLe*dl|t{NAascjaxn=SUSPTe6nRa+uikLK?RJ$5>RbAHzt^;pgF) zasYJ(DsT3C(WP(+@4gLSQ2C^4g9xW!P>|9YrO$3W{uTTA;S)e3)uj!iwuro?87()< zd%2X1opSwWS4tjrbq{(n_Lfti2w6hh2)`E!;>5L0p;OUR17ipHmx^W0L?gsTBTL>Q z`6a$RLBHO(ly4zF!;!3$<~$Vu6+YVm$pB^sBH^nBr=?|uOTK;4lqT>!y@~zjBtC&ROFXfcs^+*`nQT0_kTq-yOCvau6~)lKfm;+E-w2HlFxgvu z=f4r9DL`V$2}7S>Dv;7|%%d(StBIQxS69bZnwjv3V2Mf-jvv9xQoQl{&>P zY_-u)4yp4(p-j}Yp8u!3Vu7rOzhE`pLl#%>wo-J*c($Z&{3-=~^v+6Pid%%e9CA4sy-Od}UVAzg>n7g)C% z0Qnr^I#dOZ$}Hox~- zK-CES_8&HG==W-gAha^VLjkf?psMsqKuGjwnO@io)adC20Js%Dad; zp{F&6o>Zp_tK+; zYWNRGl2*QV_R~BgoY&ObPOl>-v{sKZxLH_z5-8xkMNz@Fni4bN(V<@o%y2L(xvPxv(>L>^4+$WmkU^## z>VrNOfrdDAg*#lT%ryDha1#(Ctv1fV=y>)VNElz(2m(kR#_zJh8ohVts;-<=;r6w3 z>zQQQREXVk>k+Q$w=biZKDGKao531yn1cG|Q51+7jkkGQaJf)L;KXgvm)tqJiO5_k zR{dXGGvxN!VYJ13b-ZAD7hmR`CHtE8g@2}~IQ~jI)&p|ug83(A4>*Oh9_nUc@n{E3 zjfJr0^G3saqu`S7vqehyZDDNZ5FC4a{1lY!shX{+W#o-|ge( zCb(wS&Qk?*5Yo7P#Rav69<6YoFoL^lo%8i<#hza}Hp8u5^K1dO`1i0I`U7R>Ir+wf zZSxBf-FNaT%5Je(4J-SwmV>sb!U`pc+9c%<3t#VXMCgouy0may%PDQr#cb&rEKqz^ zHSz5>N(HBs1_RM!{eEh zOYU1NC`Kz%O`D|Y<%I(5P=O?VeuHh)x-s6l0{e0~3r#L3t4}pLpng~y#1y7*YaNDI zA`|FA;j|~z@o%4wfN-fIF;oH>3j#vJV+t)kiUogMe+eM<^TT+yY4?43!GW7lR9`3V z5~_cI*)APH;=&Cfa1p+SkYFuMyjhrK&$H=7ktWkDGB&qQ+}qSV(yl#9uBx@vHNvYq zPS+o(Zz_r8TrAqLwr!_2#9>TiB}9)n2urGVl9%*V5*E~I9fgW*ql;Bnw}sW?s9PB5 zydd|8KflS%3{TJ2$L=>$ce_1oGs_dybwV&hlB=?mdF6VvDLJ@X%Gs8gfIsCWHUQ8l z1aV!!__y1trkh}6cMG+*Diisi7L7pQJaF|^Ggk!coJ@f(=WWEZG!lM;S!Y}v`r1qR zafT67?1Ht8mvcgjck`zn=7XfF?pS?3;dt_AT~x_$Al& zDY!IoHRBy| z(>j>2yB4JRN(MZB4r1hJ&88Mps^({25tq)K%M-p!7!t~(RH#*O`v>LT%S}>BI*eRE z(699-;_<;=HY+{tExfQ{5kxh3r9JBiKff@Qic@i4~qwFjQ2tiNy3vg8Te8mIYi-vI)Z&e`Gb! z?m+5C^GvXh!{Pje-Z~413qWk_WR9Z2O8&M1G$N~ zpf8m}spVougCp5UcP~-)>K-W4vX@Umc;JQ*Eb)8(eL4gRcpx<-Afc2(@4zBtwxZyu zmXejM=*+vwa--niIp1T5f-C6?cYsYUV$svW=d^VgANONw~BP zHod;?#I32-r-&_Fu3q~s{Q<>9wEVn%qnIxzVOl@Q^!1z188yb}4i2r6u8mz8+HB48 zhJQ9M)PbYE6`=}wh{Qoq>PbKwCM@v_YFj@LmRza#>$$?U^X!R%6P;ZeGH-5s9vXU_tZM>**oPeU@ap<-`O9Nc10|oFK96xZixWijb zdqS|$XNNE@3hkoF^WF(TaJHaK-R-72(U}$m2Q3e~G17R$XV6gB&jSXRs6b88?2H3S zSndmkWNeNjyzaen7^}-{HNYlP30E&HPmc{}Qv}M{GCZ?Po}x*8s=KVGPMA~bA!7OJ z-I2@Wiv?~w`aPO#?&v%4EEWzK}iPsMQZ6E zDUZqC-So#D3QQ6tSzCHd8&PJDW#@o~f7YLBLs?jFd0ZT?h(C2>3EDqB@5K0%P00xW z)|8CjM0FwLCYgsd=<9Cv>}aMrC$U>2(bA-Pb1ck{ArNm`V2>e82 zcCRY|(ZT=N>1tv{*+N$GWWGr4+@bOZOYVjosJuZ7Z@LD{d6~(`(@Ai8QU)!|k#pTG zHhCd6bJKfA8pK9J*zlDPr37k}E~qbC)%fXWbB8$bgn=KW=MGj{9u{uyObDVB?aV6< z0bs;;ie74Jq&YY|fJ+Vkja9`R}XF%9tK7dl&LUOVNdhw<%3OUb1xQ6YELi4zobW)rl@@JY?Ny}T-cSIMK+ za>ky~jkHTkn87tgD66ooHA;8 zxuduQY#mXfMBC7&56Z;NZZ194-ki+g&Kew1LI7v;7Q<|(uEV&?c^r9Xe&Vdq%2kwT zxnJ>9hs2NIBt97$xAoO7dU151d|O{h;zpW}<_Y*>4>AXr^Dqm|P`Tj@9GC>pOh` zhs)pqZ+wENvlLu_{h@!TF>Mv(q*(Bf+%xA8*MwyH2C2E8N83jSI!hh&im5));SMnM|Rk)R};ur4qRE;?Q!RA(`zdu` zg9C0O&DDjRfsTvsBAXZz)bHXeoRPr@4^=NwLi&GKQ_V5jvOjqFKqVY=TJnyIR+gBE zXo&`2Et=xq=&8sZwOf5lI`x&u)n<3t*Sj_T{VB-X?q(bEcM&T9zsrlo-^B{+DduWi z2%&D8pY#-k?#|G-OU^D*Z??7dhC8wP@Lkdxd<)~D5nwiby9DIL;J4vD#9F}ch}wn< z97bJ9yR>hpz-55;g=cgo{MOgNjaEn=)Zv>yKa+)sb)<7)kqK<7UGc*&!WLdR(%hur zPf^gQ*MKF*5=tV&cs4N;V0BZ)6%l;rTOoJ+Bu=cOm5(F1(5F<$eQVF+>sw<>5(a~r zI9l~#3>>JYN>`5&l((jYRGwkq#_KF+Y5GOfB5Hdi;1Yq!&sX~1ESL9!77JDE*$*7` zH7PMC`%$=y$kQ1rg!@D=0*3wlR;3VkX1(2zSSR=b7Mc#0EZOa8y$khK864#VmTT|? zU!2Fbz>BETmMW~T`rUj^K0m+OAs*O^K;P+|y+cg_1lq2yenf)RMHm!VZKMk604yVA z)p=ZY;k~@A>G14PxSNcgi*p+fXyJ68*DPhU?Z9HvO$Lx-B=ZnPyWdICx(&2hcwW zE2?>Dsu(XEzx4x4*UY!d){QH2!UV3cWV@9SBSo|)`;mGKpMr)!y%CndDM8@26Vf`6;Bp3_LM=JeEv2H}WK+`*^tFWK(-X1kyHTyfXpk8y zb-MsAj<;NzO%P33>kJ~-r@ zc8#H=S!)h6b=5DEk0tiWcTq-U8_B(G%GtKpbkZ^-fswJU`Qr)CQKzci0C)Y{(^QDd zKL_=r?z+{zz9z;;F=4bj9WPg~pEBrj7_uess0r?q=lsFUrE|;1;H>3mD+)t)vd8mGWu#C&+*LJ3?p0O=15f{O1XE>b(u% zFyG>DSgnD9v&{6kxWu%z%4`W3*y<1zU5n$`&4*!@O2R|`k7v!Jn}B7SL}ZH>DD zQ>h!<7FPkN8YK3Y6fwpzU1lRELuc_7)C5XztzR@IU4{R2ntBa${jN2*M?=i_qcJZ> zV}xq_w7)gd^6(G37VMb;N(LrdV2@f+d)?f`te)_k^l+9J2!yB>#a=`f4GkUEAnn1U zo2sQU{(0}b^;7dfaH3yUA~0^v!qyp>y}~ZKE3^HAxid4P(K@6qa+GT{HYhxxV7U}; zY!$Mqe|;T}@RXh&`*K!8OFkm(in_WQhjb|naq zDM7>Tld7uRpx|}sZe3{8GQwO+py=RAeph6&)4#?zO8hK-tITW}JtrtNj-qZ-TZ!f$ zHy<%;(b?|cL|?D@MCUaJ+th|RViMPp>6ax%4A;=i#TV z6XI8<3ukK1tPHh|Vcf*#n=&D5TRi2!eFg??T2^!=MFbkL})EcChQ7Yw5;!C1NBSLDS-_>WQbTltt9$PAi4 z1M`2XI$8(@`VW?I<$5e})fwf)vSKu-P!j%mf53cXFB4$75Kdz6a8OH_BOkTgaMnm5 zsEGD?eu#vTa4Z6BhE|RIaU5FG8BnA(#nM8Y#6!WL1BY>1tHDS3p6RQcaS?jz0pq}Z zHc0!4&HyC@@bw&2TT+K$mM4X|c5$gIj7EAlD5F2LgW1{~_!5pNTO-y`gb%?Jhz$EM zr^$d0`Y*|=n=6jm61WH<(o=#P1Z8wOjIuFr3X~b`MZVd7uaHG(P;WJ>7*8v958Z>l zLl1v%8K<)Omfk7CU;P->Xecv{sAzk$tQ9fNotf0#>*WZyXANf;=DUue;KR+thDkAh z*m%yuG7S?7spmkN^SlTzpgx0 z$A52fzk@oMZ6mHKb=1X0^w+vJ;14ltJInG?i+y}YI%wVDj@|J`Zx?!z%eQ6H6N;H^ zMxR`gtGa}oTLkF!5#lZTt`FhpoN~>67;-n?t2wUU9Kt#;_$5a9rZh%#ciPyXKKu^g z>^&Ax$(TRFA5yG`lg}zacI3~(Q;p2G4-)JjgaBufXRu3nQvBco0+3-qg(lYj7>ejR z$K6GP!`6PsEJ1QtEs&`j*bp32Ur*DymKw$P_p-zv`yU)M7IZuM<7Rm0P^=bFjrp+# zcuGVbGo%=NX4iD-ZITZS%y|Y`3uEymTjweVhuU72>GN_Nyh8<6;hh$F{%VHAQ~VT8 zY8%>xb*iXIEu5q{AzIAZozQZW8&gZKzazFXliejlKzO} z{r!pfpRYp~98LZ?f&Yz`OZ~0njb_w;G`MLXpVwFW!62Zj^Vu6R3wNZqiK~#cxxx$} z+4UxJ)r zr^y(kkn-0o^=ubo`uX%!FOVtKCm7u2atq|%CK%W+<9=K_^7`@eCSC+sG^Y?&y(p-r zNfMFn;D3AzlqKt=s{cCc@?H?FA5GdQs4O@y+Gpje_Nt2LnRz*$;43phNgK0w$-d<- z=p!=J`@zU7X`P+oJeJJMjJa}OQPi2o9o|rO=E7d_SlO+_oHED z1vB+g@VpOtO5lv#DIhrRdr?$cfq^09$NQAG8d(=3lmKdOiK9et4+m2pIe;n{|Jnm> zgRco)9~(fi!eT0Gazd1F3aR47lodbSwf@?U3s-4tDshUW&@4E*OVnm(3SRZ;V!>}tY0mFBP( z@UW@FFfD70<-)5}4i(#&y}UBzqvdv4kslW;MI;8eX27HADsu zYOlspr@yMxtz#f=o6f>ot(_pX`^!9OCR(F8*^fTwR)jwJ6%xhZ)Y z6MB~DO6d=9V-~wfmUV7d$xs{QF%e*ucu7RvP*~{ndIsQ{4H}d3&x}FIF?r`h=Yttz z^=CxgY0i+W##skpGu>C5!lvukI<`E8>-Gud?VrrHj9U^pR z@JAA1I=gN5|s?Se9`OOUSuj3VQU=fwzgN2y(`PyksFQHaOq$i`ZurAX^`dvN+I~)Vz z1ME93p8JxAC@-wdm*H!G0tF+@QXv=Z6+YXsvV;;Dhl!AML50Es!2IE={aa8t{O?$oJpxKtMRb>CRfORL-GXK(696m;n)FJJi#shqPA8vFLNE4TwP zlS3Huh1Bb>i0tc}=kvj|luA;+kc{a^`_t2$$8)^VZmU>FsugT`El+llbaj_5-xOL5 zJ|j`xk~bh)dLx99sRTOPT_jSHm-})p24V@y2Qe%$?ssZ_d{s{vwWfBg{B;;>#?N&` z!5<)(f{ylVxoiQ`&+WKtv>io{J)}f#c6rLETUI{i+%79-`O0XdBA~xtkK5%(TPcCy zk`^F}+Nm^ydDMAo5x5z~Q2u3qDBG|orQ%YtYIBsMA@H#`1Xvm&;DLtALZW6UqK3wJ z+cO8Ot-zI$G~=I&#My9#%Y5Q3fM_>ykEbLl_IgW;JoNN-!xe}yyC(DX5JR_u!IcGd z(d^*NGmV6vrs$t}TM+n)Yg$`(HqNZKNVCHB{#?DP$6Dc`nou(p3TyQ}hOydW@gZk_ zQsp~>C4)zN{*0}`^x6L1nr6R601rB#Hm5rkz)?gL@CqK5x-an#l_HHfIw{lN2aoVo zJ9=8_#q+GF8NQ!&qV>in z3q{Hi;#V{KQ)~IiS^Ge?y>H1)wW!fDwX%b}iobj^^1llquh~g6p2p0Q=;eg2&sU4( zR)!fmOF!)TbjFGS$}jN4ocpR$`o8rzVK<6&8cg8}Md+Eio!Ctb7#kGg#trBu3eYQGSB6 zkO>L-e*wJuZG+L)TAbS?9vLfz5vt4s5f<_=L8dyGZhl4~!qg9(u``-)-8)qxnSeF@ zO-V8LR*Hn=Z5@;%$w=_)v#;^iw3{*(1CL6~k6_QZjniy z{*1otsGh!pR`A4u-<5;>r$_`p|5>h56W!Fvymt>D6U#zpp3Un^EP!9nXMl^UJqETJ zfdgehot8V_6jfAzLZ^vTj<&tnt(RKaFUnubzrBNOec4%N^*iP^Nznc{zrq~U)N|sA zq^L>TG*hm`VnapH^@g1?Yb?VcEjN=a=!*Y|gs<2#z;{U$f9UiX$wMsHrZ!n9*7#a5 ztX^MybSp@sUEd;QKp87ixnlbPMpO|^)4sH`(to@mttfi%g(G^^Vr7LR0{k|u2?LbA zL&|xsFY*N25KF_e7j~_4yXsY&!w@K``t#lI#Jn=F!}9M!3Io5}O5vIXUMb%#ZCS9J zL+{Iz;=8j88*R}P2Ld=X)LiTE?9UUwBqA0M{@)_+WeO}D(OAMRd>&@%jMbhmT5##i zpP?c+mF8A6V76v0yMC@L^=w_+v#)^`9ldULqfvpJTH_r3(h5t6uzF7r$ib5QTpzS; zl39u_U0W22qsDPfp2rePwnOMJWqM3%E=@WzE=sRx#k(!(MC$KI+_SprP2@QQ7Qc?f zY){y0rb*j#Xu?iJl!&$Tshd(bh=`TBd&8BfacI~pkn`M-%F<0WqZ{Y}zcBSnPn-XU z@{@)!Y_B`m2oo{0zQts!QDzfNBkk{%T#M9r*o)d-4eL0g19#olUNQihM0~H<^WOYD z=fCIkh+j<>DJF7JS3dYb968ZU~pwejP1$B$7bi*lPPVCj>w21Xbh^6MYu6%uD!mExQ53 z*XAFh;x`v7$UnXr#CWy(#uvRjw61o4xiGMDG+em#i;1P; zF@G*?BNDo*o&5@`CLX*R8M5kgp|k)Ps|t^x2uP2l>HW9z)JS&=#qiJzrRMd5j>*=w z-E)UMf9v%UCCBB^(5CW8UD}gTne94{n0hX(X0}-DCT6VN8MZgD+QOq#+Fg#e?Uni~9Alx|^?X{7NDb@j>PJ&TuURd|yv+-%-+0ttiGq zX`(15k{dZb(($ShR4qlbRJ#2EsO}MU(&dm8T8leVONrg@9IY$IY>gt2>NUee*g`v= z^9?t}I{GphJtB~5m35VdJ(aghJQc|EnD%M$`G7WT4<@RtMc%$@mlfKZPUVYqkm*S4 zYB1;4!@8?b8l+|-#>`^eNk0*~khWZIf4*=XyLYmQI5O<9O}f;k5in=%by zUzkUdZeK||7*P8?v%F|7G%U=VjF@vF1-on&*m(OID=^LcBwm%-#ab`OBB{X;zv>_m zQI{((AM;!KdIydg&M%yR-+LBJl4qH(2xjp6Nc@939qRv~#$*%JuYJXOK@ zhs%|@EBNLj{)GW5jZfSk)@=l;E8Aa1ei5Rp+7Imt7fC{Pvx+5X(m=2F87c03wu1kL zNUlBi3V+?e^QKdaUn^p!-+`w}>+9LTpv_{1(D#XVZNW^TmEGxU$L<+{ni(PavmeUI zSn{sTaxQWPygcdU0sAh#TG0me8G}0VHW)5N;`M~=&q?k9Dz}sK?(*f0Z@NH{tV{@U zsXCWie5oc{GC+gjbA$d0R>87uwyVu&eEWdREBZJm)0N6DOTF?fW&alcd3KF_{Q2dB z?{{c*=I1)0q~jiJY_nphQqH4Y0IDE|ZRrx4Gf7%Ru>ax`8||^7&zlQK_%|Q1;NZYb z-})pM#xo{jkMnvY6e02Ls_yv$`2B1m(_ha&g$6WZ^arR4&_Yjv${hPSHE3r6GGH_SO7 zxAEB(Ly6qaRHqv&dhh2u1##5$qYo^|$`@DPy7I!e^xj>P`MHc5^H+Z|8A(Te>DC8R zs&=h8c3+S@TxQewT#p^aeGz*f=CMbPwl~ZS8u4IrhA43ig8fl$+T`mHhhB|k;gfTT z6^XTtCrV2*^weawtI9Dv`-Ev7N>`Ll%9>HkqI8w%dWi-LXvImSTVM?n}qI&p`^HM z1^?4X1?EkoJB!bz4#q$+{q_V zPg@hu*p53f(BX%7;aXZFJE1~yJpA1-+DYLR3Fc~dH4|DRN^vvfx=vg;mGvA{Ce!r- z3gn9YUda}`uauh%1spo;ITN_A4(L?pE-!kiwJ}H5M~N+20`lm1-iJw_CB~660(af3 z5;srR5K>U!kjWUg762ORF{vuN2F|Gl4qWnP&dwS#pDYW%PV1NI zyIb?Rkne#cZVq^+B#reTrz{KMhDwMp89#odWQ|rL1gIYpylFH6jNo83Kkt4YE`qC@U%+Q^W;k?htdC`U@8!RK-Nbx+lzPYJ3t|N@A4Om$&a_CfRKKv?EN7?jNkLh zg?)#LpRtm^6%2rg?qHpBB6!_SP=p>c=dmxSHH^9lOdn)NH{6grT#8_s=5AU_iS&KO zrzD*CBP(+yFX7|pMp>(&h-2SbiiK~Loq#_dn=T|Oqb6RkS6GWAsP~JC!-s-?I-$5U zF-c~n-thJZ`}x0V5Mzc+avFNk!Fyfs`;w{#C`7{x0)_~{o1fGMoF<2Hv{H8uOuE&R zk}NwRL^g@ZQV}p;d%4+OLB6WR9@z9;)wCGu_J4Tm0)L#+V9&lkREfyW@(+iXro1}bPH;m!k$YL>Fmw#xTfwgI<>q#UeGfJoPvo}sx!tY-h zw(Ldrr_s;&owJmj<>f&d4L1Mq_QD4?!|Oq<_9TbhH<v#@DF4W#@Ks)HXr+xXL7 zts?$$@(O2xwBlQZ*}fQWN$OffAi}i%kJrn+7L+*?wJdo18ju#4#bKFH{=mE$d{{v>~`)A_RMj*78|A`7XHGtI2}33-D?7^$*T&X zAZIDl0Zo8pg3x$0-3UmqD~+-(_J^WgUOU$uweTiI%2pb1pVcY`%{7c>2xO5;nvx0RLatuQR9_`xxOvr{BPBL535HXCA9M%LpD|Gt(1%s?0`lYx80 zeWqTfWuETd9b8 znr(%e=yjZ2Nsx{A5#~c-w~yUx;NQzSx#i7XBw*JO47od`rNO)FCZ3czp@0T#t^IGl&_4~qS^5??m+Sl6G#HZGK%`@}!_j>NN(PhtC&o(!P|If`&8xMf97*1ez>XzcW!R1yQtow@6o(;UG<6 z#vlf{ZXgH34n_*-TTILGf;Q2>-<3Vu@DmEFpPIw^!`P(Gwq zNLW{D1mSxs zob9UdXJ#z1k*X|c-T<)zG?rfH5D^`iJWO(;+zaeM+JM6BNa59tyu&M#Y< zUOcr(u;e@mO+95!kYkUt)!<=-&=xs(#Io@m`GUdubjz+=3Bx~edLGM-E|Qg0C=wzk z7|{>NqtY#8j{WLXh;X?}52XeS)>k_q!OaPEqCS`*6_^vo*jx&nE`9r|SLN%BJ%m|r zs`%)}^q2~SzG8T0R%6;(1v>c@YQamaggP|4dJuyF_Q)JFDL7@_eD}GL$&#+j@l{Jw zOn2C$WspQSHUccS9|QgDEKR?}$T}5=L;-NH4xO}jvP@t^FnTpLZ zACs$4fu_>t+!$yI8OK3cfgwLj@c#nkr<=0d&W#7a#*}$Lb9Ut~3+JkMD3pc|^+e?vrOA-;NOSPn zxbW`Hm8-{HgW(WA7g%49kW_t*+Vf~0ZFQGyRBcUX5h82kG^U52wxG^wR%DL6jAdW= zQ-AhVY=Lsg2#!Lo0F-p`%6SerD`mK426S1NmQ@f@aR=Z97%Pz87=4@Ah~5ol26AD5 zBhAf;facFL7%9Yis+o?@;PV%$)$;WlJbVtThW+IT2%7-6J3Dz&OX(n!=o(+35t=o> zx{Y#ZIf(RDSLKXMVWKDJWYdcGY9zEaJq~@dyU&8q#$SA-U%V)q00&S*LOJysP+u?H zXxbf8*Ij}hKNel!=@ z)K?fudU%+XVBCT^0i3}Q?H}n56j5-xl&Z86@X%0XHQ)vDaOhI(T4`J+8#wV6UeePpW#76V}zj9PPYb4`){)o6)NuZRaMW1L~f!M+{ChGNhb z^7Fy9w2Xh)6am-M-HQd~p?V&vqUuu9TJH`pLVp0+I^OwABk}Incy9pzd8tG zesl5I)Aj1CpD@LpF;+LHRx}Rc1(dPv$kvnh8D~*QKsuHczIl<~f9;&w*H22bwC0-_ zT3hW5)NB^TG>t^2myjr9;Wth+hEB~49*-YCEs__)g!?oDE9gzMETjtM=7aiBR@^+= zVk89!UD4;#nY{5x#X};`74XiljOkU~l9}{9G%JF6#;(nRg_Lj`gg)Cicbc{Qs*s8% z%hivys84$mjW{P8c1GdvDgu3$G#(aOKPx+|7pf(XH2?m!gMUmEo#T8-_V_!}x)r|_=a765?mCfb>`cr;NcR%Au))1+|s z`A41{dr%Z~2y@>r=*Ffg@*mGrS+4Fe%-4t&@2ncvr59+`e7eFll4cP{IUTswl9+31 zS_Zei9@FQs(W$Ihvyq}%R&>hlu#%`YOj|!MyE`43J2bt1LDSmL1pIY-U0cIJUy83- zx`*DrX7#9QyaGdZOpg~{Q0qxho|?uBP0yG(3fir*l7aK(a^~V_=q3uSg|x)181I)9 zeb^G52QG};RdJpyy7E&;{k4Cp`_M=n^;^2GTU3opc(H~}|Fvo3+CHtFC36}KCupbs zNIb*YGK*E$Nc9?BVQt=FcP!v zq;9N!T)DQ~(z+SC_WZc=crrkH2lS{~u6Oemp7S7$Ja*4ehFetVHfMb_HJ*Pmig?I9 ze64!@^7`ZrJH9&gW~YU?kQF-ZyVb5D8Yh16?mRM`5k{)dK> z+J?*LRdX<(i>&@@4v-_ZY*r-Gi^OJ@cz40kR+VW_^;DT>=-gcfM@( z{X$7ZdL>AWt)u7Vlu(@JwYJGGq2h`uo5ytg7C!w!t7Lb-7R7vJ-(tw~i=jdyp7k2I zv6w*UiT0#Z7tl;U%efx1mmTZLr_&{ibFZ2eThE_%A%QqLd9;|~+NA<`#-xa=VIwoz zibw(#?4?6Fp1UsNWYQkMDtXW{@N-|G}&dX~sgxJmwk5uX!Qo{3I5YIfj)rn};rm8}zeC?m4ImKQ?fw`9ei6 znHt~XI%sb85lNA}avf643$S39qfw}ept}~QV4>296gL*ezwmt_7GoC1)iLCmEBH0F z5h^(Oo0<3$oHM`A6|IFlNLTRQ-TKFVLVpwe3UP@Ea3x{C5KD91Gw};k#n3E`GpS?2 z$ADXkk-?ouBl8`Gqt0fyPkq#TL6=m#_{@m}inX0gu)Z}EQ^d#4~x!zfwzFWWY{Y#Uv+ZL`a^ZQHhO+qP}Hs&`M! z*>TQH>^ZTozWetgzZGj`KDleyp#ROEhLy$5Mh>s_c2)wz8yiV#07rtTQzK-~rTkn&rQT_8Zf=nKvxEd;zU-Ytm%r;if7 zeZL8{MW9$68*6*+W)2%74Z*Npn3ySt$Xu-=l-f;7=|LpmbZnLt z8;L1qJng7{kI_h7VC#?DRGFR^_JkqW05vPOQ_MX3S)Q`Ow)P1;-KdJW>2}Qh=xv_I z*5>3`YQrsSlI&3X;fr7JKPYHJXve^c!5dj^^*Q^wqSTkXvB91N#4n#FZf_5w~RVYl+rQ{Irod0WRO zXNAIG==k#8AkgQ%qVtkYj%@q}p4_Vf_n2;mcm9k8wkQx$Op;T0+wmbPxX@V{=y5L` z4Y{g3yePQ)_eTO#Uwg%8amuBM0e%$QqjwYi1|!;puUFK;^WXa5GzJahBTC44)C>1K zjT+w&|5X)r9rPv-!2p1~e-^O+TQ-WLlY_I7ljDDQiq#bUQweK1%IIDGP}@XwV@lM~ zXoLcEkeB+|@*u4?3apwN%Hoxge@*)CZYv~-Y+7`{!v(An-}Z3(ylyMFgPB!mUs}~S z&1Jvt*}1(b5zzf@!cJ=Kms#+mAXHr77 zXd*M#{Tx`0D|=?P+MF(u|2Qfos>zt?0`McOAdEn0TG2qb*Ha(9-@QQX|3m?|*w%(q zRfGn?2o|+!m@H>7J7QKCVG&h^7vjqrkXanWT@Kd(K?N2W`-}vTs|eOT#13i40I!pV zQU@y*C2KN1x&^oTI63qH51>pZWzp&Hu`?76{*=a+F25;z&4(u;@#=0B9v`wJJ35U*B zw%SO6a2zk(k;CuJ?mXxi*atA<6=W>Vi;Qw}3Z@u9Lq?p6osT6*$@_{?4C$ws7*0@P zSOc6Yd<^cYxIQt#96)F?)LoafM)CP|o(!-Ouhoz_rmWnaU1VsJtV^IY(%sF9snm6< zVqI6x2$G6+QC0LKR!rMmfi6W=y;N4hK$SX>yDe6aV-3r;c1jL?ZC7@O4f!@TEgXCzcX=sWSd*&ye zS$N@{a%f?~WLj#&JmLZ8%pT<5o2#HIj;~5Y&2+)MLP2^46{sB!>UqCuURcyS<~*2{ zIP4w5iP2>ke4;e^?-&EOa?cTrreW|l-nTjU=GYEUAbDEmTkCOclKNMO*(W*TyZY+tWwucNnjuO|F*t zzW5Nlg6&Y9*eL}r-pf+7fo=DOzkr1u&?F0Q332(Y%&63HFM(oZ5fD(V5`i3rE-(aB*kP^Z*WQI#9MYbr1W3d7%DU!VF z?YH9b^fAuR==(Y>mrTZ8`%=(@cf?D(G`m-O_>T3qA=KsT1s9U zJg+_9$Av5|n9eBAb}^IO=X9cq>QQ1x2~YC2E&@2Y7Lcu2G)DHQFTbNbZX(~YYUf*K zUb`p9(Txfcdl~fRbT)9s&{*27LvjxPQNa5fC3KwUry@*c;CM5`UpY&l8Y#N57yeS0fQfTo*SBAoqTC4{ z0upb%<2of&%=Xo2@*sf>-F@~_j{S@~^0Th3d3vo6?@uo+uU&cj|3+-U;AXdzPb=j1 zneUeHngafF?;MQ8Elfd~n1WJ1;R3?)QIE856Jjr^X&xV~b#Y|vRQZdEP^&>|-TR67 z-L-cU+!JOIu08yG8EKf0IpRx1E36tjom{r>f8*I_Ndls1Z~%ZntN;5p)Bh+lwKH%s zGXGDnFt3H{rf9ptEKjaJR0n~wsM?Lp&r|Bpay_xtAfW1`W^)zkLo zqU)#`d8uu{47}&xm-pE3)%&1Qu-jZ|cg>N;v{Rv%r|-v~!JkGFO|-7XaEwVB`gIa1 z@=Bgi)&75|NeA7sjA1IuLz5gCH-(||EDl9M@nm1z= z)Alo_=n3-#VG)CkivoNQ<-~IHwy23n68t}YcPh$Bm!!lbler@nNwspao>%TXl%7}4 z+I_RHU#Dnt=kCMNCrCnK#coXs?R)}QM?X!IJHj7;C`hw@UsPsgzk9qOYJ%c2^S zOqgBblUe>2D4kpyg!Ye~hp5Mvr^%xRR}6=?8CyY&4~5d?lxob(isg#W8d>mM2^WFr6>z-zsh<#!Pz)G(Zsyc?eqNj-;P;^kO=J;Z-e<w1L`#s)!89b0Z#L)|oN)8$5 z2K3x=f)7bDwLP9t%z-Mg^2_S<&Tk6>@DT6dByIUY&~=7-@6O|RJNFW$&}aQlK*314 zC6Pnsz{VxE{lZ55;3$HeX1aq)4W)P4AHgw|UGQVq)WodxEomy^!d$C>hed_akhe_- zniLJOj=>OM(xOB8L-JMwog|I(&I6xdq+QmO^hbVv`iUM#jCsAlTZl7>N8N4Zc5}bZ zCPTQdylnr@+uiQx|0AVf5AYF0t@SqcFQfdi;f@lYD8Zs%@Fb;Hl~NpK+z^#tAfgqH zMKcCM?g12zk3b^{@ApF@}RwDj>spZe{v#3)7=*T@7<5D%ZMr}287Uy3HPf>?*s zf_&|F{t4XcO7eb$^E?2#Y>PXLw>5AEW^(TbTEOMNg@w0+3A2n9c2IGskXEP@FkBy{ z(69$*5_Zt0c^@)G$GH_lvspKo#I*zbDbx=~E0opg4Q~Wj0g~o+VBIAS^}Eg*5|2rk zb>Ashu-{{6^gtQ56UhIEw;uda2C13_M0`Bh?Pg=Y$(m&S%LyE^U(KUz-)p(twJG?4 z!=v<7Y>$d~>XQj>e*z|3hiR6XN@TXN%1Cb{EG`)-oquSDY60D!Mv6I;?!gx-}pcY5J51>jJD~I8eU+$1z@iNEeA}%)pC`>gAq606$DHzngylJ z?3j?xVEmiFB;n5GoX3c&c9~* z%yw~!s8|O9ji-?&?JwZGY{SJI0mu#(ekW27lwiA(qw!@^#BZ_No86%i!h|q)&)Vuq zvQU3$lov6|{1M=z7Iyy_+TvDE>@a}Qh|k*5XS3p3} z4QBKo#t_4Mi&@!-CBr2HH)-?L9W&tO^J5OE3vyYoR#OQd0Bsr##xJ@@J!8k5haW=+ z<%C@bErD1JCj&Mj;5+VGDt!-mND)~(4wkH8&suVVmyY&9@d6E-@96rvM}9X<*HG=$ zcA`&^`8{C#P6MV=e}v_64V$5>6L!m++Vp%XLKu2tT%k-s0!^I4(Y{9Tvb%F&Nlv6l zuM!?Rn$S&8R&Z#P3ki^tbO&jJKU=)KOFoX!G{RtRRjaB@qLVQuqaop zB@ZDtRFDv`y+i?Gs$0s>tAa3f3!4PvQAuic_DU?)CWL4>hN;$bS`hte>O_cHO zaHaN)kQ%L>z%8h?Ymr1YP^K?&-mlzLetIZot!_Bb->Yy1P&9iEUlM~j2tNnrA8+iL z!8UkEkmB;VEzX{U2t^m&2iT*}^W^L?9MWN3`oeaU{i+OnEz6uEa6r5BishRt`Ul@qb zZHK;J65JQNbT3E?4GdCr(adhK1#3-Fm?=H^;|- zc;QkFeC?$!W^7E;D2}T7$?{V6Mj3rwL_kREmDmfn?LjQg3TpF9wQ$g;wMw9ZF4+MY zfK^$&Ib-qbz@j)bDg+7I1WPCUZEcW7MkbOZ4R`%4NA1fINurzgC*D6JU@u87*?`;? zZ{J{Lr1J<6Qoygbc9v=!hk4OuIM(5t!DQ|rO#2h>qQgz6exWV80|$-=LcmhZL9Xy{ zP8?DO*pg4#i072S%be*iR~;r8BU0HFo<&$oRL3*O@y!DjWwrbCWAmI%NkgFMDBqkw z8_ALr#OyUdeiIO}w?GxyG9tK;3}EQMJ-?Ipt`!Tr@k?u|TIm>Tzvg{D^}f?$EO=TH zCr)c3#}{h0{IO;Rb5f{h*@y7)v$g|+1;t^23#a|CibXfDtqbp3TF_1663Lg~mcbct zjD9$VTdIVyH91&T9zaf(K@p_t;wreiu&YJVG9ZV4y;ut%5n#6@8PVUnUalSzbBF<> zS&ujj`Q3~criVzIDt~Ih`SOi!nAR+`Ux%?&Hmgcso)LA+_}~;gS%iYzvP;smVF}f^ zmp{j&@>*6p_$`_W6)JdJ$&zsm0W6kIT|UJ(d<-VMN%>b8s1P`O9-^h-3$z*O*q?CF zo{xU&0#VZpoUBaD-!(6jLbyH|_B>yOCHsJ$H4c>S0Ttbg1cE6vnk`7$Fp82|6=f?_ zh~9y=8gw$x`iGK^1~o>&kHQ3^9_RO%F?4HQ*PuyPQX%znR0!QaE4f5>fl|I=iK5!x z#!P}aoeq3hSduNp+Ba+cr%_nq(5u?Pr@sTZjIUL2#Vlog1Ds)MMF-mAH_@@06rX^@ z*JvhzdP1=yOGf7#gnEJ0L}krF!Xa5o6rIH%va=67h-GR)Yg^wuyX?q-{y{S1d>u3` za}DObCJJ5|B}T)P(MNd(9qq8C3=$V3{XFEAz6#|MJkq7zB%C?D(KMcfcgo6_-2qDN1Al;C}*m4hA_K<)={(F;9A`;Joc~Y2@tQXWt>`5W=yZG;o_?m=Jx9A z`J1hlSsooQ*K^0z`W^nyo1^DlTi&cIm&LGv(4t5n?r5L3K%!0xVDXQ#`&_a#h! z-!eF)<^Jq4SK!ncKfzf16FyxFD)Haws*d?kz>Ey}kpinoH)vfY+Xk6FF3g5;UE$4R ze#Y_GQQ>$!yo9j`JG*BP;#_s8M=07Ke^{1<5^)#gT}tal*aDJ!3ck5zAEl|b;gzrH znNGTXW(>2t)CsQ(+%xc8{NYS^vnfLSSL|brTaT*qs8R6OoCoY@nh?_{50;u2_@zE! z;e4!T4HV~-;H6MLaTLV`pviI{%E*<7v%Gn`kWU2pM$4jVKfBW{6pHfrJPpW zCZ8pdn^G%T&ZbaG!^vXgazqIA`3S2bVDv95v?lSQRs?O(BYG(13g~_=Y?VYu_o4$* zd{8UAvZ>YNN3m!?kO7V7R;o)axF;%mU1x6rUe6_T?4?9$3)L16T1jM2&8jHR zrM!H9vAM4@nI#(9LJnus5Eko9Z3S@7Ud3d zq5(rl`ZT|4i{?J}!M7$c2R#^wtNd~?D8iX(6bsJj{BeZaf&_D`Mn9f>fB6CjDwv{C&C{vq)$1D)f z4hWupv}y*;RyXC`HRd8Yizp@8^L19nL5@A^>yVsn%LclJ+K{*8#a>w=Mdb9S%D0KQ z{AjSZ@;syByo92Aes2&?WgVJ^qJLgrXDeOS4?ogp*ZB1%9@(@l4<`!6&;G_d{XiIQ zfNChTFjg>(^HWX)MbaoVvHVSrXc%eVQW|g!%d1C={^wyj`Y=&8ulV?b|1IGSkALX; zXgM^X(>#iw5juP$L2E>>0T}sewRYeV>Ngq`8qyl)x&a=-aGMFZU0L9~R)sa#T^VDN zW&Z$NeO@?_?S#dN?8p?2Ty2H))}TRV>jcFYl3K$Ju103lS~l?6W^kA9P_1|nq;lul z*yVb^mC0eEPkZEPj7@*JteuwNgCk5bqUgMuQ*XsJk8&`zfoAyA^TZ1k;Vz?M{UOTNYNjz%R+FvCOYzmBv3C%-aRy&mJXYUmj$$`Er*h_(nNN?{8~y2|m(vV;Ug?F5Gf_l3(--eKK<7jYPz)YX0@Jim;fW{tc#X+#>~XUjD-dh>jY^5gn*ac*_o0e7or~* zneU!$T7i)%iejrN!zS1tbXp;q0RctH>G7vJMnfJMeKf@!(|UjO(I3wqox3+qRy9Fj zRz;kOlknyMPVhQiq)XmnXqw(<+Sq(Xdj&GOH}T8c<=GF04#Jx8ZrT->HJj#OCkJI( z+QRQmWy&&#z_-VEL?$?}ffmxM9!ad~F_q&mf<60CP_qR?@z3o3hYbq6>OXPO@ph*b zBJx!uFpyb8vcDX{{XkX_$`2w_VfW~X*n&5>TKLz5yWIVwFAxks_Z)xulHhjPATXLs zAf032V^gz2Om5zbIn4d`dd&#*xQ`OzR)3xlfp@XOm7(LlhAblkGQk$~+hQW~q0qOK z4brIcsWBDtVNPx?{v8)D9I2maH3mnC_e|6}ZoF##nmtmtD`Ji}1j)+xN|Y;Er|7}k z1$F)IgD6v~3D&$QjX%sRKMPlel+*u*O6n){9f2zYpq;*w#BIu-Mj!D?f~8xz236qbg5z}@XDo=cmq|c_^59;I{>JZ7QotLRMWw2pUP0; zA)jCp+z+sORt+J2R;^rnjPbD(yUQSYJZolcuv!u`&G@@%*8o{oO zxqCi8n`=Pws{`uf0go@>KhI=QY}UUjjEY*^I>xk&2vour{$Q2NV>C34w%AC?R4Wg8 znM1R`6TXUEeo~OB_AD=O%Dyb)9m1%Dmpo>(K*g5iI^3yNfA6ns4^+&Q#o$gQsMs4& z-@i0|JRi3Knxo{}&&ly*xL=ccO7Jr@9-utX8^eq82d%3wT|a-FkqIIup^HO^i^>T$ zzP5Z-a1~BYd~CHZx>Qb&ULzDxLsKqxAaz#DRUe*)BwjkSD0I|RmT}`M@46xHs7T0| zZBUN~t_=BEVPOnM{PAsFZ(TnjArXnvJ6>(&j)dHWt_@$<2INUyf0$^A3-D(i1eL4~o$Vn9A5v>v@O-5wW>gV;ODZN%dt~IV-VZj1Q z#V!NpvkPVsTzQCO70jGdm(Lyu!WD8B-pC?<@D7c%wOu4yW{6G*Iw$@T{_qa1=Z-qH zygUzYjje=*Wjv3GhZw8pzpGN_+QC3LOZV#?w`PnfcnJ(=k*yJzvM|;3nB)70Mens= za!kOmZeN0>)$K8We6sMy0@`L$xcft6cNvtmc`5?f`6TyPgX<3SU|lo!0-+-eBmq;I(TUc3liZ3bHod)K!-G@)7ETz(La=QTn>V9w)ie?}#(Uxq zA;8eZvkIrW#2!7O_F|xS>?7hRJ(*)OUm`5ebj6!oPSJY=eN`au3u`_?cXi|na}Q$= zVYX(Oe5&Y`TDWWY(Roi*77fk=lDdolXJftsTMl@5@Xep&mvJr&inYrSvDu!uI8VoW zn^buA`kaN>Aq<8htjThOnPP_7AEl^_-&y4NGDK85fxd82T(rtHzpK&KK@m`bq{wT{ zBWvit=_LPX9SiWuHPo+m%fVDG_zEK2kBfkxoqZ3^x{O|iETZIzxhan=MQWV4u+F#} ztmB_Dg!B0CP30}h@{4dv!djPPS=fB-)_gxanN3Be2mHa>GAe&Qcx-l*-h1TS-U0*! z@uKNfBS_M0y8v)u5tL27DhC5z%!Wpq2a*8t>Y+#l(dkyw-V<` z{otFoPw84CW}Rkmto&9|zia$rgVZfvD<`d4wZ!Yq%7QDj0@1k##W~5>;MAei#R-)Zt(@koTV|PJgA<${q!LPc& z<;>)#)}f5z(NO!Kbp=M4YN0(-%h~0N&ddGbW0(Mc$@WN~92Xao&Ax^c^)58x~>~W_!e{hZvyO1B3+>S^KPobTFc)}W?o#U0=ht!s-d25GW%zSglmfM@gell zvRN{vFoIS4MTx-T;)N~vYt=bI6{#HS>l6(wSPvmln?$SA)OCO;&Z~&cx%t9YYvZQy zs@A&A#l{c?tEr>=$?Iv#vZBCr*4y7fjKNEZg95=YIxluu4E?l0dA?EU^O~8z|BV9I zlIKZR@z1IO@s9%ce`{cKGIuaBFxIoOH8QZ$bF{aza5DL4+Za~Y`e)lf^;xYYOv12` z2dx#+<1x;>tO%~((ytVWEReTIk7<@HkW`fE((4_JB)Ldn(O{pVKn&+ez1@oS2uNw- zYPdQfj7$AETfbV51nBf=b!*bjJy?2sH|q{nR!+`w3S?fY-O9~2CM9KM)wmR)uJiq} ziqW;v&F16RIcYl?vTrCTkS3hSBoCRyqs?IdVVwna-|30~`Q?_H%ziDPifA@#?mtu? z;AlL9am@HTFYwVdjWZ^3LJXDq#WCZ+>lei!yYUU-TYF9A4_#ad3Iq+}jH3t4EUJAT zsd*@vs7835fPID(=LIroa7+T?_I|+xl*gpjelEXnP0Yb>spk?jZQN1$uKV~}(lNaR z^Xl-043aZgcN=?Y5-Rq)?36pusws{D-!DI`b)qb$3af;xuHwRj-)V-qfhvZQer$Eb zQw*&nqPKqbVlQ}8Ra3$VZdpg3lMy{$aXBD;n;?_|f%gp8JTw+XS+J+;dd3(4lN9_p zphy3{1n4GGLaPgcnytO^Z5nIHz04Np`D|Ix|K1jqjrOrl4 z{-p;r32TtDk|vZ1a=skTgHqbsP=&IgYNBTvIMg4N1e1#Txyy5`jW^I%b%o7%ZN+Vr zIw{SB@i5+jNYRJEJnlUmGjrIqfR(|S=3t{Mf4cg1={ACvL*MBTzVU7ieOR9?z0$1} zX!)aFwJ5<|w9%P$qe17!xSegq3|AY!pC+ETr%0!N`<;@%0-H-R3&==piMOBaH}GGO za0je2?b(Rnh8aq-A=%Oa38&}<{eb3tz{6Zwrcmr7<<~^d1KzPpCjE;^pQ_jO1qa>EBEWGR41(|=T0SZ zEIG@RX%$AIw8(%8iXcm2I{7+C#KJClUQq=8=YPICsY-&U=M9fCd;sR7nW3y_x zsr=PJTta}8X+G{+39C4zcl(Lq8FZxIRy^v$j$`d=Bs&I7Q}sJze6-&R@KJZ#v}x}e z9ORonvwygGz*94sd@7e9nrYjfV-be~6a@DF5xs2$Dh0Zg8Kn(HC( zTE$Is+-gR)(VVa+!uyzNzmIqJDgfgG^v`Z45zybn^ylv&ucx^NMvLJ=UZXgFC6*Fp z&o>jXBvCXD>h7BT;h;+*mW#v=lor*rSTlLJGfc0B&`<0x$VX9~%Z~(@rLx_Qf-*tx z+Yj`62K5^UJlX@u;mFieB+YKoR5iwkom#ega44w7Nf&cxEHT>x)(+iTI`m6~v)Hph zd;mqv&gjn|v}$UmIi3`jnrK0W`q}oumvxiCjxFDlP9Kcg-S!rSNqZK$2${#OT0-IK z9PjiBcV?`&5owYYG|?r)BU1-(y9Cg}0k8-sada$}jw+`0J z+BuT@)mQUhJ&UjgH%&57001670PyR7eg+daCkF!~C)@uZ)KzO}I3Bhk{rltnJ_{{Y zWMaQuKN3)ptVVDzj2LxU$2VhFQlbe9qPU6!O$A&l#Qe>%(e65GEP%iP#fiSDmGgr* zQdaKFFw+tlFqVw@ae6w%YVXp^u8tW$U!Hzmp1$9{xM0e_joZ9z)PtvY8;~4_SEHmx z+qF}j9Us45-wz#J%wkKBAs^M3!0B_%v@K~4da;sEW7CPL_u>C=?R{D2g`bqJMg#o!zoActt7niw3&Gm+LY z=V*YN+5||HAx|8yHopkU-nW4P8905GW$Yth%ybXx0aUX^tzTxHMl82hlth?x%2#-l z&@yMsL4E}g+TB8ONJ%CUQc3|>9@db|;?^=)V*B`*kr^n)Lu)v~7K;8F!Ma14c;OM*d9lt9`a%FD90)m4z>Dp-g&zT72D4$E_2f zABC#Qek~_SiQZW3q-`O->{x=(e`4o3PFEey%r$G4vYYOjD-*I|mosQr1t39ljz$QG)o-D=Z}T$h*=$gE72LQ^6ZK1t=Edxx!k=y zIGf&O8dijQ-pe{kdkRzu=bFo6c&g)UXH7@FpznYnSu~zh%mia#lPLj91uOqOpLVkM zNwa%4HYjcmHhfiRG+lbhvpTMjkdW>Luf-wDV9=0Qt(E64;~WyfGulbJY7p+9aaI6^ z$jdYvliCnaRkC{-1V07DZz2gnp=lR};`OHhsL?6&h^wzaG+ro$2K{p(hjn#v?u?g( zgE?SW@w8rT-#BQL_2@O^WQ};NbdE?Se0i(__91#-(1&%XQ&~P*-0&fn@JZkfRgzoA z`tF7CkaP|eHewOppe85@*Fl7!R+}fFL~|h+onwJ{pK!4of2JpwAg)udHZ|K&r4hHj zPjMw4&CxoYVIbdmir`!pc>5NvmN6VB^{icCbWUA(m#zY#vReVm3@(vL^faPgB+?r( zlvZ1b1y&SGwRxO@s~MHg!kL1|uy(qMo}jvB$=K`#&a!RjJ0M?g(E(!vzx7w-tVK!4 zI)wBvFK0=0rPga?Hh)X9M5pIx5x3A7RgaL>Za$E=`q~4$Y*F))Fqg7ubVc!1@a#gE zQQ-zexW!6++2yr!%biWl;Q)q+sCh!zhH$_dj!fmx0UWAzBDoCyB~vDrNn2B!kiWHz zm$q5MEjxTMH+{v;NF3v1hO<+Iw6T%(3eIbWb^fNI!oLPU?^;{N-o}cvIp=PNv z(B~S|My2yd_`ao3jY)=`ty`Z$e@qIWQ)c^pIkMaOcz-SZ2d|(M%8L(C`VLN(vp6s! zwV)Yt!ASg{LT4FQ!hkSUwX;8q7J9lk+4!xx`Zwn@Gu%P8!}H<{SNU7!TPIiX(_~SP z{%!IhL{ECipmM$ax~KguYJnL0N|Ci}NwS{5kutOJS`+iN_?aaJn9kKA;1MeOc2GCv zGIZ_MT(<4X`uXUazc#)-HrtgRq%z@7wgfrShbv=8`RR@HP;7|u8GV62S~}uDi;t`CNV>NzQ$JCUVQ}?4nHx5idgI?&p=!U zWp-pS_!nH>_CobG^UXt-LtWkqaU!OqzQhT*6wy^D9+!uaCw;uU9i1DZY}Nj12Ly@v zast=Yzg^kWP7h+B3^Fl7?BB1ib?vBSjG#f>{)@Uo%*3h(`J`onbYJE8_vcTQ(6b@7 zx$HO#C$$yUC3j5930vZoUt0 z=?pxzG?(8bYjrN#W#)>7=E#vgNuNNMdP=`^X7HSS5_oPlLWsIPqqsOmt1Q)mEN$3p z2{_3T3>s+vxO+B}N1kS@v1P*j1rIQ))ThBqZ>NBO?}84bNc%wF$yloa?!_tX$Y&M% z6!RFzSN+z0V#+=Iq)QuVe>Gxe5M&?H^PgAKE%`W^UijNT=I6xk!eo ztW#JjH~Yv~+j_L8u*!ddk^?X+y}C_g|JIn~DZgy6k!!}mX`!mJ*W5TcT_$l#Q<(!91XrV1FUK|1H5ld9%I&(lTi*hs zJ^~*XWPk_W8DiYqd^zSWS8VZokZsZk*Sg~0MLn)O-U`bMG$;6JkR}0yxe8oCIUvE} z4;$D{knP`KS1RHf$g3d*>*B@|J^ z*s9~O4Q7HzGN5ZrnS05S5^K11Es4@Dg0*N^ho`Pv)cKh@u3v5TYB%RPQyxNnzEPP& zTBC~0i^GRB#bb+!5Y(mmnFqqX7oZK5asOW0=v$A*5O^>fH zw4hET7cS*YA^E6v<5ThLoHy#cB&Z_j!$(8|wR%6N%E(+^3xmix*D_|%ZAS~5t|aja zyk5ghnh%=YM1+jujE_X5V9$v@jnMbNO#BX)K5i;Uacm`gHIbmwMgEcmZ_|H#uUo0c zW4pVev6={9Cy~kQ&vVa85IGqn?rd7?jr zi>nuC7vgsRh1!aU@nQ2#Rn_Oq-AzvW6$P$81`@bXAs^^$n9$pj6YI$hO* z@}8Hq!<#tMCE6H7MHPPkNe5E^aV30{ODUW>p53Ivhl>sS^CtfED<1MrS-BbZwvX_; z-7)}n!|+^S2bmBNf4}P@gBBP4Ss&Xju1GlFY)n$7O#v1|eblNyuIe>=@eN+bu|=rj zj|>^+kaSX-Mk?{?BC557g&Qno=3^Oh9)4OSe=vy!XqA|m@=5cqUak)Pe1!#;1z=r3 z!71ETD?n0$Q2)lqPZ-E!5&H_)nHHt(WB6hSdI9Zbg$YVQ^EVa{Z-rR}LE`(pSNul- zTdv0qu(bpVYxaUZ;(!*Z`sywQh#=jKFWB|psp^_x8e{IGLWw71SVN>k=1D9SFk@+8 zyNmaCDVTDh3ZhVXI53{+ZB*VqxEBxK$D9fLjzv#KnV-NAxw~5kv z^%Wj0sWBLp4n(>4?yI~?OwB#bb>AJ{6?}QkqyWacM#yJ@q-PbIo&|;Vqx)PI5}$vi zW6N~kRki!TQmhgThi`V7cVhYcXx%LZCzEqntCu~@HU21Uln%Rv!*CxsI#9BEijPKU z3m#?rM2l7sAsf?n5GJSV%QRB~QAA6;EKtv_wDb6waECyRu;%5qP2Adl){&WZcTkhw z7u&yO6-LWvPRU>&QGb^)OL7nTOIF@_F)r`8c$MO3UQ(eL4 zJ-|$%rfcWd88oRG1GDwxrNoU{NRzn8PrRBIInOyTKg?1uhCAcc{nB8jm~u|TCv@!# zVZB4)S^jmi(Bo5%|GBApchZZC3D7{G>m=nzH)K-n;I#=aoQFnmfLK~ zTQ4#}4Jf=Z*(xr)f^X&)ZZ_J}q4!XJ_FoW+eYB5#7@Rlk@tc)*13@=Tloa8lS?xvV z$fRZ3@X8kX<>irLAu$VcJb+(3xDYWjDvR533$n~xB?dzQli`X`j{)?R+c-ip%EjjE zE5bw)hUdsqkk~jncBVuZ^?^tpd+ly=1f>#VaHDSmx7;p>nfFJC-1E-Wp_vhI*+C_T z#@56V<$y=LrgD&1Z!G7?bFhx(a_G1m)B(9?LjMbSo-M8h~U7l)5E zJq{&ktP0Y`lQ_hzn$5&6%y)sM4cC$tZ%cb5MV=okiLI;1%fcbrso?e>>fiBodFNuC zg3_k%OFyqn%CH3`E-6?#06L~OU|VVTRgh)fX1ghYf6b;!m+iTgZZd?`TFm;mbepxU za_~b@ef>$1@Jwrz{u8ka`wQJ`Q|s7GKBK4G6B)N`S4~^^)^xYmAd$-Mm9N!3ysozv zsLS&=)SKIt-YE9__we4-x2dwJ$ohHB0;tgcu%Cbd>drcos6PLULzrW z$SS9XbfY?$Pmg%z>XkXt&D1S(9FrKkbhKrCGPJMMtLGjaD51p@ zx>lNpNPu*e1ugYWC|MS@lO`Uq*U3pGOLG~g7x0vzRaym1c$RONQ=I1~YR`HTFT95* zWvFwLKOyO&qR|qOv=>ewYcx!ph}CbXooqigI5hS`EGQ=^Qz+O^S{(~nH>7xG9rbD$ z53jIUnRBE1e_L?PwxNyS&mDgDXyhAb49shRs;G_NnmUnPm$Yueqix=TPVO<&GqV;) z=QJ~D^>e;m^|w~OKVNMpBi#j9*D1IgbrT6`#cK$JaHiEEJA?|tFJwpT`CnT``3n1}=b^-}Bkbe6;of!{bPhblSaqFo=$9W;Ojwy!0j ztUTI6JEO@h+_YBU zx=Yj=voSPcP5XMkf$CIIct&3~ZLU$ek)j8Y@b(5JVH!Tz>@dgk3$6rIx7(73b0%%Q z!kL7FP;M84D)NMl$rq4AM&-TfvaIf-Lw%^|Hn0jgSK{wP+HWVm5@2@Z(hM z_uTYhiHLKjs8_h?DGTSsopDMGV+``|E&iz~_0_f3j(D?2caH$wm(fA}9x3VIDQ#P} z#e@li`%ljdDw`}=8!;AxDgFHjr9?B6h<-Ijf;a4!L4zWXCc}`_vUr!X7=rD@ji-em zLZi}&BOhb4Yq%Tv-u)j=b_&<*5_-X-BX#~fU58L?cjg`mI3uQ`Kjjv;Z2ged+!tcV zc{h=FZ!Aa0gaV5`O9-)$-T(dIxOBOz`}1$2uL1s7g8P4-?Eh1+98H}5zf0%jc!fWK z42YwP@y}mcOos-DK zi(he7vgkJto{$6s z#rP|R)gq&kjZrD8f=CNNmc&&s&@>ewsWqGWNgp=CfT4;kVKu^LogN{~+O|wz$$jE^ z%pq|E#>ijIxE;!A6)7E@R8wFtV1<%1Yr>48J21O+J)3UEv%c(or~j&nCG!4-$HUhc z!#pjh;S9xi)1Rvzmg$ALg)IfaPxE95;p=>NwX{)hPg`~ExE{y)Vk!>YS~4ml8gX7w12^|B6vI(ctM_l!1+ApC8V zGP=8i6$1<>%gd~>By7g$K6`h^9uO#77I|p0@R2Q8cFz(|aHI(I`yUJSKQX{`Is9XL zy}uSiKWi(GFR;-sM?;?vpHr7VeAqq!nW4w6=Xfv{Wamz3^Z$#nbLtW-Y?gG{wry9J zZQDkdZQHhO+qP}nHoH7^*38AsHy7vZ|FARH8yOLgbbXovV`%2I!~3XHe4OCIJOvsz zsPmLY3wX+jHbvV^3@?pFnOB=(1XRMmx*62N^qa55zoi4kPI5_m>rp1kSND8G5#kjs zgtG=KAX|mSIrD869t2>^HGc(*s_v_eArg?E6Y6we+r0bk{cN-1*yu9W*v2gSL3OZv zd=usE?YU#KanEWtW~HeVf+`@@g4a>FK5@em6h$oD1|)=sQ;gka7$MDZN3j80s5WE- z8_$?x%WSo7CC@5EGujt0RPcC8xX9+YG4);c4w54>278Zx8%SBOyc-tu7Y-5b-_2!C zNW<^ls>n$Y%39z^L4i1(nX>SbeLR@iusjC}pJk|WCppr|rL7%_mb&H(T|7bnZrnwq zmFg=Sj5y30GwsT*O;>-4!w!RN-mpvjPq@9}H=D#!28W%RnHd>Gu>N_5J2?Bq$d12N zWkQM@j2%|T ztw|ZC_dV76!^>t0>p(gGVC7#&WtON!1qQtHKX7xfA`^(C-6wgIXY5Y_ZnkSLq~%RT z!#2U9fOjEji`y#p!$8lsiY8a$eiO+OmE3l1PL&zakRkHUB3!aB?c$Qh@e zqd7vnF?K-OSePj}S8J-5pRlwKh0d1-iM*Q#+(uLn!Efc9Zpf`7u96v+POR=#*#Lx< z7L;rt##n{xRK!X{PhXVRYl+tbv&V3A00K`T$td%uz-o`Tn4j9`j7USo%F7qON1}hV zR2tMy;|hyhFAIMZS0QXbzV{MPci$Ig=}>c~q2ohtRHgI7o^7KV=E09cvG@Tw25~ZQ zKFsqV4X!1jJgY{~b~~JvmaINi%{!DxHJ}ds8&NF>z*kcv>&KhZYWije$mb*L<1)T# zo5A}Y6Y7YttTY3X0f;#3XGpVW@$VeC$)fGeg)?sIEOTE_LwpAy(|3GYh* zWO@W&L|m6(XY%ULb!Di^u>t(kEeP*ngX z${V9MUigQSYWd$fxVPz-9QwnX*dl$m02lnmn>nQu_ZHFG`RfzwFu1eb`#{ci?X9in zGDRTyXjNuo%xZeeoPdT~w_L;Gi!ANNbwEec<}k}e)xA$Q=S`|g25D-e6g+C)cE_+; zt3>Y`|Fa{Z-SCP&&Say`@y~x;5IsEX1c%W8004h!PfY*aNVl*xx6%9mX3{mcrQ`n& zaG#vP19hVuj2iRVMP?R~Zv2&nuO%6kI9H}Yz>ri=1PJ^E1Qq}EZ1da#@{eo<=`CS{ zo*Hz<@A4YY`K)0-fzVm{;Hrq^1alt_v-prHpbl4-(VzyLNT{XCDDPhlp)Xs%Bie9! z(5TH@>n71b?ud}`jY0cB3?_GZ;18?lidXD)&U(c+b-h!{G37XQamTI*2lm?t4{&Ys zL_J!idu*j=j=;j|oSc25 zibc|OlaD#uXUYrPd1MAgqA5)4lunr2sdh`jISpSR@Qg~opk#r>Cuc#;J0Ga*0(jqj8plFAQuv zKQfr0D;Y&%gvVv&Fg;x?Jnskf9N)>!#$#eymrp>HSDk~_T^1oT0l?F5l%y6Eod%$Q zgGgmlh`*}+N2{04rjR1u<7Nwy)+KSWDg+f)ct{_DeVFx2 zfx_N`F`lOl5f~1*kI23X-(|n05j~L;et1J-PP_Si>OoxwS&48Dy0}~T3?^`@og#t? zg2aRuicD99aaNHY0PBqgGnG4vb&Dl+T{Yv3yc2;|_6QMdiiKoEN2eb0rVp|H)uMq^`;$d)x z8dQlrw?&?_c#^kla&B`E?T%*DJ^StlcA5{AbB7nlhzI)ZB_3Q>*NoR$reEks}G z+1k>4(z9!$jqCSES86Z2i+b5FqE@Ixz}ACFd0LW|>A1wg*G!2(jt;y$o&3zQxoP71 z`8@iq6cMD*J{W#AmdV&yPQ8NaL1P>X4_-A;us@4y8<0K0wIe+eHE^yC-v;lKB8|bM z5L2witk1cmgfBy@3N6j#yGI_U+?{p|NDM|4AXx9lp@1&JEAGb=we@FegSwd{+a;ed ze&ub#8o_)Mw=oo>=Cwq4r+>ws2DKx<-$u{ZlalkTmt|$+-)aAljMh_Zn+X=)tSrL; z_FFnQh!;wcUJUI$*VuP92e5U7pFhsMe8VwnYGV?O3u$T8#Ue(NqwDbYImAn9bR7u=>1L)qn_h=gkUT{+{?t+rfo1=Gud@{WIb4Lcc$lDD@eA@WdZ->Go z!V0A52t^jvWvs3`X%@)(pA2NF=wnosfQfq~`D%O%7 z7QBBvoJc){AjSx>Q}_pHW>*wIVen!#at)h8=~f`##5gei5}yJiDH{x5r7&&d;@>L# z@Syo|3il29t0H{HnY+DV2ZelA|7olh0;6;nElBb#5>x7Kjajgg!`4!k0R))jUxUSA zh1xh1n1+}FUU=f07{y<49~cEaI;f&sxEE1ywBAvun=~7*q;<=PS|T;TP`*9I1X;Ut zi5Cn@tKkS)kulu;abtb6<@!pW8og-*Xtx~$TR_Q(hWvS^)05bz*SB=zG z2LnoM%DR$ZJxN`PlO4v0R{t9n7S?l4^2APo(s|%U z-xV*rMZjhRUYaXQ7mx~^q!{&{$Gx;_q$*4XQ5KRAEo z=F;C|>t^;xknqgI3s{wd;Q`vbJ-sm#oG<&=#}IdQ1IZN~gY?wiPbu0=us0S+dP^L3 z@KhE{N0}>Y)F%m*Tk?f=)YOrLu`1HflKKXtNKGmOF*UZm`?R4do`;H758wj>mkRJRvSq)@A3i?8TcrAp%7 z{tk_a$aO4E=!*-rciAZ6li&pC7SY8r(Hmt}Kvn)qQ{ee7M3)az9p{~RA)+-c*xG~V z*jG`;)@Y{LbRI5$Xw9Zh=rPvo!MD%r_g&V{AsvI~DO=daY5AoJU{%{MsbrObopF3uN*RaZ=Oc%sBigX>Q4%P#gFr)Tt;MvxF`*%XD zs0^@(Xv|l3hXiBQZ-$(S!agMB$@kP?yo$ca=XiTTKkJNuAQ8R72qc=GfV>DQnezw& z9aa7!o1es-3GDyNB`oxY56Umwi}MkQ6EUR#V|T7BAiUw3Dq}ysE`->CCIe2gzg)*< zX<02TeKo2S&HJISX=2vd>GS&so<@m;I6$#h)|Q#DD$(OxTGFrr(7!xWzMj_YfLF8F zfP%PoTuJ?sQ0<1Mj+iWg;MfnD=eOtAyVU`1`#Myh7rYrzS5y=gwnV8k`RC8Jr9A=* zX@3X5ej&iUP##X^fWsab^4Wzpp`F1{3aFIY%3r)xTYsc;mD*o-R}M0G$`!~jnz(=f zQiH0U1bi2zJRD0Me+f+mD;LoJoGJM|n@6ndJ9Terz$oAErQuEcNwqcl;Xk52GVm~n$tunl;HXq8RD;I4{#MCL>)X!P|R6|O<2KuciSui z2G*%F8ByI>VW0R-SmE#`_)@W&T5Ja_$H<9hJJ(F$-xMki{T2ECs~X#;wApaPB2g8# zA(wmeUDmuB)*5IJf6D?S#)_=bf8-QjOC}&pA3wy(CzQeT85?GJdo_1wv541dfATW_ zy%em*fP63f>7U-Fb6eNQG@c^_a_cSxJ~p;_WV@c;v`SH-vJ3D)p8LqO*cbzd!DoVa zYTCo8fJhz50n|hfrY!|3m|yx4oyx!SHJ>gYXTs%;OGdRyy61Yw;Oe1VW+Ht?&d2HRttdXGsh#2{otOI+~3HML$o%aD8 zv|n`^tDC$X9(bI3e_vBzg8D{8QZj2q&%^d%UxijC0aV7XN=Ld9LkY`5Ox4yf{Iibfx+np zg;u}G%t>WpT3ny3|3VU>T)1mODsEV}_Em3%&b{7qMm*M7f=L;6a3n9Y+4l3%RfC?3T6L{ zIONEz4yicGiJjG!Imnm zI4VXLuBGlJhI{6&+_FK~sZ6~np5*7kQi=W`@4q}YWtU)GyY<5M>;Rd9F9qY+Gk%z< zXzi0P#EJ#>C)>O>6x;Y@Rh#4qhX^^&lxChLbjC{mR|)#*WG5gl;Wd&GrLb=6_|r0^ zkvZ|3*d9ygHO-%hsi6rPnaK&Sh``p?7mqXzDzmF6^eS5#Rdx6X#S*dtgw(os@8|Vf zD#r^0SiVadI(NAMym0~82#N;Nmr1_T{z8-T@q|8S}>8 zM;K}o63I>kQvwdhn4hKg*#p&O_K4@`vX#;&_Yi^RNr`>F!M$y^)*Sk-eu5JO35i~V zW*LMh^kS(i#My6dZ>Mw@tqGz;Vmpib4#*+BwTN^Wz)(hcGbh@fIBEz+7^0`;M>o;_ zCp|?d3^fUEt)gslL6USDdd#qosK|a|I6zy$6iY|Y*cX3b*c&R@ELLW%<#e+rG3Fu` zI?xqo3bh6Oia3F+f6>jLT5%J9j<%Qcm6L?HHr!@nfrcMhz39M6B4MXQ0=HTd%flSd z8ZP7dVvofaJHn3RaAHf|S*YYr$opTaXR^Aph(r(2tsVSG@M(Xco5dMM#HdTiBZI2| zHo_47u%^V|%$}=xNUF8c^l77zO51zTlvuoQnHQM5_(0hLRYCBA6hL8y&+WWZ<8aT$z}I;1buUpAEgrJvZr@rK!Q~qAeoz?Jm4lz8ipjMenQaR#(OQ zmK4S3y!EZ!8{1(z@HY>@CbGnU{5VcobjGFaWVbMpwU4g3?Xz`t+}aWEU%c0kE+$tDxP~8p*#lOh!pDrTaXYg!mX?HlS5Or8-H-@ttKYkWho<7>pe;`*U5%fgAcHM~9p&n(R?J^;1E`hX&-~)scON9( zCwb6;qXuvB1Fo5jG zU6tEZ;Lopu&*oa?Q3a%=R2NdX6*3BBpQWy_)%iltD$FY*m#Vv*_=8$A&`FQzuA z(>W3(DC$S8MPhqr0sgNGUZ7&|9%(wUYechZCvQptHkuKJ&mBejxs|PbJiR&k&K78> znpBXyGo2RLaDvKtOk-=kadTEb567vUWo;=@BrLW+Uy&~hfu6cUq})OGsN*<8%$3zX zfaQiMN0(2p5R6W%cvd$!5sRoUdGNXz1HP?N)R{-Rc^zihLW;hC_-(1US5INthXpZ? zc&g2Y1-Q^?I9tRyg|xkM1=uH;!0=6^u&3{x!q~5C`|uq!Lm{Z*#RQat>_pQ9O4c00 zi1G>!F~Y$A6m8t1_>dD}Aj1L$9}YN<$=~gxi{iW16;tW1horycBkqX{(ee|jN2YUd^}SD* zz?4tobm_tKfGZ(O>u)wwNZ>Lku+A?;=bQYQFOQ{a4-wWd)U63#Jl&dJVpzQHJ~e7u z3*Amp{)H$PK5{ca<~Q8koESU5t(#U#)zKR}j$BMCt&g~=kwv8%+KkNTfLqDz*Nc3_ zbd~2{9j2Ieo)Fw$$WAUFZ+L%Lh+gX+H+HK=rgp;63Gr}88mQTaUh@@@pp}|0c2PqD z`+6hMb0-nKSk<`OPp7{H=Ox&yy^SFwLm23*>Ln{y&Ody*O$KD!u_lhF6w*3@TZ+@(!uG_g{g zM>*fze3{s!6%a=B?33++ctfHj7*1e*o%I}|F9hI5u)5glD7Xna=Tw9HikC%uUd`Al zbJ6JMGeq3(?4cM$4~KqKlmlF!%fCfG5H=&wtX7mm(>++C)_PDrGcz zUO#7=2t?!SW4|42ne9V80Qvq{%l?||^n#$}WaUceYMcAm3h_*{ViIcID2uKnQVk~u zk!?c5-n>==SJgW4u@2wp!ZP^e3~BAxIh;~~Hq=_#1}Y1Yd`20Mu{#G!KHBONtB-$x z{@){ABI@U8EieE;(=Qr;?7vgR|F7em)PI>5doI-wb|O*6b_^nQVcPtVx%|+=ajUvx z4eAlh%#Z5w#r=yCoFl(qAM#CUj2CrR(4`*#vF}dViQ3Mg(#r^Qy6h9Q> zkfojv!{?6F-wv@e$yE3p3(n%{&*!vB2^=_Sw|6a}F5St6f|yqbRS7LM?}zY!HP*Lu zMCJ>_>Jsd(A5hCCyU zg-Wkf0X_B<|B*I7skzcCM#`s_HS0XmGLxaB!^^2-pO}N5dwNAQakS)2myXgVit-X7 zDbp|yHl-C02_uoFnnJpR_EeQbLS+YdoNWFn*i`i?tRRN-M4E<4T|)JlmsqJwCT?U1 z-FQS#spoUIBuON&`Uh6+JI4(vU#w3yLgvk>4*pj+%b3cVLG!VTkxP^lHG111uhO*j z^2?!li{AC}Yj!e*dOnh4o^Tq9F1ZI(7*Tp$Z!gva#w?WW?-)M8x4XB-JzVI+gHP-B z7K{a5p-j{J!Z~SXmF_59!^PT9+Bib|K9H^$+vPTX=hu5n^lw8pp!YyiP&0z|zeGzx zft;cM9F^jDihdY-QMoe!ak&SSzuu)2w?RlWveD~SNXq%*Zu>!2NcCbeWYxHcs%N%| z0lv(Br=%!S;Wq9Mb8s-d-Y^-R)<$EHLQhU|37$nDCJ6?hNXb|J_vE;*`*PyU;KR=t zV=Pn+;7v?@(go^%M^A_brrWIbAQ{avDUVG>2uxXnIO%UoW?8#&Wcdy@F4#>(2NTR< z;4|iOS^ARq{&3}6Nek9_PbpEfp@;y5vyv$kJR+TBF1i=p8!MvWGg76cMjB^X;EnT5 zFeD`0N-ho#HB06k=&IqE%1h;#%5Ws6C`Fk$Jqn(!d!9hSy3Q#(A(fW84NEPzTbJ!k z(;d!~zf7~Y_4$_3E0!3;-4~ROu<7X7LqN+%YG#`siq6WduM>kR;+eXNh!(^bnyaNV zl2$?9%L28&j$~bkFLzRc8!xthHszbe%JE;l_)dpCKl_=(1sIubUarrtE3vLCkslR} z#ZX+;XtN=iIp`aI>(NP-TLjp~{Bh^hIr* z&_kh9K|8A3pD{xX1ZwP`4IGhVFwMC>k^{szs%t)foDz||qEsqYNXNV4?u1v4(4 zW@Bg`Qu|?^r$WxutwryW@7I=;KR3-D@FQZydB*5@|N2H@plO%aUst_N6Ovo>HBL0% z+6yqT)VCD$V>V&(Cx_9x)i9?#aKd9b7q$bZ6#B`o{_A#XCN zQ%z;iHchHo*oG`e@EVGRwKkKun#W|42S6UC+OizLhIGLX%3}!%1C?#H4$uITkVge| ziewG5CRn+&P^=KYYG=i__BX2L@@@Ju^Agi@_8ltz!^fDj2cO>Lb+Wect+e;nymp}% zhJSy_;PdUpqNOWyd*rxn2Zgd{0SdHej)2${LDjo-7u<={^B_hO;2|A z=QBAsO*%)Tm$RVDvO71*5;t7(&`=h~;G;=5AmA!5>kzL3qr-$6+!h{0HzBI?U@bu} zcB=GmthjnXtyUB$or&UqfWXjyBGq#?NSAYCZoR(BvsKm=*8l7)8blDXG5L!zifQ)Q z?08sOw6n+f+N^_>YjAnYg}HxZl&)T@1@Tu~Fd0}@8+Y0;Hsb1zFoRTDd<*SHJ`u0@ z9whMMsIU4)VPrh60R>sKNP7Wd<0du6q&Vm-&onv>_ADY)6F1Q|!aPdy5%$RWc1}s1 zgJeiY*8vU?BtB&Vd-GOm>TGv-Ra`t@+H%>jvR1*Y59`g^cSe{$W(QcVQ&6k?(acVI z?+5|89m-ClEl*Lsq-G`H4IwiBS3@>bH-9ir9XscYEf=GiXPFva9WQNR9Jz<;oUZB{JlOy^_wVCzM&WxRy)Ck7GP zGHPWE4M!VJWX2&Zcyz%H*YvofNmF0-!>B@q)ie6lA{W=ci@$Mp|6`}{1Av(yoc&2N z8xY%RE`<6b)`ohHLT|sGaWpe6{0ij@#4FPJDJuZ(`=gawE-Vj_<6NipxYxuS{cM|x zlO_Bbqyup@LXw@wpOpE<)NF;-1ll(S;*d?%LdCjXW;|X9j@!IG5_l$3rO?Ez6ZBUr zyul0RlhC+H(wUz(hSYaS#}T`gD8XC+=oblvM9L zCNow&Xp@~}563e!G)-9xamYFhB`0}lhEkeFiTgVTETM9kWRi@%^^H8^BO z&41XZX`_^|dAmN8s2izz5TF|qfoSJPWMTU)_X2)r9QAD$j0NO}~$9_u7QG1=& zh$9|H;^%e)<6i@gWeH)TRkJ&oU{BDBWbh5t|ezQ_Quo8DfEA?cCV zuoG!y9D0}LJm0zWpAu!_mQ9W!#XsFMLj`~Tox|~lHPOx+thM&##58#_`ht)87#A6^&D?{WCeYE5X- z-sF^a{Sjal`*8IPXXz^39+lYp2VUiovJ4I)Z{1ISa;=S4uFetDctklwT8R=1n{fcM z{fGe7o&i+t&Hmj_W8$G?;+0}>hd`U=|K?>e#hbP6n-MLr=`nZKaPHuD%~FkR+oUq z>s*yvJ(dtIDFD*g?k+=P=E?;+NHT_zs{u zc;4k2qA5hRZE)`H-Re!9ci^FfngaKDXIN`wMX962EYs5c)%LV`x7fc^6BiMsWQl-6 z>;xg_+8JaR)TL%M+;5pNZEL!!_U&}hbTed6mzo(hdhY~MhZkR;WyX29sbr`v9c>0h zjuM7UD;HIre=Zwxx1FIi84R>zq$Q=^NIQzAZWxu2#pRiTs@^b zNCpq5)PH!%h1rj|k74j&xao$y-nbPTj)ts&*-g+?%=~FhaNMepgTReCC*J@nGzdM# z6iwEmvFzih$$F2;;))3cPd+zy7O|48VXp{kGROK4&u|;5qEBC4`rOL3+|ZcBn|Rzwm^$%=0TWVxxojg zDn=WzxO3~Wk+|8NDX}8fg2tW?l=AI6cVf}YAYHNMnAJVSOQ*o@B;26_(vz9XoiE1U zOR_!eydiLih!W%u_P?)C#V%wBh1tFOuJ|dFtlQ!u)7$PF1`eK;j55}X$Ra)bjgvPy zLK{5a+i|y70BM)Y879IgF#oGoybD&B6_|8<@*MlN>o9?8#s=SSUc~(ylm}1Xii0Jk z%YLIsOg1N5=K10lzO;V0bJh zE#N|%bw|_ST~z0?B(^Y-)D5-mg(sCJGbm`vM*ULQP4=}B4PWpXuPg_}wnKxc`&w@_(WozwX}O?Q*TY{a@HD-&Z{W3i=kp zN_-(b7&G;ZA}v2M%8V*p1dIS01x5vp1#!+AwI|=*cob5_B&$}lR^BD}aN+G&Twm8C zNKWcXTvB-In2~W19^d!R;Mkg7ogBTKg2v2NTbrGn9i1I*@Iqv#kCV^Zb<2dkfcb-QjD{JtI+zCK20atwicH#F{;z-=G_m+n69KfkD>a~&Q-=3d>xrncyuusy ziblpDtW06=_kS=;GlSUZvMP@^r;)zl#OF;QT&lH)Fc)KzRvqpkC?l}B{gO=W1Ddn$ zA^K}mUjB@Vc&L=Dv8g4;NKKdbj1Ma-g*jl!fK9EXJPBF;^L%qDxiDsapB^ilaH4&r zonaXM#lb1&ieG9cpYnh#sfFu|*;Y+J##W>~xGPNrnE4Aj>7c12W%m(}Iu5Z)B@+u{ zP(^x${&Ab>KQjtVaDa^%OYrN{$qTZ4=V`#kGC>H>E|Ct7Zmm04%cPmKSLPLzv{NCY zMyqDyAm4*vMHT1(Ib<3)w(a#k8FRZLO?1+tK`@Q^Ljgy(1R}Q#u5(wy*4?Z>DP3jh#FjEyf`XFdcy>GXziWCCx^7C_z7_@Hy%+KYQ#PZ z?92u%!^T(l<%EUn%NM=y}gRIZMVsZRxCA4yx zt)--m0hV3`GCv@UI6=oW5rdS3C=lfHXh5>0;@)aS;j_0cpHIlcBqBRoW>Rpe3a&g37m>;q|4e z1Dg@!JzL!~TVipH&>6I6<{M}pUAC`=E3{UXPP)l8xXTVk+0lR4O}4at1E|HSei>*P zH^wp0At{I0HT_S;I;#t-n{dkrrxh8p%cAns$jl$hxM?pfC$j=l3+^Z$=R570Rd)ef%3sN2D8`T)ufdnZf7%M0! z&6%>#_554s0l)IM`AL#+3?S#P*6Y6|4r*Li{}J0tqxICg=}7Rk2Gvf$f}2+2kd?Fa zt*W&#d$yQ<wW{~Vr70QmuX;w+TVv^$0CHte>e4Q z@Gpn6oF$wxdwGzD_(#FPJ)1blr%#|60bh_bDUFsa3o;iq1q`j<)YtiO({FQ+9~*U$ zwaTyiSH?Q~&An{Z;+}8wBzBH<-2}Ism zsm^;@ef7T!-Oq>UI2fu0vpW2=(NeQD(9@BX!NI&~T8!VKmW9vnj{3s6(7aOL_$IImXHc4>j_948?s2RflL6+FB}cN;wEGPN>EvA>RL+pH9EQ_1EuU$P+0r%IoUo(uyfLTIC6FUYtp3SIb2W)Ev2)%0aCUsoCibiN zKDpQ}@NscogI_(F*|Bx^wc#kSR9P+J;k>x{tx5!V*`)4&EUdY*R(ax)6hfz-#!FWjf;7 z?nb--Lc0jqy%S@dbU-Vnet8@o_d(jV2YZYFD~X0jcULt@q&dH$c{EB=3dF;pj@xbVjE0SINbmXF zI`_tD^dHR`kz|q?yU<|&GMj2la9RC`bs%)_!CNyh$TS<-_b*hD5fAA-W4)`S?c;%|H>kZ^XG2^Sd0>tVL%>>hB8w9$_oCI&${tYYF;b#~ zX3|5?`%I-13ISr%_csr5;WXBT`!?+|FF-K)#D%wXvlw9g? zWRwj9Qss<=4r;LsqRm3x*pX%d<)};dxO9pmlYs36_80o0nWZ_M^8ROlMQov6)SX1Q z@mt(4HER+}Wj1X_6Tm3{dA8)PFrZ0Qih$byl*iaTX~*sg-HAk#CILekF?iI9l|?^~R`eDfJ-&LJ)sqaKH< zXN~W3AAEjY3#WCN@J^5s50hiWov?`3ZD{~n5GA>bS0K5x4?-Y;wj8W1A0=NZjZMT- z!MT19dCH}P28gRpdIDpU>@$ZPv>HTV{6L9JHK|i z2c6Vch(=c7+aK5nGvbiR!*9z0hbLZEALQQ;up3{+&x8b!g#-&3Ymyu&&<<`5`o&^g zlm|zGDS3~;7bMBtQMaJhkteE?uV&`7&i6N{Z~kI|p;tj+xZZ$yLs28YxkYsj7R2$@ ztRBW4@q)mBJ%2Dj2@NSF`X?qQ*N+b&9bep@k3uziXs`|oT0>viHN&T_*Juzz5_pfF zfw0q7X;p_u5o-)4;9PNQP;b*xyjFirR}fL&5POQD3dFD*1o4or&2pX8g#ZEe9h9;( zf{SRFpQP+l^&fqjvow&6+m6E=FwXv95ru9OSJf&q)QO@2g01@yRxuE69)(@ZMzyUb z?;mvTT)*2$n>MLLAMiwxEObLSN8Z^R_xh~q12MrZbD{&kuix%-f3Ag}6|OI7M5dEt z;8V&b0S+LBR3Z*5;C*Tk7BSqDIUpTy-N67>K(NOzijA+`EXW<@M(i_2r1IdmXqfV&2{~gW2U{Yd`6v`Do-K| z|6ikOWE_Dybi&+}^-#BeC{XX9NTz8A_B6x&)Fya%uAlHYuvIGBI!nCeh(C!03>nX6 zQ*B47!U_a{X_P%I7)9P?X6YJRW3zc_nx-e{gYzF0bo-hoxa|yqpVF9-%BD#STV&;f-~22xfN9%_v8)iw&Ip0ESf ztbQmA;3Z1>zY~m}qYN@}#hsw3Qyd@q_puW)WyA)Yd_g7hmPd-lv^)w65h>#yh#L{B zM*mE!*%TrI5gSxsm!^gy>d=nagwx_t4MVPogXLL(?<9CrTze#^yo$ld3mMp4uF`7k zr9#xb&fcpumg`U6=(wmNjO|!bG-$YuEAeL2{J6v2h`)cz8eH(l}Y z&Rlf4Y#v21oXNHA;(@a9Vh$sD1oZB_NJ3%7IbW=0)|fp$MD)dV9Gwq`l9fGI!7_M1zE%m4cBO zjS>exlLolx&;*7gr_G>6LV~&#I)D(f3?t@InWrGptBpvfN0S%#xhBkcwjj`hTCIlgcHb8+_X?BcA!_n4)TNLhmyYdt@zItZc44(?J)pyugqk$FgGmqgy# zu>!4M-pH!V#H!>E%l@CgR??lyp;wB1Trp&>@Cv-cNS`=qSh0VTjwOTkhvbvxv7|bx zz9#>YJDsXcBTr@2fAu@dYN}?b5sV{Z;ATwb%o{V2B={;lmE@^d53MggC_uAiK*C*4 zEY|acBGHLQZQz}I?EXkVsqG>6^y=i~X1ncvGP7gj{9GuZ1}3Kf+NGPvt{*2-g7b)Z zaC{>W~$ ziBawb^`yy{nLMOEjGn=`40ZX%eP&R;{*TO;WZTZ0iv^ywpRoo(yiyxq-Ry zGM0aS1Q$A?k2Sb+>DBjc&OO!G$!YeHrF0&jcPyWGS{GYBZ@pM=&6cC}UA^qAj$H~5 zF^hT?Ogs^z~q zxIX-kZ`w#rSzv&Q*bIo`of#pxUu#ZVIAc_43m(OkLs-^DJ*dP?=!Zv3L%5_PUn2Gz z--e49s3!I%iZ2CqTzO(Z#3r69xpf^WvN;muA&LX!Y!+kTdmk&Xd2+^na!4rZSa;ou zRPq3b-aZ+!OZw0o3^!-Z(L1r*+HFHW<9eV}MLa9N^(1FjC10GsQ${h0hO5T0Rb;1|zbu-r{pT#BvUz@KLMpxDT%BS` zsS0Z!J7r>w6!2nO-yyVJ_B<=SW38hKKUCCn(n)k>{`{yt*?|yetX~R!{AORYh%oVo zNJ(VFB6Fef*j&;-K4ntT(}|sq%Se(=Uam8LmMBeHP4|M;GwXdOAJ2m7(N>*5y&R@> z4g*ZusK#z)fS|hJw#pMMd`E0#fwMBCIJ92aq`{3&^AG9hd%V|tNm|3HZ3kpcG#cU4 z2L*q+Dk`Je`QMs?4kYX!OtIod*;o5=JI?$PYZnU(x?T|ZP{i7WMGrwSRL2;+5zpjb zYlkt5@_A8Ra6|qQ#FD}OfTFQ+AA6rt%tflaQ?|oy9o6RV?fd~ zHrhoVX`zm(;OmS9@>GG z!RR8D2lQFqjOr{3`D|bpJPCZM>~oQGL#j+Gsn9%M*9no{OqfxBBI&G+Zbag01Rcf5mF*7W=@!p^Zdv}j$kak69E zwryv}wr$(CZQHhO+qUhb_x;kupMa~Z6%5?LWw5%?q zSJ8T_A84TC(Rl`%+(7$&$}g6JZn6^>n*9AB57~d*{9Ww{M{QzI!{JTVv%q*5MFchC z(mg^a8@OK<@&?4U@a$EfJ|!xz=IJUw*^$$)eKSK&7I2H%3U6!CQ2JLfASG{pXVbF8 z?7=R^y5i2EDT>LnZBp)*(7Nb&{_OVV*p2&>bC>}Y8vBV3viVPn$3+S1pTuw)Z;<<~2A8MliOx-760(yd?^XuBeM z(LOhZr`1_NR`6C_LNytFX_)o?KG>ugffx%Z&x;|On~VTycy(m0rPgU0Z*PUm%LGLG zc>!CZljnQHt>facT;wyPkMmA=<=lZc1RgU_iiE$ih<6vC_pl_ux)DMrs45qAG7IgX z6V&MoHY4$o`gjx?8I!pf6j%2+7J=3x?3>Z?3snGP7Eo`>^sS$j1eRV!+#s^bJhJ%^ zeu~!SFduwP`40}t2t(D${GFM@WlWx;Bu3M|OJb-f2UJU3935FUeaM@bBz4B$-wcB< z%Mx_iDY%TGQ_GUOUuwlRZccC41w8Gi-RD}cH~+=S+Vz1CiaXFfLn0o_A)9j zJcc5yr|6%jl5$w@!)p;?(0oHCH-kk>fiLP2x8d!YEWnXbD1k*m%!u5#L8s*syhD(I z;Tky4*$e&wOK{KD1fz1#0h@8puvWJ_IX2cd(a z87B?Z_MvRztTwGL8aQ#D9%6Pll2rHcYkk!W>(J^)d$SvjDVVWukQ~jQVZN89$9L*& zQi<%{mYH91nZA{9r~s-*uBhjM0*BCyy8cSVc8$(+oV~77&2^5_mZ#nQkN}x}QfF7wbMr==v zD}hVgp!@Sj*VIqlTw-JA1wt-0hf#1NFFv2B z@dIn#OxeE;TD2*c<1Rf~Q3br~RVKHCs{H~O=rf9PDp#OqWQ^NWA#j7j4Tb*X8N65%@igSfHO#H&~$(WU*Aid)ZRP_6(g*q8|Z6}H*@A>c9{W&B>r(8;qhTfe96f7zCTcNmPGue~0- zYHuy<&Ns0|6gy?fHW%T{EGp<|2{byoEUiNA_6`$p+_MQccO-=WGT2wuTv6o43JMZA z<6=0gmp5v)<8znfPWElekm8+E1h8`59W!Pit}2LX+rut7dSrBW5}dpH=TG1^yDK)W zz`e|KDEZG?Mt0j0u3;T7U7xzo#c>EkV5wocFQ&*9OTh?D)#i>{7O>Wqj^c(m1zdg@n?@e8c#uv%JxVK!z$7Lv3t|aeYj}I(NdX>~7XQ6I7T- zs!F}jMDWG+NY@Yk#ioJ%pV+5KX5I4E!9ob$(svgr-I)YxNt5(FUE4+X>D?ZbcJjmy z^6K34EZ;9hh3yQ^-Qwh`$L|uENbd2s)!;X6qK9^B=XEP3(4i@>PlkBuuQ(y&9}O>5 zV-@1a`}w~}NOI}B8#oceshA^xl;=6QWX26W03NvZ{mp75&gw@X>@!viv`8JLb;4&ep#gjWtVZWu)f@i1u-JtE9M8|Q&1IJ$GlTSDk!i5#YUf9E10{QV^N3D+!xW<#U{tG7Fhvo3^F=$Y%RfD^ zsd-2=;X#Ah`YA+~jTK@#;ZLLYg7L&4iPaaGA`Zx!r-%?%BScC!ge7<(DC3W~9+r_8 zDgdOoCn?E;(MUtQkulgxWaU+KSe>jzQex8D(K0NqvQH#7t1Yp^MM024+olMLTVaw@ z6jZ4dJIX!916*VF4`tAh`mwRZ)9P;ojyK5YN$mINbOSleHPN>Rh3AorZ7F~Kr7qWk z&JHUF*0L2Y7G42GM6eC~7|dRi;YrYC0M_cSyfphW6hDOJc7KxJ_A%I~%DjBqob{eugoQrA zWLU3i=B3vRXA{@s?LfaLy&B+jI)Ml)9Ou_d< z{bc`ECNGv4%JIKJ?5?KF9imDxX`Qq76@KYL@nBDFf(K>8=vreKxPw%1`r$S4b(+B^ zGl2TnB+%fe%#S|RUoZn;YDK7dN0g|8hYWpXa|N+f1gOi%)WL+n^t;4;ulPpd8T~WBSG;RPo|tn;|Ai z{(&zZ#MjPEdTo*raAiSuGLy>2LxAQ%i)CQIacEmnX=;$O`IEA(0n}vVRJ(QlfwtlS z7ElMo9()K$mHY|^tsC>LSC0+xLlRyDJ##~WaK%*q%L0F~;3Ga*%kP(2Gsybh1@{35 z?5X-3678~B7bcPtu1T5&7UPO|M*b4aku&iFjd6p!VpsKZep3_P8`FjXQi6qMcY<~? z3A3mNgo2|N``IIIdjT^!jjX(OHc(Gs@sBr)B_L0$mf+`E{&gdu=Y;Pr*cYd=WcMvg zK#`Ju@@5RT+tQtBFz{i^7R{tt6L^oeIkcp~|3HX}i?VVkiKBibsHdE^-!DG&IF9lCrLKO1B6A8|voUsVx;$SRyZdmabhYzm0UbdNNgisk08l)7(hEfqrcZEG zffNPcX!LqyCyc~`ZwcZEwV>2L+DuSkk%?ABgMp@U12|zt4V5S`$S6Zdd%&lpOTA#K zOU)KwI#@MRDP3B?3c+g`k$6(RREDS0a}0pJLkJ4OAc2MqF$y_7tXKADJ|Y>v*uhYBHn&x^Mw`v9odPFV9ArQ7iV) zan1lt9q?nrY;__($KWOv^Ef}oDhc`x(^J7k0433G97TxBBP)QRPoxb+ax-|J;i2Hf;3E&meMBk`71nobZxb0phpnyQO%)^UIG z@M#})*v~-TOx)L&?f_aW5cXCU<8gY`PdugzV+Q5nAB^ZJ^M!zR1CX~QUAG{?ijwW6 zzS!q0YHyJ*(bdD6>|Ee63roZ^sO4y}G4gTyY)gTS8128u<}id3MnG?Fh0}Jj+oJ7g zJ%jVfER8c%5zMSD&S)rIYhGfql1sw`;w-5OT`|9iIq4#2&jV1&i4EFjQvX~7DT>q) znT_ZwK63wN(V|i$GH%Aq<8PwNO@gf!=m}~>z_)RzfgC0G6;%^{T`>~(4fU~*K45Ok z?UF&s7etH*vLRi=^`;rHU%EyEKoKje6QIx;rjL2js2bU&XI2c~XRnRiC#fvd^oe(< z%>Afa{SjJGnkY+b=?zzK9qO zOJ$0y27ob(Ek)1@)(uBoVi+|y;ude7^e^LYO$tcfE+mPif_(#gV;D$=B40Ixrqi-y z%%PMsQhlsI5v|Hdp9t8)hlABjRB82-@uus$NuW}sToAhqPe5&JE1+lcgJ+-#Vg~p? zGwG)@4~j+Dj}UJZSYP$~kj$CKHIrFe|CNEjTp#rz1uFA_uk ze8Py3zFRA*hm_zTn8(BKWBg!gpS!Z8>CB4+EzAEZsg+>x8ra~*pXJg)DNNUrr)CM- z$EWOQg66pIjuOEo`}imXW3CBMlygDA;cM+QzMhTQFcZ^cgYXt7IHd?H8VgKDz*jh= zesqr3@MvtJVO*-2Ks@ z(K`C07A6>)tr9LaEUM6wR=%ZNc35_a2(DPN)h-O{k0r+&S$wC?MGfPsL$S(pLwD0w ztJ&9x_A4t@z5_Y`+}E%sRCE37gfHg>f~n~{5Ap27w$kagwmtLr2Yfm2;=r(wj6#(X zRn)K4Lx7r}(ReW1ao!{-dp1o#jzGU8K~i?vBn$dTCDb-KrLLrcxTMP_fCFL6xjobQ zp_z1=WOZ^o9WB@Tl!aAXe^kTH;zYLQg};$RfFCZ1Yqvgi;?#UaP6?Xah5|yZNWr6z~T9S8EGceYib2eTJTtxNhO_;5zascWzvW8!2rTj9f*+wVf~?uVXE*Qq9m!E zT7ss0g4i5%z-_e|7F0b6$r%|`(N*lV;X`mejPT@gnBJFSdcG>ZH`bBq4?g47)QK&O zkq{E1zn^oW1)|eDOWHdipTtq*VKiyJr+sLq*Z8aiM5*9E_hU*6)^~A-CbX^XP|=0H zI>R;i`iuM83-?56Bmjog-PnnX z)%w}w7dW{9Gwl-Iq|)@D-nQ5tuA{Z)@Y6FQ zPK`Z`j_Dma=Qzi)G~ht0<}3^rB;vvC_p6G+pK2h=L(b?@f;OCR*a-OBtMV8^GZA^pF|z3%Lf- zPJ{T@!xmLs;!W&#IE{_&5GThZC=Fg^_mg33X;KVK;6(el$P{ot(* zx)j}rR5k0^P`^}R8(sP7TPg8>3zViUCg8Cey@PpUXgWnUirzVojwmM?v5Yy%*WNvGR4;4zBSM6!Q+2GGhjC3{ zcJ0m&l#iw_mNK%0(T58&$Brw)s4Z{PQD%h8>!*r<$My=(0#z~W7UTJ$^pjJmHAR;` zx4HozhNn`a*p6{MLFs4>Niv;EVePS>*1FSA9ywMPpNnx86b-;VLAhUYDW#!@PxQAn0 zzW7VotK|T03791%5NvEBa3j=k%$;jS*)s)9dI&0|iM9|1zEY%4ZGOLNE}DWy>c6K! zTFfy?(mFJeQKWT2BeodQR%@6%80I%9y^gy8y}&PC{8%(KX`VMxS{7ZxW{n5CP5j(& z?E{}%8O0ld5@S@c1Vn{EPG8{j7z$@$M(+xH(C&iy+XVIip2Kz`@?%Sx=)cmP)G8Ti z_{aKe)AA|T-Ol^PUG0Y>295vY(U0fT&v{t8qj&+Q(Nv)wEew^^EN^(+^A<=RPfHdk zz|S$dw}Qo#dB!&LD<;sofhk46ZUmRhv!aLcqxms1L8Y48nZ*x#k3slX=RK7NVS7-X6q@js)3w4ZBW zCIEeIgKndpDg~TqF85(bXU(9^IGMYWfbg|Q2TwPQ>@=b6;N*0qU>&&BGs{{&z)n_W zJCKo_cuG#T+xCHpS<}WDm}zp!K^!D4C4&rO9;1Z_G<&uPI$KNn?mcfSOfi-|V&S~E zApzgK};g`p2~cl-0QWI+ZAU zmbp0X!dhdzSJURrYG;gd)4ln}5AsXslo6zO`PO%p8EN@4mfV>Vw;>dJNTXqrOLlaD zxXmcq*h~Fj7HR(s&(qMR8})s08XIuBXtkn37886{n1~u%Mtl4S<2)3chVN~bF{gug z*WWZTY?7oeLd=@;K8B|4FGmGEz3M^*tzND82P|?U8F4@a@Nn#C3UkE8l|_XAiH#o#8{w{?|6S<)MhQq4a#;1GOcZ-| zp2<)m_;yB!oLJy{8JU#}Dk-k9JO_dlP&Erd4yk3R01Swl0EKLKV7fRa$oA#wg&^$K z9_MOPI(FpNzJ%p2Cj?gx0!;}0D*(}OG&baG4mS}LE2I|b3DeUV$EZb4(SBH1sTe#o z#9EvmY*&?ui|~~8BTREefR%O9pc~=Md)*hrxfx<3jiFCjTia=ph})LCq54vrpCK|l z%Fb*;egJao#IwapwljS3e80kHz7K9rDMLN6*XcQymZH)p!GrF*RhG}}51c}G%0lQP z=8R5(p}lLwvQhA&kn}%5nqeTt$!Q8OUEpv#R!i4PdcR!LA?RT0^`gFbo!~)&8fT9v z5|+JAVGy52EFKi?mr4!vy2(*1{`sNFi%-lA^;DOF3!iXG%E5`rw~me%D(55Il)H>I z!rqdRvf39h<3cZN9h_x*X(XLStth=G-S)$s=2BHTHaKWs2$onu8=#Qj4L==wv!=0;{>n}U<3Ba9{AqvTm_ zWaStV8Y)OvVufGUN~I+yEiZTL7$aJ7V-d!P2nmcbJy}q}e8ge{qNNWO!2veJD?5gJ zSr-s{?9&shG5zFVbEe*QF@}+oGItBgq2{=xLAn#nrBU(yx9Q<^7x~#5e9Oj;a#WyJ z+!ER-Zf)j-K;B$VN9fDa7O}1>Q_%ssIJ{hF#B5XT548}62b+LraIO){dJaau(>hZJ zjwDjfd4b!_5NA|N_Fd0i@tQ&JP07YEjytct52)Uy_t2#g65S&bTXkM#Hc7mA=Q`+aworN4oMsAjVV^fW5SD>=>}-Hrb6AiQe3 zcGJ)iF+Qh_zHv(ULy?$zyYp05l?CWXJnUV*gdb z`2rqFy615cVe6d8X7zD?M=Nzkz8TX?591PBNT|eYnJEBXs9u^jZg?)+j5V|_k4fF8 z_M@8(+?Cjijl{FGN2P^h5DGY%XWVMhiAc)6J-)0S^aPX5IGEE)`4?ei!`jfE#cKW` z^ZlcdC=tf^N7w1^d+cw_gCJc?fo|n;g)P|xsKca&(z2Ukr_0rzXF@Wex>d10UdzKr zNzmSf-YNAiXZ?@gxKFwdg2j<|WM-}Zdggl0eE~?TjL~MYvD9|OB5mRO-q2yBSO!jP zE7H-#S|TRs+xa1or#M|5gB2gsEaiGhS5%N7! zUmaanDlHA@n(s><#)J<;%S1?db1D910oTzy<*KdNbMzjdV03OE#StzN-?pXIcONDH~a zZ=o0>kW|a3s;$4x=Htv(%6Odng);avV9~TD`%n%_6pc7*WP)xQj1wWthcRLGs?{{u z{v)np&>GOorJh&qZw&V-r4YHz8*fBI)Box~VTIbZ8%tRtw!J2Jdz&1QU+4q4zgKP3 zuUHhSmy^5MYMjSwd)t9iM`{N#eM=YqokKY=7RPwjr;`M1;xf|>Hl$2$irZ{%!v^7! zAxWCf8*Z=>RDK;ep)P!=WH+t68=ZKv*(1>?y;MLcElXdBo6@6Q1MS!CncvhvC?61} zspMk1GG(W!Z@?6{a!|KeC|;|-w_*S*r#G`JR8_h&KUNuF`C^x8uaU5lq-M~?aw$rS z-VGWUnD>fzEW|k2Pysb`BDa!K3L5gQ6sFk$2bXxHJz|MNiwW59myf#WHcV#t((w4; zw(vd7;0Q7RJ-~BhssLM@?jUm7E$carKH|!}y3@vjnYSflnXz z?|)64FN^cUnEX@Fa{sIYjdl&4qD>ClegqPUMcQW4KER23u|_pt2z;cg2u)$EXBG`b zBM8yp#sJ|-zHsgH8CsnTT`dzsyT4p8Vh1cH3m^H%e|b{}s>pVor&?Bw*yWgG0W@6h zr&jcZSFVxOOGL%jZF>(2(6@>kXP*&^^VsKxSRR<=Eqo0ETrGXL_^s9?4bCDVTDs9= zNBi6;B?10!Gs$w@S{jE+1BuU$QC-+psTeZc)5P3EFj;o`(6h`v z$az%JKHp;%MGVr9%M8Q=(xu4sM`aqUwkTOF`$*(xfgxM(OsMb``EVa=*ZZ^lHLT%DL=_i(?+%Eq zlUm!EBNJ_9%@uHbcl5jzIf*b%esB3tJwUWW-h3U})qA+p!f3c-RM>{S2@jUL1si4- z2}g@0*bj7~KYezBqHh(=yhX+QXE2S;B^(5{RC)I!ZP&z1f^Ems&MiWOyBz<3`LX*U zyo{nOj*s2vJ+VCab2Xom zxR>&70tz6Bu6jIZ&+HfcSdsxotv-EeHP!We>OHPBug~v&cK0FdE%z$U@KpxyHWmbR zF3Kf_HpulFt+J8njmuDitqo98SNr_^5zoPxO5Iyr9HR|;X?Z|{O1BWLKQKq>o!V)w zOu^Z<&z~yB_{f%73paN$L`#NI%^A)1DtX~Slx?W3dDZf7S4=IK6t5pfq+AhEHngPk zs{QL>bBYrdwaB*j982IMM7#q_-c}osntg6_N+FS8UQ*?W!vprugI!yjJ%6LsOe~wW zi?EzdaP#B?RkOK-{`bVR^De!PQz_8!th5MNK@3t0P^GGA&769smrO`uwE+eyykQbN znG=JHp?;}cRdoj7a+B}CPoF!Blhqtk`ti(WO2&wo?$kRd=E9!Ev>=6bBPq8a&*0NN z4{coM`0I^)ON$zRuqpT$INzgNq=y~d;?;TG91vVFy291f9@H1=i3(?>dT_E2sN5(g z*>F{e54&f`IWt2BZ}WY<>P5A8H7~Aghi`UL#KYx|oK`f4M~}z+B6xgs#zMDPOY&y% za$3vxF;3|f2C+Zy5~XomHkBjY!B=gwKj`lPPIHhck#b{n{9Z3MydXBO9_Xu(;MBNX z#38wXQ!RVJs*w|asxvS!v~N~GzFwXlo({H+zZN@RcYhX*=lG7)*JpP}XZOApdWQEC z&_XnQU~C^wx3~!;4=5T)u)g!=;A_Q+dHgYEncPC+4AeNz-Ed1r`CGV~L9zMv0l$PH zuBX}{BC1uMJjD=~PWOr;VbyBMYOmNMv-FCAXVpt3;pL($sOK}Ui+EH139tz+`gTjP zK)Wv}Pq*z(Z7_WhvL})xOkhzI54iYY&MNp##)CWHBGh7FiJpo5bp9f(J0rP{^$(wi zKHlyO=k8?pc6#}~|9r-xjg`T9BFo@;j5oWZIKCpRXrg$6BUI&>lpuV)pJg=A^ z&<^6xSmF;}%ES%Ddq!*paxKxgyzTch^>aYI{zSk@Bta^+0uRTDpisME!|2{(;u1RN z_0fsG0IMhe+9toz8B5(gc73d|O2+MOGmJv9=zL~A+cVL;5+!VFRV3;+RmzWd0yZ~WX)bg8h?QNGKhaP>HFy0Cmlz;WB> z9N4vg_xcD!&rZfSB>jm)X?9ROnim1|!hBmLD!wj+;YA%;8^lU;{1lE+N)P?1(^+zI z@0jGfhT`?I(pG;3E!bTd{cF#abG#KOr0NMe(e$>U=hQ#ED|Ze~oSbh9Z}uQxwHH(X zcVDG**8OHt2(@i#%MtgB{*86QHI_xQgHStje33VogQ44Pw88ZKop)6W9A44WYz%Zu zsp^y?dV#OHp)8Dgur~2>MWQg5%o8@s zfK7kzHA$e}A#>Ry7)r+n0QiZ9=L6V(!3SRci(M3xh3`D+bXD)gZ*$7yvugh)*vAj= z9;$n2FS|!j}#BQ%6xk{yKNDN*j`--Y7d5&j?kYNHSSaZm;e}PP)<}+>x@?oo$G{ zeMgeyl-1}l9xIwqj6xr2pB_woVJ%mF#5rR|Xy@jpJ8YgHZVhV?^^b}d0X;jCKWNC{ z9+3|;jo0g3D%-MvV!b1j(D1{QD##})an}%jq!($P9Rg_vw z<0fq$MY?U93*u?{Xu0_qB3m#rc7xvybhl}6TLhK?zqa$K6^aSmj^ZBaIIS@fo0MpU1Ql0y- zf!#$eKTbWPt7a6h;yMoxrLV6+tqQwZYaWSF5+OE3ne2!Kq}vhrNW}pETqocIG@J)J zmu3B4qfv14Xb0~4hBl7&`C>nSwm8ge;r-3S6j*sXCnUR5{9wqwG$5>dw>{Hs(o_W= zy8AqBg1dtzql@ch`t~-~$?~<(f!Ewp1FhN#OR0C!p+;8eBtzW+<|&E;9hYR3XT?VO z^;J8Dlh^kw-R34=VgT0Vk06?_W(c2k1BcK$e%b_JIo=~S)*BkJnhN0+f%aVVtTHR+ zftmEUJZS0Ke!w$;{D z%EQVob_uvgMz4zWg>{KRb|t4{(oEsd^BQW4rJPE*4?RU@@ffe>rjHUWZapH5^He$O zjt7+b82-WU%HLWf$$hqD$@B9>0=s7edU!O9^@FgYwS2AHnZn^CsB%VC1!-Z78(1TI z$-YAb^pVHg1QZYMRGRJH-Dzqg#pG4@=y_(JVI(0em|#vKL-Z9vY(rYI($7v;xOjGA znbq)@Wcv=ZS<%@r{-i9J!I z7MhEQlx;Uqn}@mUn=Ovlx=8IUPZ7@ag4+~?TcC`ePmzxGnNI5=^K$Uojdr~w2DStz zRz_~#6JFi}6U1K`@z0P1)2`|I%Yp`8>!rQz=ZkD_wKOF~OkVB==BCSq3Gd=-m@J%D zM(rTvF#m-W{J=HNZTuMUMRLiNMiE5sbkQANYa%!QOfqd=D4ghPU2*fK8n$!}ih_K^ zL%Ca-qumvGu--l_C_JW6XS)be6TdigXgCcRvT9C&=_9b8Zw@Lao1)T_ZVU;9o47v_ zSm!F=h(>y%3AV5~aaP1UU+l6Lr%uEKPt&5b#<`A+4MnHsr})5YxgHxZd><`Tct?&h z6pTvF8&z_gxkO6O#0Zq(csaoap(SIvlBkhM+g3OkfW7TI9rHz?#cg{Uluv%&+TiR!H33Lq#hmtKD;w)I1w{w|Ygu8~iyqKmLQ=ZDx}3 zQvYYXjQ(f5{9oHV|37xu(AL({+{W}D+c~VNX?eto?DJa7F0FI5=Qf$r16`EKOdy$O zl|rlyIk+cpR>KH(w5U-8_Raem#w>6 z#ljsi)FQtjI8IGvVFJyzEx$Z5K&hNrwqU_k;%7)Is}C0)S9$Z0vCQIG;|`hIex!j+ zvBjoBK7N6m0}Jyz;sn_%$t^B$-ByD_MMP&lsXApr!gQdu-&nCOW1h&vH0Y>c*>;W0 z(_pRm$c7$m;oR>?K()NsS;)m5UQoKlZqs%q6(QTvn24OCOf?Briytg_orWs45lOcF zKpbH+YSqf)efrKl*}$CE%KvcI;M#bE5}}Aya|m&#I*#EbpKz5YhDl=gBZ)Wg{^b%s zaBToaey`18TJHIGX0l6fIAeB>RJm|XQw58+!@6mW(VM-;SM}CX#g#*3Vdw3422mB) zQOxsFJ;qwHaCk~PD=Ta3PSD&W8LSe(*v9XjQk0+%+Qy_ra{n16 zL2qaH%@BW3uyFKPuTQ6OWGGo`3zVcjfdhv%ZH&ojs>sfzWqnceVFBSiV8pU-jNa0j z8$xoQc(&X0eNQ0A&=fj%oX;3G2j*(O59+M?@=eIQrp?=2P%4?zda`B`t!Ad|>4Whj zYOO)nvGc@3i{F4euLmpurE(|JRdDZu(|vGiZ}ZpejD4%SJQaATvzOgh>l6HMF5Jko zs5a=b9!p}0g2(05y!AMNpS4wxXG%fcABgbni9Y78zCX%N!tB!QJNdQ)2A&>$1W~#k z!rCS`XMa3h#4%GLeq5^SJlkJ17-$0#raAFVb#$Ru<;g8eP&Ex=PlhvI?r& zQy0@6>mX447t*ph|2F*THi*B6HY1-~4Hba%<{Xd*L#zc$1*uPfrUE9iHrcnr@ck>$ z=>7M4s`yVQ-?iQSA~Y^M>GDejc^d;8F3BG}wa@Qpu2rzyvX(zUWeH@kg)8`8;>VkF z8S()MVY3j8o>}eQ-WFuMrZT(;n%E=V$fc*!7`O*y+{nFbnl=d_i@c#MK^Dj92~#jK zAk+$Ido?!O*^=&mW<%|_jqHL^>h{48N2X$yX=*fg9;_{?XQ(x+y0tA$9s~}}W)xUE z_vc`vaLuMko5MQFWGU~st42>@(@BNIpOGE{JlH0sx z*rMJIf?nx3#5x?o^Ks5#U$^V)?Vr#@%%O24*X^2OS6~wi?^*L|^qT5v5@*0&x=L7* zBs*liS;wxCISSvjA#0&xuXNe8ZC<{)2C>F2?2zxu*)U6;M(u`kd*CTu-lr~mh8!t{ z5yKG0az%Sy$9G>Pj$&a%Q}n;^DG!F^;#KJ+cEm?i(<`SJGRNix>VDF!K2uzy*!B z6@fDhF|F*C{X}$tK1IvLKA4@bpMGceuip%Kv(~$N#M1wC%#CjZI~4Z*2X`+8LsXmp zuV{Ju--!P2(F#i&eLF{eL(6~j*;?t=a+4nZ`=nTXRqh$&3O`38``t_1NG|OFt3)bj;~JN;^v3GH~8yp zE!}Lb>-@(%g%hpb^x8Ia2Tz4rJ^u9!qUNOdY_zQ50H zBM!B1wn~#J`jV6`kphZ%k>m;ui#~D9%o^ShHl4FPCaq3>sO8=aq-xgbym|7~6y0Ft zX;X6Nd9T4PW=ML8~jCn)VNY0wN_z=es+xw$e;Uv8Cp<}*^9yl zWgV0SsEdS^BPkgq#9c?FgG;6eufosXc~U<6 zaw5YN+%qKoK!CyGG#V&P*GvjP7!YmaszQ*{AF!x@;Dw3 zSLo=CsN3vn_Zt?Lu0^l8I%=Bk^_R?CstOc#X(~t@(KBYjSP2NB;CM&vvSl}Nj#fUH z9@UphV-$4+_sZQN_z1@pnE8)w$VzQt<=!6vfX;tEIoW^Bm;7JZ{x#;N&F0v%wl0Gl zE@eA&rp-JWphjPPWAOYoaoBQ^pWYae1Nn5uK^>D2$HjF$-kj`J>Qh#G9Ckv(F}`}9 z$dg_?(<$qzkEo9Tt%!S^SL3o#g4DPxUGJy2@QznoPe%`as_kyC=jV@s{r8JBw}*hu zj`sKamsn1nVEyA|(!Weg);NZ{v!nA*=i_9SN<_41Ix?4nLihB0#+^1r5$WyTvdx*$ zcx1G0FF7p2dpuRNFH0hG?AWG)qx{}t0i`3DZfJsc9wVVIrQRUJT0P9N0x|tyU=1l}5FvtNAy7dt~(R#=kw?D0sI&iXLXkc*U}*l^)-KuP73Jc8b>&oHDAyAE*P zX!pOcf3A09Ad+UNx7rGcRf+r;h&65`a6)R4(D8=NM zL^8)}rDxrwsPxrpR1x>QqQg(Ods;Mvmz)zR4*uNFCf`&;3V7#kpn43=d4dE_4=8je zc4|RZQ)>xWwzq^@`8eMCeshu_E3;(9Qesw?glag4SFEP#r=(pIIQszI8h3K0QAdKv z7#Xty66;QA*EY0&%RS-3OQHH1IoybRmRIAPZPfB^Sb+MU(Zl+89jkUib!B4G6qat& z2<8L!6=A4?)+j4}dKwD4uCyT3a`o-SE}h4vq;(cDrLZ2NF1sMe7k;lj4pfLp&3+6B zq3i@2s}pz@BBBRS%0iV!)fpf=EOS6@`yyWVJ$D%w!sB(ovq61#fr%_qE6i19PD)UT zHo+~(g+k+WyeKc0Sg`KG7ZGnrgH%d$reg-q5O`7(!iJuQYkBkH;7Mhs22{dPYJh#* zbnTu(EEQYb!%Rti0M-!hbgN?%9+$pMpoKD7FRDTVWu$NalL`6a&y1ttP}J^^VT7%~ zl5RUHjbbfmc}LN#c*%^@wEEP^RZr5avAf?gmbd#eb5T92Tu(O+_1*&BYp#-lC42N@fA2t)}jd# z@7MI*??Wr_SuTi~%LS{7g&l%t_+-#04eZr{sKBEumDqKTDpG=mGUKmD$Bs$4;oN9& zZdEpf;Mcsn@Jxu5&#NGunf70ed)nfMo-T5U=S@&Mj3g=qN-e3>$N149Lr_t0aXJA{ zxa7S)ASA(|I|?GrDTysU0)nFk*b#;i>v05q;`0v8EHWZzuVJ_a2tKHCUh}nv%&B=f zz__4Kw;XzO+&$U)U_#|nA$B}BA^nv%ezU@Wcj0?+XnHNr91 zrkzWW@j!elnP5bbzpwoTa!O^9__5>N=(SQ#bF_O%p0MEBtA{WoJ$r92s25FnSAv& z^DLx)TbMAy-Jbj&ug&2I1%e{mNY|R7QtuEQ27#(6W?QL7{GxCQ`zF=}k^Z)HXG_S} zI8;2zDXJ|T;U=|fsGi9R6$O;YS_8s{yd3VK+KBO0UwPH4kN`FGD&_s?@A0H|1V{}@ z03HaMZ{->2#vW-o`506Ypud}bp=l8 z=oCr?mx&iQh7CyS3L*tgV6Zyq8YB>us2Zr&s58Ns!UtJHq1YP=>KYLHJFaH$XEu(= zR3GlCl67h|>oCT#SLCIgu0E0KBxJ1s5&8!ybqEHe|D%6_;!fhO0s3%ZY>Ymkns3eK zIQQ=2$_AsNUwjb^J0$fVz0q8WV?LAR61|66;Eyi`J{l_)G6GPFS0ADT=|dYri>)x@-fZ|rnzJZxk`36|2!?}{t03|}vD@!@SKN-*mPXk(m-YSs$*BF;9s zcg$ZM(72RTsz3UOIyMGwW&~!$Ld;G-g`a7jg7=ZanNn$+6&y9+qEr%rU&eD2g9=^= z>ipzU=E#C9Tspf{3(Fg=vF|R=tW%P~G&F<#cPvGuY{-qd5-H60kY9gLkzHu`s1%Ye z8Zymlg&IH5o5SqQ)VJQ0vFm<7+0c+{6Rat)@_MQcsJahf@g%C}Y-&s-!$I@y!d}(c zp|CwXkT@{)nYOa06?UF@B^bDMnY_kQ<7bkz54q9Ppp@dPEj&0kvZQHhO+}O5_8{4*R+qRwDWYW|9VYRMI5px&y3 zz0Y&l+zy7;lHQj~{GNwhcT_A66I@l(+2iINP}9Nf=abHjd~Jxi7zs||?Ac9poVoBX zObc_J*HSD$fq1?Eiw-=nTAFD^KT_KL!v762FR!RndOWUe*H9mHJ?;R&p|rjCX+2ro z=E;R{%9P2K)aO9OvF8_fx&+uCLJgn z`3c&qHt6|1NB8ViF3FHv7|7i6>6d(9PW#va`uiR+Jo!_ZXli6+ z_syq8veZg-KdB1&rRi?@&2GL>*XTPs^h)H=aGZaLrbm^?Dq>J1=0S!6g%UC#*&_cA1{W=w2>yJ=@<U_0x=x6tjPS&cY>Cf7srJ*OhLezC!;6Iqhr!1Ye4Ib-tB*Qf0%tRnuCw?+B0S7P^+0#%?L2I1M3x;j#z7?v8vE) zM;HOCPM20}-3xt9&qwbbKC~93M_%y(D3&3u9ORlWW}{;_gO2}j8DP*6&vsT!awj-K zT1^s~ifV=@AM2VZ-s_<7ya31%ZRxcGV`Eyq;F4^rb9!cp+POfjnt@Eq6Sq}x7Gno( zMK%K^k<_RfO_=_X;5@oYJw=+{6>-o#783LrwYFqSOgT4<3#s4FYRa-L+7zx|tm!?J zN~hm~1jE*0^77a1r;0G<_+$%&mnqhUdf>>%A!8I#L>UN1${WRxk5G}+b(ux}tf)f0 zX&9U6ZmpT*HIb=jF;}|4U3B3VLA%IbFl|iw>(OsrGQky0FI2%s7F(np$MfPnsG^IO z(6W8jYdb~;gHPjh)G)KItJ(U}0(#hjB@zLu=rT!S-(}};fE*J>fU1mx{kas`{X6eE zu-*(75gX7inu#Eki0cLa}!i9HA1?zL^u#`ug1dAAf)B(>%ZZJvGa_NC(lYro>zBhh7&ej2Bdbt9x1& zA01b!V)ElPR}+V1eh=Jv$C-qV+{wkQ`}-t z>pGqY_38lBGCi{#k!O@Knw6}AGiR?cxUMTq&QbiVa$@WLB+`_Ro^1;vBc6KjmfcG= z^+UZA65o!cYg#wX_{gY+x?Xi%@RuvSQ>(m1VCB(4*Hzq3<~QRVj*;$eeu!N2l`eL{ z;g=0YS;H!UYI$6r(p?8XG`wnLwn~tHrc;$-OmMlRAu+aZyO{`;2@teBHXu%g1c{+-uuQI05G$Q3jj7ay6)8rwGuuM)m-rzUJB{iJ6!VI$Hu(5ck>pyhOF}Hz9Zu@ z??dO(uVNQ_%ANlF{yJnAE6{lbBP>4X(+iCUmhUALvvsKk><*8NNjdFKG>ttkkK`Da zdu1kOmA@EN)?w@sy+Zug{4;*{*0W zbv+Qgvt1syF*gQAzpKcMZuj+~_x2|TjbWm;ZMeKb$DO*hYW}u19ut|m)oIH5k@rgE z{Yjr9v6AP^#~pl~W_eGQL8--MLKB_uQp=ShZU>s(22_A1o)@yCQ5c8*I=Vl~Z#bhR zm+b(kA+_|=_z&P28^3msn zpU=m6+DCz&G^Hkm$-;0Jz#h{+&Dc{FXM!NLd%_n+y!HJ@nv=l8%56l`TH@^*gk_63 zV`jK+YG~=9Yq9}~Xr?+6+B!hs%|1U&%L2o0RtOJwjz3{K;dS{#P7qXCmvo7~zPmfo z#nzj^N`h-Zf5rdb4-2=+Jxj{(9tS-H005@{&0#UNw==W+4}~m8{nBoe4dwf`l*8d? z&HtKI6AT56i$z+EMj#8sb_JFKECkIm0)o_v=y>RT*=;SiPQKwbKJiK?+XfmzfNb`C z=HGz2i%q3#b>r3|1@7;fn0|OlQ+)avemvdW+`qq0zq7^R@!Q2$F4bb? zMdfm-+BB_Y=a2Kn@u$!8dc+vzpneAqo|a=Jb6MK17E;N5QLA9*iTQ<^B>B)DXGk(~B<_Yo)< zu*nUySCU^rw|N&w6sotzCBrrCwpqQI0hy~-gUL^KX=QWipwiC3AwQ8t{k&NNBc03r z^NqoOvAA2UxJTJ_l>dTnh*CAk$%4U2uoWbom`B9DQGZN2| zMF6<{r+Xa2Ew^eW&1jNw`SUWM3H(Ej9 zgrfE8#!!2-1ZKqLh=&$MV5hnkmE-_3i;C@n|CA+5;iS%TB*=Hdl0$?*PMiO+vjc~% z!XGnPOS*}wX{bE{Xu?KTf(c*N(=A!lM13(_O7;91SILwf)scCL=>|+(ydFGpGhGu& zzEh3cp|<&jQ+k1|tuXh!IjJ|6FjBV)(;a>o+x^C#?4BRHpE^(|lZd6^Bm)cF35Vwj zgv3x(6NIAJKz_vqL4Nmov4~&N3hxbi^TXiY+)hYT`V1j9QO8njr{aPX>0>OonGi2NKRq=4<#~8Egg3_(R zt0gBzSqiIc__MT9IvXWGP~8=R6A9@A5*D#Y$KLu}8oM&VIZ}^Y*Vq;_LX=j-rCLnL z1G`iT;2&tRB{d|7zcCBoj+Id*;a4j6V+L|d2om#*4sgyyc7%3G4U~bAHxUx|PaZI$ zg$xD2aO5OlHldn}T(G*goKt-2wA#!0(Kw;4w;v?#G&$S8;kmz1htb!qc5WQ)vvfw1 z_;=ZQiV!xJq`}e{V@t!vvS`n$U!ZNB(p z8(uczXI1HmXHqcV{8x@=iI2XAu-0fIIqOb`&67QI9E*x>{$te%b*3$^*O&UAjopq2 zF&#%(>nLPsA-We&BK6*$bDXp>sQp{(m*i2s6%2{)^~Ccgbz4!eR&6qGOqTTcW&_aN z`ympwhJ!)Hb-bjCQdby$Y#99)-&Sii-sUTVS?cPlCK71rMip}I+wC0(r81-gY^1I+ zP(ip1XBUxw>Cc+O!7fjd@ndRhfbi_uw|HDLsDF0xx{iuB%Hs2z_6Am{~s>{~e zaxWt$ToZ;~lI(2lHv0CPK`+;tN!jtU*{-F_PsjN-Dy#$mift8M5KmJV7eMJ%|zb^uXKspdwONEAG?|Z7Lr_-Y0Y+u zZ>7ahCCQPIznr)q)-hzt%d=UVpdz<#ijd|K##!HWYvjGC4X{u#X_guThaI|vE-Tl! z#<=2C7Fe}Bvh0v|Bww@y<=))d&hZbk%XP8~wBG@gsfI8?CLjz|qtAppbJr~6Y6r0Q4-8@oKHlPhcvC8l$GozUf%9`|jv= ztZLa$0@}QT|MwtQXA+_mhy(!eDFpz){J#lu|HX3qjdIgGn|6m{ska}IkzWW%sVLm* zS3z@DHu7$X=NT@e7T7_W5I|W((oC%$ij5Mj+d+UYmV5?v`swr;&KCn6N-f;?2B0L2 zQKfP@T(8a*>BtWaoLkMq-yOt?Xi4_7J^y^gcD|3FD_7~qCiQo8zaDOemoN2q=*Y;A z9uAiEPx;X?t1)wWXDqi-I|Ey!AHpWpA8$|SC!5S&m@?!_?LQi*cUU~M^orKQ3U+=J z%w^@YA1;5s@j3eth()*OTZ+zMrzI7Oy*xMOH{rst8>sCp`RFaXZ)+e6@x8RnlWED4 zT4Br2_^vlf^h{)>B^NIOYGk9ZR){!x=di{mq0&J=$b+tdtlUE+-%CgzVu> z6Hsy6DU}}50{f)_If)a^p3_%2B_&tA^IY#SU*>d6aSi=r`)}5dq`KQIvTDXNkgPb|GBJm9S6<1G@HF^1vTZ`l5LAU{_0V z>4`Pry1h`?hF}|frhDNWJPeBP%yxUq>02b$F9Z}VO@Ejsun_8lg*w;}c#jo=P+H)u zp*X#)%SWM$I@zJ{KYs62sYvhA?;Wql(wZYE8k3$s*s1k26WSuZ{~ky$vY_rrqHn0f zp=s)(f$y`@s4D((9P*^o>(`WD00r;heazL|RlC0*Q~P79)l;{NzuQer0C{{B2o-k} zuy{vm?6LN=AJEeUh<1nTS}MgL{mV$7X2F#{qv zI2lgRu=nqN5oV8Y32zY?#LV7NItu;c+=5+`fmdP^#--&bfh4-2mt(bt`fXua3kZQ>%)2}1(Xm5peK$Sl-p zgGdkN4W4fSJRnUeM>NE(3}C`QiZR|x`NB%pqN5^XjJ+^R>Y=!cxn3qAa*L3F?jv_2CXylno^TU}MHhL&_9wK~ocWQ6K!gshhFGLaY7HHZp#ZG(&3d zeD_cjAp>`M<~hBPcd3d1!J@*KHKq27RfZwF#}>O7W&;HW@4(Ly76-ie*tuEC5At6@-J;R0h#0uK9EBL4uk z;njly89@tyj$&Y-yIcor9tYO&>H}Lb>sZ(qdDNBFq= zl@w0LO%Rg(SfS#sdo#Vv=An`vFjxWSBQDyPP=l_qQiA&F1NlL(TSV!K10gK}hwNxe za$tB~OwX9X|C$J1ExZm_j5`-F4xn^J;zZ~knqOnb_0UodD`K?y$2;uJrTXwwY4rhv z$`VYpo>jL%e|=Juz_g>7^w1Oh`4%XPLQ>N^FTp=&-xrO@JwWkjX^iUW>DkG*N3mma z<#aLz6r?8K-a5{_h0n_K*D-h7tYJg?eb|d8MW1Zeyz>U2yI5SutyQ~+-`>9l7S)_A zH`e)-B%u|5EnVG^eu>Ij7{4#&Gt8cTZ1U7WNmx6_bMn{>uqRm=5WjV&#FGW0BlNKd zi4vbPc9rHY$e3VH6=)Hu85NEgx*5;9DfJ|dK@=70tu@IAn&7O0aO^RWR}4N*PH7en z#RemZY~p_=O%%f2vW$wFb(4m7XsB?|vb`;|hFg%8z%q*rCZ$u9h|>}Y)u5Ho$nnCU z#X@i3)igugE_1b@5^tWKV_88c&-+@=6R?y9%^Ctzx0AM%T=x4l6%7XGOdjAhEC937 z5Zeh35KgLxrbgtD3GiEQ^ur7ro8$*Yf<#rO_)}zg+EZRmD#A-~D-+66*N4PpA5fir z5a4l)x6Bf*qfrs|)g?=ss4|~QrGoH8t!`m`P;sGmiF7xhB8;7lsZ%2vWrd{0 zNmZy7N!fFjid!IA&$i(~Jm6L*_B$EKP+ZWNq$eu$03)ZHquIv#-S_=!Tabk7PT*`P z@eKzWSpHk8619F4>olo(%c&=v1g9PFW;D#fb(05n3kJMNMBFqEB_qS6?gEgR*z?*V z7S18ylD@7w4?*zYCKS@p!Z2BI3jD7Wj2WL{JrvHE+5*M3QQLh3*SLs7W|p)5kNmxu zC)>#E#e(?IfAi-kOYb49;@y{DTvH1iZzW_c?Segy*837qhgtwupmlh%endFA z`UDz!8k@)mf``KGrYHhH8~Bwy1puWer000JbQcRqwY1HwETvoy_5byGB4bkd5WkRm zUPv!BROX1iWoN(F?7Vi6y{1_`jAU_y>q^qYK-F4M^nE#OHa8XI$=~Ub0#Lp=h3~gw zMhd+}hoEtei8&kJujO!TdH02OVcIcRL{0qB5ilJ7B&%cfP+_Oa0333KNoY*SL)Xar z3`Z9guRJRwXT@Kgyert5*_oeuYTPPV&=r@>awM9&J)&S`x?ZF4&0EDY)EzrK&*7zb zZdtH9ZU?gTWq539uzNH_+u5;tlLqEdU4Ey1bIuiV{+{f?QTr>rX%DTL8+jwIXW^t} zl8^x)89_3B9t4w8O3gZ7)A!)aF{JaUYg0uu+{?ZJnxBQ#fm$X5>kN-xF2($}8ii=H zeH#@;h+cmLU<0{d$}xB-pfsLXk{4RbH86FpDzL1J$N+ASa!%)5KC2!1(2%r5p*$T`ZF@yf{-Eyh|=tHs2kxKr(aY-MOh@AEk{K$g>C<@W3 zlttq0CFpn2O9o0P;H_5(PUWD!B2w>m|ch$9$c#sDFD?h8r>T#yVZuLMFq3_W2(qvl~dhKTX#zXeF zf89Ha>ID}!_|jBN2S*6|zAxuu^D@eU=RxG{Q^a}0u2r>F5=)4l!!YZ{78byc8;B7_ zMxTkK>KU!CEAQ!g(cr7@QP+C6JI8VTRB2(Z1s(;p)A+HkaTg}MeW|iJ;ekGCiJP*p zo%Ayy^4c}oj#H@Fl17R<-$lQ~eG$SlI{%4#xD3;7X|W3H&iDSb%p}|0Z1)0Buko@CWTuN3u&6@Qr3~?^YWVdzF6sWQd<9K_rau!Zvf^b>EMO zq)b~Zao9r#1Kep3!j+T(4r%n-pwsdi9)1Sxym8~6Q>zcW*Lr#!x4hVpMngf6V#I)Em9Okj1@;NA23)TM;omGFJRP0#1beN_VyfH$?< z%OZPTuRhc) zY0YEPZ4Sq1nNtl8>=Y|jD!Qc|ek`UPr~DA(dG|lN4Cymli-)?<#qciUp@bYS!wq|i z^~TW$0M`nh+-~vAuJr^V!Xbcc&7(u-tt>KP${v23(un%edjBr`{(57`Rb=NWp1~jX zyXm<{_NMK{{i?PRVj~+kGoR%&OZBZXFM=tiDuw;vLpcNG+NQV27AdLra4QnXi-k#|a>4};gg(hb920KN2oqP20Zfe%r9TMAQ>XJzh!;57eEoSzKe#KrMo9)E`9 z(RK4@7P(j_a5;hSCn=NjWVoeQ)%7D}!DU<)XE$=9@!*Pc?J0oAjykw=J5t$h`WfJl zixSxbK4|Ap98^aIV}&>iHw{DyyxG&CylHO>v%BcQSuUTTi}3u#l&JtEetM`s)L7R4 z#zaK|y>zGbk+nIi7XL;h@Dj$EKp5WaNqim%@cSX=`wqRZRzEnMVYO10nmMfv?;>9O zZBFs&ycK~l`EDlAhPOSidZ&#{2Rv-*DxEQ&W8&I%%@DFT(INhg@2aifws|_4F2!1t zS&X#VB%5wbc7^;Ap|z)|>>WAbq+r-lk+T-66m}!0Ty9Nupyv8oyIJ#eLY?t~SXt+0c@7Fb%SwoD6xFV}&w(3epyjmJ-%xT7BzoOG2(z@APBcU zwe*{r5OH`y%4O}h6C~-6bxT&-O3~5pI?%k0gD}z`<@)ac%R_zD6kNY)!eOOI=H++1 z!J01|{CxL!67lB}0P={T5o>FjxNpz4_k_l_(Za2X?|<8TXSBe^5q(91G`s=pwGPoa zLn4BS&t{YLnY$#m?Wnt(`YN)s5=AJby7U=#bXa*|WtHQ86SOCW)jXRM$0>zrU@%_J z*7FTDz_TnyQ=db%w(CjhV4Ir@_J_@f^ZN@k5HHNp)$Yt!0KN%2wryFiIMXPN6FX8x zrbBg6+8gavsfa-j7<3sDp~cxAZn4Rp%a44v1Gj(eSMEdJKH_a|Vq4co>aZz1qi}u;ZL)`c_w27 z3+5ld=D1nZT+4|;y^+mwL6F`|5~!jwyAn-VYY{cYrW8-ghk|N?dtnp-LnZcs2*{{5 z8DhTqbQr>MCT>FjmY(KKvWswdKPYR%^)q?-1i<&t+l=H0+hynNoelpZGvO7?F)#Rb z7!r=y+BjA$6TK7n@D)6ia}rZfJDb}$RkXtLt9M~hMYNl#t30O!Qa~lJt4!_j(VdWI zs-Sj#Gd8KJt%S%#?%3Q^&XfbylBWxHIrl)?%6oG0e2zPzP6{H! zYSTgZ14w}Q`CR4E(TeO>dFZb_UpUGIJ}sUAxHkXk=X^o>&dYs%9q$GB?=7vaSA zYybe%5&!^>|4mEF($>)2RNuwY=0EKs|G}=`lt}z#*L(762sl}2{>dJNQmo4$VKyhA zA6mZx3eBHWkv<3}Y9>KlXJ+sF7ycRYDqT~7e??9?kzh{N`0D^|)YSdIm#VIfuXwKc zEr#ZgdZqGUL1C1mEFpY#9bf#Z^lJ6<^ z!vj^pWFk#tS9D_&fuZ0+RS|MS?&y4WFsEx)oBR%)<;UTu`aY=v#3H2RL3j!1l2$Qt zW|EWK_~jyeL%Iwz1-YY?UJjq%`^gn4oA=Aja4(#mqkqsMr0k?S)}psW55JeA!_S@v zQVrC``PKDx4&4AIscH7mc{GYCUaAY)B2eAQk+crLI=E!|j2cAq6eYkArT+_ej@ukH zfmNnCbb|(|Y1bH;AybKEGGNB=J_*gBV`2z00Dp9S#y!(4krs-EpNH&{ap;X2eAaH50`f@d=b+z_b(W+FtJ_DN;g)WZzZ z)F3Er0LX*i&tq?&15@EK$CSBfE4ayDRUe!`5IXh^@KuimUzw_bg3*3pEhZ*d`hw*5 zbOf3c9HO69M2Fd$uZM`0x&Wr6(GV(r&d-^ugpEA8@i&%GF2(rzfTgv_8;9X~I;HzW zu{(=I(rRnG@MfW}(&?l7qy*ZbrYzt$G=ex7=9~qq8dH~?9KUsX+n|wTVIo6rqL<0Q zrNOBvX++DL47u$c5HEODYnmjQ4`2GwK8^HXaIk3v5d|U;)komWg98#51zA=0W4phH zkTOQd?A`6L70}_BIA4KFAT!KS>tX~tNADOB&FRA}PKj*G8N)mbJuy+_U!NZyf3M_+ zG7!WJb7Z)n0xVKPMS&!s7ODe;<7XsJGCV%;^5FmogBm0Ffg&PeU0Q)A#{zm&L__MXS7j=TtEq{iv7}36nI2 z#P?Wcc45S2CnU0#e8MyYpiWM&%ig!>`$idaj0_TtwUuOJUR%ZTAkT2tAW!@KH*|e3a4teXk)-;;l3&cL5~7g3qL|{|YLW9G z9e_?y%4F?3jF_dhfB(HbOi#iex|RNTzFU2L95=F?h|GZhOB$_{>Yoh?yE4DK8wln7 zybBD%!~cObjE$3jn+=MZSwD}fmD}Z%z?Axq163wu7RddK&*tId@^E;FUwq!1KRguA zUg$qOEbK4*6r&YA92|v@`+4zKe!Dy4qZx=>-;KlL@q9WvJ1cnnUXNOAJ;e8k#(?G;);OJr^0#0|NB4h2 zC(HIyM52EUEwXkbB`^!~@-DeTy_ZpO5F;jFdo(Rhr=`_DK)Un@CMKvs8SBV%JhXuZ z!u%3U$bkWis;-=_3*|NgbQBX?$*MJDPM=z2D9~mKY34Zj!YL%J2W>_qf1R9vaW%WH z_}Wt~aBuu^gc$InXIdDFR43W5I8=jhn*I#gm&k8;!04Ds| zVbR6vYDU>dg(pcGRvkE%O?jGpu2WB`2mAoycLFFXbN}gmUiWUvGuw9Wp!_AnWaK*> zwCx4Lucae7cgsjM`HYmr1#pM_{fQC?-cogeF$w|YFJJxHM5+9~Q-+Ruh|fCB=&ReL z{t(j=1{ob`JE&~%U}1@)T&WVS#NRWa{=Cfr-Auuizt3O zN&VC3KeO0#o0hj>?BLvXIiqFbO5v&gYd?S*)AseWRn%gYx$%(736rzu3xu zg_Kl7U@(h^oE6w*okC1ey>m-2@B}51wRH^~(|b%;90ki1cjVAmC4!)=fWro>9By|` zl0}k&s4YCvf}1_#6UybkuiG@&yuF=6UUW}G)P=DjrjbyNks@1@FkU}<f%1d|*QOLAw9@Y-9WEUj@}O*9XyCKE04Z*gCi$oC*&uf=3m(RzrM-E z`WA3zi=X(tyO;YI29yg@-%LbR$@P#?UI8JGyiurGSN+oHWDL4M1-tNcG(0v}&l+1} z5mV2-{663CZ@X1zbB~0F%H5Yd9>hdyO;HO>3!va4W-5?zYKj!p=_lTRct8I4cg%f? z)yuihn_8WUSWVR-PI##l97(Do9^U`_#WxktpDI(*Wak{SM~>ZfT6;dsi%LE?CT*Ry zy^b_?+{@qgF}}?4RF(uhg@dxc*K+EK(^x8F<*@uOTYV+u%7K<){(xQ<;#9mvUaL`w zvN}B%Sf*b#fst|Y#-%ouUj8*12cZq1l1H$0kf0z@n|^OK7R;DBrgeUCSXyB@0q|!1 zlQQFh&Z^D`pQdwK)_E9}%kbB{E!SZSfU%zt%dv`J|1~bFDQcXwwa5+o#eWNr3rom) zL8N-KxDyq9c!&6^!%Cr?rw30$SZZO!95Q35uIS*4;^U}#O>U{>9_IKJyA8wDZs?i| zN9b;Pc)ffN%58aBQLgmnK@*XO>w#ZI%R!}FXd zR*Y$8VNEx-Q-6MGbbE4V*j8@jkZ?bTz-STMt)n|@i@Rk1*2;V(0^cFjm*;}Z)X3B6 zct8DxieaT3UJ78N#L{ns{oi`kX{>UEI}l#U+T2KXn2Zfp5NVT^P6-2{o6WO;vgegT zYAqC3O(ss_B#m?vC4T8jvb_>`a|=?S20}A8(4i$)h0j^07Iok5^|S7cy9bk#RF$g= zvym9;Xcxm)Q#tShWkNz2^#@gXoVA}0q1d6^w{1U$F}TA>H>foXDSUN>7q%L{R`BvI z?OI8|kcQddp)^0DtEA+XGae6?eZ17AdFxzCnwQe6*;@_y&z> zfYR_LyvBua^AwuE`4Icp>mC?)@8q2e*j`C)TO78StKtM$CWu%iGLi+f8FJ32*@3bEwSUoENJs!~gFk-K!34Lc<^WL|%65WOH> z4la5%GLW-7a+}L@^9x>v_KY9Ynf1!i!XweU^+<~1-nnl-QJJ>L$ zDK%iDmLxVLUJH4&{{(#U8rRn!h@}jX~=u#_>$A-p2E`iDSYUIo{~9F-RhHW6v>J87MD|={8`I!4LooIP>A>0GepK94rEJ_Hd~rs?>TX5eKR)HY)hQg zy*KqF0K8VdCWfmyalCkrXvJifRxIik{#aXXV^b7+$Fk$Pt^ScJY3)y06fQH+zj8vm zueND!-D&RA4>qNMPYKjg7Py9L7Z`GSfR;UyL<;SGno$YnDw=$EN2^vuv-9^z!A77S zuF`1m2U^@?8kkDA@|kBD?3daG0FOkHDO^-K&I|Y^FprY~Z`K?mfl+)P!OST41)RHn znfUv_6Xuq=PehM_vs{fh5Cw05yriU2K7`a~%^5?3Q%$CMW1+ZhjpP~UU`2#2^bM4V z4at{4QUI-Z&E%!v3|99_ktnh8f>M*d<rnKU9YuJCW^NZKP`j~0H9=FYeBn4Vgo7S4Ja`|+9t|$G2X|7kp2`AZ}Wc2 zD-Gs0Q6%k{F}*e2xdu%h4@38i%~|(tF<*>P?X;v{WElZHH2tzUZM|5JG4DurSMUnu zTee)-3UsD>^O)$4Zl3mImeq<;96c#>%68hXdouY{^WnzY5*0~2b+qxGYp>UtR}1Sc zuy~#dKT17LTOSRBIU$|VNYKObe zpR0#)oBkWtsoM2Hq2LlU7EMJqwJHt<5QciJS)i(!Yc+p0D{qz8C+yA?DhNZdh)J&s zZAeeq0LY4jzgjaX{j^Ub0EFjWIOFBXiW1V)`~Gm4BT8hG`5#scJz`7}-JWn=J8bid zAnT()E%sM-5Q13zIaKazJD368pW%zOrjS#pF`#bAwnvuhT77EINB*?4=^tg0 zT0I#Q&}?GttSa*(xnO1GrQ7v)ZVT7++`fFnu0|TLIwSgi7OGm^YulCKV2@E;2FvUr z{keP*7+rj*r7yZzyjJCDlHF{&bkbm$Z)&2JXMS)1Ww*TtN;zUR%yZ7Y_@4JZlPkQi zn{iT)CP$?@En+>}&H!X30F(L<`Gpl{&mJ>g{nc0R?YYP1d1rO~hGArKbZrEWpVjU^ zy!w;5ofcW1J#1b*sZE$G!_ajH5p}cgfMj=n8;V`uUDyt-b-S={mI_~hv_-i_)WAG- z1s0j{wiG<9h?3RvG<7NiwOfiCKMhRsdh*RSgG0bma=w?P1*L<$J>v$gf53+9vn_JHLje9wF~%LoqF~Fa>zY*HxIRH% zLc*`tmfpeYI06aeRL?Z8m}yT$Lze3RGoLiw(*Aj_7Td^=`4|rcSXQ%S%kA%8{o`_c ze5q8|UU(v*?oHElSC$qQX2;MsTrdv%Bre=hyUnqAWcG9MBDRIO^O6!B`b8vfm*DH4 z23zyNr(|b)$en;C9j%^ncPgJ^pq0j^#hk5WQq4J}T5nK+J9s*~RfQ12FA$c!q}e`& zVwxKRXRI*|G_(t%#rZWdGp0fmM@epV^z?-wqpUqA`J1ujVj_t>VQ|-HGO$y6vdVT! z&L4KHhJM)`>+-*{dG7y_iMk(e5!ra_n@D*$PCqD~Mw(Sl2*E-KGM$^4y;mwEc39hR zPI*8&u~9kapfF(SK+dBRo?%WO}Qj!Rtjr0k!NFvbU`63*5ewwhn2=7^}UT3d?; zjB{TmSNgVIRCwYM^FURkcu=KY&CX}`p94IOgBCZs7b<_2B7rUu)^{X5f%WL}!7&4f zP!3WAOHWclB9mnA9(sjokp7MwGWGX^+Mk1XkqDFL;Xz*9B;IA2Lwv91gZGRPF*n>& z95}B@l(JT7&n2IvDeG8L%8)j%8`fPF)oosRFhLv5wN<>g`1Krjrw~kv=sZb`Z^PQ? zIBu%zGD=nPP=-FHR;_1gH5`oPbW}OSywZZ{^mbgwX!tGFPzF`oB9*w~{=4x9;2{vm z0=^2@#wuNJn2Rz8#fE=L6}t8SI6Zwf2h_3gK=W@AI5_jKI+^O~BiXx2u{G2I)>SX4 z4mymA@tu5-s&!~;58T5ym+37SL4BZB7rquPf=1EPf8BeH)X`L6`QOCpz4^VQ8kAlUbW|9zBCK1LalzlIkc#|n_iO!{7t#5Mda%x zQp42_2(4WVNY^Gu%uyU(KsTkwS`?2!iJd{Q=IJTzjcKb0EHHg6P zVdLBxv7SCEo8Dd{l@$DeC#h8$%l7?Y!#HM1EV=T8Pt}x%wRt%XO~&Fv(;`lL3U}3fDz*BMi<0QezJ| z=MUvyjDBVwd3q1>0L0|8SmxL;bE9XGpF@D|deaV$>`8rD9kDWv=ttQiE6U;Dtb!vH-vNXP-{a7O|0UYwq+M~8gfo&vkRi^S>oT+>oJ3!Ap=jiU1HJ`EbLUubcJ z0su@yqX?-K4spVv%y>s#A=A!U7$1@NJ(}BCaOk?UvN!x!U&z_(%-Hc5=EL|qD@MOt zZ((SNLa%p8-LN^-$L5i0m7k#mV#whD`{;i^zZPs_QnAKvh5ppjkoXgIg$JIM=+wF`XuW;ZuA~riGu1K{=lB!IvTy9 zFPmvha$C@~GzaSg_f_dKCTy1e$8hC-j(5N>dNM*XoIOP7)=c?(XvFPRmvO@Z0ZNZ{ z0IFHJc=(+{?1w!kp}=uq!n7=G*8u}cPj>$qO12qp8(R`f#y8TyrO`OxrLaZnzyq+% z**ezlfgU}L6PABdmii<20~X`&kX4a?PqmxnmytEi>vIaMsM%qeJhrpfdDyYTpQ8u( zmeMv4BwHW582!eTziNvoF4+IHC6>ubUFTrbORE1MwsM`647dS{THS2f`V~Zw^0;|f z{^yNGeEYKSb+mn5d~L)oy(#U*fU+yR0$;HEwtcv{t84ux@_5s-s^tydv1x`ZXWP~t z$*;Wqz4vK-wWdF!@U`_wM0YQI2s*6VUNp{{W!n5Vo$b$-uiK>6VVjM$R&gHYqV=ZD zx`{{e#Zn}eHg)%k%K+G=)IJhig{#i4Je5GAo;Gx*Yqq`>{cp8^dpg>-bi%{aNiJRf$#wDH2cNk2TJ*HP`1a214eirZ$n;EJuI>|M5jK_ZQ@PPH($*VmI1v`an z#tPX}_xed~%4_G^tM1k3YmCosZq#?YPi7>NkIO2jT(Pd7aO&sq+8Xq-TBppSACREC zNXp#@lqAomKym_my7EiDOpWG>GgQGx$O${win2GIZWHU1&#_h3*KEnvd{&79mt+Fj z^a6P`6wO&y38r1rM4PQv4kII$^A*ZxPxI{V))BLR?roKKby%Zz?J&9jzm9ze@B!Tg zmI|w$tblHdic>yjPI~%N*=22w&9?zEbC@3L&a29OBXFYn$LDH>Gy;j*?O)o!l&JE~ zzueE^0^3^>1_%wUvmc$m@3uxk_Zd6d43@qIrzQhJ(>p6303V$c(8g+Js z53AwEnSqnX$~wy)b79lft_QOtE2!&=y+K|{`>)ax8uJ?KrW%{%l{07eT0z{Pv%@H| z=DK?SODI@~13~omTE{>Mo!-XTV+8_kDIIk3_w6zOUy6PDbQO4U=j$K`qzHfSf*nZtF@W ze47B;qF~vFWfmV`7jZ*G#ph@#NT<2ubBP^6bSlJN_+@n%l|xqx^Z{uY&g_;bDjAP7 z-!;FoCgFK2>OmV&F8+~X>Cjf$Gap$Mo#oO(N42^K+STapV}C-ZGhjq7hs(OyBRx^< z^idu~a_F_74qv-7)b6UtTbIB``}KA^VJcBT^Ny6^Syz+Hhm(83uGto)#S~b>#M&g- z@ZeTxw(lK_5sG&NccQZO=8p|+3$LV3fYe^2M1C(5=o-{NZcbQ7ty{kG|=K95}a(Pt}ZJxw|nb>_9`v5FDSMz63=4E zX7;vk*j%#iiF<50Yx!nX^qE?1&f9{uiH9f8>b83<3eY8fT(L3u1l$pe_RRVDAO7>y zn}J@8-+^wxU-;nvO007>v~{rgRqH$)Or3vC=*?>Vc86@gQ+d>2Sy9m&Lh*UL7~2Kq z#x^lzEeROcK!OH~zq8%>5~@Xs?($w2;3)qF{w2=DC)12t6NC+g4W)(-X1wn3Jb-|8 ztfd-t*CUkTa&Ys0)DPrW60Q*BxOadTQJELAh+e{`QGS8BvTK16-HB2s|BX{ zyj{OExb0v|sA1xX;`4d8nR*a%uG#j(R=Gg_86>PNLT#T(4-PDb7pl%j08JCRCy6ZC zx5nSg1gfQEIC_7X?PZ{SS19H>)chnoS@(*MSW*I1Y@%|7!)&BUnZlqEJ_@0`zEMQ4 zbI*7es7k(oWK0@>s!oqCxa+%M+I|~5KdXSlKDL=2l;hNA*0xG<7@lPPXU|qK=q4Xa zqbd1PHc7VAiYglJeS6K9ZkzWEOfRV3Qbyzb`SG`qeA%NW{9bkJhlHu~JV;^X-P~I_qKNb!(!R?8fUCt0HcjZ{hOf;{6a{KVs z7*=#q_eY*e>)j;R9xZ?*vq~eP!&X2^;|L$`%*eE*e7)}UNBL!5n?TGx(NUT~sYjr) zu=3a9;e+l~dJ9eQUy~L~3O6LHKh|`|>`jH5B%t6(?-5$JrnI((V~!kH0BDM`AyNu; z4h-x!Gfl1>t(YYvYEq{G&@-ULU>968mOb#O?| za_KpYqj$wP!~D2pK2lD;aV&%+-AY{><5)%^KTl8#rtA)dx|{)#ZfoCelppi7Z|DZf zPFT7)Ij%0gH$^vNy;Ix2$llTl$?Dp7CzkX=OKCZ zrz;6N$o2hzh;1N>IBMNLq;KO0IE-;Rh<-i>jNZ3Y?H<~Dn-~utu<7t$ZBDz1Iu$k0 zdDC#L{;~NJTJJ-XeIi2wAx}Hz_K3r))s|u06qN_pa>@D8ww_W*V%l)Jcr3}b9j>p| zXWng+Z)$75)&vDa2Z>BvP z+V%6M&eUoeP;|gB*p7Rzbf3%js$iCLILX?vwWE4w`1q0@5f%!P#YuC|c(b(qw~ZNa zBWzPl6o`?aMvdV`673rsT~_o*rC}B!kP6J6_KR zNczAG$@Jn_xCD3~EN86<8u>WR?NDHo^XL~fOHT?y56-y@?-L*XwbDBJk@`9Dz59Ig z%~|9A-x-Z?vvjt!x6^mAw>Gu=M{bMMCjODz&NFI=k%7(Vq*Yl)2qmJk+<-tfW)%0C z3_)qqaE%DIpQ8dlF3J!U?xCL~c?$jLxFZ8-7@2A7F^zV3`R_ z__YKcRgDcDfYRydsWVm~o*j%iW9js*o!(51pvEfVWYVGjPAeTpmfOSOiczRvDo=*eG*hKk8pjTU=Qij=HbOmg^!87?Sk z76bKX1I??MVIB$^#oy{ajceQbJ&xNkLjXsDDe_H_DVAYP57`b2t6?=wc#x&t9SD8w zQQQr_=K(zN4mh~D2gLBppuJP5AiF1na`whNGSR*jB6#9ZH>6qj;qvNb3sUt04Ye_x zE>BPZrZx;fZkZ1%2u%5TcuW!Ken$-Bz8r^j)%Q zZjj(ApN;t#l_P^iZcagch{Hk?eW4(yZU8!zCABf*0dQ#VEQ(-~XbF${i`uPRaka#f z%i&-J-KU4oHDZ#S%*9}Eap6lz3`MeDMT>xJpgMa#s&1XHapMv64T6R%SiMuM!LvRT zDO?>=v3nfN#Vm_wYu0A8e95Ne8MLW5n5){m+r}xW_xf!rv}1@R|{+e2GlNH;N*$1r|N;U$z?7=!e*imj_Gzz?6M zg{*l^3U<-O>D%BZrfs-E@47KWZ?#1E)8g;P_O2@p+q@Z znOE60cl@Bf6S?!X?d-CU6YtZ>0SE3$rZ0vT`62AW{BNO9tgID?12Bq(Z?|(-l{vkt~IywKZtOR8>he^g#x>1=aD#huaiz-Yk zOe_nE*RXdkH!dm;kBBhWh)!@f_fK{NBbo6ouApaWnN}b-SZ1|?5(tj`N3;aiK`#+|>T))S@zI820WyWfa3C(v& zT_*t(*QV(VE2r}uc`5BiPF^#|5qv-ly~usE z>3vxPU`qJ2;L}-+gwBkIPv9%H{p#(PoYQUL#G199xqgs%76rHC=JiP0u?vn0DDoT$ zIY2jm#gen1nvFdvEsiz}frAKM17G+H)>-Hb5sF4z44(pmGiGha&8i?74S9==$U=tH zn<8ZetCJu#M!u}($Q<@ChdN`2tafUIEI9XW6+Em|>FF-@x=!Y6tx$-^Cpo(5gbb2&oAEEU zx@e42QQ1-W=~0#oavu3z_Or4E3+a1Xo=&b*J$*bTZ4-rB;pCtCj;AJ^8umhNVcvq6 z1&U5->msVsOC}zVM@#lwfIZLah$riCv$9p0-zwIt!Xfu2VDOlacYqclwRW@IQr&=> zReG|7i0~!+7VuVmJ>W&;K_x|(NlM?Lz3qfIFlPhMu^>tgoLr&c)OY2Xwg)v@G$(p- z9kY^e7C~b@*Jl-uZ;E#jfC)opig_ZHeRbZggB)IJtd@EmLjqcd!h!J0W&jBD+99<& zY^XwG*sCdT+=?bjI&9 zWr#x7Eq_7Cs!5soTt-E+;tvad-k3f;nBw)f^H(fude@S&Qs%rY;>@{L0HqdR8|Iwt zdJ!sav98%jw#m6`-xk3+xVyqKww~RB4Dp!gr+u@i@bx;olCv?P*N7jSZ$6LD3Tl6a z9G=<3=`TMIpSE8)u$+TgcFUC|*@=pwvqiYzVG6eQUdbL|dj^1-qtZdYcENVzM-QjZ z?71IreBtll|8@Lv#N}qX@SAah^&RJV|C1A8WND{wY2xvH8ln?=?ADmz09Rfxy)Db3 zwrJFD&~bUTPSDjHVytkW5xoj@gUCkG5CeZ72n-ObCDs#F8rR({7#0RlPQD6;^-(pJ z8%cF9uyCU!HywWv;zS=_gXqxmvDK>$ymr785G#qTY3 zqQka!xn%8~<;<0R`163Li(%Jn=-543q<}#02UxhL=kSeu`MF&w_B8s*1x(z3@TSWs zlEsA#<1RAOD_?(&O266SxIHO&l7KqgnJs7z@{W_q+%>J9BzSu=k(v7x{BVORk-vbfvlcpu!-o%|R zt*nH@8}CWJJ}Q^;5oZ6K^s}Z$s6kT^cw$!7R{o%bowE77#u5(q_!lht4`TX|mOJyJiPMx;;4Q-rf8@Rf59GtV8NP8#u0BV8` zvOKr8j+TI}1H>%1rLIQnH-yXxYmXD;T$e`}(38hC0TVZ5{Ty8#glJFldaiXztcBeR zo*#j}E=oU7o%IA{FcHu@U)$F`bcS|@1JK;YZ&o-=Ep#qCBRbdR;OoExYdO%aSx>M_ z>K(|Ys~yGWlaDbgvnZxy$+aLXApn-RhIVMIp@%Jsof}mh4wFkpT$tg=`DynbP|gG& zx|AKb?QI|?Lf&%BD7<+JC<@xm5diY@wdlTJSaZJsC>EpTc49T{G|Wjrn%*yoD2D0R zwYAgfrY2w`6Q816ZYMao z&T~#_lKx}az8NfL%Y6>&D!jgjo~v~yWn^@ouhmO)A8SO6=7--rgG;EA~ zT$tILlY@~HSuf^RfT61Tj+HKxCT29DPI99%v>)C_JmXX6E#T+-rB-qIS^{MUa*vZv}v!`bszE9FakecD8_(t}gtEA%)i$kBm z%51op^x4ndp!;{D`#DPHTc^;y)dN~5{NLjKX$0v=B2@}W=+Mf(P4nvzSQp^3+iyN?b*%hqUkat38 zQmUnjnKMOFC)dkq{rgX{SK-|9)n=@r;0eg3?MSsaA8xtNgzo4!>D_oDtUvy$FKF&w zMwT|BL~bzjr7P2Cx%p9gsOIVJJDeWyalXSYMqaNJ*h5^OC)){kVKIvHcs~mPMkGR^ z9nYv7?X7n_&&~uAGNVJ@@9O{k!4Spi$2a+l~%G2hzpen$1^^uR(I)kHh z*9tdmQ%QiyUY=eE#Ah#c&Xw+vMOb3xvSXTx=`@zWB)>_ypqkR(#So#y!JKUOy%L$f z;Cgp-aFOn`mDrZVqSRZ1gYfHNk04Rac6pL->*$UQigOw`tf$=+2Pb!yV$e2@ zVtW3Ei@|9fnTG?7`x8pj-%B?0r?phP@`ifZ9)G6j?QJNEM7nQ4Yq)nA=y~AG+v+uv z6GH)jGMd$3D|%ww8zoD*A}L18k6vMEDK+m1f7vX<ax}#eqYmYILC7x7-8q?vxJ;-6`i9|RASdR5xuNyl7C?#T8u-u6c>@A~}6a!Dw! zKBqmS5){m(cn_NHBP)8YYl}0)aE?NVZ-Fhk@X$eX@{2n(m2u@`Ror<0`<%P?r^ z0G2pb7u)f~*C%CY|J%68vq$l9=VJfwll4`-Npvp}RNuAAFLWuYkP!Gu=i)SBc%KQ3 z;^W*gv3`MyS3xO_iHDJp%i_Fh9iOrS1<&r&(%E}W!TrvaP4e_jTS;y z0uhPFv)@A7%8!%AL;PXE5IS1eZw?-<{xBC>k@Q5Mu*VH$t1ce( z+qe1l0K~cqxLf=JjiDm|emXS#v&rQoQ6pxf=Y#t(5mx$S**su(Z;yG9_o&z^s`Qxp zY71Qz5dX`77GNSUl-!%uD)hS`ttb->@jdEeH>4^tNK||eqvV>h+({TNrQqo^y{OgKCvoOy_$ZmotP!u&E>}jV9-V-v)L{!_ob`Qtv z9&6rAn`oKS;N7@3M2tsKc)QhDc@uF%x3qIHb+R!vbTc*iC$asX1~O`;Eh@pJ=@q3jgxqEy?usZiRXk# z^nr5mwQ&iwaW&&)M<*CSGEBvG-uvxf!$fx&#`IoOODu%tFvNOoJt4q0PGf;($sH+- zYO63M!N`RblwYO3_Fe_R2LBOs3*){{n7FR!DGJmvi%iHoA%luZK{o@OgVCoELs_V) zfyeObH=t!nb`gRJ=7hxzoXQanu9gajD;dB^Cde2ab8RuE5(IB&Qmdh904YaQu)V(> z2?!d-nL0-#?Xf`U)Q+Rk5@9*O3rA$T#)FZ5RZQxNjL?)89s;C=Lf)1}xh zWQgc6Lw9~y1Il+sBt_jjy0oUDjb5DZDd`{sgUhfWrHB(Ma{r;1wS@u5nK;%H6_ zh!_~D`WBHw?ZAwthDxv6mG}Hi2k4n^py$3`aq);cj?6A$4VR;?jTDPFC5+zeHjvgW zMosw~1u{tQ?Up)f`zGj>(NS69q#NO2(Ol(_nIk*BOXB4x{NUi$p>B}?mcT4A%!5%;?b15hG9r{6+)1avg6@{<<#JzDHmR-xQVEuOP z!0+ZYSGK^AQt(fxN{F0p311y3vGcb89EeP;C=eZ5iob>gqlzy@;gSD!jD%tTrrO?( zi%zfk57xe-@Lq?9w+lAMn{_URcdFt;WyOgKx*5QddN;6#RX@Jm9jpw$RF8_nuPh!I z=<>{>wuTHd)gSF@Wov<5Ta=JX5^Gj&^pqyMN`SDesT(uiY(Ay?F8&^$4zbRP4KJkk zXqR17N08qhbJl_F;nAr8T3dxTY?%3S|8Q5+9ayvi(lU$CF3ISpjIbWls6sj%Gxrk3#WS!+g>EbOb z95qqtn7H*Pd`WIKsNP%unzLY?;4e*UeLk5icttLougo?I=h&Z3UwKab+KcM6+R>(` z7KbIp)qxr*#RcJ-r9W@dp)cbR15LDhtpX*o&(qC5_hsvJ$H~!m67t|nPbq(DPgwZo z<%dq}td?}r-HeYvm!k=~Tzx4!)rv&%{nqV;aGgj_79I-Pdc`*Vm!a^;PYt zAi*MbvX9L8tL>&gW%0&I1?cr=vT0ID_x^h~|1T%S4U)Fg=u z2N7XOzl6ZRl>MD|VQB-_t3wIHwubvIb`B;1t`((kPHL#}tIa5+pt$`lnOIX}1?yVN zuK68uw}F#oL?ph}vv6!)X=1%Cl(oo~43d}4WHNDuJvM%tc1s2>ed$l#k^Vl#(B zHOuPC{ZX0*0Afnz1mfgL$mdcx?p8 zCCN1_e@wr+&rHCT5TyLJ=*?|&E`mT_F57}V;WvYH+LDBf%-dZ(_c*BoMJ34dLYG5dJ_iBxP^wRaGf8N12 z$&33DZ<~Z0q$_ijo1z@G_Rwm&05g@8KtmUlW>fL>ZQk?4mlty07O^N83qk;5MtfS- z)>%WKjTXsnR-E7(uWy#*+%*41-$oN=LMpri8>R^Y5%CG53Acx8GbpM;fIjGo|Ki5_ ziNane_8K{B)4p*eR@thf|9afHP9}@j<*Vm}#mmAI`QN}oE1S9|`tLj->-*{;@_%p? zoK4MbP3>I%X=Wy={=d!4Q8BGyt7(#v5JL}|103|? z^078!g2oGR>5U11G&ZQouBKF6y5d~Txy5EE!UnV*6g3&d>j4xK=&8Jw1}r&|;4Rry zmmxNpIkxXwM3B%Vt{O3Lt1}J(Az>IJ_4tK6ybQ5};x(n7jRt(zmX5_Myjf_jx(s@t z0X)Wa`{0K3>$#0dyN4v<8qDX%8zB`_3OKNh5C6Uh8o2ySd(w<=AbosytQHE9$#^IX{Cm)<#AoNbXe4KE_!8M{7Mz zirT@XC7BQ&*xguSbe3pM(7wMRyMayp>{NqWr=3ux6VliYSRU+E((M~p{dO{_GPj#3uSaDIjz5_6i(?%Q1nVr+7E)t*<*QV3Cz!^m zj@!ETm;U=F)6%i$nVu_2X!Ux*Q+pbIm&K$gzr>R3d_wCO|1M(jI%ybEWaBW%ELJH^ zMVFm9!X?VvQ*DNtWXQ4e>WqpIq{G4p?_+7|?5amjKia4g)Mh zDPZ2D^Pi1Q0_l|z8m_Z-715GE?T&d*S`)lb=RO)Q2c&NM8!7qEs|fuM!{_}MkHsg$`&bSHt1chE(#}xDR0HA`D+aX#ot0XY}hd@ zzjNjp4gpEX-|l(uL%wGMqzTFTX5N09{k)QDKp7GI^{@8hWczt*;n@_ym0hF3vw% zxjS)VDvj(B?W*LzPIa%*q(eIl0Y!((F}Fx^cJkf=PY~SNH1~X4PmscN3cV^nvCzp0 zp)~1L?U{h|tbTLXN{k0!+5iMBd=}%vT{MxhBYs?+xAdyaeWLQ4$gqa40-?)@h^0oL ztHy*8OgHFlpOSy(i>7H*_XqX(f4#%z*h#3zITs1Z28iY?igO?7wuKs$tyK|9hI#*|DwU_O(No7!AMPI2M?TJZck=It6Nr-dk5)FC6!C*aD^a~;&%Uv}ySn|$XS~<#FS9xyQNss* zmuz2$zd(n{hxS6`D(p?MN3jlWTaP$?QvDN4hFZR?P{BzVnq?Js)DCRp1E@P*Ws z3*WtbLe$bk6Bu<+zvJzBm+;L7*S7L84P|KqNf?3KkIkNy{BY+UmW5x+QH{@weWn@3 zS=6F)%-d<2XT_2d>6okwwZ(c%ff*JNX1&2zVOs{a^Ej)G@=kPX(MU1dxz2!O6zP(! z`B#8|+GYd^`L%4(gY@2Y4xN5|AmyEOwz+}xeSewtnGx`WS*53P{E>@Iy=n=a+(01* zf9%=sWhlv2S8KN!Jf#L=p6qlVmwSt;>&LPRRjoNU_XX7+* z_QhrBO?Sc6F2J9uEP6UrHYwLJOC9MAdudr*q1(UVcN_|!hofa%QqftvA2HR*<+@;J z1XHpx(0vrd82jekVm?N3VFkGjC;+zTQHkGZ0~vRT(L{E7^2p=R<>JxkU$|Sp&k|Nc z8p{H^GeJIZi1K?!?%VARX5KIFk9O7oO|$bcrMj9vay7FKW`r(59r#H|nO#9TnqAPIbucwlSnJ`Bf| zrqr&d`3prpUv5kyE5uGQS%pRgcD9J&Zdg%0ww1P3OurHS*NX_+wbhTT@BX3+G7yl= zf0BJUIGMUQ8Cu$XBX+)J@4xgKi!{DD(!F@g$1gPdDnp-zV`GS!WCsZD&ynPBpoa=E1kl@Z2mnI&FQb!$k; z_}+4X_SZSE%`b?i~_2dJ=O~QnQkTAgu;ldqS~5U+Z8>gd-O ziO7^RHA!-*7PmvVYvX(y&kO%DWyuvl9Y1V;-O>`INEH5($tp7qEME!Lo+92kq1lz>#_}ZUi(x5dE8-@vogy-i-8qnqVTnJu9mGC|( zxHS+9NyJ$|>D}~Ryw~b^FnMt4uZQLYr0K14=Ffl%)FTI#+OWH$=r_9|hD@ zJ|p+Bx2NC1HFk{hd~oz7s0$r~3mv7JJ*ix&y3Gzs z4sy;Dtq#C%wWMG2x45FpF3PQ71NV$Qe_;jkgIr$z-zjO$<_xr$F7PJuIPRofvdrm{ zj0^>p-$QrZn_c{Pg-%}6*baPujob#il%fem^VG665tDziyl|sY~)zwgtG226SpdQmzUyWD}LfsGIy>Mf0kG=Ky4RzQ3}6wny+4g z?vTUOjKgYWI=C^=NZ$o%h9M$SQ?iv(mQpjVfZgKAC0Po94{tnaF()yTUa;y&h__ex z`N}^;k!5SCVi)VHV@BpZ8adW#gwPNApX2<3GO-UU9EhPVaV%{DxJjbX?mD!&TsYKc zlZ!{%%KLO(HG!VZWPeAG^SQNvuDmUf(oQMMpOKLjKL{v@t0zA_C|UUT`1Gx01(>df z=na3Lq)xp%30OPv#d?OvfD%V1u-paR-4JY4Zvamv+}8m#lt)F+$?8P>`evU^2J9+u(R1?2c`@%hdR_+kbH`2^41=scjV zR$z3Ue&69F97&*Zy& z59}#J5cZ@i%DYLLZ(UVoV>e~C<$JDoR?$xOPQ;j&Do^@a=$E`ap}bb{_BOMTC6dgX zp?PP?BK)>449c(SsT>q1+YR6%aKhSi0bb2e%RoPJt5zkK?TMca2>O7-s`z`qCvJIU zY1%q8&tKWKff1a%+(c;)<4<*2x6+#GHxZ0sV$r!#Hwz`{_!=lNfoohZDPEFSIvHYw zSzal#o!7AAe&~4man$^(!V0gb9h(XF#-pf(9lPz_cS%`|z&-i*9s2FBYtgb?nwzBT zave7HHHlj5Llkk`mjttR$sTh3)O1JemXC9FIj5)m8)4G=4XZq{aozdfCc9ae`G4CG z{;xBVB>$gb(B9tW|MDaxX6EF2_P5nlsNDabfA)WWCqp|!8&5A&eG^k7*MB^UI`}@E z*Enjs&Z-2{Je6ybuoW9kwUrODG72l~StKbYb*~0XcBWBG?!-t`6Cx@+-n+Uszy*QG zey&-1W%g6t)M(H>;s)f@q^y{wP^vGHpH2C4;KU$6o?i$|>1Acrrd;v2`#*7oiRBMaFjT3Jn1Vy|i;1j&G1k>bCH_7Fef3Eim%e$V4eP}M zpy!yA`{-6SKYJyj6?yeVBt3gHQ76)=>Jv8ME`09U^D+bV@TXh+gSF+|@j{_jL+t{)%NOfz6oh%7xS3FYzV&L8 zc5t3dPOaV39&A)6HQG7E=dae#kD(DB%{PUMPr&lhg;`4s8}`hvhoKc} z;{#Wb`*n@=dG#b0lZXF^C%m@<4Et1r-xIvG1J#WerocOhW8oN8BmoI>4gJm^aMg$E zF|_qY2ZvmK4YY|c^rM@BZ`E94$Vu4Bs35o|6}+D3bfV}5mYJk@bQ3eJ+1o@-ed>A43!jbAk94RKIKLJHW(Bv1b z2hal$xEbQTF{F`TzcZTAjUr+SJDdn}d3-7xk|=`%SFfSS2zxW$HKBi)lii!(;|(-1 zCIP9eK_+pUM=%>tAJ5+6s5?vhfAEj_T1f@`HtLIbwf>B6q|AllCVD|F6=;tiBL@g~EqdImeQ@5g4 zx|&jyjtz|bx9tw8{>5~}=uNjjkTyXI_*gQXUIkWs0TPR(FKz&=Ffw*HvMHl8i@TNaMJG-4e2lt%t?fs_WB|>C^5{jmR2F|rq5H;X_EIOu* z2h#DTRv%9!B#=udJGrdv`d|ufWh)H%jO0^ECbC4lg6q>m!l;p%i(z3Gk#CjOdNz*U z?!)F(OFMw})V-wIx6G5&_X9D;;fHSBg&t|nTrB}gg$S#C{S{czI#El(km&cgxq_HP zmuKO+)1!F;EB3T4>X*R(SQ#}Dd!ZpuXNs_Uc>_DV0QBL^fBK0r461{xYixt`M}`)5 zh!Si5gxRw=WzU(*n0nN!ZqgiO&vIm6csl_BOSEON7#WSilB*CDfc5|)I^>4T#!Ton zvT%bJ;OMrX!uHFhbWPVs_d-iA9-Nv+8wq-F_r9@JdUmt!&Bp1)J=5=s>tuUC_>6?emm}e~nvkWU=5^4l?R$`d{eWu-!g@m^svP&WMAt$;z ztbH5Bq#ZRgymNec~!2Dkv(lkG_E#ki>*Js2lj?Y6yK# zRC1^G*$hc)tK6T?@_kl$XuA?ERM3L_Ue0qJHed)f%s|u^!)u7oKCd8V^ca-#%xDN0@8&LS@B3cSVVeZG-`H zhP~MBm^!ZpBh=E&2fS)@7{|cmHpJu%wek9mWJV@^a@+&x02&e&{;FIe!iTV27#~6! z;!(VSv*<2g;{f`f8#Pt&dFRprmnlE1e;W7+$^fj(9Y@07+3^>|;~02ZJwiq;mg-wM z#e4k=L{XtGyx^@j<$Ij7y&Q3>-1MI}gY@mHh|-gzci*krAS9H=yR}W#kteI;YGhy@ z23+I^Rrh7W{irm35dh6}#mCj~%P2CUzkMUzZ@uI94=IJCp@7^Tn9WkCOn%EJLY-fX zoLm?}RynWh8#x3*oI|{;%d9R8B`rh;7ES&VnlCNAqIO#;6if?aCjFWfh1aEpwc%1h zK4RCmE#BmQC1cT07>9|HnuU74<&HtER*w3M#MLIb{eVIec}_v!4G;N^jRngCquJ^9 z221Xx|D^DM!B-kmfuP$0TB6NjA{LcAJS$EA_4$hCx)KOfpLzEKu7aq0Wz6YUH5q{< zGQIjuk^v1YYL7Yui@6{ZF$feq8x4ds4a0B447{KjBtd13Lyy41;&%9kb3xSgiv_oy zr^OW#J@SLeLUavITmjlIGA19-EYt13!MKpd)I?JU(jeB$CH%otH={!XiF@GZfhL<%N7fgP0zY~&yefywHHwu1G} zg|3(smW`B|DOFl(P^XG{Xr+28`=LMg9be0k`tqOh*yvp=RZ7Qm@*>ixn}wZW~bJ$jGaV=Mx!AroQkRtzeFR?(xHSXZ+y@gIwU9a zK{|tEChQ!d`mvgL3eQ-W~ z>FsBXZE;m5+9-<&^T+L0YgQex*k2{zB;Bm&&_Hi+R?lV*~V zYRZt1;jhk-N{6-DUi`f*E=FSh)~tM)H}t!N$=~DBb(sr!;wijRV+JH&K`7$&9n=y$ z(kUx~RP5HrD>lXEA&B91#`aB!dNwYJ2RR0%mRyg_FJq=z6GG2sq}b4a^EAJseXFNlRU1D%LdaOE;E<0;8B)^|r2x$b=U0-lbQA$mj1J(1Bg5lgeAUUswR1yac-n zq5fJmi|X`^Va5)9g%TA8X>z{iLU;7=OQ8za$vPqX==C2HN0S@nGt1UFV~JQqNNXJN zN`zQP{Jz+-L>#qd6FCwq)e_GqmrM0BhjWALsf6`lB8wCEM8XT+B{_R0Vt$z+>^Fa9 z&_p?WkS(FJl)T!lYBn)U$WrdcZX8iE523q^ul0pR=%YyjS_S7-^Hr57$ zeq60CKInNYb4agR8<)~apBpl);&EWoYqmnc_;qyb81PSmmu_aPb|z_N*K9GJsHJ$A zYkSKwlQ3m*7c3G;k@eMhqK3klUJE;@6gBP2ROetH$1j*4I(iuxTJbHk;>xTKgPo!tC0`O%WADq7sK+)cB^agavJ9^nTC58V1GO7qA|JPY#kESrSsHg;a`Dbt^ij^X3nK?Nm;{mYwFki zNT)i6PVf_UpY1I}yez1N2IknK=4x-B=2Mr^rF!KE+YW*I*?iBpQF$|te{iO|Kaa$= zAR6#1JLh^UbB?V6_a!-FjLUtb(&?ed%Fhr$dzxl{H~cs`ag@eIqv7Q16Ch_90I4;6 zk6N9yjt342Tn{8=Qyhd=lBl_V_Y*~WTl|uNXB(o?)=GW{l$3(j$ZRpE7|DV~60tk1 z84O$8MrHTaCVmKomU|(a?_oP^EC?g!xlGD`iD|9K@(EY>3DSj9b4C=Kc+S z_2pu3XK#^#J!&({y;Sx=B}@+#$EZH+v-2+E~N)?j(_~#b1{&&OAttjwaoM1TKz;QnW zZ*VETskg)u-%^mOhZh+5lL>8SkIuer;iDKsed6;*{Ba6*IV{S$#86f1L6PIZ^0D49 znt)^ZB6+(8aAzrgx0b(biQi!AlWBO)5wW#g*|1vo$bf%%hFp9WZU#$)lRx!XDD2VG zi?Ez)&<1aqqLxQ}dO|IIoIVHohUDaz%J5rlwEP17m(FfhhsIMbVuC>9p{F^My}KGf ztk+XbC;=DSsjD@hdeW0(KU>us*F8pr5T_mYm8a8I7|^R$oj)0nb~=%yVpEPq>2k~# z)l$G+?MCh!>DG0mX_-~t%If_`HRVxTmZd=T~ z;r)lNt*EU7ndw?6#2wM6=Q}oQc)oJ39DjlGt+Ni9^F9BmYh}T+>Yz~e`~|d~T*}A- zSbK=4rbJaCeacY2d2S#(!+q)dOlf>u9*oUA&sC7i9tEQtTfB8w-!+rleB5%cuF1N3 zwZN!XgA5Iq-B77_0OkUFnr1YOMPOG9Rq}-!@{rwigEUps4UoWZ8%MK*RrY;D0}N|^ zwn$k5rZIRLI4{9`&MP_nj!&eWmHGSr#9~l z*NXO&ODk3W<`6rq2XPl4`7M`*q&b!hHjH_mVPX}+i@9<}R?0&Zot98qAl^$yED2Q? zKB0hVkt+Q;q$ASe@zq9Dc2#pe!9SbPXQcgUG@aD@$1VnzO1fj@#0BNB3mqGr)$;#!atz`YkVoF{?b4x+C+@*CApY^`_j!3i%Rhyon*nkfPw`@!Y4p1- z*N%-+L5h28pRLnVK@862F~(pHqKj5s-^*OxGTGRKE@PMAykj|aL0TD~XD=4P``wT@T-S+AK)J zNNXRdCYtmIcb>J7KO9$cL)$jn$^?c_e^*bPV&C)+ywU0N11%Q8NNkHY5e?6~_hh(` z#wxEsM8-b(kwmHtkheHUPis6}=WCk27}caFbU4#d(Dtd3tF8p3Z?cckB?@t=PIu07e53sflGNJ}4p=10pEK_5K@Q=M*GLv~Ah4ZQDF$+tw-D zwr$(CZQHhO+pg-<-CyrU-1nQGnGrj7&b7vv^p9`H1H1=^YL4Yn+m;hL(!(5}tz+d! z_%#=@S^lCJuS7|0<4j$+@X!y7xL;0L50*2cHlc0t?=f-qTqi^Tb#Rqw@CNfO3Y0$= zVH?-|_MIVfZ3?aF?#uW*zL-R;xM-AUQeDz3y@3@yE@$7K_*W>}8D;nj$>dUGEz=}- zCxk}oymp(+lmr(VIRNHahKMfnuc&Z0(JTQ9%fRmC=0YZl_6K!5@I_`ubUH5PB#R%Y-^12r+Ukl?F? zi!7@W*G?Bp!qFqvl}7|@9ydDL&<1-`$@zT+j5aLmSaERc6Yd%xoXzlMR|I`jgx ze&|w#5UuK65Fi{9{fi1(G<3r6-Tx^IN^Kd+xBu71{=otOK=R*XS3@&nLrXi`e+!TQ z^37>YUB}H4MBmfWw!TG3y~MC2X79PAY|e!2GH2`PDw1q{_B3KUlv*G_G;j?A=AWDG z?ask%z`QFbvh0g~r2fwB?(fe{o;OUcTGvp~9UBd_Q^?Pcqf6V;t82Mb^i=xw_s+{o z*HwC3y4`LW^4Yb|%1GatoiVMyl2dfh3e66*B90?Hp1us9CN4S;-AIYyTQ?Ua#?fh2 znI4lHgR&~EQ{3Lf%3Q{`F4a}kIxdlqiYKlwLYi#vCIzmx!?d=U9g-rX8Jb_+hu0-Z z)KYiLs9wAcD);bS$`Msn?u1yKhuuCN{hB`^J#M=XXh%6H2GdkKH`qERP%0*_2MCo% zRs(zF5kw!~9=gXA+~VVVZC2WBF35v2ZYv(sD-9^9yHLrpW}DNZR3$FOR;ZS20pk{} zw&@nwTD_cg7QNcxq6|8yRTY)#znD+mwn0Rk*CDMo_QYyBwyt-bdRl<{vY1m!9)|$lT<-0A&1x#f3vM zD8Gtb;sumS;|peW2-RboFd(y7fRzsNHejNbB1*$$DODi&XXekI?*f0Ln7$?|V)Kjl zX)*}RW%|wcpxJxPjAmez6%%+-5f81lF(H>&j9;+{avmE%Shgo!3{i)v&?-|b3*(`) zNt9e7Z-50ggMoW7ujqxwggB12x4&QI+lZEM0b0Pfwps0v{rEG3&d`EuTNz2 z67nvlXsv!$K*)%2X|9;dP)A3Fd9BHHZAGfeD6!c)%}{GuHVt$6IMKz4S-D>OqhyJO zqrG5ePOq5{>?keu3rKu>E-@n1>A)%gO zMyXs`@4kA;FTigxIScloyYwJ9!J+d9Ty$ zBGgTHSl^4!P_;|J8!(Vm^rSsR1U@Jn@5AqmHsp`RJzz7t{By%73fxPUYUt@-3s!^3 z9yyYxQ(QlD$Fu^p%@Egg#d`5d$-nLL3LC^9xP2pvaoJ4{#>x=PDW!(=Z{KrJ66r%_ z!q}IC;>?u0*TTv}cLb0s1s0?2FOSFbTMWYq z&=}AtR2<>p?+P$xH*F4raO=E6mv6>y9^CaVRk7zzd$^Fwd5GU|6>@S=hRgiRl$}+5 z77b^6rPO?JEW?9vN}<))P5!6=B^=^rVFQOAOB}A&KF87$6xDfEHR+$9~iN05Ec}2sB5?wB_}H z<%DKLLZ4-$Bcl>SoV8XlKv0NwTjzfHNlv5J0t(MtY-!KM1HP#;JX#J#{Aq5C=WRMT zV+M@$Z!FMv_EX01t8SM~b)o3r5ln^+yV$QMdYnXd*Jcw5_1EM;_NcY$;;ulsvHtn1sU@w$fhWfDdh!ql$v5W zOv!~o+^k47p9X1qiwboTvTfJSY@uNpL`8t>=J?=XMmX%^d0m3V0pjTzR}9e@O!V&%ZxvrPjT%;wRnCI9m_4=7Jb zZkcLRqnZ6q0V}jY8Pqw(#x;@WALonkpF@>DU?oWsL;Z2$N48p<59&SilS}x4%Ah#R zZv=6B*iZ_d%;kAIi@|h5_2{4vk$bgm9uaJL*CI^HH_6U?SLz*&DtOYZqjZ__s{K|P z@LA{)DR(}jW)x^-b*>*p5%KGDWL0_;Oua^0Js^=O7xetLVB}&k42=Imqj}0=4uzqG zj)VToL{^Xsw7o%pB2WsWetRCo)(3@H4rXfc*O*G9}uwD(9g(!z50>o5?2dM&& zMIh2XbM+(-CP<7`BMiSrJJ{Bm#{Ci|Y$-akypBhSixeXdvY+5(PI0nG;SHvYFpfUG z0KsCCI3~4m8C%}l(NYt=0Mdfco1j!uIUL?C0cclAlR7ZWYXe1z2*gTr*R~Kxbw8ig zAYDCqC+#Kb(PNjAcP~5{RR&-22-$8CyIm}Uf_uVV6Idp0mxa8#c5>*uB^<_9B0?u- zaB~pwM;On5l6b6p<{242QTSze&ky6^EzweYD{&OHVq!O*A806&aku%4ek=ms@MjbI zz|!SC`1TOH9Or{+w%b<*5oCJb4Var!j0gyKz zDof;I<9^NExZCMlFX!d00|}B&g>vphhdeZ2%#eP1J}PFjN>%c#exvX@$-inNV=h%F~KZ%T0h9{>Stlr!nrg+P3nu z_pxOW&(#p7=a6nsM`EnPg48{UU5{ zQL>xawd>p`IM!9-wJPohlvMDJoc=tAkVP0)IIoEN(2ZEs34<`~lyjrCNM6sxC(fV8 zp0h9^5>a-5Rn%PWr!~}o^$d^Z-=6rrSFG^^0RV54%MOs#C}3@wdKe%@M%|oqsS`>^ zE?5-|SLGVaCoYu=$1F>%Ob`=Z>R1$<}Z)Xt2p%mQ*-DT)(DAY2S zFoFBid+{8;N7+|<&JI+iO4kCTwhPCRz#PArLnnT2`w?(yDLRoFA%U~<^}Xf$JDj*b zHO@g}45#qtR>pAG8&LV~C3&e|Bn-5$a}uALz;@7hdVSSmSFVA}DgMuQl-TxH{-Q#T z2ReM;P9jXwl0x|G;tFW}DrWRQlC=p%?%mr!8c~S!v&3f=gj$4BnYQkZoudMoE;nHt zMB4Dg*DWB z9J0I8Hp@AH+JP!o#HycYondAda3%ASg112!mPQ`!CufvAXI&jfkqO}CsX@fCe3v@} z1M%cBL@JDKQGU&IG2aImfl zL|nno%?S;;^%G~|$haWSPdB63MIO%&C*Z3wyp4fy+bNxwAdefNwfQ^pTG)3uzp$bA zc$6)c7w`=T!6M`_c->L^JF|Bf=WNW9J=&;QJXx^2p&of5t7Iy)p<`?;1SORvGr(FPunblQil9xJOkkw~1@s}rWh1+JYoUmAZN`{3bS@wU-F~(T# zli@PIw4T=^R$Ccz^uTHsriG90A#m1pAUy&}bqh~FOZr`5>1r^p^*7$}6~^jv5W&0? zMgTDDalJq$#Ph@!(c1hh0WHG`)#`G_?p&XE+WSO zeQ>4I3q^D$WLsV9NJ{AQ5uA-7ikDdCDKOKYH=JO^9&7YZPU`aijxT_*`QxsI^cuWf z>3}uty;4LYWWu|HZd~l6a^Vk{(6kr?r8T=Y1t*)D)A!31UUEs!3C}NU#Ey+8WIWf` z-q|`Xi=AA@IBwKtY0T{Ih{_1?D!bCp>sz@R$M@wmmNS{UX5I5rTG0a4vBAr+Uz@Mq z2Am>rId%JIH>6zWvn;J0i_|s7W}ULSftzhvN+tBE3iit{@)*ut+~D1HD_6o3e{t#~ zV@oo!cHZ3%hheD7B@0zqY{$%xAR3}I~q)E8h!O-eeEjj&~zYg;B)nV|XpuGxGc8c!}qNaUIeISFJq%It%R|+MuxV zg07}v)=H5i;W9z{PH zp{HZ^SSd&1(*2o15xB*+Z&>K#k@}9}XFU$E9 zIsHE0*PG$Q0+LH;Lz=c)@-Affd(*)h_`I|T@Gy3=3xO5Qs*4u96pgiGH}8DRU;3l& zTTUmMk}Wd0z{_cV=Kp#plO%_2W9KC*3;d5t;>&S&b7N2Q@Su;x=LvoM14xz>JHE~U z5DUKp#sZ7N#|qE80fZ2oLz<3j#EXXw1U`P-QvOfNUBo!$n*)3a62t~d0hgrz^^0N= z1Kf{64xWz3eAYSnPLyGN=lrw|(;JyW<~x2ND&Cc1U$@_v3hVA$f+9YXOHNL^r)l)&=Z^a}Zq;O2J?YGxhvzIM;S>yJdHOH(1~lf*$&3A1)`ACs;QY<5`>W_p#Y#IfcXkta)4 z2v6aq#~UEr`j3UwF^2M)_fVB$dC<_=*!hY&P25)~Oh}~+kfb~GzlwR!D5lp|6<7KL zmbqoq2ILw|*@wr~Dc{=>dkv+OC!Xc}a^439Kqec+MX*a_$x~JDJ1^l+(Q^+f6JybP zV-g*Q&9(@5?3U%DMdDN3oZ15*v{s#U21?Ya)Mi=nQ3Q-=8JOtp()d&+9;p0L3fLQi%Gn9iDNy2_Y}QWWw2vq zPm(@C>qOcW`>?X|SeVb6K}Wo-HOo&CIoz68r{$ZZM&awENbU`)-LBtzLP$<=&+4Mh z&3tVK?c3iLO7lA3;mw|`zg?&UOM?7(7y%XUxBQ`&4=UWKCQb*Mt3Iv)5&zWaz#E@j zxag%=y5e~muUBk(5v9n4&U3^7SclCwY1S>r#gnmh*kjWha&wgmh>h1xVZ3VoeI7{P_^S(zvfhuAaj~BT%H9Qmum4Lhz#_aC#}zSL6iKp93=AO!D(MD4DHQ-H8a&&VVeQ-I1^wi&y^JK5My-UfCW&KBL~J zWuy5_-&E=gK#Dk3lhe1w{@gLi29~^9?^?d;vRtSZvTnf!P+SfTk?dx(p2POcuJ5({ zqMia$QQp!ilWA_5#r*by>`5v(Tb6okfs_V*bnzgao z<-Q+bEF8arN&8eV6(Qs~Xkif|JDj8J}|IT6?pRhtE~~-3>1D ze!9UW!Ig4Uc8$OPR~H8lB+BqH?<{G%cf!b%gf&R~&p3a!Vh!nhs8C!|EEf1Kj&|-5 z)3IMOu{Vx*&G$FN{ed}n3vb@eQ27NkB)ZL%-H+#^z-uc`u4>IV|q7X>nghMjISZb64Xg0kj1nl zn1cZCw$4;DETYh=?DVDw{l6Qpv!;#eY9nO zV3)CH@Va}BS$ur`TaTV3#FMY~>yykt$On>^`5`CxYl69lgl+i+DM`}U47>?EN7|3QKu)ny1eVm} z@?0Bo)@|!aDccjd>~d|#=<~wEqmrTm9XGv-6X(SIMP`VWN3 zi%4faJ)G>HA!$Itl42>;1!PT)gFk3*d@!b5F@7)|Ly;AUHfV$<8e*H*Ia%+9h{kkh zmV&l7n|i-d>k>LRg_0jG%yYUD>Cc)WW$M*N(b+;I6xXm7zhX$S4h7x6uX zZ~8Y~M)L?WbfsiVf0tv`4pWMJ@v^#I!TIL1Op~18xbrWD*ZDrK@c@w!mf%KT*Su1# zO%KynuLqq=8*Xo516xDubTDf1gDzl(gZ16l!nn>i#Kv{UM_ViD=pGW-iwk^1T&8MN z(-RAn)Vj@3Y2)quTz>051T_b-V9?eFC`G}k?Fj^aypG(I@MvC^P4`lT<$KzyU8^aR z9is(kSI^P@>XEJ(@c4_lM{6hfrv~Y1d$26B|BO*-3~fPYc%J}6Ho62K`9(*uBrkZ; z@zM1?E0EgpF5Mv{W}l04o(5&cHWhn{=_711u;Ioz=U@8$CEj{>h~mEIpi^TBLw5*+ zoRshh?rO227s%6xsJ72BN_UqtYy=%D(i*j87>+N0r?>>s^oV~gg|AO*LFX``xw!3q z8h(8bm$1#6#GeYbWOF6P!*8fE56DM6x}r0bI9CFAWq4$Z+_+8LlCJK={f-pfxwdzS zLF92Q`^JkQ1w!_5AFs;O1+8)q09j%Yx}|b{*VleaSHGGTgVrIx;E~xA3yVz8IQ#;sXpb^m%Hev)wL6x8@8i28r&egxl)Wql8il zN@2@Ubq4rqwRqho3rA?c;Fo3QkBs&$CDyhgX2s32S*5+!NGy$zsP^9H~I@U zo~eJ*Fm*+&E~+S8O;c$nG1-9D_~n8y{qrAZwSc2&+ARzKKnXJd0RDf!SzFsW|Nl84 zwx#1{+kK}GXiUzzMD-9wyAfw0=FSrOB*!dAi`WB8_qZEIL=Z_1WdLa(d&TANjy8Z- zpa;9T=NEz0kU{<0E6^qKLkj68vsN|e`6uv$#n;7joYwdE!PP-hS9i{~)~u{8eNFAv z4rKan*0-S(<_GhMlM~{cTF_)%3G&0@m(|nYne~xz){DnJnSMDz)ah>9ov>|s18lxI z7E^()oG&L27kMR?q>4zr9>3Kc=U z%z#_+U{@gnP`wkx9#l)-bgbZSPpOIZ*#t6Wq;-&PzwYki>XJ?fqBDndDaQc36QbL| z;oq#mHJnh+KXW6zB@V>hue~}uI&Ju}rm91PXg}R;9;Y*MyK>vQCB5zFsx?7v?O7eI zrA)712M(OA9^E^C^LcXRBC@@Lx)-=Gkk4Hfi0>#QQwaQxGHX45F1&*(gZ zNuhq73CoI@xX5y!8N|SgUIbjatoMrkk-WU=;*)GCw&h?-6dKR!4M|&z*Kv1ux=!Lj zsgjc0e*oYz###&GBGE*?1+ZCo_mS!o)54Lo9c$dlc*&2jn*X}JqC6!Abd+cjL@U}$ zU^vY%*FGVdwF`y!>sTtCqvw)YmoAmSyaalhU&P19EJ1USHo|0Gg-CjR%X(C6uH~#i zKJ3}g8QTd^!j1`1teC|rc8;0O{mF~1Gq%ZNO$P$`w--63>a`FS0un?mK`KrqplCfQ zKA88M67Z4s3VIV_AYE4e9e1^sm1HnzQsbw?v07s{o*m}Z0n+;@L;u*e#)$y`Vl zsMutOo_=Z5ARZ*HQS-G>R(=NWl2OnP)w_`v<El6<^=so;IpoS@C4X9 z?+!`Eod=W>ygXmN;x9V}p43??lYyM#X96ztz^}i_91qpK2DQg_3 z?5{3?G3bz&8|VJMu#WHqZ#KW+gr0!-iQB4SMSMDO{KsQ2<3Tw$vI&9`II(tc%yD># zWAY$xa6Oy9q)tD8YQBuyKe72W9q4gWb4}=Atq6zFXi$&TX%!i2iaWK7o(ZBr5o|33 zW=`NNl5;*TW{UrhIZOo*Rf9s4J9G*pE?G{Z{+bX#SFap z2-Y-A16d($k^Y=mvoEDXrWc-=kC+>W`*=l9fY=!g~i2fEe%n&EU-ppOMWUL!UUfqFBSZFBq}5FbOx=ecs=<`8Q_JMQPiJw$3eMO zS4vIF5&JZMGJOn}rxJv#jO| z2Y&1lz6$Cm0#y8XI`gPQGTDJRf!tq#| zLnGBImGrJQ>rkEqsfIJJbhn87NpK+s)4qsZzg(@Yl!bH%C%(`M{C*DnQ_Nd7+_Lyn z^!lB$;z|?wCPtzUZ@6*as)K-)-eBt5*|c|ejZ5JD_~QckeHTCiZ}v7HgVq_Kiq{qa ziy;69p%V&PGq@1OY|uC=%1r`utpjHydU-Pg|wPY!X}o~+RGW1)`UTKXyI-y%FD>4XN;|>)8o;OA zZ!)?G3B5{Z5)=4=*!`KM+y@!A>Sw`@Ut!#r<`RHhFMGJe(oFkj^YZKZ`107r@8mM&G5L!te}Ko5R-zl8gD zGL-}`H3GBt?T4wv_V430H5uGwN@cR4SW73YISNAMp%WkeH3r=Sr{Plb{a*ehXT=a5 zB2GL$jsg%jlltgmdY~kzSIpEd{7Gwj8xZNz==Xz_Cf-3NfIV9~FBgd8`OVWmpK)il z9@_+@J9d)d!|K)5AkD9jha*!rCsQ^$UY7O`z3h69Gm49Fu}2@Ink8I^xw{N2NzFT6 z&86MpuXf^Xz6R=Xb9o0fNsu4tvfI_74{H2Zv)A(iqO$MoX-c46MVx{~58D)l?$G)Ig$pWJS5vd=d zVei$Y$S{wr8+zi+@$4l4j!(7sQ~?@W)%ts*L2%}^8Kj`C*M&Zms2^xSA6d}1>LM*< z#^??btwaoA&G?4udzuQ`qONqHwkX!r*2aky*iL)JLU{fR;l1*Swn`a8^JKR9IBUTY zbqXLOp;%Ebxs-9x6-$D0H1|dM`6XyZN-i(Yj^I*X!pxw@9=%}$+%mZcS%a>ZP+JX) zy-Wq%eqcuFdj7{A^p0FoRoP{NMGf6%o97=gu3GC{Yu3GhPmB&6DI&k=a{+p7V{CDC z`L(FI(DsDdVF)Yvz zO$bH5B=McSfAeZ5yrVhUVlFqC!l8N-OjeD!;M3h&MXS7q4ppUm_oBue17)AD3O35) zwStrQS^Wrehnz=#hssHuOf5tDGvJKuQp-^sNtOgwDk4wt&@yUqq}own zCh8xe1IDsbx4<|@#InAFQ20{nI;~UWyh00x}_s>5iNn9RQ$6832Ir zzyB-!le5_AI~e~zD%{fXuq}S~o7(OPT%i+XmC4gHSt;3N7IQOP>9L|dv70tFLP#oZ zm_>we!J<TsE4$$+r z^)m9(@#SQ&G!~yXdsoNT4L>KVAu|^Jm&+q_%1z~d{yEFkxk22G!^F3%9pjtbBXQ(1 zA~%bql8D}tHTKdiWkh+HdO68Rx!<(8ACahLbaPR;cDMV;e`5Ynwkt>tGZD= z`)Hk(`C@cVnTKzd70@&DuX7(S${K?#uuCVMBAl;XREB;ES)-V9W4P*hI6&(GyuJ2` z`WTZA@7^v^O{%#=tx4M##4C?Cy7?CiJxjw6WZ-42{jl%x>sUm@s_K2!%%lu|>`u}p z$1k)ULHC*GMifdk+dWh`2j-0!q1VFUfO-xs(saJ->FRA^CA=*?9iPbU?5;K*yy@^I zhVSN)b_8Goz)?NFQ_E4~sCZ0tf>pej$T&3u#{Czd#&qfSN`~GuV#mQF3EaD*HqtD! z`9l(6rUeH{oC@D1M0O3N;Ae}U0n%-DKgt7bd z&M7AP-942#(3=tgnh+>qxt`{-J?r5d)*Q&734*LbkV4bM>1C1_Vy1MOYcCjCKSk}i zsWkuhZ%@o3jZS~q%NMk*rNCD3=1Q2hkhXtOkp5c!K%i)CD2QVrJQuM!BXBdT=8%VS zlPXbtm%{WuT}?au2UWJJA*-B89+_UXWt)V6DGcsqndo?NBY((n2yMdODpFX~JOxpV zI<>hF;Bwr*xmS9$uq?siFfMo96dp&J6c0dPeEOq)c8-nZ8oQIzkhDUc&w7=E88z)T z(uq;PRM36$EY_Kju8E939gjAAT3&Z0fQF?}e1#4tj!5|on-=ltf#P*8l1{Whe*5>F z)yZpE8GK%@-j0snKXR-ULF=!5#eU=Yh9~v6!&n*wp&nWNIvA7$e{LPg+G~fPum*M= z*F-(`vRRs4Cu~yv$ZK#tBlOc(oqHlKjnGQZ!JH7z!Z6b9di8aS2IMXFrOrTc_4;}l zI$>(70V;0fjc*?uTCDi2cU^@yLYI*Zq$41Fqj)@Feq5nAi$aFG zbcW&eB>W-JO3Tt&``*=gN0(;=<{F`lDtO8P2q?PtS>~B1Cy|PRyrUOc0E)8XOQz`G zcUL9GPY3ZFqFDsBzE>VioN*6BM(w~~9hwM^Zx3%7F0>D(y>m(ry_+Bt1)AbS^&&7( zfJinDy090F-w+N$){ZU{Bo+lU8NrrEeCXeR%T7)+hS#&ca{RVyEw83?9<1br=x%`&+oSFK}q#HSq_O7uVk8ymHMuha3*ByXw>y=g>2YQjUFhs-v^ z$B$2CBP{g@o5=x5NPMIdh!V{c!S6#+;foSguel{rGbbZ!JfPT6cGiLue6a+A`r|&D%&lKUf}bt#dff33xZbph z)Dyb|ZPJ%N9}}PGa-+V84kRhFp5L`R;pnz1M&k}kevV-K6l!0p7#KX9)X&u?IlGG=g@m* zx~ls*$>&7fM$J>ql?+`#G~EdFI^E>5+2lFigZNU%OyJP99+V1u^? z-vjT#86>DC)80}O@Z3H4et^w!9{J~1%I;`CgzV6|`@Mh>$9wSom3ydTJLJc5gbB1% z0&W78)l6%qsEvPF0?aHyob36_r-W`vKTt}t?-AMMkCh--c2uY@aP-A8MFPC6 zOi^?gp|ZT;YFzukYz=^bp(!}w>ZWB2VMD-;=DPv*D8hG*l&*E?QG{U}MJwC>Kw@ht zgaBxWure9?w!h!yyr8IAOYEHXuSop~dnEV^79I2Mm!5~ANzioXk{iz}k5Yyt?*o`% z*$B#+nlCIdghI7Bk5m-*-F!guJEf5s7cT-?nLDwksC1}I1l6_OX8i(u($(z zmKpXW6ZA2XA*S{Vkqy?EO_L;xDf*fTA;Tau^wSSwed9CmaUEJ5owN>XrMR?4@(4-e zW<=wbDpxcM!%G+mlw6f@FK#JeY1Y_qlTXo)C|>tSX~CPlJ}hGxz`99UO|DudG3K3g z0(T^4l3{*Zll-@$sYV=RdjRa)&F9lpt;P#>vtrC2H|dCgE4iR+@sA_~jCMZ;Oplx@&b;}vDp)4}r zV>Eguk*dxECmz9;-dHq7)RDw}HTJr}FA>W~=6sW^zgk%}LB7@es@f^WIv=v+a76N~ zkxV3j$}&AbwFquB%hJ>^aS)Jyg=5GQ$gz=E>?#s};RS4{cAU;kP3I=rSCKQd1B@?H zBP~A&LDxLSNz@-D%cdkwL4pd5vd>n($zlRi@3xPx--q9?-=~kS-RK#sje_e<1zreHU(>7c{duEiqsy_|%O|WO0HOEHN&5Mt2OY1nVelwhexAl-@~%`O z1npqcoN=vG=El0A2)(vs;E`2wlvoFOOzC~)aH=d7=16?teJmpcBm2EC|Il%e;`W-m z8c}X#K5CeoQaKm@pyUD!Gkf7wc)Z8}*B&TVbv+Miw_-tZZq~_~txvxmLV_}pt#-f> z$Bov0s#V+dYB8efvoguUPdDDsCGss=WhSPBCnZx^)%jOh8{SoGaSb<{l zS85(@{00a(4N4B;qz8*R>q4dw$Af&Yz&|gObW{pYg`3V)N>>mqIH^Q6Z$ehP96h?TBb#ku2;QR-~?7?jK3>r%z16SY^R+AcINiSj%i{D02DYT%#%R(gAH2>!Hy23HKIY5h`Rs`=Z0d zbEgWl2^cySJ#^tDPJ5jfoCU{SY@xUxNvKCANJl1{3lOpdi^LI;q=;HwHRcFL{8K8N z{>vohbnfJzjJ^I^4mc)Z72o@~gfPF@`QVjgIV#UQH#4-+OU#K}^!d7&bn ziCmAsJm4K(k~1P-D5|iC!XubS=Ge=TqLt$j0W8Ts&ccv2lSOx(4b%-2iao%#?9F}R zp%BTa&V=9ZhJ%V4J}1Ahmf#euFl!vJfbn^sE)tRXwzW)q8brltK-d2=!Sr&&dq%AZ zlj*Bz=Tpxnm^!X=2D5(zpBMbzWr=D5B)UXvcMjZk=ZZYw$Oy}@=`9`~h-^7g_NN6* zBfAy)^gPj{{t7={s*+;$uw1@r^4>z(XWNw|eZV<$$vuzUKQUTe=$s50c%!x$e-q7P zY1tU{A^%7K@f@w_Sktq{F>PjCn;5@;#1>%Iy3 zoB2V`8i~mIM4sDJe9vsv*-*o~OZ=4d>d4p}{Ve(j*FbTjt}qZx7j}fe}yU zmk5)vi9ER|YqR@5>o!@;lg#4cjG64!W}kHG8ng&r>3mTmX}1K`AR?DBK7$_WG5Nb- zhum`_r$zZuP_d*R+2bkyQn{b#5kDiYFq^|3S|$jj$~>yaAyu6L-=D#6!Qj9&H~*&; z*l=b4W^Ukh*0(DlPB&(4?h3noZ0$WF13t|!RDQ4?Ty8dPPQ9EyAwav>p(}vIC4$Ea zI*TefnIKn1+^DYDXcw6r@BoeH}?Py*9<@Na1 zNX&yd2hHVLjQz|X4E!mrmNgf15<{yc4*3WdMr(`dTmB<9diZLmq1=(Di71c05Gw!-b~ zSIy#C*m4>o{eNe`=FK$LFT!KOmWhRRd;ZDVKB!}!Wd#|=-LLyy%y(eA5vlS^cyj+n z=-dmRJ%jZ<0YK_&p>St@_K2lx6nlA}WrRU#3i_m+#uAhWsV%M`UfH=BRIMu$B?s-z zev9H%yS64l;YDjJG<&Zb{sWpi`BfRk^b@>Xkguf+)D53A0WsXtLQ`g#Q~S1o_O8fb z#5{F|njGSSRlitd6)Uu}F-XtVjG#|kO6s@Vhk_v3=j5L2fWM2~XM&YE8P>3()?tN_ zpq;(nIJj2bRHwr%zBo25B3wQfvO^t#8oqQgfU-O#3Z_mh6%)AW7C{6}BqdY}BtKlI z7J+?DN~qZTAUb1?(!2u7lREo*9Z+SwL$EQWYWX))!#n{o{@o1rj5DG&Ct)&a(30OH zMy=VxEjod!OQijAH*(BkA)HB<9FSVZ>5MczsEb%07m=X7=HS@*X&#M6Hp>xeli$aL zwJET2jdN7ZydcQw=ZuTBA#X`2mQX<*Ry!cOk=bQpw?^E?i$H>#lLF5cQMdmFrO_BA z81QEd!{_;KazMfgb>|yaND`SYsLXb|$JxwOvLsA9(lOUie5T&w`@5nw?9QR)6$sGj ze8pYhczC-5mjtBhimz0qVi-{bIf!2C+6bJ`GkFc?eJy12a)rGtcYp*c6+B|c#WobU z$|siwETijYtY-I|o!^5r^+zKOV~{9v?BHo7DM8lABq%^KL(I2Csyymorzkh$!iu#3 zi^iSJ11y59YrA?nMM~e}aqn7GA_@5;fK%rzKx^gI;v$(vv2LY7xlwH)FA}>!j>1?1 z;hed=v9|R1H4p8uwra!dpttAI`JNNC#gE98SP+k)+K}~kD;K^mJ`29YSoIp7F*egy zE)A*4NyT7GCU3JfvYtZ&t^@#Rdj?1by32&>n9hJsLgC+W-eA05?`Hu@rUgmP~~{Bb1I6&DK%%5&3N!FYeRF(CYVI8t~e$ zxt?k?tu9|2J=Mf&xz}6a*KbUqxJs`o(pI{GZg5nsE**+IYdS_E5R(^WZl9#Mttc3{ z=HQ*k0P1)`kyD(L%x#I!fp+VQ-2jc`@Zj`Y#zuC`#em#Hz+@>9z+^NXMj?`kSrc~A z$YJX;e4*o96PKMI`>DTJz3=DMF{(Q>nyK=4;yRt;(BZX1(cN(tbwTdVuA|aW$C^s{ ztM)2u1g`teXntpaa&P8#ZwgUfjn zBW+S8CQm-v2|VhHe}4&OwnX%!%TnJRnlN$46~lD8ji>4haCJuwT52oOgXGPOETv;+Zcrd)Bdh<4mM-3E-_*e$W+I zA+*_Wk&|2^gPvr520$KRzErl}u)%nAB3(X%!@)UQdVlT7yIu}LgnA|}cJ^N472IEV zS?7CSM`le!v$y4%x;gVU?!65S2oRhNtN9fKv*&%F`GKzmrV6V1!WP~VWxiPJCam$@ z4t(_utz`7ak8H#yj92;6TUd~J9sU3$c)QGW1x^1nfw^m!M|e?#kM;(TAPzoux+5El__17Gghx#CTBolVld7gB>*>W-P}e>{_0WT|1Y@P$I6=$YY?BXwZ; zim}+cnud-q&4c#~{1693cRFIgcE#93Y@l1ks^P>bGQxVBday0+?{hf#ipezU?kp`= z8M2vK>y8UL^G&7IfLvEA>>y)xo+{QNDRxKDE%!xWKKJLRvR~Vng^)ODlYCyvHMy~i zqG~tw27<}tWV=(c?jTrx)Q_#WzUd_})wq8Y)IUB^?)?2gar4Kwsvnx_k7PS5FQXeB$_B1v>oteFqEz$gGEhW3yzD!p{#42$v^t9`Wz9D9 z|1kCrOrk~Il3>}kZPzW^wr$(C?Yd>#wr#s^*|t6PW+r;#>*($`e<9C_y>n%*oq1P4 z)EZ&5>Q@Q*^DOVM=QFb%zM3}-&2!Cy(X^^xx0n*ZmJ4>l1`A#LvCeQ8QeEf-Q4>D* zZHvnJ8{dMt;S&pF3{7azRN`qbCc@kDS;m~`AJ6!s0H#E z3K^Qd9GskdflA2~P{8y&sqXCzE?1!m%2LU$b#QVhh}xq?AA5-D%XoGO(iEUk<8MZ? z%fD+7hC!?sjGyy~{GTd&48|2L4hR6iJoNuDf%X4Y+4XEpj4cfQYw2HAChoUmx%)#6 zVWMBPUU38Es^8z=b_GtKU|nZObRh>s5=+9GM3JD}Hk#z8dk}v7ezhg)Twi0~{&+f* zO{km5tzbh+bvc9PKECVw?5URR@cY2Dxt-Jv9v=^%ck`MR{^jlxb)C27JAAtyr!CN2 z31GF#m>~6V+7w*38Dw8Er zg%m21JD4cc$7r?p}p~A;hArz}m@6R{|1{#P zza6Us>(zQb&~BnV8V+1UOYFM|uR0;Xi7u;pN2@#&hmke`63Zf_b`UM$g>E#)l(y~` z6e@Bf;+zI~p(`j$DoD&*ic=UGem&emlV9kWAD4Z$-vYhaIF%TiC0Is*J9uB4ob7Et z!OEb6y#8PX{@rx^)GYuN(8GOkO!YSenwRrF!l-?m!0ci) z+~Eu?%!#|qTH+5O)(f0SvL_+UI0Y^=7p{KkIxowU1`&`%Pl6ZCCJUF?IP*aJbhtMI zbZTrZf%_w^^mx8i#EqufSqWoso@bTbpVt_N2WzC-rU&BbD*(drH~9$0SfJ6Rr)0aD z5_by|=_yZG+#JF62!9OO(JEZ@d?f*oIw`uTv)9G<>2p`NPMw<+V&hWwF{}F9AxvGk z!t^|pXp)X^Bm)N=_h7s}inu{7KGBilrbB4+C0Gl8wGP>6q=BP^_V%2#8lDaVGtr~{ zsTu8&t>KVR(>ao~v@wTX3I1h@Z-bR=-3l~03-YC+gc)wmxocNjq>P9l_Spx<)f|DK zY%^J+CJy=F*S#MmGJ%&wY%_4R_S1IJb!|=sBTbaEdVsjIt5b`}#b51?dAz7Qb8;m7}iGzAaMa3<1znRN^yj*bL&JwAF(gSL0iD@91#XrT7MP_?cJh&12 zXZZg-VD@Qo1IWPv0Lp)3p8wA_o~wn4+wakmt?p&_d$j!Qe2J*i6j9#I{AE$KBDI|DAu2uj7UzSXN_aNy&Zdd z{^6NC)2ts4M*kb1VXVB!Tq?<^(yLh6eC+X6L@KkMRq~P)f&x0f5t%Brk0v*1Lrgi% zfv~JCSW+2bLb;Y`NjVYzSDH>lgK1~3U_wLOmYJ4`s&Qu>r&&STuYeL_Ew)iCmO?_L zeK&@xJuls>IV?^^xz>@lmV>Tl&jbF7s1#nX!hG=*#ijM*>G7WG?eQ)3A&P!{`K%Vb zHgzyk(k|&A1X*!>jA)y)ts0%ZL7Z~aUlwgC1f!ix!W{^YUgDoE?raS?88S>Q{wAd2 z;vfD8mP`YAp#dr3ndrU`e`&vgo%P_pVf0{XT`CC82^uN$L0liuF-S~x&MpuFRz?yM zvg1|_PJ&g%sWa}KKwal?US{gL0K%AwXmpa;aY#V#HaANVYF96J@b`p`!uYJ=t{ zA!oPPa; zVA5z^MIK=kN}TJ#$7Z1K{{oQ-W3BE%OkTg?!w5Ni@ttuE<(r|dZc6Sd9cJvY|HMel zNbUf2P?skX{k1Ai2cis{mzYiT8lhrnGA#d9V(8up9I?Dll!^E{XnPw=V2PErV00g7 z&o)|+emz0i{gOD4;Sm#C%|8iV6i%fkYNj!m(D6uAD;?7P%HaYaNa{r#79j_UJzF$mL%j(_N;C;+f*MwZerGbKt>4fq2&m!K^Xa(jh zlVriisaAu6{QkzM29&YjKe=OkS9;LL`}(h(;rk z^Ofq4bqxU!_YdGXDFr6|8Aw=*qU8xxfo7DK_ zlJ=XQTL-_jt8Y!R!f{CMXg)()3+u~(2ZH>VviS9nxS1_b=Uy{%*MvDv+!GS8Rp3Lw}@)dfvCZ2rPJp48E6o;JoldgGm50id#K@LvClUZBHUO81{qT==w8vfu(G?ngO+q@#YB6Ksf` zb_$wDV=neZIQH-V6*#leNDkEW%-ErCI=Vb}o`pd@?TnUq&225l8U;W5P-A0^(}Ic! zEocP?)Yt~b22hB^5#{gw`xLrQ9e&Nh%=nR%_Qr@k8&Dx9-N_RsNWGm7X|lw=%v{l^ zluod_*xUHphu%6!W6}nFy}87}s|P=Y?Ew3Locs34%;`6o-WvrDetzX+{0l6;azDD% zvnyZ^kMSFOqR+}qzpE3MoTcsXXkpJcxdb`9?Sa`C;yKgg-t5LL#lDFBU>HwHHOBkz zDQd23b-BWKQ{y+LB{w5J50RtOEcK@^ibuaY;6@w>2y*gPha_{SP4 z1`j1)5Fq+7+PRFH>;ntkA6AmSd9D`V3KXZ0?7bh+#G5a$L8Z_jnkFSEPzqw~8nq$s zAjMti5rXNIiC4}b+de1M^a6OEN?w@#d^}GDz}*nhL(d{=<^IF}spIH8`*Mm55C8!C zx1gTwukpZ@Ku7A}#iM`U|Ke0j&!9vWi1pr2|VPjUy$zoCvKi%|Bk&s(FSLIvw z7Mzy`D}G3^w$`( z@`9$&?Xeg-4Hn6Cr@e7f)zjvEW(BXoZPItK^tF>~qzMd`BTL8b(R0PM{ZY-HFF!$V zocuot4GbC$`PwhU?7wi6{x?t?+1c3G+5VCml~_5eAby15n+M9!!vciV#04n=@IpMa zsz}LGm%>U7`=7c_yw7JA3+uD2?N4{6n=8h7(|W@$FPIL5IJ?m}U^KOi1&pMInh8!@ z3k`%dBc)odR=qzg$|V!!cI1-d0r9gj zp41sENLxAHmLcT;thUc(p=oH>^7#dggXPACjb%LN?hWr^|0gs$Qd;TPzmRDC|AEHE z$=S~47aE;dIjO<_g64xV^l>3jqiDalHBB3+h0+F5V5p`u$=Kj0cs*jy&Mv$q+zB&{ z?dEG6Xo6jP%#~+Qu~y%w*S9NZOG;-MG#|6_&q1&IcvjU&JZ%*Wcw>dg zX7^0}2;SptDNP>YB1{|xN8&}W35Cb~5=99H30RTMh745^dz}1inSp@xwK=u&sWF@B5E$D8q-nqUx%DnYs!X4)d;FSD@x>0v+g^{R}YGGFp-* z5`RzKvk88_Ohb!=6%+9-oHl^WP&6`eW?;-hABaAJw+@GUq*v*B!89%O5Z@}fi*({o> zUx!5+DDM^PWJnCZcs6XWJv4NxCcFm=K0uWYc6{BC`)(wY=LN0X!XJu*5kH8=S6I|r z;4$)QD^?XS>+n5BCXEs6auX#%E=33vxuX{Rw``)42XlP@f4-@M=Anav3}*`>O+xUH zJlexiIX>T6%k@J_AZbjNsp#2IYdSjP{jqyuh11Ol8|p%PVb$Bl*fLGOwz6~D+uGC# zZmRI!IH;g>J)69m(=CAc9!Ak&cvX66(Y@gcb`LyXlr}e$Utt?*J~{Mkm3T9C<{~rF z>wo>66z}2V2NJ0v?`suq9nh`0NY;7qGE+b61sVngx&r2E1=V>lX8m1?_O+Dspa~^? zq-nOl;ld5_|5n2x*<(On@J6iv9wSnC*7m$v-?E%CPNgVlh<=1({ee92p6xCL)G9b6 zyUj%Nc?;R;lL-Ut%mI81Vq{e`@=E=fCKR32CP-Cik$q$QrlY5~ySE{8=TI+t+h5gD zbcg_ep*L=A-<2Lh#L$~s_XC?3Qe=Fs9ZQm~GwGQW?7Kzm#MJ}0mi7wr*dNc7-Mb*F zB{}f*XHrQl&u#RV0bhXtWWH@o_e)8pVn~uJoiGTa5<97Ei1bkQ{X-^(D-Abm_D8o^ zV(%-!5j%RO;O9B6pw?wcZU+)31K2+2%y-)zdC~wB!)*d2dCeZ&TiH;T^rnqNa?%_h zg|&pN>kT(>)_Q7y!MbakNU*oOC+u{cI z0W24!XEKr8cuYKb3uZfXZYBZ(lRLg7YPqgMatf4^>>7}>V+XNp0O!-Cgn?LphJaZV z$yc4Oa?l5uC`9wH>aB;wWa@)A^)6fa8cUcla&^06Pcq*)+Baatq(NdQ&YCbhR7(n; zY+sK!Z*$@WDeTF3WHHWA+=oFDBz6p@*FZ%eJ%f}c5^ajvSx&$0*q|DAcyJa< z@ET-e^<5&$46{tZYRte5l6ZAOEwMXB^ZQqasKBqLDQ8XdHt}tCoamr zlrpoxP}N3y5C^ykA38&I922P)F5zq%w<1}_4Et*Dv-uJbiaoc+sL@D##}r8n=ZDZy zc`$wQ(R^AL1hQmO%;0_N{_*y?ga?}#YH20M$!WtQ(eOTj-0~N46U;Um3nK>E`s74W zy2Vc1wD=L>$cX8Hy(zPuG8dK%^6YScMgJIY^()L*a~gozl^mRjabyT)mtlq>N%S8c zC(L6TTq$NI(-hk7x|twmVR1g6Aa#ZPa&4Un`>~1$N2n{3OmkK+$o+UnD?^6h8Jd5$ zZ`xFgKS2*5qs%l}GT#7UY&&CHGPVq(}0YrhPA<|4Pit9jYa2F*#P z>0O`Qp$4w_yVp2l;4Nh_=3f z%^6uD2Br7me|Aiz;JJJZp0p^rl~~o#@O$hYFmnC(9?-zpB5)ea{;bBA1X2RS4JZ4? zhd37{H;dn<5yA`Eq#y_zK4W5!3-R8G2JBGUes-M+60c(b$dMZ=$g)7LW%>_1g&?B( z`WxTzYJNgFuLYXvLsPy19h9v|mT1l=l|q4#`%UPuC^M$dN#!uIFK(F`Y%Vl&8-KkI zj_pLvur=}|fllbj84%XMe*#iR_wRe&z2Rb>(9QJoB@}Ww48p;iOXwPL5WLK;yYPI3 zS~A%n%oxXQ|0thXm5xKfpoMqR)Vk%5Qf z&BhALuvf)Ak~GTHu*Fk@*F6pe+gT!};m&?4kAQ1S&YFRDW-xM1VU#~fDy@S7O3GA+I^HCLE$&unQ&VHHclHJ@(3!lMTIMOUP2f&+kvR-mP)^^`- zX1y?SR>gQ6IaxDm?V&cHLa4_|MzgmqR9?0j?oLi1mLIvg+7AJsHPi~?y}mRe z)kDWo`_D$GRL)?kj5O3lSV}lgL1zyw5~LwuUTv_Z1ZB&f`1P<`)5x`?Z?)QmH_yZ@ z2Q>`4Oh};6Xtco2*hx_*@l?wt9xU0<`CxNmj`69jmfJ-lM(f^ZEi#?o*A!rH*#+C2 zx$$;{d>1!6aJ5@5fE(Tl<&tWS1t5#HCF+!CV)BHqzx?1Ys2nP+-psNnI=4i(=FFeo zPB7NG%MP$cgc@HC5H?kQQ!Z7bQ zKqkMN&%*S_eCc+k4B`#f?s=p|Hph0)Jps+@47PJnwNH7jHX<&L>2LO*>RHb#K>V+o ze;hw!vj@fJ0R-R~SNJXtFQaG%6|{vgj_Z=xBjrO(bgM zlg=H@hxW{V&c<_?=Tn*L12CK$Q|=2?mEUmz+b7t_+7K6o#B|lN*yXg z0}1k})uxJik|kr0AS-bSj|6xLW+@KRWR#ulqdG)GPQE%(*5-iE%2RGw8FtkOPAcF!s7!y`?ml1Z*h(_ zF2C>ifL;b9C-HkY|H$Q{tM|W?(w}{kdwA_qW z>xhbUu{{Fl3e0)!NOnsl-K!3$Ex=&RrikTZxw!HyDGyWfXjp|WOtRoqLe7n$Pwzz^ zKLw<~?c>ttg9*~#>_e@oy>pApzh$)ahp1KSXXygM75hs9PY~zWuyqOl+|p5E(E%y_ z>2cyI(#Rk}_P67%OjlCtsm5TW6?I;P^sB26ljD5!3);sbUPVx+^a&sxN>%qo!r6lvtyv=_3d*)DajKu@!R|mH!sod+~St=j{ zb-PC$?C97IR_l2d#3k7(lM2okmIKDea!%x#A?*nKzIeBo&dBt~G$*!~x%8avn*HTy zt*r0LY&b4Q$E`JF?q^Q{`#csQhx&ejC!0Vh{rExob7^!W+kfo5i9?H{XK|@XdSqy0 z$JY|GZCQvd(ELKz60!!y6;hR{bfX&BtlFiQ0{pJAgL#u>F0+eI@OUsy1(csf>&u_> zh(Yyc?RU6izg&}$4GX)uGrod(Kfv>Ju{@K*zhZIw>7?s}ZPtKy1;mQ)Q7#BzrwudT z{ZZ5bI61Reg{^)6c^}Yvzu-+?&;j8t#hp}1sQwUyk??LZB2P!Txg?H&+x1kv8f0wrY|LGt$CXQzR z_q!ntEjw&BG~d-)cIw_Kl)?zw8?+F4&09#FFfd#AIU9F~1wa z6G=D|O6)u^W2r2s9;aS;-k^)FwMO@D#NA-J=-JVq``zb*%jrv~nj5_98@!t1#!f9i zd$70{yKgQ}^f2ZFXXYq#vV9flbr4?A0$9dhHzGyOV44=6*up>?eNrm=K?H%4^4 zsTWY*n%D*#&C3A$vB>vR-{+a!Vq=*L;DkndrM>aiS;n}BDlpV&WADF>$x>+^aWi(1 zpUbS!pg9Re$2lf}4McL`+`hSDOF=}kfhaq71f(I#y$z(rRzw&;3(*bDa5%Q@{E1MH znm!q;adC}O0?Bt7y?frZGvGl2X$UiWJbxCB^9!RN8>hs&TOnP@f%|gNTnlh|80J!d zMbPF^K?Jgfi6C%flm4!rFoV!gL*9qkAx=1afeqN;6RMB0rws_=f=+7?+~+Z`Bhu|c zLWDQQewqNgg7qo_8swDx+27I!94AsAmH?V1Sy$YyJ5^R{n`W}~!+?0GA85?dps=cj zb<#pF*9So@xIM~@O-=q&nL#iDG!~MW2dr$&l8~KO=n>%-FSt&qnS~ATBg6OI9Wm)PT%Jv$V1l<;wP7FE@D9 zjOo|4vpx8O1Lqf$OWD$n2611?XC6+>Jkv(N?iP_pz>4xgY?O@dfi`ZTWx2;D{{V|6 ziIg)4=0x1j{+IpLEku+71=%IIOhc$W4y&mrrRSh{EuIs$-fV_bZMLfM+`~;Wj3kI0 zs*OaLyQQ9Hwa{YhuOP9X$+nhgaoz zJb{H#D`0DA>EtaXgt^ZqwBu``oLNE(Pfe58Drl+hk;fAWBF`KrdIB zUYKpBhx>g*U72Kyj#w?qOSF4fc?K%)meR>WkT{Xd-2>Hl#Oqz|bwJ;Bm+KdmANI4T zHVF5(w3PDz#yZ!`%#M6JX93oCDLWE&D>33l1-0DDKZmjGSvHUhBjy|$cq2rb$v_50 zI!q~AKutBg%=zNZE>ZYR0IR9cVK$@%#`MSX1u8O33p9=+@)vml+^0@`uSteKo_cmp zI*(e1ibDCF@n}HbLN8RfwC&YM&#dT;P2%Q}le{tKQ_JK~-e=HONG;z9DtFQj!_|(; zqURC8eiPE6o?F-8rP47ZBf~y1NIBFmFMaqziF1*oU=Nm>-CEH+81&$#UzuCqyY_0# z203}}Bf4c>8K?6s`aJe?Qg~&W&a|*1LN^Ybx4Y*tlami`&B z@gZp>28mARO?a8n^%&?UcviUT~b_=CE_X7=CJAhSISLFm9ct# zhoru1JQDHg)_MJXym$9~?8*D#-tgo5{_$z1it;?^SemgIeKIvRGEo=_6i$nLp~=R1 zigNrLpOrj$Fn#dp{)20Z!lq9b`eke1(d4PaBe1CyO^YbvTiSI2|H>($-8P&`Ii0e+ z!^dhpzrfY#y2qG%(7gYtwbWo{aWbHJwjTMac`;XGXL%DZvD({l3YMJ@M?)I1=_w2s zH6H?wE~QMbby3}h>E{DFeWc0)`iSpB^6#`@hwppqV9!yP)5v}Kk}(R}`tBd#v*&Nc z+(-{e!TQPEgc?IilwtR7d5}`F4Q%rE0!OsW?%2v4=@(zvR>p9GHo`@e&c}3t9%~lA zBF^b-WW+6h*|ExGay!lNe&xD{lwvc5b}ankZQ?Pa(UJJI4M<&0`nV4`_L@$|@kuxh z={ms)L5|evLMPT7Z9E5V&s%4KYMT{1Jep?ubs*V8=QA(&Rlvt&klD4rx5t>{wf0N2 zww#~R8Z%*dy5t8gp;Abb_Iz1Ar0R{HFG)6%N{xCeVPd%)8OLsR*JSom7DeBL3P`Pb z?auXIP2;Cs+)h4b*zFIP(p9MQ1iPj`cvlX zGyCcOb)e?!Eir-_1vAHBRgevhb$q?Qp{f9g8VE&*nLy%(ZF{W)z z<5}1FIN`d%rF0JZx~27YbY3lC0omPj$*rIyk&q(42qQ`ONDdLJyU!f@Hbl#(AWhzb zMhQp*k$_5-kPs4$tfXRG46N7 zXFkC0hf*QNSd=ne0A)bc0hCZ2E0i)>p}Cv%$(NI(KX4m*p4DEA0lhWmr(XQ~M_Q#b zK41P02c;5eFiefskcOINVli#-O}&A&!-*jFDxJj7H`6l^ssTA5;bsd*%91QTIQ8;y zjPp=zvgl}>Nwu*|s#?slrf?~A=3}xw5ZEvNB zYJFS__|8zy@5PSN{p>Z!{6e+8RqY|Qr<#g#S?PvsWmzu1K@<7CuA0!|fO^$xi}%&! z`#*(T4G|IWRQz(Vu3rvD_`edh|HoH1G;sPa0yg|hz&2P>elrOPlEXC0iJWnt4f})^ zjcI{kWaizk00t3^BU&QHk_i?fx(R-KPU4e=C7Y5AfsezMN@k9@UWA^awf`>Xn_k3_ z9MF*5Z1;Sy5asr6Y42z!R&Lbl+4kzr*xK@q7BV@$9KF{@@u`cp=b^K493@s`RFfYL zZaodtqpOK@AUiW_E+3^7O-9$*sH`Mv`?f{yw#K_;jMM(j{IkleF;2QQ8Ayw*=+q$N z(NWS|k{A|w=RzHgPjnyQs9D-jHLaE+aws}5-;jscUQI&np($+KQhBTIS~j&~M_Knd zzE>a6UVNdvLK>$qh4rn2dM2b1<(>|eaabPgXI$vTO?3OFKKGR^O&YT_+vyx7IrRb_ zY+t?psF86JAIcnblLZ^FzOGe$fVXW}Dw5&GQu?xJ#!sud^7^vA{; z54@28XBr0XWHSpC@%5XqT4UnKS@p(TEGL=k%&7lC$vWDr9pi-|MpPpKN`TjYr)Skd zW&R(#!{xiPt(9KWUar0BmxCO6yX$O=p_=;EMC?_#`FN^ z#NsR(%6;X9VbfMkRCQ=rCJX?mDBv6uF7X;#jVt%ivs#kL7*T|zPDd#&CrMo#<}Ba^ ziZjq7t9HVpiX*|zJOd?Op#5Bsatjf{s|Sotc17+`NonWe>A2xY2M8gYQRqQYHIe`! zvLxZv@si|}(yQ~KP>CzbJYaf4c+gMX<5`DRD;2dsaxEyZUngh$|N zD~_xxo4U?H)&*twJ;tq(!D9gqB3@Ky?-D&e{f|9f zb&Yfz47uf``;fSZ_3csDF7?ZvG^YgvELh{dg~)Aq1D#cf8M4j0X$eKPJPOZ|My0%J zs1SUOoWQoi2V{!W-r85gf*VR!U1`XV2RsC$2!HIgy@n0Q1MOeGA~C?Y^PtxIjCCh; zWj*qMZ1_OqOQFk4*@+CRjivz>8^Ksr%8clUxA>+n6%kw1q9V5T;+E>5Kg0<~dEC+J zl%{iz5ZhP`-ta!H?xx6Z(0WuS9fN}*izdO3F~U-^NqC~G^K#bQkbBj+$Pr@hbxxT! zd!gaIIxh%lpNdio0w91*C^}|Ju1f;&E8h)%z);(E++<30-Z}x2%0HX#~x6sAK&6o8;SQlJg6F+N?f4X3NO7#l+G;vm5ar7iHv^8{ha0;{8f z2iWe_IGXswk3-nqV&|%}8ot@006N9qYIZS)tbA@JA#34cO(3xPDY@)e#EJ&GQ`Hqc zs8X=SwX6v^s@-}nRul1))bdf$0@=>UKy~SI=qq-63+Tv&i8@0(yYq5=%DY4K7T4J# zgV8Y?xprW4v?7YYyzwkpX1B`5UHWlMwzhRd7omSHSZT30&`258wJvf8C?twXrak(| zI~Q_0bX}Wc*7+OqW=Nc_FSn?@OV9?gFYR^FK66UM?NaH#0q&X)*)wt5=Fs*PNOsaH zWf`CxpB0t|{-KoZBEE_sCHt}x37xWzjiR}d>aBf*{p-gasGBL@_{atkCViA>rGH?0 zwr%AA%mMM|=?R*Rxh}O~!K?yZwAbf1vtGLKago)>TI_dh{o~aN{H5>icxNG-D5A&o-JnDo1lOlPh$;MCmgk z&(|Hj0~n){yTu2NYe+B51#awPALOWy64?N};35asPv^PxL&f=8F4-(7=SQGyawIxo z)5se|Zyn`F5WmXIlK=TbY~j&)x4s0W1Y+;EqsBbg)YvR$4D0K+z&$Ils*xtVCI3%M z_l$w#YH#@D&SLAU3@@2k!pKVNl<3 z*K2&w9v7iiTH03HDOtAs02=W$EW#v94Zyd~s@!z=@K!PyUq`a0E&LL^iNVx@bUd4e z`pEvVSnsN#RjTyg=Ms!eqZS}(s;6LeI{@qW6E+hcJ9XS91)qiu*I~$qn85=nNUikl&LcFZG zWw+Ue=(p1wxRuvuE33N6^=J|R8{g;^5@6-X-kAy%ntvdfoEoG0FETl)`r_BZekDvq zYHo4r2|MrCh+rDr^ZqmyKSt%SdJaq3u_e_wl)AqE`_QACq2Equgy;Kt+&gho zL&x7PZB#qFFMaUK{v2~sg?4pRsMEYpY;+ar>E-5oyFRQc$kuPf+_}nBMbeGpnLMhp z&csa!>wz1M$$fSqV%4%hJ z(>sHl#O~UB(t>tX&1N(uWepEf&wbgW=|S;M_}iP!xb2@X6BpXB(1u{ zpZGqL6bqbwvAPQBceaDrWrESrV;2};d|0rYBZ|=vPZBcZL`g6Q;wIpJ3cfkXI!1#U z$Mpw)!78hv5i7zCkbYz`7tMYtb_n=zzn!I=M}g1i(pKNUuG%q(nVAPZd`JP7+rDJ&3hhF-mLud`M}PoP z=ICoPeXyhOp?m|MnpcEVNr1^g{Yb3!#qEDQBJWwH{G_f?>Z6ua(JP>OCv;Ij3Z4!Y zS1-pT2e9OUjCC$p)@8}1-Tf4PV9r-^8N{_-y*5-FPh%~Q&wm1 zbLs=VB~10XS9ETHtpUCWX!YOA-gB8osTUDCKysdbDk+ zkKS7o#L-12d{^9Wn73^6(45GIfP4iejhacad=*f9{#2`;eP;0&%E$j<^TXrOqZE*7p6Ces_D_4y2xs(L}h&^j524~1s zopEbuhrjze45sh7PyikD6a-_gbH%p8+RN#{RQ`q<$;wwhf0b>9r|>Fp4&*y9t{uA$ zF%>Ixz3Gs+0LXStv`$r4B#!N@?O*=40E9Z~qc*heDj5_C#7pfhb;+Yl_?fmXu}1Ro z!r{!W(`rlI-Abc^)})GZ9Mq})#!=yn;8$m~Mz|0>umyZuJw&VYmn-*7N$UqCdbfkk zo!%=GJ#qtvh31Uco_P}j!8lk3-{VZIU{Ph+~{470Po=nBcclAMD4F5MnWPwG5qAVxrv1?BUJV_GQ!fRX%)1|a>SJ&YA`P~?J>EhSo!yMRf#{% zKeFQEU1|I@OaLtKHAZf4l)LvMKHR6<{;R%&GW6VQ(%o5JdVrv3OnRi;14eYnxRawF zi{#HEL?-3K7XmZ8nl`7nnr9316XBso!Kd}8Hq}>eT+RIloFQdRUO<*R4|aC(z`djoZAcKTQY|zQC8*eLZPwLH+7ZM$}lcWYisQM z4$WF@U6Dw5yqwjL@q00=c+$8XR&~mJj2c&Wu(@$ho=vuxt)l8YTQTX7D7N3X8Dv=} zSI94h3GMGTYO)vBbpfYr`tDR0B;7=2O(W%+qLt_T*Xom#z!kwK+@40@A^s}X;95W~ zMI@B&VFN2OO!_IsIb+b%Taeso1jdS5WdwXMZevo(4 zJqv7c$4k1|;|I+^3E3TP1p0f}&CZLeZ@fQa;?sTA?w^%STP}YOIq#ry8udw+NVMHK zkyN&$PRZG*4U`{*MF;k}R%EmHRKHBz32jMi{1@55xi z1V8*3*#Yd;DA%q;?UISX%`Eum@qrW7seK;R`&oo~KjwC=v`$hjGxf?$@K%8yjklk^ z7NK2h=og!1ovU`^5>_w{`@_ymR|md+TskSgcYPO!YI?YuNg;@=oC@IkZh-&dD}$uS zfU6({3jNSW@SAp3e?qBo+kTElf5Sk_YR%ZWqS&`9;%_T-iewINgEt(9Pd%tW%>A61 z7YlG^xnFzo>0>$vF)FTTc1#J%9+zMI?t}Y$d0M}Jl=)oJh*C&DkBPDt@imYhUQ@Xs z?q!({@kdQR4Iu}It@V#94V8P`2tAu@lGiLlcVRsG6B_LVYK49e`ZJUjr?Au1tvn7= zI&a;OGuZT{z4!O(1TeIwQWx^W+kv3dDXzW*&H0FLf|g)j?%*SNL3Y4F>lxZ*(1!VX zsn3}8{!enu4s;SF$gcpmfcW3CG;0&vUt01%SlY1qPFyBCg6~N!Mm@cJy?O0&x3FZf zO^cgmxn<>+ikBh^A$S%DsdEc@t__s50611zm zx82*Ny+VEUo?vaf1k8mq#Sq@a$GRxc%0Y6sWJ(37HBzbsE4{C?Vfe=DyZM0V1d`i% zG*Y*e_-Du%A+4Dr6F+(-<0d^ZM!bF3G44iZxQC@drnTxwqaURwdFwI9$7myR=dlp7 zu3$`4^?)q>khbDz9o^Scv>qqsXLF2RdpqKbsUXgCJ5`s(P>r{fzu_POyYi}o?e)Wn z&wvIl5Zs8KhL%dfd~|>vBCyNQ2HK8_4yutN<3eMDG`#w)IRYJU7dbTFmk(zGjmCR~ zdoA%F+Ifv4DW*c=iTxZ@%V`FW8aT>U>JB@D`{5ArqnBk9X9EjYKiXdcIv|$f@0`~p zmAK2LSfHSQ_ps&0xh9Q|9$eYBb@RP#xa8$z&6|qQ{GUbUt+tK^fhkA(tx3L8Zouks z>w#RjF}Go8qlFfW3SY7d%ml(PG&ryK_7=4qgG1%;Yor4EITSJk&ZzO>f4V(EG{njc zGz-L!OYFxHOGO3fyhiBEuP{3C#TrC4Ay zWpTMS`SJ1E$=cs^_;YLEgk_L0Au8Sa@7BSDT>-_`t=C(sBvtNNpMuH6pD|q1o+d>5 zY7#tKb;mR;j1qF9=A%igY~tq@0AYd%HF-pSYf?y>GZMcSA@a9-CykhkWwdPB-Z1F&20uh^LuyANNH!h9 z2fuNUy}Cn$xRGA{>MD+~-2un$#zf}op-QJrH4XbpXmZd_2|$0$Kjd+{o4VOM>^Hy2 zN#zSdPS0t{ln=X)s5skD)H(EndnmLbL`2Y(4O!AZ<>R$o6!XBXu(`{P_RX~~6(d>i zwJ*)^rN2Xg|M_Z-j7I;#1GhpNjXd7>)~`;mD#U2qjD(yo>lxBNuJvCPbHR$m0E1(t z9-rUBxJ50|5Ht;kg|g@NrL-B5)8C*pqr|lu0>3~t4RI_DyDXpogD~ySDq2=Bu(WYS z!ZfK4Bq(--VM<(`Eg*06uxR70-#H5ip5jl&ypv) zqWCO~2rl3`0odG#r!ke7<+hoO1*9PSXb^`y+Q>2jLv0g#5#az+qPHm>HZl9$mLc^@ z*d>Z@UFMgF3qRv7?Uq_TkkXPc-&X*%q1JPG;+*aMRVNEiKmfis3X`UoD+YrrRR{{Sh#?}foKBRsJWAoQscc8Ax9|^%84v~H5dWY z%aZeR0akPQ6yQ&@*eLvrWGk!^?6sbec#cZ%dC`!hPWC&+xC^ipU{RT_Jn>}Ik{)_z zz4y+Nc57hOKpCC06nlvl-|Oz3Klty;k|76h_RFG2cnsEYKFgGhcx+!Z?xquS<7NB1 z-GnA%L6uO0Py{XvqTK>WbF!C$w`TFe=iKg9q$#l_{lB9wD(uCtE3X*r`(QorY}C{F zF&&eTEhK%i;M#9#*F3W*xX(YC{eh9_{xdfj$=uT}FlyNSxysJZ{7IX4$q&T37s^hT zmGCV}eB-3Re@57p3@(lptgp``cris>B;Wkvd2>bOoQvI$GqBgG_3ijrgZ}89Tk@VQ z`!0ND-Jq9?oobcW3*}vWW7R@HZhdE8_c4?2@H_}fw(SLevz{KLywB9)R)SP|q>%R9 z4nb(Q<^(W0^N!TGSM}=R^R&#L%$$^ONl)E+iZdH-tH)--Q~R~)pbi*iR$%gX&ULQJ z!D#vNnO(>Imb!ud=fux;jn#4T+ckuU3IIU-f1dbVOifK3e@Fddj1Aij*2gWqeoFX8 zFcAvQOA-IDqXsz4$fB}FSAR2z1`!Tn6UpK-lA060ojn;6(sc<&9ug#v>z4y2XO(3x zQC4^ynN>p1kK-$P#Evg#Cm*MPnrmL)whwm)KPNYAxtY1o!Ka;*n{li~u-u))G)F~* zDI{l(Uf!R#&x)ykU~*)jl`Y{nmV#{I))$e0N05Ohf!9QDjxsCS-CqHerkJ&94-zZq z2cey6Hxh*o*OszW8Gp~MVtZTCYU@F6%t}Q<)7uWn0_#?>jQ*5KyFwIb7Kzm_7 z=_fAh4Q09}u(FOZ!~8ZOerkqhopKy-@;#o@swKUv0b=JoE7s7bV4lt1TvX-*tz!x6 zs%q+wZExxJz8Zh?;)$C2xosL~%hLXA`diq)gWixOdHF<|ie6ag&j8JNG|)AcFfPm< zy_smCq{)O?qk%CA9@}7I&N`<~G9-k;2JQk?lh7)oitDf8Y~#0(q}a3Iw6WI^78es*aQYr#oRLFRl$Bdun2k7bv7x+Urd{*E8o`a) zY+D)3_0Y=~0S_>o6ATW37;Ew@Se%pG?nSyK#k_)-_p86$Y8z2{!3C$q zsMG(%@KAg_#qe@N*bnd@LO3Q>HN^4|WX{ZI!#kv?pRu7cR$x4YfG1=sNT3(p!K)Vs z&5(;JC(UqtT*^n%EX`6J!s|d{$8%~E;q*wPyDGAo9&P*JbU+<|6vG<3!!m?Whk|)X zQJ_si#o&AB>676rB|bc9q+EkmszjfIOp&tBhsZ()VrIobt(}8qqL-IgEOZ=#%h?4^sxh{SFp05-VqI!O_&~Q0Uo|Y+i6w!<k((5>We7Vo_Cf@_%1Are=0V>@4Ye0v@?1F@uHsb7paV!1C^FY8 zmZAk~O8EstW^8^;D;0zDY`o|^$b1(2ClA6B@LH$JxED?l_lV z-2X+`J2qz)a9g{vZJQn2wr$(F)9Ki@ZQHhO+Z{XU*m?8RsXAxxeX92RY5jpU*P0mP z8d~d5G{^t63$%pDBzaqt;zATptrfFW)2uQ6%1GHu5mZveR;2Mp@Fv^#)<$STjGoeD z#JoD{_Ut(KGy^#=aSgsBV^_n?Ju9>QvP6E^5_DA4pX*Ji-2~oiI$;GsMW|)6sX|_@ zGBZX4`OjyooQ{yHYKiw2;V-d9JyaOy7@C^LRJ)WF93cQhQG+KrvHJE|ZZkt%#2*tn zkwFIU4+9I`q6lwHJ;4n#gU}4x4FiI_boRpUcd;uCff>^V2%7f!s3jg?@LCvEOxPr~ zN45z}*K_kPBEH7vxoIZt+ngOlng(p@(F-V}9HHNm{Po<(xg>SUuP8bGxEvsTH`=v< z2ZduuHF%T01-fsgZfC|W^RDWNM;IF?<4Q#+(RpiAqAmN?E-VM*S8Oicm$7Rf81dCN z-t*b~o1=FGy4$Sp^*fukQJjtCSjoRyreb>(y|iAM)X5$C1LlQ&Aj*TY=^wHg+aAxN z(J5VARgguK{|RD`@vk1dKm>s=>zk!Shc2cXBqx&hM6^smqU`rEy3N7!lz2)?LSm-o|@!+Kn|H*?uT__G+D&uU%t;xlUF|JYp3p+}`#)=L$*?pJgp27GAFs z9225?JznjGoHh#-U+Ix>c8C|+j~HtPFTd}jn{ACIXmCw|75+8E;+-^mpD0u|eZBG1 zvW0NqkPznu2llj&3$`1Jat?f&?Bu$OVPu)E(^ZXkL< zfPi{`<_?<(us1Ps`Y%I=x9W||AtTb~POov9!A9XGYV3PMUBLA~y+c{qslR0S zl@ol)xPHPpUC#Hr+(wCjZg+#shUrdbDn}w-pR%2627rL>XWWnomDok)?9{4Vp{(QW z?c(g^ZP(AuE$#=_)vNNE_PN6=?qBZ^v~nL;n*IA=&Gyc&1EY=$7oc1Tb7?<$1mV`K zwlNUCMP9exF1H6My~L|vXs5(d=>)RcNw>qeO-!@JSeY!Vw-+*^TxU@%_tpnz4&KPB zso*u7vmV%|#*+y|6ERh1O~bfo8HV3Tx#cbVQSg9jt>C=rMVCzbdd%WyzDehz%zK@* z1K13f7e~@{B{BTVff?*W9~DS62F?;vmCDtK|7(M?;~wj-%(L?b{}S`&z#Zt}hfE!{ z8oTB9T%HK$o(Mz!39s5*w>5rH196y($YHn)aYcHbWGTL=W=G3wIB4J~)drRn>G}6C zNEIUuNyKf2OiZM*R#HdmK}Gq}FQcdmJV|L>`Kb zNc*2am>$9!XKR6xGcTqq+wUkiYftmKQN23XrGoHIz|Hfod+xt5XCWgF*YxKrxT}@D zHWXx)jFTndA~(3e*Iv$@58t&Xr)}NQGBV8GX3s$>e;$QM3Q?Rgu=5ZY(oLdKLnM1X zb!YhJGKorLl;}v5>Vtq5%k!2uA?I>e4_HM=Z*AgZGP(qQoVVEDG-9AdITmE8mi8rX ziv$1;ga8pu06Z6s%#!8HbOht=1k$ZDG2^6>VR-;2Gb=b9@nq7V7REU`8(8&ZTdtPD z0$m#d1Y8j*VTc&$9_?zH93L1iNn|7w`x56o(cHg6jDTPkiu6a*Q7>u;;GBMqUs_cM z?zwVMIuxSL8h|%Z34+@Dy(*nuVp+{I2Z)4qRuV(2@wUldCE&>Y+K?p#jY=S^G#hUF z%UElSWDVBbzl^WQ2UFCIVoLkO)*OCyut5kWOY}S0<)OCyf=*SzKVqMGG95h3Z-mI# z*+3B?OUQOd##i^jBImPzlm2Hq+pTrzPgBpv+xqW`V{}X+fhFp6s#LJGe98&m{IVWe z!g6Ov`2iWvXUTYhPOJn+ag?VPPm!Tx`H#GeecT^EWZl;6VXJ9J*<9xtFjN%_5dUb= z{y;-YCfMChbqdBJn^SFYCUP-msq=o5sp6KENQ(uhi>VT5{;9v+1|S=8Rk3$e$9&k; z;NEnoUEc2j@Hu(g7=ISFbndEt4M6d`{p#@)65$4Jx46NL>X%_crCC`PQ*SiWT!hq~ zjU_QSM0pA(eRoI^ez_6;5|}}r5UPS0@^)Ct1ssl2h*4MK9$^#W1D6sEcLgmN0_{ZH=9TiH69IsTV#X0^J~|Ff~wh&qL>g5rL3VvzJ)U{_Lzzjmp#ui>MPD8rOB#%Id1 z>xgc7-v!u4^%KXxDH5R6S@fj89^($;YO-HR)qM&HsSe#frMC8xJA7MpYwz~o{XGZ` zjdjg@9Q4NN@*hR4^u_RP2ym3byhv~y{v|F3aYLXHX@EJI;gkR&M4mxfrZV|LuvTvu z6u{#nZ76k~!Mn`rk4~cs;;SeRPJ^OJ^DE3+6a=x_7@pD@BvVC309p4f^B@A8Z-Gy# z;BWcU)`9Pjfg(EY3^ksWA$&l9ZODtvz`04S*8_a79r;_Csm+UBVKPL+)^}PcuBDj$ zaPJx6M&KEjT{qxwtexCySJ7^OVWP}ERI7-Q5@%KLZzM1=xuT%wDs=>sj6!=z!o$&> zE4*0yKbRK4d*~s5kNe2ry4IF}eQm_ak*K)K>&7O6f-4S!O9)j#lh7dRxKQ0oX_=&J*w$1{z4bCl;Ic|&YF*yDzmb!5n4C!Lx6SQ{1KW{1z>F7< zrD`ZJ=r{PToFR_@1+p&wecd$sJZ(`p51lWLl!GFEB*Y0Biy-%yvmNpzJE}svE0(DU*c&Q zA+p{PR)>E_>{mY7^7HXRI;s_XYi3JuA!84v|Vf zD~9=;rNn%B8df-D!`|U@8RhRDF}nR9gTwrfnYsE$JNofM3;R)(0NI&Y7&$ot>>14L zf807h&;M6DYGLwU>b_W0>xYQ&pHyq2tWfD{jkS`~Nf7F~b!TNzE0oI)E=4pLx@c1u zGi4%bKvv6lHw&4wmWy_tPaU}<=LrYoXV7-pTb@{ZGwp=&)5G0aEbPMn7RBVuFmT^<%5! zy#j4k2ofGoh(2oBcu{jKamluQ2K5=<-Flv+SO@*)^L&Gj|HN_un?CS__5F((fNeRJ z38$%G_3DP@T-4-4*+Mal~+TH#mJq*Z%f3RbH^~$(v%USe#_bYRE(2 zHOqm%^|s6cN098}H#7BoeI-6FoOFUIv*Rsm zP+IwThd|xaJ9JgGKT?;v4hw$^o^p6}S|gNp{UPJUX7W-V4^#e!3aLn5`CJe6HCOde zLsk-Z?BxE{g9eTUCtWL_j74Wzm1`4d(oHQtAQSZ9t`CMgyK(PWyAni37P}0tp?#ik;VjU1hu*E-eJhGUlhUZ)NeCJPhr2BnoK2Frx>c+u{>@H z@9ZAb6UYI39JvhKG7g)l4V&2sr-7Z{!c-t(MBl#JC|0d69{gZ)5J3@_h>Ls`SAU;6 z*u^}F^Hix;POBWgcCDA#@OQ#MW8Ad8Uztp6NC%_XTdJ0FfZXyiBARUhlHv)o_o=}t zDGax9nra{zK_VElQ4niN;U-BllvzJ!q?@$w-Bi)QP!y|t&~J#GW8qxNONc>Hc{{5! zA|t>gd@Dx!7?bN06=O{XtgIn@dL^%c(Kk!J(R`&aff7nQh9LI?ia^2-Ac_cnw~TM7 zu`xhCnB}iKLHWyNx=~*5U{EfSurmcGrH)9#*Mh2g*JGV{>QlM!=cz0edUMRk@wt+6 z^&-5J=`B~_MnflisqQfLpZr@XIP?{HDSHxtyKRC3R|$r1UsjRsdz2ls#vN{C&Z@%@ zc}Gpu;B%34-uULo9(JZ3(n zm{X6iU~Z$DT19T)L)ByUxOuqY^T2JA`G>57Y?@na$zp8pVCXNZ%aki2Ld{|4b#Bzk zW-BD9HKh%*dlHuOlbz2cIb9%DjIce)85lRyP4Qz*XdQjp#XdT=U+E^nRY2~Ca9N6F z=n>&j8$zeNcK>3t_f=?;g{9;yN|QgmT=BqgqY;O8<9;rz>6->?p9WPI z0jf=5`}NajY$;7fl^9f;%e^TL$%7+!ZUpqe&=qI;mALy^f2nqBSO3+C@x&D@gTQnn z1#vc6Mrb!LEr^m;JXe7{pJK1EI2hARq2d|UG+f=L@z+ZXXb8+M03vzn_ET8v0coWA z5|w2)wA{g$g~ypFDX8z~0+T7bE%#?CJ_ybX8aNB-I*A0qzBW~rUY4{I$rVI@Vw9AK z@+hp&(#nv@iq$m{FtTn5zU&WrYdkxSix7LpfY`4rRBBV0%6ZuQVv?0h&kU#)XZfwbs)RZ@k}lOG zz}iHTgx$+^N_z^G6r37HQRuRvMkrjG*9DJ$7}V*#+j>PH>!;0lkCEsy4 z#sdWJ|7rVx5z9=49fIl4?S+r}#z_?Jd#ZvuJwUvvmAG$+hU`2=?_*q>97N!OV_RtE zTQx9(cdbXC+({$*wQjL)5CaTTL^z715&P=C9k4y_XE94hpoZK8i@$uf_Vqf48KECw znVyf3sFXmDv0YrLsI)aRTx2&}faO3ei#psJO@83?*tq(l&LcGaQE06TYiF5qB@>ru={$j&EPJ*fwj2ewUH(! zLoXOVdZ?laOzV+ z-3FV83(7PPoQ!fTyEVV#n3FJ0DY8{qBzCpjMKH$gK7!OL=*DxsK!#!d$*CFR65n z9H12@-0^t+kGuc~==<_?c6WSb$jZ-6$EJSsehRZFvmZYEqBN6%l|HBD z?rrD&JUAPMaD`8oJ$2DorYkUOB-P@fKDUnCD8;AKkfaYr1cvwO0xXB<=Cp4&Z%61_z=bo|Ld{3P%(u2&uQ~&DP*%0{}o#1#Wx8E zO`#w>&sj#jt-6xg=%jjW7q69bbFV}RE{(%P=e(nW?ZseyQB1Anus(VoC-pU?h|S8+ zI00r3nRggschI_F2wrMEAKprj+GitfG!vsWp&r+3cF2)haG2*Mf!(dM5uuw@AYU-1&J=V64!h3OsUBzMqq zmg_pcm57u^6|MU$Hf+9BQm$&t4I9xd& zj367H8$p_D|3I*M(Stn7=Am(&VFv2Po36(8byj|23Dpo2>wM0Z30W?t2@X3F>bNoz zSgXwj(b2|2fs)Qa19gSRCQ%h;rSN##y4X>TLswDwd_=s0W0RM#Tr)XUCAFwTm_vA& zmrVeBk#4gM*hz{SEQanXQkh8*?h>Nvkd5>cVi!&t2>yp zLKL|$lr6ZB5ebM{3q=R$1F9vs5JTB!m5q3ybgdI^%1~_?4GUVTL3-{T(bp#^7t8zQ zrBvc2e17zqkl-(+a2}FqM1VP(BjLcdHLNcaCi;+5KTA=&Xt}y}`=0$qqNElp z9Pa+Xpeju3dD}goxQ+g;`B-2txoq63-CDQC=PRISsEj@PXhx{8&fMD)Vv)Y66Q4*e zeZA@Sgv^#0t#!yyoQl1Ey&%NAZ|!7^ryM?nSPgA^nmHzVmiV7d%V(!6gftpWhIpqo zvY>hi@OLnJUPa(oXs8x(y5(G>$8uV9O!L+qQR{^^{vi=qJ=KjLqTP`SLqvleElL9t zSb@lbCyJ9Al+r&an zyGTA(t!6z(GWc~Psy!)zUN$5-VsO<-Uy%Umx!r%L6-;(vf!W?AhfJAY%v4B#>r;8K z9D_6+OENKRG0=1Q)nGoL1k-wSJ9NF6?!R3f@ownbec_ z^qFbqK+`#gfOmY5A@~`<88!%z2Af|;jS-IIQU5&8{ zL-@C)$(bo)UDr6;U0I1o%T(_?(aP%sEEI{;xr=4&8?5&brYd4zSmt;-;K46LtRQwO zbSTqZ>iYQ)B~`@lqpAtxphg4${0Zf0rezlKlWUiqj=EktWmv0IRXm6mharVarrCbQ zWrx^U+>=ebX;^sHFLY**A-!riwm0SA?SVOqHkTs9OT>k=0AOpqW&)wU0UTA^7V_LY z26QLas?Aw3iJ3T&Vp;cOiwV}auqq78^2jvLt}{tg@t#HJclsmXv5TGgPMjqajQ)@O zp3JN0Ed)cP#?Le^axF+0Y8`}jLE2oUF6z0wZaeOmnm~zfH1%3zbFopujy4VW&J(E)N__C778^jK4P)5%fZ#pP8)yO;!hk!u2UfJkh*N5x@@BCR^R?dM)^C z7CdfDXlBd?Gl#c4M5=Msgn~`RjEy8_A>+i?zgi0hvRp@Z2g{@Bkcrb<5CCEFBBi7La zYW;wyMAU-(*GzX%Evk|z9?SfkFzI9i)TdF1u{#IbolFT z-H5Xj*WB)@VL)JH5XT~y3%I;R(V23Yjh%1#G{5q#kXOgT0*#gq$37kxk5f%W)BTKxRdnG zKrXz75xmYU_xzi%ycF2(v?3ZHm-4k!-95F_!k{{IhCx1Q@LYrwlA?M(680JcE=*tH z8foa941}`dgO~DgrIZ$4M+6Ui)tfS$A~zo~FCgah>|m=q4}TxZrAGD!PMdEAgPYf& zja^thUvB!iDumcrap@9c43RHqp1f3RgYEm&s#-TyG?W{7INQQ@d(`fCR@+CO;bO^r zX1Be3W7sM5Zkl@}BrYR*Eht2Y-{ax;Y<2n8OB~;nNJ5h+G+%TWB@q8$503`4qPHpR zA|9OGU{n=aNM4Umrv#aiV$(YlQ%kbxkJ&#$Ae2=NHZtvVTz~!U;Zh=tUD^;}A5O9w ziM<`$TfPE%d2ptN1phDxo*-~QrZmH zIUti7`Vf)l^H|imsfP(To^1s)Y*5|3)Y&iMl3Bc3l%L>}Qep2j%<_DWzaS^dZVl*c!=8Y%p zHyVFBnWSHW3$gjR7wsJ(3KJULs+UtKqM%a4pCl&v&j2~!;^ak~pXdJ4&1l!@Hrbv* zXmobFb5p{}*tmRQhH*TLDjm*(lq0deLn^1eoQ$1g)KN^JkO#P4)}&zwh+hkc+!yeyn0DME|!- zsMUWq`ai7G|FYey*0TGLTkPi&8ldTW`YU;P3;omB5ccp=-)JubtlpE_5Q|&8z$M(W zn2vmSr{wj<`$hhI*v0=o^ildhtWyGhmcyL*$tO1yuhyIK63MYxfbWNl^v<`3xASL= zuFuQEzrj-1)>XnCi%J&9`+w&KC%v};Z1_CA^jeGe5iGxjd^*~>zwcko#nvi=5>MtxIxi+~zL;z83JPO#rdmG1yOs?NAZI7(M{5 z_C@LDU(4j}oR~`u>!Po9K`K4+l~Q%IciU6#b#^|_bCrftIju(BIs|de8ZpcVRkqKF`AguDJLSzLZNTYjUt-c8 z>CNe1tkbUndDsX-oqxg5mFJBf0Y88jIys~VH`5Vxo*PQ9?<&YFT#kw0ixN`>>hqvW6RA-IPT15elBYK4RMP$I@=X@a+2>{;cp|B$MYF^S*fyp{^m9<6iIqC z&XtSDs~p!M^FFY^y~g7z(j+ucbyuS0IW!czKEyL{yN(KUpv9c{8q6pHNgd>Bg2ar3 zEY3<9vw(yUKKWv1_|pi(&ZHoYxC^J9Y?fef3`kaPWvHEd5Qk8$jl^BffJDMmrrM0% zL(T&Si9VDoY&a*Juj;w|4K+9^!oxvR{;5IKv#xPVDZ^~&%euFub%*MuRKx4w&2d*( z7vGMuT~gG<)L+2Q(hAcV*lb<#*h#t4W7Sj|=EavHn=2*~cf$Q21OEoHH{M(mw6BrA zRUHu@Jb2u~Aa9~|R+Q-beUM2E5IrlCRzd4(TlqB^O@oq`ix)C9P*Cs{d$lCqR;>!i zWz^&GZrpNDGH932we*Uw#|;mRMN#zmC5y{D+wi9z(jmQ%WxwIGFeNCroum?qC;dSJx*HVyqMql z)H3Ncak%`nBsZ5R6d+_V7A!#xG8iPEL z7J~h9ly^f8&7oB$4EU$@Ox$+yobn+n`GImMLuqA?Wlyt$=@|-W&xzom9jAz>)fCImODsXRF_p1&$F{F>I0sTo~w@a8x4qtGw zh`+XbND^%(gH7#$2Stv2%_fK5<{J`5trK>+v!Cz!$M7`J6-7y>t~&)Ub4?N9DuQUM zQ!SD7X?HpD?2IGFQgJ|J-r3=u*L;0m?JU5cHH;u5ED}m965kp-O+oeVfS5Wr?!Na| zpKjC&K0A!2&0KhyyVfCT3{1h=FYlc!+3XZ$7CdFQ@Vh7vxvxsNOq_DOew~|8;BP3@ zjgI2^zL3?NA-m`(*K16k-QRAFr{CMqgDyup3Eu&$+)&3vNL>5zJ|!0hvI#vQhG=-t zR9u8jfzxlrR)pgRIhdYuRw)rMNS#$(e|H2(9hSq3BJ7;`%IcGQ8!XctybaH_t>o`Y z&uaeaDvlNR-o{->qsf!Uo?f)mlMNGr$#1ooIYeQ%%7^g@Kdz9D8(MxjJ9s5dpKW)$ zP86d5QG?+4<$4`PkX54^Sj>NO5ON5|hVkoRpkU2$Rk0I3(xnU>w9~ohkG{AHnH>q) zKH01yOI~}L;*)Ju4zx7IBkPg**I1|0NTO+h)Q^;rHLQ$6Y|t(9w?iTlQtmUJP>0xG zNU)zaMfqQRO16D*tZ>7g9`m&Pas*Ia`RD{3u{8>V$6#>3{cf5dB@2d^<1vFedm{@k z=K|(P@@^zE!g?eMz?|_VN^bDw9z{hH!bhBl!CKtEPs<%LOu&8|5fwUEM&U7!vs4TB zu83jQb8*7hBO^UL{Myul2OfZ|$As!lFlA6tuI(sfB)vZs`vx(*ecomcz2#EZrA^P; z;|$xu*tmz?d%$jlH<$!Q-8sjRb1YaD(^mgkcUf+YY2yK%+)C3%(YHR9g#^_E=a!l!{PtT3n_FZaLO6 z;Zhd$O?G@H!-wU$_?AG~{Y1=}&Om`!qSC{`%D@+OKnRH#R2U9I7*B2;cn|z93DLcX zPH#^~0>x;ShCJmjwuYl;;LL_qTbySLO3?~^t>*U!$m)gqrN3#d!HklRr>@KT@nU5Tfj0xXLqk* zq8=i%cix&`|L16KL1(;Q{7H6gTgPqArU0QB?6EjF)w%kYI*I!?;@z8#1z#3>CVE>z zp*I7FV3BDxYZa#7Avp^Lhvdk6ZW@fUNglZXRVD$VO*@{ zvRhnK5okaIi=$^_0;GPZEc?R0<7qVD-D4)EZbrX(3+~p}%+O#TiK9LPaJj9Vt;(ox zO?WrjDG#&UCvODlTD9^^-$=En`=i;r+0w~LaGah{3D3OQoJ-0V+}$2qkek-U%(Nu} z%c(T?x16eNK)@mm(6dU;8-sxh3niI1B}~9#;2j!sw`Oa^QtDnF$_LP2A9f}V3T~vQ z74HX@hSZ$baEVqY1_e3s>`cA+m6^}pYb}tNTMIhZo`xOS%nBl$y9B@nUK&d>!3*Jk z@5;vK=gnMy^l9Bl|2J!xt(Dymp6UOyhFxh~*=(_6eP8zqb4IOK9++dlJ3fF|4U7m$ zNjo+LqC`2SFcff)Cmx!%(tp1bXeHRK_n6l)JsE`@(4IYa^`8Di?m&9fU$_bsAEyv2 zr7RUNKRq8jcXW=SuL0~B>}J(!tZKMB1=;T!d^vu^*h6U?x~ZYmSc!$}=C!t_jtbxz zFm6$yh$W-(6g{&Y>4>^E$Khn(dxy}UiCdgn41^bD!d0`zYr>}IKlV@%);T5dGz@FQ z(5lM2M2f=HxZm4tC+8_DDn+MKwQi0zI~3-3%CCPPps@GPdzK>qzZi7 z?q-SH0R0DlDCfaTaB(1Gso}J$_y}OPQ6QilBWU`i4PCbho44!-qQsW4dxq3T7dj$T z^c1kzu889@$)z3sdQt!iA10KH%?WTpsGiFtfAe9Q5Z{*MkJ5LQ%>xo)M8%zg$y}Z! z+OIX}CF0&geY=$FJIlnx3dV_U_5g>NS?A~r%IBE5{6iU5MY^q&sVm<4}3ikpF;uPxn(X;lv~4sHwDDR7y%O9X)i5a5O2 z0+DQ_#uVJnmYu0ooahoiWyPyXDF=DvJiSgZ-{K^t_(TDjr%xt?<5BkIfCOGMH3Jh_a0@#`TuHve<-W&E!0$DDRdb&*JrE?Zi+k?)|fdXW`=(`))i^ z9C&>9WdmDw2(s^JPjGy>y(Kl{B!8u`t-Ls6`yey`uM%^m~@+v_vf`L}=$)n1 z>M8pT>2hx@v-_Dv%B5jhp3!*x*k8~zH5cki$bWSDwCT~@BR<(#YjEA84_I++y+4H6 z_SO1i>HJHc7&5)$NkLc3xd{H|)tlj+MF-A@okEtZE@3(gpUSpBDsB~7bHpVf^9yVn zI;9Yu!7k>@Ji$u;*wQY&MEwkJ+u`|_V-pw0CGIludi?T|HTemBL5KzU=}&AH&F7dC=G%5{ zinMx{4~^hriBbLu{WVLMF92SzcRZwNI$tbQIHHp;i(E<>#u~0e>9Knk*8j|`#tsw0 zEu=M+;A!V24jnpxaK!D;9)z<4j2iAxP8xq9i-qy*6BVmDdIKbc+m(E?Yo>_pej?{3 zo^?@zn8J$-&o73ydL5HT(6?~=WK%@`>$0b+)6-q!O6pY3p9=-dmIsoFmI@KQ9m)#b zY?0$Mwjj-q@A^dR0@#a0xa}|{Zt12i*Qj9PT@91(c;R-_O34a%g;PPsV= zg~D|f+lkx8>94tHyVJ0D;k@c(=-hx1a3Bht(&rIPkM@8txbF3YsMk0GQ^&B+xz zxn8?hweB~>svDt##>VvL!andfxU&V}h@@W8io{;~`gcA!K8G4D?V@k^G0KXtO`S1T zV!t@RLP~!bDY@r0n1A(H)1>VZSgu97)?#v<%-MFDQN@`!z!{{q4i@5hoe0%@bY;=}8X%6MyqB6Z!ioAQcIujG?OrgW@YOTeUYO{Q%O^s|QcNHag(zOk9N2i0*J=u>dBgAZ~I29Rw z*e0JX{(6K(t;%n(&(g`c$fP;z>DZCcht)po+_VRzZxcLSB3hSyk@`_-4Z&6~#@LnF zTE~;@!hZoWI;_0Ozr$zgt%=tu3zt(O3XHV|wUK6-yP?`c(o@hsp}@{S?X=Vtp}&jJ ze~Sq0_b@yz2F?9A+dDRSE`KaMa0jCuuYEt;eFbO8h!iiD*1Y_8K9-6#XMw>RfyYSw zXx!p~WU95&Alo#!v!*sSh4Bp>F6|AA$u{5YjJmYRdeF zD`YrnmI8mLt)2{uh$>X6bj;mzBlVpFg#FTs;}Zf3G4;ft-E3R6Fe!@Ypb_ynRc&Af zuU|}JK~n~l_EtMP1Mv%9yBmhy8rz}@B9t!m79z-{JqQn0^T$XqL^Jei4AiT^>j`q!?9? zr2R@8-9ub-#@=(f}yp^2>An>_|qPX+!9i*`-Zz_!$klq&ytf{1t&Az&U(NJH_=> zN(Rok9xO3olr9uXxcdFNSnZ+95S~W1tn^S^;uQOf+AUJ&UozD3mnEV79pcN1(2m&e zjlw@hJm|4U8=aASPzuq*t0NCzlfUDD*I$&vGUJz6*q-^%h8_g@-H#cdm!&k1Feb(t ztMYhS=u;7deO#74_R)m8JjyS^3HbAouQu7XRUbjCopx=SaaV?6eHoA>)&vXK8sC&e zR3ASIVRN7mW-kT9a8(=gJ5F>>WQ-#wEl7cUNEH9kRhc2cqc{Y;u`4 zlpT%jxd|!E7;>jE1vgGX+VCK|Wj2@u`ruAiqIyorkC{~uImC0=LN1no5R(R8oPj~e z3)(!zuL7NR-RN9BjvcWh30l$1twj;I%7Hp2&G&;1^?=37-@1mfh{nI_tUzO)h5jBZ z=n8JvG_DaNfGquntzBGnn%LZNZ1E6{Iprjt7hV2=X+iK04MN-j!9!@fOAznCG+)r~ zmdM0L)+Q9e%LjYw2Ne#*Mzv>g-(R3vYx)@!_CJKz^o__fUHb|B6&APcSoPCTo5&lL zfFKlo*^Dfm;U`{$ipR<1+boq0BU<*ue6_m5wVb#r6N(!~Jal%5x6d3X1yfOZ@ZdU@ zF)zmpoc+W7v>5%A(fk^T6dCVGSAu_8)btKQT z355ZJZ(4+fL6!fnTo9x|Z>4EN6@6@U`U6b4T z#AJ7(!p$4kasTlL9JIj)Dug~-3Kc_2t-p)(fvTFVb5wYdmeJqmSykX& z{ikP|$%ocQL(a3p_OaVlUJS$1Obs*U#BOnBC#u>%xSl@`YC0!S5WX|L(aT=cvGb9s zmkFoSq=5U9@x&OX1P;fjo3XGCao%mo9GtW7GD^#Q5%}nH9$-5<6nv8W6=EV!vD{8N zB@Z2+%`YIg_@KYupY0(|YN7}# zZ7>+cut+=jmv1u0iHd%+wUTi&zvUT*IuV{HhoYu8x&90Ah4Hz|`PMo1HZ*lw=YW8v zN{Rk~hslX5$rtgpH^7kC;~yg*EAsl@u|Ai0@_wM>*@SsMt6##AIpEdC2kh!4_UBGG zdR$h;n1UnhlHc>V6Vr15pI&RQ#oU3!iQSChK=O0ld_*QD5oS#M=Pk;YUD_TYcx(;7 z!|H4Ptr@%VeTy)W>txurL)hA1C9>6x=&V+JqPX#)AeUJ2wyt6xuGpme8uri|qmGmO zQP9JB3i3d(h6TTm+>GEYleMlpffxLqbf@0q!REYuQpeIAXoB=8YI7Ygg4rKB3o)NW zq`m`N<#jChNX{zGADEf&wr+sWZod**%ywVoeU|F|AzfJ$`?MgqlgrHWkE>eD@_7r()J&hk!n8!YnHK2WN%`urIgh6@&>=ZB8o5l`Z{xas zn^N@DXX{koE$0sd9Bw_)RjcU5VI1mIkfxLyVQa_5Ft;7{#VM629Zd-+0`m;XwKAHD z-BnIhgFPH8SU5tX7)Tu7U`u>(klZK|p+^dc~7X@UaRDZ-ARRi7nr&<0<3tg?7W4y@6PRyv$%12U; zUxEHt!6~`Ff^samOzVzNM-mlq;`dx3*=OGk&H$a-1;KNkg>7(R5xM5anBLE2C2FcW zJgRLaoEU&#4s`Ix}o-onNKXe=0t;sBwjFMl^|72QtpdZ~F zn4}*p+{589cW+X6*Z%#cF3&quRkk$JF)n58lJ2JDA!d(kmC^N;I_GV%^rh10VW=t5 zgG@FqZaIfTOOChyY2)NjYzEB+69-y0)`p^bsX+b>r<(szo*)SV?q%Xcfb9hWoc$)JDxXzBkI0*!~k?mKM4@e2 zykcNNW$D6fo9Sa-5O1`Wj-8k|6Ec>TRNS@JU(uK+6WQ1w>|Qs@n}jamw!g4u{o>EL zOS(2Pku@x;Eo<4Ve(co|RsfE(2{2mLb@KoTJVYGp6&%(3<%h>Z34#a;ic5!~KHVX- z#a`XiCb#NkK}- z=WJi2o{g13Rp~&A< z6uR(P{w5jwx1U z!fGA`wmnpB)5b!#6NvWkW`k)v$nrA5e(Q`I{Ce(rnJl5WT-I(0-)X{qLmB2QWGK>S zz$k;s;oJ4@ojRgg**C0ZxN$TqGg7UsV)jtOe1J_qzd#vTotVi&@v6z2=Dwc zF_*E$tQyZ88EQ9dFH_A{drAo!GTnwYqBEoxp^kdQSm|_`I^l=-&ax3no{v%25!BCt z*|Y!BQ1q3OnO=E9`N#P(ewXonRe6SyRfyV0r)h7?gz&=1dO-(jMou7m@KBG6yGAFu zzS4f|-7J(>9v{3vA9_)WAQ&KijGdM|V_$Pn)jUFrxf~&U)eD5^J>0-f3f3Z|hX@k^ z%wh}=LTI5MafkB7CxkM3aSMDH5=?CtvV1GwA&p`71Ha2HNLYN3yLP=M_7d;o{FnG4 zz)k9P=-kFQ^xT3JTb(EUnjT;yjnRrmj>VF?i<=KnKG>?ya{lxemxS6ER_k+Xm{<;bI{@q@#PeD38zCYJjALy&<_Wp5X=7)ow z>OUO*$>dEoXK$)flXSCA8#LpS@PB81ocuJ0e5J{gebm)u#gi&8Pfu#c*zeQ3Y^<|f zlB+A+N;X$LJ;jISY=m|nivH2B;io$)4rK@3m#cd&-1UCTNpB44E?dNV4&c{v`JT#d zM{(L;(EXig+m&Da$oj^DF3^|4(3CkwmRpG19@bS(MJ(zbyeX^5Qd@W>=F@emMKi%p zw-j83wKbIOI4%I>(M0qQnzul1b;%vU^>HOT!QYo3yP@VAsfb>5*&6B8(mu+A0t8HO zx4|peEws9}Bwqa-%L_=Q>4lcH*zYI{GUL1<{rpOVmyil{D%r1;IXeMue@Jq5h~%3D z+Yh1O*iN^xU}{IR7?--lzsa+n)K+K{YbPzu@NbKZ-g3fxBqR~J{~j4^(A|3L z_@_u3#E^fbZZQIzz4mQ@EMKt$_mIzU#|vGiE9!LGBd zxX-kcB)gIdLm~}$fuVv;%bXjBWLtGeVSq-)IE*s~`_Y`;c9mvsuOA5*_0|Bc$OB`f zk+gkViO>gV6LwPtWmk4Sa-gN6`=-TFe1o=t1bYHfk~j=VhDi?4XF8GQD~$|Im>)fO z(5%hSeO5cFDp2iDD3v#$pz+GVM$TNg&cb{P1Nt$9kC0ao)7xPEQg`8zZmVh5lt+5g z7OSfM1(clvB>Zkci<7EgZ3qZ9!IUXr^y3R&F*bxFJeA!XPXa^)GTI#o6MR2OjfM6% zBwNYfOlRtTYM`))u1oHctGtD*!jlwf71|nO2(w*SJc_BkB;z5*-2oGQ2iO?JcplK( zBy4<)%L2t5m0g4!_C-o(XC0k^zO|6j>-SN`EVuFbjh1)Y3|Hwxk8u4s;tWw&#A(76 z<{{X8*#-`JF9?$|f|dP~=8Ps5CgAKS;{saEm>B~9g#4-`xkH&U=Q=#v1vDbeco`Z! zjb}bx3*lNe@bJ)YkWp7eYyo0!o4!zYnSg1|M#||iEU!TqEp(M5UFTf zsHHgL!7=5a^?;*_M6 zq;$wV#S8}@(a{07aoI?9Q+JX)>uNd|>n#eWM+M)DlraGLz6*l2re&E#+`%P(M^7Vr zBE!o%#-PQmHZFQInmk@GH-(f@y%4`F$Yk|v4nx-}P z0eZ*&OwC(BVn>KwVR<6ATFptUy5He3r@4jyV@<5Dz0S~kF%4QFaS16G5Qgltz9clp z84rG(m9O(cK2f-*Jv%XZ%T{jkZJV{m6Z1Fbgyb=b>_cOm&-|VxQnzhO?2+6bUcvUp z6RYhktcuZL=RK5di7P~;9B?%0PwwZB7Gh(CtkbO;#{cPFW`3xD6GW4# zQ)V`7H@`0OlHf^1%ru%7xcD|G%8}MI3tDFrvO-?37Q9)otLl-E6z=RX^cyIom{*S+ zqs|)kov)X{{qmML@a=gie|7G;a4)%Z$F>vKsE{BMy$4eoXK)m2Hp6Qmsn$eJg#Xt? zf1A%@pBRov%WdV$b`XMs6UR>n>oe27l!>fh2+Go%=RhDkU2k0QdTPBfb(}b{qqNS? zn`c!&`7Y1$UusHRLqU{U9CErVhu_Oyh~4H5fGOwHZjhow3w~G z$)lqoni$liEh|+!%>xj;sHUh@?&j_H{@S(6f#Lzyo1_cmRcTzSco7xC>$3zKVGzi1Hk>s^v zRn9FjL$(|87V0t>NZ*t;o!yLb)8COHzNt)Yny`yado!h-ro~3iS|mQPc|IO7a4{t{ zTxJyJ@WUWX`XljBX9)o^D{T>Z#AM1!;d8qgT#zukgKQC|N9HAx2z8H?06BAA-W28< zPy=|v&zv0o)|QaJwk-k%wDQSuh7mx3k+GVJ<2nP27?1P^QOBpO!EUFPOMU0;t;7ai z!J&$kUPjFBd!UxtA&4l?0OTTE3rz1+)l!-QWY%!QW)zYqXXbhx4|Z7!LSAoe_2(g6Pg3T8iRcP zvbw5Dcb;SS(XyBb5FlCW`Vz&C2l94XrX7hIW(FaE>zn1eYYhHDR;j)cG^dg%xZH+A zM*KEtVp!VwTfej8m{_W%_nZK1EoVz*t=Dfuw}MZ*+>LU9>yO=;Igyo9p#|-Sb_LBx z(h5jA+Cc*~?%e{2rf4QaI)n610x08w)nLBAYkH-SOy4HrsHd5TdweX>kSTDnH&(Ev zp^ua6)?3~g$*5p&R2L6d1O5|29};<+uc_|pSl^($2g=5rwEW0=@^548ETeN34IA#K z7zu;`&)gY&+wn(6`IlkOpRxi?0 zmOi9Bl&56=X`|B~?b2Y8mO9)SQ9{&|EvC57gXXZJ#etsL2jRrC(tpl* zJfItYPIye)?0Fd3wcV4Z9Eb_oQOHMpz8IZ7{YwN)HYC1)ajsotd`wv4GwNUH3 zqoJep$eu|DX4pNm^*7a{c6z`W0D~R{I7>G7h!5^gc3CwN0X6vG0mil!zk~b$RpGE^ zpz)_rRa{iHNiz|;9zO$CYko5HDotMv@ubJ#5k?xR=_SqN^zS@M&BwY+Re9Pj?l3=W zC_u2`t$N!})LLv+3d-QI_CNqLob3ze`{PWeOBrOLz3RR%?rdCWm$pjX624{h2knT@ zkwkNzqNT&g>Ml~8k}(pqKjWitprl zRfa*LB_UP$u1#2X()Mk5m@`;!drdYFCd1oL`9#i;6OsN~G`>!4?QPvpo}}%fHr{h5 zVMtW?q=hPrOjRjpf<=Q%Toj)UX8r6Qu?~al_rfs->DFw@(&%&1F1&Rd4?CJ^Zz<%- zE~qQgj# z^K69sz*MVZf_A+Xn+?y`J_7c8^wCzh{bZ;V#^v$*I`h?cdy8vWMbQZwTiYEL%iRz7`+o*@);Gzp<<*|EL_r{-zw>v}ury8{aFdxVAiV zj49*GyOZ;gsk0f?#v+=2Bx$Yw`bXLPl7kWAg^`|G`Cl$ttv8P?PV*<=O%P8&tLES zYD)OfYZ5X(n$e)mw?5~e$sc!`JxD-lj_wD4g$LcPYuT0V2hW~+MG8-`0e+Fknyd5 zTErpH&Im=<5Wwg=aJ4q*XzP#7dR8A%kYS5FvA5d=is+!=P6qrS^yeGbDhFHJAn_1W z*;FlMv=L=E(mu_BXL&rs^G`R$U&pI(H5$MDSma9tYCWZ#3K5@PpUpFIJ@9~u3Qg83i*rvW~Jwz2~2NCcgY%I zHWSUB4f_6e_oI)0>_B#K2hh>UC!wT~b-l^?LZibZGAa+oV>ZRYPN?(|K_1mAp z#x;Wgmw|m8D8%#0$EO++tFnXr>_9x8g((<+?^Erm{}pUxJNaBWL(kn1FAKn=77-m| zU7p=h#n}POw|Yf_y!leQHVh*g@v(Lz5KXupF0hRfbnaj=KDqJLxqY>ia;#*fER!y` zHw1*wWq|BV(=)7@uXdd}Mg!Y=9d*orBa1QpLrKE-^CxnDoq1KP$K%PG3c+#1AQm_r z3XB6~?d_IoYYH?Ydetc>kT_lf&o{T2xUOEcH>1&(5_?Y$YBeS>N!q$>tnv^i)1-Ya z{vB0F!ND_lPSNq>C^ECGq>8>}gO9H1B~DEa-QBQDs_8zYH1u*NO7QNHBVXwI&71_hbeli^L*nQ4lqqlG6^w6lR_BQ@fpnzZeq! zj0GZU%W*b!NI9ljPQbmYvH9%@>sZoB=^Fj<_i1)_%zCRPrWN8ST@H65A?WSw_UJjP zF3vInyJdtkCFvg^voFtm+0oZdk?yjxa|@$MLwRVkMHEz4G^8Dbu>BK~q{gD~dSFWX zd3c>muHo^Gq9MD;qjy_rQisZfF>aAdXg;O{i@6GFNl{Fd65ZlD1TiMUP=71}H9EC{ z$^UkFfLRe2?;xV-7UIcx;!y;9sj-fp-_0`|XDKUAHvwo*AT`h5JZ~YD)F{FUR7yiu zNbLujN_I>x4uD13b2$Ys!C&zKDQ*>pF^s2`L!@ zmaXKhcFYyw=4ja*g$6DX0i>Hci8_Y{BUgxMQ~VJPh@v@Pn;8~+77}c7FB_7r>Oi8t zE}8&}q+6&OtGKlwJ7g_x#Af}R-QOpjalBH`)?ITCwubD*7Q?NsP;N1cldTpp4c~_zEOhfB$ z#JaF)rg=r&Z){6 ze$l|ooy+zy*#Xwgn5?y{X0q!T8-sGTL@!U)z)@(X+p=8}+)qyFiCpatXm}5I(uhGW z9vTuM%Rv}EhP^~DPWos|ElAcu2ly|Q!(0`n%Y$~$M%=y7w;wy@Emu+477G>;v4QM1 z#$$QD{Bz!G%l00r$v;Z+0FkEPZJe4?;)$s)3W%Cy;*agLqMhe={WD;le&sK6*SfQ< zFQOr37=LTXL-ijGR0q<})?wB`?xYjLO5@eK9y~um4tTiApW5?bz`Qa+HnktFt}B5* z-wn)AEl``pj(jx!aJ*i!UuLc@FV9@j#s9NRTO|GzFZJ75iVOt+!2G{z%FY(Qp!?sp zQltNucmLm{d``w2EX-x;fj= z8*c8I@J#@3O+B2=m%GlLt#!TtNkuzSeO?L(G(sF)Jm1LOOE0$E+`hxl`~A=RLn}8w zG;eT*-i&Vy-vekA)HpL~lqho?Q=syPv(pEQq(<4$y!3G+SSxL(dY~RU=!#tt8{&-n z(5PK%s1zT>ewjj($^=9m+X)=YwKR`ui*MaqAq{S89Y#`UMC)eMSk0L{D9z!Q>H(K= z6xd2fW^?&VzER8W_b|Zq{tn7R;tA<`FX6}a%g%*lA(Ky8(D4Z*+77XslyCd<*q21t zjcl5W&y7FFHnJ?aS+ujAQq;`hj)U5}?ACs#p4&v`s-O?xN%l?EF9!H?w#8DKb^h+Q zl^fHY40i#1J1;f5brOw2L0$rNZxNCM%tfd_p^y%RQ#3V@vXr2b@ji{iXi@xzTG%pS zLDvpTEtX~`N>l?rnuy4OH(7VYx88iL^jeOQm?f0xI;EBdYA`BOjem02C6aQp&KX>a zY-6?-%CA>D^fMYHbBHQKX)4mzB1Os&FjVlF{{gKjkOec?88-+eUie&236bAwfD6K$ z_irUgq%BAxRyKK{3C0g@(bD1%oMOyC1x8wzrGq&w9&eux-2gf>OLYLJDr$QYpA*8{ zLU2MwPpw(FzTFR(pnPW-jMd=DI5cexlPTJk1oZs0nKw2br{UrHrj(`qJ*K#umu;E>ev}xc!e8~J9`UMPSqWkr8geYU_0$V*MS&4sP)jX{Yn+B zAP&FN1070=JmNhbejr%riFcU2RCE6zU~|IID6hpjUIySJZT*OTM}92=s(scym;{pM zN>Ei?p)B^cS{veRzB_;$#b1OBuWkP6+vRUqy>Xzs9F46QEvQ^sIZatpqsX@`* zc_#sBuDD6yh`W)AWiRFZ$IY$b*7f}PRIoh#&o$9ce+lBW-LduLq;)V@9v% zcwzKp7IqR;+c-wz`y?gnkadg&jD+x(24@XiPHf96Y|G0vyyzf-dIcl__}($3bF4!;HzyfNKSzkcFk zFFeDd=%Hd!rP0~TaDhceQfF#HT`SUGkYu6HSJ*i3viRULkcApk?xiQY1PhEMACGOp z=VVfe!iuf3TgU2(>X6WaY)85J5(hSk(QxMCyn9*wl4+lP(Td%)Oe2qfszKj~h?h;o zVrKATN9LJEF@bvtAig2~0aX!23w*;vBs0bP4X&UI1a1*Urmw;O=UZXea(rL@E;v=c zB&7eH6z%?B{3Da!e)j+WFiy9cwEY1aLeGUdOpdUv02-Qh6F4wkLhR;%Kq3v2u`&S! zh**)$025`xqgU+J_Y0hU5uW3Mvg_c`I4yr7Yn?=Zy z15pxlkE$f;FyLc*Km4ut>SxYC6e?FJKV^9mDrGfmst9!EfNpW-Zi$?sv)w5=5puX= zfp$^pZl>b;{(liqg(@#YDYDW<4)Wp1@W#~dl@?RAy8#@CI@VH)o-XG7^^wq>lox~^ zzO|mD9lAYuwnzZvkDD~CGlB+kAN&Ox2;DZ!F-*NhqES`>vM@SQ-5TM;(C44u7=?X;QStCo;6}|68-;4|5wxlp7tSrcXVjMN|~I+9&%w=ET|Tu$tz4!tQ628hm`R7y`PP%G)`!ZS zBpLXpaM5?N*h*k1mWQZlp~z97Jh$)Lu7rZZ?Y9cZbucj_)pE)Sofh>DQvZZK^rONczuK02L+Idg1X%^2)U^+6 z>_UHxMdOSMv}mR^k5KFOBXw=Vxhm3ojXUT|2&Fav&S`7V1||pxWSy6W?%?)kj8#QA zA!6XbNdFIAX~N=mPtA%Xhj?FSY;Fkf8lC{&l1E|anlLQ~RDtT_*@w01L-AFy$?=iD3j=Ou2fq{X_bHAPIHF5DRrh!vx%s*t#0ZYp=Hx04<|4!C?vXYu{|cRiZtrfNlT~pU2DO6S=u3mMme>M) z0GZMj25EINE&G*G?Ud{Tr2MZT5f8U#ht*r-q8?H$7SWwvc9!xJ$*VjTRCN_C ztau=&`c4kUw##3aIj284zueqXylWRc%eU_64`VjT|LMom45Xy8_1it4iU9z?_22Ro z{r?Y_O3O~i{Ck<0x-m0Qo13t(vKbqhuo%*rSUS7VTH2Y})60sg2#N@*2-aw9Ds4(& z^{gsCCPBj)j~g&|s_ISA0hf#@gJ>8e)kw(-N?UF6N?4V4n%=79S-BzJ#S3-yqK8m3BBTw%X@qL#X zt?Yc+C0+aftdhvRcsl4Fajrf9dMi8IVWp{OrW)2N^`_i$m? z;j7X~@gd@}HMg(?+qn~6HANMR9BndSRIMFo%a#vlq8`}T8 z&JLLc;hHd$o-Xo7XUdS03x?ymai(U{NIz?3nNYECQ;dEu5@|Y+AuD1a=JBOzVcguo zuaX<`^UQdd4k7njX9AWr&flHAJy`}wBgB`^0lO(eP1n$nU3%G7kF5ikuiB`T6rsGT zu8}$gFwXPLM()L+Cqrodols?XHU#B--`f^Isw;|zEK^6%4^BIH-mw%mo ziePaY)E`G8#FNr5ELV{vkAI!-VeD>rQo@CM&)Y2_rN<*iI&E{gotafHFtCLf2>5dR zi+%rUi71-b1d_@$JqMI3WZtmenmOT_Rg-6>H8?Evc>h2cvx-( zM^xU!Fs!ghC~xERZOVejNI$I501tU#y`uLw56W0~7-PF=oNn3G%!m>xrJeTyvv@B4 zvT&D_6Yg1$>S!9VGMU?R;6E4rdTJBNF<*d^A}Lw$;)#A+&|c*!ZbG#zf|?}Vh~mU~ z{vK*Ylj>E~;@QJ7r_!TGqHQ@UXsX@Jn)lMKoU<56vL=2kVo_-6T3Pn^qy%S{V2VGHnS;}L&EN>%}NHXnk6bY+e90o+n6Pp6?)-hp_fs= zz*%{&Oj9*dDN-8>a+-$Fw_|Hq)|*zXp}DM|WcF@rm}b3{R;s!eN{RF!`VG2)4#Ha3 z#RI>Az|U+0Mm|8Im4-&L$GCv|9L{dxX#ggS-nr^qi5Eq{@hXiGcBzr-OiQjj*^*?t zvU%mZW9XM@G>WdCj;=1wUL5^E(bmKB=Dv2xLaZfs4r<_dW@zYvf#aZNz=~t1p=yVN z=kiAAQtC9~K}J6808CUphaC=R48Hiayf|wg3chvLV}9)$mAQO@FX0{6hGjOIH>o4Bl7!eH9a%5Tuh%}ri>anM*ipS@U>n+w>PDK#jS_iqtQ z#~wPY&<#|4o@oP)$jyaQ$moGnx=bTY)CK!@&+epbg_%M_`bGrd=C1PWYCQNmTy7Oq z!RWo?@YT>9c42Pcvc96DO0x$_WQJ1wo?3VC_Lex^T10ikWY2y z3((?jhBlX}KX&T)a4yEWf+{1F?A(M~9(H!pMw59hXjg1DkVFz7EtdQeFz_`B0N3~? zjzaib)sU141M4|?z5kAiQn)1NNOJ^`X`ApGa;2oCltwv(O|(#qdx_{^%?=RVeT}n{ zjNatB66S6wl8pb1Q;vZ!_bw*<22;%_?3F`+F^Gq&G`It7*f1n0yHjo&a2%Qm>8e; z7B}|^I(zw2+%_bMUr?RmC1n9M8>?l#=sac0li(j>IbTx2d9pRQ7UG=^yxY^K=EEb| zx0r#0QdFUmv7$0pk_NiiQkNpQgI$d&+$xo@x-PNvLkaf?frBXp3sK|Rzh5WE8-`*8 z&C)%B~mZN_d8+eQch|(^SYD9iCuC>OK_j2Sdiy*`-dG4r+t2wP7I7*&1p~Pi>Kj} zsNS^~C`fI>6-p186qeYgtWZ;tvPf%acsNFm!0k^S(NUQh#x=A7<6xHYu0T}MX*Gq= zwH<`bncH(sTijkPs*tKS?-2NP^lx~!v!`sW*I^NeiST<<>T=<)swfcT%{j{dj3n%Zx&r2aeD|4&Jix`e2x4D_so z?DRaWG%eN4)NG>?!xHnZll-(atrX2TeS^}N1dZg(*!Yw*_;2GK&{5F+gR~28v@5Vw zw30JZay5!nl$3J25Yl1|3KUhW>%;wnk}L89RXf0c{uY|1A-J&qhRFRpg#J&Vn%cQI zdFngZTiUre)46!KM9a<0QqRmv#?s5tP0&zK#vMuhSIo1s%5|z^va<3tloPXy!BA(U zCno1dWM;>jfe>{6BdCOBwN;JZYDK@v|1~HZOJh?zXH#c-8A)MLIc3o+9c!n9vBYov z^0ACq4QXVAr_-|*?F$DCqC&JX$|UFnT56oQJR${>dLTe7_=@%3OK%VOI`lQ*%MraH z^7SdI002paeQTV17St^yNdQgFzx2KDRMA_B0fMI=d&%f9{VzC(LqM-4``$!hBp}xx z47WZ$$zZSCVGNj&r{ja2K@5o3q{dGUOtD>i6oJGi7<4mq+g5Y$CTude=eAXgEswS9 zcFP!zAs54vhfj+`(7?Na(3Bf@KoNs1P9dKqH=sOOMasV>ZM);rVhCdQ_8DY0I33%4 z&K#I?0pdVpL|ERCUO>qdK94#+i7_5Qy%Vg?1AR}Yg&Fh0#)$<7C#)Inkk;&DzK%lZ zH_`_nM2QZMi7@wmuM+7EP|N^Upf;?q^xlkY1l#d!QTryj$G-hYxcT13h|6ok4bU1Y z6|hd|bxw9sc3&`h0hKtJj>6=)(FA**;JHSdlX&5b@*;IEa)O9iAn zZwR_DuNn#LlfppVbasihH>8a1fpOH{hA77U%BRp+lytBXh;6SG1%iQbF^(`nR@o$A z=)68CbeR~ttjk!A91?kd78@Bs?nF3PB5pb3!_-6u$rXSp<4jRCUPhYw?eyJmTqXg0 zw(2)dd*=hHJ}R?LYruN!U1=XGS8cV~KF-LuMo#>ms68d^w}S7&<4o7snrIp9xn+@U z3mv8HU1bMX`fQf7yTjJs3jnhWYTVGXj%(J3H#=-$JJ`jBjpus8vA)X*=nyVpyA zG`qLjTc_aJb=xf2vBN}r(ybxY!MG~)Sz!}M4Qa8=Xmo4$)YvlG^x0L! z%`x$@Ov8&+*(I>sHaMAYj4(c@S}wwtQ9a9zGI=*lEQz0{dTC5=nLmK7zQCP zg1~`6W<8AyK|h)&POFo_2tjV4N3fe?!=nE(kY0klbmFw)T8!gt+>CD{9!iJYTmDD9 zzz(?fF*jATgxJHW2giS6-!JD8@N_D6SWF+}hXspyF-69)D;h#RpFhB`56D$XXaE7J zJQE=lqj;RAk&)yvkd8GRJy$i zTQg}8;T*b6Rt(0`R5*N{WC$eUsLC;Q4uS8b7V~k4#Ilz<| zut58mtevDVPyr9TA!+juU);Lfo->&u(43*lD#1tmX+k<|gz1##fS=4DJ2`DUt*wXI zFl22a{%+l%ZFIFYq(s6~-|O5KzM<6?yyaO^H|&f-+knY;PBB`(EkEe(fQi$H1%f8M zOa+w72K?8&+_S6^W_yJrtR2=MBn#`z8`L`I9T|`*Xo2B{Y>OB*Np;L_A0P-f6BIt_NNb8nkphVsR2dvY9bMm)gUaIgM+;&9LryYpT7Vp(bi{=DK%5 z@tEkOwy?$tH_3KVH;bUr!W>wt*~qhp&Pnt6Mem=?@$%C9Nxm61OsB|}m5ANAmZ~ti zVpK29T4PXuNs0zd->C6S6kTJJ|lz ziPlLs0eT#EJnV7AK%BI4q4jMn~*o4F5u&k)! zW{wNuXGUw%tuVke(3OiJlripxCJX_z_G~1fD815SZCf6iQVg-F23=v37~7n{6IHj> zltA-ePXtWSwHYl4)rc!`XifGg7akz=KTA|xQcR4aJ`x&hkMD^6{}e}U@D@8X zKWF{>-#%V?e;FEna9@;b1A-D^WH+M)`l{QXZf*~*B>%>|1I)16h)0-O9Y&-6N3NL}-?n*X*o7}|^Z@`(}i!oq`7(u(iT=T`@9ZEj-C+kI4m{<(GS-Zgnk69_A zW=KF`0s^xM35B+45Vl<$UQM*%z$&XO2^%aTobKhphC$w!LZqXhbx9g@A?R)3CRsDW zC~pZZxtWX>HW5d6(IBs_CK@>)h1DamZDu@bH5@nY9#WneV zPi_%s|Dl_xB;h&}v7*9oMCT%=H=?7OiFY0wAe66fXljyTIAz3q9MgYMEHK5wVY-#v znvgD$Nr!eJVM?XcK=XJ2GmkEmr!LBi33RTXWjjv`v?$eIKg1!#GYm(PGZO^rh$F#^6dVS>A821ZqPaNeQrzbFxeXiq1@Mu%>?l#I%?vN zgu`Q`G0Vb_W!Lrj8>G-WXW@_{Kf0li|aiG-Y+!W+k1svyDPmuXiL^lRRC<%+0p$Hdi9sx+J0rM;vz>->H?THzph#l>-?TBVr48j){k%-+FJhDLB15oA`ggC#VT z(l{6k)bz&0uGF_8D@FQWWz+?|e=E}@o=i>Uxw=h{@f-i9(;#|dVz*SIJ`MF>vNu;{ zzc1s`)5Q4a%lYSx@=qQV0kdwXO$fMdMJ6~dXv^9Ugmj=7P~1|fFb5TB%o1Vx$@pdz zy8-3}Ge+N#4ho1;cn~4=Z#DMleYKFrDMuA2aN`WbBVUvL&`cQHfE+{SI^)w77D+Xzyjdr`f1iE0bCB5&{>6hMJe3HPx-#QP z$=tBS8#IRf!zthQ%5?wEq4fC1_<1j%7H>@rrAQ9tD9e~5G}PCi=S@{@M!hEC< zYnr1_8Dcf$nl%P(T+xEwu}MRKPT7yu39sznLC0ZazIQ8aHqEL=x;kLt5hW&H-=>_^ zIceF6>Z+>+(=8bO$}#ZG6;ZqyRa{x?jVw;-nT|U&Pdhx%I%8T*8z)2y*3E|;mv>_V_bI!vt|Pp2NYW!)>mHoq|S zFZen?nQ+z3L)0S9MIPw~?B|aDA~>FZC>ea%r$fZ$?~UFk44?g|ZNKn0l1ncJ*vNXx z6Id5MbHD;Bj9bnzVWpVidH4~4e~0OlK~HNLfA=Ii0Keski6bwjYgf93juY~gQPiZ-jHAkUqY zaQcpa60nLFr*^|}*8PH6)!Vsf?QUx}=(DbXH;3bWMGlAdeu(Hco}4N^;bat3%f+P` zk&DGV$nfeRM7;i$2EJ+k^CL##9kX0Bf}Z5UsHvGb-G{*O4D(-4j(d5?o@*w|D|Q_+ z<1Qf2zJG($=og3|_9VRjU!NM^qh#b*$OiKXkS>K0w;c9swKs8Y+$*xQ)P9Wx;=6?`^8q*Pd zjsSChxBnG+kqR)Y7EK~4RE0)cB*o*vEZf`&B>>hm{t7k1FMHEndKCF%ugCue} zGfpxjS>9xrRk)Q{Ch;Ji{Ir2bKGG(Uqq zaJ299)>Jr_PgI=G-rS9?WH`hd3&=fbB>UmkgS@~yQH~0ktxl-H@Hn8C;Fb?s>(20+ zQhWK$t>@2jqBL^F{k&ZdjOeS@Ou0ii_uwA*LAV5H5v(MPS`4r==qxVBm%k6>rEu*T z;W|_Peg#~IM&Fn18JG&H{`u!X5~{wLU==aQT!Ykl9%p>eOFs_m$<#PMJ#^7E$9)rq?v?ycrGVVn#)&Mi`U`;S`iR_eCQgZq5?_) zD%`sSp*I5Ive@r-`76uVwSTcRD_~k1g|=&K9CQt*xFd)I#7n@TwGX)tNKNc@z~2G{ zmu~9zMw7k^aOHe>8Ya;-=z_3nd!kMoo0#{ayioZKEN^h6Lt`bJEn6>Y9H8_B7LUB^>g&a z1$ksk=*gRzeG6@I3eZ$Q#yRRusPee7 zRfK)n?A9}!-EUi-?KT~TKri4h8eUFCuR4d1m8b8)wAE&%b%p3R7e5fwQ1aV*IU=8r z2fUcBK$jP}&CzQg9WH=F3=VJVOmSgEPd{6BNBcffuaKFnJ}}WyZE3@yI^Ep0c$n8C zhY~iRuHbmV-@;AZb1Gp!+#EZ4V7KmJGh!7-Q(dI!uq-Umc14B&es=aDHf)%{^oUxszr^3vBe+f)hyp#rXJ1^_*l!Wz6RXv zyV|YWkYT(xdAz8FlV%*)q?h|hlm1DqygeF5+|O|g)HIt&)~*kt`?b_L1cxH)=iQxu zih4P`?_$;=s>y>Qknl&nr@V}FToNy?ifB4g+%RTAVeWM)?e;`_Z%ydESv8=+h;PCC z9FltwH4}7i^u1s+xe`4IkNd~%>dYg>l{k>5%?7{SUmLyx->QSXY~Lp%{0vGJBait2 z%OsgDF0T?CsJF6o#@FgENt*lO=I;%8)Jb7|rqA!@)g@a!wFRsdJ&RV8Nt;T6l`4WJER% zcNadY0q^eINo1#bE7`pMDv1n&r9<*Xi9Q_6u?9BYh!gW+w%QFF$+Qh=L_3UzP5?#;JcNV|>6V>Bewe#3YNqJusN=># zQ&{@Kxpwmf&1eHBjkKYKMh_BNXUr#(T(%D)?j!5qFILjFB;_{w;xo(_kVa2je!7YF z(cyNW^bZ^RD(w`#E}CU&zJfKq7~6ux%_m)qWAE$}>G!!lH+|O};XDJ)fs?UusU|Wd#Ah^1QLCq&6oD|>7DcNVbkFu0qPPxn{a z$Va=5FY(q^+|y>Pdk+AjD2i(JI3vy)aoU6ESq0j0sW^oYny&dS*{3qlA4cW3|EIGv zkB74R0ys%_B1$48q_QOxNh!)+C`y^e7!1ZRW8X)#5lNBk8QF;x$zHOQELpS5Qr2wQ z`Cawq{f!<|Z%-ee>96lO_uO;Ox%b@Xxpyo@#NA(1?d#UYJ;w{H?>6{|a@<0Rou5v< z%6I{MuzRymw8ln6-QGjYM!xV{_QU7j+5<-^`)E0{JK1pX=+@l&nLrxDYvPDAcQa12 zP2K(aQ6*z$Sc-K`Q0p9M@13B2anK2D|Yx;SKtQ+#&D@bLtcbaQN!=aHC1 zFCRN5+c4)h^dhpyA!M@9niez1U%btY*EmNCW%sx;bc@~#hNg$)N&J#&`i_FfOa-Uh zuHUXrF>{;$e9(3=l{Y=4g!f|3;Vlnn)Fb#euv+_T6cv^7G(FSjt-SC7Ztz5kK@r~M zH{7u&C*QK8QL~tNGG5w8za&rFY2a@6vp~(@;>hO5CPfv+aoO>O_aqo!r}{228q{~_ zE^ktw+KTNtE%Yw;OsIEYdm*hjC*DYJhwC|VIz+ly#^{G!;jmVpjRmKdMO&9Bp0_TH zGH3-M2c43Ql}ilx0}=(;{Lh6nGx6UOSa#T;e8-GX7gCebdbWbI`IZvu;rdMdyX{jG$77^!Uz*=K+AwZGm4{Qv0xwN# zOwcsXHiX2Tozwq>(X0$HqFM4}TuMngoglSm275AMd*{p4CJy9?eL8n?zc5AAR}Ji} zvJP4yhq_IizP;D8rX<W%~#CebbgbsZc)FYXR4lMtl_De$mUv%d^@7g%bHma?&;oT{RcYgI8{ZyRIvbi(wCc+=1q~R0FU%t6GZaeOqB6Hz% znYwv(bxPcU0s|f1o1;uKZ&0Ptw$+oL_6N-r8)|=Ee8&6Ivih+pAt&^agn1E0!K1Ny zhk+bxk6Hj%r+uhhuzE+6qmJCEcl;@SnQ7>>$uu-esOV2FH!EF3J}vj;-V%w~;y>M2l@gt;UH;Bxt}%!E%6;qy=FsbiEq)7aE|(?w zqiKz(rN`#(+qoiqYoAbaHcr3N{J?QqbC0IFRnS9UOPpczgru~!=A}CZ1_yf&sxPt( zNi?AJ>vt`6`8__j`F-S+15Z!wgod2zpyPQ5SHqogd<4(2u?Im}+V3Pa4^h=i!uZfh zexD7pa!=L09JRaZoHo$hlkY@(rycg5;ZTHb6s63S1D78*gwH!ka*Hux$EC>^tG*3f z?U|l*Fw+1ZT`tU7)!xu-no~LvoMrsD>~&rDhWLG$4w~muKlh@?ykHx>A;BA!*ZGxUh9g)e*{4Jg57!*9*~C#`u)36${hVk()4Z^Y?iv zrs+&$p3%Dz6gSUbml~2L&pSIjA05l#Qd)XS+xKi|$emL&VUb03G3VK29N#(FSby5u z7WSlET267{slvyp;m+yN5M?{wUXhKx6Ta;?4)i5%zAuC;W8U^w@`i}zTf>Qj^!Gdt z$I|nz7I8iFKVx}RKr%A9>};lf_*PZ83+hJZ`}ldXVuyhtvyrQNKbk)9L(t$<>d_IW zUFy~Has1rMuV#;W3>zrUp&ly6Z_TK<@iLSC>g6XPZZ{GS8!=U18=*z<@MB;Xue@M9 zXn(9cZ&q>pri7PqHit5qPUX4G#q@OEy+B~fH=u`6a>A}gEgWrgRM{SWqgYzA!`wEX z4gv@;*%h4p7`_L6S&c)k;mD|dl8^=4w$^g_4rM#7 z&_Zj|^z!l?%z;fuWENjd*TbXA)XHiaVDqIZbh`5ioWm3wMi~fh9!s|?j&zJBo@>8{ z$5iZ(%zAUQfox#ObnKMRM}yjPb0G;v+tTDY;^aG*tEmcNgW2#0bj%LWxr8^<>g|h8 zx`hZ95xHmDogng6O{_XmR=kzGnaeHjg9|HHKjuZ4o(p^Rhr?!^#drni>NS-3 ztN5cAsYLb%n0oj(HsBPRBJR2h&NH|(K3*bsM%nb_Z9&aVxi%f>6%xMqBI)wpk3;M} zg_VaR<0#aI+bn0fmge!QJ#(X${EER+@5x5oCZ5c&p&}$Zcm#(c_}^T}h)Q03I*s|{ z^R$;jfydp!uQsivdhFH{C;3{(o23UzJISRsvj_z+j+3{)`;5bCv+u}m)9xA9YCGxM zY5AdahI?n<@Pu=lns4g~9N%UGQJDZP$Kn3UGkD|i5UF<5{$knK*wd-qj2o3ku9b>h zmna@R_^C{zPObCcFiWMXGL?Xx(ZxsHl(-6-`(*wS{97!nK2}Jm4AFEjNZJ<<=<~PJ z1?i;4N+h1+NBSQ(d~P&<(ffT_G*x@K;enl_c#)psj}srsJZjuEOU$@3bxXWuZ{$o^i(Dg?j(o=vCbpJxZ^YCTw%>r17Ei z=B}$a8uNT+zHO)N?zwO0#@4`6^h1oj-`AU6S)E|hI#cbvoJ`V|8)d97Gc#q3Rb6sx zKX7f_R!5a^x<wb$w_LZ=@O#swD)T0PR*ik}p(6CQu!`da z7`YNIJ;3p~l97u=iAM9x>+xZo>|_f2>D+#fi%(jQPQ4v;VG(-uoPS?6_mQc0dxxy^ z&EIT1oSmc=h;R?Pv+yOLz0A%0I=MA^yLp4zgm^Jh;-UQ+uheonS0&>j+UZbJ)+`gX zhFy-wc+F1HE^^hw+?}}3mvdaX(i`#|*7J0SLkWVb=XKi@)!WEQYaI;I?`5LVD)!Yzj^o+}gKE{memvm)JgzIK3sWk4b|1SKJ07tUqJ*@H?*z z`74hEKDD^CEIm!Jbb8kHgnVjrZij){7jN;H8mgjBE|rjc|=JI%&yw#hQf+Id++y>_hzJ#=`z@=&saKc@dU$ zF4)~z=16yql*ajVorr4`8(MM&FTleqWqupFm29>OIhM|-u4Em=W51E zt`v{KdnEUdof9~@MUkoM;&J62Whc9XMn2Z3)!+gi_en*2E^l+Y9DHuT!YOTgyJL$N zlfc$OSk9xY!>J7=yNWg!P2Ql&8`)SDpDliPw}LA@;_LF8?#j~3deV#RXj%79pDhFt z^^&?buNB}q4Sw7@^T}k z_30*31KNUnoDtT~U4wL(OJ3Sp8lSA7)z#kaaWM@SD`Nl-Lk(R_{cg$u#!WV}<(16tp8-7cfss`g=DBaYWua@@_9v z(JsLNFTO>V08dBbXx>f@CD_v?mtDvrEaisIz^ zH>NYLBu3*ILTiw?3g3E|Rh4l70)?4-Ct z>sZ)%S@L7fxo!367dsB!#l~*V(GQH(WGQs7O>p7274A4R>T67W=e_KOMnBdBv?2!6 zwTz!0%vfOkDrOD8c!?{_BxBc6mvgH0Q@wSFV_`B?+hmxF40hAN?nvu4%d$CBap&Z_ z6&PW{X|jr6v~Opp*-q!_~zOTHy=Y*YWw&YVYW<60$A6kjZBhKNO{>H|-_}ngxyQBT-N?-3STRu)- zm(3b_%Z5C3riy8>Sak2#2YxAq?GdMSWp!sg0-du=9)Fv^YhP98a<{bm3CHY9g{@bc zgp5%*Pwf&M{gn(B`%Bo?482z|RMZ_W$!51K220heab6AKqRu~{XBlO@z>=ujF0a4; z-l5muSmE)0&*H{LN^jP*Z|q}uF&!8lZFBRK?~B;uLQI^Le03MMq+M%lPewSnMQIjG zxGug~(Y;EW6!LBQ3 zcYP}5>d%X=ci(eu&s^P{SKz3Mj%M=F5)SXun8KjFn$iQz97&E&H{gN&mS4xOjeHVj zH;apzznORcY{#&DLxE(o-0P&iVftF8v*%O$CqG*?1}fgw%@dSXd{vgI2}{b5J@&di zJU)^Wu05}uGjiFP&ugJM?8-f5i^wt+{ruyL0UG^5uTjHF#WWY+mv7k*U0ypvrAXx4b_8_`Lte59Yv=M zSU!i%?sXZoT_ol$fSLgLrK1MrkO;R}^!Me#h;BqFM z62_)_P(iLXcua>fP~?k2$1H_0M#f|`^{r;mF5wSy{kv@P8y^XipL;{$9Wcl4#22m^ zv2!;;>EPV~Kk5VKMl)qu8gp;aq61B{;-BVjUqA9_HakCK^Hed%8T`qoDFduu$vHn$ z4zr9P(GSPRY5n!Lc197Ai`X zWPCsJCY-;o-n&2SSwQ2RozzDHEWIP*t1dn;-^rROXfGB`rDM+BI2xa}0r^3!%km<5 z(w*Y;Ss`cFpn;uNpJs@n_C$o_>>fBj%evqEMcSjfdG56Oq0bYMoR>3tOVBL^aiv@$ zshxVpg|JGkT_TxH3gn@+w}QR->8r$iAL_T#mprql87D(HYbP&T^`a)e$cSC}q_9=T zIF!#mfKbD1?_at3*o{*y&DcqTvJ$uTeXSj z;`GP19@#6e!@wLF&sJ2g+FvKmm`4mV9!~ zllkd}9IJ}(dB*}#4bP1QA62Ja&ie(4j+GdE(z(Zw{UoAj`vVlU0S!&(0_Ny!$Ag(S zT=hGg4=q$JgFAc3H?UHGGwaE~$A)AIXTA)goP5EdHK*9f$iR;3>O;V<{1=Thf!km) zmZJZBg(wPxp)A4mfiUnj_FB>;?~Icu_~`jE&yUZ!f1(KZH-P^O$q)-iAQ2b~3%C^y z34ya7udQy}L`GJFX|Z~t*}T0+y;fQgn%Vl2-Tng>=l6RB!Y!)hkzkhv_n|HaX1K8^zgLAP0$%| zV15oFmf{X578Z-aLUDW>_~;VAhv(wp4mF^}>O;V9+6~2VgrhAW)WNt&Uh)GsN$vnv z;aY{P*u75>0%MH9n!`*n7;^|z2UB{|5I{`cIs&+iNGw2Y& z%>Wz^5zZdGnD9#zvB;HgIB+&tBpeN)+``UFZom2v@WbG{kzZ++P>${s z>2YZsK#qZ4=UGLpbl})Y2*^K|>Dk(#ETFu~D)zdq3h;DeV1M9_s}BLss{u)~gBw~v zjpuI*4a<+fMHz&_wd0wfxjtxp%fG1d(@p|PO@li)h{y9aHz*#)7K=cF=w*z7FuWe6 z(P12Dn+^CQ9y^wJD3}f02C9uLu+t?^fR&gs$;kLtO|@crc25Y5jiVLPXeDCVA+a`) zo^RMic!*;lBRjZnZFHuIfB^nHQ39fcC%*}&lQDvD3J$>`9*fVDph#d?LuwR{vE}Uq zO?w1^p8~_!Aek2~)bT6S{aJ%oPWk58rSt_^sAc$G>4r>Rt zSMK^Okb$6M4sUAxMMI$0;a_6TbltkRmD~uSnif@nNaSSF&t-uLn6&dbwlhy)Q13YKGG}xwN%FT-%x+zAzJ;h{LG{= z&@&WRapg4spAP}w`}sFibEKm&7LG=)Gg`@=1#W)?Q-cmLR0V!w!7l-Sf8aM{5O|^c zTk((`gAQP5fxne=qJBOE{L``DFu?$}fuk&uSR5%My9`p81_4vh0>z01=#Brm0QAaq z3}+69u=jT75XM+wZzGV7#D1y|ehGN_sXrHBg~b>`g|_g)&C0xhUIv=5i->+;8iJ01 zBTSK`%n@;s%q0?pZbJ|*iHCmR%(^^7IOJ4uH@+Al!3tWgMn)z?q?;uf`LAj7hi=9w zkk`Ne7kpERftm@)F!oDPfJC|j=m&D?>O;VP1{29&vhn|6aW)t%lo7%8a@F5}0l7iL ziH#`CwJy;T^FvLDF8_b0(S7_?;-TM z$6wU@80b?5{yG-xqHvW7N7U!jSSao$7!t2IzuBZ5KOFif#!eeilf$9nuKGpk#M_ z9Q*;a6hOfYaP=YJi}grA(Y6*gD1<2ne47okvxdqQQq0*m*?{+RfMpBuEbgH`2`~l$ zwLH666(zq6w7d%9#(`BGSBBD%0SOomg+^Jxu_zlyFe@iz1#71x@8&`3u>*d;!=A1g0eRqU~VH_e;w889hKJ!{C-iFzEgnb9kov5p;GE$YsRo@hqretP3>+vsoht zQ7y&eC)E{2&_-+PJ=1DAH*$ahIY8GCJ15rq_gE5zBnuG{hEQOD7O+kM>-*J*fVai| z9{fXX5|OWqg_b=6;O~LiBXJynIsGwy-FYM@ZG}$(P+0_6g*dOMdXk`<;V_mKq_V(d zi)3#n=s*E58Cf|;;^#xaYhC><7qJpyVc%ttFbw&FliCU(#l{2A+w+qTehGMQ-`|Qr zeWydjunq}}p;|E3n!wsjl!(d@{D)N7y4IG|k72O@uB`&%7%_Z*#Bbq0>;cMrR&dDK znrhpqyd~(yZV-2f;l0r$;Pyzve~OW$`g4%g(M1k;p&T%a6p>k?V*U&cH!;DklxH#6 z^(L^f517=qf_cbWFzShAh$sD-jPH|(e;WkRy*gbRc5?$G<$}MJ<1T+b1bpc8Ka;WU zRO)H^74Aw944Z-X9RJA$zXUvG_Mf4Rkj8M33t>h-miaJ}Eup;GU!4Sk__Hl*#fiWC zZ{kc=s^usIX>r`m?{We_YaW3=8KTz77X6txB-#q(&Yy*EM%utvTnNH*Oeg{zY(aQhLjAbm z_-88!_@&K%246q6dRLz?qydWC0>>dvX9Yrk#{Q>n{m;;bP+wX*>s>3TxOsrG&cUC$ zgI@xEhuD7-v2OAI+>?73F`%Xzp|I9K$_jrd!4!^#u16(b7jMi5iDwDKZ{qN<@$4V6 zZNQWe;7NOs0T%Z56Nq|Iz*3+(zxojH9AL4u?&R?QJP7P&*doA@RKM;BhN+PG?g#aq zs}^g=nDV7RV}r2^1H}PQOM)81&!e105`peiz=?=$RAT>!eAAVw1=wU**)hUdTc9Az z`a?;zZ%Tn>9)ldWYt=$4o>b}zAqR~#LcvMpetP&wraG9haDllT9}(|8*yj4h=0A8~ zH3N1`LAHl5baL<;+)9@80frzhwO+chE^ws)hI7Q(Aknb($7u+C$4z_ShSxwSBW}7o zSeUM(Dv6jVeK4K>K-KTM|1);3{nO?`AW<81kmy-(^@BXy8j#M&-A(O zAa`=IQYZo8l^lc-38K!TdH5UlN(5ic&+7^ZWLqdr0|`14jC27a0o>`o5wI@vln2NA zATTn5L3AQslFPC{7)W(u69v33M{V7>k_W4b&yk_pbg>1|GM2zX9i%H-2n(| z9Bor<3ImaW02~lZv{xSj{&Xt@+Xjv^2ZuU;-*+PA)3^7u2nPY)+v2q~)P3#0!>&^& zefashb7ZQvEe3%rfg6u(-5NK*vZ;)kB1sWI@SMT6no2FS$ik74*7)!Y)i{Q>TB zhA?YGT1-+W2$C-W6D&wq9|GQI^Y5_hj*DYE7u6?$p02g_Ub& z(8uJ!%ZX>+c)kB1<)8WG${sz`+@@mj!BQpY>M`J=hgY3*Wlk`9`9DapL*YO<9cF_u z2ghhanC$;|c#aT>*<$z~#H@_o-|SC5sP0Gwxq(a=c$hqqq@ur)^rM$nHnJg_Rc0zldXpsz<85|=LRBiGXg231T?`q)Ido7HG z64zY^nsf;`#c?8O_LhGk?S~2f*>Zts9H-Q&h$xVCQo+)mxXTi;e-2 zBNPlRMWVLF+5d$wOFJ~E`omCYll9wYX7Myj7_=`OnDYdY$PA~y7l}ifput(XP^0aA z?odu35UBEtL65org+L725(Rb&u~xQF!dSnW$%O;g)B&?7HKNvudHsbjFoy>V1W>cL zH2LS1qIKQ$|AWddBFWO=g1cX4LpET0oa7GWwy^iD5v*in-WqFD_1pD%a7(y_qZ8Po zL>k&cZ6*dke;sWNGTj^~59}sVGBs*lCBZ@wVgCJyWQgJNdeeL!7I^O*hy`HfwfYe7 z@bq%4+8mR9Rv1z7CAu$Z#eWr2hRwIg*Lr}SVHtgjw}X#?>N(#za+S!j$cB+QHD ztDqowcaU&y?WSpe`MR+6*MGO!hlM%8h^PUJ6XIEzde`r8uqwM=E4ht+IIr* z8Q#@ySjnBKy`-RaD5O0EZXkT3WtST;=pUGXp%lnzjeQ06VUAt z0P_>+=srRkiol@JkS`5iR@Dy~0VuG;x(4}h^fyRb@J0#L3!}^$cPbb_dGre?e+Uq1 z`SvSmBpPmI1u_|gIb~0sol^t76U(u7;BhYfhKa;N?z|f)adbZgjrIc3j@XuolpBAI zH}=>cx{+EPz23zmmk5;11xoS~H9U%n6mp%FQS{jIO#_f+#(@9qCBofgB8>yJsh>-H z2wQg01qy8ht*!+zo|r4Tog{ZFdl*lKcH`LkLCZ8%8(f zgX~@alqFty$MBN^f(a3LC4z*e9$(vh2SEVg1r{J)aV72{1x8w75T>Y=`~Zs6(2II5 z`*`$IFy@&+Y$DdV6U-WZNn@*S1qxr_bpsOMnn#VdQ-E_VfcOL)V)Y^5Tg6F%O%NoY zLD<~b2SC#*U^?%X-{W7<;O;~&5$-i=r-T50i z-|s(JD`f3mCI1Q;{3{`AZ!TFYZ|yZC|H|_S_?t$py_96FytT)N{wwd$qrZ{&<1o=P Ws+1skf}J$*@1it_YXu-^lKl^CROikB diff --git a/examples/mimo/vendor/old_energon_multimodal_provider.py b/examples/mimo/vendor/old_energon_multimodal_provider.py deleted file mode 100644 index a0ca5aaf083..00000000000 --- a/examples/mimo/vendor/old_energon_multimodal_provider.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - -"""Previous-branch MIMO Energon encoder fixture for dataloader parity checks. - -Bridges energon's ``MultiModalPackingEncoder`` (which produces 1 image-placeholder -per ```` tag) to MIMO's contract (which requires N placeholders per image, -where N = num_tiles * embeddings_per_tile, for 1:1 ``masked_scatter_`` alignment). -""" - -import warnings -from typing import List, Optional - -import torch - -from megatron.energon.task_encoder.multimodal import ( - MultiModalPackingEncoder, - PackingConfig, - VisionConfig, -) -from megatron.energon.task_encoder.multimodal.sample_types import PackedSample -from megatron.energon.task_encoder.multimodal.vision_tokens import get_num_image_embeddings - - -# --------------------------------------------------------------------------- -# Tokenizer adapter: Megatron tokenizer → energon TokenizerProtocol -# --------------------------------------------------------------------------- -class _TokenizerAdapter: - """Wraps a Megatron tokenizer to satisfy energon's ``TokenizerProtocol``. - - Handles both HuggingFaceTokenizer (single wrapper) and MultimodalTokenizer - (double wrapper) by walking the ``_tokenizer`` / ``tokenizer`` chain. - """ - - def __init__(self, megatron_tokenizer): - self._tok = megatron_tokenizer - # Walk the wrapper chain to reach the HF PreTrainedTokenizerFast. - # Chain: DefaultTokenizerVision._tokenizer → MegatronMultimodalTokenizer - # MegatronMultimodalTokenizer.tokenizer → HF AutoTokenizer - # IMPORTANT: Do NOT drill into PreTrainedTokenizerFast.tokenizer — that's - # the raw Rust tokenizer whose encode() returns tokenizers.Encoding, not list[int]. - inner = megatron_tokenizer - # Unwrap DefaultTokenizerVision → MegatronMultimodalTokenizer - if hasattr(inner, '_tokenizer'): - inner = inner._tokenizer - # Unwrap MegatronMultimodalTokenizer → HF AutoTokenizer - if hasattr(inner, 'tokenizer'): - inner = inner.tokenizer - self._hf = inner - - @property - def pad_token_id(self) -> int: - return self._tok.pad - - @property - def eos_token_id(self) -> int: - return self._tok.eod - - def encode(self, text: str, add_special_tokens: bool = True) -> list: - return self._hf.encode(text, add_special_tokens=add_special_tokens) - - def decode(self, token_ids, skip_special_tokens: bool = False) -> str: - return self._hf.decode(token_ids, skip_special_tokens=skip_special_tokens) - - def convert_tokens_to_ids(self, tokens): - return self._tok.convert_tokens_to_ids(tokens) - - -# --------------------------------------------------------------------------- -# MIMO-specific MultiModalPackingEncoder subclass -# --------------------------------------------------------------------------- -class MimoMultiModalPackingEncoder(MultiModalPackingEncoder): - """Subclass that remaps energon batch output to MIMO's forward() signature. - - Key transformation: expand each single ``image_token_id`` placeholder in the - token stream into ``num_tiles * embeddings_per_tile`` copies so that MIMO's - ``align_embeddings_by_token_positions`` can do a strict 1:1 scatter. - """ - - def __init__( - self, - vision_config: VisionConfig, - packing_config: PackingConfig, - tokenizer, - encoder_name: str = "radio_encoder", - encoder_input_key: str = "x", - target_seq_length: Optional[int] = None, - ): - super().__init__(vision_config, packing_config, tokenizer) - self.encoder_name = encoder_name - self.encoder_input_key = encoder_input_key - self._target_seq_length = target_seq_length - - # Compute embeddings per tile using the standalone math function. - self._embeddings_per_tile = get_num_image_embeddings( - img_h=vision_config.img_h, - img_w=vision_config.img_w, - patch_dim=vision_config.patch_dim, - class_token_len=vision_config.class_token_len, - disable_vision_class_token=vision_config.disable_vision_class_token, - pixel_shuffle=vision_config.pixel_shuffle, - conv_merging=vision_config.conv_merging, - use_tile_tags=vision_config.use_tile_tags, - max_num_tiles=vision_config.max_num_tiles, - use_image_break_token=vision_config.use_image_break_token, - ) - - def batch(self, samples: List[PackedSample]) -> dict: - """Override to expand image placeholders, build packing_kwargs, and remap to MIMO format. - - Energon's token stream has 1 placeholder per image. - MIMO needs ``num_tiles * embeddings_per_tile`` placeholders per image - for 1:1 ``masked_scatter_`` alignment. - - The base class pipeline (preencode → pack_selected_samples) already - computes ``cu_lengths`` using expanded ``total_len`` values, so the - cumulative lengths are correct for the MIMO-expanded token stream. - - When ``target_seq_length`` is set, samples whose expanded length would - exceed the limit are **right-truncated** at image boundaries. - - Returns dict with keys: input_ids, labels, loss_mask, position_ids, - modality_inputs, and optionally packing_kwargs. - """ - image_token_id = self.packing_config.image_token_id - ignore_index = self.packing_config.ignore_index - pad_id = self.packing_config.pad_id - emb_per_tile = self._embeddings_per_tile - - expanded_tokens_list = [] - expanded_labels_list = [] - all_images = [] - - for sample in samples: - tokens = sample.tokens - labels = sample.labels - num_tiles = sample.num_tiles # e.g. [5, 3, 1] for 3 images - - budget = self._target_seq_length # None means unlimited - - # Expand each single image placeholder → N copies, respecting budget. - new_tokens = [] - new_labels = [] - img_idx = 0 - truncated = False - kept_tile_count = 0 - - for i, tok in enumerate(tokens.tolist()): - if tok == image_token_id: - n_tiles = num_tiles[img_idx] if img_idx < len(num_tiles) else 1 - n_tokens = n_tiles * emb_per_tile - if budget is not None and len(new_tokens) + n_tokens > budget: - truncated = True - break - new_tokens.extend([image_token_id] * n_tokens) - new_labels.extend([ignore_index] * n_tokens) - kept_tile_count += n_tiles - img_idx += 1 - else: - if budget is not None and len(new_tokens) + 1 > budget: - truncated = True - break - new_tokens.append(tok) - new_labels.append(labels[i].item()) - - if truncated: - warnings.warn( - f"Sample truncated to fit target_seq_length " - f"({self._target_seq_length}): kept {len(new_tokens)} of " - f"~{len(tokens)} original tokens, {img_idx}/{len(num_tiles)} " - f"images ({kept_tile_count} tiles). " - f"Consider increasing --total-seq-length or reducing " - f"--max-num-tiles.", - stacklevel=2, - ) - - all_images.extend(sample.images[:kept_tile_count]) - expanded_tokens_list.append(torch.tensor(new_tokens, dtype=torch.long)) - expanded_labels_list.append(torch.tensor(new_labels, dtype=torch.long)) - - # Pad to target length or max length in batch - max_len = max(len(t) for t in expanded_tokens_list) - if self._target_seq_length is not None: - max_len = self._target_seq_length - - B = len(samples) - tokens_batch = torch.full((B, max_len), pad_id, dtype=torch.long) - labels_batch = torch.full((B, max_len), ignore_index, dtype=torch.long) - - for i, (t, l) in enumerate(zip(expanded_tokens_list, expanded_labels_list)): - tokens_batch[i, : len(t)] = t - labels_batch[i, : len(l)] = l - - loss_mask = (labels_batch != ignore_index).float() - # Don't train the model to predict tokens — they are special - # placeholders replaced by vision embeddings, never naturally generated. - loss_mask[labels_batch == image_token_id] = 0.0 - position_ids = torch.arange(max_len).unsqueeze(0).expand(B, -1).contiguous() - - result = { - "input_ids": tokens_batch, - "labels": labels_batch, - "loss_mask": loss_mask, - "position_ids": position_ids, - } - - # Only include modality_inputs when there are actual images. - if all_images: - imgs = self.tiling_strategy.stack(all_images)[0] # (total_tiles, C, H, W) - result["modality_inputs"] = { - "images": {self.encoder_name: {self.encoder_input_key: imgs}} - } - - # Build packing_kwargs from base class cu_lengths when packing is active. - # The base class pipeline computes cu_lengths using expanded total_len, - # so they match our MIMO-expanded token stream. - is_packed = any(len(s.cu_lengths) > 2 for s in samples) - if is_packed: - # Build per-sample cu_seqlens from PackedSample.cu_lengths. - # With micro_batch_size=1 (required for packing), B==1. - assert B == 1, f"Packing requires micro_batch_size=1, got B={B}" - sample = samples[0] - cu_seqlens = sample.cu_lengths.to(dtype=torch.int32) - - # Clamp to actual sequence length (cu_lengths are based on expanded - # total_len which should match, but clamp for safety). - cu_seqlens = cu_seqlens.clamp(max=max_len) - - # Ensure starts at 0 and ends at max_len. - if cu_seqlens[0] != 0: - cu_seqlens = torch.cat([torch.tensor([0], dtype=torch.int32), cu_seqlens]) - if cu_seqlens[-1] != max_len: - cu_seqlens = torch.cat([cu_seqlens, torch.tensor([max_len], dtype=torch.int32)]) - - # Compute per-segment lengths and max segment length. - segment_lens = cu_seqlens[1:] - cu_seqlens[:-1] - max_seqlen = segment_lens.max() - - result["packing_kwargs"] = { - "cu_seqlens_q": cu_seqlens, - "cu_seqlens_kv": cu_seqlens, - "cu_seqlens_q_padded": cu_seqlens, - "cu_seqlens_kv_padded": cu_seqlens, - "max_seqlen_q": max_seqlen, - "max_seqlen_kv": max_seqlen, - "total_tokens": torch.tensor(max_len, dtype=torch.int32), - } - - return result diff --git a/megatron/core/distributed/finalize_model_grads.py b/megatron/core/distributed/finalize_model_grads.py index 2857cb5c99a..dff535a0809 100644 --- a/megatron/core/distributed/finalize_model_grads.py +++ b/megatron/core/distributed/finalize_model_grads.py @@ -489,10 +489,6 @@ def finalize_model_grads( config.timers('embedding-grads-all-reduce').stop() if config.moe_router_enable_expert_bias: - if tp_dp_cp_group is None: - raise RuntimeError( - "pg_collection.tp_dp_cp is required when moe_router_enable_expert_bias is enabled" - ) _update_router_expert_bias(model, config, tp_dp_cp_group=tp_dp_cp_group) reset_model_temporary_tensors(config, model) diff --git a/pyproject.toml b/pyproject.toml index e40fb704f7e..a223bf6ada4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,7 +204,7 @@ transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "17ae86b64d7f75653351664f5d8c9e466faede00" } emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "v0.2.0" } nvidia-resiliency-ext = { git = "https://github.com/NVIDIA/nvidia-resiliency-ext.git", rev = "b2bb3d728a18795807d9f76c535e005a609a1b01" } -megatron-energon = { path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" } +megatron-energon = { git = "https://gitlab-master.nvidia.com/sasatheesh/Megatron-Energon", rev = "d456cbd4a9a8a760b20be51194a0209c9a945b0a" } [tool.isort] profile = "black" # black-compatible diff --git a/uv.lock b/uv.lock index ced46cb032f..4f86c47869f 100644 --- a/uv.lock +++ b/uv.lock @@ -2792,8 +2792,8 @@ requires-dist = [ { name = "hypercorn", marker = "extra == 'dev'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, - { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'dev'", path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" }, - { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'lts'", path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" }, + { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'dev'", git = "https://gitlab-master.nvidia.com/sasatheesh/Megatron-Energon?rev=d456cbd4a9a8a760b20be51194a0209c9a945b0a" }, + { name = "megatron-energon", extras = ["av-decode", "multimodal"], marker = "extra == 'lts'", git = "https://gitlab-master.nvidia.com/sasatheesh/Megatron-Energon?rev=d456cbd4a9a8a760b20be51194a0209c9a945b0a" }, { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy" }, @@ -2880,7 +2880,7 @@ test = [ [[package]] name = "megatron-energon" version = "7.3.3.dev30+gd456cbd4a" -source = { path = "examples/mimo/vendor/megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl" } +source = { git = "https://gitlab-master.nvidia.com/sasatheesh/Megatron-Energon?rev=d456cbd4a9a8a760b20be51194a0209c9a945b0a#d456cbd4a9a8a760b20be51194a0209c9a945b0a" } dependencies = [ { name = "braceexpand" }, { name = "click" }, @@ -2898,9 +2898,6 @@ dependencies = [ { name = "tqdm" }, { name = "webdataset" }, ] -wheels = [ - { filename = "megatron_energon-7.3.3.dev30+gd456cbd4a-py3-none-any.whl", hash = "sha256:34c309d2baa623eabedf0cc6e94d430fc5b0abb09d9f2b5d20da2464b54cce54" }, -] [package.optional-dependencies] av-decode = [ @@ -2916,47 +2913,6 @@ multimodal = [ { name = "transformers" }, ] -[package.metadata] -requires-dist = [ - { name = "av", marker = "extra == 'av-decode'", specifier = ">=14.4.0" }, - { name = "bitstring", marker = "extra == 'av-decode'", specifier = ">=4.2.3" }, - { name = "braceexpand" }, - { name = "click" }, - { name = "ebmlite", marker = "extra == 'av-decode'", specifier = ">=3.3.1" }, - { name = "einops", marker = "extra == 'multimodal'" }, - { name = "filetype", specifier = ">=1.0.0" }, - { name = "filetype", marker = "extra == 'av-decode'", specifier = ">=1.2.0" }, - { name = "mfusepy" }, - { name = "multi-storage-client", specifier = ">=0.33.0" }, - { name = "multi-storage-client", extras = ["aistore"], marker = "extra == 'aistore'" }, - { name = "multi-storage-client", extras = ["azure-storage-blob"], marker = "extra == 'azure-storage-blob'" }, - { name = "multi-storage-client", extras = ["boto3"], marker = "extra == 's3'" }, - { name = "multi-storage-client", extras = ["google-cloud-storage"], marker = "extra == 'google-cloud-storage'" }, - { name = "multi-storage-client", extras = ["huggingface"], marker = "extra == 'huggingface'" }, - { name = "multi-storage-client", extras = ["oci"], marker = "extra == 'oci'" }, - { name = "myst-parser", marker = "extra == 'dev'" }, - { name = "numba", marker = "extra == 'tar-patcher'" }, - { name = "numpy" }, - { name = "pillow", specifier = ">=10.0.1" }, - { name = "pyyaml" }, - { name = "rapidyaml", specifier = ">=0.10.0" }, - { name = "ruff", marker = "extra == 'dev'" }, - { name = "s3fs" }, - { name = "sortedcontainers", marker = "extra == 'av-decode'", specifier = ">=2.4.0" }, - { name = "soundfile", marker = "extra == 'dev'" }, - { name = "sphinx", marker = "extra == 'dev'" }, - { name = "sphinx-click", marker = "extra == 'dev'" }, - { name = "sphinx-rtd-theme", marker = "extra == 'dev'" }, - { name = "sphinxcontrib-napoleon", marker = "extra == 'dev'" }, - { name = "torch" }, - { name = "torchvision", marker = "extra == 'multimodal'" }, - { name = "torchvision", marker = "extra == 'transforms'" }, - { name = "tqdm" }, - { name = "transformers", marker = "extra == 'multimodal'" }, - { name = "webdataset" }, -] -provides-extras = ["aistore", "av-decode", "azure-storage-blob", "dev", "google-cloud-storage", "huggingface", "multimodal", "oci", "s3", "tar-patcher", "transforms"] - [[package]] name = "mfusepy" version = "3.1.1" From 67df146ed53ea3f203b6b77fabd594dd978cbb1c Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 18:35:09 +0000 Subject: [PATCH 11/44] NMFW-464 avoid partial hybrid logging groups --- megatron/core/models/hybrid/hybrid_model.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index ba33352533c..8d781473526 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -186,11 +186,15 @@ def __init__( self.mtp_pattern = parsed.mtp_pattern self.mtp_num_depths = parsed.mtp_num_depths - logging_tp_group = getattr(self.pg_collection, 'tp', None) - logging_dp_cp_group = getattr(self.pg_collection, 'dp_cp', None) - if logging_tp_group is None or logging_dp_cp_group is None: - logging_tp_group = None - logging_dp_cp_group = None + logging_pg_kwargs = {} + if ( + getattr(self.pg_collection, 'tp', None) is not None + and getattr(self.pg_collection, 'dp_cp', None) is not None + ): + logging_pg_kwargs = { + 'tp_group': self.pg_collection.tp, + 'dp_cp_group': self.pg_collection.dp_cp, + } layer_type_list, layer_offset = select_pipeline_segment( parsed.main_pattern or '', @@ -198,8 +202,7 @@ def __init__( vp_stage, first_stage_layers=self.config.num_layers_in_first_pipeline_stage, last_stage_layers=self.config.num_layers_in_last_pipeline_stage, - tp_group=logging_tp_group, - dp_cp_group=logging_dp_cp_group, + **logging_pg_kwargs, ) # Determine if MTP is needed (based on pattern parsing) From 7423b4de581957ee3ad91c9551e7344f4220cb0b Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 19:13:40 +0000 Subject: [PATCH 12/44] NMFW-464 remove hetero runtime wrapper --- examples/mimo/training/hetero/loop.py | 32 ++++++++++++------------ examples/mimo/training/hetero/runtime.py | 27 ++++++-------------- examples/mimo/training/hetero/step.py | 23 +++++++---------- 3 files changed, 32 insertions(+), 50 deletions(-) diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 55d29695718..667a80e9e90 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -9,16 +9,11 @@ import torch -from megatron.core.models.mimo.optimizer import get_mimo_optimizer -from megatron.core.optimizer.optimizer_config import OptimizerConfig -from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator -from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage - from examples.mimo.data.hetero_mock import MockVLMIterator from examples.mimo.training.hetero.args import prepare_args from examples.mimo.training.hetero.distributed import print_rank_0 from examples.mimo.training.hetero.logging import HeteroTrainingLogger -from examples.mimo.training.hetero.runtime import HeteroRuntime, build_mimo_runtime +from examples.mimo.training.hetero.runtime import build_mimo_runtime from examples.mimo.training.hetero.scheduler import build_optimizer_param_scheduler from examples.mimo.training.hetero.step import train_step, wire_training_hooks from examples.mimo.training.hetero.topology import ( @@ -28,6 +23,11 @@ is_rank_in_grid, ) from examples.mimo.utils.hetero import debug_rank +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.models.mimo.optimizer import get_mimo_optimizer +from megatron.core.optimizer.optimizer_config import OptimizerConfig +from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage def run_train_loop(args: argparse.Namespace) -> None: @@ -36,18 +36,18 @@ def run_train_loop(args: argparse.Namespace) -> None: encoder_size, llm_size = prepare_args(args, world_size) topology: Optional[HeteroTopology] = None - runtime: Optional[HeteroRuntime] = None + model: Optional[MimoModel] = None try: topology = create_topology(args, encoder_size, llm_size) torch.manual_seed(args.seed) debug_rank("building MIMO model") - runtime = build_mimo_runtime(args, topology) + model = build_mimo_runtime(args, topology) debug_rank("wiring training hooks") - wire_training_hooks(runtime, topology) + wire_training_hooks(model, topology) debug_rank("building MIMO optimizer") - optimizer = build_optimizer(args, runtime) + optimizer = build_optimizer(args, model) opt_param_scheduler = build_optimizer_param_scheduler(args, optimizer) debug_rank("MIMO optimizer ready") @@ -55,7 +55,7 @@ def run_train_loop(args: argparse.Namespace) -> None: communicator = MultiModulePipelineCommunicator( topology.module_to_grid_map, topology.module_dependency_map, - runtime.model.config, + model.config, dim_mapping={"s": 0, "h": 2, "b": 1}, module_output_ndim={topology.encoder_name: 2}, ) @@ -75,22 +75,22 @@ def run_train_loop(args: argparse.Namespace) -> None: for iteration in range(1, args.train_iters + 1): debug_rank(f"iteration {iteration}: train step start") result = train_step( - args, runtime, topology, optimizer, opt_param_scheduler, communicator, data_iterator + args, model, topology, optimizer, opt_param_scheduler, communicator, data_iterator ) logger.record_step(result) logger.maybe_log(iteration, optimizer, result) debug_rank(f"iteration {iteration}: train step complete") finally: - if runtime is not None: - runtime.destroy() + if model is not None: + model.destroy() if topology is not None: topology.destroy() -def build_optimizer(args: argparse.Namespace, runtime: HeteroRuntime): +def build_optimizer(args: argparse.Namespace, model: MimoModel): """Build the MIMO optimizer for active hetero module optimizers.""" return get_mimo_optimizer( - runtime.model, + model, OptimizerConfig( optimizer="adam", lr=args.lr, diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index fe389dc7c0a..78b97c253a3 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -6,17 +6,10 @@ import argparse from contextlib import ExitStack, contextmanager -from dataclasses import dataclass from typing import Optional import torch -from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig -from megatron.core.models.mimo.config.base_configs import MimoModelConfig -from megatron.core.models.mimo.model.base import MimoModel -from megatron.core.process_groups_config import ProcessGroupCollection -from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed - from examples.mimo.model_providers.nemotron_moe_vlm import ( get_vision_encoder_module, iter_vision_projection_modules, @@ -25,20 +18,14 @@ ) from examples.mimo.training.hetero.topology import HeteroTopology, is_rank_in_grid from examples.mimo.utils.hetero import debug_rank, get_group_rank_or +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.models.mimo.config.base_configs import MimoModelConfig +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed -@dataclass -class HeteroRuntime: - """Runtime-owned model state for a hetero MIMO training run.""" - - model: MimoModel - - def destroy(self) -> None: - """Destroy runtime-owned model communication state.""" - self.model.destroy() - - -def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> HeteroRuntime: +def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> MimoModel: """Build the MIMO model and wrap active modules in MCore DDP.""" language_pg = topology.language_pg vision_pg = topology.vision_pg @@ -82,7 +69,7 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> He wrap_active_modules(args, mimo_model, topology) broadcast_active_params(mimo_model) - return HeteroRuntime(model=mimo_model) + return mimo_model def wrap_active_modules( diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 82665e656ad..6a3fab2ceeb 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -13,20 +13,16 @@ import torch.distributed as dist import megatron.core.pipeline_parallel.schedules as schedule +from examples.mimo.training.hetero.runtime import build_no_sync_func, zero_active_grad_buffers +from examples.mimo.training.hetero.scheduler import get_global_batch_size +from examples.mimo.training.hetero.topology import HeteroTopology +from examples.mimo.utils.hetero import debug_rank, is_process_group_member from megatron.core.distributed.finalize_model_grads import finalize_model_grads from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY +from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator from megatron.core.pipeline_parallel.utils import is_pp_last_stage -from examples.mimo.training.hetero.runtime import ( - HeteroRuntime, - build_no_sync_func, - zero_active_grad_buffers, -) -from examples.mimo.training.hetero.scheduler import get_global_batch_size -from examples.mimo.training.hetero.topology import HeteroTopology -from examples.mimo.utils.hetero import debug_rank, is_process_group_member - @dataclass class TrainStepResult: @@ -39,9 +35,8 @@ class TrainStepResult: num_zeros_in_grad: Optional[int] -def wire_training_hooks(runtime: HeteroRuntime, topology: HeteroTopology) -> None: +def wire_training_hooks(mimo_model: MimoModel, topology: HeteroTopology) -> None: """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" - mimo_model = runtime.model language_pg = topology.language_pg vision_pg = topology.vision_pg @@ -166,7 +161,7 @@ def move_batch_to_cuda(value): def train_step( args: argparse.Namespace, - runtime: HeteroRuntime, + model: MimoModel, topology: HeteroTopology, optimizer, opt_param_scheduler, @@ -174,14 +169,14 @@ def train_step( data_iterator, ) -> TrainStepResult: """Run one Megatron-shaped hetero training step.""" - zero_active_grad_buffers(runtime.model) + zero_active_grad_buffers(model) optimizer.zero_grad() debug_rank("starting forward/backward schedule") losses = schedule.forward_backward_pipelining_without_interleaving( forward_step_func=forward_step, data_iterator=data_iterator, - model=[runtime.model], + model=[model], num_microbatches=args.num_microbatches, seq_length=args.seq_length, micro_batch_size=args.micro_batch_size, From 532acaa337093bfa7be3cd41ed32e07ee13eed8e Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 19:24:47 +0000 Subject: [PATCH 13/44] NMFW-464 keep encoder grad overlap disabled --- examples/mimo/training/hetero/args.py | 9 ++------- examples/mimo/training/hetero/runtime.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 238e8b2a57c..b6bb1fb3f15 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -71,8 +71,8 @@ def parse_args() -> argparse.Namespace: action=argparse.BooleanOptionalAction, default=False, help=( - "Enable DDP gradient-reduce overlap. The hetero example defaults this off " - "to match Megatron's conservative DDP default and the 20L reference script." + "Enable DDP gradient-reduce overlap for the language module. Vision encoder DDP " + "keeps overlap disabled because actual-data batches may be text-only." ), ) train.add_argument( @@ -145,11 +145,6 @@ def validate_energon_data_args(args: argparse.Namespace) -> None: "energon_multimodal currently requires --encoder-dp == --llm-dp so the " "encoder and LLM grids consume matching DP-lane samples" ) - if args.overlap_grad_reduce: - raise ValueError( - "energon_multimodal currently requires --no-overlap-grad-reduce because " - "the blend can yield text-only batches on vision ranks" - ) if args.packing_buffer_size is not None and args.packing_buffer_size > 0: if args.micro_batch_size != 1: raise ValueError( diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 78b97c253a3..f3641bfc8fb 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -76,18 +76,23 @@ def wrap_active_modules( args: argparse.Namespace, mimo_model: MimoModel, topology: HeteroTopology ) -> None: """Freeze and DDP-wrap active local MIMO modules.""" - ddp_config = DistributedDataParallelConfig( + language_ddp_config = DistributedDataParallelConfig( overlap_grad_reduce=args.overlap_grad_reduce, bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, use_distributed_optimizer=True, ) + vision_ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=False, + bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, + use_distributed_optimizer=True, + ) if mimo_model.language_model is not None: if args.freeze_lm: set_module_requires_grad(mimo_model.language_model, False) debug_rank("wrapping language model in DDP") mimo_model.language_model = DistributedDataParallel( config=mimo_model.language_model.config, - ddp_config=ddp_config, + ddp_config=language_ddp_config, module=mimo_model.language_model, pg_collection=topology.language_pg, ) @@ -107,7 +112,7 @@ def wrap_active_modules( debug_rank("wrapping vision submodule in DDP") mimo_model.modality_submodules[topology.encoder_name] = DistributedDataParallel( config=encoder_module.config, - ddp_config=ddp_config, + ddp_config=vision_ddp_config, module=submodule, pg_collection=topology.vision_pg, ) From db85eaad85543b57b356bb43df8a42f10a992317 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 19:37:42 +0000 Subject: [PATCH 14/44] NMFW-464 simplify embedding group lifecycle --- examples/mimo/training/hetero/topology.py | 69 +++++++++++++---------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py index 6f5da1ee3e1..923c7fd6283 100644 --- a/examples/mimo/training/hetero/topology.py +++ b/examples/mimo/training/hetero/topology.py @@ -10,6 +10,7 @@ import torch.distributed as dist +from examples.mimo.utils.hetero import debug_rank, is_process_group_member from megatron.core.hyper_comm_grid import HyperCommGrid from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator @@ -19,11 +20,8 @@ ProcessGroupCollection, ) -from examples.mimo.utils.hetero import debug_rank, is_process_group_member - ENCODER_MODULE_NAME = "images" - -_EMBEDDING_PG_CACHE: dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] = {} +EmbeddingGroupMap = dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] @dataclass @@ -35,6 +33,7 @@ class HeteroTopology: language_pg: ProcessGroupCollection vision_pg: ProcessGroupCollection schedule_pg_collection: MultiModuleProcessGroupCollection + embedding_groups: EmbeddingGroupMap encoder_size: int llm_size: int encoder_name: str = ENCODER_MODULE_NAME @@ -51,7 +50,7 @@ def module_dependency_map(self) -> dict[str, list[str]]: def destroy(self) -> None: """Destroy all process groups owned by this topology.""" - destroy_embedding_groups() + destroy_embedding_groups(self.embedding_groups) self.encoder_grid.destroy() self.llm_grid.destroy() BridgeCommunicator.destroy_broadcast_pgs() @@ -61,6 +60,7 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) """Create all rank-global process groups in one deterministic order.""" encoder_grid = None llm_grid = None + embedding_groups: Optional[EmbeddingGroupMap] = None try: debug_rank("creating encoder grid") encoder_grid = create_hypercomm_grid( @@ -85,11 +85,15 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) expt_dp=args.llm_expt_dp, ) debug_rank("creating embedding groups") - create_all_embedding_groups([encoder_grid, llm_grid]) + embedding_groups = create_embedding_groups([encoder_grid, llm_grid]) debug_rank("embedding groups ready") - language_pg = get_pg_collection_with_embedding_groups(llm_grid, is_language_model=True) - vision_pg = get_pg_collection_with_embedding_groups(encoder_grid, is_language_model=False) + language_pg = populate_embedding_groups( + get_pg_collection(llm_grid), embedding_groups, is_language_model=True + ) + vision_pg = populate_embedding_groups( + get_pg_collection(encoder_grid), embedding_groups, is_language_model=False + ) schedule_pg_collection = build_schedule_pg_collection( ENCODER_MODULE_NAME, encoder_grid, llm_grid, vision_pg, language_pg ) @@ -100,11 +104,13 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) language_pg=language_pg, vision_pg=vision_pg, schedule_pg_collection=schedule_pg_collection, + embedding_groups=embedding_groups, encoder_size=encoder_size, llm_size=llm_size, ) except Exception: - destroy_embedding_groups() + if embedding_groups is not None: + destroy_embedding_groups(embedding_groups) if encoder_grid is not None: encoder_grid.destroy() if llm_grid is not None: @@ -196,8 +202,13 @@ def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: ) -def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: - """Create PP-derived embedding groups in a consistent global order.""" +def create_embedding_groups(grids: list[HyperCommGrid]) -> EmbeddingGroupMap: + """Create PP-derived embedding groups and return handles by PP rank tuple. + + These groups must be created collectively by all ranks in one deterministic order before + module-local ProcessGroupCollections are populated. + """ + embedding_groups: EmbeddingGroupMap = {} pp_rank_sets: list[tuple[int, ...]] = [] seen_pp_rank_sets = set() for grid in sorted(grids, key=lambda candidate: (candidate.rank_offset, candidate.size)): @@ -208,8 +219,8 @@ def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: pp_rank_sets.append(pp_rank_tuple) seen_pp_rank_sets.add(pp_rank_tuple) - for pp_ranks in pp_rank_sets: - if pp_ranks not in _EMBEDDING_PG_CACHE: + try: + for pp_ranks in pp_rank_sets: pos_embd_ranks = [pp_ranks[0]] embd_ranks = [pp_ranks[0]] if pp_ranks[-1] != pp_ranks[0]: @@ -219,34 +230,41 @@ def create_all_embedding_groups(grids: list[HyperCommGrid]) -> None: try: pos_embd_pg = dist.new_group(ranks=pos_embd_ranks) embd_pg = dist.new_group(ranks=embd_ranks) - _EMBEDDING_PG_CACHE[pp_ranks] = (pos_embd_pg, embd_pg) + embedding_groups[pp_ranks] = (pos_embd_pg, embd_pg) except Exception: destroy_process_group_if_member(pos_embd_pg) destroy_process_group_if_member(embd_pg) raise + except Exception: + destroy_embedding_groups(embedding_groups) + raise + + return embedding_groups -def destroy_embedding_groups() -> None: - """Destroy cached embedding process groups created by this module.""" +def destroy_embedding_groups(embedding_groups: EmbeddingGroupMap) -> None: + """Destroy embedding process groups returned by create_embedding_groups.""" destroyed_embedding_pgs = set() - for pos_embd_pg, embd_pg in _EMBEDDING_PG_CACHE.values(): + for pos_embd_pg, embd_pg in embedding_groups.values(): for pg in (pos_embd_pg, embd_pg): if id(pg) in destroyed_embedding_pgs: continue destroy_process_group_if_member(pg) destroyed_embedding_pgs.add(id(pg)) - _EMBEDDING_PG_CACHE.clear() + embedding_groups.clear() -def add_embedding_groups( - pg_collection: ProcessGroupCollection, is_language_model: bool = False +def populate_embedding_groups( + pg_collection: ProcessGroupCollection, + embedding_groups: EmbeddingGroupMap, + is_language_model: bool = False, ) -> ProcessGroupCollection: - """Attach cached embedding process groups to a ProcessGroupCollection.""" + """Populate Megatron's required embedding fields on a ProcessGroupCollection.""" if not is_process_group_member(getattr(pg_collection, "pp", None)): return pg_collection pp_ranks = tuple(dist.get_process_group_ranks(pg_collection.pp)) - pos_embd_pg, embd_pg = _EMBEDDING_PG_CACHE[pp_ranks] + pos_embd_pg, embd_pg = embedding_groups[pp_ranks] pg_collection.pos_embd = pos_embd_pg if is_pp_first_stage(pg_collection.pp) else None if is_language_model: @@ -261,13 +279,6 @@ def add_embedding_groups( return pg_collection -def get_pg_collection_with_embedding_groups( - grid: HyperCommGrid, is_language_model: bool = False -) -> ProcessGroupCollection: - """Build a ProcessGroupCollection and add PP-derived embedding groups.""" - return add_embedding_groups(get_pg_collection(grid), is_language_model=is_language_model) - - def build_schedule_pg_collection( encoder_name: str, encoder_grid: HyperCommGrid, From 8db3fc556edf6cc085e5a56d7d959bd47e113d30 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 20:14:54 +0000 Subject: [PATCH 15/44] NMFW-464 align hetero logging reductions --- examples/mimo/training/hetero/logging.py | 40 +++++++----------------- examples/mimo/training/hetero/step.py | 6 ++-- megatron/core/models/mimo/optimizer.py | 18 ++++++++++- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py index 6654442b288..b46f76c3efa 100644 --- a/examples/mimo/training/hetero/logging.py +++ b/examples/mimo/training/hetero/logging.py @@ -68,9 +68,6 @@ def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: elapsed_ms = (elapsed / interval_iters) * 1000.0 loss_value = self.loss_total / self.loss_count if self.loss_count else None learning_rate = get_canonical_lr_for_logging(optimizer.param_groups) - learning_rate = reduce_max_optional_float(learning_rate) - grad_norm = reduce_max_optional_float(result.grad_norm) - num_zeros_in_grad = reduce_max_optional_float(result.num_zeros_in_grad) loss_scale = optimizer.get_loss_scale().item() if is_language_log_rank(self.topology): @@ -84,10 +81,10 @@ def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: if loss_value is not None: log_string += f" lm loss: {loss_value:.6E} |" log_string += f" loss scale: {loss_scale:.1f} |" - if grad_norm is not None: - log_string += f" grad norm: {grad_norm:.3f} |" - if num_zeros_in_grad is not None: - log_string += f" num zeros: {int(num_zeros_in_grad)} |" + if result.grad_norm is not None: + log_string += f" grad norm: {result.grad_norm:.3f} |" + if result.num_zeros_in_grad is not None: + log_string += f" num zeros: {int(result.num_zeros_in_grad)} |" log_string += " number of skipped iterations: {:3d} |".format(self.skipped_iterations) log_string += " number of nan iterations: {:3d} |".format(self.nan_iterations) sys.stdout.write(f"{log_string}\n") @@ -104,6 +101,7 @@ def reset_interval(self) -> None: self.interval_start = time.time() +@torch.no_grad() def reduce_language_loss(losses: list[dict], topology: HeteroTopology) -> Optional[float]: """Reduce raw loss/token vectors over the language DP/CP logging group.""" language_pg = topology.language_pg @@ -118,16 +116,11 @@ def reduce_language_loss(losses: list[dict], topology: HeteroTopology) -> Option if losses: for loss_dict in losses: - loss_sum = loss_dict.get("lm loss sum") - num_tokens = loss_dict.get("lm tokens") - if isinstance(loss_sum, torch.Tensor): - loss_acc[0] += loss_sum.float() - elif loss_sum is not None: - loss_acc[0] += float(loss_sum) - if isinstance(num_tokens, torch.Tensor): - loss_acc[1] += num_tokens.float() - elif num_tokens is not None: - loss_acc[1] += float(num_tokens) + loss = loss_dict.get("lm loss") + if isinstance(loss, torch.Tensor): + loss_acc += loss.detach().to(device="cuda", dtype=torch.float32).view(2) + elif loss is not None: + loss_acc += torch.tensor(loss, dtype=torch.float32, device="cuda").view(2) dist.all_reduce(loss_acc, op=dist.ReduceOp.SUM, group=language_pg.dp_cp) return loss_acc[0].item() / loss_acc[1].item() if loss_acc[1].item() else None @@ -142,15 +135,4 @@ def is_language_log_rank(topology: HeteroTopology) -> bool: and language_pg.tp.rank() == 0 ): return False - language_group_ranks = dist.get_process_group_ranks(language_pg.dp_cp) - return dist.get_rank() == min(language_group_ranks) - - -def reduce_max_optional_float(value) -> Optional[float]: - """Reduce optional scalar stats so the language log rank can see non-local optimizers.""" - if isinstance(value, torch.Tensor): - value = value.item() - local_value = -1.0 if value is None else float(value) - stat = torch.tensor([local_value], dtype=torch.float32, device="cuda") - dist.all_reduce(stat, op=dist.ReduceOp.MAX) - return None if stat.item() == -1.0 else stat.item() + return dist.get_rank() == dist.get_global_rank(language_pg.dp_cp, 0) diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 6a3fab2ceeb..acbe7b5200b 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -102,7 +102,7 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): if output_tensor is None: zero = torch.tensor(0.0, device="cuda", requires_grad=True) zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + return zero, zero_count, {"lm loss": torch.stack((zero.detach(), zero_count.float()))} if isinstance(output_tensor, dict): output = output_tensor.get( @@ -114,7 +114,7 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): if output is None: zero = torch.tensor(0.0, device="cuda", requires_grad=True) zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss sum": zero.detach(), "lm tokens": zero_count} + return zero, zero_count, {"lm loss": torch.stack((zero.detach(), zero_count.float()))} output = output.float() if loss_mask is None: @@ -131,7 +131,7 @@ def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): return ( loss_sum, num_tokens, - {"lm loss sum": loss_sum.detach(), "lm tokens": num_tokens.detach()}, + {"lm loss": torch.stack((loss_sum.detach(), num_tokens.detach().float()))}, ) diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index c0d46179927..9d22c75060a 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -51,6 +51,7 @@ def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: Optimiz @torch.no_grad() def prepare_grads(self) -> bool: + """Prepare gradients for every active module optimizer.""" found_inf = False for opt in self._active_optimizers: found_inf |= opt.prepare_grads() @@ -72,6 +73,7 @@ def get_grad_norm(self) -> float: @torch.no_grad() def step(self) -> Tuple[bool, Optional[float], Optional[int]]: + """Run one optimizer step across active module optimizers.""" found_inf = self.prepare_grads() # Synchronize found_inf across all ranks to prevent deadlock: # if encoder ranks detect inf but LLM ranks don't, the early return @@ -104,22 +106,34 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: @torch.no_grad() def step_with_ready_grads(self) -> bool: + """Apply updates after gradients have been prepared.""" success = True for opt in self._active_optimizers: success &= opt.step_with_ready_grads() return success def zero_grad(self, set_to_none: bool = True): + """Clear gradients on all active module optimizers.""" for opt in self._active_optimizers: opt.zero_grad(set_to_none) def get_loss_scale(self) -> torch.Tensor: + """Return the active optimizer loss scale, or one for stub ranks.""" if self._active_optimizers: return self._active_optimizers[0].get_loss_scale() return torch.tensor([1.0], dtype=torch.float32, device="cuda") def count_zeros(self) -> int: - return sum(opt.count_zeros() for opt in self._active_optimizers) + """Count zero gradients across all MIMO modules.""" + num_modules = len(self.module_infos) + zeros_by_module = torch.zeros(num_modules, device="cuda", dtype=torch.float32) + + for i, (name, info) in enumerate(sorted(self.module_infos.items())): + if info.is_active and info.optimizer: + zeros_by_module[i] = float(info.optimizer.count_zeros()) + + torch.distributed.all_reduce(zeros_by_module, op=torch.distributed.ReduceOp.MAX) + return int(zeros_by_module.sum().item()) @property def param_groups(self) -> List[dict]: @@ -132,6 +146,7 @@ def param_groups(self) -> List[dict]: # Checkpointing def state_dict(self): + """Return per-module optimizer state dicts.""" return { name: info.optimizer.state_dict() if info.is_active and info.optimizer else None for name, info in self.module_infos.items() @@ -183,6 +198,7 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, return sharded_state def reload_model_params(self, state_dict=None): + """Reload model params in each active module optimizer.""" for opt in self._active_optimizers: opt.reload_model_params(state_dict) From 9a30ad8baf39782241ecb37bf333788e608a185c Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 20:36:11 +0000 Subject: [PATCH 16/44] NMFW-464 organize hetero training modules --- examples/mimo/training/hetero/data.py | 85 ++++++++++++ examples/mimo/training/hetero/grad_sync.py | 101 ++++++++++++++ examples/mimo/training/hetero/logging.py | 7 +- examples/mimo/training/hetero/loop.py | 124 +++--------------- .../hetero/{scheduler.py => optimizer.py} | 24 +++- examples/mimo/training/hetero/runtime.py | 23 ---- examples/mimo/training/hetero/step.py | 70 +--------- 7 files changed, 233 insertions(+), 201 deletions(-) create mode 100644 examples/mimo/training/hetero/data.py create mode 100644 examples/mimo/training/hetero/grad_sync.py rename examples/mimo/training/hetero/{scheduler.py => optimizer.py} (66%) diff --git a/examples/mimo/training/hetero/data.py b/examples/mimo/training/hetero/data.py new file mode 100644 index 00000000000..23ffdc5050b --- /dev/null +++ b/examples/mimo/training/hetero/data.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Data iterator selection for heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +from typing import Optional + +from examples.mimo.data.hetero_mock import MockVLMIterator +from examples.mimo.training.hetero.topology import ( + HeteroTopology, + get_grid_coordinate, + is_rank_in_grid, +) +from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage + + +def select_data_iterator(args: argparse.Namespace, topology: HeteroTopology) -> Optional[object]: + """Create the per-role data iterator needed by local ranks.""" + if args.dataset_provider == "mock": + return select_mock_data_iterator(args, topology) + if args.dataset_provider == "energon_multimodal": + from examples.mimo.data.hetero_energon import build_energon_iterator + + return build_energon_iterator(args, topology) + raise ValueError(f"unsupported dataset provider: {args.dataset_provider}") + + +def validate_data_iterator( + args: argparse.Namespace, data_iterator, topology: HeteroTopology +) -> None: + """Run data-provider checks that must happen outside the pipeline schedule.""" + if args.dataset_provider == "energon_multimodal": + from examples.mimo.data.hetero_energon import validate_energon_data_alignment + + validate_energon_data_alignment(data_iterator, topology) + + +def select_mock_data_iterator( + args: argparse.Namespace, topology: HeteroTopology +) -> Optional[MockVLMIterator]: + """Create the per-role mock-data iterator needed by local ranks.""" + llm_mbs = args.micro_batch_size + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") + encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp + + encoder_grid = topology.encoder_grid + llm_grid = topology.llm_grid + encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( + encoder_grid.get_pg("pp") + ) + llm_needs_data = is_rank_in_grid(llm_grid) and ( + is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) + ) + + if encoder_needs_data and not llm_needs_data: + return MockVLMIterator( + args, + encoder_mbs, + topology.encoder_name, + get_mock_data_seed(args, encoder_grid, module_seed_offset=0), + ) + if llm_needs_data and not encoder_needs_data: + return MockVLMIterator( + args, + llm_mbs, + topology.encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + if encoder_needs_data and llm_needs_data: + return MockVLMIterator( + args, + llm_mbs, + topology.encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + return None + + +def get_mock_data_seed(args: argparse.Namespace, grid, module_seed_offset: int) -> int: + """Seed mock data by data-parallel lane so PP/TP stages see coherent batches.""" + dp_lane = get_grid_coordinate(grid, "dp") if "dp" in grid.dim_names else 0 + return args.seed + module_seed_offset + dp_lane diff --git a/examples/mimo/training/hetero/grad_sync.py b/examples/mimo/training/hetero/grad_sync.py new file mode 100644 index 00000000000..f9904dccd6d --- /dev/null +++ b/examples/mimo/training/hetero/grad_sync.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Gradient finalization and DDP sync helpers for heterogeneous MIMO training.""" + +from __future__ import annotations + +from contextlib import ExitStack, contextmanager + +import torch +import torch.distributed as dist + +from examples.mimo.training.hetero.runtime import active_ddp_modules +from examples.mimo.training.hetero.topology import HeteroTopology +from examples.mimo.utils.hetero import debug_rank, is_process_group_member +from megatron.core.distributed.finalize_model_grads import finalize_model_grads +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.pipeline_parallel.utils import is_pp_last_stage + + +def configure_grad_sync(mimo_model: MimoModel, topology: HeteroTopology) -> None: + """Configure grad-finalization callbacks consumed by the pipeline schedule.""" + language_pg = topology.language_pg + vision_pg = topology.vision_pg + + def is_token_source_rank() -> bool: + return ( + is_process_group_member(getattr(language_pg, "pp", None)) + and is_process_group_member(getattr(language_pg, "tp", None)) + and is_pp_last_stage(language_pg.pp) + and language_pg.tp.rank() == 0 + ) + + def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): + if num_tokens is None: + raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") + + token_count = torch.zeros(1, dtype=torch.float32, device="cuda") + if is_token_source_rank(): + token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() + dist.all_reduce(token_count, op=dist.ReduceOp.SUM) + global_num_tokens = token_count.item() + + if mimo_model.language_model is not None: + debug_rank("finalizing language grads") + finalize_model_grads( + [mimo_model.language_model], + num_tokens=None, + pg_collection=language_pg, + force_all_reduce=force_all_reduce, + ) + debug_rank("language grads finalized") + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + debug_rank("finalizing vision grads") + finalize_model_grads( + [submodule], + num_tokens=None, + pg_collection=vision_pg, + force_all_reduce=force_all_reduce, + ) + debug_rank("vision grads finalized") + + if global_num_tokens > 0: + scale = 1.0 / global_num_tokens + if mimo_model.language_model is not None: + debug_rank("scaling language grads") + mimo_model.language_model.scale_gradients(scale) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + debug_rank("scaling vision grads") + submodule.scale_gradients(scale) + + mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) + mimo_model.config.finalize_model_grads_func = finalize_grads_func + mimo_model.config.grad_scale_func = lambda loss: ( + torch.tensor(loss, dtype=torch.float32, device="cuda", requires_grad=True) + if isinstance(loss, (int, float)) + else loss + ) + + +def zero_active_grad_buffers(mimo_model: MimoModel) -> None: + """Clear MCore DDP grad buffers before each training iteration.""" + for module in active_ddp_modules(mimo_model): + module.zero_grad_buffer() + + +def build_no_sync_func(mimo_model: MimoModel): + """Build a no_sync context spanning all active MIMO submodules.""" + + @contextmanager + def no_sync_func(): + with ExitStack() as stack: + if mimo_model.language_model is not None: + stack.enter_context(mimo_model.language_model.no_sync()) + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + stack.enter_context(submodule.no_sync()) + yield + + return no_sync_func diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py index b46f76c3efa..3bc9552f648 100644 --- a/examples/mimo/training/hetero/logging.py +++ b/examples/mimo/training/hetero/logging.py @@ -15,13 +15,12 @@ import torch import torch.distributed as dist -from megatron.core.optimizer_param_scheduler import get_canonical_lr_for_logging -from megatron.core.pipeline_parallel.utils import is_pp_last_stage - -from examples.mimo.training.hetero.scheduler import get_global_batch_size +from examples.mimo.training.hetero.optimizer import get_global_batch_size from examples.mimo.training.hetero.step import TrainStepResult from examples.mimo.training.hetero.topology import HeteroTopology from examples.mimo.utils.hetero import is_process_group_member +from megatron.core.optimizer_param_scheduler import get_canonical_lr_for_logging +from megatron.core.pipeline_parallel.utils import is_pp_last_stage @dataclass diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 667a80e9e90..649cd164642 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -9,25 +9,18 @@ import torch -from examples.mimo.data.hetero_mock import MockVLMIterator from examples.mimo.training.hetero.args import prepare_args +from examples.mimo.training.hetero.data import select_data_iterator, validate_data_iterator from examples.mimo.training.hetero.distributed import print_rank_0 +from examples.mimo.training.hetero.grad_sync import configure_grad_sync from examples.mimo.training.hetero.logging import HeteroTrainingLogger +from examples.mimo.training.hetero.optimizer import build_optimizer, build_optimizer_param_scheduler from examples.mimo.training.hetero.runtime import build_mimo_runtime -from examples.mimo.training.hetero.scheduler import build_optimizer_param_scheduler -from examples.mimo.training.hetero.step import train_step, wire_training_hooks -from examples.mimo.training.hetero.topology import ( - HeteroTopology, - create_topology, - get_grid_coordinate, - is_rank_in_grid, -) +from examples.mimo.training.hetero.step import train_step +from examples.mimo.training.hetero.topology import HeteroTopology, create_topology from examples.mimo.utils.hetero import debug_rank from megatron.core.models.mimo.model.base import MimoModel -from megatron.core.models.mimo.optimizer import get_mimo_optimizer -from megatron.core.optimizer.optimizer_config import OptimizerConfig from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator -from megatron.core.pipeline_parallel.utils import is_pp_first_stage, is_pp_last_stage def run_train_loop(args: argparse.Namespace) -> None: @@ -43,8 +36,8 @@ def run_train_loop(args: argparse.Namespace) -> None: torch.manual_seed(args.seed) debug_rank("building MIMO model") model = build_mimo_runtime(args, topology) - debug_rank("wiring training hooks") - wire_training_hooks(model, topology) + debug_rank("configuring gradient sync") + configure_grad_sync(model, topology) debug_rank("building MIMO optimizer") optimizer = build_optimizer(args, model) @@ -52,13 +45,7 @@ def run_train_loop(args: argparse.Namespace) -> None: debug_rank("MIMO optimizer ready") debug_rank("building pipeline communicator") - communicator = MultiModulePipelineCommunicator( - topology.module_to_grid_map, - topology.module_dependency_map, - model.config, - dim_mapping={"s": 0, "h": 2, "b": 1}, - module_output_ndim={topology.encoder_name: 2}, - ) + communicator = build_pipeline_communicator(model, topology) debug_rank("selecting data iterator") data_iterator = select_data_iterator(args, topology) validate_data_iterator(args, data_iterator, topology) @@ -87,89 +74,14 @@ def run_train_loop(args: argparse.Namespace) -> None: topology.destroy() -def build_optimizer(args: argparse.Namespace, model: MimoModel): - """Build the MIMO optimizer for active hetero module optimizers.""" - return get_mimo_optimizer( - model, - OptimizerConfig( - optimizer="adam", - lr=args.lr, - min_lr=args.min_lr, - weight_decay=args.weight_decay, - adam_beta1=args.adam_beta1, - adam_beta2=args.adam_beta2, - clip_grad=args.clip_grad, - bf16=not args.fp32, - use_distributed_optimizer=True, - log_num_zeros_in_grad=args.log_num_zeros_in_grad, - ), - ) - - -def select_data_iterator(args: argparse.Namespace, topology: HeteroTopology) -> Optional[object]: - """Create the per-role data iterator needed by local ranks.""" - if args.dataset_provider == "mock": - return select_mock_data_iterator(args, topology) - if args.dataset_provider == "energon_multimodal": - from examples.mimo.data.hetero_energon import build_energon_iterator - - return build_energon_iterator(args, topology) - raise ValueError(f"unsupported dataset provider: {args.dataset_provider}") - - -def validate_data_iterator( - args: argparse.Namespace, data_iterator, topology: HeteroTopology -) -> None: - """Run data-provider checks that must happen outside the pipeline schedule.""" - if args.dataset_provider == "energon_multimodal": - from examples.mimo.data.hetero_energon import validate_energon_data_alignment - - validate_energon_data_alignment(data_iterator, topology) - - -def select_mock_data_iterator( - args: argparse.Namespace, topology: HeteroTopology -) -> Optional[MockVLMIterator]: - """Create the per-role mock-data iterator needed by local ranks.""" - llm_mbs = args.micro_batch_size - if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: - raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") - encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp - - encoder_grid = topology.encoder_grid - llm_grid = topology.llm_grid - encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( - encoder_grid.get_pg("pp") - ) - llm_needs_data = is_rank_in_grid(llm_grid) and ( - is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) +def build_pipeline_communicator( + model: MimoModel, topology: HeteroTopology +) -> MultiModulePipelineCommunicator: + """Build the MIMO pipeline communicator used by the train schedule.""" + return MultiModulePipelineCommunicator( + topology.module_to_grid_map, + topology.module_dependency_map, + model.config, + dim_mapping={"s": 0, "h": 2, "b": 1}, + module_output_ndim={topology.encoder_name: 2}, ) - - if encoder_needs_data and not llm_needs_data: - return MockVLMIterator( - args, - encoder_mbs, - topology.encoder_name, - get_mock_data_seed(args, encoder_grid, module_seed_offset=0), - ) - if llm_needs_data and not encoder_needs_data: - return MockVLMIterator( - args, - llm_mbs, - topology.encoder_name, - get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), - ) - if encoder_needs_data and llm_needs_data: - return MockVLMIterator( - args, - llm_mbs, - topology.encoder_name, - get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), - ) - return None - - -def get_mock_data_seed(args: argparse.Namespace, grid, module_seed_offset: int) -> int: - """Seed mock data by data-parallel lane so PP/TP stages see coherent batches.""" - dp_lane = get_grid_coordinate(grid, "dp") if "dp" in grid.dim_names else 0 - return args.seed + module_seed_offset + dp_lane diff --git a/examples/mimo/training/hetero/scheduler.py b/examples/mimo/training/hetero/optimizer.py similarity index 66% rename from examples/mimo/training/hetero/scheduler.py rename to examples/mimo/training/hetero/optimizer.py index 80f7e0b2793..fb3bcca8ca1 100644 --- a/examples/mimo/training/hetero/scheduler.py +++ b/examples/mimo/training/hetero/optimizer.py @@ -1,14 +1,36 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -"""Optimizer scheduler helpers for heterogeneous MIMO training.""" +"""Optimizer and scheduler construction for heterogeneous MIMO training.""" from __future__ import annotations import argparse +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.models.mimo.optimizer import get_mimo_optimizer +from megatron.core.optimizer.optimizer_config import OptimizerConfig from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler +def build_optimizer(args: argparse.Namespace, model: MimoModel): + """Build the MIMO optimizer for active hetero module optimizers.""" + return get_mimo_optimizer( + model, + OptimizerConfig( + optimizer="adam", + lr=args.lr, + min_lr=args.min_lr, + weight_decay=args.weight_decay, + adam_beta1=args.adam_beta1, + adam_beta2=args.adam_beta2, + clip_grad=args.clip_grad, + bf16=not args.fp32, + use_distributed_optimizer=True, + log_num_zeros_in_grad=args.log_num_zeros_in_grad, + ), + ) + + def get_global_batch_size(args: argparse.Namespace) -> int: """Return the language-side global batch size for scheduler accounting.""" derived_global_batch_size = args.micro_batch_size * args.num_microbatches * args.llm_dp diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index f3641bfc8fb..6a295a996df 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse -from contextlib import ExitStack, contextmanager from typing import Optional import torch @@ -169,25 +168,3 @@ def broadcast_active_params(mimo_model: MimoModel) -> None: """Synchronize initial parameters across each module's DP groups.""" for module in active_ddp_modules(mimo_model): module.broadcast_params() - - -def zero_active_grad_buffers(mimo_model: MimoModel) -> None: - """Clear MCore DDP grad buffers before each training iteration.""" - for module in active_ddp_modules(mimo_model): - module.zero_grad_buffer() - - -def build_no_sync_func(mimo_model: MimoModel): - """Build a no_sync context spanning all active MIMO submodules.""" - - @contextmanager - def no_sync_func(): - with ExitStack() as stack: - if mimo_model.language_model is not None: - stack.enter_context(mimo_model.language_model.no_sync()) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - stack.enter_context(submodule.no_sync()) - yield - - return no_sync_func diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index acbe7b5200b..0833a62a2fa 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -13,15 +13,13 @@ import torch.distributed as dist import megatron.core.pipeline_parallel.schedules as schedule -from examples.mimo.training.hetero.runtime import build_no_sync_func, zero_active_grad_buffers -from examples.mimo.training.hetero.scheduler import get_global_batch_size +from examples.mimo.training.hetero.grad_sync import zero_active_grad_buffers +from examples.mimo.training.hetero.optimizer import get_global_batch_size from examples.mimo.training.hetero.topology import HeteroTopology -from examples.mimo.utils.hetero import debug_rank, is_process_group_member -from megatron.core.distributed.finalize_model_grads import finalize_model_grads +from examples.mimo.utils.hetero import debug_rank from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator -from megatron.core.pipeline_parallel.utils import is_pp_last_stage @dataclass @@ -35,68 +33,6 @@ class TrainStepResult: num_zeros_in_grad: Optional[int] -def wire_training_hooks(mimo_model: MimoModel, topology: HeteroTopology) -> None: - """Attach MIMO-specific grad sync hooks expected by the pipeline schedule.""" - language_pg = topology.language_pg - vision_pg = topology.vision_pg - - def is_token_source_rank() -> bool: - return ( - is_process_group_member(getattr(language_pg, "pp", None)) - and is_process_group_member(getattr(language_pg, "tp", None)) - and is_pp_last_stage(language_pg.pp) - and language_pg.tp.rank() == 0 - ) - - def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): - if num_tokens is None: - raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") - - token_count = torch.zeros(1, dtype=torch.float32, device="cuda") - if is_token_source_rank(): - token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() - dist.all_reduce(token_count, op=dist.ReduceOp.SUM) - global_num_tokens = token_count.item() - - if mimo_model.language_model is not None: - debug_rank("finalizing language grads") - finalize_model_grads( - [mimo_model.language_model], - num_tokens=None, - pg_collection=language_pg, - force_all_reduce=force_all_reduce, - ) - debug_rank("language grads finalized") - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("finalizing vision grads") - finalize_model_grads( - [submodule], - num_tokens=None, - pg_collection=vision_pg, - force_all_reduce=force_all_reduce, - ) - debug_rank("vision grads finalized") - - if global_num_tokens > 0: - scale = 1.0 / global_num_tokens - if mimo_model.language_model is not None: - debug_rank("scaling language grads") - mimo_model.language_model.scale_gradients(scale) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("scaling vision grads") - submodule.scale_gradients(scale) - - mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) - mimo_model.config.finalize_model_grads_func = finalize_grads_func - mimo_model.config.grad_scale_func = lambda loss: ( - torch.tensor(loss, dtype=torch.float32, device="cuda", requires_grad=True) - if isinstance(loss, (int, float)) - else loss - ) - - def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): """Return raw loss sum, local token count, and logging tensors.""" if output_tensor is None: From 60e07e7879a04d52aebf387794231e119e0a2b3c Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 21:13:16 +0000 Subject: [PATCH 17/44] NMFW-464 fix token-count grad scaling --- examples/mimo/training/hetero/grad_sync.py | 19 ++++++--- megatron/core/models/mimo/model/base.py | 47 ++++++++++++++-------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/examples/mimo/training/hetero/grad_sync.py b/examples/mimo/training/hetero/grad_sync.py index f9904dccd6d..14ae94543e5 100644 --- a/examples/mimo/training/hetero/grad_sync.py +++ b/examples/mimo/training/hetero/grad_sync.py @@ -34,11 +34,18 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar if num_tokens is None: raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") - token_count = torch.zeros(1, dtype=torch.float32, device="cuda") + global_num_tokens = torch.zeros(1, dtype=torch.float32, device="cuda") if is_token_source_rank(): - token_count[0] = num_tokens.to(device="cuda", dtype=torch.float32).sum() - dist.all_reduce(token_count, op=dist.ReduceOp.SUM) - global_num_tokens = token_count.item() + # MCore has already summed loss-mask token counts across microbatches + # for this gradient-accumulation step. Match Megatron's normalization + # domain by reducing the language last-stage count over DP and CP. + token_count = num_tokens.to(device="cuda", dtype=torch.float32).sum().view(1) + dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=language_pg.dp_cp) + if dist.get_rank(language_pg.dp_cp) == 0: + global_num_tokens.copy_(token_count) + # Publish the already DP/CP-reduced language token count to encoder ranks too. + dist.all_reduce(global_num_tokens, op=dist.ReduceOp.MAX) + global_num_tokens_value = global_num_tokens.item() if mimo_model.language_model is not None: debug_rank("finalizing language grads") @@ -60,8 +67,8 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar ) debug_rank("vision grads finalized") - if global_num_tokens > 0: - scale = 1.0 / global_num_tokens + if global_num_tokens_value > 0: + scale = 1.0 / global_num_tokens_value if mimo_model.language_model is not None: debug_rank("scaling language grads") mimo_model.language_model.scale_gradients(scale) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index 3edd3e10010..4e4018f5dc1 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -2,7 +2,7 @@ import logging import warnings -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple import torch @@ -379,16 +379,14 @@ def forward( return self._forward_encoders(input_ids, modality_inputs, input_tensors), loss_mask if self.role.has_language_module: - return ( - self._forward_language_module( - input_ids, - position_ids, - attention_mask, - labels, - input_tensors, - packing_kwargs, - ), + return self._forward_language_module( + input_ids, + position_ids, + attention_mask, loss_mask, + labels, + input_tensors, + packing_kwargs, ) raise RuntimeError(f"Rank has no modules assigned in role: {self.role}") @@ -463,21 +461,23 @@ def _forward_language_module( input_ids: torch.Tensor, position_ids: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], labels: Optional[torch.Tensor], input_tensors: Optional[Dict[str, torch.Tensor]], packing_kwargs: Optional[dict] = None, - ) -> torch.Tensor: + ) -> Tuple[Any, Optional[torch.Tensor]]: """Forward pass for language module on this rank. Args: input_ids: Token IDs position_ids: Position IDs attention_mask: Attention mask + loss_mask: Loss mask for per-token loss normalization labels: Labels for loss computation input_tensors: Hidden states or embeddings from previous stage Returns: - Language model output (hidden states, logits, or loss depending on stage) + Tuple of language model output and the matching, possibly sharded loss mask. """ lang_name = MIMO_LANGUAGE_MODULE_KEY packed_seq_params = self._build_packed_seq_params(packing_kwargs) @@ -506,11 +506,12 @@ def _forward_language_module( if self.partition_adapter is not None: combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() - combined_embeddings, labels, _, attention_mask, packed_seq_params = ( + shard_loss_inputs = self.role.is_last_stage(lang_name) + combined_embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( self.partition_adapter.shard( embeddings=combined_embeddings, - labels=labels, - loss_mask=None, + labels=labels if shard_loss_inputs else None, + loss_mask=loss_mask if shard_loss_inputs else None, attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) @@ -530,6 +531,18 @@ def _forward_language_module( # Non-first stage: receive hidden states from previous LM stage hidden_states = input_tensors.get(lang_name) if input_tensors else None + if self.partition_adapter is not None: + shard_loss_inputs = self.role.is_last_stage(lang_name) + _, labels, loss_mask, attention_mask, packed_seq_params = ( + self.partition_adapter.shard( + embeddings=None, + labels=labels if shard_loss_inputs else None, + loss_mask=loss_mask if shard_loss_inputs else None, + attention_mask=attention_mask, + packed_seq_params=packed_seq_params, + ) + ) + # Set input tensor on language model for PP (unwrap DDP to reach GPTModel) if hidden_states is not None: underlying_lm = unwrap_model(self.language_model) @@ -547,9 +560,9 @@ def _forward_language_module( # Key output for non-last stages so schedule can route to next LM stage if not self.role.is_last_stage(lang_name): - return {lang_name: lm_output} + return {lang_name: lm_output}, loss_mask - return lm_output + return lm_output, loss_mask @staticmethod def _build_packed_seq_params(packing_kwargs: Optional[dict]) -> Optional[PackedSeqParams]: From c6082098816efb2121b20e26b85981eaa0f6a30d Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:04:15 +0000 Subject: [PATCH 18/44] NMFW-464 remove eager param broadcast --- examples/mimo/training/hetero/runtime.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 6a295a996df..cda13865ddb 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -67,7 +67,6 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi debug_rank("MimoModel moved to target dtype/device") wrap_active_modules(args, mimo_model, topology) - broadcast_active_params(mimo_model) return mimo_model @@ -162,9 +161,3 @@ def active_ddp_modules(mimo_model: MimoModel) -> list[DistributedDataParallel]: if isinstance(submodule, DistributedDataParallel) ) return modules - - -def broadcast_active_params(mimo_model: MimoModel) -> None: - """Synchronize initial parameters across each module's DP groups.""" - for module in active_ddp_modules(mimo_model): - module.broadcast_params() From f80ccbd71d6316f85973f3f3dfc3faeb171dc01a Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:12:32 +0000 Subject: [PATCH 19/44] NMFW-464 keep embedding groups language-only --- examples/mimo/training/hetero/topology.py | 105 ++++++++++------------ 1 file changed, 45 insertions(+), 60 deletions(-) diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py index 923c7fd6283..cc4cc5a175e 100644 --- a/examples/mimo/training/hetero/topology.py +++ b/examples/mimo/training/hetero/topology.py @@ -21,7 +21,7 @@ ) ENCODER_MODULE_NAME = "images" -EmbeddingGroupMap = dict[tuple[int, ...], tuple[dist.ProcessGroup, dist.ProcessGroup]] +LanguageEmbeddingGroups = dict[tuple[int, ...], Optional[dist.ProcessGroup]] @dataclass @@ -33,7 +33,7 @@ class HeteroTopology: language_pg: ProcessGroupCollection vision_pg: ProcessGroupCollection schedule_pg_collection: MultiModuleProcessGroupCollection - embedding_groups: EmbeddingGroupMap + language_embedding_groups: LanguageEmbeddingGroups encoder_size: int llm_size: int encoder_name: str = ENCODER_MODULE_NAME @@ -50,7 +50,7 @@ def module_dependency_map(self) -> dict[str, list[str]]: def destroy(self) -> None: """Destroy all process groups owned by this topology.""" - destroy_embedding_groups(self.embedding_groups) + destroy_embedding_groups(self.language_embedding_groups) self.encoder_grid.destroy() self.llm_grid.destroy() BridgeCommunicator.destroy_broadcast_pgs() @@ -60,7 +60,7 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) """Create all rank-global process groups in one deterministic order.""" encoder_grid = None llm_grid = None - embedding_groups: Optional[EmbeddingGroupMap] = None + language_embedding_groups: Optional[LanguageEmbeddingGroups] = None try: debug_rank("creating encoder grid") encoder_grid = create_hypercomm_grid( @@ -84,16 +84,14 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) expt_tp=args.llm_expt_tp, expt_dp=args.llm_expt_dp, ) - debug_rank("creating embedding groups") - embedding_groups = create_embedding_groups([encoder_grid, llm_grid]) - debug_rank("embedding groups ready") + debug_rank("creating language embedding groups") + language_embedding_groups = create_language_embedding_groups(llm_grid) + debug_rank("language embedding groups ready") - language_pg = populate_embedding_groups( - get_pg_collection(llm_grid), embedding_groups, is_language_model=True - ) - vision_pg = populate_embedding_groups( - get_pg_collection(encoder_grid), embedding_groups, is_language_model=False + language_pg = populate_language_embedding_groups( + get_pg_collection(llm_grid), language_embedding_groups ) + vision_pg = clear_embedding_groups(get_pg_collection(encoder_grid)) schedule_pg_collection = build_schedule_pg_collection( ENCODER_MODULE_NAME, encoder_grid, llm_grid, vision_pg, language_pg ) @@ -104,13 +102,13 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) language_pg=language_pg, vision_pg=vision_pg, schedule_pg_collection=schedule_pg_collection, - embedding_groups=embedding_groups, + language_embedding_groups=language_embedding_groups, encoder_size=encoder_size, llm_size=llm_size, ) except Exception: - if embedding_groups is not None: - destroy_embedding_groups(embedding_groups) + if language_embedding_groups is not None: + destroy_embedding_groups(language_embedding_groups) if encoder_grid is not None: encoder_grid.destroy() if llm_grid is not None: @@ -202,37 +200,26 @@ def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: ) -def create_embedding_groups(grids: list[HyperCommGrid]) -> EmbeddingGroupMap: - """Create PP-derived embedding groups and return handles by PP rank tuple. +def create_language_embedding_groups(grid: HyperCommGrid) -> LanguageEmbeddingGroups: + """Create language-model embedding groups keyed by PP rank tuple. - These groups must be created collectively by all ranks in one deterministic order before - module-local ProcessGroupCollections are populated. + A language grid has one PP group per TP/CP/DP lane, so the rank tuple is the stable key used + to attach the matching first/last-stage embedding group to each ProcessGroupCollection. """ - embedding_groups: EmbeddingGroupMap = {} - pp_rank_sets: list[tuple[int, ...]] = [] - seen_pp_rank_sets = set() - for grid in sorted(grids, key=lambda candidate: (candidate.rank_offset, candidate.size)): + embedding_groups: LanguageEmbeddingGroups = {} + + try: for pp_ranks in grid.get_rank_enum("pp"): pp_rank_tuple = tuple(pp_ranks) - if pp_rank_tuple in seen_pp_rank_sets: + if pp_rank_tuple[0] == pp_rank_tuple[-1]: + embedding_groups[pp_rank_tuple] = None continue - pp_rank_sets.append(pp_rank_tuple) - seen_pp_rank_sets.add(pp_rank_tuple) - try: - for pp_ranks in pp_rank_sets: - pos_embd_ranks = [pp_ranks[0]] - embd_ranks = [pp_ranks[0]] - if pp_ranks[-1] != pp_ranks[0]: - embd_ranks.append(pp_ranks[-1]) - pos_embd_pg = None embd_pg = None try: - pos_embd_pg = dist.new_group(ranks=pos_embd_ranks) - embd_pg = dist.new_group(ranks=embd_ranks) - embedding_groups[pp_ranks] = (pos_embd_pg, embd_pg) + embd_pg = dist.new_group(ranks=[pp_rank_tuple[0], pp_rank_tuple[-1]]) + embedding_groups[pp_rank_tuple] = embd_pg except Exception: - destroy_process_group_if_member(pos_embd_pg) destroy_process_group_if_member(embd_pg) raise except Exception: @@ -242,40 +229,38 @@ def create_embedding_groups(grids: list[HyperCommGrid]) -> EmbeddingGroupMap: return embedding_groups -def destroy_embedding_groups(embedding_groups: EmbeddingGroupMap) -> None: - """Destroy embedding process groups returned by create_embedding_groups.""" +def destroy_embedding_groups(embedding_groups: LanguageEmbeddingGroups) -> None: + """Destroy embedding process groups returned by create_language_embedding_groups.""" destroyed_embedding_pgs = set() - for pos_embd_pg, embd_pg in embedding_groups.values(): - for pg in (pos_embd_pg, embd_pg): - if id(pg) in destroyed_embedding_pgs: - continue - destroy_process_group_if_member(pg) - destroyed_embedding_pgs.add(id(pg)) + for embd_pg in embedding_groups.values(): + if embd_pg is None or id(embd_pg) in destroyed_embedding_pgs: + continue + destroy_process_group_if_member(embd_pg) + destroyed_embedding_pgs.add(id(embd_pg)) embedding_groups.clear() -def populate_embedding_groups( +def populate_language_embedding_groups( pg_collection: ProcessGroupCollection, - embedding_groups: EmbeddingGroupMap, - is_language_model: bool = False, + embedding_groups: LanguageEmbeddingGroups, ) -> ProcessGroupCollection: - """Populate Megatron's required embedding fields on a ProcessGroupCollection.""" + """Populate language embedding fields required by finalize_model_grads.""" + pg_collection.pos_embd = None + pg_collection.embd = None if not is_process_group_member(getattr(pg_collection, "pp", None)): return pg_collection pp_ranks = tuple(dist.get_process_group_ranks(pg_collection.pp)) - pos_embd_pg, embd_pg = embedding_groups[pp_ranks] - - pg_collection.pos_embd = pos_embd_pg if is_pp_first_stage(pg_collection.pp) else None - if is_language_model: - pg_collection.embd = ( - embd_pg - if is_pp_last_stage(pg_collection.pp) or is_pp_first_stage(pg_collection.pp) - else None - ) - else: - pg_collection.embd = None + if is_pp_last_stage(pg_collection.pp) or is_pp_first_stage(pg_collection.pp): + pg_collection.embd = embedding_groups[pp_ranks] + + return pg_collection + +def clear_embedding_groups(pg_collection: ProcessGroupCollection) -> ProcessGroupCollection: + """Populate embedding fields with None for modules that do not share embeddings.""" + pg_collection.pos_embd = None + pg_collection.embd = None return pg_collection From 226dcc378e0005ad42a3dd55faa2d5603911fd05 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:19:02 +0000 Subject: [PATCH 20/44] NMFW-464 guard hetero parallel state ownership --- examples/mimo/training/hetero/distributed.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/examples/mimo/training/hetero/distributed.py b/examples/mimo/training/hetero/distributed.py index 22dd2003ebd..ad617c0f472 100644 --- a/examples/mimo/training/hetero/distributed.py +++ b/examples/mimo/training/hetero/distributed.py @@ -20,6 +20,7 @@ def initialize_distributed() -> None: torch.cuda.set_device(local_rank) if not dist.is_initialized(): dist.init_process_group(backend="nccl") + assert_megatron_parallel_state_uninitialized() try: parallel_state.get_global_memory_buffer() except AssertionError: @@ -27,6 +28,31 @@ def initialize_distributed() -> None: dist.barrier() +def assert_megatron_parallel_state_uninitialized() -> None: + """Ensure this standalone hetero path owns Megatron process-group setup.""" + initialized_groups = [] + if parallel_state.is_initialized(): + initialized_groups.append("data_parallel") + if parallel_state.get_model_parallel_group(check_initialized=False) is not None: + initialized_groups.append("model_parallel") + if parallel_state.get_tensor_model_parallel_group(check_initialized=False) is not None: + initialized_groups.append("tensor_model_parallel") + if parallel_state.get_pipeline_model_parallel_group(check_initialized=False) is not None: + initialized_groups.append("pipeline_model_parallel") + if parallel_state.get_context_parallel_group(check_initialized=False) is not None: + initialized_groups.append("context_parallel") + if parallel_state.get_embedding_group(check_initialized=False) is not None: + initialized_groups.append("embedding") + if parallel_state.get_position_embedding_group(check_initialized=False) is not None: + initialized_groups.append("position_embedding") + + if initialized_groups: + raise RuntimeError( + "train_hetero.py expects Megatron parallel_state process groups to be " + f"uninitialized, but found: {', '.join(initialized_groups)}" + ) + + def print_rank_0(message: str) -> None: """Print only on global rank zero.""" if not dist.is_initialized() or dist.get_rank() == 0: From de2ca39ae4ac072de98427651649f334fb3f6a44 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:24:52 +0000 Subject: [PATCH 21/44] NMFW-464 simplify hetero loss contract --- examples/mimo/training/hetero/step.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 0833a62a2fa..934a7d347d7 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -17,7 +17,6 @@ from examples.mimo.training.hetero.optimizer import get_global_batch_size from examples.mimo.training.hetero.topology import HeteroTopology from examples.mimo.utils.hetero import debug_rank -from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator @@ -34,25 +33,16 @@ class TrainStepResult: def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): - """Return raw loss sum, local token count, and logging tensors.""" + """Return terminal language-model loss sum, local token count, and logging tensors.""" if output_tensor is None: - zero = torch.tensor(0.0, device="cuda", requires_grad=True) - zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss": torch.stack((zero.detach(), zero_count.float()))} - - if isinstance(output_tensor, dict): - output = output_tensor.get( - MIMO_LANGUAGE_MODULE_KEY, next(iter(output_tensor.values()), None) + raise RuntimeError("terminal language stage returned no loss tensor") + if not isinstance(output_tensor, torch.Tensor): + raise TypeError( + "loss_func expects the terminal language stage to return a tensor, " + f"got {type(output_tensor).__name__}" ) - else: - output = output_tensor - - if output is None: - zero = torch.tensor(0.0, device="cuda", requires_grad=True) - zero_count = torch.tensor(0, device="cuda", dtype=torch.int) - return zero, zero_count, {"lm loss": torch.stack((zero.detach(), zero_count.float()))} - output = output.float() + output = output_tensor.float() if loss_mask is None: raise RuntimeError("train_hetero.py requires a loss_mask for per-token loss") if output.shape != loss_mask.shape: From 88560928d8d246cb8f80f929061410f74d98d181 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:28:04 +0000 Subject: [PATCH 22/44] NMFW-464 clarify hetero loss function signature --- examples/mimo/training/hetero/step.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 934a7d347d7..2de59ca7b4e 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -32,7 +32,7 @@ class TrainStepResult: num_zeros_in_grad: Optional[int] -def loss_func(loss_mask: Optional[torch.Tensor], output_tensor): +def loss_func(output_tensor: torch.Tensor, *, loss_mask: torch.Tensor): """Return terminal language-model loss sum, local token count, and logging tensors.""" if output_tensor is None: raise RuntimeError("terminal language stage returned no loss tensor") @@ -69,7 +69,7 @@ def forward_step(data_iterator, model): debug_rank("forward_step model call start") output_tensor, loss_mask = model(**batch) debug_rank("forward_step model call done") - return output_tensor, partial(loss_func, loss_mask) + return output_tensor, partial(loss_func, loss_mask=loss_mask) def move_batch_to_cuda(value): From 69fa21fe2f1fde771c756a156bd0d983131b254e Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Mon, 11 May 2026 22:51:52 +0000 Subject: [PATCH 23/44] NMFW-464 clarify hetero runtime setup --- examples/mimo/training/hetero/grad_sync.py | 11 ++--- examples/mimo/training/hetero/runtime.py | 50 ++++++++++------------ 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/examples/mimo/training/hetero/grad_sync.py b/examples/mimo/training/hetero/grad_sync.py index 14ae94543e5..be5ee7b0c78 100644 --- a/examples/mimo/training/hetero/grad_sync.py +++ b/examples/mimo/training/hetero/grad_sync.py @@ -9,7 +9,7 @@ import torch import torch.distributed as dist -from examples.mimo.training.hetero.runtime import active_ddp_modules +from examples.mimo.training.hetero.runtime import iter_active_ddp_modules from examples.mimo.training.hetero.topology import HeteroTopology from examples.mimo.utils.hetero import debug_rank, is_process_group_member from megatron.core.distributed.finalize_model_grads import finalize_model_grads @@ -88,7 +88,7 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar def zero_active_grad_buffers(mimo_model: MimoModel) -> None: """Clear MCore DDP grad buffers before each training iteration.""" - for module in active_ddp_modules(mimo_model): + for module in iter_active_ddp_modules(mimo_model): module.zero_grad_buffer() @@ -98,11 +98,8 @@ def build_no_sync_func(mimo_model: MimoModel): @contextmanager def no_sync_func(): with ExitStack() as stack: - if mimo_model.language_model is not None: - stack.enter_context(mimo_model.language_model.no_sync()) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - stack.enter_context(submodule.no_sync()) + for module in iter_active_ddp_modules(mimo_model): + stack.enter_context(module.no_sync()) yield return no_sync_func diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index cda13865ddb..10ba4eea0ac 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -5,7 +5,7 @@ from __future__ import annotations import argparse -from typing import Optional +from typing import Iterator, Optional import torch @@ -34,12 +34,12 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi "building model specs " f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" ) + # The CUDA RNG tracker is process-global; this runtime assumes non-colocated module grids, + # so each rank configures RNG state for exactly one module role. if rank_in_language_grid: - set_model_init_seed(args, language_pg, role_offset=20_000) - initialize_model_parallel_rng(args, language_pg) + configure_module_rng(args, language_pg, role_seed_offset=20_000) elif rank_in_encoder_grid: - set_model_init_seed(args, vision_pg, role_offset=10_000) - initialize_model_parallel_rng(args, vision_pg) + configure_module_rng(args, vision_pg, role_seed_offset=10_000) mimo_config = MimoModelConfig( language_model_spec=language_model_spec( @@ -66,11 +66,11 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi mimo_model.to(torch.bfloat16) debug_rank("MimoModel moved to target dtype/device") - wrap_active_modules(args, mimo_model, topology) + wrap_active_modules_with_ddp(args, mimo_model, topology) return mimo_model -def wrap_active_modules( +def wrap_active_modules_with_ddp( args: argparse.Namespace, mimo_model: MimoModel, topology: HeteroTopology ) -> None: """Freeze and DDP-wrap active local MIMO modules.""" @@ -125,24 +125,22 @@ def set_module_requires_grad(module: Optional[torch.nn.Module], requires_grad: b param.requires_grad = requires_grad -def set_model_init_seed( - args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_offset: int +def configure_module_rng( + args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_seed_offset: int ) -> None: - """Seed CPU model init consistently across TP/DP peers for one module role.""" - pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) - torch.manual_seed(args.seed + role_offset + (100 * pp_rank)) - + """Seed module init and CUDA RNG tracker for the active module role. -def initialize_model_parallel_rng( - args: argparse.Namespace, pg_collection: ProcessGroupCollection -) -> None: - """Initialize CUDA RNG tracker using the active module's hetero process groups.""" + The seed is identical across DP/CP replicas for a module PP stage, and differs across + module roles and PP stages. + """ pp_rank = get_group_rank_or(getattr(pg_collection, "pp", None)) tp_rank = get_group_rank_or(getattr(pg_collection, "tp", None)) ep_rank = get_group_rank_or(getattr(pg_collection, "ep", None)) expt_tp_rank = get_group_rank_or(getattr(pg_collection, "expt_tp", None)) + seed = args.seed + role_seed_offset + (100 * pp_rank) + torch.manual_seed(seed) model_parallel_cuda_manual_seed( - args.seed + (100 * pp_rank), + seed, tp_rank=tp_rank, ep_rank=ep_rank, etp_rank=expt_tp_rank, @@ -150,14 +148,10 @@ def initialize_model_parallel_rng( ) -def active_ddp_modules(mimo_model: MimoModel) -> list[DistributedDataParallel]: - """Return active DDP-wrapped submodules owned by this rank.""" - modules = [] +def iter_active_ddp_modules(mimo_model: MimoModel) -> Iterator[DistributedDataParallel]: + """Yield active DDP-wrapped submodules owned by this rank.""" if isinstance(mimo_model.language_model, DistributedDataParallel): - modules.append(mimo_model.language_model) - modules.extend( - submodule - for submodule in mimo_model.modality_submodules.values() - if isinstance(submodule, DistributedDataParallel) - ) - return modules + yield mimo_model.language_model + for submodule in mimo_model.modality_submodules.values(): + if isinstance(submodule, DistributedDataParallel): + yield submodule From 4eb48fbfe5cb61c6ee3da92bc944f18ff4ae3fdb Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Tue, 12 May 2026 02:16:23 +0000 Subject: [PATCH 24/44] NMFW-464 support Energon encoder DP fan-out --- examples/mimo/data/hetero_energon.py | 241 +++++++++++++++--- .../run_hetero_nemotron_20l_energon_train.sh | 34 ++- examples/mimo/training/hetero/args.py | 20 +- examples/mimo/training/hetero/topology.py | 37 ++- megatron/core/hyper_comm_grid.py | 2 +- megatron/core/models/mimo/model/base.py | 20 ++ megatron/core/models/mimo/optimizer.py | 65 +---- .../pipeline_parallel/bridge_communicator.py | 66 ++++- megatron/core/process_groups_config.py | 51 ---- .../models/test_mimo_1f1b_schedule.py | 37 ++- .../models/test_mimo_bridge_split_sizes.py | 18 ++ .../test_bridge_communicator.py | 46 +++- tests/unit_tests/test_hetero_energon.py | 32 +++ .../unit_tests/test_process_groups_config.py | 99 ------- 14 files changed, 451 insertions(+), 317 deletions(-) create mode 100644 tests/unit_tests/models/test_mimo_bridge_split_sizes.py diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 0c3a7125534..5a2fb75e32d 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -6,7 +6,7 @@ import hashlib import random -from typing import Optional +from typing import Callable, Optional import torch import torch.distributed as dist @@ -29,47 +29,129 @@ def build_energon_iterator(args, topology): ) if encoder_needs_data: - return _build_iterator_for_grid(args, encoder_grid) + return _build_encoder_iterator(args, encoder_grid) if llm_needs_data: - return _build_iterator_for_grid(args, llm_grid) + return _build_llm_iterator(args, llm_grid) return None -def validate_energon_data_alignment(data_iterator, topology) -> None: +def validate_energon_data_alignment(data_iterator, _topology) -> None: """Check the first actual-data batch aligns across non-colocated module grids.""" if not dist.is_initialized(): return - signature = data_iterator.peek_signature() if data_iterator is not None else None - local = (get_current_dp_lane(topology), signature) gathered = [None for _ in range(dist.get_world_size())] - dist.all_gather_object(gathered, local) + dist.all_gather_object( + gathered, data_iterator.peek_alignment() if data_iterator is not None else None + ) - signatures_by_lane = {} - for lane, candidate in gathered: - if lane < 0 or candidate is None: + encoder_signatures_by_lane = {} + llm_signatures_by_lane = {} + for candidate in gathered: + if candidate is None: continue - signatures_by_lane.setdefault(lane, set()).add(candidate) - mismatched = {lane: values for lane, values in signatures_by_lane.items() if len(values) > 1} + target = ( + encoder_signatures_by_lane + if candidate["role"] == "encoder" + else llm_signatures_by_lane + ) + for lane, signature in zip(candidate["llm_lanes"], candidate["signatures"]): + target.setdefault(lane, set()).add(signature) + + mismatched = {} + for lane in sorted(set(encoder_signatures_by_lane) | set(llm_signatures_by_lane)): + encoder_values = encoder_signatures_by_lane.get(lane, set()) + llm_values = llm_signatures_by_lane.get(lane, set()) + if len(encoder_values) != 1 or len(llm_values) != 1 or encoder_values != llm_values: + mismatched[lane] = { + "encoder": sorted(encoder_values), + "llm": sorted(llm_values), + } if mismatched: raise RuntimeError(f"hetero Energon data loaders diverged across grids: {mismatched}") -def get_current_dp_lane(topology) -> int: - """Return the active module DP lane for this rank, or -1 for inactive ranks.""" - if is_rank_in_grid(topology.encoder_grid): - return get_grid_coordinate(topology.encoder_grid, "dp") - if is_rank_in_grid(topology.llm_grid): - return get_grid_coordinate(topology.llm_grid, "dp") - return -1 +def _build_llm_iterator(args, grid): + """Build the single-lane LLM iterator for this grid coordinate.""" + tp_group = grid.get_pg("tp") + if get_grid_coordinate(grid, "tp") != 0: + lane = get_grid_coordinate(grid, "dp") + return EnergonIterator( + None, + tp_group=tp_group, + source_rank=False, + alignment_role="llm", + llm_lanes=[lane], + ) + + lane = get_grid_coordinate(grid, "dp") + return _build_single_lane_iterator( + args, + tp_group=tp_group, + lane=lane, + role="llm", + random_seed=args.seed + lane, + ) -def _build_iterator_for_grid(args, grid): - """Build a deterministic per-DP-lane loader for one module grid.""" +def _build_encoder_iterator(args, grid): + """Build the encoder iterator, composing LLM-lane samples for DP fan-out.""" tp_group = grid.get_pg("tp") + encoder_dp_rank = get_grid_coordinate(grid, "dp") + llm_lanes = _llm_lanes_for_encoder_rank(args, encoder_dp_rank) if get_grid_coordinate(grid, "tp") != 0: - return EnergonIterator(None, tp_group=tp_group, source_rank=False) + return EnergonIterator( + None, + tp_group=tp_group, + source_rank=False, + alignment_role="encoder", + llm_lanes=llm_lanes, + ) + + if len(llm_lanes) == 1: + return _build_single_lane_iterator( + args, + tp_group=tp_group, + lane=llm_lanes[0], + role="encoder", + random_seed=args.seed + llm_lanes[0], + ) + + lane_iterators = [ + _build_single_lane_iterator( + args, + tp_group=None, + lane=lane, + role="encoder-component", + random_seed=args.seed + lane, + ) + for lane in llm_lanes + ] + + def next_encoder_batch(): + batches = [next(iterator) for iterator in lane_iterators] + signatures = [EnergonIterator._batch_signature(batch) for batch in batches] + return _combine_encoder_batches(batches), signatures + + return EnergonIterator( + None, + tp_group=tp_group, + source_rank=True, + local_batch_fn=next_encoder_batch, + alignment_role="encoder", + llm_lanes=llm_lanes, + ) + + +def _llm_lanes_for_encoder_rank(args, encoder_dp_rank: int) -> list[int]: + """Return the contiguous LLM DP lanes owned by one encoder DP lane.""" + scale = args.llm_dp // args.encoder_dp + start = encoder_dp_rank * scale + return list(range(start, start + scale)) + +def _build_single_lane_iterator(args, tp_group, lane: int, role: str, random_seed: int): + """Build a deterministic loader for one LLM data lane.""" from examples.mimo.data.energon_multimodal_provider import build_multimodal_encoder from megatron.energon import WorkerConfig, get_loader, get_train_dataset @@ -80,13 +162,12 @@ def _build_iterator_for_grid(args, grid): encoder_name=getattr(args, "vision_encoder_key", "radio_encoder"), encoder_input_key="x", ) - dp_rank = get_grid_coordinate(grid, "dp") worker_config = WorkerConfig( - rank=dp_rank, world_size=args.llm_dp, num_workers=args.num_workers, data_parallel_group=None + rank=lane, world_size=args.llm_dp, num_workers=args.num_workers, data_parallel_group=None ) debug_rank( "building energon dataloader " - f"dp_rank={dp_rank} dp_world={args.llm_dp} batch_size={args.micro_batch_size}" + f"role={role} lane={lane} dp_world={args.llm_dp} batch_size={args.micro_batch_size}" ) dataset = get_train_dataset( args.data_path, @@ -98,10 +179,54 @@ def _build_iterator_for_grid(args, grid): max_samples_per_sequence=args.max_samples_per_sequence, ) return EnergonIterator( - get_loader(dataset), tp_group=tp_group, source_rank=True, random_seed=args.seed + dp_rank + get_loader(dataset), + tp_group=tp_group, + source_rank=True, + random_seed=random_seed, + alignment_role="encoder" if role.startswith("encoder") else "llm", + llm_lanes=[lane], ) +def _combine_encoder_batches(batches: list[dict]) -> dict: + """Combine LLM-lane batches into one encoder batch and drop LLM-only metadata.""" + if not batches: + raise RuntimeError("cannot combine an empty encoder batch list") + + combined = {} + for key in ("input_ids", "labels", "loss_mask", "position_ids"): + values = [batch.get(key) for batch in batches if batch.get(key) is not None] + if values: + combined[key] = torch.cat(values, dim=0) + + modality_values = [ + batch.get("modality_inputs") for batch in batches if batch.get("modality_inputs") is not None + ] + if modality_values: + combined["modality_inputs"] = _concat_nested_tensors(modality_values) + + return combined + + +def _concat_nested_tensors(values): + """Concatenate a list of matching nested tensor structures along leading dim.""" + present = [value for value in values if value is not None] + if not present: + return None + first = present[0] + if isinstance(first, torch.Tensor): + return torch.cat(present, dim=0) + if isinstance(first, dict): + keys = set().union(*(value.keys() for value in present if isinstance(value, dict))) + merged = {} + for key in sorted(keys): + value = _concat_nested_tensors([item.get(key) for item in present]) + if value is not None: + merged[key] = value + return merged + raise TypeError(f"cannot concatenate encoder batch value of type {type(first).__name__}") + + def _build_tokenizer(args): from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( MegatronMultimodalTokenizer, @@ -120,13 +245,25 @@ class EnergonIterator: """Endless wrapper around an Energon dataloader with TP-rank-0 ownership.""" def __init__( - self, dataloader, tp_group=None, source_rank: bool = True, random_seed: Optional[int] = None + self, + dataloader, + tp_group=None, + source_rank: bool = True, + random_seed: Optional[int] = None, + local_batch_fn: Optional[Callable[[], dict]] = None, + alignment_role: Optional[str] = None, + llm_lanes: Optional[list[int]] = None, ) -> None: self._dataloader = dataloader - self._iterator = iter(dataloader) if dataloader is not None else None + self._iterator = None self._tp_group = tp_group self._source_rank = source_rank + self._local_batch_fn = local_batch_fn + self._alignment_role = alignment_role + self._llm_lanes = llm_lanes or [] self._prefetched = None + self._prefetched_component_signatures = None + self._local_component_signatures = None self._python_random_state = None if random_seed is not None: rng = random.Random(random_seed) @@ -142,40 +279,74 @@ def __next__(self): return batch batch = self._next_local_batch() if self._source_rank else None + component_signatures = self._current_component_signatures(batch) if is_process_group_member(self._tp_group) and self._tp_group.size() > 1: - obj = [batch] + obj = [(batch, component_signatures)] dist.broadcast_object_list(obj, src=self._tp_source_rank(), group=self._tp_group) - batch = obj[0] + batch, component_signatures = obj[0] + self._prefetched_component_signatures = component_signatures return batch - def peek_signature(self): - """Read and retain the next batch, returning a compact deterministic signature.""" + def peek_alignment(self): + """Read and retain the next batch, returning lane signatures from TP source ranks.""" if self._prefetched is None: self._prefetched = next(self) - return self._batch_signature(self._prefetched) + if not self._source_rank or self._alignment_role is None: + return None + signatures = self._prefetched_component_signatures + if signatures is None: + signatures = [self._batch_signature(self._prefetched)] + return { + "role": self._alignment_role, + "llm_lanes": self._llm_lanes, + "signatures": signatures, + } def _next_local_batch(self): """Read the next local Energon batch on the TP source rank.""" if self._python_random_state is None: - return self._read_next_local_batch() + result = self._read_next_local_batch() + return self._extract_batch_and_signatures(result) global_random_state = random.getstate() try: random.setstate(self._python_random_state) - batch = self._read_next_local_batch() + result = self._read_next_local_batch() + batch = self._extract_batch_and_signatures(result) self._python_random_state = random.getstate() return batch finally: random.setstate(global_random_state) + def _extract_batch_and_signatures(self, result): + """Handle local batch providers that also return component signatures.""" + self._local_component_signatures = None + if isinstance(result, tuple) and len(result) == 2: + batch, signatures = result + self._local_component_signatures = signatures + return batch + return result + def _read_next_local_batch(self): """Read from the underlying dataloader, cycling at epoch boundaries.""" + if self._local_batch_fn is not None: + return self._local_batch_fn() + if self._iterator is None: + self._iterator = iter(self._dataloader) try: return next(self._iterator) except StopIteration: self._iterator = iter(self._dataloader) return next(self._iterator) + def _current_component_signatures(self, batch): + """Return per-lane signatures for the current batch if they can be inferred.""" + if batch is None: + return None + if self._local_component_signatures is not None: + return self._local_component_signatures + return [self._batch_signature(batch)] + def _tp_source_rank(self) -> int: """Return the global source rank for the local TP batch broadcast.""" if hasattr(dist, "get_global_rank"): diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh index 43f6d6d1f82..6b5d4335a50 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh @@ -16,11 +16,26 @@ case "${TRAINING_STAGE}" in ;; esac -GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +GPUS_PER_NODE="${GPUS_PER_NODE:-}" TRAIN_ITERS="${TRAIN_ITERS:-100}" NUM_MICROBATCHES="${NUM_MICROBATCHES:-4}" MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" -LLM_DP=2 +ENCODER_TP="${ENCODER_TP:-2}" +ENCODER_PP="${ENCODER_PP:-1}" +ENCODER_DP="${ENCODER_DP:-2}" +LLM_TP="${LLM_TP:-2}" +LLM_PP="${LLM_PP:-1}" +LLM_DP="${LLM_DP:-2}" +LLM_EP="${LLM_EP:-4}" +ENCODER_SIZE=$((ENCODER_TP * ENCODER_PP * ENCODER_DP)) +LLM_SIZE=$((LLM_TP * LLM_PP * LLM_DP)) +LLM_OFFSET="${LLM_OFFSET:-${ENCODER_SIZE}}" +EXPECTED_WORLD_SIZE=$((ENCODER_SIZE + LLM_SIZE)) +GPUS_PER_NODE="${GPUS_PER_NODE:-${EXPECTED_WORLD_SIZE}}" +if [[ "${GPUS_PER_NODE}" -ne "${EXPECTED_WORLD_SIZE}" ]]; then + echo "ERROR: GPUS_PER_NODE=${GPUS_PER_NODE} but hetero layout requires ${EXPECTED_WORLD_SIZE}" >&2 + exit 1 +fi GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP))}" LR_WARMUP_ITERS="${LR_WARMUP_ITERS:-2}" LR_DECAY_ITERS="${LR_DECAY_ITERS:-10}" @@ -45,6 +60,7 @@ fi echo "=== Hetero MIMO Nemotron6-MoE VLM 20L Energon training ===" echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} gbs=${GLOBAL_BATCH_SIZE}" +echo "layout=encoder(tp=${ENCODER_TP},pp=${ENCODER_PP},dp=${ENCODER_DP}) llm(tp=${LLM_TP},pp=${LLM_PP},dp=${LLM_DP},ep=${LLM_EP}) world=${EXPECTED_WORLD_SIZE}" echo "data=${DATA_PATH}" echo "tokenizer=${TOKENIZER_MODEL}" echo "===========================================================" @@ -65,14 +81,14 @@ fi --model-provider nemotron-moe-vlm-20l \ --dataset-provider energon_multimodal \ --training-stage "${TRAINING_STAGE}" \ - --encoder-tp 2 \ - --encoder-pp 1 \ - --encoder-dp 2 \ - --llm-offset 4 \ - --llm-tp 2 \ - --llm-pp 1 \ + --encoder-tp "${ENCODER_TP}" \ + --encoder-pp "${ENCODER_PP}" \ + --encoder-dp "${ENCODER_DP}" \ + --llm-offset "${LLM_OFFSET}" \ + --llm-tp "${LLM_TP}" \ + --llm-pp "${LLM_PP}" \ --llm-dp "${LLM_DP}" \ - --llm-ep 4 \ + --llm-ep "${LLM_EP}" \ --llm-expt-tp 1 \ --llm-expt-dp 1 \ --vocab-size 131072 \ diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index b6bb1fb3f15..7249cfa3640 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -140,18 +140,22 @@ def validate_energon_data_args(args: argparse.Namespace) -> None: raise ValueError("energon_multimodal is currently wired for the Nemotron 20L VLM provider") if args.encoder_pp != 1 or args.llm_pp != 1: raise ValueError("energon_multimodal currently supports encoder and LLM PP size 1") - if args.encoder_dp != args.llm_dp: + if args.encoder_dp > args.llm_dp: raise ValueError( - "energon_multimodal currently requires --encoder-dp == --llm-dp so the " - "encoder and LLM grids consume matching DP-lane samples" + "energon_multimodal currently supports fan-out only: --encoder-dp must be " + "<= --llm-dp" + ) + if args.llm_dp % args.encoder_dp != 0: + raise ValueError( + "energon_multimodal fan-out requires --llm-dp to be divisible by --encoder-dp" + ) + if args.encoder_dp != args.llm_dp and args.micro_batch_size != 1: + raise ValueError( + "energon_multimodal fan-out currently requires --micro-batch-size 1 so bridge " + "splits map one encoder sample to one LLM DP lane" ) if args.packing_buffer_size is not None and args.packing_buffer_size > 0: if args.micro_batch_size != 1: raise ValueError( "Energon packed multimodal batches currently require --micro-batch-size 1" ) - encoder_micro_batch_size = args.micro_batch_size * args.llm_dp // args.encoder_dp - if encoder_micro_batch_size != args.micro_batch_size: - raise ValueError( - "energon_multimodal currently requires equal encoder and LLM microbatch sizes" - ) diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py index cc4cc5a175e..66eb3f8572a 100644 --- a/examples/mimo/training/hetero/topology.py +++ b/examples/mimo/training/hetero/topology.py @@ -179,25 +179,24 @@ def create_hypercomm_grid( def get_pg_collection(grid: HyperCommGrid) -> ProcessGroupCollection: """Build a ProcessGroupCollection from a populated HyperCommGrid.""" - return ProcessGroupCollection.from_hyper_comm_grid( - grid, - required_pgs=[ - "tp", - "cp", - "pp", - "dp", - "dp_cp", - "tp_cp", - "mp", - "tp_dp_cp", - "ep", - "expt_tp", - "expt_dp", - "tp_ep", - "tp_ep_pp", - "intra_dist_opt", - ], - ) + pg = ProcessGroupCollection() + pg.tp = grid.get_pg("tp") + pg.cp = grid.get_pg("cp") + pg.pp = grid.get_pg("pp") + pg.dp = grid.get_pg("dp") + pg.dp_cp = grid.get_pg(["dp", "cp"]) + pg.intra_dp_cp = pg.dp_cp + pg.tp_cp = grid.get_pg(["tp", "cp"]) + pg.mp = grid.get_pg(["tp", "pp"]) + pg.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) + pg.ep = grid.get_pg("ep") + pg.expt_tp = grid.get_pg("expt_tp") + pg.expt_dp = grid.get_pg("expt_dp") + pg.intra_expt_dp = pg.expt_dp + pg.tp_ep = grid.get_pg(["expt_tp", "ep"]) + pg.tp_ep_pp = grid.get_pg(["expt_tp", "ep", "pp"]) + pg.intra_dist_opt = grid.get_pg(["tp", "cp", "dp", "pp"]) + return pg def create_language_embedding_groups(grid: HyperCommGrid) -> LanguageEmbeddingGroups: diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py index 74813304132..b745f52b75a 100644 --- a/megatron/core/hyper_comm_grid.py +++ b/megatron/core/hyper_comm_grid.py @@ -218,7 +218,7 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG rank_enum = self._gen_rank_enum_for_layout(ordered_dims, layout_name) pg, _ = dist.new_subgroups_by_enumeration(rank_enum, backend=self.backend, **kwargs) - if not dist.is_initialized() or dist.get_rank() == 0: + if dist.is_initialized() and dist.get_rank() == 0: logging.info( f"Generated process group for {unique_group_key} with enumeration {rank_enum}" ) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index 4e4018f5dc1..d7323a6c487 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -426,10 +426,30 @@ def _forward_encoders( output = self._empty_modality_output(submodule) if output is not None: + self._attach_modality_split_sizes(output, input_ids, encoder_name) outputs[encoder_name] = output return outputs + def _attach_modality_split_sizes( + self, output: torch.Tensor, input_ids: Optional[torch.Tensor], encoder_name: str + ) -> None: + """Annotate flat modality outputs with per-sample split sizes for bridge fan-out.""" + if ( + not isinstance(output, torch.Tensor) + or output.ndim != 2 + or input_ids is None + or input_ids.ndim != 2 + or encoder_name not in self.special_token_ids + or input_ids.size(0) <= 1 + ): + return + + token_id = self.special_token_ids[encoder_name] + split_sizes = (input_ids == token_id).sum(dim=1).to(torch.long).tolist() + if sum(split_sizes) == output.size(0): + output._mimo_bridge_split_sizes = split_sizes + def _has_encoder_tokens(self, input_ids: Optional[torch.Tensor], encoder_name: str) -> bool: """Return whether the batch contains tokens for an encoder module.""" if input_ids is None or encoder_name not in self.special_token_ids: diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 9d22c75060a..41fad0d5dd8 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -2,6 +2,8 @@ """Optimizer for MIMO models with heterogeneous parallelism.""" +# pylint: disable=missing-function-docstring + from __future__ import annotations from copy import deepcopy @@ -51,7 +53,6 @@ def __init__(self, module_infos: Dict[str, ModuleOptimizerInfo], config: Optimiz @torch.no_grad() def prepare_grads(self) -> bool: - """Prepare gradients for every active module optimizer.""" found_inf = False for opt in self._active_optimizers: found_inf |= opt.prepare_grads() @@ -73,7 +74,6 @@ def get_grad_norm(self) -> float: @torch.no_grad() def step(self) -> Tuple[bool, Optional[float], Optional[int]]: - """Run one optimizer step across active module optimizers.""" found_inf = self.prepare_grads() # Synchronize found_inf across all ranks to prevent deadlock: # if encoder ranks detect inf but LLM ranks don't, the early return @@ -106,25 +106,21 @@ def step(self) -> Tuple[bool, Optional[float], Optional[int]]: @torch.no_grad() def step_with_ready_grads(self) -> bool: - """Apply updates after gradients have been prepared.""" success = True for opt in self._active_optimizers: success &= opt.step_with_ready_grads() return success def zero_grad(self, set_to_none: bool = True): - """Clear gradients on all active module optimizers.""" for opt in self._active_optimizers: opt.zero_grad(set_to_none) def get_loss_scale(self) -> torch.Tensor: - """Return the active optimizer loss scale, or one for stub ranks.""" if self._active_optimizers: return self._active_optimizers[0].get_loss_scale() return torch.tensor([1.0], dtype=torch.float32, device="cuda") def count_zeros(self) -> int: - """Count zero gradients across all MIMO modules.""" num_modules = len(self.module_infos) zeros_by_module = torch.zeros(num_modules, device="cuda", dtype=torch.float32) @@ -146,7 +142,6 @@ def param_groups(self) -> List[dict]: # Checkpointing def state_dict(self): - """Return per-module optimizer state dicts.""" return { name: info.optimizer.state_dict() if info.is_active and info.optimizer else None for name, info in self.module_infos.items() @@ -198,7 +193,6 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, return sharded_state def reload_model_params(self, state_dict=None): - """Reload model params in each active module optimizer.""" for opt in self._active_optimizers: opt.reload_model_params(state_dict) @@ -301,63 +295,18 @@ def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: Only fetches process groups required by the optimizer. Assumes all groups are pre-created in the grid via grid.create_pg() - does not create any new groups. - - For HyperCommGrid instances with registered expert dimensions, the following - groups must be pre-created before calling this function: - grid.create_pg(["dp"]) - grid.create_pg(["dp", "cp"]) - grid.create_pg(["tp"]) - grid.create_pg(["pp"]) - grid.create_pg(["tp", "pp"]) - grid.create_pg(["expt_tp", "ep", "pp"]) - grid.create_pg("expt_dp") - grid.create_pg(["tp", "cp", "dp", "pp"]) - - Args: - grid: HyperCommGrid with pre-created process groups. - - Returns: - ProcessGroupCollection containing optimizer-required groups: - - dp: Data parallel group - - dp_cp: Data parallel with context parallel - - tp: Tensor parallel group - - mp: Model parallel group (tp × pp) - - tp_ep_pp: Expert tensor-model-pipeline group - - expt_dp: Expert data parallel group """ - try: - return ProcessGroupCollection.from_hyper_comm_grid( - grid, - create=False, - required_pgs=['dp', 'dp_cp', 'tp', 'pp', 'mp', 'tp_ep_pp', 'expt_dp', 'intra_dist_opt'], - ) - except (KeyError, ValueError) as exc: - if hasattr(grid, 'has_layout') and grid.has_layout('expert'): - raise exc - # Backward-compatible fallback for older tests/grids that encoded EP - # directly in the base Cartesian layout. - pass - pg = ProcessGroupCollection() - - # Core groups needed by optimizer and checkpointing pg.dp = grid.get_pg("dp") pg.dp_cp = grid.get_pg(["dp", "cp"]) + pg.intra_dp_cp = pg.dp_cp pg.tp = grid.get_pg("tp") pg.pp = grid.get_pg("pp") pg.mp = grid.get_pg(["tp", "pp"]) - - # Expert groups - pg.tp_ep_pp = grid.get_pg(["tp", "ep", "pp"]) - pg.expt_dp = grid.get_pg(["dp", "ep"]) - - # Distributed optimizer grad stats group: must span all dimensions so grad norm - # and found-inf all-reduces see every unique gradient shard. TP/PP/EP ranks hold - # different parameters, DP ranks hold different optimizer shards after reduce-scatter. - # This mirrors standard Megatron's intra_distributed_optimizer_instance_group which - # spans the full world when num_distributed_optimizer_instances == 1. - pg.intra_dist_opt = grid.get_pg(["tp", "cp", "ep", "pp", "dp"]) - + pg.tp_ep_pp = grid.get_pg(["expt_tp", "ep", "pp"]) + pg.expt_dp = grid.get_pg("expt_dp") + pg.intra_expt_dp = pg.expt_dp + pg.intra_dist_opt = grid.get_pg(["tp", "cp", "dp", "pp"]) return pg diff --git a/megatron/core/pipeline_parallel/bridge_communicator.py b/megatron/core/pipeline_parallel/bridge_communicator.py index 515ddf1743a..bc028970ff4 100644 --- a/megatron/core/pipeline_parallel/bridge_communicator.py +++ b/megatron/core/pipeline_parallel/bridge_communicator.py @@ -350,7 +350,7 @@ def send_forward(self, tensor_to_send: torch.Tensor): num_sends = len(rank_info.send_to_ranks) if num_sends > 0: tensor_splits = self._split_tensor_at_batch_dim(tensor_to_send, num_sends) - self._communicate_shapes(tensor_to_send_next=tensor_splits[0]) + self._communicate_shapes(tensor_to_send_next=tensor_splits) for dest_rank, tensor_split in zip(rank_info.send_to_ranks, tensor_splits): logging.debug( f"[Bridge Comunicator] [send_forward] Rank {self.current_rank} " @@ -480,7 +480,7 @@ def send_backward(self, grad_tensor: torch.Tensor): # Send gradients back to source ranks num_receives = len(rank_info.recv_from_ranks) tensor_splits = self._split_tensor_at_batch_dim(grad_tensor, num_receives) - self._communicate_shapes(tensor_to_send_prev=tensor_splits[0]) + self._communicate_shapes(tensor_to_send_prev=tensor_splits) if num_receives > 0: for src_rank, tensor_split in zip(rank_info.recv_from_ranks, tensor_splits): # Send the gradient split back to the source rank @@ -618,7 +618,7 @@ def send_forward_recv_backward( activation_splits = self._split_tensor_at_batch_dim(input_tensor, num_sends) # Communicate shapes for both directions (send forward, receive backward) recv_forward_shapes, recv_grad_shapes = self._communicate_shapes( - tensor_to_send_next=activation_splits[0], recv_next=True + tensor_to_send_next=activation_splits, recv_next=True ) logging.debug( f"[Bridge Communicator] [send_forward_recv_backward] Rank {self.current_rank} " @@ -737,7 +737,7 @@ def send_backward_recv_forward( gradient_splits = self._split_tensor_at_batch_dim(grad_tensor, num_receives) # Communicate shapes for both directions (send backward, receive forward) recv_forward_shapes, recv_grad_shapes = self._communicate_shapes( - tensor_to_send_prev=gradient_splits[0], recv_prev=True + tensor_to_send_prev=gradient_splits, recv_prev=True ) logging.debug( f"[Bridge Communicator] [send_backward_recv_backward] Rank {self.current_rank} " @@ -848,8 +848,10 @@ def _communicate_shapes( when dealing with variable sequence lengths or dynamic shapes. Args: - tensor_to_send_next: The tensor to send to the next rank (None if not sending) - tensor_to_send_prev: The tensor to send to the previous rank (None if not sending) + tensor_to_send_next: Tensor shape source for next ranks. Pass a single tensor when + every peer receives the same shape, or a list with one tensor per peer. + tensor_to_send_prev: Tensor shape source for previous ranks. Pass a single tensor when + every peer receives the same shape, or a list with one tensor per peer. recv_next: Whether to receive from the next rank (None if not receiving) recv_prev: Whether to receive from the previous rank (None if not receiving) @@ -876,12 +878,14 @@ def _communicate_shapes( if rank_info.role == CommRole.SENDER: # Prepare send operations for forward shapes if tensor_to_send_next is not None: - send_shape = tensor_to_send_next.shape - send_shape_tensor = torch.tensor( - send_shape, device=torch.cuda.current_device(), dtype=torch.int64 + tensors_to_send = self._as_per_peer_tensors( + tensor_to_send_next, len(rank_info.send_to_ranks) ) # Add send operations for each destination - for dest_rank in rank_info.send_to_ranks: + for dest_rank, tensor in zip(rank_info.send_to_ranks, tensors_to_send): + send_shape_tensor = torch.tensor( + tensor.shape, device=torch.cuda.current_device(), dtype=torch.int64 + ) ops.append( torch.distributed.P2POp( torch.distributed.isend, send_shape_tensor, dest_rank @@ -918,12 +922,14 @@ def _communicate_shapes( # If we need to send gradient shapes back, prepare send operations if tensor_to_send_prev is not None: - grad_shape = tensor_to_send_prev.shape - grad_shape_tensor = torch.tensor( - grad_shape, device=torch.cuda.current_device(), dtype=torch.int64 + tensors_to_send = self._as_per_peer_tensors( + tensor_to_send_prev, len(rank_info.recv_from_ranks) ) - for src_rank in rank_info.recv_from_ranks: + for src_rank, tensor in zip(rank_info.recv_from_ranks, tensors_to_send): + grad_shape_tensor = torch.tensor( + tensor.shape, device=torch.cuda.current_device(), dtype=torch.int64 + ) ops.append( torch.distributed.P2POp( torch.distributed.isend, grad_shape_tensor, src_rank @@ -947,6 +953,17 @@ def _communicate_shapes( return recv_forward_shapes, recv_grad_shapes + @staticmethod + def _as_per_peer_tensors(tensors, expected_count: int) -> List[torch.Tensor]: + """Return one tensor per peer from either a shared tensor or a per-peer tensor list.""" + if isinstance(tensors, torch.Tensor): + return [tensors for _ in range(expected_count)] + if len(tensors) != expected_count: + raise ValueError( + f"expected {expected_count} tensors for shape communication, got {len(tensors)}" + ) + return list(tensors) + def _split_tensor_at_batch_dim( self, aggregated_tensor: torch.Tensor, num_splits: int ) -> List[torch.Tensor]: @@ -962,6 +979,27 @@ def _split_tensor_at_batch_dim( if num_splits <= 0: raise ValueError(f"num_splits must be positive, got {num_splits}") + split_sizes = getattr(aggregated_tensor, "_mimo_bridge_split_sizes", None) + if split_sizes is not None: + if num_splits == 1: + return [aggregated_tensor.contiguous()] + split_sizes = [int(size) for size in split_sizes] + if len(split_sizes) != num_splits: + raise ValueError( + f"bridge split metadata has {len(split_sizes)} entries, " + f"but communication requires {num_splits} splits" + ) + batch_dim_size = int(aggregated_tensor.shape[self._batch_dim]) + if sum(split_sizes) != batch_dim_size: + raise ValueError( + f"bridge split metadata sums to {sum(split_sizes)}, " + f"but tensor batch dimension is {batch_dim_size}" + ) + return [ + split.contiguous() + for split in torch.split(aggregated_tensor, split_sizes, dim=self._batch_dim) + ] + splits = torch.tensor_split(aggregated_tensor, num_splits, dim=self._batch_dim) # PyTorch p2p requires the tensors to be contiguous return [split.contiguous() for split in splits] diff --git a/megatron/core/process_groups_config.py b/megatron/core/process_groups_config.py index 914722ade29..6c1e3651387 100644 --- a/megatron/core/process_groups_config.py +++ b/megatron/core/process_groups_config.py @@ -259,57 +259,6 @@ def use_mpu_process_groups(cls, required_pgs: Optional[List[str]] = None): return cls(**init_dict) - @classmethod - def from_hyper_comm_grid( - cls, - grid, - create: bool = False, - required_pgs: Optional[List[str]] = None, - num_distributed_optimizer_instances: int = 1, - ): - """Build a ProcessGroupCollection from a HyperCommGrid with expert dimensions.""" - if num_distributed_optimizer_instances != 1: - raise ValueError( - "ProcessGroupCollection.from_hyper_comm_grid only supports " - "num_distributed_optimizer_instances == 1" - ) - - pg_specs = { - 'tp': 'tp', - 'cp': 'cp', - 'pp': 'pp', - 'dp': 'dp', - 'dp_cp': ['dp', 'cp'], - 'tp_cp': ['tp', 'cp'], - 'mp': ['tp', 'pp'], - 'tp_dp_cp': ['tp', 'dp', 'cp'], - 'ep': 'ep', - 'expt_tp': 'expt_tp', - 'expt_dp': 'expt_dp', - 'tp_ep': ['expt_tp', 'ep'], - 'tp_ep_pp': ['expt_tp', 'ep', 'pp'], - 'intra_dist_opt': grid.dim_names, - } - if required_pgs is None: - required_pgs = list(pg_specs) - - invalid_pgs = [pg for pg in required_pgs if pg not in pg_specs] - if invalid_pgs: - raise ValueError(f"Invalid process groups requested: {invalid_pgs}") - - def get_or_create(dims): - return grid.create_pg(dims) if create else grid.get_pg(dims) - - init_dict = {pg_name: get_or_create(pg_specs[pg_name]) for pg_name in required_pgs} - - if 'dp_cp' in init_dict: - init_dict.setdefault('intra_dp_cp', init_dict['dp_cp']) - if 'expt_dp' in init_dict: - init_dict.setdefault('intra_expt_dp', init_dict['expt_dp']) - init_dict.setdefault('inter_dist_opt', None) - - return cls(**init_dict) - @staticmethod def setup_process_groups_for_optimizer( pg_collection: Optional['ProcessGroupCollection'], diff --git a/tests/unit_tests/models/test_mimo_1f1b_schedule.py b/tests/unit_tests/models/test_mimo_1f1b_schedule.py index 09595b94131..21fb1eb1510 100644 --- a/tests/unit_tests/models/test_mimo_1f1b_schedule.py +++ b/tests/unit_tests/models/test_mimo_1f1b_schedule.py @@ -129,25 +129,24 @@ def destroy_all_grids(): def get_pg_collection(grid): """Get ProcessGroupCollection from grid.""" - return ProcessGroupCollection.from_hyper_comm_grid( - grid, - required_pgs=[ - 'tp', - 'cp', - 'pp', - 'dp', - 'dp_cp', - 'tp_cp', - 'mp', - 'tp_dp_cp', - 'ep', - 'expt_tp', - 'expt_dp', - 'tp_ep', - 'tp_ep_pp', - 'intra_dist_opt', - ], - ) + pg = ProcessGroupCollection() + pg.tp = grid.get_pg("tp") + pg.cp = grid.get_pg("cp") + pg.pp = grid.get_pg("pp") + pg.dp = grid.get_pg("dp") + pg.dp_cp = grid.get_pg(["dp", "cp"]) + pg.intra_dp_cp = pg.dp_cp + pg.tp_cp = grid.get_pg(["tp", "cp"]) + pg.mp = grid.get_pg(["tp", "pp"]) + pg.tp_dp_cp = grid.get_pg(["tp", "dp", "cp"]) + pg.ep = grid.get_pg("ep") + pg.expt_tp = grid.get_pg("expt_tp") + pg.expt_dp = grid.get_pg("expt_dp") + pg.intra_expt_dp = pg.expt_dp + pg.tp_ep = grid.get_pg(["expt_tp", "ep"]) + pg.tp_ep_pp = grid.get_pg(["expt_tp", "ep", "pp"]) + pg.intra_dist_opt = grid.get_pg(["tp", "cp", "dp", "pp"]) + return pg def create_all_embedding_groups(grids): diff --git a/tests/unit_tests/models/test_mimo_bridge_split_sizes.py b/tests/unit_tests/models/test_mimo_bridge_split_sizes.py new file mode 100644 index 00000000000..efcdceaa00e --- /dev/null +++ b/tests/unit_tests/models/test_mimo_bridge_split_sizes.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +import torch + +from megatron.core.models.mimo.model.base import MimoModel + + +def test_attach_modality_split_sizes_includes_zero_image_lanes(): + """Bridge split metadata follows per-sample image token counts.""" + model = object.__new__(MimoModel) + model.special_token_ids = {"images": 18} + + input_ids = torch.tensor([[18, 1, 2, 3], [4, 5, 6, 7], [18, 18, 8, 9]]) + output = torch.empty((3, 4)) + + model._attach_modality_split_sizes(output, input_ids, "images") + + assert output._mimo_bridge_split_sizes == [1, 0, 2] diff --git a/tests/unit_tests/pipeline_parallel/test_bridge_communicator.py b/tests/unit_tests/pipeline_parallel/test_bridge_communicator.py index 326ac8b5890..43911370950 100644 --- a/tests/unit_tests/pipeline_parallel/test_bridge_communicator.py +++ b/tests/unit_tests/pipeline_parallel/test_bridge_communicator.py @@ -521,11 +521,49 @@ def test_2d_fan_out_fwd_bwd(self): ) rank = dist.get_rank() + split_sizes = [257, 577, 773, 989] + total_rows = sum(split_sizes) if bridge.is_current_rank_in_grid(src_grid): - tensor = torch.randn(577 * 4, 128, device='cuda') + tensor = torch.cat( + [ + torch.full((split_size, 128), float(index), device='cuda') + for index, split_size in enumerate(split_sizes) + ], + dim=0, + ) + tensor._mimo_bridge_split_sizes = split_sizes grad = bridge.send_forward_recv_backward(tensor) - assert grad.shape == (577 * 4, 128) + assert grad.shape == (total_rows, 128) + expected_grad = torch.cat( + [ + torch.full( + (split_size, 128), float(dest_grid.rank_offset + index), device='cuda' + ) + for index, split_size in enumerate(split_sizes) + ], + dim=0, + ) + assert torch.equal(grad, expected_grad) else: - grad = torch.full((577, 128), float(rank), device='cuda') + split_index = rank - dest_grid.rank_offset + grad = torch.full((split_sizes[split_index], 128), float(rank), device='cuda') activation = bridge.send_backward_recv_forward(grad) - assert activation.shape == (577, 128) + assert activation.shape == (split_sizes[split_index], 128) + assert torch.equal(activation, torch.full_like(activation, float(split_index))) + + def test_2d_metadata_split_allows_zero_size_chunks(self): + """Metadata split supports text-only lanes that have no image embeddings.""" + bridge = BridgeCommunicator.__new__(BridgeCommunicator) + bridge.tensor_ndim = 2 + bridge.dim_mapping = {'s': 0, 'h': 1, 'b': 0} + + tensor = torch.arange(20, device='cuda').reshape(10, 2) + tensor._mimo_bridge_split_sizes = [3, 0, 4, 3] + + splits = bridge._split_tensor_at_batch_dim(tensor, 4) + + assert [split.shape[0] for split in splits] == [3, 0, 4, 3] + assert torch.equal(splits[0], tensor[:3]) + assert splits[1].numel() == 0 + assert torch.equal(splits[2], tensor[3:7]) + assert torch.equal(splits[3], tensor[7:]) diff --git a/tests/unit_tests/test_hetero_energon.py b/tests/unit_tests/test_hetero_energon.py index 32264232f6c..d0da5509f01 100644 --- a/tests/unit_tests/test_hetero_energon.py +++ b/tests/unit_tests/test_hetero_energon.py @@ -2,6 +2,9 @@ import random +import torch + +from examples.mimo.data import hetero_energon from examples.mimo.data.hetero_energon import EnergonIterator @@ -35,3 +38,32 @@ def test_energon_iterator_uses_isolated_python_random_state(): assert first_values == second_values assert len(set(first_values)) > 1 + + +def test_combine_encoder_batches_drops_packing_and_concatenates_modalities(): + """Encoder fan-out combines whole packed samples without carrying LLM packing metadata.""" + first = { + "input_ids": torch.tensor([[1, 2, 3]]), + "labels": torch.tensor([[2, 3, 4]]), + "loss_mask": torch.tensor([[1.0, 1.0, 0.0]]), + "position_ids": torch.tensor([[0, 1, 2]]), + "packing_kwargs": {"cu_seqlens_q": torch.tensor([0, 3])}, + "modality_inputs": {"images": {"radio_encoder": {"x": torch.ones(1, 3, 4, 4)}}}, + } + second = { + "input_ids": torch.tensor([[5, 6, 7]]), + "labels": torch.tensor([[6, 7, 8]]), + "loss_mask": torch.tensor([[1.0, 0.0, 0.0]]), + "position_ids": torch.tensor([[0, 1, 2]]), + "packing_kwargs": {"cu_seqlens_q": torch.tensor([0, 3])}, + "modality_inputs": {"images": {"radio_encoder": {"x": torch.zeros(2, 3, 4, 4)}}}, + } + + combined = hetero_energon._combine_encoder_batches([first, second]) + + assert "packing_kwargs" not in combined + assert combined["input_ids"].tolist() == [[1, 2, 3], [5, 6, 7]] + images = combined["modality_inputs"]["images"]["radio_encoder"]["x"] + assert images.shape == (3, 3, 4, 4) + assert torch.all(images[:1] == 1) + assert torch.all(images[1:] == 0) diff --git a/tests/unit_tests/test_process_groups_config.py b/tests/unit_tests/test_process_groups_config.py index fa1c883cd35..b49962b1a5a 100644 --- a/tests/unit_tests/test_process_groups_config.py +++ b/tests/unit_tests/test_process_groups_config.py @@ -3,7 +3,6 @@ import pytest import torch.distributed as dist -from megatron.core.hyper_comm_grid import HyperCommGrid from megatron.core.process_groups_config import ProcessGroupCollection from tests.unit_tests.test_utilities import Utils @@ -105,104 +104,6 @@ def test_repr_with_list_process_groups(self, mocker): assert "ProcessGroupCollection(" in repr_str assert "hcp([2, 4])" in repr_str - def test_from_hyper_comm_grid_reads_required_groups(self, mocker): - """Test mapping from an extended HyperCommGrid to ProcessGroupCollection.""" - grid = mocker.Mock() - grid.dim_names = ["tp", "cp", "dp", "pp"] - - pgs = {} - - def pg_for(dims): - key = tuple(dims) if isinstance(dims, list) else dims - pgs.setdefault(key, mocker.Mock(spec=dist.ProcessGroup)) - return pgs[key] - - grid.get_pg.side_effect = pg_for - - collection = ProcessGroupCollection.from_hyper_comm_grid( - grid, - required_pgs=['tp', 'pp', 'dp', 'dp_cp', 'mp', 'expt_dp', 'tp_ep_pp', 'intra_dist_opt'], - ) - - assert collection.tp is pgs['tp'] - assert collection.pp is pgs['pp'] - assert collection.dp is pgs['dp'] - assert collection.dp_cp is pgs[('dp', 'cp')] - assert collection.mp is pgs[('tp', 'pp')] - assert collection.expt_dp is pgs['expt_dp'] - assert collection.tp_ep_pp is pgs[('expt_tp', 'ep', 'pp')] - assert collection.intra_dist_opt is pgs[('tp', 'cp', 'dp', 'pp')] - assert collection.intra_dp_cp is collection.dp_cp - assert collection.intra_expt_dp is collection.expt_dp - assert collection.inter_dist_opt is None - - def test_from_hyper_comm_grid_rejects_multi_instance_distopt(self, mocker): - """Phase 1 helper does not support multiple distributed optimizer instances.""" - grid = mocker.Mock() - with pytest.raises(ValueError, match="num_distributed_optimizer_instances == 1"): - ProcessGroupCollection.from_hyper_comm_grid(grid, num_distributed_optimizer_instances=2) - - def test_from_hyper_comm_grid_creates_from_real_extended_grid(self, mocker, monkeypatch): - """Test helper against a real HyperCommGrid expert layout without distributed init.""" - monkeypatch.setenv("WORLD_SIZE", "16") - mocker.patch('torch.distributed.get_rank', return_value=0) - mock_new_subgroups = mocker.patch('torch.distributed.new_subgroups_by_enumeration') - - created = [] - - def make_pg(rank_enum, **_kwargs): - pg = mocker.Mock(spec=dist.ProcessGroup) - pg.size.return_value = len(rank_enum[0]) - created.append((rank_enum, pg)) - return pg, None - - mock_new_subgroups.side_effect = make_pg - - grid = HyperCommGrid([2, 1, 4, 2], ["tp", "cp", "dp", "pp"]) - grid.register_layout("expert", [1, 4, 2, 2], ["expt_tp", "ep", "expt_dp", "pp"]) - - collection = ProcessGroupCollection.from_hyper_comm_grid( - grid, - create=True, - required_pgs=[ - 'tp', - 'dp', - 'dp_cp', - 'mp', - 'ep', - 'expt_tp', - 'expt_dp', - 'tp_ep', - 'tp_ep_pp', - 'intra_dist_opt', - ], - ) - - assert collection.tp is grid.get_pg("tp") - assert collection.dp is grid.get_pg("dp") - assert collection.dp_cp is grid.get_pg(["dp", "cp"]) - assert collection.mp is grid.get_pg(["tp", "pp"]) - assert collection.ep is grid.get_pg("ep") - assert collection.expt_tp is grid.get_pg("expt_tp") - assert collection.expt_dp is grid.get_pg("expt_dp") - assert collection.tp_ep is grid.get_pg(["expt_tp", "ep"]) - assert collection.tp_ep_pp is grid.get_pg(["expt_tp", "ep", "pp"]) - assert collection.intra_dist_opt is grid.get_pg(["tp", "cp", "dp", "pp"]) - assert collection.intra_dp_cp is collection.dp_cp - assert collection.intra_expt_dp is collection.expt_dp - assert collection.inter_dist_opt is None - - def test_from_hyper_comm_grid_rejects_missing_expert_pp(self, monkeypatch): - """tp_ep_pp requires the registered expert layout to include pp.""" - monkeypatch.setenv("WORLD_SIZE", "4") - grid = HyperCommGrid([2, 2], ["tp", "pp"]) - grid.register_layout("expert", [2, 2], ["ep", "expert_pp"]) - - with pytest.raises(ValueError, match="Dimensions .*pp"): - ProcessGroupCollection.from_hyper_comm_grid( - grid, create=True, required_pgs=['tp_ep_pp'] - ) - class TestPGConfigDefaultInitialization: From 931874611a837059cbbbb4add76cfbe45fd8c4c5 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Tue, 12 May 2026 04:46:08 +0000 Subject: [PATCH 25/44] NMFW-464 simplify MIMO partition layout --- megatron/core/models/mimo/model/base.py | 69 +++++------ megatron/core/models/mimo/partition/utils.py | 66 ++++------- .../models/test_mimo_embedding_alignment.py | 56 ++++----- tests/unit_tests/models/test_mimo_model.py | 22 +++- .../unit_tests/models/test_mimo_partition.py | 110 +++++++++--------- 5 files changed, 160 insertions(+), 163 deletions(-) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index d7323a6c487..23c91c66da4 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -152,7 +152,7 @@ def align_embeddings_by_token_positions( special_token_ids: Dictionary mapping modality names to their special token IDs Returns: - Combined embeddings tensor. Shape: (S, B, H) + Combined embeddings tensor. Shape: (B, S, H) """ # Ensure we have at least one modality if not modality_embeddings: @@ -170,7 +170,7 @@ def align_embeddings_by_token_positions( batch_size, seq_length = input_ids.size() # input_ids is [B, S] logger.debug( - f"Combined output tensor will have shape: [{seq_length}, {batch_size}, {hidden_dim}]" + f"Combined output tensor will have shape: [{batch_size}, {seq_length}, {hidden_dim}]" ) combined_embeddings = torch.zeros( @@ -199,7 +199,7 @@ def align_embeddings_by_token_positions( expanded_mask = mask.unsqueeze(-1).expand_as(combined_embeddings) combined_embeddings.masked_scatter_(expanded_mask, modality_emb.flatten()) - return combined_embeddings.transpose(0, 1).contiguous() # [S, B, H] + return combined_embeddings def _initialize_submodules(self) -> None: """Initialize modality submodules from the ModuleSpec configurations. @@ -304,10 +304,23 @@ def get_text_embeddings( position_ids[batch_idx, seq_idx].unsqueeze(0) if position_ids is not None else None ) - text_embeddings = ( - unwrap_model(self.language_model) - .embedding(input_ids=input_ids_text, position_ids=position_ids_text) - .squeeze(1) + language_model = unwrap_model(self.language_model) + embedding_layer = language_model.embedding + if ( + self.partition_adapter is not None + and self.partition_adapter.cfg.seq_parallel + and getattr(embedding_layer, 'scatter_to_sequence_parallel', False) + ): + raise RuntimeError( + "MIMO sequence parallelism requires language embedding scatter to be disabled; " + "pass scatter_embedding_sequence_parallel=False when constructing the " + "language model" + ) + + text_embeddings = embedding_layer( + input_ids=input_ids_text, position_ids=position_ids_text + ).squeeze( + 1 ) # Shape: [num_text_tokens, hidden_dim] return text_embeddings @@ -326,7 +339,9 @@ def forward( Args: input_ids: Input token IDs. Shape: (B, S) position_ids: Position IDs. Shape: (B, S) - attention_mask: Attention mask. Shape: (B, S) + attention_mask: Accepted for API compatibility. This path currently relies on + the language model's causal/packed-sequence masking and does not forward + dataloader attention masks to the language model. loss_mask: Loss mask. Shape: (B, S) labels: Labels for training. Shape: (B, S) modality_inputs: Dictionary mapping modality names to encoder inputs. For example: @@ -491,7 +506,7 @@ def _forward_language_module( Args: input_ids: Token IDs position_ids: Position IDs - attention_mask: Attention mask + attention_mask: Accepted for API compatibility; not forwarded to the language model. loss_mask: Loss mask for per-token loss normalization labels: Labels for loss computation input_tensors: Hidden states or embeddings from previous stage @@ -525,19 +540,17 @@ def _forward_language_module( ) if self.partition_adapter is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() shard_loss_inputs = self.role.is_last_stage(lang_name) - combined_embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( + combined_embeddings, labels, loss_mask, packed_seq_params = ( self.partition_adapter.shard( embeddings=combined_embeddings, labels=labels if shard_loss_inputs else None, loss_mask=loss_mask if shard_loss_inputs else None, - attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) ) - if combined_embeddings is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + else: + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() lm_output = self.language_model( input_ids=None, @@ -553,14 +566,11 @@ def _forward_language_module( if self.partition_adapter is not None: shard_loss_inputs = self.role.is_last_stage(lang_name) - _, labels, loss_mask, attention_mask, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=None, - labels=labels if shard_loss_inputs else None, - loss_mask=loss_mask if shard_loss_inputs else None, - attention_mask=attention_mask, - packed_seq_params=packed_seq_params, - ) + _, labels, loss_mask, packed_seq_params = self.partition_adapter.shard( + embeddings=None, + labels=labels if shard_loss_inputs else None, + loss_mask=loss_mask if shard_loss_inputs else None, + packed_seq_params=packed_seq_params, ) # Set input tensor on language model for PP (unwrap DDP to reach GPTModel) @@ -574,7 +584,7 @@ def _forward_language_module( position_ids=None, decoder_input=None, labels=labels, - attention_mask=attention_mask, + attention_mask=None, packed_seq_params=packed_seq_params, ) @@ -696,24 +706,17 @@ def _forward_all_modules( logger.debug(f"Combined embeddings shape: {combined_embeddings.shape}") # 3. If sharding is needed, apply PartitionAdapter. - # combined_embeddings is [S, B, H]; transpose to [B, S, H] for shard() which expects - # batch-first layout (required by get_batch_on_this_cp_rank). After CP sharding each - # rank holds [B, S/cp, H]; transpose back to [S/cp, B, H] for the language model. if self.partition_adapter is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() # [B, S, H] - combined_embeddings, labels, loss_mask, _, packed_seq_params = ( + combined_embeddings, labels, loss_mask, packed_seq_params = ( self.partition_adapter.shard( embeddings=combined_embeddings, labels=labels, loss_mask=loss_mask, - attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) ) - # shard() returns embeddings in [B, S/cp, H]; transpose to [S/cp, B, H] - # which is what the language model expects. - if combined_embeddings is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + else: + combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() # 5. Forward pass through language model lm_output = self.language_model( diff --git a/megatron/core/models/mimo/partition/utils.py b/megatron/core/models/mimo/partition/utils.py index cbb6d4cba5d..f593894ab03 100644 --- a/megatron/core/models/mimo/partition/utils.py +++ b/megatron/core/models/mimo/partition/utils.py @@ -89,7 +89,7 @@ def from_mp_config( class PartitionAdapter: - """Shard batch-first embeddings & label tensors for Context and Sequence Parallelism.""" + """Shard batch-first MIMO inputs and return language-model-ready embeddings.""" def __init__(self, cfg: PartitionConfig): """Initialize the partition adapter. @@ -100,22 +100,26 @@ def __init__(self, cfg: PartitionConfig): def shard( self, - embeddings: torch.Tensor, - labels: torch.Tensor, - loss_mask: torch.Tensor, - attention_mask: torch.Tensor, + embeddings: Optional[torch.Tensor], + labels: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], packed_seq_params: Optional[PackedSeqParams] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[PackedSeqParams]]: + ) -> Tuple[ + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[PackedSeqParams], + ]: """ Apply context parallel (CP) and sequence parallel (SP) sharding to input tensors. - All input tensors must be in batch-first layout: + Tensor inputs use the dataloader layout: - embeddings: (B, S, H) - - labels / loss_mask / attention_mask: (B, S) + - labels / loss_mask: (B, S) - After this call embeddings are still in (B, S/cp, H) batch-first layout. - The caller is responsible for transposing to (S/cp, B, H) if the language model - requires sequence-first tensors. + Embeddings are returned in language-model layout (S, B, H), with CP and SP applied + along the sequence dimension. Labels and loss masks are CP-sharded, but not + SP-sharded, because the language-model loss consumes the gathered TP sequence. Args: embeddings (torch.Tensor): @@ -124,29 +128,21 @@ def shard( Labels tensor. Shape: (B, S) loss_mask (torch.Tensor): Loss mask tensor. Shape: (B, S) - attention_mask (torch.Tensor): - Attention mask tensor. Shape: (B, S) packed_seq_params (PackedSeqParams, optional): Packed sequence parameters. Defaults to None. Returns: Tuple containing: - - embeddings (torch.Tensor): Sharded embeddings. Shape: (B, S/cp, H) + - embeddings (torch.Tensor): Sharded embeddings. Shape: (S/(cp*tp), B, H) - labels (torch.Tensor): Possibly sharded labels. Shape: (B, S/cp) - loss_mask (torch.Tensor): Possibly sharded loss mask. Shape: (B, S/cp) - - attention_mask (torch.Tensor): Possibly sharded attention mask. Shape: (B, S/cp) - packed_seq_params (PackedSeqParams, optional): Updated packed sequence parameters. """ - if not (self.cfg.use_cp or self.cfg.seq_parallel): - return embeddings, labels, loss_mask, attention_mask, packed_seq_params - # Sanity-check the sequence length before any sharding happens. if embeddings is not None: shard_factor = None seq_dim = None # which dimension holds the token sequence - # MimoModel.forward() passes embeddings in batch-first layout - # [B, S, H], so the token sequence dimension is always 1 here. if self.cfg.use_cp and self.cfg.seq_parallel: shard_factor = get_pg_size(self.cfg.tp_group) * get_pg_size(self.cfg.cp_group) * 2 seq_dim = 1 # embeddings shape: [B, S, H] @@ -173,35 +169,30 @@ def shard( ) if self.cfg.use_cp: - embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( - self._apply_context_parallel( - embeddings, labels, loss_mask, attention_mask, packed_seq_params - ) + embeddings, labels, loss_mask, packed_seq_params = self._apply_context_parallel( + embeddings, labels, loss_mask, packed_seq_params ) if self.cfg.seq_parallel and embeddings is not None: - # GPT/Hybrid output layers gather sequence-parallel hidden states - # before per-token loss, so labels/loss_mask remain full sequence. embeddings = embeddings.transpose(0, 1).contiguous() embeddings = tensor_parallel.scatter_to_sequence_parallel_region( embeddings, group=self.cfg.tp_group ) + elif embeddings is not None: embeddings = embeddings.transpose(0, 1).contiguous() - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params def _apply_context_parallel( self, embeddings: Optional[torch.Tensor], labels: Optional[torch.Tensor], loss_mask: Optional[torch.Tensor], - attention_mask: Optional[torch.Tensor], packed_seq_params: Optional[PackedSeqParams], ) -> Tuple[ Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor], - Optional[torch.Tensor], Optional[PackedSeqParams], ]: """ @@ -214,8 +205,6 @@ def _apply_context_parallel( Labels tensor. Shape: (B, S) loss_mask (Optional[torch.Tensor]): Loss mask tensor. Shape: (B, S) - attention_mask (Optional[torch.Tensor]): - Attention mask tensor. Shape: (B, S) packed_seq_params (PackedSeqParams, optional): Packed sequence parameters. Defaults to None. @@ -224,12 +213,10 @@ def _apply_context_parallel( - embeddings (Optional[torch.Tensor]): Sharded embeddings. Shape: (B, S/cp, H) - labels (Optional[torch.Tensor]): Possibly sharded labels. Shape: (B, S/cp) - loss_mask (Optional[torch.Tensor]): Possibly sharded loss mask. Shape: (B, S/cp) - - attention_mask (Optional[torch.Tensor]): Possibly sharded attention mask. - Shape: (B, S/cp) - packed_seq_params (PackedSeqParams, optional): Updated packed sequence parameters. """ if not self.cfg.use_cp: - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params # Distribute sequence across CP ranks batch = dict() @@ -239,11 +226,9 @@ def _apply_context_parallel( batch["labels"] = labels if loss_mask is not None: batch["loss_mask"] = loss_mask - if attention_mask is not None: - batch["attention_mask"] = attention_mask if packed_seq_params is None or getattr(packed_seq_params, 'qkv_format', 'sbhd') == 'sbhd': - batch = get_batch_on_this_cp_rank(batch) + batch = get_batch_on_this_cp_rank(batch, cp_group=self.cfg.cp_group) else: assert _HAVE_TEX and is_te_min_version("1.10.0"), ( "Please update Transformer Engine to >= 1.10 " @@ -258,11 +243,10 @@ def _apply_context_parallel( ) batch[key] = data.index_select(1, index) - # Extract sharded tensors; embeddings remain in [B, S/cp, H] — the caller - # is responsible for transposing to [S/cp, B, H] for the language model. + # Extract sharded tensors; shard() transposes embeddings to language-model + # layout after CP and before optional SP. embeddings = batch.get("embeddings", None) labels = batch.get("labels", None) loss_mask = batch.get("loss_mask", None) - attention_mask = batch.get("attention_mask", None) - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params diff --git a/tests/unit_tests/models/test_mimo_embedding_alignment.py b/tests/unit_tests/models/test_mimo_embedding_alignment.py index 688ebe4832b..c6e0f105cdf 100644 --- a/tests/unit_tests/models/test_mimo_embedding_alignment.py +++ b/tests/unit_tests/models/test_mimo_embedding_alignment.py @@ -107,17 +107,17 @@ def test_basic_alignment(self): ) # Check output shape - assert combined.shape == (seq_length, batch_size, hidden_dim) + assert combined.shape == (batch_size, seq_length, hidden_dim) # Check special token positions have the correct embeddings # First vision token (Batch 0, Seq 1) should have the first vision embedding - assert combined[1, 0, 0] == 10.0 # First marker - assert torch.all(combined[1, 0, 1:] == 0.0), "Non-zero values found after marker" + assert combined[0, 1, 0] == 10.0 # First marker + assert torch.all(combined[0, 1, 1:] == 0.0), "Non-zero values found after marker" # Second vision token (Batch 1, Seq 3) should have the second vision embedding - assert combined[3, 1, 1] == 20.0 # Second marker - assert torch.all(combined[3, 1, :1] == 0.0), "Non-zero values found before marker" - assert torch.all(combined[3, 1, 2:] == 0.0), "Non-zero values found after marker" + assert combined[1, 3, 1] == 20.0 # Second marker + assert torch.all(combined[1, 3, :1] == 0.0), "Non-zero values found before marker" + assert torch.all(combined[1, 3, 2:] == 0.0), "Non-zero values found after marker" # Verify text positions have only zeros text_positions = [ @@ -138,7 +138,7 @@ def test_basic_alignment(self): ] for s, b in text_positions: - assert torch.all(combined[s, b] == 0.01) + assert torch.all(combined[b, s] == 0.01) def test_multiple_modalities(self): """Test alignment with multiple modalities with special tokens at different positions.""" @@ -215,27 +215,27 @@ def test_multiple_modalities(self): ) # Check output shape - assert combined.shape == (seq_length, batch_size, hidden_dim) + assert combined.shape == (batch_size, seq_length, hidden_dim) # Check that special token positions have the correct markers and only at correct positions # Batch 0 markers - assert torch.isclose(combined[1, 0, 0], torch.tensor(10.0, device=self.device)) # Vision - assert torch.isclose(combined[4, 0, 1], torch.tensor(30.0, device=self.device)) # Audio - assert torch.isclose(combined[8, 0, 2], torch.tensor(50.0, device=self.device)) # Video + assert torch.isclose(combined[0, 1, 0], torch.tensor(10.0, device=self.device)) # Vision + assert torch.isclose(combined[0, 4, 1], torch.tensor(30.0, device=self.device)) # Audio + assert torch.isclose(combined[0, 8, 2], torch.tensor(50.0, device=self.device)) # Video # Batch 1 markers - assert torch.isclose(combined[2, 1, 0], torch.tensor(20.0, device=self.device)) # Vision - assert torch.isclose(combined[5, 1, 1], torch.tensor(40.0, device=self.device)) # Audio - assert torch.isclose(combined[7, 1, 2], torch.tensor(60.0, device=self.device)) # Video + assert torch.isclose(combined[1, 2, 0], torch.tensor(20.0, device=self.device)) # Vision + assert torch.isclose(combined[1, 5, 1], torch.tensor(40.0, device=self.device)) # Audio + assert torch.isclose(combined[1, 7, 2], torch.tensor(60.0, device=self.device)) # Video # Also check that markers are ONLY at their specific positions # For vision in batch 0 (position 1, value at index 0) - assert torch.all(combined[1, 0, 1:] == 0.0), "Non-zero values found after marker" + assert torch.all(combined[0, 1, 1:] == 0.0), "Non-zero values found after marker" # For audio in batch 1 (position 5, value at index 1) - assert torch.all(combined[5, 1, :1] == 0.0), "Non-zero values found before marker" - assert torch.all(combined[5, 1, 2:] == 0.0), "Non-zero values found after marker" + assert torch.all(combined[1, 5, :1] == 0.0), "Non-zero values found before marker" + assert torch.all(combined[1, 5, 2:] == 0.0), "Non-zero values found after marker" def test_multiple_images_with_variable_length(self): """Test handling multiple images per sample with variable sequence lengths. @@ -322,31 +322,31 @@ def test_multiple_images_with_variable_length(self): ) # Check output shape - assert combined.shape == (seq_length, batch_size, hidden_dim) + assert combined.shape == (batch_size, seq_length, hidden_dim) # Verify vision token embeddings are placed correctly # Batch 0, first image embeddings (3 patches) - assert torch.isclose(combined[1, 0, 0], torch.tensor(101.0, device=self.device)) - assert torch.isclose(combined[2, 0, 1], torch.tensor(102.0, device=self.device)) - assert torch.isclose(combined[3, 0, 2], torch.tensor(103.0, device=self.device)) + assert torch.isclose(combined[0, 1, 0], torch.tensor(101.0, device=self.device)) + assert torch.isclose(combined[0, 2, 1], torch.tensor(102.0, device=self.device)) + assert torch.isclose(combined[0, 3, 2], torch.tensor(103.0, device=self.device)) # Batch 0, second image embeddings (2 patches) - assert torch.isclose(combined[5, 0, 3], torch.tensor(104.0, device=self.device)) - assert torch.isclose(combined[6, 0, 4], torch.tensor(105.0, device=self.device)) + assert torch.isclose(combined[0, 5, 3], torch.tensor(104.0, device=self.device)) + assert torch.isclose(combined[0, 6, 4], torch.tensor(105.0, device=self.device)) # Batch 1, image embeddings (4 patches) - assert torch.isclose(combined[2, 1, 5], torch.tensor(201.0, device=self.device)) - assert torch.isclose(combined[3, 1, 6], torch.tensor(202.0, device=self.device)) - assert torch.isclose(combined[4, 1, 7], torch.tensor(203.0, device=self.device)) - assert torch.isclose(combined[5, 1, 8], torch.tensor(204.0, device=self.device)) + assert torch.isclose(combined[1, 2, 5], torch.tensor(201.0, device=self.device)) + assert torch.isclose(combined[1, 3, 6], torch.tensor(202.0, device=self.device)) + assert torch.isclose(combined[1, 4, 7], torch.tensor(203.0, device=self.device)) + assert torch.isclose(combined[1, 5, 8], torch.tensor(204.0, device=self.device)) # Verify that each embedding only has one non-zero value for b in range(batch_size): # Check positions with special tokens positions = [(1, 2, 3, 5, 6), (2, 3, 4, 5)][b] for s in positions: - emb = combined[s, b].clone() + emb = combined[b, s].clone() # Find the non-zero position nonzero_indices = torch.nonzero(emb) # Make sure we actually have non-zero values diff --git a/tests/unit_tests/models/test_mimo_model.py b/tests/unit_tests/models/test_mimo_model.py index 0ef62ff570f..487704fa183 100644 --- a/tests/unit_tests/models/test_mimo_model.py +++ b/tests/unit_tests/models/test_mimo_model.py @@ -220,6 +220,18 @@ def test_get_text_embeddings(self): ) assert text_embeddings.shape == (self.batch_size * self.seq_len, self.hidden_size) + def test_get_text_embeddings_rejects_embedding_sp_scatter(self): + """MIMO owns SP scatter after multimodal alignment.""" + mimo_model = self._make_avlm() + mimo_model.partition_adapter = MagicMock() + mimo_model.partition_adapter.cfg.seq_parallel = True + mimo_model.language_model.embedding.scatter_to_sequence_parallel = True + + with pytest.raises(RuntimeError, match="embedding scatter"): + mimo_model.get_text_embeddings( + self._make_input_ids(), self._make_position_ids(), self.special_token_ids + ) + def test_forward_text_only(self): """Test forward pass with only text input.""" mimo_model = self._make_vlm() @@ -357,7 +369,7 @@ def test_forward_with_packing_kwargs(self): text_emb = torch.zeros(self.batch_size * self.seq_len, self.hidden_size, device=self.device) combined_emb = torch.zeros( - self.seq_len, self.batch_size, self.hidden_size, device=self.device + self.batch_size, self.seq_len, self.hidden_size, device=self.device ) captured = {} @@ -387,22 +399,22 @@ def capture_lm_forward(*args, **kwargs): assert packed_seq_params.cu_seqlens_kv.dtype == torch.int32 def test_forward_with_partition_adapter(self): - """Test that partition_adapter.shard() is called and embeddings are transposed correctly.""" + """Test that partition_adapter.shard() receives batch-first embeddings.""" mimo_model = self._make_vlm() input_ids = self._make_input_ids() position_ids = self._make_position_ids() sharded_seq_len = self.seq_len // 2 sharded_emb = torch.zeros( - self.batch_size, sharded_seq_len, self.hidden_size, device=self.device + sharded_seq_len, self.batch_size, self.hidden_size, device=self.device ) mock_adapter = MagicMock() - mock_adapter.shard.return_value = (sharded_emb, None, None, None, None) + mock_adapter.shard.return_value = (sharded_emb, None, None, None) mimo_model.partition_adapter = mock_adapter text_emb = torch.zeros(self.batch_size * self.seq_len, self.hidden_size, device=self.device) combined_emb = torch.zeros( - self.seq_len, self.batch_size, self.hidden_size, device=self.device + self.batch_size, self.seq_len, self.hidden_size, device=self.device ) captured = {} diff --git a/tests/unit_tests/models/test_mimo_partition.py b/tests/unit_tests/models/test_mimo_partition.py index 23fb2ef7ed3..34d5eccabf1 100644 --- a/tests/unit_tests/models/test_mimo_partition.py +++ b/tests/unit_tests/models/test_mimo_partition.py @@ -154,61 +154,67 @@ def _make_tensors(self, B=2, S=8, H=16): embeddings = torch.rand(B, S, H) labels = torch.randint(0, 100, (B, S)) loss_mask = torch.ones(B, S) - attention_mask = torch.ones(B, S) - return embeddings, labels, loss_mask, attention_mask + return embeddings, labels, loss_mask - def test_noop_when_both_disabled(self): - """No sharding when neither CP nor SP is enabled — inputs returned as-is.""" + def test_lm_layout_when_both_disabled(self): + """Even without CP/SP, shard() returns embeddings in language-model layout.""" cfg = self._make_cfg(use_cp=False, seq_parallel=False) adapter = PartitionAdapter(cfg) - embeddings, labels, loss_mask, attention_mask = self._make_tensors() - out = adapter.shard(embeddings, labels, loss_mask, attention_mask) - assert out[0] is embeddings + embeddings, labels, loss_mask = self._make_tensors() + out = adapter.shard(embeddings, labels, loss_mask) + torch.testing.assert_close(out[0], embeddings.transpose(0, 1).contiguous()) assert out[1] is labels assert out[2] is loss_mask - assert out[3] is attention_mask - assert out[4] is None + assert out[3] is None def test_cp_only_shards_sequence(self): mock_cp_group = MagicMock() cfg = self._make_cfg(use_cp=True, max_seq_len=8, cp_group=mock_cp_group) adapter = PartitionAdapter(cfg) - embeddings, labels, loss_mask, attention_mask = self._make_tensors(B=2, S=8, H=16) + embeddings, labels, loss_mask = self._make_tensors(B=2, S=8, H=16) sharded = { 'embeddings': embeddings[:, :4, :], 'labels': labels[:, :4], 'loss_mask': loss_mask[:, :4], - 'attention_mask': attention_mask[:, :4], } with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), patch( 'megatron.core.models.mimo.partition.utils.get_batch_on_this_cp_rank', return_value=sharded, - ), + ) as mock_cp_shard, ): - out = adapter.shard(embeddings, labels, loss_mask, attention_mask) - assert out[0].shape == (2, 4, 16) + out = adapter.shard(embeddings, labels, loss_mask) + mock_cp_shard.assert_called_once_with( + {'embeddings': embeddings, 'labels': labels, 'loss_mask': loss_mask}, + cp_group=mock_cp_group, + ) + assert out[0].shape == (4, 2, 16) assert out[1].shape == (2, 4) def test_sp_only_scatters(self): mock_tp_group = MagicMock() cfg = self._make_cfg(seq_parallel=True, max_seq_len=8, tp_group=mock_tp_group) adapter = PartitionAdapter(cfg) - embeddings = torch.rand(2, 8, 16) - labels = torch.randint(0, 100, (2, 8)) - loss_mask = torch.ones(2, 8) - attention_mask = torch.ones(2, 8) - scattered = torch.rand(4, 2, 16) + embeddings = torch.rand(1, 8, 16) + labels = torch.randint(0, 100, (1, 8)) + loss_mask = torch.ones(1, 8) + scattered = torch.rand(4, 1, 16) + + def scatter(input_, group=None): + assert input_.shape == (8, 1, 16) + assert group is mock_tp_group + return scattered + with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), patch( 'megatron.core.models.mimo.partition.utils.tensor_parallel.scatter_to_sequence_parallel_region', - return_value=scattered, + side_effect=scatter, ), ): - out = adapter.shard(embeddings, labels, loss_mask, attention_mask) - assert out[0].shape == (2, 4, 16) + out = adapter.shard(embeddings, labels, loss_mask) + assert out[0] is scattered assert out[1] is labels assert out[2] is loss_mask @@ -218,13 +224,12 @@ def test_sp_only_leaves_labels_and_loss_mask_without_embeddings(self): adapter = PartitionAdapter(cfg) labels = torch.arange(16).view(2, 8) loss_mask = torch.arange(16, dtype=torch.float32).view(2, 8) - attention_mask = torch.ones(2, 8) with patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2): - out = adapter.shard(None, labels, loss_mask, attention_mask) + out = adapter.shard(None, labels, loss_mask) assert out[0] is None assert out[1] is labels assert out[2] is loss_mask - assert out[3] is attention_mask + assert out[3] is None def test_cp_and_sp_combined(self): mock_cp_group = MagicMock() @@ -241,12 +246,10 @@ def test_cp_and_sp_combined(self): embeddings = torch.rand(2, 16, 16) labels = torch.randint(0, 100, (2, 16)) loss_mask = torch.ones(2, 16) - attention_mask = torch.ones(2, 16) cp_sharded = { 'embeddings': embeddings[:, :8, :], 'labels': labels[:, :8], 'loss_mask': loss_mask[:, :8], - 'attention_mask': attention_mask[:, :8], } scattered = torch.rand(4, 2, 16) @@ -255,14 +258,18 @@ def test_cp_and_sp_combined(self): patch( 'megatron.core.models.mimo.partition.utils.get_batch_on_this_cp_rank', return_value=cp_sharded, - ), + ) as mock_cp_shard, patch( 'megatron.core.models.mimo.partition.utils.tensor_parallel.scatter_to_sequence_parallel_region', return_value=scattered, ), ): - out = adapter.shard(embeddings, labels, loss_mask, attention_mask) - assert out[0].shape == (2, 4, 16) + out = adapter.shard(embeddings, labels, loss_mask) + mock_cp_shard.assert_called_once_with( + {'embeddings': embeddings, 'labels': labels, 'loss_mask': loss_mask}, + cp_group=mock_cp_group, + ) + assert out[0].shape == (4, 2, 16) torch.testing.assert_close(out[1], labels[:, :8]) torch.testing.assert_close(out[2], loss_mask[:, :8]) @@ -273,12 +280,11 @@ def test_seq_not_divisible_raises(self): embeddings = torch.rand(2, 7, 16) # 7 % (2*2) != 0 labels = torch.randint(0, 100, (2, 7)) loss_mask = torch.ones(2, 7) - attention_mask = torch.ones(2, 7) with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), pytest.raises(AssertionError, match="divisible"), ): - adapter.shard(embeddings, labels, loss_mask, attention_mask) + adapter.shard(embeddings, labels, loss_mask) def test_tp_comm_overlap_seq_len_assertion(self): mock_tp_group = MagicMock() @@ -290,12 +296,11 @@ def test_tp_comm_overlap_seq_len_assertion(self): embeddings = torch.rand(2, 8, 16) labels = torch.randint(0, 100, (2, 8)) loss_mask = torch.ones(2, 8) - attention_mask = torch.ones(2, 8) with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), pytest.raises(AssertionError, match="TP Comm overlap"), ): - adapter.shard(embeddings, labels, loss_mask, attention_mask) + adapter.shard(embeddings, labels, loss_mask) def test_thd_format_skips_divisibility_check(self): """PackedSeqParams with qkv_format='thd' bypasses the divisibility assertion.""" @@ -307,7 +312,6 @@ def test_thd_format_skips_divisibility_check(self): embeddings = torch.rand(2, 7, 16) # seq_len=7 not divisible by cp*2, but THD skips check labels = torch.randint(0, 100, (2, 7)) loss_mask = torch.ones(2, 7) - attention_mask = torch.ones(2, 7) packed_seq_params = MagicMock(spec=PackedSeqParams) packed_seq_params.qkv_format = 'thd' packed_seq_params.cu_seqlens_q_padded = torch.tensor([0, 4, 7], dtype=torch.int32) @@ -321,7 +325,7 @@ def test_thd_format_skips_divisibility_check(self): ): mock_tex.thd_get_partitioned_indices.return_value = fake_index # Should NOT raise AssertionError about divisibility - out = adapter.shard(embeddings, labels, loss_mask, attention_mask, packed_seq_params) + out = adapter.shard(embeddings, labels, loss_mask, packed_seq_params) assert out[0] is not None def test_none_embeddings_skips_shard_factor_check(self): @@ -331,12 +335,7 @@ def test_none_embeddings_skips_shard_factor_check(self): adapter = PartitionAdapter(cfg) labels = torch.randint(0, 100, (2, 7)) loss_mask = torch.ones(2, 7) - attention_mask = torch.ones(2, 7) - cp_sharded = { - 'labels': labels[:, :4], - 'loss_mask': loss_mask[:, :4], - 'attention_mask': attention_mask[:, :4], - } + cp_sharded = {'labels': labels[:, :4], 'loss_mask': loss_mask[:, :4]} with ( patch('megatron.core.models.mimo.partition.utils.get_pg_size', return_value=2), patch( @@ -344,7 +343,7 @@ def test_none_embeddings_skips_shard_factor_check(self): return_value=cp_sharded, ), ): - out = adapter.shard(None, labels, loss_mask, attention_mask) + out = adapter.shard(None, labels, loss_mask) assert out[0] is None @@ -367,12 +366,11 @@ def test_returns_unchanged_when_cp_disabled(self): embeddings = torch.rand(2, 8, 16) labels = torch.randint(0, 100, (2, 8)) loss_mask = torch.ones(2, 8) - attention_mask = torch.ones(2, 8) - out = adapter._apply_context_parallel(embeddings, labels, loss_mask, attention_mask, None) + out = adapter._apply_context_parallel(embeddings, labels, loss_mask, None) assert out[0] is embeddings assert out[1] is labels assert out[2] is loss_mask - assert out[3] is attention_mask + assert out[3] is None def test_sbhd_path_calls_get_batch_on_this_cp_rank(self): mock_cp_group = MagicMock() @@ -381,21 +379,20 @@ def test_sbhd_path_calls_get_batch_on_this_cp_rank(self): embeddings = torch.rand(2, 8, 16) labels = torch.randint(0, 100, (2, 8)) loss_mask = torch.ones(2, 8) - attention_mask = torch.ones(2, 8) sharded = { 'embeddings': embeddings[:, :4, :], 'labels': labels[:, :4], 'loss_mask': loss_mask[:, :4], - 'attention_mask': attention_mask[:, :4], } with patch( 'megatron.core.models.mimo.partition.utils.get_batch_on_this_cp_rank', return_value=sharded, ) as mock_fn: - out = adapter._apply_context_parallel( - embeddings, labels, loss_mask, attention_mask, None + out = adapter._apply_context_parallel(embeddings, labels, loss_mask, None) + mock_fn.assert_called_once_with( + {'embeddings': embeddings, 'labels': labels, 'loss_mask': loss_mask}, + cp_group=mock_cp_group, ) - mock_fn.assert_called_once() assert out[0].shape == (2, 4, 16) assert out[1].shape == (2, 4) @@ -406,8 +403,8 @@ def test_all_none_inputs_produces_none_outputs(self): with patch( 'megatron.core.models.mimo.partition.utils.get_batch_on_this_cp_rank', return_value={} ): - out = adapter._apply_context_parallel(None, None, None, None, None) - assert all(v is None for v in out[:4]) + out = adapter._apply_context_parallel(None, None, None, None) + assert all(v is None for v in out) def test_only_non_none_tensors_added_to_batch(self): """None tensors must not appear in the batch dict passed to get_batch_on_this_cp_rank.""" @@ -418,7 +415,8 @@ def test_only_non_none_tensors_added_to_batch(self): sharded = {'embeddings': embeddings[:, :4, :]} captured = {} - def mock_fn(batch): + def mock_fn(batch, cp_group=None): + assert cp_group is mock_cp_group captured.update(batch) return sharded @@ -426,7 +424,7 @@ def mock_fn(batch): 'megatron.core.models.mimo.partition.utils.get_batch_on_this_cp_rank', side_effect=mock_fn, ): - out = adapter._apply_context_parallel(embeddings, None, None, None, None) + out = adapter._apply_context_parallel(embeddings, None, None, None) assert 'embeddings' in captured assert 'labels' not in captured @@ -448,4 +446,4 @@ def test_thd_path_raises_when_te_unavailable(self): patch('megatron.core.models.mimo.partition.utils._HAVE_TEX', False), pytest.raises(AssertionError, match="Transformer Engine"), ): - adapter._apply_context_parallel(embeddings, None, None, None, packed_seq_params) + adapter._apply_context_parallel(embeddings, None, None, packed_seq_params) From 3d6da4a5fde0ec58bb3b89ce1f895b8a664ccd89 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Tue, 12 May 2026 04:56:51 +0000 Subject: [PATCH 26/44] NMFW-464 address MIMO base review comments --- .../mimo/data/energon_multimodal_provider.py | 7 +- examples/mimo/data/hetero_energon.py | 33 ++--- megatron/core/models/mimo/model/base.py | 140 +++++++++--------- tests/unit_tests/models/test_mimo_model.py | 17 ++- 4 files changed, 98 insertions(+), 99 deletions(-) diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py index 653e0017ecd..19a2a9f0c8d 100644 --- a/examples/mimo/data/energon_multimodal_provider.py +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -234,13 +234,14 @@ def _build_packing_kwargs(sample: PackedSample, max_len: int) -> dict[str, torch ) max_seqlen = segment_lens.max() return { + "qkv_format": "thd", "cu_seqlens_q": cu_seqlens, "cu_seqlens_kv": cu_seqlens, "cu_seqlens_q_padded": cu_seqlens, "cu_seqlens_kv_padded": cu_seqlens, - "max_seqlen_q": max_seqlen, - "max_seqlen_kv": max_seqlen, - "total_tokens": torch.tensor(max_len, dtype=torch.int32), + "max_seqlen_q": int(max_seqlen.item()), + "max_seqlen_kv": int(max_seqlen.item()), + "total_tokens": int(max_len), } diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 5a2fb75e32d..76e9ae4ebdd 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -51,9 +51,7 @@ def validate_energon_data_alignment(data_iterator, _topology) -> None: if candidate is None: continue target = ( - encoder_signatures_by_lane - if candidate["role"] == "encoder" - else llm_signatures_by_lane + encoder_signatures_by_lane if candidate["role"] == "encoder" else llm_signatures_by_lane ) for lane, signature in zip(candidate["llm_lanes"], candidate["signatures"]): target.setdefault(lane, set()).add(signature) @@ -63,10 +61,7 @@ def validate_energon_data_alignment(data_iterator, _topology) -> None: encoder_values = encoder_signatures_by_lane.get(lane, set()) llm_values = llm_signatures_by_lane.get(lane, set()) if len(encoder_values) != 1 or len(llm_values) != 1 or encoder_values != llm_values: - mismatched[lane] = { - "encoder": sorted(encoder_values), - "llm": sorted(llm_values), - } + mismatched[lane] = {"encoder": sorted(encoder_values), "llm": sorted(llm_values)} if mismatched: raise RuntimeError(f"hetero Energon data loaders diverged across grids: {mismatched}") @@ -77,20 +72,12 @@ def _build_llm_iterator(args, grid): if get_grid_coordinate(grid, "tp") != 0: lane = get_grid_coordinate(grid, "dp") return EnergonIterator( - None, - tp_group=tp_group, - source_rank=False, - alignment_role="llm", - llm_lanes=[lane], + None, tp_group=tp_group, source_rank=False, alignment_role="llm", llm_lanes=[lane] ) lane = get_grid_coordinate(grid, "dp") return _build_single_lane_iterator( - args, - tp_group=tp_group, - lane=lane, - role="llm", - random_seed=args.seed + lane, + args, tp_group=tp_group, lane=lane, role="llm", random_seed=args.seed + lane ) @@ -119,11 +106,7 @@ def _build_encoder_iterator(args, grid): lane_iterators = [ _build_single_lane_iterator( - args, - tp_group=None, - lane=lane, - role="encoder-component", - random_seed=args.seed + lane, + args, tp_group=None, lane=lane, role="encoder-component", random_seed=args.seed + lane ) for lane in llm_lanes ] @@ -200,7 +183,9 @@ def _combine_encoder_batches(batches: list[dict]) -> dict: combined[key] = torch.cat(values, dim=0) modality_values = [ - batch.get("modality_inputs") for batch in batches if batch.get("modality_inputs") is not None + batch.get("modality_inputs") + for batch in batches + if batch.get("modality_inputs") is not None ] if modality_values: combined["modality_inputs"] = _concat_nested_tensors(modality_values) @@ -403,6 +388,8 @@ def _checksum_packing_kwargs(cls, packing_kwargs: Optional[dict]) -> int: value_checksum = cls._checksum_tensor(value) elif value is None: value_checksum = 0 + elif isinstance(value, str): + value_checksum = sum(value.encode("utf-8")) else: value_checksum = int(value) checksum = (checksum * 131 + value_checksum) % 2_147_483_647 diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index 23c91c66da4..ce0c0d6de98 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -438,7 +438,7 @@ def _forward_encoders( raise RuntimeError( f"{encoder_name} inputs are missing, but matching special tokens exist" ) - output = self._empty_modality_output(submodule) + output = self._empty_modality_output(submodule, input_ids) if output is not None: self._attach_modality_split_sizes(output, input_ids, encoder_name) @@ -450,17 +450,10 @@ def _attach_modality_split_sizes( self, output: torch.Tensor, input_ids: Optional[torch.Tensor], encoder_name: str ) -> None: """Annotate flat modality outputs with per-sample split sizes for bridge fan-out.""" - if ( - not isinstance(output, torch.Tensor) - or output.ndim != 2 - or input_ids is None - or input_ids.ndim != 2 - or encoder_name not in self.special_token_ids - or input_ids.size(0) <= 1 - ): + token_id = self.special_token_ids.get(encoder_name) + if token_id is None or input_ids is None or output.ndim != 2 or input_ids.size(0) <= 1: return - token_id = self.special_token_ids[encoder_name] split_sizes = (input_ids == token_id).sum(dim=1).to(torch.long).tolist() if sum(split_sizes) == output.size(0): output._mimo_bridge_split_sizes = split_sizes @@ -471,24 +464,23 @@ def _has_encoder_tokens(self, input_ids: Optional[torch.Tensor], encoder_name: s return False return bool((input_ids == self.special_token_ids[encoder_name]).any().item()) - @staticmethod - def _empty_modality_output(submodule: torch.nn.Module) -> torch.Tensor: + def _empty_modality_output( + self, submodule: torch.nn.Module, input_ids: Optional[torch.Tensor] + ) -> torch.Tensor: """Return an empty projected activation for text-only non-colocated batches.""" - unwrapped = unwrap_model(submodule) - projections = getattr(unwrapped, "input_projections", None) - if not projections: - raise RuntimeError("cannot build empty modality output without input projections") - - projection = projections[0] - hidden_size = getattr(getattr(projection, "config", None), "hidden_size", None) - if hidden_size is None: - hidden_size = getattr(projection, "out_features", None) - if hidden_size is None: - raise RuntimeError("cannot infer hidden size for empty modality output") - - param = next(projection.parameters(), None) - device = param.device if param is not None else torch.device("cuda") - dtype = param.dtype if param is not None else torch.float32 + hidden_size = self.config.hidden_size + param = next(submodule.parameters(), None) + if param is not None: + device = param.device + elif input_ids is not None: + device = input_ids.device + else: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + dtype = ( + param.dtype + if param is not None + else getattr(self.config, 'params_dtype', None) or torch.float32 + ) return torch.empty((0, hidden_size), device=device, dtype=dtype, requires_grad=True) def _forward_language_module( @@ -539,18 +531,15 @@ def _forward_language_module( special_token_ids=self.special_token_ids, ) - if self.partition_adapter is not None: - shard_loss_inputs = self.role.is_last_stage(lang_name) - combined_embeddings, labels, loss_mask, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=combined_embeddings, - labels=labels if shard_loss_inputs else None, - loss_mask=loss_mask if shard_loss_inputs else None, - packed_seq_params=packed_seq_params, - ) + combined_embeddings, labels, loss_mask, packed_seq_params = ( + self._prepare_language_inputs( + embeddings=combined_embeddings, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + shard_loss_inputs=self.role.is_last_stage(lang_name), ) - else: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + ) lm_output = self.language_model( input_ids=None, @@ -564,14 +553,13 @@ def _forward_language_module( # Non-first stage: receive hidden states from previous LM stage hidden_states = input_tensors.get(lang_name) if input_tensors else None - if self.partition_adapter is not None: - shard_loss_inputs = self.role.is_last_stage(lang_name) - _, labels, loss_mask, packed_seq_params = self.partition_adapter.shard( - embeddings=None, - labels=labels if shard_loss_inputs else None, - loss_mask=loss_mask if shard_loss_inputs else None, - packed_seq_params=packed_seq_params, - ) + _, labels, loss_mask, packed_seq_params = self._prepare_language_inputs( + embeddings=None, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + shard_loss_inputs=self.role.is_last_stage(lang_name), + ) # Set input tensor on language model for PP (unwrap DDP to reach GPTModel) if hidden_states is not None: @@ -596,22 +584,42 @@ def _forward_language_module( @staticmethod def _build_packed_seq_params(packing_kwargs: Optional[dict]) -> Optional[PackedSeqParams]: - """Build packed-sequence params from dataloader kwargs.""" + """Build packed-sequence params from dataloader-provided metadata.""" if packing_kwargs is None: return None - converted_kwargs = {} - for key, value in packing_kwargs.items(): - if 'cu_seqlens' in key and value is not None: - converted_kwargs[key] = value.to(dtype=torch.int32) - elif key == 'total_tokens' and isinstance(value, torch.Tensor): - converted_kwargs[key] = int(value.item()) - else: - converted_kwargs[key] = value - packed_seq_params = PackedSeqParams(**converted_kwargs) - packed_seq_params.qkv_format = 'thd' + if isinstance(packing_kwargs, PackedSeqParams): + return packing_kwargs + packed_seq_params = PackedSeqParams(**packing_kwargs) logger.debug(f"Packed sequence parameters: {packed_seq_params}") return packed_seq_params + def _prepare_language_inputs( + self, + embeddings: Optional[torch.Tensor], + labels: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], + packed_seq_params: Optional[PackedSeqParams], + *, + shard_loss_inputs: bool, + ) -> Tuple[ + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[PackedSeqParams], + ]: + """Return LM-layout embeddings and matching loss tensors.""" + if self.partition_adapter is None: + if embeddings is not None: + embeddings = embeddings.transpose(0, 1).contiguous() + return embeddings, labels, loss_mask, packed_seq_params + + return self.partition_adapter.shard( + embeddings=embeddings, + labels=labels if shard_loss_inputs else None, + loss_mask=loss_mask if shard_loss_inputs else None, + packed_seq_params=packed_seq_params, + ) + def _build_colocated_communicators(self): grid_map = self.mimo_config.module_to_grid_map if any( @@ -706,17 +714,13 @@ def _forward_all_modules( logger.debug(f"Combined embeddings shape: {combined_embeddings.shape}") # 3. If sharding is needed, apply PartitionAdapter. - if self.partition_adapter is not None: - combined_embeddings, labels, loss_mask, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=combined_embeddings, - labels=labels, - loss_mask=loss_mask, - packed_seq_params=packed_seq_params, - ) - ) - else: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + combined_embeddings, labels, loss_mask, packed_seq_params = self._prepare_language_inputs( + embeddings=combined_embeddings, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + shard_loss_inputs=True, + ) # 5. Forward pass through language model lm_output = self.language_model( diff --git a/tests/unit_tests/models/test_mimo_model.py b/tests/unit_tests/models/test_mimo_model.py index 487704fa183..b92c688295a 100644 --- a/tests/unit_tests/models/test_mimo_model.py +++ b/tests/unit_tests/models/test_mimo_model.py @@ -355,7 +355,7 @@ def test_partition_adapter_none_by_default(self): assert mimo_model.partition_adapter is None def test_forward_with_packing_kwargs(self): - """Test that packing_kwargs builds PackedSeqParams with qkv_format='thd' and int32 seqlens.""" + """Test that dataloader-provided packing metadata reaches the language model.""" from megatron.core.packed_seq_params import PackedSeqParams mimo_model = self._make_vlm() @@ -363,9 +363,16 @@ def test_forward_with_packing_kwargs(self): position_ids = self._make_position_ids() cu_seqlens = torch.tensor( - [0, self.seq_len, 2 * self.seq_len], dtype=torch.int64, device=self.device + [0, self.seq_len, 2 * self.seq_len], dtype=torch.int32, device=self.device ) - packing_kwargs = {"cu_seqlens_q": cu_seqlens.clone(), "cu_seqlens_kv": cu_seqlens.clone()} + packing_kwargs = { + "qkv_format": "thd", + "cu_seqlens_q": cu_seqlens, + "cu_seqlens_kv": cu_seqlens, + "max_seqlen_q": self.seq_len, + "max_seqlen_kv": self.seq_len, + "total_tokens": 2 * self.seq_len, + } text_emb = torch.zeros(self.batch_size * self.seq_len, self.hidden_size, device=self.device) combined_emb = torch.zeros( @@ -395,8 +402,8 @@ def capture_lm_forward(*args, **kwargs): packed_seq_params = captured['packed_seq_params'] assert isinstance(packed_seq_params, PackedSeqParams) assert packed_seq_params.qkv_format == 'thd' - assert packed_seq_params.cu_seqlens_q.dtype == torch.int32 - assert packed_seq_params.cu_seqlens_kv.dtype == torch.int32 + assert packed_seq_params.cu_seqlens_q is cu_seqlens + assert packed_seq_params.cu_seqlens_kv is cu_seqlens def test_forward_with_partition_adapter(self): """Test that partition_adapter.shard() receives batch-first embeddings.""" From ad7a5175c5d3cf8c8a90b0e1b6839cadd1dfe6bd Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Tue, 12 May 2026 05:10:46 +0000 Subject: [PATCH 27/44] NMFW-464 clarify text-only encoder bridge payload --- megatron/core/models/mimo/model/base.py | 31 ++++++++++--------- .../models/test_mimo_bridge_split_sizes.py | 15 +++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index ce0c0d6de98..bdd9022d07e 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -438,7 +438,7 @@ def _forward_encoders( raise RuntimeError( f"{encoder_name} inputs are missing, but matching special tokens exist" ) - output = self._empty_modality_output(submodule, input_ids) + output = self._empty_encoder_output(submodule, input_ids) if output is not None: self._attach_modality_split_sizes(output, input_ids, encoder_name) @@ -464,24 +464,25 @@ def _has_encoder_tokens(self, input_ids: Optional[torch.Tensor], encoder_name: s return False return bool((input_ids == self.special_token_ids[encoder_name]).any().item()) - def _empty_modality_output( + def _empty_encoder_output( self, submodule: torch.nn.Module, input_ids: Optional[torch.Tensor] ) -> torch.Tensor: - """Return an empty projected activation for text-only non-colocated batches.""" - hidden_size = self.config.hidden_size + """Return the bridge payload for text-only non-colocated batches.""" param = next(submodule.parameters(), None) - if param is not None: - device = param.device - elif input_ids is not None: - device = input_ids.device - else: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - dtype = ( - param.dtype - if param is not None - else getattr(self.config, 'params_dtype', None) or torch.float32 + reference = param if param is not None else input_ids + device = ( + reference.device + if reference is not None + else torch.device("cuda" if torch.cuda.is_available() else "cpu") + ) + dtype = param.dtype if param is not None else self.config.params_dtype or torch.float32 + + # The bridge schedule communicates every module edge each microbatch. + # For a text-only batch, send shape [0, H] so the LLM receives no + # modality embeddings without changing the communication schedule. + return torch.empty( + (0, self.config.hidden_size), device=device, dtype=dtype, requires_grad=True ) - return torch.empty((0, hidden_size), device=device, dtype=dtype, requires_grad=True) def _forward_language_module( self, diff --git a/tests/unit_tests/models/test_mimo_bridge_split_sizes.py b/tests/unit_tests/models/test_mimo_bridge_split_sizes.py index efcdceaa00e..b515877dbf0 100644 --- a/tests/unit_tests/models/test_mimo_bridge_split_sizes.py +++ b/tests/unit_tests/models/test_mimo_bridge_split_sizes.py @@ -3,6 +3,7 @@ import torch from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator def test_attach_modality_split_sizes_includes_zero_image_lanes(): @@ -16,3 +17,17 @@ def test_attach_modality_split_sizes_includes_zero_image_lanes(): model._attach_modality_split_sizes(output, input_ids, "images") assert output._mimo_bridge_split_sizes == [1, 0, 2] + + +def test_bridge_split_sizes_allow_text_only_encoder_output(): + """The bridge can split a text-only encoder payload with no modality tokens.""" + bridge = BridgeCommunicator.__new__(BridgeCommunicator) + bridge.tensor_ndim = 2 + bridge.dim_mapping = {'s': 0, 'h': 1, 'b': 0} + + output = torch.empty((0, 4)) + output._mimo_bridge_split_sizes = [0, 0] + + splits = bridge._split_tensor_at_batch_dim(output, 2) + + assert [tuple(split.shape) for split in splits] == [(0, 4), (0, 4)] From e61d7ea678ef165ea133b77f96a4432f1d23b199 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Tue, 12 May 2026 19:23:58 +0000 Subject: [PATCH 28/44] Pass attention mask through MIMO language model --- megatron/core/models/mimo/model/base.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index bdd9022d07e..9bbaf61e9ce 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -339,9 +339,7 @@ def forward( Args: input_ids: Input token IDs. Shape: (B, S) position_ids: Position IDs. Shape: (B, S) - attention_mask: Accepted for API compatibility. This path currently relies on - the language model's causal/packed-sequence masking and does not forward - dataloader attention masks to the language model. + attention_mask: Attention mask. Shape: (B, S) loss_mask: Loss mask. Shape: (B, S) labels: Labels for training. Shape: (B, S) modality_inputs: Dictionary mapping modality names to encoder inputs. For example: @@ -499,7 +497,7 @@ def _forward_language_module( Args: input_ids: Token IDs position_ids: Position IDs - attention_mask: Accepted for API compatibility; not forwarded to the language model. + attention_mask: Attention mask loss_mask: Loss mask for per-token loss normalization labels: Labels for loss computation input_tensors: Hidden states or embeddings from previous stage @@ -547,7 +545,7 @@ def _forward_language_module( position_ids=None, decoder_input=combined_embeddings, labels=labels, - attention_mask=None, + attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) else: @@ -573,7 +571,7 @@ def _forward_language_module( position_ids=None, decoder_input=None, labels=labels, - attention_mask=None, + attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) @@ -729,7 +727,7 @@ def _forward_all_modules( position_ids=None, decoder_input=combined_embeddings, labels=labels, - attention_mask=None, + attention_mask=attention_mask, packed_seq_params=packed_seq_params, ) From 65026c74ee5a3ccdfbf0a675e350203d8a1492f9 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Wed, 13 May 2026 09:36:51 -0700 Subject: [PATCH 29/44] Add 54L Nemotron MoE VLM provider (#20) * Add 54L Nemotron MoE VLM provider * Address 54L provider review comments * Inline static Nemotron provider config * Hardcode Nemotron language architecture config --- .../mimo/model_providers/nemotron_moe_vlm.py | 49 +++++++++++++------ .../run_hetero_nemotron_20l_energon_train.sh | 12 +++++ examples/mimo/train_hetero.py | 4 ++ examples/mimo/training/hetero/args.py | 17 +++++-- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/examples/mimo/model_providers/nemotron_moe_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py index 981a09438c9..a09aff1228c 100644 --- a/examples/mimo/model_providers/nemotron_moe_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -49,7 +49,7 @@ MOCK_MODEL_PROVIDER = "mock" NEMOTRON_20L_MODEL_PROVIDER = "nemotron-moe-vlm-20l" -NEMOTRON_20L_HYBRID_PATTERN = "MEMEM*EMEMEM*EMEMEM*" +NEMOTRON_54L_MODEL_PROVIDER = "nemotron-moe-vlm-54l" NEMOTRON_20L_IMAGE_SEQ_PER_TILE = 256 NEMOTRON_20L_MAX_NUM_TILES = 12 NEMOTRON_20L_DEFAULT_STAGE = "stage2" @@ -62,12 +62,17 @@ def is_nemotron_20l(args: argparse.Namespace) -> bool: return args.model_provider == NEMOTRON_20L_MODEL_PROVIDER +def is_nemotron_moe_vlm(args: argparse.Namespace) -> bool: + """Return whether a Nemotron6-MoE VLM provider is active.""" + return args.model_provider in (NEMOTRON_20L_MODEL_PROVIDER, NEMOTRON_54L_MODEL_PROVIDER) + + def add_model_provider_args(parser: argparse.ArgumentParser) -> None: """Register model-provider arguments for hetero MIMO examples.""" provider = parser.add_argument_group("model provider") provider.add_argument( "--model-provider", - choices=[MOCK_MODEL_PROVIDER, NEMOTRON_20L_MODEL_PROVIDER], + choices=[MOCK_MODEL_PROVIDER, NEMOTRON_20L_MODEL_PROVIDER, NEMOTRON_54L_MODEL_PROVIDER], default=MOCK_MODEL_PROVIDER, ) provider.add_argument("--hidden-size", type=int, default=128) @@ -85,6 +90,11 @@ def add_model_provider_args(parser: argparse.ArgumentParser) -> None: provider.add_argument("--force-system-message", action="store_true") provider.add_argument("--num-moe-experts", type=int, default=4) provider.add_argument("--moe-router-topk", type=int, default=1) + provider.add_argument( + "--moe-router-force-load-balancing", + action="store_true", + help="Use random router logits to force MoE load balancing for benchmark/debug runs.", + ) provider.add_argument("--moe-grouped-gemm", action="store_true") provider.add_argument("--img-h", type=int, default=512) provider.add_argument("--img-w", type=int, default=512) @@ -115,20 +125,25 @@ def prepare_model_provider_args(args: argparse.Namespace) -> None: apply_training_stage(args) resolve_image_token_id(args) args.vision_encoder_key = get_encoder_module_name(args) - args.vision_input_mode = "pixels" if is_nemotron_20l(args) else "hidden_states" + args.vision_input_mode = "pixels" if is_nemotron_moe_vlm(args) else "hidden_states" def apply_model_provider_defaults(args: argparse.Namespace) -> None: - """Apply the exact Nemotron6-MoE VLM 20L model defaults.""" - if not is_nemotron_20l(args): + """Apply Nemotron6-MoE VLM model defaults.""" + if not is_nemotron_moe_vlm(args): return - args.num_layers = 20 + args.num_layers = 54 if args.model_provider == NEMOTRON_54L_MODEL_PROVIDER else 20 args.hidden_size = 2688 args.num_attention_heads = 32 args.num_moe_experts = 128 args.moe_router_topk = 6 args.moe_grouped_gemm = True + args.hybrid_layer_pattern = ( + "MEMEM*EMEM*EMEM*EMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEME" + if args.model_provider == NEMOTRON_54L_MODEL_PROVIDER + else "MEMEM*EMEMEM*EMEMEM*" + ) args.seq_length = 8192 args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles args.pixel_shuffle = True @@ -139,7 +154,7 @@ def apply_model_provider_defaults(args: argparse.Namespace) -> None: def apply_training_stage(args: argparse.Namespace) -> None: """Apply stage-specific freeze flags for the Nemotron VLM recipe.""" - if not is_nemotron_20l(args): + if not is_nemotron_moe_vlm(args): return stage = args.training_stage or NEMOTRON_20L_DEFAULT_STAGE @@ -155,7 +170,7 @@ def apply_training_stage(args: argparse.Namespace) -> None: def resolve_image_token_id(args: argparse.Namespace) -> None: """Resolve image, pad, and vocab ids from the configured tokenizer.""" - if not is_nemotron_20l(args) or not args.tokenizer_model: + if not is_nemotron_moe_vlm(args) or not args.tokenizer_model: return from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( @@ -269,7 +284,7 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): def get_encoder_module_name(args: argparse.Namespace) -> str: """Return the concrete encoder key for the active vision provider.""" - return NEMOTRON_VISION_ENCODER_KEY if is_nemotron_20l(args) else MOCK_VISION_ENCODER_KEY + return NEMOTRON_VISION_ENCODER_KEY if is_nemotron_moe_vlm(args) else MOCK_VISION_ENCODER_KEY def get_vision_encoder_module(args: argparse.Namespace, vision_submodule): @@ -307,15 +322,15 @@ def nemotron_projection_layer_spec() -> ModuleSpec: def nemotron_language_config( args: argparse.Namespace, tp_size: int, pp_size: int, ep_size: int, expt_tp_size: int ) -> TransformerConfig: - """Build the exact Nemotron6-MoE 20L language TransformerConfig.""" + """Build the Nemotron6-MoE language TransformerConfig.""" bf16 = not args.fp32 dtype = torch.bfloat16 if bf16 else torch.float32 config = TransformerConfig( - num_layers=20, + num_layers=54 if args.model_provider == NEMOTRON_54L_MODEL_PROVIDER else 20, hidden_size=2688, num_attention_heads=32, attention_backend=AttnBackend.flash, - num_query_groups=2, + num_query_groups=8, ffn_hidden_size=1856, kv_channels=128, activation_func=squared_relu, @@ -353,7 +368,9 @@ def nemotron_language_config( moe_router_enable_expert_bias=True, moe_router_dtype="fp32", moe_router_load_balancing_type="seq_aux_loss", - moe_aux_loss_coeff=0.0001, + moe_router_force_load_balancing=args.moe_router_force_load_balancing, + moe_router_fusion=True, + moe_aux_loss_coeff=1.0e-9, moe_shared_expert_intermediate_size=3712, moe_shared_expert_overlap=True, moe_token_dispatcher_type="alltoall", @@ -450,7 +467,7 @@ def language_model_spec( tp_size = get_group_size_or(tp_pg, fallback_tp_size) ep_size = get_group_size_or(ep_pg, args.llm_ep) expt_tp_size = get_group_size_or(expt_tp_pg, args.llm_expt_tp or fallback_tp_size) - if is_nemotron_20l(args): + if is_nemotron_moe_vlm(args): config = nemotron_language_config(args, tp_size, pp_size, ep_size, expt_tp_size) require_per_token_loss(config) return ModuleSpec( @@ -462,7 +479,7 @@ def language_model_spec( "max_sequence_length": args.seq_length, "pre_process": pp_rank == 0, "post_process": pp_rank == pp_size - 1, - "hybrid_override_pattern": NEMOTRON_20L_HYBRID_PATTERN, + "hybrid_layer_pattern": args.hybrid_layer_pattern, "position_embedding_type": "none", "scatter_embedding_sequence_parallel": False, "pg_collection": pg_collection, @@ -530,7 +547,7 @@ def vision_submodules_spec( pp_rank = get_group_rank_or(pp_pg) bf16 = not args.fp32 - if is_nemotron_20l(args): + if is_nemotron_moe_vlm(args): vision_config = radio_vision_config(args, tp_size, pp_size) vision_encoder_spec = ModuleSpec( module=RADIOEncoderWrapper, diff --git a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh index 6b5d4335a50..6f4050a966c 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh @@ -27,6 +27,8 @@ LLM_TP="${LLM_TP:-2}" LLM_PP="${LLM_PP:-1}" LLM_DP="${LLM_DP:-2}" LLM_EP="${LLM_EP:-4}" +ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" +MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-0}" ENCODER_SIZE=$((ENCODER_TP * ENCODER_PP * ENCODER_DP)) LLM_SIZE=$((LLM_TP * LLM_PP * LLM_DP)) LLM_OFFSET="${LLM_OFFSET:-${ENCODER_SIZE}}" @@ -61,6 +63,8 @@ fi echo "=== Hetero MIMO Nemotron6-MoE VLM 20L Energon training ===" echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} gbs=${GLOBAL_BATCH_SIZE}" echo "layout=encoder(tp=${ENCODER_TP},pp=${ENCODER_PP},dp=${ENCODER_DP}) llm(tp=${LLM_TP},pp=${LLM_PP},dp=${LLM_DP},ep=${LLM_EP}) world=${EXPECTED_WORLD_SIZE}" +echo "enable_experimental=${ENABLE_EXPERIMENTAL}" +echo "moe_router_force_load_balancing=${MOE_ROUTER_FORCE_LOAD_BALANCING}" echo "data=${DATA_PATH}" echo "tokenizer=${TOKENIZER_MODEL}" echo "===========================================================" @@ -73,6 +77,13 @@ DATA_LOADER_ARGS=( if [[ "${PACKING_BUFFER_SIZE}" != "0" ]]; then DATA_LOADER_ARGS+=(--packing-buffer-size "${PACKING_BUFFER_SIZE}") fi +MODEL_ARGS=() +if [[ "${ENABLE_EXPERIMENTAL}" == "1" || "${ENABLE_EXPERIMENTAL}" == "true" ]]; then + MODEL_ARGS+=(--enable-experimental) +fi +if [[ "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "1" || "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "true" ]]; then + MODEL_ARGS+=(--moe-router-force-load-balancing) +fi "${PYTHON_BIN}" -m torch.distributed.run \ --standalone \ @@ -91,6 +102,7 @@ fi --llm-ep "${LLM_EP}" \ --llm-expt-tp 1 \ --llm-expt-dp 1 \ + "${MODEL_ARGS[@]}" \ --vocab-size 131072 \ --max-num-tiles 12 \ --data-path "${DATA_PATH}" \ diff --git a/examples/mimo/train_hetero.py b/examples/mimo/train_hetero.py index 963f439ecb3..af46e3235ea 100644 --- a/examples/mimo/train_hetero.py +++ b/examples/mimo/train_hetero.py @@ -11,6 +11,8 @@ if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) +from megatron.core.config import set_experimental_flag + from examples.mimo.training.hetero.args import parse_args from examples.mimo.training.hetero.distributed import ( initialize_distributed, @@ -23,6 +25,8 @@ def main() -> None: """Program entrypoint.""" args = parse_args() + if args.enable_experimental: + set_experimental_flag(True) initialize_distributed() try: run_train_loop(args) diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 7249cfa3640..b68655908ce 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -8,6 +8,8 @@ from examples.mimo.data.hetero_mock import validate_mock_data_args from examples.mimo.model_providers.nemotron_moe_vlm import ( + NEMOTRON_20L_MODEL_PROVIDER, + NEMOTRON_54L_MODEL_PROVIDER, add_model_provider_args, prepare_model_provider_args, validate_model_provider_args, @@ -39,10 +41,17 @@ def parse_args() -> argparse.Namespace: grid.add_argument("--llm-dp", type=int, default=2) grid.add_argument("--llm-ep", type=int, default=2) grid.add_argument("--llm-expt-tp", type=int, default=1) - grid.add_argument("--llm-expt-dp", type=int, default=1) + grid.add_argument("--llm-expt-dp", type=int, default=None) add_model_provider_args(parser) + runtime = parser.add_argument_group("runtime") + runtime.add_argument( + "--enable-experimental", + action="store_true", + help="Enable Megatron experimental kernels/features used by some MoE performance paths.", + ) + data = parser.add_argument_group("data") data.add_argument("--dataset-provider", choices=["mock", "energon_multimodal"], default="mock") data.add_argument("--data-path", type=str, default=None) @@ -136,8 +145,10 @@ def validate_energon_data_args(args: argparse.Namespace) -> None: raise ValueError("--data-path is required for --dataset-provider energon_multimodal") if not args.tokenizer_model: raise ValueError("--tokenizer-model is required for --dataset-provider energon_multimodal") - if args.model_provider != "nemotron-moe-vlm-20l": - raise ValueError("energon_multimodal is currently wired for the Nemotron 20L VLM provider") + if args.model_provider not in (NEMOTRON_20L_MODEL_PROVIDER, NEMOTRON_54L_MODEL_PROVIDER): + raise ValueError( + "energon_multimodal is currently wired for Nemotron MoE VLM providers" + ) if args.encoder_pp != 1 or args.llm_pp != 1: raise ValueError("energon_multimodal currently supports encoder and LLM PP size 1") if args.encoder_dp > args.llm_dp: From ba5276594c26f0bf30ad741bdc8510c05d08a4d8 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Wed, 13 May 2026 09:46:02 -0700 Subject: [PATCH 30/44] Guard Energon alignment validation (#21) --- examples/mimo/training/hetero/args.py | 9 +++++++++ examples/mimo/training/hetero/data.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index b68655908ce..96706182554 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -59,6 +59,15 @@ def parse_args() -> argparse.Namespace: data.add_argument("--packing-buffer-size", type=int, default=None) data.add_argument("--shuffle-buffer-size", type=int, default=100) data.add_argument("--max-samples-per-sequence", type=int, default=100) + data.add_argument( + "--validate-energon-data-alignment", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "Check that encoder and LLM Energon readers start from matching samples. " + "This is disabled by default because the validation all-gather is expensive at scale." + ), + ) train = parser.add_argument_group("training") train.add_argument("--micro-batch-size", type=int, default=2) diff --git a/examples/mimo/training/hetero/data.py b/examples/mimo/training/hetero/data.py index 23ffdc5050b..419e902839a 100644 --- a/examples/mimo/training/hetero/data.py +++ b/examples/mimo/training/hetero/data.py @@ -31,7 +31,7 @@ def validate_data_iterator( args: argparse.Namespace, data_iterator, topology: HeteroTopology ) -> None: """Run data-provider checks that must happen outside the pipeline schedule.""" - if args.dataset_provider == "energon_multimodal": + if args.dataset_provider == "energon_multimodal" and args.validate_energon_data_alignment: from examples.mimo.data.hetero_energon import validate_energon_data_alignment validate_energon_data_alignment(data_iterator, topology) From e032dd7b023bee3b88700fa6cc5ccabc2c47544a Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Wed, 13 May 2026 19:44:17 -0700 Subject: [PATCH 31/44] Add hetero pipeline timeline tracing (#22) --- examples/mimo/training/hetero/args.py | 35 ++++ examples/mimo/training/hetero/loop.py | 12 ++ examples/mimo/training/hetero/timeline.py | 110 +++++++++++ .../multimodule_communicator.py | 47 ++++- megatron/core/pipeline_parallel/schedules.py | 172 ++++++++++------ megatron/core/pipeline_parallel/timeline.py | 185 ++++++++++++++++++ 6 files changed, 491 insertions(+), 70 deletions(-) create mode 100644 examples/mimo/training/hetero/timeline.py create mode 100644 megatron/core/pipeline_parallel/timeline.py diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 96706182554..12b7f9f041d 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -51,6 +51,39 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable Megatron experimental kernels/features used by some MoE performance paths.", ) + runtime.add_argument( + "--timeline-profile", + action="store_true", + help="Write rank-local 1F1B timeline JSONL traces for selected debug ranks.", + ) + runtime.add_argument( + "--timeline-dir", + type=str, + default=None, + help="Directory for rank-local timeline JSONL traces.", + ) + runtime.add_argument( + "--timeline-ranks", + type=str, + default="dp-replica", + help="'dp-replica', 'all', or comma-separated global ranks to trace.", + ) + runtime.add_argument( + "--timeline-dp-replica", + type=int, + default=0, + help="Dense data-parallel replica to trace when --timeline-ranks=dp-replica.", + ) + runtime.add_argument( + "--timeline-cuda-events", + action="store_true", + help="Also record CUDA event elapsed time for compute events.", + ) + runtime.add_argument( + "--timeline-nvtx", + action="store_true", + help="Push NVTX ranges with timeline event names for Nsight Systems.", + ) data = parser.add_argument_group("data") data.add_argument("--dataset-provider", choices=["mock", "energon_multimodal"], default="mock") @@ -117,6 +150,8 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: raise ValueError("Phase 2 mock training currently supports CP=1 only") if args.log_interval < 1: raise ValueError("--log-interval must be >= 1") + if args.timeline_dp_replica < 0: + raise ValueError("--timeline-dp-replica must be >= 0") validate_model_provider_args(args) if args.dataset_provider == "mock": diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 649cd164642..5e9574ccfe9 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -17,10 +17,16 @@ from examples.mimo.training.hetero.optimizer import build_optimizer, build_optimizer_param_scheduler from examples.mimo.training.hetero.runtime import build_mimo_runtime from examples.mimo.training.hetero.step import train_step +from examples.mimo.training.hetero.timeline import configure_hetero_timeline from examples.mimo.training.hetero.topology import HeteroTopology, create_topology from examples.mimo.utils.hetero import debug_rank from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.timeline import ( + close_pipeline_timeline, + flush_pipeline_timeline, + set_pipeline_timeline_iteration, +) def run_train_loop(args: argparse.Namespace) -> None: @@ -32,6 +38,9 @@ def run_train_loop(args: argparse.Namespace) -> None: model: Optional[MimoModel] = None try: topology = create_topology(args, encoder_size, llm_size) + timeline_summary = configure_hetero_timeline(args, topology) + if timeline_summary is not None: + print_rank_0(timeline_summary) torch.manual_seed(args.seed) debug_rank("building MIMO model") @@ -61,13 +70,16 @@ def run_train_loop(args: argparse.Namespace) -> None: for iteration in range(1, args.train_iters + 1): debug_rank(f"iteration {iteration}: train step start") + set_pipeline_timeline_iteration(iteration) result = train_step( args, model, topology, optimizer, opt_param_scheduler, communicator, data_iterator ) + flush_pipeline_timeline() logger.record_step(result) logger.maybe_log(iteration, optimizer, result) debug_rank(f"iteration {iteration}: train step complete") finally: + close_pipeline_timeline() if model is not None: model.destroy() if topology is not None: diff --git a/examples/mimo/training/hetero/timeline.py b/examples/mimo/training/hetero/timeline.py new file mode 100644 index 00000000000..1fff25782bb --- /dev/null +++ b/examples/mimo/training/hetero/timeline.py @@ -0,0 +1,110 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Timeline tracing configuration for standalone heterogeneous MIMO training.""" + +from __future__ import annotations + +import argparse +import os +from typing import Optional + +import torch.distributed as dist + +from examples.mimo.training.hetero.topology import HeteroTopology +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.pipeline_parallel.timeline import configure_pipeline_timeline + + +def configure_hetero_timeline(args: argparse.Namespace, topology: HeteroTopology) -> Optional[str]: + """Configure rank-local pipeline timeline tracing and return a rank-0 summary.""" + enabled = args.timeline_profile or env_flag_enabled("MIMO_TIMELINE") + if not enabled: + configure_pipeline_timeline( + enabled=False, + output_dir=args.timeline_dir or "", + rank=dist.get_rank(), + world_size=dist.get_world_size(), + role="", + ) + return None + + rank = dist.get_rank() + world_size = dist.get_world_size() + scope = os.environ.get("MIMO_TIMELINE_RANKS", args.timeline_ranks) + dp_replica = int(os.environ.get("MIMO_TIMELINE_DP_REPLICA", args.timeline_dp_replica)) + output_dir = args.timeline_dir or os.environ.get("MIMO_TIMELINE_DIR", "mimo_timeline") + selected_ranks = select_timeline_ranks(scope, dp_replica, topology, world_size) + role, coords = rank_role_and_coords(rank, topology) + + configure_pipeline_timeline( + enabled=rank in selected_ranks, + output_dir=output_dir, + rank=rank, + world_size=world_size, + role=role, + metadata={ + "rank_scope": scope, + "timeline_dp_replica": dp_replica, + **coords, + }, + cuda_events=args.timeline_cuda_events or env_flag_enabled("MIMO_TIMELINE_CUDA_EVENTS"), + nvtx=args.timeline_nvtx or env_flag_enabled("MIMO_TIMELINE_NVTX"), + ) + + if rank != 0: + return None + return ( + "Pipeline timeline enabled: " + f"dir={output_dir}, scope={scope}, selected_ranks={len(selected_ranks)}" + ) + + +def select_timeline_ranks( + scope: str, dp_replica: int, topology: HeteroTopology, world_size: int +) -> set[int]: + """Select ranks to trace.""" + scope = scope.strip().lower() + if scope == "all": + return set(range(world_size)) + if scope == "dp-replica": + ranks = ranks_for_dp_replica(topology.encoder_grid, dp_replica) + ranks.update(ranks_for_dp_replica(topology.llm_grid, dp_replica)) + return ranks + return {int(item) for item in scope.split(",") if item.strip()} + + +def ranks_for_dp_replica(grid: HyperCommGrid, dp_replica: int) -> set[int]: + """Return all ranks that belong to one dense DP replica of a grid.""" + ranks = set() + for rank in range(grid.rank_offset, grid.rank_offset + grid.size): + coords = grid_coords(grid, rank) + if coords.get("dp") == dp_replica: + ranks.add(rank) + return ranks + + +def rank_role_and_coords( + rank: int, topology: HeteroTopology +) -> tuple[str, dict[str, int | str]]: + """Return role and dense-grid coordinates for timeline metadata.""" + for role, grid in (("encoder", topology.encoder_grid), ("llm", topology.llm_grid)): + if grid.rank_offset <= rank < grid.rank_offset + grid.size: + coords = grid_coords(grid, rank) + role_coords = {f"{role}_{key}": value for key, value in coords.items()} + return role, {"module": role, **role_coords} + return "unknown", {"module": "unknown"} + + +def grid_coords(grid: HyperCommGrid, rank: int) -> dict[str, int]: + """Decode a global rank into dense HyperCommGrid coordinates.""" + local_rank = rank - grid.rank_offset + coords = {} + for dim_name, dim_size in zip(grid.dim_names, grid.shape): + coords[dim_name] = local_rank % dim_size + local_rank //= dim_size + return coords + + +def env_flag_enabled(name: str) -> bool: + """Return whether an environment flag is set to a truthy value.""" + return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"} diff --git a/megatron/core/pipeline_parallel/multimodule_communicator.py b/megatron/core/pipeline_parallel/multimodule_communicator.py index b2e5682a29d..65d33dfaea9 100644 --- a/megatron/core/pipeline_parallel/multimodule_communicator.py +++ b/megatron/core/pipeline_parallel/multimodule_communicator.py @@ -11,6 +11,7 @@ from megatron.core.model_parallel_config import ModelParallelConfig from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator +from megatron.core.pipeline_parallel.timeline import timeline_event # Types Shape = Union[List[int], torch.Size] @@ -342,7 +343,12 @@ def recv_forward( # If first stage, and has incoming modules, receive forward activation # from incoming modules. for bridge_comm in rank_module_info.bridge_comms_as_dest_module: - received_tensor = bridge_comm.recv_forward() + with timeline_event( + "bridge.recv_forward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + received_tensor = bridge_comm.recv_forward() input_dict[bridge_comm.src_module_name] = received_tensor else: # If not first stage, receive forward activation tensor from P2P communicator. @@ -364,7 +370,12 @@ def send_forward(self, output_dict: Dict[str, torch.Tensor], is_last_stage: bool # If last stage, and has outgoing modules, send forward activation # by using bridge communicator. for bridge_comm in rank_module_info.bridge_comms_as_src_module: - bridge_comm.send_forward(output_dict[module_name]) + with timeline_event( + "bridge.send_forward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + bridge_comm.send_forward(output_dict[module_name]) else: # If not last stage, send forward activation by using P2P communicator. tensor_to_send = _prepare_tensor_for_comm(output_dict[module_name]) @@ -391,7 +402,12 @@ def send_forward_recv_backward( # If last stage, and has outgoing modules, send forward activation and # receive backward gradient by using bridge communicator. for bridge_comm in rank_module_info.bridge_comms_as_src_module: - grad = bridge_comm.send_forward_recv_backward(output_dict[module_name]) + with timeline_event( + "bridge.send_forward_recv_backward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + grad = bridge_comm.send_forward_recv_backward(output_dict[module_name]) grad_dict[bridge_comm.src_module_name] = grad else: # If not last stage, send forward activation and receive backward gradient @@ -424,9 +440,14 @@ def send_backward_recv_forward( for bridge_comm in rank_module_info.bridge_comms_as_dest_module: # If first stage, and has incoming modules, send backward gradient and # receive forward activation by using bridge communicator. - received_tensor = bridge_comm.send_backward_recv_forward( - grad_dict[bridge_comm.src_module_name] - ) + with timeline_event( + "bridge.send_backward_recv_forward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + received_tensor = bridge_comm.send_backward_recv_forward( + grad_dict[bridge_comm.src_module_name] + ) input_dict[bridge_comm.src_module_name] = received_tensor else: # If not first stage, send backward gradient and receive forward activation @@ -459,7 +480,12 @@ def recv_backward( # If last stage, and has incoming modules, receive backward gradient # by using bridge communicator. for bridge_comm in rank_module_info.bridge_comms_as_src_module: - grad = bridge_comm.recv_backward() + with timeline_event( + "bridge.recv_backward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + grad = bridge_comm.recv_backward() grad_dict[bridge_comm.src_module_name] = grad else: # If not last stage, receive backward gradient by using P2P communicator. @@ -480,7 +506,12 @@ def send_backward(self, grad_dict: Dict[str, torch.Tensor], is_first_stage: bool # If first stage, and has incoming modules, send backward activation # by using bridge communicator. for bridge_comm in rank_module_info.bridge_comms_as_dest_module: - bridge_comm.send_backward(grad_dict[bridge_comm.src_module_name]) + with timeline_event( + "bridge.send_backward", + src_module=bridge_comm.src_module_name, + dest_module=bridge_comm.dest_module_name, + ): + bridge_comm.send_backward(grad_dict[bridge_comm.src_module_name]) else: # If not first stage, send backward activation by using P2P communicator. grad_to_send = _prepare_tensor_for_comm(grad_dict[module_name]) diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index 14fc6041574..835f92d9d12 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -13,6 +13,7 @@ ) from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator +from megatron.core.pipeline_parallel.timeline import timeline_event from megatron.core.pipeline_parallel.utils import ( is_pp_first_stage, is_pp_last_stage, @@ -2231,25 +2232,28 @@ def enable_grad_sync(): else: checkpoint_activations_microbatch = None - input_tensor = p2p_communicator.recv_forward( - recv_tensor_shapes, p2p_communicator.is_pp_first_stage - ) - output_tensor, num_tokens = forward_step( - forward_step_func, - data_iterator, - model, - num_microbatches, - input_tensor, - forward_data_store, - config, - cp_group_size=cp_size, - collect_non_loss_data=collect_non_loss_data, - checkpoint_activations_microbatch=checkpoint_activations_microbatch, - is_first_microbatch=check_first_val_step(first_val_step, forward_only, i == 0), - current_microbatch=i, - is_last_stage=p2p_communicator.is_pp_last_stage, - ) - p2p_communicator.send_forward(output_tensor, p2p_communicator.is_pp_last_stage) + with timeline_event("schedule.recv_forward", phase="warmup", microbatch=i): + input_tensor = p2p_communicator.recv_forward( + recv_tensor_shapes, p2p_communicator.is_pp_first_stage + ) + with timeline_event("schedule.forward", phase="warmup", microbatch=i, cuda=True): + output_tensor, num_tokens = forward_step( + forward_step_func, + data_iterator, + model, + num_microbatches, + input_tensor, + forward_data_store, + config, + cp_group_size=cp_size, + collect_non_loss_data=collect_non_loss_data, + checkpoint_activations_microbatch=checkpoint_activations_microbatch, + is_first_microbatch=check_first_val_step(first_val_step, forward_only, i == 0), + current_microbatch=i, + is_last_stage=p2p_communicator.is_pp_last_stage, + ) + with timeline_event("schedule.send_forward", phase="warmup", microbatch=i): + p2p_communicator.send_forward(output_tensor, p2p_communicator.is_pp_last_stage) total_num_tokens += num_tokens if not forward_only: @@ -2261,13 +2265,18 @@ def enable_grad_sync(): # If all microbatches are run in warmup / cooldown phase, then no need to # receive this tensor here. if num_microbatches_remaining > 0: - input_tensor = p2p_communicator.recv_forward( - recv_tensor_shapes, p2p_communicator.is_pp_first_stage - ) + with timeline_event( + "schedule.recv_forward", phase="steady_prefetch", microbatch=num_warmup_microbatches + ): + input_tensor = p2p_communicator.recv_forward( + recv_tensor_shapes, p2p_communicator.is_pp_first_stage + ) # Run 1F1B in steady state. for i in range(num_microbatches_remaining): last_iteration = i == (num_microbatches_remaining - 1) + forward_microbatch = i + num_warmup_microbatches + backward_microbatch = i # Decide to checkpoint all layers' activations of the current micro-batch if max_outstanding_backprops is not None: @@ -2277,35 +2286,50 @@ def enable_grad_sync(): else: checkpoint_activations_microbatch = None - output_tensor, num_tokens = forward_step( - forward_step_func, - data_iterator, - model, - num_microbatches, - input_tensor, - forward_data_store, - config, - cp_group_size=cp_size, - collect_non_loss_data=collect_non_loss_data, - checkpoint_activations_microbatch=checkpoint_activations_microbatch, - is_first_microbatch=check_first_val_step( - first_val_step, forward_only, (i == 0) and (num_warmup_microbatches == 0) - ), - current_microbatch=i + num_warmup_microbatches, - is_last_stage=p2p_communicator.is_pp_last_stage, - ) + with timeline_event( + "schedule.forward", phase="steady", microbatch=forward_microbatch, cuda=True + ): + output_tensor, num_tokens = forward_step( + forward_step_func, + data_iterator, + model, + num_microbatches, + input_tensor, + forward_data_store, + config, + cp_group_size=cp_size, + collect_non_loss_data=collect_non_loss_data, + checkpoint_activations_microbatch=checkpoint_activations_microbatch, + is_first_microbatch=check_first_val_step( + first_val_step, forward_only, (i == 0) and (num_warmup_microbatches == 0) + ), + current_microbatch=forward_microbatch, + is_last_stage=p2p_communicator.is_pp_last_stage, + ) total_num_tokens += num_tokens if forward_only: - p2p_communicator.send_forward(output_tensor, p2p_communicator.is_pp_last_stage) + with timeline_event( + "schedule.send_forward", phase="steady", microbatch=forward_microbatch + ): + p2p_communicator.send_forward(output_tensor, p2p_communicator.is_pp_last_stage) if not last_iteration: - input_tensor = p2p_communicator.recv_forward( - recv_tensor_shapes, p2p_communicator.is_pp_first_stage - ) + with timeline_event( + "schedule.recv_forward", phase="steady", microbatch=forward_microbatch + 1 + ): + input_tensor = p2p_communicator.recv_forward( + recv_tensor_shapes, p2p_communicator.is_pp_first_stage + ) else: - output_tensor_grad = p2p_communicator.send_forward_recv_backward( - output_tensor, send_tensor_shapes, p2p_communicator.is_pp_last_stage - ) + with timeline_event( + "schedule.send_forward_recv_backward", + phase="steady", + microbatch=forward_microbatch, + backward_microbatch=backward_microbatch, + ): + output_tensor_grad = p2p_communicator.send_forward_recv_backward( + output_tensor, send_tensor_shapes, p2p_communicator.is_pp_last_stage + ) # Add input_tensor and output_tensor to end of list. input_tensors.append(input_tensor) @@ -2323,23 +2347,36 @@ def enable_grad_sync(): if config.grad_sync_func is None or p2p_communicator.is_pp_first_stage: enable_grad_sync() - input_tensor_grad = backward_func( - input_tensor, output_tensor, output_tensor_grad, config - ) + with timeline_event( + "schedule.backward", phase="steady", microbatch=backward_microbatch, cuda=True + ): + input_tensor_grad = backward_func( + input_tensor, output_tensor, output_tensor_grad, config + ) if last_iteration: input_tensor = None - p2p_communicator.send_backward( - input_tensor_grad, p2p_communicator.is_pp_first_stage - ) + with timeline_event( + "schedule.send_backward", phase="steady", microbatch=backward_microbatch + ): + p2p_communicator.send_backward( + input_tensor_grad, p2p_communicator.is_pp_first_stage + ) else: - input_tensor = p2p_communicator.send_backward_recv_forward( - input_tensor_grad, recv_tensor_shapes, p2p_communicator.is_pp_first_stage - ) + with timeline_event( + "schedule.send_backward_recv_forward", + phase="steady", + microbatch=backward_microbatch, + recv_microbatch=forward_microbatch + 1, + ): + input_tensor = p2p_communicator.send_backward_recv_forward( + input_tensor_grad, recv_tensor_shapes, p2p_communicator.is_pp_first_stage + ) # Run cooldown backward passes. if not forward_only: for i in range(num_warmup_microbatches): + backward_microbatch = num_microbatches_remaining + i # Enable async grad reduction in the last backward pass # Note: If grad sync function is provided, only enable @@ -2353,15 +2390,26 @@ def enable_grad_sync(): input_tensor = input_tensors.pop(0) output_tensor = output_tensors.pop(0) - output_tensor_grad = p2p_communicator.recv_backward( - send_tensor_shapes, p2p_communicator.is_pp_last_stage - ) + with timeline_event( + "schedule.recv_backward", phase="cooldown", microbatch=backward_microbatch + ): + output_tensor_grad = p2p_communicator.recv_backward( + send_tensor_shapes, p2p_communicator.is_pp_last_stage + ) - input_tensor_grad = backward_func( - input_tensor, output_tensor, output_tensor_grad, config - ) + with timeline_event( + "schedule.backward", phase="cooldown", microbatch=backward_microbatch, cuda=True + ): + input_tensor_grad = backward_func( + input_tensor, output_tensor, output_tensor_grad, config + ) - p2p_communicator.send_backward(input_tensor_grad, p2p_communicator.is_pp_first_stage) + with timeline_event( + "schedule.send_backward", phase="cooldown", microbatch=backward_microbatch + ): + p2p_communicator.send_backward( + input_tensor_grad, p2p_communicator.is_pp_first_stage + ) # Launch any remaining grad reductions. if no_sync_context is not None: diff --git a/megatron/core/pipeline_parallel/timeline.py b/megatron/core/pipeline_parallel/timeline.py new file mode 100644 index 00000000000..051408c257c --- /dev/null +++ b/megatron/core/pipeline_parallel/timeline.py @@ -0,0 +1,185 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Low-overhead rank-local timeline tracing for pipeline debug runs.""" + +from __future__ import annotations + +import contextlib +import json +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterator, Optional, TextIO + +import torch + + +@dataclass +class PipelineTimelineRecorder: + """Collect rank-local timeline events and write them as JSONL.""" + + output_dir: Path + rank: int + world_size: int + role: str + metadata: dict[str, Any] = field(default_factory=dict) + cuda_events: bool = False + nvtx: bool = False + iteration: Optional[int] = None + _records: list[dict[str, Any]] = field(default_factory=list) + _context_stack: list[dict[str, Any]] = field(default_factory=list) + _file: Optional[TextIO] = None + + @contextlib.contextmanager + def record(self, event: str, cuda: bool = False, **metadata) -> Iterator[None]: + """Record one event duration without synchronizing by default.""" + event_metadata = {} + for context_metadata in self._context_stack: + event_metadata.update(context_metadata) + event_metadata.update(metadata) + nvtx_enabled = self.nvtx and torch.cuda.is_available() + cuda_start = None + cuda_end = None + use_cuda_events = self.cuda_events and cuda and torch.cuda.is_available() + + if nvtx_enabled: + torch.cuda.nvtx.range_push(self._format_nvtx(event, event_metadata)) + if use_cuda_events: + cuda_start = torch.cuda.Event(enable_timing=True) + cuda_end = torch.cuda.Event(enable_timing=True) + cuda_start.record() + + start_time_ns = time.time_ns() + start_perf_ns = time.perf_counter_ns() + ok = True + self._context_stack.append(event_metadata) + try: + yield + except Exception: + ok = False + raise + finally: + end_perf_ns = time.perf_counter_ns() + self._context_stack.pop() + if use_cuda_events: + cuda_end.record() + if nvtx_enabled: + torch.cuda.nvtx.range_pop() + + record = { + "event": event, + "iteration": self.iteration, + "rank": self.rank, + "world_size": self.world_size, + "role": self.role, + "start_time_ns": start_time_ns, + "duration_us": (end_perf_ns - start_perf_ns) / 1000.0, + "ok": ok, + } + record.update(self.metadata) + record.update(event_metadata) + if use_cuda_events: + record["_cuda_start"] = cuda_start + record["_cuda_end"] = cuda_end + self._records.append(record) + + def flush(self) -> None: + """Write pending events for this rank.""" + if not self._records: + return + self.output_dir.mkdir(parents=True, exist_ok=True) + if self._file is None: + path = self.output_dir / f"rank{self.rank:05d}.jsonl" + self._file = path.open("a", encoding="utf-8") + + for record in sorted(self._records, key=lambda item: item["start_time_ns"]): + cuda_start = record.pop("_cuda_start", None) + cuda_end = record.pop("_cuda_end", None) + if cuda_start is not None and cuda_end is not None: + cuda_end.synchronize() + record["cuda_ms"] = cuda_start.elapsed_time(cuda_end) + self._file.write(json.dumps(_jsonable(record), sort_keys=True) + "\n") + self._file.flush() + self._records.clear() + + def close(self) -> None: + """Flush and close the rank-local output file.""" + self.flush() + if self._file is not None: + self._file.close() + self._file = None + + def _format_nvtx(self, event: str, metadata: dict[str, Any]) -> str: + microbatch = metadata.get("microbatch") + if microbatch is None: + return f"{event}/iter={self.iteration}/role={self.role}" + return f"{event}/iter={self.iteration}/mb={microbatch}/role={self.role}" + + +_RECORDER: Optional[PipelineTimelineRecorder] = None + + +def configure_pipeline_timeline( + *, + enabled: bool, + output_dir: str, + rank: int, + world_size: int, + role: str, + metadata: Optional[dict[str, Any]] = None, + cuda_events: bool = False, + nvtx: bool = False, +) -> None: + """Configure the process-local pipeline timeline recorder.""" + global _RECORDER + close_pipeline_timeline() + if not enabled: + _RECORDER = None + return + _RECORDER = PipelineTimelineRecorder( + output_dir=Path(output_dir), + rank=rank, + world_size=world_size, + role=role, + metadata=metadata or {}, + cuda_events=cuda_events, + nvtx=nvtx, + ) + + +def set_pipeline_timeline_iteration(iteration: int) -> None: + """Set the current training iteration attached to subsequent events.""" + if _RECORDER is not None: + _RECORDER.iteration = iteration + + +def flush_pipeline_timeline() -> None: + """Flush pending rank-local timeline events.""" + if _RECORDER is not None: + _RECORDER.flush() + + +def close_pipeline_timeline() -> None: + """Close the process-local timeline recorder.""" + global _RECORDER + if _RECORDER is not None: + _RECORDER.close() + _RECORDER = None + + +def timeline_event(event: str, cuda: bool = False, **metadata): + """Return a no-op or recording context manager for one timeline event.""" + if _RECORDER is None: + return contextlib.nullcontext() + return _RECORDER.record(event, cuda=cuda, **metadata) + + +def _jsonable(value): + """Convert common non-JSON values used in trace metadata.""" + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, torch.Size): + return list(value) + return value From e03f8aa3a3b3ee9205e46e882836cfbbbc858c17 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Wed, 13 May 2026 20:31:00 -0700 Subject: [PATCH 32/44] Add HEL MIMO launch scripts (#23) --- .../1t_phase1var_moresft_wrapper.yaml | 6 + .../text_omnicorpus_blend_10_90_hel.yaml | 365 ++++++++++++++++++ .../mimo/blend_files/text_only_1t_hel.yaml | 8 + .../run_hetero_nemotron_54l_hel_train.sh | 273 +++++++++++++ ..._hetero_nemotron_20l_hel_1n_text_vision.sh | 113 ++++++ .../sbatch_hetero_nemotron_54l_hel_9n.sh | 200 ++++++++++ ...ch_hetero_nemotron_54l_hel_9n_text_only.sh | 37 ++ ..._hetero_nemotron_54l_hel_9n_text_vision.sh | 37 ++ examples/mimo/training/hetero/step.py | 7 +- 9 files changed, 1044 insertions(+), 2 deletions(-) create mode 100644 examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml create mode 100644 examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml create mode 100644 examples/mimo/blend_files/text_only_1t_hel.yaml create mode 100755 examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_20l_hel_1n_text_vision.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh diff --git a/examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml b/examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml new file mode 100644 index 00000000000..ed19e692d6f --- /dev/null +++ b/examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml @@ -0,0 +1,6 @@ +# RKarimi 3B-nano SOTA 1T text subset blend. +# The 3B-nano baseline uses TRAIN_SAMPLES=122070313 and SEQ_LEN=8192, +# which is 1,000,000,004,096 tokens. +__module__: megatron.energon +__class__: McoreBlend +mcore_json: /scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/rkarimimahab/workspace/blends/1T-phase1var-moresft.json diff --git a/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml b/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml new file mode 100644 index 00000000000..66f1f4ccb70 --- /dev/null +++ b/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml @@ -0,0 +1,365 @@ +# 90% RKarimi 1T text subset + 10% OmniCorpus (CC-MAIN-2021-25 excluded - corrupt tar) +__module__: megatron.energon +__class__: MetadatasetV2 +splits: + train: + blend: + - weight: 0.9 + path: __MEGATRON_ROOT__/examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2013-20 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2013-48 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-10 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-15 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-23 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-35 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-41 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-42 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-49 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2014-52 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-06 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-11 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-14 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-18 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-22 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-27 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-32 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-35 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-40 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2015-48 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-07 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-18 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-22 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-26 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-30 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-36 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-40 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-44 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2016-50 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-04 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-09 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-13 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-17 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-22 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-26 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-30 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-34 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-39 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-43 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-47 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2017-51 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-05 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-09 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-13 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-17 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-26 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-30 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-34 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-39 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-43 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-47 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2018-51 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-04 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-09 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-13 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-18 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-22 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-26 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-30 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-35 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-39 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-43 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-47 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2019-51 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-05 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-10 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-16 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-24 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-29 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-34 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-40 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-45 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2020-50 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-04 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-10 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-17 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-21 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-31 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-39 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-43 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2021-49 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2022-05 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2022-21 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2022-27 + subflavors: + cook: omnicorpus + - weight: 0.001163 + path: /lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2022-33 + subflavors: + cook: omnicorpus + val: + blend: + - path: __MULTIMODAL_DATA_ROOT__/validation/text_arxiv_math/data.bin + subflavors: + cook: bin_idx + - path: __MULTIMODAL_DATA_ROOT__/validation/text_cc/data.bin + subflavors: + cook: bin_idx + - path: __MULTIMODAL_DATA_ROOT__/validation/text_python/data.bin + subflavors: + cook: bin_idx + - path: __MULTIMODAL_DATA_ROOT__/validation/mint_arxiv + subflavors: + cook: interleaved + - path: __MULTIMODAL_DATA_ROOT__/validation/mint_pdf + subflavors: + cook: interleaved diff --git a/examples/mimo/blend_files/text_only_1t_hel.yaml b/examples/mimo/blend_files/text_only_1t_hel.yaml new file mode 100644 index 00000000000..8ee3ced115b --- /dev/null +++ b/examples/mimo/blend_files/text_only_1t_hel.yaml @@ -0,0 +1,8 @@ +# HEL text-only Energon blend for MIMO jitter isolation. +__module__: megatron.energon +__class__: MetadatasetV2 +splits: + train: + blend: + - weight: 1.0 + path: __MEGATRON_ROOT__/examples/mimo/blend_files/1t_phase1var_moresft_wrapper.yaml diff --git a/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh new file mode 100755 index 00000000000..74be1ed2134 --- /dev/null +++ b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# Run non-colocated heterogeneous MIMO Nemotron6-MoE VLM 54L training on HEL data. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +cd "${REPO_ROOT}" + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export NCCL_SHM_DISABLE="${NCCL_SHM_DISABLE:-1}" +export NCCL_PROTO="${NCCL_PROTO:-simple}" +export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" +export TORCH_NCCL_AVOID_RECORD_STREAMS="${TORCH_NCCL_AVOID_RECORD_STREAMS:-0}" +export NVTE_ALLOW_NONDETERMINISTIC_ALGO="${NVTE_ALLOW_NONDETERMINISTIC_ALGO:-1}" +export PYTHONNOUSERSITE=1 + +if [[ -z "${LOCAL_RANK:-}" && -n "${SLURM_LOCALID:-}" ]]; then + export LOCAL_RANK="${SLURM_LOCALID}" +fi +if [[ -z "${RANK:-}" && -n "${SLURM_PROCID:-}" ]]; then + export RANK="${SLURM_PROCID}" +fi +if [[ -z "${WORLD_SIZE:-}" && -n "${SLURM_NTASKS:-}" ]]; then + export WORLD_SIZE="${SLURM_NTASKS}" +fi +if [[ -z "${MASTER_ADDR:-}" && -n "${SLURM_NODELIST:-}" ]] && command -v scontrol >/dev/null 2>&1; then + export MASTER_ADDR="$(scontrol show hostnames "${SLURM_NODELIST}" | head -n 1)" +fi +export MASTER_PORT="${MASTER_PORT:-29500}" + +TRAINING_STAGE="${TRAINING_STAGE:-stage2}" +MODEL_PROVIDER="${MODEL_PROVIDER:-nemotron-moe-vlm-54l}" +case "${TRAINING_STAGE}" in + stage1|stage2|stage3) + ;; + *) + echo "ERROR: Unknown TRAINING_STAGE='${TRAINING_STAGE}'. Use stage1, stage2, or stage3." >&2 + exit 1 + ;; +esac + +TRAIN_ITERS="${TRAIN_ITERS:-100}" +NUM_MICROBATCHES="${NUM_MICROBATCHES:-12}" +MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +ENCODER_TP="${ENCODER_TP:-1}" +ENCODER_CP="${ENCODER_CP:-1}" +ENCODER_PP="${ENCODER_PP:-1}" +ENCODER_DP="${ENCODER_DP:-8}" +ENCODER_EP="${ENCODER_EP:-1}" +LLM_TP="${LLM_TP:-4}" +LLM_CP="${LLM_CP:-1}" +LLM_PP="${LLM_PP:-1}" +LLM_DP="${LLM_DP:-64}" +LLM_EP="${LLM_EP:-16}" +LLM_EXPT_TP="${LLM_EXPT_TP:-1}" +ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" +MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-0}" + +ENCODER_SIZE=$((ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP)) +LLM_SIZE=$((LLM_TP * LLM_CP * LLM_PP * LLM_DP)) +LLM_OFFSET="${LLM_OFFSET:-${ENCODER_SIZE}}" +EXPECTED_WORLD_SIZE=$((ENCODER_SIZE + LLM_SIZE)) +LLM_EXPT_DP="${LLM_EXPT_DP:-$((LLM_SIZE / (LLM_EXPT_TP * LLM_EP * LLM_PP)))}" + +if [[ $((LLM_EXPT_TP * LLM_EP * LLM_PP * LLM_EXPT_DP)) -ne "${LLM_SIZE}" ]]; then + echo "ERROR: LLM expert layout does not cover LLM ranks." >&2 + echo " llm_size=${LLM_SIZE} etp=${LLM_EXPT_TP} ep=${LLM_EP} pp=${LLM_PP} edp=${LLM_EXPT_DP}" >&2 + exit 1 +fi +if [[ -n "${WORLD_SIZE:-}" && "${WORLD_SIZE}" -ne "${EXPECTED_WORLD_SIZE}" ]]; then + echo "ERROR: WORLD_SIZE=${WORLD_SIZE} but hetero layout requires ${EXPECTED_WORLD_SIZE}" >&2 + echo " Submit with nodes*tasks_per_node=${EXPECTED_WORLD_SIZE}." >&2 + exit 1 +fi + +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP))}" +LR_WARMUP_ITERS="${LR_WARMUP_ITERS:-10}" +LR_DECAY_ITERS="${LR_DECAY_ITERS:-${TRAIN_ITERS}}" +PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" +NUM_WORKERS="${NUM_WORKERS:-1}" +SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" +MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" +LOG_INTERVAL="${LOG_INTERVAL:-10}" + +if [[ -z "${PYTHON_BIN:-}" ]]; then + if command -v python >/dev/null 2>&1; then + PYTHON_BIN=python + else + PYTHON_BIN=python3 + fi +fi + +SCRATCH_ROOT="${SCRATCH_ROOT:-/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch}" +TOKENIZER_MODEL="${TOKENIZER_MODEL:-${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff}" +DATA_TEMPLATE="${DATA_PATH:-${REPO_ROOT}/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml}" +RUN_DIR="${RUN_DIR:-${SCRATCH_ROOT}/runs/mimo_54l_hel/${SLURM_JOB_ID:-local}}" +RESOLVED_CONFIG_DIR="${RESOLVED_CONFIG_DIR:-${RUN_DIR}/resolved_configs}" +DATA_TEMPLATE_BASENAME="$(basename "${DATA_TEMPLATE}")" +DATA_TRAIN="${DATA_TRAIN:-${RESOLVED_CONFIG_DIR}/${DATA_TEMPLATE_BASENAME%.yaml}.train.yaml}" +DATA_READY_FILE="${DATA_TRAIN}.ready" +RANK_ID="${RANK:-${SLURM_PROCID:-0}}" +DATA_READY_TIMEOUT="${DATA_READY_TIMEOUT:-600}" +TMPDIR="${TMPDIR:-${RUN_DIR}/tmp/rank-${RANK_ID}}" +mkdir -p "${TMPDIR}" +export TMPDIR +if [[ -z "${TRITON_CACHE_DIR:-}" ]]; then + export TRITON_CACHE_DIR="${TRITON_CACHE_DIR_BASE:-${RUN_DIR}/triton-cache}/rank-${RANK_ID}" +fi +mkdir -p "${TRITON_CACHE_DIR}" + +if [[ ! -r "${DATA_TEMPLATE}" ]]; then + echo "ERROR: Cannot read DATA_PATH template: ${DATA_TEMPLATE}" >&2 + exit 1 +fi + +if [[ "${RESOLVE_TRAIN_ONLY_CONFIG:-1}" == "1" ]]; then + if [[ "${RANK_ID}" -eq 0 ]]; then + mkdir -p "${RESOLVED_CONFIG_DIR}" + rm -f "${DATA_READY_FILE}" + DATA_TEMPLATE="${DATA_TEMPLATE}" \ + DATA_TRAIN="${DATA_TRAIN}" \ + REPO_ROOT="${REPO_ROOT}" \ + USER_HOME="${USER_HOME:-/home/${USER:-ykarnati}}" \ + MULTIMODAL_DATA_ROOT="${MULTIMODAL_DATA_ROOT:-/home/${USER:-ykarnati}/data/multimodal_data}" \ + "${PYTHON_BIN}" - <<'PY' +import os +from pathlib import Path + +src = Path(os.environ["DATA_TEMPLATE"]) +dst = Path(os.environ["DATA_TRAIN"]) +text = src.read_text() +for key, value in { + "__MEGATRON_ROOT__": os.environ["REPO_ROOT"], + "__USER_HOME__": os.environ["USER_HOME"], + "__MULTIMODAL_DATA_ROOT__": os.environ["MULTIMODAL_DATA_ROOT"], +}.items(): + text = text.replace(key, value) + +train_only = [] +for line in text.splitlines(): + if line.startswith(" val:") or line.startswith(" test:"): + break + train_only.append(line) +text = "\n".join(train_only) + "\n" + +dst.parent.mkdir(parents=True, exist_ok=True) +tmp = dst.with_suffix(dst.suffix + f".tmp.{os.getpid()}") +tmp.write_text(text) +tmp.replace(dst) +PY + touch "${DATA_READY_FILE}" + else + waited=0 + until [[ -f "${DATA_READY_FILE}" ]]; do + sleep 2 + waited=$((waited + 2)) + if [[ "${waited}" -gt "${DATA_READY_TIMEOUT}" ]]; then + echo "ERROR: Timed out waiting for resolved data config: ${DATA_READY_FILE}" >&2 + exit 1 + fi + done + fi +else + DATA_TRAIN="${DATA_TEMPLATE}" +fi + +if [[ ! -r "${DATA_TRAIN}" ]]; then + echo "ERROR: Cannot read resolved data config: ${DATA_TRAIN}" >&2 + exit 1 +fi +if [[ ! -r "${TOKENIZER_MODEL}/tokenizer.json" ]]; then + echo "ERROR: Cannot read tokenizer.json under TOKENIZER_MODEL=${TOKENIZER_MODEL}" >&2 + exit 1 +fi + +if [[ "${CHECK_HEL_PATHS:-1}" == "1" ]]; then + TEXT_MCORE_JSON="/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/rkarimimahab/workspace/blends/1T-phase1var-moresft.json" + OMNICORPUS_SAMPLE="/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/multimodal/datasets/OmniCorpus-CC-210M/webdataset/CC-MAIN-2013-20" + if [[ ! -r "${TEXT_MCORE_JSON}" ]]; then + echo "ERROR: Cannot read text MCore blend JSON: ${TEXT_MCORE_JSON}" >&2 + exit 1 + fi + if [[ ! -d "${OMNICORPUS_SAMPLE}" ]]; then + echo "ERROR: Cannot find OmniCorpus HEL sample directory: ${OMNICORPUS_SAMPLE}" >&2 + exit 1 + fi +fi + +if [[ "${RANK_ID}" -eq 0 ]]; then + echo "=== Hetero MIMO Nemotron6-MoE VLM 54L HEL training ===" + echo "model_provider=${MODEL_PROVIDER}" + echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} mbs=${MICRO_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} gbs=${GLOBAL_BATCH_SIZE}" + echo "layout=encoder(tp=${ENCODER_TP},cp=${ENCODER_CP},pp=${ENCODER_PP},dp=${ENCODER_DP},ep=${ENCODER_EP}) llm(tp=${LLM_TP},cp=${LLM_CP},pp=${LLM_PP},dp=${LLM_DP},ep=${LLM_EP},etp=${LLM_EXPT_TP},edp=${LLM_EXPT_DP}) world=${EXPECTED_WORLD_SIZE}" + echo "enable_experimental=${ENABLE_EXPERIMENTAL}" + echo "moe_router_force_load_balancing=${MOE_ROUTER_FORCE_LOAD_BALANCING}" + echo "moe_router_fusion=model-provider-default" + echo "data=${DATA_TRAIN}" + echo "tokenizer=${TOKENIZER_MODEL}" + echo "run_dir=${RUN_DIR}" + echo "==========================================================" +fi + +DATA_LOADER_ARGS=( + --num-workers "${NUM_WORKERS}" + --shuffle-buffer-size "${SHUFFLE_BUFFER_SIZE}" + --max-samples-per-sequence "${MAX_SAMPLES_PER_SEQUENCE}" +) +if [[ "${PACKING_BUFFER_SIZE}" != "0" ]]; then + DATA_LOADER_ARGS+=(--packing-buffer-size "${PACKING_BUFFER_SIZE}") +fi +MODEL_ARGS=() +if [[ "${ENABLE_EXPERIMENTAL}" == "1" || "${ENABLE_EXPERIMENTAL}" == "true" ]]; then + MODEL_ARGS+=(--enable-experimental) +fi +if [[ "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "1" || "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "true" ]]; then + MODEL_ARGS+=(--moe-router-force-load-balancing) +fi + +CMD=( + "${PYTHON_BIN}" -u examples/mimo/train_hetero.py + --model-provider "${MODEL_PROVIDER}" + --dataset-provider energon_multimodal + --training-stage "${TRAINING_STAGE}" + --encoder-tp "${ENCODER_TP}" + --encoder-cp "${ENCODER_CP}" + --encoder-pp "${ENCODER_PP}" + --encoder-dp "${ENCODER_DP}" + --encoder-ep "${ENCODER_EP}" + --llm-offset "${LLM_OFFSET}" + --llm-tp "${LLM_TP}" + --llm-cp "${LLM_CP}" + --llm-pp "${LLM_PP}" + --llm-dp "${LLM_DP}" + --llm-ep "${LLM_EP}" + --llm-expt-tp "${LLM_EXPT_TP}" + --llm-expt-dp "${LLM_EXPT_DP}" + "${MODEL_ARGS[@]}" + --vocab-size 131072 + --max-num-tiles 12 + --data-path "${DATA_TRAIN}" + "${DATA_LOADER_ARGS[@]}" + --tokenizer-model "${TOKENIZER_MODEL}" + --tokenizer-prompt-format nemotron6-moe + --image-token "" + --micro-batch-size "${MICRO_BATCH_SIZE}" + --global-batch-size "${GLOBAL_BATCH_SIZE}" + --num-microbatches "${NUM_MICROBATCHES}" + --lr 2e-4 + --min-lr 2e-6 + --lr-decay-style cosine + --lr-warmup-iters "${LR_WARMUP_ITERS}" + --lr-decay-iters "${LR_DECAY_ITERS}" + --weight-decay 0.05 + --adam-beta1 0.9 + --adam-beta2 0.95 + --clip-grad 1.0 + --no-overlap-grad-reduce + --ddp-bucket-size 0 + --log-interval "${LOG_INTERVAL}" + --train-iters "${TRAIN_ITERS}" + "$@" +) + +if [[ "${DRY_RUN:-0}" == "1" ]]; then + printf '%q ' "${CMD[@]}" + printf '\n' + exit 0 +fi + +exec "${CMD[@]}" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_20l_hel_1n_text_vision.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_20l_hel_1n_text_vision.sh new file mode 100755 index 00000000000..ec5fdee7c94 --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_20l_hel_1n_text_vision.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Submit a one-node HEL 20L heterogeneous MIMO smoke run on the 90% text / 10% vision blend. +# +# Intended use from a Cog-synced nb-hel workspace: +# sbatch examples/mimo/scripts/sbatch_hetero_nemotron_20l_hel_1n_text_vision.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p interactive +#SBATCH -N 1 +#SBATCH --gres=gpu:8 +#SBATCH --time=00:30:00 +#SBATCH -J mimo20l1ntv +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -z "${REPO_ROOT:-}" ]]; then + if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" + else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + fi +fi + +SCRATCH_ROOT="${SCRATCH_ROOT:-/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch}" +CONTAINER_IMAGE="${CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/e4b4805e816ada20.sqsh}" +ENV_ROOT="${ENV_ROOT:-${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39}" + +RUN_NAME="${RUN_NAME:-mimo20l-hel-1n-text-vision-10-90}" +RUN_DIR="${RUN_DIR:-${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}}" + +mkdir -p \ + "${RUN_DIR}/tmp" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/home" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" \ + "${SCRATCH_ROOT}/uv-cache/megatron_lm" + +if [[ ! -r "${CONTAINER_IMAGE}" ]]; then + echo "ERROR: Cannot read CONTAINER_IMAGE=${CONTAINER_IMAGE}" >&2 + exit 1 +fi +if [[ ! -d "${ENV_ROOT}/.venv" ]]; then + echo "ERROR: Cannot find uv environment at ENV_ROOT=${ENV_ROOT}" >&2 + exit 1 +fi + +export SCRATCH_ROOT +export REPO_ROOT +export RUN_DIR +export TMPDIR="${RUN_DIR}/tmp" +export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" +export XDG_CACHE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" +export XDG_DATA_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" +export XDG_STATE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" +export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-${RUN_DIR}/triton-cache}" +export CUDA_CACHE_PATH="${CUDA_CACHE_PATH:-${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache}" +export PYTHONPATH="${REPO_ROOT}" +export PYTHONNOUSERSITE=1 +export PIP_CONSTRAINT="" +export UV_LINK_MODE=copy +export UV_CACHE_DIR="${SCRATCH_ROOT}/uv-cache/megatron_lm" +export UV_PROJECT_ENVIRONMENT="${ENV_ROOT}/.venv" +export VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}" +export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" + +export DATA_PATH="${DATA_PATH:-${REPO_ROOT}/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml}" +export TOKENIZER_MODEL="${TOKENIZER_MODEL:-${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff}" +export TRAIN_ITERS="${TRAIN_ITERS:-30}" +export NUM_MICROBATCHES="${NUM_MICROBATCHES:-4}" +export MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +export NUM_WORKERS="${NUM_WORKERS:-0}" +export SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" +export PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" +export MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" +export VERIFY_ENERGON="${VERIFY_ENERGON:-1}" +export ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" +export MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-1}" + +CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" +if [[ "${REPO_ROOT}" != "${SCRATCH_ROOT}"/* ]]; then + CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" +fi +if [[ -n "${CONTAINER_MOUNTS_EXTRA:-}" ]]; then + CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${CONTAINER_MOUNTS_EXTRA}" +fi + +echo "=== HEL 20L heterogeneous MIMO sbatch ===" +echo "repo=${REPO_ROOT}" +echo "run_dir=${RUN_DIR}" +echo "container_image=${CONTAINER_IMAGE}" +echo "env_root=${ENV_ROOT}" +echo "data=${DATA_PATH}" +echo "tokenizer=${TOKENIZER_MODEL}" +echo "train_iters=${TRAIN_ITERS} microbatches=${NUM_MICROBATCHES}" +echo "================================================" + +srun --kill-on-bad-exit=1 \ + --ntasks=1 \ + --container-image="${CONTAINER_IMAGE}" \ + --no-container-mount-home \ + --container-mounts="${CONTAINER_MOUNTS}" \ + --container-workdir="${REPO_ROOT}" \ + bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_20l_energon_train.sh "$@"' \ + bash "$@" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh new file mode 100755 index 00000000000..a2a6ba7590d --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# Submit the 9-node HEL 54L heterogeneous MIMO run. +# +# Intended use from a Cog-synced nb-hel workspace: +# sbatch examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 9 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=00:45:00 +#SBATCH -J mimo54l9n +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -z "${REPO_ROOT:-}" ]]; then + if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" + else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + fi +fi + +SCRATCH_ROOT="${SCRATCH_ROOT:-/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch}" +CONTAINER_IMAGE="${CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/e4b4805e816ada20.sqsh}" +ENV_ROOT="${ENV_ROOT:-${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39}" + +TRAIN_ITERS="${TRAIN_ITERS:-30}" +NUM_MICROBATCHES="${NUM_MICROBATCHES:-12}" +MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-192}" +LOG_INTERVAL="${LOG_INTERVAL:-1}" + +RUN_NAME="${RUN_NAME:-mimo54l-hel-9n-gbs${GLOBAL_BATCH_SIZE}}" +RUN_DIR="${RUN_DIR:-${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}}" +TIMELINE_DIR="${TIMELINE_DIR:-${RUN_DIR}/timeline}" + +mkdir -p \ + "${RUN_DIR}/logs/app" \ + "${RUN_DIR}/logs/torchrun" \ + "${RUN_DIR}/checkpoints" \ + "${RUN_DIR}/tensorboard" \ + "${RUN_DIR}/data_cache" \ + "${RUN_DIR}/tmp" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/home" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" \ + "${SCRATCH_ROOT}/uv-cache/megatron_lm" + +if [[ ! -r "${CONTAINER_IMAGE}" ]]; then + echo "ERROR: Cannot read CONTAINER_IMAGE=${CONTAINER_IMAGE}" >&2 + exit 1 +fi +if [[ ! -d "${ENV_ROOT}/.venv" ]]; then + echo "ERROR: Cannot find uv environment at ENV_ROOT=${ENV_ROOT}" >&2 + exit 1 +fi + +export SCRATCH_ROOT +export REPO_ROOT +export RUN_DIR +export OUTPUT_PATH="${RUN_DIR}" +export LOG_DIR="${RUN_DIR}/logs/app" +export APP_LOG_DIR="${RUN_DIR}/logs/app" +export TORCHRUN_LOG_DIR="${RUN_DIR}/logs/torchrun" +export CHECKPOINT_SAVE_PATH="${RUN_DIR}/checkpoints" +export CHECKPOINT_LOAD_PATH="${RUN_DIR}/checkpoints" +export CHECKPOINT_DIR="${RUN_DIR}/checkpoints" +export TENSORBOARD_PATH="${RUN_DIR}/tensorboard" +export TB_DIR="${RUN_DIR}/tensorboard" +export DATA_CACHE_DIR="${RUN_DIR}/data_cache" +export ARTIFACT_MANIFEST="${RUN_DIR}/artifacts.json" +export TMPDIR="${RUN_DIR}/tmp" + +export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" +export XDG_CACHE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" +export XDG_DATA_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" +export XDG_STATE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" +export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache}" +export TRITON_CACHE_DIR_BASE="${TRITON_CACHE_DIR_BASE:-${RUN_DIR}/triton-cache}" +export CUDA_CACHE_PATH="${CUDA_CACHE_PATH:-${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache}" +export TORCHINDUCTOR_COMPILE_THREADS="${TORCHINDUCTOR_COMPILE_THREADS:-4}" +export PYTHONPATH="${REPO_ROOT}" +export PYTHONNOUSERSITE=1 +export PIP_CONSTRAINT="" +export UV_LINK_MODE=copy +export UV_CACHE_DIR="${SCRATCH_ROOT}/uv-cache/megatron_lm" +export UV_PROJECT_ENVIRONMENT="${ENV_ROOT}/.venv" +export VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}" +export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" + +# Runtime/perf settings carried over from the validated HEL run plus the v3 +# multimodal recipe flags that matter for Transformer Engine and NCCL behavior. +export CUDA_DEVICE_MAX_CONNECTIONS="${CUDA_DEVICE_MAX_CONNECTIONS:-1}" +export NVTE_FWD_LAYERNORM_SM_MARGIN="${NVTE_FWD_LAYERNORM_SM_MARGIN:-16}" +export NVTE_BWD_LAYERNORM_SM_MARGIN="${NVTE_BWD_LAYERNORM_SM_MARGIN:-16}" +export NCCL_P2P_NET_CHUNKSIZE="${NCCL_P2P_NET_CHUNKSIZE:-2097152}" +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export NCCL_SHM_DISABLE="${NCCL_SHM_DISABLE:-1}" +export NCCL_PROTO="${NCCL_PROTO:-simple}" +export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" +export TORCH_NCCL_AVOID_RECORD_STREAMS="${TORCH_NCCL_AVOID_RECORD_STREAMS:-0}" +export TORCH_FR_BUFFER_SIZE="${TORCH_FR_BUFFER_SIZE:-1048576}" +export TORCH_NCCL_TRACE_BUFFER_SIZE="${TORCH_NCCL_TRACE_BUFFER_SIZE:-1048576}" +export TORCH_NCCL_TRACE_CPP_STACK="${TORCH_NCCL_TRACE_CPP_STACK:-1}" +export TORCH_NCCL_DUMP_ON_TIMEOUT="${TORCH_NCCL_DUMP_ON_TIMEOUT:-1}" +export TORCH_NCCL_DESYNC_DEBUG="${TORCH_NCCL_DESYNC_DEBUG:-1}" +export NVTE_ALLOW_NONDETERMINISTIC_ALGO="${NVTE_ALLOW_NONDETERMINISTIC_ALGO:-1}" + +export TRAINING_STAGE="${TRAINING_STAGE:-stage2}" +export MODEL_PROVIDER="${MODEL_PROVIDER:-nemotron-moe-vlm-54l}" +export TRAIN_ITERS +export NUM_MICROBATCHES +export MICRO_BATCH_SIZE +export GLOBAL_BATCH_SIZE +export LR_WARMUP_ITERS="${LR_WARMUP_ITERS:-2}" +export LR_DECAY_ITERS="${LR_DECAY_ITERS:-${TRAIN_ITERS}}" +export LOG_INTERVAL + +export ENCODER_TP="${ENCODER_TP:-1}" +export ENCODER_CP="${ENCODER_CP:-1}" +export ENCODER_PP="${ENCODER_PP:-1}" +export ENCODER_DP="${ENCODER_DP:-8}" +export ENCODER_EP="${ENCODER_EP:-1}" +export LLM_TP="${LLM_TP:-4}" +export LLM_CP="${LLM_CP:-1}" +export LLM_PP="${LLM_PP:-1}" +export LLM_DP="${LLM_DP:-16}" +export LLM_EP="${LLM_EP:-16}" +export LLM_EXPT_TP="${LLM_EXPT_TP:-1}" + +export NUM_WORKERS="${NUM_WORKERS:-0}" +export SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" +export PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" +export MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" +export CHECK_HEL_PATHS="${CHECK_HEL_PATHS:-1}" + +export ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" +export MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-1}" + +WORLD_SIZE=$((ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP + LLM_TP * LLM_CP * LLM_PP * LLM_DP)) +if [[ "${WORLD_SIZE}" -ne 72 ]]; then + echo "ERROR: This 9-node sbatch expects 72 ranks, but layout computed WORLD_SIZE=${WORLD_SIZE}" >&2 + exit 1 +fi +if [[ -n "${SLURM_NTASKS:-}" && "${SLURM_NTASKS}" -ne "${WORLD_SIZE}" ]]; then + echo "ERROR: SLURM_NTASKS=${SLURM_NTASKS}, expected ${WORLD_SIZE}" >&2 + exit 1 +fi + +CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" +if [[ "${REPO_ROOT}" != "${SCRATCH_ROOT}"/* ]]; then + CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" +fi +if [[ -n "${CONTAINER_MOUNTS_EXTRA:-}" ]]; then + CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${CONTAINER_MOUNTS_EXTRA}" +fi + +TRAIN_LAUNCH_ARGS=() +if [[ "${ENABLE_TIMELINE:-1}" == "1" || "${ENABLE_TIMELINE:-1}" == "true" ]]; then + TRAIN_LAUNCH_ARGS+=(--timeline-profile --timeline-dir "${TIMELINE_DIR}") + TRAIN_LAUNCH_ARGS+=(--timeline-ranks "${TIMELINE_RANKS:-dp-replica}") +fi +if [[ "${TIMELINE_CUDA_EVENTS:-0}" == "1" || "${TIMELINE_CUDA_EVENTS:-0}" == "true" ]]; then + TRAIN_LAUNCH_ARGS+=(--timeline-cuda-events) +fi +if [[ "${TIMELINE_NVTX:-0}" == "1" || "${TIMELINE_NVTX:-0}" == "true" ]]; then + TRAIN_LAUNCH_ARGS+=(--timeline-nvtx) +fi +TRAIN_LAUNCH_ARGS+=("$@") + +echo "=== HEL 54L heterogeneous MIMO sbatch ===" +echo "repo=${REPO_ROOT}" +echo "run_dir=${RUN_DIR}" +echo "container_image=${CONTAINER_IMAGE}" +echo "env_root=${ENV_ROOT}" +echo "world_size=${WORLD_SIZE}" +echo "gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} train_iters=${TRAIN_ITERS}" +echo "layout=encoder(tp=${ENCODER_TP},dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP},etp=${LLM_EXPT_TP})" +echo "timeline=${ENABLE_TIMELINE:-1} timeline_dir=${TIMELINE_DIR}" +echo "================================================" + +srun --kill-on-bad-exit=1 \ + --ntasks="${WORLD_SIZE}" \ + --ntasks-per-node=8 \ + --container-image="${CONTAINER_IMAGE}" \ + --no-container-mount-home \ + --container-mounts="${CONTAINER_MOUNTS}" \ + --container-workdir="${REPO_ROOT}" \ + bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"' \ + bash "${TRAIN_LAUNCH_ARGS[@]}" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh new file mode 100755 index 00000000000..e750c930d98 --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Submit the 9-node HEL 54L heterogeneous MIMO run on the text-only blend. +# +# Intended use from a Cog-synced nb-hel workspace: +# sbatch examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 9 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=00:45:00 +#SBATCH -J mimo54l9nt +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -z "${REPO_ROOT:-}" ]]; then + if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" + else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + fi +fi + +export REPO_ROOT +export DATA_PATH="${DATA_PATH:-${REPO_ROOT}/examples/mimo/blend_files/text_only_1t_hel.yaml}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-192}" +export TRAIN_ITERS="${TRAIN_ITERS:-30}" +export NUM_MICROBATCHES="${NUM_MICROBATCHES:-12}" +export LOG_INTERVAL="${LOG_INTERVAL:-1}" +export RUN_NAME="${RUN_NAME:-mimo54l-hel-9n-text-only-gbs${GLOBAL_BATCH_SIZE}}" + +exec bash "${REPO_ROOT}/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh" "$@" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh new file mode 100755 index 00000000000..31c11b6a872 --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Submit the 9-node HEL 54L heterogeneous MIMO run on the 90% text / 10% vision blend. +# +# Intended use from a Cog-synced nb-hel workspace: +# sbatch examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 9 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=00:45:00 +#SBATCH -J mimo54l9ntv +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -z "${REPO_ROOT:-}" ]]; then + if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" + else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + fi +fi + +export REPO_ROOT +export DATA_PATH="${DATA_PATH:-${REPO_ROOT}/examples/mimo/blend_files/text_omnicorpus_blend_10_90_hel.yaml}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-192}" +export TRAIN_ITERS="${TRAIN_ITERS:-30}" +export NUM_MICROBATCHES="${NUM_MICROBATCHES:-12}" +export LOG_INTERVAL="${LOG_INTERVAL:-1}" +export RUN_NAME="${RUN_NAME:-mimo54l-hel-9n-text-vision-10-90-gbs${GLOBAL_BATCH_SIZE}}" + +exec bash "${REPO_ROOT}/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh" "$@" diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 2de59ca7b4e..f136de45873 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -19,6 +19,7 @@ from examples.mimo.utils.hetero import debug_rank from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from megatron.core.pipeline_parallel.timeline import timeline_event @dataclass @@ -63,8 +64,10 @@ def loss_func(output_tensor: torch.Tensor, *, loss_mask: torch.Tensor): def forward_step(data_iterator, model): """Forward step consumed by the MCore pipeline schedule.""" - batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} - batch = move_batch_to_cuda(batch) + with timeline_event("data.next"): + batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} + with timeline_event("data.to_cuda", cuda=True): + batch = move_batch_to_cuda(batch) debug_rank("forward_step batch prepared") debug_rank("forward_step model call start") output_tensor, loss_mask = model(**batch) From 23bf04a2c30f52f84d7c19f737a38d4aa01f62a6 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Fri, 15 May 2026 15:10:41 -0700 Subject: [PATCH 33/44] NMFW-464: Distributed checkpoint save/load for the hetero MIMO training loop (#26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add distributed-checkpoint save/load to the hetero MIMO training loop Adds the standalone `examples/mimo/training/hetero/checkpointing.py` module plus the CLI surface and loop wiring needed to round-trip MimoModel, MimoOptimizer (ChainedOptimizer-of-DistributedOptimizers in the MoE recipe) and the LR/WD scheduler through `megatron.core.dist_checkpointing` without depending on the `parallel_state` singleton. Layout stays compatible with `megatron/training/checkpointing.py` output: `/latest_checkpointed_iteration.txt` plus per-iteration directories containing `common.pt`, `metadata.json`, `.metadata`, and torch_dist shards. Common state now carries `args`, `checkpoint_version=3.0`, the LR scheduler state, and a per-branch `mimo.{branch}.rng_state` ShardedObject; the tracker read uses a cross-rank MAX reduce to mirror megatron's `read_metadata`. Fixes three pre-existing dist-ckpt bugs that hetero usage uncovered: - `megatron/core/ssm/mamba_mixer.py` was calling `make_sharded_tensors_for_checkpoint` without passing `tp_group` and `dp_cp_group`, which fell back to the parallel_state singleton and asserted in hetero mode (gated_delta_net was already correct). - `MimoOptimizer.sharded_state_dict` now applies `add_prefix_for_sharding(module_sd, f'mimo.{name}.')` to each per-branch optimizer sub-dict so two modules' identical internal ShardedObject keys (e.g. `chained_0.optimizer.distributed.dp_group_idx_0.*`) don't collide. - `_get_replica_id` now folds in `tp_rank` so two TP ranks within DP=0 don't both claim primary writer for the same shard. Also routes DistributedOptimizer's per-module `param_state_sharding_type` config string through a new ShardedObject (`_extract_*` helpers) so the non-rank-0 module owner doesn't lose it when only rank 0's common.pt is authoritative. A `_propagate_tp_groups_for_checkpoint` walker stamps `self.tp_group` on descendants that omit it (e.g. `ExtendedRMSNorm`, RADIO submodules) so the default `MegatronModule.sharded_state_dict` path doesn't fall through to `parallel_state.get_tensor_model_parallel_group`. Validated end-to-end on cw-dfw 8-GPU 20L mock (stage2): - Save iter 3 (DistributedOptimizer + EP=4 + TP=2 + 2-module Chained) - Reload iter 3 → resume at iter 4 with cosine LR continuation (1.59e-4 → 1.32e-4 → 1.01e-4), losses match prior trajectory. New flags: `--save`, `--load`, `--save-interval`, `--no-save-optim`, `--no-load-optim`, `--no-load-scheduler`, `--no-save-rng`, `--no-load-rng`, `--finetune`, `--dist-ckpt-optim-fully-reshardable`. Co-Authored-By: Claude Opus 4.7 (1M context) * Adopt MimoOptimizer checkpoint patterns from NVIDIA/Megatron-LM#4801 Three convergent simplifications to MimoOptimizer's distributed-checkpoint path, matching Kamran's MimoOptimizer fixes PR: 1. Replace the ShardedObject-based round-trip for `param_state_sharding_type` with a metadata stash. The sharding type is not per-rank state — it's a load-time interpretation hint that the caller supplies via the `metadata` kwarg on `sharded_state_dict()`. We stash that metadata in `self._last_sharded_metadata` at save and re-inject the sharding type into each per-module sub state-dict during `load_state_dict()` for ranks that lost it via dist_checkpointing's common-state path (i.e. non-rank-0 module owners in non-colocated layouts). Drops `_extract_param_state_sharding_type` / `_restore_param_state_sharding_type` along with their ShardedObject keys. 2. `_restore_param_groups` now uses `setdefault('optimizer', {})` before writing back `param_groups`. After `_extract_param_groups` deletes `param_groups` at save time, the leftover empty `'optimizer'` dict can be dropped by the common-state round-trip on ranks whose active module wasn't on rank 0 at save. The setdefault makes the restore path tolerant of that drop. 3. `_get_replica_id` reorders to `(tp_rank, pp_rank, dp_rank)` to match the convention used by `make_sharded_object_for_checkpoint` in `megatron/core/transformer/utils.py:168-172`. Dedup math is unchanged — `(0, 0, 0)` is still the primary replica — but the order is now consistent with the rest of the codebase. Validated on cw-dfw 1-node 8-GPU 20L mock (stage2, DistributedOptimizer + ChainedOptimizer + EP=4 + TP=2): save iter 3, reload, resume iter 4 with cosine LR continuation (1.59e-4 → 1.32e-4 → 1.01e-4) and matching loss trajectory. Save exit 0, load exit 0. Co-Authored-By: Claude Opus 4.7 (1M context) * Document _propagate_tp_groups_for_checkpoint escapees from no-walker run Disabled `_propagate_tp_groups_for_checkpoint` and re-ran the 20L mock to enumerate exactly which modules fall through to `parallel_state.get_tensor_model_parallel_group()` and assert. Confirmed both branches escape: - RADIO encoder internals (first failure, reached via `nemotron_moe_vlm.RadioEncoder.sharded_state_dict` → HF radio_model leaves with no tp_group + no own sharded_state_dict). - `MambaLayer.__init__` in `megatron/core/ssm/mamba_layer.py` plumbs pg_collection to the mixer but never sets `self.tp_group`. - `ExtendedRMSNorm` at `megatron/core/ssm/mamba_mixer.py:93` never sees pg_collection at all. Fixing each at the source would mean patches across core (Mamba) plus a partial walk of RADIO's HF wrapper, validated against all existing non-hetero users of those modules. The walker is the smaller intervention: one place, hasattr-guarded, applied per branch with the correct pg. Re-enables the walker (it was already in PR1; this commit only updates the docstring to record the experiment's findings). Co-Authored-By: Claude Opus 4.7 (1M context) * Switch back to ShardedObject for param_state_sharding_type (NVIDIA/Megatron-LM#4791) Kamran reverted the metadata-stash approach in #4801 (discussion r3250847203) and adopted Li Ding's PR #4791 pattern, which is the same ShardedObject round-trip we had originally. Align our MimoOptimizer with that final shape: - Restore `_extract_param_state_sharding_type` / `_restore_param_state_sharding_type` helpers. Hooks back into the existing `_iter_optimizer_sub_dicts` loop. - Add `if not opt_sub: del sub_sd['optimizer']` to `_extract_param_groups` (from #4791) so the now-empty `'optimizer'` wrapper doesn't round-trip through common-state with undefined behavior on the load side. - Drop `self._last_sharded_metadata` and the metadata-stash recover path from `load_state_dict` / `sharded_state_dict`. The ShardedObject route is self-contained and doesn't need caller-state coupling. Kept (not in #4791, specific to our non-colocated hetero layout): - `add_prefix_for_sharding(module_sd, f'mimo.{name}.')` so the two branches' identical inner ShardedObject keys (e.g. `chained_0.optimizer.distributed.dp_group_idx_0.*`) don't collide. - `_get_replica_id` returning `(tp_rank, pp_rank, dp_rank)` (from #4801). Validated on cw-dfw 1-node 8-GPU 20L mock (stage2, DistributedOptimizer + ChainedOptimizer + EP=4 + TP=2): save iter 3 exit 0, reload + resume iter 4 with cosine LR continuation (1.59e-4 → 1.32e-4 → 1.01e-4), matching loss trajectory across the boundary. Co-Authored-By: Claude Opus 4.7 (1M context) * Drop _stamp_tp_group walker; fix the three constructors at the source Three modules under our hetero save path don't store `self.tp_group` in their constructors and therefore trip `MegatronModule.sharded_state_dict`'s parallel_state fallback (`megatron/core/transformer/module.py:85`) in heterogeneous-parallelism layouts where parallel_state is intentionally not initialized. Fix them at the source instead of papering over with the hasattr-guarded walker: - `megatron/core/models/vision/radio.py:RADIOViTModel.__init__` — already extracts `tp_group` at line 129 for the embedder; now also stamps `self.tp_group = tp_group`. - `megatron/core/ssm/mamba_layer.py:MambaLayer.__init__` — takes pg_collection and plumbs it into the mixer; now also stores `self.tp_group = pg_collection.tp` on the layer itself. - `megatron/core/ssm/mamba_mixer.py:ExtendedRMSNorm` — adds an `__init__(*args, tp_group=None, **kwargs)` override that stores `self.tp_group` eagerly, and updates the single call site at line ~369 to pass `tp_group=self.pg_collection.tp`. The lazy `hasattr` fallback inside `sharded_state_dict` is preserved for callers that don't pass tp_group. With these three constructor fixes in place, the `_propagate_tp_groups_for_checkpoint` walker (and `_stamp_tp_group` helper) in `examples/mimo/training/hetero/runtime.py` is no longer needed. Removed entirely. Validated on cw-dfw 1-node 8-GPU 20L mock with the walker disabled: - save iter 3 exit 0 (DistributedOptimizer + ChainedOptimizer + EP=4 + TP=2) - reload iter 3 → resume iter 4-5 with cosine LR continuation (1.59e-4 → 1.32e-4 → 1.01e-4), exit 0 - losses match prior runs (iter 1: 12.187, iter 2: 12.190, iter 3: 12.177, resume iter 4: 11.817, iter 5: 11.264) The downstream check `if not hasattr(self, 'tp_group')` in subsequent descendants (TransformerBlock, TransformerLayer, Attention, MLP, ColumnParallelLinear) was already satisfied by their own constructors; verified by reading those files. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- examples/mimo/training/hetero/args.py | 75 +++- .../mimo/training/hetero/checkpointing.py | 334 ++++++++++++++++++ examples/mimo/training/hetero/loop.py | 27 +- examples/mimo/training/hetero/runtime.py | 6 +- megatron/core/models/mimo/optimizer.py | 77 +++- megatron/core/models/vision/radio.py | 5 + megatron/core/ssm/mamba_layer.py | 4 + megatron/core/ssm/mamba_mixer.py | 14 + .../unit_tests/models/test_mimo_checkpoint.py | 76 +++- 9 files changed, 589 insertions(+), 29 deletions(-) create mode 100644 examples/mimo/training/hetero/checkpointing.py diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 12b7f9f041d..90010d1a468 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -135,6 +135,72 @@ def parse_args() -> argparse.Namespace: train.add_argument("--seed", type=int, default=12345) train.add_argument("--log-interval", type=int, default=1) + ckpt = parser.add_argument_group("checkpointing") + ckpt.add_argument( + "--save", + type=str, + default=None, + help="Directory to save distributed checkpoints into. Each save creates iter_NNNNNNN/.", + ) + ckpt.add_argument( + "--load", + type=str, + default=None, + help=( + "Directory to resume from. If the directory has no completed checkpoint, " + "training starts from iteration 0." + ), + ) + ckpt.add_argument( + "--save-interval", + type=int, + default=None, + help=( + "Iteration interval between checkpoint saves. When unset, --save still " + "produces exactly one checkpoint at --train-iters (the final iter). Set to " + "an integer >=1 for periodic saves; the final iter is also always saved." + ), + ) + ckpt.add_argument("--no-save-optim", action="store_true", help="Skip optimizer state on save.") + ckpt.add_argument( + "--no-load-optim", + action="store_true", + help="Skip optimizer state on load (fresh optimizer at the loaded iteration).", + ) + ckpt.add_argument( + "--no-load-scheduler", action="store_true", help="Skip LR/WD scheduler state on load." + ) + ckpt.add_argument( + "--no-save-rng", action="store_true", help="Skip Python/NumPy/Torch/CUDA RNG state on save." + ) + ckpt.add_argument( + "--no-load-rng", + action="store_true", + help="Skip Python/NumPy/Torch/CUDA RNG state on load (start with fresh RNG).", + ) + ckpt.add_argument( + "--finetune", + action="store_true", + help=( + "Treat the load directory as a pretrained checkpoint: restart from iteration 0 and " + "skip optimizer + scheduler state regardless of the other flags." + ), + ) + ckpt.add_argument( + "--dist-ckpt-optim-fully-reshardable", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "Use the 'fully_reshardable' DistributedOptimizer sharding type so a saved " + "checkpoint can be reloaded under a different TP/EP layout. Defaults to False " + "('dp_reshardable', DP-only reshardable, lower save-time memory). Enable when " + "you intend to change --llm-tp / --llm-ep on resume. WARNING: this gathers " + "the full per-DP optimizer state on DP rank 0 during save; on <80 GB GPUs " + "(or when running near peak memory) the gather will OOM. Prefer leaving this " + "False unless you actually need cross-TP/EP resharding." + ), + ) + return parser.parse_args() @@ -163,6 +229,11 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") + if args.save_interval is not None and args.save_interval < 1: + raise ValueError("--save-interval must be >= 1 when set") + if args.save_interval is not None and args.save is None: + raise ValueError("--save-interval requires --save") + encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) @@ -190,9 +261,7 @@ def validate_energon_data_args(args: argparse.Namespace) -> None: if not args.tokenizer_model: raise ValueError("--tokenizer-model is required for --dataset-provider energon_multimodal") if args.model_provider not in (NEMOTRON_20L_MODEL_PROVIDER, NEMOTRON_54L_MODEL_PROVIDER): - raise ValueError( - "energon_multimodal is currently wired for Nemotron MoE VLM providers" - ) + raise ValueError("energon_multimodal is currently wired for Nemotron MoE VLM providers") if args.encoder_pp != 1 or args.llm_pp != 1: raise ValueError("energon_multimodal currently supports encoder and LLM PP size 1") if args.encoder_dp > args.llm_dp: diff --git a/examples/mimo/training/hetero/checkpointing.py b/examples/mimo/training/hetero/checkpointing.py new file mode 100644 index 00000000000..90a40c42059 --- /dev/null +++ b/examples/mimo/training/hetero/checkpointing.py @@ -0,0 +1,334 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Distributed checkpoint save/load for the heterogeneous MIMO training loop. + +Wraps `megatron.core.dist_checkpointing` so the standalone hetero loop can +persist MimoModel + MimoOptimizer + LR scheduler state without depending on +`megatron.training.checkpointing` (which assumes the parallel_state singleton). + +Stays intentionally close to the layout that `megatron/training/checkpointing.py` +produces so existing inspection tooling keeps working: + + / + latest_checkpointed_iteration.txt + iter_0000010/ + common.pt # args, checkpoint_version, iteration, scheduler + metadata.json # backend + version + sharding-type content_metadata + ...torch_dist shards... +""" + +from __future__ import annotations + +import argparse +import os +import random +from pathlib import Path +from typing import Any, Dict, Optional + +import numpy as np +import torch +import torch.distributed as dist + +from examples.mimo.training.hetero.distributed import print_rank_0 +from examples.mimo.training.hetero.topology import HeteroTopology, is_rank_in_grid +from examples.mimo.utils.hetero import is_process_group_member +from megatron.core import dist_checkpointing, tensor_parallel +from megatron.core.dist_checkpointing.mapping import ShardedObject +from megatron.core.dist_checkpointing.utils import _clean_metadata_for_serialization +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.models.mimo.optimizer import MimoOptimizer +from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + +_TRACKER_FILE = "latest_checkpointed_iteration.txt" +_CHECKPOINT_VERSION = 3.0 + + +def _iter_directory(root: str, iteration: int) -> str: + return os.path.join(root, f"iter_{iteration:07d}") + + +def _tracker_path(root: str) -> str: + return os.path.join(root, _TRACKER_FILE) + + +def _build_optim_metadata(args: argparse.Namespace) -> Dict[str, Any]: + """Optimizer-side metadata controlling DistributedOptimizer sharding format.""" + metadata: Dict[str, Any] = {"chained_optim_avoid_prefix": True, "singleton_local_shards": False} + if args.dist_ckpt_optim_fully_reshardable: + metadata["distrib_optim_sharding_type"] = "fully_reshardable" + else: + metadata["distrib_optim_sharding_type"] = "dp_reshardable" + return metadata + + +def _pg_rank_size(pg: Optional[dist.ProcessGroup]) -> tuple[int, int]: + """Return (rank, size) for a process group, or (0, 1) when this rank isn't a member.""" + if pg is not None and is_process_group_member(pg): + return pg.rank(), pg.size() + return 0, 1 + + +def _collect_rng_state(topology: HeteroTopology) -> Optional[Dict[str, ShardedObject]]: + """Collect this rank's Python/NumPy/Torch/CUDA RNG state, sharded by (pp, tp). + + Mirrors `megatron.training.checkpointing.get_rng_state` but reads pp/tp/dp + groups from the active hetero branch's pg_collection instead of parallel_state. + The returned dict has a single per-branch entry: encoder ranks publish + ``mimo..rng_state`` and LLM ranks publish ``mimo.language.rng_state`` + so the two branches don't collide on the same ShardedObject key. + Returns None when the rank is not in any branch (should not happen in + non-colocated layouts, but defensive). + """ + if is_rank_in_grid(topology.llm_grid): + pg = topology.language_pg + branch_name = "language" + elif is_rank_in_grid(topology.encoder_grid): + pg = topology.vision_pg + branch_name = topology.encoder_name + else: + return None + + rng_state = { + "random_rng_state": random.getstate(), + "np_rng_state": np.random.get_state(), + "torch_rng_state": torch.get_rng_state(), + "cuda_rng_state": torch.cuda.get_rng_state(), + "rng_tracker_states": tensor_parallel.get_cuda_rng_tracker().get_states(), + } + + pp_rank, pp_size = _pg_rank_size(getattr(pg, "pp", None)) + tp_rank, tp_size = _pg_rank_size(getattr(pg, "tp", None)) + dp_rank, _ = _pg_rank_size(getattr(pg, "dp", None)) + + key = f"mimo.{branch_name}.rng_state" + # One RNG snapshot per (pp, tp) shard; replicated across DP within that shard. + return { + key: ShardedObject( + key, [rng_state], (pp_size, tp_size), (pp_rank, tp_rank), replica_id=dp_rank + ) + } + + +def _restore_rng_state(rng_state_obj) -> None: + """Apply RNG state previously captured by `_collect_rng_state`.""" + if rng_state_obj is None: + return + rng_state_list = rng_state_obj + if isinstance(rng_state_list, list) and rng_state_list and isinstance(rng_state_list[0], dict): + rng_state = rng_state_list[0] + elif isinstance(rng_state_list, dict): + rng_state = rng_state_list + else: + # Unknown payload shape — skip silently rather than crash the run. + return + + random.setstate(rng_state["random_rng_state"]) + np.random.set_state(rng_state["np_rng_state"]) + torch.set_rng_state(rng_state["torch_rng_state"]) + torch.cuda.set_rng_state(rng_state["cuda_rng_state"]) + if rng_state.get("rng_tracker_states"): + tensor_parallel.get_cuda_rng_tracker().set_states(rng_state["rng_tracker_states"]) + + +def _assemble_state_dict( + model: MimoModel, + optimizer: Optional[MimoOptimizer], + opt_param_scheduler: Optional[OptimizerParamScheduler], + iteration: Optional[int], + args: argparse.Namespace, + topology: HeteroTopology, + include_optimizer: bool, + include_scheduler: bool, + include_rng: bool, + include_args: bool, + is_loading: bool, +) -> Dict[str, Any]: + """Build the (sharded) state dict consumed by `dist_checkpointing.save`/`load`. + + The MimoModel and MimoOptimizer already inject per-submodule `dp_cp_group` + from each module's pg_collection, so no global dp_cp_group needs to be set. + """ + state_dict: Dict[str, Any] = {"checkpoint_version": _CHECKPOINT_VERSION} + if iteration is not None: + state_dict["iteration"] = iteration + + if include_args: + # Stored as a plain dict (not vars(args) directly) so torch.save can pickle it + # via the common-state path. argparse.Namespace round-trips fine; using a dict + # gives us better forward-compat across argparse internals. + state_dict["args"] = dict(vars(args)) + + state_dict["model"] = model.sharded_state_dict() + + if include_optimizer and optimizer is not None and not optimizer.is_stub_optimizer: + optim_kwargs = {"metadata": _build_optim_metadata(args)} + state_dict["optimizer"] = optimizer.sharded_state_dict( + state_dict, is_loading=is_loading, **optim_kwargs + ) + + if include_scheduler and opt_param_scheduler is not None: + state_dict["opt_param_scheduler"] = opt_param_scheduler.state_dict() + + if include_rng: + rng = _collect_rng_state(topology) + if rng is not None: + # The dict contains exactly one entry; merge it at top level so each + # branch's ShardedObject lives under its own key (no cross-branch collision). + state_dict.update(rng) + + return state_dict + + +def save_checkpoint( + iteration: int, + model: MimoModel, + optimizer: Optional[MimoOptimizer], + opt_param_scheduler: Optional[OptimizerParamScheduler], + args: argparse.Namespace, + topology: HeteroTopology, +) -> None: + """Save a hetero MIMO checkpoint at iteration `iteration` under `args.save`.""" + if not args.save: + return + + save_root = args.save + target_dir = _iter_directory(save_root, iteration) + + # mkdir on every rank with exist_ok=True so a single rank's mkdir failure + # doesn't strand peers behind a barrier. + Path(target_dir).mkdir(parents=True, exist_ok=True) + dist.barrier() + + print_rank_0(f"saving hetero checkpoint at iteration {iteration} to {target_dir}") + + state_dict = _assemble_state_dict( + model=model, + optimizer=optimizer, + opt_param_scheduler=opt_param_scheduler, + iteration=iteration, + args=args, + topology=topology, + include_optimizer=not args.no_save_optim, + include_scheduler=True, + include_rng=not args.no_save_rng, + include_args=True, + is_loading=False, + ) + + content_metadata = _clean_metadata_for_serialization(_build_optim_metadata(args)) + + dist_checkpointing.save(state_dict, target_dir, content_metadata=content_metadata) + + if dist.get_rank() == 0: + tracker_tmp = _tracker_path(save_root) + ".tmp" + with open(tracker_tmp, "w") as f: + f.write(str(iteration)) + os.replace(tracker_tmp, _tracker_path(save_root)) + dist.barrier() + print_rank_0(f"hetero checkpoint at iteration {iteration} saved") + + +def _read_tracker(load_root: str) -> Optional[int]: + """Return the iteration recorded in the tracker file (max-reduced across ranks). + + Mirrors `megatron.training.checkpointing.read_metadata`: each rank reads the + local file and we agree on the largest value. None if no checkpoint exists + at this path on any rank. + """ + tracker = _tracker_path(load_root) + local_iter = -1 + if os.path.isfile(tracker): + with open(tracker) as f: + contents = f.read().strip() + if contents: + try: + local_iter = int(contents) + except ValueError as e: + raise RuntimeError(f"Tracker file {tracker} is corrupted: {contents!r}") from e + + if dist.is_available() and dist.is_initialized(): + iters_cuda = torch.tensor([local_iter], dtype=torch.long, device="cuda") + dist.all_reduce(iters_cuda, op=dist.ReduceOp.MAX) + max_iter = int(iters_cuda[0].item()) + else: + max_iter = local_iter + + return max_iter if max_iter >= 0 else None + + +def load_checkpoint( + model: MimoModel, + optimizer: Optional[MimoOptimizer], + opt_param_scheduler: Optional[OptimizerParamScheduler], + args: argparse.Namespace, + topology: HeteroTopology, +) -> int: + """Restore a hetero MIMO checkpoint from `args.load` and return the resume iteration. + + Returns 0 if `--load` is not set or no completed checkpoint exists at that path. + With `--finetune`, model state is loaded but iteration/optimizer/scheduler/rng + are reset. + """ + if not args.load: + return 0 + + load_root = args.load + iteration = _read_tracker(load_root) + if iteration is None: + print_rank_0(f"no checkpoint found at {load_root}; starting from iteration 0") + return 0 + + source_dir = _iter_directory(load_root, iteration) + if not os.path.isdir(source_dir): + raise RuntimeError( + f"Tracker at {load_root} points to iteration {iteration} but " + f"{source_dir} is missing" + ) + + is_finetune = bool(args.finetune) + include_optimizer = (not args.no_load_optim) and not is_finetune + include_scheduler = (not args.no_load_scheduler) and not is_finetune + include_rng = (not args.no_load_rng) and not is_finetune + + print_rank_0( + f"loading hetero checkpoint from {source_dir}" + f" (optimizer={'yes' if include_optimizer else 'no'}," + f" scheduler={'yes' if include_scheduler else 'no'}," + f" rng={'yes' if include_rng else 'no'}," + f" finetune={is_finetune})" + ) + + sharded_state_dict = _assemble_state_dict( + model=model, + optimizer=optimizer, + opt_param_scheduler=opt_param_scheduler, + iteration=iteration, + args=args, + topology=topology, + include_optimizer=include_optimizer, + include_scheduler=include_scheduler, + include_rng=include_rng, + include_args=False, # args round-trips via common.pt, not via the request dict + is_loading=True, + ) + + loaded = dist_checkpointing.load(sharded_state_dict, source_dir) + + model.load_state_dict(loaded["model"], strict=True) + + if include_optimizer and optimizer is not None and not optimizer.is_stub_optimizer: + optimizer.load_state_dict(loaded["optimizer"]) + + if include_scheduler and opt_param_scheduler is not None and "opt_param_scheduler" in loaded: + opt_param_scheduler.load_state_dict(loaded["opt_param_scheduler"]) + + if include_rng: + # Find this rank's per-branch rng key in the loaded dict. + for key, value in loaded.items(): + if key.startswith("mimo.") and key.endswith(".rng_state"): + _restore_rng_state(value) + break + + resume_iter = 0 if is_finetune else int(loaded.get("iteration", iteration)) + print_rank_0(f"resuming hetero training at iteration {resume_iter}") + return resume_iter diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 5e9574ccfe9..425eea4dc66 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -10,6 +10,7 @@ import torch from examples.mimo.training.hetero.args import prepare_args +from examples.mimo.training.hetero.checkpointing import load_checkpoint, save_checkpoint from examples.mimo.training.hetero.data import select_data_iterator, validate_data_iterator from examples.mimo.training.hetero.distributed import print_rank_0 from examples.mimo.training.hetero.grad_sync import configure_grad_sync @@ -61,14 +62,24 @@ def run_train_loop(args: argparse.Namespace) -> None: logger = HeteroTrainingLogger(args=args, topology=topology) debug_rank("training setup ready") + start_iteration = load_checkpoint(model, optimizer, opt_param_scheduler, args, topology) + if start_iteration >= args.train_iters: + print_rank_0( + f"Resume iteration ({start_iteration}) >= --train-iters ({args.train_iters}); " + "nothing to train." + ) + return + print_rank_0( "Starting hetero MIMO training: " f"world_size={world_size}, encoder_size={topology.encoder_size}, " - f"llm_size={topology.llm_size}, train_iters={args.train_iters}, " + f"llm_size={topology.llm_size}, " + f"iters={start_iteration + 1}..{args.train_iters}, " f"dataset_provider={args.dataset_provider}" ) - for iteration in range(1, args.train_iters + 1): + last_saved = start_iteration + for iteration in range(start_iteration + 1, args.train_iters + 1): debug_rank(f"iteration {iteration}: train step start") set_pipeline_timeline_iteration(iteration) result = train_step( @@ -78,6 +89,18 @@ def run_train_loop(args: argparse.Namespace) -> None: logger.record_step(result) logger.maybe_log(iteration, optimizer, result) debug_rank(f"iteration {iteration}: train step complete") + + if ( + args.save + and args.save_interval + and iteration % args.save_interval == 0 + and iteration != args.train_iters + ): + save_checkpoint(iteration, model, optimizer, opt_param_scheduler, args, topology) + last_saved = iteration + + if args.save and last_saved != args.train_iters: + save_checkpoint(args.train_iters, model, optimizer, opt_param_scheduler, args, topology) finally: close_pipeline_timeline() if model is not None: diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 10ba4eea0ac..64b967b2832 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -140,11 +140,7 @@ def configure_module_rng( seed = args.seed + role_seed_offset + (100 * pp_rank) torch.manual_seed(seed) model_parallel_cuda_manual_seed( - seed, - tp_rank=tp_rank, - ep_rank=ep_rank, - etp_rank=expt_tp_rank, - force_reset_rng=True, + seed, tp_rank=tp_rank, ep_rank=ep_rank, etp_rank=expt_tp_rank, force_reset_rng=True ) diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 41fad0d5dd8..737a5c0f482 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -13,6 +13,7 @@ import torch from megatron.core.dist_checkpointing.mapping import ShardedObject +from megatron.core.dist_checkpointing.utils import add_prefix_for_sharding from megatron.core.optimizer.clip_grads import clip_grad_by_total_norm_fp32 from megatron.core.optimizer.optimizer import MegatronOptimizer from megatron.core.optimizer.optimizer_config import OptimizerConfig @@ -150,9 +151,9 @@ def state_dict(self): def load_state_dict(self, state_dict: Dict): """Load per-module optimizer state dicts. - Reassembles param_groups and grad_scaler that were extracted and saved - as ShardedObjects by sharded_state_dict(), then delegates to each - per-module optimizer's load_state_dict. + Reassembles param_groups, grad_scaler, and param_state_sharding_type + that were extracted and saved as ShardedObjects by sharded_state_dict(), + then delegates to each per-module optimizer's load_state_dict. """ for name, info in self.module_infos.items(): if not (info.is_active and info.optimizer): @@ -163,14 +164,16 @@ def load_state_dict(self, state_dict: Dict): for sub_sd, inner_opt in _iter_optimizer_sub_dicts(module_sd, info.optimizer): _restore_param_groups(sub_sd, inner_opt, name) + _restore_param_state_sharding_type(sub_sd) _restore_grad_scaler(sub_sd) info.optimizer.load_state_dict(module_sd) def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, **kwargs): - """Build sharded state dict, routing param_groups and grad_scaler - through distributed save as ShardedObjects (common.pt is rank-0 only, - which misses LLM optimizer state in non-colocated mode). + """Build sharded state dict, routing param_groups, grad_scaler, and + param_state_sharding_type through distributed save as ShardedObjects + (common.pt is rank-0 only, which misses non-colocated LLM optimizer + state). """ sharded_state = {} for name, info in self.module_infos.items(): @@ -185,8 +188,14 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, ): suffix = f'.{idx}' if idx > 0 else '' _extract_param_groups(sub_sd, name, suffix, replica_id) + _extract_param_state_sharding_type(sub_sd, name, suffix, replica_id) _extract_grad_scaler(sub_sd, name, suffix, replica_id) + # Namespace every internal ShardedBase key with the submodule name + # so two module optimizers (e.g. 'language' + 'images') don't collide + # on identical inner keys like 'chained_0.optimizer.distributed.*'. + add_prefix_for_sharding(module_sd, f'mimo.{name}.') + sharded_state[name] = module_sd else: sharded_state[name] = {} @@ -228,6 +237,14 @@ def _extract_param_groups(sub_sd, module_name, suffix, replica_id): replica_id=replica_id, ) del opt_sub['param_groups'] + # Drop the now-empty `optimizer` wrapper. If we left it in place, the + # empty dict would round-trip through dist_checkpointing's common-state + # path with no defined behavior on the load side; explicitly removing + # it pairs with the `setdefault` in `_restore_param_groups` so the load + # path always rebuilds a clean wrapper. Pattern from + # https://github.com/NVIDIA/Megatron-LM/pull/4791. + if not opt_sub: + del sub_sd['optimizer'] def _extract_grad_scaler(sub_sd, module_name, suffix, replica_id): @@ -242,6 +259,25 @@ def _extract_grad_scaler(sub_sd, module_name, suffix, replica_id): ) +def _extract_param_state_sharding_type(sub_sd, module_name, suffix, replica_id): + """Save: extract param_state_sharding_type into a ShardedObject. + + Plain non-tensor scalars at the per-module level otherwise travel through + dist_checkpointing's common-state path (rank 0 only), so for non-colocated + MIMO they are lost on ranks whose module is inactive on rank 0. + `DistributedOptimizer.load_state_dict` asserts on the missing key, so it + must round-trip explicitly. Pattern from NVIDIA/Megatron-LM#4791. + """ + if 'param_state_sharding_type' in sub_sd: + sub_sd[f'_mimo_param_state_sharding_type{suffix}'] = ShardedObject( + f'optimizer.mimo.{module_name}{suffix}.param_state_sharding_type', + sub_sd.pop('param_state_sharding_type'), + (1,), + (0,), + replica_id=replica_id, + ) + + def _restore_param_groups(sub_sd, inner_optimizer, module_name): """Load: restore param_groups with current param IDs from the inner optimizer.""" # Find the _mimo_param_groups key (may have a suffix for chained optimizers) @@ -263,7 +299,14 @@ def _restore_param_groups(sub_sd, inner_optimizer, module_name): ) for loaded_g, current_g in zip(loaded_pg, current_pg): loaded_g['params'] = current_g['params'] - sub_sd['optimizer']['param_groups'] = loaded_pg + # `sub_sd['optimizer']` may be absent on load: when the per-module state_dict + # produced by `DistributedOptimizer.state_dict()` only contains + # `param_groups` under the 'optimizer' key, `_extract_param_groups` deletes + # `param_groups` at save time, and the resulting empty dict can be dropped + # by dist_checkpointing's common-state round-trip on ranks whose active + # module wasn't on rank 0. `setdefault` lets the restored `param_groups` + # land in the right place regardless. Pattern from NVIDIA/Megatron-LM#4801. + sub_sd.setdefault('optimizer', {})['param_groups'] = loaded_pg def _restore_grad_scaler(sub_sd): @@ -274,20 +317,34 @@ def _restore_grad_scaler(sub_sd): break +def _restore_param_state_sharding_type(sub_sd): + """Load: restore param_state_sharding_type from its ShardedObject key.""" + for k in list(sub_sd.keys()): + if k.startswith('_mimo_param_state_sharding_type'): + sub_sd['param_state_sharding_type'] = sub_sd.pop(k) + break + + def _get_replica_id(pg_collection: Optional[ProcessGroupCollection]) -> tuple: """Build replica_id tuple for ShardedObject deduplication. - Includes pp_rank so only one PP stage writes the metadata, - and dp_rank so only dp_rank=0 writes (others are replicas). + Returns ``(tp_rank, pp_rank, dp_rank)`` so only ``(0, 0, 0)`` within each + module's parallelism group is the main replica; all other ranks in the same + module are non-main replicas of the same object. Order matches + `make_sharded_object_for_checkpoint` in + `megatron/core/transformer/utils.py:168-172` and NVIDIA/Megatron-LM#4801. """ assert pg_collection is not None, "pg_collection required for checkpoint replica_id" + assert ( + hasattr(pg_collection, 'tp') and pg_collection.tp is not None + ), "pg_collection.tp must be set for checkpoint deduplication" assert ( hasattr(pg_collection, 'pp') and pg_collection.pp is not None ), "pg_collection.pp must be set for checkpoint deduplication" assert ( hasattr(pg_collection, 'dp') and pg_collection.dp is not None ), "pg_collection.dp must be set for checkpoint deduplication" - return (0, pg_collection.pp.rank(), pg_collection.dp.rank()) + return (pg_collection.tp.rank(), pg_collection.pp.rank(), pg_collection.dp.rank()) def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: diff --git a/megatron/core/models/vision/radio.py b/megatron/core/models/vision/radio.py index b7d245da5c1..3eb3e227e67 100644 --- a/megatron/core/models/vision/radio.py +++ b/megatron/core/models/vision/radio.py @@ -127,6 +127,11 @@ def __init__( # Using non-TE version so we can force gather_output tp_group = getattr(pg_collection, "tp", None) if pg_collection is not None else None + # Store tp_group on self so MegatronModule.sharded_state_dict doesn't + # fall back to parallel_state.get_tensor_model_parallel_group(), which + # isn't initialized in heterogeneous-parallelism layouts that pass + # pg_collection explicitly. + self.tp_group = tp_group self.embedder = ColumnParallelLinear( input_size=3 * self.patch_dim * self.patch_dim, output_size=self.visual_hidden_size, diff --git a/megatron/core/ssm/mamba_layer.py b/megatron/core/ssm/mamba_layer.py index 17903cebf3b..6147d6ae689 100644 --- a/megatron/core/ssm/mamba_layer.py +++ b/megatron/core/ssm/mamba_layer.py @@ -80,6 +80,10 @@ def __init__( self.submodules_config = submodules self.layer_number = layer_number self.hidden_dropout = config.hidden_dropout + # Store tp_group so MegatronModule.sharded_state_dict doesn't fall back + # to parallel_state.get_tensor_model_parallel_group(); the hetero + # MIMO loop never initializes parallel_state. + self.tp_group = pg_collection.tp self.mixer = build_module( submodules.mixer, self.config, diff --git a/megatron/core/ssm/mamba_mixer.py b/megatron/core/ssm/mamba_mixer.py index 727c6ef5fd6..a99545b4e3c 100644 --- a/megatron/core/ssm/mamba_mixer.py +++ b/megatron/core/ssm/mamba_mixer.py @@ -95,6 +95,17 @@ class ExtendedRMSNorm(RMSNormGated): RMSNormGated with sharded state dict. """ + def __init__(self, *args, tp_group=None, **kwargs): + super().__init__(*args, **kwargs) + # Store tp_group eagerly so MegatronModule.sharded_state_dict and the + # method below don't have to fall back to + # parallel_state.get_tensor_model_parallel_group() — that fallback is + # unavailable in heterogeneous layouts that don't initialize + # parallel_state. Callers that don't pass tp_group keep the old lazy + # fallback behavior via `hasattr` in `sharded_state_dict`. + if tp_group is not None: + self.tp_group = tp_group + def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 0, bias not sharded""" if not hasattr(self, 'tp_group'): @@ -373,6 +384,7 @@ def __init__( norm_before_gate=self.norm_before_gate, device=torch.cuda.current_device(), dtype=config.params_dtype, + tp_group=self.pg_collection.tp, ) setattr(self.norm.weight, "tensor_model_parallel", True) setattr(self.norm.weight, "partition_dim", 0) @@ -1284,6 +1296,8 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): "D": 0, }, # parameters sharded across TP sharded_offsets=sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) # Submodules for name, module in self.named_children(): diff --git a/tests/unit_tests/models/test_mimo_checkpoint.py b/tests/unit_tests/models/test_mimo_checkpoint.py index 3dc75a05a87..800759a5e6c 100644 --- a/tests/unit_tests/models/test_mimo_checkpoint.py +++ b/tests/unit_tests/models/test_mimo_checkpoint.py @@ -56,10 +56,25 @@ def _randomize_params(model, seed): p.random_() -def _create_model_and_optimizer(encoder_grid, llm_grid, hidden_size, num_layers, vocab_size, seed): +def _create_model_and_optimizer( + encoder_grid, + llm_grid, + hidden_size, + num_layers, + vocab_size, + seed, + use_distributed_optimizer=False, +): """Create MIMO model with DDP + optimizer, do a fake step to populate optimizer state. Caller must call create_all_embedding_groups() before this function. + + With ``use_distributed_optimizer=False`` (default) the inner optimizer is + Float16Optimizer, which exercises the MIMO-specific param_groups/grad_scaler + extraction in sharded_state_dict. With ``use_distributed_optimizer=True`` the + inner is a ChainedOptimizer of DistributedOptimizers, exercising the + ``mimo.{name}.`` prefix walk and the ``_extract_param_state_sharding_type`` + helper that wraps DistributedOptimizer's per-module sharding-type string. """ torch.manual_seed(seed) @@ -74,22 +89,27 @@ def _create_model_and_optimizer(encoder_grid, llm_grid, hidden_size, num_layers, ) _randomize_params(mimo_model, seed) - # Use Float16Optimizer (not DistributedOptimizer) to exercise the MIMO-specific - # param_groups/grad_scaler extraction in sharded_state_dict. DistributedOptimizer - # handles its own checkpointing internally and our code is transparent to it. opt_config = OptimizerConfig( optimizer='adam', lr=1e-4, weight_decay=0.01, clip_grad=1.0, bf16=True, - use_distributed_optimizer=False, + use_distributed_optimizer=use_distributed_optimizer, ) optimizer = get_mimo_optimizer(mimo_model, opt_config) - # Fake backward + step to populate optimizer state (Adam m/v) + # Fake backward + step to populate optimizer state (Adam m/v). + # DistributedOptimizer reads grads from each DDP wrapper's main_grad buffer + # rather than from param.grad, so populate the buffer directly. for param in mimo_model.parameters(): - param.grad = torch.randn_like(param) + if not param.requires_grad: + continue + grad = torch.randn_like(param) + if hasattr(param, "main_grad") and param.main_grad is not None: + param.main_grad.copy_(grad.to(param.main_grad.dtype)) + else: + param.grad = grad optimizer.step() return mimo_model, optimizer @@ -107,6 +127,7 @@ def run_checkpoint_test( hidden_size=256, num_layers=2, vocab_size=1000, + use_distributed_optimizer=False, ): """Save model + optimizer checkpoint, load into fresh instances, verify match.""" # Clear NVTE env vars that the conftest set_env fixture sets to '0'. @@ -123,7 +144,13 @@ def run_checkpoint_test( # --- Create model A + optimizer, snapshot state --- model_a, optimizer_a = _create_model_and_optimizer( - encoder_grid, llm_grid, hidden_size, num_layers, vocab_size, seed=1 + encoder_grid, + llm_grid, + hidden_size, + num_layers, + vocab_size, + seed=1, + use_distributed_optimizer=use_distributed_optimizer, ) params_a = {name: p.clone() for name, p in model_a.named_parameters()} @@ -147,7 +174,13 @@ def run_checkpoint_test( # --- Create model B + optimizer with different weights (reuse same grids) --- model_b, optimizer_b = _create_model_and_optimizer( - encoder_grid, llm_grid, hidden_size, num_layers, vocab_size, seed=2 + encoder_grid, + llm_grid, + hidden_size, + num_layers, + vocab_size, + seed=2, + use_distributed_optimizer=use_distributed_optimizer, ) # Load model @@ -271,3 +304,28 @@ def test_encoder_tp2_pp2_llm_tp2_pp2(self): hidden_size=256, num_layers=2, ) + + def test_encoder_tp2_llm_tp2_pp3_distributed_optimizer(self): + """Same shape as test_encoder_tp2_llm_tp2_pp3 but with DistributedOptimizer. + + Exercises the `mimo.{name}.` ShardedObject-key prefix walk that prevents + the two branches' optimizers from colliding, and the new + `_extract_param_state_sharding_type` helper that re-routes + DistributedOptimizer's top-level sharding-type string through a per-module + ShardedObject (so non-rank-0 owners don't lose it on reload). + """ + if self.world_size != 8: + pytest.skip(f"Requires 8 GPUs, got {self.world_size}") + run_checkpoint_test( + encoder_tp=2, + encoder_pp=1, + encoder_dp=1, + encoder_offset=0, + llm_tp=2, + llm_pp=3, + llm_dp=1, + llm_offset=2, + hidden_size=256, + num_layers=3, + use_distributed_optimizer=True, + ) From edc6037a6b4d6966337b2f4c76554b39f896b095 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Fri, 15 May 2026 21:31:09 -0700 Subject: [PATCH 34/44] NMFW-464: Skip MIMO optimizer build for all-frozen modules; scope _get_param_groups all-gather (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two convergent fixes for MIMO + frozen modules under non-colocated parallelism, cherry-picking the pattern from NVIDIA/Megatron-LM#4790 (Li Ding, "Fix MIMO optimizer setup for frozen modules") with comments updated to cite the source and link to NMFW-464 context. ## Problem 1 — placeholder optimizer for all-frozen modules `get_mimo_optimizer` previously called `get_megatron_optimizer` on every non-None module, including modules whose params are all frozen on every rank in the module's group. The most common trigger is `--training-stage stage1` (the LLaVA projector-only recipe), which sets `--freeze-vit --freeze-lm` and leaves the language model with zero trainable parameters on the LLM ranks. The resulting placeholder DistributedOptimizer either crashes in downstream setup or behaves silently incorrectly (e.g., LR scheduler advancing an empty group, save of a degenerate optimizer state). ## Problem 2 — `_get_param_groups` all-gathers over WORLD `_get_param_groups` reconciles param-group keys across ranks of the same model via `all_gather_object(params_key, …)` over the global default group. In non-colocated MIMO, encoder ranks and LLM ranks are disjoint and own different params (RADIO vs Mamba), so a WORLD-group all-gather pollutes both branches with the other branch's keys. ## Fixes - `megatron/core/models/mimo/optimizer.py`: add `_module_has_any_trainable_parameters(module, pg_collection)` — an all-reduce-MAX over `pg_collection.intra_dist_opt` of the local trainable-param count. Gate the `get_megatron_optimizer` call on it. When false, leave `info.optimizer = None` so `MimoOptimizer.is_stub_optimizer` handles the branch (that path was already designed for this; the gating never triggered it before). - `megatron/core/optimizer/__init__.py`: add an optional `process_group=None` kwarg to `_get_param_groups` / `_get_param_groups_and_buffers`, plumbed through `_get_megatron_emerging_optimizer` and `get_megatron_optimizer`, so the cross-rank `all_gather_object` can target a specific group. MIMO passes `pg_collection.intra_dist_opt`. Default `None` preserves the current WORLD-group behavior for every existing non-MIMO caller. ## Validation cw-dfw 1-node 8-GPU 20L mock, stage1 (vit + lm frozen, projector-only): - 3 iters, exit 0 - grad norm 0.025 / 0.020 / 0.016 (consistent with only projector params having live grads) - The `learning rate: ...` field in the iteration log is now absent — the previous-PR-era smoke printed it, because the placeholder language optimizer was being queried for an LR; with this fix the language optimizer is correctly `None` and the logger has nothing to query. That's the visible signature of the fix landing. Co-authored-by: Claude Opus 4.7 (1M context) --- megatron/core/models/mimo/optimizer.py | 36 +++++++++++++++++++++++++- megatron/core/optimizer/__init__.py | 24 ++++++++++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 737a5c0f482..054f9677dd3 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -347,6 +347,31 @@ def _get_replica_id(pg_collection: Optional[ProcessGroupCollection]) -> tuple: return (pg_collection.tp.rank(), pg_collection.pp.rank(), pg_collection.dp.rank()) +def _module_has_trainable_parameters(module) -> bool: + """Return whether this rank owns any trainable parameters for a module.""" + return module is not None and any(param.requires_grad for param in module.parameters()) + + +def _module_has_any_trainable_parameters(module, pg_collection: ProcessGroupCollection) -> bool: + """Return whether any rank in the module optimizer group has trainable parameters. + + Without this cross-rank check, `get_mimo_optimizer` would call + `get_megatron_optimizer` on a module whose params are all frozen on every + rank (e.g. the language model under stage1 = ``--freeze-vit --freeze-lm``), + producing a placeholder optimizer that breaks downstream setup. Pattern + from NVIDIA/Megatron-LM#4790. + """ + local_has_params = torch.tensor( + [int(_module_has_trainable_parameters(module))], + device=torch.cuda.current_device(), + dtype=torch.int, + ) + torch.distributed.all_reduce( + local_has_params, op=torch.distributed.ReduceOp.MAX, group=pg_collection.intra_dist_opt + ) + return bool(local_has_params.item()) + + def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: """Create ProcessGroupCollection from HyperCommGrid for optimizer use. @@ -390,7 +415,16 @@ def get_mimo_optimizer(mimo_model: "MimoModel", config: OptimizerConfig) -> Mimo else: module = mimo_model.modality_submodules[module_name] - if module is not None: + # Skip the optimizer build when no rank in this module's + # intra-dist-opt group has any trainable parameters (e.g. the + # language model under stage1 = `--freeze-vit --freeze-lm`). + # Leaving `optimizer = None` lets `MimoOptimizer.is_stub_optimizer` + # handle the branch correctly, instead of constructing a + # placeholder DistributedOptimizer that breaks downstream setup. + module_has_trainable_params = _module_has_any_trainable_parameters( + module, pg_collection + ) + if module is not None and module_has_trainable_params: assert ( not hasattr(module, 'ddp_config') or module.ddp_config is None diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py index c6d3e41aed5..fe4d48d0c2b 100644 --- a/megatron/core/optimizer/__init__.py +++ b/megatron/core/optimizer/__init__.py @@ -300,6 +300,7 @@ def _get_param_groups( model_chunks: List[MegatronModule], config: OptimizerConfig, config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], + process_group: Optional[torch.distributed.ProcessGroup] = None, ) -> List[Dict]: """Create parameter groups for optimizer. @@ -360,8 +361,10 @@ def _get_param_groups( # so we need to align the param groups across ranks, otherwise we may have # runtime error when loading the checkpoint or numerical error when resuming training. params_key = list(params_map.keys()) - gathered_params_key = [None for _ in range(torch.distributed.get_world_size())] - torch.distributed.all_gather_object(gathered_params_key, params_key) + gathered_params_key = [ + None for _ in range(torch.distributed.get_world_size(group=process_group)) + ] + torch.distributed.all_gather_object(gathered_params_key, params_key, group=process_group) for keys in gathered_params_key: for key in keys: if key not in params_key: @@ -419,6 +422,7 @@ def _get_param_groups_and_buffers( config_overrides: Optional[Dict[ParamKey, ParamGroupOverride]], filter_fn: Callable, buffer_name: str, + process_group: Optional[torch.distributed.ProcessGroup] = None, ) -> Tuple[List[Dict], Dict[int, List[_ParamAndGradBuffer]]]: """Returns parameter groups and buffer for optimizer. @@ -437,7 +441,9 @@ def _get_param_groups_and_buffers( Returns: List of parameter groups and dictionary of model chunk IDs to buffers. """ - param_groups = _get_param_groups(model_chunks, config, config_overrides) + param_groups = _get_param_groups( + model_chunks, config, config_overrides, process_group=process_group + ) param_groups = list(filter(filter_fn, param_groups)) buffers = {} for model_chunk_idx, model_chunk in enumerate(model_chunks): @@ -762,6 +768,7 @@ def _get_megatron_emerging_optimizer( if config.fp16: raise ValueError('emerging optimizer with fp16 is not supported.') + uses_explicit_pg_collection = pg_collection is not None if pg_collection is None: pg_collection = ProcessGroupCollection.use_mpu_process_groups() @@ -783,7 +790,12 @@ def _get_megatron_emerging_optimizer( # Build param groups and bucket by (optimizer_name, is_expert_parallel). # Layer-wise distributed optimizer handles expert params internally so we skip that split. - all_param_groups = _get_param_groups(model_chunks, config, config_overrides) + param_group_process_group = ( + getattr(pg_collection, 'intra_dist_opt', None) if uses_explicit_pg_collection else None + ) + all_param_groups = _get_param_groups( + model_chunks, config, config_overrides, process_group=param_group_process_group + ) grouped_param_groups = defaultdict(list) for group in all_param_groups: opt_name = group.get('optimizer', eopt_name) @@ -926,6 +938,7 @@ def get_megatron_optimizer( intra_dp_cp_group_gloo = process_groups_dict['intra_dp_cp_group_gloo'] intra_expt_dp_group_gloo = process_groups_dict['intra_expt_dp_group_gloo'] intra_dist_opt_group = process_groups_dict['intra_dist_opt_group'] + param_group_process_group = intra_dist_opt_group if pg_collection is not None else None model_parallel_rank = get_pg_rank(mp_group) @@ -949,6 +962,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: True, buffer_name='buffers', + process_group=param_group_process_group, ) optimizer_part = _get_megatron_optimizer_based_on_param_groups( @@ -999,6 +1013,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: not g['is_expert_parallel'], buffer_name='buffers', + process_group=param_group_process_group, ) for model_chunk in dense_model_chunks: model_chunk.overlap_param_gather_with_optimizer_step = ( @@ -1036,6 +1051,7 @@ def get_megatron_optimizer( config_overrides=config_overrides, filter_fn=lambda g: g['is_expert_parallel'], buffer_name='expert_parallel_buffers', + process_group=param_group_process_group, ) if dump_param_to_param_group_map is not None: for param_group in moe_param_groups: From 2491b43aa1bfc7ce1ea1fecfc4b55a8c3321b56f Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Fri, 15 May 2026 22:15:22 +0000 Subject: [PATCH 35/44] NMFW-464: Add LLM-only hetero MIMO launch path --- examples/mimo/data/hetero_energon.py | 6 ++- .../run_hetero_nemotron_54l_hel_train.sh | 12 ++++- .../sbatch_hetero_nemotron_54l_hel_9n.sh | 15 ++++-- ..._mimo_nemotron_54l_hel_8n_text_only_llm.sh | 50 +++++++++++++++++++ examples/mimo/training/hetero/args.py | 38 +++++++++++--- examples/mimo/training/hetero/data.py | 23 +++++++-- examples/mimo/training/hetero/loop.py | 5 +- examples/mimo/training/hetero/runtime.py | 27 +++++++--- examples/mimo/training/hetero/timeline.py | 6 ++- examples/mimo/training/hetero/topology.py | 47 ++++++++++------- 10 files changed, 185 insertions(+), 44 deletions(-) create mode 100755 examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 76e9ae4ebdd..2b7deaf9d5a 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -21,8 +21,10 @@ def build_energon_iterator(args, topology): encoder_grid = topology.encoder_grid llm_grid = topology.llm_grid - encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( - encoder_grid.get_pg("pp") + encoder_needs_data = ( + encoder_grid is not None + and is_rank_in_grid(encoder_grid) + and is_pp_first_stage(encoder_grid.get_pg("pp")) ) llm_needs_data = is_rank_in_grid(llm_grid) and ( is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) diff --git a/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh index 74be1ed2134..8cceef1d231 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh @@ -56,12 +56,18 @@ LLM_PP="${LLM_PP:-1}" LLM_DP="${LLM_DP:-64}" LLM_EP="${LLM_EP:-16}" LLM_EXPT_TP="${LLM_EXPT_TP:-1}" +LLM_ONLY="${LLM_ONLY:-0}" ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-0}" ENCODER_SIZE=$((ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP)) LLM_SIZE=$((LLM_TP * LLM_CP * LLM_PP * LLM_DP)) -LLM_OFFSET="${LLM_OFFSET:-${ENCODER_SIZE}}" +if [[ "${LLM_ONLY}" == "1" || "${LLM_ONLY}" == "true" ]]; then + ENCODER_SIZE=0 + LLM_OFFSET="${LLM_OFFSET:-0}" +else + LLM_OFFSET="${LLM_OFFSET:-${ENCODER_SIZE}}" +fi EXPECTED_WORLD_SIZE=$((ENCODER_SIZE + LLM_SIZE)) LLM_EXPT_DP="${LLM_EXPT_DP:-$((LLM_SIZE / (LLM_EXPT_TP * LLM_EP * LLM_PP)))}" @@ -193,6 +199,7 @@ if [[ "${RANK_ID}" -eq 0 ]]; then echo "=== Hetero MIMO Nemotron6-MoE VLM 54L HEL training ===" echo "model_provider=${MODEL_PROVIDER}" echo "stage=${TRAINING_STAGE} train_iters=${TRAIN_ITERS} mbs=${MICRO_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} gbs=${GLOBAL_BATCH_SIZE}" + echo "llm_only=${LLM_ONLY}" echo "layout=encoder(tp=${ENCODER_TP},cp=${ENCODER_CP},pp=${ENCODER_PP},dp=${ENCODER_DP},ep=${ENCODER_EP}) llm(tp=${LLM_TP},cp=${LLM_CP},pp=${LLM_PP},dp=${LLM_DP},ep=${LLM_EP},etp=${LLM_EXPT_TP},edp=${LLM_EXPT_DP}) world=${EXPECTED_WORLD_SIZE}" echo "enable_experimental=${ENABLE_EXPERIMENTAL}" echo "moe_router_force_load_balancing=${MOE_ROUTER_FORCE_LOAD_BALANCING}" @@ -218,6 +225,9 @@ fi if [[ "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "1" || "${MOE_ROUTER_FORCE_LOAD_BALANCING}" == "true" ]]; then MODEL_ARGS+=(--moe-router-force-load-balancing) fi +if [[ "${LLM_ONLY}" == "1" || "${LLM_ONLY}" == "true" ]]; then + MODEL_ARGS+=(--llm-only) +fi CMD=( "${PYTHON_BIN}" -u examples/mimo/train_hetero.py diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh index a2a6ba7590d..9c6813d4c3e 100755 --- a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh @@ -131,6 +131,7 @@ export ENCODER_CP="${ENCODER_CP:-1}" export ENCODER_PP="${ENCODER_PP:-1}" export ENCODER_DP="${ENCODER_DP:-8}" export ENCODER_EP="${ENCODER_EP:-1}" +export LLM_ONLY="${LLM_ONLY:-0}" export LLM_TP="${LLM_TP:-4}" export LLM_CP="${LLM_CP:-1}" export LLM_PP="${LLM_PP:-1}" @@ -147,10 +148,15 @@ export CHECK_HEL_PATHS="${CHECK_HEL_PATHS:-1}" export ENABLE_EXPERIMENTAL="${ENABLE_EXPERIMENTAL:-1}" export MOE_ROUTER_FORCE_LOAD_BALANCING="${MOE_ROUTER_FORCE_LOAD_BALANCING:-1}" -WORLD_SIZE=$((ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP + LLM_TP * LLM_CP * LLM_PP * LLM_DP)) -if [[ "${WORLD_SIZE}" -ne 72 ]]; then - echo "ERROR: This 9-node sbatch expects 72 ranks, but layout computed WORLD_SIZE=${WORLD_SIZE}" >&2 - exit 1 +if [[ "${LLM_ONLY}" == "1" || "${LLM_ONLY}" == "true" ]]; then + export LLM_OFFSET="${LLM_OFFSET:-0}" + WORLD_SIZE=$((LLM_TP * LLM_CP * LLM_PP * LLM_DP)) +else + WORLD_SIZE=$((ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP + LLM_TP * LLM_CP * LLM_PP * LLM_DP)) + if [[ "${WORLD_SIZE}" -ne 72 ]]; then + echo "ERROR: This 9-node sbatch expects 72 ranks, but layout computed WORLD_SIZE=${WORLD_SIZE}" >&2 + exit 1 + fi fi if [[ -n "${SLURM_NTASKS:-}" && "${SLURM_NTASKS}" -ne "${WORLD_SIZE}" ]]; then echo "ERROR: SLURM_NTASKS=${SLURM_NTASKS}, expected ${WORLD_SIZE}" >&2 @@ -185,6 +191,7 @@ echo "container_image=${CONTAINER_IMAGE}" echo "env_root=${ENV_ROOT}" echo "world_size=${WORLD_SIZE}" echo "gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} train_iters=${TRAIN_ITERS}" +echo "llm_only=${LLM_ONLY}" echo "layout=encoder(tp=${ENCODER_TP},dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP},etp=${LLM_EXPT_TP})" echo "timeline=${ENABLE_TIMELINE:-1} timeline_dir=${TIMELINE_DIR}" echo "================================================" diff --git a/examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh b/examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh new file mode 100755 index 00000000000..68b35dea017 --- /dev/null +++ b/examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Submit the HEL 54L MIMO LLM-only baseline that matches the 9-node VLM run's LLM grid. +# +# Intended use from a Cog-synced nb-hel workspace: +# sbatch examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 8 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=00:45:00 +#SBATCH -J mimo54l8nt +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -z "${REPO_ROOT:-}" ]]; then + if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" + else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + fi +fi + +export REPO_ROOT +export DATA_PATH="${DATA_PATH:-${REPO_ROOT}/examples/mimo/blend_files/text_only_1t_hel.yaml}" +export RUN_NAME="${RUN_NAME:-mimo54l-hel-8n-text-only-llm-tp2-ep8-gbs192}" +export TRAIN_ITERS="${TRAIN_ITERS:-30}" +export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-192}" +export MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +export NUM_MICROBATCHES="${NUM_MICROBATCHES:-6}" +export LOG_INTERVAL="${LOG_INTERVAL:-1}" + +export LLM_ONLY=1 +export LLM_OFFSET=0 +export LLM_TP="${LLM_TP:-2}" +export LLM_CP="${LLM_CP:-1}" +export LLM_PP="${LLM_PP:-1}" +export LLM_DP="${LLM_DP:-32}" +export LLM_EP="${LLM_EP:-8}" +export LLM_EXPT_TP="${LLM_EXPT_TP:-1}" + +export ENABLE_TIMELINE="${ENABLE_TIMELINE:-0}" +export NUM_WORKERS="${NUM_WORKERS:-4}" + +exec bash "${REPO_ROOT}/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh" "$@" diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 90010d1a468..9991803e96e 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -42,6 +42,14 @@ def parse_args() -> argparse.Namespace: grid.add_argument("--llm-ep", type=int, default=2) grid.add_argument("--llm-expt-tp", type=int, default=1) grid.add_argument("--llm-expt-dp", type=int, default=None) + grid.add_argument( + "--llm-only", + action="store_true", + help=( + "Run only the MIMO language module on the LLM grid. This keeps the MIMO " + "training/data path but does not create encoder ranks or bridge communicators." + ), + ) add_model_provider_args(parser) @@ -226,16 +234,30 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: validate_energon_data_args(args) if args.num_moe_experts > 0 and args.num_moe_experts % args.llm_ep != 0: raise ValueError("--num-moe-experts must be divisible by --llm-ep") - if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: - raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") - if args.save_interval is not None and args.save_interval < 1: raise ValueError("--save-interval must be >= 1 when set") if args.save_interval is not None and args.save is None: raise ValueError("--save-interval requires --save") - encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp + if args.llm_only: + if args.llm_offset != 0: + raise ValueError( + "--llm-only requires --llm-offset 0 so language ranks cover WORLD_SIZE" + ) + llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) + all_ranks = set(range(world_size)) + if llm_ranks != all_ranks: + raise ValueError( + "--llm-only requires the language grid to cover every torchrun rank exactly " + f"once; covered={sorted(llm_ranks)}, world={sorted(all_ranks)}" + ) + return 0, llm_size + + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: + raise ValueError("--micro-batch-size * --llm-dp must be divisible by --encoder-dp") + + encoder_size = args.encoder_tp * args.encoder_cp * args.encoder_pp * args.encoder_dp encoder_ranks = set(range(args.encoder_offset, args.encoder_offset + encoder_size)) llm_ranks = set(range(args.llm_offset, args.llm_offset + llm_size)) all_ranks = set(range(world_size)) @@ -262,8 +284,12 @@ def validate_energon_data_args(args: argparse.Namespace) -> None: raise ValueError("--tokenizer-model is required for --dataset-provider energon_multimodal") if args.model_provider not in (NEMOTRON_20L_MODEL_PROVIDER, NEMOTRON_54L_MODEL_PROVIDER): raise ValueError("energon_multimodal is currently wired for Nemotron MoE VLM providers") - if args.encoder_pp != 1 or args.llm_pp != 1: - raise ValueError("energon_multimodal currently supports encoder and LLM PP size 1") + if args.llm_pp != 1: + raise ValueError("energon_multimodal currently supports LLM PP size 1") + if args.llm_only: + return + if args.encoder_pp != 1: + raise ValueError("energon_multimodal currently supports encoder PP size 1") if args.encoder_dp > args.llm_dp: raise ValueError( "energon_multimodal currently supports fan-out only: --encoder-dp must be " diff --git a/examples/mimo/training/hetero/data.py b/examples/mimo/training/hetero/data.py index 419e902839a..b79f5272353 100644 --- a/examples/mimo/training/hetero/data.py +++ b/examples/mimo/training/hetero/data.py @@ -31,7 +31,11 @@ def validate_data_iterator( args: argparse.Namespace, data_iterator, topology: HeteroTopology ) -> None: """Run data-provider checks that must happen outside the pipeline schedule.""" - if args.dataset_provider == "energon_multimodal" and args.validate_energon_data_alignment: + if ( + args.dataset_provider == "energon_multimodal" + and args.validate_energon_data_alignment + and topology.encoder_grid is not None + ): from examples.mimo.data.hetero_energon import validate_energon_data_alignment validate_energon_data_alignment(data_iterator, topology) @@ -42,12 +46,25 @@ def select_mock_data_iterator( ) -> Optional[MockVLMIterator]: """Create the per-role mock-data iterator needed by local ranks.""" llm_mbs = args.micro_batch_size + encoder_grid = topology.encoder_grid + llm_grid = topology.llm_grid + if encoder_grid is None: + llm_needs_data = is_rank_in_grid(llm_grid) and ( + is_pp_first_stage(llm_grid.get_pg("pp")) or is_pp_last_stage(llm_grid.get_pg("pp")) + ) + if llm_needs_data: + return MockVLMIterator( + args, + llm_mbs, + topology.encoder_name, + get_mock_data_seed(args, llm_grid, module_seed_offset=100_000), + ) + return None + if (args.micro_batch_size * args.llm_dp) % args.encoder_dp != 0: raise ValueError("micro_batch_size * llm_dp must be divisible by encoder_dp") encoder_mbs = args.micro_batch_size * args.llm_dp // args.encoder_dp - encoder_grid = topology.encoder_grid - llm_grid = topology.llm_grid encoder_needs_data = is_rank_in_grid(encoder_grid) and is_pp_first_stage( encoder_grid.get_pg("pp") ) diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index 425eea4dc66..ac3d723eda3 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -113,10 +113,13 @@ def build_pipeline_communicator( model: MimoModel, topology: HeteroTopology ) -> MultiModulePipelineCommunicator: """Build the MIMO pipeline communicator used by the train schedule.""" + module_output_ndim = {} + if topology.encoder_grid is not None: + module_output_ndim[topology.encoder_name] = 2 return MultiModulePipelineCommunicator( topology.module_to_grid_map, topology.module_dependency_map, model.config, dim_mapping={"s": 0, "h": 2, "b": 1}, - module_output_ndim={topology.encoder_name: 2}, + module_output_ndim=module_output_ndim, ) diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 64b967b2832..00de2c7aacc 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -29,7 +29,9 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi language_pg = topology.language_pg vision_pg = topology.vision_pg rank_in_language_grid = is_rank_in_grid(topology.llm_grid) - rank_in_encoder_grid = is_rank_in_grid(topology.encoder_grid) + rank_in_encoder_grid = topology.encoder_grid is not None and is_rank_in_grid( + topology.encoder_grid + ) debug_rank( "building model specs " f"rank_in_encoder={rank_in_encoder_grid} rank_in_language={rank_in_language_grid}" @@ -39,18 +41,23 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi if rank_in_language_grid: configure_module_rng(args, language_pg, role_seed_offset=20_000) elif rank_in_encoder_grid: + assert vision_pg is not None configure_module_rng(args, vision_pg, role_seed_offset=10_000) + modality_submodules_spec = {} + special_token_ids = {} + if topology.encoder_grid is not None: + modality_submodules_spec[topology.encoder_name] = vision_submodules_spec( + args, vision_pg if rank_in_encoder_grid else None, topology.encoder_grid + ) + special_token_ids[topology.encoder_name] = args.image_token_id + mimo_config = MimoModelConfig( language_model_spec=language_model_spec( args, language_pg if rank_in_language_grid else None, topology.llm_grid ), - modality_submodules_spec={ - topology.encoder_name: vision_submodules_spec( - args, vision_pg if rank_in_encoder_grid else None, topology.encoder_grid - ) - }, - special_token_ids={topology.encoder_name: args.image_token_id}, + modality_submodules_spec=modality_submodules_spec, + special_token_ids=special_token_ids, module_to_grid_map=topology.module_to_grid_map, ) @@ -96,7 +103,11 @@ def wrap_active_modules_with_ddp( ) debug_rank("language model DDP ready") - if topology.encoder_name in mimo_model.modality_submodules: + if ( + topology.encoder_grid is not None + and topology.encoder_name in mimo_model.modality_submodules + ): + assert topology.vision_pg is not None submodule = mimo_model.modality_submodules[topology.encoder_name] if submodule is None: return diff --git a/examples/mimo/training/hetero/timeline.py b/examples/mimo/training/hetero/timeline.py index 1fff25782bb..76f84fc9434 100644 --- a/examples/mimo/training/hetero/timeline.py +++ b/examples/mimo/training/hetero/timeline.py @@ -73,8 +73,10 @@ def select_timeline_ranks( return {int(item) for item in scope.split(",") if item.strip()} -def ranks_for_dp_replica(grid: HyperCommGrid, dp_replica: int) -> set[int]: +def ranks_for_dp_replica(grid: Optional[HyperCommGrid], dp_replica: int) -> set[int]: """Return all ranks that belong to one dense DP replica of a grid.""" + if grid is None: + return set() ranks = set() for rank in range(grid.rank_offset, grid.rank_offset + grid.size): coords = grid_coords(grid, rank) @@ -88,6 +90,8 @@ def rank_role_and_coords( ) -> tuple[str, dict[str, int | str]]: """Return role and dense-grid coordinates for timeline metadata.""" for role, grid in (("encoder", topology.encoder_grid), ("llm", topology.llm_grid)): + if grid is None: + continue if grid.rank_offset <= rank < grid.rank_offset + grid.size: coords = grid_coords(grid, rank) role_coords = {f"{role}_{key}": value for key, value in coords.items()} diff --git a/examples/mimo/training/hetero/topology.py b/examples/mimo/training/hetero/topology.py index 66eb3f8572a..138b9a5a4cc 100644 --- a/examples/mimo/training/hetero/topology.py +++ b/examples/mimo/training/hetero/topology.py @@ -28,10 +28,10 @@ class HeteroTopology: """Process groups and rank topology for one hetero MIMO run.""" - encoder_grid: HyperCommGrid + encoder_grid: Optional[HyperCommGrid] llm_grid: HyperCommGrid language_pg: ProcessGroupCollection - vision_pg: ProcessGroupCollection + vision_pg: Optional[ProcessGroupCollection] schedule_pg_collection: MultiModuleProcessGroupCollection language_embedding_groups: LanguageEmbeddingGroups encoder_size: int @@ -41,17 +41,22 @@ class HeteroTopology: @property def module_to_grid_map(self) -> dict[str, HyperCommGrid]: """Return the MIMO module-to-grid mapping consumed by schedules and models.""" + if self.encoder_grid is None: + return {MIMO_LANGUAGE_MODULE_KEY: self.llm_grid} return {self.encoder_name: self.encoder_grid, MIMO_LANGUAGE_MODULE_KEY: self.llm_grid} @property def module_dependency_map(self) -> dict[str, list[str]]: """Return the static encoder-to-language MIMO dependency graph.""" + if self.encoder_grid is None: + return {MIMO_LANGUAGE_MODULE_KEY: []} return {self.encoder_name: [MIMO_LANGUAGE_MODULE_KEY], MIMO_LANGUAGE_MODULE_KEY: []} def destroy(self) -> None: """Destroy all process groups owned by this topology.""" destroy_embedding_groups(self.language_embedding_groups) - self.encoder_grid.destroy() + if self.encoder_grid is not None: + self.encoder_grid.destroy() self.llm_grid.destroy() BridgeCommunicator.destroy_broadcast_pgs() @@ -62,17 +67,18 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) llm_grid = None language_embedding_groups: Optional[LanguageEmbeddingGroups] = None try: - debug_rank("creating encoder grid") - encoder_grid = create_hypercomm_grid( - offset=args.encoder_offset, - tp=args.encoder_tp, - cp=args.encoder_cp, - pp=args.encoder_pp, - dp=args.encoder_dp, - ep=args.encoder_ep, - expt_tp=args.encoder_expt_tp, - expt_dp=args.encoder_expt_dp, - ) + if not args.llm_only: + debug_rank("creating encoder grid") + encoder_grid = create_hypercomm_grid( + offset=args.encoder_offset, + tp=args.encoder_tp, + cp=args.encoder_cp, + pp=args.encoder_pp, + dp=args.encoder_dp, + ep=args.encoder_ep, + expt_tp=args.encoder_expt_tp, + expt_dp=args.encoder_expt_dp, + ) debug_rank("creating language grid") llm_grid = create_hypercomm_grid( offset=args.llm_offset, @@ -91,7 +97,11 @@ def create_topology(args: argparse.Namespace, encoder_size: int, llm_size: int) language_pg = populate_language_embedding_groups( get_pg_collection(llm_grid), language_embedding_groups ) - vision_pg = clear_embedding_groups(get_pg_collection(encoder_grid)) + vision_pg = ( + None + if encoder_grid is None + else clear_embedding_groups(get_pg_collection(encoder_grid)) + ) schedule_pg_collection = build_schedule_pg_collection( ENCODER_MODULE_NAME, encoder_grid, llm_grid, vision_pg, language_pg ) @@ -265,15 +275,16 @@ def clear_embedding_groups(pg_collection: ProcessGroupCollection) -> ProcessGrou def build_schedule_pg_collection( encoder_name: str, - encoder_grid: HyperCommGrid, + encoder_grid: Optional[HyperCommGrid], llm_grid: HyperCommGrid, - vision_pg: ProcessGroupCollection, + vision_pg: Optional[ProcessGroupCollection], language_pg: ProcessGroupCollection, ) -> MultiModuleProcessGroupCollection: """Build the schedule-facing process group collection for this rank.""" module_pgs = {} language_model_module_name = None - if is_rank_in_grid(encoder_grid): + if encoder_grid is not None and is_rank_in_grid(encoder_grid): + assert vision_pg is not None module_pgs[encoder_name] = vision_pg if is_rank_in_grid(llm_grid): module_pgs[MIMO_LANGUAGE_MODULE_KEY] = language_pg From 2f6b2c05a5d25fa123891b3baaf7a94e62e2ba00 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati Date: Sat, 16 May 2026 04:56:56 +0000 Subject: [PATCH 36/44] NMFW-478: Wire hetero MIMO training parity with pre-vlm-05 VLM recipe Reproduces Sanjeev's `examples/multimodal/v3/pretrain_3b_nano_vlm_sota_90t_10v.sh` (`sasatheesh/megatron-lm!45`, job 202967) on 9 HEL nodes with our hetero MIMO training loop, modulo three items deferred to follow-up PRs (load_vision_from, correct_encoder_grad_for_partial_participation, dataloader_save) and MTP. Layout - LLM grid: TP=2, EP=16, DP=32 (encoder grid unchanged at TP=1, DP=8, EP=1). - 9n sbatch wrapper bumps `LLM_TP 4 -> 2`, `LLM_DP 16 -> 32`, `NUM_WORKERS 0 -> 2`. Model provider (`nemotron_moe_vlm.py`) - `moe_aux_loss_coeff: 1e-9 -> 1e-4` for non-trivial router pressure. - `bias_dropout_fusion: False -> True` for args-dump parity (inert under add_bias_linear=False but matches Sanjeev). - Make `mamba_num_groups=8`, `mamba_state_dim=128`, `linear_conv_kernel_dim=4` explicit in `nemotron_language_config` (mcore defaults already match; we declare to be defensive). - Pass `share_embeddings_and_output_weights=False` to `MambaModel` for explicit `untie_embeddings_and_output_weights=True` parity. Vision encoder / data path - Add `--dynamic-resolution / --no-dynamic-resolution`, `--dynamic-resolution-min-patches`, `--dynamic-resolution-max-patches` CLI flags. Default `dynamic_resolution=True` for `nemotron-moe-vlm-*` providers. - Pin `args.use_thumbnail=False` and `args.use_tiling=False` under dynamic resolution so `DynamicResolutionImageTilingStrategy` does not emit an extra thumbnail tile (Sanjeev's run has both False). - Thread `dynamic_resolution_min/max_patches` and `dynamic_resolution_min/ max_side` through `VisionConfig` in `energon_multimodal_provider.py`. Optimizer + WSD scheduler - Add `--train-samples`, `--lr-warmup-samples`, `--lr-decay-samples`, `--lr-wsd-decay-samples`, `--lr-wsd-decay-style`. Extend `--lr-decay-style` choices to include `WSD`. - `validate_args` derives `train_iters = ceil(train_samples / gbs)` and enforces WSD requires both wsd_decay_samples and wsd_decay_style. - `build_optimizer_param_scheduler` honors the sample-based knobs (taking precedence over iter-based) and passes `wsd_decay_steps` / `lr_wsd_decay_style` to `OptimizerParamScheduler`. - `run_hetero` lifts `LR/MIN_LR/WEIGHT_DECAY/LR_DECAY_STYLE` to env-var defaults and threads optional sample-based knobs through. DDP - Add `--overlap-param-gather`, `--ddp-num-buckets`, and `--ddp-pad-buckets-for-high-nccl-busbw` CLI flags. - `runtime._resolve_bucket_size` derives DDP bucket_size from num_parameters // num_buckets when `--ddp-num-buckets` is set, else honors `--ddp-bucket-size`, else returns None for mcore auto-default. - Vision DDP keeps both overlap knobs forced OFF (partial-participation safety: text-only batches leave some encoder DP ranks with zero grads). Self-contained parity sbatch - `sbatch_hetero_nemotron_54l_hel_9n_parity.sh`: every training value pinned inline (no `${VAR:-default}` fallbacks). GBS is canonical and NUM_MICROBATCHES is derived as `GBS / (MBS * LLM_DP) = 24`. - Uses `megatron-venv-baked-206674.sqsh`; staged tokenizer + post-c-radio-omni encoder under `agents-scratch`. - Sanjeev-faithful values: lr=1.2e-3, min_lr=1.2e-5, wd=0.1, WSD with minus_sqrt tail, warmup 1.024M samples, decay 35.6M samples, wsd-decay 5.49M samples, train_samples 36.62M (~300B tokens, 47684 iters at gbs=768), log_interval=100, save_interval=1000, seed=1234, class_token_len=10, image_tag_type=internvl, max_num_tiles=1, overlap-grad-reduce, overlap-param-gather, ddp-num-buckets=8, ddp-pad-buckets. - `--load-vision-from` commented out pending PR_load_vision_from. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../mimo/data/energon_multimodal_provider.py | 4 + .../mimo/model_providers/nemotron_moe_vlm.py | 44 ++- .../run_hetero_nemotron_54l_hel_train.sh | 35 ++- .../sbatch_hetero_nemotron_54l_hel_9n.sh | 6 +- ...batch_hetero_nemotron_54l_hel_9n_parity.sh | 254 ++++++++++++++++++ examples/mimo/training/hetero/args.py | 96 ++++++- examples/mimo/training/hetero/optimizer.py | 25 +- examples/mimo/training/hetero/runtime.py | 57 +++- 8 files changed, 493 insertions(+), 28 deletions(-) create mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py index 19a2a9f0c8d..372173c6241 100644 --- a/examples/mimo/data/energon_multimodal_provider.py +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -271,6 +271,10 @@ def build_multimodal_encoder( use_image_break_token=getattr(args, "image_break_token", None) is not None, use_area_weighted_aspect_ratio=getattr(args, "use_area_weighted_aspect_ratio", False), dynamic_resolution=getattr(args, "dynamic_resolution", False), + dynamic_resolution_min_patches=getattr(args, "dynamic_resolution_min_patches", 4), + dynamic_resolution_max_patches=getattr(args, "dynamic_resolution_max_patches", 0), + dynamic_resolution_min_side=getattr(args, "dynamic_resolution_min_side", None), + dynamic_resolution_max_side=getattr(args, "dynamic_resolution_max_side", None), ) packing_config = PackingConfig( seq_length=target_seq_length, pad_id=pad_id, image_token_id=image_token_id diff --git a/examples/mimo/model_providers/nemotron_moe_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py index a09aff1228c..2129d943045 100644 --- a/examples/mimo/model_providers/nemotron_moe_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -112,6 +112,28 @@ def add_model_provider_args(parser: argparse.ArgumentParser) -> None: provider.add_argument("--disable-vision-class-token", action="store_true") provider.add_argument("--use-tiling", action="store_true") provider.add_argument("--use-thumbnail", action="store_true") + provider.add_argument( + "--dynamic-resolution", + action=argparse.BooleanOptionalAction, + default=None, + help=( + "Patchify each image at its native aspect ratio with a token budget instead of " + "fixed-tile resize. Enabled by default for Nemotron6-MoE VLM providers (matches " + "Sanjeev's pre-vlm-05 recipe). Pass --no-dynamic-resolution to disable." + ), + ) + provider.add_argument( + "--dynamic-resolution-min-patches", + type=int, + default=4, + help="Lower bound on per-image patch count under dynamic resolution.", + ) + provider.add_argument( + "--dynamic-resolution-max-patches", + type=int, + default=0, + help="Upper bound on per-image patch count under dynamic resolution; 0 = uncapped.", + ) provider.add_argument("--freeze-lm", action="store_true") provider.add_argument("--freeze-vit", action="store_true") provider.add_argument("--freeze-projection", action="store_true") @@ -148,8 +170,18 @@ def apply_model_provider_defaults(args: argparse.Namespace) -> None: args.image_seq_length = NEMOTRON_20L_IMAGE_SEQ_PER_TILE * args.num_image_tiles args.pixel_shuffle = True args.disable_vision_class_token = True - args.use_tiling = True - args.use_thumbnail = True + if args.dynamic_resolution is None: + args.dynamic_resolution = True + if args.dynamic_resolution: + # Dynamic-resolution strategy reads `use_thumbnail` inside + # `DynamicResolutionImageTilingStrategy` and emits an extra thumbnail + # tile when True. `use_tiling` is inert in this branch (the fixed-tile + # path is unreachable), but pin it False for args-dump parity. + args.use_tiling = False + args.use_thumbnail = False + else: + args.use_tiling = True + args.use_thumbnail = True def apply_training_stage(args: argparse.Namespace) -> None: @@ -356,7 +388,7 @@ def nemotron_language_config( bias_activation_fusion=False, masked_softmax_fusion=True, persist_layer_norm=True, - bias_dropout_fusion=False, + bias_dropout_fusion=True, recompute_granularity="selective", recompute_modules=["core_attn"], moe_ffn_hidden_size=1856, @@ -370,7 +402,7 @@ def nemotron_language_config( moe_router_load_balancing_type="seq_aux_loss", moe_router_force_load_balancing=args.moe_router_force_load_balancing, moe_router_fusion=True, - moe_aux_loss_coeff=1.0e-9, + moe_aux_loss_coeff=1.0e-4, moe_shared_expert_intermediate_size=3712, moe_shared_expert_overlap=True, moe_token_dispatcher_type="alltoall", @@ -379,6 +411,9 @@ def nemotron_language_config( is_hybrid_model=True, mamba_num_heads=64, mamba_head_dim=64, + mamba_num_groups=8, + mamba_state_dim=128, + linear_conv_kernel_dim=4, ) config.position_embedding_type = "none" config.seq_length = 8192 @@ -481,6 +516,7 @@ def language_model_spec( "post_process": pp_rank == pp_size - 1, "hybrid_layer_pattern": args.hybrid_layer_pattern, "position_embedding_type": "none", + "share_embeddings_and_output_weights": False, "scatter_embedding_sequence_parallel": False, "pg_collection": pg_collection, }, diff --git a/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh index 8cceef1d231..3d206961c43 100755 --- a/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh +++ b/examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh @@ -85,6 +85,16 @@ fi GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP))}" LR_WARMUP_ITERS="${LR_WARMUP_ITERS:-10}" LR_DECAY_ITERS="${LR_DECAY_ITERS:-${TRAIN_ITERS}}" +LR="${LR:-2e-4}" +MIN_LR="${MIN_LR:-2e-6}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.05}" +LR_DECAY_STYLE="${LR_DECAY_STYLE:-cosine}" +# Sample-based scheduler knobs (set to enable Sanjeev-style WSD). Empty = unused. +LR_WARMUP_SAMPLES="${LR_WARMUP_SAMPLES:-}" +LR_DECAY_SAMPLES="${LR_DECAY_SAMPLES:-}" +LR_WSD_DECAY_SAMPLES="${LR_WSD_DECAY_SAMPLES:-}" +LR_WSD_DECAY_STYLE="${LR_WSD_DECAY_STYLE:-}" +TRAIN_SAMPLES="${TRAIN_SAMPLES:-}" PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" NUM_WORKERS="${NUM_WORKERS:-1}" SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" @@ -258,12 +268,12 @@ CMD=( --micro-batch-size "${MICRO_BATCH_SIZE}" --global-batch-size "${GLOBAL_BATCH_SIZE}" --num-microbatches "${NUM_MICROBATCHES}" - --lr 2e-4 - --min-lr 2e-6 - --lr-decay-style cosine + --lr "${LR}" + --min-lr "${MIN_LR}" + --lr-decay-style "${LR_DECAY_STYLE}" --lr-warmup-iters "${LR_WARMUP_ITERS}" --lr-decay-iters "${LR_DECAY_ITERS}" - --weight-decay 0.05 + --weight-decay "${WEIGHT_DECAY}" --adam-beta1 0.9 --adam-beta2 0.95 --clip-grad 1.0 @@ -271,8 +281,23 @@ CMD=( --ddp-bucket-size 0 --log-interval "${LOG_INTERVAL}" --train-iters "${TRAIN_ITERS}" - "$@" ) +if [[ -n "${LR_WARMUP_SAMPLES}" ]]; then + CMD+=(--lr-warmup-samples "${LR_WARMUP_SAMPLES}") +fi +if [[ -n "${LR_DECAY_SAMPLES}" ]]; then + CMD+=(--lr-decay-samples "${LR_DECAY_SAMPLES}") +fi +if [[ -n "${LR_WSD_DECAY_SAMPLES}" ]]; then + CMD+=(--lr-wsd-decay-samples "${LR_WSD_DECAY_SAMPLES}") +fi +if [[ -n "${LR_WSD_DECAY_STYLE}" ]]; then + CMD+=(--lr-wsd-decay-style "${LR_WSD_DECAY_STYLE}") +fi +if [[ -n "${TRAIN_SAMPLES}" ]]; then + CMD+=(--train-samples "${TRAIN_SAMPLES}") +fi +CMD+=("$@") if [[ "${DRY_RUN:-0}" == "1" ]]; then printf '%q ' "${CMD[@]}" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh index 9c6813d4c3e..865ebdd60a9 100755 --- a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh @@ -132,14 +132,14 @@ export ENCODER_PP="${ENCODER_PP:-1}" export ENCODER_DP="${ENCODER_DP:-8}" export ENCODER_EP="${ENCODER_EP:-1}" export LLM_ONLY="${LLM_ONLY:-0}" -export LLM_TP="${LLM_TP:-4}" +export LLM_TP="${LLM_TP:-2}" export LLM_CP="${LLM_CP:-1}" export LLM_PP="${LLM_PP:-1}" -export LLM_DP="${LLM_DP:-16}" +export LLM_DP="${LLM_DP:-32}" export LLM_EP="${LLM_EP:-16}" export LLM_EXPT_TP="${LLM_EXPT_TP:-1}" -export NUM_WORKERS="${NUM_WORKERS:-0}" +export NUM_WORKERS="${NUM_WORKERS:-2}" export SHUFFLE_BUFFER_SIZE="${SHUFFLE_BUFFER_SIZE:-100}" export PACKING_BUFFER_SIZE="${PACKING_BUFFER_SIZE:-128}" export MAX_SAMPLES_PER_SEQUENCE="${MAX_SAMPLES_PER_SEQUENCE:-100}" diff --git a/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh new file mode 100755 index 00000000000..8f6ff13d024 --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# Parity reproduction of Sanjeev's pre-vlm-05 VLM training, scaled to 9 HEL +# nodes. Every training value is pinned inline in this file; nothing falls back +# to user-set env vars. GBS is canonical; NUM_MICROBATCHES is derived from +# (GBS / (MBS * LLM_DP)). +# +# Submit from the worktree root: +# sbatch examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 9 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=04:00:00 +#SBATCH -J mimo54l9n-parity +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +# ----------------------------------------------------------------------------- +# Repo + cluster paths (pinned) +# ----------------------------------------------------------------------------- +if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" +else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +fi + +SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch +CONTAINER_IMAGE="${SCRATCH_ROOT}/images/megatron-venv-baked-206674.sqsh" +ENV_ROOT="${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39" +TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" +VISION_CKPT="${SCRATCH_ROOT}/encoders/post-c-radio-omni" + +RUN_NAME="mimo54l-hel-9n-parity" +RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" + +# ----------------------------------------------------------------------------- +# Parallelism (pinned — Sanjeev's TP=2 EP=16 with our 9-node shape) +# Encoder grid: 8 ranks. LLM grid: 64 ranks. Total: 72. +# ----------------------------------------------------------------------------- +ENCODER_TP=1 +ENCODER_CP=1 +ENCODER_PP=1 +ENCODER_DP=8 +ENCODER_EP=1 +LLM_TP=2 +LLM_CP=1 +LLM_PP=1 +LLM_DP=32 +LLM_EP=16 +LLM_EXPT_TP=1 +LLM_ONLY=0 + +# ----------------------------------------------------------------------------- +# Batch — GBS canonical, NUM_MICROBATCHES derived +# ----------------------------------------------------------------------------- +MICRO_BATCH_SIZE=1 +GLOBAL_BATCH_SIZE=768 # Sanjeev's +NUM_MICROBATCHES=$(( GLOBAL_BATCH_SIZE / (MICRO_BATCH_SIZE * LLM_DP) )) # = 24 + +# ----------------------------------------------------------------------------- +# Sanjeev-faithful optimizer + WSD schedule (sample-based) +# ----------------------------------------------------------------------------- +LR=1.2e-3 +MIN_LR=1.2e-5 +WEIGHT_DECAY=0.1 +LR_DECAY_STYLE=WSD +LR_WARMUP_SAMPLES=1024000 +LR_DECAY_SAMPLES=35597094 +LR_WSD_DECAY_SAMPLES=5493164 +LR_WSD_DECAY_STYLE=minus_sqrt +TRAIN_SAMPLES=36621094 + +# validate_args derives args.train_iters = ceil(TRAIN_SAMPLES / GBS) when +# --train-samples is set, so this sentinel is only used pre-derivation. +TRAIN_ITERS=$(( (TRAIN_SAMPLES + GLOBAL_BATCH_SIZE - 1) / GLOBAL_BATCH_SIZE )) + +# ----------------------------------------------------------------------------- +# Logging / checkpointing +# ----------------------------------------------------------------------------- +LOG_INTERVAL=100 +SAVE_INTERVAL=1000 + +# ----------------------------------------------------------------------------- +# Provider / data knobs +# ----------------------------------------------------------------------------- +TRAINING_STAGE=stage2 +MODEL_PROVIDER=nemotron-moe-vlm-54l +ENABLE_EXPERIMENTAL=1 +MOE_ROUTER_FORCE_LOAD_BALANCING=0 +NUM_WORKERS=2 +PACKING_BUFFER_SIZE=128 +SHUFFLE_BUFFER_SIZE=100 +MAX_SAMPLES_PER_SEQUENCE=100 +CHECK_HEL_PATHS=1 + +# ----------------------------------------------------------------------------- +# Sanity checks (fail fast before srun) +# ----------------------------------------------------------------------------- +[[ -r "${CONTAINER_IMAGE}" ]] || { echo "ERROR: missing ${CONTAINER_IMAGE}" >&2; exit 1; } +[[ -d "${ENV_ROOT}/.venv" ]] || { echo "ERROR: missing ${ENV_ROOT}/.venv" >&2; exit 1; } +[[ -d "${TOKENIZER_MODEL}" ]] || { echo "ERROR: missing ${TOKENIZER_MODEL}" >&2; exit 1; } +[[ -d "${VISION_CKPT}" ]] || { echo "ERROR: missing ${VISION_CKPT}" >&2; exit 1; } + +WORLD_SIZE=$(( ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP \ + + LLM_TP * LLM_CP * LLM_PP * LLM_DP )) +[[ "${WORLD_SIZE}" -eq 72 ]] || { echo "ERROR: derived world_size=${WORLD_SIZE} (expected 72 for 9n)" >&2; exit 1; } +if [[ -n "${SLURM_NTASKS:-}" && "${SLURM_NTASKS}" -ne "${WORLD_SIZE}" ]]; then + echo "ERROR: SLURM_NTASKS=${SLURM_NTASKS}, expected ${WORLD_SIZE}" >&2 + exit 1 +fi + +# GBS / NUM_MICROBATCHES consistency +if (( MICRO_BATCH_SIZE * NUM_MICROBATCHES * LLM_DP != GLOBAL_BATCH_SIZE )); then + echo "ERROR: GBS=${GLOBAL_BATCH_SIZE} != MBS*NUM_MICROBATCHES*LLM_DP=$((MICRO_BATCH_SIZE*NUM_MICROBATCHES*LLM_DP))" >&2 + exit 1 +fi + +# ----------------------------------------------------------------------------- +# Output directories +# ----------------------------------------------------------------------------- +mkdir -p \ + "${RUN_DIR}/logs/app" \ + "${RUN_DIR}/logs/torchrun" \ + "${RUN_DIR}/checkpoints" \ + "${RUN_DIR}/tensorboard" \ + "${RUN_DIR}/data_cache" \ + "${RUN_DIR}/tmp" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/home" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" \ + "${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" \ + "${SCRATCH_ROOT}/uv-cache/megatron_lm" + +# ----------------------------------------------------------------------------- +# Exports consumed by run_hetero_nemotron_54l_hel_train.sh +# ----------------------------------------------------------------------------- +export REPO_ROOT RUN_DIR SCRATCH_ROOT +export OUTPUT_PATH="${RUN_DIR}" +export LOG_DIR="${RUN_DIR}/logs/app" +export APP_LOG_DIR="${RUN_DIR}/logs/app" +export TORCHRUN_LOG_DIR="${RUN_DIR}/logs/torchrun" +export CHECKPOINT_SAVE_PATH="${RUN_DIR}/checkpoints" +export CHECKPOINT_LOAD_PATH="${RUN_DIR}/checkpoints" +export CHECKPOINT_DIR="${RUN_DIR}/checkpoints" +export TENSORBOARD_PATH="${RUN_DIR}/tensorboard" +export TB_DIR="${RUN_DIR}/tensorboard" +export DATA_CACHE_DIR="${RUN_DIR}/data_cache" +export TMPDIR="${RUN_DIR}/tmp" + +export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" +export XDG_CACHE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" +export XDG_DATA_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" +export XDG_STATE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" +export TORCHINDUCTOR_CACHE_DIR="${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" +export TRITON_CACHE_DIR_BASE="${RUN_DIR}/triton-cache" +export CUDA_CACHE_PATH="${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" +export TORCHINDUCTOR_COMPILE_THREADS=4 +export PYTHONPATH="${REPO_ROOT}" +export PYTHONNOUSERSITE=1 +export PIP_CONSTRAINT="" +export UV_LINK_MODE=copy +export UV_CACHE_DIR="${SCRATCH_ROOT}/uv-cache/megatron_lm" +export UV_PROJECT_ENVIRONMENT="${ENV_ROOT}/.venv" +export VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}" +export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" + +# Runtime / NCCL / TE env +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NVTE_FWD_LAYERNORM_SM_MARGIN=16 +export NVTE_BWD_LAYERNORM_SM_MARGIN=16 +export NCCL_P2P_NET_CHUNKSIZE=2097152 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export NCCL_DEBUG=WARN +export NCCL_SHM_DISABLE=1 +export NCCL_PROTO=simple +export NCCL_NVLS_ENABLE=0 +export TORCH_NCCL_AVOID_RECORD_STREAMS=0 +export TORCH_FR_BUFFER_SIZE=1048576 +export TORCH_NCCL_TRACE_BUFFER_SIZE=1048576 +export TORCH_NCCL_TRACE_CPP_STACK=1 +export TORCH_NCCL_DUMP_ON_TIMEOUT=1 +export TORCH_NCCL_DESYNC_DEBUG=1 +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 + +# Training-knob exports picked up by run_hetero +export TRAINING_STAGE MODEL_PROVIDER ENABLE_EXPERIMENTAL MOE_ROUTER_FORCE_LOAD_BALANCING +export TRAIN_ITERS NUM_MICROBATCHES MICRO_BATCH_SIZE GLOBAL_BATCH_SIZE LOG_INTERVAL +export ENCODER_TP ENCODER_CP ENCODER_PP ENCODER_DP ENCODER_EP +export LLM_TP LLM_CP LLM_PP LLM_DP LLM_EP LLM_EXPT_TP LLM_ONLY +export LR MIN_LR WEIGHT_DECAY LR_DECAY_STYLE +export LR_WARMUP_SAMPLES LR_DECAY_SAMPLES LR_WSD_DECAY_SAMPLES LR_WSD_DECAY_STYLE TRAIN_SAMPLES +export NUM_WORKERS PACKING_BUFFER_SIZE SHUFFLE_BUFFER_SIZE MAX_SAMPLES_PER_SEQUENCE CHECK_HEL_PATHS +export TOKENIZER_MODEL +export VISION_CKPT + +# ----------------------------------------------------------------------------- +# Extra CLI args appended to train_hetero.py via run_hetero "$@". +# Argparse processes in order; these override the hardcoded values inside +# run_hetero_nemotron_54l_hel_train.sh:CMD[]. +# ----------------------------------------------------------------------------- +TRAIN_LAUNCH_ARGS=( + --class-token-len 10 # RADIO vit_huge_patch16_224 has 1 CLS + 9 registers + --image-tag-type internvl # Sanjeev wraps with internvl markers + --max-num-tiles 1 # single-image dynamic-resolution; no LLaVA-NeXT tile grid + --overlap-grad-reduce # BooleanOptionalAction; later value wins over --no-overlap-grad-reduce + --overlap-param-gather # distributed-optimizer param all-gather overlap + --ddp-num-buckets 8 # match Sanjeev; mutually exclusive with --ddp-bucket-size>0 + --ddp-pad-buckets-for-high-nccl-busbw # pad to 2^16 for NCCL busbw at large DP + --seed 1234 # Sanjeev's value + --save "${CHECKPOINT_SAVE_PATH}" + --save-interval "${SAVE_INTERVAL}" + # --load-vision-from "${VISION_CKPT}" # TODO: enable once PR_load_vision_from lands +) + +CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" +if [[ "${REPO_ROOT}" != "${SCRATCH_ROOT}"/* ]]; then + CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" +fi + +# ----------------------------------------------------------------------------- +# Summary banner +# ----------------------------------------------------------------------------- +echo "=== HEL 54L hetero MIMO parity sbatch (9n, Sanjeev recipe) ===" +echo "repo=${REPO_ROOT}" +echo "run_dir=${RUN_DIR}" +echo "container_image=${CONTAINER_IMAGE}" +echo "world_size=${WORLD_SIZE}" +echo "mbs=${MICRO_BATCH_SIZE} gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} (gbs/(mbs*llm_dp))" +echo "layout=encoder(tp=${ENCODER_TP},dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP})" +echo "lr=${LR} min_lr=${MIN_LR} wd=${WEIGHT_DECAY} schedule=${LR_DECAY_STYLE}/${LR_WSD_DECAY_STYLE}" +echo "train_samples=${TRAIN_SAMPLES} train_iters=${TRAIN_ITERS}" +echo "log_interval=${LOG_INTERVAL} save_interval=${SAVE_INTERVAL}" +echo "tokenizer=${TOKENIZER_MODEL}" +echo "vision_ckpt=${VISION_CKPT}" +echo "extra_args=${TRAIN_LAUNCH_ARGS[*]}" +echo "===============================================================" + +srun --kill-on-bad-exit=1 \ + --ntasks="${WORLD_SIZE}" \ + --ntasks-per-node=8 \ + --container-image="${CONTAINER_IMAGE}" \ + --no-container-mount-home \ + --container-mounts="${CONTAINER_MOUNTS}" \ + --container-workdir="${REPO_ROOT}" \ + bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"' \ + bash "${TRAIN_LAUNCH_ARGS[@]}" diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index 9991803e96e..efd4f87136c 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -115,11 +115,54 @@ def parse_args() -> argparse.Namespace: train.add_argument("--global-batch-size", type=int, default=None) train.add_argument("--num-microbatches", type=int, default=2) train.add_argument("--train-iters", type=int, default=2) + train.add_argument( + "--train-samples", + type=int, + default=None, + help=( + "Total training budget in consumed samples. When set, --train-iters is " + "re-derived as ceil(train_samples / global_batch_size). Matches Sanjeev's " + "samples-based recipe." + ), + ) train.add_argument("--lr", type=float, default=1.0e-4) train.add_argument("--min-lr", type=float, default=None) - train.add_argument("--lr-decay-style", type=str, default="constant") + train.add_argument( + "--lr-decay-style", + type=str, + default="constant", + choices=["constant", "linear", "cosine", "inverse-square-root", "WSD"], + ) train.add_argument("--lr-warmup-iters", type=int, default=0) train.add_argument("--lr-decay-iters", type=int, default=None) + train.add_argument( + "--lr-warmup-samples", + type=int, + default=None, + help="LR warmup duration in consumed samples. Overrides --lr-warmup-iters when set.", + ) + train.add_argument( + "--lr-decay-samples", + type=int, + default=None, + help="LR decay duration in consumed samples. Overrides --lr-decay-iters when set.", + ) + train.add_argument( + "--lr-wsd-decay-samples", + type=int, + default=None, + help=( + "Length of the WSD decay tail in consumed samples. Required when " + "--lr-decay-style=WSD." + ), + ) + train.add_argument( + "--lr-wsd-decay-style", + type=str, + default=None, + choices=["linear", "cosine", "exponential", "minus_sqrt"], + help="Decay-style applied during the WSD tail.", + ) train.add_argument("--weight-decay", type=float, default=0.01) train.add_argument("--adam-beta1", type=float, default=0.9) train.add_argument("--adam-beta2", type=float, default=0.999) @@ -134,11 +177,38 @@ def parse_args() -> argparse.Namespace: "keeps overlap disabled because actual-data batches may be text-only." ), ) + train.add_argument( + "--overlap-param-gather", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "Enable distributed-optimizer param all-gather overlap with forward compute. " + "Requires --use-distributed-optimizer (already on for the hetero loop)." + ), + ) train.add_argument( "--ddp-bucket-size", type=int, default=10000, - help="DDP bucket size. Use 0 for a single unbounded bucket.", + help="DDP bucket size in parameters. Use 0 for a single unbounded bucket.", + ) + train.add_argument( + "--ddp-num-buckets", + type=int, + default=None, + help=( + "If set, DDP bucket_size is derived from num_parameters // ddp_num_buckets " + "(mutually exclusive with --ddp-bucket-size > 0)." + ), + ) + train.add_argument( + "--ddp-pad-buckets-for-high-nccl-busbw", + action=argparse.BooleanOptionalAction, + default=False, + help=( + "Pad DDP bucket sizes to a multiple of 2^16 so NCCL collectives have high " + "bus bandwidth at large DP counts." + ), ) train.add_argument("--seed", type=int, default=12345) train.add_argument("--log-interval", type=int, default=1) @@ -239,6 +309,28 @@ def validate_args(args: argparse.Namespace, world_size: int) -> tuple[int, int]: if args.save_interval is not None and args.save is None: raise ValueError("--save-interval requires --save") + # Sample-based scheduler resolution: when --train-samples is set, derive + # --train-iters from it using the (now-known) global batch size. The + # OptimizerParamScheduler tracks "steps" in units of consumed samples, so + # the sample-based knobs flow through unchanged downstream. + if args.train_samples is not None: + derived_gbs = args.micro_batch_size * args.num_microbatches * args.llm_dp + gbs = args.global_batch_size if args.global_batch_size is not None else derived_gbs + if gbs <= 0: + raise ValueError( + "--train-samples requires a positive derived/explicit --global-batch-size" + ) + import math as _math + + derived_iters = _math.ceil(args.train_samples / gbs) + args.train_iters = derived_iters + + if args.lr_decay_style == "WSD": + if args.lr_wsd_decay_samples is None: + raise ValueError("--lr-decay-style=WSD requires --lr-wsd-decay-samples") + if args.lr_wsd_decay_style is None: + raise ValueError("--lr-decay-style=WSD requires --lr-wsd-decay-style") + llm_size = args.llm_tp * args.llm_cp * args.llm_pp * args.llm_dp if args.llm_only: if args.llm_offset != 0: diff --git a/examples/mimo/training/hetero/optimizer.py b/examples/mimo/training/hetero/optimizer.py index fb3bcca8ca1..2e214b1943b 100644 --- a/examples/mimo/training/hetero/optimizer.py +++ b/examples/mimo/training/hetero/optimizer.py @@ -46,16 +46,31 @@ def get_global_batch_size(args: argparse.Namespace) -> int: def build_optimizer_param_scheduler(args: argparse.Namespace, optimizer) -> OptimizerParamScheduler: - """Build the MCore optimizer parameter scheduler using Megatron train-iters semantics.""" + """Build the MCore optimizer parameter scheduler. + + The scheduler tracks "steps" in units of consumed samples (incremented by the + global batch size per call). Sample-based knobs take precedence when set; + iter-based knobs are converted via iter * global_batch_size for back-compat. + """ global_batch_size = get_global_batch_size(args) - lr_decay_iters = args.lr_decay_iters if args.lr_decay_iters is not None else args.train_iters + if args.lr_warmup_samples is not None: + lr_warmup_steps = args.lr_warmup_samples + else: + lr_warmup_steps = args.lr_warmup_iters * global_batch_size + if args.lr_decay_samples is not None: + lr_decay_steps = args.lr_decay_samples + else: + lr_decay_iters = ( + args.lr_decay_iters if args.lr_decay_iters is not None else args.train_iters + ) + lr_decay_steps = lr_decay_iters * global_batch_size return OptimizerParamScheduler( optimizer, init_lr=0.0, max_lr=args.lr, min_lr=args.min_lr if args.min_lr is not None else 0.0, - lr_warmup_steps=args.lr_warmup_iters * global_batch_size, - lr_decay_steps=lr_decay_iters * global_batch_size, + lr_warmup_steps=lr_warmup_steps, + lr_decay_steps=lr_decay_steps, lr_decay_style=args.lr_decay_style, start_wd=args.weight_decay, end_wd=args.weight_decay, @@ -63,4 +78,6 @@ def build_optimizer_param_scheduler(args: argparse.Namespace, optimizer) -> Opti wd_incr_style="constant", use_checkpoint_opt_param_scheduler=False, override_opt_param_scheduler=True, + wsd_decay_steps=args.lr_wsd_decay_samples, + lr_wsd_decay_style=args.lr_wsd_decay_style, ) diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 00de2c7aacc..0e340ef6473 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -77,23 +77,50 @@ def build_mimo_runtime(args: argparse.Namespace, topology: HeteroTopology) -> Mi return mimo_model +def _resolve_bucket_size( + args: argparse.Namespace, module: Optional[torch.nn.Module] +) -> Optional[int]: + """Resolve DDP bucket_size for a module. + + Precedence: + 1. --ddp-num-buckets (set): bucket_size = num_params // num_buckets. + 2. --ddp-bucket-size > 0: use that value. + 3. Else None (mcore auto-default = max(40M, 1M * dp_size)). + """ + num_buckets = getattr(args, "ddp_num_buckets", None) + if num_buckets is not None: + if num_buckets <= 0: + raise ValueError("--ddp-num-buckets must be > 0 when set") + if args.ddp_bucket_size and args.ddp_bucket_size > 0: + raise ValueError( + "--ddp-num-buckets and --ddp-bucket-size are mutually exclusive" + ) + if module is None: + return None + num_params = sum(p.numel() for p in module.parameters()) + if num_params <= 0: + return None + return max(1, num_params // num_buckets) + if args.ddp_bucket_size and args.ddp_bucket_size > 0: + return args.ddp_bucket_size + return None + + def wrap_active_modules_with_ddp( args: argparse.Namespace, mimo_model: MimoModel, topology: HeteroTopology ) -> None: """Freeze and DDP-wrap active local MIMO modules.""" - language_ddp_config = DistributedDataParallelConfig( - overlap_grad_reduce=args.overlap_grad_reduce, - bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, - use_distributed_optimizer=True, - ) - vision_ddp_config = DistributedDataParallelConfig( - overlap_grad_reduce=False, - bucket_size=args.ddp_bucket_size if args.ddp_bucket_size > 0 else None, - use_distributed_optimizer=True, - ) + pad_buckets = getattr(args, "ddp_pad_buckets_for_high_nccl_busbw", False) if mimo_model.language_model is not None: if args.freeze_lm: set_module_requires_grad(mimo_model.language_model, False) + language_ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=args.overlap_grad_reduce, + overlap_param_gather=getattr(args, "overlap_param_gather", False), + bucket_size=_resolve_bucket_size(args, mimo_model.language_model), + pad_buckets_for_high_nccl_busbw=pad_buckets, + use_distributed_optimizer=True, + ) debug_rank("wrapping language model in DDP") mimo_model.language_model = DistributedDataParallel( config=mimo_model.language_model.config, @@ -118,6 +145,16 @@ def wrap_active_modules_with_ddp( if args.freeze_projection: for projection in iter_vision_projection_modules(submodule): set_module_requires_grad(projection, False) + # Vision DDP keeps all overlap off: actual-data batches may be text-only, + # so some encoder DP ranks see zero grads/params per step; overlap'd + # collectives are not safe under that partial participation. + vision_ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=False, + overlap_param_gather=False, + bucket_size=_resolve_bucket_size(args, submodule), + pad_buckets_for_high_nccl_busbw=pad_buckets, + use_distributed_optimizer=True, + ) debug_rank("wrapping vision submodule in DDP") mimo_model.modality_submodules[topology.encoder_name] = DistributedDataParallel( config=encoder_module.config, From 6f3deb3e1196dfc9f0b2c44d72a1da841fb0f405 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Mon, 18 May 2026 09:22:35 -0700 Subject: [PATCH 37/44] Hetero MIMO: arg-parity + correctness fixes (Sanjeev-202967) (#29) * Add Nemotron-format ckpt loader for hetero MIMO - Add load_nemotron_vlm_ckpt_hetero and --load-nemotron-checkpoint flag for loading pre-vlm-05 Nemotron-format VLM dist-ckpts from the hetero pipeline. - After the custom load, refresh the DistributedOptimizer's FP32 main-param shards via optimizer.reload_model_params(); the standard load_checkpoint path does this automatically, but the custom loader bypasses it. Without this the optimizer steps with the model-provider init weights instead of the loaded ckpt weights. - Seed python random + numpy + torch in the hetero entry to match Megatron's _set_random_seed (energon's text_packing shuffle uses the global random module). - Add --correct-encoder-grad-for-partial-participation flag (consumed in a follow-up commit by grad_sync.py). - Add --train-samples flag (samples-based budget; --train-iters is derived). - Fix RADIO pos-embedding bilinear interpolation to align_corners=False to match upstream RADIO. Co-Authored-By: Claude Opus 4.7 (1M context) * Hetero correctness fixes: fp32 grad reduce + encoder grad participation - runtime.py: set grad_reduce_in_fp32=True on both language and vision DDP configs (mirrors --accumulate-allreduce-grads-in-fp32). The default False produces bf16 main_grad, which drifts step-2 weights after Adam. - grad_sync.py: when only some encoder DP ranks process images in a step, scale vision grads post-DP-reduce by encoder_dp_size / participation_count. Without this the vision encoder learns at a diluted rate. - nemotron_moe_vlm.py: set moe_router_fusion=False to match the TransformerConfig default. The fused softmax/topk kernel takes a different bf16 reduction path, slightly perturbing router probs. Co-Authored-By: Claude Opus 4.7 (1M context) * Hetero data pipeline parity - energon_multimodal_provider.py: forward HF tokenizer's chat_template and apply_chat_template through TokenizerAdapter so energon's tokenize_and_prepare can find them. Add _supported_kwargs filter on VisionConfig so the same recipe args work across energon versions whose VisionConfig accepts different kwargs. - hetero_energon.py: use get_savable_loader (SavableDatasetWrapper sets a worker_id_offset and per-worker init that affects step-0 sample order). Use the unsalted seed in the single-lane iterator; energon's WorkerConfig(rank=lane, world_size=llm_dp) already salts per-rank, so adding a +lane offset over-salts and de-aligns sample ordering. Co-Authored-By: Claude Opus 4.7 (1M context) * Parity sbatch scripts + broaden modelopt import guard - Add sbatch_hetero_parity_100step.sh and sbatch_sanjeev_parity_100step.sh as paired 150-step train-loss parity drivers (hetero MIMO vs reference recipe). Settings match the Sanjeev-202967 hetero arg-parity table. - training.py: broaden the modelopt distill-plugin import guard from ImportError to Exception. Some container builds ship modelopt against a transformers version that removed transformers.modeling_utils.Conv1D, so importing the distill plugin raises AttributeError during module init. Distill is optional, so skip safely. Co-Authored-By: Claude Opus 4.7 (1M context) * Address PR review comments - Revert training.py modelopt import-guard change (not needed for hetero correctness; container-specific compat issue tracked elsewhere). - Trim verbose docstrings in model_helpers.py (_load_submodule_from_ckpt, load_nemotron_vlm_ckpt_hetero) and args.py --load-nemotron-checkpoint help. - hetero_energon.py: drop the WorkerConfig-salting comment and the over- defensive try/except wrapping get_savable_loader. Match pre-vlm-05 exactly: one direct call with cache_pool=NoCachePool() and the watchdog kwargs. - grad_sync.py: replace expensive (buffer.grad_data != 0).any() scans with a one-bool participation flag set by forward_step from batch.images. Combine the per-token normalization and the partial-participation correction into a single scale_gradients call per submodule (was two separate kernel launches before). - step.py: call mark_modality_participation in forward_step and reset_modality_participation at the top of each train_step. - loop.py: extract the --load-nemotron-checkpoint branch into load_and_refresh_nemotron_checkpoint helper in model_helpers.py. Co-Authored-By: Claude Opus 4.7 (1M context) * Point sanj parity sbatch at clean pre-vlm-05 clone SANJEEV_REPO now defaults to ${SCRATCH_ROOT}/sanjeev-repos/megatron-lm-clean, a fresh checkout of sasatheesh/pre-vlm-05 with only the two correctness changes needed for the parity baseline (model.py calculate_per_token_loss honors --calculate-per-token-loss; recipe sh passes the flag). All NMFW debug instrumentation that lived on the old sanjeev-repos/megatron-lm checkout is dropped from this baseline. Co-Authored-By: Claude Opus 4.7 (1M context) * RADIO: set mtp_num_layers=0 so final layernorm applies With post_process=False on the RADIO TransformerBlock and mtp_num_layers defaulting to None, has_final_layernorm_in_this_stage short-circuits to False and the final layernorm is dropped. The sanj recipe passes --mtp-num-layers 0, which takes the alternate branch and keeps the layernorm on the stage that holds layer.num_layers. Co-Authored-By: Claude Opus 4.7 (1M context) * RADIO: pass ln_post_impl=TENorm to RADIOViTModel The RADIO post-encoder layernorm lives on RADIOViTModel as self.ln_post, applied after the decoder in radio.py:239-240. The ckpt converter writes RADIO's inner.norm.{weight,bias} into ln_post.{weight,bias}; without ln_post_impl=TENorm the wrapper leaves self.ln_post=None and the loader silently drops the ckpt entries (StrictHandling.LOG_UNEXPECTED), so the vision tower output is the raw decoder hidden state instead of its post-norm. Sanj-side llava_model.py:309-311 sets the same impl. Revert previous mtp_num_layers=0 attempt -- that was targeting TransformerBlock.final_layernorm, which is a different module and not where the ckpt weight lands. Co-Authored-By: Claude Opus 4.7 (1M context) * Revert RADIO layernorm experiments Both attempts targeted layernorms that aren't actually missing for this RADIO variant: - mtp_num_layers=0 (0b5996cad) targeted TransformerBlock.final_layernorm, but that gating returns False here for unrelated reasons. - ln_post_impl=TENorm (82ce39618) builds RADIOViTModel.ln_post, but the sanj iter_1000 ckpt has no ln_post.* keys -- llava_model.py only sets ln_post_impl=TENorm for vision_model_type=='radio-g', not for the cradio variant we run. Confirmed by 266840's load failure: model requested vision_model.ln_post.* but dist_checkpointing flagged them as unexpected (not found in ckpt). The actual sanj parity baseline (266771, num_layers=54) still sits at iter1=2.898, hetero 266780 at iter1=2.757 -- same pattern as prior runs, no missing-norm regression. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../mimo/data/energon_multimodal_provider.py | 28 +- examples/mimo/data/hetero_energon.py | 18 +- .../mimo/model_providers/nemotron_moe_vlm.py | 21 +- .../scripts/sbatch_hetero_parity_100step.sh | 180 ++++++++++ .../scripts/sbatch_sanjeev_parity_100step.sh | 100 ++++++ examples/mimo/training/hetero/args.py | 24 +- examples/mimo/training/hetero/grad_sync.py | 109 ++++-- examples/mimo/training/hetero/loop.py | 18 +- examples/mimo/training/hetero/runtime.py | 4 + examples/mimo/training/hetero/step.py | 8 +- examples/mimo/utils/model_helpers.py | 319 +++++++++++++++++- megatron/core/models/vision/radio.py | 8 +- 12 files changed, 779 insertions(+), 58 deletions(-) create mode 100755 examples/mimo/scripts/sbatch_hetero_parity_100step.sh create mode 100755 examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py index 372173c6241..07530923945 100644 --- a/examples/mimo/data/energon_multimodal_provider.py +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -11,9 +11,23 @@ from __future__ import annotations +import inspect import warnings from typing import Optional + +def _supported_kwargs(fn, kwargs): + """Drop kwargs the target callable doesn't accept. + + Lets the caller pass a superset of recipe args without erroring on fields + that the installed energon's VisionConfig doesn't recognize. + """ + params = inspect.signature(fn).parameters + if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params.values()): + return kwargs + return {key: value for key, value in kwargs.items() if key in params} + + import torch from megatron.energon.task_encoder.multimodal import ( @@ -59,6 +73,15 @@ def convert_tokens_to_ids(self, tokens): """Convert tokens to ids with the wrapped Megatron tokenizer.""" return self._tok.convert_tokens_to_ids(tokens) + @property + def chat_template(self): + """Forward HuggingFace chat_template so energon's tokenize_and_prepare can find it.""" + return getattr(self._hf, "chat_template", None) + + def apply_chat_template(self, *args, **kwargs): + """Forward to underlying HuggingFace tokenizer for energon's chat-template path.""" + return self._hf.apply_chat_template(*args, **kwargs) + class MimoMultiModalPackingEncoder(MultiModalPackingEncoder): """Remap Energon multimodal packed samples to MIMO batch inputs.""" @@ -255,7 +278,7 @@ def build_multimodal_encoder( image_token_id = tokenizer.convert_tokens_to_ids(getattr(args, "image_token", "")) pad_id = getattr(args, "pad_token_id", tokenizer.pad) - vision_config = VisionConfig( + vision_config_kwargs = dict( img_h=args.img_h, img_w=args.img_w, patch_dim=args.patch_dim, @@ -276,6 +299,9 @@ def build_multimodal_encoder( dynamic_resolution_min_side=getattr(args, "dynamic_resolution_min_side", None), dynamic_resolution_max_side=getattr(args, "dynamic_resolution_max_side", None), ) + # Drop kwargs the installed energon's VisionConfig doesn't accept (e.g. + # dynamic_resolution_max_side is only on newer forks). + vision_config = VisionConfig(**_supported_kwargs(VisionConfig, vision_config_kwargs)) packing_config = PackingConfig( seq_length=target_seq_length, pad_id=pad_id, image_token_id=image_token_id ) diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 2b7deaf9d5a..901a725f8af 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -103,12 +103,14 @@ def _build_encoder_iterator(args, grid): tp_group=tp_group, lane=llm_lanes[0], role="encoder", - random_seed=args.seed + llm_lanes[0], + # energon's WorkerConfig(rank=lane, world_size=llm_dp) already + # salts per-rank, so the seed here must be unsalted. + random_seed=args.seed, ) lane_iterators = [ _build_single_lane_iterator( - args, tp_group=None, lane=lane, role="encoder-component", random_seed=args.seed + lane + args, tp_group=None, lane=lane, role="encoder-component", random_seed=args.seed ) for lane in llm_lanes ] @@ -138,7 +140,7 @@ def _llm_lanes_for_encoder_rank(args, encoder_dp_rank: int) -> list[int]: def _build_single_lane_iterator(args, tp_group, lane: int, role: str, random_seed: int): """Build a deterministic loader for one LLM data lane.""" from examples.mimo.data.energon_multimodal_provider import build_multimodal_encoder - from megatron.energon import WorkerConfig, get_loader, get_train_dataset + from megatron.energon import WorkerConfig, get_savable_loader, get_train_dataset tokenizer = _build_tokenizer(args) encoder = build_multimodal_encoder( @@ -163,8 +165,16 @@ def _build_single_lane_iterator(args, tp_group, lane: int, role: str, random_see shuffle_buffer_size=args.shuffle_buffer_size, max_samples_per_sequence=args.max_samples_per_sequence, ) + from megatron.energon.cache.no_cache import NoCachePool + + loader = get_savable_loader( + dataset, + cache_pool=NoCachePool(), + watchdog_timeout_seconds=5 * 60, + watchdog_initial_timeout_seconds=5 * 60, + ) return EnergonIterator( - get_loader(dataset), + loader, tp_group=tp_group, source_rank=True, random_seed=random_seed, diff --git a/examples/mimo/model_providers/nemotron_moe_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py index 2129d943045..4487d123fe0 100644 --- a/examples/mimo/model_providers/nemotron_moe_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -10,6 +10,13 @@ import torch +from examples.mimo.utils.hetero import ( + debug_rank, + get_grid_dim_size, + get_group_rank_or, + get_group_size_or, + is_process_group_member, +) from megatron.core.activations import fast_gelu, squared_relu from megatron.core.hyper_comm_grid import HyperCommGrid from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec @@ -28,14 +35,6 @@ from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.utils import sharded_state_dict_default -from examples.mimo.utils.hetero import ( - debug_rank, - get_grid_dim_size, - get_group_rank_or, - get_group_size_or, - is_process_group_member, -) - try: from megatron.core.extensions.transformer_engine import ( TEColumnParallelLinear, @@ -118,8 +117,8 @@ def add_model_provider_args(parser: argparse.ArgumentParser) -> None: default=None, help=( "Patchify each image at its native aspect ratio with a token budget instead of " - "fixed-tile resize. Enabled by default for Nemotron6-MoE VLM providers (matches " - "Sanjeev's pre-vlm-05 recipe). Pass --no-dynamic-resolution to disable." + "fixed-tile resize. Enabled by default for Nemotron6-MoE VLM providers. " + "Pass --no-dynamic-resolution to disable." ), ) provider.add_argument( @@ -401,7 +400,7 @@ def nemotron_language_config( moe_router_dtype="fp32", moe_router_load_balancing_type="seq_aux_loss", moe_router_force_load_balancing=args.moe_router_force_load_balancing, - moe_router_fusion=True, + moe_router_fusion=False, moe_aux_loss_coeff=1.0e-4, moe_shared_expert_intermediate_size=3712, moe_shared_expert_overlap=True, diff --git a/examples/mimo/scripts/sbatch_hetero_parity_100step.sh b/examples/mimo/scripts/sbatch_hetero_parity_100step.sh new file mode 100755 index 00000000000..5cb74f7bcbf --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_parity_100step.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# 150-step train-loss parity run — hetero MIMO side. +# 3 nodes (1n encoder DP=8 + 2n LLM TP=2 DP=8 EP=16), GBS=8. +# Paired with sbatch_sanjeev_parity_100step.sh. +# +# Uses --load-nemotron-checkpoint to route ckpt-load through +# examples/mimo/utils/model_helpers.py:load_nemotron_vlm_ckpt_hetero and +# start training at iter 0 with fresh optimizer + RNG. + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 3 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=02:00:00 +#SBATCH -J mimo-parity-100step +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" +else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +fi + +SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch +# Default to the energon-baked container so the energon swap below finds +# /usr/local/lib/python3.12/dist-packages/megatron. Skip uv for the same +# reason. Override via HETERO_CONTAINER_IMAGE / HETERO_SKIP_UV if needed. +CONTAINER_IMAGE="${HETERO_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" +export HETERO_SKIP_UV="${HETERO_SKIP_UV:-1}" +ENV_ROOT="${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39" +TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" +VISION_CKPT="${SCRATCH_ROOT}/encoders/post-c-radio-omni" + +NEMOTRON_CKPT="${NEMOTRON_CKPT:-/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511/checkpoints/iter_0001000}" + +RUN_NAME="mimo-parity-100step" +RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" + +# ---- topology: TP=2 EP=16 LLM + TP=1 DP=8 encoder lane ---------------------- +ENCODER_TP=1; ENCODER_CP=1; ENCODER_PP=1; ENCODER_DP=8; ENCODER_EP=1 +LLM_TP=2; LLM_CP=1; LLM_PP=1; LLM_DP=8; LLM_EP=16; LLM_EXPT_TP=1 +LLM_ONLY=0 + +# ---- batch / schedule pinned for the parity diff ---------------------------- +MICRO_BATCH_SIZE=1 +GLOBAL_BATCH_SIZE=8 +NUM_MICROBATCHES=$(( GLOBAL_BATCH_SIZE / (MICRO_BATCH_SIZE * LLM_DP) )) # = 1 +TRAIN_ITERS=150 +LOG_INTERVAL=1 +SAVE_INTERVAL=99999999 # don't save during the parity run + +# WSD schedule (matches the comparison recipe). LR_WARMUP_SAMPLES=0 so both +# sides hit full LR from iter 1 and there's no warmup-ramp difference. +LR=1.2e-3 +MIN_LR=1.2e-5 +WEIGHT_DECAY=0.1 +LR_DECAY_STYLE=WSD +LR_WARMUP_SAMPLES=0 +LR_DECAY_SAMPLES=121046313 +LR_WSD_DECAY_SAMPLES=1 +LR_WSD_DECAY_STYLE=minus_sqrt +TRAIN_SAMPLES=$(( TRAIN_ITERS * GLOBAL_BATCH_SIZE )) + +TRAINING_STAGE=stage2 +MODEL_PROVIDER=nemotron-moe-vlm-54l +ENABLE_EXPERIMENTAL=1 +MOE_ROUTER_FORCE_LOAD_BALANCING=0 +NUM_WORKERS=2 +PACKING_BUFFER_SIZE=4 # smaller pool: less likelihood of multi-image packs exceeding seq_length after MIMO expansion (see energon_multimodal_provider.py:150 defensive check) +SHUFFLE_BUFFER_SIZE=100 +MAX_SAMPLES_PER_SEQUENCE=100 +CHECK_HEL_PATHS=1 + +WORLD_SIZE=$(( ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP \ + + LLM_TP * LLM_CP * LLM_PP * LLM_DP )) +[[ "${WORLD_SIZE}" -eq 24 ]] || { echo "ERROR: derived world_size=${WORLD_SIZE} (expected 24)" >&2; exit 1; } + +mkdir -p "${RUN_DIR}/logs/app" "${RUN_DIR}/logs/torchrun" "${RUN_DIR}/checkpoints" \ + "${RUN_DIR}/tensorboard" "${RUN_DIR}/data_cache" "${RUN_DIR}/tmp" + +export REPO_ROOT RUN_DIR SCRATCH_ROOT +export OUTPUT_PATH="${RUN_DIR}" LOG_DIR="${RUN_DIR}/logs/app" APP_LOG_DIR="${RUN_DIR}/logs/app" +export TORCHRUN_LOG_DIR="${RUN_DIR}/logs/torchrun" +export CHECKPOINT_SAVE_PATH="${RUN_DIR}/checkpoints" CHECKPOINT_LOAD_PATH="${NEMOTRON_CKPT}" +export CHECKPOINT_DIR="${RUN_DIR}/checkpoints" TENSORBOARD_PATH="${RUN_DIR}/tensorboard" TB_DIR="${RUN_DIR}/tensorboard" +export DATA_CACHE_DIR="${RUN_DIR}/data_cache" +# DataLoader workers use TMPDIR for AF_UNIX IPC sockets; the path must be +# below the 108-char Unix-socket limit. RUN_DIR under our lustre scratch +# is already ~93 chars and python's multiprocessing appends more, blowing +# the limit. Use /tmp (node-local, short, exists by default on every +# node — /dev/shm would also work but only if pre-created on every +# node, which the sbatch head-node mkdir doesn't do). +export TMPDIR="/tmp" + +export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" +export XDG_CACHE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" +export XDG_DATA_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" +export XDG_STATE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" +export TORCHINDUCTOR_CACHE_DIR="${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" +export TRITON_CACHE_DIR_BASE="${RUN_DIR}/triton-cache" +export CUDA_CACHE_PATH="${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" +export PYTHONPATH="${REPO_ROOT}" PYTHONNOUSERSITE=1 PIP_CONSTRAINT="" +export UV_CACHE_DIR="${SCRATCH_ROOT}/uv-cache/megatron_lm" UV_LINK_MODE=copy +export UV_PROJECT_ENVIRONMENT="${ENV_ROOT}/.venv" +export VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}" +export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NVTE_FWD_LAYERNORM_SM_MARGIN=16 NVTE_BWD_LAYERNORM_SM_MARGIN=16 +export NCCL_P2P_NET_CHUNKSIZE=2097152 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export NCCL_DEBUG=WARN NCCL_SHM_DISABLE=1 NCCL_PROTO=simple NCCL_NVLS_ENABLE=0 +export TORCH_NCCL_AVOID_RECORD_STREAMS=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 + +export TRAINING_STAGE MODEL_PROVIDER ENABLE_EXPERIMENTAL MOE_ROUTER_FORCE_LOAD_BALANCING +export TRAIN_ITERS NUM_MICROBATCHES MICRO_BATCH_SIZE GLOBAL_BATCH_SIZE LOG_INTERVAL +export ENCODER_TP ENCODER_CP ENCODER_PP ENCODER_DP ENCODER_EP +export LLM_TP LLM_CP LLM_PP LLM_DP LLM_EP LLM_EXPT_TP LLM_ONLY +export LR MIN_LR WEIGHT_DECAY LR_DECAY_STYLE +export LR_WARMUP_SAMPLES LR_DECAY_SAMPLES LR_WSD_DECAY_SAMPLES LR_WSD_DECAY_STYLE TRAIN_SAMPLES +export NUM_WORKERS PACKING_BUFFER_SIZE SHUFFLE_BUFFER_SIZE MAX_SAMPLES_PER_SEQUENCE CHECK_HEL_PATHS +export TOKENIZER_MODEL VISION_CKPT + +export DUMP_DATA_ONLY="${DUMP_DATA_ONLY:-0}" +export DUMP_N_STEPS="${DUMP_N_STEPS:-5}" +export DUMP_OUTPUT_DIR="${DUMP_OUTPUT_DIR:-${RUN_DIR}/dumps}" + +TRAIN_LAUNCH_ARGS=( + --class-token-len 10 + --image-tag-type internvl + --max-num-tiles 1 + --overlap-grad-reduce --overlap-param-gather + --ddp-num-buckets 8 --ddp-pad-buckets-for-high-nccl-busbw + --correct-encoder-grad-for-partial-participation + --seed 1234 + --save "${CHECKPOINT_SAVE_PATH}" + --save-interval "${SAVE_INTERVAL}" + --no-load-optim --no-load-rng + --load-nemotron-checkpoint "${NEMOTRON_CKPT}" + --no-dynamic-resolution # parity step 1: fixed-res images on both sides +) + +CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" +[[ "${REPO_ROOT}" == "${SCRATCH_ROOT}"/* ]] || CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" + +echo "=== hetero parity 100-step ===" +echo "repo=${REPO_ROOT} run_dir=${RUN_DIR}" +echo "world_size=${WORLD_SIZE} gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} train_iters=${TRAIN_ITERS}" +echo "layout: encoder(dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP})" +echo "ckpt=${NEMOTRON_CKPT}" +echo "=============================" + +srun --kill-on-bad-exit=1 \ + --ntasks="${WORLD_SIZE}" \ + --ntasks-per-node=8 \ + --container-image="${CONTAINER_IMAGE}" \ + --no-container-mount-home \ + --container-mounts="${CONTAINER_MOUNTS}" \ + --container-workdir="${REPO_ROOT}" \ + bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; + # Optionally swap the container-baked energon for an editable clone. + if [ -n "${USE_HETERO_ENERGON_OVERRIDE:-}" ]; then + ESRC="${HETERO_ENERGON_SRC:-/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/megatron-energon-editable/src/megatron/energon}" + EDST="/usr/local/lib/python3.12/dist-packages/megatron/energon" + SENT="/tmp/.energon_installed_${SLURM_JOB_ID}" + if [ "${SLURM_LOCALID}" -eq 0 ]; then + echo "[hetero_sbatch] swapping energon: $ESRC -> $EDST" + rm -rf "$EDST"; cp -r "$ESRC" "$EDST"; touch "$SENT" + else + while [ ! -f "$SENT" ]; do sleep 1; done + fi + fi + if [ -n "${HETERO_SKIP_UV:-}" ]; then export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"; exec bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; else exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; fi + ' \ + bash "${TRAIN_LAUNCH_ARGS[@]}" diff --git a/examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh b/examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh new file mode 100755 index 00000000000..6009c579bb6 --- /dev/null +++ b/examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# 150-step train-loss parity run — Sanjeev-side recipe. +# Thin wrapper around the pretrain script with env overrides for parity: +# 2 nodes, GBS=8, --calculate-per-token-loss enabled. +# +# Submit from anywhere; this script cds into the reference repo on the +# cluster and shells out to its pretrain entry point. + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 2 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=02:00:00 +#SBATCH -J sanj-parity-100step +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch +SANJEEV_REPO="${SANJEEV_REPO:-${SCRATCH_ROOT}/sanjeev-repos/megatron-lm-clean}" +CONTAINER_IMAGE="${SANJEEV_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" +TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" + +# Ckpt to resume from. Flip CKPT_RUN_ROOT / CKPT_STEP for a different ckpt. +CKPT_RUN_ROOT="/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511" +CKPT_STEP="${CKPT_STEP:-1000}" + +RUN_NAME="sanj-parity-100step" +RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" +mkdir -p "${RUN_DIR}/logs" "${RUN_DIR}/save" "${RUN_DIR}/tb" + +# ---- Overrides passed to the pretrain script -------------------------------- +# Topology: TP=2 EP=16. EDP/DP fall out from world_size; we constrain TP/EP/MBS/GBS. +export VISION_MODEL_TYPE=radio +export RADIO_ENCODER_DIR=post-c-radio-omni +export TP=2 +export EP=16 +export NUM_EXPERTS=128 +export MOE_ROUTER_TOPK=6 +export MBS=1 +export GBS=8 +export NUM_WORKERS=2 +export PACKING_BUFFER_SIZE=4 # Match hetero side. Larger buffers reorder samples differently → per-rank batches diverge. +export SEQ_LEN=8192 +export DECODER_SEQ_LEN=8192 +# 150 optimizer steps after the resume + the ckpt's iter_1000 offset. +export TRAIN_SAMPLES=$(( 150 * GBS + 1000 * GBS )) +# Override LR_WARMUP_SAMPLES so LR_DECAY_SAMPLES stays positive. With +# --no-load-rng / --no-load-optim we restart the scheduler at iter 0 anyway. +export LR_WARMUP_SAMPLES=0 +export LR_WSD_DECAY_SAMPLES=1 +export EXIT_MIN=240 +export LOG_INTERVAL=1 +export EVAL_INTERVAL=99999999999 +export EVAL_ITERS=0 +export SAVE_INTERVAL=99999999999 +export USE_DYNAMIC_RES=0 # parity step 1: fixed-res images on both sides +export SEQUENCE_PARALLEL=1 + +# Ckpt paths (the pretrain script reads these and routes via --load / --save). +export LOAD_CHECKPOINT_DIR="${CKPT_RUN_ROOT}/checkpoints" +export SAVE_CHECKPOINT_DIR="${RUN_DIR}/save" +export OUTPUT="${RUN_DIR}" +export LOGS_DIR="${RUN_DIR}/logs" +export TENSORBOARD_DIR="${RUN_DIR}/tb" +export WANDB_DIR="${RUN_DIR}/wandb" +export RUN_NAME + +# --ckpt-step pins the resume iter. --calculate-per-token-loss aligns gradient +# formula with hetero. --no-load-rng restarts the scheduler from seed=1234. +# Disable MTP and selective recompute for parity (hetero side runs without +# MTP, and --mtp-num-layers 0 also sidesteps an MTP-block autograd bug under +# this resume config). +export HYBRID_LAYER_PATTERN="MEMEM*EMEM*EMEM*EMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEME" +export DISABLE_RECOMPUTE=1 +# --pixel-shuffle: gated by USE_DYNAMIC_RES=1 in the recipe but the ckpt was +# trained with pixel-shuffle on (vision_projection input dim 5120 = 4*1280), +# so re-add it explicitly when USE_DYNAMIC_RES=0. +export EXTRA_MEGATRON_ARGS="--ckpt-step ${CKPT_STEP} --calculate-per-token-loss --no-load-rng --no-load-optim --mtp-num-layers 0 --pixel-shuffle" + +# Container + repo +export CONTAINER_IMAGE_OVERRIDE="${CONTAINER_IMAGE}" +export TOKENIZER_MODEL="${TOKENIZER_MODEL}" +export MEGATRON_ROOT="${SANJEEV_REPO}" +export SBATCH_NODES=2 + +# The blend yaml's val: section eagerly post_initializes both splits even with +# --eval-iters 0. /home is NFS-mounted so reading from the cluster-shared home +# makes the val split resolvable cheaply; it's never actually iterated. +export MULTIMODAL_DATA_ROOT=/home/sasatheesh/data/multimodal_data + +export DUMP_DATA_ONLY="${DUMP_DATA_ONLY:-0}" +export DUMP_N_STEPS="${DUMP_N_STEPS:-5}" +export DUMP_OUTPUT_DIR="${DUMP_OUTPUT_DIR:-${RUN_DIR}/dumps}" + +cd "${SANJEEV_REPO}" +exec bash "${SANJEEV_REPO}/examples/multimodal/v3/pretrain_3b_nano_vlm_sota_90t_10v.sh" diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index efd4f87136c..bdbdd07aea0 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -121,8 +121,7 @@ def parse_args() -> argparse.Namespace: default=None, help=( "Total training budget in consumed samples. When set, --train-iters is " - "re-derived as ceil(train_samples / global_batch_size). Matches Sanjeev's " - "samples-based recipe." + "re-derived as ceil(train_samples / global_batch_size)." ), ) train.add_argument("--lr", type=float, default=1.0e-4) @@ -210,6 +209,18 @@ def parse_args() -> argparse.Namespace: "bus bandwidth at large DP counts." ), ) + train.add_argument( + "--correct-encoder-grad-for-partial-participation", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "When some encoder DP ranks see text-only batches, scale vision " + "grads post-DP-reduce by encoder_dp_size / participation_count so " + "the vision encoder learns at full rate instead of being diluted. " + "Default on; pass --no-correct-encoder-grad-for-partial-participation " + "to disable." + ), + ) train.add_argument("--seed", type=int, default=12345) train.add_argument("--log-interval", type=int, default=1) @@ -264,6 +275,15 @@ def parse_args() -> argparse.Namespace: "skip optimizer + scheduler state regardless of the other flags." ), ) + ckpt.add_argument( + "--load-nemotron-checkpoint", + type=str, + default=None, + help=( + "Path to a flat Nemotron-format VLM dist-ckpt. Loads weights and " + "starts training at iter 0; mutually exclusive with --load." + ), + ) ckpt.add_argument( "--dist-ckpt-optim-fully-reshardable", action=argparse.BooleanOptionalAction, diff --git a/examples/mimo/training/hetero/grad_sync.py b/examples/mimo/training/hetero/grad_sync.py index be5ee7b0c78..2e72196c415 100644 --- a/examples/mimo/training/hetero/grad_sync.py +++ b/examples/mimo/training/hetero/grad_sync.py @@ -16,11 +16,53 @@ from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.utils import is_pp_last_stage - -def configure_grad_sync(mimo_model: MimoModel, topology: HeteroTopology) -> None: +# Sentinel attribute set on a modality submodule by forward_step when its rank +# processed image input this step. Used instead of scanning grad buffers. +_PARTICIPATED_ATTR = "_mimo_rank_processed_input" + + +def mark_modality_participation(model, batch) -> None: + """Tag each modality submodule with whether this rank has image input + this step. Called from forward_step before the model forward. + """ + if not hasattr(model, "modality_submodules"): + return + images = batch.get("images") if isinstance(batch, dict) else None + if isinstance(images, torch.Tensor): + had_input = images.numel() > 0 + elif isinstance(images, (list, tuple)): + had_input = len(images) > 0 + else: + had_input = False + for submodule in model.modality_submodules.values(): + if submodule is not None: + setattr(submodule, _PARTICIPATED_ATTR, had_input) + + +def reset_modality_participation(mimo_model: MimoModel) -> None: + """Clear per-step participation flags at the top of each train_step.""" + for submodule in mimo_model.modality_submodules.values(): + if submodule is not None: + setattr(submodule, _PARTICIPATED_ATTR, False) + + +def _vision_participation_count(submodule, vision_dp_group) -> float: + """All-reduce a 1-element bool across the vision DP group to get the + number of DP ranks that processed image input this step. + """ + val = 1.0 if getattr(submodule, _PARTICIPATED_ATTR, False) else 0.0 + indicator = torch.tensor([val], dtype=torch.float32, device="cuda") + dist.all_reduce(indicator, op=dist.ReduceOp.SUM, group=vision_dp_group) + return float(indicator.item()) + + +def configure_grad_sync(args, mimo_model: MimoModel, topology: HeteroTopology) -> None: """Configure grad-finalization callbacks consumed by the pipeline schedule.""" language_pg = topology.language_pg vision_pg = topology.vision_pg + correct_encoder_grad = bool( + getattr(args, "correct_encoder_grad_for_partial_participation", False) + ) def is_token_source_rank() -> bool: return ( @@ -32,18 +74,18 @@ def is_token_source_rank() -> bool: def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwargs): if num_tokens is None: - raise RuntimeError("train_hetero.py expects calculate_per_token_loss=True") + raise RuntimeError("hetero train loop expects calculate_per_token_loss=True") global_num_tokens = torch.zeros(1, dtype=torch.float32, device="cuda") if is_token_source_rank(): # MCore has already summed loss-mask token counts across microbatches - # for this gradient-accumulation step. Match Megatron's normalization - # domain by reducing the language last-stage count over DP and CP. + # for this gradient-accumulation step. Reduce over DP/CP to match + # Megatron's normalization domain. token_count = num_tokens.to(device="cuda", dtype=torch.float32).sum().view(1) dist.all_reduce(token_count, op=dist.ReduceOp.SUM, group=language_pg.dp_cp) if dist.get_rank(language_pg.dp_cp) == 0: global_num_tokens.copy_(token_count) - # Publish the already DP/CP-reduced language token count to encoder ranks too. + # Publish the language-side count to encoder ranks too. dist.all_reduce(global_num_tokens, op=dist.ReduceOp.MAX) global_num_tokens_value = global_num_tokens.item() @@ -56,26 +98,43 @@ def finalize_grads_func(_model_list, num_tokens, force_all_reduce=False, **_kwar force_all_reduce=force_all_reduce, ) debug_rank("language grads finalized") + + # Combine the per-token normalization with any partial-participation + # correction into a single scale_gradients call per submodule. + lang_scale = 1.0 / global_num_tokens_value if global_num_tokens_value > 0 else 0.0 + if lang_scale != 0.0 and mimo_model.language_model is not None: + debug_rank("scaling language grads") + mimo_model.language_model.scale_gradients(lang_scale) + for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("finalizing vision grads") - finalize_model_grads( - [submodule], - num_tokens=None, - pg_collection=vision_pg, - force_all_reduce=force_all_reduce, - ) - debug_rank("vision grads finalized") - - if global_num_tokens_value > 0: - scale = 1.0 / global_num_tokens_value - if mimo_model.language_model is not None: - debug_rank("scaling language grads") - mimo_model.language_model.scale_gradients(scale) - for submodule in mimo_model.modality_submodules.values(): - if submodule is not None: - debug_rank("scaling vision grads") - submodule.scale_gradients(scale) + if submodule is None: + continue + vision_scale = lang_scale + if correct_encoder_grad and vision_pg is not None: + vision_dp_group = getattr(vision_pg, "dp", None) + if is_process_group_member(vision_dp_group): + vision_dp_size = dist.get_world_size(vision_dp_group) + if vision_dp_size > 1: + participation = _vision_participation_count( + submodule, vision_dp_group + ) + debug_rank( + f"vision participation: {participation}/{vision_dp_size}" + ) + if 0.0 < participation < vision_dp_size: + vision_scale *= vision_dp_size / participation + + debug_rank("finalizing vision grads") + finalize_model_grads( + [submodule], + num_tokens=None, + pg_collection=vision_pg, + force_all_reduce=force_all_reduce, + ) + debug_rank("vision grads finalized") + if vision_scale != 0.0: + debug_rank("scaling vision grads") + submodule.scale_gradients(vision_scale) mimo_model.config.no_sync_func = build_no_sync_func(mimo_model) mimo_model.config.finalize_model_grads_func = finalize_grads_func diff --git a/examples/mimo/training/hetero/loop.py b/examples/mimo/training/hetero/loop.py index ac3d723eda3..882db12cb18 100644 --- a/examples/mimo/training/hetero/loop.py +++ b/examples/mimo/training/hetero/loop.py @@ -5,8 +5,10 @@ from __future__ import annotations import argparse +import random from typing import Optional +import numpy as np import torch from examples.mimo.training.hetero.args import prepare_args @@ -21,6 +23,7 @@ from examples.mimo.training.hetero.timeline import configure_hetero_timeline from examples.mimo.training.hetero.topology import HeteroTopology, create_topology from examples.mimo.utils.hetero import debug_rank +from examples.mimo.utils.model_helpers import load_and_refresh_nemotron_checkpoint from megatron.core.models.mimo.model.base import MimoModel from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator from megatron.core.pipeline_parallel.timeline import ( @@ -43,11 +46,17 @@ def run_train_loop(args: argparse.Namespace) -> None: if timeline_summary is not None: print_rank_0(timeline_summary) + # Match Megatron's _set_random_seed: seed python random and numpy + # too. energon's text_packing.random.shuffle uses the global random + # module, so dataset-construction RNG draws would diverge otherwise. + random.seed(args.seed) + np.random.seed(args.seed) torch.manual_seed(args.seed) + debug_rank("building MIMO model") model = build_mimo_runtime(args, topology) debug_rank("configuring gradient sync") - configure_grad_sync(model, topology) + configure_grad_sync(args, model, topology) debug_rank("building MIMO optimizer") optimizer = build_optimizer(args, model) @@ -62,7 +71,12 @@ def run_train_loop(args: argparse.Namespace) -> None: logger = HeteroTrainingLogger(args=args, topology=topology) debug_rank("training setup ready") - start_iteration = load_checkpoint(model, optimizer, opt_param_scheduler, args, topology) + nemotron_ckpt = getattr(args, "load_nemotron_checkpoint", None) + if nemotron_ckpt: + load_and_refresh_nemotron_checkpoint(model, optimizer, topology, args) + start_iteration = 0 + else: + start_iteration = load_checkpoint(model, optimizer, opt_param_scheduler, args, topology) if start_iteration >= args.train_iters: print_rank_0( f"Resume iteration ({start_iteration}) >= --train-iters ({args.train_iters}); " diff --git a/examples/mimo/training/hetero/runtime.py b/examples/mimo/training/hetero/runtime.py index 0e340ef6473..0550099d066 100644 --- a/examples/mimo/training/hetero/runtime.py +++ b/examples/mimo/training/hetero/runtime.py @@ -120,6 +120,9 @@ def wrap_active_modules_with_ddp( bucket_size=_resolve_bucket_size(args, mimo_model.language_model), pad_buckets_for_high_nccl_busbw=pad_buckets, use_distributed_optimizer=True, + # Keep main_grad in fp32. Default False → bf16 main_grad → step-2 + # weight drift after Adam. + grad_reduce_in_fp32=getattr(args, "accumulate_allreduce_grads_in_fp32", True), ) debug_rank("wrapping language model in DDP") mimo_model.language_model = DistributedDataParallel( @@ -154,6 +157,7 @@ def wrap_active_modules_with_ddp( bucket_size=_resolve_bucket_size(args, submodule), pad_buckets_for_high_nccl_busbw=pad_buckets, use_distributed_optimizer=True, + grad_reduce_in_fp32=getattr(args, "accumulate_allreduce_grads_in_fp32", True), ) debug_rank("wrapping vision submodule in DDP") mimo_model.modality_submodules[topology.encoder_name] = DistributedDataParallel( diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index f136de45873..494ecd9c730 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -13,7 +13,11 @@ import torch.distributed as dist import megatron.core.pipeline_parallel.schedules as schedule -from examples.mimo.training.hetero.grad_sync import zero_active_grad_buffers +from examples.mimo.training.hetero.grad_sync import ( + mark_modality_participation, + reset_modality_participation, + zero_active_grad_buffers, +) from examples.mimo.training.hetero.optimizer import get_global_batch_size from examples.mimo.training.hetero.topology import HeteroTopology from examples.mimo.utils.hetero import debug_rank @@ -68,6 +72,7 @@ def forward_step(data_iterator, model): batch = next(data_iterator) if data_iterator is not None else {"input_ids": None} with timeline_event("data.to_cuda", cuda=True): batch = move_batch_to_cuda(batch) + mark_modality_participation(model, batch) debug_rank("forward_step batch prepared") debug_rank("forward_step model call start") output_tensor, loss_mask = model(**batch) @@ -99,6 +104,7 @@ def train_step( ) -> TrainStepResult: """Run one Megatron-shaped hetero training step.""" zero_active_grad_buffers(model) + reset_modality_participation(model) optimizer.zero_grad() debug_rank("starting forward/backward schedule") diff --git a/examples/mimo/utils/model_helpers.py b/examples/mimo/utils/model_helpers.py index 2872158a9f1..744963054cc 100644 --- a/examples/mimo/utils/model_helpers.py +++ b/examples/mimo/utils/model_helpers.py @@ -1,31 +1,63 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -""" -Utility helpers for mimo models. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Helpers to load non-MIMO Nemotron VLM checkpoints into hetero MIMO models. + +Vision and language live on disjoint rank grids in hetero, so encoder ranks +load only ``vision_model.*`` / ``vision_projection.*`` and LLM ranks load +only ``language_model.*``. """ +from __future__ import annotations + +import os +from typing import Any + import torch + +from examples.mimo.utils.hetero import is_process_group_member from megatron.core import dist_checkpointing +from megatron.core.dist_checkpointing.validation import StrictHandling + + +def _resolve_ckpt_dir(ckpt_dir: str) -> str: + """Resolve a checkpoint path to the actual iteration directory. + + If ``ckpt_dir`` contains ``latest_checkpointed_iteration.txt``, read it + and return the corresponding ``iter_NNNNNNN`` subdirectory. Otherwise + return ``ckpt_dir`` unchanged (assumed to already point at an iter dir). + """ + tracker = os.path.join(ckpt_dir, "latest_checkpointed_iteration.txt") + if os.path.isfile(tracker): + with open(tracker) as f: + iteration = int(f.read().strip()) + iter_dir = os.path.join(ckpt_dir, f"iter_{iteration:07d}") + if not os.path.isdir(iter_dir): + raise FileNotFoundError( + f"Checkpoint tracker points to iteration {iteration} but " + f"{iter_dir} does not exist" + ) + return iter_dir + return ckpt_dir def load_submodule_ckpt(module: torch.nn.Module, ckpt_dir: str): - """Load *ckpt_dir* into *module* using Megatron distributed-checkpointing.""" + """Load ``ckpt_dir`` into ``module`` using a flat ``module.*`` prefix. - # 1) Ask for tensors using a `module.` prefix so they match checkpoint keys. + Retained from the original POC; not used by the hetero loader below. + Kept so older inference scripts continue to import successfully. + """ sharded_sd_with_prefix = module.sharded_state_dict(prefix="module.") - # Remove fp8 extra_state tensors – they may not exist in older checkpoints. for k in list(sharded_sd_with_prefix.keys()): if "extra_state" in k: del sharded_sd_with_prefix[k] - # 2) Wrap it under a root key just as in user snippet; this becomes the state - # dict returned by `load` so we can easily strip the prefix afterwards. - wrapper_sd = dict(state_dict=sharded_sd_with_prefix) + wrapper_sd = {"state_dict": sharded_sd_with_prefix} loaded = dist_checkpointing.load( sharded_state_dict=wrapper_sd, checkpoint_dir=ckpt_dir, + strict=StrictHandling.LOG_UNEXPECTED, ) - # 3) Remove the prefix and push into the module. cleaned = {k.removeprefix("module."): v for k, v in loaded["state_dict"].items()} incompatible = module.load_state_dict(cleaned, strict=False) @@ -35,3 +67,270 @@ def load_submodule_ckpt(module: torch.nn.Module, ckpt_dir: str): raise RuntimeError( f"load_state_dict had unexpected mismatch. Missing: {missing}, Unexpected: {unexpected}" ) + + +def _load_submodule_from_ckpt( + module: torch.nn.Module, + ckpt_dir: str, + ckpt_prefix: str, + dp_cp_group=None, +) -> tuple[int, int]: + """Load one submodule from ``ckpt_dir`` under ``ckpt_prefix``. Returns + ``(n_loaded, n_total)`` parameter-tensor counts. ``dp_cp_group`` must be + passed when ``parallel_state`` isn't initialized.""" + metadata = {"dp_cp_group": dp_cp_group} if dp_cp_group is not None else None + sharded_sd = module.sharded_state_dict(prefix=ckpt_prefix, metadata=metadata) + + for k in list(sharded_sd.keys()): + if "extra_state" in k: + del sharded_sd[k] + + wrapper_sd = {"state_dict": sharded_sd} + loaded = dist_checkpointing.load( + sharded_state_dict=wrapper_sd, + checkpoint_dir=ckpt_dir, + strict=StrictHandling.LOG_UNEXPECTED, + ) + + cleaned = {k.removeprefix(ckpt_prefix): v for k, v in loaded["state_dict"].items()} + + model_sd = module.state_dict() + shape_mismatches = [] + for k, v in cleaned.items(): + if k in model_sd and isinstance(v, torch.Tensor) and isinstance(model_sd[k], torch.Tensor): + if v.shape != model_sd[k].shape: + shape_mismatches.append( + f" {k}: ckpt={list(v.shape)} vs model={list(model_sd[k].shape)}" + ) + if shape_mismatches: + raise RuntimeError( + f"Shape mismatches loading prefix '{ckpt_prefix}':\n" + "\n".join(shape_mismatches) + ) + + incompatible = module.load_state_dict(cleaned, strict=False) + unexpected = [k for k in incompatible.unexpected_keys if "extra_state" not in k] + missing = [k for k in incompatible.missing_keys if "extra_state" not in k] + if unexpected or missing: + raise RuntimeError( + f"load mismatch for prefix '{ckpt_prefix}'. " + f"Missing: {missing}, Unexpected: {unexpected}" + ) + + n_loaded = sum(1 for k in cleaned if k in model_sd and "extra_state" not in k) + n_total = sum(1 for k in model_sd if "extra_state" not in k) + return n_loaded, n_total + + +def load_nemotron_vlm_ckpt_hetero( + mimo_model, + ckpt_dir: str, + encoder_name: str, + radio_encoder_key: str = "radio_encoder", + *, + has_encoder: bool, + has_language: bool, + language_dp_cp_group=None, + encoder_dp_cp_group=None, + skip_projection: bool = False, +) -> None: + """Load a flat ``vision_model.* / vision_projection.* / language_model.*`` + ckpt into a hetero MIMO model. Each rank loads only the submodules its + grid owns: encoder ranks load vision_model + vision_projection; LLM + ranks load language_model.""" + ckpt_dir = _resolve_ckpt_dir(ckpt_dir) + rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0 + if rank == 0: + print(f"[load-nemotron-vlm-ckpt] resolved iter_dir: {ckpt_dir}", flush=True) + + # Build a SINGLE combined sharded state dict containing all submodules + # this rank participates in. We then issue ONE `dist_checkpointing.load` + # call across all world ranks. This is required because mcore's + # `dist_checkpointing.load` internally does world-collectives — splitting + # the load into separate calls per grid deadlocks (encoder ranks finish + # and hit a world barrier while LLM ranks are still inside the load). + combined_sd: dict[str, Any] = {} + targets: list[tuple[torch.nn.Module, str, str]] = [] + + def _drill_through_ddp(mod): + """Unwrap DDP / Float16Module wrappers so we hit the raw nn.Module.""" + try: + from megatron.core.distributed import DistributedDataParallel as _DDP + except Exception: # pylint: disable=broad-except + _DDP = () + seen = set() + while True: + inner = getattr(mod, "module", None) + if inner is None or id(inner) in seen: + return mod + seen.add(id(inner)) + mod = inner + + if has_language: + if not hasattr(mimo_model, "language_model") or mimo_model.language_model is None: + raise RuntimeError( + "has_language=True but mimo_model.language_model is None on this rank." + ) + if language_dp_cp_group is None: + raise RuntimeError( + "has_language=True requires language_dp_cp_group (our hetero loop does " + "not initialize megatron.core.parallel_state)." + ) + # After wrap_active_modules_with_ddp, mimo_model.language_model is a + # DistributedDataParallel wrapper; drill through to call + # sharded_state_dict on the raw model. + lm_raw = _drill_through_ddp(mimo_model.language_model) + lm_sd = lm_raw.sharded_state_dict( + prefix="language_model.", metadata={"dp_cp_group": language_dp_cp_group} + ) + for k in list(lm_sd.keys()): + if "extra_state" in k: + del lm_sd[k] + combined_sd.update(lm_sd) + targets.append((lm_raw, "language_model.", "language_model")) + + if has_encoder: + submodules = getattr(mimo_model, "modality_submodules", None) + if submodules is None or encoder_name not in submodules: + raise RuntimeError( + f"has_encoder=True but mimo_model.modality_submodules[{encoder_name!r}] missing." + ) + # Same DDP-unwrap dance for the encoder-side submodule. + vision_submodule = _drill_through_ddp(submodules[encoder_name]) + encoders = getattr(vision_submodule, "encoders", None) + if encoders is None or radio_encoder_key not in encoders: + raise RuntimeError(f"vision submodule missing encoders[{radio_encoder_key!r}].") + radio_wrapper = encoders[radio_encoder_key] + radio_model = getattr(radio_wrapper, "radio_model", None) + if radio_model is None: + raise RuntimeError( + f"encoders[{radio_encoder_key!r}].radio_model is None on this rank." + ) + if encoder_dp_cp_group is None: + raise RuntimeError( + "has_encoder=True requires encoder_dp_cp_group (our hetero loop does " + "not initialize megatron.core.parallel_state)." + ) + radio_sd = radio_model.sharded_state_dict( + prefix="vision_model.", metadata={"dp_cp_group": encoder_dp_cp_group} + ) + for k in list(radio_sd.keys()): + if "extra_state" in k: + del radio_sd[k] + combined_sd.update(radio_sd) + targets.append( + (radio_model, "vision_model.", f"encoders.{radio_encoder_key}.radio_model") + ) + + if not skip_projection: + projectors = getattr(vision_submodule, "input_projections", None) + if projectors is None or len(projectors) == 0: + raise RuntimeError("vision submodule has no input_projections to load into.") + proj_sd = projectors[0].sharded_state_dict( + prefix="vision_projection.", metadata={"dp_cp_group": encoder_dp_cp_group} + ) + for k in list(proj_sd.keys()): + if "extra_state" in k: + del proj_sd[k] + combined_sd.update(proj_sd) + targets.append((projectors[0], "vision_projection.", "input_projections[0]")) + + # Even ranks with no local targets (shouldn't happen in non-colocated + # hetero, but defensive) participate in the load so the world-collective + # has the full barrier population. + wrapper_sd = {"state_dict": combined_sd} + if rank == 0: + print( + f"[load-nemotron-vlm-ckpt] rank=0 combined_sd has {len(combined_sd)} keys " + f"across {len(targets)} target submodules", + flush=True, + ) + + loaded = dist_checkpointing.load( + sharded_state_dict=wrapper_sd, + checkpoint_dir=ckpt_dir, + strict=StrictHandling.LOG_UNEXPECTED, + ) + loaded_sd = loaded.get("state_dict", {}) + + # Apply loaded tensors back to each submodule by stripping its checkpoint prefix. + for module, prefix, label in targets: + cleaned = { + k.removeprefix(prefix): v for k, v in loaded_sd.items() if k.startswith(prefix) + } + model_sd = module.state_dict() + shape_mismatches = [] + for k, v in cleaned.items(): + if ( + k in model_sd + and isinstance(v, torch.Tensor) + and isinstance(model_sd[k], torch.Tensor) + ): + if v.shape != model_sd[k].shape: + shape_mismatches.append( + f" {k}: ckpt={list(v.shape)} vs model={list(model_sd[k].shape)}" + ) + if shape_mismatches: + raise RuntimeError( + f"Shape mismatches for prefix '{prefix}':\n" + "\n".join(shape_mismatches) + ) + + incompatible = module.load_state_dict(cleaned, strict=False) + unexpected = [k for k in incompatible.unexpected_keys if "extra_state" not in k] + missing = [k for k in incompatible.missing_keys if "extra_state" not in k] + if unexpected or missing: + raise RuntimeError( + f"load mismatch for prefix '{prefix}'. Missing: {missing}, " + f"Unexpected: {unexpected}" + ) + + n_loaded = sum(1 for k in cleaned if k in model_sd and "extra_state" not in k) + n_total = sum(1 for k in model_sd if "extra_state" not in k) + if rank == 0 or has_encoder: + print( + f"[load-nemotron-vlm-ckpt] rank={rank} '{prefix}*' -> {label}" + f" ({n_loaded}/{n_total} param tensors)", + flush=True, + ) + + +def load_and_refresh_nemotron_checkpoint(model, optimizer, topology, args) -> None: + """Load a Nemotron-format ckpt into a hetero MIMO model and resync the + optimizer's FP32 main params. DistributedOptimizer is built before this + custom load runs, so its shards otherwise hold the model-provider init + weights; ``reload_model_params`` syncs them to the loaded weights.""" + from examples.mimo.model_providers.nemotron_moe_vlm import NEMOTRON_VISION_ENCODER_KEY + + if args.load: + raise ValueError( + "--load and --load-nemotron-checkpoint are mutually exclusive; pick one" + ) + + rank_in_llm = topology.language_pg is not None and is_process_group_member( + getattr(topology.language_pg, "dp_cp", None) + ) + rank_in_enc = topology.vision_pg is not None and is_process_group_member( + getattr(topology.vision_pg, "dp_cp", None) + ) + has_encoder = ( + rank_in_enc + and topology.encoder_name in getattr(model, "modality_submodules", {}) + and model.modality_submodules[topology.encoder_name] is not None + ) + has_language = rank_in_llm and getattr(model, "language_model", None) is not None + + load_nemotron_vlm_ckpt_hetero( + model, + args.load_nemotron_checkpoint, + encoder_name=topology.encoder_name, + radio_encoder_key=NEMOTRON_VISION_ENCODER_KEY, + has_encoder=has_encoder, + has_language=has_language, + language_dp_cp_group=( + getattr(topology.language_pg, "dp_cp", None) if has_language else None + ), + encoder_dp_cp_group=( + getattr(topology.vision_pg, "dp_cp", None) if has_encoder else None + ), + skip_projection=False, + ) + optimizer.reload_model_params() diff --git a/megatron/core/models/vision/radio.py b/megatron/core/models/vision/radio.py index 3eb3e227e67..62ed3b6c59d 100644 --- a/megatron/core/models/vision/radio.py +++ b/megatron/core/models/vision/radio.py @@ -345,8 +345,12 @@ def window_select(pos_embed): ).to(pos_embed.dtype) else: max_dim = max(input_dims) + # Use align_corners=False on bilinear pos-embedding interpolation + # to match upstream RADIO. align_corners=True drifts numerics + # and breaks parity for any ckpt trained against the standard + # RADIO implementation. pos_embed = F.interpolate( - pos_embed.float(), size=(max_dim, max_dim), align_corners=True, mode="bilinear" + pos_embed.float(), size=(max_dim, max_dim), align_corners=False, mode="bilinear" ).to(pos_embed.dtype) pos_embed = window_select(pos_embed) @@ -355,7 +359,7 @@ def window_select(pos_embed): if pos_embed.shape[-2:] != input_dims: pos_embed = F.interpolate( - pos_embed.float(), size=input_dims, align_corners=True, mode="bilinear" + pos_embed.float(), size=input_dims, align_corners=False, mode="bilinear" ).to(pos_embed.dtype) pos_embed = pos_embed.flatten(2).permute(0, 2, 1) From aef90f1d59eae72d87654853c4f6ad756abaab76 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Tue, 19 May 2026 15:07:43 -0700 Subject: [PATCH 38/44] NMFW-464: route encoder samples through one Energon iterator per encoder rank (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-LLM-lane Energon iterators owned by each encoder DP rank with a single multiplexed iterator whose worker pool is sized ``args.num_workers * lanes_per_encoder``. Samples are routed back to their owning LLM lane using the producing worker's ``WorkerConfig.global_worker_id``, which the MIMO multimodal encoder now stamps onto every batch when ``attach_provenance=True``. At scale (encoder_dp small relative to llm_dp), the per-lane construction path issues ``lanes_per_encoder × num_workers`` shard-open events at iterator creation; collapsing to one iterator per encoder rank cuts the open-burst by ``lanes_per_encoder``× and avoids the previous workaround scripts that staggered loader construction across encoder ranks. The reshape preserves bit-wise sample parity with the per-lane path: - ``global_workers = world_size * num_workers`` is invariant under ``(world_size, num_workers) → (world_size/k, num_workers*k)``. ``WebdatasetSharder.split_samples_to_workers`` partitions shards by global worker index over ``global_workers``, so equal global_worker_ids ⇒ equal shards in equal order. - ``WorkerConfig.worker_seed`` hashes only ``(global_worker_id, seed_offset)`` (see ``megatron/energon/worker.py``); ``seed_offset`` is unchanged. - The routed-iterator's worker W on encoder rank E has ``global_worker_id = E * (num_workers * lanes_per_encoder) + W``, which equals the per-lane worker w on lane L=E*lanes_per_encoder + W//num_workers, w = W%num_workers. ``test_hetero_energon.py`` adds unit tests for ``_route_samples_to_lanes`` (round-robin fill, surplus FIFO, lane-offset shift, pull-budget overflow, out-of-range worker id, missing provenance) plus an algebraic parity test asserting the global_worker_id equivalence above and a global_workers-invariant sweep across (encoder_dp, llm_dp, num_workers) shapes. Co-authored-by: Claude Opus 4.7 (1M context) --- .../mimo/data/energon_multimodal_provider.py | 24 +- examples/mimo/data/hetero_energon.py | 199 +++++++++++++++- tests/unit_tests/test_hetero_energon.py | 225 +++++++++++++++++- 3 files changed, 438 insertions(+), 10 deletions(-) diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py index 07530923945..b3ddcb5223f 100644 --- a/examples/mimo/data/energon_multimodal_provider.py +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -30,6 +30,7 @@ def _supported_kwargs(fn, kwargs): import torch +from megatron.energon import WorkerConfig from megatron.energon.task_encoder.multimodal import ( MultiModalPackingEncoder, PackingConfig, @@ -86,6 +87,12 @@ def apply_chat_template(self, *args, **kwargs): class MimoMultiModalPackingEncoder(MultiModalPackingEncoder): """Remap Energon multimodal packed samples to MIMO batch inputs.""" + # Key under which the producing Energon worker's ``global_worker_id`` is + # stamped on each output batch when ``attach_provenance`` is enabled. + # Hetero MIMO uses this to route samples back to their LLM data lane when + # a single encoder-side Energon iterator multiplexes several lanes. + PROVENANCE_KEY = "__encoder_provenance__" + def __init__( self, vision_config: VisionConfig, @@ -94,11 +101,13 @@ def __init__( encoder_name: str = "radio_encoder", encoder_input_key: str = "x", target_seq_length: Optional[int] = None, + attach_provenance: bool = False, ) -> None: super().__init__(vision_config, packing_config, tokenizer) self.encoder_name = encoder_name self.encoder_input_key = encoder_input_key self._target_seq_length = target_seq_length + self._attach_provenance = attach_provenance self._embeddings_per_tile = get_num_image_embeddings( img_h=vision_config.img_h, img_w=vision_config.img_w, @@ -216,6 +225,14 @@ def batch(self, samples: list[PackedSample]) -> dict: raise RuntimeError(f"Packing requires micro_batch_size=1, got {batch_size}") result["packing_kwargs"] = _build_packing_kwargs(samples[0], max_len) + if self._attach_provenance: + active = WorkerConfig.active_worker_config + if active is None: + raise RuntimeError( + "attach_provenance=True requires an active Energon worker context" + ) + result[self.PROVENANCE_KEY] = active.global_worker_id() + return result @@ -269,7 +286,11 @@ def _build_packing_kwargs(sample: PackedSample, max_len: int) -> dict[str, torch def build_multimodal_encoder( - args, tokenizer, encoder_name: str = "radio_encoder", encoder_input_key: str = "x" + args, + tokenizer, + encoder_name: str = "radio_encoder", + encoder_input_key: str = "x", + attach_provenance: bool = False, ) -> MimoMultiModalPackingEncoder: """Build the MIMO Energon encoder from train args.""" target_seq_length = _resolve_target_seq_length(args) @@ -312,6 +333,7 @@ def build_multimodal_encoder( encoder_name=encoder_name, encoder_input_key=encoder_input_key, target_seq_length=target_seq_length, + attach_provenance=attach_provenance, ) diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 901a725f8af..5b481a078da 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -6,6 +6,7 @@ import hashlib import random +from collections import deque from typing import Callable, Optional import torch @@ -108,22 +109,204 @@ def _build_encoder_iterator(args, grid): random_seed=args.seed, ) - lane_iterators = [ - _build_single_lane_iterator( - args, tp_group=None, lane=lane, role="encoder-component", random_seed=args.seed + return _build_routed_encoder_iterator( + args, tp_group=tp_group, encoder_dp_rank=encoder_dp_rank, llm_lanes=llm_lanes + ) + + +def _route_samples_to_lanes( + loader_iter, + *, + lanes_per_encoder: int, + lane_offset: int, + num_workers_per_lane: int, + encoder_dp_rank: int, + pending_by_lane: list, + max_pulls_per_step: int, + provenance_key: str, +) -> tuple[list, int]: + """Pull samples from a single multiplexed loader and route each one to its LLM lane. + + Samples are routed by reading the producing worker's + ``WorkerConfig.global_worker_id()``, which the encoder batcher stamps under + ``provenance_key``. The mapping from worker id back to local lane is: + + global_worker_id = encoder_dp_rank * num_workers_enc + local_worker_id + global_llm_lane = global_worker_id // num_workers_per_lane + local_lane = global_llm_lane - lane_offset + + Surplus samples (a worker yields a second sample for a lane that's already + filled this step) are stashed in ``pending_by_lane`` and consumed on the + next encoder step. ``max_pulls_per_step`` bounds the loop so a stuck or + skewed worker pool fails loudly instead of silently stalling. + + Returns ``(lane_batches, pulls)`` where ``lane_batches[lane]`` is the sample + routed to local lane ``lane``. + """ + lane_batches: list = [None] * lanes_per_encoder + filled = 0 + for lane in range(lanes_per_encoder): + if pending_by_lane[lane]: + lane_batches[lane] = pending_by_lane[lane].popleft() + filled += 1 + pulls = 0 + while filled < lanes_per_encoder: + if pulls >= max_pulls_per_step: + missing = [i for i, b in enumerate(lane_batches) if b is None] + raise RuntimeError( + f"encoder dataloader did not yield samples for local_lanes={missing} " + f"in {max_pulls_per_step} pulls (encoder_dp_rank={encoder_dp_rank}); " + "check Energon worker rotation contract" + ) + sample = next(loader_iter) + pulls += 1 + wid = sample.pop(provenance_key, None) + if wid is None: + raise RuntimeError( + f"encoder sample missing {provenance_key!r}; " + "ensure build_multimodal_encoder was called with attach_provenance=True" + ) + global_llm_lane = wid // num_workers_per_lane + local_lane = global_llm_lane - lane_offset + if not (0 <= local_lane < lanes_per_encoder): + raise RuntimeError( + f"worker_id={wid} maps to global_llm_lane={global_llm_lane}, " + f"outside encoder rank {encoder_dp_rank} range " + f"[{lane_offset}, {lane_offset + lanes_per_encoder})" + ) + if lane_batches[local_lane] is None: + lane_batches[local_lane] = sample + filled += 1 + else: + pending_by_lane[local_lane].append(sample) + return lane_batches, pulls + + +def _build_routed_encoder_iterator(args, tp_group, encoder_dp_rank, llm_lanes): + """Build one Energon iterator per encoder rank and route samples back to LLM lanes. + + The previous implementation built ``lanes_per_encoder`` independent Energon + iterators per encoder rank — one per LLM data lane — which produces + ``lanes_per_encoder × num_workers`` shard-open events at construction. + This collapses that to a single Energon iterator with + ``num_workers = args.num_workers * lanes_per_encoder``; each emitted batch + is routed to its owning lane using the producing worker's + ``WorkerConfig.global_worker_id()`` that the encoder batcher stamps onto + every batch. + + Bit-wise sample parity with the per-lane iterator path is preserved by + Energon's design: ``global_workers = world_size * num_workers`` is invariant + under this reshape and per-worker seeds depend only on ``global_worker_id`` + and ``seed_offset`` (see ``megatron/energon/worker.py``), so each worker + here produces the same shards in the same order as the per-lane worker it + replaces. + """ + from examples.mimo.data.energon_multimodal_provider import ( + MimoMultiModalPackingEncoder, + build_multimodal_encoder, + ) + from megatron.energon import WorkerConfig, get_savable_loader, get_train_dataset + from megatron.energon.cache.no_cache import NoCachePool + + if args.num_workers < 1: + raise ValueError( + "routed encoder iterator requires args.num_workers >= 1 " + "(global_worker_id -> lane mapping divides by num_workers_per_lane); " + f"got {args.num_workers}" ) - for lane in llm_lanes - ] + lanes_per_encoder = len(llm_lanes) + num_workers_per_lane = args.num_workers + num_workers_enc = num_workers_per_lane * lanes_per_encoder + lane_offset = llm_lanes[0] + + tokenizer = _build_tokenizer(args) + encoder = build_multimodal_encoder( + args, + tokenizer, + encoder_name=getattr(args, "vision_encoder_key", "radio_encoder"), + encoder_input_key="x", + attach_provenance=True, + ) + worker_config = WorkerConfig( + rank=encoder_dp_rank, + world_size=args.encoder_dp, + num_workers=num_workers_enc, + data_parallel_group=None, + ) + debug_rank( + "building routed encoder dataloader " + f"encoder_dp_rank={encoder_dp_rank} encoder_dp={args.encoder_dp} " + f"num_workers_enc={num_workers_enc} lanes_per_encoder={lanes_per_encoder} " + f"lane_offset={lane_offset}" + ) + dataset = get_train_dataset( + args.data_path, + batch_size=args.micro_batch_size, + task_encoder=encoder, + worker_config=worker_config, + packing_buffer_size=args.packing_buffer_size, + shuffle_buffer_size=args.shuffle_buffer_size, + max_samples_per_sequence=args.max_samples_per_sequence, + ) + loader = get_savable_loader( + dataset, + cache_pool=NoCachePool(), + watchdog_timeout_seconds=5 * 60, + watchdog_initial_timeout_seconds=5 * 60, + ) + + loader_iter_holder: list = [iter(loader)] + # Dense integer keys (0..lanes_per_encoder-1) → use a list so the hot-path + # routing in ``_route_samples_to_lanes`` does O(1) array indexing rather + # than dict probing. + pending_by_lane: list[deque] = [deque() for _ in range(lanes_per_encoder)] + # Energon's SavableDataLoader rotates through every worker in one round, + # so a step worst case needs ``num_workers_enc`` pulls to fill every lane + # (one batch per worker, including the surplus to lanes that filled + # early). The 4× factor adds slack for transient rotation skew; we cap + # below by 2*num_workers_enc so configurations with high + # ``num_workers_per_lane`` aren't bounded too tightly. A genuine stall + # surfaces as a loud failure in ``_route_samples_to_lanes``. + max_pulls_per_step = max(4 * lanes_per_encoder, 2 * num_workers_enc) + provenance_key = MimoMultiModalPackingEncoder.PROVENANCE_KEY def next_encoder_batch(): - batches = [next(iterator) for iterator in lane_iterators] - signatures = [EnergonIterator._batch_signature(batch) for batch in batches] - return _combine_encoder_batches(batches), signatures + try: + lane_batches, _pulls = _route_samples_to_lanes( + loader_iter_holder[0], + lanes_per_encoder=lanes_per_encoder, + lane_offset=lane_offset, + num_workers_per_lane=num_workers_per_lane, + encoder_dp_rank=encoder_dp_rank, + pending_by_lane=pending_by_lane, + max_pulls_per_step=max_pulls_per_step, + provenance_key=provenance_key, + ) + except StopIteration: + # One-shot per epoch on savable-loader exhaustion. Any partial + # ``lane_batches`` accumulated before the exception is dropped — + # those samples count against the worker's seed sequence and are + # never delivered. Acceptable because webdataset is streamed as + # a pseudo-infinite source; this branch is rarely hit in practice. + loader_iter_holder[0] = iter(loader) + lane_batches, _pulls = _route_samples_to_lanes( + loader_iter_holder[0], + lanes_per_encoder=lanes_per_encoder, + lane_offset=lane_offset, + num_workers_per_lane=num_workers_per_lane, + encoder_dp_rank=encoder_dp_rank, + pending_by_lane=pending_by_lane, + max_pulls_per_step=max_pulls_per_step, + provenance_key=provenance_key, + ) + signatures = [EnergonIterator._batch_signature(batch) for batch in lane_batches] + return _combine_encoder_batches(lane_batches), signatures return EnergonIterator( None, tp_group=tp_group, source_rank=True, + random_seed=args.seed, local_batch_fn=next_encoder_batch, alignment_role="encoder", llm_lanes=llm_lanes, diff --git a/tests/unit_tests/test_hetero_energon.py b/tests/unit_tests/test_hetero_energon.py index d0da5509f01..833cd8d624f 100644 --- a/tests/unit_tests/test_hetero_energon.py +++ b/tests/unit_tests/test_hetero_energon.py @@ -1,11 +1,16 @@ # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. import random +from collections import deque +import pytest import torch from examples.mimo.data import hetero_energon -from examples.mimo.data.hetero_energon import EnergonIterator +from examples.mimo.data.energon_multimodal_provider import MimoMultiModalPackingEncoder +from examples.mimo.data.hetero_energon import EnergonIterator, _route_samples_to_lanes + +_PROVENANCE_KEY = MimoMultiModalPackingEncoder.PROVENANCE_KEY class RandomLoader: @@ -67,3 +72,221 @@ def test_combine_encoder_batches_drops_packing_and_concatenates_modalities(): assert images.shape == (3, 3, 4, 4) assert torch.all(images[:1] == 1) assert torch.all(images[1:] == 0) + + +# --------------------------------------------------------------------------- +# Routed encoder iterator — _route_samples_to_lanes tests +# --------------------------------------------------------------------------- + + +def _stamped(worker_id: int, payload: object) -> dict: + """Build a fake encoder batch carrying a provenance stamp.""" + return {_PROVENANCE_KEY: worker_id, "payload": payload} + + +def _make_loader(samples): + """Wrap a list of pre-stamped samples in an iterator the routing code can consume.""" + return iter(samples) + + +def test_route_samples_to_lanes_round_robin_assigns_workers_to_lanes(): + """Workers 0..NW-1 feed lane 0, NW..2NW-1 feed lane 1, etc.""" + # encoder_dp_rank=0, world hosts lanes 0..3 (lane_offset=0). + # num_workers_per_lane=2 → workers [0,1]→lane0, [2,3]→lane1, [4,5]→lane2, [6,7]→lane3. + samples = [_stamped(w, f"w{w}") for w in (0, 2, 4, 6)] + pending = [deque() for _ in range(4)] + lane_batches, pulls = _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=4, + lane_offset=0, + num_workers_per_lane=2, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=16, + provenance_key=_PROVENANCE_KEY, + ) + assert pulls == 4 + assert [b["payload"] for b in lane_batches] == ["w0", "w2", "w4", "w6"] + assert all(len(q) == 0 for q in pending.values()) + + +def test_route_samples_to_lanes_surplus_lands_in_pending_fifo(): + """A second sample for an already-filled lane is queued for next step.""" + # 2 lanes, NW=2: workers 0,1→lane0; 2,3→lane1. + # Loader yields w0 (lane0), w1 (lane0 surplus), w2 (lane1) — first step fills. + samples = [_stamped(0, "a"), _stamped(1, "b"), _stamped(2, "c")] + pending = [deque() for _ in range(2)] + lane_batches, pulls = _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=2, + lane_offset=0, + num_workers_per_lane=2, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=8, + provenance_key=_PROVENANCE_KEY, + ) + assert pulls == 3 + assert [b["payload"] for b in lane_batches] == ["a", "c"] + assert len(pending[0]) == 1 + assert pending[0][0]["payload"] == "b" + assert len(pending[1]) == 0 + + +def test_route_samples_to_lanes_drains_pending_before_pulling(): + """Pending lane-0 sample is consumed first; loader is only pulled for empty lanes.""" + pending = [deque([_stamped(0, "stashed")]), deque()] + # Loader has one new sample for lane 1. + samples = [_stamped(2, "fresh")] + lane_batches, pulls = _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=2, + lane_offset=0, + num_workers_per_lane=2, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=8, + provenance_key=_PROVENANCE_KEY, + ) + assert pulls == 1 + assert [b["payload"] for b in lane_batches] == ["stashed", "fresh"] + assert len(pending[0]) == 0 + + +def test_route_samples_to_lanes_lane_offset_shifts_global_lane(): + """Encoder rank E>0 owns a non-zero lane_offset; routing subtracts it correctly.""" + # encoder_dp_rank=1, encoder_dp=2, llm_dp=4, NW=1 → lane_offset=2, lanes 2,3 local. + # Workers in this encoder: global ids 2,3 (E*NW*k + W where k=2, NW=1 → 2 + W). + # global_llm_lane = global_worker_id // NW = 2,3 → local_lane = 0,1. + samples = [_stamped(2, "L2"), _stamped(3, "L3")] + pending = [deque() for _ in range(2)] + lane_batches, _ = _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=2, + lane_offset=2, + num_workers_per_lane=1, + encoder_dp_rank=1, + pending_by_lane=pending, + max_pulls_per_step=8, + provenance_key=_PROVENANCE_KEY, + ) + assert [b["payload"] for b in lane_batches] == ["L2", "L3"] + + +def test_route_samples_to_lanes_raises_on_pull_budget_exhaustion(): + """When the loader can't fill every lane in the budget, fail loudly.""" + # 2 lanes but loader only delivers to lane 0. + samples = [_stamped(0, "x"), _stamped(0, "y"), _stamped(1, "z")] + pending = [deque() for _ in range(2)] + with pytest.raises(RuntimeError, match="did not yield samples for local_lanes"): + _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=2, + lane_offset=0, + num_workers_per_lane=2, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=3, + provenance_key=_PROVENANCE_KEY, + ) + + +def test_route_samples_to_lanes_raises_on_out_of_range_worker(): + """A worker id from a foreign encoder rank surfaces as a hard error.""" + # encoder_dp_rank=0 owns lanes 0..1 with NW=2, so global_worker_id 0..3 are valid. + # A stray sample stamped with worker 4 (which belongs to rank 1) should fail. + samples = [_stamped(0, "ok"), _stamped(4, "stray")] + pending = [deque() for _ in range(2)] + with pytest.raises(RuntimeError, match="outside encoder rank"): + _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=2, + lane_offset=0, + num_workers_per_lane=2, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=8, + provenance_key=_PROVENANCE_KEY, + ) + + +def test_route_samples_to_lanes_raises_when_provenance_missing(): + """Samples without a provenance stamp fail with a clear message.""" + samples = [{"payload": "missing"}] + pending = [deque()] + with pytest.raises(RuntimeError, match="attach_provenance"): + _route_samples_to_lanes( + _make_loader(samples), + lanes_per_encoder=1, + lane_offset=0, + num_workers_per_lane=1, + encoder_dp_rank=0, + pending_by_lane=pending, + max_pulls_per_step=4, + provenance_key=_PROVENANCE_KEY, + ) + + +# --------------------------------------------------------------------------- +# Bit-wise parity: routed iterator must produce the same per-lane sample +# sequence as the previous per-lane iterators would have. +# --------------------------------------------------------------------------- + + +def test_routed_iterator_matches_per_lane_global_worker_ids(): + """For the (rank, world_size, num_workers) reshape used by the routed iterator, + the producing global_worker_id at the encoder side equals the per-lane + global_worker_id, lane-by-lane, sample-by-sample. + + This is the algebraic property that gives bit-wise sample parity with the + previous multi-iterator path. ``megatron.energon.worker.WorkerConfig.worker_seed`` + hashes only ``global_worker_id`` and ``seed_offset``, and + ``WebdatasetSharder.split_samples_to_workers`` partitions shards by global + worker index over ``global_workers = world_size * num_workers``, so equal + global_worker_ids ⇒ equal shards ⇒ equal samples in equal order. + """ + # llm_dp=8, encoder_dp=2 → lanes_per_encoder=4, NW=2 per lane. + llm_dp = 8 + encoder_dp = 2 + num_workers_per_lane = 2 + lanes_per_encoder = llm_dp // encoder_dp + num_workers_enc = num_workers_per_lane * lanes_per_encoder + + # OLD scheme: for lane L, the workers have global_worker_ids + # L*NW + w for w in [0, NW). + old_by_lane = { + lane: [lane * num_workers_per_lane + w for w in range(num_workers_per_lane)] + for lane in range(llm_dp) + } + + # NEW scheme: for encoder rank E, worker W → global_worker_id = E*num_workers_enc + W, + # routed to local_lane = (global_worker_id // NW) - lane_offset (= W // NW). + for encoder_dp_rank in range(encoder_dp): + lane_offset = encoder_dp_rank * lanes_per_encoder + new_by_local_lane: dict[int, list[int]] = {lane: [] for lane in range(lanes_per_encoder)} + for W in range(num_workers_enc): + gid_new = encoder_dp_rank * num_workers_enc + W + local_lane = (gid_new // num_workers_per_lane) - lane_offset + new_by_local_lane[local_lane].append(gid_new) + + for local_lane in range(lanes_per_encoder): + global_lane = lane_offset + local_lane + assert new_by_local_lane[local_lane] == old_by_lane[global_lane], ( + f"global_worker_id mismatch at encoder_dp_rank={encoder_dp_rank}, " + f"local_lane={local_lane}: new={new_by_local_lane[local_lane]} " + f"vs old={old_by_lane[global_lane]}" + ) + + +def test_routed_iterator_preserves_global_workers_invariant(): + """The reshape preserves the total global worker count, which is what makes + Energon's per-worker shard partitioning identical between the per-lane and + routed configurations (see split_samples_to_workers).""" + for llm_dp, encoder_dp, num_workers in [(8, 2, 2), (16, 1, 4), (32, 8, 2), (128, 16, 4)]: + lanes_per_encoder = llm_dp // encoder_dp + old_global_workers = llm_dp * num_workers + new_global_workers = encoder_dp * (num_workers * lanes_per_encoder) + assert old_global_workers == new_global_workers, ( + f"global_workers diverged for llm_dp={llm_dp} encoder_dp={encoder_dp}: " + f"old={old_global_workers} new={new_global_workers}" + ) From 7e3223d03cfeec9542b11df6b8d9e79eb61755ec Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Tue, 19 May 2026 16:14:38 -0700 Subject: [PATCH 39/44] NMFW-464: dynamic-resolution + RADIO final_layernorm parity for hetero MIMO VLM (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1) Dynamic-resolution RADIO ViT Patchify each image at its native aspect ratio with a token budget, interleave per-tile class tokens, run THD-packed attention through the transformer block. Line-for-line equivalent to sanj's pre-vlm-05 RADIOViTModel.forward dynres branch. - megatron/core/models/vision/radio.py: add dynamic_resolution kwarg, imgs_sizes + packed_seq_params forward kwargs, per-tile apply_pos_enc, interleaved CLS-token concat, packed_seq_params.cu_seqlens shift. - examples/mimo/data/energon_multimodal_provider.py: per-image n_tokens under dynamic_resolution; surface imgs_sizes + PackedSeqParams. - examples/mimo/model_providers/nemotron_moe_vlm.py: RADIOEncoderWrapper accepts dynres kwargs; interleaved-CLS removal mask; per-tile pixel-shuffle. - examples/mimo/training/hetero/step.py: PackedSeqParams cu_seqlens / max_seqlen tensors moved to CUDA before the encoder forward (TE THD attention hangs on H2D-sync of these tensors otherwise). 2) RADIO encoder final_layernorm parity with sanj Sanj's --mtp-num-layers 0 propagates to vision_config.mtp_num_layers via core_transformer_config_from_args. In TransformerBlock.has_final_layernorm_in_this_stage, the else-branch (mtp_num_layers is not None) allocates final_layernorm when the last decoder layer is in this stage — independent of post_process. Without this, hetero's frozen RADIO output magnitude was ~150x larger than sanj's (5344 vs 35.75) because the final LN never ran. The downstream projection + LLM were trained against the LN-normalized magnitude (gamma=1 / bias=0 from ckpt). - examples/mimo/model_providers/nemotron_moe_vlm.py:radio_vision_config sets config.mtp_num_layers = 0. - examples/mimo/utils/model_helpers.py:load_nemotron_vlm_ckpt_hetero switches dist_checkpointing.load to StrictHandling.RETURN_ALL and raises on any non-extra_state key the model requests but the ckpt does not have (mcore's "unexpected" set, opposite of PyTorch's "missing"). Weaker modes silently keep random-init values and masked this bug. Verified at iter 1, GBS=8 (3-node hetero + 2-node sanj parity pair): vision_proj hetero absmax=584 sanj absmax=584 cos=1.000010 combined_embeddings cos=1.000011 iter-1 lm_loss hetero=2.756 sanj=2.750 Δ=+0.22% iter-2/3 lm_loss rel Δ < 1.1% Co-authored-by: Claude Opus 4.7 (1M context) --- .../mimo/data/energon_multimodal_provider.py | 41 ++++- .../mimo/model_providers/nemotron_moe_vlm.py | 84 +++++++++- .../scripts/sbatch_hetero_parity_gbs192.sh | 148 ++++++++++++++++++ ...0step.sh => sbatch_hetero_parity_gbs32.sh} | 63 ++------ .../scripts/sbatch_sanjeev_parity_gbs192.sh | 75 +++++++++ ...step.sh => sbatch_sanjeev_parity_gbs32.sh} | 54 ++----- examples/mimo/training/hetero/step.py | 18 +++ examples/mimo/utils/model_helpers.py | 18 ++- megatron/core/models/vision/radio.py | 112 ++++++++++--- 9 files changed, 495 insertions(+), 118 deletions(-) create mode 100755 examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh rename examples/mimo/scripts/{sbatch_hetero_parity_100step.sh => sbatch_hetero_parity_gbs32.sh} (70%) create mode 100755 examples/mimo/scripts/sbatch_sanjeev_parity_gbs192.sh rename examples/mimo/scripts/{sbatch_sanjeev_parity_100step.sh => sbatch_sanjeev_parity_gbs32.sh} (53%) diff --git a/examples/mimo/data/energon_multimodal_provider.py b/examples/mimo/data/energon_multimodal_provider.py index b3ddcb5223f..0e754a52758 100644 --- a/examples/mimo/data/energon_multimodal_provider.py +++ b/examples/mimo/data/energon_multimodal_provider.py @@ -30,6 +30,7 @@ def _supported_kwargs(fn, kwargs): import torch +from megatron.core.packed_seq_params import PackedSeqParams from megatron.energon import WorkerConfig from megatron.energon.task_encoder.multimodal import ( MultiModalPackingEncoder, @@ -120,6 +121,12 @@ def __init__( max_num_tiles=vision_config.max_num_tiles, use_image_break_token=vision_config.use_image_break_token, ) + # Stashed so batch() can compute per-image embedding counts under + # dynamic resolution (where the constant emb_per_tile doesn't apply). + self._dynamic_resolution = getattr(vision_config, "dynamic_resolution", False) + self._patch_dim = vision_config.patch_dim + self._pixel_shuffle = vision_config.pixel_shuffle + self._conv_merging = vision_config.conv_merging def batch(self, samples: list[PackedSample]) -> dict: """Expand image placeholders and return a MIMO-compatible batch.""" @@ -147,7 +154,20 @@ def batch(self, samples: list[PackedSample]) -> dict: for idx, token in enumerate(tokens.tolist()): if token == image_token_id: n_tiles = num_tiles[img_idx] if img_idx < len(num_tiles) else 1 - n_tokens = n_tiles * emb_per_tile + if self._dynamic_resolution: + # Each image produces (h/p) * (w/p) patches; pixel_shuffle and + # conv_merging each halve both axes => divide by 4 each. + img_pix = sample.images[img_idx] + h_pix = img_pix.shape[-2] + w_pix = img_pix.shape[-1] + per_image = (h_pix // self._patch_dim) * (w_pix // self._patch_dim) + if self._pixel_shuffle: + per_image //= 4 + if self._conv_merging: + per_image //= 4 + n_tokens = per_image + else: + n_tokens = n_tiles * emb_per_tile if budget is not None and len(new_tokens) + n_tokens > budget: truncated = True break @@ -214,9 +234,24 @@ def batch(self, samples: list[PackedSample]) -> dict: } if all_images: - images = self.tiling_strategy.stack(all_images)[0] + images, imgs_sizes, cu_lengths, max_seqlen = self.tiling_strategy.stack(all_images) + encoder_inputs = {self.encoder_input_key: images} + if imgs_sizes is not None: + encoder_inputs["imgs_sizes"] = imgs_sizes.to(torch.int32) + if cu_lengths is not None and max_seqlen is not None: + # THD packing metadata for RADIO's variable-length attention. + # Class-token offsets get applied inside RADIO.forward. + cu = cu_lengths.to(torch.int32) + max_q = max_seqlen.to(torch.int32) if torch.is_tensor(max_seqlen) else torch.tensor(int(max_seqlen), dtype=torch.int32) + encoder_inputs["packed_seq_params"] = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu, + cu_seqlens_kv=cu, + max_seqlen_q=max_q, + max_seqlen_kv=max_q, + ) result["modality_inputs"] = { - "images": {self.encoder_name: {self.encoder_input_key: images}} + "images": {self.encoder_name: encoder_inputs} } is_packed = any(len(sample.cu_lengths) > 2 for sample in samples) diff --git a/examples/mimo/model_providers/nemotron_moe_vlm.py b/examples/mimo/model_providers/nemotron_moe_vlm.py index 4487d123fe0..cc50f5a2b61 100644 --- a/examples/mimo/model_providers/nemotron_moe_vlm.py +++ b/examples/mimo/model_providers/nemotron_moe_vlm.py @@ -237,6 +237,42 @@ def validate_model_provider_args(args: argparse.Namespace) -> None: raise ValueError("--pad-token-id must be within --vocab-size") +def _pixel_shuffle_dynamic_res(x, imgs_sizes, patch_dim, scale_factor=0.5, version=2): + """Pixel shuffle for dynamic resolution (variable tile sizes). + + Splits the packed sequence by per-tile lengths, applies pixel shuffle to each + tile, then re-concatenates. Mirrors sasatheesh/pre-vlm-05's + llava_model.pixel_shuffle_dynamic_res; vendored here to avoid touching the + upstream-owned llava_model.py. + """ + seq_lens = torch.prod(imgs_sizes // patch_dim, dim=-1) + splits = torch.split(x, seq_lens.tolist(), dim=-2) + + out = [] + for i, sv in enumerate(splits): + h = imgs_sizes[i][0] // patch_dim + w = imgs_sizes[i][1] // patch_dim + sv = sv.reshape(sv.shape[0], h, w, -1) + + n, h, w, c = sv.size() + sv = sv.view(n, h, int(w * scale_factor), int(c / scale_factor)) + sv = sv.permute(0, 2, 1, 3).contiguous() + sv = sv.view( + n, + int(w * scale_factor), + int(h * scale_factor), + int(c / (scale_factor * scale_factor)), + ) + + if version == 2: + sv = sv.permute(0, 2, 1, 3).contiguous() + + sv = sv.reshape(sv.shape[0], -1, sv.shape[-1]) + out.append(sv) + + return torch.cat(out, dim=-2) + + class RADIOEncoderWrapper(torch.nn.Module): """RADIO encoder wrapper matching the Nemotron6-MoE VLM provider.""" @@ -252,12 +288,14 @@ def __init__( drop_class_token: bool = True, apply_pixel_shuffle: bool = True, force_eval_mode: bool = False, + dynamic_resolution: bool = False, ) -> None: super().__init__() self.class_token_len = class_token_len self.drop_class_token = drop_class_token self.apply_pixel_shuffle = apply_pixel_shuffle self.force_eval_mode = force_eval_mode + self.dynamic_resolution = dynamic_resolution self.radio_model = RADIOViTModel( transformer_config=transformer_config, transformer_layer_spec=transformer_layer_spec, @@ -270,6 +308,7 @@ def __init__( max_img_w=2048, has_cpe=True, embedder_bias=False, + dynamic_resolution=dynamic_resolution, pg_collection=pg_collection, ) if self.force_eval_mode: @@ -287,19 +326,53 @@ def config(self): """Expose the underlying RADIO config for DDP wrapping.""" return self.radio_model.config - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward( + self, + x: torch.Tensor, + imgs_sizes: Optional[torch.Tensor] = None, + packed_seq_params=None, + ) -> torch.Tensor: """Run RADIO, drop class tokens, and apply pixel shuffle.""" context = torch.no_grad() if self.force_eval_mode else nullcontext() debug_rank(f"RADIO forward start: input_shape={tuple(x.shape)}") with context: x = x.to(dtype=self.radio_model.embedder.weight.dtype) - embeddings = self.radio_model(x) + embeddings = self.radio_model( + x, imgs_sizes=imgs_sizes, packed_seq_params=packed_seq_params + ) debug_rank(f"RADIO forward done: output_shape={tuple(embeddings.shape)}") if self.drop_class_token: - embeddings = embeddings[:, self.class_token_len :, :] + if ( + self.dynamic_resolution + and imgs_sizes is not None + and self.class_token_len > 0 + ): + # Class tokens are interleaved between tiles; build mask to remove them. + remove_mask = torch.full( + (embeddings.shape[-2],), True, dtype=torch.bool, device=embeddings.device + ) + patch_dim = self.radio_model.patch_dim + if torch.is_tensor(imgs_sizes): + seq_lens = torch.prod(imgs_sizes // patch_dim, dim=-1) + else: + seq_lens = torch.tensor( + [(h // patch_dim) * (w // patch_dim) for h, w in imgs_sizes] + ) + current_length = 0 + for sl in seq_lens: + remove_mask[current_length : current_length + self.class_token_len] = False + current_length += int(sl) + self.class_token_len + embeddings = embeddings[:, remove_mask, :] + else: + embeddings = embeddings[:, self.class_token_len :, :] debug_rank(f"RADIO class tokens dropped: output_shape={tuple(embeddings.shape)}") if self.apply_pixel_shuffle: - embeddings = pixel_shuffle(embeddings, scale_factor=0.5) + if self.dynamic_resolution and imgs_sizes is not None: + embeddings = _pixel_shuffle_dynamic_res( + embeddings, imgs_sizes, self.radio_model.patch_dim + ) + else: + embeddings = pixel_shuffle(embeddings, scale_factor=0.5) debug_rank(f"RADIO pixel shuffle done: output_shape={tuple(embeddings.shape)}") return embeddings @@ -458,6 +531,8 @@ def radio_vision_config(args: argparse.Namespace, tp_size: int, pp_size: int) -> config.attention_softmax_in_fp32 = True config.attention_dropout = 0.0 config.hidden_dropout = 0.0 + # Trigger TransformerBlock's final_layernorm allocation (matches sanj path). + config.mtp_num_layers = 0 return config @@ -597,6 +672,7 @@ def vision_submodules_spec( "drop_class_token": True, "apply_pixel_shuffle": True, "force_eval_mode": args.freeze_vit, + "dynamic_resolution": bool(getattr(args, "dynamic_resolution", False)), }, ) vision_projection_spec = ModuleSpec( diff --git a/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh b/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh new file mode 100755 index 00000000000..845dad69cd8 --- /dev/null +++ b/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# Long hetero-MIMO parity run vs Sanjeev pre-vlm-05. +# 9 nodes (1n encoder DP=8 + 8n LLM TP=2 DP=32 EP=16), GBS=192, 5000 iters, 4h. +# Paired with sbatch_sanjeev_parity_gbs192.sh. + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 9 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=04:00:00 +#SBATCH -J mimo-parity-gbs192 +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +if [[ -n "${SLURM_SUBMIT_DIR:-}" && -d "${SLURM_SUBMIT_DIR}/examples/mimo" ]]; then + REPO_ROOT="${SLURM_SUBMIT_DIR}" +else + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +fi + +SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch +CONTAINER_IMAGE="${HETERO_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" +export HETERO_SKIP_UV="${HETERO_SKIP_UV:-1}" +ENV_ROOT="${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39" +TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" +VISION_CKPT="${SCRATCH_ROOT}/encoders/post-c-radio-omni" + +NEMOTRON_CKPT="${NEMOTRON_CKPT:-/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511/checkpoints/iter_0001000}" + +RUN_NAME="mimo-parity-gbs192" +RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" + +# ---- topology: TP=2 EP=16 LLM (8 nodes, DP=32) + TP=1 DP=8 encoder lane (1 node) +ENCODER_TP=1; ENCODER_CP=1; ENCODER_PP=1; ENCODER_DP=8; ENCODER_EP=1 +LLM_TP=2; LLM_CP=1; LLM_PP=1; LLM_DP=32; LLM_EP=16; LLM_EXPT_TP=1 +LLM_ONLY=0 + +MICRO_BATCH_SIZE=1 +GLOBAL_BATCH_SIZE=192 +NUM_MICROBATCHES=$(( GLOBAL_BATCH_SIZE / (MICRO_BATCH_SIZE * LLM_DP) )) # = 6 +TRAIN_ITERS=5000 +LOG_INTERVAL=1 +SAVE_INTERVAL=99999999 + +LR=1.2e-3 +MIN_LR=1.2e-5 +WEIGHT_DECAY=0.1 +LR_DECAY_STYLE=WSD +LR_WARMUP_SAMPLES=0 +LR_DECAY_SAMPLES=121046313 +LR_WSD_DECAY_SAMPLES=1 +LR_WSD_DECAY_STYLE=minus_sqrt +TRAIN_SAMPLES=$(( TRAIN_ITERS * GLOBAL_BATCH_SIZE )) + +TRAINING_STAGE=stage2 +MODEL_PROVIDER=nemotron-moe-vlm-54l +ENABLE_EXPERIMENTAL=1 +MOE_ROUTER_FORCE_LOAD_BALANCING=0 +NUM_WORKERS=2 +PACKING_BUFFER_SIZE=4 +SHUFFLE_BUFFER_SIZE=100 +MAX_SAMPLES_PER_SEQUENCE=100 +CHECK_HEL_PATHS=1 + +WORLD_SIZE=$(( ENCODER_TP * ENCODER_CP * ENCODER_PP * ENCODER_DP \ + + LLM_TP * LLM_CP * LLM_PP * LLM_DP )) +[[ "${WORLD_SIZE}" -eq 72 ]] || { echo "ERROR: derived world_size=${WORLD_SIZE} (expected 72)" >&2; exit 1; } + +mkdir -p "${RUN_DIR}/logs/app" "${RUN_DIR}/logs/torchrun" "${RUN_DIR}/checkpoints" \ + "${RUN_DIR}/tensorboard" "${RUN_DIR}/data_cache" "${RUN_DIR}/tmp" + +export REPO_ROOT RUN_DIR SCRATCH_ROOT +export OUTPUT_PATH="${RUN_DIR}" LOG_DIR="${RUN_DIR}/logs/app" APP_LOG_DIR="${RUN_DIR}/logs/app" +export TORCHRUN_LOG_DIR="${RUN_DIR}/logs/torchrun" +export CHECKPOINT_SAVE_PATH="${RUN_DIR}/checkpoints" CHECKPOINT_LOAD_PATH="${NEMOTRON_CKPT}" +export CHECKPOINT_DIR="${RUN_DIR}/checkpoints" TENSORBOARD_PATH="${RUN_DIR}/tensorboard" TB_DIR="${RUN_DIR}/tensorboard" +export DATA_CACHE_DIR="${RUN_DIR}/data_cache" +export TMPDIR="/tmp" + +export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" +export XDG_CACHE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/cache" +export XDG_DATA_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/data" +export XDG_STATE_HOME="${SCRATCH_ROOT}/runtime/megatron_lm/xdg/state" +export TORCHINDUCTOR_CACHE_DIR="${SCRATCH_ROOT}/runtime/megatron_lm/torchinductor-cache" +export TRITON_CACHE_DIR_BASE="${RUN_DIR}/triton-cache" +export CUDA_CACHE_PATH="${SCRATCH_ROOT}/runtime/megatron_lm/cuda-cache" +export PYTHONPATH="${REPO_ROOT}" PYTHONNOUSERSITE=1 PIP_CONSTRAINT="" +export UV_CACHE_DIR="${SCRATCH_ROOT}/uv-cache/megatron_lm" UV_LINK_MODE=copy +export UV_PROJECT_ENVIRONMENT="${ENV_ROOT}/.venv" +export VIRTUAL_ENV="${UV_PROJECT_ENVIRONMENT}" +export PATH="${UV_PROJECT_ENVIRONMENT}/bin:${PATH}" + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NVTE_FWD_LAYERNORM_SM_MARGIN=16 NVTE_BWD_LAYERNORM_SM_MARGIN=16 +export NCCL_P2P_NET_CHUNKSIZE=2097152 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export NCCL_DEBUG=WARN NCCL_SHM_DISABLE=1 NCCL_PROTO=simple NCCL_NVLS_ENABLE=0 +export TORCH_NCCL_AVOID_RECORD_STREAMS=0 NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 + +export TRAINING_STAGE MODEL_PROVIDER ENABLE_EXPERIMENTAL MOE_ROUTER_FORCE_LOAD_BALANCING +export TRAIN_ITERS NUM_MICROBATCHES MICRO_BATCH_SIZE GLOBAL_BATCH_SIZE LOG_INTERVAL +export ENCODER_TP ENCODER_CP ENCODER_PP ENCODER_DP ENCODER_EP +export LLM_TP LLM_CP LLM_PP LLM_DP LLM_EP LLM_EXPT_TP LLM_ONLY +export LR MIN_LR WEIGHT_DECAY LR_DECAY_STYLE +export LR_WARMUP_SAMPLES LR_DECAY_SAMPLES LR_WSD_DECAY_SAMPLES LR_WSD_DECAY_STYLE TRAIN_SAMPLES +export NUM_WORKERS PACKING_BUFFER_SIZE SHUFFLE_BUFFER_SIZE MAX_SAMPLES_PER_SEQUENCE CHECK_HEL_PATHS +export TOKENIZER_MODEL VISION_CKPT + +TRAIN_LAUNCH_ARGS=( + --class-token-len 10 + --image-tag-type internvl + --max-num-tiles 1 + --overlap-grad-reduce --overlap-param-gather + --ddp-num-buckets 8 --ddp-pad-buckets-for-high-nccl-busbw + --correct-encoder-grad-for-partial-participation + --seed 1234 + --save "${CHECKPOINT_SAVE_PATH}" + --save-interval "${SAVE_INTERVAL}" + --no-load-optim --no-load-rng + --load-nemotron-checkpoint "${NEMOTRON_CKPT}" + --dynamic-resolution +) + +CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" +[[ "${REPO_ROOT}" == "${SCRATCH_ROOT}"/* ]] || CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" + +echo "=== hetero parity GBS=192 (${TRAIN_ITERS} iters, ~4h) ===" +echo "repo=${REPO_ROOT} run_dir=${RUN_DIR}" +echo "world_size=${WORLD_SIZE} gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES}" +echo "layout: encoder(dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP})" +echo "ckpt=${NEMOTRON_CKPT}" +echo "========================================================" + +srun --kill-on-bad-exit=1 \ + --ntasks="${WORLD_SIZE}" \ + --ntasks-per-node=8 \ + --container-image="${CONTAINER_IMAGE}" \ + --no-container-mount-home \ + --container-mounts="${CONTAINER_MOUNTS}" \ + --container-workdir="${REPO_ROOT}" \ + bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; + if [ -n "${HETERO_SKIP_UV:-}" ]; then export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"; exec bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; else exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; fi + ' \ + bash "${TRAIN_LAUNCH_ARGS[@]}" diff --git a/examples/mimo/scripts/sbatch_hetero_parity_100step.sh b/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh similarity index 70% rename from examples/mimo/scripts/sbatch_hetero_parity_100step.sh rename to examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh index 5cb74f7bcbf..c8e2d58f53c 100755 --- a/examples/mimo/scripts/sbatch_hetero_parity_100step.sh +++ b/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh @@ -1,19 +1,15 @@ #!/bin/bash -# 150-step train-loss parity run — hetero MIMO side. -# 3 nodes (1n encoder DP=8 + 2n LLM TP=2 DP=8 EP=16), GBS=8. -# Paired with sbatch_sanjeev_parity_100step.sh. -# -# Uses --load-nemotron-checkpoint to route ckpt-load through -# examples/mimo/utils/model_helpers.py:load_nemotron_vlm_ckpt_hetero and -# start training at iter 0 with fresh optimizer + RNG. +# Short hetero-MIMO parity run vs Sanjeev pre-vlm-05. +# 3 nodes (1n encoder DP=8 + 2n LLM TP=2 DP=8 EP=16), GBS=32, 20 iters. +# Paired with sbatch_sanjeev_parity_gbs32.sh. #SBATCH -A nemotron_n4_pre #SBATCH -p batch #SBATCH -N 3 #SBATCH --ntasks-per-node=8 #SBATCH --gres=gpu:8 -#SBATCH --time=02:00:00 -#SBATCH -J mimo-parity-100step +#SBATCH --time=00:30:00 +#SBATCH -J mimo-parity-gbs32 #SBATCH --exclusive #SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out #SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err @@ -28,9 +24,6 @@ else fi SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch -# Default to the energon-baked container so the energon swap below finds -# /usr/local/lib/python3.12/dist-packages/megatron. Skip uv for the same -# reason. Override via HETERO_CONTAINER_IMAGE / HETERO_SKIP_UV if needed. CONTAINER_IMAGE="${HETERO_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" export HETERO_SKIP_UV="${HETERO_SKIP_UV:-1}" ENV_ROOT="${SCRATCH_ROOT}/envs/megatron_lm/01f0da7539da4b39" @@ -39,7 +32,7 @@ VISION_CKPT="${SCRATCH_ROOT}/encoders/post-c-radio-omni" NEMOTRON_CKPT="${NEMOTRON_CKPT:-/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511/checkpoints/iter_0001000}" -RUN_NAME="mimo-parity-100step" +RUN_NAME="mimo-parity-gbs32" RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" # ---- topology: TP=2 EP=16 LLM + TP=1 DP=8 encoder lane ---------------------- @@ -47,16 +40,13 @@ ENCODER_TP=1; ENCODER_CP=1; ENCODER_PP=1; ENCODER_DP=8; ENCODER_EP=1 LLM_TP=2; LLM_CP=1; LLM_PP=1; LLM_DP=8; LLM_EP=16; LLM_EXPT_TP=1 LLM_ONLY=0 -# ---- batch / schedule pinned for the parity diff ---------------------------- MICRO_BATCH_SIZE=1 -GLOBAL_BATCH_SIZE=8 -NUM_MICROBATCHES=$(( GLOBAL_BATCH_SIZE / (MICRO_BATCH_SIZE * LLM_DP) )) # = 1 -TRAIN_ITERS=150 +GLOBAL_BATCH_SIZE=32 +NUM_MICROBATCHES=$(( GLOBAL_BATCH_SIZE / (MICRO_BATCH_SIZE * LLM_DP) )) # = 4 +TRAIN_ITERS=20 LOG_INTERVAL=1 -SAVE_INTERVAL=99999999 # don't save during the parity run +SAVE_INTERVAL=99999999 -# WSD schedule (matches the comparison recipe). LR_WARMUP_SAMPLES=0 so both -# sides hit full LR from iter 1 and there's no warmup-ramp difference. LR=1.2e-3 MIN_LR=1.2e-5 WEIGHT_DECAY=0.1 @@ -72,7 +62,7 @@ MODEL_PROVIDER=nemotron-moe-vlm-54l ENABLE_EXPERIMENTAL=1 MOE_ROUTER_FORCE_LOAD_BALANCING=0 NUM_WORKERS=2 -PACKING_BUFFER_SIZE=4 # smaller pool: less likelihood of multi-image packs exceeding seq_length after MIMO expansion (see energon_multimodal_provider.py:150 defensive check) +PACKING_BUFFER_SIZE=4 SHUFFLE_BUFFER_SIZE=100 MAX_SAMPLES_PER_SEQUENCE=100 CHECK_HEL_PATHS=1 @@ -90,12 +80,7 @@ export TORCHRUN_LOG_DIR="${RUN_DIR}/logs/torchrun" export CHECKPOINT_SAVE_PATH="${RUN_DIR}/checkpoints" CHECKPOINT_LOAD_PATH="${NEMOTRON_CKPT}" export CHECKPOINT_DIR="${RUN_DIR}/checkpoints" TENSORBOARD_PATH="${RUN_DIR}/tensorboard" TB_DIR="${RUN_DIR}/tensorboard" export DATA_CACHE_DIR="${RUN_DIR}/data_cache" -# DataLoader workers use TMPDIR for AF_UNIX IPC sockets; the path must be -# below the 108-char Unix-socket limit. RUN_DIR under our lustre scratch -# is already ~93 chars and python's multiprocessing appends more, blowing -# the limit. Use /tmp (node-local, short, exists by default on every -# node — /dev/shm would also work but only if pre-created on every -# node, which the sbatch head-node mkdir doesn't do). +# DataLoader worker AF_UNIX sockets must stay under 108 chars; RUN_DIR is too long. export TMPDIR="/tmp" export HOME="${SCRATCH_ROOT}/runtime/megatron_lm/home" @@ -126,10 +111,6 @@ export LR_WARMUP_SAMPLES LR_DECAY_SAMPLES LR_WSD_DECAY_SAMPLES LR_WSD_DECAY_STYL export NUM_WORKERS PACKING_BUFFER_SIZE SHUFFLE_BUFFER_SIZE MAX_SAMPLES_PER_SEQUENCE CHECK_HEL_PATHS export TOKENIZER_MODEL VISION_CKPT -export DUMP_DATA_ONLY="${DUMP_DATA_ONLY:-0}" -export DUMP_N_STEPS="${DUMP_N_STEPS:-5}" -export DUMP_OUTPUT_DIR="${DUMP_OUTPUT_DIR:-${RUN_DIR}/dumps}" - TRAIN_LAUNCH_ARGS=( --class-token-len 10 --image-tag-type internvl @@ -142,18 +123,18 @@ TRAIN_LAUNCH_ARGS=( --save-interval "${SAVE_INTERVAL}" --no-load-optim --no-load-rng --load-nemotron-checkpoint "${NEMOTRON_CKPT}" - --no-dynamic-resolution # parity step 1: fixed-res images on both sides + --dynamic-resolution ) CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" [[ "${REPO_ROOT}" == "${SCRATCH_ROOT}"/* ]] || CONTAINER_MOUNTS="${CONTAINER_MOUNTS},${REPO_ROOT}:${REPO_ROOT}" -echo "=== hetero parity 100-step ===" +echo "=== hetero parity GBS=32 (${TRAIN_ITERS} iters) ===" echo "repo=${REPO_ROOT} run_dir=${RUN_DIR}" -echo "world_size=${WORLD_SIZE} gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES} train_iters=${TRAIN_ITERS}" +echo "world_size=${WORLD_SIZE} gbs=${GLOBAL_BATCH_SIZE} microbatches=${NUM_MICROBATCHES}" echo "layout: encoder(dp=${ENCODER_DP}) llm(tp=${LLM_TP},dp=${LLM_DP},ep=${LLM_EP})" echo "ckpt=${NEMOTRON_CKPT}" -echo "=============================" +echo "==================================================" srun --kill-on-bad-exit=1 \ --ntasks="${WORLD_SIZE}" \ @@ -163,18 +144,6 @@ srun --kill-on-bad-exit=1 \ --container-mounts="${CONTAINER_MOUNTS}" \ --container-workdir="${REPO_ROOT}" \ bash -lc 'set -euo pipefail; cd "${REPO_ROOT}"; - # Optionally swap the container-baked energon for an editable clone. - if [ -n "${USE_HETERO_ENERGON_OVERRIDE:-}" ]; then - ESRC="${HETERO_ENERGON_SRC:-/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/megatron-energon-editable/src/megatron/energon}" - EDST="/usr/local/lib/python3.12/dist-packages/megatron/energon" - SENT="/tmp/.energon_installed_${SLURM_JOB_ID}" - if [ "${SLURM_LOCALID}" -eq 0 ]; then - echo "[hetero_sbatch] swapping energon: $ESRC -> $EDST" - rm -rf "$EDST"; cp -r "$ESRC" "$EDST"; touch "$SENT" - else - while [ ! -f "$SENT" ]; do sleep 1; done - fi - fi if [ -n "${HETERO_SKIP_UV:-}" ]; then export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}"; exec bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; else exec uv run --no-sync bash examples/mimo/scripts/run_hetero_nemotron_54l_hel_train.sh "$@"; fi ' \ bash "${TRAIN_LAUNCH_ARGS[@]}" diff --git a/examples/mimo/scripts/sbatch_sanjeev_parity_gbs192.sh b/examples/mimo/scripts/sbatch_sanjeev_parity_gbs192.sh new file mode 100755 index 00000000000..a73fa68ce78 --- /dev/null +++ b/examples/mimo/scripts/sbatch_sanjeev_parity_gbs192.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Long Sanjeev pre-vlm-05 parity run (reference side). +# 8 nodes, GBS=192, 5000 iters, 4h. Paired with sbatch_hetero_parity_gbs192.sh. + +#SBATCH -A nemotron_n4_pre +#SBATCH -p batch +#SBATCH -N 8 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --time=04:00:00 +#SBATCH -J sanj-parity-gbs192 +#SBATCH --exclusive +#SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out +#SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err + +set -euo pipefail + +SCRATCH_ROOT=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch +SANJEEV_REPO="${SANJEEV_REPO:-${SCRATCH_ROOT}/sanjeev-repos/megatron-lm-clean}" +CONTAINER_IMAGE="${SANJEEV_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" +TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" + +CKPT_RUN_ROOT="/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511" +CKPT_STEP="${CKPT_STEP:-1000}" + +RUN_NAME="sanj-parity-gbs192" +RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" +mkdir -p "${RUN_DIR}/logs" "${RUN_DIR}/save" "${RUN_DIR}/tb" + +export VISION_MODEL_TYPE=radio +export RADIO_ENCODER_DIR=post-c-radio-omni +export TP=2 +export EP=16 +export NUM_EXPERTS=128 +export MOE_ROUTER_TOPK=6 +export MBS=1 +export GBS=192 + +export NUM_WORKERS=2 +export PACKING_BUFFER_SIZE=4 +export SEQ_LEN=8192 +export DECODER_SEQ_LEN=8192 +# 5000 optimizer steps after the resume + the ckpt's iter_1000 offset. +export TRAIN_SAMPLES=$(( 5000 * GBS + CKPT_STEP * GBS )) +export LR_WARMUP_SAMPLES=0 +export LR_WSD_DECAY_SAMPLES=1 +export EXIT_MIN=240 +export LOG_INTERVAL=1 +export EVAL_INTERVAL=99999999999 +export EVAL_ITERS=0 +export SAVE_INTERVAL=99999999999 +export USE_DYNAMIC_RES=1 +export SEQUENCE_PARALLEL=1 + +export LOAD_CHECKPOINT_DIR="${CKPT_RUN_ROOT}/checkpoints" +export SAVE_CHECKPOINT_DIR="${RUN_DIR}/save" +export OUTPUT="${RUN_DIR}" +export LOGS_DIR="${RUN_DIR}/logs" +export TENSORBOARD_DIR="${RUN_DIR}/tb" +export WANDB_DIR="${RUN_DIR}/wandb" +export RUN_NAME + +export HYBRID_LAYER_PATTERN="MEMEM*EMEM*EMEM*EMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEME" +export DISABLE_RECOMPUTE=1 +export EXTRA_MEGATRON_ARGS="--ckpt-step ${CKPT_STEP} --calculate-per-token-loss --no-load-rng --no-load-optim --mtp-num-layers 0" + +export CONTAINER_IMAGE_OVERRIDE="${CONTAINER_IMAGE}" +export TOKENIZER_MODEL="${TOKENIZER_MODEL}" +export MEGATRON_ROOT="${SANJEEV_REPO}" +export SBATCH_NODES=8 + +export MULTIMODAL_DATA_ROOT=/home/sasatheesh/data/multimodal_data + +cd "${SANJEEV_REPO}" +exec bash "${SANJEEV_REPO}/examples/multimodal/v3/pretrain_3b_nano_vlm_sota_90t_10v.sh" diff --git a/examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh b/examples/mimo/scripts/sbatch_sanjeev_parity_gbs32.sh similarity index 53% rename from examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh rename to examples/mimo/scripts/sbatch_sanjeev_parity_gbs32.sh index 6009c579bb6..e45360a0921 100755 --- a/examples/mimo/scripts/sbatch_sanjeev_parity_100step.sh +++ b/examples/mimo/scripts/sbatch_sanjeev_parity_gbs32.sh @@ -1,18 +1,14 @@ #!/bin/bash -# 150-step train-loss parity run — Sanjeev-side recipe. -# Thin wrapper around the pretrain script with env overrides for parity: -# 2 nodes, GBS=8, --calculate-per-token-loss enabled. -# -# Submit from anywhere; this script cds into the reference repo on the -# cluster and shells out to its pretrain entry point. +# Short Sanjeev pre-vlm-05 parity run (reference side). +# 2 nodes, GBS=32, 20 iters. Paired with sbatch_hetero_parity_gbs32.sh. #SBATCH -A nemotron_n4_pre #SBATCH -p batch #SBATCH -N 2 #SBATCH --ntasks-per-node=8 #SBATCH --gres=gpu:8 -#SBATCH --time=02:00:00 -#SBATCH -J sanj-parity-100step +#SBATCH --time=00:30:00 +#SBATCH -J sanj-parity-gbs32 #SBATCH --exclusive #SBATCH --output=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.out #SBATCH --error=/lustre/fsw/portfolios/nemotron/users/ykarnati/agents-scratch/runs/%x-%j.err @@ -24,16 +20,13 @@ SANJEEV_REPO="${SANJEEV_REPO:-${SCRATCH_ROOT}/sanjeev-repos/megatron-lm-clean}" CONTAINER_IMAGE="${SANJEEV_CONTAINER_IMAGE:-${SCRATCH_ROOT}/images/m_lm_energon_0506.sqsh}" TOKENIZER_MODEL="${SCRATCH_ROOT}/tokenizers/sanjeevnv-multimodal-pretraining-26f81d5db838eb6dee2ff8692db83a2fbc76f3ff" -# Ckpt to resume from. Flip CKPT_RUN_ROOT / CKPT_STEP for a different ckpt. CKPT_RUN_ROOT="/scratch/fsw/portfolios/llmservice/projects/llmservice_fm_text/users/sasatheesh/workspace/output/3b_nano_vlm_sota_mtp2_90t10v_post_c_radio_omni_96n_tp2_ep16_selective_300b_20260511" CKPT_STEP="${CKPT_STEP:-1000}" -RUN_NAME="sanj-parity-100step" +RUN_NAME="sanj-parity-gbs32" RUN_DIR="${SCRATCH_ROOT}/runs/${RUN_NAME}/${SLURM_JOB_ID:-local}" mkdir -p "${RUN_DIR}/logs" "${RUN_DIR}/save" "${RUN_DIR}/tb" -# ---- Overrides passed to the pretrain script -------------------------------- -# Topology: TP=2 EP=16. EDP/DP fall out from world_size; we constrain TP/EP/MBS/GBS. export VISION_MODEL_TYPE=radio export RADIO_ENCODER_DIR=post-c-radio-omni export TP=2 @@ -41,15 +34,13 @@ export EP=16 export NUM_EXPERTS=128 export MOE_ROUTER_TOPK=6 export MBS=1 -export GBS=8 +export GBS=32 + export NUM_WORKERS=2 -export PACKING_BUFFER_SIZE=4 # Match hetero side. Larger buffers reorder samples differently → per-rank batches diverge. +export PACKING_BUFFER_SIZE=4 export SEQ_LEN=8192 export DECODER_SEQ_LEN=8192 -# 150 optimizer steps after the resume + the ckpt's iter_1000 offset. -export TRAIN_SAMPLES=$(( 150 * GBS + 1000 * GBS )) -# Override LR_WARMUP_SAMPLES so LR_DECAY_SAMPLES stays positive. With -# --no-load-rng / --no-load-optim we restart the scheduler at iter 0 anyway. +export TRAIN_SAMPLES=$(( 20 * GBS + CKPT_STEP * GBS )) export LR_WARMUP_SAMPLES=0 export LR_WSD_DECAY_SAMPLES=1 export EXIT_MIN=240 @@ -57,10 +48,9 @@ export LOG_INTERVAL=1 export EVAL_INTERVAL=99999999999 export EVAL_ITERS=0 export SAVE_INTERVAL=99999999999 -export USE_DYNAMIC_RES=0 # parity step 1: fixed-res images on both sides +export USE_DYNAMIC_RES=1 export SEQUENCE_PARALLEL=1 -# Ckpt paths (the pretrain script reads these and routes via --load / --save). export LOAD_CHECKPOINT_DIR="${CKPT_RUN_ROOT}/checkpoints" export SAVE_CHECKPOINT_DIR="${RUN_DIR}/save" export OUTPUT="${RUN_DIR}" @@ -69,32 +59,22 @@ export TENSORBOARD_DIR="${RUN_DIR}/tb" export WANDB_DIR="${RUN_DIR}/wandb" export RUN_NAME -# --ckpt-step pins the resume iter. --calculate-per-token-loss aligns gradient -# formula with hetero. --no-load-rng restarts the scheduler from seed=1234. -# Disable MTP and selective recompute for parity (hetero side runs without -# MTP, and --mtp-num-layers 0 also sidesteps an MTP-block autograd bug under -# this resume config). +# --ckpt-step pins the resume iter. --calculate-per-token-loss aligns the +# gradient formula with hetero. --no-load-rng / --no-load-optim restart the +# scheduler at iter 0. --mtp-num-layers 0 disables MTP (hetero side runs +# without MTP too). export HYBRID_LAYER_PATTERN="MEMEM*EMEM*EMEM*EMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEME" export DISABLE_RECOMPUTE=1 -# --pixel-shuffle: gated by USE_DYNAMIC_RES=1 in the recipe but the ckpt was -# trained with pixel-shuffle on (vision_projection input dim 5120 = 4*1280), -# so re-add it explicitly when USE_DYNAMIC_RES=0. -export EXTRA_MEGATRON_ARGS="--ckpt-step ${CKPT_STEP} --calculate-per-token-loss --no-load-rng --no-load-optim --mtp-num-layers 0 --pixel-shuffle" +export EXTRA_MEGATRON_ARGS="--ckpt-step ${CKPT_STEP} --calculate-per-token-loss --no-load-rng --no-load-optim --mtp-num-layers 0" -# Container + repo export CONTAINER_IMAGE_OVERRIDE="${CONTAINER_IMAGE}" export TOKENIZER_MODEL="${TOKENIZER_MODEL}" export MEGATRON_ROOT="${SANJEEV_REPO}" export SBATCH_NODES=2 -# The blend yaml's val: section eagerly post_initializes both splits even with -# --eval-iters 0. /home is NFS-mounted so reading from the cluster-shared home -# makes the val split resolvable cheaply; it's never actually iterated. +# /home is NFS-mounted; the blend yaml's val: section eagerly post_initializes +# both splits even with --eval-iters 0. export MULTIMODAL_DATA_ROOT=/home/sasatheesh/data/multimodal_data -export DUMP_DATA_ONLY="${DUMP_DATA_ONLY:-0}" -export DUMP_N_STEPS="${DUMP_N_STEPS:-5}" -export DUMP_OUTPUT_DIR="${DUMP_OUTPUT_DIR:-${RUN_DIR}/dumps}" - cd "${SANJEEV_REPO}" exec bash "${SANJEEV_REPO}/examples/multimodal/v3/pretrain_3b_nano_vlm_sota_90t_10v.sh" diff --git a/examples/mimo/training/hetero/step.py b/examples/mimo/training/hetero/step.py index 494ecd9c730..c383fc9b0b8 100644 --- a/examples/mimo/training/hetero/step.py +++ b/examples/mimo/training/hetero/step.py @@ -90,6 +90,24 @@ def move_batch_to_cuda(value): return [move_batch_to_cuda(item) for item in value] if isinstance(value, tuple): return tuple(move_batch_to_cuda(item) for item in value) + # PackedSeqParams is a dataclass carrying tensors that TE attention needs + # on the GPU. Recurse through its tensor-valued fields so cu_seqlens_q/kv + # and max_seqlen_q/kv land on cuda alongside the rest of the batch. + from megatron.core.packed_seq_params import PackedSeqParams + + if isinstance(value, PackedSeqParams): + for attr in ( + "cu_seqlens_q", + "cu_seqlens_kv", + "cu_seqlens_q_padded", + "cu_seqlens_kv_padded", + "max_seqlen_q", + "max_seqlen_kv", + ): + sub = getattr(value, attr, None) + if isinstance(sub, torch.Tensor) and not sub.is_cuda: + setattr(value, attr, sub.cuda(non_blocking=True)) + return value return value diff --git a/examples/mimo/utils/model_helpers.py b/examples/mimo/utils/model_helpers.py index 744963054cc..48e61ac92eb 100644 --- a/examples/mimo/utils/model_helpers.py +++ b/examples/mimo/utils/model_helpers.py @@ -245,11 +245,25 @@ def _drill_through_ddp(mod): flush=True, ) - loaded = dist_checkpointing.load( + # RETURN_ALL semantics (megatron/core/dist_checkpointing/validation.py:267-274): + # third return = keys we requested but ckpt does NOT have + # (DANGEROUS — those tensors would silently keep their + # random-init values; PyTorch load_state_dict calls this + # set "missing" but mcore returns it as "unexpected"). + # Weaker modes (LOG_UNEXPECTED, ASSUME_OK_UNEXPECTED) skip this check + # entirely. Raise loudly on any non-extra_state mismatch. + loaded, _ckpt_only_keys, request_only_keys = dist_checkpointing.load( sharded_state_dict=wrapper_sd, checkpoint_dir=ckpt_dir, - strict=StrictHandling.LOG_UNEXPECTED, + strict=StrictHandling.RETURN_ALL, ) + missing_in_ckpt = sorted(k for k in request_only_keys if "extra_state" not in k) + if missing_in_ckpt: + raise RuntimeError( + f"checkpoint is missing {len(missing_in_ckpt)} keys the model requested; " + f"these would silently keep random-init values. " + f"First 30: {missing_in_ckpt[:30]}" + ) loaded_sd = loaded.get("state_dict", {}) # Apply loaded tensors back to each submodule by stripping its checkpoint prefix. diff --git a/megatron/core/models/vision/radio.py b/megatron/core/models/vision/radio.py index 62ed3b6c59d..1438418b552 100644 --- a/megatron/core/models/vision/radio.py +++ b/megatron/core/models/vision/radio.py @@ -1,7 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import math -from typing import Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F @@ -9,6 +9,7 @@ from megatron.core.config_logger import has_config_logger_enabled, log_config_to_disk from megatron.core.models.common.vision_module.vision_module import VisionModule +from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear from megatron.core.transformer.enums import ModelType @@ -64,6 +65,7 @@ def __init__( pos_dropout: int = 0, has_cpe: bool = True, embedder_bias: bool = False, + dynamic_resolution: bool = False, pg_collection: Optional[ProcessGroupCollection] = None, vp_stage: Optional[int] = None, ) -> None: @@ -124,6 +126,7 @@ def __init__( ) self.pos_dropout = pos_dropout self.has_cpe = has_cpe + self.dynamic_resolution = dynamic_resolution # Using non-TE version so we can force gather_output tp_group = getattr(pg_collection, "tp", None) if pg_collection is not None else None @@ -181,49 +184,108 @@ def set_input_tensor(self, input_tensor: torch.Tensor) -> None: self.decoder.set_input_tensor(input_tensor) def forward( - self, x: torch.Tensor, attention_mask: Optional[torch.Tensor] = None + self, + x: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + imgs_sizes: Optional[Union[List[Tuple[int, int]], torch.Tensor]] = None, + packed_seq_params: Optional[PackedSeqParams] = None, ) -> torch.Tensor: """Forward function of the RADIO ViT Model. This function passes the input tensors through the embedding layer and then the transformer. Args: - x (torch.Tensor): input data of shape [batch, img_h, img_w] + x (torch.Tensor): input data of shape [batch, img_h, img_w] or + [batch, total_patches, patch_features] when dynamic_resolution=True. attention_mask (torch.Tensor with dtype=bool): Attention mask to use. + imgs_sizes: Per-tile (H, W) pixel sizes for dynamic resolution. + packed_seq_params: Packed sequence params for THD attention. Returns: x (torch.Tensor): output after final transformer block of shape [b, s, h]. """ + if not self.dynamic_resolution: + if not HAVE_EINOPS: + raise ImportError( + "einops is required for RADIOViTModel, please install it with " + "`pip install einops`" + ) - if not HAVE_EINOPS: - raise ImportError( - "einops is required for RADIOViTModel, please install it with `pip install einops`" + input_size = x.shape[2:] + py = x.shape[-2] // self.patch_dim + px = x.shape[-1] // self.patch_dim + x = rearrange( + x, + "b c (py yy) (px xx) -> b (py px) (c yy xx)", + py=py, + yy=self.patch_dim, + px=px, + xx=self.patch_dim, ) - - input_size = x.shape[2:] - py = x.shape[-2] // self.patch_dim - px = x.shape[-1] // self.patch_dim - x = rearrange( - x, - "b c (py yy) (px xx) -> b (py px) (c yy xx)", - py=py, - yy=self.patch_dim, - px=px, - xx=self.patch_dim, - ) x, _ = self.embedder(x) # [batch, seq_length, hidden_size] - x, _ = self.apply_pos_enc(x, input_size=input_size) + # Apply position encoding -- per-tile for dynamic resolution, global otherwise. + if self.dynamic_resolution: + if torch.is_tensor(imgs_sizes): + seq_lens = torch.prod(imgs_sizes // self.patch_dim, dim=-1).tolist() + sizes_iter = [tuple(sz.tolist()) for sz in imgs_sizes] + else: + seq_lens = [(h // self.patch_dim) * (w // self.patch_dim) for h, w in imgs_sizes] + sizes_iter = imgs_sizes + + assert sum(seq_lens) == x.shape[1], f"{sum(seq_lens)} != {x.shape[1]}" + + chunks = torch.split(x, seq_lens, dim=1) + chunks = [ + self.apply_pos_enc(chunk, input_size=size)[0] + for chunk, size in zip(chunks, sizes_iter) + ] + x = torch.cat(chunks, dim=1) + else: + x, _ = self.apply_pos_enc(x, input_size=input_size) if self.add_class_token: class_token = self.class_token.expand( x.shape[0], -1, -1 ) # [batch, class_token_len, hidden_size] + if self.dynamic_resolution: + # Interleave class tokens between tiles for dynamic resolution. + out = [] + current_length = 0 + for input_size in imgs_sizes: + if torch.is_tensor(input_size): + seq_length = ( + input_size[0] // self.patch_dim * input_size[1] // self.patch_dim + ) + else: + seq_length = ( + input_size[0] // self.patch_dim * input_size[1] // self.patch_dim + ) + out.append(class_token) + out.append(x[:, current_length : current_length + seq_length, :]) + current_length += int(seq_length) + x = torch.cat(out, dim=1) + if packed_seq_params is not None: + # Update packed_seq_params to account for added class tokens. + add_cu = torch.full_like( + packed_seq_params.cu_seqlens_q, self.class_token_len, dtype=torch.int32 + ) + add_cu[0] = 0 + add_cu = torch.cumsum(add_cu, dim=-1, dtype=torch.int32) + packed_seq_params.cu_seqlens_q = packed_seq_params.cu_seqlens_q + add_cu + packed_seq_params.cu_seqlens_kv = packed_seq_params.cu_seqlens_kv + add_cu + packed_seq_params.max_seqlen_q = ( + packed_seq_params.max_seqlen_q + self.class_token_len + ) + packed_seq_params.max_seqlen_kv = ( + packed_seq_params.max_seqlen_kv + self.class_token_len + ) + else: + x = torch.cat( + [class_token, x], dim=1 + ) # [batch, seq_length + class_token_len, hidden_size] - x = torch.cat( - [class_token, x], dim=1 - ) # [batch, seq_length + class_token_len, hidden_size] - - assert x.shape[1] == self.seq_length, f"{x.shape[1]} != {self.seq_length}" + if not self.dynamic_resolution: + assert x.shape[1] == self.seq_length, f"{x.shape[1]} != {self.seq_length}" if self.ln_pre: x = self.ln_pre(x) @@ -231,7 +293,7 @@ def forward( x = x.permute(1, 0, 2) # [b, s, h] -> [s, b, h] x = x.contiguous() - x = self.decoder(x, attention_mask=attention_mask) + x = self.decoder(x, attention_mask=attention_mask, packed_seq_params=packed_seq_params) x = x.permute(1, 0, 2) # [s, b, h] -> [b, s, h] x = x.contiguous() From 7e0834d49abfcc75bb00854981cf27a409ddec23 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Tue, 19 May 2026 16:27:53 -0700 Subject: [PATCH 40/44] Add tensorboard logging to hetero train loop (#33) - args.py: new --tensorboard-dir flag. - logging.py: HeteroTrainingLogger creates a torch.utils.tensorboard SummaryWriter on the language logging rank when --tensorboard-dir is set. Emits the same scalar keys Megatron's standard training_log uses (lm loss, learning-rate, grad-norm, batch-size, loss-scale, num-zeros) plus iteration-time-ms, both per-iter and "vs samples", so TB plots overlay cleanly against the reference run's logs. - sbatch_hetero_parity_100step.sh: pass --tensorboard-dir "${RUN_DIR}/tensorboard" so parity runs auto-log to a per-run TB dir. Co-authored-by: Claude Opus 4.7 (1M context) --- .../scripts/sbatch_hetero_parity_gbs192.sh | 1 + .../scripts/sbatch_hetero_parity_gbs32.sh | 1 + examples/mimo/training/hetero/args.py | 9 +++ examples/mimo/training/hetero/logging.py | 71 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh b/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh index 845dad69cd8..260b4ab1173 100755 --- a/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh +++ b/examples/mimo/scripts/sbatch_hetero_parity_gbs192.sh @@ -123,6 +123,7 @@ TRAIN_LAUNCH_ARGS=( --no-load-optim --no-load-rng --load-nemotron-checkpoint "${NEMOTRON_CKPT}" --dynamic-resolution + --tensorboard-dir "${RUN_DIR}/tensorboard" ) CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" diff --git a/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh b/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh index c8e2d58f53c..39c73aedfa4 100755 --- a/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh +++ b/examples/mimo/scripts/sbatch_hetero_parity_gbs32.sh @@ -124,6 +124,7 @@ TRAIN_LAUNCH_ARGS=( --no-load-optim --no-load-rng --load-nemotron-checkpoint "${NEMOTRON_CKPT}" --dynamic-resolution + --tensorboard-dir "${RUN_DIR}/tensorboard" ) CONTAINER_MOUNTS="${SCRATCH_ROOT}:${SCRATCH_ROOT},/lustre/fsw/portfolios/llmservice:/lustre/fsw/portfolios/llmservice,/scratch/fsw/portfolios/llmservice:/scratch/fsw/portfolios/llmservice" diff --git a/examples/mimo/training/hetero/args.py b/examples/mimo/training/hetero/args.py index bdbdd07aea0..e7f7c3b07cb 100644 --- a/examples/mimo/training/hetero/args.py +++ b/examples/mimo/training/hetero/args.py @@ -223,6 +223,15 @@ def parse_args() -> argparse.Namespace: ) train.add_argument("--seed", type=int, default=12345) train.add_argument("--log-interval", type=int, default=1) + train.add_argument( + "--tensorboard-dir", + type=str, + default=None, + help="Directory for tensorboard scalar logs. When set, the language " + "logging rank writes lm_loss/grad-norm/learning-rate/etc. each log " + "interval, matching the scalar keys used by Megatron's standard " + "training_log so hetero and reference runs can be diffed in TB.", + ) ckpt = parser.add_argument_group("checkpointing") ckpt.add_argument( diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py index 3bc9552f648..31ce5842a62 100644 --- a/examples/mimo/training/hetero/logging.py +++ b/examples/mimo/training/hetero/logging.py @@ -21,6 +21,8 @@ from examples.mimo.utils.hetero import is_process_group_member from megatron.core.optimizer_param_scheduler import get_canonical_lr_for_logging from megatron.core.pipeline_parallel.utils import is_pp_last_stage +from megatron.core.transformer.moe.moe_utils import track_moe_metrics +from megatron.core.num_microbatches_calculator import get_num_microbatches @dataclass @@ -36,6 +38,16 @@ class HeteroTrainingLogger: loss_total: float = 0.0 loss_count: int = 0 interval_start: float = field(default_factory=time.time) + _tb_writer: Optional[object] = field(default=None, init=False, repr=False) + _moe_total_loss_dict: dict = field(default_factory=dict, init=False, repr=False) + + def __post_init__(self) -> None: + # Only the language logging rank owns the writer; other ranks no-op. + tb_dir = getattr(self.args, "tensorboard_dir", None) + if tb_dir and is_language_log_rank(self.topology): + from torch.utils.tensorboard import SummaryWriter + + self._tb_writer = SummaryWriter(log_dir=tb_dir) def record_step(self, result: TrainStepResult) -> Optional[float]: """Update interval state from one train step and return this iteration's loss.""" @@ -88,6 +100,65 @@ def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: log_string += " number of nan iterations: {:3d} |".format(self.nan_iterations) sys.stdout.write(f"{log_string}\n") sys.stdout.flush() + # MoE aux-loss tracking (seq_load_balancing_loss etc.) — mirrors + # megatron.training.training.training_log:2196-2237. Reduces the + # per-microbatch tracker across the LLM lane and emits per-iter scalars. + # Runs on every LLM-lane rank because track_moe_metrics issues + # internal collectives across pg_collection — but only the language + # log rank's writer actually persists scalars. + if getattr(self.args, "num_experts", None) and is_process_group_member( + getattr(self.topology.language_pg, "dp_cp", None) + ): + track_names = [] + lb_type = getattr(self.args, "moe_router_load_balancing_type", "") + if "aux_loss" in lb_type: + track_names.append("load_balancing_loss") + if "seq_aux_loss" in lb_type: + track_names.append("seq_load_balancing_loss") + if "global_aux_loss" in lb_type: + track_names.append("global_load_balancing_loss") + track_moe_metrics( + loss_scale=1.0 / max(1, get_num_microbatches()), + iteration=iteration, + writer=self._tb_writer, + wandb_writer=None, + total_loss_dict=self._moe_total_loss_dict, + per_layer_logging=False, + force_initialize=True, + track_names=track_names, + num_layers=getattr(self.args, "num_layers", 0), + moe_layer_freq=getattr(self.args, "moe_layer_freq", None), + mtp_num_layers=getattr(self.args, "mtp_num_layers", None), + pg_collection=self.topology.language_pg, + ) + + if self._tb_writer is not None: + batch_size = get_global_batch_size(self.args) + samples = self.consumed_train_samples + if loss_value is not None: + self._tb_writer.add_scalar("lm loss", loss_value, iteration) + self._tb_writer.add_scalar("lm loss vs samples", loss_value, samples) + if learning_rate is not None: + self._tb_writer.add_scalar("learning-rate", learning_rate, iteration) + self._tb_writer.add_scalar( + "learning-rate vs samples", learning_rate, samples + ) + self._tb_writer.add_scalar("batch-size", batch_size, iteration) + self._tb_writer.add_scalar("batch-size vs samples", batch_size, samples) + self._tb_writer.add_scalar("loss-scale", loss_scale, iteration) + if result.grad_norm is not None: + self._tb_writer.add_scalar("grad-norm", result.grad_norm, iteration) + self._tb_writer.add_scalar( + "grad-norm vs samples", result.grad_norm, samples + ) + if result.num_zeros_in_grad is not None: + self._tb_writer.add_scalar( + "num-zeros", result.num_zeros_in_grad, iteration + ) + self._tb_writer.add_scalar( + "iteration-time-ms", elapsed_ms, iteration + ) + self._tb_writer.flush() self.reset_interval() def reset_interval(self) -> None: From 1e24f4d0177dbd6fde95bfcdcb7520081f62dd54 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Tue, 19 May 2026 17:24:51 -0700 Subject: [PATCH 41/44] NMFW-464: emit seq_load_balancing_loss to TB on the hetero side (#34) PR #33's MoE-tracker block gated on args.num_experts + args.moe_router_load_balancing_type, but hetero stores those as args.num_moe_experts and on the TransformerConfig (not args). The gate never fired, so seq_load_balancing_loss never reached TB. Fix: gate on args.num_moe_experts; hardcode track_names to ["seq_load_balancing_loss"] (the only LB type Nemotron6-MoE uses); compute num_moe_layers from args.hybrid_layer_pattern's E-count so the per-iter average over MoE layers matches sanj's training_log output (54L pattern has 27 'E' layers, not 54). Co-authored-by: Claude Opus 4.7 (1M context) --- examples/mimo/training/hetero/logging.py | 31 +++++++++--------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/examples/mimo/training/hetero/logging.py b/examples/mimo/training/hetero/logging.py index 31ce5842a62..a4f04766d8c 100644 --- a/examples/mimo/training/hetero/logging.py +++ b/examples/mimo/training/hetero/logging.py @@ -22,7 +22,6 @@ from megatron.core.optimizer_param_scheduler import get_canonical_lr_for_logging from megatron.core.pipeline_parallel.utils import is_pp_last_stage from megatron.core.transformer.moe.moe_utils import track_moe_metrics -from megatron.core.num_microbatches_calculator import get_num_microbatches @dataclass @@ -100,34 +99,26 @@ def maybe_log(self, iteration: int, optimizer, result: TrainStepResult) -> None: log_string += " number of nan iterations: {:3d} |".format(self.nan_iterations) sys.stdout.write(f"{log_string}\n") sys.stdout.flush() - # MoE aux-loss tracking (seq_load_balancing_loss etc.) — mirrors - # megatron.training.training.training_log:2196-2237. Reduces the - # per-microbatch tracker across the LLM lane and emits per-iter scalars. - # Runs on every LLM-lane rank because track_moe_metrics issues - # internal collectives across pg_collection — but only the language - # log rank's writer actually persists scalars. - if getattr(self.args, "num_experts", None) and is_process_group_member( + num_moe_experts = getattr(self.args, "num_moe_experts", None) + if num_moe_experts and is_process_group_member( getattr(self.topology.language_pg, "dp_cp", None) ): - track_names = [] - lb_type = getattr(self.args, "moe_router_load_balancing_type", "") - if "aux_loss" in lb_type: - track_names.append("load_balancing_loss") - if "seq_aux_loss" in lb_type: - track_names.append("seq_load_balancing_loss") - if "global_aux_loss" in lb_type: - track_names.append("global_load_balancing_loss") + hybrid_pat = getattr(self.args, "hybrid_layer_pattern", None) + if hybrid_pat: + num_moe_layers = hybrid_pat.count("E") + else: + num_moe_layers = getattr(self.args, "num_layers", 0) track_moe_metrics( - loss_scale=1.0 / max(1, get_num_microbatches()), + loss_scale=1.0 / max(1, getattr(self.args, "num_microbatches", 1)), iteration=iteration, writer=self._tb_writer, wandb_writer=None, total_loss_dict=self._moe_total_loss_dict, per_layer_logging=False, force_initialize=True, - track_names=track_names, - num_layers=getattr(self.args, "num_layers", 0), - moe_layer_freq=getattr(self.args, "moe_layer_freq", None), + track_names=["seq_load_balancing_loss"], + num_layers=num_moe_layers, + moe_layer_freq=None, mtp_num_layers=getattr(self.args, "mtp_num_layers", None), pg_collection=self.topology.language_pg, ) From 01e5491f9aac125854f81c279751c35999d3e2a6 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Tue, 19 May 2026 23:22:30 -0700 Subject: [PATCH 42/44] NMFW-464: fix routed encoder iterator merge for dynres PackedSeqParams (#35) When encoder_dp < llm_dp, the routed encoder iterator merges N per-lane batches into one encoder forward via _combine_encoder_batches -> _concat_nested_tensors. After PR #32 (dynres) added a PackedSeqParams dataclass into modality_inputs.images..packed_seq_params, two bugs in this merge path surface together on multi-lane configurations (e.g. 9n GBS=192: ENCODER_DP=8, LLM_DP=32, lanes_per_encoder=4): 1) TypeError: cannot concatenate encoder batch value of type PackedSeqParams. PR #31 only handled torch.Tensor and dict; the dataclass fell through to the final raise. 2) RuntimeError: Sizes of tensors must match except in dimension 0. Per-lane packed image buffers have shape (1, T_lane, C) -- dim 0 is the constant lane batch and T_lane varies. PR #31's blanket torch.cat(present, dim=0) requires every other dim to match and fails on the T axis. The combination of (1) + (2) produced the 0-15-silent / 16-71-NCCL- timeout cascade observed on the 9n parity sbatch: encoder rank 0's loader worker died in torch.cat, lanes 0-3 stalled at the bridge, the remaining LLM ranks reached the DDP allgather and timed out after 10 minutes. Fix: - _concat_packed_seq_params merges N per-lane PackedSeqParams into one set covering the merged flat buffer. cu_seqlens_{q,kv}[_padded] concatenate with running offset = sum of prior lanes' cu_seqlens_q[-1] (the same offset-shift rule used in megatron.energon.task_encoder.multimodal.encoder.py); max_seqlen takes element-wise max; total_tokens sums. qkv_format, local_cp_size, cp_group are asserted equal across lanes. seq_idx is left to PackedSeqParams.__post_init__. - _concat_first_varying_dim concatenates plain tensors along the first dimension whose size differs across lanes, defaulting to dim 0 when all shapes agree. This handles (1, T_lane, C) packed image buffers (dim 1) and (N_images_lane, 2) imgs_sizes (dim 0) uniformly without sibling-key context, and preserves the prior behavior on non-dynres batches. Verified by running the existing 9n GBS=192 parity sbatch end-to-end: 30-iter smoke reaches steady-state iter time around 4.5-6s with a clean lm-loss trajectory; previously the same sbatch crashed inside _combine_encoder_batches before iteration 1. --- examples/mimo/data/hetero_energon.py | 90 +++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 5b481a078da..7af9b9da8d9 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -14,6 +14,7 @@ from examples.mimo.training.hetero.topology import get_grid_coordinate, is_rank_in_grid from examples.mimo.utils.hetero import debug_rank, is_process_group_member +from megatron.core.packed_seq_params import PackedSeqParams def build_energon_iterator(args, topology): @@ -389,13 +390,21 @@ def _combine_encoder_batches(batches: list[dict]) -> dict: def _concat_nested_tensors(values): - """Concatenate a list of matching nested tensor structures along leading dim.""" + """Concatenate a list of matching nested tensor structures along the + first dimension that varies across lanes. + + For most per-lane tensors the varying dim is 0 (batch). For dynres-packed + image buffers the per-lane shape is ``(1, T_lane, C)`` — dim 0 is the + fixed lane batch size and dim 1 is the variable token axis, so the right + concatenation is along dim 1. Selecting the first-varying dim handles + both uniformly without any sibling-key context. + """ present = [value for value in values if value is not None] if not present: return None first = present[0] if isinstance(first, torch.Tensor): - return torch.cat(present, dim=0) + return _concat_first_varying_dim(present) if isinstance(first, dict): keys = set().union(*(value.keys() for value in present if isinstance(value, dict))) merged = {} @@ -404,9 +413,86 @@ def _concat_nested_tensors(values): if value is not None: merged[key] = value return merged + if isinstance(first, PackedSeqParams): + return _concat_packed_seq_params(present) raise TypeError(f"cannot concatenate encoder batch value of type {type(first).__name__}") +def _concat_first_varying_dim(tensors): + """Concatenate ``tensors`` along the first dimension whose size differs + across the list; default to dim 0 if all shapes match.""" + if len(tensors) == 1: + return tensors[0] + first = tensors[0] + for d in range(first.dim()): + if any(t.shape[d] != first.shape[d] for t in tensors[1:]): + return torch.cat(tensors, dim=d) + return torch.cat(tensors, dim=0) + + +def _concat_packed_seq_params(values: list) -> PackedSeqParams: + """Merge per-lane PackedSeqParams into one set covering the merged flat buffer. + + The dim-0 image buffers from each lane are concatenated by the surrounding + tensor merge; here we re-number cu_seqlens so they index into that merged + buffer. Mirrors the offset-shift rule in + ``megatron.energon.task_encoder.multimodal.encoder``. + """ + first = values[0] + for v in values[1:]: + if v.qkv_format != first.qkv_format: + raise ValueError( + f"qkv_format mismatch across encoder lanes: " + f"{first.qkv_format!r} vs {v.qkv_format!r}" + ) + if v.local_cp_size != first.local_cp_size or v.cp_group is not first.cp_group: + raise ValueError("CP fields mismatch across encoder lanes; refusing to merge") + + def _concat_cu(attr: str): + per_lane = [getattr(v, attr) for v in values] + if per_lane[0] is None: + if not all(x is None for x in per_lane): + raise ValueError(f"{attr} present on some lanes but not others") + return None + merged = [per_lane[0]] + offset = int(per_lane[0][-1].item()) + for cu in per_lane[1:]: + merged.append(cu[1:] + offset) + offset += int(cu[-1].item()) + return torch.cat(merged) + + def _max_scalar(attr: str): + per_lane = [getattr(v, attr) for v in values] + if per_lane[0] is None: + if not all(x is None for x in per_lane): + raise ValueError(f"{attr} present on some lanes but not others") + return None + if torch.is_tensor(per_lane[0]): + return torch.stack([x.reshape(()) for x in per_lane]).max() + return max(per_lane) + + def _sum_or_none(attr: str): + per_lane = [getattr(v, attr) for v in values] + if all(x is None for x in per_lane): + return None + if any(x is None for x in per_lane): + raise ValueError(f"{attr} present on some lanes but not others") + return sum(per_lane) + + return PackedSeqParams( + qkv_format=first.qkv_format, + cu_seqlens_q=_concat_cu("cu_seqlens_q"), + cu_seqlens_kv=_concat_cu("cu_seqlens_kv"), + cu_seqlens_q_padded=_concat_cu("cu_seqlens_q_padded"), + cu_seqlens_kv_padded=_concat_cu("cu_seqlens_kv_padded"), + max_seqlen_q=_max_scalar("max_seqlen_q"), + max_seqlen_kv=_max_scalar("max_seqlen_kv"), + total_tokens=_sum_or_none("total_tokens"), + local_cp_size=first.local_cp_size, + cp_group=first.cp_group, + ) + + def _build_tokenizer(args): from megatron.core.tokenizers.vision.libraries.multimodal_tokenizer import ( MegatronMultimodalTokenizer, From b759183c8930f41ef23ccf460b76b18b0660083a Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Wed, 20 May 2026 21:21:37 -0700 Subject: [PATCH 43/44] NMFW-464: explicit per-key schema for routed-iter modality_inputs merge (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #35 introduced ``_concat_first_varying_dim`` that selected the concat dim at runtime by looking for the first dim whose size differed across per-lane tensors, falling back to dim 0 when no dim varied. That worked in the common case but fails when two participating lanes happen to produce identically-shaped packed image buffers in the same step: the (1, T, C) buffer falls back to dim-0 cat and becomes (2, T, C) instead of (1, 2T, C). RADIO then splits the buffer using imgs_sizes (correctly cat'd to (2N, 2)) and asserts ``sum(seq_lens) != x.shape[1]`` with an exact 2x ratio — visible in the 1-node standalone smoke at lanes_per_encoder=16 as AssertionError: 15984 != 7992 at radio.py:235 In production hetero training, the rank dies silently in encoder forward; its served LLM lanes stall on bridge recv; the cluster cascades into a 600 s NCCL watchdog timeout that looked like a different bug. The probability of two lanes producing identical (1, T, C) per step grows with ``lanes_per_encoder``: rare at 4 (9n), common at 16 (33n). All "33n hangs" we chased were instances of this. Replace the runtime inference with a schema-aware merger that knows the fixed structure of ``modality_inputs``: packed image buffer (1, T_lane, C) -> torch.cat dim 1 imgs_sizes (N_images, 2) -> torch.cat dim 0 packed_seq_params PackedSeqParams -> _concat_packed_seq_params Anything unrecognized raises a loud ``TypeError`` so a future schema change has to be handled in ``_merge_encoder_inputs`` rather than silently miscompiled by a heuristic. Validated end-to-end: * 1-node standalone (lanes_per_encoder=16, 200 steps): all 8 ranks complete with no AssertionError, no hang. Previously failed at step 27 on rank 5. * 9n GBS=192 production smoke: 25/25 iters, ~5.5 s/iter steady state. * 17n GBS=384 production smoke: 25/25 iters, ~6.2 s/iter steady state. * 33n GBS=768 production smoke: 25/25 iters, ~7.0 s/iter steady state (matches the scaling-study report's 33n target of 7.11 s/iter). Previously hung at iter 5 across multiple attempts. --- examples/mimo/data/hetero_energon.py | 121 ++++++++++++++++++--------- 1 file changed, 82 insertions(+), 39 deletions(-) diff --git a/examples/mimo/data/hetero_energon.py b/examples/mimo/data/hetero_energon.py index 7af9b9da8d9..1fb2380f91e 100644 --- a/examples/mimo/data/hetero_energon.py +++ b/examples/mimo/data/hetero_energon.py @@ -384,50 +384,93 @@ def _combine_encoder_batches(batches: list[dict]) -> dict: if batch.get("modality_inputs") is not None ] if modality_values: - combined["modality_inputs"] = _concat_nested_tensors(modality_values) + combined["modality_inputs"] = _merge_modality_inputs(modality_values) return combined -def _concat_nested_tensors(values): - """Concatenate a list of matching nested tensor structures along the - first dimension that varies across lanes. - - For most per-lane tensors the varying dim is 0 (batch). For dynres-packed - image buffers the per-lane shape is ``(1, T_lane, C)`` — dim 0 is the - fixed lane batch size and dim 1 is the variable token axis, so the right - concatenation is along dim 1. Selecting the first-varying dim handles - both uniformly without any sibling-key context. +# --------------------------------------------------------------------------- +# Schema-aware merge of ``modality_inputs`` across LLM lanes served by one +# encoder rank. The structure produced by the dataset is fixed: +# +# modality_inputs = { +# "": { # e.g. "images" +# "": { # e.g. "radio_encoder" +# : Tensor of shape (1, T_lane, C), +# "imgs_sizes": Tensor of shape (N_images_lane, 2), +# "packed_seq_params": PackedSeqParams describing the T axis, +# } +# } +# } +# +# Each per-lane tensor has a known concat semantics; we encode them +# explicitly rather than inferring from runtime shape variation: +# +# * packed image buffer: leading dim is always 1 (lane batch == MBS=1); +# dim 1 is the variable token axis -> concat along dim 1. +# * ``imgs_sizes``: dim 0 = per-lane image count -> concat along dim 0. +# * ``packed_seq_params``: cu_seqlens need offset-shifting -> custom merge. +# --------------------------------------------------------------------------- + + +def _merge_modality_inputs(per_lane_modality_inputs): + """Merge the ``modality_inputs`` field of N per-lane batches.""" + merged = {} + modality_types = set().union( + *(p.keys() for p in per_lane_modality_inputs if isinstance(p, dict)) + ) + for mod_type in sorted(modality_types): + per_lane_mod = [p[mod_type] for p in per_lane_modality_inputs if mod_type in p] + merged_per_encoder = {} + encoder_names = set().union( + *(p.keys() for p in per_lane_mod if isinstance(p, dict)) + ) + for enc_name in sorted(encoder_names): + per_lane_enc = [p[enc_name] for p in per_lane_mod if enc_name in p] + merged_per_encoder[enc_name] = _merge_encoder_inputs(per_lane_enc) + merged[mod_type] = merged_per_encoder + return merged + + +def _merge_encoder_inputs(per_lane_enc_inputs): + """Merge per-lane encoder-input dicts using a key-explicit schema. + + Keys are categorized by name / value type: + * ``packed_seq_params`` -> ``_concat_packed_seq_params`` + * ``imgs_sizes`` -> ``torch.cat(..., dim=0)`` + * any other ``Tensor`` -> packed image buffer ``(1, T, C)``, + concat along dim 1 + Anything else triggers a loud error so a future schema change has to be + handled here rather than guessed at by a heuristic. """ - present = [value for value in values if value is not None] - if not present: - return None - first = present[0] - if isinstance(first, torch.Tensor): - return _concat_first_varying_dim(present) - if isinstance(first, dict): - keys = set().union(*(value.keys() for value in present if isinstance(value, dict))) - merged = {} - for key in sorted(keys): - value = _concat_nested_tensors([item.get(key) for item in present]) - if value is not None: - merged[key] = value - return merged - if isinstance(first, PackedSeqParams): - return _concat_packed_seq_params(present) - raise TypeError(f"cannot concatenate encoder batch value of type {type(first).__name__}") - - -def _concat_first_varying_dim(tensors): - """Concatenate ``tensors`` along the first dimension whose size differs - across the list; default to dim 0 if all shapes match.""" - if len(tensors) == 1: - return tensors[0] - first = tensors[0] - for d in range(first.dim()): - if any(t.shape[d] != first.shape[d] for t in tensors[1:]): - return torch.cat(tensors, dim=d) - return torch.cat(tensors, dim=0) + merged = {} + keys = set().union(*(p.keys() for p in per_lane_enc_inputs if isinstance(p, dict))) + for key in sorted(keys): + vals = [p[key] for p in per_lane_enc_inputs if key in p] + if not vals: + continue + first = vals[0] + if isinstance(first, PackedSeqParams): + merged[key] = _concat_packed_seq_params(vals) + elif key == "imgs_sizes": + assert all(isinstance(v, torch.Tensor) for v in vals), ( + f"imgs_sizes must be tensors, got {[type(v).__name__ for v in vals]}" + ) + merged[key] = torch.cat(vals, dim=0) + elif isinstance(first, torch.Tensor): + # Packed image buffer: leading dim is the lane batch (==1); the + # variable token axis is dim 1. + assert first.dim() >= 2 and first.shape[0] == 1, ( + f"unexpected packed-buffer shape for encoder key {key!r}: " + f"{tuple(first.shape)} (expected leading dim 1)" + ) + merged[key] = torch.cat(vals, dim=1) + else: + raise TypeError( + f"unsupported encoder-input value for key {key!r}: " + f"{type(first).__name__}; extend _merge_encoder_inputs" + ) + return merged def _concat_packed_seq_params(values: list) -> PackedSeqParams: From 59155fd98794459220b83e55bd58b30dc3c94c90 Mon Sep 17 00:00:00 2001 From: Yashaswi Karnati <144376261+yashaswikarnati@users.noreply.github.com> Date: Sat, 23 May 2026 11:20:49 -0700 Subject: [PATCH 44/44] NMFW-464: production hetero scaling sbatches (33n / 68n / 100n, EP=8, Sanjeev LR schedule) (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * NMFW-464: production hetero scaling sbatches at 33n / 68n / 100n Adds three production-grade hetero MIMO Nemotron6-MoE VLM training sbatches that mirror Sanjeev's pretrain_3b_nano_vlm_sota_90t_10v.sh schedule: * sbatch_hetero_prod_gbs768_33n_ep8.sh — 33 nodes (1 enc + 32 LLM) * sbatch_hetero_prod_gbs768_68n_ep8.sh — 68 nodes (4 enc + 64 LLM) * sbatch_hetero_prod_gbs768_100n.sh — 100 nodes (4 enc + 96 LLM) Pinned from Sanjeev's baseline: - TRAIN_SAMPLES=122070313 - LR_WARMUP_SAMPLES=1024000 - LR_DECAY_SAMPLES = TRAIN_SAMPLES - LR_WARMUP_SAMPLES = 121046313 - LR_WSD_DECAY_SAMPLES=18310547 - LR_WSD_DECAY_STYLE=minus_sqrt - PACKING_BUFFER_SIZE=128 - NUM_WORKERS=1 - LOG_INTERVAL=100, SAVE_INTERVAL=1000 - --load-nemotron-checkpoint pointing at sasatheesh iter_1000 Deviations from Sanjeev's baseline: - LLM_EP=8 (vs Sanjeev's EP=16) - Hetero topology TP=2 (vs Sanjeev's TP=4); explicit encoder grid - MOE_ROUTER_FORCE_LOAD_BALANCING=0 (natural seq_aux_loss) - No MTP layers - Wall time 4h (Sanjeev's; restartable from save-interval-1000 checkpoints) These sbatches are derived from the scaling-study templates (sbatch_hetero_parity_gbs768_{33n_ep8,68n_ep8,100n}.sh on the ykarnati/nmfw-464-encoder-stall-profiling branch). The timeline-profile instrumentation and the num-distributed-optimizer-instances flag from that branch are dropped here — they are debug-only and not present in the production codebase on ykarnati/nmfw-464-nemotron-vlm-with-hetero-parallel. Co-Authored-By: Claude Opus 4.7 (1M context) * NMFW-464: drop obsolete early-development 9n/8n smoke sbatches Removes five sbatch scripts that were only used during early hetero MIMO development and have no external references in the repo (no docs, no Python code, no CI). They are superseded by the new production sbatches (33n/68n/100n) added in this PR, plus the existing parity templates sbatch_hetero_parity_gbs192.sh and sbatch_hetero_parity_gbs32.sh. Removed: * sbatch_hetero_nemotron_54l_hel_9n.sh 30-iter 9n smoke test. Smoke-only; no production use. * sbatch_hetero_nemotron_54l_hel_9n_text_only.sh text-only data-blend smoke test. exec'd _9n.sh — orphaned by its removal. * sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh 90/10 text-vision blend smoke test. exec'd _9n.sh — orphaned by its removal. * sbatch_hetero_nemotron_54l_hel_9n_parity.sh Standalone 9n Sanjeev-parity reproduction. Functionally superseded by sbatch_hetero_parity_gbs192.sh (also 9n + Sanjeev recipe, paired with sbatch_sanjeev_parity_gbs192.sh). * sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh 8n LLM-only text-only variant. exec'd _9n.sh — orphaned by its removal. Co-Authored-By: Claude Opus 4.7 (1M context) * NMFW-464: add concise README for the hetero sbatch directory One table covering the 9n parity sbatch and the three production sbatches (33n / 68n / 100n) with their topology, GBS, and purpose. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- examples/mimo/scripts/README.md | 12 + .../sbatch_hetero_nemotron_54l_hel_9n.sh | 207 -------------- ...batch_hetero_nemotron_54l_hel_9n_parity.sh | 254 ------------------ ...ch_hetero_nemotron_54l_hel_9n_text_only.sh | 37 --- ..._hetero_nemotron_54l_hel_9n_text_vision.sh | 37 --- .../scripts/sbatch_hetero_prod_gbs768_100n.sh | 159 +++++++++++ .../sbatch_hetero_prod_gbs768_33n_ep8.sh | 165 ++++++++++++ .../sbatch_hetero_prod_gbs768_68n_ep8.sh | 160 +++++++++++ ..._mimo_nemotron_54l_hel_8n_text_only_llm.sh | 50 ---- 9 files changed, 496 insertions(+), 585 deletions(-) create mode 100644 examples/mimo/scripts/README.md delete mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n.sh delete mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_parity.sh delete mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_only.sh delete mode 100755 examples/mimo/scripts/sbatch_hetero_nemotron_54l_hel_9n_text_vision.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_prod_gbs768_100n.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_prod_gbs768_33n_ep8.sh create mode 100755 examples/mimo/scripts/sbatch_hetero_prod_gbs768_68n_ep8.sh delete mode 100755 examples/mimo/scripts/sbatch_mimo_nemotron_54l_hel_8n_text_only_llm.sh diff --git a/examples/mimo/scripts/README.md b/examples/mimo/scripts/README.md new file mode 100644 index 00000000000..3b8ef84e91c --- /dev/null +++ b/examples/mimo/scripts/README.md @@ -0,0 +1,12 @@ +# MIMO hetero training sbatches + +| Script | Nodes | Layout | GBS | Purpose | +|---|---|---|---|---| +| sbatch_hetero_parity_gbs192.sh | 9 | 1 enc + 8 LLM, TP=2 EP=16 | 192 | 9n Sanjeev parity test (5000 iters, paired with sbatch_sanjeev_parity_gbs192.sh) | +| sbatch_hetero_prod_gbs768_33n_ep8.sh | 33 | 1 enc + 32 LLM, TP=2 EP=8 | 768 | 33n production | +| sbatch_hetero_prod_gbs768_68n_ep8.sh | 68 | 4 enc + 64 LLM, TP=2 EP=8 | 768 | 68n production | +| sbatch_hetero_prod_gbs768_100n.sh | 100 | 4 enc + 96 LLM, TP=2 EP=8 | 768 | 100n production | + +Production sbatches use Sanjeev's WSD schedule (`TRAIN_SAMPLES=122070313`, `LR_WARMUP_SAMPLES=1024000`, `LR_WSD_DECAY_SAMPLES=18310547`) with EP=8 (vs Sanjeev's EP=16), no MTP, no force-LB. Load LLM weights via `--load-nemotron-checkpoint` from sasatheesh's `iter_0001000`. + +Launch: `sbatch examples/mimo/scripts/