diff --git a/examples/wan2/.gitignore b/examples/wan2/.gitignore new file mode 100644 index 000000000..ddf7d25d3 --- /dev/null +++ b/examples/wan2/.gitignore @@ -0,0 +1,4 @@ +# Exported ONNX artifacts (regenerate via export_*.py) and generated videos. +model/ +*.mp4 +_tiny_sym.onnx diff --git a/examples/wan2/README.md b/examples/wan2/README.md new file mode 100644 index 000000000..d08a2383d --- /dev/null +++ b/examples/wan2/README.md @@ -0,0 +1,299 @@ +# Wan-AI/Wan2.1-T2V-1.3B-Diffusers — ONNX denoiser export + +End-to-end walkthrough for exporting the **denoising transformer** of +`Wan-AI/Wan2.1-T2V-1.3B-Diffusers` (text-to-video) to **fp16 ONNX** and running +the full diffusers pipeline with that graph on **ONNX Runtime (CUDA)**. + +Only the transformer is exported. In a diffusion pipeline the transformer is the +module that runs on *every* denoising step (100 calls for a 50-step, +classifier-free-guidance run), so it dominates runtime. The T5 text encoder and +the VAE run **once** per video and stay in PyTorch. + +``` +text prompt ──► T5 (umt5-xxl) ──►┐ + │ ┌────────── denoise loop (N steps) ──────────┐ +random latent noise ──────────────┼──►│ WanTransformer3DModel ← THIS is exported │──► latent ──► VAE decode ──► frames ──► mp4 +[1,16,21,60,104] │ │ (fp16 ONNX / ORT-CUDA) │ (PyTorch fp32, tiled) + │ └─────────────────────────────────────────────┘ + └── stays in PyTorch (runs once) +``` + +--- + +## Files + +| File | Purpose | +|---|---| +| `diffusers_t2v_sample.py` | Baseline: generate a video with the stock PyTorch pipeline (bf16). | +| `export_transformer_onnx.py` | Export the transformer to fp32 ONNX, then convert to fp16. | +| `verify_onnx.py` | Numerical parity check: ORT fp16 graph vs PyTorch fp16 module. | +| `run_onnx_pipeline.py` | Full pipeline with the ONNX transformer swapped in via ORT-CUDA. | +| `test_mha_symbolic.py` | Minimal, self-contained demo of the fused-attention export trick. | +| `export_vae_decoder_onnx.py` | Export the VAE **decoder** (single-tile graph, dynamic spatial) to fp32 ONNX. | +| `run_vae_onnx.py` | `OrtVaeDecoder`: portable tiled VAE decode on any ORT execution provider. | +| `verify_vae_onnx.py` | Parity check: ORT tiled decode vs PyTorch `AutoencoderKLWan.decode`. | +| `requirements.txt` | Pinned dependency versions used for the numbers below. | + +--- + +## 0. Setup + +Use a CUDA-enabled `torch` (the numbers below are cu128 on an RTX 5090 D, 32 GB). + +```powershell +uv venv --python 3.12 +uv pip install torch --index-url https://download.pytorch.org/whl/cu128 +uv pip install -r requirements.txt +``` + +All scripts download the model from the HuggingFace Hub on first run and resolve +their own paths, so they can be run from anywhere. + +--- + +## 1. Baseline (PyTorch) + +Confirms the model works and gives a reference video + timing. + +```powershell +python diffusers_t2v_sample.py +``` + +Writes `output.mp4` (480×832, 81 frames, 16 fps). Settings mirror the model card: +`flow_shift=3.0`, `guidance_scale=6.0`, 50 steps (diffusers default). + +--- + +## 2. Export the transformer to ONNX + +```powershell +python export_transformer_onnx.py +``` + +Produces, under `model/`: + +- `wan_transformer_fp32.onnx` (+ `.data`, ~5.4 GB) +- `wan_transformer_fp16.onnx` (+ `.data`, ~2.7 GB) ← used for inference + +The export is **static-shaped** for 480P / 81 frames, batch 1 +(latent `[1,16,21,60,104]`, sequence length `21×30×52 = 32,760` tokens). + +### How the export works (and why it's done this way) + +The naive `softmax(QK^T)V` attention is the crux of the whole problem: at 32,760 +tokens each layer materializes a **~26 GB** score tensor, which makes the ONNX +graph unusable in ORT (it hangs / near-OOMs). The fix is to emit a single +**fused `com.microsoft::MultiHeadAttention` contrib node** per attention, which +ORT executes with a flash-attention kernel. + +The node is emitted directly with `torch.onnx.ops.symbolic` from a monkeypatched +`WanAttnProcessor.__call__` (see `wan_mha_call` in `export_transformer_onnx.py`): + +```python +out = torch.onnx.ops.symbolic( + "com.microsoft::MultiHeadAttention", + (query, key, value), # each packed [B, S, num_heads*head_dim] + attrs={"num_heads": int(attn.heads)}, + dtype=query.dtype, + shape=(B, Sq, D), + version=1, +) +``` + +`test_mha_symbolic.py` is a 60-line standalone proof of this mechanism — it +exports one node and shows ORT matches PyTorch SDPA to ~1e-3. Run it first if you +want to understand the trick in isolation: + +```powershell +python test_mha_symbolic.py +# nodes: ['MultiHeadAttention(com.microsoft)'] +# max_abs_diff vs torch SDPA: ~9e-4 +``` + +Other things the export script handles, each of which otherwise breaks: + +- **Dynamo (`torch.export`) exporter, not the legacy TorchScript one.** Dynamo + traces with fake tensors, so the multi-GB activations are never allocated. The + legacy tracer allocates real activations across all 30 layers and OOMs (~75 GB). +- **ScatterND-free RoPE.** The diffusers reference uses `out[..., 0::2] = ...` + slice-assignment, which lowers to *hundreds* of `ScatterND` ops. It is replaced + by `torch.stack((o1, o2), -1).flatten(-2)` (0 ScatterND in the final graph). +- **RMSNorm swap.** `torch.nn.RMSNorm` dispatches to `aten._fused_rms_norm`, + which the exporter can't lower; `OnnxRMSNorm` computes the same thing with + primitive ops. +- **Uniform fp32 export, then convert to fp16.** Exporting directly in mixed + precision crashes the type-promotion pass at `scale_shift_table + temb`. + fp16 conversion uses `onnxconverter_common.float16` with + `disable_shape_infer=True` (required for >2 GB models). +- **RoPE buffers cast to fp32** (they are float64 by default; no float64 in the graph). + +The final fp16 graph has **60 MultiHeadAttention nodes, 0 Softmax, 0 ScatterND**. + +#### Approaches that do NOT work (don't retry these) + +| Attempt | Result | +|---|---| +| Naive `softmax(QK^T)V` attention | ORT hang / near-OOM (26 GB score tensor per layer) | +| Legacy TorchScript exporter | CUDA OOM (~75 GB real activations across 30 layers) | +| Dynamo + onnxscript `custom_translation_table` | version-converter inliner crash: "number of values and replacements must match" | +| ORT `optimize_model` transformer fusion | did not fuse the attention | + +--- + +## 3. Verify parity + +```powershell +python verify_onnx.py +``` + +Compares the ORT fp16 graph against the PyTorch fp16 module on the same random +input. Expected: `max_abs_diff ≈ 0.013`, `mean rel error ≈ 0.29%` — i.e. within +fp16 noise. + +--- + +## 4. Run the full pipeline on ONNX Runtime + +```powershell +# denoiser in ONNX; VAE decode in PyTorch (tiled, CUDA only) +python run_onnx_pipeline.py --steps 50 --frames 81 --vae torch --out output_onnx.mp4 + +# denoiser AND VAE decode in ONNX/ORT (portable path, default) +python run_onnx_pipeline.py --steps 50 --frames 81 --vae onnx --out output_onnx.mp4 +``` + +`OrtTransformer` is a drop-in for `WanTransformer3DModel` exposing only what the +`WanPipeline` touches (`config`, `dtype`, a no-op `cache_context`, and a +`__call__` returning `(noise_pred,)`). T5 always stays PyTorch. + +With `--vae onnx` (default) the runner drives the pipeline with +`output_type="latent"`, then reproduces WanPipeline's latent un-scaling +(`latents / std + mean`) and decodes with `OrtVaeDecoder` (see §5) — so **both** +the per-step denoiser and the VAE decode run on ONNX Runtime, and the torch VAE +weights are moved off the GPU. With `--vae torch` the VAE decode stays in PyTorch. + +### VRAM management (needed to fit in 32 GB) + +- **T5 stays bf16/fp32** — umt5-xxl overflows in fp16. +- **VAE decode is tiled** either way — a full fp32 decode of 81 frames OOMs 32 GB + (`vae.enable_tiling()` for `--vae torch`; the single-tile ONNX graph for `--vae onnx`). +- **ORT arenas capped** — `gpu_mem_limit = 8 GB` + `arena_extend_strategy=kSameAsRequested` + on both the denoiser and VAE sessions, so ORT doesn't grab all the VRAM. +- With `--vae onnx`, the denoiser ORT session is freed before the VAE session decodes. + +Peak VRAM ≈ **23 GB**, occurring during VAE decode (T5 ~11 GB resident + ORT +arena + tiled VAE working set). + +--- + +## 5. Export the VAE decoder (needed for `--vae onnx`, and for non-CUDA GPUs) + +The T5 encoder stays in PyTorch, so with `--vae torch` the *decode* path only runs +on CUDA. To run the VAE decode on ONNX Runtime — required by the default +`--vae onnx` in §4, and the only way to run the decode on **other GPU backends** +(DirectML, OpenVINO, ROCm, …) — the decoder has to be ONNX too. For T2V only the +**decoder** matters (the encoder is unused). + +```powershell +python export_vae_decoder_onnx.py # -> model/wan_vae_decoder_fp32.onnx (+ .data, ~280 MB) +python verify_vae_onnx.py # parity vs PyTorch tiled decode +``` + +### Why the VAE is trickier than the transformer + +The Wan VAE decode is a **stateful, sequential** process, not a single forward: +`AutoencoderKLWan._decode` walks the latent frames one chunk at a time, threading +a **causal temporal feature cache** through 33 `WanCausalConv3d` layers, and +`tiled_decode` wraps that in a spatial tile loop with overlap blending. The ops +themselves (Conv3d, `F.normalize` RMSNorm, nearest `Upsample`, a small ~1024-token +mid-block attention) all export cleanly — the challenge is the state + the loops. + +### How the export works + +`export_vae_decoder_onnx.py` exports the decode of **one spatial tile** as a +single graph: + +- The **21-frame loop is unrolled** inside the traced forward, so the temporal + feature cache becomes ordinary intermediate tensors — there is no 33-tensor + cache to plumb in/out, and the `first_chunk` / cache-padding branches collapse + to compile-time constants. Numerically identical to PyTorch. +- The cache is threaded through **local Python lists** (not module attributes) so + the dynamo exporter traces it without side-effect errors. +- Frames are fixed at 21 (→81 output); **spatial H/W are dynamic** (`torch.export.Dim`) + so the smaller edge tiles decode at their true size — exact parity, no padding. +- `post_quant_conv` is folded in; the graph has 817 Convs, 21 small Softmaxes, and + **0 ScatterND**. + +`run_vae_onnx.py` provides `OrtVaeDecoder`, which reproduces `tiled_decode`'s outer +spatial tiling + `blend_v`/`blend_h` orchestration in ~40 lines of Python/NumPy +around the single-tile ORT session — so all heavy compute runs on the chosen EP: + +```python +from run_vae_onnx import OrtVaeDecoder +dec = OrtVaeDecoder(providers=["DmlExecutionProvider"]) # or OpenVINO/ROCm/CPU +frames = dec.decode(latents) # [1,16,21,60,104] -> [1,3,81,480,832] +``` + +### Notes + +- **fp32, portable.** The Wan VAE prefers fp32 for stability, and fp32 is the + safest common denominator across EPs. Parity vs PyTorch tiled decode: + **max_abs_diff 0.0042, mean rel error 0.037%**. +- **Memory-bounded by tiling.** The single-tile graph keeps peak memory small, + which matters on lower-VRAM non-NVIDIA GPUs. +- **Verify 3D-conv support on your target EP.** These are the standard ONNX ops, + but 5D `Conv` coverage varies by backend (e.g. DirectML historically) — run + `verify_vae_onnx.py` with your providers before relying on it. + +--- + +## 6. Why T5 stays in PyTorch + +The T5 (umt5-xxl) text encoder is deliberately **not** exported. It is the one +component where ONNX adds cost without benefit: + +- **It runs once per video, not per step.** T5 encodes the prompt a single time; + the ~180 s run is dominated by the 100 denoiser calls. Its latency is + irrelevant — even 4.4 s on CPU is noise next to the denoise loop. +- **It's big and fp16-hostile.** umt5-xxl is ~5.6 B params and overflows in fp16, + so an ONNX export would be fp32 (~22 GB) or bf16 (~11 GB) — a huge artifact for + a once-per-video op. +- **PyTorch-CPU already covers portability.** The point of exporting was the + *per-step GPU compute*. On a non-CUDA machine you simply run T5 in PyTorch on + CPU (4.4 s); ORT wouldn't make that faster. + +Only export T5 if you need a **100% pure-ORT deployment** with zero PyTorch +anywhere — and even then, INT8-dynamic-quantize it (halves the size) rather than +shipping fp32. If your prompts are fixed, a simpler win is to **precompute and +cache the T5 embeddings once** and skip the encoder entirely at inference. + +--- + +## Benchmarks + +RTX 5090 D (32 GB), 50 steps, 81 frames, 480×832. + +| Phase | GPU | CPU (fp32) | Notes | +|---|---|---|---| +| T5 text encode (both prompts) | 0.64 s | 4.4 s (~14×) | once per video | +| Denoise (per transformer call) | 1.82 s | — | 100 calls / 50 steps → ~98% of runtime | +| VAE decode (81 frames, PyTorch, tiled) | 11 s | 397 s (~36×) | once per video | +| VAE decode (81 frames, ONNX/ORT-CUDA, tiled) | 11.3 s | — | portable graph, matches PyTorch | +| **Total pipeline** | **~186 s** | — | vs ~289 s for the bf16 PyTorch baseline | + +The ONNX/ORT denoiser is faster than the bf16 PyTorch transformer here, but the +main reason to export is **portability**, not raw speed. + +--- + +## Recommendation + +- **Export the denoiser always.** It runs every step and dominates runtime. +- **Export the VAE decoder only if you need non-CUDA portability.** On NVIDIA it's + a wash with PyTorch (11.3 s vs 11 s); its value is running on DirectML / OpenVINO + / ROCm via ORT. For T2V you never need the encoder. +- **T5 (umt5-xxl) is the remaining PyTorch piece.** It runs once and must stay + bf16/fp32 (fp16 overflows). Export it too only if the *whole* pipeline must + leave CUDA; otherwise keep it in PyTorch. ONNX is for portability, not speed — + CPU execution is 14–36× slower, and only INT8 quantization would meaningfully + help there. diff --git a/examples/wan2/diffusers_t2v_sample.py b/examples/wan2/diffusers_t2v_sample.py new file mode 100644 index 000000000..32d2464c8 --- /dev/null +++ b/examples/wan2/diffusers_t2v_sample.py @@ -0,0 +1,47 @@ +"""Minimal Diffusers text-to-video sample for Wan-AI/Wan2.1-T2V-1.3B-Diffusers. + +Adapted from the repo README. Uses 480P settings recommended for the 1.3B model. +""" +import torch +from diffusers import AutoencoderKLWan, WanPipeline +from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler +from diffusers.utils import export_to_video + +model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" + +# 1.3B is trained at 480P; flow_shift=3.0 recommended for 480P. +flow_shift = 3.0 + +vae = AutoencoderKLWan.from_pretrained( + model_id, subfolder="vae", torch_dtype=torch.float32) +scheduler = UniPCMultistepScheduler( + prediction_type="flow_prediction", + use_flow_sigmas=True, + num_train_timesteps=1000, + flow_shift=flow_shift, +) +pipe = WanPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16) +pipe.scheduler = scheduler +pipe.to("cuda") + +prompt = ("Two anthropomorphic cats in comfy boxing gear and bright gloves " + "fight intensely on a spotlighted stage.") +negative_prompt = ( + "Bright tones, overexposed, static, blurred details, subtitles, style, " + "works, paintings, images, static, overall gray, worst quality, low " + "quality, JPEG compression residue, ugly, incomplete, extra fingers, " + "poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen " + "limbs, fused fingers, still picture, messy background, three legs, many " + "people in the background, walking backwards") + +output = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + height=480, + width=832, + num_frames=81, + guidance_scale=6.0, +).frames[0] + +export_to_video(output, "output.mp4", fps=16) +print("Saved output.mp4") diff --git a/examples/wan2/export_transformer_onnx.py b/examples/wan2/export_transformer_onnx.py new file mode 100644 index 000000000..3ab73c3c0 --- /dev/null +++ b/examples/wan2/export_transformer_onnx.py @@ -0,0 +1,214 @@ +"""Export the Wan2.1-T2V-1.3B denoiser (WanTransformer3DModel) to fp16 ONNX. + +Strategy: + 1. Replace each self/cross-attention with a single fused + `com.microsoft.MultiHeadAttention` contrib node, emitted via + torch.onnx.ops.symbolic. The naive softmax(QK^T)V export materializes a + ~26GB score tensor per layer over the 32,760-token sequence and is unusable + in ORT; the fused node lets ORT use a flash-attention kernel instead. + 2. Export the transformer in uniform fp32 with the dynamo (torch.export) + exporter. Uniform dtype avoids the mixed fp16/fp32 adds around + scale_shift_table that break the type-promotion pass, and the fake-tensor + tracing avoids the multi-GB activation blow-up that OOMs the legacy tracer. + 3. Convert the fp32 ONNX graph to fp16 with onnxconverter-common. + +Only the denoising transformer is exported -- it is the module that runs on +every diffusion step. The T5 text encoder and the VAE stay in PyTorch. + +Static shapes for 480P / 81 frames, batch=1 (the diffusers WanPipeline calls the +transformer once for cond and once for uncond, each with batch=1). +""" +import os + +import onnx +import torch +from diffusers import WanTransformer3DModel +from diffusers.models.transformers import transformer_wan as twan +from onnxconverter_common import float16 + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +OUT_DIR = os.path.join(os.path.dirname(__file__), "model") +FP32_PATH = os.path.join(OUT_DIR, "wan_transformer_fp32.onnx") +FP16_PATH = os.path.join(OUT_DIR, "wan_transformer_fp16.onnx") +OPSET = 18 + +# 480P / 81 frames latent geometry: vae temporal stride 4, spatial stride 8. +LAT_FRAMES = (81 - 1) // 4 + 1 # 21 +LAT_H = 480 // 8 # 60 +LAT_W = 832 // 8 # 104 + + +# --------------------------------------------------------------------------- +# Fused attention -> com.microsoft.MultiHeadAttention +# --------------------------------------------------------------------------- + + +def _apply_rotary_emb_onnx(hs, freqs_cos, freqs_sin): + """ScatterND-free rotary embedding (interleaved layout). + + Equivalent to the diffusers reference which uses out[..., 0::2]/out[..., 1::2] + slice-assignment (exports to hundreds of ScatterND ops). + """ + xshaped = hs.unflatten(-1, (-1, 2)) + x1 = xshaped[..., 0] + x2 = xshaped[..., 1] + cos = freqs_cos[..., 0::2] + sin = freqs_sin[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x1 * sin + x2 * cos + out = torch.stack((o1, o2), dim=-1).flatten(-2) + return out.type_as(hs) + + +def wan_mha_call(self, attn, hidden_states, encoder_hidden_states=None, + attention_mask=None, rotary_emb=None): + """Drop-in WanAttnProcessor.__call__ emitting the fused MHA op (T2V only).""" + query, key, value = twan._get_qkv_projections( + attn, hidden_states, encoder_hidden_states) + + query = attn.norm_q(query) + key = attn.norm_k(key) + + query = query.unflatten(2, (attn.heads, -1)) + key = key.unflatten(2, (attn.heads, -1)) + value = value.unflatten(2, (attn.heads, -1)) + + if rotary_emb is not None: + query = _apply_rotary_emb_onnx(query, *rotary_emb) + key = _apply_rotary_emb_onnx(key, *rotary_emb) + + B, Sq, N, H = query.shape + Sk = key.shape[1] + D = N * H + query = query.reshape(B, Sq, D) + key = key.reshape(B, Sk, D) + value = value.reshape(B, Sk, D) + + out = torch.onnx.ops.symbolic( + "com.microsoft::MultiHeadAttention", + (query, key, value), + attrs={"num_heads": int(attn.heads)}, + dtype=query.dtype, + shape=(B, Sq, D), + version=1, + ) + out = out.type_as(query) + out = attn.to_out[0](out) + out = attn.to_out[1](out) + return out + + +class TransformerWrapper(torch.nn.Module): + """Fixed-signature wrapper that returns just the predicted-noise tensor.""" + + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, hidden_states, timestep, encoder_hidden_states): + return self.model( + hidden_states=hidden_states, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + return_dict=False, + )[0] + + +class OnnxRMSNorm(torch.nn.Module): + """Explicit RMSNorm equivalent to torch.nn.RMSNorm. + + torch.nn.RMSNorm dispatches to aten._fused_rms_norm, which the dynamo ONNX + exporter cannot lower. This computes the same result with basic ops. + """ + + def __init__(self, weight, eps): + super().__init__() + self.weight = torch.nn.Parameter(weight.detach().clone()) + self.eps = 1e-6 if eps is None else float(eps) + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.eps) + return self.weight * hidden_states.to(input_dtype) + + +def replace_rmsnorm(module): + for name, child in module.named_children(): + if isinstance(child, torch.nn.RMSNorm): + setattr(module, name, OnnxRMSNorm(child.weight, child.eps)) + else: + replace_rmsnorm(child) + + +def export_fp32(): + device = "cuda" + print("Loading transformer (fp32) ...") + model = WanTransformer3DModel.from_pretrained( + MODEL_ID, subfolder="transformer", torch_dtype=torch.float32) + model.eval().to(device) + + cfg = model.config + text_dim = cfg.text_dim + in_ch = cfg.in_channels + print(f"config: in_channels={in_ch} text_dim={text_dim} " + f"num_layers={cfg.num_layers}") + + # Rotary buffers are float64 by default -> force fp32 (no float64 in graph). + model.rope.freqs_cos = model.rope.freqs_cos.to(torch.float32) + model.rope.freqs_sin = model.rope.freqs_sin.to(torch.float32) + + # Swap torch.nn.RMSNorm (fused op) for an ONNX-exportable implementation. + replace_rmsnorm(model) + model.to(device) + + # Emit fused MultiHeadAttention contrib nodes instead of naive softmax. + twan.WanAttnProcessor.__call__ = wan_mha_call + + wrapper = TransformerWrapper(model).eval() + + hidden_states = torch.randn( + 1, in_ch, LAT_FRAMES, LAT_H, LAT_W, dtype=torch.float32, device=device) + timestep = torch.tensor([999.0], dtype=torch.float32, device=device) + encoder_hidden_states = torch.randn( + 1, 512, text_dim, dtype=torch.float32, device=device) + + # Note: with the fused MHA op, eager output values are placeholders; only + # the traced graph is meaningful, so we skip a numeric ref forward here. + print("Exporting to ONNX fp32 (dynamo / torch.export) ...") + onnx_program = torch.onnx.export( + wrapper, + (hidden_states, timestep, encoder_hidden_states), + input_names=["hidden_states", "timestep", "encoder_hidden_states"], + output_names=["noise_pred"], + opset_version=OPSET, + dynamo=True, + ) + onnx_program.save(FP32_PATH, external_data=True) + print(f"Saved fp32 ONNX to {FP32_PATH}") + + +def convert_fp16(): + print("Converting fp32 ONNX -> fp16 ...") + model = onnx.load(FP32_PATH) + model16 = float16.convert_float_to_float16( + model, keep_io_types=False, disable_shape_infer=True) + onnx.save( + model16, + FP16_PATH, + save_as_external_data=True, + all_tensors_to_one_file=True, + location=os.path.basename(FP16_PATH) + ".data", + ) + print(f"Saved fp16 ONNX to {FP16_PATH}") + + +def main(): + os.makedirs(OUT_DIR, exist_ok=True) + export_fp32() + convert_fp16() + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/export_vae_decoder_onnx.py b/examples/wan2/export_vae_decoder_onnx.py new file mode 100644 index 000000000..e32a30eb9 --- /dev/null +++ b/examples/wan2/export_vae_decoder_onnx.py @@ -0,0 +1,95 @@ +"""Export the Wan2.1 VAE decoder (AutoencoderKLWan) to portable fp32 ONNX. + +For text-to-video only the *decoder* is used (latent -> frames); the encoder is +skipped. The diffusers decode is a stateful, sequential process: it walks the +latent frames one chunk at a time, threading a causal temporal feature cache +(``feat_cache``) through 33 ``WanCausalConv3d`` layers, and -- when tiling is on +-- wraps that in a spatial tile loop with overlap blending. + +We export the decode of a *single spatial tile* as one graph: + + * The 21-frame loop is **unrolled** inside the traced forward, so the temporal + feature cache becomes ordinary intermediate tensors -- no 33-tensor cache to + plumb in/out, and the ``first_chunk`` / cache-padding branches collapse to + compile-time constants. This is numerically identical to PyTorch. + * ``post_quant_conv`` is folded into the graph. + * Spatial height/width are **dynamic** so the smaller edge tiles decode at + their true size (exact parity with tiled_decode, no padding). + +The outer spatial tiling + blend loop stays in Python (see run_vae_onnx.py), so +all heavy compute (convs, group/RMS norms, upsamples) runs in ORT on any EP. +""" +import os + +import torch +from diffusers import AutoencoderKLWan +from torch.export import Dim + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model") +FP32_PATH = os.path.join(OUT_DIR, "wan_vae_decoder_fp32.onnx") +OPSET = 18 + +# 480P / 81 frames: 81 output frames -> 21 latent frames (temporal stride 4). +LAT_FRAMES = (81 - 1) // 4 + 1 # 21 + + +class VaeDecoderTile(torch.nn.Module): + """Decode one latent tile [1, 16, LAT_FRAMES, h, w] -> sample [1, 3, 81, 8h, 8w]. + + Mirrors the inner loop of AutoencoderKLWan.tiled_decode for a single tile, + with the frame loop unrolled and the causal cache kept in local lists so the + dynamo exporter can trace it without module-attribute side effects. + """ + + def __init__(self, vae, num_latent_frames): + super().__init__() + self.vae = vae + self.num_frames = num_latent_frames + vae.clear_cache() + self.conv_num = vae._conv_num + + def forward(self, z_tile): + vae = self.vae + feat_map = [None] * self.conv_num + outs = [] + for k in range(self.num_frames): + conv_idx = [0] + tile = vae.post_quant_conv(z_tile[:, :, k : k + 1, :, :]) + decoded = vae.decoder( + tile, feat_cache=feat_map, feat_idx=conv_idx, first_chunk=(k == 0)) + outs.append(decoded) + return torch.cat(outs, dim=2) + + +def main(): + os.makedirs(OUT_DIR, exist_ok=True) + print("Loading VAE (fp32) ...") + vae = AutoencoderKLWan.from_pretrained( + MODEL_ID, subfolder="vae", torch_dtype=torch.float32).eval() + + model = VaeDecoderTile(vae, LAT_FRAMES).eval() + + # A min-size latent tile (32x32) is enough to trace the graph; spatial dims + # are exported dynamic so any tile size (incl. smaller edge tiles) works. + z = torch.randn(1, vae.config.z_dim, LAT_FRAMES, 32, 32, dtype=torch.float32) + + h = Dim("h", min=2, max=256) + w = Dim("w", min=2, max=256) + + print("Exporting VAE decoder to ONNX fp32 (dynamo / torch.export) ...") + onnx_program = torch.onnx.export( + model, + (z,), + input_names=["z_tile"], + output_names=["sample_tile"], + dynamic_shapes={"z_tile": {3: h, 4: w}}, + opset_version=OPSET, + dynamo=True, + ) + onnx_program.save(FP32_PATH, external_data=True) + print(f"Saved fp32 VAE decoder ONNX to {FP32_PATH}") + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/requirements.txt b/examples/wan2/requirements.txt new file mode 100644 index 000000000..2c11548ad --- /dev/null +++ b/examples/wan2/requirements.txt @@ -0,0 +1,15 @@ +# ONNX export + ORT inference environment for Wan2.1-T2V-1.3B. +# Tested with Python 3.12 on Windows (RTX 5090 D, CUDA 12.8). +# Install torch from the CUDA 12.8 index, e.g.: +# uv pip install torch --index-url https://download.pytorch.org/whl/cu128 +torch==2.11.0 # +cu128 build +diffusers==0.39.0 +transformers==5.14.1 +accelerate +ftfy # T5/umt5 tokenizer text cleaning +imageio-ffmpeg # export_to_video encoder +numpy +onnx==1.22.0 +onnxruntime-gpu==1.27.0 +onnxscript==0.7.1 +onnxconverter-common==1.16.0 diff --git a/examples/wan2/run_onnx_pipeline.py b/examples/wan2/run_onnx_pipeline.py new file mode 100644 index 000000000..f172fc42c --- /dev/null +++ b/examples/wan2/run_onnx_pipeline.py @@ -0,0 +1,179 @@ +"""Run the Wan2.1-T2V-1.3B pipeline end-to-end with the ONNX denoiser in ORT. + +The T5 text encoder stays in PyTorch (it must not run in fp16). The diffusion +transformer -- the module executed on every denoising step -- is always replaced +by the fp16 ONNX graph running on ONNX Runtime (CUDA). + +The VAE decode can run either in PyTorch (``--vae torch``, tiled) or, for a fully +portable path, on the ONNX decoder via ONNX Runtime (``--vae onnx``, the default; +see export_vae_decoder_onnx.py / run_vae_onnx.py). +""" +import argparse +import gc +import os +import time +from contextlib import contextmanager + +import numpy as np +import onnxruntime as ort +import torch +from diffusers import AutoencoderKLWan, WanPipeline +from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler +from diffusers.utils import export_to_video + +from run_vae_onnx import OrtVaeDecoder + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +FP16_PATH = os.path.join(os.path.dirname(__file__), "model", "wan_transformer_fp16.onnx") + + +class OrtTransformer: + """Drop-in replacement for WanTransformer3DModel backed by an ORT session. + + Implements just the surface the WanPipeline touches: ``config``, ``dtype``, + a no-op ``cache_context`` and a ``__call__`` returning ``(noise_pred,)``. + """ + + def __init__(self, onnx_path, config, dtype, device): + self.config = config + self.dtype = dtype + self._device = device + so = ort.SessionOptions() + cuda_opts = { + "device_id": 0, + # Keep ORT's arena small so the fp32 VAE decode has GPU headroom. + "arena_extend_strategy": "kSameAsRequested", + "gpu_mem_limit": 8 * 1024 * 1024 * 1024, + } + self.sess = ort.InferenceSession( + onnx_path, so, + providers=[("CUDAExecutionProvider", cuda_opts), + "CPUExecutionProvider"]) + self._itypes = {i.name: i.type for i in self.sess.get_inputs()} + self.n_calls = 0 + self.total_s = 0.0 + + @contextmanager + def cache_context(self, name): + yield + + def _np(self, name, t): + a = t.detach().to("cpu") + if "float16" in self._itypes[name]: + return a.to(torch.float16).numpy() + return a.to(torch.float32).numpy() + + def __call__(self, hidden_states, timestep, encoder_hidden_states, + attention_kwargs=None, return_dict=False, **kwargs): + feeds = { + "hidden_states": self._np("hidden_states", hidden_states), + "timestep": self._np("timestep", timestep), + "encoder_hidden_states": self._np("encoder_hidden_states", + encoder_hidden_states), + } + t0 = time.time() + out = self.sess.run(["noise_pred"], feeds)[0] + self.total_s += time.time() - t0 + self.n_calls += 1 + noise_pred = torch.from_numpy(np.ascontiguousarray(out)).to( + self._device, hidden_states.dtype) + return (noise_pred,) + + +def onnx_vae_decode(pipe, latents, ort_vae): + """Replicate WanPipeline's latent un-scaling, then decode with the ONNX VAE. + + Mirrors the ``output_type != "latent"`` branch of WanPipeline.__call__: + un-normalize with the VAE's latents_mean/std, decode, then postprocess. + """ + cfg = pipe.vae.config + z_dim = cfg.z_dim + latents = latents.to(torch.float32).cpu() + mean = torch.tensor(cfg.latents_mean).view(1, z_dim, 1, 1, 1) + inv_std = 1.0 / torch.tensor(cfg.latents_std).view(1, z_dim, 1, 1, 1) + latents = latents / inv_std + mean + video = ort_vae.decode(latents) # [1, 3, F, H, W] in [-1, 1] + return pipe.video_processor.postprocess_video(video, output_type="np")[0] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--steps", type=int, default=50) + ap.add_argument("--frames", type=int, default=81) + ap.add_argument("--out", default="output_onnx.mp4") + ap.add_argument("--vae", choices=["onnx", "torch"], default="onnx", + help="VAE decode backend (onnx = portable ORT decoder).") + args = ap.parse_args() + + device = "cuda" + vae = AutoencoderKLWan.from_pretrained( + MODEL_ID, subfolder="vae", torch_dtype=torch.float32) + scheduler = UniPCMultistepScheduler( + prediction_type="flow_prediction", use_flow_sigmas=True, + num_train_timesteps=1000, flow_shift=3.0) + pipe = WanPipeline.from_pretrained(MODEL_ID, vae=vae, torch_dtype=torch.bfloat16) + pipe.scheduler = scheduler + pipe.to(device) + + ort_vae = None + if args.vae == "onnx": + # Decode via the ONNX VAE; the torch VAE weights are only needed for + # config/un-scaling, so move them off the GPU to free VRAM. + pipe.vae.to("cpu") + ort_vae = OrtVaeDecoder(gpu_mem_limit_gb=8) + else: + # Tile the VAE decode so 81 fp32 frames fit alongside T5 + the ORT session. + pipe.vae.enable_tiling() + + # Swap the torch transformer for the ONNX session; free the torch weights. + cfg = pipe.transformer.config + pipe.transformer = None + gc.collect() + torch.cuda.empty_cache() + print("Creating ORT session (CUDA) ...") + pipe.transformer = OrtTransformer(FP16_PATH, cfg, torch.float16, device) + + prompt = ("Two anthropomorphic cats in comfy boxing gear and bright gloves " + "fight intensely on a spotlighted stage.") + negative_prompt = ( + "Bright tones, overexposed, static, blurred details, subtitles, style, " + "works, paintings, images, static, overall gray, worst quality, low " + "quality, JPEG compression residue, ugly, incomplete, extra fingers, " + "poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen " + "limbs, fused fingers, still picture, messy background, three legs, many " + "people in the background, walking backwards") + + # For the ONNX VAE path, stop the pipeline at the latent and decode with ORT. + output_type = "latent" if args.vae == "onnx" else "np" + + t0 = time.time() + result = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + height=480, + width=832, + num_frames=args.frames, + num_inference_steps=args.steps, + guidance_scale=6.0, + output_type=output_type, + ).frames + tr = pipe.transformer + + if args.vae == "onnx": + # Free the denoiser ORT session before the VAE session decodes. + pipe.transformer = None + gc.collect() + torch.cuda.empty_cache() + output = onnx_vae_decode(pipe, result, ort_vae) + else: + output = result[0] + dt = time.time() - t0 + + export_to_video(output, args.out, fps=16) + print(f"Saved {args.out} (vae={args.vae})") + print(f"Total pipeline time: {dt:.1f}s | ORT denoiser calls: {tr.n_calls} | " + f"avg ORT/call: {tr.total_s / max(tr.n_calls, 1):.3f}s") + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/run_vae_onnx.py b/examples/wan2/run_vae_onnx.py new file mode 100644 index 000000000..f9e1e0445 --- /dev/null +++ b/examples/wan2/run_vae_onnx.py @@ -0,0 +1,125 @@ +"""Tiled VAE decode on ONNX Runtime, portable across execution providers. + +Wraps the single-tile ONNX decoder (see export_vae_decoder_onnx.py) with the +spatial tiling + overlap-blend orchestration from AutoencoderKLWan.tiled_decode. +The heavy compute runs in ORT; only the cheap tiling/blend bookkeeping is Python. + +The tiling constants below are the AutoencoderKLWan defaults for this model +(spatial compression 8, 256px min tile, 192px stride, patch_size=None). +""" +import os + +import numpy as np +import onnxruntime as ort +import torch + +FP32_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "model", "wan_vae_decoder_fp32.onnx") + +# AutoencoderKLWan defaults for Wan2.1-T2V-1.3B. +SPATIAL_COMP = 8 +TILE_SAMPLE_MIN = 256 +TILE_SAMPLE_STRIDE = 192 +TILE_LATENT_MIN = TILE_SAMPLE_MIN // SPATIAL_COMP # 32 +TILE_LATENT_STRIDE = TILE_SAMPLE_STRIDE // SPATIAL_COMP # 24 +BLEND_EXTENT = TILE_SAMPLE_MIN - TILE_SAMPLE_STRIDE # 64 (patch_size is None) + + +def _blend_v(a, b, blend_extent): + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = ( + a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + + b[:, :, :, y, :] * (y / blend_extent)) + return b + + +def _blend_h(a, b, blend_extent): + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = ( + a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + + b[:, :, :, :, x] * (x / blend_extent)) + return b + + +class OrtVaeDecoder: + """Portable tiled VAE decode backed by an ONNX Runtime session. + + Args: + onnx_path: path to the single-tile decoder ONNX. + providers: ORT execution providers, e.g. ``["DmlExecutionProvider"]`` or + ``["OpenVINOExecutionProvider"]``. Defaults to CUDA then CPU. + """ + + def __init__(self, onnx_path=FP32_PATH, providers=None, gpu_mem_limit_gb=None): + if providers is None: + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + # Optionally cap the CUDA arena so the decoder coexists with other GPU + # sessions (e.g. the ONNX denoiser) without grabbing all the VRAM. + if gpu_mem_limit_gb is not None: + providers = [ + ("CUDAExecutionProvider", { + "device_id": 0, + "arena_extend_strategy": "kSameAsRequested", + "gpu_mem_limit": int(gpu_mem_limit_gb * 1024 * 1024 * 1024), + }) if p == "CUDAExecutionProvider" else p + for p in providers + ] + self.sess = ort.InferenceSession(onnx_path, providers=providers) + + def _decode_tile(self, z_tile): + out = self.sess.run(["sample_tile"], {"z_tile": z_tile})[0] + return torch.from_numpy(np.ascontiguousarray(out)) + + def decode(self, z): + """z: [1, 16, LAT_FRAMES, H, W] fp32 torch/np -> sample [1, 3, F, 8H, 8W].""" + if isinstance(z, torch.Tensor): + z = z.detach().cpu().float().numpy() + z = np.ascontiguousarray(z, dtype=np.float32) + _, _, _, height, width = z.shape + sample_height, sample_width = height * SPATIAL_COMP, width * SPATIAL_COMP + + rows = [] + for i in range(0, height, TILE_LATENT_STRIDE): + row = [] + for j in range(0, width, TILE_LATENT_STRIDE): + tile = z[:, :, :, i : i + TILE_LATENT_MIN, j : j + TILE_LATENT_MIN] + row.append(self._decode_tile(np.ascontiguousarray(tile))) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = _blend_v(rows[i - 1][j], tile, BLEND_EXTENT) + if j > 0: + tile = _blend_h(row[j - 1], tile, BLEND_EXTENT) + result_row.append( + tile[:, :, :, :TILE_SAMPLE_STRIDE, :TILE_SAMPLE_STRIDE]) + result_rows.append(torch.cat(result_row, dim=-1)) + dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width] + return torch.clamp(dec, -1.0, 1.0) + + +def main(): + import argparse + import time + + ap = argparse.ArgumentParser() + ap.add_argument("--frames", type=int, default=21, help="latent frames (21 -> 81)") + ap.add_argument("--lat-h", type=int, default=60) + ap.add_argument("--lat-w", type=int, default=104) + args = ap.parse_args() + + dec = OrtVaeDecoder() + print("providers:", dec.sess.get_providers()) + z = torch.randn(1, 16, args.frames, args.lat_h, args.lat_w, dtype=torch.float32) + t0 = time.time() + out = dec.decode(z) + print(f"decoded {tuple(out.shape)} in {time.time() - t0:.1f}s") + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/test_mha_symbolic.py b/examples/wan2/test_mha_symbolic.py new file mode 100644 index 000000000..191c0dd2d --- /dev/null +++ b/examples/wan2/test_mha_symbolic.py @@ -0,0 +1,65 @@ +"""Validate torch.onnx.ops.symbolic -> com.microsoft.MultiHeadAttention (dynamo).""" +import os + +import numpy as np +import onnx +import onnxruntime as ort +import torch +import torch.nn.functional as F + +TINY_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_tiny_sym.onnx") + + +def sdpa_ref(q, k, v, num_heads): + B, Sq, D = q.shape + H = D // num_heads + Sk = k.shape[1] + qh = q.view(B, Sq, num_heads, H).transpose(1, 2) + kh = k.view(B, Sk, num_heads, H).transpose(1, 2) + vh = v.view(B, Sk, num_heads, H).transpose(1, 2) + o = F.scaled_dot_product_attention(qh, kh, vh) + return o.transpose(1, 2).reshape(B, Sq, D) + + +class Tiny(torch.nn.Module): + def forward(self, q, k, v): + B, Sq, D = q.shape + return torch.onnx.ops.symbolic( + "com.microsoft::MultiHeadAttention", + (q, k, v), + attrs={"num_heads": 2}, + dtype=q.dtype, + shape=(B, Sq, D), + version=1, + ) + + +def main(): + torch.manual_seed(0) + B, Sq, Sk, N, H = 1, 8, 8, 2, 4 + D = N * H + q = torch.randn(B, Sq, D) + k = torch.randn(B, Sk, D) + v = torch.randn(B, Sk, D) + + ref = sdpa_ref(q, k, v, N).detach().numpy() + + prog = torch.onnx.export( + Tiny().eval(), (q, k, v), + input_names=["q", "k", "v"], output_names=["o"], + opset_version=18, dynamo=True, + ) + prog.save(TINY_PATH) + + model = onnx.load(TINY_PATH) + print("nodes:", [n.op_type + "(" + n.domain + ")" for n in model.graph.node]) + + sess = ort.InferenceSession( + TINY_PATH, + providers=["CUDAExecutionProvider", "CPUExecutionProvider"]) + out = sess.run(["o"], {"q": q.numpy(), "k": k.numpy(), "v": v.numpy()})[0] + print("max_abs_diff vs torch SDPA:", np.abs(out - ref).max()) + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/verify_onnx.py b/examples/wan2/verify_onnx.py new file mode 100644 index 000000000..6bc411bd4 --- /dev/null +++ b/examples/wan2/verify_onnx.py @@ -0,0 +1,58 @@ +"""Verify the fp16 ONNX denoiser matches the PyTorch transformer in ORT-CUDA.""" +import os + +import numpy as np +import onnxruntime as ort +import torch +from diffusers import WanTransformer3DModel + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +FP16_PATH = os.path.join(os.path.dirname(__file__), "model", "wan_transformer_fp16.onnx") + +LAT_FRAMES = (81 - 1) // 4 + 1 +LAT_H, LAT_W = 480 // 8, 832 // 8 + + +def main(): + torch.manual_seed(0) + device = "cuda" + + model = WanTransformer3DModel.from_pretrained( + MODEL_ID, subfolder="transformer", torch_dtype=torch.float16).eval().to(device) + text_dim = model.config.text_dim + + hidden = torch.randn(1, model.config.in_channels, LAT_FRAMES, LAT_H, LAT_W, + dtype=torch.float16, device=device) + timestep = torch.tensor([987.0], dtype=torch.float32, device=device) + enc = torch.randn(1, 512, text_dim, dtype=torch.float16, device=device) + + with torch.no_grad(): + ref = model(hidden_states=hidden, timestep=timestep, + encoder_hidden_states=enc, return_dict=False)[0] + ref = ref.float().cpu().numpy() + + sess = ort.InferenceSession( + FP16_PATH, providers=["CUDAExecutionProvider", "CPUExecutionProvider"]) + print("ORT providers:", sess.get_providers()) + itypes = {i.name: i.type for i in sess.get_inputs()} + print("ONNX input types:", itypes) + + def cast(name, t): + return t.astype(np.float16) if "float16" in itypes[name] else t.astype(np.float32) + + feeds = { + "hidden_states": cast("hidden_states", hidden.float().cpu().numpy()), + "timestep": cast("timestep", timestep.float().cpu().numpy()), + "encoder_hidden_states": cast("encoder_hidden_states", enc.float().cpu().numpy()), + } + out = sess.run(["noise_pred"], feeds)[0].astype(np.float32) + + diff = np.abs(out - ref) + denom = np.abs(ref).mean() + print(f"shape={out.shape} dtype_onnx_out={out.dtype}") + print(f"max_abs_diff={diff.max():.4f} mean_abs_diff={diff.mean():.5f} " + f"mean_abs_ref={denom:.4f} rel={diff.mean()/denom:.4%}") + + +if __name__ == "__main__": + main() diff --git a/examples/wan2/verify_vae_onnx.py b/examples/wan2/verify_vae_onnx.py new file mode 100644 index 000000000..5443e6373 --- /dev/null +++ b/examples/wan2/verify_vae_onnx.py @@ -0,0 +1,37 @@ +"""Verify the ONNX tiled VAE decode matches PyTorch AutoencoderKLWan.decode.""" +import numpy as np +import torch +from diffusers import AutoencoderKLWan + +from run_vae_onnx import OrtVaeDecoder + +MODEL_ID = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +LAT_FRAMES = (81 - 1) // 4 + 1 # 21 +LAT_H, LAT_W = 60, 104 + + +def main(): + torch.manual_seed(0) + z = torch.randn(1, 16, LAT_FRAMES, LAT_H, LAT_W, dtype=torch.float32) + + print("PyTorch tiled decode (reference) ...") + vae = AutoencoderKLWan.from_pretrained( + MODEL_ID, subfolder="vae", torch_dtype=torch.float32).eval() + vae.enable_tiling() + with torch.no_grad(): + ref = vae.decode(z, return_dict=False)[0].float().cpu().numpy() + + print("ONNX Runtime tiled decode ...") + dec = OrtVaeDecoder() + print("providers:", dec.sess.get_providers()) + out = dec.decode(z).float().cpu().numpy() + + diff = np.abs(out - ref) + denom = np.abs(ref).mean() + print(f"shapes ref={ref.shape} onnx={out.shape}") + print(f"max_abs_diff={diff.max():.5f} mean_abs_diff={diff.mean():.6f} " + f"mean_abs_ref={denom:.4f} rel={diff.mean() / denom:.4%}") + + +if __name__ == "__main__": + main()