From e29606f07e6cb295dea40a2bc18f8bd0b46ec9d3 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:28:46 +0300 Subject: [PATCH 001/306] hcq2: copy kernel (#17480) * hcq2: copy with kernel * test * x --- .github/workflows/test.yml | 2 ++ extra/hcq2/ops_amd2.py | 6 +++--- test/device/test_hcq2.py | 14 ++++++++++++++ tinygrad/device.py | 2 ++ tinygrad/runtime/support/hcq2.py | 16 ++++++++++++++-- 5 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 test/device/test_hcq2.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45dfa564918cf..7bd3063e41bc1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -527,6 +527,8 @@ jobs: TestMultiTensor.test_backward_sum TestMultiTensor.test_matmul_shard_0_0 - name: Run HCQ2 JIT tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_jit.py + - name: Run HCQ2 unit tests + run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest test/device/test_hcq2.py testmockam: name: Linux (am) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 0c8e99d693e2f..fbdef9f4444ed 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -288,10 +288,10 @@ def amd_build_program(prg:UOp) -> UOp: class AMDAllocator(HCQAllocator['AMDDevice']): def __init__(self, dev:AMDDevice): - super().__init__(dev, supports_copy_from_disk=dev.has_sdma_queue, supports_transfer=dev.has_sdma_queue and not dev.is_usb()) + super().__init__(dev, supports_copy_from_disk=dev.has_copy_queue, supports_transfer=dev.has_copy_queue and not dev.is_usb()) def _alloc(self, size:int, options:BufferSpec) -> HCQ2Buffer: - return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_sdma_queue) + return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_copy_queue) def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque) @@ -581,7 +581,7 @@ def __init__(self, device:str=""): self.max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000 self.sdma_queues:dict = {} - self.has_sdma_queue = True # self.sdma_queue(0) is not None, TODO: think of this + self.has_copy_queue = not getenv("AMD_DISABLE_SDMA") super().__init__(device, AMDAllocator(self), [HIPRenderer, AMDLLVMRenderer, HIPCCRenderer], None, can_recover=self.is_am(), arch=self.arch) diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py new file mode 100644 index 0000000000000..19d72d7bb8609 --- /dev/null +++ b/test/device/test_hcq2.py @@ -0,0 +1,14 @@ +import unittest, numpy as np +from unittest.mock import patch +from tinygrad import Device, Tensor +from tinygrad.helpers import getenv +from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in + +@unittest.skipUnless(getenv("HCQ2") and all_devices_in(Device.DEFAULT, HCQ_DEVS), "hcq2 device required") +class TestHCQ2(unittest.TestCase): + def test_copy_without_copy_queue(self): + with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): + np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + +if __name__ == "__main__": + unittest.main() diff --git a/tinygrad/device.py b/tinygrad/device.py index 570cd79813bd4..6f796e07a6f17 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -336,6 +336,8 @@ class Compiled: pm_lower:Any = None pm_bufferize:Any = None + has_copy_queue:bool = True + def __init__(self, device:str, allocator:Allocator, renderers:list[type[Renderer]], runtime:type[Program[Self]]|None, graph=None, arch=None): from tinygrad.renderer import Renderer self.device, self.allocator, self.runtime_t, self.graph, self.renderers = device, allocator, runtime, graph, renderers or [Renderer] diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 9e2429e304020..9b0372d0354ed 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -96,12 +96,24 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS) and not all_devices_in(b.device, HCQ_P2P_DEVS) +def hcq_call_devs(call:UOp) -> Any|None: return next((b.device for b in call.src[1:] if all_devices_in(b.device, HCQ_DEVS)), None) + def stage_copy(dst:UOp, src:UOp) -> UOp|None: if not (_need_staging(src, dst) or _need_staging(dst, src)): return None stage = UOp.new_buffer("CPU", src.max_numel() * src.dtype.itemsize, dtypes.uint8) return UOp(Ops.LINEAR, src=(src.copy_to_device("CPU").call(stage, src), stage.copy_to_device(dst.device).call(dst, stage))) -pm_insert_copy_staging = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy)]) + +def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: + if (devs:=hcq_call_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None + d, s = (UOp.param(i, dst.dtype, (n:=dst.max_numel(),), device=devs) for i in range(2)) + ast = d.index(r:=UOp.range(n, 0)).store(s.index(r).load()).end(r).sink(arg=KernelInfo(name="copy"), tag=1) + return call.replace(src=(to_program(ast, Device[dev].renderer), dst, src)) + +pm_insert_copy_staging = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy), + (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src")), name="call"), kernel_copy) +]) # ***************** # 2. deps @@ -217,7 +229,7 @@ def sched_hcq_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] batch:list[tuple[UOp, tuple[str, ...]]] = [] for call in l.src: - if (devs:=next((b.device for b in call.src[1:] if all_devices_in(b.device, HCQ_DEVS)), None)) is not None: batch.append((call, to_tuple(devs))) + if (devs:=hcq_call_devs(call)) is not None: batch.append((call, to_tuple(devs))) else: srcs, batch = srcs + _finalize_batch(batch, profile) + [call], [] return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) From d41ca5e60fc55550b38c0aa0d6c01d9c94397895 Mon Sep 17 00:00:00 2001 From: Raine Date: Mon, 10 Aug 2026 12:39:14 -0300 Subject: [PATCH 002/306] Fix WMMA CI (#17479) * init * split into sub tests * trigger ci --- test/backend/test_linearizer.py | 6 ++---- test/opt/test_tensor_cores.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 958a9be84070a..5a43d6801168f 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -437,7 +437,7 @@ def reset_bufs(bufs:list[Buffer]): for buf in bufs: buf.copy_from(Buffer("PYTHON", buf.size, buf.dtype, opaque=memoryview(bytearray(buf.nbytes)))) def _helper_linearizer_opt_ast(realized_ast:UOp, real_bufs:list[Buffer], opts=[], - apply_tc=False, atol=1e-4, rtol=1e-4, color_sizes=[], wanna_output=[]): + apply_tc=False, atol=1e-4, rtol=1e-4, color_sizes=[], wanna_output=[], check_default_opt=True): outbufs = real_bufs[:len(realized_ast.src)] wanna_output = [np.array(x).flatten() for x in wanna_output] buf_uops = [UOp.new_buffer(b.device, b.size, b.dtype) for b in real_bufs] @@ -459,9 +459,7 @@ def check_opt(opts): for buf,want in zip(copyout_outputs(outbufs), wanna_output): np.testing.assert_allclose(buf, want, atol=atol, rtol=rtol) # Check correctness of handcoded optimiztions. - reset_bufs(outbufs) - run_prg(opts=None) - for buf,want in zip(copyout_outputs(outbufs), wanna_output): np.testing.assert_allclose(buf, want, atol=atol, rtol=rtol) + if check_default_opt: check_opt(None) for x in opts: # Check custom transformations if any. check_opt(([Opt(OptOps.TC, 0, (TC_SELECT.value, TC_OPT.value, 1))] if apply_tc else [])+x) diff --git a/test/opt/test_tensor_cores.py b/test/opt/test_tensor_cores.py index c25eec469aa15..0d9b5db3d1f31 100644 --- a/test/opt/test_tensor_cores.py +++ b/test/opt/test_tensor_cores.py @@ -79,7 +79,8 @@ class TestTensorCores(unittest.TestCase): @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") def test_tensor_cores(self): for tc in Device[Device.DEFAULT].renderer.tensor_cores: - helper_tc_allclose(tc.dims[0], tc.dims[1], tc.dims[2], tc.dtype_in, tc.dtype_out, axis=0, tc_opt=0) + with self.subTest(tc=tc): + helper_tc_allclose(tc.dims[0], tc.dims[1], tc.dims[2], tc.dtype_in, tc.dtype_out, axis=0, tc_opt=0) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") def test_tensor_cores_nested_reduce(self): @@ -185,10 +186,10 @@ def test_tensor_cores_unroll_phi(self): # skip fp8 tcs: the unoptimized ALU baseline quantizes products to fp8 (JAX promotion), which legitimately # differs from the MFMA path (f32 accumulation), so the baseline-vs-TC numerical gate can't hold for fp8. tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in not in dtypes.fp8s) - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) + ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: assert u.src[-1].src[0].op != Ops.STORE @@ -199,10 +200,10 @@ def test_tensor_cores_unroll_phi(self): @unittest.skipIf(Device.DEFAULT in {"CPU"}, "CPU does not support using a different type for accumulation") def test_tensor_cores_unroll_casted_phi(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) + ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) @@ -215,10 +216,10 @@ def test_tensor_cores_unroll_casted_phi(self): def test_tensor_cores_unroll_casted_phi_with_children(self): # all STORE children are outside the loop tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] - x, y = Tensor.rand(64, 64, dtype=tc.dtype_in), Tensor.rand(64, 64, dtype=tc.dtype_in) + x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out).relu() opts = [Opt(OptOps.UNROLL, 0, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3) + ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) From 22722ea2e7d7cff1db906369ec9de8c5c9e0320f Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:01:28 +0800 Subject: [PATCH 003/306] llama: correct optim_dtype for mxfp4 (#17482) --- .../llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh | 2 +- .../llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh index db5daa5f2feb1..b472e3093987a 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh @@ -26,7 +26,7 @@ export FUSED_SILU_W13=${FUSED_SILU_W13:-1} export SPLIT_W13=${SPLIT_W13:-0} export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-0} -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=${DP:-8} MP=${MP:-1} BS=${BS:-16} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2} export GBS=$((BS * GRADIENT_ACC_STEPS)) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index 0bfe3ae827ce8..bb53cc0ded6cb 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -26,7 +26,7 @@ export FUSED_SILU_W13=${FUSED_SILU_W13:-1} export SPLIT_W13=${SPLIT_W13:-0} export OFFLOAD_OPTIM=${OFFLOAD_OPTIM:-0} -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=${DP:-8} MP=${MP:-1} BS=${BS:-16} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2} export GBS=$((BS * GRADIENT_ACC_STEPS)) From 7edf80a48abc3956c47be4ab9525f711d17a889f Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:58:49 -0700 Subject: [PATCH 004/306] small changes from new rangeify + remove flaky tc tests (#17486) * small changes from new rangeify * remove test/opt/test_tensor_cores.py --- .github/workflows/test.yml | 2 +- test/helpers.py | 4 ++-- tinygrad/uop/symbolic.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bd3063e41bc1..7fe9471032f38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -591,7 +591,7 @@ jobs: if: ${{ matrix.backend == 'amd' && matrix.arch == 'gfx950' }} run: PYTHONPATH=. DEV=NULL:HIP:gfx950 MXFP4=1 LLAMA_LAYERS=2 BENCHMARK=3 NULL_ALLOW_COPYOUT=1 NO_HIPCC=1 ROCM_PATH=/opt/rocm JITBEAM=0 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/profile.sh - name: Run pytest (amd) - run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM test/opt/test_tensor_cores.py --durations=20 + run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM --durations=20 - name: Run disk copy tests run: python -m pytest test/unit/test_disk_tensor.py -k test_copy_from_disk - name: Run TRANSCENDENTAL math diff --git a/test/helpers.py b/test/helpers.py index ea900d505c074..21febed6bc366 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -48,6 +48,8 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te else: assert isinstance(t, UOp), f"can't schedule {t}" linear, var_vals = Tensor(t).linear_with_vars() + # test compiling the linear + compile_linear(linear) kernel_cnt = sum((len(call.device) if isinstance(call.device, tuple) else 1) for call in linear.src if call.src[0].op is Ops.SINK or not filter_sink) if kernel_cnt != allowed: @@ -57,8 +59,6 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te print("kernel", i+1) print(call.src[0]) raise KernelCountException(allowed, kernel_cnt) - # test compiling the linear - compile_linear(linear) return linear, var_vals def assert_kernel_count(expected:int): diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 1eda875236084..f3d78ebfb5d66 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -293,8 +293,8 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: (UPat(Ops.AFTER, name="x"), lambda x: x.replace(src=(x.src[0],)+ tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.FUNCTION, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} else y.src for y in x.src[1:]]))))), - # after with 1 src is just src[0] - (UPat(Ops.AFTER, src=(UPat.var("s"),)), lambda s: s), + # after/end with 1 src is just src[0] + (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), ])+div_and_mod_symbolic # ******** we take a small aside to "simplify_valid" to rewrite valids ******** From 115bf9940fa3236a2949c101e95959cbb3f37e2e Mon Sep 17 00:00:00 2001 From: Raine Date: Mon, 10 Aug 2026 21:04:38 -0300 Subject: [PATCH 005/306] add kwargs to group (#17484) --- tinygrad/uop/ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 9ae599b987e32..723e15fdffeab 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -568,9 +568,9 @@ def gettuple(self, idx:int) -> UOp: in_tuple = self.src[0] if self.op is Ops.FUNCTION else self assert in_tuple.op is Ops.TUPLE, f"gettuple requires FUNCTION or TUPLE source, got {self.op}" return UOp(Ops.GETTUPLE, src=(self,), arg=idx) - def group(*srcs:UOp|None): # pylint: disable=no-self-argument + def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] - return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None])) + return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None]), **kwargs) def index(self, *srcs:UOp|int|None, **kwargs): new_srcs: list[UOp] = [UOp.const(x) if isinstance(x, int) else x for x in srcs if x is not None] if len(new_srcs) == 1 and new_srcs[0].op is Ops.CONST and self.op is Ops.STACK: return self.src[new_srcs[0].val] From ad2fdeae696a38bd8b865563a30919ef5384af15 Mon Sep 17 00:00:00 2001 From: Raine Date: Mon, 10 Aug 2026 21:05:21 -0300 Subject: [PATCH 006/306] move WMMA pms to codegen (#17485) * move wmma pms to codegen * lint tabs --- tinygrad/codegen/opt/tc.py | 37 ++++++++++++++++++++++++++++++++++ tinygrad/renderer/llvmir.py | 40 +++---------------------------------- 2 files changed, 40 insertions(+), 37 deletions(-) diff --git a/tinygrad/codegen/opt/tc.py b/tinygrad/codegen/opt/tc.py index 52df9ce5601f4..6d13624a1dae2 100644 --- a/tinygrad/codegen/opt/tc.py +++ b/tinygrad/codegen/opt/tc.py @@ -1,6 +1,7 @@ import math, functools from dataclasses import dataclass from tinygrad.dtype import DType, dtypes +from tinygrad.uop.ops import PatternMatcher, UOp, UPat, Ops @dataclass(frozen=True) class TensorCore: # D = A * B + C, A is (M x K), B is (K x N), C and D are (M x N) @@ -135,6 +136,42 @@ def get_cuda(arch): return cuda_sm89 if (ver:=int(arch[3:])) >= 89 else cuda_sm8 def get_amd(arch): return {"gfx942": amd_cdna3, "gfx950": amd_cdna4, "gfx1200": amd_rdna4, "gfx1201": amd_rdna4}.get(arch, amd_rdna3) +pm_validate_wmma_rdna3 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( + src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) + if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) + for j in range(x.max_numel()*2)))), + arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) + for i in range(x.max_numel()))) if x.max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x"), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), +]) + +pm_validate_wmma_rdna4 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( + dtype=dtypes.uint16, + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) + .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) +]) + +pm_validate_wmma_cdna = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), +]) # ***** Apple Metal ***** metal = [TensorCore(dims=(8,8,8), threads=32, elements_per_thread=(2,2,2), dtype_in=di, dtype_out=do, diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 99da4d1345575..05f6e72033cbb 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -279,43 +279,9 @@ def __init__(self, target:Target): (UPat(Ops.WMMA, name="wmma"), lambda ctx, wmma, rdna4=AMDLLVMRenderer.is_rdna4(target.arch), cdna=self.is_cdna: render_wmma_amd(ctx, wmma, cdna, rdna4)) ]) - if self.is_cdna: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), - ]) - if target.arch in {"gfx1100", "gfx1151"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( - src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) - if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) - for j in range(x.max_numel()*2)))), - arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) - for i in range(x.max_numel()))) if x.max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x"), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), - ]) - if target.arch in {"gfx1200", "gfx1201"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( - dtype=dtypes.uint16, - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) - .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) - ]) + if self.is_cdna: self.extra_matcher += tc.pm_validate_wmma_cdna + if target.arch in {"gfx1100", "gfx1151"}: self.extra_matcher += tc.pm_validate_wmma_rdna3 + if target.arch in {"gfx1200", "gfx1201"}: self.extra_matcher += tc.pm_validate_wmma_rdna4 def supported_dtypes(self): return {d for d in super().supported_dtypes() if (d not in dtypes.fp8_ocp or self.target.arch == "gfx950") and d not in dtypes.fp8_fnuz} From 0c6a2c7dd611e03543cdae588e4bb07dd2bf7ce7 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 10 Aug 2026 20:37:17 -0700 Subject: [PATCH 007/306] slice is just shrink (#17483) --- tinygrad/engine/realize.py | 2 +- tinygrad/runtime/graph/metal.py | 2 +- tinygrad/runtime/support/hcq2.py | 2 +- tinygrad/tensor.py | 52 +++++++++++++++++--------------- tinygrad/uop/ops.py | 42 ++++++++++++++------------ 5 files changed, 53 insertions(+), 47 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 92006e87d6f20..e91e83ea75793 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -140,7 +140,7 @@ class ExecContext: cache: bool = True def _resolve(b:UOp, inputs:tuple[UOp, ...]) -> UOp: - if b.op in (Ops.SLICE, Ops.MSELECT) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) + if b.op in (Ops.SLICE, Ops.MSELECT, Ops.SHRINK) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) if b.op is Ops.MSTACK: return b.replace(src=tuple(_resolve(x, inputs) for x in b.src)) return inputs[b.arg.slot] if b.op is Ops.PARAM else b def resolve_params(call:UOp, inputs:tuple[UOp, ...]) -> list[UOp]: return [_resolve(b, inputs) for b in get_call_arg_uops(call)] diff --git a/tinygrad/runtime/graph/metal.py b/tinygrad/runtime/graph/metal.py index 409cfb973dac4..8ad6152c4ff89 100644 --- a/tinygrad/runtime/graph/metal.py +++ b/tinygrad/runtime/graph/metal.py @@ -113,5 +113,5 @@ def __del__(self): @staticmethod def supports_uop(batch_devs, new_call:UOp) -> bool: # Metal ICB replay encodes offsets as uint32; reject if any Metal buffer offset exceeds 32-bit range. - if any(b.op is Ops.SLICE and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False + if any(b.op in {Ops.SLICE, Ops.SHRINK} and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False return GraphRunner.supports_uop(batch_devs, new_call) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 9b0372d0354ed..767291e4f8894 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -383,7 +383,7 @@ def resolve_getaddr_slice(bv:UOp, g:UOp) -> UOp: return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64) pm_early_simplify = PatternMatcher([ - (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat(Ops.SLICE, name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), + (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat((Ops.SLICE, Ops.SHRINK), name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), (UPat(Ops.INDEX, src=(UPat(Ops.SLICE, name="bv"),), allow_any_len=True, name="x"), lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))), ]) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 304d1fe1a3481..c2b654a7afe27 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -23,6 +23,7 @@ class AllocCtx: bases: set[UOp] = field(default_factory=set) assigns: list[UOp] = field(default_factory=list) replacements: list[UOp] = field(default_factory=list) + views: set[UOp] = field(default_factory=set) def tag_uop(ctx:AllocCtx, x:UOp): if x.tag is not None: return None @@ -63,40 +64,37 @@ def replace_contig_with_store_after(u:UOp): def replace_store_after_with_contig(u:UOp, src:UOp): assigned_to = u while assigned_to.op in {Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: assigned_to = assigned_to.src[0].base - if assigned_to.op not in {Ops.BUFFER, Ops.SLICE}: return src.contiguous(tag=u.tag) + if assigned_to.op is not Ops.BUFFER: return src.contiguous(tag=u.tag) def _make_buffer_view(src:UOp) -> UOp|None: - """If movement ops on src collapse to a contiguous range, return SLICE. Otherwise None.""" - if (offset := src.contiguous_view_offset()) is None: return None + if (cv := src.contiguous_view()) is None: return None + (buf, offset), size = cv, src.max_numel() * src.element_size() // cv[0].element_size() + if buf.op is not Ops.BUFFER: return None + # NB: make offset a UOp.variable here to do the offset computation in the kernels + return buf[offset:offset+size].bitcast(src.dtype) + +def contiguous_mops_to_view(ctx:AllocCtx, c:UOp, src:UOp): + """MOPS(BUFFER) → SHRINK when movement ops collapse to a contiguous range.""" buf = src.base - if buf.op is Ops.SLICE: - byte_offset = buf.src[1].val * buf.src[0].dtype.itemsize + offset * src.dtype.itemsize - buf = buf.src[0] - if byte_offset % buf.dtype.itemsize != 0: return None - offset = byte_offset // buf.dtype.itemsize - return UOp(Ops.SLICE, src.dtype, (buf, UOp.const(offset)), src.numel()) - -def contiguous_mops_to_view(c:UOp, src:UOp): - """MOPS(BUFFER) → SLICE when movement ops collapse to a contiguous range.""" - buf = src.base - if buf.op not in {Ops.BUFFER, Ops.SLICE, Ops.UNSHARD}: return None - if src.op is Ops.RESHAPE and src.src[0].op in {Ops.BUFFER, Ops.SLICE} and c.op is not Ops.BITCAST: return None - if c.op is not Ops.BITCAST and src.op is Ops.BUFFER: return None + while buf.op is Ops.BITCAST: buf = buf.src[0].base + if buf.op not in {Ops.BUFFER, Ops.UNSHARD}: return None # no symbolic shape if not all_int(c.shape): return None if buf.op is not Ops.UNSHARD and (view := _make_buffer_view(src)) is not None: - view = (view.replace(dtype=c.dtype, arg=c.numel()) if c.op is Ops.BITCAST else view).reshape(c.shape) - return c.replace(src=(view,)) if c.op is Ops.COPY else view + ctx.views.add(view) + view = view.reshape(c.shape) + return c.replace(src=(view,)+c.src[1:]) if c.op in {Ops.COPY, Ops.STORE} else view - # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then create SLICE on the resolved result + # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then create SHRINK on the resolved result if not isinstance(c.device, str): from tinygrad.schedule.multi import multi_pm resolved = graph_rewrite(src, multi_pm, name="multi_buffer_view") if resolved.op is not Ops.UNSHARD: return None if (view := _make_buffer_view(resolved.src[0])) is None: return None - return view.reshape(resolved.src[0].shape).unshard(resolved.arg, resolved.src[1:]).contiguous(tag=c.tag) + ctx.views.add(view) + return view.reshape(resolved.src[0].shape).unshard(resolved.arg, resolved.src[1:]) return None @@ -151,8 +149,9 @@ def transform_precompiled_call(c:UOp) -> UOp|None: # resolve TUPLE+GETTUPLE (for precompiled calls) (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - # fold MOPS+BITCAST over BUFFER/SLICE into SLICE when movement ops collapse to contiguous range - (UPat((Ops.BITCAST, Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BUFFER}, name="src"),), name="c"), contiguous_mops_to_view), + # fold MOPS+BITCAST over BUFFER into SHRINK when movement ops collapse to contiguous range + (UPat((Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BITCAST}, name="src"),), name="c"), contiguous_mops_to_view), + (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, name="src"), UPat()), name="c", allow_any_len=True), contiguous_mops_to_view), # remove contiguous on movement ops before a copy on disk (UPat(GroupOp.Movement-{Ops.SHRINK, Ops.RESHAPE}, name="x").f(Ops.CONTIGUOUS).f(Ops.COPY, name="copy"), lambda x,copy: @@ -201,6 +200,8 @@ def replace_input_buffer(ctx:AllocCtx, b:UOp): return UOp.param(len(ctx.replacements)-1, b.dtype, b.shape, b.device, addrspace=b.addrspace if b.addrspace is not None else AddrSpace.GLOBAL) +def replace_input_view(ctx:AllocCtx, b:UOp): return replace_input_buffer(ctx, b) if b in ctx.views else None + pm_finalize_call = PatternMatcher([ (UPat(Ops.AFTER, name="x"), finalize_after), (UPat(Ops.COPY, name="x"), lambda ctx,x: ctx.assigns.append(x) if isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS")) else None), @@ -210,8 +211,9 @@ def replace_input_buffer(ctx:AllocCtx, b:UOp): # replace BUFFER with PARAM for cache key normalization (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), - # replace SLICE with PARAM. this rewrite is bottom up so BUFFERs we don't need won't be in the input - (UPat(Ops.SLICE, src=(UPat(Ops.BUFFER), UPat(Ops.CONST, dtype=dtypes.weakint)), name="b"), replace_input_buffer), + # replace SHRINK with PARAM + (UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), name="b", allow_any_len=True), replace_input_view), + (UPat(Ops.BITCAST, src=(UPat.any(UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), allow_any_len=True), UPat(Ops.BUFFER)),), name="b"), replace_input_view), # strip value from BIND for cache key normalization, so different values hit same cache (UPat(Ops.BIND, src=(UPat(Ops.PARAM), UPat(Ops.CONST)), name="b"), replace_input_buffer), ]) @@ -229,7 +231,7 @@ def transform_to_call(big_sink:UOp) -> tuple[UOp, dict[UOp, UOp]]: big_sink = graph_rewrite(big_sink, add_tags, ctx=ctx, bottom_up=True, name="number the uops") # here we can break the tensor graph. this is the only place you need to maintain numbered tags - big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, name="early transform tensor graph") + big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, ctx=ctx, name="early transform tensor graph") # here we construct the final buffer_map: as-built nodes -> their final storage. values are never keys graph_rewrite(big_sink, pm_finalize_call, ctx=ctx, name="finalize call") diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 723e15fdffeab..28e2a047bafb8 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -822,7 +822,7 @@ def _mop(self, op:Ops, arg) -> UOp: unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self + if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.SHRINK, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): @@ -901,8 +901,7 @@ def buf_uop(self) -> UOp: while len(s.src) and s.op not in {Ops.BUFFER, Ops.PARAM, Ops.STAGE, Ops.MSTACK}: s = s.src[0] return s - def contiguous_view_offset(self) -> int|None: - """If movement ops on a BUFFER collapse to a contiguous range, return `offset` in elements. Otherwise None.""" + def contiguous_view(self) -> tuple[UOp, int]|None: from tinygrad.schedule.rangeify import pm_mops from tinygrad.uop.symbolic import symbolic @@ -915,7 +914,10 @@ def contiguous_view_offset(self) -> int|None: idx = self.flatten().index(UOp.range(self.numel(), 0)) out = graph_rewrite(idx, pm_mops+symbolic+pm_contiguous_view_offset, ctx=self, name="contiguous_view_offset") - return out.val if out.op is Ops.CONST and isinstance(out.val, int) else None + if out.op is not Ops.INDEX or not (b:=out.src[0]).tag or (c:=out.src[1]).op is not Ops.CONST or not isinstance(c.val, int): return None + return b.rtag(None), c.val + + def contiguous_view_offset(self) -> int|None: return None if (view := self.contiguous_view()) is None else view[1] def has_buffer_identity(self, after_ok=False): """Check if this UOp has a concrete buffer identity in the graph (RESHAPE/UNSHARD -> BUFFER chain).""" @@ -932,18 +934,16 @@ def _base_buffer_is_realized(self) -> bool: @property def buffer(self) -> Buffer|MultiBuffer: - if self.op in {Ops.CONTIGUOUS, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer + if self.op in {Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer # this buffer can process disk tensors and simple movement ops - if self is not self.base: - buf = self.base.buffer - assert isinstance(buf, Buffer), "must be a Buffer for movement ops" - offset = self.contiguous_view_offset() - if offset is None: raise RuntimeError(f"non-contiguous view is not supported for {buf.device} buffer") - return buf.view(prod(self.max_shape), self.dtype, offset*self.dtype.itemsize) - if self.op is Ops.BITCAST: - buf = self.src[0].buffer - assert isinstance(buf, Buffer), "must be a Buffer for BITCAST" - return buf.view(prod(self.max_shape), self.dtype, 0) + if self is not self.base or self.op is Ops.BITCAST: + if (cv := self.contiguous_view()) is None: raise RuntimeError(f"non-contiguous view is not supported for {self.device} buffer") + buf, offset = (b:=cv[0]).base.buffer, cv[1] + if isinstance(buf, MultiBuffer): + mbuf = MultiBuffer.__new__(MultiBuffer) + mbuf.bufs = [x.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) for x in buf.bufs] + return mbuf + return buf.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) if self.op is Ops.SLICE: if (cret:=buffers.get(self)) is not None: return cret buf = self.src[0].buffer @@ -1775,10 +1775,14 @@ def do_unbind(ctx:dict[Variable, int], x:UOp): # ctx is source UOp for which we are finding a contiguous view for. used in contiguous_view_offset pm_contiguous_view_offset = PatternMatcher([ - (UPat(Ops.INDEX, src=(UPat(),)), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE))), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda c: c), - (UPat(Ops.INDEX, src=(UPat(), UPat.cvar('c'))), lambda ctx, c: c if resolve(ctx.numel() == 1, False) else None), + # normalize to 1d bitcasts + (UPat(Ops.BITCAST, name="b"), lambda b: b.src[0].flatten().bitcast(b.dtype).reshape(b.shape) if len(b.shape) != 1 else None), + (UPat(Ops.BITCAST, name="b").index(UPat.cvar("c")), lambda ctx, b, c: + b.src[0].flatten().index(UOp.range(ctx.numel() * (osz:=b.element_size())//(isz:=b.src[0].element_size()), 0) + (c * osz//isz)) if b.tag else None), + (UPat(Ops.INDEX, src=(UPat.var("b"),)), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE))), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c) if resolve(ctx.numel() == 1, False) else None), ]) # *** what was symbolic.py *** From 55e4f9d4f3561ba73482142f54d92de9d5f69ff2 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:28 +0300 Subject: [PATCH 008/306] hcq2: proper unmap (#17489) --- extra/hcq2/ops_amd2.py | 20 ++++++++++++++++---- tinygrad/runtime/support/hcq2.py | 4 +++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index fbdef9f4444ed..6d69671d813f0 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -297,6 +297,8 @@ def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque) def _do_map(self, buf:HCQ2Buffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _do_unmap(self, buf:HCQ2Buffer): self.dev.iface.unmap(buf) + @dataclass class AMDQueueDesc: ring: Buffer; read_ptr: Buffer; write_ptr: Buffer; doorbell: Buffer; put_value: Buffer # noqa: E702 @@ -388,15 +390,24 @@ def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguo return hcqbuf def free(self, mem): + self._unmap(mem) + if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) + kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def unmap(self, mem): + self._unmap(mem) + if getattr(mem, '_owns_kfd_handle', False): kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def _unmap(self, mem): gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(gpus), n_devices=1) assert stm.n_success == 1 - if mem.owner == self.dev: - if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) - kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) def map(self, mem): - if mem.owner is not None and mem.owner._is_cpu(): return self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + if mem.owner is not None and mem.owner._is_cpu(): + mapped = self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + mapped._owns_kfd_handle = True + return mapped c_gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(c_gpus), n_devices=1) @@ -468,6 +479,7 @@ def p2p_paddrs(self, paddrs:list[tuple[int,int]]) -> tuple[list[tuple[int,int]], def require_profile_mode(self): return True def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return True # TODO: account for WGP disablement on some asics. + def unmap(self, mem): self.free(mem) def _compute_props(self): self.ip_versions = self.dev_impl.ip_ver diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 767291e4f8894..2464928ca56ea 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -623,6 +623,8 @@ def _map(self, buf:HCQ2Buffer) -> HCQ2Buffer: if not hasattr(self, '_do_map'): raise NotImplementedError("map failed: no method implemented") return self._do_map(buf) + def _do_unmap(self, mb): self.dev.iface.free(mb) + @suppress_finalizing def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None): if options is not None and options.external_ptr is not None: return @@ -631,6 +633,6 @@ def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None): def _unmap(self, mb): self.dev.synchronize() - self.dev.iface.free(mb) + self._do_unmap(mb) def _offset(self, buf, size:int, offset:int) -> HCQ2Buffer: return buf.offset(offset=offset, size=size) From ffef35c53e259e111b00991c658e114d6c218722 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:22:43 +0300 Subject: [PATCH 009/306] hcq2: fix deps (#17481) * hcq2: proper unmap * hcq2: fix deps * x * x --- .github/workflows/test.yml | 7 ++++--- tinygrad/runtime/support/hcq2.py | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7fe9471032f38..907d73a2fa49d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -522,9 +522,10 @@ jobs: run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/test_tiny.py - name: Run HCQ2 multi-device tests run: | - HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_multitensor.py \ - TestMultiTensor.test_simple_add TestMultiTensor.test_shard_reduce \ - TestMultiTensor.test_backward_sum TestMultiTensor.test_matmul_shard_0_0 + HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest -n=auto test/backend/test_multitensor.py \ + -k "not (multitensor_jit_input or multitensor_inside_jit or four_add or elementwise_dtype or stack or \ + test_2d_shard_basic or test_2d_shard_elementwise or test_2d_shard_sum_non_sharded_axis or test_2d_shard_matmul or test_numpy or \ + data_parallel_simple_train_step or multi_tensor_jit_graph_assign_updates_each_shard or test_transformer or test_shrink_2d)" - name: Run HCQ2 JIT tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_jit.py - name: Run HCQ2 unit tests diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 2464928ca56ea..dd045c5e19e16 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -141,14 +141,14 @@ def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int] # opt2: keep latest dep per (dep device, queue, cur lane) latest = {((dep[0][dlane], dep[1]), lane): (dep, dlane) for dep, dlane, lane in sorted(dep_lanes, key=lambda x: x[0][2])} - deps:dict[tuple, list[int|None]] = collections.defaultdict(lambda: [None]*len(devices)) - for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane] = dlane + deps:dict[tuple, dict[int, list[int]]] = collections.defaultdict(lambda: collections.defaultdict(list)) + for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane].append(dlane) waits = [] - for (ddevs, dqueue, dtag), lanes in deps.items(): - sig = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) - for dl, d in zip(lanes, devices)]) - waits.append(UOp(Ops.INS, arg="wait", src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) + for (ddevs, dqueue, dtag), by_lane in deps.items(): + for ls in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): + s = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) for dl, d in zip(ls, devices)]) + waits.append(UOp(Ops.INS, arg="wait", src=(s, UOp.const(dtag + 1, dtypes.uint64)))) return waits, {dtag for _, _, dtag in deps} def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[tuple[tuple[str, ...], str]], From a8c84ab34ee6e61e947ec1f07f4fb8c8cca79ed7 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:47:33 +0300 Subject: [PATCH 010/306] hcq2: enable all multitesnor tests (#17490) --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 907d73a2fa49d..e39096664749d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -521,11 +521,7 @@ jobs: - name: Run HCQ2 tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/test_tiny.py - name: Run HCQ2 multi-device tests - run: | - HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest -n=auto test/backend/test_multitensor.py \ - -k "not (multitensor_jit_input or multitensor_inside_jit or four_add or elementwise_dtype or stack or \ - test_2d_shard_basic or test_2d_shard_elementwise or test_2d_shard_sum_non_sharded_axis or test_2d_shard_matmul or test_numpy or \ - data_parallel_simple_train_step or multi_tensor_jit_graph_assign_updates_each_shard or test_transformer or test_shrink_2d)" + run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest -n=auto test/backend/test_multitensor.py - name: Run HCQ2 JIT tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_jit.py - name: Run HCQ2 unit tests From e11df72e0f05b0ba724c36733cfbaead01d81ca3 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:22:11 -0700 Subject: [PATCH 011/306] notes from digitalocean_mi350x (#17494) * notes from digitalocean_mi350x * cleanup * revert non-doc changes on digitalocean_mi350x branch --- extra/runbook_digitalocean_mi350x.md | 235 +++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 extra/runbook_digitalocean_mi350x.md diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md new file mode 100644 index 0000000000000..b76e9c77de038 --- /dev/null +++ b/extra/runbook_digitalocean_mi350x.md @@ -0,0 +1,235 @@ +# Runbook: Llama 3 8B Training on DigitalOcean MI350X + +## Machine Specs +- 8x MI350X GPUs (gfx950, device ID 75b0), 288GB VRAM each +- 2TB RAM, 192 CPUs, 2TB disk +- ROCm 7.14 at `/opt/rocm` (NOT `/opt/rocm-7.1.1` like the submission scripts assume) +- Python 3.12 + +## Phase 1: System Setup + +### 1.1 Install packages +```bash +apt-get update +apt-get install -y python3-pip python3-venv git tmux rclone clang +``` + +### 1.2 Install Python deps +```bash +python3 -m pip install --break-system-packages numpy tqdm wandb tiktoken sentencepiece +``` + +### 1.3 Install ROCm dev headers +The base image has ROCm runtime but NOT the HIP dev headers. Need: +```bash +apt-get install -y amdrocm-core-dev +``` +This installs `hip/hip_runtime.h` at `/opt/rocm/core-7.14/include/hip/hip_runtime.h`. +The symlink `/opt/rocm/include` → `/opt/rocm/core-7.14/include` makes it available at `/opt/rocm/include/hip/hip_runtime.h`. + +### 1.4 Configure ROCm comgr +ROCm 7.14 ships comgr 3.3 at `/opt/rocm/lib/libamd_comgr.so`. tinygrad's DLL loader needs explicit env vars to find it (it searches for `libcomgr.so*` by default, not `libamd_comgr.so*`). Set these in the run command: +```bash +export COMGR_PATH=/opt/rocm/lib/libamd_comgr.so +export COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so +``` +Also add ROCm libs to ldconfig so comgr's shared library dependencies resolve: +```bash +cat > /etc/ld.so.conf.d/rocm.conf << 'EOF' +/opt/rocm/lib +/opt/rocm/lib/llvm/lib +/opt/rocm/lib/rocm_sysdeps/lib +EOF +ldconfig +``` + +### 1.5 Install geohot tmux config +```bash +curl -sL https://raw.githubusercontent.com/geohot/configuration/master/.tmux.conf -o ~/.tmux.conf +``` + +### 1.6 Reload amdgpu driver +tinygrad's HCQ backend needs `/dev/kfd` which is created by the amdgpu kernel driver. +If the driver was unloaded, reload it: +```bash +modprobe amdgpu +ls /dev/kfd # should exist +``` + +## Phase 2: Clone tinygrad +```bash +cd /root +git clone https://github.com/tinygrad/tinygrad.git +cd tinygrad +python3 -m pip install --break-system-packages -e . +``` + +## Phase 3: Download C4 Dataset + +The C4 data is on the MLCommons Cloudflare R2 bucket in Megatron-LM indexed format. + +```bash +rclone config create mlc-training s3 provider=Cloudflare \ + access_key_id=76ea42eadb867e854061a1806220ee1e \ + secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 \ + endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com + +mkdir -p /root/datasets/c4-8b +rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /root/datasets/c4-8b/ -P +``` + +Files downloaded (~85GB total, ~6 minutes): +- `c4-train.en_6_text_document.bin` (79 GB) +- `c4-train.en_6_text_document.idx` (870 MB) +- `c4-validation-91205-samples.en_text_document.bin` (159 MB) +- `c4-validation-91205-samples.en_text_document.idx` (1.8 MB) +- `LICENSE.txt`, `NOTICE.txt` + +### Symlink for the submission script +The `dev_run.sh` script hardcodes `BASEDIR="/raid/datasets/c4-8b/"`. Symlink: +```bash +mkdir -p /raid/datasets +ln -s /root/datasets/c4-8b /raid/datasets/c4-8b +``` + +## Phase 4: wandb Login +```bash +wandb login +``` +Enter API key from https://wandb.ai/authorize + +## Phase 5: Run Training + +### 5.1 Smoke test (beam search, 2 layers, fake data) +Always run beam first to validate the pipeline: +```bash +cd /root/tinygrad +COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ +COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ +CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ +DEV=AMD:HIP \ +ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ + bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +``` + +### 5.2 Full training run +```bash +cd /root/tinygrad +COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ +COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ +CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ +DEV=AMD:HIP \ +ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ +WANDB=1 \ + bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +``` + +## Environment Variable Reference + +| Variable | Value | Why | +|---|---|---| +| `COMGR_PATH` | `/opt/rocm/lib/libamd_comgr.so` | tinygrad's DLL loader needs explicit path to find comgr 3.3 | +| `COMGR_3_PATH` | `/opt/rocm/lib/libamd_comgr.so` | comgr 3.x uses a separate `comgr_3` module with its own path var | +| `CC` | `/opt/rocm/core-7.14/lib/llvm/bin/clang` | System clang doesn't know gfx950; must use ROCm's bundled clang | +| `DEV` | `AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | +| `ROCM_PATH` | `/opt/rocm` | Script defaults to `/opt/rocm-7.1.1` which doesn't exist | +| `BASEDIR` | `/root/datasets/c4-8b/` | Where C4 dataset was downloaded (script hardcodes `/raid/datasets/c4-8b/`) | +| `WANDB` | `1` | Enable wandb logging (off by default) | + +## Architecture + +| Component | Source file | +|---|---| +| Model | `examples/mlperf/models/flat_llama.py` — FlatTransformer, FP8 MXFP4 weights, fused QKV, flash attention | +| Trainer | `examples/mlperf/model_train.py` → `train_llama3()` | +| Optimizer | `examples/mlperf/optim.py` — GradAccClipAdamW, master weights, FP8 re-quant | +| LR schedule | `examples/mlperf/lr_schedulers.py` — CosineAnnealingLRWithWarmup | +| Dataloader | `examples/mlperf/dataloader.py` — Megatron-LM indexed bin format | +| ASM GEMM | `extra/gemm/cdna_asm_gemm.py` — gfx950 MFMA assembly, MXFP4 | +| Flash attention | `extra/thunder/amd/fa.py` | +| Fused kernels | `extra/llama_kernels/` — rmsnorm, silu, quantize, fused_ce | +| GPU driver | `tinygrad/runtime/ops_amd.py` — HCQ, direct KFD ioctl | +| Renderer | `tinygrad/renderer/cstyle.py` — HIPRenderer for gfx950 | +| comgr compiler | `tinygrad/runtime/support/compiler_amd.py` — HIPCompiler using comgr 3.3 | + +## Troubleshooting + +### `'hip/hip_runtime.h' file not found` +Install `amdrocm-core-dev`: +```bash +apt-get install -y amdrocm-core-dev +``` + +### `'gfx950' is not a recognized processor` + LLVM crash +System clang doesn't know gfx950. Set `CC=/opt/rocm/core-7.14/lib/llvm/bin/clang`. + +### `comgr not available: try setting COMGR_PATH?` +Add ROCm libs to ldconfig and set `COMGR_PATH` and `COMGR_3_PATH`: +```bash +# /etc/ld.so.conf.d/rocm.conf should contain /opt/rocm/lib paths +ldconfig +``` + +### `comgr not available: try setting COMGR_3_PATH?` +comgr 3.x uses a separate module. Set `COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so` too. + +### `FileNotFoundError: '/raid/datasets/c4-8b/...'` +Script hardcodes `BASEDIR`. Either symlink or edit the script: +```bash +mkdir -p /raid/datasets && ln -s /root/datasets/c4-8b /raid/datasets/c4-8b +``` + +### `No such file or directory: 'clang'` +Install clang: `apt-get install -y clang` (for CPU compilation). +For gfx950 HIP compilation, comgr (not clang) is used — ensure the ROCm 7.14 comgr 3.3 is properly loaded via `COMGR_PATH` and `COMGR_3_PATH`. + +## Appendix: KVM Virtualization Observations + +### Virtualization detection +``` +$ systemd-detect-virt +kvm +$ lspci -nn | grep AMD +83:00.0 ... Device [1002:75b0] +``` +CPU flags include `hypervisor`. `dmesg` shows `Hypervisor detected: KVM`. + +### PCI device ID +`lspci -v` shows device ID `0x75b0` and subsystem ID `0x75a0`: +``` +83:00.0 Processing accelerators: ... Device 75b0 + Subsystem: ... Device 75a0 +``` +tinygrad's `PCIIface` in `ops_amd.py` and `hive_reset.py` did not list `0x75b0`, so the GPU was not found. Adding `0x75b0` to the device ID list in both files fixes the detection. + +### amdgpu driver behavior +On first boot, amdgpu loaded and bound to all 8 GPUs. On one boot it failed to initialize: +``` +[ 799.780369] amdgpu 0000:83:00.0: Failed to alloc msi vectors +[ 799.781476] amdgpu 0000:83:00.0: sw_init of IP block failed -22 +[ 799.782724] amdgpu 0000:83:00.0: amdgpu_device_ip_init failed +[ 799.793885] amdgpu 0000:83:00.0: Fatal error during GPU init +``` +On a subsequent boot, amdgpu initialized successfully (SMU initialized, VRAM ready). After unbinding all 8 GPUs from amdgpu, `rmmod amdgpu` wedged the module (stuck in "Unloading" state in `/proc/modules`), requiring a full VM reboot. + +### `/dev/kfd` +`/dev/kfd` exists when amdgpu is loaded. Opening it returns `OSError: [Errno 22] Invalid argument`. + +### VRAM BAR reads all 0xFF +After amdgpu initializes the GPU and is then unbound, reading the VRAM BAR (via `/sys/bus/pci/devices/0000:83:00.0/resource0`) returns all `0xFF` at all offsets — including the discovery table at `vram_size - 64KB`. tinygrad's `AMDev._run_discovery()` fails with `AssertionError: discovery signatures mismatch`. + +A PCI reset (`echo 1 > /sys/bus/pci/devices/0000:83:00.0/reset`) did not change the VRAM contents — still all `0xFF`. + +VRAM was also all `0xFF` when read via `/dev/mem` at the BAR physical address (`0xa0000000000`). + +### VFIO attempt +Bound the GPU to `vfio-pci` with `enable_unsafe_noiommu_mode=1`. The GPU bound successfully and `/dev/vfio/noiommu-0` appeared. Running tinygrad with `VFIO=1` still failed with the same `discovery signatures mismatch` — VRAM BAR still reads all `0xFF`. + +### No IOMMU in guest +`dmesg` has no `AMD-Vi` entries. PCI devices have no `iommu_group` symlink. + +### No fan control +No `fan*` or `pwm*` hwmon entries exist. Only `temp*`, `power*`, `freq*` are exposed. GPU temps read 56-63°C, power ~265W per GPU. + +### Current status: NOT WORKING +tinygrad's `PCIIface` finds the GPU (after adding `0x75b0`) but `AMDev._run_discovery()` fails because the VRAM discovery table reads all `0xFF`. This was observed with the GPU unbound from any driver, after PCI reset, and with VFIO bound. From 2b5018e86aac548e2ed5d5578855c1b7102fc47b Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:25:44 +0300 Subject: [PATCH 012/306] hcq2: fix debug 2 info (#17491) * hcq2: fix debug 2 info * x * x * x --- tinygrad/engine/realize.py | 35 +++++++++++++++++--------------- tinygrad/runtime/support/hcq2.py | 14 ++++++------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index e91e83ea75793..2d73ea8bb1f67 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -5,7 +5,7 @@ from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, wait_cond from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, buffers, graph_rewrite -from tinygrad.device import Device, Buffer, MultiBuffer +from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.renderer import Estimates from tinygrad.codegen import to_program from tinygrad.codegen.opt.postrange import args_from_ast @@ -210,27 +210,30 @@ def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: return t[0] def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - if (inputs:=call.arg.aux.inputs) is not None: + if (info:=call.arg.aux).inputs is not None: bufs = [_resolve(ctx.input_uops[i], ctx.input_uops).buffer for i in call.arg.aux.input_idxs] - table = call.src[1+inputs].buffer + table = call.src[1+info.inputs].buffer for j,dev in enumerate(call.arg.aux.device): addrs = array.array('Q', [(b.bufs[j] if isinstance(b, MultiBuffer) else b).get_buf(dev).va_addr for b in bufs]) mv = (table.bufs[j] if isinstance(table, MultiBuffer) else table).ensure_allocated()._buf.cpu_view().view(fmt='Q') wait_cond(lambda: mv[0], value=0, timeout_ms=ctx.timeout or getenv("HCQDEV_WAIT_TIMEOUT_MS", 30000), msg=f"{dev} hang detected") mv[:len(addrs)] = addrs - exec_kernel(replace(ctx, update_stats=False), call, ast) - - tms:list[float|None] = [] - for e in (aux:=call.arg.aux).prof: cast(Any, Device[e.device]).prof_ents[e.st_id] = e - for d in [cast(Any, Device[x]) for x in aux.device]: - with track_stats(ctx, call, d.device, [], ctx.var_vals) as et: - if ctx.wait: - d.synchronize(timeout=ctx.timeout) - ts = [d.signal(i)._buf.cpu_view().view(fmt='Q')[0] for e in aux.prof if e.device == d.device for i in (e.st_id, e.en_id)] - if ts: et[0] = float(max(ts)-min(ts))/d.timestamp_divider/1e6 - tms += et - return tms[0] + exec_kernel(replace(ctx, update_stats=DEBUG>=3), call, ast) + + tms = [] + for devices,name,estimates,prof in info.kernels: + for device in devices: + d, tm = cast(Any, Device[device]), None + if prof: + d.prof_ents[prof[0]] = ProfileGraphEntry(device, name, *prof) + if ctx.wait: + d.synchronize(timeout=ctx.timeout) + st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) + tms.append(tm:=float(en-st)/d.timestamp_divider/1e6) + with track_stats(ctx, call.replace(arg=replace(call.arg, name=name, aux=replace(info, estimates=estimates))), d.device, [], ctx.var_vals) as et: + et[0] = tm + return max(tms) if tms else None # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src pm_flatten_linear = PatternMatcher([ @@ -276,7 +279,7 @@ def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:li if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) linear = graph_rewrite(linear, pm_compile, name="precompile kernels", walk=True) - if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE) if profile is None else profile) + if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) return graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if getenv("HCQ2") else linear diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index dd045c5e19e16..b3bd74f8366ca 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -33,7 +33,7 @@ class HCQInfo: input_idxs:tuple[int, ...] = () # indexes into input_uops used by this call inputs:int|None = None - prof:tuple[ProfileGraphEntry, ...] = () # st_id/en_id are timestamp signal slots until collect + kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...]], ...] = () def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c @@ -199,10 +199,10 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li signal_tags |= cur_signal_tags # build fences and finalizers - fences, finalizers, finalizer_signal_tags = _build_finalizers(batch, batch_info, deps_tracker, slots) + fences, fins, finalizer_signal_tags = _build_finalizers(batch, batch_info, deps_tracker, slots) signal_tags |= finalizer_signal_tags - src, prof = [], [] + src, kerns = [], [] for tag, ((call, _), (devices, queue), q) in enumerate(zip(batch, batch_info, call_waits)): # first queue use, sync prior device work with the device timeline if batch_info.index((devices, queue)) == tag: @@ -212,18 +212,18 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] - prof += [ProfileGraphEntry(d, name, *ts_ids) for d in devices if ts_ids] + kerns.append((devices, name, info.estimates, tuple(ts_ids))) ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_signal(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] # signal the queue if someone waits for us if tag in signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_signal(devices, slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] - src.append(make_call(name, make_submit(*q, devs=devices, queue=queue).sink(), info)) + src.append(make_call(f"submit {name}", make_submit(*q, devs=devices, queue=queue).sink(), info)) # append batch timestamps to finalizers - finalizers = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, prof=tuple(e for e in prof if e.device in a.device)))) for f in finalizers] - return fences + src + finalizers + fins = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, kernels=tuple(x for x in kerns if set(x[0]) & set(a.device))))) for f in fins] + return fences + src + fins def sched_hcq_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] From 479ffb0cda3c4616be7d5e38a9ba5fb1b94d1aa4 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 11 Aug 2026 15:50:04 -0700 Subject: [PATCH 013/306] remove Ops.SLICE (#17492) --- test/null/test_memory_planner.py | 11 +++++--- test/unit/test_metal_graph.py | 44 ++++++++++++-------------------- tinygrad/engine/jit.py | 4 +-- tinygrad/engine/realize.py | 17 +++--------- tinygrad/runtime/graph/metal.py | 3 ++- tinygrad/runtime/support/hcq2.py | 15 ++++++----- tinygrad/schedule/indexing.py | 10 +++----- tinygrad/schedule/memory.py | 6 ++--- tinygrad/uop/__init__.py | 2 +- tinygrad/uop/ops.py | 33 ++++++------------------ tinygrad/uop/spec.py | 7 ----- tinygrad/viz/serve.py | 2 +- 12 files changed, 54 insertions(+), 100 deletions(-) diff --git a/test/null/test_memory_planner.py b/test/null/test_memory_planner.py index 63be203320a1d..c879630fac35c 100644 --- a/test/null/test_memory_planner.py +++ b/test/null/test_memory_planner.py @@ -27,10 +27,15 @@ def _make_linear(buffer_lists, copies=None): calls.append(UOp(Ops.CALL, src=(src0, *bufs))) return UOp(Ops.LINEAR, src=tuple(calls)) +def _get_planned_view(buf:UOp) -> tuple[UOp, int, int]|None: + view = buf.src[0] if buf.op is Ops.BITCAST else buf + if view.op is not Ops.SHRINK or view.src[0].op is not Ops.BUFFER: return None + return (arena:=view.src[0]), view.src[1].val * arena.dtype.itemsize, view.src[2].val * arena.dtype.itemsize + def _get_arena(buf, linear, result): for orig_si, new_si in zip(linear.src, result.src): for orig, new in zip(orig_si.src[1:], new_si.src[1:]): - if orig is buf and new.op is Ops.SLICE: return new.src[0] + if orig is buf and (planned:=_get_planned_view(new)) is not None: return planned[0] return None def check_assign(buffer_lists, copies=None): @@ -41,8 +46,8 @@ def check_assign(buffer_lists, copies=None): replace_map: dict[int, tuple[UOp, int, int]] = {} for orig_si, new_si in zip(linear.src, result.src): for orig, new in zip(orig_si.src[1:], new_si.src[1:]): - if new.op is Ops.SLICE and id(orig) not in replace_map: - replace_map[id(orig)] = (new.src[0], new.src[1].val * new.src[0].dtype.itemsize, new.arg * new.dtype.itemsize) + if (planned:=_get_planned_view(new)) is not None and id(orig) not in replace_map: + replace_map[id(orig)] = planned # verify pinned buffers are not planned for buf in held_bufs: diff --git a/test/unit/test_metal_graph.py b/test/unit/test_metal_graph.py index a85616ff7f762..c879a9fc2ff2b 100644 --- a/test/unit/test_metal_graph.py +++ b/test/unit/test_metal_graph.py @@ -1,5 +1,4 @@ import unittest -from unittest.mock import MagicMock from tinygrad import Device from tinygrad.uop.ops import Ops, UOp from tinygrad.dtype import dtypes @@ -11,36 +10,27 @@ def setUp(self): self.MetalGraph = MetalGraph self.dev = Device[Device.DEFAULT] - def metal_buf(self, offset): - buf = MagicMock() - if offset > 0: - buf.op = Ops.SLICE - src = MagicMock() - src.dtype = dtypes.uint8 - buf.src = (src, UOp.const(offset)) - buf.dtype = dtypes.uint8 - else: - buf.op = Ops.BUFFER - buf.device = Device.DEFAULT - return buf - - def call(self, *bufs): - c = MagicMock() - c.src = (MagicMock(op=Ops.PROGRAM),) + tuple(bufs) - return c + def metal_buf(self, offset, bitcast=False): + size = 4 if bitcast else 1 + buf = UOp.new_buffer(Device.DEFAULT, offset+size, dtypes.uint8) + if offset: buf = buf[offset:offset+size] + return buf.bitcast(dtypes.float32) if bitcast else buf + + def supports_uop(self, *bufs): + return self.MetalGraph.supports_uop([self.dev], UOp(Ops.PROGRAM, src=(UOp.sink(),)).call(*bufs)) def test_supports_uop_normal_offset(self): - assert self.MetalGraph.supports_uop([self.dev], self.call(self.metal_buf(0), self.metal_buf(100), self.metal_buf(0xFFFFFFFF))) is True + assert self.supports_uop(self.metal_buf(0), self.metal_buf(100), self.metal_buf(0xFFFFFFFF)) is True def test_supports_uop_overflow_offset(self): - assert self.MetalGraph.supports_uop([self.dev], self.call(self.metal_buf(0), self.metal_buf(0x100000000))) is False - - def test_supports_uop_nonmetal_buf(self): - # non-SLICE ops should not be checked for offset - buf = MagicMock() - buf.op = Ops.BUFFER - buf.device = Device.DEFAULT - self.MetalGraph.supports_uop([self.dev], self.call(buf)) + assert self.supports_uop(self.metal_buf(0), self.metal_buf(0x100000000)) is False + + def test_supports_uop_non_view_buf(self): + assert self.supports_uop(self.metal_buf(0)) is True + + def test_supports_uop_bitcast(self): + assert self.supports_uop(self.metal_buf(0xFFFFFFFF, bitcast=True)) is True + assert self.supports_uop(self.metal_buf(0x100000000, bitcast=True)) is False if __name__ == "__main__": unittest.main() diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index fe11805c17151..d67197c98b9b6 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -44,8 +44,6 @@ def flush_batch(): current_batch, current_batch_devs = [], [] for si in linear.src: - if si.src[0].op is Ops.SLICE: continue - devs = dedup([Device[x] for b in si.src[1:] if b.op is not Ops.BIND for x in (b.device if isinstance(b.device, tuple) else (b.device,))]) graph_t = graph_class(devs[0]) if devs[0].graph is not None else None @@ -180,7 +178,7 @@ def _written_uops(self) -> set[UOp]: if call.op is not Ops.CALL: continue arg_uops = get_call_arg_uops(call) outs, ins = get_call_outs_ins(call) - out |= {arg_uops[k] for k in set(outs) - set(ins) if arg_uops[k].op in (Ops.BUFFER, Ops.SLICE)} + out |= {b for k in set(outs) - set(ins) if (b:=u if (cv:=(u:=arg_uops[k]).contiguous_view()) is None else cv[0]).op is Ops.BUFFER} return out def __call__(self, input_uops:list[UOp], var_vals:dict[str, int]) -> ReturnType: diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 2d73ea8bb1f67..8723d0de81fb7 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, replace, field from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, wait_cond -from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, buffers, graph_rewrite +from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.renderer import Estimates from tinygrad.codegen import to_program @@ -17,7 +17,7 @@ def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.src[0] if ast.op is Ops.PROGRAM: return tuple(ast.arg.outs), tuple(ast.arg.ins) - if ast.op in (Ops.COPY, Ops.SLICE): return (0,), (1,) + if ast.op is Ops.COPY: return (0,), (1,) if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return (0,), tuple(range(1, len(get_call_arg_uops(call)))) return (), () @@ -27,9 +27,6 @@ def _dev_str(buf:Buffer|UOp) -> str: return ', '.join(d[:7] for d in to_tuple(bu ast, arg_uops = call.src[0], get_call_arg_uops(call) if ast.op is Ops.PROGRAM: return ast.arg.name - if ast.op is Ops.SLICE: - offset = ast.src[1].val * arg_uops[1].dtype.itemsize - return colored(f"view {_uop_sz_to_str(arg_uops[0]):>10} @ {offset:<10d}", "yellow") if ast.op is Ops.COPY: return colored(f"copy {_uop_sz_to_str(arg_uops[0]):>10}, {_dev_str(bufs[0]):>7s} <- {_dev_str(bufs[1]):7s}", "yellow") if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return colored(f"enc/dec {_uop_sz_to_str(arg_uops[0])}", "yellow") if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return colored(f"batched {len(ast.src[0].src)}", "cyan") @@ -140,7 +137,7 @@ class ExecContext: cache: bool = True def _resolve(b:UOp, inputs:tuple[UOp, ...]) -> UOp: - if b.op in (Ops.SLICE, Ops.MSELECT, Ops.SHRINK) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) + if b.op in (Ops.MSELECT, Ops.SHRINK) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) if b.op is Ops.MSTACK: return b.replace(src=tuple(_resolve(x, inputs) for x in b.src)) return inputs[b.arg.slot] if b.op is Ops.PARAM else b def resolve_params(call:UOp, inputs:tuple[UOp, ...]) -> list[UOp]: return [_resolve(b, inputs) for b in get_call_arg_uops(call)] @@ -154,13 +151,6 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d for x in call.src[0].toposort()) for j, per_dev in enumerate(zip(*[cast(MultiBuffer, b).bufs for b in bufs])): yield list(per_dev), {"_device_num": j} if has_dnum else {} -def exec_view(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - resolved = resolve_params(call, ctx.input_uops) - bufs = [cast(Buffer, b.buffer) for b in resolved] - bv = bufs[1].view(resolved[0].max_numel(), ast.dtype, ast.src[1].val*bufs[1].dtype.itemsize) - with track_stats(ctx, call, bv.device, [bv, bufs[1]], ctx.var_vals): buffers[resolved[0]] = bv - return None - def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): dest, src = bufs[0].ensure_allocated(), bufs[1].ensure_allocated() @@ -264,7 +254,6 @@ def _validate(call:UOp, sink:UOp) -> UOp: ]) pm_exec = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.SLICE, name="ast"),), name="call", allow_any_len=True), exec_view), (UPat(Ops.CALL, src=(UPat(Ops.COPY, name="ast"),), name="call", allow_any_len=True), exec_copy), (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="ast"),), name="call", allow_any_len=True), exec_kernel), (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="encdec", name="ast"),), name="call", allow_any_len=True), exec_encdec), diff --git a/tinygrad/runtime/graph/metal.py b/tinygrad/runtime/graph/metal.py index 8ad6152c4ff89..d38e43043277d 100644 --- a/tinygrad/runtime/graph/metal.py +++ b/tinygrad/runtime/graph/metal.py @@ -113,5 +113,6 @@ def __del__(self): @staticmethod def supports_uop(batch_devs, new_call:UOp) -> bool: # Metal ICB replay encodes offsets as uint32; reject if any Metal buffer offset exceeds 32-bit range. - if any(b.op in {Ops.SLICE, Ops.SHRINK} and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False + for shrink in [s for src in new_call.src[1:] if (s:=src.src[0] if src.op is Ops.BITCAST else src).op is Ops.SHRINK]: + if shrink.src[1].val * shrink.src[0].dtype.itemsize > 0xFFFFFFFF: return False return GraphRunner.supports_uop(batch_devs, new_call) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index b3bd74f8366ca..bdb03c7bb09b8 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -39,7 +39,7 @@ def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for def unwrap_mstack(u): if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s)) - return unwrap_mstack(u.src[0]) if u.op in {Ops.MSELECT, Ops.SLICE} else (u,) + return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,) def is_value_known_at_link(val:UOp) -> bool: runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)] @@ -377,14 +377,15 @@ def replace_params(call:UOp) -> UOp|None: # ***************** -def resolve_getaddr_slice(bv:UOp, g:UOp) -> UOp: +def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp: base = bv.src[0].after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()) - itemsize = bv.src[0].dtype.itemsize if bv.src[0].without_after.op in (Ops.BUFFER, Ops.SLICE, Ops.MSTACK, Ops.MSELECT) else bv.dtype.itemsize + if bv.op is Ops.BITCAST: return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + itemsize = bv.src[0].dtype.itemsize if bv.src[0].without_after.op in (Ops.BUFFER, Ops.MSTACK, Ops.MSELECT) else bv.dtype.itemsize return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64) pm_early_simplify = PatternMatcher([ - (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat((Ops.SLICE, Ops.SHRINK), name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), - (UPat(Ops.INDEX, src=(UPat(Ops.SLICE, name="bv"),), allow_any_len=True, name="x"), + (UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv").or_after(),), name="g"), resolve_getaddr_view), + (UPat(Ops.INDEX, src=(UPat(Ops.SHRINK, name="bv"),), allow_any_len=True, name="x"), lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))), ]) @@ -402,7 +403,7 @@ def pack_hcq_placeholders(call:UOp) -> UOp|None: sizes[b.tag] = offs[b] + b.max_numel() counts = collections.Counter(b.tag for b in bufs) bases = {b.tag:UOp.placeholder((sizes[b.tag],), b.dtype, next(UOp.unique_num), device=b.device).rtag(b.tag) for b in bufs if counts[b.tag] > 1} - subs = {b:UOp(Ops.SLICE, b.dtype, (bases[b.tag], UOp.const(offs.get(b, 0))), b.max_numel()) for b in bufs if b.tag in bases} + subs = {b:bases[b.tag][(off:=offs.get(b, 0)):off+b.max_numel()] for b in bufs if b.tag in bases} return call.replace(src=(call.src[0].substitute(subs, walk=True), *call.src[1:])) if subs else None pm_pack_placeholders = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), pack_hcq_placeholders)]) @@ -493,7 +494,7 @@ def resolve_getaddr(buf:UOp, g:UOp) -> UOp: (UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True) .store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()).index(UPat(Ops.RANGE), allow_any_len=True).load()) .end(UPat(Ops.RANGE)), fold_binary), - (UPat({Ops.BUFFER, Ops.SLICE, Ops.MSTACK}, name="buf").index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), + (UPat({Ops.BUFFER, Ops.MSTACK}, name="buf").index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), ]) pm_assert_no_afters = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: panic(RuntimeError, f"AFTER left at hcq_link: {a.src[0].op}"))]) diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index b084b1fb51fe7..dcaa3f7f78e29 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -23,7 +23,7 @@ def new_range(self, s:sint, axistype:AxisType=AxisType.WEAK) -> UOp: return UOp.range(s, next(self.range_idx), axistype) if resolve(s!=1) else UOp.const(0) -ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, Ops.SLICE, +ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, Ops.CONST, Ops.BIND, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, Ops.LOAD, Ops.CALL, Ops.FUNCTION} @@ -34,10 +34,6 @@ def realize_srcs(ctx:IndexingContext, rb:UOp) -> None: if s.base.op not in ALWAYS_CONTIGUOUS: ctx.realize_map[s] = None def realize_store_after_src(ctx:IndexingContext, dest:UOp, src:UOp): - # don't realize SLICE when it's the direct source of STORE+AFTER — the target buffer is the output - if src.op is Ops.SLICE and src in ctx.realize_map \ - and not dest.op_in_backward_slice_with_self(Ops.SHRINK, Ops.PERMUTE, Ops.FLIP, Ops.PAD): - del ctx.realize_map[src] # you don't usually have to do this for assign unless there's a WAR hazard like TestAssign.test_assign_double_diamond_reduce if dest.base in src.backward_slice_with_self: ctx.realize_map[src] = None @@ -74,7 +70,7 @@ def broadcast_rngs(x:UOp, src:UOp, rngs:tuple[UOp, ...]) -> tuple[UOp, ...]: # TODO: srcs contain (real data srcs, something else, ranges) and the boundary is confusing. see range_start def data_srcs(op:Ops, src:tuple[UOp, ...]) -> tuple[UOp, ...]: if op in {Ops.PARAM, Ops.BUFFER, Ops.RANGE, Ops.SPECIAL, Ops.BIND}: return () - if op in GroupOp.Movement|{Ops.INDEX, Ops.SLICE, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END}: return src[:1] + if op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END}: return src[:1] return src def create_bufferize_and_index_srcs(ctx:IndexingContext, x:UOp) -> list[UOp]: @@ -84,7 +80,7 @@ def create_bufferize_and_index_srcs(ctx:IndexingContext, x:UOp) -> list[UOp]: for i, s in enumerate(x.src): new_src = s src_rngs = broadcast_rngs(x, s, ctx.range_map[x][0]) if x in ctx.range_map else () - if s.op in {Ops.PARAM, Ops.BUFFER, Ops.SLICE, Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: + if s.op in {Ops.PARAM, Ops.BUFFER, Ops.MSTACK, Ops.MSELECT, Ops.AFTER}: if x in ctx.range_map and i < data_src_count: new_src = new_src.index(*src_rngs) elif s in ctx.realize_map: realized_ranges = ctx.realize_map[s] diff --git a/tinygrad/schedule/memory.py b/tinygrad/schedule/memory.py index a16acb09f95ce..e0883b4c91e17 100644 --- a/tinygrad/schedule/memory.py +++ b/tinygrad/schedule/memory.py @@ -52,11 +52,9 @@ def _key(b:UOp): return (b.device, 1 if b in copy_bufs else 0) peaks[_key(buf)] = (max(peaks[_key(buf)][0], offsets[buf] + buf.max_numel() * buf.dtype.itemsize), peaks[_key(buf)][1]) arena_sizes = {key: round_up(peak, block_size) for key, (peak, _) in peaks.items()} - # build replace_map: each buffer becomes a SLICE into a shared per-device-lane arena + # build replace_map: each buffer becomes a SHRINK/BITCAST into a shared per-device-lane arena arenas = {key: UOp.new_buffer(key[0], sz, dtypes.int8) for key, sz in arena_sizes.items()} - replace_map:dict[UOp, UOp] = {} - for buf_uop, offset in offsets.items(): - replace_map[buf_uop] = UOp(Ops.SLICE, buf_uop.dtype, (arenas[_key(buf_uop)], UOp.const(offset)), buf_uop.max_numel()) + replace_map = {buf_uop:arenas[_key(buf_uop)][offset:offset+buf_uop.nbytes()].bitcast(buf_uop.dtype) for buf_uop, offset in offsets.items()} if DEBUG >= 1 and (omem:=sum(nbytes.values()) / 1e6) != (nmem:=sum(arena_sizes.values()) / 1e6): print(f"memory reduced from {omem:.2f} MB -> {nmem:.2f} MB, {len(first_appearance)} -> {len(arenas)} bufs") diff --git a/tinygrad/uop/__init__.py b/tinygrad/uop/__init__.py index e39fc8b876cec..2252c8b926578 100644 --- a/tinygrad/uop/__init__.py +++ b/tinygrad/uop/__init__.py @@ -93,7 +93,7 @@ class Ops(FastEnum): CONTIGUOUS = auto(); CONTIGUOUS_BACKWARD = auto(); DETACH = auto() # buffer ops - STAGE = auto(); COPY = auto(); SLICE = auto(); MSELECT = auto(); MSTACK = auto(); CUSTOM_FUNCTION = auto() + STAGE = auto(); COPY = auto(); MSELECT = auto(); MSTACK = auto(); CUSTOM_FUNCTION = auto() # the core 6 movement ops! these only exist in the tensor graph RESHAPE = auto(); PERMUTE = auto(); EXPAND = auto(); PAD = auto(); FLIP = auto() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 28e2a047bafb8..0350b103f282a 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -45,8 +45,7 @@ def __repr__(self): axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.THREAD: 0, AxisType.GLOBAL: 0, AxisType.WARP: 1, AxisType.LOCAL: 2, AxisType.UPCAST: 3, AxisType.GROUP_REDUCE: 2, AxisType.REDUCE: 4, AxisType.UNROLL: 5} -range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.FUNCTION: 1, - Ops.SLICE: 2, Ops.LINEAR: 0} +range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.FUNCTION: 1, Ops.LINEAR: 0} # https://en.wikipedia.org/wiki/Identity_element def identity_element(op:Ops, dt:DType) -> PyConst: return dt.const({Ops.ADD:0, Ops.MUL:1, Ops.MAX:dt.min}[op]) @@ -171,9 +170,6 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: return arg.dtype case Ops.BINARY: return dtypes.uint8 - case Ops.SLICE: - # TODO: slice just shouldn't exist - return None case Ops.CAST | Ops.BITCAST: assert isinstance(arg, DType), f"CAST/BITCAST arg must be DType, got {arg}" return arg @@ -221,7 +217,7 @@ def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg return created # some uops map to other stuff -buffers:weakref.WeakKeyDictionary[UOp, Buffer|MultiBuffer] = weakref.WeakKeyDictionary() # this maps BUFFER/SLICE uops to their device Buffers +buffers:weakref.WeakKeyDictionary[UOp, Buffer|MultiBuffer] = weakref.WeakKeyDictionary() # this maps BUFFER/view uops to their device Buffers all_metadata:weakref.WeakKeyDictionary[UOp, tuple[Metadata, ...]] = weakref.WeakKeyDictionary() # TODO: should this be here? # recursive_property replaces functools.cached_property in recursive UOp functions to prevent RecursionError @@ -386,10 +382,6 @@ def _shape(self) -> tuple[sint, ...]|None: case Ops.BUFFER: if len(self.src): return self.src[0].as_shape return () - case Ops.SLICE: - # HACK: SLICE is used inside kernels, so we set the shape to () if it's on an INDEX - if self.src[0].op is Ops.INDEX: return () - return (self.arg,) case Ops.CUSTOM | Ops.CUSTOMI: if self.dtype is dtypes.void: return None input_shapes = [x._shape for x in self.src if x._shape is not None] @@ -822,7 +814,7 @@ def _mop(self, op:Ops, arg) -> UOp: unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.SHRINK, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self + if self.without_after.op not in {Ops.BUFFER, Ops.SHRINK, Ops.BITCAST, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): @@ -924,7 +916,7 @@ def has_buffer_identity(self, after_ok=False): # TODO: this is confusing because UOp.variable('v', 0, 1, dtypes.weakfloat) is True for jit to work, but it doesn't have a buffer if self.op in {Ops.RESHAPE, Ops.UNSHARD, Ops.MSELECT}: return self.src[0].has_buffer_identity(after_ok) if after_ok and self.op == Ops.AFTER: return self.src[0].has_buffer_identity(after_ok) - return self.op in {Ops.BUFFER, Ops.SLICE, Ops.PARAM} + return self.op in {Ops.BUFFER, Ops.PARAM} def _base_buffer_is_realized(self) -> bool: """Walk through AFTER chain to find if the underlying buffer is realized (has allocated memory).""" @@ -937,25 +929,16 @@ def buffer(self) -> Buffer|MultiBuffer: if self.op in {Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer # this buffer can process disk tensors and simple movement ops if self is not self.base or self.op is Ops.BITCAST: + if (cret:=buffers.get(self)) is not None: return cret if (cv := self.contiguous_view()) is None: raise RuntimeError(f"non-contiguous view is not supported for {self.device} buffer") buf, offset = (b:=cv[0]).base.buffer, cv[1] if isinstance(buf, MultiBuffer): mbuf = MultiBuffer.__new__(MultiBuffer) mbuf.bufs = [x.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) for x in buf.bufs] - return mbuf - return buf.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) - if self.op is Ops.SLICE: - if (cret:=buffers.get(self)) is not None: return cret - buf = self.src[0].buffer - offset = self.src[1].val - if isinstance(buf, MultiBuffer): - mbuf = MultiBuffer.__new__(MultiBuffer) - mbuf.bufs = [b.view(self.arg, self.dtype, offset * self.src[0].dtype.itemsize) for b in buf.bufs] buffers[self] = mbuf return mbuf - assert isinstance(buf, Buffer), "must be a Buffer for SLICE" - buffers[self] = bv = buf.view(self.arg, self.dtype, offset * self.src[0].dtype.itemsize) - return bv + buffers[self] = buf.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) + return buffers[self] if self.op is Ops.MSELECT: ret = self.src[0].buffer assert isinstance(ret, MultiBuffer) @@ -1181,7 +1164,7 @@ def param_like(self, slot:int): def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, src=src, arg=name) # opaque bodies stay as Ops.CALL; value-producing bodies become Ops.FUNCTION (wrapped in TUPLE) - _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.SLICE, Ops.CUSTOM_FUNCTION} + _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.CUSTOM_FUNCTION} def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None, name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: if ret_dtype is not None: return UOp(Ops.CALL, ret_dtype, src=(self,)+srcs) diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 654f491cd7727..4c19457aaf954 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -233,13 +233,6 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < spec_full = PatternMatcher([ (UPat(Ops.REWRITE_ERROR, dtypes.void, name="x"), lambda x: isinstance(x.arg, str)), - # SLICE on BUFFER is allowed if BUFFER is - (UPat(Ops.SLICE, src=(UPat(GroupOp.Movement.union({Ops.BUFFER, Ops.PARAM, Ops.STAGE, Ops.AFTER})), - UPat(Ops.CONST, dtype=dtypes.weakint)), allow_any_len=True, name="bv"), - lambda bv: isinstance(bv.arg, int)), - - (UPat(Ops.CALL, dtypes.void, src=(UPat((Ops.SLICE,)),), allow_any_len=True), lambda: True), - # codegen may end ranges after gpudims has replaced RANGE with SPECIAL. (UPat(Ops.END, src=(UPat(), UPat()), allow_any_len=True), lambda: True), diff --git a/tinygrad/viz/serve.py b/tinygrad/viz/serve.py index 506fcac00c06e..86ba657d7b1d9 100755 --- a/tinygrad/viz/serve.py +++ b/tinygrad/viz/serve.py @@ -50,7 +50,7 @@ def stream_json(self, source:Generator): Ops.INDEX: "#CEF9B7", Ops.STACK: "#D8F9E4", Ops.WMMA: "#efefc0", Ops.UNSHARD: "#f6ccff", Ops.INS: "#eec4ff", **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", - Ops.SLICE: "#E5EAFF", Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", + Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", Ops.CALL: "#00B7C8", Ops.FUNCTION: "#C07788", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", Ops.LINEAR: "#7DF4FF", Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D", From 4a253db9b473366913e01c90d191f2ea4c9765b5 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:06:25 -0700 Subject: [PATCH 014/306] minor cleanups to improve import speed (#17495) * minor cleanups to improve import speed * dumb --- tinygrad/helpers.py | 16 ++++++++-------- tinygrad/nn/state.py | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index fab620deaaea9..66014d594cb37 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -1,9 +1,9 @@ from __future__ import annotations import time START_TIME = time.perf_counter() -import os, functools, platform, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc +import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc from collections import defaultdict -import subprocess, shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib +import shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib from dataclasses import dataclass, field, replace from typing import ClassVar, Iterable, Any, TypeVar, Callable, Sequence, TypeGuard, Iterator, Generic, Generator, cast, overload @@ -13,8 +13,7 @@ def prod(x:Iterable[T]) -> T|int: return functools.reduce(operator.mul, x, 1) # NOTE: helpers is not allowed to import from anything else in tinygrad -OSX, WIN = platform.system() == "Darwin", sys.platform == "win32" -ARCH_X86 = any(x in platform.processor() for x in ("Intel", "i386", "x86_64")) +OSX, WIN = sys.platform == "darwin", sys.platform == "win32" BASEDIR = pathlib.Path(__file__).parent # fix colors on Windows, https://stackoverflow.com/questions/12492810/python-how-can-i-make-the-ansi-escape-codes-to-work-also-in-windows @@ -231,7 +230,7 @@ def target(self, dev:str, **kwargs) -> Target: DEV, DEBUG, BEAM, NOOPT = _DEV("DEV", ""), ContextVar("DEBUG", 0), ContextVar("BEAM", 0), ContextVar("NOOPT", 0) IMAGE, FLOAT16, OPENPILOT_HACKS = ContextVar("IMAGE", 0), ContextVar("FLOAT16", 0), ContextVar("OPENPILOT_HACKS", 0) -JIT, JIT_BATCH_SIZE = ContextVar("JIT", 2 if OSX and ARCH_X86 else 1), ContextVar("JIT_BATCH_SIZE", 32) +JIT, JIT_BATCH_SIZE = ContextVar("JIT", 1), ContextVar("JIT_BATCH_SIZE", 32) CHUNK_SIZE = 2**20 # TinyFS content-addressed store: blob chunk + hash-tree node granularity WINO, CAPTURING, TRACEMETA, NO_COLOR = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("TRACEMETA", 1), ContextVar("NO_COLOR", 0) TRAINING = ContextVar("TRAINING", 0) @@ -454,9 +453,9 @@ def _ensure_downloads_dir() -> pathlib.Path: if pathlib.Path("/etc/tinybox-release").is_file(): # try creating dir with sudo if not (downloads_dir := pathlib.Path("/raid/downloads")).exists(): - subprocess.run(["sudo", "mkdir", "-p", downloads_dir], check=True) - subprocess.run(["sudo", "chown", "tiny:root", downloads_dir], check=True) - subprocess.run(["sudo", "chmod", "775", downloads_dir], check=True) + system(f"sudo mkdir -p {downloads_dir}") + system(f"sudo chown tiny:root {downloads_dir}") + system(f"sudo chmod 775 {downloads_dir}") return downloads_dir return pathlib.Path(cache_dir) / "downloads" @@ -497,6 +496,7 @@ def fetch_fw(path:str, name:str, sha256:str) -> bytes: # *** Exec helpers def system(cmd:str, **kwargs) -> str: + import subprocess st = time.perf_counter() try: ret = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, **kwargs).decode().strip() except subprocess.CalledProcessError as e: diff --git a/tinygrad/nn/state.py b/tinygrad/nn/state.py index 17ffc538dca79..ac18bc038cc12 100644 --- a/tinygrad/nn/state.py +++ b/tinygrad/nn/state.py @@ -1,4 +1,4 @@ -import json, math, pathlib, zipfile, pickle, tarfile, struct, functools, io, zlib +import json, math, pathlib, struct, functools, io, zlib from collections import OrderedDict from typing import Any, Callable, BinaryIO, Iterable, cast from tinygrad.tensor import Tensor @@ -219,6 +219,7 @@ def __init__(self): @accept_filename def zip_extract(t: Tensor) -> dict[str, Tensor]: + import zipfile files: dict[str, Tensor] = {} with zipfile.ZipFile(TensorIO(t), "r") as myzip: # sadly, the extra length needs to be read from the local header of each file. @@ -249,6 +250,7 @@ def tar_extract(t: Tensor) -> dict[str, Tensor]: tensors = nn.state.tar_extract(Tensor(pathlib.Path("archive.tar"))) ``` """ + import tarfile with tarfile.open(fileobj=TensorIO(t), mode="r") as tar: return {member.name:t[member.offset_data:member.offset_data+member.size] for member in tar if member.type == tarfile.REGTYPE} @@ -303,6 +305,7 @@ def __setstate__(self, state): self.tensor = state[0] "FloatTensor": None, "Parameter": Parameter} whitelist = {"torch", "collections", "numpy", "_codecs"} # NOTE: this is not for security, only speed class Dummy: pass + import pickle, zipfile, tarfile class TorchPickle(pickle.Unpickler): def find_class(self, module, name): module_root = module.split(".")[0] From 3686a1758f3b853e6e348ff76639b7c216379838 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:23:11 -0700 Subject: [PATCH 015/306] mac/rdma imports lazy (#17496) * mac/rdma imports lazy * ish * fixes --- tinygrad/runtime/autogen/__init__.py | 5 +++-- tinygrad/runtime/autogen/mesa.py | 2 +- tinygrad/runtime/graph/hcq.py | 4 ++-- tinygrad/runtime/ops_cpu.py | 2 +- tinygrad/runtime/support/system.py | 6 ++++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tinygrad/runtime/autogen/__init__.py b/tinygrad/runtime/autogen/__init__.py index 3a1920662facf..8462337be3b39 100644 --- a/tinygrad/runtime/autogen/__init__.py +++ b/tinygrad/runtime/autogen/__init__.py @@ -1,4 +1,4 @@ -import glob, importlib, os, pathlib, shutil, subprocess, tarfile, tempfile +import glob, importlib, os, pathlib, subprocess from tinygrad.helpers import fetch, flatten, system, getenv root = (here:=pathlib.Path(__file__).parent).parents[2] @@ -31,6 +31,7 @@ def load(name, files, **kwargs): if not (f:=(root/(path:=kwargs.pop("path", __name__)).replace('.','/')/f"{name}.py")).exists() or getenv('REGEN'): files, kwargs['args'] = files() if callable(files) else files, args() if callable(args:=kwargs.get('args', [])) else args if (srcs:=kwargs.pop('srcs', None)): + import tempfile, tarfile srcpath = (td:=tempfile.TemporaryDirectory(f"autogen-src-{name.replace('/','-')}")).name + "/" for src in (srcs if isinstance(srcs, list) else [srcs]): if 'tar' in src: @@ -157,7 +158,7 @@ def __getattr__(nm): *[f"python3 src/compiler/nir/nir_{s}_h.py --outdir gen" for s in ["intrinsics", "intrinsics_indices"]]]), cwd=path, shell=True, check=True), srcs="https://gitlab.freedesktop.org/mesa/mesa/-/archive/mesa-25.2.7/mesa-25.2.7.tar.gz", dll=f"'tinymesa_cpu' if DEV.renderer == 'LVP' else 'tinymesa', {tinymesa_path}, emsg='pip install tinymesa==25.2.7.2'", - prolog=["from tinygrad.helpers import DEV", "import gzip, base64, platform, sysconfig, os"], + prolog=["from tinygrad.helpers import DEV", "import gzip, base64, sysconfig, os"], epilog=lambda path: [system(f"{root}/extra/mesa/lvp_nir_options.sh {path}")]) case "libclang": return load("libclang", diff --git a/tinygrad/runtime/autogen/mesa.py b/tinygrad/runtime/autogen/mesa.py index 6b74bf01d8f59..98d7ee2c6abc2 100644 --- a/tinygrad/runtime/autogen/mesa.py +++ b/tinygrad/runtime/autogen/mesa.py @@ -5,7 +5,7 @@ from tinygrad.runtime.support.c import _IO, _IOW, _IOR, _IOWR from tinygrad.runtime.support import c from tinygrad.helpers import DEV -import gzip, base64, platform, sysconfig, os +import gzip, base64, sysconfig, os dll = c.DLL('mesa', 'tinymesa_cpu' if DEV.renderer == 'LVP' else 'tinymesa', os.path.join(sysconfig.get_paths()['platlib'], 'tinymesa'), emsg='pip install tinymesa==25.2.7.2') class struct_u_printf_info(c.Struct): pass u_printf_info: TypeAlias = struct_u_printf_info diff --git a/tinygrad/runtime/graph/hcq.py b/tinygrad/runtime/graph/hcq.py index dbe321159a668..e9d5e73f6275d 100644 --- a/tinygrad/runtime/graph/hcq.py +++ b/tinygrad/runtime/graph/hcq.py @@ -6,7 +6,6 @@ from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops, Variable from tinygrad.engine.jit import GraphRunner, MultiGraphRunner -from tinygrad.runtime.ops_rdma import RDMACopyQueue class HCQGraph(MultiGraphRunner): def __init__(self, *args, **kwargs): @@ -50,7 +49,7 @@ def __init__(self, *args, **kwargs): self.comp_queues: dict[HCQCompiled, HWQueue] = {dev: unwrap(dev.hw_compute_queue_t)() for dev in self.devices} self.copy_queues: dict[tuple[HCQCompiled, int], HWQueue] = {} # lazy allocation, keyed by (device, queue_idx) - self.rdma_queues: dict[tuple[HCQCompiled, HCQCompiled], RDMACopyQueue] = {} # lazy allocation, keyed by device pair + self.rdma_queues: dict[tuple[HCQCompiled, HCQCompiled], "RDMACopyQueue"] = {} # lazy allocation, keyed by device pair self.num_copy_queues: int = getenv("HCQ_NUM_SDMA", min(len(self.devices), 8) if ALL2ALL >= 1 else 1) self.num_rdma_ops: dict[tuple[HCQCompiled, HCQCompiled], int] = collections.defaultdict(int) @@ -104,6 +103,7 @@ def __init__(self, *args, **kwargs): elif is_rdma: enqueue_queue = self.comp_queues[enqueue_dev] rdma_key = (cast(HCQCompiled, Device[bufs[0].device]).rdma_dev(), enqueue_dev.rdma_dev()) + from tinygrad.runtime.ops_rdma import RDMACopyQueue self.rdma_queues.setdefault(rdma_key, RDMACopyQueue(enqueue_dev.rdma_dev())) else: assert (enqueue_dev.hw_copy_queue_t is not None), "device must implement a copy queue" diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 3ecb49a54ecd2..eec7575f7964d 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -1,5 +1,5 @@ from __future__ import annotations -import platform, sys, os, ctypes, functools, mmap, threading, array, itertools +import platform, sys, os, ctypes, ctypes.util, functools, mmap, threading, array, itertools from dataclasses import replace from typing import cast from tinygrad.helpers import to_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, partition diff --git a/tinygrad/runtime/support/system.py b/tinygrad/runtime/support/system.py index 44e560618675a..cf6839503038a 100644 --- a/tinygrad/runtime/support/system.py +++ b/tinygrad/runtime/support/system.py @@ -1,7 +1,8 @@ from __future__ import annotations -import os, mmap, array, functools, ctypes, select, contextlib, dataclasses, sys, itertools, struct, socket, subprocess, time, enum, atexit +import os, mmap, array, functools, ctypes, ctypes.util, select, contextlib, dataclasses, sys, itertools, struct, socket +import subprocess, time, enum, atexit from tinygrad.helpers import round_up, getenv, OSX, temp, ceildiv, unwrap, fetch, system, _ensure_downloads_dir, DEBUG, flatten, pluralize -from tinygrad.runtime.autogen import libc, pci, vfio, iokit, corefoundation +from tinygrad.runtime.autogen import libc, pci, vfio from tinygrad.runtime.support.hcq import FileIOInterface, MMIOInterface, HCQBuffer, hcq_filter_visible_devices from tinygrad.runtime.support.memory import VirtMapping, AddrSpace, BumpAllocator from tinygrad.runtime.support.usb import USB3, CustomASM24Controller, USBMMIOInterface @@ -55,6 +56,7 @@ def system_paddrs(self, vaddr:int, size:int) -> list[int]: def pci_scan_bus(self, vendor:int, devices:tuple[tuple[int, tuple[int, ...]], ...], base_class:int|None=None) -> list[str]: all_devs = [] if OSX: + from tinygrad.runtime.autogen import iokit, corefoundation def read_prop(svc, key) -> int: cfkey = corefoundation.CFStringCreateWithCString(None, key.encode(), corefoundation.kCFStringEncodingUTF8) cfdata = ctypes.cast(iokit.IORegistryEntryCreateCFProperty(svc, ctypes.cast(cfkey, iokit.CFStringRef), None, 0), corefoundation.CFDataRef) From f891f5ffd019bbaac80cc1ad3cf4427cf5bcc7a6 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Wed, 12 Aug 2026 09:32:50 +0800 Subject: [PATCH 016/306] gptoss: route lm_head thru asm_gemm (#17497) --- examples/mlperf/models/gpt_oss.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/mlperf/models/gpt_oss.py b/examples/mlperf/models/gpt_oss.py index 1e157a94da1cd..2fb03ad603d1b 100644 --- a/examples/mlperf/models/gpt_oss.py +++ b/examples/mlperf/models/gpt_oss.py @@ -12,7 +12,7 @@ from tinygrad.uop.ops import Ops, UOp from extra.models.llama import apply_rotary_emb from extra.llama_kernels.rmsnorm import rmsnorm -from extra.gemm.cdna_asm_gemm import _mx_block_scale, _mx_block_scale_3d, quantize_mxfp8 +from extra.gemm.cdna_asm_gemm import _mx_block_scale, _mx_block_scale_3d, quantize_mxfp8, asm_gemm, can_use_asm_gemm from extra.gemm.moe_gemm import grouped_mx_gemm from extra.gemm.moe_routing import route, dispatch, combine @@ -263,7 +263,11 @@ def __call__(self, tokens:Tensor, save:bool=True): w_down=self.w_down[i], w_down_scale=self.w_down_scale[i], w_down_bias=self.w_down_bias[i]) h, *_ = self.run_layer(h, freqs_cis, mask_full, i % 2 == 0, attn_kwargs, ffn_kwargs, save=save) - logits = self.norm(h) @ self.output.T + h_normed = self.norm(h) + pad = (-self.dim) % 256 + h_padded, w_padded = h_normed.pad((None, None, (0, pad))), self.output.pad(((0, 0), (0, pad))) + if ASM_GEMM and can_use_asm_gemm(h_padded, w_padded.T): logits = asm_gemm(h_padded, w_padded.T) + else: logits = h_normed @ self.output.T return logits def _get_pads(uop:UOp) -> list[UOp]: From e1013a63568448213c5df15bf22fc511976912d4 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:03:39 +0800 Subject: [PATCH 017/306] llama: create dataset cache by default in dev_beam (#17500) --- .../llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh index b472e3093987a..570d299ee45a2 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh @@ -46,7 +46,7 @@ export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 -export FAKEDATA=${FAKEDATA:-1} BENCHMARK=${BENCHMARK:-10} +export FAKEDATA=${FAKEDATA:-$([[ "$DEV" == NULL:* ]] && echo 1 || echo 0)} BENCHMARK=${BENCHMARK:-10} if [ -z "$FULL_LAYERS" ]; then export LLAMA_LAYERS=${LLAMA_LAYERS:-2} fi From 2e5a9a4121951bd3722bc8988fb12e314a20c24c Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:19:25 +0800 Subject: [PATCH 018/306] no hardcoded device names in test_sliced_buffer_function (#17501) --- test/backend/test_custom_kernel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/backend/test_custom_kernel.py b/test/backend/test_custom_kernel.py index f92a253a7da93..1c8be94cdfb13 100644 --- a/test/backend/test_custom_kernel.py +++ b/test/backend/test_custom_kernel.py @@ -422,9 +422,8 @@ def run(x:Tensor) -> Tensor: return Tensor.custom_kernel(y, x, fxn=custom_add_one_kernel)[0] GlobalCounters.reset() y = run(x[0]).realize() - # it's copying the input and the output - # TODO: subbuffer usage has runtime specific behavior, this will be fixed after the removal of SLICE. - assert_kernel_count(2 if y.device in ("CL", "WEBGPU") else 1) + # backends that support contiguous views don't launch extra kernels + assert_kernel_count(2 if x[0].uop.contiguous_view() is None else 1) self.assertEqual(y.tolist(), [1, 2, 3, 4]) @Context(DEV="CPU") From 04c271ac414c42a0bbfe58fdcaa4ae0a5bc797a2 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:00:04 +0800 Subject: [PATCH 019/306] simplify digitalocean_mi350x (#17502) * simplify digitalocean_mi350x * no hardcoded rocm path --- .../tinybox_8xMI350X/dev_run.sh | 4 ++-- extra/runbook_digitalocean_mi350x.md | 24 ++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index bb53cc0ded6cb..6ec564f12db30 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash export PYTHONPATH="." -export PATH="/opt/rocm-7.1.1/bin:$PATH" -export ROCM_PATH="/opt/rocm-7.1.1" +export ROCM_PATH=${ROCM_PATH:-/opt/rocm-7.1.1} +export PATH="$ROCM_PATH/bin:$PATH" export DEV=${DEV:-AMD} export CHECK_OOB=0 export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000 diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md index b76e9c77de038..edb09e6f2f45f 100644 --- a/extra/runbook_digitalocean_mi350x.md +++ b/extra/runbook_digitalocean_mi350x.md @@ -74,8 +74,8 @@ rclone config create mlc-training s3 provider=Cloudflare \ secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 \ endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com -mkdir -p /root/datasets/c4-8b -rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /root/datasets/c4-8b/ -P +mkdir -p /raid/datasets/c4-8b +rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /raid/datasets/c4-8b/ -P ``` Files downloaded (~85GB total, ~6 minutes): @@ -85,13 +85,6 @@ Files downloaded (~85GB total, ~6 minutes): - `c4-validation-91205-samples.en_text_document.idx` (1.8 MB) - `LICENSE.txt`, `NOTICE.txt` -### Symlink for the submission script -The `dev_run.sh` script hardcodes `BASEDIR="/raid/datasets/c4-8b/"`. Symlink: -```bash -mkdir -p /raid/datasets -ln -s /root/datasets/c4-8b /raid/datasets/c4-8b -``` - ## Phase 4: wandb Login ```bash wandb login @@ -100,7 +93,7 @@ Enter API key from https://wandb.ai/authorize ## Phase 5: Run Training -### 5.1 Smoke test (beam search, 2 layers, fake data) +### 5.1 Smoke test (beam search, 2 layers, real data) Always run beam first to validate the pipeline: ```bash cd /root/tinygrad @@ -108,7 +101,7 @@ COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ +ROCM_PATH=/opt/rocm \ bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh ``` @@ -119,7 +112,7 @@ COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ +ROCM_PATH=/opt/rocm \ WANDB=1 \ bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh ``` @@ -133,7 +126,6 @@ WANDB=1 \ | `CC` | `/opt/rocm/core-7.14/lib/llvm/bin/clang` | System clang doesn't know gfx950; must use ROCm's bundled clang | | `DEV` | `AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | | `ROCM_PATH` | `/opt/rocm` | Script defaults to `/opt/rocm-7.1.1` which doesn't exist | -| `BASEDIR` | `/root/datasets/c4-8b/` | Where C4 dataset was downloaded (script hardcodes `/raid/datasets/c4-8b/`) | | `WANDB` | `1` | Enable wandb logging (off by default) | ## Architecture @@ -173,12 +165,6 @@ ldconfig ### `comgr not available: try setting COMGR_3_PATH?` comgr 3.x uses a separate module. Set `COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so` too. -### `FileNotFoundError: '/raid/datasets/c4-8b/...'` -Script hardcodes `BASEDIR`. Either symlink or edit the script: -```bash -mkdir -p /raid/datasets && ln -s /root/datasets/c4-8b /raid/datasets/c4-8b -``` - ### `No such file or directory: 'clang'` Install clang: `apt-get install -y clang` (for CPU compilation). For gfx950 HIP compilation, comgr (not clang) is used — ensure the ROCm 7.14 comgr 3.3 is properly loaded via `COMGR_PATH` and `COMGR_3_PATH`. From 4f106ebe877f051846943511c397cef0cfea14a2 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:08:53 +0300 Subject: [PATCH 020/306] hcq2: enqueue speed (#17504) --- tinygrad/engine/realize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 8723d0de81fb7..4313a8b9daec8 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -214,15 +214,15 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: tms = [] for devices,name,estimates,prof in info.kernels: for device in devices: - d, tm = cast(Any, Device[device]), None + tm = None if prof: - d.prof_ents[prof[0]] = ProfileGraphEntry(device, name, *prof) + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, name, *prof) if ctx.wait: d.synchronize(timeout=ctx.timeout) st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) tms.append(tm:=float(en-st)/d.timestamp_divider/1e6) - with track_stats(ctx, call.replace(arg=replace(call.arg, name=name, aux=replace(info, estimates=estimates))), d.device, [], ctx.var_vals) as et: - et[0] = tm + stat_call = call.replace(arg=replace(call.arg, name=name, aux=replace(info, estimates=estimates, kernels=()))) + with track_stats(ctx, stat_call, device, [], ctx.var_vals) as et: et[0] = tm return max(tms) if tms else None # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src From de04781b36d59676d1dad8a19c0f3bc89aaa219e Mon Sep 17 00:00:00 2001 From: Raine Date: Wed, 12 Aug 2026 12:39:29 -0300 Subject: [PATCH 021/306] simplify equivalent const max (#17505) * add const max folds * add regression test * move --- test/null/test_uop_symbolic.py | 5 +++++ tinygrad/uop/symbolic.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 575fe9a55abee..d13afaa2eb98f 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -948,6 +948,11 @@ def test_where_removal(self): self.helper_test_variable(cond.where(u0, u1), 0, 1, "((a<2)!=True)") self.helper_test_variable(cond.where(u0, u1).where(u0, u1), 0, 1, "(a<2)") + def test_equivalent_const_max(self): + x = Variable("x", -10, 10) + self.helper_test_variable((x < 0).where(0, x), 0, 10, "x.maximum(0)") + self.helper_test_variable((0 < x).where(x, 0), 0, 10, "x.maximum(0)") + def test_where_combine(self): cond = Variable("x", 0, 3) < 2 a = Variable("a", 0, 3) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index f3d78ebfb5d66..a2e82fb18099b 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -249,6 +249,8 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), (UPat(Ops.RANGE, src=(UPat(Ops.CONST,)), name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), # max folding + ((UPat.cvar("a") < UPat.var("b")).where(UPat.var("b"), UPat.cvar("c")), lambda a,b,c: UOp.maximum(a,b) if a.val == c.val else None), + ((UPat.var("a") < UPat.cvar("b")).where(UPat.cvar("c"), UPat.var("a")), lambda a,b,c: UOp.maximum(a,b) if b.val == c.val else None), (UPat.maximum(UPat.var("x"), UPat.var("y")), lambda x,y: x if x.vmin >= y.vmax else y if x.vmax <= y.vmin else None), # TODO: why does this rule break beautiful_mnist? #((UPat.var("x")+UPat.var("z")).maximum(UPat.var("y")+UPat.var("z")), lambda x,y,z: x.maximum(y) + z), From ed8297a1027e85fbd0882803155ac22249608b27 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:07:30 +0800 Subject: [PATCH 022/306] kerenl opts test from nan in llama 8b (#17510) * all2all * nan * remove that * less * has_local * only the nan change here * use nice getitem syntax for INDEX * work * remove * even simpler --- test/opt/test_kernel_opts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index af90b69329e9b..0c347bef212f0 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -239,6 +239,15 @@ def test_padto_sum_ok(self): helper_linearizer_opt(a.sum().exp(), [[Opt(OptOps.PADTO, 0, 32)],]) helper_linearizer_opt(a.sum(0).exp(), [[Opt(OptOps.PADTO, 1, 32)],]) + @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") + @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared") + @unittest.expectedFailure + def test_padto_group_full_unroll_sum(self): + a = Tensor.ones(2, 28, 4096, dtype=dtypes.bfloat16).realize() + out = ((a * 0.5).float().square()).sum(axis=(0, 2)) + opts_to_apply = [Opt(OptOps.GROUPTOP, 1, 256), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.UNROLL, 2, 0), Opt(OptOps.UPCAST, 0, 7)] + helper_linearizer_opt(out, [opts_to_apply], check_default_opt=False) + def test_padto_sum(self): N = 18 # NOTE: this setup prevents 17 * 17 contiguous merged into one dimension From ff0cb28c21732cce01b4050c31762f5f2298914e Mon Sep 17 00:00:00 2001 From: George Hotz Date: Wed, 12 Aug 2026 13:12:37 -0700 Subject: [PATCH 023/306] skip slow whisper tests --- test/models/test_whisper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/models/test_whisper.py b/test/models/test_whisper.py index e6e3e507c0b30..3c0fa2c18012a 100644 --- a/test/models/test_whisper.py +++ b/test/models/test_whisper.py @@ -74,6 +74,7 @@ def assertWER(self, actual: str, expected: str, threshold: float): err ) + @slow def test_transcribe_file1(self): self.assertEqual(transcribe_file(self.model, self.enc, TEST_FILE_1), TRANSCRIPTION_1) @@ -89,6 +90,7 @@ def test_transcribe_batch12(self): self.assertEqual(TRANSCRIPTION_1, transcriptions[0]) self.assertEqual(TRANSCRIPTION_2, transcriptions[1]) + @slow def test_transcribe_batch21(self): waveforms = [load_file_waveform(TEST_FILE_2), load_file_waveform(TEST_FILE_1)] transcriptions = transcribe_waveform(self.model, self.enc, waveforms) From cd6d0d6ee32135a3cb0d761cc3cb327a5875e62b Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 12 Aug 2026 19:27:55 -0700 Subject: [PATCH 024/306] allow running QCOMCL compiler in docker (#17499) --- .github/actions/setup-tinygrad/action.yml | 20 +++++++------- .github/workflows/platform.yml | 32 ----------------------- .github/workflows/test.yml | 26 +++++++++--------- tinygrad/runtime/support/compiler_qcom.py | 26 +++++++++++++++--- 4 files changed, 45 insertions(+), 59 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 6f164a5003864..49c532b233c70 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -41,14 +41,14 @@ inputs: description: "Install LLVM?" required: false default: 'false' - tinydreno: - description: "Install tinydreno" - required: false - default: 'false' qemu: description: "Install qemu" required: false default: 'false' + docker-qemu: + description: "Setup docker to use qemu" + required: false + default: 'false' runs: using: "composite" steps: @@ -277,12 +277,6 @@ runs: shell: bash run: brew install llvm@20 - # *** tinydreno *** - - name: Install tinydreno (linux) - if: inputs.tinydreno == 'true' && runner.os == 'Linux' - shell: bash - run: sudo curl -fL https://github.com/sirhcm/tinydreno/raw/refs/heads/master/libllvm-qcom.so -o /usr/lib/libllvm-qcom.so - # *** OpenCL *** - name: Install rusticl if: inputs.opencl == 'true' @@ -292,3 +286,9 @@ runs: sudo mkdir -p /etc/OpenCL/vendors echo "/usr/lib/libRusticlOpenCL.so" | sudo tee /etc/OpenCL/vendors/rusticl.icd echo "RUSTICL_ENABLE=llvmpipe" >> "$GITHUB_ENV" + + - name: Setup docker to use qemu + if: inputs.docker-qemu == 'true' + uses: docker/setup-qemu-action@v4 + with: + platforms: arm64 diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index bb27f446d1d81..b48e8081b327e 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -179,35 +179,3 @@ jobs: - name: Run test_tiny shell: bash run: python -m pytest -n=auto test/test_tiny.py --durations=20 - - - qcomclcompiletests: - name: Compile-only (QCOM CL) - runs-on: ubuntu-24.04-arm - timeout-minutes: 15 - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Setup Environment - uses: ./.github/actions/setup-tinygrad - with: - key: compile-qcomcl - deps: testing_unit - tinydreno: 'true' - - name: Set env - shell: bash - run: printf "DEV=NULL:QCOMCL:a630\nNULL_ALLOW_COPYOUT=1" >> $GITHUB_ENV - - name: Run test_ops - shell: bash - run: | - python -c "from tinygrad import Device; assert Device.DEFAULT == 'NULL'" - DEBUG=4 python3 test/backend/test_ops.py TestOps.test_add - python -m pytest -n=auto test/backend/test_ops.py --durations=20 - - name: Run test_ops (IMAGE) - shell: bash - env: - IMAGE: 1 - DEV: "NULL:QCOMCL:a630,IMAGE_PITCH_ALIGNMENT=64" - run: | - DEBUG=4 python test/backend/test_ops.py TestOps.test_gemm | grep read_imagef - python -m pytest -n=auto test/backend/test_ops.py --durations=20 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e39096664749d..63f8b56e30cc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -637,10 +637,17 @@ jobs: strategy: fail-fast: false matrix: - backend: [ir3, nak] - name: Compile-only (${{ matrix.backend }}) + dev: + - 'NULL:IR3:a630' + - 'NULL:QCOMCL:a630' + - 'NULL:NAK:sm_120' + name: Compile-only (DEV=${{ matrix.dev }}) runs-on: *linux timeout-minutes: 15 + env: + NULL_ALLOW_COPYOUT: 1 + DEV: ${{ matrix.dev }}${{ contains(matrix.dev, 'a630') && ',IMAGE_PITCH_ALIGNMENT=64' || '' }} + IMAGE: ${{ contains(matrix.dev, 'a630') && '1' || '0' }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -649,21 +656,14 @@ jobs: with: key: compile-${{ matrix.backend }} deps: "testing_unit mesa" - - name: Set env + docker-qemu: ${{ contains(matrix.dev, 'QCOMCL') }} + - name: Test IMAGE shell: bash - run: printf "NULL_ALLOW_COPYOUT=1\n${{ matrix.backend == 'ir3' && 'DEV=NULL:IR3:a630' || matrix.backend == 'nak' && 'DEV=NULL:NAK:sm_120' }}" >> $GITHUB_ENV + if: contains(matrix.dev, 'a630') + run: DEBUG=7 python3 test/backend/test_ops.py TestOps.test_gemm | grep isam - name: Run test_ops shell: bash run: | python -c "from tinygrad import Device; assert Device.DEFAULT == 'NULL'" DEBUG=4 python3 test/backend/test_ops.py TestOps.test_add python -m pytest -n=auto test/backend/test_ops.py --durations=20 - - name: Run test_ops (IMAGE) - if: matrix.backend == 'ir3' - shell: bash - env: - IMAGE: 1 - DEV: "NULL:IR3:a630,IMAGE_PITCH_ALIGNMENT=64" - run: | - DEBUG=4 python3 test/backend/test_ops.py TestOps.test_gemm | grep image_load - python -m pytest -n=auto test/backend/test_ops.py --durations=20 diff --git a/tinygrad/runtime/support/compiler_qcom.py b/tinygrad/runtime/support/compiler_qcom.py index ceb8f270b78f9..2bf972689c3bb 100644 --- a/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad/runtime/support/compiler_qcom.py @@ -1,6 +1,6 @@ -import ctypes, struct +import ctypes, struct, platform, pathlib, subprocess, sys from tinygrad.device import Compiler -from tinygrad.helpers import DEBUG, system +from tinygrad.helpers import DEBUG, system, fetch, unwrap from tinygrad.runtime.support.compiler_mesa import disas_adreno # see https://github.com/sirhcm/tinydreno from tinygrad.runtime.autogen import llvm_qcom @@ -10,10 +10,14 @@ def _read_lib(lib, off) -> int: return struct.unpack("I", lib[off:off+4])[0] class QCOMCompiler(Compiler): def __init__(self, arch:str): assert arch.split(',')[0] == "a630", "only a630 supported" - self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() + if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() + else: self.arch, self.chip_id, self.compiler_process = arch, 0x6030001, subprocess.Popen( + (f"docker run --rm -i --platform linux/aarch64 -e PYTHONPATH=/ -e QEMU_CPU=max,pauth=off -v {pathlib.Path(__file__).parents[2]}:/tinygrad " + f"-v {fetch('https://github.com/sirhcm/tinydreno/raw/refs/heads/master/libllvm-qcom.so')}:/lib/libllvm-qcom.so python:3.12-slim " + f"python /tinygrad/runtime/support/compiler_qcom.py {arch}").split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) super().__init__(f"compile_qcomcl_{arch}") - def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) + def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) if platform.machine() == "aarch64" else self.compiler_process.kill() def __reduce__(self): return QCOMCompiler, (self.arch,) @@ -25,6 +29,10 @@ def checked(self, handle): return handle def compile(self, src) -> bytes: + if platform.machine() != "aarch64": + unwrap(self.compiler_process.stdin).write(struct.pack("I", len(src.encode())) + src.encode()) + if (lib:=unwrap(self.compiler_process.stdout).read(struct.unpack("I", unwrap(self.compiler_process.stdout).read(4))[0])): return lib + raise RuntimeError("QCOM Compilation Error") ch = self.checked(llvm_qcom.cl_compiler_compile_source(self.llvm_inst, self.chip_id, llvm_qcom.CL_MODE_64BIT, b"", 0, 0, 0, src.encode(), 0, llvm_qcom.CL_SRC_STR, None)) if DEBUG >= 8: print(system("llvm-dis", input=ctypes.string_at((comp:=ch.contents.compiled.contents).llvm_bitcode, comp.llvm_bitcode_size))) @@ -36,3 +44,13 @@ def compile(self, src) -> bytes: return ret def disassemble(self, lib: bytes): disas_adreno(lib[(ofs:=_read_lib(lib, 0xc0)):ofs+_read_lib(lib, 0x100)], self.chip_id) + +if __name__ == "__main__": + compiler = QCOMCompiler(sys.argv[1]) + while (amt:=sys.stdin.buffer.read(4)): + try: lib = compiler.compile(sys.stdin.buffer.read(struct.unpack("I", amt)[0]).decode()) + except Exception as e: + lib = b"" + print(e, file=sys.stderr, flush=True) + sys.stdout.buffer.write(struct.pack("I", len(lib)) + lib) + sys.stdout.buffer.flush() From 229711854192751540b685ddd0847f302db67034 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:29:32 -0700 Subject: [PATCH 025/306] lil llm improvements (#17513) --- test/unit/test_llm_server.py | 12 ++++++++++++ tinygrad/llm/model.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test/unit/test_llm_server.py b/test/unit/test_llm_server.py index 4eee0e5225fa5..466d59d055782 100644 --- a/test/unit/test_llm_server.py +++ b/test/unit/test_llm_server.py @@ -19,6 +19,12 @@ def generate(tokens): with patch.object(model, "generate", generate): model.warmup() self.assertEqual(calls, [[0], [0]]) + def test_warmup_then_generate_with_default_chunk(self): + # warmup must not capture JIT graphs that generate()'s default chunk_size then rejects + model = Transformer(TEST_CONFIG) + model.warmup() + self.assertIsInstance(next(model.generate([5, 6, 7, 8])), int) + def test_first_recurrent_generate_before_state_init(self): model = Transformer(TEST_CONFIG) model.has_recurrent_block = True @@ -178,6 +184,12 @@ def test_temperature_high_produces_variety(self): # with temperature=2.0, we should see at least 2 distinct outputs across 5 runs self.assertGreater(len(runs), 1, "high temperature should produce varied outputs") + def test_recurrent_temperature_high_produces_variety(self): + model = Transformer(TEST_CONFIG) + model.has_recurrent_block = True + outputs = {model.forward(Tensor([[1]]), 0, Tensor([2.0])).item() for _ in range(5)} + self.assertGreater(len(outputs), 1) + def test_temperature_passed_to_forward(self): """Temperature from generate should be passed through to __call__.""" model = Transformer(TEST_CONFIG) diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index 1f4feb9698236..c91e17c8c93d2 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -326,7 +326,8 @@ def __init__(self, config:TransformerConfig): def forward(self, tokens:Tensor, start_pos:int|UOp, temperature:Tensor) -> Tensor: x = self.token_embd(tokens).float() # (B, T, D) for block in self.blk: x = block(x, start_pos) - logits = self.output(self.output_norm(x))[:, -1, :] + # only run the output projection on the last token + logits = self.output(self.output_norm(x[:, -1:]))[:, -1, :] # Gumbel-max trick: argmax(logits/temp - log(-log(uniform))) is equivalent to sampling from softmax(logits/temp) return (logits / temperature.maximum(1e-12) - (Tensor.rand_like(logits).maximum(1e-12).log().neg()).log()).argmax(-1, keepdim=True) From e103fb2a10aa977f88b87e394b597a723f87b2b9 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:13:20 -0700 Subject: [PATCH 026/306] more lil llm improvements (#17514) * more lil llm improvements * default float --- test/unit/test_llm_server.py | 11 ++++++++++- tinygrad/llm/model.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/test/unit/test_llm_server.py b/test/unit/test_llm_server.py index 466d59d055782..e6b2df7cd4029 100644 --- a/test/unit/test_llm_server.py +++ b/test/unit/test_llm_server.py @@ -13,7 +13,7 @@ class TestTransformerGenerate(unittest.TestCase): def test_warmup(self): model, calls = Transformer(TEST_CONFIG), [] - def generate(tokens): + def generate(tokens, **kwargs): calls.append(tokens) yield from (1, 2) with patch.object(model, "generate", generate): model.warmup() @@ -44,6 +44,15 @@ def mock_call(self, tokens, start_pos, temperature, **kwargs): next(model.generate([1, 2, 3, 4, 5, 42, 10])) self.assertEqual(calls, [((1, 1), V_START_POS.bind(5)), ((1, 1), V_START_POS.bind(6))]) + def test_recurrent_divergent_prompt_restarts(self): + model, calls = Transformer(TEST_CONFIG), [] + model.has_recurrent_block, model._cached_tokens = True, [1, 2, 9] + def mock_call(self, tokens, start_pos, temperature): + calls.append(start_pos) + return Tensor([[42]]) + with patch.object(Transformer, '__call__', mock_call): next(model.generate([1, 2, 10, 11])) + self.assertEqual(calls[0], V_START_POS.bind(0)) + def test_template_starts_reasoning(self): router = StreamRouter(reasoning=True) self.assertEqual(list(router.route("reasoninganswer")), diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index c91e17c8c93d2..95ce646e819e6 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -1,7 +1,7 @@ from __future__ import annotations import functools, itertools, pathlib from dataclasses import dataclass, replace -from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function +from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function, dtypes from tinygrad.nn import Linear from tinygrad.llm.gguf import gguf_load from tinygrad.uop.ops import resolve @@ -187,8 +187,8 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: def _init_state(self, x:Tensor): if not hasattr(self, "cache_kv"): - # TODO: how is the dtype of this determined? - self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, device=x.device) + self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, + dtype=dtypes.default_float, device=x.device) self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device) class MLATransformerBlock(FFNBlock): @@ -261,13 +261,14 @@ def __init__(self, config:TransformerConfig, ssm:SSMConfig): def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: B, T, _ = x.shape assert T == 1, "GatedDeltaNetBlock currently only supports T=1" + is_kda = hasattr(self, "ssm_g_a") # input processing x = x.half() - out_gate = self.ssm_g_b(self.ssm_g_a(x)) if hasattr(self, "ssm_g_a") else self.attn_gate(x) + out_gate = self.ssm_g_b(self.ssm_g_a(x)) if is_kda else self.attn_gate(x) out_gate = out_gate.reshape(B, 1, self.num_v_heads, self.head_v_dim) beta = self.ssm_beta(x).sigmoid().reshape(B, self.num_v_heads, 1, 1) - alpha = self.ssm_f_b(self.ssm_f_a(x)) if hasattr(self, "ssm_f_a") else self.ssm_alpha(x) + alpha = self.ssm_f_b(self.ssm_f_a(x)) if is_kda else self.ssm_alpha(x) alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, self.num_v_heads, -1) * self.ssm_a.reshape(1, self.num_v_heads, -1)).exp().unsqueeze(-2) @@ -291,14 +292,13 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: # output core_attn_out = self.ssm_norm((recurrent_state@q).squeeze(-1).reshape(B, 1, self.num_v_heads, self.head_v_dim)) - out_gate = out_gate.sigmoid() if hasattr(self, "ssm_g_a") else out_gate.silu() + out_gate = out_gate.sigmoid() if is_kda else out_gate.silu() return self.ssm_out((core_attn_out * out_gate).reshape(B, 1, -1).cast(x.dtype)) # recurrent state can't be partially reused after divergence, force a full rebuild def _state_reset_ops(self): return [self.conv_state.assign(self.conv_state.const_like(0)), self.recurrent_state.assign(self.recurrent_state.const_like(0))] if hasattr(self, "conv_state") else [] - def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return 0 if prefix_len != cached_len else prefix_len def _init_state(self, x): if not hasattr(self, "conv_state"): @@ -421,6 +421,10 @@ def warmup(self): for _ in range(2): list(zip(range(2), self.generate([0]))) def get_start_pos(self, tokens:list[int]) -> int: + # recurrent state can't be partially reused after divergence: reuse it only when tokens extend the cached prefix + if self.has_recurrent_block: + return len(self._cached_tokens) if self._cached_tokens and len(self._cached_tokens) < len(tokens) \ + and tokens[:len(self._cached_tokens)] == self._cached_tokens else 0 prefix_len = sum(1 for _ in itertools.takewhile(lambda ab: ab[0] == ab[1], zip(tokens[:-1], self._cached_tokens))) return min(block._reusable_prefix_len(prefix_len, len(self._cached_tokens)) for block in self.blk) From 39d144546e8ea69889806eaf0213de2050c12afd Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:48:15 +0800 Subject: [PATCH 027/306] fix mxfp4 mem estimate (#17515) * add mem estimates * rename * move --- extra/gemm/cdna_asm_gemm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extra/gemm/cdna_asm_gemm.py b/extra/gemm/cdna_asm_gemm.py index e9cb78e98a9ab..e94cf6c2520cf 100644 --- a/extra/gemm/cdna_asm_gemm.py +++ b/extra/gemm/cdna_asm_gemm.py @@ -122,7 +122,8 @@ def custom_mxfp4_gemm(C:UOp, A:UOp, B:UOp, scale_a:UOp, scale_b:UOp, *extra:UOp, groups_x, groups_y = UOp.special(ceildiv(N, tile_n), "gidx0"), UOp.special(ceildiv(M, tile_m), "gidx1") lds = UOp.placeholder((163840,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(C.base, A.base, B.base, scale_a.base, scale_b.base, *(x.base for x in extra), lds, threads, groups_x, groups_y, - arg=KernelInfo(f"custom_mxfp4_gemm_{M}_{N}_{K}", estimates=Estimates(ops=2*M*N*K))) + arg=KernelInfo(f"mxfp4_gemm_{M}_{N}_{K}", + estimates=Estimates(ops=2*M*N*K, mem=(M*half_k+N*half_k)*A.dtype.itemsize+M*N*C.dtype.itemsize))) insts = build_kernel(M, N, K, tile_m, tile_n) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=x) for x in insts)))) From 1b7f04098437735260d182f9e06ea7e41362439c Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Thu, 13 Aug 2026 14:45:57 +0800 Subject: [PATCH 028/306] fa: paas through window (#17517) --- extra/thunder/amd/fa.py | 64 +++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/extra/thunder/amd/fa.py b/extra/thunder/amd/fa.py index bbb4d46857bb7..685a1b8e77860 100644 --- a/extra/thunder/amd/fa.py +++ b/extra/thunder/amd/fa.py @@ -110,7 +110,49 @@ def _sharded_empty_like(ref:Tensor, axis:int|None=None) -> Tensor: return _sharded_empty(ref.shape, ref, axis) @functools.cache -def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink): +def _windowed_lse(xq:Tensor, xk:Tensor, sinks, W:int) -> Tensor: + B, N, H, hd = xq.shape + H_KV = xk.shape[2]; R = H // H_KV; nb = N // W; sm = hd ** -0.5 + q = xq.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() + k = xk.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() + k_prev = k.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] + sc_d = (q @ k.transpose(-1, -2)) * sm + sc_p = (q @ k_prev.transpose(-1, -2)) * sm + li, lj = Tensor.arange(W).reshape(W, 1), Tensor.arange(W).reshape(1, W) + pv = (Tensor.arange(nb).reshape(nb, 1, 1) >= 1) + sc_d = (lj <= li).where(sc_d, -float("inf")) + sc_p = ((li < lj) & pv).where(sc_p, -float("inf")) + m = sc_d.max(-1, keepdim=True).maximum(sc_p.max(-1, keepdim=True)) + if sinks is not None: m = m.maximum(sinks.reshape(1, H_KV, R, 1, 1, 1).float()) + denom = (sc_d - m).exp().sum(-1, keepdim=True) + (sc_p - m).exp().sum(-1, keepdim=True) + if sinks is not None: denom = denom + (sinks.reshape(1, H_KV, R, 1, 1, 1).float() - m).exp() + return (m + denom.log()).reshape(B, H, N).unsqueeze(2) # (B, H, 1, N), matches saved l_vec + +def _windowed_delta(xq:Tensor, xk:Tensor, xv:Tensor, do:Tensor, sinks, W:int) -> Tensor: + B, N, H, hd = xq.shape + H_KV = xk.shape[2]; R = H // H_KV; nb = N // W; sm = hd ** -0.5 + q = xq.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() + k = xk.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() + v = xv.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() + dob = do.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() + k_prev = k.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] + v_prev = v.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] + sc_d = (q @ k.transpose(-1, -2)) * sm + sc_p = (q @ k_prev.transpose(-1, -2)) * sm + li, lj = Tensor.arange(W).reshape(W, 1), Tensor.arange(W).reshape(1, W) + pv = (Tensor.arange(nb).reshape(nb, 1, 1) >= 1) + sc_d = (lj <= li).where(sc_d, -float("inf")) + sc_p = ((li < lj) & pv).where(sc_p, -float("inf")) + m = sc_d.max(-1, keepdim=True).maximum(sc_p.max(-1, keepdim=True)) + if sinks is not None: m = m.maximum(sinks.reshape(1, H_KV, R, 1, 1, 1).float()) + e_d, e_p = (sc_d - m).exp(), (sc_p - m).exp() + denom = e_d.sum(-1, keepdim=True) + e_p.sum(-1, keepdim=True) + if sinks is not None: denom = denom + (sinks.reshape(1, H_KV, R, 1, 1, 1).float() - m).exp() + o = ((e_d / denom) @ v) + ((e_p / denom) @ v_prev) + delta = (dob * o).sum(-1) + return delta.reshape(B, H, N).unsqueeze(2) + +def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=0): def grad(dou:UOp, ker:UOp) -> tuple: do = Tensor(dou, device=dou.device) attn = Tensor(ker.src[1].after(ker), device=ker.src[1].device) @@ -118,6 +160,8 @@ def grad(dou:UOp, ker:UOp) -> tuple: xq = Tensor(ker.src[3], device=ker.src[3].device) xk = Tensor(ker.src[4], device=ker.src[4].device) xv = Tensor(ker.src[5], device=ker.src[5].device) + if window: + l_vec = _windowed_lse(xq, xk, Tensor(ker.src[6], device=ker.src[6].device) if has_sink else None, window) dq = _sharded_empty((B, H, N, D), xq, axis=shard_axis_t) GROUP_SIZE = H_local // H_KV_local @@ -128,8 +172,10 @@ def grad(dou:UOp, ker:UOp) -> tuple: # delta_vec = (do * attn).sum(-1, dtype=dtypes.float32).transpose(1, 2).unsqueeze(-2).detach() delta_vec = _sharded_empty((B, H, 1, N), xq, dtype=dtypes.float32, axis=shard_axis_t) delta_vec, dq = Tensor.custom_kernel(delta_vec, dq, attn, do, fxn=functools.partial(custom_fa_backward_pre, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D))[:2] + if window: + delta_vec = _windowed_delta(xq, xk, xv, do, Tensor(ker.src[6], device=ker.src[6].device) if has_sink else None, window) - dq, dk_partial, dv_partial = Tensor.custom_kernel(dq, dk_partial, dv_partial, do, xq, xk, xv, l_vec, delta_vec, fxn=functools.partial(custom_fa_backward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D))[:3] + dq, dk_partial, dv_partial = Tensor.custom_kernel(dq, dk_partial, dv_partial, do, xq, xk, xv, l_vec, delta_vec, fxn=functools.partial(custom_fa_backward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, window=window))[:3] if D == 64: dq = dq.reshape(B, H, N//16, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2).permute(0, 1, 2, 8, 9, 10, 11, 3, 4, 6, 7, 5, 12).reshape(B, H, N, D).transpose(1, 2) @@ -149,7 +195,7 @@ def grad(dou:UOp, ker:UOp) -> tuple: return grad # TODO: remove write_flat once scheduler can remove reshapes between custom_kernel. TestCustomKernel.test_simple_reshape -def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False, write_flat:bool=False, sinks:Tensor|None=None): +def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False, write_flat:bool=False, sinks:Tensor|None=None, window:int=0): assert attn_mask is None, "attn_mask not supported" assert is_causal, "only causal attention supported" @@ -176,18 +222,18 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False attn = _sharded_empty((B, N, H * D), xq, axis=shard_axis) if write_flat else _sharded_empty_like(xq, axis=shard_axis) l_vec = _sharded_empty((B, H, 1, N), xq, dtype=dtypes.float32, axis=shard_axis_t) - grad = _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink) + grad = _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=window) fwd_inputs = (attn, l_vec, xq, xk, xv) + ((sinks,) if has_sink else ()) - attn, l_vec = Tensor.custom_kernel(*fwd_inputs, fxn=functools.partial(custom_fa_forward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, has_sink=has_sink), grad_fxn=grad)[:2] + attn, l_vec = Tensor.custom_kernel(*fwd_inputs, fxn=functools.partial(custom_fa_forward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, has_sink=has_sink, window=window), grad_fxn=grad)[:2] return attn, attn, l_vec @functools.cache -def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None, *, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, has_sink:bool=True): +def custom_fa_forward(o:UOp, l_vec:UOp, q:UOp, k:UOp, v:UOp, sinks:UOp|None=None, *, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, has_sink:bool=True, window:int=0): code = (pathlib.Path(__file__).parent / "fa_fwd_causal.cpp").read_text() compile_args = [f"-I{(pathlib.Path(__file__).parent / 'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", "-ffast-math", - f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DATTN_SINK={int(has_sink)}"] + f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DATTN_SINK={int(has_sink)}", f"-DWINDOW={window}"] Q_BLOCK_SIZE = 32 NUM_WARPS = 8 @@ -247,10 +293,10 @@ def custom_fa_backward_pre(delta_vec:UOp, dq:UOp, o:UOp, do:UOp, device:str, arc src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) @functools.cache -def custom_fa_backward(dq:UOp, dk:UOp, dv:UOp, do:UOp, q:UOp, k:UOp, v:UOp, l_vec:UOp, delta_vec:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): +def custom_fa_backward(dq:UOp, dk:UOp, dv:UOp, do:UOp, q:UOp, k:UOp, v:UOp, l_vec:UOp, delta_vec:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int, window:int=0): code = (pathlib.Path(__file__).parent / "fa_bwd_causal.cpp").read_text() compile_args = [f"-I{(pathlib.Path(__file__).parent / 'include').as_posix()}", "-std=c++20", "-DKITTENS_CDNA4", "-DHIP_ENABLE_WARP_SYNC_BUILTINS", "-ffast-math", - f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}"] + f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DWINDOW={window}"] BLOCK_SIZE_KV = 256 GROUP_SIZE = H // H_KV From 16c5ff249026ae97a31413ddaf7a3818207954a4 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:50:43 +0800 Subject: [PATCH 029/306] add external_benchmark_all2all.py (#17507) * add external_benchmark_all2all.py * mv * more minimal * less * fix space --- test/external/external_benchmark_all2all.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 test/external/external_benchmark_all2all.py diff --git a/test/external/external_benchmark_all2all.py b/test/external/external_benchmark_all2all.py new file mode 100644 index 0000000000000..e278dfad735c3 --- /dev/null +++ b/test/external/external_benchmark_all2all.py @@ -0,0 +1,17 @@ +from tinygrad import Device, Tensor, TinyJit, dtypes +from tinygrad.helpers import Timing, Context + +GPUS, DEPTH, SZ = 8, 4, 128 * 2**20 +WARMUP, ITERS = 3, 5 +devs = tuple(f"{Device.DEFAULT}:{i}" for i in range(GPUS)) +bufs = tuple(Tensor.empty(SZ, dtype=dtypes.uint8, device=dev).contiguous().realize() for _ in range(DEPTH) for dev in devs) + +@TinyJit +def all_to_all(*srcs:Tensor): return Tensor.realize(*(src.to(dst) for i,src in enumerate(srcs) for j,dst in enumerate(devs) if i % GPUS != j)) + +if __name__ == "__main__": + with Context(ALL2ALL=1, JIT_BATCH_SIZE=0): + for i in range(-WARMUP, ITERS): + with Timing("ALL2ALL ", lambda ns: f" {SZ*GPUS*(GPUS-1)*DEPTH/ns:.2f} GB/s", enabled=i>=0): + all_to_all(*bufs) + for dev in devs: Device[dev].synchronize() From cc6d33bde783969caa436857a2eb8d0d1dc1345d Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:24:45 +0800 Subject: [PATCH 030/306] amd: fix ALL2ALL speed on amdgpu (gpt) (#17518) --- tinygrad/runtime/ops_amd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 7404a1fca3c6f..d87d094b06b7c 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -790,10 +790,14 @@ def map(self, mem): def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0, xcc_id=0, idx=0): + if queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA and idx and self.kfd_ver >= (1,17): + queue_type = kfd.KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID + sdma_engine_id = idx % (self.props['num_sdma_engines'] + self.props.get('num_sdma_xgmi_engines', 0)) + else: sdma_engine_id = 0 queue = kfd.AMDKFD_IOC_CREATE_QUEUE(KFDIface.kfd, ring_base_address=ring.va_addr, ring_size=ring.size, gpu_id=self.gpu_id, queue_type=queue_type, queue_percentage=kfd.KFD_MAX_QUEUE_PERCENTAGE|(xcc_id<<8), queue_priority=getenv("AMD_KFD_QUEUE_PRIORITY", 7), eop_buffer_address=eop_buffer.va_addr if eop_buffer else 0, eop_buffer_size=eop_buffer.size if eop_buffer else 0, ctl_stack_size=ctl_stack_size, - ctx_save_restore_address=cwsr_buffer.va_addr if cwsr_buffer else 0, ctx_save_restore_size=ctx_save_restore_size, + ctx_save_restore_address=cwsr_buffer.va_addr if cwsr_buffer else 0, ctx_save_restore_size=ctx_save_restore_size, sdma_engine_id=sdma_engine_id, write_pointer_address=gart.va_addr+wptr, read_pointer_address=gart.va_addr+rptr+8*xcc_id) if not hasattr(self, 'doorbells'): From 0252cb8fa7ef7b9b0d68485ece353fc6a298c794 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:17:22 -0700 Subject: [PATCH 031/306] remove anchors from CI flow (#17521) * ci: remove yaml anchors from test.yml for gitea actions compatibility Gitea Actions does not support YAML anchors/aliases, which causes the workflow to fail parsing. Replace the &linux/*linux anchor with a plain runs-on: ubuntu-24.04. * ci: keep runner selection, inline expression instead of anchors Instead of replacing the anchored runs-on with a plain ubuntu-24.04 (which drops the namespace-profile-tinygrad routing for collaborator PRs), inline the full ${{ }} expression at every job. No YAML anchors, works with runners that can't parse them (gitea runner), and identical behavior on GitHub Actions. --- .github/workflows/test.yml | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63f8b56e30cc3..edca331e143cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ concurrency: jobs: docs: name: Docs - runs-on: &linux ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 env: CHECK_OOB: 0 @@ -61,7 +61,7 @@ jobs: torchbackend: name: Torch Backend Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -88,7 +88,7 @@ jobs: bepython: name: Python Backend - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -126,7 +126,7 @@ jobs: linter: name: Linters - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 steps: @@ -157,7 +157,7 @@ jobs: nulltest: name: Null Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: @@ -191,7 +191,7 @@ jobs: unittest: name: Unit Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: @@ -228,7 +228,7 @@ jobs: matrix: group: [1, 2] name: SPEC=2 (${{ matrix.group }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -244,7 +244,7 @@ jobs: fuzzing: name: Fuzzing - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 10 steps: - name: Checkout Code @@ -260,7 +260,7 @@ jobs: testopenclimage: name: CL IMAGE Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -280,7 +280,7 @@ jobs: testopenpilot: name: openpilot Compile Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -309,7 +309,7 @@ jobs: testonnxcpu: name: ONNX (CPU) Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: @@ -328,7 +328,7 @@ jobs: testoptim: name: Optimization Tests - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: - name: Checkout Code @@ -360,7 +360,7 @@ jobs: testllm: name: Test LLM - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: CHECK_OOB: 0 @@ -387,7 +387,7 @@ jobs: testmodels: name: Models - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -407,7 +407,7 @@ jobs: testdsp: name: Linux (DSP) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 steps: - name: Checkout Code @@ -435,7 +435,7 @@ jobs: - 'WEBGPU' name: Linux (DEV=${{ matrix.dev }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 steps: - name: Checkout Code @@ -461,7 +461,7 @@ jobs: testamdasm: name: AMD ASM IDE - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 env: DEV: MOCKKFD+AMD @@ -507,7 +507,7 @@ jobs: hcq2: name: hcq2 - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 5 steps: - name: Checkout Code @@ -529,7 +529,7 @@ jobs: testmockam: name: Linux (am) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: DEV: MOCKPCI+AMD @@ -565,7 +565,7 @@ jobs: arch: [gfx1100, gfx1201, gfx950] name: Linux (${{ matrix.backend }} ${{ matrix.arch }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: DEV: MOCKKFD+AMD:${{ matrix.backend == 'amdllvm' && 'LLVM' || '' }}:${{ matrix.arch }} @@ -603,7 +603,7 @@ jobs: backend: [ptx, nv] name: Linux (${{ matrix.backend }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 20 env: FORWARD_ONLY: 1 @@ -642,7 +642,7 @@ jobs: - 'NULL:QCOMCL:a630' - 'NULL:NAK:sm_120' name: Compile-only (DEV=${{ matrix.dev }}) - runs-on: *linux + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} timeout-minutes: 15 env: NULL_ALLOW_COPYOUT: 1 From 849074f0dbdc9e97095aef09335f85cbea848089 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Fri, 14 Aug 2026 01:39:22 +0800 Subject: [PATCH 032/306] gptoss: use fa swa (#17522) --- examples/mlperf/models/gpt_oss.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/mlperf/models/gpt_oss.py b/examples/mlperf/models/gpt_oss.py index 2fb03ad603d1b..d58d615ef0e76 100644 --- a/examples/mlperf/models/gpt_oss.py +++ b/examples/mlperf/models/gpt_oss.py @@ -182,12 +182,12 @@ def attention(self, x:Tensor, freqs_cis:Tensor, mask:Tensor, sliding:bool, *, at xq, xk = apply_rotary_emb(xq, xk, freqs_cis) xq, xk, xv = xq.cast(dtypes.bfloat16), xk.cast(dtypes.bfloat16), xv.cast(dtypes.bfloat16) # (B,N,H,D)/(B,N,KV,D) - if sliding: - attn = self._sliding_attention(xq, xk, xv, sinks) - elif getenv("HK_FLASH_ATTENTION"): + if getenv("HK_FLASH_ATTENTION"): from extra.thunder.amd.fa import flash_attention - attn, *_ = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks) + attn, *_ = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks, window=self.sliding_window if sliding else 0) attn = attn.reshape(bsz, seqlen, self.n_heads * self.head_dim) + elif sliding: + attn = self._sliding_attention(xq, xk, xv, sinks) else: xqm = xq.reshape(bsz, seqlen, self.n_kv_heads, self.n_rep, self.head_dim).permute(0, 2, 3, 1, 4) xkm, xvm = xk.permute(0, 2, 1, 3).unsqueeze(2), xv.permute(0, 2, 1, 3).unsqueeze(2) From 673c6463f9defd9497f80c23cefca6638dd5e734 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:14:40 -0700 Subject: [PATCH 033/306] disable HCQ2 for AMD CI (#17523) --- .github/workflows/benchmark.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 739f7242c05dc..b805c996941d7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,7 +94,6 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -149,7 +148,6 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -202,7 +200,6 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -252,7 +249,6 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code From 303d1677b3e8efa6bd363d028ea64e05d31854f4 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 13 Aug 2026 17:16:02 -0700 Subject: [PATCH 034/306] qcomcl: use qemu for compilation (#17524) --- .github/actions/setup-tinygrad/action.yml | 10 ---------- .github/workflows/test.yml | 2 +- tinygrad/runtime/support/compiler_qcom.py | 13 ++++++++----- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 49c532b233c70..354bf0001ce01 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -45,10 +45,6 @@ inputs: description: "Install qemu" required: false default: 'false' - docker-qemu: - description: "Setup docker to use qemu" - required: false - default: 'false' runs: using: "composite" steps: @@ -286,9 +282,3 @@ runs: sudo mkdir -p /etc/OpenCL/vendors echo "/usr/lib/libRusticlOpenCL.so" | sudo tee /etc/OpenCL/vendors/rusticl.icd echo "RUSTICL_ENABLE=llvmpipe" >> "$GITHUB_ENV" - - - name: Setup docker to use qemu - if: inputs.docker-qemu == 'true' - uses: docker/setup-qemu-action@v4 - with: - platforms: arm64 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index edca331e143cf..6f73444da1da5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -656,7 +656,7 @@ jobs: with: key: compile-${{ matrix.backend }} deps: "testing_unit mesa" - docker-qemu: ${{ contains(matrix.dev, 'QCOMCL') }} + qemu: ${{ contains(matrix.dev, 'QCOMCL') }} - name: Test IMAGE shell: bash if: contains(matrix.dev, 'a630') diff --git a/tinygrad/runtime/support/compiler_qcom.py b/tinygrad/runtime/support/compiler_qcom.py index 2bf972689c3bb..0c4c3bff41a82 100644 --- a/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad/runtime/support/compiler_qcom.py @@ -1,4 +1,4 @@ -import ctypes, struct, platform, pathlib, subprocess, sys +import ctypes, struct, platform, pathlib, shutil, subprocess, sys, tarfile, tempfile from tinygrad.device import Compiler from tinygrad.helpers import DEBUG, system, fetch, unwrap from tinygrad.runtime.support.compiler_mesa import disas_adreno @@ -11,10 +11,13 @@ class QCOMCompiler(Compiler): def __init__(self, arch:str): assert arch.split(',')[0] == "a630", "only a630 supported" if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() - else: self.arch, self.chip_id, self.compiler_process = arch, 0x6030001, subprocess.Popen( - (f"docker run --rm -i --platform linux/aarch64 -e PYTHONPATH=/ -e QEMU_CPU=max,pauth=off -v {pathlib.Path(__file__).parents[2]}:/tinygrad " - f"-v {fetch('https://github.com/sirhcm/tinydreno/raw/refs/heads/master/libllvm-qcom.so')}:/lib/libllvm-qcom.so python:3.12-slim " - f"python /tinygrad/runtime/support/compiler_qcom.py {arch}").split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + else: + self.arch, self.chip_id, self.fs = arch, 0x6030001, tempfile.TemporaryDirectory() + with tarfile.open(fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz')) as t: t.extractall(fs:=self.fs.name) + if (qemu:=shutil.which("qemu-aarch64-static")): argv = f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3 {__file__} {arch}" + else: argv = (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {pathlib.Path(__file__).parents[2]}:/tinygrad " + f"-e PYTHONPATH=/ -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3 /tinygrad/runtime/support/compiler_qcom.py {arch}") + self.compiler_process = subprocess.Popen(argv.split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) super().__init__(f"compile_qcomcl_{arch}") def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) if platform.machine() == "aarch64" else self.compiler_process.kill() From 95ca5081fe02d33522a186e622fce15afc1672a1 Mon Sep 17 00:00:00 2001 From: George Hotz Date: Thu, 13 Aug 2026 17:23:47 -0700 Subject: [PATCH 035/306] hotfix: update extra/runbook_digitalocean_mi350x --- extra/runbook_digitalocean_mi350x.md | 80 +++++++++++++--------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md index edb09e6f2f45f..69d59215f0618 100644 --- a/extra/runbook_digitalocean_mi350x.md +++ b/extra/runbook_digitalocean_mi350x.md @@ -16,8 +16,9 @@ apt-get install -y python3-pip python3-venv git tmux rclone clang ### 1.2 Install Python deps ```bash -python3 -m pip install --break-system-packages numpy tqdm wandb tiktoken sentencepiece +python3 -m pip install --break-system-packages --ignore-installed typing-extensions numpy tqdm wandb tiktoken sentencepiece ``` +Note: `--ignore-installed typing-extensions` is needed because the base image ships typing-extensions 4.10.0 without a RECORD file, so pip cannot uninstall it. ### 1.3 Install ROCm dev headers The base image has ROCm runtime but NOT the HIP dev headers. Need: @@ -85,36 +86,45 @@ Files downloaded (~85GB total, ~6 minutes): - `c4-validation-91205-samples.en_text_document.idx` (1.8 MB) - `LICENSE.txt`, `NOTICE.txt` +**Wait for rclone to fully complete before starting training.** Starting training while the dataset is still downloading will read a truncated .bin file, causing `ValueError: all input arrays must have the same shape` in the dataloader. The stale `.index_cache` and `.blend_cache` files must also be deleted if this happens: +```bash +rm -f /raid/datasets/c4-8b/*.index_cache /raid/datasets/c4-8b/*.blend_cache +``` + ## Phase 4: wandb Login ```bash wandb login ``` Enter API key from https://wandb.ai/authorize +Alternatively, pass the key directly: +```bash +wandb login +``` + ## Phase 5: Run Training +Run training in tmux so it survives SSH disconnects: +```bash +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' +``` +Attach with `tmux attach -t train`. + ### 5.1 Smoke test (beam search, 2 layers, real data) Always run beam first to validate the pipeline: ```bash -cd /root/tinygrad -COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ -COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ -CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ -DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm \ - bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' ``` +The beam test runs 10 training steps with 2 layers. Expected results: +- ~0.29s per step after warmup +- ~700K GFLOPS, ~7% MFU (low because only 2 layers) +- ~380 GB VRAM used +- Loss stable at ~12.55 with random init + ### 5.2 Full training run ```bash -cd /root/tinygrad -COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ -COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ -CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ -DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm \ -WANDB=1 \ - bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' ``` ## Environment Variable Reference @@ -180,13 +190,18 @@ $ lspci -nn | grep AMD ``` CPU flags include `hypervisor`. `dmesg` shows `Hypervisor detected: KVM`. -### PCI device ID -`lspci -v` shows device ID `0x75b0` and subsystem ID `0x75a0`: -``` -83:00.0 Processing accelerators: ... Device 75b0 - Subsystem: ... Device 75a0 -``` -tinygrad's `PCIIface` in `ops_amd.py` and `hive_reset.py` did not list `0x75b0`, so the GPU was not found. Adding `0x75b0` to the device ID list in both files fixes the detection. +### Working path: amdgpu driver (KFDIface) +The amdgpu driver loads on boot and binds to all 8 GPUs, creating `/dev/kfd` and 64 renderD nodes (`/dev/dri/renderD128` through `/dev/dri/renderD191`). tinygrad's `KFDIface` enumerates GPUs through `/sys/devices/virtual/kfd/kfd/topology/nodes` and uses `/dev/kfd` for ioctl. No PCI device ID patching is needed — the KFD path does not use `PCIIface` or `AMDev._run_discovery()`. + +This is the working configuration. No code changes to tinygrad are required. + +### PCIIface path (does not work on this VM) +For reference, the `PCIIface` path was also explored but does not work in this KVM guest: + +- `PCIIface` in `ops_amd.py` does not list device ID `0x75b0`. Adding it allows PCI detection but `AMDev._run_discovery()` fails because the VRAM BAR reads all `0xFF`. +- This was observed with the GPU unbound from any driver, after PCI reset, and with VFIO bound. +- VFIO binding (`vfio-pci` with `enable_unsafe_noiommu_mode=1`) succeeded but VRAM BAR still reads all `0xFF`. +- No IOMMU in guest — `dmesg` has no `AMD-Vi` entries, PCI devices have no `iommu_group` symlink. ### amdgpu driver behavior On first boot, amdgpu loaded and bound to all 8 GPUs. On one boot it failed to initialize: @@ -198,24 +213,5 @@ On first boot, amdgpu loaded and bound to all 8 GPUs. On one boot it failed to i ``` On a subsequent boot, amdgpu initialized successfully (SMU initialized, VRAM ready). After unbinding all 8 GPUs from amdgpu, `rmmod amdgpu` wedged the module (stuck in "Unloading" state in `/proc/modules`), requiring a full VM reboot. -### `/dev/kfd` -`/dev/kfd` exists when amdgpu is loaded. Opening it returns `OSError: [Errno 22] Invalid argument`. - -### VRAM BAR reads all 0xFF -After amdgpu initializes the GPU and is then unbound, reading the VRAM BAR (via `/sys/bus/pci/devices/0000:83:00.0/resource0`) returns all `0xFF` at all offsets — including the discovery table at `vram_size - 64KB`. tinygrad's `AMDev._run_discovery()` fails with `AssertionError: discovery signatures mismatch`. - -A PCI reset (`echo 1 > /sys/bus/pci/devices/0000:83:00.0/reset`) did not change the VRAM contents — still all `0xFF`. - -VRAM was also all `0xFF` when read via `/dev/mem` at the BAR physical address (`0xa0000000000`). - -### VFIO attempt -Bound the GPU to `vfio-pci` with `enable_unsafe_noiommu_mode=1`. The GPU bound successfully and `/dev/vfio/noiommu-0` appeared. Running tinygrad with `VFIO=1` still failed with the same `discovery signatures mismatch` — VRAM BAR still reads all `0xFF`. - -### No IOMMU in guest -`dmesg` has no `AMD-Vi` entries. PCI devices have no `iommu_group` symlink. - ### No fan control No `fan*` or `pwm*` hwmon entries exist. Only `temp*`, `power*`, `freq*` are exposed. GPU temps read 56-63°C, power ~265W per GPU. - -### Current status: NOT WORKING -tinygrad's `PCIIface` finds the GPU (after adding `0x75b0`) but `AMDev._run_discovery()` fails because the VRAM discovery table reads all `0xFF`. This was observed with the GPU unbound from any driver, after PCI reset, and with VFIO bound. From 81dc8ec232945f8fd293cdacd1bb63d27ca8d022 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:39:14 +0800 Subject: [PATCH 036/306] Revert "amd: fix ALL2ALL speed on amdgpu (gpt) (#17518)" (#17526) This reverts commit cc6d33bde783969caa436857a2eb8d0d1dc1345d. --- tinygrad/runtime/ops_amd.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index d87d094b06b7c..7404a1fca3c6f 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -790,14 +790,10 @@ def map(self, mem): def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0, xcc_id=0, idx=0): - if queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA and idx and self.kfd_ver >= (1,17): - queue_type = kfd.KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID - sdma_engine_id = idx % (self.props['num_sdma_engines'] + self.props.get('num_sdma_xgmi_engines', 0)) - else: sdma_engine_id = 0 queue = kfd.AMDKFD_IOC_CREATE_QUEUE(KFDIface.kfd, ring_base_address=ring.va_addr, ring_size=ring.size, gpu_id=self.gpu_id, queue_type=queue_type, queue_percentage=kfd.KFD_MAX_QUEUE_PERCENTAGE|(xcc_id<<8), queue_priority=getenv("AMD_KFD_QUEUE_PRIORITY", 7), eop_buffer_address=eop_buffer.va_addr if eop_buffer else 0, eop_buffer_size=eop_buffer.size if eop_buffer else 0, ctl_stack_size=ctl_stack_size, - ctx_save_restore_address=cwsr_buffer.va_addr if cwsr_buffer else 0, ctx_save_restore_size=ctx_save_restore_size, sdma_engine_id=sdma_engine_id, + ctx_save_restore_address=cwsr_buffer.va_addr if cwsr_buffer else 0, ctx_save_restore_size=ctx_save_restore_size, write_pointer_address=gart.va_addr+wptr, read_pointer_address=gart.va_addr+rptr+8*xcc_id) if not hasattr(self, 'doorbells'): From faba071b1dee6c199b061ac0f3f759fcc34d6831 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:05:04 +0800 Subject: [PATCH 037/306] don't enter CALL body in assign fixups (#17527) * fix python time regression in mxfp4 * this saves even more time * s_nop test * itertools count * cleanup --- test/external/external_test_schedule_scaling.py | 17 +++++++++++++++-- tinygrad/schedule/indexing.py | 2 +- tinygrad/schedule/rangeify.py | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/test/external/external_test_schedule_scaling.py b/test/external/external_test_schedule_scaling.py index e7026508ee0d5..e97697ae32045 100644 --- a/test/external/external_test_schedule_scaling.py +++ b/test/external/external_test_schedule_scaling.py @@ -1,5 +1,5 @@ -import unittest, time -from tinygrad import Tensor +import unittest, time, itertools +from tinygrad import Tensor, Context class TestScheduleScaling(unittest.TestCase): """Test that .schedule() scales linearly with graph size (no O(n^2) behavior).""" @@ -130,5 +130,18 @@ def concat_chain(n): return parts[0].cat(*parts[1:]) self._assert_linear(concat_chain) + @Context(DEV="NULL:HIP:gfx1100") + def test_custom_kernel_assign_scaling(self): + from tinygrad.uop.ops import UOp, Ops, KernelInfo + from tinygrad.runtime.autogen.amd.rdna3.ins import s_nop + count = itertools.count(0) + def custom_kernel_assign(n): + def custom_asm(out): + return UOp(Ops.PROGRAM, src=(UOp.sink(out, arg=KernelInfo(f"fxn_{next(count)}")), + UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=s_nop(i)) for i in range(n*8))))) + call = Tensor.custom_kernel(Tensor.empty(1), fxn=custom_asm)[0] + return Tensor.cat(*[Tensor.empty(1).assign(call+i) for i in range(n)]) + self._assert_linear(custom_kernel_assign, n_small=50, n_large=500) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index dcaa3f7f78e29..e1f8304adc830 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -35,7 +35,7 @@ def realize_srcs(ctx:IndexingContext, rb:UOp) -> None: def realize_store_after_src(ctx:IndexingContext, dest:UOp, src:UOp): # you don't usually have to do this for assign unless there's a WAR hazard like TestAssign.test_assign_double_diamond_reduce - if dest.base in src.backward_slice_with_self: ctx.realize_map[src] = None + if dest.base in src.toposort(enter_calls=False): ctx.realize_map[src] = None def realize_custom_kernel_srcs(ctx:IndexingContext, c:UOp) -> None: for s in c.src[1:]: diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 1dd683e3c0c13..2314ad7b2d27c 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -60,7 +60,7 @@ def _mop_index(r:UOp, idx:UOp): # 0. do some cleanup rewrites, mostly copied from the old stuff def fix_store_hazard(target:UOp, src:UOp): - if (base:=target.base) not in src.backward_slice_with_self: return None + if (base:=target.base) not in src.toposort(enter_calls=False): return None # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) reaches_base: dict[UOp, bool] = {} From b1859805b1496e5196b84305b9e759c29c06eccf Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:52:16 -0700 Subject: [PATCH 038/306] remove Ops.BIND (#17511) * remove Ops.BIND * param arg * simplify that * simplify * cleaner * props, not functions * param and buffer can share --- extra/export_model.py | 2 +- extra/gemm/triton_nv_matmul.py | 2 +- test/backend/test_linearizer.py | 4 +- test/null/test_graph_rewrite.py | 4 +- test/null/test_simplify_valid_idx.py | 2 +- test/null/test_uop_graph.py | 4 +- test/null/test_uop_symbolic.py | 21 +++++----- test/null/test_validate_oob.py | 4 +- test/null/test_viz.py | 8 ++-- tinygrad/codegen/gpudims.py | 4 +- tinygrad/codegen/late/coalesce.py | 2 +- tinygrad/codegen/simplify.py | 2 +- tinygrad/engine/jit.py | 2 +- tinygrad/engine/realize.py | 2 +- tinygrad/function.py | 2 +- tinygrad/runtime/support/hcq2.py | 4 +- tinygrad/schedule/__init__.py | 18 ++++---- tinygrad/schedule/indexing.py | 6 ++- tinygrad/schedule/rangeify.py | 18 ++++---- tinygrad/tensor.py | 12 +++--- tinygrad/uop/__init__.py | 3 -- tinygrad/uop/ops.py | 61 +++++++++++++++++----------- tinygrad/uop/render.py | 3 +- tinygrad/uop/spec.py | 19 ++++----- tinygrad/uop/symbolic.py | 6 +-- tinygrad/uop/validate.py | 3 +- tinygrad/uop/weak.py | 2 +- 27 files changed, 117 insertions(+), 103 deletions(-) diff --git a/extra/export_model.py b/extra/export_model.py index a0024494ba9dc..f5d5a09a840db 100644 --- a/extra/export_model.py +++ b/extra/export_model.py @@ -35,7 +35,7 @@ def name_of(bu:UOp, is_out:bool) -> str: return name for call in iter_kernel_calls(linear): - arg_uops = [b for b in call.src[1:] if b.op is not Ops.BIND] + arg_uops = [b for b in call.src[1:] if not b.is_bound_var] prg = to_program(call.src[0], Device[arg_uops[0].device].renderer) info = prg.arg functions[info.function_name] = prg.src[2].arg diff --git a/extra/gemm/triton_nv_matmul.py b/extra/gemm/triton_nv_matmul.py index 133f09227b762..8477b5cba486c 100644 --- a/extra/gemm/triton_nv_matmul.py +++ b/extra/gemm/triton_nv_matmul.py @@ -79,7 +79,7 @@ def matmul_kernel(c_ptr, a_ptr, b_ptr, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: linear, var_vals = C.linear_with_vars() last_call = linear.src[-1] ast = last_call.src[0] - bufs = [s.buffer for s in last_call.src[1:] if s.op is not Ops.BIND] + bufs = [s.buffer for s in last_call.src[1:] if not s.is_bound_var] src = compiled.asm["ptx"] # specify the shared memory here so we don't need to do it dynamically diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 5a43d6801168f..0f36811baaba0 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -30,7 +30,7 @@ def test_arg_dedup(self): c = ((a.shrink(((0, 2),)) - a.shrink(((2, 4),))) - (b.shrink(((0, 2),)) - b.shrink(((2, 4),)))) linear = c.schedule_linear() run_linear(linear) - rawbufs = [s.buffer for s in linear.src[-1].src[1:] if s.op is not Ops.BIND] + rawbufs = [s.buffer for s in linear.src[-1].src[1:] if not s.is_bound_var] assert len(rawbufs) == 3 and set(rawbufs[1:]) == {a.uop.base.realized, b.uop.base.realized} np_c = (np_a[:2] - np_a[2:]) - (np_b[:2] - np_b[2:]) np.testing.assert_allclose(np_c, c.numpy(), atol=1e-4, rtol=1e-4) @@ -411,7 +411,7 @@ def helper_realized_ast(r:Tensor|list[Tensor]) -> tuple[UOp, list[Buffer]]: last_call = linear.src[-1] ast = last_call.src[0] assert ast.op is Ops.SINK, f"helper_realized_ast expects a SINK {last_call}" - last_bufs = [s.buffer for s in last_call.src[1:] if s.op is not Ops.BIND] + last_bufs = [s.buffer for s in last_call.src[1:] if not s.is_bound_var] # now all input buffers in last_call should be realized # create fresh buffers for the outputs bufs = [Buffer(x.device, x.size, x.dtype).allocate() if i < len(ast.src) else x for i,x in enumerate(last_bufs)] diff --git a/test/null/test_graph_rewrite.py b/test/null/test_graph_rewrite.py index 9194cc01bd3ee..06aa090359010 100644 --- a/test/null/test_graph_rewrite.py +++ b/test/null/test_graph_rewrite.py @@ -143,13 +143,13 @@ def test_full_graph_rewrite_modulo_large_divisor(self): self.assertIs(apply_rewrite(x_var_uop.cast(dtypes.weakint) % 10).render(simplify=False), x_var_uop.render(simplify=False)) def test_full_graph_rewrite_division_with_remainder(self): - x_var_uop = UOp.variable('x', 7, 9) + x_var_uop = UOp.variable('x', 7, 9, param=True) optimized_sink = apply_rewrite(x_var_uop // 2) for x_value in range(7, 10): self.assertEqual(x_value // 2, evaluate_uop(optimized_sink, {'x': x_value})) def test_full_graph_rewrite_complex_mod_div_expression(self): - x_var_uop = UOp.variable('x', 1, 10) + x_var_uop = UOp.variable('x', 1, 10, param=True) optimized_sink = apply_rewrite(((x_var_uop * 5) % 3) // 2) for x_value in range(1, 11): original_result = ((x_value * 5) % 3) // 2 diff --git a/test/null/test_simplify_valid_idx.py b/test/null/test_simplify_valid_idx.py index 754d713fe23fd..a5dc1d83d2908 100644 --- a/test/null/test_simplify_valid_idx.py +++ b/test/null/test_simplify_valid_idx.py @@ -25,7 +25,7 @@ def get_load_image_uop(image_shape:tuple[int, ...], valid:UOp, idx:tuple[UOp, UO )) def Special(expr, nmax): return UOp(Ops.SPECIAL, src=(UOp.const(nmax),), arg=expr) -def Variable(expr, nmin, nmax): return UOp.variable(expr, nmin, nmax) +def Variable(expr, nmin, nmax): return UOp.variable(expr, nmin, nmax, param=True) def Range(n, nmax): return UOp.range(nmax, n) class TestValidIdxSimplification(unittest.TestCase): diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index b7ff3fbebbf56..6e77bcec12ebe 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -157,7 +157,7 @@ def test_magic_4(self): self.assertEqual(nout.val, 3.0) def test_depth_2_fold(self): - v = UOp.variable("v", 0, 1, dtypes.float) + v = UOp.variable("v", 0, 1, dtypes.float, param=True) c1 = UOp.const(1.0) c2 = UOp.const(2.0) nout = graph_rewrite(v+c1+c2, simple_pm) @@ -339,7 +339,7 @@ def test_double_cast_fold(self): self.assertEqual(len([x for x in uops if x.op is Ops.CAST]), 1) def test_depth_2_const_fold(self): - v = UOp.variable("tmp", 0, 1, dtypes.int) + v = UOp.variable("tmp", 0, 1, dtypes.int, param=True) c2 = UOp.const(2, dtypes.int) c4 = UOp.const(4, dtypes.int) vc = v+c2 diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index d13afaa2eb98f..6c267257baaf3 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -16,7 +16,8 @@ def check_uop_against_string(self, v:UOp, s:str): s_eval = graph_rewrite(s_eval, commutative, name="cannonicalize eval") self.assertIs(s_eval, v, f"eval did not match simplified: {s_eval} != {v.render()} for {s}") -def Variable(name: str, min_val: ConstType, max_val: ConstType, dtype: DType=dtypes.weakint): return UOp.variable(name,min_val,max_val,dtype) +def Variable(name: str, min_val: ConstType, max_val: ConstType, dtype: DType=dtypes.weakint): + return UOp.variable(name, min_val, max_val, dtype, param=True) def uconst(val): return UOp.const(val) def usum(ops): return functools.reduce(lambda x,y: x+y, ops) def uand(ops): return functools.reduce(lambda x,y: x*y, ops) @@ -442,7 +443,7 @@ def test_and_remove(self): self.helper_test_variable(uand([uconst(1), Variable("a", 0, 1)]), 0, 1, "a") def test_masked_shr_fold(self): - x = UOp.variable('x', 0, 255, dtype=dtypes.uint32) + x = UOp.variable('x', 0, 255, dtype=dtypes.uint32, param=True) self.helper_test_variable((x & -4) >> 2, 0, 63, "(x>>2)") def test_bool_or_not_tautology(self): @@ -483,12 +484,12 @@ def test_mul_div(self): def test_div_drop_small_terms(self): # from openpilot, shouldnt simplify - gidx0 = UOp.variable("gidx0", 0, 10) - gidx1 = UOp.variable("gidx1", 0, 10) - lidx0 = UOp.variable("lidx0", 0, 1) - lidx1 = UOp.variable("lidx1", 0, 1) - ridx1005 = UOp.variable("ridx1005", 0, 2) - ridx1006 = UOp.variable("ridx1006", 0, 2) + gidx0 = UOp.variable("gidx0", 0, 10, param=True) + gidx1 = UOp.variable("gidx1", 0, 10, param=True) + lidx0 = UOp.variable("lidx0", 0, 1, param=True) + lidx1 = UOp.variable("lidx1", 0, 1, param=True) + ridx1005 = UOp.variable("ridx1005", 0, 2, param=True) + ridx1006 = UOp.variable("ridx1006", 0, 2, param=True) self.helper_test_variable((lidx1+((gidx1*18)+(ridx1005*18)+(lidx0*162))+(gidx0*2)+(ridx1006*2)+-40)//18, -3, 20, "(gidx1+ridx1005+lidx0*9+(gidx0+ridx1006+7)//9+-3)") @@ -997,7 +998,7 @@ def test_bool_ne_false(self): self.helper_test_variable(cond.ne(False), 0, 1, "(x<2)") def test_bitcast_chain(self): - a = UOp.variable("a", 0, 3, dtype=dtypes.int32) + a = UOp.variable("a", 0, 3, dtype=dtypes.int32, param=True) self.assertIs(graph_rewrite(a.bitcast(dtypes.float32).bitcast(a.dtype), sym), a) def test_negation_in_where(self): @@ -1180,7 +1181,7 @@ def test_dedup(self): assert (a//4 + a//6).variables() == [a] def test_variable_min_eq_max_bind_folds(self): - b = Variable("x", 1, 1).bind(1) + b = UOp.variable("x", 1, 1).bind(1) s = b.simplify() self.assertEqual(s.op, Ops.CONST) self.assertEqual(s.val, 1) diff --git a/test/null/test_validate_oob.py b/test/null/test_validate_oob.py index 2d52f8e70e044..abc7a9bc06419 100644 --- a/test/null/test_validate_oob.py +++ b/test/null/test_validate_oob.py @@ -1,10 +1,12 @@ import unittest -from tinygrad import dtypes, Variable +from tinygrad import dtypes from tinygrad.dtype import AddrSpace from tinygrad.helpers import Context from tinygrad.uop.ops import Ops, UOp, AxisType from test.helpers import to_uops_list +def Variable(name, nmin, nmax): return UOp.variable(name, nmin, nmax, param=True) + class TestValidateOOB(unittest.TestCase): """Test z3 validation of index bounds for different ALU ops and patterns.""" diff --git a/test/null/test_viz.py b/test/null/test_viz.py index c6cdb6213c925..49fbff61406c8 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -305,10 +305,10 @@ def assertStepEqual(self, step:dict, want:dict): def test_tree_view(self): with save_viz() as viz: - a = UOp.variable("a",0,10) - b = UOp.variable("b",0,10) - c = UOp.variable("c",0,10) - d = UOp.variable("d",0,10) + a = UOp.variable("a",0,10,param=True) + b = UOp.variable("b",0,10,param=True) + c = UOp.variable("c",0,10,param=True) + d = UOp.variable("d",0,10,param=True) sink = UOp.sink(a+b, c+d) def tree_rewrite(): return graph_rewrite(sink, root, name="root") tree_rewrite() diff --git a/tinygrad/codegen/gpudims.py b/tinygrad/codegen/gpudims.py index 0a22c56874ca9..f479007c820bb 100644 --- a/tinygrad/codegen/gpudims.py +++ b/tinygrad/codegen/gpudims.py @@ -57,7 +57,7 @@ def add_gpudims(ctx:Renderer, s:UOp): # get the idxs ki: KernelInfo = s.arg - if ctx.has_threads: idxs = [UOp.variable("core_id", 0, int(global_shape[0])-1, dtypes.int).cast(dtypes.weakint)] + if ctx.has_threads: idxs = [UOp.variable("core_id", 0, int(global_shape[0])-1, dtypes.int, param=True).cast(dtypes.weakint)] elif ki.dont_use_locals: assert not local_dims, "can't use locals if there's no local dims" idxs = get_grouped_dims("idx", global_shape, ctx.global_max, reverse=True) @@ -89,7 +89,7 @@ def add_gpudims(ctx:Renderer, s:UOp): pm_device_to_var = PatternMatcher([ # the DEVICE axis is not a program axis, it's bound per device at launch. lower it to the _device_num variable (like SPECIAL for devices) - (UPat(Ops.RANGE, name="r"), lambda r: UOp.variable("_device_num", 0, r.vmax, dtype=r.dtype) if r.arg[-1] is AxisType.DEVICE else None), + (UPat(Ops.RANGE, name="r"), lambda r: UOp.variable("_device_num", 0, r.vmax, dtype=r.dtype, param=True) if r.arg[-1] is AxisType.DEVICE else None), # ENDs that closed a DEVICE range no longer close it (UPat(Ops.END, name="e"), lambda e: e.replace(src=(e.src[0],)+tuple(s for s in e.src[1:] if s.op is not Ops.PARAM)) if any(s.op is Ops.PARAM and s.arg.name == '_device_num' for s in e.src[1:]) else None), diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index f72c951c16ee0..64e65f2320541 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -26,7 +26,7 @@ def _drop_valid_stmts(valid:UOp, idx:UOp, height:int, width:int) -> list[UOp]: # check if idx is out of bound when X is on the wrong side of the bound: X in [c+1, vmax] or [vmin, c-1] lo, hi = (c + 1, X.vmax) if is_upper_bound else (X.vmin, c - 1) if lo <= hi: - fake = UOp.variable(f"fake{i}", lo, hi, X.dtype) + fake = UOp.variable(f"fake{i}", lo, hi, X.dtype, param=True) subs = [{X: fake}] # idx may not have X itself, so also substitute a term of X: v -> fake - (X - v) terms = list(X.split_uop(Ops.ADD)) diff --git a/tinygrad/codegen/simplify.py b/tinygrad/codegen/simplify.py index 879dd0e6c9921..a1d9d4f01b492 100644 --- a/tinygrad/codegen/simplify.py +++ b/tinygrad/codegen/simplify.py @@ -137,7 +137,7 @@ def reduce_collapse(red:UOp, u:UOp, pm:PatternMatcher=pm_reduce_collapse) -> UOp for u in included: for s in u.src: if s in included or s in replaces or s.op in {Ops.CONST, Ops.PARAM, Ops.BUFFER}: continue - replaces[s] = UOp.variable(f'in{len(replaces)}', s.vmin, s.vmax, s.dtype) + replaces[s] = UOp.variable(f'in{len(replaces)}', s.vmin, s.vmax, s.dtype, param=True) collapse_fxn = u.substitute(replaces).reduce(r, arg=Ops.ADD) sink = graph_rewrite(collapse_fxn, pm, name="reduce_collapse") if not no_range(sink): return None diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index d67197c98b9b6..f812f0b9eba4c 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -44,7 +44,7 @@ def flush_batch(): current_batch, current_batch_devs = [], [] for si in linear.src: - devs = dedup([Device[x] for b in si.src[1:] if b.op is not Ops.BIND for x in (b.device if isinstance(b.device, tuple) else (b.device,))]) + devs = dedup([Device[x] for b in si.src[1:] if not b.is_bound_var for x in (b.device if isinstance(b.device, tuple) else (b.device,))]) graph_t = graph_class(devs[0]) if devs[0].graph is not None else None can_graph = graph_t is not None and graph_t.supports_uop(devs, si) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 4313a8b9daec8..11458d7696ebf 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -12,7 +12,7 @@ # **************** Helpers **************** -def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if s.op is not Ops.BIND) +def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if not s.is_bound_var) def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.src[0] diff --git a/tinygrad/function.py b/tinygrad/function.py index edb1691d9d974..b11ba4c25bbc4 100644 --- a/tinygrad/function.py +++ b/tinygrad/function.py @@ -12,7 +12,7 @@ def add_to_ctx(ctx, x:UOp): return ret pm_ctx = PatternMatcher([ - (UPat((Ops.BUFFER, Ops.BIND), name="x"), add_to_ctx), + (UPat(Ops.BUFFER, name="x"), add_to_ctx), (UPat((Ops.AFTER, Ops.CONTIGUOUS), name="x"), lambda ctx,x: add_to_ctx(ctx,x) if not x.op_in_backward_slice_with_self(Ops.PARAM) and x.op_in_backward_slice_with_self(Ops.BUFFER) else None), ]) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index bdb03c7bb09b8..d57cc11d99b7f 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -87,8 +87,8 @@ def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp|None: bufs, slots = ctx for s in call.src[1:]: - if s.op not in (Ops.PARAM, Ops.BIND) and slots.setdefault(s, len(bufs)) == len(bufs): bufs.append(s) - return call.replace(src=call.src[:1] + tuple(s if s.op in (Ops.PARAM, Ops.BIND) else s.param_like(slots[s]) for s in call.src[1:])) + if s.op is not Ops.PARAM and not s.is_bound_var and slots.setdefault(s, len(bufs)) == len(bufs): bufs.append(s) + return call.replace(src=call.src[:1] + tuple(s if s.op is Ops.PARAM or s.is_bound_var else s.param_like(slots[s]) for s in call.src[1:])) pm_replace_buffers = PatternMatcher([(UPat(Ops.CALL, name="call"), replace_call_buffers)]) # ***************** diff --git a/tinygrad/schedule/__init__.py b/tinygrad/schedule/__init__.py index a98e97b7c40ca..2a87a48fb8f41 100644 --- a/tinygrad/schedule/__init__.py +++ b/tinygrad/schedule/__init__.py @@ -8,14 +8,13 @@ # unwrap VIEW/CAST/etc to find the actual data source (kernel output, buffer, or multi-device op) def _unwrap_src(s: UOp) -> UOp: - while len(s.src) and s.op not in {Ops.AFTER, Ops.BUFFER, Ops.PARAM, Ops.MSELECT, Ops.MSTACK, Ops.BIND}: s = s.src[0] + while len(s.src) and s.op not in {Ops.AFTER, Ops.BUFFER, Ops.PARAM, Ops.MSELECT, Ops.MSTACK}: s = s.src[0] return s -# a buffer state is AFTER | BUFFER | PARAM. MSELECT/MSTACK join per-device states, BIND is not a buffer dependency +# a buffer state is AFTER | BUFFER | PARAM. MSELECT/MSTACK join per-device states def _states(s: UOp) -> list[UOp]: s = _unwrap_src(s) if s.op in {Ops.MSELECT, Ops.MSTACK}: return [st for ss in s.src for st in _states(ss)] - if s.op is Ops.BIND: return [] assert s.op in {Ops.AFTER, Ops.BUFFER, Ops.PARAM}, f"input to kernel must resolve to a buffer state, not {s.op}" return [s] @@ -71,7 +70,7 @@ def create_schedule(sched_sink:UOp) -> UOp: else: k = rk.src[0] if rk.op is Ops.END else rk assert k.op is Ops.CALL, f"unexpected op in queue: {k.op}" - buf_uops = tuple(_unwrap_src(s).buf_uop for s in k.src[1:] if s.op is not Ops.BIND) + buf_uops = tuple(_unwrap_src(s).buf_uop for s in k.src[1:] if not s.is_bound_var) linearized.append(k.src[0].call(*buf_uops)) for x in children.get(rk, []): in_degree[x] -= 1 @@ -100,7 +99,8 @@ def create_new_buffer(ctx:tuple[dict[UOp, UOp], tuple[UOp, ...]], b:UOp): def resolve_linear_call(linear_call:UOp): linear = graph_rewrite(linear_call.src[0], pm_post_sched_cache, ctx=({}, linear_call.src[1:]), walk=True, name="params to buffers") - binds = {f"p{i}":x.src[0] for i,x in enumerate(linear_call.src[1:]) if x.op is Ops.BIND} + # map the call body params back to the original Variables stored in the call args + binds = {f"p{i}":x.src[0].replace(op=Ops.PARAM) for i,x in enumerate(linear_call.src[1:]) if x.is_bound_var} return linear.substitute({v:binds[v.expr] for v in linear.variables() if v.expr in binds}, enter_calls=True, name="resolve scalar params") pm_resolve_linear_call = PatternMatcher([ @@ -184,13 +184,13 @@ def create_linear_with_vars(big_sink:UOp) -> tuple[UOp, dict[str, int]]: # vars used in the schedule used_vars = set().union(*[{v.expr for v in si.src[0].variables()} for si in linear.src]) - # get var_vals + # get var_vals from the bound Variables in the call args var_vals: dict[str, int] = {} for b in big_sink.src[1:]: - if b.op is Ops.BIND: - nm = b.src[0].expr + if b.is_bound_var: + v, val = b.unbind() + nm = v.expr if nm not in used_vars: continue - val = b.src[1].val if var_vals.get(nm, val) != val: raise RuntimeError(f"bind mismatch on {nm}, {var_vals[nm]} != {val}") var_vals[nm] = val diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index e1f8304adc830..430542a932cb1 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -24,7 +24,7 @@ def new_range(self, s:sint, axistype:AxisType=AxisType.WEAK) -> UOp: ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, - Ops.CONST, Ops.BIND, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, + Ops.CONST, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, Ops.LOAD, Ops.CALL, Ops.FUNCTION} def realize(ctx:IndexingContext, tr:UOp) -> None: ctx.realize_map[tr] = None @@ -69,7 +69,9 @@ def broadcast_rngs(x:UOp, src:UOp, rngs:tuple[UOp, ...]) -> tuple[UOp, ...]: # TODO: srcs contain (real data srcs, something else, ranges) and the boundary is confusing. see range_start def data_srcs(op:Ops, src:tuple[UOp, ...]) -> tuple[UOp, ...]: - if op in {Ops.PARAM, Ops.BUFFER, Ops.RANGE, Ops.SPECIAL, Ops.BIND}: return () + if op in {Ops.PARAM, Ops.BUFFER, Ops.RANGE, Ops.SPECIAL}: return () + # the store of a bound Variable only carries the input value, it has no data srcs + if op is Ops.STORE and src[0].is_variable: return () if op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.REDUCE, Ops.AFTER, Ops.END}: return src[:1] return src diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 2314ad7b2d27c..bf64d55761462 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -461,11 +461,12 @@ def remove_noop_afters(x:UOp) -> UOp|None: class LocalAddBufferContext: dg:int = 0 map:dict = field(default_factory=dict) - vars:dict = field(default_factory=dict) range:int = 0 opts:tuple|None = None def debuf(ctx:LocalAddBufferContext, buf:UOp): + # Variables (ALU buffers with a value range) are scalar symbolic values, not real buffers: they become ALU params with no slot + if buf.is_variable: return buf.replace(op=Ops.PARAM) param = UOp(Ops.PARAM, src=(UOp.const(prod(buf.max_shape)),), arg=ParamArg(ctx.dg, buf.dtype, addrspace=buf.addrspace, device=buf.device)) ret = param.reshape(buf.max_shape) @@ -475,10 +476,6 @@ def debuf(ctx:LocalAddBufferContext, buf:UOp): ctx.dg += 1 return ret -def unbind_kernel(ctx:LocalAddBufferContext, b:UOp): - ctx.vars[b] = None - return b.src[0] - def handle_after(ctx:LocalAddBufferContext, after:UOp): if after.addrspace == AddrSpace.LOCAL: return None buf = after.buf_uop @@ -502,8 +499,7 @@ def find_bufs(x:UOp): (UPat(Ops.STORE, name="x"), find_bufs), (UPat((Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), name="buf"), debuf), (UPat(Ops.PARAM, name="v"), lambda v: - UOp.variable(v.arg.name, v.arg.vmin_vmax[0], v.arg.vmin_vmax[1], v.dtype, multiple_of=v.arg.multiple_of) - if v.arg.name is not None and v.arg.vmin_vmax is not None else None), + v.replace(arg=replace(v.arg, slot=-1)) if v.arg.name is not None and v.arg.vmin_vmax is not None and v.arg.slot != -1 else None), # this renumbers the params (UPat(Ops.PARAM, name="buf"), lambda ctx, buf: @@ -512,7 +508,8 @@ def find_bufs(x:UOp): # ALU params are scalar symbolic values, not buffers. (UPat(Ops.INDEX, src=(UPat(Ops.PARAM, name="v"),)), lambda v: v if v.addrspace == AddrSpace.ALU else None), - (UPat(Ops.BIND, name="b"), unbind_kernel), + # bound Variables are stores into Variable buffers: strip the store, the buffer becomes an ALU param via debuf + (UPat(Ops.AFTER, name="b"), lambda b: b.src[0] if b.is_bound_var else None), (UPat(Ops.AFTER, name="after"), handle_after), # remove device from local BUFFERIZE @@ -541,13 +538,16 @@ def get_contiguous(ctx:LocalAddBufferContext, x:UOp): def split_store(x:UOp) -> UOp|None: # if we have any open ranges here, we don't split. open DEVICE ranges are fine, they are bound per device at launch if any(r.arg[-1] is not AxisType.DEVICE for r in x.ranges): return None + # the store of a bound Variable is an input value, not a kernel + st = x.src[0] if x.op is Ops.END else x + if st.op is Ops.STORE and st.src[0].is_variable: return None # local kernel rewrite lctx = LocalAddBufferContext() ret = graph_rewrite(x, to_define_global+pm_flatten_range+rangeify_codegen, ctx=lctx, name="kernel split", bottom_up=True) # create the Kernel. NOTE: buffers can be on different devices here now, they are compiled to SDMA copies later by schedule - return ret.sink(arg=KernelInfo(opts_to_apply=lctx.opts)).call(*lctx.map.values(), *lctx.vars.keys()) + return ret.sink(arg=KernelInfo(opts_to_apply=lctx.opts)).call(*lctx.map.values()) split_kernels = PatternMatcher([ (UPat((Ops.STORE, Ops.END), name="x"), split_store), diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index c2b654a7afe27..11885a387ac9c 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -109,7 +109,7 @@ def _precompiled_output_redirect(s:UOp, t:UOp) -> UOp|None: def transform_precompiled_call(c:UOp) -> UOp|None: if not c.arg.precompile: return None assert c.src[0].op is Ops.TUPLE, f"expected TUPLE body for precompiled FUNCTION, got {c.src[0].op}" - input_buffers = tuple(x.contiguous() if x.op not in {Ops.AFTER, Ops.BIND} else x for x in c.src[1:]) + input_buffers = tuple(x.contiguous() if x.op is not Ops.AFTER else x for x in c.src[1:]) # add the outputs to the call srcs = c.src[0].src @@ -176,6 +176,8 @@ def transform_precompiled_call(c:UOp) -> UOp|None: ]) def finalize_after(ctx:AllocCtx, x:UOp): + # bound Variables are call inputs, not assigns: they stay in the graph and pm_replace_buf turns them into call args + if x.is_bound_var: return None # untagged: record as an assign for the call body if x.tag is None: ctx.assigns.append(x) @@ -196,7 +198,7 @@ def finalize_after(ctx:AllocCtx, x:UOp): def replace_input_buffer(ctx:AllocCtx, b:UOp): ctx.replacements.append(b) - if b.op is Ops.BIND: return b.param_like(len(ctx.replacements)-1) + if b.is_bound_var or b.is_variable: return b.param_like(len(ctx.replacements)-1) return UOp.param(len(ctx.replacements)-1, b.dtype, b.shape, b.device, addrspace=b.addrspace if b.addrspace is not None else AddrSpace.GLOBAL) @@ -214,8 +216,8 @@ def replace_input_view(ctx:AllocCtx, b:UOp): return replace_input_buffer(ctx, b) # replace SHRINK with PARAM (UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), name="b", allow_any_len=True), replace_input_view), (UPat(Ops.BITCAST, src=(UPat.any(UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), allow_any_len=True), UPat(Ops.BUFFER)),), name="b"), replace_input_view), - # strip value from BIND for cache key normalization, so different values hit same cache - (UPat(Ops.BIND, src=(UPat(Ops.PARAM), UPat(Ops.CONST)), name="b"), replace_input_buffer), + # strip the stored value from bound Variables for cache key normalization, so different values hit same cache + (UPat(Ops.AFTER, name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if b.is_bound_var else None), ]) @rewrite_group(lambda _,ret: f"Callify {pluralize('Buffer', len(ret[1]))}") @@ -741,7 +743,7 @@ def decode_hevc_frame(self, frame_pos:Variable, shape:tuple[int,...], state:Tens the reference frames (`ref_frames`). """ ref_frames = [x.contiguous() for x in ref_frames or []] - assert frame_pos.op is Ops.BIND, "frame_pos must be a bound Variable" + assert frame_pos.is_bound_var, "frame_pos must be a bound Variable" srcs = (out:=Tensor.empty(*shape, device=self.device, dtype=self.dtype), self.contiguous(), state.contiguous(), *ref_frames) fn = UOp(Ops.CUSTOM_FUNCTION, src=(frame_pos.src[0], *[UOp.const(s, dtypes.int) for s in shape]), arg="encdec") return Tensor(out.uop.after(fn.call(*[s.uop for s in srcs], frame_pos))) diff --git a/tinygrad/uop/__init__.py b/tinygrad/uop/__init__.py index 2252c8b926578..d3a516e32a7fe 100644 --- a/tinygrad/uop/__init__.py +++ b/tinygrad/uop/__init__.py @@ -13,9 +13,6 @@ def _generate_next_value_(_, __, ___, last_values): return 1 + max([0, *last_val class Ops(FastEnum): # ** 1 -- defines/special ** - # BIND pairs a symbolic PARAM with a concrete value - BIND = auto() - # this is a RANGE for GPU dimensions, similar to symbolic shapes but not exactly SPECIAL = auto() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 0350b103f282a..fadcdb866ed88 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -150,9 +150,6 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: case Ops.STACK: if len(src) == 0: return dtypes.void return promo_dtype(src) - case Ops.BIND: - assert src[0].dtype == src[1].dtype, f"bind dtype mismatch {src[0].dtype} != {src[1].dtype}" - return src[0].dtype case Ops.WMMA: # WMMA output dtype is the accumulator dtype (src[2]) return src[2].dtype @@ -377,7 +374,7 @@ def _shape(self) -> tuple[sint, ...]|None: # some ops init the shape case Ops.GETADDR: return () - case Ops.BIND | Ops.RANGE | Ops.SPECIAL: return () + case Ops.RANGE | Ops.SPECIAL: return () case Ops.BINARY: return (len(self.arg),) case Ops.BUFFER: if len(self.src): return self.src[0].as_shape @@ -960,8 +957,8 @@ def realized(self) -> Buffer|MultiBuffer|None: if self.op is Ops.UNSHARD: return self.src[0].realized # only these can be realized if self.op not in (Ops.BUFFER, Ops.MSTACK): return None - # LOCAL/REG scratch buffers are never realized - if self.op is Ops.BUFFER and self.addrspace in (AddrSpace.LOCAL, AddrSpace.REG): return None + # LOCAL/REG scratch buffers are never realized, and Variables (ALU) have no real storage + if self.op is Ops.BUFFER and self.addrspace in (AddrSpace.LOCAL, AddrSpace.REG, AddrSpace.ALU): return None # an unbacked intermediate BUFFER (directly or as an MSTACK source) is not realized if any(b.op is Ops.BUFFER and buffers.get(b) is None for b in self.backward_slice_with_self): return None # NOTE: this is used by the JIT to determine which inputs we capture @@ -972,29 +969,41 @@ def is_realized(self) -> bool: return self.base.realized is not None # *** uop Variable stuff *** @staticmethod - def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint, multiple_of:int=1) -> UOp: - return UOp(Ops.PARAM, src=(shape_to_shape_arg(()),), - arg=ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), multiple_of=multiple_of, addrspace=AddrSpace.ALU)) + def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weakint, multiple_of:int=1, param:bool=False) -> UOp: + # a Variable is a 0-d BUFFER in the ALU addrspace; binding it is storing a CONST into it + # param=True creates the kernel-side form directly: an ALU PARAM (what the BUFFER becomes inside kernels) + arg = ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), multiple_of=multiple_of, addrspace=AddrSpace.ALU) + return UOp(Ops.PARAM if param else Ops.BUFFER, src=(shape_to_shape_arg(()),), arg=arg) + @property + def is_variable(self) -> bool: + # a Variable is a 0-d BUFFER in the ALU addrspace that carries a value range (it becomes a PARAM inside kernels) + return self.op is Ops.BUFFER and isinstance(self.arg, ParamArg) and \ + self.arg.vmin_vmax is not None and self.arg.addrspace is AddrSpace.ALU and self._shape == () + @property + def is_bound_var(self) -> bool: + # a bound Variable is bind()'s AFTER(var, STORE(var, CONST)) + return self.op is Ops.AFTER and self.src[0].is_variable and self.src[1].op is Ops.STORE and \ + self.src[1].src[0] is self.src[0] and self.src[1].src[1].op is Ops.CONST and len(self.src) == 2 @property def expr(self) -> str: - assert self.op is Ops.PARAM + assert self.op in {Ops.PARAM, Ops.BUFFER} return unwrap(self.arg.name) def bind(self, val:int|UOp): - assert self.op is Ops.PARAM and self.addrspace is AddrSpace.ALU, f"op is {self.op}, need PARAM" + assert self.is_variable, f"op is {self.op}, need Variable" uval = self.const_like(val) if isinstance(val, int) else val assert self.vmin <= uval.vmin and uval.vmax <= self.vmax, f"bind {val} not in range [{self.vmin}, {self.vmax}]" assert uval.divides(self.arg.multiple_of) is not None, f"bind {val} not divisible by {self.arg.multiple_of}" - return UOp(Ops.BIND, src=(self, uval)) + return self.after(self.store(uval)) def unbind(self) -> tuple[Variable, int]: - assert self.op is Ops.BIND and self.src[0].op is Ops.PARAM and self.src[1].op is Ops.CONST, f"can't unbind {self}" - return self.src[0], self.src[1].val + assert self.is_bound_var, f"can't unbind {self}" + return self.src[0], self.src[1].src[1].val def unbind_all(self) -> tuple[UOp, dict[Variable, int]]: ret:dict[Variable, int] = {} return graph_rewrite(self, pm_unbind, ctx=ret), ret def variables(self) -> list[Variable]: - return sorted({x if x.op is Ops.PARAM else UOp.variable("_device_num", 0, x.vmax, dtype=x.dtype) - for x in self.backward_slice_with_self if (x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or x.op is Ops.PARAM - and x.arg.addrspace is AddrSpace.ALU}, key=lambda v: v.expr) + return sorted({x if x.op in {Ops.PARAM, Ops.BUFFER} else UOp.variable("_device_num", 0, x.vmax, dtype=x.dtype, param=True) + for x in self.backward_slice_with_self if (x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or + (x.op is Ops.PARAM and x.arg.addrspace is AddrSpace.ALU) or x.is_variable}, key=lambda v: v.expr) # *** uop symbolic stuff *** @@ -1005,7 +1014,7 @@ def const_factor(self) -> int: if self.op is Ops.STACK: return math.gcd(*[x.const_factor() for x in self.src]) if self.op is Ops.ADD: return math.gcd(self.src[0].const_factor(), self.src[1].const_factor()) if self.op is Ops.MUL: return self.src[0].val if self.src[0].op is Ops.CONST else self.src[1].val if self.src[1].op is Ops.CONST else 1 - if self.op is Ops.PARAM and self.arg.multiple_of is not None: return self.arg.multiple_of + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.multiple_of is not None: return self.arg.multiple_of return 1 def divides(self, v:int) -> UOp|None: if v==1: return self @@ -1017,7 +1026,8 @@ def divides(self, v:int) -> UOp|None: if self.op is Ops.MUL: if (d0:=self.src[0].divides(v)) is not None: return d0 * self.src[1] if (d1:=self.src[1].divides(v)) is not None: return self.src[0] * d1 - if self.op is Ops.PARAM and self.arg.multiple_of is not None: return self // v if self.arg.multiple_of%v == 0 else None + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.multiple_of is not None: + return self // v if self.arg.multiple_of%v == 0 else None return None # generic None if we aren't sure def pop_const(self, op=Ops.ADD) -> tuple[UOp, PyConst]: # NOTE: assume Invalid ALU is resolved return (self.src[0], self.src[1].val) if self.op is op and self.src[1].op is Ops.CONST else (self, identity_element(op, self.dtype)) @@ -1082,9 +1092,9 @@ def _min_max(self) -> tuple[PyConst, PyConst]: # float has NAN issue and we use explicit NAN in transcendental if self.op is Ops.WHERE and dtypes.is_int(self.dtype): return min(self.src[1].vmin, self.src[2].vmin), max(self.src[1].vmax, self.src[2].vmax) # NOTE: returned UOp is assumed to be CONST - if self.op is Ops.PARAM and self.arg.vmin_vmax is not None: return self.arg.vmin_vmax + if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.vmin_vmax is not None: return self.arg.vmin_vmax if self.op in (Ops.RANGE, Ops.SPECIAL) and self.dtype is not dtypes.void: return 0, (self.src[0]-1).vmax - if self.op is Ops.BIND: return self.src[0]._min_max # ignore the bound value + if self.op is Ops.AFTER: return self.src[0]._min_max if self.op is Ops.STACK: return min(x.vmin for x in self.src), max(x.vmax for x in self.src) if self.op is Ops.CONST and self.val is not Invalid: return self.val, self.val if self.op is Ops.INDEX: return self.src[0]._min_max @@ -1101,7 +1111,7 @@ def _min_max(self) -> tuple[PyConst, PyConst]: def _sym_fxn(self): from tinygrad.uop.render import _render_with_splits, renderer_infer sself = self.simplify() - varnames = tuple(dedup(x.expr for x in sself.toposort() if x.op is Ops.PARAM and x.arg.addrspace == AddrSpace.ALU)) + varnames = tuple(dedup(x.expr for x in sself.toposort() if (x.op is Ops.PARAM and x.arg.addrspace == AddrSpace.ALU) or x.is_variable)) # TODO: sanitize varnames, or don't use naked eval while staying fast ret = _render_with_splits(list(sself.toposort()), renderer_infer, {sself}) lines = [f" {k}={v}" for k,v in ret.items() if k != "ast"] + [f" return {ret['ast']}"] @@ -1156,7 +1166,10 @@ def param(slot:int, dtype:DType, shape:tuple[sint, ...]|None=None, device=None, src: tuple[UOp, ...] = (UOp(Ops.NOOP) if shape is None else shape_to_shape_arg(shape),) return UOp(Ops.PARAM, src=src, arg=ParamArg(slot, dtype, vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) def param_like(self, slot:int): - if self.op is Ops.BIND: return self.src[0].replace(arg=replace(self.src[0].arg, slot=slot, name=f"p{slot}")) + # Variables become ALU params in the call body; the stored value (if bound) stays in the call args + if self.is_bound_var or self.is_variable: + b = self.src[0] if self.op is Ops.AFTER else self + return UOp(Ops.PARAM, src=b.src, arg=replace(b.arg, slot=slot, name=f"p{slot}")) addrspace = self.addrspace if self.addrspace is not None else AddrSpace.GLOBAL return UOp.param(slot, self.dtype, self.shard_shape if self.axis is not None else self._shape, self.device, addrspace=addrspace, axis=self.axis) @@ -1754,7 +1767,7 @@ def do_unbind(ctx:dict[Variable, int], x:UOp): v,i = x.unbind() ctx[v] = i return v -pm_unbind = PatternMatcher([(UPat(Ops.BIND, name="x"), do_unbind)]) +pm_unbind = PatternMatcher([(UPat(Ops.AFTER, name="x"), lambda ctx,x: do_unbind(ctx,x) if x.is_bound_var else None)]) # ctx is source UOp for which we are finding a contiguous view for. used in contiguous_view_offset pm_contiguous_view_offset = PatternMatcher([ diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 03133e1cea285..6718d4636e03b 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -33,12 +33,13 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: renderer = PatternMatcher([ (UPat(Ops.PARAM, name="x"), lambda x: x.arg.name if x.arg.name is not None else f"p{x.arg.slot}"), + (UPat(Ops.BUFFER, name="x"), lambda x: x.arg.name if isinstance(x.arg, ParamArg) and x.arg.name is not None else f"b{x.arg.slot}"), + (UPat(Ops.AFTER, name="x"), lambda ctx,x: ctx[x.src[0]]), (UPat((Ops.SPECIAL), name="x"), lambda x: x.arg), (UPat(Ops.RANGE, dtypes.void, name="x"), lambda x: f"loop{x.arg[0]}"), (UPat(Ops.RANGE, name="x"), lambda x: f"r{range_str(x)}"), (UPat(Ops.CONST, name="x"), lambda x: str(x.val)), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({str(x.dtype)[7:]})({ctx[x.src[0]]})"), - (UPat(Ops.BIND, name="x"), lambda ctx,x: ctx[x.src[0]]), (UPat(Ops.NEG, name="x"), lambda ctx,x: f"(-{ctx[x.src[0]]})"), (UPat(Ops.RECIPROCAL, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), (UPat(Ops.MAX, name="x"), lambda ctx,x: f"max({ctx[x.src[0]]}, {ctx[x.src[1]]})"), diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 4c19457aaf954..35fa4bb94f202 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -141,9 +141,8 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (isinstance(buf.dtype, DType) and matches_dtype(buf.src[0], dtypes.weakint) and is_device(buf.arg.device)) if isinstance(buf.arg, ParamArg) and buf.addrspace is AddrSpace.GLOBAL else None), - # Tensor variable bindings - (UPat(Ops.BIND, (dtypes.int, dtypes.long, dtypes.weakint,), (UPat(Ops.PARAM), UPat.cvar(dtype=(dtypes.int,dtypes.long,dtypes.weakint,))), arg=None), - lambda: True), + # a Variable is a 0-d ALU BUFFER with a value range and no device + (UPat(Ops.BUFFER, src=(UPat(),), name="buf"), lambda buf: buf.arg.device is None if buf.is_variable else None), # custom function (UPat(Ops.CUSTOM_FUNCTION, name="x"), lambda x: isinstance(x.arg, str)), @@ -241,9 +240,6 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # all loads/stores (UPat((Ops.LOAD, Ops.STORE)), lambda: True), - - # while BIND is being casted - (UPat(Ops.BIND, (dtypes.int, dtypes.weakint), (UPat(), UPat()), arg=None), lambda: True), ])+spec_tensor+spec_program+spec_hcq # ***** kernel graph spec ***** @@ -251,17 +247,16 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < spec_kernel_graph = PatternMatcher([ # sink (UPat(Ops.SINK, dtypes.void), lambda: True), - # bind - (UPat(Ops.BIND), lambda: True), - # const + stack to make vconsts + # the store of a bound Variable binds it: AFTER(BUFFER, STORE(BUFFER, CONST)) in call args + (UPat(Ops.STORE, dtypes.void, (UPat(Ops.BUFFER, name="b"), UPat(Ops.CONST))), lambda b: b.is_variable), + # const + stack to make vconsts and shape args (UPat(Ops.CONST, src=()), lambda: True), - (UPat(Ops.STACK, src=()), lambda: True), - (UPat(Ops.STACK, src=UPat((Ops.CONST, Ops.BIND, Ops.PARAM))), lambda: True), + (UPat(Ops.STACK, name="s"), lambda s: all(x.op in (Ops.CONST, Ops.PARAM) or x.is_variable or x.is_bound_var for x in s.src) or None), # linear for more kernels (TODO: we should enter non sink calls) #(UPat(Ops.LINEAR), lambda: True), # param is outside buffer, buffer is local buffer (UPat(Ops.PARAM, name="x"), lambda x: isinstance(x.arg, ParamArg)), - (UPat(Ops.BUFFER, name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace == AddrSpace.GLOBAL), + (UPat(Ops.BUFFER, name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.GLOBAL, AddrSpace.ALU)), # RESHAPE/BITCAST are NOOPs in the kernel graph (do we need them?) (UPat((Ops.RESHAPE, Ops.BITCAST)), lambda: True), # mstack/mselect diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index a2e82fb18099b..924325f426cbb 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -245,7 +245,7 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: # complementary zero branches under the same condition select directly (UPat.var("c").where(UPat.var("t"), 0) + UPat.var("c").where(0, UPat.var("f")), lambda c,t,f: c.where(t, f)), # ALU/variable min==max -> CONST - (UPat({Ops.CMPLT, Ops.CMPNE, Ops.FLOORDIV, Ops.FLOORMOD, Ops.PARAM, Ops.BIND, Ops.SPECIAL}, name="x"), + (UPat({Ops.CMPLT, Ops.CMPNE, Ops.FLOORDIV, Ops.FLOORMOD, Ops.PARAM, Ops.AFTER, Ops.SPECIAL}, name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), (UPat(Ops.RANGE, src=(UPat(Ops.CONST,)), name="x"), lambda x: x.const_like(x.vmin) if x.vmin == x.vmax else None), # max folding @@ -330,14 +330,14 @@ def uop_given_valid(valid:UOp, uop:UOp, try_simplex=True) -> UOp: for i,(expr,v) in enumerate(bounds.items()): v0, v1 = (expr.vmin if v[0] is None else v[0], expr.vmax if v[1] is None else v[1]) # try checking the whole clause - all_candidates.append((expr, UOp.variable(f"fake{i}", v0, v1, expr.dtype))) + all_candidates.append((expr, UOp.variable(f"fake{i}", v0, v1, expr.dtype, param=True))) if try_simplex: # every candidate is a set of constrained UOp based on valid, and if every item in a set simplifies the uop into a same output, we rewrite uop candidates = [[all_candidates[-1]]] if expr.op is Ops.ADD and v0 == 1 and all(u.op in GroupOp.Irreducible for u in expr.split_uop(Ops.ADD)): # if the constraint is a simplex: X0 + X1 + ... > 0, we can check if all Xi > 0 simplify into the same output - candidates.append([(Xi, UOp.variable(f"fake{i}", 1, Xi.vmax, Xi.dtype)) for Xi in expr.split_uop(Ops.ADD)]) + candidates.append([(Xi, UOp.variable(f"fake{i}", 1, Xi.vmax, Xi.dtype, param=True)) for Xi in expr.split_uop(Ops.ADD)]) for candidate in candidates: # if every branch in candidate gives the same simplified uop, we can rewrite the uop diff --git a/tinygrad/uop/validate.py b/tinygrad/uop/validate.py index d87fce5a31a56..2dfb9025ea14c 100644 --- a/tinygrad/uop/validate.py +++ b/tinygrad/uop/validate.py @@ -37,6 +37,7 @@ def create_bounded(name:str, vmin:int, vmax:int, z3ctx:z3.Context) -> tuple[z3.A # variables (UPat(Ops.SPECIAL, name="x"), lambda x,ctx: create_bounded(x.arg, 0, ctx[1][x.src[0]]-1, ctx[0])), (UPat(Ops.PARAM, name="x"), lambda x,ctx: create_bounded(x.arg.name, x.vmin, x.vmax, ctx[0])), + (UPat(Ops.BUFFER, name="x"), lambda x,ctx: create_bounded(x.arg.name, x.vmin, x.vmax, ctx[0]) if x.is_variable else None), (UPat(Ops.RANGE, name="x"), lambda x,ctx: create_bounded(x.render(simplify=False), 0, ctx[1][x.src[0]]-1, ctx[0])), # loads are variables bounded by the min/max of the dtype. non-pointer INDEX is also a LOAD (UPat((Ops.LOAD, Ops.INDEX), dtypes.ints+(dtypes.weakint,), name="x"), lambda x,ctx: @@ -60,7 +61,7 @@ def create_bounded(name:str, vmin:int, vmax:int, z3ctx:z3.Context) -> tuple[z3.A def uops_to_z3(solver:z3.Solver, *uops: UOp) -> list[z3.ExprRef]: # gate on upstream memory addressing, but keep INDEX as an unknown LOAD - lst = list(UOp.sink(*uops).toposort(gate=lambda x: x.op not in {Ops.AFTER, Ops.BUFFER, Ops.SHRINK} and \ + lst = list(UOp.sink(*uops).toposort(gate=lambda x: x.op not in {Ops.AFTER, Ops.SHRINK} and (x.op is not Ops.BUFFER or x.is_variable) and \ (x.dtype in dtypes.ints+(dtypes.bool, dtypes.weakint) or x.op is Ops.SINK)))[:-1] z3map: dict[UOp, z3.ExprRef] = {} for u in lst: diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 0b0e578cae317..0460845664e01 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -23,7 +23,7 @@ def lower_weak_node(u:UOp) -> UOp|None: # Binary can widen from the bounds, all other nodes derive from the lowered sources. # a weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition (UPat(GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL}, name="u"), lower_weak_node), - (UPat(Ops.PARAM, dtype=dtypes.weakint, name="u"), + (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=select_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), ]) From 6b352206223c867d2b1835c59764ed7ab5fd2287 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:06:03 +0300 Subject: [PATCH 039/306] cpu hcq2 (#17503) * cpu hcq2 * temp * slop * test with backpressure * x * x * x * x * x * x * Dx * save reverts * um? * x * x * call from py * x? * x * submitters gone * x * x * z * Dx * Dx * x * x * fixes * repl * x * f * for now keep hcqbuffer --- extra/hcq2/ops_amd2.py | 8 +- test/backend/test_jit.py | 8 +- test/backend/test_multitensor.py | 10 +- test/helpers.py | 2 +- test/unit/test_hcq_graph.py | 6 +- tinygrad/runtime/ops_cpu.py | 249 ++++++++++++++++--------------- tinygrad/runtime/ops_python.py | 11 +- tinygrad/runtime/support/hcq.py | 3 +- tinygrad/runtime/support/hcq2.py | 54 ++++--- 9 files changed, 190 insertions(+), 161 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 6d69671d813f0..77dc0d3964e7d 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -3,7 +3,7 @@ import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit assert sys.platform != 'win32' from dataclasses import dataclass -from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, HCQ2Buffer, encode_kernargs_clike, make_cmdbuf +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, encode_kernargs_clike, make_cmdbuf from tinygrad.runtime.support.hcq2 import make_binary_patch from tinygrad.uop.ops import sint, UOp from tinygrad.device import Compiled, BufferSpec, Buffer, Device @@ -290,14 +290,14 @@ class AMDAllocator(HCQAllocator['AMDDevice']): def __init__(self, dev:AMDDevice): super().__init__(dev, supports_copy_from_disk=dev.has_copy_queue, supports_transfer=dev.has_copy_queue and not dev.is_usb()) - def _alloc(self, size:int, options:BufferSpec) -> HCQ2Buffer: + def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_copy_queue) def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque) - def _do_map(self, buf:HCQ2Buffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _do_map(self, buf:HCQBuffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) - def _do_unmap(self, buf:HCQ2Buffer): self.dev.iface.unmap(buf) + def _do_unmap(self, buf:HCQBuffer): self.dev.iface.unmap(buf) @dataclass class AMDQueueDesc: diff --git a/test/backend/test_jit.py b/test/backend/test_jit.py index 997d176ee1263..23db717e2638f 100644 --- a/test/backend/test_jit.py +++ b/test/backend/test_jit.py @@ -360,7 +360,7 @@ def f(inp, inp_cpu): self.expect(f, inp, inp_cpu, graph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()], multigraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()], - hcqgraph=[self.ji_graph(4)]) + hcqgraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp()]) # cpu is hcq2 now, it does not join hcq graphs def test_jit_cpu_several(self): if Device.DEFAULT == "CPU": raise unittest.SkipTest("CPU is not a valid default device for this test") @@ -377,9 +377,9 @@ def f(inp, inp_cpu): inp = Tensor.randn(10, 10, device=Device.DEFAULT).realize() inp_cpu = Tensor.randn(10, 10, device="CPU").realize() self.expect(f, inp, inp_cpu, - graph=[self.ji_graph(2), self.ji_graph(2), self.ji_comp()], - multigraph=[self.ji_graph(2), self.ji_graph(2), self.ji_comp()], - hcqgraph=[self.ji_graph(5)]) + graph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()], + multigraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()], + hcqgraph=[self.ji_graph(2), self.ji_comp(), self.ji_comp(), self.ji_comp()]) def test_jit_multidev(self): if Device.DEFAULT == "CPU": raise unittest.SkipTest("CPU is not a valid default device for this test") diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index 1b5d4896fa618..2f60958977bdb 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -1,9 +1,9 @@ import unittest, random from tinygrad import Tensor, Device, nn, GlobalCounters, TinyJit, dtypes, Variable -from tinygrad.uop.ops import Ops, UOp, AxisType +from tinygrad.uop.ops import Ops, UOp, AxisType, graph_rewrite from tinygrad.helpers import getenv, prod, Context from tinygrad.nn.state import get_parameters -from tinygrad.engine.realize import run_linear, compile_linear +from tinygrad.engine.realize import run_linear, compile_linear, pm_beam, pm_compile import numpy as np from hypothesis import given, strategies as strat, settings from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count @@ -79,9 +79,9 @@ def test_shard_no_recompile(self): def test_shard_beam(self): cpu_2 = ("CPU:1", "CPU:2") src = Tensor.ones(16).shard(cpu_2, 0).realize() - pad = src.to(cpu_2[::-1]).schedule_linear().src[0] - with Context(BEAM=1, IGNORE_BEAM_CACHE=1): prg = compile_linear(UOp(Ops.LINEAR, src=(pad,))).src[0].src[0] - self.assertNotEqual(prg.src[0].arg.applied_opts, ()) + lin = UOp(Ops.LINEAR, src=(src.to(cpu_2[::-1]).schedule_linear().src[0],)) + with Context(BEAM=1, IGNORE_BEAM_CACHE=1): call = graph_rewrite(graph_rewrite(lin, pm_beam, ctx=1, walk=True), pm_compile, walk=True).src[0] + self.assertNotEqual(call.src[0].src[0].arg.applied_opts, ()) def test_shard_same_device(self): X = Tensor.ones(256).contiguous().realize() diff --git a/test/helpers.py b/test/helpers.py index 21febed6bc366..e90b25c525a09 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -86,7 +86,7 @@ def assert_jit_cache_len(fxn, expected_len): if linear is None or not linear.src: if expected_len != 0: raise KernelCountException(expected_len, 0) return - if expected_len and all(call_is_hcq(call) for call in linear.src): expected_len = 3 # HCQ2: merged same-queue calls + finalizer + bumps + if expected_len and all(call_is_hcq(call) for call in linear.src): expected_len = 4 # HCQ2: fence + reset + merged same-queue calls + finalizer if call_is_graph(linear.src[0]): if len(linear.src) != 1: raise KernelCountException(1, len(linear.src)) inner = linear.src[0].src[0].src[0] # LINEAR UOp inside CUSTOM_FUNCTION diff --git a/test/unit/test_hcq_graph.py b/test/unit/test_hcq_graph.py index 2ca6fd7ef186b..fd37375764b89 100644 --- a/test/unit/test_hcq_graph.py +++ b/test/unit/test_hcq_graph.py @@ -25,10 +25,10 @@ def f(inp, inp_cpu): cpu_call = UOp(Ops.PROGRAM, src=(UOp.sink(),)).call(UOp.new_buffer("CPU", 1, dtypes.float)) gpu_devs = [d0] - # local MMIO: GPU works alone and with CPU in batch (cpu_support=True) + # CPU uses HCQ2 and is no longer batched into legacy HCQ graphs. assert HCQGraph.supports_uop(gpu_devs, gpu_call) is True - assert HCQGraph.supports_uop(gpu_devs, cpu_call) is True - assert HCQGraph.supports_uop(gpu_devs + [cpu_dev], gpu_call) is True + assert HCQGraph.supports_uop(gpu_devs, cpu_call) is False + assert HCQGraph.supports_uop(gpu_devs + [cpu_dev], gpu_call) is False # USB MMIO: GPU-only still works, but CPU batching must be rejected (cpu_support=False) orig_view = d0.timeline_signal.base_buf.view diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index eec7575f7964d..1ffd2347f0991 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -1,11 +1,11 @@ from __future__ import annotations -import platform, sys, os, ctypes, ctypes.util, functools, mmap, threading, array, itertools -from dataclasses import replace -from typing import cast -from tinygrad.helpers import to_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, partition -from tinygrad.device import Buffer, BufferSpec, TinyELF -from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, HCQArgsState, HCQSignal, HCQProgram, MMIOInterface -from tinygrad.runtime.support.hcq import CLikeArgsState +import platform, sys, os, ctypes, functools, mmap, threading, array, struct, time +from dataclasses import dataclass, replace +from typing import cast, Callable +from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, to_tuple +from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device +from tinygrad.runtime.support.hcq import HCQBuffer, MMIOInterface +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_cmdbuf, make_signal from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer from tinygrad.renderer.nir import LVPRenderer @@ -13,11 +13,15 @@ from tinygrad.runtime.support.elf import jit_loader from tinygrad.runtime.autogen import libc from tinygrad.codegen import do_to_program +from tinygrad.engine.realize import pm_flatten_linear, get_call_arg_uops, get_runtime from tinygrad import UOp, dtypes from tinygrad.dtype import AddrSpace -from tinygrad.uop.ops import sint, KernelInfo, Ops, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher, graph_rewrite -MAX_ARGS, CMD_SIZE, RING_SLOTS = 63, 64, (16 << 10) +MAX_ARGS, CMD_SIZE, RING_SLOTS, FUNCS = 63, 64, (16 << 10), (() if WIN else ('clock_gettime', 'sem_wait', 'sem_post')) + +# ***************** +# 1. workers def signal_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) @@ -35,79 +39,86 @@ def timestamp_prog(): val = ts.after(call)[0].load() * 1_000_000_000 + ts.after(call)[1].load() return UOp.param(0, dtypes.uint64, (1,))[0].store(val) -def quit_prog(): - fn = UOp.param(0, dtypes.uint64, (1 if WIN else 3,)) - if WIN: return fn[0].load().call(UOp.const(0, dtypes.uint64), ret_dtype=dtypes.void) # ExitThread(0) - sem = UOp.param(1, dtypes.uint64, (1,)) - - close = fn[2].load().call(sem[0], ret_dtype=dtypes.void) # sem_close(sem) - return fn.after(close)[0].load().call(UOp.const(0, dtypes.uint64), ret_dtype=dtypes.void) # pthread_exit(0) - def worker_prog(): ring = UOp.param(0, dtypes.uint64, (RING_SLOTS * CMD_SIZE,), volatile=True) - wait, sem = UOp.param(1, dtypes.uint64, (1,), volatile=True), UOp.param(2, dtypes.uint64, (1,)) - cur = UOp.range(2**64-1, 0, dtype=dtypes.uint64) + wait, done = UOp.param(1, dtypes.uint64, (1,), volatile=True), UOp.param(2, dtypes.uint64, (1,), volatile=True) + sem, cur = UOp.param(3, dtypes.uint64, (1,)), UOp.range(2**64-1, 0, dtype=dtypes.uint64) # sem is unused on windows, it has to come last # spin on windows, sem_wait to sleep on posix if WIN: ready = (v:=wait.after(lw:=UOp.loop(1), cur)[0].load()).end(lw, v <= cur) else: ready = (rv:=wait.after(lw:=UOp.loop(1), cur)[0].load().call(sem.after(cur)[0], ret_dtype=dtypes.int)).end(lw, rv != 0) entry = [ring.after(ready).index((cur % RING_SLOTS) * CMD_SIZE + i).load() for i in range(CMD_SIZE)] - return entry[0].call(*entry[1:], ret_dtype=dtypes.void).end(cur) - -def host_wait(ctx, dst:UOp, val:UOp) -> UOp: - return (cur:=dst.after(loop:=UOp.loop(next(ctx))).index(UOp.const(0, dtypes.int)).load()).end(loop, cur < val) - -pm_host_opsel = PatternMatcher([(UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), host_wait)]) - -def encode_host_queue(q:UOp) -> UOp: - # TODO: subset of hcq2 for now - spins, (store,) = partition(graph_rewrite(q, pm_host_opsel, ctx=itertools.count(), walk=True, name="host opsel").src, lambda u: u.op is Ops.END) - assert store.op is Ops.INS and store.arg == "store", f"host queue cannot encode {store.op} {store.arg}" - return store.src[0].after(*spins).index(UOp.const(0, dtypes.int)).store(store.src[1]) - -class CPUComputeQueue(HWQueue): - def __init__(self, dev): - super().__init__() - self.dev = dev - def _cmd(self, prog, args=(), vals=()): return self.exec(prg:=self.dev.prgs[prog], prg.fill_kernargs(args, vals), None, None) - def memory_barrier(self): return self - def exec(self, prg:CPUProgram, args_state:HCQArgsState, global_size, local_size): - if (lvp:=isinstance(args_state, LVPArgsState)): self.bind_args_state(args_state) - args:list[sint|None] = [args_state.buf.va_addr] if lvp else [*[x.va_addr for x in args_state.bufs], *args_state.vals] - assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" - for tid in range(1 if lvp else (global_size or (1,))[0]): - if not lvp and 'core_id' in prg.runtimevars: args[prg.runtimevars['core_id']] = tid - self.q(prg, *[unwrap(x) for x in args], *([0] * (MAX_ARGS - len(args)))) - return self - def wait(self, signal, value=0): return self._cmd(wait_prog, (signal.base_buf,), (value,)) - def timestamp(self, signal): return self._cmd(timestamp_prog, (signal.base_buf.offset(8, 8), self.dev.func_table._buf.offset(0, 8))) - def signal(self, signal, value:sint=0): return self._cmd(signal_prog, (signal.base_buf,), (value,)) - def _submit(self, dev): - dev.ensure_worker() - ring_view = dev.ring.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - for off in range(0, len(self._q), CMD_SIZE): - entry = [self._q[off].addr, *self._q[off+1:off+CMD_SIZE]] - ring_view[(base:=(dev.ring_pos % RING_SLOTS) * CMD_SIZE):base+CMD_SIZE] = array.array('Q', (int(x) & ((1<<64)-1) for x in entry)) - dev.ring_pos += 1 - if WIN: dev.sys.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = dev.ring_pos - else: assert libc.sem_post(dev.sem) == 0 - -class LVPArgsState(CLikeArgsState): - def __init__(self, buf, prg, bufs, vals=()): super().__init__(buf, prg, bufs, vals, [*data64_le(buf.va_addr + 12), (len(bufs) + len(vals)) * 2]) + return done.after(entry[0].call(*entry[1:], ret_dtype=dtypes.void)).index(0).store(cur + 1).end(cur) + +@dataclass +class CPUWorker: ring:Buffer; put:Buffer; sem:Buffer; sys:Buffer; done:Buffer; thread:threading.Thread # noqa: E702 + +# ***************** +# 2. queue encoders + +def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: + progs = [get_runtime(d, prog) if isinstance(prog, UOp) else cast(CPUDevice, Device[d]).prgs[prog] for d in devs] + addrs = tuple(UOp.const(p.addr, dtypes.uint64) for p in progs) + words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, dtypes.uint64, addrs)),) + args + return UOp(Ops.INS, dtypes.void, words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg="cmd") + +def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: + args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in prg.arg.vars] + if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(ctx, prg, *args) + + la = [cpu_cmd(ctx,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t, dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] + return UOp(Ops.LINEAR, dtypes.void, tuple(la)) + +pm_cpu_opsel = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), + + (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP, dtypes.void, ())), + (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), + lambda ctx, dst, val: cpu_cmd(ctx, wait_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), + (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), + lambda ctx, dst, val: cpu_cmd(ctx, signal_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), + (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),)), + lambda ctx, dst: cpu_cmd(ctx, timestamp_prog, dst.getaddr(ctx), *(() if WIN else (make_signal(ctx, tag="func:clock_gettime").getaddr(ctx),)))), +]) + +def encode_queue(q:UOp) -> UOp: + devs, queue = to_tuple(q.arg[0]), q.arg[1] + lin = graph_rewrite(q, pm_cpu_opsel+pm_flatten_linear, ctx=devs, walk=True, name=f"{queue} opsel") + + cnt = sum(len(ins.src) for ins in lin.src) // CMD_SIZE + assert cnt < RING_SLOTS, f"submit of {cnt} entries doesn't fit the ring" + cmdbuf = make_cmdbuf(lin, devs, buf=UOp.placeholder((cnt*CMD_SIZE,), dtypes.uint64, next(UOp.unique_num), device=devs).rtag("cmdbuf")) + ring = UOp.placeholder((ring_words:=RING_SLOTS*CMD_SIZE,), dtypes.uint64, 0, device=devs, volatile=True).rtag(f"{queue}_ring") + put, done, sem, sysbuf = (make_signal(devs, tag=f"{queue}_{name}") for name in ("put", "done", "sem", "sys")) + + # submits are serialized on the submitter, so they can bump put without atomics + ran = done.after(l:=UOp.loop(next(UOp.unique_num))).index(0).load() + room = ran.end(l, put.index(0).load() - ran > RING_SLOTS - cnt) # wait until cnt entries fit in the ring + base = ((put.after(room).index(0).load() % RING_SLOTS) * CMD_SIZE).cast(dtypes.int) + e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(cmdbuf, ring)) + copy = UOp.group(*[ring.index((base + e*CMD_SIZE + w) % ring_words).store(cmdbuf.index(e*CMD_SIZE + w).load()) for w in range(CMD_SIZE)]) + + # wake the worker after each entry, keeping the post with the stores stops it from hoisting out of the loop + wake = copy.end(e) if WIN else make_signal(devs, tag="func:sem_post").after(copy).index(0).load().call(sem.index(0), ret_dtype=dtypes.void).end(e) + bumped = put.after(wake).index(0).store(put.index(0).load() + cnt) + return sysbuf.after(bumped).index(0).store(put.index(0).load() + cnt) if WIN else bumped + +# ***************** # NOTE: MAP_JIT is added to mmap module in python 3.13 MAP_JIT = 0x0800 -class CPUProgram(HCQProgram['CPUDevice']): +class CPUProgram(Program['CPUDevice']): rt_lib = None try: rt_lib = ctypes.CDLL(ctypes.util.find_library('System' if OSX else 'kernel32') if OSX or WIN else 'libgcc_s.so.1') except OSError: pass def __init__(self, dev:CPUDevice, obj:TinyELF): - self.signature, self.runtimevars = obj.signature, {name:slot for name,slot,*_ in obj.signature if name == 'core_id'} + self.dev, self.name, self.signature = dev, obj.name, obj.signature + self.runtimevars = {name:slot for name,slot,*_ in obj.signature if name == 'core_id'} + self.lvp = obj.target.renderer == "LVP" - LVP = obj.target.renderer == "LVP" if sys.platform == "win32": # mypy doesn't understand when WIN is used here PAGE_EXECUTE_READWRITE, MEM_COMMIT, MEM_RESERVE = 0x40, 0x1000, 0x2000 ctypes.windll.kernel32.VirtualAlloc.restype = ctypes.c_void_p @@ -117,7 +128,7 @@ def __init__(self, dev:CPUDevice, obj:TinyELF): ctypes.windll.kernel32.GetCurrentProcess.restype = ctypes.c_void_p proc = ctypes.windll.kernel32.GetCurrentProcess() ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(obj.lib))) - self.fxn = ctypes.CFUNCTYPE(None)(self.addr) + self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) else: # On apple silicon with SPRR enabled (it always is in macos) RWX pages are unrepresentable: https://blog.svenpeter.dev/posts/m1_sprr_gxf/ # MAP_JIT allows us to easily flip pages from RW- to R-X and vice versa. It is a noop on intel cpus. (man pthread_jit_write_protect_np) @@ -125,7 +136,7 @@ def __init__(self, dev:CPUDevice, obj:TinyELF): self.addr = mv_address(self.mem) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(False) - lib = jit_loader(obj.lib, base=ctypes.addressof(ctypes.c_void_p.from_buffer(self.mem)), link_libs=['m']) if LVP else obj.lib + lib = jit_loader(obj.lib, base=ctypes.addressof(ctypes.c_void_p.from_buffer(self.mem)), link_libs=['m']) if self.lvp else obj.lib self.mem.write(lib) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(True) @@ -138,15 +149,30 @@ def __init__(self, dev:CPUDevice, obj:TinyELF): # msync should be a universal POSIX way to do this libc.msync(ctypes.c_void_p(self.addr), len(lib), libc.MS_SYNC | libc.MS_INVALIDATE) - self.fxn = ctypes.CFUNCTYPE(None)(self.addr) - - super().__init__(LVPArgsState if LVP else HCQArgsState, dev, obj, kernargs_alloc_size=12+256 if LVP else 0) + self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) + + def __call__(self, *bufs:HCQBuffer, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), + vals:tuple[int|None, ...]=(), wait:bool=False, timeout:int|None=None) -> float|None: + st = time.perf_counter() + if self.lvp: + lvp_args = bytearray(12 + (len(bufs) + len(vals)) * 8) + addr = mv_address(lvp_args) + struct.pack_into(f'<3I{len(bufs)}Q', lvp_args, 0, *data64_le(addr+12), (len(bufs)+len(vals))*2, *[b.va_addr for b in bufs]) + for v,(off,dt) in zip(vals, TinyELF.iter_sig(self.signature[-len(vals):], len(bufs)*8)): struct.pack_into(f'<{dt.fmt}', lvp_args, 12+off, v) + self.fxn(addr) + else: + args = [*[cast(int, b.va_addr) for b in bufs], *cast(tuple[int, ...], vals)] + assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" + for tid in range(global_size[0]): + if 'core_id' in self.runtimevars: args[self.runtimevars['core_id']] = tid + self.fxn(*[ctypes.c_uint64(x) for x in args]) + return time.perf_counter() - st if wait else None @suppress_finalizing def __del__(self): if sys.platform == 'win32': ctypes.windll.kernel32.VirtualFree(ctypes.c_void_p(self.addr), ctypes.c_size_t(0), 0x8000) #0x8000 - MEM_RELEASE -class CPUAllocator(HCQAllocator): +class CPUAllocator(HCQAllocator['CPUDevice']): def __init__(self, dev:CPUDevice): super().__init__(dev, supports_copy_from_disk=False, supports_transfer=False) def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: if options.external_ptr is not None: addr, buf = options.external_ptr, None @@ -154,68 +180,55 @@ def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: else: addr = mv_address(buf:=mmap.mmap(-1, size, mmap.MAP_ANON | mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE)) return HCQBuffer(va:=addr, sz:=size, meta=buf, view=MMIOInterface(va, sz, fmt='B'), owner=self.dev) def _as_buffer(self, src) -> memoryview: return to_mv(src.va_addr, src.size) + def _copyin(self, dest:HCQBuffer, src:memoryview): + self.dev.synchronize() + ctypes.memmove(int(dest.va_addr), from_mv(src), len(src)) + def _copyout(self, dest:memoryview, src:HCQBuffer): + self.dev.synchronize() + ctypes.memmove(from_mv(dest), int(src.va_addr), len(dest)) def _do_map(self, buf:HCQBuffer): if buf.view is None or not isinstance(buf.view, MMIOInterface): raise RuntimeError("Cannot map buffer without view to cpu") return HCQBuffer(buf.view.addr, buf.size, view=buf.view, owner=buf.owner) def _unmap(self, mb): pass # CPU _do_map returns a view wrapper, nothing to release -class CPUDevice(HCQCompiled): - pm_lower = PatternMatcher([ - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_host_queue)]) - - pm_bufferize = PatternMatcher([ - (UPat(Ops.PARAM, tag="sentinel_signal"), lambda ctx: ctx[0].signal("sentinel", (1 << 64) - 1)), - (UPat(Ops.PARAM, tag="timeline_signal"), lambda ctx: ctx[0].signal("timeline")), - (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1)), - (UPat(Ops.PARAM, tag="signal", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot)), - ]) - - @functools.cache - def signal(self, name:str|int, init_value:int=0) -> Buffer: - (buf:=Buffer(self.device, 1, dtypes.uint64, preallocate=True)).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = init_value - return buf +class CPUDevice(HCQ2Compiled): + wait_timeout_ms, has_copy_queue = 30000, False + pm_lower = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue)]) def __init__(self, device:str=""): - super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, HCQSignal, - functools.partial(CPUComputeQueue, self), arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") + super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, + arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") - self.ring_pos = 0 + self.pm_bufferize = PatternMatcher( + [(UPat(Ops.PARAM, tag=f"COMPUTE:0_{n}"), lambda ctx, n=n: getattr(ctx[0].worker, n)) for n in ("ring", "put", "sem", "sys", "done")] + + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize - # posix uses sem to put cpus into sleep - self.sem_addr = 0 - if not WIN: - self.sem = libc.sem_open(sem_name:=f"/tinygrad-{os.getpid()}-{id(self):x}".encode(), os.O_CREAT|os.O_EXCL, 0o600, 0) # type: ignore[call-arg] - self.sem_addr = unwrap(ctypes.cast(self.sem, ctypes.c_void_p).value) - if self.sem_addr == ctypes.c_void_p(-1).value or libc.sem_unlink(sem_name): raise OSError(ctypes.get_errno(), "semaphore") - - # TODO: move to hcq2 with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): - prgs = {f: f().sink(arg=KernelInfo(f.__name__), tag=1) for f in (signal_prog, wait_prog, timestamp_prog, quit_prog, worker_prog)} - self.prgs = {f: self.runtime(do_to_program(v, ClangRenderer(replace(self.renderer.target, renderer="CLANG"))).to_elf()) for f,v in prgs.items()} + clang = ClangRenderer(replace(self.renderer.target, renderer="CLANG")) + self.prgs:dict[Callable, CPUProgram] = {f: CPUProgram(self, do_to_program(f().sink(arg=KernelInfo(f.__name__), tag=1), clang).to_elf()) + for f in (signal_prog, wait_prog, timestamp_prog, worker_prog)} - @functools.cached_property - def ring(self) -> Buffer: return Buffer(self.device, RING_SLOTS * CMD_SIZE, dtypes.uint64, preallocate=True) - @functools.cached_property - def sys(self) -> Buffer: return Buffer(self.device, 1, dtypes.uint64, preallocate=True) - @functools.cached_property - def sem_buf(self) -> Buffer: return Buffer(self.device, 1, dtypes.uint8, options=BufferSpec(external_ptr=self.sem_addr), preallocate=True) + def func_ptr(self, name:str) -> Buffer: return self.func_table.view(1, dtypes.uint64, FUNCS.index(name)*8).ensure_allocated() - # TODO: move to hcq2 infra @functools.cached_property def func_table(self) -> Buffer: - fns = ([0, ctypes.windll.kernel32.ExitThread, 0, 0] if WIN else # type: ignore[attr-defined] - [libc.dll.clock_gettime, libc.dll.pthread_exit, libc.dll.sem_wait, libc.dll.sem_close]) - addrs = array.array('Q', [unwrap(ctypes.cast(f, ctypes.c_void_p).value) if f else 0 for f in fns]) - (ft:=Buffer(self.device, len(fns), dtypes.uint64, preallocate=True)).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[:] = addrs + lib = ctypes.windll.kernel32 if sys.platform == "win32" else libc.dll # type: ignore[attr-defined] + (ft:=Buffer(self.device, len(FUNCS), dtypes.uint64, preallocate=True))._buf.cpu_view().view(fmt='Q')[:] = \ + array.array('Q', [unwrap(ctypes.cast(getattr(lib, f), ctypes.c_void_p).value) for f in FUNCS]) return ft - @functools.cache - def ensure_worker(self): - threading.Thread(target=cast(CPUProgram, self.prgs[worker_prog]).fxn, daemon=True, args=[ctypes.c_uint64(x) for x in - [self.ring._buf.va_addr, self.sys._buf.va_addr if WIN else self.func_table._buf.va_addr+16, self.sem_addr]]).start() + @functools.cached_property + def worker(self) -> CPUWorker: + ring, put, sysbuf, done = (Buffer(self.device, sz, dtypes.uint64, preallocate=True) for sz in (RING_SLOTS*CMD_SIZE, 1, 1, 1)) + addr, hsem = 0, None - def finalize(self): - if self.ring_pos == 0: return # the worker starts with the first submit - ft = self.func_table._buf - CPUComputeQueue(self)._cmd(quit_prog, (ft.offset(8, 8),) if WIN else (ft.offset(8, 24), self.sem_buf._buf)).submit(self) - self.ring_pos = 0 + # sem are posix-only + if not WIN: + hsem = libc.sem_open(nm:=f"/tinygrad-{os.getpid()}-{id(ring):x}".encode(), os.O_CREAT|os.O_EXCL, 0o600, 0) # type: ignore[call-arg] + if (addr:=unwrap(ctypes.cast(hsem, ctypes.c_void_p).value)) == ctypes.c_void_p(-1).value or libc.sem_unlink(nm): + raise OSError(ctypes.get_errno(), "semaphore") + sem = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(external_ptr=addr), preallocate=True) + + worker_args = [ring._buf.va_addr, sysbuf._buf.va_addr if WIN else self.func_ptr('sem_wait')._buf.va_addr, done._buf.va_addr, addr] + (worker:=threading.Thread(target=self.prgs[worker_prog].fxn, daemon=True, args=[ctypes.c_uint64(x) for x in worker_args])).start() + return CPUWorker(ring, put, sem, sysbuf, done, worker) diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index aa1390f04a235..0b2126deaae2b 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -3,10 +3,10 @@ # works to test the tensor cores, and all the uops in general # this is the (living) definition of uops from typing import Any, TYPE_CHECKING -import pickle, base64, itertools, time, sys, functools +import pickle, base64, itertools, time, sys, functools, ctypes from dataclasses import replace from tinygrad.dtype import bitcast, DType, dtypes, AddrSpace, truncate, storage_fmt_for_dtype, to_storage_scalar, from_storage_scalar -from tinygrad.helpers import all_same, getenv, flatten, Target, IMAGE, is_image_shape, cpu_profile +from tinygrad.helpers import all_same, getenv, flatten, Target, IMAGE, is_image_shape, cpu_profile, mv_address from tinygrad.device import Buffer, Compiled, Compiler, Allocator, Program, TinyELF from tinygrad.codegen.opt import tc from tinygrad.uop.ops import exec_alu, python_alu, Ops, UOp, GroupOp @@ -134,6 +134,13 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup for k in range(len(src_values))], j, u.dtype) for j in range(load_sz)] else: values[u] = load(src_values, 0, u.dtype) + elif u.op is Ops.CALL: + assert u.dtype is dtypes.void + cfunc = ctypes.CFUNCTYPE(None, *[ctypes.c_uint64] * (len(src_values)-1)) + values[u] = [] + for args,gate in zip(zip(*src_values), exec_masks[-1]): + call_args = [(mv_address(x[0]) + x[1]*dt.itemsize) if isinstance(x, tuple) else x for x,dt in zip(args, src_dtypes)] + values[u].append(cfunc(call_args[0])(*call_args[1:]) if gate else None) elif u.op is Ops.WMMA: first_src_dtype = u.src[0].dtype assert isinstance(first_src_dtype, DType) # mypy diff --git a/tinygrad/runtime/support/hcq.py b/tinygrad/runtime/support/hcq.py index e22a436b0a82a..6ff4effc25a25 100644 --- a/tinygrad/runtime/support/hcq.py +++ b/tinygrad/runtime/support/hcq.py @@ -517,8 +517,7 @@ def finalize(self): if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() class HCQBuffer: - def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None, - owner:HCQCompiled|None=None): + def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None, owner:Any=None): self.va_addr, self.size, self.meta, self._base, self.view = va_addr, size, meta, _base, view self._devs, self.owner = ([owner] if owner is not None else []), owner self._mappings:dict[HCQCompiled, HCQBuffer] = {} # mapping to the other devices diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index d57cc11d99b7f..73f9129da7172 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import cast, Callable, TypeVar, Generic, Any, Sequence +from typing import cast, Callable, TypeVar, Generic, Any, Sequence, Iterable import struct, functools, time, collections, itertools, decimal, statistics from dataclasses import replace, dataclass from tinygrad.helpers import DEV, getenv, select_first_inited, select_by_name, suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE @@ -9,7 +9,7 @@ from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const from tinygrad.dtype import dtypes, truncate -from tinygrad.runtime.support.hcq import MMIOInterface +from tinygrad.runtime.support.hcq import MMIOInterface, HCQBuffer from tinygrad.runtime.support.memory import BumpAllocator from tinygrad.renderer import Renderer, Estimates from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop @@ -22,7 +22,7 @@ HCQ_RUNTIME_DEV = ContextVar("HCQ_RUNTIME_DEV", "CPU") -HCQ_DEVS = frozenset(("AMD",)) +HCQ_DEVS = frozenset(("AMD", "CPU")) HCQ_P2P_DEVS = HCQ_DEVS | frozenset(("CPU",)) HCQ_CACHE_TAGS = frozenset(("program", "systems", "template")) @@ -94,9 +94,12 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| # ***************** # 1.1. prep: staging copies -def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS) and not all_devices_in(b.device, HCQ_P2P_DEVS) +def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_P2P_DEVS) -def hcq_call_devs(call:UOp) -> Any|None: return next((b.device for b in call.src[1:] if all_devices_in(b.device, HCQ_DEVS)), None) +def _get_enqueue_devs(call:UOp) -> Any|None: + if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_P2P_DEVS) for b in bufs): return None + devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device + return devs if all_devices_in(devs, HCQ_DEVS) else None def stage_copy(dst:UOp, src:UOp) -> UOp|None: if not (_need_staging(src, dst) or _need_staging(dst, src)): return None @@ -105,7 +108,7 @@ def stage_copy(dst:UOp, src:UOp) -> UOp|None: return UOp(Ops.LINEAR, src=(src.copy_to_device("CPU").call(stage, src), stage.copy_to_device(dst.device).call(dst, stage))) def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: - if (devs:=hcq_call_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None + if (devs:=_get_enqueue_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None d, s = (UOp.param(i, dst.dtype, (n:=dst.max_numel(),), device=devs) for i in range(2)) ast = d.index(r:=UOp.range(n, 0)).store(s.index(r).load()).end(r).sink(arg=KernelInfo(name="copy"), tag=1) return call.replace(src=(to_program(ast, Device[dev].renderer), dst, src)) @@ -136,7 +139,7 @@ def _get_deps(ctx:DepsTracker, bufs_by_lane:list[list[Any]], write, key:tuple[tu def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int]], devices:tuple[str, ...], queue:str) -> tuple[list[UOp], set[int]]: # opt1: same-queue ops are fifo-ordered - if devices[0].split(":")[0] in {"AMD", "QCOM"} or queue.startswith("COPY"): + if devices[0].split(":")[0] in {"AMD", "QCOM", "CPU"} or queue.startswith("COPY"): dep_lanes = [(dep, dlane, lane) for dep, dlane, lane in dep_lanes if (dep[0][dlane], dep[1]) != (devices[lane], queue)] # opt2: keep latest dep per (dep device, queue, cur lane) @@ -159,7 +162,7 @@ def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[t for b in itertools.chain.from_iterable(_get_call_bufs_by_lane(call, devices)): for bd in to_tuple(b.device): dev_bufs[bd][id(b)] = b - n, fences, fins, signal_tags = len(batch_info), [], [], set() + n, fences, resets, fins, signal_tags = len(batch_info), [], [], [], set() for _, devgroup in itertools.groupby(sorted(dev_bufs), key=lambda d: d.split(":")[0]): devs = tuple(devgroup) @@ -173,16 +176,17 @@ def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[t fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() - # fence once per device group on this schedule's previous epoch, then reset any queue signals used by the group + # fence once per device group on this schedule's previous epoch qs = dedup([qn for bdevs, qn in batch_info if set(bdevs) & set(devs)]) sched_epoch = make_signal(devs, next(UOp.unique_num)) wait_device_epoch = (done:=tl_signal.after(loop:=UOp.loop(0)).index(0).load()).end(loop, done < sched_epoch.index(0).load()) - resets = [make_signal(devs, slots[q]).after(wait_device_epoch).index(0).store(0) for q in qs] + fences.append(make_call("hcq_fence", UOp.sink(wait_device_epoch), HCQInfo(devs))) - fences.append(make_call("hcq_fence", UOp.sink(*(resets or [wait_device_epoch])), HCQInfo(devs))) + # queues of other groups wait on these signals, so reset them only after every group reached its epoch + if qs: resets.append(make_call("hcq_reset", UOp.sink(*[make_signal(devs, slots[q]).index(0).store(0) for q in qs]), HCQInfo(devs))) fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) - return fences, fins, signal_tags + return fences + resets, fins, signal_tags def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: batch_info = [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch] @@ -229,7 +233,7 @@ def sched_hcq_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] batch:list[tuple[UOp, tuple[str, ...]]] = [] for call in l.src: - if (devs:=hcq_call_devs(call)) is not None: batch.append((call, to_tuple(devs))) + if (devs:=_get_enqueue_devs(call)) is not None: batch.append((call, to_tuple(devs))) else: srcs, batch = srcs + _finalize_batch(batch, profile) + [call], [] return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) @@ -354,9 +358,13 @@ def split_patches(call:UOp) -> UOp|None: # ***************** +def _rank_ranges(uops:Iterable[UOp]) -> dict[UOp, UOp]: + return {r: r.replace(arg=(i,)+r.arg[1:]) for i,r in enumerate(sorted([u for u in uops if u.op is Ops.RANGE], key=lambda r: r.arg))} + def replace_params(call:UOp) -> UOp|None: body, variables, param_ops = call.src[0], call.src[0].variables(), {Ops.PARAM, Ops.MSTACK} - args = dedup([s for u in body.toposort(gate=lambda u: u.op not in param_ops) for s in u.src if s.op in param_ops and s not in variables]) + tops = body.toposort(gate=lambda u: u.op not in param_ops) + args = dedup([s for u in tops for s in u.src if s.op in param_ops and s not in variables]) patched, refhold = partition(call.src[1:], lambda x: x.src[0] in args) by_root = {p.src[0]: p for p in patched} @@ -368,7 +376,7 @@ def replace_params(call:UOp) -> UOp|None: refhold += [a for a in addrs if a not in held and all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(a))] sub = {(b:=u.without_after): UOp.param(i, u.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=b.op is Ops.PARAM and b.arg.volatile) - for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} + for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} | _rank_ranges(tops) info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args) if u.without_after.tag == "inputs"), None)) return call.replace(src=(body.substitute(sub).replace(arg="hcq_args"), *c_args, *refhold), arg=replace(call.arg, aux=info)) # TODO: call.after(*refhold)? @@ -521,6 +529,7 @@ def hcq_link(linear:UOp, cache=True) -> UOp: class HCQ2Compiled(Compiled): timestamp_divider: float = 1000.0 + wait_timeout_ms: float = 30000.0 def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Renderer]], runtime, can_recover:bool=False, arch=None): self.device_id:int = int(device.split(":")[1]) if ":" in device else 0 @@ -577,9 +586,10 @@ def synchronize(self, timeout:int|None=None): sig = self.signal("timeline").as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') tl = self.signal("value", 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') timeout = timeout if timeout is not None and self.can_recover else None - st = time.perf_counter() - while sig[0] < tl[0] - 1: - if time.perf_counter() - st > (timeout or 3000) / 1000: self.on_device_hang() + st, done = time.perf_counter(), sig[0] + while done < tl[0] - 1: + if done != (done:=sig[0]): st = time.perf_counter() + elif time.perf_counter() - st > (timeout or self.wait_timeout_ms) / 1000: self.on_device_hang() if self.prof_ents: self.collect_prof() def on_device_hang(self): raise RuntimeError(f"{self.device} hang detected") @@ -617,17 +627,17 @@ def offset(self, offset:int, size:int) -> HCQ2Buffer: return HCQ2Buffer(self.va_addr+offset, meta=self.meta, view=(self.view.view(offset=offset, size=size) if self.view is not None else None)) class HCQAllocator(LRUAllocator[HCQDeviceType], Generic[HCQDeviceType]): - def _as_buffer(self, buf:HCQ2Buffer) -> memoryview: + def _as_buffer(self, buf:HCQBuffer) -> memoryview: return unwrap(buf.view).mv - def _map(self, buf:HCQ2Buffer) -> HCQ2Buffer: + def _map(self, buf:HCQBuffer) -> HCQBuffer: if not hasattr(self, '_do_map'): raise NotImplementedError("map failed: no method implemented") return self._do_map(buf) def _do_unmap(self, mb): self.dev.iface.free(mb) @suppress_finalizing - def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None): + def _free(self, buf:HCQBuffer, options:BufferSpec|None=None): if options is not None and options.external_ptr is not None: return self.dev.synchronize() if hasattr(self, '_do_free'): self._do_free(buf, options) @@ -636,4 +646,4 @@ def _unmap(self, mb): self.dev.synchronize() self._do_unmap(mb) - def _offset(self, buf, size:int, offset:int) -> HCQ2Buffer: return buf.offset(offset=offset, size=size) + def _offset(self, buf, size:int, offset:int) -> HCQBuffer: return buf.offset(offset=offset, size=size) From 25c3bd027b1dad6cc2c0a00757db191ec9c12640 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 09:10:05 -0400 Subject: [PATCH 040/306] remove pm_fold_cast_const in simplify_merge_adjacent [pr] (#17530) --- tinygrad/codegen/simplify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinygrad/codegen/simplify.py b/tinygrad/codegen/simplify.py index a1d9d4f01b492..b25de5089fc0b 100644 --- a/tinygrad/codegen/simplify.py +++ b/tinygrad/codegen/simplify.py @@ -1,7 +1,7 @@ import itertools from typing import Callable from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, graph_rewrite, _substitute, range_start, AxisType -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const, invalid_gate +from tinygrad.uop.symbolic import symbolic, invalid_gate from tinygrad.helpers import partition from tinygrad.dtype import dtypes @@ -32,7 +32,7 @@ def simplify_merge_adjacent(u:UOp) -> UOp|None: s0, s1 = r0.src[0], r1.src[0] # do the merge new_range = r0.replace(src=(s0*s1,)) - nidx = graph_rewrite(u, _substitute+symbolic+pm_fold_cast_const+pm_flatten_range, ctx={r0:new_range//s1, r1:new_range%s1}, + nidx = graph_rewrite(u, _substitute+symbolic+pm_flatten_range, ctx={r0:new_range//s1, r1:new_range%s1}, name=f"check_merge_{r0.arg[0]}_{r1.arg[0]}") # check if it simplifies From 89ab344c42bc7f72730f4bb597c9bb4f8b3819b1 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 09:23:01 -0400 Subject: [PATCH 041/306] fix assign into bitcast with no explicit realize (#17531) --- test/unit/test_assign.py | 4 ++++ tinygrad/tensor.py | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/unit/test_assign.py b/test/unit/test_assign.py index 7847ee95b09e2..61b9b5266dc43 100644 --- a/test/unit/test_assign.py +++ b/test/unit/test_assign.py @@ -540,6 +540,10 @@ def test_assign_bitcast(self): c = Tensor([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32).realize() c[0:2].bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000], dtype=dtypes.uint32)).realize() np.testing.assert_allclose(c.numpy(), [4.0, 3.0, 3.0, 4.0]) + # without .realize() + a = Tensor([1.0, 2.0, 3.0, 4.0], dtype=dtypes.float32).realize() + a.bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000, 0x40000000, 0x3f800000], dtype=dtypes.uint32)) + np.testing.assert_allclose(a.numpy(), [4.0, 3.0, 2.0, 1.0]) def test_assign_bitcast_different_size(self): # assign to a shape-changing bitcast view (only works on DISK currently) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 11885a387ac9c..ee7424cf7f715 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -453,12 +453,11 @@ def assign(self, x:Tensor|PyConst|list|tuple) -> Tensor: return self # STORE+AFTER: STORE is the write effect (void), AFTER wraps the view for correct shape/ranging assign = self.uop.after(self.uop.store(x.uop)) - if (base := self.uop.base).op in {Ops.BUFFER, Ops.AFTER} and self.uop is not base and not self.uop.has_buffer_identity(): + ib = self.uop + while not ib.has_buffer_identity() and ib.op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH}: ib = ib.src[0] + if ib is not self.uop and ib.has_buffer_identity(after_ok=True): # view assign: replace at the buffer-identity level (e.g. RESHAPE(BUFFER)) so @function's substitution catches it - ib = self.uop - while not ib.has_buffer_identity() and ib is not base: ib = ib.src[0] - assigned_ib = ib.after(assign) - _apply_map_to_tensors({ib: assigned_ib}, name="Embed View Assign") + _apply_map_to_tensors({ib: ib.after(assign)}, name="Embed View Assign") else: # simple assign self.uop = assign From adacaa3e170adf77b6c363e9ba80f63bfcc5bf5d Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:23:59 +0300 Subject: [PATCH 042/306] hcq2 fix hangs (#17529) --- tinygrad/renderer/cstyle.py | 1 + tinygrad/renderer/llvmir.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index e25ab0c605fc3..3b555ac177446 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -264,6 +264,7 @@ class ClangRenderer(CStyleLanguage): nan = '__builtin_nanf("")' # language options + barrier = "__atomic_thread_fence(__ATOMIC_SEQ_CST);" buffer_suffix = " restrict" type_map = {dtypes.bool:"_Bool", dtypes.half:"__fp16"} code_for_op = {**({k:v for k,v in CStyleLanguage.code_for_op.items() if k not in [Ops.EXP2, Ops.SIN, Ops.LOG2, Ops.TRUNC, Ops.RECIPROCAL]}), diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 05f6e72033cbb..838e1c95f07a5 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -142,7 +142,7 @@ def _bf16(dt:DType): return dtypes.ushort if dt is dtypes.bfloat16 else dt (UPat(Ops.IF, name="x"), lambda ctx,x: f" br i1 {ctx[x.src[0]]}, label %ifbody_{ctx[x][1:]}, label %ifskip_{ctx[x][1:]}\nifbody_{ctx[x][1:]}:"), (UPat(Ops.ENDIF, name="x"), lambda ctx,x: f" br label %ifskip_{ctx[x.src[0]][1:]}\nifskip_{ctx[x.src[0]][1:]}:"), - (UPat(Ops.BARRIER), lambda ctx: "") + (UPat(Ops.BARRIER), lambda ctx: " fence seq_cst") ]) class LLVMRenderer(Renderer): From 80169c67584749a19fbe62c8730ce6eb51a91fb0 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 12:57:56 -0400 Subject: [PATCH 043/306] remove where push cast to branches from sym [pr] (#17533) * remove where push cast to branches from sym [pr] not really needed and one less place that generates casted weak const when it's not needed * fix --- test/mockgpu/amd/emu.py | 36 +++++++++++++++------------------- test/null/test_uop_symbolic.py | 27 ++++++++++--------------- tinygrad/uop/symbolic.py | 6 ++---- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/test/mockgpu/amd/emu.py b/test/mockgpu/amd/emu.py index 3f36e331e9a18..7c6fe9038d21d 100644 --- a/test/mockgpu/amd/emu.py +++ b/test/mockgpu/amd/emu.py @@ -260,19 +260,6 @@ def _cond(cond, if_true, if_false): def _cond_hi16(cond, val: UOp) -> UOp: return _cond(cond, _hi16(val), val) def _apply_opsel(val: UOp, sel_bit: int, opsel: int) -> UOp: return _hi16(val) if opsel & (1 << sel_bit) else val -def _set_lane_bit(old: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> UOp: - """Set/clear a single bit in a mask based on lane index, respecting exec mask.""" - if old.dtype in (dtypes.uint64, dtypes.int64): - dt = dtypes.uint64 - mask = UOp.const(1, dt) << lane.cast(dt) - new_bit = _to_u32(val).cast(dt) << lane.cast(dt) - cleared = old.cast(dt) & (mask ^ UOp.const(0xFFFFFFFFFFFFFFFF, dt)) - return _lane_active(exec_mask, lane).where(cleared | new_bit, old.cast(dt)) - mask = _c(1) << lane.cast(dtypes.uint32) - new_bit = _to_u32(val) << lane.cast(dtypes.uint32) - cleared = old & (mask ^ _c(MASK32)) - return _lane_active(exec_mask, lane).where(cleared | new_bit, old) - def _val_to_u32(val: UOp) -> UOp: """Convert any value to uint32 for storage (bitcast floats, cast ints).""" if val.dtype == dtypes.uint32: return val @@ -532,6 +519,19 @@ def wmask(self, reg: UOp, val: UOp) -> list[UOp]: return [self.wsgpr_dyn(reg, lo), self.wsgpr_dyn(reg + _c(1), hi)] return [self.wsgpr_dyn(reg, val)] + def wmask_lane_bit(self, reg: UOp, lane: UOp, val: UOp, exec_mask: UOp) -> list[UOp]: + """Set/clear bit `lane` of the mask at `reg` from val for exec-active lanes, preserving memory for inactive lanes""" + active, bit = _lane_active(exec_mask, lane), _to_u32(val) + if self.wave_size <= 32: + old = self.rsgpr_dyn(reg) + mask = _c(1) << lane.cast(dtypes.uint32) + return [self.wsgpr_dyn(reg, active.where((old & (mask ^ _c(MASK32))) | (bit << lane.cast(dtypes.uint32)), old))] + off = (lane & _c(31, dtypes.int)).cast(dtypes.uint32) + mask = _c(1) << off + def half(old: UOp, sel: UOp) -> UOp: return sel.where(active.where((old & (mask ^ _c(MASK32))) | (bit << off), old), old) + return [self.wsgpr_dyn(reg, half(self.rsgpr_dyn(reg), lane < _c(32, dtypes.int))), + self.wsgpr_dyn(reg + _c(1), half(self.rsgpr_dyn(reg + _c(1)), _c(32, dtypes.int) <= lane))] + def rmask(self, reg: UOp) -> UOp: """Read a lane mask (VCC/EXEC). Combines lo/hi for wave64.""" if self.wave_size > 32: return _u64(self.rsgpr_dyn(reg), self.rsgpr_dyn(reg + _c(1))) @@ -718,9 +718,7 @@ def to_wide(x): return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.it raw_stores.append(('vgpr_direct', self.vgpr.index(val[0].valid(active)).store(new_val))) continue if 'D0' in dest and '[laneId]' in dest: - old_vcc = self.rmask(_c(VCC_LO.offset)) - new_vcc = _set_lane_bit(old_vcc, lane, val, exec_mask) - raw_stores.extend([('vcc', s) for s in self.wmask(_c(VCC_LO.offset), new_vcc)]) + raw_stores.extend([('vcc', s) for s in self.wmask_lane_bit(_c(VCC_LO.offset), lane, val, exec_mask)]) elif dest.startswith('D0'): dest_suffix = re.match(r'D0\.(\w+)', dest) if dest_suffix is not None: @@ -1039,13 +1037,11 @@ def get_cmp_bit(lane) -> UOp: result = _sdwa_write(old, result, dst_sel, dst_unused) stores.append(ctx.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) elif dest.startswith('VCC'): - old_vcc = ctx.rmask(_c(VCC_LO.offset)) - stores.extend(ctx.wmask(_c(VCC_LO.offset), _set_lane_bit(old_vcc, lane, val, exec_mask))) + stores.extend(ctx.wmask_lane_bit(_c(VCC_LO.offset), lane, val, exec_mask)) if vcc_val is not None: # Initialize sdst to 0 before lane loop (old value may be unrelated data), then set lane bits in loop init_stores = [ctx.wsgpr_dyn(sdst_off, _c(0)), ctx.wsgpr_dyn(sdst_off + _c(1), _c(0))] - old_sdst = ctx.rmask(sdst_off) - stores.extend(ctx.wmask(sdst_off, _set_lane_bit(old_sdst, lane, vcc_val, exec_mask))) + stores.extend(ctx.wmask_lane_bit(sdst_off, lane, vcc_val, exec_mask)) if stores: return UOp.sink(*init_stores, UOp.sink(*stores).end(lane), *ctx.inc_pc()) return UOp.sink(*init_stores, *ctx.inc_pc()) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 6c267257baaf3..20a740bfbb992 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -3,7 +3,6 @@ import z3 from tinygrad.dtype import dtypes, ConstType, DType, Invalid -from test.helpers import get_uops from tinygrad.uop.ops import UOp, Ops, graph_rewrite, sym_infer from tinygrad.uop.spec import spec_shared, type_verify from tinygrad.uop.symbolic import sym, pm_fold_cast_const, commutative, pm_simplify_valid, pm_move_where_on_load @@ -1013,22 +1012,6 @@ def test_neg_in_comp(self): b = Variable("b", 0, 3) self.helper_test_variable(-a<-b, False, True, "(b 2 @@ -1375,6 +1358,16 @@ def test_alu_invalid_vconst(self): c2 = UOp.const((1, Invalid, 1, 1)) self.assertIs((c1+c2).simplify(), UOp.const((2, Invalid, Invalid, Invalid))) + def test_gated_load_keeps_index_valid(self): + # the load executes even on gated-off iterations: gated_given_valid must not erase its mask (PADTO OOB shape) + buf = UOp.param(0, dtypes.bool, (17,)) + ridx = Variable("ridx", 0, 31) + cond = ridx < 17 + load = buf.index(ridx.valid(cond)) + out = graph_rewrite(cond.where(load.where(uconst(2), uconst(0)), UOp.invalid()), sym) + idx = next(u for u in out.toposort() if u.op is Ops.INDEX) + self.assertIs(idx.src[1].get_valid(), cond.simplify()) + class TestStoreLoadFolding(unittest.TestCase): """Tests for store(index, load(index)) -> NOOP rule. This rule matches patterns that EMERGE during simplification.""" def test_store_load_folding(self): diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 924325f426cbb..212bf5122d55e 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -407,7 +407,8 @@ def can_move(c:UOp) -> bool: ]) def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: - if x.dtype is not dtypes.weakint: return None + # pure index math only: a LOAD in x executes even where cond is false, so its INDEX valid must survive the assumption + if x.dtype is not dtypes.weakint or x.op_in_backward_slice_with_self(Ops.INDEX): return None # Skip if x contains DIV/MOD AND IMAGE mode is enabled -> image index e.g. openpilot if IMAGE.value > 0 and x.op_in_backward_slice_with_self(Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD): return None return cond.where(uop_given_valid(cond, x, try_simplex=False), i) @@ -432,9 +433,6 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: # reorder ALU/VECTORIZE (UPat(GroupOp.ALU, src=(UPat(Ops.STACK, src=UPat(name='x')), UPat(Ops.STACK, src=UPat(name='y'))), name='alu'), lambda x,y,alu: UOp(Ops.STACK, src=(UOp(alu.op, src=(x,y)),))), - # ** where ** - # push cast to branches - (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), lambda s,a,b,cast: s.where(a.cast(cast.dtype), b.cast(cast.dtype))), # ** pow ** ((UPat(Ops.POW, name="p"), lambda p: xpow(*p.src))), # ** load/store folding ** From ac7067ac603b707f233b799a8a1e6290f8709c2d Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 13:38:34 -0400 Subject: [PATCH 044/306] fix deconstruct_function for python 3.11 (#17534) --- test/backend/test_pickle.py | 7 ++++++- tinygrad/uop/ops.py | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/backend/test_pickle.py b/test/backend/test_pickle.py index cbb0aa3782f27..7033a4dd6b125 100644 --- a/test/backend/test_pickle.py +++ b/test/backend/test_pickle.py @@ -2,7 +2,7 @@ import numpy as np from tinygrad import Tensor, Device, TinyJit, Variable, dtypes from tinygrad.helpers import GlobalCounters, ContextVar, Context, DEV -from tinygrad.uop.ops import PatternMatcher, UPat, UOp +from tinygrad.uop.ops import PatternMatcher, UPat, UOp, deconstruct_function class TestPickle(unittest.TestCase): def test_pickle_code_object(self): @@ -11,6 +11,11 @@ def test_pickle_code_object(self): fxn = types.FunctionType(pickle.loads(code_str), globals()) self.assertEqual(fxn(2), 4) + def test_deconstruct_function_nested_comprehension(self): + # pre PEP 709, each comprehension is its own code object, so dtypes here is referenced two code objects deep + def fxn(): return [[dtypes.int for _ in range(2)] for _ in range(2)] + self.assertEqual(types.FunctionType(*deconstruct_function(fxn))(), fxn()) + def test_pickle_pattern_matcher(self): pm = PatternMatcher([(UPat.cvar('x'), lambda x: x*2)]) sink = UOp.const(2) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index fadcdb866ed88..6611ef12bf792 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1423,9 +1423,9 @@ def match(self:UPat, uop:UOp, store:dict[str, UOp]) -> list[dict[str, UOp]]: return res def deconstruct_function(fxn:Callable) -> tuple: - new_globals = {k:v for k,v in fxn.__globals__.items() if k in fxn.__code__.co_names} - for co in fxn.__code__.co_consts: - if isinstance(co, types.CodeType): new_globals.update({k:v for k,v in fxn.__globals__.items() if k in co.co_names}) + # globals can be referenced from arbitrarily nested code objects (comprehensions/lambdas, pre PEP 709) + def names(co:types.CodeType) -> set: return set(co.co_names).union(*(names(c) for c in co.co_consts if isinstance(c, types.CodeType))) + new_globals = {k:v for k,v in fxn.__globals__.items() if k in names(fxn.__code__)} # NOTE: optional round trip through pickle! assert fxn.__closure__ is None, "closures are not supported in pattern matchers" ret = fxn.__code__, new_globals, fxn.__name__, fxn.__defaults__ From 13c381b0c032616763e04b0659e969fc507352c1 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 14:14:40 -0400 Subject: [PATCH 045/306] remove pm_fold_cast_const from initial symbolic [pr] (#17535) interestingly it gives more accurate numerics when composing const like log10 --- test/null/test_const_folding.py | 8 ++++++++ tinygrad/codegen/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index 5e365c423366c..b781c6bfe1dde 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -51,6 +51,14 @@ def test_weakfloat_math(self): def test_invalid_poison(self): self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) + def test_single_rounding_log10_backward(self): + # log10 backward folds log10(2)/log(2) = 1/log(10) in one rounding, not the double-rounded 1/float32(log(10)) + x = Tensor([1.0, 2.0, 3.0]) + ast = next(s.src[0] for s in x.log10().sum().gradient(x)[0].schedule_linear().src if s.src[0].op is Ops.SINK) + const = next(u.arg for u in full_rewrite(ast).toposort() if u.op is Ops.CONST and u.dtype is dtypes.float32) + # correctly rounded: within half a float32 ulp of the exact value (folding at float32 lands 0.66 ulp off) + self.assertLess(abs(const - 1/math.log(10)), 2**-26) + def test_cast_commits_to_dtype_grid(self): # committing a weak const to a stated width puts the value on that width's grid, same as storage packing and native compilers v = 1/123008 # not representable in float16 diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 7d8c0dbd3701c..0f0877d9f250c 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -301,7 +301,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_split_ranges+pm_flatten_range, ctx={}, name="split ranges") # symbolic (NOTE: this is a requirement for pm_simplify_ranges to be correct) - sink = graph_rewrite(sink, sym+pm_fold_cast_const+pm_flatten_range, name="initial symbolic") + sink = graph_rewrite(sink, sym+pm_flatten_range, name="initial symbolic") # optimize (schedule) the AST sink = graph_rewrite(sink, pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges") From 4a1f32977cc39700c8263b8a8051f83ac4ec57f9 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Sat, 15 Aug 2026 03:24:37 +0800 Subject: [PATCH 046/306] gptoss: default GROUPED_MOE=1 (#17537) --- .../gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh | 1 + .../gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh b/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh index 35216e667b58d..a6348b34df8e1 100755 --- a/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_beam.sh @@ -11,6 +11,7 @@ export DEVICE_IN_FUNCTION_BUG=1 export DEBUG=${DEBUG:-2} export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1} export ASM_GEMM=${ASM_GEMM:-1} +export GROUPED_MOE=${GROUPED_MOE:-1} export ALL2ALL=${ALL2ALL:-1} export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0} export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1} diff --git a/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh index 9e071ea63653e..d7ae4cdcce914 100755 --- a/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.1/tinycorp/benchmarks/gpt_oss/implementations/tinybox_8xMI350X/dev_run.sh @@ -11,6 +11,7 @@ export DEVICE_IN_FUNCTION_BUG=1 export DEBUG=${DEBUG:-0} export HK_FLASH_ATTENTION=${HK_FLASH_ATTENTION:-1} export ASM_GEMM=${ASM_GEMM:-1} +export GROUPED_MOE=${GROUPED_MOE:-1} export ALL2ALL=${ALL2ALL:-1} export LATE_ALLREDUCE=${LATE_ALLREDUCE:-0} export ALLREDUCE_CAST=${ALLREDUCE_CAST:-1} From 0725acc392c74e1949b4fa07f7ff040d6a03ba74 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:13:18 +0300 Subject: [PATCH 047/306] reenable hcq2 ci (#17532) --- .github/workflows/benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b805c996941d7..739f7242c05dc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,6 +94,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -148,6 +149,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -200,6 +202,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -249,6 +252,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code From 6ea665ed66a8fc02543f6893e73b76fe68619e0f Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 16:24:51 -0400 Subject: [PATCH 048/306] remove pm_fold_cast_const from dtype decomp [pr] (#17536) * remove pm_fold_cast_const from dtype decomp [pr] * fix --- tinygrad/codegen/__init__.py | 2 +- tinygrad/codegen/late/gater.py | 2 +- tinygrad/renderer/llvmir.py | 4 ++-- tinygrad/renderer/nir.py | 3 ++- tinygrad/uop/weak.py | 1 + 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 0f0877d9f250c..02150e4320c66 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -357,7 +357,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # floordiv+mod / dtype decomp (early) supported_ops = tuple(ren.code_for_op.keys()) - pm_decomp = symbolic_simple+pm_fold_cast_const+get_simplifying_rewrite_patterns(supported_ops) + pm_decomp = symbolic_simple+get_simplifying_rewrite_patterns(supported_ops) sink = graph_rewrite(sink, pm_decomp, name="early decompositions") # late decomps + move gates from unrenderable INVALID where diff --git a/tinygrad/codegen/late/gater.py b/tinygrad/codegen/late/gater.py index b9b3a4efb7e1e..e9b46ecfd52b2 100644 --- a/tinygrad/codegen/late/gater.py +++ b/tinygrad/codegen/late/gater.py @@ -3,7 +3,7 @@ from tinygrad.dtype import Invalid, dtypes def move_where_load(gate, l, a, w): - return l.replace(src=(l.src[0], l.vconst_like(0) if a.is_invalid else + return l.replace(src=(l.src[0], l.vconst_like(0) if a.is_invalid else l.const_like(a.val) if a.op is Ops.CONST else a.src[0] if a.op is Ops.CAST and a.src[0].dtype == l.dtype else a.cast(l.dtype), l.src[2])).cast(w.dtype) pm_move_gates_from_index = PatternMatcher([ diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 838e1c95f07a5..3ce6fc8919b6a 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -238,8 +238,8 @@ class AMDLLVMRenderer(LLVMRenderer): (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f" {ctx[x]} = call i8 @f32_to_fp8({ldt(x.src[0].dtype)} {ctx[x.src[0]]}, i1 {'1' if x.dtype == dtypes.fp8e5m2 else '0'})"), (UPat(Ops.CAST, dtypes.float, (UPat.var("y", dtypes.fp8s),), name="x",), lambda ctx,x,y: - f" {ctx[x.src[0]]}_i32 = zext i8 {ctx[x.src[0]]} to i32\n" - f" {ctx[x]} = call float @llvm.amdgcn.cvt.f32.{'bf8' if y.dtype == dtypes.fp8e5m2 else 'fp8'}(i32 {ctx[x.src[0]]}_i32, i32 0)"), + f" {ctx[x]}_i32 = zext i8 {ctx[x.src[0]]} to i32\n" + f" {ctx[x]} = call float @llvm.amdgcn.cvt.f32.{'bf8' if y.dtype == dtypes.fp8e5m2 else 'fp8'}(i32 {ctx[x]}_i32, i32 0)"), ]) + base_rewrite extra_matcher = LLVMRenderer.extra_matcher + create_non_native_float_pats(dtypes.fp8s) + PatternMatcher([ # amd llvm intrinsics llvm.log2/llvm.exp2 don't support double diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index b0d3a38f2ce63..a2153394f66b2 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -137,7 +137,8 @@ class NIRRenderer(Renderer): (UPat(Ops.CAST, (dtypes.uchar, dtypes.ushort), src=(UPat.var("x", dtypes.floats),), name="c"), lambda x,c: x.cast(dtypes.int32).cast(c.dtype)), # load/store use pointer arithmetic, and the cast does nothing. NOTE: this doesn't apply to image indexing cause it's 1-D (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), lambda x,buf,off: x.replace( - src=(buf,off.cast(dtypes.long))+x.src[2:]) if buf.addrspace != AddrSpace.REG and not is_image_shape(buf._shape) else None), + src=(buf,UOp.const(off.val, dtypes.long) if off.op is Ops.CONST else off.cast(dtypes.long))+x.src[2:]) + if buf.addrspace != AddrSpace.REG and not is_image_shape(buf._shape) else None), # images need index to be int for nir (coordinates only: the INDEX keeps its access dtype) (UPat.var("buf").index(UPat.var("idx_y"), UPat.var("idx_x"), name="x"), lambda x,buf,idx_y,idx_x: x.replace(src=(buf, idx_y.cast(dtypes.int), idx_x.cast(dtypes.int)))), diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 0460845664e01..398c68fb3ce74 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -65,6 +65,7 @@ def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: pm_cast_weak = PatternMatcher([ (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), + (UPat(Ops.CAST, name="c", src=(UPat(Ops.CONST, dtype=dtypes.weaks, name="u"),)), lambda c,u: commit_weak(u, c.dtype)), ]) pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ From 64ccbde3bbe2c23463622fd9eb90f1971edf01ac Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 21:07:01 -0400 Subject: [PATCH 049/306] clean up STACK with a const [PR] (#17539) --- tinygrad/uop/ops.py | 4 +--- tinygrad/uop/weak.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 6611ef12bf792..58df80338acc6 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -796,11 +796,9 @@ def _mop(self, op:Ops, arg) -> UOp: case Ops.PAD | Ops.SHRINK: src_args = list(zip(*arg)) case Ops.PERMUTE | Ops.FLIP: src_args = [] case Ops.STACK: - # arg is the other srcs; all are cast to the promoted dtype, spec requires STACK srcs to match its dtype srcs = (self,)+tuple(arg) dtype = cast(DType, dtype_from_uop(Ops.STACK, srcs, None)) - # TODO: why cast here? - return UOp(Ops.STACK, dtype, tuple(u if u.base.is_invalid else u.cast(dtype) for u in srcs)) + return UOp(Ops.STACK, dtype, tuple(u if u.base.is_invalid else UOp.const(u.val, dtype) if u.op is Ops.CONST else u.cast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] if len(usrcs) == 0: return UOp(op, src=(self,), arg=arg) diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 398c68fb3ce74..d5410aa3f6010 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -12,7 +12,7 @@ def lower_weak_node(u:UOp) -> UOp|None: if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None dt = strong_dtype(least_upper_dtype(select_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary else unwrap(dtype_from_uop(u.op, src, u.arg))) - return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else s.cast(dt) for s in src[start:])).cast(u.dtype) + return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else commit_weak(s, dt) for s in src[start:])).cast(u.dtype) pm_lower_weak = PatternMatcher([ (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, select_dtype(u)).cast(u.dtype)), @@ -40,7 +40,7 @@ def lower(s:UOp) -> UOp: return None if ret is u else ret def commit_weak(s:UOp, dt:DType) -> UOp: - # a bare weak CONST commits directly (the value stays mathematical, emission truncates), a weak non-const src takes the demand cast + # a CONST commits directly at dt (the value stays mathematical, emission truncates), a non-const src takes the cast return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) def commit_weak_srcs(u:UOp) -> UOp|None: From 4b0525e59417cf0c18fee132b244e9b0a2f4cd2f Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 14 Aug 2026 21:39:47 -0400 Subject: [PATCH 050/306] no pm_fold_cast_const in UOp.simplify and hcq2 [pr] (#17540) --- test/null/test_const_folding.py | 13 +------------ tinygrad/runtime/support/hcq2.py | 10 +++++----- tinygrad/uop/ops.py | 8 +++++--- tinygrad/uop/symbolic.py | 10 +++++++--- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index b781c6bfe1dde..a4ac45e350b5c 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -1,6 +1,6 @@ import unittest, itertools, math from tinygrad import Tensor, dtypes, Context -from tinygrad.dtype import DType, ConstType, truncate +from tinygrad.dtype import DType, ConstType from tinygrad.uop.ops import Ops, UOp from test.helpers import full_rewrite import numpy as np @@ -59,17 +59,6 @@ def test_single_rounding_log10_backward(self): # correctly rounded: within half a float32 ulp of the exact value (folding at float32 lands 0.66 ulp off) self.assertLess(abs(const - 1/math.log(10)), 2**-26) - def test_cast_commits_to_dtype_grid(self): - # committing a weak const to a stated width puts the value on that width's grid, same as storage packing and native compilers - v = 1/123008 # not representable in float16 - out = UOp.const(v).cast(dtypes.half).simplify() - self.assertEqual((out.op, out.dtype, out.val), (Ops.CONST, dtypes.half, truncate[dtypes.half](v))) - self.assertNotEqual(out.val, v) - # the grid commit preserves the sign of zero - self.assertEqual(math.copysign(1, UOp.const(-0.0).cast(dtypes.half).simplify().val), -1) - # observable at tensor level: the const-folded comparison agrees with the committed value - self.assertTrue((Tensor(-3.2).cast(dtypes.float32) <= truncate[dtypes.float32](-3.2)).item()) - class TestBinaryOpsConstFolding(unittest.TestCase): def test_add_literal_zero(self): _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + 0) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 73f9129da7172..68d6d4f5a4ce4 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -7,7 +7,7 @@ from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const +from tinygrad.uop.symbolic import symbolic from tinygrad.dtype import dtypes, truncate from tinygrad.runtime.support.hcq import MMIOInterface, HCQBuffer from tinygrad.runtime.support.memory import BumpAllocator @@ -445,7 +445,7 @@ def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: linear = graph_rewrite(linear, pm_encode_cmdbufs+pm_pack_placeholders, walk=True, name="encode and pack", enter_calls=True) # patches and runtime uops - linear = graph_rewrite(linear, pm_early_simplify+symbolic+pm_fold_cast_const, bottom_up=False, name="simplify patches", enter_calls=True) + linear = graph_rewrite(linear, pm_early_simplify+symbolic, bottom_up=False, name="simplify patches", enter_calls=True) linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") # and compile it @@ -477,7 +477,7 @@ def fold_binary(buf:UOp, blob:UOp) -> UOp: def fold_const_store(buf:UOp, off:UOp, val:UOp) -> UOp: for off,val in zip(off.src, val.src): for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): - data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype](v.val)) + data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype]((v.src[0] if v.op is Ops.CAST else v).val)) b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[(bo:=off.val*buf.dtype.itemsize):bo+len(data)] = data return UOp(Ops.NOOP) @@ -492,7 +492,7 @@ def resolve_getaddr(buf:UOp, g:UOp) -> UOp: pm_resolve_patches = PatternMatcher([ # multi - (UPat(GroupOp.ALU, src=[UPat(Ops.STACK, name="s"), UPat(Ops.CONST)], name="op"), push_stack), + (UPat(GroupOp.ALU, src=[UPat(Ops.STACK, name="s"), UPat.any(UPat(Ops.CONST), UPat(Ops.CAST, src=(UPat(Ops.CONST),)))], name="op"), push_stack), (UPat(Ops.CAST, src=(UPat(Ops.STACK, name="s"),), name="op"), push_stack), # getaddr @@ -518,7 +518,7 @@ def hcq_link(linear:UOp, cache=True) -> UOp: bufs = {(j,i):a for j,c in enumerate(linear.src) for i,a in enumerate(c.src[1:], 1) if a.op is Ops.AFTER and unwrap_mstack(a.src[0])[0].tag in HCQ_CACHE_TAGS} linear = linear.substitute({x:link_buf_cache[k] for a in bufs.values() if (k:=link_buf_key(a)) in link_buf_cache for x in (a, a.src[0])}, walk=True) - linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_fold_cast_const+pm_assert_no_afters, bpm=pm_bufferize, ctx=cache, bottom_up=False, + linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_assert_no_afters, bpm=pm_bufferize, ctx=cache, bottom_up=False, name="resolve patches") for (j,i),a in bufs.items(): link_buf_cache.setdefault(link_buf_key(a), linear.src[j].src[i]) if cache: link_linear_cache[linear_key] = linear diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 58df80338acc6..81f347524f265 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -517,10 +517,12 @@ def simplify(self, tracked=False): if self.op is Ops.CONST: return self if self.op is Ops.SINK and all(s.op is Ops.CONST or (s.op is Ops.STACK and len(s.src) == 0) for s in self.src): return self # late import! - from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const + from tinygrad.uop.symbolic import symbolic with Context(TRACK_MATCH_STATS=0 if not tracked else TRACK_MATCH_STATS.value): - return graph_rewrite(self, symbolic+pm_fold_cast_const, name="simplify") - def ssimplify(self) -> UOp|ConstType: return ret.val if (ret:=self.simplify()).op is Ops.CONST else ret + return graph_rewrite(self, symbolic, name="simplify") + def ssimplify(self) -> UOp|ConstType: + if (ret := self.simplify()).op is Ops.CAST and ret.src[0].op is Ops.CONST: return ret.dtype.const(ret.src[0].val) + return ret.val if ret.op is Ops.CONST else ret def _eval(self, dtype, expected_type:Type[T]) -> T: assert self.dtype in dtype, f"eval with wrong dtype {self}" vmin, vmax = (simple_self:=self.simplify())._min_max diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 212bf5122d55e..02349330ee417 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -23,8 +23,12 @@ def fold_bitcast(root:UOp, c:UOp) -> UOp|None: if c.dtype.fmt is None or root.dtype.fmt is None or c.dtype.itemsize != root.dtype.itemsize: return None return root.const_like(bitcast(c.val, c.dtype, root.dtype)) +# const folding works for CONST, STACK, and casted CONST +const_folding_pat = UPat.any(UPat((Ops.CONST, Ops.STACK)), UPat(Ops.CAST, src=(UPat(Ops.CONST),))) + def const_arg(u:UOp) -> ConstType|tuple[ConstType, ...]|None: if u.op is Ops.CONST: return u.val + if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return u.dtype.const(u.src[0].val) if u.op is Ops.STACK and all(s.op is Ops.CONST for s in u.src): return tuple(s.val for s in u.src) return None @@ -136,10 +140,10 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) # ** constant folding ** - (UPat(GroupOp.Unary, src=(UPat((Ops.CONST, Ops.STACK)),), name="a"), fold_const_alu), + (UPat(GroupOp.Unary, src=(const_folding_pat,), name="a"), fold_const_alu), # NOTE: THREEFRY(const,const) folds via its decomposition - (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(UPat((Ops.CONST, Ops.STACK)),)*2, name="a"), fold_const_alu), - (UPat(GroupOp.Ternary, src=(UPat((Ops.CONST, Ops.STACK)),)*3, name="a"), fold_const_alu), + (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(const_folding_pat,)*2, name="a"), fold_const_alu), + (UPat(GroupOp.Ternary, src=(const_folding_pat,)*3, name="a"), fold_const_alu), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), (UPat.var('x', dtype=dtypes.bool) + UPat.var('y', dtype=dtypes.bool), lambda x,y: x|y), From e6f5bb9c0949d60895455b90efaf4f636b5396c7 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Sat, 15 Aug 2026 10:00:21 +0800 Subject: [PATCH 051/306] simple test for Invalid clone cache miss regression (#17541) * simple test for Invalid clone cache miss regression * xfail * _ --- test/unit/test_schedule_cache.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/unit/test_schedule_cache.py b/test/unit/test_schedule_cache.py index 078d51214829d..c304eab63641c 100644 --- a/test/unit/test_schedule_cache.py +++ b/test/unit/test_schedule_cache.py @@ -1,6 +1,6 @@ import unittest import functools -from tinygrad import Tensor, Variable, UOp +from tinygrad import Tensor, Variable, UOp, function from tinygrad.uop.ops import KernelInfo from tinygrad.schedule import schedule_cache @@ -65,5 +65,24 @@ def test_simple(self): print(num) self.assertEqual(len(schedule_cache), start_len_schedule_cache) + @unittest.expectedFailure + def test_simple_precompile(self): + @function(precompile=True) + def f(x:Tensor) -> Tensor: + out = Tensor.invalids(*x.shape, dtype=x.dtype, device=x.device) + out = Tensor.custom_kernel(out, fxn=functools.partial(custom_set0_kernel, num=10))[0] + return out + x + + # warmup + x = Tensor.ones(1).realize() + _ = f(x).realize() + + # use the cache next time function is called + start_len_schedule_cache = len(schedule_cache) + for _ in range(3): + num = f(x).realize() + self.assertEqual(num.item(), 11) + self.assertEqual(len(schedule_cache), start_len_schedule_cache) + if __name__ == "__main__": unittest.main() From 5c43a89fb11bc19d6a5dda724d1920bbe9e56d43 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:25:31 +0800 Subject: [PATCH 052/306] precompile_backward tests for sched_cache (#17544) * work * back * work * keep + --- test/unit/test_schedule_cache.py | 45 ++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/test/unit/test_schedule_cache.py b/test/unit/test_schedule_cache.py index c304eab63641c..6b3235903e0d5 100644 --- a/test/unit/test_schedule_cache.py +++ b/test/unit/test_schedule_cache.py @@ -4,8 +4,13 @@ from tinygrad.uop.ops import KernelInfo from tinygrad.schedule import schedule_cache -def custom_set0_kernel(A:UOp, num:int) -> UOp: - return A[0].set(num).sink(arg=KernelInfo(f"custom_set0_{num}")) +def custom_add_kernel(A:UOp, B:UOp, num:int=0) -> UOp: + return A[0].set(B[0] + num).sink(arg=KernelInfo(f"custom_add_{num}")) + +def custom_add_backward(grad_output:UOp, _) -> tuple[None, UOp]: + grad = Tensor.invalids(*grad_output.shape, dtype=grad_output.dtype, device=grad_output.device) + grad = Tensor.custom_kernel(grad, Tensor(grad_output, device=grad_output.device), fxn=functools.partial(custom_add_kernel, num=0))[0] + return None, grad.uop class TestScheduleCache(unittest.TestCase): def test_bound_variable_reuses_cache(self): @@ -25,27 +30,27 @@ def test_bound_variable_reuses_cache(self): def test_custom_kernel(self): for i in range(4): - a = Tensor.empty(1) - a = Tensor.custom_kernel(a, fxn=functools.partial(custom_set0_kernel, num=i))[0] + a, b = Tensor.empty(1), Tensor.ones(1) + a = Tensor.custom_kernel(a, b, fxn=functools.partial(custom_add_kernel, num=i))[0] a.realize() - self.assertEqual(a.item(), i) + self.assertEqual(a.item(), i+1) def test_same_custom_function_reuses_cache(self): schedule_cache.clear() - fxn = functools.partial(custom_set0_kernel, num=10) + fxn = functools.partial(custom_add_kernel, num=10) # first run - a = Tensor.empty(1) - a = Tensor.custom_kernel(a, fxn=fxn)[0] + a, x = Tensor.empty(1), Tensor.ones(1) + a = Tensor.custom_kernel(a, x, fxn=fxn)[0] a.realize() - self.assertEqual(a.item(), 10) + self.assertEqual(a.item(), 11) cache_size_after_first = len(schedule_cache) # second run with same function should reuse cache - b = Tensor.empty(1) - b = Tensor.custom_kernel(b, fxn=fxn)[0] + b, x = Tensor.empty(1), Tensor.ones(1) + b = Tensor.custom_kernel(b, x, fxn=fxn)[0] b.realize() - self.assertEqual(b.item(), 10) + self.assertEqual(b.item(), 11) self.assertEqual(len(schedule_cache), cache_size_after_first) def test_simple(self): @@ -67,21 +72,27 @@ def test_simple(self): @unittest.expectedFailure def test_simple_precompile(self): - @function(precompile=True) + @function(precompile=True, precompile_backward=True) def f(x:Tensor) -> Tensor: out = Tensor.invalids(*x.shape, dtype=x.dtype, device=x.device) - out = Tensor.custom_kernel(out, fxn=functools.partial(custom_set0_kernel, num=10))[0] + out = Tensor.custom_kernel(out, x, fxn=functools.partial(custom_add_kernel, num=10), grad_fxn=custom_add_backward)[0] return out + x # warmup x = Tensor.ones(1).realize() - _ = f(x).realize() + out = f(x) + out.backward(x) + self.assertEqual(out.item(), 12) + self.assertEqual(x.grad.item(), 2) # use the cache next time function is called start_len_schedule_cache = len(schedule_cache) for _ in range(3): - num = f(x).realize() - self.assertEqual(num.item(), 11) + x = Tensor.ones(1).realize() + out = f(x) + out.backward(x) + self.assertEqual(out.item(), 12) + self.assertEqual(x.grad.item(), 2) self.assertEqual(len(schedule_cache), start_len_schedule_cache) if __name__ == "__main__": From a57569349cbf0ec77b6ab9e136966e77fd872e55 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Sat, 15 Aug 2026 17:00:54 +0800 Subject: [PATCH 053/306] renumber invalids before callify (#17542) * renumber invalids before callify * change * Revert "change" This reverts commit 6f4df1541e79721a85ee3f5801114454f264c973. * renumber in tensor * scope renumber_invalid_outputs * cleanup --- test/unit/test_schedule_cache.py | 1 - tinygrad/function.py | 6 ++++++ tinygrad/mixin/gradient.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/test/unit/test_schedule_cache.py b/test/unit/test_schedule_cache.py index 6b3235903e0d5..eae741b5735f4 100644 --- a/test/unit/test_schedule_cache.py +++ b/test/unit/test_schedule_cache.py @@ -70,7 +70,6 @@ def test_simple(self): print(num) self.assertEqual(len(schedule_cache), start_len_schedule_cache) - @unittest.expectedFailure def test_simple_precompile(self): @function(precompile=True, precompile_backward=True) def f(x:Tensor) -> Tensor: diff --git a/tinygrad/function.py b/tinygrad/function.py index b11ba4c25bbc4..813fcd2ec83a6 100644 --- a/tinygrad/function.py +++ b/tinygrad/function.py @@ -1,4 +1,5 @@ import functools, time +from dataclasses import replace from typing import Generic, TypeVar, Callable, cast, overload from tinygrad.helpers import Context, dedup, getenv, DEBUG from tinygrad.uop.ops import UOp, Ops, graph_rewrite, PatternMatcher, UPat @@ -23,6 +24,10 @@ def invalid_outputs(uret:UOp) -> set[UOp]: return {u.src[0].buf_uop for u in uret.backward_slice_with_self if u.op is Ops.STORE and u.src[1].base.is_invalid and not u.src[0].buf_uop.is_realized} +def renumber_invalid_outputs(uret:UOp) -> UOp: + return uret.substitute({b:b.replace(arg=replace(b.arg, slot=i)) + for i,b in enumerate(x for x in uret.toposort(enter_calls=False) if x in invalid_outputs(uret))}) + ReturnType = TypeVar('ReturnType') class _function(Generic[ReturnType]): depth = 0 @@ -65,6 +70,7 @@ def __call__(self, *args, **kwargs) -> ReturnType: # the BUFFERs that are left are the implicit inputs num_explicit = len(call_uops) uret = graph_rewrite(uret, pm_ctx, (call_uops, invalid_outputs(uret)), bottom_up=True, name="get_implicit_inputs") + uret = renumber_invalid_outputs(uret) name = getattr(self.fxn, '__qualname__', None) or type(self.fxn).__qualname__ if not self.allow_implicit: implicit_buffers = [x for x in call_uops[num_explicit:] if x.op is Ops.BUFFER] diff --git a/tinygrad/mixin/gradient.py b/tinygrad/mixin/gradient.py index 14fa98c00b44e..b73e63e45a687 100644 --- a/tinygrad/mixin/gradient.py +++ b/tinygrad/mixin/gradient.py @@ -3,6 +3,7 @@ from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, all_metadata, broadcast_axes from tinygrad.helpers import argsort from tinygrad.dtype import sum_acc_dtype +from tinygrad.function import renumber_invalid_outputs def reduce_gradient(ctx:UOp, ret:UOp, op:Ops): if op == Ops.ADD: return (ctx._broadcast_to(ret.src[0].shape),) @@ -40,6 +41,7 @@ def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else # collect needed gradient bodies, compact unused params, create a single backward CALL grad_bodies = [(i, grads[p]) for i in needed if (p:=params.get(i)) is not None and p in grads] bwd_body = UOp.maketuple(*(gb for _, gb in grad_bodies)).substitute(fwd_subs, walk=True) + bwd_body = renumber_invalid_outputs(bwd_body) bwd_body, compact_args = _compact_params(bwd_body, (*args, *grad_args, *fwd_outs)) bwd_call = bwd_body.call(*compact_args, name=(k.arg.name or "")+"_backward", precompile=k.arg.precompile_backward) gb_map = {i: idx for idx, (i, _) in enumerate(grad_bodies)} From 539a03343a45a94893f07ececeea6e80b651e8dc Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 15 Aug 2026 07:47:40 -0400 Subject: [PATCH 054/306] no casted const from sub and div [pr] (#17543) --- test/unit/test_dtype_weak.py | 5 +++++ tinygrad/mixin/elementwise.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index a4e9740eb9bd1..ea6a612a3c583 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -76,6 +76,11 @@ def test_weak_expression_anchors_at_strong_lub(self): committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_lower_index_dtype, ctx={}) self.assertEqual([u.dtype for u in committed.toposort() if u.op is Ops.ADD], [dtypes.float32]) + def test_div_sub_operand_kept_weak(self): + a = Tensor.empty(4, dtype=dtypes.float32) + for t in (a / 1, a - 0): + self.assertEqual(t.uop.src[1].dtype, dtypes.weakfloat) + def test_cast_weak_expression_commits_at_cast_floor(self): # the floor never narrows: a cast BELOW the default does not pull the compute width down with it with Context(DEFAULT_FLOAT=dtypes.float32): diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index f06fd5aa52256..3818ef0b5e2b8 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -115,7 +115,8 @@ def sub(self, x: Self | ConstType, reverse: bool = False) -> Self: ``` """ a, b = self._broadcasted(x, reverse) - return a + (-b) + # alu, not +: _broadcasted already promoted these, and a second promote would cast -b (only a bare weak CONST is kept weak) + return a.alu(Ops.ADD, -b) def mul(self, x: Self | ConstType, reverse: bool = False) -> Self: """ @@ -245,8 +246,9 @@ def div(self, x: 'Self|ConstType|UOp', reverse: bool = False, rounding_mode: Lit if dtypes.is_int(a.dtype) and dtypes.is_int(b.dtype): if rounding_mode == "trunc": return a.alu(Ops.CDIV, b) if rounding_mode == "floor": return a.alu(Ops.FLOORDIV, b) - a = a.cast(dtypes.default_float) - d = a * b.reciprocal() + if dtypes.is_int(a.dtype) or a.dtype == dtypes.bool: a = a.cast(dtypes.default_float) + # alu, not *: _broadcasted already promoted these, and a second promote would cast 1/b (only a bare weak CONST is kept weak) + d = a.alu(Ops.MUL, b.reciprocal()) if rounding_mode is None: return d if rounding_mode == "trunc": return d.trunc() if rounding_mode == "floor": return d.floor() From c17849a1f8888910cc2f25af86206121662ff756 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Sat, 15 Aug 2026 15:03:28 +0300 Subject: [PATCH 055/306] disk: neable polling (#17538) --- tinygrad/runtime/ops_disk.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tinygrad/runtime/ops_disk.py b/tinygrad/runtime/ops_disk.py index 801bd35b68d11..0759f01a242cc 100644 --- a/tinygrad/runtime/ops_disk.py +++ b/tinygrad/runtime/ops_disk.py @@ -49,7 +49,8 @@ def _iouring_setup(self): DiskDevice._tried_io_uring_init = True if sys.platform == 'linux' and not hasattr(sys, "getandroidapilevel"): - fd = libc.syscall(io_uring.NR_io_uring_setup, 4096, ctypes.byref(p:=io_uring.struct_io_uring_params())) + p = io_uring.struct_io_uring_params(flags=io_uring.IORING_SETUP_SQPOLL, sq_thread_idle=0xffffffff) + fd = libc.syscall(io_uring.NR_io_uring_setup, 4096, ctypes.byref(p)) if fd < 0: return sq_ptr = libc.mmap(0, p.sq_off.array + p.sq_entries * 4, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | MAP_POPULATE, fd, 0) @@ -67,6 +68,7 @@ def u32ptr(val): return ctypes.cast(val, ctypes.POINTER(ctypes.c_uint32)) kring_mask=u32ptr(sq_ptr+p.cq_off.ring_mask), cqes=ctypes.cast(cq_ptr+p.cq_off.cqes, ctypes.POINTER(io_uring.struct_io_uring_cqe))) DiskDevice.io_uring = io_uring.struct_io_uring(ring_fd=fd, sq=sqdesc, cq=cqdesc) # type: ignore + libc.syscall(io_uring.NR_io_uring_enter, fd, 0, 0, io_uring.IORING_ENTER_SQ_WAKEUP) class DiskBuffer: def __init__(self, device:DiskDevice, size:int, offset=0): @@ -124,7 +126,6 @@ def _copyout_sharded(self, src:DiskBuffer, size:int, _get_free_buf:Callable, seg # Send sqe DiskDevice.io_uring.sq.array[sqe_index] = sqe_index DiskDevice.io_uring.sq.ktail[0] = tail + 1 - libc.syscall(io_uring.NR_io_uring_enter, DiskDevice.io_uring.ring_fd, 1, 1, io_uring.IORING_ENTER_GETEVENTS) reqs.append((copy_batch, copied_in, minor_offset, real_copy_size:=min(sqe.len - minor_offset, size - copied_in))) next_read_offset += sqe.len From fae893753b1c642f67f534cbd339746d1707d67e Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 15 Aug 2026 09:57:05 -0400 Subject: [PATCH 056/306] no pm_fold_cast_const in get_kernel_graph [pr] (#17546) * no pm_fold_cast_const in get_kernel_graph [pr] * maybe --- test/backend/test_ops.py | 6 ++++-- tinygrad/schedule/rangeify.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 9eacc5b524730..27beab433157c 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -720,10 +720,11 @@ def get_torch_gradient(x, c): return torch.autograd.grad(t ** c, t)[0].item() for x in [-math.inf, 0, 1, math.inf]: for c in [-1, 0, 0.3, 1, 2]: - tiny_out = get_tiny_gradient(x, c) torch_out = get_torch_gradient(x, c) + # the pow backward routes through exp2/log2, whose 0/inf behavior is undefined on WEBGPU + if Device.DEFAULT == "WEBGPU" and not math.isfinite(torch_out): continue + tiny_out = get_tiny_gradient(x, c) if math.isnan(tiny_out): - if Device.DEFAULT == "WEBGPU": continue # TODO: WEBGPU issue with nan assert math.isnan(torch_out) else: self.assertAlmostEqual(tiny_out, torch_out, msg=f"{x}, {c}") @@ -749,6 +750,7 @@ def test_pow_zero_tensor(self): def test_exp2_log2_zero_times_negative(self): # gallivm's exp2/log2 have "undefined behavior with infs, 0s and nans", so exp2(log2(0)*y) returns 0 instead of inf helper_test_op(None, lambda x,y: (x.log2()*y).exp2(), lambda x,y: (x.log2()*y).exp2(), vals=[[0.0], [-0.7]], forward_only=True) + @unittest.skipIf(Device.DEFAULT == "WEBGPU", "pow at 0 routes through exp2/log2, whose 0/inf behavior is undefined on WEBGPU") def test_pow_zero_const(self): helper_test_op(None, lambda x: x**0.3, vals=[[0.0]]) helper_test_op(None, lambda x: x**0.0, vals=[[0.0]]) diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index bf64d55761462..c16b73811f618 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -4,7 +4,7 @@ from tinygrad.dtype import dtypes, AddrSpace, Invalid, to_dtype, strong_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg, shape_to_shape_arg from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group, identity_element -from tinygrad.uop.symbolic import symbolic, pm_fold_cast_const +from tinygrad.uop.symbolic import symbolic from tinygrad.uop.movement import mop_cleanup from tinygrad.helpers import prod, getenv, dedup, all_int, DEBUG, SPLIT_REDUCEOP, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC from tinygrad.helpers import PCONTIG, FLOAT16, OPENPILOT_HACKS, argsort, partition, get_single_element @@ -584,7 +584,7 @@ def get_kernel_graph(sink:UOp) -> UOp: tsink, rctx = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) tsink = graph_rewrite(tsink, - symbolic+pm_fold_cast_const+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize+pm_no_indexing_calls, + symbolic+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize+pm_no_indexing_calls, name="symbolic+reduce_collapse+debuf") tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=rctx, name="limit buffers") From 26cbadd69a29dba20c99832b9cddcefb13527615 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 15 Aug 2026 10:18:59 -0400 Subject: [PATCH 057/306] no pm_fold_cast_const in full_rewrite_to_sink [PR] (#17547) --- test/null/test_uop_symbolic.py | 5 +++-- tinygrad/codegen/__init__.py | 4 ++-- tinygrad/uop/symbolic.py | 4 ---- tinygrad/uop/weak.py | 3 +++ 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 20a740bfbb992..697d3c8890595 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -5,7 +5,8 @@ from tinygrad.dtype import dtypes, ConstType, DType, Invalid from tinygrad.uop.ops import UOp, Ops, graph_rewrite, sym_infer from tinygrad.uop.spec import spec_shared, type_verify -from tinygrad.uop.symbolic import sym, pm_fold_cast_const, commutative, pm_simplify_valid, pm_move_where_on_load +from tinygrad.uop.symbolic import sym, commutative, pm_simplify_valid, pm_move_where_on_load +from tinygrad.uop.weak import pm_cast_weak from tinygrad.uop.validate import uops_to_z3 def check_uop_against_string(self, v:UOp, s:str): @@ -35,7 +36,7 @@ def check_equal_z3(self, expr1, expr2): self.assertEqual(solver.check(expr1 != expr2), z3.unsat, "simplified expression not equal to original") def helper_test_variable(self, v, n, m, s, test_z3:bool=True): - v_simplified = graph_rewrite(v, sym+pm_fold_cast_const, name="simplify symbolic uop") + v_simplified = graph_rewrite(v, sym+pm_cast_weak, name="simplify symbolic uop") if test_z3: self.check_equal_z3(v, v_simplified) nmin, nmax = v_simplified.vmin, v_simplified.vmax check_uop_against_string(self, v_simplified, s) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 02150e4320c66..05d666a72410a 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -12,7 +12,7 @@ # import all pattern matchers here from tinygrad.codegen.gpudims import pm_add_gpudims -from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_fold_cast_const, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid +from tinygrad.uop.symbolic import sym, symbolic_simple, symbolic, pm_move_where_on_load, pm_clean_up_group_sink, pm_remove_invalid from tinygrad.uop.movement import mop_cleanup from tinygrad.codegen.decomp.dtype import pm_dtype_decomps from tinygrad.codegen.decomp.op import get_late_rewrite_patterns, get_simplifying_rewrite_patterns @@ -346,7 +346,7 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # lower index dtype # NOTE: we need indexing_simplify to remove the cast to long using the Invalid - sink = graph_rewrite(sink, symbolic_simple+pm_fold_cast_const+pm_lower_index_dtype+indexing_simplify, ctx={}, name="lower all index dtypes") + sink = graph_rewrite(sink, symbolic_simple+pm_lower_index_dtype+indexing_simplify, ctx={}, name="lower all index dtypes") # final symbolic before decomp sink = graph_rewrite(sink, symbolic, name="final symbolic") diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 02349330ee417..de3556bff6257 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -100,10 +100,6 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: if any(x.is_invalid for x in s.src) else None), ]) -# the one rule that collapses the pair CAST(dt, CONST(v)) into a typed CONST -# TODO: delete this once CONST has no dtype -pm_fold_cast_const = PatternMatcher([(UPat(Ops.CAST, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val))]) - symbolic_simple = pm_data_invalid + PatternMatcher([ # ** self folding ** (UPat.var("x") + 0, lambda x: x), # x+0 -> x diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index d5410aa3f6010..068b5b3fc02c4 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -69,6 +69,9 @@ def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: ]) pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ + # a CAST between two concrete dtypes over a CONST is a value conversion: evaluate it once, at the width the CAST states + # TODO: delete this once CONST has no dtype + (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c", dtypes.all),)), lambda root, c: root.const_like(c.val)), (UPat(GroupOp.All, name="u"), lambda ctx,u: lower_weak_srcs(ctx, u) if u.dtype not in dtypes.weaks and any(s.dtype in dtypes.weaks for s in u.src) else None), # a valid index into an n-element buffer lives in [0,n): a gated long index narrows when n-1 fits int32 (out-of-gate wraps, discarded) From 5ca87f1bacc07306037757e95db9e5d46ef04d48 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 15 Aug 2026 12:39:54 -0400 Subject: [PATCH 058/306] fix cast to weak twice [pr] (#17548) also no gradient for weak target --- test/unit/test_dtype_weak.py | 7 +++++++ test/unit/test_gradient.py | 10 +++++++--- tinygrad/mixin/op.py | 1 + tinygrad/uop/weak.py | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index ea6a612a3c583..90b8d16b9015a 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -93,6 +93,13 @@ def test_cast_weak_expression_value_uses_cast_floor(self): out = Tensor(1.0, dtype=dtypes.float32, device="CPU") / denom self.assertAlmostEqual(out.item(), 1 / (70000 + 1e-5), places=10) + def test_stacked_weak_casts_convert_each_kind(self): + # each weak cast is a kind conversion: weakint truncates before weakfloat re-lifts (neither is only a marker) + x = Tensor([2.5, -3.7], dtype=dtypes.float32, device="CPU") + stacked = x.cast(dtypes.weakint).cast(dtypes.weakfloat) + self.assertIs(stacked.dtype, dtypes.weakfloat) + self.assertEqual(stacked.tolist(), [2.0, -3.0]) + def test_uop_scalar_const_lifts_kind(self): for dtype, value, out_dtype, const_dtype in ((dtypes.weakint, 1, dtypes.weakint, dtypes.weakint), (dtypes.int32, 1, dtypes.int32, dtypes.weakint), diff --git a/test/unit/test_gradient.py b/test/unit/test_gradient.py index 8a1f91c16b224..5f704aeb338b1 100644 --- a/test/unit/test_gradient.py +++ b/test/unit/test_gradient.py @@ -51,6 +51,10 @@ def test_non_float_tensor_raise(self): with self.assertRaises(RuntimeError): x.sum().gradient(x) with self.assertRaises(RuntimeError): x.float().sum().gradient(x) + def test_const_target_raise(self): + t = Tensor(2.0) + with self.assertRaises(RuntimeError): (t * 2.0).gradient(t) + def test_copy_to_device_gradient(self): t = Tensor([1.0, 2, 3]).realize() t.to("CPU:1").square().sum().backward() @@ -100,7 +104,7 @@ def test_gradient_through_chained_unrealized_setitem(self): def test_implicit_broadcast_where_gradient(self): # WHERE with a bare ()-shape branch: the scalar's gradient counts the positions where it is selected - cond, x, w = Tensor([True, False, True]), Tensor([1.0, 2.0, 3.0]), Tensor(4.0) + cond, x, w = Tensor([True, False, True]), Tensor([1.0, 2.0, 3.0]), Tensor(4.0, dtype=dtypes.float32) dw = Tensor(cond.uop.alu(Ops.WHERE, x.uop, w.uop)).sum().gradient(w)[0] self.assertEqual(dw.shape, ()) self.assertEqual(dw.item(), 1.0) @@ -109,7 +113,7 @@ def test_implicit_broadcast_where_gradient(self): def test_implicit_broadcast_alu_gradient(self): # MUL with a bare ()-shape src, no EXPAND in the graph - x, w = Tensor([1.0, 2.0, 3.0]), Tensor(2.0) + x, w = Tensor([1.0, 2.0, 3.0]), Tensor(2.0, dtype=dtypes.float32) m = x.uop.alu(Ops.MUL, w.uop) self.assertIs(m.src[1], w.uop) dw = Tensor(m).sum().gradient(w)[0] @@ -118,7 +122,7 @@ def test_implicit_broadcast_alu_gradient(self): def test_implicit_broadcast_intermediate_accumulation(self): # s is used directly and through an implicit broadcast edge, each edge's gradient reduces to s's shape before they sum - x, p = Tensor([1.0, 2.0, 3.0]), Tensor(0.5) + x, p = Tensor([1.0, 2.0, 3.0]), Tensor(0.5, dtype=dtypes.float32) s = p.sin() z = Tensor(x.uop.alu(Ops.MUL, s.uop)).sum() + s dp = z.gradient(p)[0] diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 15c9c0d1edede..94b1dbe4efeaf 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -460,6 +460,7 @@ def gradient(self, *targets:Self, gradient:Self|None=None) -> list[Self]: """ assert gradient is not None or self.shape == tuple(), "when no gradient is provided, backward must be called on a scalar tensor" if not (self.is_floating_point() and all(t.is_floating_point() for t in targets)): raise RuntimeError("only float Tensors have gradient") + if any(t.dtype in dtypes.weaks for t in targets): raise RuntimeError("cannot take gradient wrt a weak Tensor") from tinygrad.mixin.gradient import compute_gradient if gradient is None: gradient = self.const_like(1.0) target_uops = [t._uop for t in targets] diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 068b5b3fc02c4..0b60cdb5604cb 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -16,10 +16,10 @@ def lower_weak_node(u:UOp) -> UOp|None: pm_lower_weak = PatternMatcher([ (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, select_dtype(u)).cast(u.dtype)), - # two stacked weak casts are a weakint value used as weakfloat (or vice versa): resolve the inner one at the outer kind's default. + # two stacked weak casts are two kind conversions: each resolves at its own kind's default # a SINGLE weak cast is never rewritten here, each consumer absorbs it on its own edge (see lower_weak_srcs) (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), - lambda u,x: x.cast(select_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), + lambda u,x: x.cast(select_dtype(u.src[0])).cast(select_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), # Binary can widen from the bounds, all other nodes derive from the lowered sources. # a weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition (UPat(GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL}, name="u"), lower_weak_node), From 417563ca20caaec68c8497ee56008d97096c149a Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 15 Aug 2026 16:31:19 -0400 Subject: [PATCH 059/306] fix webgpu is_nan [pr] (#17551) --- test/backend/test_const_folding.py | 1 - tinygrad/renderer/wgsl.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/backend/test_const_folding.py b/test/backend/test_const_folding.py index 0e4239ecfe63c..0c2c80156063c 100644 --- a/test/backend/test_const_folding.py +++ b/test/backend/test_const_folding.py @@ -188,7 +188,6 @@ def test_truth_table(self): np.testing.assert_equal((Tensor(True) < Tensor(False)).numpy(), False) np.testing.assert_equal((Tensor(True) < Tensor(True)).numpy(), False) - @unittest.skipIf(Device.DEFAULT == "WEBGPU", "WEBGPU doesn't support NaN comparison correctly") def test_a_eq_a(self): # self eq is always true for int or bool a = Tensor([1, 2, 3]) diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 4cecf8116ebb1..83a9355e6b754 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -50,8 +50,9 @@ def is_nan(a): (UPat.store(UPat.var("b"), UPat.var("var"), name="s"), lambda b,var,s: packed_store(b,var) if is_packed(s) else None), (UPat.var("a") << UPat.var("b"),lambda a,b:(a.bitcast(dtypes.uint32)<> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), - # fix nan check: 'a != a -> is_nan()' - (UPat.var("a") != UPat.var("a"), is_nan), + # fix nan check: 'a != a -> is_nan()'. the decomp rewrites (a != a).logical_not() to CMPEQ, so match both forms + (UPat.var("a", dtypes.floats) != UPat.var("a"), is_nan), + (UPat.var("a", dtypes.floats).alu(Ops.CMPEQ, UPat.var("a")), lambda a: is_nan(a).ne(True)), ]) class WGSLRenderer(CStyleLanguage): From 97022960ae1fe3d898b758fd67e3dbe14a985fee Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:01:36 +0300 Subject: [PATCH 060/306] device: fix remap (#17549) --- test/device/test_hcq2.py | 8 ++++++++ tinygrad/device.py | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index 19d72d7bb8609..a8ffe60a29604 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -10,5 +10,13 @@ def test_copy_without_copy_queue(self): with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + def test_overlapping_device_tuples(self): + # an op on a wide device tuple followed by an op on an overlapping smaller tuple used to MMU-fault the smaller one + d4, d2 = tuple(f"{Device.DEFAULT}:{i}" for i in range(4)), tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) + ref = Tensor.arange(16).contiguous().realize() + Tensor(ref.uop.copy_to_device(d4)).realize() + out = Tensor.ones(8).shard(d2, axis=0).contiguous().realize() + np.testing.assert_equal(out.numpy(), np.ones(8)) + if __name__ == "__main__": unittest.main() diff --git a/tinygrad/device.py b/tinygrad/device.py index 6f796e07a6f17..6176ced789439 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -103,7 +103,7 @@ class Buffer: def __init__(self, device:str, size:int, dtype:DType, opaque:Any=None, options:BufferSpec|None=None, initial_value:bytes|pickle.PickleBuffer|None=None, uop_refcount=0, base:Buffer|None=None, offset:int=0, preallocate=False): assert isinstance(dtype, DType) - self.device, self.size, self.dtype, self.options, self.offset, self.allocated_views = device, size, dtype, options, offset, 0 + self.device, self.size, self.dtype, self.options, self.offset, self.allocated_views = Device.canonicalize(device), size, dtype, options, offset, 0 self._bufs: dict[str, Any] = {} if base is None: assert offset == 0, "base buffers can't have offset" @@ -116,7 +116,7 @@ def __init__(self, device:str, size:int, dtype:DType, opaque:Any=None, options:B if isinstance(initial_value, pickle.PickleBuffer): initial_value.release() else: assert base._base is None, "base can't have a base" - assert device == base.device, "base must have the same device" + assert self.device == base.device, "base must have the same device" self._base = base if preallocate: self.allocate() @property @@ -133,7 +133,7 @@ def is_initialized(self) -> bool: return self.is_allocated() and self.device in # check if the underlying buffer is allocated, possibly from the base object def is_allocated(self) -> bool: return self.base.is_allocated() if self._base is not None else self.device in self._bufs def get_buf(self, device: str) -> Any: - if device not in self._bufs: + if (device:=Device.canonicalize(device)) not in self._bufs: allocator = Device[device].allocator if device == self.device: self.ensure_allocated() elif self._base is not None: self._bufs[device] = allocator._offset(self._base.get_buf(device), self.nbytes, self.offset) From e688e077586ff0f6b554b864a928ec295e1e7e9a Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Sat, 15 Aug 2026 20:05:10 -0700 Subject: [PATCH 061/306] add max_shape/max_numel to mixins + pad_to (#17553) --- test/backend/test_tensor.py | 10 ++++++++++ tinygrad/mixin/movement.py | 10 ++++++++++ tinygrad/mixin/op.py | 6 ++++++ tinygrad/uop/ops.py | 4 ---- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/test/backend/test_tensor.py b/test/backend/test_tensor.py index c44a2b9e08311..c77b8d649bdc0 100644 --- a/test/backend/test_tensor.py +++ b/test/backend/test_tensor.py @@ -653,9 +653,19 @@ def test_pad(self): np.testing.assert_equal(Tensor([[1, 2]]).pad_to(2, 3).numpy(), [[1, 2, 0], [0, 0, 0]]) np.testing.assert_equal(Tensor([[1, 2]]).pad_to(1, 3).numpy(), [[1, 2, 0]]) np.testing.assert_equal(Tensor([[1, 2]]).pad_to(None, 3).numpy(), [[1, 2, 0]]) + np.testing.assert_equal(Tensor([1, 2]).pad_to(4, value=2).numpy(), [1, 2, 2, 2]) + np.testing.assert_equal(Tensor([[1, 2]]).pad_to(2, 3, value=-1).numpy(), [[1, 2, -1], [-1, -1, -1]]) + np.testing.assert_equal(Tensor([1, 2]).pad_to(None, value=5).numpy(), [1, 2]) # no-op pad ignores the fill with self.assertRaises(ValueError): Tensor([1, 2]).pad_to(2, 3) with self.assertRaises(ValueError): Tensor([[1, 2]]).pad_to(3) + def test_max_shape(self): + from tinygrad import UOp + t = Tensor.empty(2, UOp.variable('v', 1, 32), 4) + self.assertEqual(t.max_shape, (2, 32, 4)) + self.assertEqual(t.max_numel(), 2*32*4) + self.assertEqual(Tensor.empty(2, 3).max_shape, (2, 3)) + def test_shrink_into_zero(self): t = Tensor.rand(3, 4).realize() assert t.shrink((None, (2, 2))).realize().shape == (3, 0) diff --git a/tinygrad/mixin/movement.py b/tinygrad/mixin/movement.py index b916f8af9daa8..c900993019b9d 100644 --- a/tinygrad/mixin/movement.py +++ b/tinygrad/mixin/movement.py @@ -46,6 +46,16 @@ def numel(self) -> sint: """ return prod(self.shape) + @property + def max_shape(self) -> tuple[int, ...]: + """The shape with every symbolic dimension replaced by its maximum.""" + from tinygrad.uop.ops import to_max_shape # deferred: ops.py imports the mixins + return to_max_shape(self.shape) + + def max_numel(self) -> int: + """The number of elements in `max_shape`.""" + return prod(self.max_shape) + def size(self, dim:int|None=None) -> sint|tuple[sint, ...]: """ Returns the size of the tensor. If `dim` is specified, return the length along dimension `dim`. Otherwise return the shape of the tensor. diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 94b1dbe4efeaf..9f65649042f75 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -289,6 +289,12 @@ def _pad_constant(self, pX, value:ConstType) -> Self: if value == 0: return base return MovementMixin.pad(X.const_like(True, dtypes.bool), pads).where(base, value) + def pad_to(self, shape, *args, value:ConstType=0) -> Self: + # same mask trick as _pad_constant so the fill survives backends that realize PAD as 0-fill + ret = MovementMixin.pad_to(self, shape, *args) + if value == 0 or ret is self: return ret + return MovementMixin.pad_to(self.const_like(True, dtypes.bool), shape, *args).where(ret, value) + def _pad_circular(self, pX:tuple[tuple[sint, sint], ...]) -> Self: # shrink first for negative pads, then wrap the non-negative remainder X = self.shrink(tuple((-smin(pB,0), smin(pA+sh,sh)) for (pB,pA),sh in zip(pX, self.shape))) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 81f347524f265..40363b03504ba 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -470,10 +470,6 @@ def shape(self) -> tuple[sint, ...]: if (ret:=self._shape) is None: raise RuntimeError(f"shape requested, but {self.op} doesn't have a shape") return ret - @property - def max_shape(self) -> tuple[int, ...]: return to_max_shape(self.shape) - def max_numel(self) -> int: return prod(self.max_shape) - @property def shard_shape(self) -> tuple[sint, ...]: if not isinstance(self.device, tuple) or self.axis is None: return self.shape From e7bf2a811d0584b6379e452774911d5e56f8607b Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:09:24 +0300 Subject: [PATCH 062/306] iface in device (#17554) * iface in device * drop * move * sorry --- extra/hcq2/ops_amd2.py | 4 +--- tinygrad/device.py | 25 ++++++++++++++++++------- tinygrad/runtime/ops_amd.py | 4 +--- tinygrad/runtime/ops_cl.py | 26 +++++++++++++------------- tinygrad/runtime/ops_nv.py | 3 +-- tinygrad/runtime/ops_rdma.py | 3 +-- tinygrad/runtime/support/hcq.py | 23 +++-------------------- tinygrad/runtime/support/hcq2.py | 28 ++++++---------------------- 8 files changed, 44 insertions(+), 72 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 77dc0d3964e7d..11db4f3f72c15 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -561,9 +561,7 @@ def is_am(self) -> bool: return isinstance(self.iface, (PCIIface,)) def is_usb(self) -> bool: return False def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - - self.iface = self._select_iface() + self.iface = self._select_iface(device) self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target diff --git a/tinygrad/device.py b/tinygrad/device.py index 6176ced789439..de6cc53b97227 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, replace from collections import defaultdict -from typing import Any, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING +from typing import Any, Callable, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal from tinygrad.helpers import LRU, getenv, diskcache_get, diskcache_put, DEBUG, GlobalCounters, PROFILE, temp, colored from tinygrad.helpers import Context, CCACHE, ALLOW_DEVICE_USAGE, MAX_BUFFER_SIZE, cpu_events, ProfileEvent, ProfilePointEvent, suppress_finalizing @@ -331,17 +331,18 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup wait=False) -> float|None: pass class Compiled: + ifaces:list[Callable] = [] profile_events:list[ProfileEvent] = [ProfileDeviceEvent("CPU")] # NOTE: CPU is the default device. + has_copy_queue:bool = True + pm_lower:Any = None pm_bufferize:Any = None - has_copy_queue:bool = True - def __init__(self, device:str, allocator:Allocator, renderers:list[type[Renderer]], runtime:type[Program[Self]]|None, graph=None, arch=None): from tinygrad.renderer import Renderer self.device, self.allocator, self.runtime_t, self.graph, self.renderers = device, allocator, runtime, graph, renderers or [Renderer] - self.arch = arch + self.device_id, self.arch = (int(idx) if ":" in device and (idx:=device.split(":")[1]).isdigit() else 0), arch self.cached_renderer:dict[Any, Renderer] = {} @property @@ -364,11 +365,21 @@ def _select_renderer(self) -> Renderer: return select_first_inited(select_by_name(self.renderers, self._renderer_name, t.renderer, f"{self.device} has no renderer {t.renderer!r}"), f"No renderer for {self.device} is available", self.cached_renderer, t) + def _select_iface(self, device:str): + self.device_id = int(device.split(":")[1]) if ":" in device else 0 + assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ + f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" + t = DEV.target(dev:=type(self).__name__[:-6]) + filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") + filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fallback to mock ifaces + return select_first_inited([functools.partial(iface, self, self.device_id) for iface in filtered], + f"No interface for {dev}:{self.device_id} is available") + def count(self) -> int: """ Returns the number of physical accelerators available to the runtime. """ - return 1 + return self.iface.count if hasattr(self, 'iface') else 1 def synchronize(self): """ @@ -386,7 +397,7 @@ def finalize(self): """ Called at the end of process lifetime to allow the device to finalize. """ - # override this in your device implementation + if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() if PROFILE: @atexit.register @@ -408,7 +419,7 @@ def enumerate_devices_str() -> Generator[str, None, None]: ren_results, iface_results = [], [] try: d = Device[device] - for iface in [i for i in getattr(d, 'ifaces', []) if not i.__name__.startswith("MOCK")]: + for iface in [i for i in d.ifaces if not i.__name__.startswith("MOCK")]: try: name = iface.__name__[:-5] default_text, count = ("(default)", d.count()) if type(d.iface) is iface else (f"(DEV={name}+{device} to make default)", iface(d, 0).count) # type: ignore diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 7404a1fca3c6f..6c0878af25005 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -944,9 +944,7 @@ def is_am(self) -> bool: return isinstance(self.iface, (PCIIface, USBIface)) def is_usb(self) -> bool: return isinstance(self.iface, USBIface) def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - - self.iface = self._select_iface() + self.iface = self._select_iface(device) self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target diff --git a/tinygrad/runtime/ops_cl.py b/tinygrad/runtime/ops_cl.py index 5087996e31fdc..33180f2c5c84e 100644 --- a/tinygrad/runtime/ops_cl.py +++ b/tinygrad/runtime/ops_cl.py @@ -24,10 +24,10 @@ def __init__(self, dev:CLDevice, compile_key:str): super().__init__(f"compile_cl_{compile_key}") def compile(self, src:str) -> bytes: program = checked(cl.clCreateProgramWithSource(self.dev.context, 1, to_char_p_p([src.encode()]), None, status := ctypes.c_int32()), status) - build_status: int = cl.clBuildProgram(program, 1, self.dev.device_id, None, BP_CB(), None) + build_status: int = cl.clBuildProgram(program, 1, self.dev.cl_dev, None, BP_CB(), None) if build_status != 0: - cl.clGetProgramBuildInfo(program, self.dev.device_id, cl.CL_PROGRAM_BUILD_LOG, 0, None, log_size := ctypes.c_size_t()) - cl.clGetProgramBuildInfo(program, self.dev.device_id, cl.CL_PROGRAM_BUILD_LOG, + cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, 0, None, log_size := ctypes.c_size_t()) + cl.clGetProgramBuildInfo(program, self.dev.cl_dev, cl.CL_PROGRAM_BUILD_LOG, log_size.value, mstr := ctypes.create_string_buffer(log_size.value), None) raise CompileError(f"OpenCL Compile Error\n\n{mstr.value.decode()}") check(cl.clGetProgramInfo(program, cl.CL_PROGRAM_BINARY_SIZES, ctypes.sizeof(ctypes.c_size_t), binary_sizes := (ctypes.c_size_t * 1)(), None)) @@ -39,11 +39,11 @@ def compile(self, src:str) -> bytes: class CLProgram(Program['CLDevice']): def __init__(self, device:CLDevice, obj:TinyELF): self.dev, self.lib, self.signature = device, device.cl_compiler.compile_cached(obj.lib.decode()), obj.signature - self.program = checked(cl.clCreateProgramWithBinary(device.context, 1, device.device_id, (ctypes.c_size_t * 1)(len(self.lib)), + self.program = checked(cl.clCreateProgramWithBinary(device.context, 1, device.cl_dev, (ctypes.c_size_t * 1)(len(self.lib)), to_char_p_p([self.lib], ctypes.c_ubyte), binary_status := ctypes.c_int32(), errcode_ret := ctypes.c_int32()), errcode_ret) check(binary_status.value) - check(cl.clBuildProgram(self.program, 1, device.device_id, None, BP_CB(), None)) # NOTE: OSX requires this + check(cl.clBuildProgram(self.program, 1, device.cl_dev, None, BP_CB(), None)) # NOTE: OSX requires this self.kernel = checked(cl.clCreateKernel(self.program, obj.name.encode(), status := ctypes.c_int32()), status) def __del__(self): @@ -101,17 +101,17 @@ def __init__(self, device:str=""): CLDevice.device_ids = c.init_c_var((cl.cl_device_id * num_devices.value), lambda x: check(cl.clGetDeviceIDs(platform_ids[0], device_type, num_devices, x, None))) - self.device_id = CLDevice.device_ids[0 if ":" not in device else int(device.split(":")[1])] - self.device_name = (cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_NAME, 256, + self.cl_dev = CLDevice.device_ids[0 if ":" not in device else int(device.split(":")[1])] + self.device_name = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_NAME, 256, buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] - self.driver_version = (cl.clGetDeviceInfo(self.device_id, cl.CL_DRIVER_VERSION, 256, + self.driver_version = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DRIVER_VERSION, 256, buf:=ctypes.create_string_buffer(256), None), buf.value.decode())[1] if DEBUG >= 1: print(f"CLDevice: opening {self.device_name} with version {self.driver_version}") - self.context = checked(cl.clCreateContext(None, 1, self.device_id, CC_CB(), None, status := ctypes.c_int32()), status) - self.queue = checked(cl.clCreateCommandQueue(self.context, self.device_id, cl.CL_QUEUE_PROFILING_ENABLE, status), status) + self.context = checked(cl.clCreateContext(None, 1, self.cl_dev, CC_CB(), None, status := ctypes.c_int32()), status) + self.queue = checked(cl.clCreateCommandQueue(self.context, self.cl_dev, cl.CL_QUEUE_PROFILING_ENABLE, status), status) self.pending_copyin: list[memoryview] = [] - check(cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_EXTENSIONS, 0, None, ctypes.byref(exts_len:=ctypes.c_size_t()))) - self.device_exts = (cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_EXTENSIONS, exts_len.value, + check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, 0, None, ctypes.byref(exts_len:=ctypes.c_size_t()))) + self.device_exts = (cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_EXTENSIONS, exts_len.value, ctypes.byref(buf := ctypes.create_string_buffer(exts_len.value)), None), ctypes.string_at(buf).decode().split())[1] @@ -119,7 +119,7 @@ def __init__(self, device:str=""): arch = ",".join(self.device_exts) if "cl_khr_image2d_from_buffer" in self.device_exts: - check(cl.clGetDeviceInfo(self.device_id, cl.CL_DEVICE_IMAGE_PITCH_ALIGNMENT, 4, ctypes.byref(ipa := ctypes.c_uint32()), None)) + check(cl.clGetDeviceInfo(self.cl_dev, cl.CL_DEVICE_IMAGE_PITCH_ALIGNMENT, 4, ctypes.byref(ipa := ctypes.c_uint32()), None)) arch += f",IMAGE_PITCH_ALIGNMENT={ipa.value}" super().__init__(device, CLAllocator(self), [OpenCLRenderer], CLProgram, arch=arch) diff --git a/tinygrad/runtime/ops_nv.py b/tinygrad/runtime/ops_nv.py index 2c274a6dd726f..59d6db6b2047a 100644 --- a/tinygrad/runtime/ops_nv.py +++ b/tinygrad/runtime/ops_nv.py @@ -588,8 +588,7 @@ class NVDevice(HCQCompiled[NVSignal]): def is_nvd(self) -> bool: return isinstance(self.iface, PCIIface) def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - self.iface = self._select_iface() + self.iface = self._select_iface(device) device_params = nv_gpu.NV0080_ALLOC_PARAMETERS(deviceId=self.iface.gpu_instance, hClientShare=self.iface.root, vaMode=nv_gpu.NV_DEVICE_ALLOCATION_VAMODE_OPTIONAL_MULTIPLE_VASPACES) diff --git a/tinygrad/runtime/ops_rdma.py b/tinygrad/runtime/ops_rdma.py index 6f7a46600393f..c4be2d0f9654a 100644 --- a/tinygrad/runtime/ops_rdma.py +++ b/tinygrad/runtime/ops_rdma.py @@ -101,6 +101,5 @@ def _transfer(self, dest:HCQBuffer, src:HCQBuffer, sz:int, src_dev:HCQCompiled, class RDMADevice(HCQCompiled): def __init__(self, device:str=""): - self.device_id = int(device.split(":")[1]) if ":" in device else 0 - self.iface = MLXIface(self, self.device_id) + self.iface = MLXIface(self, int(device.split(":")[1]) if ":" in device else 0) super().__init__(device, RDMAAllocator(self), [], None, signal_t=None) diff --git a/tinygrad/runtime/support/hcq.py b/tinygrad/runtime/support/hcq.py index 6ff4effc25a25..ec68b106feb76 100644 --- a/tinygrad/runtime/support/hcq.py +++ b/tinygrad/runtime/support/hcq.py @@ -1,10 +1,9 @@ from __future__ import annotations from typing import cast, Callable, Type, TypeVar, Generic, Any -import contextlib, decimal, statistics, time, ctypes, array, os, struct, collections, functools, itertools -from dataclasses import replace +import contextlib, decimal, statistics, time, ctypes, array, os, struct, collections, itertools try: import fcntl # windows misses that except ImportError: fcntl = None #type:ignore[assignment] -from tinygrad.helpers import DEV, PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, select_first_inited, select_by_name, unwrap +from tinygrad.helpers import DEV, PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, unwrap from tinygrad.helpers import suppress_finalizing, pluralize, TracingKey from tinygrad.device import Device, BufferSpec, Compiled, LRUAllocator, ProfileDeviceEvent, ProfileProgramEvent, Program, TinyELF from tinygrad.uop.ops import sym_infer, sint, UOp @@ -393,8 +392,6 @@ class HCQCompiled(Compiled, Generic[SignalType]): def __init__(self, device:str, allocator:HCQAllocatorBase, compilers:list[type[Renderer]], runtime:type[Program]|None, signal_t:Type[SignalType]|None=None, comp_queue_t:Callable[..., HWQueue]|None=None, copy_queue_t:Callable[..., HWQueue]|None=None, kernargs_size=(16 << 20), sigalloc_size=0x1000, can_recover:bool=False, arch=None): - self.device_id:int = int(device.split(":")[1]) if ":" in device else 0 - from tinygrad.runtime.graph.hcq import HCQGraph super().__init__(device, allocator, compilers, runtime, HCQGraph, arch=arch) @@ -424,8 +421,6 @@ def __init__(self, device:str, allocator:HCQAllocatorBase, compilers:list[type[R if self._is_cpu(): HCQCompiled.cpu_devices.append(self) - def count(self) -> int: return self.iface.count if hasattr(self, 'iface') else 1 - def synchronize(self, timeout:int|None=None): if self.error_state is not None: raise self.error_state if not hasattr(self, 'timeline_signal'): return @@ -491,16 +486,6 @@ def _realloc(self, oldbuf:HCQBuffer|None, new_size:int, options:BufferSpec|None= buf, realloced = self.allocator.alloc(oldbuf.size if oldbuf is not None else new_size, options=options), False return buf, realloced - def _select_iface(self): - assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ - f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" - assert hasattr(self, "ifaces"), "must have ifaces to select an iface" - t = DEV.target(dev:=type(self).__name__[:-6]) - filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") - filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fallback to mock ifaces - return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in filtered], - f"No interface for {dev}:{self.device_id} is available") - def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU" def rdma_dev(self): @@ -512,9 +497,7 @@ def rdma_dev(self): def finalize(self): try: self.synchronize() # Try to finalize device in any case. except RuntimeError as e: print(f"{self.device} synchronization failed before finalizing: {e}") - - # If the device has an interface, call its device_fini method to clean up resources. - if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() + super().finalize() class HCQBuffer: def __init__(self, va_addr:sint, size:int, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None, owner:Any=None): diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 68d6d4f5a4ce4..5b468a8741f6c 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -1,8 +1,8 @@ from __future__ import annotations -from typing import cast, Callable, TypeVar, Generic, Any, Sequence, Iterable +from typing import cast, TypeVar, Generic, Any, Sequence, Iterable import struct, functools, time, collections, itertools, decimal, statistics from dataclasses import replace, dataclass -from tinygrad.helpers import DEV, getenv, select_first_inited, select_by_name, suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE +from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE from tinygrad.helpers import to_tuple, round_up, partition, data64_le, panic, ContextVar, perf_counter_us, Context from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent @@ -23,8 +23,7 @@ HCQ_RUNTIME_DEV = ContextVar("HCQ_RUNTIME_DEV", "CPU") HCQ_DEVS = frozenset(("AMD", "CPU")) -HCQ_P2P_DEVS = HCQ_DEVS | frozenset(("CPU",)) -HCQ_CACHE_TAGS = frozenset(("program", "systems", "template")) +HCQ_CACHE_TAGS = frozenset(("program", "systems")) @dataclass(frozen=True) class HCQInfo: @@ -94,10 +93,10 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| # ***************** # 1.1. prep: staging copies -def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_P2P_DEVS) +def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_DEVS) def _get_enqueue_devs(call:UOp) -> Any|None: - if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_P2P_DEVS) for b in bufs): return None + if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device return devs if all_devices_in(devs, HCQ_DEVS) else None @@ -532,7 +531,6 @@ class HCQ2Compiled(Compiled): wait_timeout_ms: float = 30000.0 def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Renderer]], runtime, can_recover:bool=False, arch=None): - self.device_id:int = int(device.split(":")[1]) if ":" in device else 0 self.can_recover = can_recover self.pm_bufferize = PatternMatcher([ @@ -596,26 +594,12 @@ def on_device_hang(self): raise RuntimeError(f"{self.device} hang detected") def device_props(self) -> dict[str,Any]: return {} # to be overridden if needed. dict keys are backend dependent. - def count(self) -> int: return self.iface.count if hasattr(self, 'iface') else 1 - - def _select_iface(self): - assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ - f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" - assert hasattr(self, "ifaces"), "must have ifaces to select an iface" - t = DEV.target(dev:=type(self).__name__[:-6]) - filtered = select_by_name(self.ifaces, lambda i: i.__name__[:-5], t.interface, f"{dev} has no interface {t.interface!r}") - filtered = [i for i in filtered if t.interface.startswith("MOCK") or not i.__name__[:-5].startswith("MOCK")] # never fall back to mock ifaces - return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in filtered], - f"No interface for {dev}:{self.device_id} is available") - def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU" def finalize(self): try: self.synchronize() # try to finalize the device in any case except RuntimeError as e: print(f"{self.device} synchronization failed before finalizing: {e}") - - # if the device has an interface, call device_fini to clean up resources - if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini() + super().finalize() @dataclass class HCQ2Buffer: From c30bf116b764ada4f33ede853833f74caf0a7536 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 16 Aug 2026 20:07:25 -0400 Subject: [PATCH 063/306] few torch_backend fix (#17558) * few torch_backend fix * fix --- extra/torch_backend/backend.py | 56 ++++++++++++++++++++++----- extra/torch_backend/test.py | 69 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/extra/torch_backend/backend.py b/extra/torch_backend/backend.py index 4b600a4c0a3bd..fda5b0a5aeeb0 100644 --- a/extra/torch_backend/backend.py +++ b/extra/torch_backend/backend.py @@ -124,9 +124,10 @@ def _try_simple_reshape_view_write(base: Tensor, view: Tensor, val: Tensor) -> b if not (ops := _get_view_ops(view)): return False shapes = [base.shape] for fn, args, _ in ops: - if fn is Tensor.reshape: - if not (next_shape := _reshape_target_shape(shapes[-1], args)): return False - shapes.append(next_shape) + if fn is Tensor.detach: continue # detach leaves every element where it was (a tracked view only on torch<2.10) + if fn is not Tensor.reshape: return False + if not (next_shape := _reshape_target_shape(shapes[-1], args)): return False + shapes.append(next_shape) if shapes[-1] != view.shape: return False for s in reversed(shapes[:-1]): val = val.reshape(s) base.assign(val) @@ -134,11 +135,12 @@ def _try_simple_reshape_view_write(base: Tensor, view: Tensor, val: Tensor) -> b def _view_write(base: Tensor, view: Tensor, value: Tensor) -> None: val = value if value.dtype == base.dtype else value.cast(base.dtype) - if view.shape == base.shape: return base.assign(val) if _try_simple_reshape_view_write(base, view, val): return idx_base = Tensor.arange(base.numel(), dtype=dtypes.int32).reshape(base.shape) idx_view = _apply_view_ops(idx_base, _get_view_ops(view)).reshape(-1) - flat_base = base.reshape(base.numel()).contiguous() + # clone, not contiguous: contiguous() on a base that already owns its buffer returns the base itself, and scattering + # into that is an in-place write to a buffer other tensors still hold, which setitem refuses + flat_base = base.reshape(base.numel()).clone() flat_base[idx_view] = val.reshape(-1) base.assign(flat_base.reshape(base.shape)) @@ -301,6 +303,34 @@ def slice_tensor(self, dim=0, start=None, end=None, step=1): slices[dim] = slice(start, end, step) return self[slices] +# the functional scatters. without an impl aten falls back to a path that assumes a real storage: "self.has_storage() INTERNAL ASSERT FAILED" +def _scatter_into(self, src, dim, index): + out = unwrap(self).clone() + slices = [slice(None)] * out.ndim + slices[dim] = index + out[slices] = unwrap(src).cast(out.dtype) # torch casts src to self's dtype, tinygrad setitem demands they already match + return wrap(out) + +@torch.library.impl("aten::slice_scatter", "privateuseone") +def slice_scatter(self, src, dim=0, start=None, end=None, step=1): return _scatter_into(self, src, dim, slice(start, end, step)) + +@torch.library.impl("aten::select_scatter", "privateuseone") +def select_scatter(self, src, dim, index): return _scatter_into(self, src, dim, index) + +@torch.library.impl("aten::diagonal_scatter", "privateuseone") +def diagonal_scatter(self, src, offset=0, dim1=0, dim2=1): + # a diagonal is not one axis, so scatter through the flat indices it picks out + base, out = unwrap(self), unwrap(self).clone().reshape(-1) + idx = Tensor.arange(base.numel(), dtype=dtypes.int32).reshape(base.shape).diagonal(offset, dim1, dim2).reshape(-1) + out[idx] = unwrap(src).cast(base.dtype).reshape(-1) + return wrap(out.reshape(base.shape)) + +# the functional copy_. without an impl the fallback segfaults on a tensor with no storage +@torch.library.impl("aten::copy", "privateuseone") +def copy(self, src, non_blocking=False): + dest = unwrap(self) + return wrap(unwrap(src).cast(dest.dtype).to(dest.device).expand(dest.shape)) + @torch.library.impl("aten::slice_backward", "privateuseone") def slice_backward(grad_out, input_sizes, dim, start, end, step): grad_input = Tensor.zeros(input_sizes).contiguous() @@ -341,7 +371,9 @@ def pad_backward(grad_out, self, padding, mode): torch.library.impl(f"aten::{pad_type}_pad{dim}d", "privateuseone")(functools.partial(pad_forward, mode=mode)) torch.library.impl(f"aten::{pad_type}_pad{dim}d_backward", "privateuseone")(functools.partial(pad_backward, mode=mode)) -def upsample(self, size, align_corners=False, mode=None): return wrap(Tensor.interpolate(unwrap(self), size, mode=mode, align_corners=align_corners)) +# the schemas are all positional: (self, output_size, align_corners, *scales) for linear, (self, output_size, *scales) for nearest. +def upsample(self, size, *args, mode=None): + return wrap(Tensor.interpolate(unwrap(self), size, mode=mode, align_corners=args[0] if mode == "linear" else False)) for i,pre in enumerate(["", "bi", "tri"]): torch.library.impl(f"aten::upsample_{pre}linear{i+1}d", "privateuseone")(functools.partial(upsample, mode="linear")) torch.library.impl(f"aten::upsample_nearest{i+1}d", "privateuseone")(functools.partial(upsample, mode="nearest")) @@ -452,7 +484,9 @@ def _linalg_svd(self, full_matrices=False): aten.hardswish, aten.hardswish_backward, aten.hardtanh, aten.hardtanh_backward, aten.gelu, aten.gelu_backward, - aten.logical_and, + # NOTE: no aten.logical_or here, its decomposition reaches aten.bitwise_or through a path that checks aliasing by + # reading storage, which a tiny tensor has none of. it gets a direct impl below instead + aten.logical_and, aten.logical_xor, aten.randint, aten.eye, aten.hardsigmoid_backward, @@ -579,8 +613,8 @@ def _inplace_op(t, new_value): # inplace ops using replace for fusion "aten.zero_": lambda x: x.const_like(0), "aten.fill_.Scalar": lambda x, y: x.const_like(y), - "aten.add_.Tensor": lambda self, other, alpha=1.0: self + other * alpha, - "aten.add_.Scalar": lambda self, other, alpha=1.0: self + other * alpha, + "aten.add_.Tensor": lambda self, other, alpha=1: self + other * alpha, + "aten.add_.Scalar": lambda self, other, alpha=1: self + other * alpha, "aten.mul_.Tensor": lambda self, other: self * other, "aten.mul_.Scalar": lambda self, other: self * other, # relu doesn't have an out form? @@ -613,7 +647,9 @@ def _inplace_op(t, new_value): # these don't work in out form, they have size 0 "aten.abs": Tensor.abs, "aten.logical_not": Tensor.logical_not, - "aten.logical_or_": lambda x, y: x | y, + # compare against zero first: logical_* is bool-valued for any input dtype, while | is bitwise + "aten.logical_or": lambda x, y: (x != 0) | (y != 0), + "aten.logical_or_": lambda x, y: (x != 0) | (y != 0), "aten.multinomial": Tensor.multinomial, "aten.masked_fill_.Scalar": lambda self, mask, value: self.masked_fill(mask, value), "aten.masked_fill_.Tensor": lambda self, mask, value: self.masked_fill(mask, value), diff --git a/extra/torch_backend/test.py b/extra/torch_backend/test.py index 0b45cbf7340ff..74886a608da02 100644 --- a/extra/torch_backend/test.py +++ b/extra/torch_backend/test.py @@ -796,6 +796,75 @@ def test_biased_conv2d(self): np.testing.assert_allclose(w_tiny.grad.cpu().numpy(), w_cpu.grad.numpy(), atol=1e-4, rtol=1e-3) np.testing.assert_allclose(b_tiny.grad.cpu().numpy(), b_cpu.grad.numpy(), atol=1e-4, rtol=1e-3) + def test_write_through_detach_of_unrealized(self): + # how every module parameter is initialized under set_default_device("tiny"). on torch<2.10 detach is a tracked view, + # so this writes through a view whose shape equals its base's, and the base has no buffer of its own yet + a = torch.empty(4, device=device) + a.detach().fill_(3) + np.testing.assert_equal(a.cpu().numpy(), [3, 3, 3, 3]) + + def test_square_transpose_inplace(self): + # a same-shape transpose is not a reshape: writing the transposed values straight back would scramble the base + a = torch.tensor([[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], device=device) + a.transpose(0, 1).add_(100) + np.testing.assert_equal(a.cpu().numpy(), [[100., 101., 102.], [103., 104., 105.], [106., 107., 108.]]) + + def test_interpolate(self): + a = torch.arange(4, dtype=torch.float32, device=device).reshape(1, 1, 2, 2) + nearest = torch.nn.functional.interpolate(a, scale_factor=2.0) + np.testing.assert_equal(nearest.cpu().numpy()[0, 0], [[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3], [2, 2, 3, 3]]) + linear = torch.nn.functional.interpolate(a, size=(4, 4), mode="bilinear", align_corners=False) + ref = torch.nn.functional.interpolate(a.cpu(), size=(4, 4), mode="bilinear", align_corners=False) + np.testing.assert_allclose(linear.cpu().numpy(), ref.numpy(), rtol=1e-5) + + @unittest.expectedFailure + def test_interpolate_inexact_scale(self): + # torch forwards the raw scale_factor, Tensor.interpolate recomputes it from output_size, and they disagree here + a = torch.arange(6, dtype=torch.float32, device=device).reshape(1, 1, 2, 3) + tiny = torch.nn.functional.interpolate(a, scale_factor=2.5, mode="bilinear") + ref = torch.nn.functional.interpolate(a.cpu(), scale_factor=2.5, mode="bilinear") + np.testing.assert_allclose(tiny.cpu().numpy(), ref.numpy(), rtol=1e-5) + + def test_logical_or_xor(self): + a = torch.tensor([True, True, False, False], device=device) + b = torch.tensor([True, False, True, False], device=device) + np.testing.assert_equal(torch.logical_or(a, b).cpu().numpy(), [True, True, True, False]) + np.testing.assert_equal(torch.logical_xor(a, b).cpu().numpy(), [False, True, True, False]) + # bool-valued whatever the input dtype, so this is not | and ^ + i, j = torch.tensor([2, 0, 5, 0], device=device), torch.tensor([0, 0, 1, 1], device=device) + np.testing.assert_equal(torch.logical_or(i, j).cpu().numpy(), [True, False, True, True]) + np.testing.assert_equal(torch.logical_xor(i, j).cpu().numpy(), [True, False, False, True]) + + def test_slice_scatter(self): + # the scatters are functional: they return a new tensor and must leave the one they were given alone + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + out = torch.slice_scatter(a, torch.ones(1, 4, device=device), 0, 0, 1) + np.testing.assert_equal(out.cpu().numpy(), [[1, 1, 1, 1], [4, 5, 6, 7], [8, 9, 10, 11]]) + np.testing.assert_equal(a.cpu().numpy(), np.arange(12, dtype=np.float32).reshape(3, 4)) + + def test_slice_scatter_casts_src(self): + a = torch.zeros(3, 4, device=device) + out = torch.slice_scatter(a, torch.ones(1, 4, dtype=torch.int32, device=device), 0, 0, 1) + self.assertEqual(out.dtype, torch.float32) + np.testing.assert_equal(out.cpu().numpy()[0], np.ones(4, dtype=np.float32)) + + def test_select_scatter(self): + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + out = torch.select_scatter(a, torch.ones(4, device=device), 0, 1) + np.testing.assert_equal(out.cpu().numpy(), [[0, 1, 2, 3], [1, 1, 1, 1], [8, 9, 10, 11]]) + + def test_diagonal_scatter(self): + a = torch.zeros(3, 3, device=device) + out = torch.diagonal_scatter(a, torch.arange(3, dtype=torch.float32, device=device)) + np.testing.assert_equal(out.cpu().numpy(), np.diag([0., 1., 2.])) + np.testing.assert_equal(a.cpu().numpy(), np.zeros((3, 3), dtype=np.float32)) + + def test_copy_functional(self): + # without an impl this segfaults rather than fails: a regression here takes the whole run down + a = torch.arange(4, dtype=torch.float32, device=device) + out = torch.ops.aten.copy(a, torch.zeros(4, device=device)) + np.testing.assert_equal(out.cpu().numpy(), [0., 0., 0., 0.]) + np.testing.assert_equal(a.cpu().numpy(), [0., 1., 2., 3.]) from tinygrad import Tensor class TestBackendHelpers(unittest.TestCase): From 057a18a07c987708638eec9a4178c096161b87a9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 16 Aug 2026 20:52:42 -0400 Subject: [PATCH 064/306] fix emulated long cast to double (#17559) --- test/backend/test_dtype.py | 3 +++ tinygrad/codegen/decomp/dtype.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/backend/test_dtype.py b/test/backend/test_dtype.py index 594ebd0e23431..94d37090e54e7 100644 --- a/test/backend/test_dtype.py +++ b/test/backend/test_dtype.py @@ -340,6 +340,9 @@ class TestUint64DType(TestDType): DTYPE = dtypes.uint64 def test_uint64_load(self): assert Tensor(2**64 - 1, dtype=dtypes.uint64).numpy() == 2**64 - 1 + @unittest.skipIf(dtypes.double not in supported_dtypes, "needs float64") + def test_uint64_cast_double(self): + assert Tensor([2**32 + 1], dtype=dtypes.uint64).cast(dtypes.double).numpy() == 2**32 + 1 @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") class TestEmulatedUInt64DType(TestUint64DType): diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 52b8e463046eb..db85526c83d37 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -33,7 +33,8 @@ def l2i(op: Ops, dt: DType, *uops:UOp): return (lo:=uops[0].cast(l2i_dt[dt])), (uops[0] / 2**32).cast(l2i_dt[dt]) - ((uops[0] < 0) & lo.ne(0)) case Ops.CAST if dt in dtypes.floats: small = (a1.eq(0) & (a0 >= 0)) | (a1.eq(-1) & (a0 < 0)) - return small.where(a0.cast(dt), ((a1.cast(dtypes.float32) * (2**32)) + a0.bitcast(dtypes.uint).cast(dtypes.float32)).cast(dt)) + cdt = dt if dt == dtypes.float64 else dtypes.float32 + return small.where(a0.cast(dt), ((a1.cast(cdt) * (2**32)) + a0.bitcast(dtypes.uint).cast(cdt)).cast(dt)) case Ops.CAST: return a0.bitcast(dtypes.uint).cast(dt) case Ops.BITCAST: return a0.bitcast(dt), a1.bitcast(dt) case Ops.SHL: From bfd4048abf1f37de15b15aeba5e2528ef3de4c02 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 16 Aug 2026 21:06:38 -0400 Subject: [PATCH 065/306] no dtype in vconst_like [PR] (#17560) --- tinygrad/uop/ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 40363b03504ba..63eabaa0b54da 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -583,9 +583,9 @@ def _wrap_uop(cls, u:UOp) -> UOp: return u def const_like(self, b:ConstLike, dtype:DType|None=None): ret = UOp.const(b, dtype or self.dtype) return ret._mop(Ops.EXPAND, arg=self._shape) if self._shape and ret._shape != self._shape else ret - def vconst_like(self, b:ConstLike, dtype:DType|None=None): + def vconst_like(self, b:ConstLike): # for use after movement ops have been removed - return UOp.const(b, dtype or self.dtype).broadcast(self.max_numel()) + return UOp.const(b, self.dtype).broadcast(self.max_numel()) def ufix(self, x): if isinstance(x, UOp): return x return UOp.const(x) @@ -1403,7 +1403,7 @@ def match(self:UPat, uop:UOp, store:dict[str, UOp]) -> list[dict[str, UOp]]: if self.is_any: return flatten([x.match(uop, store.copy()) for x in self.src[0]]) if (self.op is not None and uop.op not in self.op) or \ (self.name is not None and store.setdefault(self.name, uop) is not uop) or \ - (self.match_dtype is not None and uop.dtype not in self.match_dtype and uop.dtype.scalar() not in self.match_dtype) or \ + (self.match_dtype is not None and uop.dtype not in self.match_dtype) or \ (self.arg is not None and self.arg != uop.arg) or \ (self.match_tag is not None and uop.tag not in self.match_tag) or \ (len(uop.src) < self.required_len) or \ From 138fb4a783d82f4e877ad2fe3692aaf8d1de2e46 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 16 Aug 2026 21:12:17 -0400 Subject: [PATCH 066/306] delete dead DType.scalar [PR] (#17561) --- extra/thunder/tiny/tk/tiles.py | 2 +- test/backend/test_isel.py | 2 +- test/backend/test_quantize_onnx.py | 2 +- test/null/test_dtype_spec.py | 4 ---- test/null/test_tensor.py | 2 +- test/opt/test_gen_float4.py | 8 ++++---- test/unit/test_allreduce.py | 2 +- tinygrad/dtype.py | 1 - tinygrad/renderer/__init__.py | 8 ++++---- tinygrad/renderer/cstyle.py | 12 ++++++------ tinygrad/renderer/isa/x86.py | 8 ++++---- tinygrad/renderer/ptx.py | 26 +++++++++++++------------- tinygrad/runtime/support/hcq2.py | 2 +- 13 files changed, 37 insertions(+), 42 deletions(-) diff --git a/extra/thunder/tiny/tk/tiles.py b/extra/thunder/tiny/tk/tiles.py index 98cff727c7e76..f27ee77fc7db1 100644 --- a/extra/thunder/tiny/tk/tiles.py +++ b/extra/thunder/tiny/tk/tiles.py @@ -209,7 +209,7 @@ def create(cls, shape, dtype:DType, layout:TileLayout, base_shape:STBaseShape, k return cls(uop, rows, cols, layout, base_shape, ker) def swizzle(self, row, col): - swizzled_offset = self.base_shape.swizzle(row, col, self._uop.dtype.scalar()) + swizzled_offset = self.base_shape.swizzle(row, col, self._uop.dtype) row = swizzled_offset // self.base_shape.cols col = swizzled_offset % self.base_shape.cols diff --git a/test/backend/test_isel.py b/test/backend/test_isel.py index 6965a5db1711a..ed38be7c3b38d 100644 --- a/test/backend/test_isel.py +++ b/test/backend/test_isel.py @@ -7,7 +7,7 @@ from tinygrad.renderer.isa import IselContext # INDEX on a register value with a constant index extracts a single element (the old GEP) -def lane(y:UOp, i:int) -> UOp: return y.index(UOp.const(i, dtypes.int), dtype=y.dtype.scalar()) +def lane(y:UOp, i:int) -> UOp: return y.index(UOp.const(i, dtypes.int), dtype=y.dtype) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only x86") class TestIselX86(unittest.TestCase): diff --git a/test/backend/test_quantize_onnx.py b/test/backend/test_quantize_onnx.py index 587b03f8845c5..8528caa9c4512 100644 --- a/test/backend/test_quantize_onnx.py +++ b/test/backend/test_quantize_onnx.py @@ -82,7 +82,7 @@ def test_quant_128(self, sz=128): linear = run_onnx({"input":inp})["output"].schedule_linear() prg = to_program(linear.src[-2].src[0], renderer=Device[Device.DEFAULT].renderer) daccs = [u for u in tuple(prg.src[1].src) if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG] - assert all(u.dtype.scalar() is dtypes.int for u in daccs) + assert all(u.dtype is dtypes.int for u in daccs) @unittest.skipIf(Device.DEFAULT != "DSP", "only tests for DSP") class TestQuantizeOnnx(unittest.TestCase): diff --git a/test/null/test_dtype_spec.py b/test/null/test_dtype_spec.py index 2aaebefe05c86..e963ffb6c0622 100644 --- a/test/null/test_dtype_spec.py +++ b/test/null/test_dtype_spec.py @@ -51,10 +51,6 @@ def test_fp8s_are_float(self): assert dtypes.is_float(dtypes.fp8e4m3) assert dtypes.is_float(dtypes.fp8e5m2) - @given(strat.sampled_from([d for d in DTYPES_DICT.values() if dtypes.is_float(d) or dtypes.is_int(d)])) - def test_scalar(self, dtype): - assert dtype.scalar() == dtype - def test_from_py(self): assert dtypes.from_py(True) == dtypes.bool assert dtypes.from_py(Invalid) == dtypes.bool diff --git a/test/null/test_tensor.py b/test/null/test_tensor.py index 4156b37177c3a..32ccc255e3fae 100644 --- a/test/null/test_tensor.py +++ b/test/null/test_tensor.py @@ -69,7 +69,7 @@ def _assert(self, dtype: DType, a: Tensor): if not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, NIRRenderer)): assert idx.op is Ops.INDEX idx_val = idx.src[1] - self.assertFalse(idx_val.overflows(idx_val.dtype.scalar())) + self.assertFalse(idx_val.overflows(idx_val.dtype)) # use expand to generate kernel that uses large idx def do_op_then_assert(self, dtype: DType, dim1, dim2, dim3): diff --git a/test/opt/test_gen_float4.py b/test/opt/test_gen_float4.py index b946cb6bd14b6..b09aabe50f271 100644 --- a/test/opt/test_gen_float4.py +++ b/test/opt/test_gen_float4.py @@ -10,12 +10,12 @@ class TestFloat4(unittest.TestCase): @staticmethod def count_float4(uops: list[UOp], n=4): - return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype.scalar() == dtypes.float and uop.shape == (4,)]), - len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype.scalar() == dtypes.float and uop.shape == (4,)])) + return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype == dtypes.float and uop.shape == (4,)]), + len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype == dtypes.float and uop.shape == (4,)])) @staticmethod def count_half4(uops: list[UOp]): - return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype.scalar() == dtypes.half and uop.shape == (4,)]), - len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype.scalar() == dtypes.half and uop.shape == (4,)])) + return (len([uop for uop in uops if uop.op is Ops.LOAD and uop.dtype == dtypes.half and uop.shape == (4,)]), + len([uop for uop in uops if uop.op is Ops.STORE and uop.src[1].dtype == dtypes.half and uop.shape == (4,)])) def test_float4_basic(self): a = Tensor.empty(2, 8).realize() diff --git a/test/unit/test_allreduce.py b/test/unit/test_allreduce.py index 0997e55979090..84dce582429d1 100644 --- a/test/unit/test_allreduce.py +++ b/test/unit/test_allreduce.py @@ -64,7 +64,7 @@ def _get_copy_dtypes(self, dtype, allreduce_cast): with Context(ALLREDUCE_CAST=allreduce_cast, RING=0, SCACHE=0): t = Tensor.empty(4, 4, dtype=dtype).shard(ds, axis=0) linear = t.sum(0).linear_with_vars()[0] - return {si.src[1].buffer.dtype.scalar() for si in linear.src if si.src[0].op is Ops.COPY} + return {si.src[1].buffer.dtype for si in linear.src if si.src[0].op is Ops.COPY} def test_allreduce_cast_bf16(self): # with ALLREDUCE_CAST, allreduce copies stay in bfloat16 instead of promoting to float32 diff --git a/tinygrad/dtype.py b/tinygrad/dtype.py index 9c0b336a58353..e160cb6cddcb0 100644 --- a/tinygrad/dtype.py +++ b/tinygrad/dtype.py @@ -66,7 +66,6 @@ def new(priority:int, bitsize:int, name:str, fmt:FmtStr|None): return DType(prio def __reduce__(self): return type(self), tuple(getattr(self, f.name) for f in fields(self)) def __repr__(self): return f"dtypes.{INVERSE_DTYPES_DICT[self.name]}" def __lt__(self, o:DType): return (self.priority, self.bitsize, self.name, self.fmt) < (o.priority, o.bitsize, o.name, o.fmt) - def scalar(self) -> DType: return self @functools.cached_property def min(self): if dtypes.is_int(self): return 0 if dtypes.is_unsigned(self) else -2**(self.bitsize-1) diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 438af612be1bd..728e6148d3ddc 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -35,8 +35,8 @@ def from_uops(uops:tuple[UOp, ...], ignore_indexing=False) -> Estimates: while len(buf.src) and buf.op is not Ops.PARAM: buf = buf.src[0] if buf.op is Ops.PARAM: # u.src[0] is INDEX, cap at buffer size for re-reads (e.g. matmul) - accessed = mem.get((buf, u.op), 0) + u.src[0].max_numel() * u.src[0].dtype.scalar().itemsize * mults - mem[(buf, u.op)] = smin(accessed, buf.max_numel() * buf.dtype.scalar().itemsize) + accessed = mem.get((buf, u.op), 0) + u.src[0].max_numel() * u.src[0].dtype.itemsize * mults + mem[(buf, u.op)] = smin(accessed, buf.max_numel() * buf.dtype.itemsize) if u.op is Ops.RANGE: mult_stack.append(mults) if u.dtype is not dtypes.void: # unbounded loop, unknown trip count @@ -47,9 +47,9 @@ def from_uops(uops:tuple[UOp, ...], ignore_indexing=False) -> Estimates: elif u.op is Ops.SPECIAL: mults *= cast(sint, u.src[0].ssimplify()) # NOTE: we don't push to the mult_stack here, you can't end these elif u.op is Ops.PARAM and u.arg.addrspace == AddrSpace.ALU and u.expr == 'core_id': mults *= int(u.vmax) + 1 elif u.op is Ops.LOAD and u.src[0].addrspace != AddrSpace.REG: - lds += u.max_numel() * u.dtype.scalar().itemsize * mults + lds += u.max_numel() * u.dtype.itemsize * mults elif u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG: - lds += u.max_numel() * u.src[1].dtype.scalar().itemsize * mults + lds += u.max_numel() * u.src[1].dtype.itemsize * mults elif u.op in GroupOp.ALU and u not in excluded: flops += (mults * (2 if u.op is Ops.MULACC else 1)) * u.max_numel() elif u.op is Ops.WMMA and u not in excluded: diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 3b555ac177446..7e220512ac2be 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -107,11 +107,11 @@ def uops_to_dtypes(uops:list[UOp]) -> list[tuple[DType, int]]: def _wmma_name(u:UOp) -> str: # sanitize spaces in DType.name (int8 = "signed char") - return f"WMMA_{'_'.join(map(str, u.arg[0]))}_{u.arg[1].name}_{u.dtype.scalar().name}".replace(" ", "_") + return f"WMMA_{'_'.join(map(str, u.arg[0]))}_{u.arg[1].name}_{u.dtype.name}".replace(" ", "_") # (name, dims, dtype_in, dtype_out, device, threads, upcast_sizes) def wmma_args(uops:list[UOp]): - return dedup((_wmma_name(uop), uop.arg[0], uop.arg[1], uop.dtype.scalar(), *(uop.arg[2:4]), + return dedup((_wmma_name(uop), uop.arg[0], uop.arg[1], uop.dtype, *(uop.arg[2:4]), tuple(uop.src[i].shape[-1] for i in range(3))) for uop in uops if uop.op is Ops.WMMA) @@ -182,8 +182,8 @@ def _render_dtype(self, dtype:DType, sz:int=1, addrspace=AddrSpace.ALU, mutable= if addrspace in (AddrSpace.LOCAL, AddrSpace.GLOBAL) or override_ptr: suffix = "*" if sz > 1: - return prefix + self.type_map.get(scalar:=dtype.scalar(), scalar.name).replace(" ", "_") + str(sz) + suffix - return prefix + self.type_map.get(scalar:=dtype.scalar(), scalar.name) + suffix + return prefix + self.type_map.get(dtype, dtype.name).replace(" ", "_") + str(sz) + suffix + return prefix + self.type_map.get(dtype, dtype.name) + suffix def render_type(self, u:UOp): return self._render_dtype(u.dtype, u.max_numel(), u.addrspace, shape=u._shape) def render_access(self, u:UOp): @@ -472,7 +472,7 @@ def supported_dtypes(self): class NVCCRenderer(CUDARenderer): def __init__(self, target:Target): super().__init__(target, use_nvcc=True) -def fp8_index(dtype: DType): return (dtypes.fp8e4m3, dtypes.fp8e5m2).index(dtype.scalar()) +def fp8_index(dtype: DType): return (dtypes.fp8e4m3, dtypes.fp8e5m2).index(dtype) def _ocml(op): return lambda x,dtype: f"__ocml_{op}_f{ {dtypes.half:16, dtypes.double:64}.get(dtype, 32)}({x})" class HIPRenderer(CStyleLanguage): @@ -546,7 +546,7 @@ def render_kernel(self, function_name, kernel, bufs, uops, prefix=None) -> str: ockl = [(f"__ockl_get_{name}", "unsigned int", "size_t", "const") for name in ["local_id", "group_id", "local_size"]] ocml_ops = {Ops.EXP2: ("exp2", "pure"), Ops.LOG2: ("log2", "pure"), Ops.SQRT: ("sqrt", "const"), Ops.SIN: ("sin", ""), Ops.TRUNC: ("trunc", "")} ocml = [(f"__ocml_{ocml_ops[op][0]}_f{dt.bitsize}", dt.name, dt.name, ocml_ops[op][1]) - for op, dt in dedup((u.op, u.dtype.scalar()) for u in uops) if op in ocml_ops and dt in (dtypes.half, dtypes.float, dtypes.double)] + for op, dt in dedup((u.op, u.dtype) for u in uops) if op in ocml_ops and dt in (dtypes.half, dtypes.float, dtypes.double)] if any(dt == dtypes.bfloat16 for dt, _ in used_dtypes): prefix.append(f"typedef {'__bf16' if self.is_cdna4(self.target.arch) else 'unsigned short'} hip_bfloat16;") if any(dt == dtypes.half for dt, _ in used_dtypes): prefix.append("#define half _Float16") diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 1346a026785ab..85b4ece0d238a 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -165,7 +165,7 @@ def scratch_buffer(elem_dt:DType, count:int, slot:int) -> UOp: return UOp.placeholder((count,), elem_dt, slot, AddrSpace.LOCAL) def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): - local = scratch_buffer(addr.src[0].dtype.scalar(), x.max_numel(), next(ctx)) + local = scratch_buffer(addr.src[0].dtype, x.max_numel(), next(ctx)) local_idx = local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64) # the selected address is a 64bit value, the AFTER orders the load after the scratch store and carries the element dtype for the encoder sel = gate.where(addr.replace(dtype=dtypes.uint64), local_idx) @@ -173,7 +173,7 @@ def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): return ptr.load(dtype=x.dtype) def gated_store(addr:UOp, gate:UOp, val:UOp): - local = scratch_buffer(addr.src[0].dtype.scalar(), val.max_numel(), -1) + local = scratch_buffer(addr.src[0].dtype, val.max_numel(), -1) sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64)) return UOp(Ops.AFTER, addr.dtype, (sel,)).store(val) @@ -237,7 +237,7 @@ def cmp(x:UOp) -> UOp: return x.ins(X86Ops.CMP, dtype=dtypes.void) if (i:=to_imm(x.src[1])) is None else x.ins(X86Ops.CMPi, dtype=dtypes.void, src=(x.src[0], i)) def vcmp(x:UOp) -> UOp: v = imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) - if x.dtype.scalar() is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, src=x.src + (v,)) + if x.dtype is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, src=x.src + (v,)) return x.ins(X86Ops.VCMPSD if x.max_numel() == 1 else X86Ops.VCMPPD, src=x.src + (v,)) # vinsertps xmm2, xmm0, xmm1, imm @@ -252,7 +252,7 @@ def _insert(ret:UOp, i:int) -> UOp: # vpinsq xmm2, xmm0, rax, imm # inserts element in rax into any position in xmm0, result is written to xmm2 according to imm def vpins(x:UOp) -> UOp: - op = {1: X86Ops.VPINSRB, 2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD, 8: X86Ops.VPINSRQ}[x.dtype.scalar().itemsize] + op = {1: X86Ops.VPINSRB, 2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD, 8: X86Ops.VPINSRQ}[x.dtype.itemsize] return functools.reduce(lambda ret,i: x.ins(op, src=(ret, x.src[i], imm(dtypes.uint8, i))), range(len(x.src)), def_reg(x.dtype)) # we don't call ctx.vreg on the srcs to avoid duplicates, a rewrite will assign the tuple of valid registers to a vreg diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index 1b6b77859aebb..30c73c2f930a5 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -64,7 +64,7 @@ def render_wmma(ctx: "PTXRenderer", wmma: UOp): for src, regs in zip(wmma.src, ctx.wmma_r): for i, reg in enumerate(regs): # pack input and acc registers - if (elems_per_reg := 4 // src.dtype.scalar().itemsize) == 1: yield f"mov.b32 {reg}, {ctx.r[src][i]};" + if (elems_per_reg := 4 // src.dtype.itemsize) == 1: yield f"mov.b32 {reg}, {ctx.r[src][i]};" else: yield f"mov.b32 {reg}, {{{', '.join(ctx.r[src][i * elems_per_reg : (i+1) * elems_per_reg])}}};" dt_map_in, dt_map_out = {dtypes.float: "tf32", dtypes.half: "f16"}, {dtypes.float: "f32", dtypes.half: "f16"} @@ -101,17 +101,17 @@ def modifier(a: DType, b: DType): return '.rzi' if dtypes.is_int(a) and dtypes.i if loc.addrspace == AddrSpace.REG else None), (UPat(Ops.STORE, src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"), UPat.var("var"))), lambda ctx, loc, var: f"st.{mem_type(loc)}" + \ - f"{f'.v{cnt}' if ((cnt:=var.max_numel())>1) else ''}.{ctx.mem_types[var.dtype.scalar()]} " + \ + f"{f'.v{cnt}' if ((cnt:=var.max_numel())>1) else ''}.{ctx.mem_types[var.dtype]} " + \ f"[{ctx.r[loc]}+0], {('{' + ', '.join(ctx.r[var]) + '}') if var.max_numel() > 1 else ctx.r[var]};"), (UPat(Ops.LOAD, name="x", src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"), UPat.var("alt"), UPat.var("gate"))), lambda ctx, x, loc, alt, gate: flatten([ - [f"mov.{ctx.mem_types[x.dtype.scalar()]} {v}, {render_val(0, x.dtype.scalar())};" for v in ctx.r[x]], - [f"@{ctx.r[gate]} ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype.scalar()]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];"] + [f"mov.{ctx.mem_types[x.dtype]} {v}, {render_val(0, x.dtype)};" for v in ctx.r[x]], + [f"@{ctx.r[gate]} ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];"] ]) if alt.max_numel() > 1 else [ - f"@{ctx.r[gate]} ld.{mem_type(loc)}.{ctx.mem_types[x.dtype.scalar()]} {ctx.r[x]}, [{ctx.r[loc]}+0];", - f"@!{ctx.r[gate]} mov.b{ctx.types[x.dtype.scalar()][1:]} {ctx.r[x]}, {ctx.r[alt]};"]), + f"@{ctx.r[gate]} ld.{mem_type(loc)}.{ctx.mem_types[x.dtype]} {ctx.r[x]}, [{ctx.r[loc]}+0];", + f"@!{ctx.r[gate]} mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {ctx.r[alt]};"]), (UPat(Ops.LOAD, name="x", src=(UPat((Ops.INDEX, Ops.SHRINK), name="loc"),)), - lambda ctx, x, loc: f"ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype.scalar()]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];" \ + lambda ctx, x, loc: f"ld.{mem_type(loc)}.v{x.max_numel()}.{ctx.mem_types[x.dtype]} {{{', '.join(ctx.r[x])}}}, [{ctx.r[loc]}+0];" \ if x.max_numel() > 1 else f"ld.{mem_type(loc)}.{ctx.mem_types[x.dtype]} {ctx.r[x]}, [{ctx.r[loc]}+0];"), # simple (UPat(Ops.BUFFER, name="x"), lambda ctx, x: [] if x.addrspace == AddrSpace.REG else [ @@ -197,7 +197,7 @@ def ssa(prefix:str, u:UOp|None=None, dtype:str|None=None) -> str: r[u] = [cast(str,r[x]) for x in u.src] continue if u.op is Ops.BUFFER and u.addrspace == AddrSpace.REG: - r[u] = [ssa("reg", u, self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] + r[u] = [ssa("reg", u, self.types[u.dtype]) for _ in range(u.max_numel())] continue if u.op in {Ops.INDEX, Ops.SHRINK, Ops.LOAD} and u.src[0].addrspace in (AddrSpace.REG, AddrSpace.ALU): # on REG, INDEX/SHRINK pick the register (must be CONST) and LOAD is a noop @@ -207,14 +207,14 @@ def ssa(prefix:str, u:UOp|None=None, dtype:str|None=None) -> str: continue if u.op is Ops.SPECIAL: r[u] = "%" + u.arg elif u.op is Ops.LOAD: - r[u] = [ssa('val', dtype=self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] if u.max_numel() > 1 else ssa('val', u) + r[u] = [ssa('val', dtype=self.types[u.dtype]) for _ in range(u.max_numel())] if u.max_numel() > 1 else ssa('val', u) elif u.op is Ops.PARAM: bufs.append((f"data{u.arg.slot}", u)) elif u.op is Ops.WMMA: # registers for packing/unpacking input and acc - self.wmma_r = [[ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[0]]), 4 // u.src[0].dtype.scalar().itemsize)], - [ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[1]]), 4 // u.src[0].dtype.scalar().itemsize)], - [ssa("wmma_acc", dtype="b32") for _ in range(0, len(r[u.src[2]]), 4 // u.dtype.scalar().itemsize)]] - r[u] = [ssa("wmma", dtype=self.types[u.dtype.scalar()]) for _ in range(u.max_numel())] + self.wmma_r = [[ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[0]]), 4 // u.src[0].dtype.itemsize)], + [ssa("wmma_in", dtype="b32") for _ in range(0, len(r[u.src[1]]), 4 // u.src[0].dtype.itemsize)], + [ssa("wmma_acc", dtype="b32") for _ in range(0, len(r[u.src[2]]), 4 // u.dtype.itemsize)]] + r[u] = [ssa("wmma", dtype=self.types[u.dtype]) for _ in range(u.max_numel())] prefix, dtype = {Ops.CAST: ("cast", None), Ops.BITCAST: ("cast", None), Ops.END: ("pred", "pred"), Ops.RANGE: ("ridx", None), Ops.CONST: ("const", None), Ops.BUFFER: ("local", "u64"), Ops.INDEX: ("bidx", "u64"), Ops.SHRINK: ("bidx", "u64"), Ops.PARAM: ("dat", "u64" if u.addrspace is AddrSpace.GLOBAL else None), **{op: ("alu", None) for op in GroupOp.ALU}}.get(u.op, (None, None)) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 5b468a8741f6c..0d42960a39486 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -466,7 +466,7 @@ def bufferize_buf(ctx:bool, buf:UOp) -> UOp|None: # 7. resolve patches def push_stack(op, s): return UOp(Ops.STACK, - src=tuple(op.replace(dtype=op.dtype.scalar(), src=tuple(x if y is s else y for y in op.src)) for x in s.src)) + src=tuple(op.replace(dtype=op.dtype, src=tuple(x if y is s else y for y in op.src)) for x in s.src)) def fold_binary(buf:UOp, blob:UOp) -> UOp: for b in (m.bufs if isinstance(m:=buf.buffer, MultiBuffer) else (m,)): From e25f86721ddecaed62db1db1e6ba230a3d7bc0f2 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 16 Aug 2026 21:31:53 -0400 Subject: [PATCH 067/306] more torch backend fixups (#17562) --- extra/torch_backend/backend.py | 48 +++++++++++----------------------- extra/torch_backend/test.py | 38 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/extra/torch_backend/backend.py b/extra/torch_backend/backend.py index fda5b0a5aeeb0..71a983390e75d 100644 --- a/extra/torch_backend/backend.py +++ b/extra/torch_backend/backend.py @@ -4,7 +4,7 @@ # A006 Lambda argument `input` is shadowing a Python builtin from tinygrad import Tensor, dtypes, Device from tinygrad.uop.ops import Ops, GroupOp -from tinygrad.helpers import getenv, prod, strides_for_shape, argfix +from tinygrad.helpers import getenv, prod, strides_for_shape import torch.lib TORCH_DEBUG = getenv("TORCH_DEBUG") import torch, pathlib, operator, functools, weakref @@ -99,38 +99,11 @@ def _apply_view_ops(target, ops): for fn, args, kwargs in ops: target = fn(target, *args, **kwargs) return target -# similar to https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/InferSize.h -def _reshape_target_shape(shape:tuple[int, ...], args) -> tuple[int, ...]|None: - if not (req := argfix(*args)): return None - new_shape, infer_idx = [], -1 - for i, s in enumerate(req): - if s is None: s = shape[i] if i < len(shape) else None - if not isinstance(s, int): return None - if s == -1: - if infer_idx != -1: return None - infer_idx = len(new_shape) - new_shape.append(s) - total = prod(shape) - if infer_idx != -1: - known = prod(x for x in new_shape if x != -1) - if known == 0: - if total != 0: return None - new_shape[infer_idx] = 0 - else: new_shape[infer_idx] = total // known - return tuple(new_shape) if prod(new_shape) == total else None - -# TODO: can we get rid of this? only for test_flatten_reshape_add +# a chain of reshapes (and detaches, which move nothing) is undone by reshaping the value back to the base def _try_simple_reshape_view_write(base: Tensor, view: Tensor, val: Tensor) -> bool: if not (ops := _get_view_ops(view)): return False - shapes = [base.shape] - for fn, args, _ in ops: - if fn is Tensor.detach: continue # detach leaves every element where it was (a tracked view only on torch<2.10) - if fn is not Tensor.reshape: return False - if not (next_shape := _reshape_target_shape(shapes[-1], args)): return False - shapes.append(next_shape) - if shapes[-1] != view.shape: return False - for s in reversed(shapes[:-1]): val = val.reshape(s) - base.assign(val) + if any(fn not in (Tensor.reshape, Tensor.detach) for fn, _, _ in ops): return False + base.assign(val.reshape(base.shape)) return True def _view_write(base: Tensor, view: Tensor, value: Tensor) -> None: @@ -445,6 +418,7 @@ def _linalg_svd(self, full_matrices=False): from torch._decomp import get_decompositions decomps = [ aten.native_layer_norm_backward, + aten.native_group_norm_backward, aten.linalg_cross, aten.addmm, aten.addcmul, @@ -479,7 +453,13 @@ def _linalg_svd(self, full_matrices=False): aten._softmax_backward_data, aten.embedding_dense_backward, aten.linalg_vector_norm, aten.binary_cross_entropy, aten.binary_cross_entropy_backward, + # the C++ mse/smooth_l1 kernels resize their out tensor, and a tiny tensor has no storage to resize + aten.mse_loss, aten.mse_loss_backward, + aten.smooth_l1_loss, aten.smooth_l1_loss_backward, aten.upsample_nearest2d.out, + # NOTE: only the "out" overload, the "vec" one is CompositeImplicitAutograd and overriding it loses the autograd kernel + aten.upsample_bicubic2d.out, + aten._adaptive_avg_pool2d, # activations aten.hardswish, aten.hardswish_backward, aten.hardtanh, aten.hardtanh_backward, @@ -661,8 +641,9 @@ def _inplace_op(t, new_value): "aten.acos": Tensor.acos, "aten.any": Tensor.any, "aten.bitwise_not": Tensor.bitwise_not, - "aten.argmax": Tensor.argmax, - "aten.argmin": Tensor.argmin, + # tinygrad indexes with int32, torch's arg reduces return int64 + "aten.argmax": lambda self, dim=None, keepdim=False: self.argmax(dim, keepdim).cast(dtypes.int64), + "aten.argmin": lambda self, dim=None, keepdim=False: self.argmin(dim, keepdim).cast(dtypes.int64), "aten.asinh": Tensor.asinh, "aten.mul": Tensor.mul, "aten.atanh": Tensor.atanh, @@ -688,6 +669,7 @@ def _inplace_op(t, new_value): self.ones_like(**{k: v for k, v in {"dtype": _from_torch_dtype(dtype) if dtype else None, "device": _from_torch_device(device) if device else None}.items() if v is not None}), "aten.max.dim": lambda self, dim, keepdim=False: (self.max(dim, keepdim), self.argmax(dim, keepdim).cast(dtype=dtypes.int64)), + "aten.min.dim": lambda self, dim, keepdim=False: (self.min(dim, keepdim), self.argmin(dim, keepdim).cast(dtype=dtypes.int64)), "aten.cummax": lambda self, dim: ((r := self.cummax(dim))[0], r[1].cast(dtypes.int64)), "aten.cummin": lambda self, dim: ((r := self.cummin(dim))[0], r[1].cast(dtypes.int64)), "aten.nonzero": Tensor.nonzero, diff --git a/extra/torch_backend/test.py b/extra/torch_backend/test.py index 74886a608da02..69d7ba1798659 100644 --- a/extra/torch_backend/test.py +++ b/extra/torch_backend/test.py @@ -166,6 +166,15 @@ def test_amin(self): expected = np.array([[1.5, 5.2, 9.0], [13.2, 17.1, 18.4]], dtype=np.float32) np.testing.assert_equal(y3.cpu().numpy(), expected) + def test_argmax_argmin(self): + a = torch.arange(12, dtype=torch.float32, device=device).reshape(3, 4) + c = a.cpu() + for got, want in [(a.argmax(), c.argmax()), (a.argmin(0), c.argmin(0)), (a.argmax(1, keepdim=True), c.argmax(1, keepdim=True)), + (torch.min(a, 1).indices, torch.min(c, 1).indices), (torch.max(a, 1).indices, torch.max(c, 1).indices), + (torch.min(a, 1).values, torch.min(c, 1).values), (torch.min(a, 1, keepdim=True).indices, torch.min(c, 1, keepdim=True).indices)]: + self.assertEqual(got.dtype, want.dtype) # torch's arg reduces are int64, tinygrad's are int32 + np.testing.assert_equal(got.cpu().numpy(), want.numpy()) + def test_isfinite(self): a = torch.ones(4, device=device) np.testing.assert_equal(torch.isfinite(a).cpu().numpy(), [True, True, True, True]) @@ -373,6 +382,22 @@ def run(dev, bwd_eps): for bwd_eps in [1e-5, 0.3]: for got, want in zip(run(device, bwd_eps), run("cpu", bwd_eps)): np.testing.assert_allclose(got, want, atol=1e-4, rtol=1e-3) + def test_groupnorm_backward(self): + def run(dev): + x = torch.arange(24., device=dev).reshape(2, 4, 3).requires_grad_() + w = torch.linspace(0.5, 2.0, 4).to(dev).requires_grad_() + torch.nn.functional.group_norm(x, 2, w, torch.zeros(4, device=dev)).square().sum().backward() + return x.grad.cpu().numpy(), w.grad.cpu().numpy() + for got, want in zip(run(device), run("cpu")): np.testing.assert_allclose(got, want, atol=1e-4, rtol=1e-3) + + def test_mse_smooth_l1_loss_backward(self): + def run(dev, loss): + x = torch.arange(4., device=dev).requires_grad_() + loss(x, torch.ones(4, device=dev)).backward() + return x.grad.cpu().numpy() + for loss in [torch.nn.functional.mse_loss, torch.nn.functional.smooth_l1_loss]: + np.testing.assert_allclose(run(device, loss), run("cpu", loss), atol=1e-6) + def test_batchnorm_unsqueeze(self): bn = torch.nn.BatchNorm2d(4).to(device) x = torch.randn(8, 4, 3, 3, device=device) @@ -817,6 +842,19 @@ def test_interpolate(self): ref = torch.nn.functional.interpolate(a.cpu(), size=(4, 4), mode="bilinear", align_corners=False) np.testing.assert_allclose(linear.cpu().numpy(), ref.numpy(), rtol=1e-5) + def test_interpolate_bicubic_area(self): + a = torch.arange(32, dtype=torch.float32, device=device).reshape(1, 2, 4, 4) + for mode, scale in [("bicubic", 2.0), ("area", 0.5)]: + ref = torch.nn.functional.interpolate(a.cpu(), scale_factor=scale, mode=mode) + np.testing.assert_allclose(torch.nn.functional.interpolate(a, scale_factor=scale, mode=mode).cpu().numpy(), ref.numpy(), atol=1e-4) + + @unittest.expectedFailure + def test_interpolate_bicubic_backward(self): + # the forward comes from a decomposition, but aten::upsample_bicubic2d_backward has none (nor does + # aten::_adaptive_avg_pool2d_backward, for area), so training through these modes needs a real kernel + x = torch.arange(32., dtype=torch.float32, device=device).reshape(1, 2, 4, 4).requires_grad_() + torch.nn.functional.interpolate(x, scale_factor=2.0, mode="bicubic").sum().backward() + @unittest.expectedFailure def test_interpolate_inexact_scale(self): # torch forwards the raw scale_factor, Tensor.interpolate recomputes it from output_size, and they disagree here From 37a54dc7cf764bbb9a74f6f72aa7345d481f0f87 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:39:37 -0700 Subject: [PATCH 068/306] add some dels to jit for OOM fixes (#17566) --- tinygrad/engine/jit.py | 4 ++++ tinygrad/uop/ops.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index f812f0b9eba4c..a543623072fa5 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -269,10 +269,14 @@ def __call__(self, *args, **kwargs) -> ReturnType: big_linear, onetime_linear = prune_linear(big_linear, set(input_buf_uops)) if DEBUG >= 1: print(f"pruned from {len(big_linear.src) + len(onetime_linear.src)} -> {len(big_linear.src)} kernels") run_linear(onetime_linear, var_vals) + del onetime_linear # hold all buffers reachable from live Tensors (e.g. lazy .grad created during capture), the memory planner can't suballocate those held_bufs = set(buffers) | {u for tref in list(all_tensors) if (t:=tref()) is not None for u in t.uop.toposort() if u.op is Ops.BUFFER} linear = jit_lower(big_linear, held_bufs, input_buf_uops) + # drop the pre-planning graph: it keeps the whole capture-time working set allocated (big_linear) or referenced (held_bufs). + # the planned linear only uses the arena/held buffers, so the intermediates must be freed before linking and first exec + del big_linear, held_bufs self.captured = CapturedJit(ret, linear, names, expected_input_info) ret = self.captured(input_buf_uops, var_vals) elif self.cnt >= 2: diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 63eabaa0b54da..f1f641fe541f7 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -242,9 +242,10 @@ class UOp(RandMixin, metaclass=UOpMetaClass): arg:Any = None tag:Any = None def __del__(self): - if Ops is not None and self.op is Ops.BUFFER and (buffer:=buffers.get(self)) is not None: buffer.ref(-1) + # NOTE: getattr because this object may be partially constructed (e.g. if __init__ raised, like the BEAM timeout SIGALRM) + if Ops is not None and getattr(self, 'op', None) is Ops.BUFFER and (buffer:=buffers.get(self)) is not None: buffer.ref(-1) try: del UOpMetaClass.ucache[(self.op, self.dtype, self.src, self.arg, self.tag)] - except AttributeError: pass + except (AttributeError, KeyError): pass def __reduce__(self): args = [self.op, self.dtype, self.src, self.arg, self.tag, self.metadata] if self.op is Ops.BUFFER and self.realized is not None: args.append(self.realized) From 821e80ff9a4d8894dabdc3e6edb1c53a7547ae59 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 07:33:05 -0400 Subject: [PATCH 069/306] remove torch backend detach hack (#17565) --- extra/torch_backend/backend.py | 7 ++----- extra/torch_backend/test.py | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/extra/torch_backend/backend.py b/extra/torch_backend/backend.py index 71a983390e75d..b7c7a8fac9385 100644 --- a/extra/torch_backend/backend.py +++ b/extra/torch_backend/backend.py @@ -88,9 +88,6 @@ def _wrap(*args, **kwargs): "aten.diagonal": Tensor.diagonal, } -# torch 2.10 handles this natively -if tuple(map(int, torch.__version__.split('.')[:2])) < (2, 10): view_ops.update({"aten.detach": Tensor.detach}) - for k,v in view_ops.items(): torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(wrap_view_op(v)) def _get_view_ops(view): return getattr(view, "_view_ops", []) @@ -99,10 +96,10 @@ def _apply_view_ops(target, ops): for fn, args, kwargs in ops: target = fn(target, *args, **kwargs) return target -# a chain of reshapes (and detaches, which move nothing) is undone by reshaping the value back to the base +# a chain of reshapes is undone by reshaping the value back to the base def _try_simple_reshape_view_write(base: Tensor, view: Tensor, val: Tensor) -> bool: if not (ops := _get_view_ops(view)): return False - if any(fn not in (Tensor.reshape, Tensor.detach) for fn, _, _ in ops): return False + if any(fn is not Tensor.reshape for fn, _, _ in ops): return False base.assign(val.reshape(base.shape)) return True diff --git a/extra/torch_backend/test.py b/extra/torch_backend/test.py index 69d7ba1798659..05e02b1fcad1e 100644 --- a/extra/torch_backend/test.py +++ b/extra/torch_backend/test.py @@ -822,8 +822,6 @@ def test_biased_conv2d(self): np.testing.assert_allclose(b_tiny.grad.cpu().numpy(), b_cpu.grad.numpy(), atol=1e-4, rtol=1e-3) def test_write_through_detach_of_unrealized(self): - # how every module parameter is initialized under set_default_device("tiny"). on torch<2.10 detach is a tracked view, - # so this writes through a view whose shape equals its base's, and the base has no buffer of its own yet a = torch.empty(4, device=device) a.detach().fill_(3) np.testing.assert_equal(a.cpu().numpy(), [3, 3, 3, 3]) From 42714e13992e81305d28d6f38566cb18d44a5b6e Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 07:33:16 -0400 Subject: [PATCH 070/306] update a few is CONST check to check device None [pr] (#17563) * update a few is CONST check to check device None [pr] * clone --- README.md | 2 +- test/external/external_uop_gc.py | 2 +- tinygrad/mixin/gradient.py | 2 +- tinygrad/schedule/multi.py | 6 +++--- tinygrad/schedule/rangeify.py | 6 +++--- tinygrad/tensor.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4e05df80e4051..0e7375ce03410 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ Documentation along with a quick start guide can be found on the [docs website]( ```python from tinygrad import Tensor -x = Tensor.eye(3) +x = Tensor.eye(3).clone() # clone to make it a buffer y = Tensor([[2.0,0,-2.0]]) z = y.matmul(x).sum() z.backward() diff --git a/test/external/external_uop_gc.py b/test/external/external_uop_gc.py index e52500fc195b4..35f58c1481160 100644 --- a/test/external/external_uop_gc.py +++ b/test/external/external_uop_gc.py @@ -44,7 +44,7 @@ def realized_matmul(): z = y.matmul(x) Tensor.realize(z) def realized_gradient(): - x = Tensor.eye(3) + x = Tensor.eye(3).clone() y = Tensor([[2.0,0,-2.0]]) z = y.matmul(x).sum() z.backward() diff --git a/tinygrad/mixin/gradient.py b/tinygrad/mixin/gradient.py index b73e63e45a687..4b10a4393ca2c 100644 --- a/tinygrad/mixin/gradient.py +++ b/tinygrad/mixin/gradient.py @@ -33,7 +33,7 @@ def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else params = {x.arg.slot:x for x in fxn.toposort(enter_calls=False) if x.op == Ops.PARAM} grad_args = ctx.src root_grad = UOp(Ops.TUPLE, src=tuple(UOp(Ops.NOOP) if g.op is Ops.NOOP else - g if g.base.op is Ops.CONST else g.param_like(len(args)+i) for i,g in enumerate(grad_args))) + g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args))) grads = compute_gradient(fxn, root_grad, set(params.values())) # for precompiled calls, substitute forward outputs with params so intermediates aren't recomputed fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(fxn.src)} if k.arg.precompile else {} diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index 14d4126812866..0d415d56cbc09 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -22,14 +22,14 @@ def mstack_early_shrink(ms:UOp, shrink:UOp): def lower_broadcast_copy(c:UOp, x:UOp): if not (isinstance(c.device, tuple) and isinstance(x.device, str)): return None - if (sx:=x.simplify()).device is None and sx.base.op is Ops.CONST: return UOp(Ops.MSTACK, src=(sx,)*len(c.device)) + if (sx:=x.simplify()).device is None: return UOp(Ops.MSTACK, src=(sx,)*len(c.device)) return UOp(Ops.MSTACK, src=tuple(x.copy_to_device(d) for d in c.device)) replace_allreduce = PatternMatcher([ # BROADCAST: explicitly expand broadcast copies and combine with MSTACK - (UPat(Ops.COPY, name="c", src=(UPat(GroupOp.All-{Ops.CONST}, name="x"),)), lower_broadcast_copy), + (UPat(Ops.COPY, name="c", src=(UPat(name="x"),)), lower_broadcast_copy), # COPY_TO_ONE: if copying from multidevice to one, MSELECT the first (TODO: a little from each?) - (UPat(Ops.COPY, name="c", src=(UPat(GroupOp.All-{Ops.CONST}, name="x"),)), lambda c,x: + (UPat(Ops.COPY, name="c", src=(UPat(name="x"),)), lambda c,x: x.mselect(0).copy_to_device(c.device) if isinstance(c.device, str) and isinstance(x.device, tuple) else None), # MSELECT on MSTACK is replaced with nothing (UPat(Ops.MSELECT, src=(UPat(Ops.MSTACK, name="mstack"),), name="ms"), lambda mstack, ms: mstack.src[ms.arg]), diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index c16b73811f618..4b2e36f434489 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -313,9 +313,9 @@ def after_all_invalid(after:UOp): lambda idx,after: idx.const_like(Invalid) if after_all_invalid(after) else None), # hack if a noop turned to a const (UPat(Ops.NOOP, src=(UPat.cvar("c"),)), lambda c: c), - # mstack on CONST is CONST - (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True), - lambda s: c if (c:=s.base).op is Ops.CONST else None), + # a deviceless MSTACK src is the same value on every device, so indexing the stack is just indexing that value + (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True, name="idx"), + lambda s,idx: idx.replace(src=(s,)+idx.src[1:]) if s.device is None else None), ]) pm_remove_bufferize = PatternMatcher([ diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index ee7424cf7f715..d23c6ce33ac76 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -665,13 +665,13 @@ def backward(self, gradient:Tensor|None=None) -> Tensor: ``` """ all_uops = self.uop.toposort() - # backward fills .grad for every in-scope non-CONST float tensor + # backward fills .grad for every in-scope float tensor with a device tensors_need_grad: list[Tensor] = [t for tref in all_tensors if (t:=tref()) is not None and \ - t.uop in all_uops and t.is_floating_point() and t.uop.op is not Ops.CONST] + t.uop in all_uops and t.is_floating_point() and t.device is not None] # clear contexts for t,g in zip(tensors_need_grad, self.gradient(*tensors_need_grad, gradient=gradient)): assert g.shape == t.shape, f"grad shape must match tensor shape, {g.shape!r} != {t.shape!r}" - if g.device is None and t.device is not None: g = g.clone(device=t.device) + if g.device is None: g = g.clone(device=t.device) if t.grad is None: t.grad = g else: t.grad.assign(t.grad + g.to(t.grad.device)) return self From 7fe8e350c5f0c9b7b3fdbfcfe6cc01866d5299a9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 07:56:05 -0400 Subject: [PATCH 071/306] delete bad torch backend function override (#17568) --- extra/torch_backend/backend.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/extra/torch_backend/backend.py b/extra/torch_backend/backend.py index b7c7a8fac9385..927a163ac6164 100644 --- a/extra/torch_backend/backend.py +++ b/extra/torch_backend/backend.py @@ -506,7 +506,7 @@ def _linalg_svd(self, full_matrices=False): # reduce "all", "any", "argmax", "argmin", "cumsum", "cumprod", # complex - "avg_pool2d", "linspace"] + "linspace"] tiny_backend_out = {**{f"aten.{x}.out":getattr(Tensor,x) for x in simple_tensor_methods}, **{ "aten.add.out": lambda input,other,alpha=1: input+alpha*other, @@ -635,15 +635,7 @@ def _inplace_op(t, new_value): "aten.masked_select": Tensor.masked_select, "aten.all": Tensor.all, "aten.sgn": Tensor.sign, - "aten.acos": Tensor.acos, "aten.any": Tensor.any, - "aten.bitwise_not": Tensor.bitwise_not, - # tinygrad indexes with int32, torch's arg reduces return int64 - "aten.argmax": lambda self, dim=None, keepdim=False: self.argmax(dim, keepdim).cast(dtypes.int64), - "aten.argmin": lambda self, dim=None, keepdim=False: self.argmin(dim, keepdim).cast(dtypes.int64), - "aten.asinh": Tensor.asinh, - "aten.mul": Tensor.mul, - "aten.atanh": Tensor.atanh, "aten.fill_.Tensor": lambda self, value: self.const_like(value.reshape(()).item()), "aten.flip": Tensor.flip, "aten.scatter_reduce.two": Tensor.scatter_reduce, @@ -656,8 +648,7 @@ def _inplace_op(t, new_value): Tensor.linspace(start, stop, steps, **({"dtype": _from_torch_dtype(dtype)} if dtype is not None else {})), "aten.topk": Tensor.topk, "aten.constant_pad_nd": lambda self, padding, value=0.0: self.pad(padding, mode="constant", value=value).contiguous(), - # TODO: input contiguous is needed to prevent CFGContext circular dependency assertion for shapes >512 (see test_cumsum_arange_large) - "aten.cumsum": lambda self, dim: self.contiguous().cumsum(dim), + "aten.cumsum": lambda self, dim: self.cumsum(dim), "aten.logsumexp": lambda self, axis, keepdim=False: self.logsumexp(axis[0], keepdim=keepdim), "aten.roll": Tensor.roll, "aten.logcumsumexp": Tensor.logcumsumexp, From 954d4f7797e3b5dd774e5b437e96197b9634a429 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 08:23:34 -0400 Subject: [PATCH 072/306] add back beautiful_mnist_torch in CI (#17569) --- .github/workflows/test.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f73444da1da5..84c6823c3c7a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -86,6 +86,27 @@ jobs: - name: Custom tests run: DEV=CPU:LLVM GPUS=4 TINY_BACKEND=1 python3 -m pytest -nauto extra/torch_backend/test.py extra/torch_backend/test_inplace.py extra/torch_backend/test_multigpu.py extra/torch_backend/test_kernel_fusion.py --durations=20 + torchbackendtrain: + name: Torch Backend Training + runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} + timeout-minutes: 15 + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: Setup Environment + uses: ./.github/actions/setup-tinygrad + with: + key: torch-backend-pillow-torchvision-et-pt + deps: testing_unit + pydeps: "pillow torchvision expecttest" + llvm: 'true' + - name: Install ninja + run: | + sudo apt update || true + sudo apt install -y --no-install-recommends ninja-build + - name: Test beautiful_mnist in torch with TINY_BACKEND + run: STEPS=20 DEV=CPU TARGET_EVAL_ACC_PCT=90.0 MAX_BUFFER_SIZE=0 TINY_BACKEND=1 python3 examples/other_mnist/beautiful_mnist_torch.py + bepython: name: Python Backend runs-on: ${{ github.repository == 'tinygrad/tinygrad' && github.event_name == 'pull_request' && github.event.pull_request.author_association == 'COLLABORATOR' && 'namespace-profile-tinygrad' || 'ubuntu-24.04' }} From 58edff61d91b1da18de1216ce086ad495de3c663 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:08:19 +0300 Subject: [PATCH 073/306] hcq2: one submitter (#17556) * hcq2: c submitter * x * x * x * simpler * simpler * x * x * Dx * revrt * Dx * x * fst * fix --- test/helpers.py | 4 +- tinygrad/engine/realize.py | 36 +++++++------ tinygrad/runtime/ops_cpu.py | 32 ++++++++---- tinygrad/runtime/support/hcq2.py | 89 +++++++++++++++++++++----------- 4 files changed, 103 insertions(+), 58 deletions(-) diff --git a/test/helpers.py b/test/helpers.py index e90b25c525a09..e6d9916d4ea95 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -86,7 +86,9 @@ def assert_jit_cache_len(fxn, expected_len): if linear is None or not linear.src: if expected_len != 0: raise KernelCountException(expected_len, 0) return - if expected_len and all(call_is_hcq(call) for call in linear.src): expected_len = 4 # HCQ2: fence + reset + merged same-queue calls + finalizer + if expected_len and all(call_is_hcq(call) for call in linear.src): # HCQ2: one batch submitter, or fence + reset + merged calls + finalizer + from tinygrad.runtime.support.hcq2 import HCQ_RUNTIME_DEV + expected_len = 1 if HCQ_RUNTIME_DEV.value == "CPU" else 4 if call_is_graph(linear.src[0]): if len(linear.src) != 1: raise KernelCountException(1, len(linear.src)) inner = linear.src[0].src[0].src[0] # LINEAR UOp inside CUSTOM_FUNCTION diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 11458d7696ebf..cbb2bd459b9a1 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -3,9 +3,10 @@ import time, random, itertools, math, contextlib, weakref, array from dataclasses import dataclass, replace, field from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple -from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, wait_cond +from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry +from tinygrad.dtype import dtypes from tinygrad.renderer import Estimates from tinygrad.codegen import to_program from tinygrad.codegen.opt.postrange import args_from_ast @@ -13,7 +14,9 @@ # **************** Helpers **************** def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call.src[1:] if not s.is_bound_var) - +def get_call_var_uops(call:UOp, prg:UOp) -> list[UOp]: + bound = {s.src[0].expr: s.src[1].src[1] for s in call.src[1:] if s.is_bound_var} + return [bound.get(v.expr, v) for v in prg.arg.vars] def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.src[0] if ast.op is Ops.PROGRAM: return tuple(ast.arg.outs), tuple(ast.arg.ins) @@ -166,9 +169,10 @@ def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: et = None - for device, (bufs, device_vars) in zip(to_tuple(call.src[1].device), unwrap_multi(call, resolve_params(call, ctx.input_uops))): + resolved = resolve_params(call, ctx.input_uops) + for device, (bufs, device_vars) in zip(to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): var_vals = {**ctx.var_vals, **device_vars} - prg_bufs = [bufs[i].ensure_allocated() for i in ast.arg.globals] + prg_bufs = [b.ensure_allocated() for b in bufs] rt = get_runtime(device, ast, cache=ctx.cache) global_size, local_size = ast.arg.launch_dims(var_vals) with track_stats(ctx, call, device, prg_bufs, var_vals) as tm: @@ -200,28 +204,26 @@ def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: return t[0] def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - if (info:=call.arg.aux).inputs is not None: - bufs = [_resolve(ctx.input_uops[i], ctx.input_uops).buffer for i in call.arg.aux.input_idxs] - table = call.src[1+info.inputs].buffer - for j,dev in enumerate(call.arg.aux.device): - addrs = array.array('Q', [(b.bufs[j] if isinstance(b, MultiBuffer) else b).get_buf(dev).va_addr for b in bufs]) - mv = (table.bufs[j] if isinstance(table, MultiBuffer) else table).ensure_allocated()._buf.cpu_view().view(fmt='Q') - wait_cond(lambda: mv[0], value=0, timeout_ms=ctx.timeout or getenv("HCQDEV_WAIT_TIMEOUT_MS", 30000), msg=f"{dev} hang detected") - mv[:len(addrs)] = addrs + dev = cast(Any, Device[(info:= call.arg.aux).device[0]]) + addrs = [(b.bufs[j] if isinstance(b:=_resolve(ctx.input_uops[k], ctx.input_uops).buffer, MultiBuffer) else b).get_buf(dev_name).va_addr + for devs, idxs in info.input_idxs for j, dev_name in enumerate(devs) for k in idxs] + dev.rt_buffer._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) - exec_kernel(replace(ctx, update_stats=DEBUG>=3), call, ast) + tables = [UOp.from_buffer(dev.rt_buffer.view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) + for devs, idxs in info.input_idxs for j in range(len(devs))] + if info.inputs is not None: call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) + exec_kernel(replace(ctx, update_stats=DEBUG>=3, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer._buf.va_addr + base}), call, ast) tms = [] - for devices,name,estimates,prof in info.kernels: + for devices, stat_call, prof in info.kernels: for device in devices: tm = None if prof: - (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, name, *prof) + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, *prof) if ctx.wait: d.synchronize(timeout=ctx.timeout) st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) tms.append(tm:=float(en-st)/d.timestamp_divider/1e6) - stat_call = call.replace(arg=replace(call.arg, name=name, aux=replace(info, estimates=estimates, kernels=()))) with track_stats(ctx, stat_call, device, [], ctx.var_vals) as et: et[0] = tm return max(tms) if tms else None @@ -262,7 +264,7 @@ def _validate(call:UOp, sink:UOp) -> UOp: (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="validate", name="ast"),), name="call", allow_any_len=True), exec_validate), ]) -if getenv("HCQ2"): from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link # noqa: E402 # down here, hcq2 imports the helpers above +if getenv("HCQ2"): from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:list[UOp]|None=None, profile:bool|None=None) -> UOp: if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 1ffd2347f0991..0aed3949c35fc 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -13,7 +13,7 @@ from tinygrad.runtime.support.elf import jit_loader from tinygrad.runtime.autogen import libc from tinygrad.codegen import do_to_program -from tinygrad.engine.realize import pm_flatten_linear, get_call_arg_uops, get_runtime +from tinygrad.engine.realize import pm_flatten_linear, get_call_arg_uops, get_call_var_uops, get_runtime from tinygrad import UOp, dtypes from tinygrad.dtype import AddrSpace from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher, graph_rewrite @@ -64,7 +64,7 @@ def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: return UOp(Ops.INS, dtypes.void, words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg="cmd") def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: - args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in prg.arg.vars] + args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(ctx, prg, *args) la = [cpu_cmd(ctx,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t, dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] @@ -99,10 +99,11 @@ def encode_queue(q:UOp) -> UOp: e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(cmdbuf, ring)) copy = UOp.group(*[ring.index((base + e*CMD_SIZE + w) % ring_words).store(cmdbuf.index(e*CMD_SIZE + w).load()) for w in range(CMD_SIZE)]) - # wake the worker after each entry, keeping the post with the stores stops it from hoisting out of the loop - wake = copy.end(e) if WIN else make_signal(devs, tag="func:sem_post").after(copy).index(0).load().call(sem.index(0), ret_dtype=dtypes.void).end(e) - bumped = put.after(wake).index(0).store(put.index(0).load() + cnt) - return sysbuf.after(bumped).index(0).store(put.index(0).load() + cnt) if WIN else bumped + bumped = put.after(copy.end(e)).index(0).store(put.index(0).load() + cnt) + if WIN: return sysbuf.after(bumped).index(0).store(put.after(bumped).index(0).load()) + + e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(bumped,)) + return make_signal(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) # ***************** @@ -196,11 +197,13 @@ class CPUDevice(HCQ2Compiled): pm_lower = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue)]) def __init__(self, device:str=""): + self.workers:list[CPUWorker] = [] super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") self.pm_bufferize = PatternMatcher( - [(UPat(Ops.PARAM, tag=f"COMPUTE:0_{n}"), lambda ctx, n=n: getattr(ctx[0].worker, n)) for n in ("ring", "put", "sem", "sys", "done")] + + [(UPat(Ops.PARAM, tag=f"{q}_{n}"), lambda ctx, q=q,n=n: getattr(ctx[0].worker(q), n)) + for q in ("COMPUTE:0", "SUBMIT:0") for n in ("ring", "put", "sem", "sys", "done")] + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): @@ -210,6 +213,12 @@ def __init__(self, device:str=""): def func_ptr(self, name:str) -> Buffer: return self.func_table.view(1, dtypes.uint64, FUNCS.index(name)*8).ensure_allocated() + def synchronize(self, timeout:int|None=None): + for worker in self.workers: + put, done = (getattr(worker, x)._buf.cpu_view().view(fmt='Q') for x in ("put", "done")) + while done[0] < put[0]: self._wait_signal(done, put[0], timeout) + super().synchronize(timeout) + @functools.cached_property def func_table(self) -> Buffer: lib = ctypes.windll.kernel32 if sys.platform == "win32" else libc.dll # type: ignore[attr-defined] @@ -217,8 +226,8 @@ def func_table(self) -> Buffer: array.array('Q', [unwrap(ctypes.cast(getattr(lib, f), ctypes.c_void_p).value) for f in FUNCS]) return ft - @functools.cached_property - def worker(self) -> CPUWorker: + @functools.cache + def worker(self, queue:str) -> CPUWorker: ring, put, sysbuf, done = (Buffer(self.device, sz, dtypes.uint64, preallocate=True) for sz in (RING_SLOTS*CMD_SIZE, 1, 1, 1)) addr, hsem = 0, None @@ -230,5 +239,6 @@ def worker(self) -> CPUWorker: sem = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(external_ptr=addr), preallocate=True) worker_args = [ring._buf.va_addr, sysbuf._buf.va_addr if WIN else self.func_ptr('sem_wait')._buf.va_addr, done._buf.va_addr, addr] - (worker:=threading.Thread(target=self.prgs[worker_prog].fxn, daemon=True, args=[ctypes.c_uint64(x) for x in worker_args])).start() - return CPUWorker(ring, put, sem, sysbuf, done, worker) + (thread:=threading.Thread(target=self.prgs[worker_prog].fxn, daemon=True, args=[ctypes.c_uint64(x) for x in worker_args])).start() + self.workers.append(worker:=CPUWorker(ring, put, sem, sysbuf, done, thread)) + return worker diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 0d42960a39486..639b948c2bb96 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -30,9 +30,9 @@ class HCQInfo: device:tuple[str, ...] estimates:Estimates = Estimates() - input_idxs:tuple[int, ...] = () # indexes into input_uops used by this call - inputs:int|None = None - kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...]], ...] = () + input_idxs:tuple[tuple[tuple[str, ...], tuple[int, ...]], ...] = () # per inputs table: (devices, indexes into input_uops) + inputs:int|None = None # index of the inputs table in call.src + kernels:tuple[tuple[tuple[str, ...], UOp, tuple[int, ...]], ...] = () # per kernel: (devices, a call carrying its name and estimates, timestamps) def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c @@ -215,7 +215,7 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] - kerns.append((devices, name, info.estimates, tuple(ts_ids))) + kerns.append((devices, make_call(name, call.src[0], info), tuple(ts_ids))) ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_signal(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] @@ -345,14 +345,11 @@ def split_patches(call:UOp) -> UOp|None: scatter = make_scatter_loops(input_patches, tables[0], lt_patches) body = body.substitute({p:p.substitute(scatter | reads) for p in rt_patches}) - if inputs: # fence inputs - fills.append((t:=tables[0][0]).after(make_binary_patch(t, bytes(t.max_numel() * 8)))) # zeroed at link, slot 0 is the host fence - body = body.replace(src=(UOp.sink(*body.src[0].src, t.after(*body.src[0].src).index(0).store(0)),)) # open it once consumed - lt_srcs = collections.defaultdict(list) for p in lt_patches: lt_srcs[p.buf_uop].append(p) return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), - arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=tuple(sorted(dedup(b.arg.slot for g in inputs for b in unwrap_mstack(g.buf_uop))))))) + arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=((call.arg.aux.device, + tuple(sorted(dedup(b.arg.slot for g in inputs for b in unwrap_mstack(g.buf_uop))))),) if inputs else call.arg.aux.input_idxs))) pm_split_patches = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), split_patches)]) # ***************** @@ -371,14 +368,13 @@ def replace_params(call:UOp) -> UOp|None: # keep buffers whose addresses become link-time constants alive and mapped held = args + [r.without_after for r in refhold] - addrs = dedup([g.src[0].without_after for x in call.src for g in x.toposort() if g.op is Ops.GETADDR]) + addrs = dedup([g.src[0].without_after for g in call.toposort() if g.op is Ops.GETADDR]) refhold += [a for a in addrs if a not in held and all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(a))] sub = {(b:=u.without_after): UOp.param(i, u.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=b.op is Ops.PARAM and b.arg.volatile) for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} | _rank_ranges(tops) - info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args) if u.without_after.tag == "inputs"), None)) - return call.replace(src=(body.substitute(sub).replace(arg="hcq_args"), *c_args, *refhold), - arg=replace(call.arg, aux=info)) # TODO: call.after(*refhold)? + info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args + refhold) if u.without_after.tag == "inputs"), None)) + return call.replace(src=(body.substitute(sub).replace(arg="hcq_args"), *c_args, *refhold), arg=replace(call.arg, aux=info)) pm_replace_params = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), replace_params)]) @@ -424,8 +420,44 @@ def callify_hcq(call:UOp, cf:UOp) -> UOp: pm_callify_hcq = PatternMatcher([(UPat(Ops.CALL, src=( UPat(Ops.CUSTOM_FUNCTION, arg="hcq_args", src=(UPat(Ops.SINK),), name="cf"),), name="call", allow_any_len=True), callify_hcq)]) +# ***************** +# 9. merge submitters + +def _lane_arg(a:UOp, lane:int, table:UOp) -> UOp: return table if a.tag == "inputs" else a.mselect(lane) if len(to_tuple(a.device)) > 1 else a + +def merge_batch(batch:list[UOp]) -> UOp: + tables = UOp.variable("hcq_inputs_ptr", 0, 2**64-1, dtypes.uint64, param=True) + lanes = [(c, j, sum(len(idxs) * 8 for _, idxs in c.arg.aux.input_idxs)) for c in batch for j in range(len(c.arg.aux.device))] # (call, lane, bytes) + offs = itertools.accumulate((table_bytes for _, _, table_bytes in lanes), initial=0) # every lane owns the next table of the region + cmds = [c.src[0].src[0].call(*[_lane_arg(a.without_after, j, tables + off) for a in c.src[1:]], UOp.variable("_device_num", 0, 1 << 30).bind(j)) + for (c, j, _), off in zip(lanes, offs)] + + info = HCQInfo((HCQ_RUNTIME_DEV.value,), sum((c.arg.aux.estimates for c in batch), start=Estimates()), + input_idxs=tuple(x for c in batch for x in c.arg.aux.input_idxs), kernels=tuple(k for c in batch for k in c.arg.aux.kernels)) + body = UOp.custom_function("hcq", make_submit(*cmds, devs=HCQ_RUNTIME_DEV.value, queue="SUBMIT:0").sink()) + return body.call(*[s for c in batch for s in c.src[1:] if s.without_after.tag != "inputs"], name=f"hcq_submitter ({len(batch)})", aux=info) + +def merge_submitters(linear:UOp) -> UOp: + batches = [(k, list(g)) for k, g in itertools.groupby(linear.src, key=lambda c: isinstance(c.arg.aux, HCQInfo))] + return linear.replace(src=tuple(c for is_hcq, b in batches for c in ([merge_batch(b)] if is_hcq else b))) + +# ***************** +# hcq schedule + hcq_compile_cache:dict[tuple[bytes, bool], UOp] = {} +def hcq_lower(linear:UOp, pm_encode:PatternMatcher) -> UOp: + # lowering to hcq ir + linear = graph_rewrite(linear, pm_encode, walk=True, name="encode and pack", enter_calls=True) + + # patches and runtime uops + linear = graph_rewrite(linear, pm_early_simplify+symbolic, bottom_up=False, name="simplify patches", enter_calls=True) + linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") + + # and compile it + linear = graph_rewrite(linear, pm_replace_params, name="replace params") + return graph_rewrite(linear, pm_callify_hcq, name="callify hcq", enter_calls=True) + @rewrite_group(lambda linear,input_uops,profile,ret: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: if input_uops is not None: @@ -440,16 +472,9 @@ def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: # schedule linear = graph_rewrite(linear, pm_schedule_and_merge, ctx=({s:p for p,s in back_map.items()}, profile), walk=True, name="schedule and merge hcq") - # lowering to hcq ir - linear = graph_rewrite(linear, pm_encode_cmdbufs+pm_pack_placeholders, walk=True, name="encode and pack", enter_calls=True) - - # patches and runtime uops - linear = graph_rewrite(linear, pm_early_simplify+symbolic, bottom_up=False, name="simplify patches", enter_calls=True) - linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") - - # and compile it - linear = graph_rewrite(linear, pm_replace_params, name="replace params") - final_linear = hcq_compile_cache[cache_key] = graph_rewrite(linear, pm_callify_hcq, name="callify hcq", enter_calls=True) + # lower to hcq programs, then pack the programs of every batch into one C submitter (needs a C runtime device for the program addresses) + linear = hcq_lower(linear, pm_encode_cmdbufs+pm_pack_placeholders) + final_linear = hcq_compile_cache[cache_key] = hcq_lower(merge_submitters(linear), pm_encode_cmdbufs) if HCQ_RUNTIME_DEV.value == "CPU" else linear return final_linear @@ -543,7 +568,6 @@ def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Rende super().__init__(device, allocator, compilers, runtime, None, arch=arch) - self.rt_buffer = Buffer(self.device, 64 << 20, dtypes.uint8, options=BufferSpec(uncached=True, cpu_access=True)) self.rt_allocator = BumpAllocator(64 << 20) self.prof_ents:dict[int, ProfileGraphEntry] = {} @@ -567,6 +591,10 @@ def _at_profile_finalize(self): tdiffs.append((st+perf_counter_us())/2 - gpu) Compiled.profile_events.append(ProfileDeviceEvent(self.device, statistics.median(tdiffs), self.device_props())) + @functools.cached_property + def rt_buffer(self) -> Buffer: + return Buffer(self.device, self.rt_allocator.size, dtypes.uint8, options=BufferSpec(uncached=True, cpu_access=True), preallocate=True) + def new_buffer(self, b:UOp, cache:bool) -> Buffer: if cache or b.tag in HCQ_CACHE_TAGS: return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=True, cpu_access=True, nolru=True)) @@ -578,16 +606,19 @@ def signal(self, name:str|int, init_value:int=0) -> Buffer: buf.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = init_value return buf + def _wait_signal(self, sig:memoryview, value:int, timeout:int|None=None): + timeout = timeout if timeout is not None and self.can_recover else None + st, done = time.perf_counter(), sig[0] + while done < value: + if done != (done:=sig[0]): st = time.perf_counter() + elif time.perf_counter() - st > (timeout or self.wait_timeout_ms) / 1000: self.on_device_hang() + def synchronize(self, timeout:int|None=None): if HCQ_RUNTIME_DEV.value != self.device: Device[HCQ_RUNTIME_DEV.value].synchronize() sig = self.signal("timeline").as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') tl = self.signal("value", 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - timeout = timeout if timeout is not None and self.can_recover else None - st, done = time.perf_counter(), sig[0] - while done < tl[0] - 1: - if done != (done:=sig[0]): st = time.perf_counter() - elif time.perf_counter() - st > (timeout or self.wait_timeout_ms) / 1000: self.on_device_hang() + self._wait_signal(sig, tl[0] - 1, timeout) if self.prof_ents: self.collect_prof() def on_device_hang(self): raise RuntimeError(f"{self.device} hang detected") From 2776c5b36905084d10aac0e3ed83cec2cf8f8633 Mon Sep 17 00:00:00 2001 From: b1tg <33436708+b1tg@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:57:50 +0800 Subject: [PATCH 074/306] fix call arg indexing in shard scheduling (#17519) --- test/unit/test_multitensor.py | 6 ++++++ tinygrad/schedule/rangeify.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/unit/test_multitensor.py b/test/unit/test_multitensor.py index 139c4588ff535..d93019bc6015e 100644 --- a/test/unit/test_multitensor.py +++ b/test/unit/test_multitensor.py @@ -390,6 +390,12 @@ def test_symbolic_broadcast_copy(self): self.assertEqual(out.shape, (rows, 8)) np.testing.assert_equal(out[:3].to(Device.DEFAULT).numpy(), np.ones((3, 8))) + def test_symbolic_broadcast_consumed(self): + rows = Variable("rows", 1, 4).bind(3) + out = (Tensor.ones(rows).to(devices_2) + 1).realize() + self.assertEqual(out.shape, (rows,)) + np.testing.assert_equal(out[:3].to(Device.DEFAULT).numpy(), np.full(3, 2)) + def test_multitensor_jit_in_list(self): # test MULTI tensor inside a list container - exercises the container unpacking + MULTI unpacking @TinyJit diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 4b2e36f434489..430fb52a6a900 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -327,6 +327,9 @@ def after_all_invalid(after:UOp): (UPat(Ops.END, src=(UPat(Ops.NOOP, name="x"),), allow_any_len=True), lambda x: x), ]) +def strip_zero_offset_shrink(x:UOp) -> UOp: + return x.src[0] if x.op is Ops.SHRINK and all(resolve(start == 0, False) for start,_ in x.marg) else x + def no_indexing_calls(u:UOp): new_srcs = [] for x in u.src: @@ -336,8 +339,9 @@ def no_indexing_calls(u:UOp): new_srcs.append(x.src[0]) elif x.op is Ops.SHRINK: # SHRINK with offset 0 is fine - # TODO: check offset - new_srcs.append(x.src[0]) + new_srcs.append(strip_zero_offset_shrink(x)) + elif x.op is Ops.MSTACK: + new_srcs.append(x.replace(src=tuple(strip_zero_offset_shrink(s) for s in x.src))) else: # everything else we pass through new_srcs.append(x) @@ -584,7 +588,7 @@ def get_kernel_graph(sink:UOp) -> UOp: tsink, rctx = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) tsink = graph_rewrite(tsink, - symbolic+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize+pm_no_indexing_calls, + symbolic+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize, name="symbolic+reduce_collapse+debuf") tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=rctx, name="limit buffers") @@ -595,6 +599,7 @@ def get_kernel_graph(sink:UOp) -> UOp: paramarg_start: int = max([-1]+slots) + 1 tsink = graph_rewrite(tsink, pm_add_buffers+pm_add_param_range_tags, ctx=itertools.count(paramarg_start), bottom_up=True, name="stage to store") tsink = graph_rewrite(tsink, split_kernels, bottom_up=True, name="split kernels") + tsink = graph_rewrite(tsink, pm_no_indexing_calls, name="remove indexing from call args") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Kernel Graph") if SPEC: From 00d6eed43ca1660196de780f6d750ac38e989f6a Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:03:44 +0300 Subject: [PATCH 075/306] hcq2: speed (#17570) * hcq2: speed * x x --- tinygrad/device.py | 2 +- tinygrad/engine/realize.py | 3 ++- tinygrad/runtime/support/hcq2.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tinygrad/device.py b/tinygrad/device.py index de6cc53b97227..e7cae72e12ec4 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -133,7 +133,7 @@ def is_initialized(self) -> bool: return self.is_allocated() and self.device in # check if the underlying buffer is allocated, possibly from the base object def is_allocated(self) -> bool: return self.base.is_allocated() if self._base is not None else self.device in self._bufs def get_buf(self, device: str) -> Any: - if (device:=Device.canonicalize(device)) not in self._bufs: + if device not in self._bufs and (device:=Device.canonicalize(device)) not in self._bufs: allocator = Device[device].allocator if device == self.device: self.ensure_allocated() elif self._base is not None: self._bufs[device] = allocator._offset(self._base.get_buf(device), self.nbytes, self.offset) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index cbb2bd459b9a1..b703a95a37778 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -270,8 +270,9 @@ def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:li if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) linear = graph_rewrite(linear, pm_compile, name="precompile kernels", walk=True) + linear = graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) - return graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) + return linear def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if getenv("HCQ2") else linear diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 639b948c2bb96..3655834410cdb 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -97,6 +97,7 @@ def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and def _get_enqueue_devs(call:UOp) -> Any|None: if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None + if call.src[0].op is Ops.COPY: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device return devs if all_devices_in(devs, HCQ_DEVS) else None @@ -597,7 +598,7 @@ def rt_buffer(self) -> Buffer: def new_buffer(self, b:UOp, cache:bool) -> Buffer: if cache or b.tag in HCQ_CACHE_TAGS: - return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=True, cpu_access=True, nolru=True)) + return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=b.tag != "program", cpu_access=True, nolru=True)) return self.rt_buffer.view(b.max_numel(), b.dtype, self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) @functools.cache From b757437f6433ea4655c54914d22f99eeffe3923c Mon Sep 17 00:00:00 2001 From: b1tg <33436708+b1tg@users.noreply.github.com> Date: Tue, 18 Aug 2026 03:19:33 +0800 Subject: [PATCH 076/306] llm: respect expert_gating_func (#17458) * llm: respect expert_gating_func * test * enum * clean --- test/unit/test_llm_moe.py | 29 ++++++++++++++++++++++++++++- tinygrad/llm/model.py | 33 ++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/test/unit/test_llm_moe.py b/test/unit/test_llm_moe.py index 56a486ab56abd..b5d6aafcbb664 100644 --- a/test/unit/test_llm_moe.py +++ b/test/unit/test_llm_moe.py @@ -2,7 +2,7 @@ import numpy as np from dataclasses import replace from tinygrad import Tensor -from tinygrad.llm.model import TransformerBlock, TransformerConfig +from tinygrad.llm.model import ExpertGating, TransformerBlock, TransformerConfig def _moe_config(dim=8, hidden=16, n_heads=2, num_experts=4, num_experts_per_tok=2): return TransformerConfig( @@ -96,5 +96,32 @@ def test_moe_feed_forward_shared_expert(self): expected = moe_expected + shared_expected np.testing.assert_allclose(out.numpy(), expected, rtol=1e-2) + def test_moe_feed_forward_gating_funcs(self): + dim, hidden, n_heads = 8, 16, 2 + num_experts, k = 4, 2 + logits = np.array([4.0, 3.0, 0.0, -1.0], dtype=np.float32) + def softmax(x): + probs = np.exp(x - x.max()) + return probs / probs.sum() + for gating_func in ExpertGating: + for norm_topk_prob in (False, True): + block = TransformerBlock(replace(_moe_config(dim, hidden, n_heads, num_experts, k), + expert_gating_func=gating_func, norm_topk_prob=norm_topk_prob)) + block.ffn_gate_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) for _ in range(num_experts)]) + block.ffn_up_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) * (i + 1) for i in range(num_experts)]) + block.ffn_down_exps.weight = Tensor.stack(*[Tensor.eye(dim, hidden) for _ in range(num_experts)]) + block.ffn_gate_inp.weight = Tensor((logits / dim)[None, :].repeat(dim, 0).T) + out = block._feed_forward(Tensor.ones(1, 1, dim)).numpy()[0, 0, 0] + + if gating_func == ExpertGating.SOFTMAX: selection_scores = softmax(logits) + elif gating_func == ExpertGating.SIGMOID: selection_scores = 1 / (1 + np.exp(-logits)) + elif gating_func == ExpertGating.SOFTMAX_WEIGHT: selection_scores = logits + else: selection_scores = np.sqrt(np.logaddexp(0, logits)) + sel = np.argsort(selection_scores)[-k:] + weights = softmax(logits[sel]) if gating_func == ExpertGating.SOFTMAX_WEIGHT else selection_scores[sel] + if norm_topk_prob: weights /= weights.sum() + expected = (weights * (sel + 1)).sum() / (1 + np.exp(-1)) + np.testing.assert_allclose(out, expected, rtol=1e-3) + if __name__ == '__main__': unittest.main() diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index 95ce646e819e6..7d2034a0180a4 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -1,11 +1,17 @@ from __future__ import annotations -import functools, itertools, pathlib +import enum, functools, itertools, pathlib from dataclasses import dataclass, replace from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function, dtypes from tinygrad.nn import Linear from tinygrad.llm.gguf import gguf_load from tinygrad.uop.ops import resolve +class ExpertGating(enum.IntEnum): + SOFTMAX = 1 + SIGMOID = 2 + SOFTMAX_WEIGHT = 3 # softmax over the top-k selected logits + SQRT_SOFTPLUS = 4 + @functools.cache def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, device:str|None=None) -> Tensor: freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2)[:(dim // 2)] / dim)) @@ -61,6 +67,7 @@ class TransformerConfig: num_experts: int = 0 num_experts_per_tok: int = 0 norm_topk_prob: bool = False + expert_gating_func: ExpertGating = ExpertGating.SOFTMAX q_lora_rank: int = 0 kv_lora_rank: int = 0 shared_expert_dim: int = 0 @@ -103,14 +110,21 @@ def _feed_forward(self, x:Tensor) -> Tensor: if hasattr(self, 'ffn_gate_exps'): h = x.unsqueeze(2) # (B, T, 1, D) - add expert dim for broadcasting logits = self.ffn_gate_inp(x) - if hasattr(self, 'exp_probs_b'): - probs = logits.sigmoid() - _, sel = pairwise_topk(probs + self.exp_probs_b["bias"], self.config.num_experts_per_tok) - probs = probs.gather(-1, sel) - if self.config.norm_topk_prob: probs = probs / probs.sum(axis=-1, keepdim=True) - else: - vals, sel = pairwise_topk(logits, self.config.num_experts_per_tok) - probs = vals.softmax(-1) if self.config.norm_topk_prob else logits.softmax(-1).gather(-1, sel) + bias = self.exp_probs_b["bias"] if hasattr(self, 'exp_probs_b') else None + gating, normalize_topk = self.config.expert_gating_func, self.config.norm_topk_prob + # fast path: without selection bias, normalized SOFTMAX is equivalent to SOFTMAX_WEIGHT + if gating == ExpertGating.SOFTMAX and bias is None and normalize_topk: + gating, normalize_topk = ExpertGating.SOFTMAX_WEIGHT, False + if gating == ExpertGating.SOFTMAX_WEIGHT: scores = logits + elif gating == ExpertGating.SOFTMAX: scores = logits.softmax(-1) + elif gating == ExpertGating.SIGMOID: scores = logits.sigmoid() + elif gating == ExpertGating.SQRT_SOFTPLUS: scores = logits.softplus().sqrt() + + _, sel = pairwise_topk(scores if bias is None else scores + bias, self.config.num_experts_per_tok) + probs = scores.gather(-1, sel) + # SOFTMAX_WEIGHT applies softmax after top-k selection + if gating == ExpertGating.SOFTMAX_WEIGHT: probs = probs.softmax(-1) + if normalize_topk: probs = probs / probs.sum(axis=-1, keepdim=True) probs = probs * self.config.routed_scaling_factor x_down = self.ffn_down_exps(sel, (self.ffn_gate_exps(sel, h).silu() * self.ffn_up_exps(sel, h)).contiguous()) # (B, T, k, D) out = (x_down * probs.unsqueeze(-1)).sum(axis=2) # (B, T, D) @@ -398,6 +412,7 @@ def from_gguf(gguf:Tensor|str|pathlib.Path, max_context:int|None=None, qk_norm=int(state_dict['blk.0.attn_q_norm.weight'].shape[0]) if 'blk.0.attn_q_norm.weight' in state_dict else 0, num_experts=kv.get(f'{arch}.expert_count', 0), num_experts_per_tok=kv.get(f'{arch}.expert_used_count', 0), norm_topk_prob=kv.get(f'{arch}.expert_weights_norm', arch in ('qwen3moe', 'qwen35moe', 'kimi-linear')), + expert_gating_func=ExpertGating(kv.get(f'{arch}.expert_gating_func', ExpertGating.SOFTMAX)), kv_lora_rank=kv_lora_rank, q_lora_rank=kv.get(f'{arch}.attention.q_lora_rank', 0), leading_dense_blocks=kv.get(f'{arch}.leading_dense_block_count', 0), shared_expert_dim=kv.get( From 901d257a26f9ee1afe0c2391d02974bacc553f96 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 15:19:57 -0400 Subject: [PATCH 077/306] some more torch backend cleanups (#17571) * some more torch backend cleanups * fix --- extra/torch_backend/backend.py | 120 +++++++++++---------------------- extra/torch_backend/test.py | 15 +++++ 2 files changed, 53 insertions(+), 82 deletions(-) diff --git a/extra/torch_backend/backend.py b/extra/torch_backend/backend.py index 927a163ac6164..85466aa8253be 100644 --- a/extra/torch_backend/backend.py +++ b/extra/torch_backend/backend.py @@ -73,6 +73,12 @@ def _wrap(*args, **kwargs): return wrap(ret) return _wrap +# NOTE: list assignment raises IndexError on an out of range dim, and the index must be a tuple: a list of all ints is one advanced index +def _index_dim(self, dim, idx): + idxs = [slice(None)] * self.ndim + idxs[dim] = idx + return self[tuple(idxs)] + view_ops = { "aten.view": Tensor.reshape, "aten._unsafe_view": Tensor.reshape, # when are views unsafe, and do we care? @@ -82,10 +88,11 @@ def _wrap(*args, **kwargs): "aten.transpose.int": Tensor.transpose, "aten.squeeze.dim": Tensor.squeeze, "aten.unsqueeze": Tensor.unsqueeze, - "aten.select.int": lambda self, dim, idx: self[(slice(None),) * (dim%self.ndim) + (idx,)], + "aten.select.int": _index_dim, "aten.permute": Tensor.permute, "aten.alias": lambda self: self, "aten.diagonal": Tensor.diagonal, + "aten.slice.Tensor": lambda self, dim=0, start=None, end=None, step=1: _index_dim(self, dim, slice(start, end, step)), } for k,v in view_ops.items(): torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(wrap_view_op(v)) @@ -138,11 +145,6 @@ def _index_put_impl_(self, indices, values, accumulate=False, unsafe=False): def index_put(self, indices, values, accumulate=False): return aten.index_put(self.cpu(), [z.cpu() if isinstance(z, torch.Tensor) else None for z in indices], values.clone().cpu(), accumulate).tiny() -@torch.library.impl("aten::isin.Tensor_Tensor_out", "privateuseone") -def isin_tensor_tensor_out(x, y, *, assume_unique=False, invert=False, out=None): - result = (unwrap(x).unsqueeze(-1) == unwrap(y).flatten()).any(-1) - return out.copy_(wrap(~result if invert else result)) - @torch.library.impl("aten::randperm.generator_out", "privateuseone") def randperm_generator(n, generator=None, out=None): if generator is not None: raise NotImplementedError("tinygrad torch backend does not support torch.Generator for randperm") @@ -203,49 +205,6 @@ def as_strided(tensor:torch.Tensor, size, stride, storage_offset=None): def _reshape_alias(tensor:torch.Tensor, size, stride): return _as_strided(tensor, size, stride) -@torch.library.impl("aten::empty_strided", "privateuseone") -def empty_strided(size, stride, dtype=None, layout=None, device=None, pin_memory=False): - if TORCH_DEBUG: print(f"empty_strided {size=} {stride=} {dtype=} {layout=} {device=} {pin_memory=}") - ret = Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) - # TODO: should return with requested strides - return wrap(ret) - -@torch.library.impl("aten::empty.memory_format", "privateuseone") -def empty_memory_format(size, dtype=None, layout=None, device=None, pin_memory=False, memory_format=None): - if TORCH_DEBUG: print(f"empty.memory_format {size=} {dtype=} {layout=} {device=} {pin_memory=} {memory_format=}") - ret = Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) - return wrap(ret) - -@torch.library.impl("aten::max_pool2d_with_indices", "privateuseone") -def max_pool2d_with_indices(self:torch.Tensor, kernel_size:tuple[int, ...], stride=None, padding=0, dilation=1, ceil_mode=False): - # TODO: supprt stride [] in tinygrad? - if stride is not None and len(stride) == 0: stride = None - ret, idx = unwrap(self).max_pool2d(kernel_size, stride, dilation, padding, ceil_mode, return_indices=True) - return (wrap(ret), wrap(idx.cast(dtypes.int64))) - -@torch.library.impl("aten::max_pool2d_with_indices_backward", "privateuseone") -def max_pool2d_with_indices_backward(grad_out:torch.Tensor, self:torch.Tensor, kernel_size:tuple[int, ...], stride=None, padding=0, dilation=1, ceil_mode=False, indices=None): - return wrap(Tensor.max_unpool2d(unwrap(grad_out), unwrap(indices), output_size=unwrap(self).shape)) - -@torch.library.impl("aten::max_unpool2d", "privateuseone") -def max_unpool2d(self:torch.Tensor, indices:torch.Tensor, output_size): - return wrap(unwrap(self).max_unpool2d(unwrap(indices), output_size=output_size)) - -@torch.library.impl("aten::arange", "privateuseone") -def arange(end, dtype=None, device=None, pin_memory=None): - has_float = isinstance(end, float) - return wrap(Tensor.arange(0, end, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - -@torch.library.impl("aten::arange.start", "privateuseone") -def arange_start(start, end, dtype=None, device=None, pin_memory=None): - has_float = any(isinstance(x, float) for x in (start, end)) - return wrap(Tensor.arange(start, end, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - -@torch.library.impl("aten::arange.start_step", "privateuseone") -def arange_start_step(start, end, step, dtype=None, device=None, pin_memory=None): - has_float = any(isinstance(x, float) for x in (start, end, step)) - return wrap(Tensor.arange(start, end, step, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if has_float else torch.int64)))) - @torch.library.impl("aten::convolution_overrideable", "privateuseone") def convolution_overrideable(input, weight, bias, stride, padding, dilation, transposed, output_padding, groups): if TORCH_DEBUG >= 1: @@ -266,13 +225,6 @@ def convolution_backward_overrideable(grad_out, input, weight, stride, padding, grads = out.gradient(*[t for t,m in zip([input, weight, bias], output_mask) if m], gradient=grad_out) return tuple([wrap(grads.pop(0)) if m else None for m in output_mask]) -@torch.library.impl("aten::slice.Tensor", "privateuseone") -@wrap_view_op -def slice_tensor(self, dim=0, start=None, end=None, step=1): - slices = [slice(None)] * self.ndim - slices[dim] = slice(start, end, step) - return self[slices] - # the functional scatters. without an impl aten falls back to a path that assumes a real storage: "self.has_storage() INTERNAL ASSERT FAILED" def _scatter_into(self, src, dim, index): out = unwrap(self).clone() @@ -295,12 +247,6 @@ def diagonal_scatter(self, src, offset=0, dim1=0, dim2=1): out[idx] = unwrap(src).cast(base.dtype).reshape(-1) return wrap(out.reshape(base.shape)) -# the functional copy_. without an impl the fallback segfaults on a tensor with no storage -@torch.library.impl("aten::copy", "privateuseone") -def copy(self, src, non_blocking=False): - dest = unwrap(self) - return wrap(unwrap(src).cast(dest.dtype).to(dest.device).expand(dest.shape)) - @torch.library.impl("aten::slice_backward", "privateuseone") def slice_backward(grad_out, input_sizes, dim, start, end, step): grad_input = Tensor.zeros(input_sizes).contiguous() @@ -349,13 +295,6 @@ def upsample(self, size, *args, mode=None): torch.library.impl(f"aten::upsample_nearest{i+1}d", "privateuseone")(functools.partial(upsample, mode="nearest")) torch.library.impl(f"aten::_upsample_nearest_exact{i+1}d", "privateuseone")(functools.partial(upsample, mode="nearest-exact")) -@torch.library.impl("aten::scatter_add.out", "privateuseone") -def scatter_add(self, dim, index, src, out): - self, index, src, out_unwrapped = unwrap(self), unwrap(index), unwrap(src), unwrap(out) - if self.shape == (): _apply_inplace(out_unwrapped, src) - else: _apply_inplace(out_unwrapped, Tensor.scatter_reduce(self, dim, index, src, reduce='sum')) - return out - def _copy_between_devices(src, dest, cast_dtype, to_device, non_blocking=False): if src.is_tiny and dest.is_tiny: src_t, dest_t = unwrap(src), unwrap(dest) @@ -406,11 +345,6 @@ def sort_values(input, dim=-1, descending=False, stable=True, values=None, indic _apply_inplace(unwrap(indices), out_indices.cast(dtypes.int64)) return values, indices -@torch.library.impl("aten::_linalg_svd", "privateuseone") -def _linalg_svd(self, full_matrices=False): - U, S, Vh = unwrap(self).svd(full_matrices) - return wrap(U), wrap(S), wrap(Vh) - # register some decompositions from torch._decomp import get_decompositions decomps = [ @@ -551,6 +485,8 @@ def _linalg_svd(self, full_matrices=False): "aten.where.self_out": Tensor.where, "aten.prod.int_out": Tensor.prod, "aten.scatter.src_out": Tensor.scatter, + "aten.scatter_add.out": lambda self,dim,index,src: src if self.shape == () else Tensor.scatter_reduce(self, dim, index, src, reduce="sum"), + "aten.isin.Tensor_Tensor_out": lambda x,y,assume_unique=False,invert=False: (x.unsqueeze(-1)==y.flatten()).any(-1) != invert, # NOTE: axis=[] in torch means all, change tinygrad? "aten.sum.IntList_out": lambda self,axis,keepdim=False,dtype=None: self.sum(axis if axis is None or len(axis) else None, keepdim, @@ -566,10 +502,9 @@ def _wrap_out(*args, **kwargs): assert out.shape == assigned.shape, f"shape mismatch: {assigned.shape} -> {out.shape}" assert out.device == assigned.device or out.device is None or assigned.device is None, f"device mismatch: {assigned.device} -> {out.device}" assert out.dtype == assigned.dtype, f"dtype mismatch: {assigned.dtype} -> {out.dtype}" - # an out= that is a view has to be written through its base, and _apply_inplace gives a deviceless base its buffer first - if canonical_base(out) is not out: return _apply_inplace(out, assigned) or out - if out.device is None and assigned.device is not None: out.replace(out.empty_like(device=assigned.device)) - return out.assign(assigned) + # writing out= is an in-place write like any other: through the base if it is a view, refreshing any derived views + _apply_inplace(out, assigned) + return out return _wrap_out def _inplace_op(t, new_value): @@ -577,7 +512,14 @@ def _inplace_op(t, new_value): else: _apply_inplace(t, new_value) return t -tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{ +# the three arange overloads are one function at different arity, and dtype/layout/device/pin_memory are keyword only in all of them +def _arange(*args, dtype=None, **_): + return Tensor.arange(*args, dtype=_from_torch_dtype(dtype or (torch.get_default_dtype() if any(isinstance(x, float) for x in args) else torch.int64))) + +def _empty(size, dtype=None, device=None, **_): + return Tensor.empty(*size, dtype=_from_torch_dtype(dtype or torch.get_default_dtype()), device=_from_torch_device(device)) + +tiny_backend = {**tiny_backend_out, **{ "aten.remainder.Scalar_Tensor": lambda x,y: x%y, "aten.floor_divide": lambda x,y: x//y, "aten.floor_divide_.Tensor": lambda x,y: x//y, @@ -646,6 +588,19 @@ def _inplace_op(t, new_value): "aten.add.Tensor": lambda input,other,alpha=1: input+alpha*other, "aten.linspace": lambda start, stop, steps, dtype=None, **kwargs: Tensor.linspace(start, stop, steps, **({"dtype": _from_torch_dtype(dtype)} if dtype is not None else {})), + # the functional copy_. without an impl the fallback segfaults on a tensor with no storage + "aten.copy": lambda self,src,non_blocking=False: src.cast(self.dtype).to(self.device).expand(self.shape), + "aten.arange": lambda end, **kwargs: _arange(0, end, **kwargs), + "aten.arange.start": _arange, + "aten.arange.start_step": _arange, + # empty_strided takes the strides and drops them: we always allocate contiguous + "aten.empty_strided": lambda size, stride, **kwargs: _empty(size, **kwargs), + "aten.empty.memory_format": _empty, + # TODO: supprt stride [] in tinygrad? + "aten.max_pool2d_with_indices": lambda self,kernel_size,stride=None,padding=0,dilation=1,ceil_mode=False: ((r:=Tensor.max_pool2d(self, kernel_size, stride or None, dilation, padding, ceil_mode, return_indices=True))[0], r[1].cast(dtypes.int64)), + "aten.max_pool2d_with_indices_backward": lambda grad_out,self,kernel_size,stride=None,padding=0,dilation=1,ceil_mode=False,indices=None: Tensor.max_unpool2d(grad_out, indices, output_size=self.shape), + "aten.max_unpool2d": lambda self,indices,output_size: Tensor.max_unpool2d(self, indices, output_size=output_size), + "aten._linalg_svd": lambda self,full_matrices=False: Tensor.svd(self, full_matrices), "aten.topk": Tensor.topk, "aten.constant_pad_nd": lambda self, padding, value=0.0: self.pad(padding, mode="constant", value=value).contiguous(), "aten.cumsum": lambda self, dim: self.cumsum(dim), @@ -719,15 +674,16 @@ def nf(*args, **kwargs): return nf # the aten schema says how an op is called: an inplace view retargets the view, a writable first arg is inplace, -# and a writable out arg must have come from tiny_backend_out so that wrap_out was applied +# and a writable out arg gets wrap_out's dtype cast, shape assert, and view write-through for k,v in tiny_backend.items(): name, _, overload = k.removeprefix("aten.").partition(".") op = getattr(getattr(aten, name), overload or "default") writes = [a.name for a in op._schema.arguments if a.alias_info is not None and a.alias_info.is_write] if torch.Tag.inplace_view in op.tags: fxn = wrap_inplace_view_op(v) elif writes == [op._schema.arguments[0].name] and op._schema.returns: fxn = wrap_inplace(v) - elif not writes or (writes == ["out"] and k in tiny_backend_out): fxn = wrap_fxn(k, v) - else: raise RuntimeError(f"{k} writes {writes}: expected an inplace first arg, or an out arg with {k} in tiny_backend_out") + elif not writes: fxn = wrap_fxn(k, v) + elif writes == ["out"]: fxn = wrap_fxn(k, wrap_out(v)) + else: raise RuntimeError(f"{k} writes {writes}: unhandled writable arg in schema") torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(fxn) @torch.library.impl("aten::equal", "privateuseone") diff --git a/extra/torch_backend/test.py b/extra/torch_backend/test.py index 05e02b1fcad1e..b6f53f944af32 100644 --- a/extra/torch_backend/test.py +++ b/extra/torch_backend/test.py @@ -83,6 +83,12 @@ def test_storage_offset_through_aliases(self): torch.add(torch.ones(5, device=device), torch.ones(5, device=device), out=a) self.assertEqual(a.detach().storage_offset(), 3) + def test_out_refreshes_views_of_base(self): + a = torch.zeros(4, device=device) + v = a[2:] + torch.add(torch.ones(4, device=device), torch.ones(4, device=device), out=a) + np.testing.assert_equal(v.cpu().numpy(), [2., 2.]) + @unittest.expectedFailure # TODO: storage offset assumes a contiguous source, use UOp.contiguous_view_offset def test_storage_offset_non_contiguous_source(self): a = torch.arange(12., device=device).reshape(3,4) @@ -541,6 +547,15 @@ def test_slice_with_step(self): cpu_res = torch.arange(20, dtype=torch.float32)[::2][1:4].numpy() np.testing.assert_equal(torch_res, cpu_res) + def test_select_out_of_range_dim(self): + a = torch.arange(12, dtype=torch.int32, device=device).reshape(3, 4) + with self.assertRaises(IndexError): a.select(5, 0) + + def test_select_collapses_the_only_dim(self): + a = torch.arange(3, dtype=torch.int32, device=device) + self.assertEqual(a.select(0, 1).shape, ()) + np.testing.assert_equal(a.select(0, 1).cpu().numpy(), 1) + def test_slice_negative_dim(self): a = torch.arange(13, dtype=torch.int32, device=device).repeat(8, 1) torch_chunks = a.chunk(3, -1) From 34c9b9d434e7b7853af4f5e9a11fe654fec675c4 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 15:27:20 -0400 Subject: [PATCH 078/306] add back cast where rule [pr] (#17572) --- test/null/test_uop_symbolic.py | 7 +++++++ tinygrad/uop/symbolic.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 697d3c8890595..369ea785feca6 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -1013,6 +1013,13 @@ def test_neg_in_comp(self): b = Variable("b", 0, 3) self.helper_test_variable(-a<-b, False, True, "(b 2 diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index de3556bff6257..647460639d5bb 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -6,6 +6,7 @@ from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup +from tinygrad.uop.weak import commit_weak # TODO: symbolic shouldn't be importing from codegen from tinygrad.codegen.decomp.transcendental import xpow @@ -433,6 +434,10 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: # reorder ALU/VECTORIZE (UPat(GroupOp.ALU, src=(UPat(Ops.STACK, src=UPat(name='x')), UPat(Ops.STACK, src=UPat(name='y'))), name='alu'), lambda x,y,alu: UOp(Ops.STACK, src=(UOp(alu.op, src=(x,y)),))), + # ** where ** + # push cast to branches + (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), + lambda s,a,b,cast: s.where(commit_weak(a, cast.dtype), commit_weak(b, cast.dtype))), # ** pow ** ((UPat(Ops.POW, name="p"), lambda p: xpow(*p.src))), # ** load/store folding ** From d4a1f39038659ff07840ddf87336f5984c1aae61 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 18:01:11 -0400 Subject: [PATCH 079/306] clean up cstyle render inf and nan [PR] (#17573) --- tinygrad/renderer/cstyle.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 7e220512ac2be..fd2e58499822b 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -32,9 +32,8 @@ (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"{ctx.code_for_workitem[x.arg[0]](x.arg[-1])}; /* {(x.src[0]).render()} */"), # const - (UPat(Ops.CONST, arg=math.inf, name="x"), lambda ctx, x: f"({ctx.render_cast(x, ctx.infinity)})"), - (UPat(Ops.CONST, arg=-math.inf, name="x"), lambda ctx, x: f"({ctx.render_cast(x, f'-{ctx.infinity}')})"), - (UPat(Ops.CONST, dtype=dtypes.floats, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx.nan)})" if math.isnan(x.val) else None), + (UPat(Ops.CONST, dtype=dtypes.floats, name="x"), lambda ctx,x: None if math.isfinite(v:=x.val) else \ + f"({ctx.render_cast(x, ctx.nan if math.isnan(v) else ctx.infinity if v > 0 else f'-{ctx.infinity}')})"), (UPat(Ops.CONST, dtype=dtypes.float, name="x"), lambda ctx,x: f"{x.val}f"), (UPat(Ops.CONST, dtype=dtypes.int64, name="x"), lambda ctx,x: f"{x.val}l"), (UPat(Ops.CONST, dtype=dtypes.uint64, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}ul"), @@ -495,10 +494,9 @@ def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 = (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}," f" {fp8_index(x.src[0].dtype)}, {fp8_index(x.src[0].dtype)}, 0, 0, 0, 0)" if x.arg[0][2] == 128 else None), (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}, 0, 0, 0)"), - (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: f"f32_to_fp8({ctx.nan}, {fp8_index(x.dtype)})" if math.isnan(x.val) else None), - (UPat(Ops.CONST, dtypes.fp8s, arg=math.inf, name="x"), lambda ctx,x: f"f32_to_fp8({ctx.infinity}, {fp8_index(x.dtype)})"), - (UPat(Ops.CONST, dtypes.fp8s, arg=-math.inf, name="x"), lambda ctx,x: f"f32_to_fp8(-{ctx.infinity}, {fp8_index(x.dtype)})"), - (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: f"f32_to_fp8({x.val}f, {fp8_index(x.dtype)})"), + (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: + f"f32_to_fp8({ctx.nan if math.isnan(v:=x.val) else ctx.infinity if v == math.inf else f'-{ctx.infinity}' if v == -math.inf else f'{v}f'}," + f" {fp8_index(x.dtype)})"), (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f"f32_to_fp8({ctx[x.src[0]]}, {fp8_index(x.dtype)})"), (UPat(Ops.CAST, dtypes.float, (UPat.var("y", dtypes.fp8s),), name="x",), From cb892e1b92d58e6c80d4b00182bdf4799a06f19c Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 17 Aug 2026 20:55:44 -0400 Subject: [PATCH 080/306] base_rewrite reorder [PR] (#17575) put const before cast, const will become casted const later --- tinygrad/renderer/cstyle.py | 23 +++++++++++------------ tinygrad/uop/weak.py | 12 ++++++------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index fd2e58499822b..0014952bdf7e2 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -7,7 +7,6 @@ from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, float_to_bf16 from tinygrad.renderer import Renderer - base_rewrite = PatternMatcher([ # local/reg buffers (UPat(Ops.BUFFER, name="x"), lambda ctx,x: ctx.render_buffer(x)), @@ -20,17 +19,6 @@ (UPat(Ops.IF, name="x"), lambda ctx,x: f"if ({ctx[x.src[0]]}) {{"), (UPat((Ops.ENDIF, Ops.END)), lambda ctx: "}"), - # casting - (UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \ - if x.max_numel() > 1 and x.addrspace is AddrSpace.REG else None), - (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx[x.src[0]])})"), - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: ctx[x.src[0]] if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"__builtin_bit_cast({ctx.render_type(x)}, ({ctx.render_type(x.src[0])})({ctx[x.src[0]]}))"), - - # GPU stuff - (UPat(Ops.BARRIER), lambda ctx: ctx.barrier), - (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"{ctx.code_for_workitem[x.arg[0]](x.arg[-1])}; /* {(x.src[0]).render()} */"), - # const (UPat(Ops.CONST, dtype=dtypes.floats, name="x"), lambda ctx,x: None if math.isfinite(v:=x.val) else \ f"({ctx.render_cast(x, ctx.nan if math.isnan(v) else ctx.infinity if v > 0 else f'-{ctx.infinity}')})"), @@ -46,6 +34,17 @@ # default const render (UPat(Ops.CONST, name="x"), lambda ctx,x: str(x.val)), + # casting + (UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \ + if x.max_numel() > 1 and x.addrspace is AddrSpace.REG else None), + (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx[x.src[0]])})"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: ctx[x.src[0]] if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"__builtin_bit_cast({ctx.render_type(x)}, ({ctx.render_type(x.src[0])})({ctx[x.src[0]]}))"), + + # GPU stuff + (UPat(Ops.BARRIER), lambda ctx: ctx.barrier), + (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"{ctx.code_for_workitem[x.arg[0]](x.arg[-1])}; /* {(x.src[0]).render()} */"), + # SHRINK/INDEX (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var('idx')), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), (UPat(Ops.SHRINK, src=(UPat.var("buf"), UPat.var('idx'), UPat.cvar()), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 0b60cdb5604cb..23001e80e85be 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -3,28 +3,28 @@ from tinygrad.helpers import unwrap from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, graph_rewrite, dtype_from_uop -def select_dtype(u:UOp): +def default_dtype(u:UOp): if u.dtype is dtypes.weakfloat: return dtypes.default_float return dtypes.long if u.overflows(dtypes.int32) else dtypes.int def lower_weak_node(u:UOp) -> UOp|None: start, src = (1 if u.op is Ops.WHERE else 0), tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None - dt = strong_dtype(least_upper_dtype(select_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary + dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary else unwrap(dtype_from_uop(u.op, src, u.arg))) return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else commit_weak(s, dt) for s in src[start:])).cast(u.dtype) pm_lower_weak = PatternMatcher([ - (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, select_dtype(u)).cast(u.dtype)), + (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, default_dtype(u)).cast(u.dtype)), # two stacked weak casts are two kind conversions: each resolves at its own kind's default # a SINGLE weak cast is never rewritten here, each consumer absorbs it on its own edge (see lower_weak_srcs) (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), - lambda u,x: x.cast(select_dtype(u.src[0])).cast(select_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), + lambda u,x: x.cast(default_dtype(u.src[0])).cast(default_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), # Binary can widen from the bounds, all other nodes derive from the lowered sources. # a weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition (UPat(GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL}, name="u"), lower_weak_node), (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), - lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=select_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), + lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), ]) def lower_weak_srcs(ctx:dict[UOp, UOp]|None, u:UOp) -> UOp|None: @@ -60,7 +60,7 @@ def commit_weak_srcs(u:UOp) -> UOp|None: # a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - dt = least_upper_dtype(c.dtype, select_dtype(u)) + dt = least_upper_dtype(c.dtype, default_dtype(u)) return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)).cast(c.dtype) pm_cast_weak = PatternMatcher([ From 8d2cc64b696939890dedb3d62a8afac2db7997af Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:24:03 -0700 Subject: [PATCH 081/306] llm: refactor delta attention (#17564) * refactor delta attention * cleanups * bugfixes * stack * recurrent w chunk_size 1 * revert that * extra test --- test/unit/test_attention.py | 82 ++++++++++++++++++++++++++++------ tinygrad/llm/model.py | 87 ++++++++++++++++++++++--------------- 2 files changed, 121 insertions(+), 48 deletions(-) diff --git a/test/unit/test_attention.py b/test/unit/test_attention.py index 622712cc6ed92..8203caf0d9564 100644 --- a/test/unit/test_attention.py +++ b/test/unit/test_attention.py @@ -1,6 +1,6 @@ import unittest import numpy as np -from tinygrad import Tensor, dtypes +from tinygrad import Tensor, dtypes, nn from tinygrad.llm.model import ( GatedDeltaNetBlock, SSMConfig, TransformerBlock, TransformerConfig, apply_rope as apply_rope_new, precompute_freqs_cis, pairwise_topk, @@ -45,10 +45,10 @@ def _tensor_linspace(self, start:float, stop:float, shape:tuple[int, ...]) -> Te return Tensor.linspace(start, stop, int(np.prod(shape)), dtype=dtypes.float32).reshape(*shape) def _make_config(self, **kwargs): - return TransformerConfig(**({"num_blocks":1, "dim":4, "hidden_dim":8, "n_heads":1, "n_kv_heads":1, - "norm_eps":1e-5, "vocab_size":32, "head_dim":4, "rope_theta":10000.0, - "rope_dim":4, "v_head_dim":4, "max_context":4, "ssm_layers":(True,), - "ssm":SSMConfig(conv_kernel=2, state_size=2, group_count=1, time_step_rank=1, inner_size=2)} | kwargs)) + return TransformerConfig(**({"num_blocks":1, "dim":32, "hidden_dim":64, "n_heads":1, "n_kv_heads":1, + "norm_eps":1e-5, "vocab_size":32, "head_dim":32, "rope_theta":10000.0, + "rope_dim":32, "v_head_dim":32, "max_context":4, "ssm_layers":(True,), + "ssm":SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32)} | kwargs)) def _make_block(self, config:TransformerConfig) -> GatedDeltaNetBlock: block = GatedDeltaNetBlock(config, config.ssm) @@ -79,6 +79,10 @@ def _cache_views(self, block:GatedDeltaNetBlock) -> tuple[np.ndarray, np.ndarray recurrent_state = cache[:, conv_flat:].reshape(cache.shape[0], block.num_v_heads, block.head_v_dim, block.head_v_dim) return conv_state, recurrent_state + def _reset_state(self, block:GatedDeltaNetBlock): + Tensor.realize(block.conv_state.assign(block.conv_state.const_like(0)), + block.recurrent_state.assign(block.recurrent_state.const_like(0))) + def _linear_np(self, x:np.ndarray, weight:np.ndarray) -> np.ndarray: return x.astype(np.float32) @ weight.T.astype(np.float32) @@ -86,7 +90,7 @@ def _rms_norm_np(self, x:np.ndarray, weight:np.ndarray, eps:float) -> np.ndarray x_float = x.astype(np.float32) return (x_float / np.sqrt((x_float * x_float).mean(axis=-1, keepdims=True) + eps)) * weight.astype(np.float32) - def _normalize_np(self, x:np.ndarray, eps:float=1e-12) -> np.ndarray: + def _normalize_np(self, x:np.ndarray, eps:float=1e-6) -> np.ndarray: return x / np.maximum(np.sqrt((x * x).sum(axis=-1, keepdims=True)), eps) def _softplus_np(self, x:np.ndarray) -> np.ndarray: @@ -148,6 +152,12 @@ def test_gatedeltanet_reference_and_reset(self): x = Tensor.linspace(-1.0, 1.0, 3 * config.dim, dtype=dtypes.float32).reshape(1, 3, config.dim) expected_outs, expected_conv, expected_recurrent = self._naive_attention(block, x) + out = self._run_attention(block, x, 0) + conv_state, recurrent_state = self._cache_views(block) + np.testing.assert_allclose(out, np.concatenate(expected_outs, axis=1), rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(conv_state, expected_conv[-1], rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(recurrent_state, expected_recurrent[-1], rtol=1e-3, atol=1e-3) + self._reset_state(block) for step in range(x.shape[1]): out = self._run_attention(block, x[:, step:step+1], step) @@ -163,7 +173,7 @@ def test_gatedeltanet_reference_and_reset(self): prompt = Tensor.linspace(0.75, -0.75, 2 * config.dim, dtype=dtypes.float32).reshape(1, 2, config.dim) for i in range(warmup.shape[1]): self._run_attention(block, warmup[:, i:i+1], i) - Tensor.realize(*block._state_reset_ops()) + self._reset_state(block) expected_outs, expected_conv, expected_recurrent = self._naive_attention(block, prompt) for step in range(prompt.shape[1]): @@ -177,18 +187,64 @@ def test_gatedeltanet_reference_and_reset(self): err_msg=f"GatedDeltaNet reset recurrent cache mismatch at step {step}") def test_kda_channel_decay(self): - config = self._make_config(n_heads=2, ssm=SSMConfig(conv_kernel=2, state_size=2, group_count=2, time_step_rank=2, inner_size=4, kda=True)) - block, x = GatedDeltaNetBlock(config, config.ssm), Tensor([[[1., 2., 0., 0.]]]) - # f_b(f_a(x)) = [1, 2, 3, 4] + config = self._make_config(dim=4, hidden_dim=8, n_heads=2, head_dim=4, rope_dim=4, v_head_dim=4, + ssm=SSMConfig(conv_kernel=2, state_size=2, group_count=2, time_step_rank=2, inner_size=4, kda=True)) + block, x = GatedDeltaNetBlock(config, config.ssm), Tensor([[[1., 2., 0., 0.], [2., 1., 0., 0.]]]) block.ssm_f_a.weight = Tensor([[1., 0., 0., 0.], [0., 1., 0., 0.]]) block.ssm_f_b.weight = Tensor([[1., 0.], [0., 1.], [1., 1.], [2., 1.]]) block._init_state(x) initial_state = Tensor.arange(8, dtype=dtypes.float32).reshape(1, 2, 2, 2) block.recurrent_state.assign(initial_state).realize() block.ssm_a = Tensor([[-1.], [-1.]]) - block._attention(x, 0).realize() - alpha = np.exp(-self._softplus_np(np.arange(1, 5)).reshape(1, 2, 1, 2)) - np.testing.assert_allclose(block.recurrent_state.numpy(), initial_state.numpy() * alpha, rtol=1e-5, atol=1e-5) + block._attention(x, x.shape[1]).realize() + alpha = np.exp(-self._softplus_np(np.array([[1, 2, 3, 4], [2, 1, 3, 5]])).reshape(2, 2, 2)).prod(0) + np.testing.assert_allclose(block.recurrent_state.numpy(), initial_state.numpy() * alpha[..., None], rtol=1e-5, atol=1e-5) + + def test_kda_prefill_matches_decode(self): + config = self._make_config(ssm=SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32, kda=True)) + block = GatedDeltaNetBlock(config, config.ssm) + for p in nn.state.get_parameters(block): + p.replace(self._tensor_linspace(-0.05, 0.05, p.shape) if len(p.shape) > 1 else self._tensor_linspace(0.05, 0.1, p.shape)) + x = self._tensor_linspace(-0.5, 0.5, (1, 3, config.dim)) + prefill = self._run_attention(block, x, 0) + prefill_conv, prefill_recurrent = self._cache_views(block) + self._reset_state(block) + decode = np.concatenate([self._run_attention(block, x[:, i:i+1], i) for i in range(3)], axis=1) + decode_conv, decode_recurrent = self._cache_views(block) + np.testing.assert_allclose(prefill, decode, rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(prefill_conv, decode_conv, rtol=1e-3, atol=1e-3) + np.testing.assert_allclose(prefill_recurrent, decode_recurrent, rtol=1e-3, atol=1e-3) + + def test_varied_chunk_sizes_match_decode(self): + for kda in (False, True): + ssm = SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32, kda=kda) + config = self._make_config(ssm=ssm) + if kda: + block = GatedDeltaNetBlock(config, config.ssm) + for p in nn.state.get_parameters(block): + p.replace(self._tensor_linspace(-0.05, 0.05, p.shape) if len(p.shape) > 1 else self._tensor_linspace(0.05, 0.1, p.shape)) + else: block = self._make_block(config) + x = self._tensor_linspace(-0.5, 0.5, (1, 4, config.dim)) + decode = np.concatenate([self._run_attention(block, x[:, i:i+1], i) for i in range(4)], axis=1) + decode_conv, decode_recurrent = self._cache_views(block) + for chunking in ([4], [2, 2], [1, 3], [3, 1], [2, 1, 1]): + self._reset_state(block) + outs, start = [], 0 + for size in chunking: + outs.append(self._run_attention(block, x[:, start:start+size], start)) + start += size + chunked_conv, chunked_recurrent = self._cache_views(block) + np.testing.assert_allclose(np.concatenate(outs, axis=1), decode, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + np.testing.assert_allclose(chunked_conv, decode_conv, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + np.testing.assert_allclose(chunked_recurrent, decode_recurrent, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") + + def test_start_zero_resets_realized_state(self): + config, x = self._make_config(max_context=3), self._tensor_linspace(-1, 1, (1, 3, 32)) + block = self._make_block(config) + self._run_attention(block, x, 0) + restarted = self._run_attention(block, x[:, :2], 0) + fresh = self._run_attention(self._make_block(config), x[:, :2], 0) + np.testing.assert_allclose(restarted, fresh, rtol=1e-3, atol=1e-3) class TestPairwiseTopk(unittest.TestCase): def test_basic_topk(self): diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index 7d2034a0180a4..0d167f43133ef 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -138,8 +138,6 @@ def _feed_forward(self, x:Tensor) -> Tensor: # given the token-prefix match, return how much cached state this block can still reuse def _reusable_prefix_len(self, prefix_len:int, cached_len:int) -> int: return prefix_len - # return writes that reset this block's state after a cache mismatch - def _state_reset_ops(self) -> list[Tensor]: return [] def _init_state(self, x:Tensor): raise NotImplementedError def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: raise NotImplementedError @@ -274,45 +272,65 @@ def __init__(self, config:TransformerConfig, ssm:SSMConfig): def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: B, T, _ = x.shape - assert T == 1, "GatedDeltaNetBlock currently only supports T=1" + # bind ints to a variable so the reset flag stays a runtime value (it toggles when generation restarts at position 0) + start_pos = start_pos if isinstance(start_pos, UOp) else UOp.variable("start_pos", 0, self.config.max_context-1).bind(start_pos) + initial = Tensor(start_pos).eq(0) is_kda = hasattr(self, "ssm_g_a") + symbolic = isinstance(T, UOp) + T_pad = x.max_shape[1] # symbolic chunks are padded to their max size: one graph serves every size # input processing x = x.half() out_gate = self.ssm_g_b(self.ssm_g_a(x)) if is_kda else self.attn_gate(x) - out_gate = out_gate.reshape(B, 1, self.num_v_heads, self.head_v_dim) - beta = self.ssm_beta(x).sigmoid().reshape(B, self.num_v_heads, 1, 1) + out_gate = out_gate.reshape(B, T, self.num_v_heads, self.head_v_dim) + beta = self.ssm_beta(x).sigmoid().reshape(B, T, self.num_v_heads) alpha = self.ssm_f_b(self.ssm_f_a(x)) if is_kda else self.ssm_alpha(x) - alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, self.num_v_heads, -1) * - self.ssm_a.reshape(1, self.num_v_heads, -1)).exp().unsqueeze(-2) - - # qkv conv - conv_window = self.conv_state.cat(self.attn_qkv(x), dim=1) - conv_out = (conv_window * self.ssm_conv1d["weight"].T.unsqueeze(0)).sum(1).silu() + log_alpha = ((alpha.float() + self.ssm_dt["bias"]).softplus().reshape(B, T, self.num_v_heads, -1) * + self.ssm_a.reshape(self.num_v_heads, -1)) + + # qkv conv, conv_state is reset when starting from position 0 + conv_state = initial.where(0, self.conv_state) + # assemble the conv window in a static-size buffer: [conv_state | qkv rows | zero-pad]. + # padded steps are exact no-ops: beta=0 (delta rule off), log_alpha=0 (decay 1 after exp) + win = Tensor.zeros(B, self.ssm_conv_kernel-1 + T_pad, self.conv_channels).uop + win = win.after(win[:, :self.ssm_conv_kernel-1].store(conv_state.cast(win.dtype).uop)) + win = win.after(win[:, self.ssm_conv_kernel-1:self.ssm_conv_kernel-1+T].store(self.attn_qkv(x).cast(win.dtype).uop)) + conv_window = Tensor(win) + # the last conv_kernel-1 columns of the window become the next conv state + conv_state_store = self.conv_state.uop.store(conv_window[:, T:T+self.ssm_conv_kernel-1].cast(self.conv_state.dtype).uop) + + conv_out = functools.reduce(lambda a,b: a+b, + (conv_window[:, i:i+T_pad] * self.ssm_conv1d["weight"][:, i] for i in range(self.ssm_conv_kernel))).silu() + if symbolic: + out_gate = out_gate.pad_to((B, T_pad, self.num_v_heads, self.head_v_dim)) + beta, log_alpha = beta.pad_to((B, T_pad, self.num_v_heads)), log_alpha.pad_to((B, T_pad, *log_alpha.shape[2:])) q, k, v = conv_out.split([self.q_dim, self.q_dim, self.conv_channels - 2*self.q_dim], dim=-1) - q = q.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1) - k = k.reshape(B, self.num_k_heads, self.head_k_dim).normalize(dim=-1).repeat(1, self.num_v_heads//self.num_k_heads, 1) - v = v.reshape(B, self.num_v_heads, self.head_v_dim) - q, k, v = q.mul(self.head_k_dim**-0.5).unsqueeze(-1), k.unsqueeze(-1), v.unsqueeze(-1) - - # recurrent - recurrent_state = self.recurrent_state * alpha - recurrent_state = recurrent_state + ((v - recurrent_state@k) * beta)@k.transpose(-1, -2) - - # store the updated state - conv_state_store = self.conv_state.uop.store(conv_window[:, 1:, :].cast(self.conv_state.dtype).uop) - recurrent_state_store = self.recurrent_state.uop.store(recurrent_state.cast(self.recurrent_state.dtype).uop) - recurrent_state = Tensor(self.recurrent_state.uop.after(recurrent_state_store, conv_state_store)) - - # output - core_attn_out = self.ssm_norm((recurrent_state@q).squeeze(-1).reshape(B, 1, self.num_v_heads, self.head_v_dim)) - out_gate = out_gate.sigmoid() if is_kda else out_gate.silu() - return self.ssm_out((core_attn_out * out_gate).reshape(B, 1, -1).cast(x.dtype)) - - # recurrent state can't be partially reused after divergence, force a full rebuild - def _state_reset_ops(self): - return [self.conv_state.assign(self.conv_state.const_like(0)), - self.recurrent_state.assign(self.recurrent_state.const_like(0))] if hasattr(self, "conv_state") else [] + qk_eps = 1e-12 if is_kda else 1e-6 + q, k = (z.reshape(B, T_pad, self.num_k_heads, self.head_k_dim).normalize(dim=-1, eps=qk_eps) + .repeat(1, 1, self.num_v_heads//self.num_k_heads, 1) for z in (q, k)) + v = v.reshape(B, T_pad, self.num_v_heads, self.head_v_dim) + # layout the per-step operands to broadcast against the (B, H, V, K) state + q, k, v, beta = (z.transpose(1, 2).float() for z in (q, k, v, beta)) + q, k, v, beta = q.unsqueeze(-2) * self.head_k_dim**-0.5, k.unsqueeze(-2), v.unsqueeze(-1), beta.unsqueeze(-1).unsqueeze(-1) + alpha = log_alpha.transpose(1, 2).exp().unsqueeze(-1) # per-channel decay for kda, per-head otherwise (B, H, T, V|1, 1) + + # recurrent: scan over the (padded) tokens, updating the recurrent state. collect the per-step outputs + state = Tensor(self.recurrent_state.uop.after(conv_state_store)).float() # carry the conv write into this graph + state = initial.where(0, state) + outs = [] + for t in range(T_pad): + s1 = state * alpha[:, :, t] # decay the state + delta = (v[:, :, t] - (s1*k[:, :, t]).sum(-1, keepdim=True)) * beta[:, :, t] # the delta rule update + state = s1 + delta * k[:, :, t] + outs.append((state * q[:, :, t]).sum(-1)) + + # store the updated recurrent state in place, then read the stacked outputs after the write + core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop))) + + # output; undo the padding before the output projection + z = (self.ssm_norm(core) * (out_gate.sigmoid() if is_kda else out_gate.silu())).cast(x.dtype).contiguous() + if symbolic: z = z[:, :T] + return self.ssm_out(z.reshape(B, T, -1)) def _init_state(self, x): if not hasattr(self, "conv_state"): @@ -453,7 +471,6 @@ def generate(self, tokens:list[int], chunk_size:int=32, temperature:float=0.0): t = Tensor(tokens + [0] * (self.max_context - len(tokens)), dtype="int32").reshape(1, self.max_context) # recompute start_pos from what's currently valid in the caches start_pos = self.get_start_pos(tokens) - if start_pos < len(self._cached_tokens) and (resets := [r for b in self.blk for r in b._state_reset_ops()]): Tensor.realize(*resets) out, prompt_len = None, len(tokens) while len(tokens) < self.max_context: n_toks = min(chunk_size, len(tokens) - start_pos) From a746861ac08ebcf017e3c40023f0ca86a7b3ea7b Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 17 Aug 2026 19:55:59 -0700 Subject: [PATCH 082/306] compile server for cuda on mac (#17574) --- tinygrad/device.py | 10 ++++++++- tinygrad/runtime/support/compiler_cuda.py | 17 +++++++++----- tinygrad/runtime/support/compiler_qcom.py | 27 ++++++----------------- tinygrad/runtime/support/compileserver.py | 13 +++++++++++ 4 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 tinygrad/runtime/support/compileserver.py diff --git a/tinygrad/device.py b/tinygrad/device.py index e7cae72e12ec4..85742adf3f0cf 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, replace from collections import defaultdict from typing import Any, Callable, Generic, TypeVar, Iterator, Generator, Self, TYPE_CHECKING -import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal +import importlib, inspect, functools, pathlib, os, contextlib, re, atexit, pickle, decimal, subprocess, struct from tinygrad.helpers import LRU, getenv, diskcache_get, diskcache_put, DEBUG, GlobalCounters, PROFILE, temp, colored from tinygrad.helpers import Context, CCACHE, ALLOW_DEVICE_USAGE, MAX_BUFFER_SIZE, cpu_events, ProfileEvent, ProfilePointEvent, suppress_finalizing from tinygrad.helpers import select_by_name, select_first_inited, DEV, TracingKey, size_to_str, pluralize, Target, unwrap, round_up @@ -310,6 +310,14 @@ def compile_cached(self, src:str) -> bytes: if self.cachekey is not None: diskcache_put(self.cachekey, src, lib) return lib def disassemble(self, lib:bytes): pass + def server(self, cmd:str, arch:str, *args) -> subprocess.Popen: + argv = f"{cmd} {pathlib.Path(__file__).parent}/runtime/support/compileserver.py {type(self).__module__}:{type(self).__name__} {arch}" + return subprocess.Popen(argv.split() + [str(a) for a in args], stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + def compile_server(self, src:str, proc:subprocess.Popen) -> bytes: + unwrap(proc.stdin).write(struct.pack("I", len(src.encode())) + src.encode()) + if (lib:=unwrap(proc.stdout).read(struct.unpack("I", unwrap(proc.stdout).read(4))[0])): return lib + raise CompileError("Compilation Error") + @dataclass class TinyELF: diff --git a/tinygrad/runtime/support/compiler_cuda.py b/tinygrad/runtime/support/compiler_cuda.py index 1ba245a9c023a..81124b11696a1 100644 --- a/tinygrad/runtime/support/compiler_cuda.py +++ b/tinygrad/runtime/support/compiler_cuda.py @@ -1,10 +1,12 @@ import hashlib, tempfile, ctypes, re, pathlib -from tinygrad.helpers import to_char_p_p, colored, getenv, system +from tinygrad.helpers import to_char_p_p, colored, getenv, system, OSX from tinygrad.runtime.support.c import init_c_var from tinygrad.runtime.autogen import nvrtc, nvjitlink as jitlink from tinygrad.device import Compiler, CompileError CUDA_PATH = getenv("CUDA_PATH", "") +root = pathlib.Path(__file__).parents[3] +osx_docker_cmd = f"docker run --rm -i -v {root}:{root} -e PYTHONPATH={root} ghcr.io/tinygrad/cuda-arm64:v2.3" def _get_bytes(arg, get_str, get_sz, check) -> bytes: x = ctypes.create_string_buffer(init_c_var(ctypes.c_size_t, lambda x: check(get_sz(arg, ctypes.byref(x)))).value) @@ -44,11 +46,14 @@ def cuda_disassemble(lib:bytes, arch:str, ptx=False): class NVRTCCompiler(Compiler): def __init__(self, arch:str, ptx=True, cache_key:str="cuda"): self.ptx, self.arch, self.compile_options = ptx, arch, [f'--gpu-architecture={arch}'] - self.compile_options += [f"-I{CUDA_PATH}/include"] if CUDA_PATH else ["-I/usr/local/cuda/include", "-I/usr/include", "-I/opt/cuda/include"] - nvrtc_check(nvrtc.nvrtcVersion((nvrtcMajor := ctypes.c_int()), (nvrtcMinor := ctypes.c_int()))) - if (nvrtcMajor.value, nvrtcMinor.value) >= (12, 4): self.compile_options.append("--minimal") + if OSX: self.compiler_process = self.server(osx_docker_cmd, arch, ptx) + else: + self.compile_options += [f"-I{CUDA_PATH}/include"] if CUDA_PATH else ["-I/usr/local/cuda/include", "-I/usr/include", "-I/opt/cuda/include"] + nvrtc_check(nvrtc.nvrtcVersion((nvrtcMajor := ctypes.c_int()), (nvrtcMinor := ctypes.c_int()))) + if (nvrtcMajor.value, nvrtcMinor.value) >= (12, 4): self.compile_options.append("--minimal") super().__init__(f"compile_{cache_key}_{self.arch}") def compile(self, src:str) -> bytes: + if OSX: return self.compile_server(src, self.compiler_process) nvrtc_check(nvrtc.nvrtcCreateProgram(ctypes.byref(prog := nvrtc.nvrtcProgram()), src.encode(), "".encode(), 0, None, None)) nvrtc_check(nvrtc.nvrtcCompileProgram(prog, len(self.compile_options), to_char_p_p([o.encode() for o in self.compile_options])), prog) data = _get_bytes(prog, nvrtc.nvrtcGetPTX if self.ptx else nvrtc.nvrtcGetCUBIN, @@ -80,9 +85,11 @@ def disassemble(self, lib:bytes): cuda_disassemble(lib, self.arch, ptx=True) class NVPTXCompiler(PTXCompiler): def __init__(self, arch:str): - jitlink_check(jitlink.nvJitLinkVersion(ctypes.byref(ctypes.c_uint()), ctypes.byref(ctypes.c_uint()))) + if OSX: self.compiler_process = self.server(osx_docker_cmd, arch) + else: jitlink_check(jitlink.nvJitLinkVersion(ctypes.byref(ctypes.c_uint()), ctypes.byref(ctypes.c_uint()))) super().__init__(arch, cache_key="nv_ptx") def compile(self, src:str) -> bytes: + if OSX: return self.compile_server(src, self.compiler_process) jitlink_check(jitlink.nvJitLinkCreate(handle := jitlink.nvJitLinkHandle(), 1, to_char_p_p([f'-arch={self.arch}'.encode()])), handle) jitlink_check(jitlink.nvJitLinkAddData(handle, jitlink.NVJITLINK_INPUT_PTX, ptxsrc:=super().compile(src), len(ptxsrc), "".encode()), handle) jitlink_check(jitlink.nvJitLinkComplete(handle), handle) diff --git a/tinygrad/runtime/support/compiler_qcom.py b/tinygrad/runtime/support/compiler_qcom.py index 0c4c3bff41a82..64a7cba8d6960 100644 --- a/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad/runtime/support/compiler_qcom.py @@ -1,6 +1,6 @@ -import ctypes, struct, platform, pathlib, shutil, subprocess, sys, tarfile, tempfile +import ctypes, struct, platform, pathlib, shutil, tarfile, tempfile from tinygrad.device import Compiler -from tinygrad.helpers import DEBUG, system, fetch, unwrap +from tinygrad.helpers import DEBUG, system, fetch from tinygrad.runtime.support.compiler_mesa import disas_adreno # see https://github.com/sirhcm/tinydreno from tinygrad.runtime.autogen import llvm_qcom @@ -12,12 +12,11 @@ def __init__(self, arch:str): assert arch.split(',')[0] == "a630", "only a630 supported" if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() else: - self.arch, self.chip_id, self.fs = arch, 0x6030001, tempfile.TemporaryDirectory() + self.arch, self.chip_id, self.fs, root = arch, 0x6030001, tempfile.TemporaryDirectory(), pathlib.Path(__file__).parents[3] with tarfile.open(fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz')) as t: t.extractall(fs:=self.fs.name) - if (qemu:=shutil.which("qemu-aarch64-static")): argv = f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3 {__file__} {arch}" - else: argv = (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {pathlib.Path(__file__).parents[2]}:/tinygrad " - f"-e PYTHONPATH=/ -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3 /tinygrad/runtime/support/compiler_qcom.py {arch}") - self.compiler_process = subprocess.Popen(argv.split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + self.compiler_process = self.server(f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3" if (qemu:=shutil.which("qemu-aarch64-static")) + else (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {root}:{root} " + f"-e PYTHONPATH={root} -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3"), arch) super().__init__(f"compile_qcomcl_{arch}") def __del__(self): llvm_qcom.cl_compiler_destroy_llvm_instance(self.llvm_inst) if platform.machine() == "aarch64" else self.compiler_process.kill() @@ -32,10 +31,7 @@ def checked(self, handle): return handle def compile(self, src) -> bytes: - if platform.machine() != "aarch64": - unwrap(self.compiler_process.stdin).write(struct.pack("I", len(src.encode())) + src.encode()) - if (lib:=unwrap(self.compiler_process.stdout).read(struct.unpack("I", unwrap(self.compiler_process.stdout).read(4))[0])): return lib - raise RuntimeError("QCOM Compilation Error") + if platform.machine() != "aarch64": return self.compile_server(src, self.compiler_process) ch = self.checked(llvm_qcom.cl_compiler_compile_source(self.llvm_inst, self.chip_id, llvm_qcom.CL_MODE_64BIT, b"", 0, 0, 0, src.encode(), 0, llvm_qcom.CL_SRC_STR, None)) if DEBUG >= 8: print(system("llvm-dis", input=ctypes.string_at((comp:=ch.contents.compiled.contents).llvm_bitcode, comp.llvm_bitcode_size))) @@ -48,12 +44,3 @@ def compile(self, src) -> bytes: def disassemble(self, lib: bytes): disas_adreno(lib[(ofs:=_read_lib(lib, 0xc0)):ofs+_read_lib(lib, 0x100)], self.chip_id) -if __name__ == "__main__": - compiler = QCOMCompiler(sys.argv[1]) - while (amt:=sys.stdin.buffer.read(4)): - try: lib = compiler.compile(sys.stdin.buffer.read(struct.unpack("I", amt)[0]).decode()) - except Exception as e: - lib = b"" - print(e, file=sys.stderr, flush=True) - sys.stdout.buffer.write(struct.pack("I", len(lib)) + lib) - sys.stdout.buffer.flush() diff --git a/tinygrad/runtime/support/compileserver.py b/tinygrad/runtime/support/compileserver.py new file mode 100644 index 0000000000000..b41bd7ab6ac27 --- /dev/null +++ b/tinygrad/runtime/support/compileserver.py @@ -0,0 +1,13 @@ +import ast, struct, sys +from tinygrad.helpers import fromimport + +if __name__ == "__main__": + assert len(sys.argv) >= 3, f"usage: {sys.argv[0]} []" + compiler = fromimport(*sys.argv[1].split(':'))(sys.argv[2], *(ast.literal_eval(arg) for arg in sys.argv[3:])) + while (amt:=sys.stdin.buffer.read(4)): + try: lib = compiler.compile(sys.stdin.buffer.read(struct.unpack("I", amt)[0]).decode()) + except Exception as e: + lib = b"" + print(e, file=sys.stderr, flush=True) + sys.stdout.buffer.write(struct.pack("I", len(lib)) + lib) + sys.stdout.buffer.flush() From 7cbe8e0d1525a7a0a92ad9c71d1d1ab957718e93 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:45:43 +0800 Subject: [PATCH 083/306] viz: expanding srcs should not override history (#17577) --- tinygrad/viz/js/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/viz/js/index.js b/tinygrad/viz/js/index.js index 8ea2ef384a7fa..44c0f5dfd9679 100644 --- a/tinygrad/viz/js/index.js +++ b/tinygrad/viz/js/index.js @@ -886,7 +886,7 @@ const evtSources = []; // context: collection of steps const state = {currentCtx:-1, currentStep:0, currentRewrite:0, expandSteps:false, callSrcMask:new Set(), expandedNodes:new Set()}; function setState(ns) { - saveToHistory(state); + if (["currentCtx", "currentStep", "currentRewrite"].some(k => k in ns && state[k] !== ns[k])) saveToHistory(state); const { ctx:prevCtx, step:prevStep } = select(state.currentCtx, state.currentStep); const prevRewrite = state.currentRewrite; Object.assign(state, ns); From 0b757bb9bc98a66e7c0f1eb179e75a9bc35f8a84 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:31:25 +0300 Subject: [PATCH 084/306] Revert "disk: neable polling (#17538)" (#17578) This reverts commit c17849a1f8888910cc2f25af86206121662ff756. --- tinygrad/runtime/ops_disk.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tinygrad/runtime/ops_disk.py b/tinygrad/runtime/ops_disk.py index 0759f01a242cc..801bd35b68d11 100644 --- a/tinygrad/runtime/ops_disk.py +++ b/tinygrad/runtime/ops_disk.py @@ -49,8 +49,7 @@ def _iouring_setup(self): DiskDevice._tried_io_uring_init = True if sys.platform == 'linux' and not hasattr(sys, "getandroidapilevel"): - p = io_uring.struct_io_uring_params(flags=io_uring.IORING_SETUP_SQPOLL, sq_thread_idle=0xffffffff) - fd = libc.syscall(io_uring.NR_io_uring_setup, 4096, ctypes.byref(p)) + fd = libc.syscall(io_uring.NR_io_uring_setup, 4096, ctypes.byref(p:=io_uring.struct_io_uring_params())) if fd < 0: return sq_ptr = libc.mmap(0, p.sq_off.array + p.sq_entries * 4, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | MAP_POPULATE, fd, 0) @@ -68,7 +67,6 @@ def u32ptr(val): return ctypes.cast(val, ctypes.POINTER(ctypes.c_uint32)) kring_mask=u32ptr(sq_ptr+p.cq_off.ring_mask), cqes=ctypes.cast(cq_ptr+p.cq_off.cqes, ctypes.POINTER(io_uring.struct_io_uring_cqe))) DiskDevice.io_uring = io_uring.struct_io_uring(ring_fd=fd, sq=sqdesc, cq=cqdesc) # type: ignore - libc.syscall(io_uring.NR_io_uring_enter, fd, 0, 0, io_uring.IORING_ENTER_SQ_WAKEUP) class DiskBuffer: def __init__(self, device:DiskDevice, size:int, offset=0): @@ -126,6 +124,7 @@ def _copyout_sharded(self, src:DiskBuffer, size:int, _get_free_buf:Callable, seg # Send sqe DiskDevice.io_uring.sq.array[sqe_index] = sqe_index DiskDevice.io_uring.sq.ktail[0] = tail + 1 + libc.syscall(io_uring.NR_io_uring_enter, DiskDevice.io_uring.ring_fd, 1, 1, io_uring.IORING_ENTER_GETEVENTS) reqs.append((copy_batch, copied_in, minor_offset, real_copy_size:=min(sqe.len - minor_offset, size - copied_in))) next_read_offset += sqe.len From a1366e2f6c9cf193ee92894dbe12d5e283f2faf4 Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 18 Aug 2026 09:08:35 -0400 Subject: [PATCH 085/306] alu(long, weakint) can do math in int too [pr] (#17579) * alu(long, weakint) can do math in int too [pr] * remove --- test/unit/test_dtype_weak.py | 16 +++++++++- tinygrad/uop/symbolic.py | 4 +-- tinygrad/uop/weak.py | 58 ++++++++++++++++++------------------ 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 90b8d16b9015a..59cc8f40f3bd3 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -3,11 +3,12 @@ from tinygrad import Tensor, dtypes, TinyJit from tinygrad.helpers import Context from tinygrad.dtype import least_upper_float -from tinygrad.uop.ops import UOp, Ops, dtype_from_uop, graph_rewrite +from tinygrad.uop.ops import UOp, Ops, GroupOp, dtype_from_uop, graph_rewrite from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak from tinygrad.uop.symbolic import symbolic_simple from tinygrad.uop.spec import spec_shared, type_verify from tinygrad.engine.jit import JitError +from test.helpers import full_rewrite class TestWeakPromotion(unittest.TestCase): @@ -288,5 +289,18 @@ def test_weakfloat_lowers(self): self.assertAlmostEqual((i64 + u64).sin().item(), math.sin(2), places=5) # Unary lowers before transcendental +class TestNoRedundantWide(unittest.TestCase): + def wide_alu(self, t:Tensor) -> int: + return sum(sum(1 for u in full_rewrite(call.src[0]).toposort() if u.op in GroupOp.ALU and u.dtype in {dtypes.long, dtypes.ulong}) + for call in t.schedule_linear().src if call.src[0].op is Ops.SINK) + + def test_unbounded_long_stays_long(self): + self.assertGreater(self.wide_alu(Tensor.empty(16, dtype=dtypes.long)*3 + 1), 0) + + def test_fancy_index_has_no_wide_alu(self): + j, o = Tensor([0, 1, 2]).reshape(3, 1), Tensor([0, 1]).reshape(1, 2) + self.assertEqual(self.wide_alu(Tensor.empty(8, 9, 10, 11, 12)[1, j, 2, o, 2]), 0) + + if __name__ == "__main__": unittest.main() diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 647460639d5bb..3f4950d14db1b 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -287,10 +287,10 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: (UPat.var('x', dtypes.ints+(dtypes.weakint,)).cast(dtypes.ints+(dtypes.weakint,), name="a").cast(name="b"), lambda x,a,b: x.cast(b.dtype) if a.dtype.min<=x.vmin and x.vmax<=a.dtype.max else None), # try to do math in int instead of long, keep weak const weak - (UPat(GroupOp.Binary, src=(UPat.var("x", dtypes.long), UPat.var("y", dtypes.long)), name="u"), lambda u,x,y: + (UPat(GroupOp.Binary, src=(UPat.var("x", (dtypes.long, dtypes.weakint)), UPat.var("y", (dtypes.long, dtypes.weakint))), name="u"), lambda u,x,y: (UOp.const(x.val) if x.op is Ops.CONST else x.cast(dtypes.int)).alu(u.op, UOp.const(y.val) if y.op is Ops.CONST else y.cast(dtypes.int)).cast(u.dtype) - if not any(v.overflows(dtypes.int) for v in (u,x,y)) else None), + if dtypes.long in (x.dtype, y.dtype) and not any(v.overflows(dtypes.int) for v in (u,x,y)) else None), ((UPat.var("x", dtypes.weakint) + UPat.cvar("c")).cast(dtypes.sints, name="cast"), lambda x,c,cast:x.cast(cast.dtype)+cast.const_like(c.val)), # only RANGE/IF/STORE/KERNEL have side effects (UPat(Ops.AFTER, name="x"), lambda x: x.replace(src=(x.src[0],)+ diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 23001e80e85be..8b237e5ca5f55 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -7,6 +7,35 @@ def default_dtype(u:UOp): if u.dtype is dtypes.weakfloat: return dtypes.default_float return dtypes.long if u.overflows(dtypes.int32) else dtypes.int +def commit_weak(s:UOp, dt:DType) -> UOp: + # a CONST commits directly at dt (the value stays mathematical, emission truncates), a non-const src takes the cast + return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) + +def commit_weak_srcs(u:UOp) -> UOp|None: + if not any(s.dtype in dtypes.weaks for s in u.src): return None + if (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None + # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too + return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) + +# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer +pm_commit_weak = PatternMatcher([ + (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), + # demand from the destination: a STORE's weak value commits at the destination's dtype + (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), + lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), +]) + +# a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing +def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: + if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None + dt = least_upper_dtype(c.dtype, default_dtype(u)) + return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)).cast(c.dtype) + +pm_cast_weak = PatternMatcher([ + (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), + (UPat(Ops.CAST, name="c", src=(UPat(Ops.CONST, dtype=dtypes.weaks, name="u"),)), lambda c,u: commit_weak(u, c.dtype)), +]) + def lower_weak_node(u:UOp) -> UOp|None: start, src = (1 if u.op is Ops.WHERE else 0), tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None @@ -39,35 +68,6 @@ def lower(s:UOp) -> UOp: ret = lower(u) if u.op in GroupOp.Comparison else u.replace(src=tuple(lower(s) if s.dtype in dtypes.weaks else s for s in u.src)) return None if ret is u else ret -def commit_weak(s:UOp, dt:DType) -> UOp: - # a CONST commits directly at dt (the value stays mathematical, emission truncates), a non-const src takes the cast - return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) - -def commit_weak_srcs(u:UOp) -> UOp|None: - if not any(s.dtype in dtypes.weaks for s in u.src): return None - if (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None - # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) - -# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer -pm_commit_weak = PatternMatcher([ - (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), - # demand from the destination: a STORE's weak value commits at the destination's dtype - (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), - lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), -]) - -# a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing -def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: - if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - dt = least_upper_dtype(c.dtype, default_dtype(u)) - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)).cast(c.dtype) - -pm_cast_weak = PatternMatcher([ - (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), - (UPat(Ops.CAST, name="c", src=(UPat(Ops.CONST, dtype=dtypes.weaks, name="u"),)), lambda c,u: commit_weak(u, c.dtype)), -]) - pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ # a CAST between two concrete dtypes over a CONST is a value conversion: evaluate it once, at the width the CAST states # TODO: delete this once CONST has no dtype From af2a43c850ed30f27dda4bde291ad775ff82615d Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:52:18 +0300 Subject: [PATCH 086/306] hcq2: 64bit addresses (#17576) --- extra/hcq2/ops_amd2.py | 9 +++-- tinygrad/codegen/__init__.py | 4 +-- tinygrad/codegen/late/coalesce.py | 2 +- tinygrad/runtime/ops_python.py | 4 ++- tinygrad/runtime/support/hcq2.py | 55 ++++++++++++++----------------- 5 files changed, 34 insertions(+), 40 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 11db4f3f72c15..be8872c65cbcd 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -179,11 +179,10 @@ class SDMAOps(FastEnum): COPY = auto(); POLL_REGMEM = auto(); FENCE = auto(); TR def sdma_copy(ctx, call): sz = call.src[2].max_numel() * call.src[2].dtype.itemsize - src_addr, dst_addr = call.src[2].getaddr(ctx.devs), call.src[1].getaddr(ctx.devs) - return call.ins(SDMAOps.COPY, src=tuple(UOp.const(x, dtypes.uint32) for off in range(0, sz, ctx.max_copy_size) for x in ( - ctx.sdma.SDMA_OP_COPY | ctx.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_COPY_LINEAR), - ctx.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(min(sz-off, ctx.max_copy_size)-1), 0, - *data64_le(src_addr+UOp.const(off, dtypes.uint64)), *data64_le(dst_addr+UOp.const(off, dtypes.uint64))))) + hdr = ctx.sdma.SDMA_OP_COPY | ctx.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_COPY_LINEAR) + return call.ins(SDMAOps.COPY, src=tuple(x for off in range(0, sz, ctx.max_copy_size) for x in ( + *(UOp.const(v, dtypes.uint32) for v in (hdr, ctx.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(min(sz-off, ctx.max_copy_size)-1), 0)), + *(a + UOp.const(off, dtypes.uint64) if off else a for a in (call.src[2].getaddr(ctx.devs), call.src[1].getaddr(ctx.devs)))))) def sdma_wait(ctx, ins, dst, val): op = ctx.sdma.SDMA_OP_POLL_REGMEM | ctx.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) \ diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 05d666a72410a..982ab35fa9d14 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -153,8 +153,8 @@ def do_stack_wmma(u:UOp): # unpack WMMA (UPat(Ops.WMMA, name="u"), do_stack_wmma), # stacked INDEX is many INDEX - (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.STACK, name="s"))), - lambda b,s: UOp.stack(*[b.index(u) for u in s.src])), + (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.STACK, name="s")), name="x"), + lambda b,s,x: UOp.stack(*[x.replace(src=(b,u)) for u in s.src])), # INDEX into RESHAPE moves the RESHAPE (UPat(Ops.INDEX, src=(UPat((Ops.PARAM, Ops.BUFFER), name="b"), UPat(Ops.RESHAPE, name="s"))), lambda b,s: b.index(s.src[0]).reshape(s.shape)), diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index 64e65f2320541..bac011e4ec1c1 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -149,7 +149,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: grp = full_grp[:length] # NOTE: we apply the valid again after we determine the length offset = offset.valid(valid) if valid is not None else offset - idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(len(grp)))) if len(grp) > 1 else buf.index(offset) + idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(len(grp)))) if len(grp) > 1 else buf.index(offset, dtype=offsets[grp[0]][0].src[0].dtype) if op == Ops.STORE: datas = [] for i,g in enumerate(grp): diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index 0b2126deaae2b..0171dabd58c9d 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -23,7 +23,9 @@ def load(inp, j, dtype: DType): def _store(m, i, v, dtype: DType): if i < 0 or i >= len(m): raise IndexError(f"store out of bounds, size is {len(m)}, access is {i}, value is {v}") - m[i] = to_storage_scalar(v, dtype) + if (w:=m.nbytes // len(m)) >= dtype.itemsize: m[i] = to_storage_scalar(v, dtype) + else: + for k in range(dtype.itemsize // w): m[i+k] = (v >> 8*w*k) & ((1 << 8*w) - 1) # here are the models for the WMMA instruction on the different hardware def generic_wmma_helper(inp, warp_size, WARP_THREADS, K, NUM_A, NUM_B, NUM_C, a_elem, b_elem, c_map): diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 3655834410cdb..9d195fbf8ee9f 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -3,7 +3,7 @@ import struct, functools, time, collections, itertools, decimal, statistics from dataclasses import replace, dataclass from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE -from tinygrad.helpers import to_tuple, round_up, partition, data64_le, panic, ContextVar, perf_counter_us, Context +from tinygrad.helpers import to_tuple, round_up, partition, panic, ContextVar, perf_counter_us, Context from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp @@ -48,9 +48,16 @@ def is_value_known_at_link(val:UOp) -> bool: return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs) def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]: - return tuple(buf.index(UOp(Ops.STACK, dtypes.int, tuple(UOp.const(off // buf.dtype.itemsize, dtypes.int) for off,_ in ps))) - .store(UOp(Ops.STACK, buf.dtype, tuple(val.cast(buf.dtype) for _,val in ps))).rtag(tag) - for ps, tag in zip(partition(patches, lambda p: is_value_known_at_link(p[1])), ("link", None)) if ps) + def _mk_store(ps:list[tuple[sint, UOp]], tag:str|None) -> UOp: + offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const(off // buf.dtype.itemsize, dtypes.int) for off,_ in ps)) + vals = UOp(Ops.STACK, ps[0][1].dtype, tuple(val for _,val in ps)) + return buf.index(offs, dtype=vals.dtype).store(vals).rtag(tag) + + patches = [(off, val.cast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val) for off, val in patches] + link, runtime = partition(patches, lambda p: is_value_known_at_link(p[1])) + inputs, runtime = partition(runtime, lambda p: p[1].op is Ops.GETADDR) + return tuple(_mk_store(list(ps), tag) for cls, tag in ((link, "link"), (inputs, "inputs"), (runtime, None)) + for _, ps in itertools.groupby(sorted(cls, key=lambda p: p[1].dtype), key=lambda p: p[1].dtype)) def make_binary_patch(buf:UOp, blob:bytes) -> UOp: data = UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype) @@ -77,8 +84,8 @@ def make_call(name:str, body:UOp, info:HCQInfo) -> UOp: return UOp.custom_functi def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: data, info = prg.arg buf = UOp.placeholder((data.kernargs_alloc_size // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("kernargs") - words = [w for gi in info.globals for w in data64_le(get_call_arg_uops(call)[gi].getaddr(devs))] + list(info.vars) - return buf.after(*make_patches(buf, [(i * 4, w) for i, w in enumerate(words)])) + words = [get_call_arg_uops(call)[gi].getaddr(devs) for gi in info.globals] + list(info.vars) + return buf.after(*make_patches(buf, list(zip(itertools.accumulate((w.dtype.itemsize for w in words), initial=0), words)))) # ***************** # 0.1. prep: replace buffers with params @@ -307,27 +314,15 @@ def make_addr_table(call:UOp, gaddrs:list[UOp], name:str) -> tuple[UOp, dict[UOp fills = (table.after(*make_patches(table, [(i*table.dtype.itemsize, addr) for addr, i in slots.items()])),) if slots else () return table, reads, fills, {g:slots[bare[g]] for g in gaddrs} -def is_bare_addr(val:UOp) -> bool: return val.op is Ops.CAST and val.src[0].op in (Ops.AND, Ops.SHR) and val.src[0].src[0].op is Ops.GETADDR - -def make_scatter_loops(patches:list[UOp], inputs_table:tuple, lt_patches:list[UOp]) -> dict[UOp, UOp]: - table, _, _, slots = inputs_table - subs, by_dst = {}, collections.defaultdict(list) - for p in patches: by_dst[p.buf_uop].append(p) - for dst, patches in by_dst.items(): - data = [] - for p in patches: - words = [(off, val, get_getaddrs(val)) for off,val in zip(p.src[0].src[1].src, p.src[1].src)] - data += [(off.val, slots[gaddrs[0]]) for off,_,gaddrs in words if gaddrs][::2] - scalars = [(off.val*dst.dtype.itemsize, val) for off,val,gaddrs in words if not gaddrs] - subs[p] = UOp.group(*make_patches(dst, scalars)) if scalars else UOp(Ops.NOOP) - - word_table, slot_table = (UOp.placeholder((len(data),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") for _ in range(2)) - ridx = UOp.range(len(data), next(UOp.unique_num), dtype=dtypes.int, src=(word_table, slot_table, dst)) - widx, slot = ((p.index(ridx).load() % bound).cast(dtypes.int) for p,bound in ((word_table, dst.max_numel()-1), (slot_table, table.max_numel()))) - loop = UOp.group(*[dst.index(widx+i).store((table.index(slot).load() >> 32*i).cast(dtypes.uint32)) for i in range(2)]).end(ridx) - lt_patches += [make_binary_patch(buf, struct.pack(f'<{len(data)}I', *vals)) for buf,vals in zip((word_table, slot_table), zip(*data))] - subs[patches[0]] = UOp.group(loop, subs[patches[0]]) - return subs +def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp]) -> dict[UOp, UOp]: + (dst,), words = dedup(p.buf_uop for p in patches), [(off.val, slots[val]) for p in patches for off, val in zip(p.src[0].src[1].src, p.src[1].src)] + + # build a runtime loop that writes every input address + pairs = UOp.placeholder((2*len(words),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") + lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words)))) + r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst)) + off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()))) + return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: dst.index(off, dtype=table.dtype).store(table.index(slot).load()).end(r)} def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) @@ -341,10 +336,8 @@ def split_patches(call:UOp) -> UOp|None: runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) tables = [make_addr_table(call, gs, n) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec - input_patches = [p for p in rt_patches if (gs:=get_getaddrs(p)) and all(map(is_input_addr, gs)) - and all(is_bare_addr(v) for v in p.src[1].src if get_getaddrs(v))] - scatter = make_scatter_loops(input_patches, tables[0], lt_patches) - body = body.substitute({p:p.substitute(scatter | reads) for p in rt_patches}) + gathers = make_gather_loop(ipathces, tables[0][0], tables[0][3], lt_patches) if (ipathces:=[p for p in rt_patches if p.tag == "inputs"]) else {} + body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}) lt_srcs = collections.defaultdict(list) for p in lt_patches: lt_srcs[p.buf_uop].append(p) From 72280bb218f06aada6a45cfbae264c9cdba0f3f3 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Wed, 19 Aug 2026 05:28:57 +0800 Subject: [PATCH 087/306] gptoss: zero-2 optim (#17581) --- examples/mlperf/model_train.py | 4 ++-- examples/mlperf/models/gpt_oss.py | 1 + extra/gemm/moe_gemm.py | 27 +++++++++++++++++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index 47c4977e3994b..19ba390c6e675 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -1742,8 +1742,8 @@ def train_gptoss(): ) for p in optim.params: - grad_dtype = dtypes.bfloat16 if p.dtype == FP8_DTYPE else p.dtype - p.grad = p.zeros_like(dtype=grad_dtype).contiguous() + p.grad = p.zeros_like(dtype=dtypes.bfloat16 if p.dtype == FP8_DTYPE else p.dtype).contiguous() + if getattr(p, "_zero2", False): p.grad = optim.optimizers[0]._zero_shard(p.grad) grads = [p.grad for p in optim.params] from extra.gemm.cdna_asm_gemm import _mx_block_scale diff --git a/examples/mlperf/models/gpt_oss.py b/examples/mlperf/models/gpt_oss.py index d58d615ef0e76..14822db6dbb7b 100644 --- a/examples/mlperf/models/gpt_oss.py +++ b/examples/mlperf/models/gpt_oss.py @@ -146,6 +146,7 @@ def _one(*s:int): return w_q, w_e8.is_param_(False) if moe: qs = [_one(*shape[1:]) for _ in range(shape[0])] + for q in qs: q[0]._zero2 = True # grad arrives sharded on the expert axis under ZeRO-2 (moe_gemm) return [q[0] for q in qs], [q[1] for q in qs] return _one(*shape) diff --git a/extra/gemm/moe_gemm.py b/extra/gemm/moe_gemm.py index 2b89f752ee2db..4fba527d19e6c 100644 --- a/extra/gemm/moe_gemm.py +++ b/extra/gemm/moe_gemm.py @@ -1,10 +1,32 @@ import functools, pathlib from tinygrad import Tensor, dtypes -from tinygrad.uop.ops import UOp, Ops, KernelInfo +from tinygrad.uop.ops import UOp, Ops, KernelInfo, AxisType +from tinygrad.helpers import getenv from tinygrad.renderer import Estimates from tinygrad.runtime.support.compiler_amd import HIPCCCompiler from extra.gemm.cdna_asm_gemm import quantize_mxfp8, _mx_block_scale, _mx_block_scale_3d +ZERO_OPTIM = getenv("ZERO_OPTIM", 0) + +def reduce_scatter_devaxis(out:Tensor, shard_axis:int=0) -> Tensor: + # out: sharded on the device axis, shape (ndev, *rest); return the device-axis sum left sharded on shard_axis. + u = out.uop + devs, rest = u.device, u.shape[1:] + assert rest[shard_axis] % len(devs) == 0, f"reduce_scatter needs even shards: {rest[shard_axis]} % {len(devs)}" + # reach the raw per-device buffer below the UNSHARD, keeping the AFTERs so reads stay ordered after the kernel writes + node, barriers = u, [] + while node.op is not Ops.UNSHARD: + if node.op is Ops.AFTER: barriers += node.src[1:] + node = node.src[0] + mbuf = node.src[0].after(*barriers) if barriers else node.src[0] + sz = rest[shard_axis] // len(devs) + shards = [] + for i in range(len(devs)): + bounds = tuple((0,s) if a != shard_axis else (i*sz,(i+1)*sz) for a,s in enumerate(rest)) + contribs = [mbuf.mselect(j).reshape(rest).shrink(bounds).copy_to_device(devs[i]) for j in range(len(devs))] + shards.append(functools.reduce(lambda a,b: a.alu(Ops.ADD, b), contribs)) + return Tensor(UOp.mstack(*shards).unshard(shard_axis, UOp.range(len(devs), -1, AxisType.DEVICE)), device=devs) + @functools.cache def custom_hk_grouped_mxfp8_gemm(C:UOp, A:UOp, B:UOp, scale_A:UOp, scale_B:UOp, *extra:UOp, dname:str, n_experts:int) -> UOp: M, K = A.shape @@ -58,7 +80,8 @@ def grouped_mx_wgrad(g:Tensor, xg:Tensor, expert_off:Tensor, n_experts:int) -> T out = Tensor(inv.uop.unshard(0), device=g.device) if is_multi else inv out = Tensor.custom_kernel(out, gT, xT, g_si, x_si, expert_off, fxn=functools.partial(custom_hk_grouped_mxfp8_wgrad, dname=dname, n_experts=n_experts))[0] - out = out.sum(0) if is_multi else out.squeeze(0) + if is_multi and ZERO_OPTIM: out = reduce_scatter_devaxis(out, 0) + else: out = out.sum(0) if is_multi else out.squeeze(0) return out.reshape(n_experts, N, K) def mx_pack_3d(e8:Tensor) -> Tensor: From 49778d9a487b04bd60b3cd451cf57381c08fce6d Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 18 Aug 2026 17:43:27 -0400 Subject: [PATCH 088/306] start renderer casted const migration [pr] (#17582) before rendering, rewrite strong typed const to casted weak const and have renderer adopt the new UOp. starting with PYTHON --- test/backend/test_linearizer.py | 3 ++- tinygrad/codegen/__init__.py | 11 +++++++++-- tinygrad/renderer/__init__.py | 2 ++ tinygrad/runtime/ops_python.py | 1 + tinygrad/uop/render.py | 2 +- tinygrad/uop/spec.py | 6 ++++++ tinygrad/uop/validate.py | 2 ++ 7 files changed, 23 insertions(+), 4 deletions(-) diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 0f36811baaba0..ab63e732b5e3f 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -252,7 +252,7 @@ def test_simple_unroll_no_between_phi_dependencies(self): for u in uops: if u.op is Ops.STORE and u.src[0].addrspace is AddrSpace.REG: if uops.index(u) < begin_range: - assert u.src[1].op is Ops.CONST + assert u.src[1].op not in GroupOp.ALU else: assert u.src[1].op in GroupOp.ALU assert begin_range < uops.index(u) < end_range @@ -261,6 +261,7 @@ def test_simple_unroll_no_between_phi_dependencies(self): assert end_range < uops.index(u) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") + @unittest.skipIf(Device[Device.DEFAULT].renderer.casted_consts, "reads a literal, which is casted here. TODO: flip this") def test_default_global_reversed(self): # shrink so that the dims do not collapse t = Tensor.ones(5, 6, 7).contiguous().realize().shrink(((0, 4), (0, 5), (0, 6))) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 982ab35fa9d14..fb62365aa4d89 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -5,7 +5,7 @@ from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak, pm_cast_weak from tinygrad.uop.render import pyrender -from tinygrad.uop.spec import type_verify, spec_tensor, spec_program +from tinygrad.uop.spec import type_verify, spec_tensor, spec_program, spec_program_casted_consts from tinygrad.renderer import Renderer, Estimates from tinygrad.renderer.isa import ISARenderer, IselContext, PreRegAllocContext from tinygrad.dtype import dtypes, AddrSpace @@ -281,6 +281,10 @@ def add_war_barrier(end:UOp): (UPat(Ops.END, name="end"), add_war_barrier), ]) +pm_casted_consts = PatternMatcher([ + (UPat(Ops.CONST, dtypes.all, name="c"), lambda c: UOp(Ops.CAST, c.dtype, src=(UOp.const(c.val),), arg=c.dtype)), +]) + def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: if VIZ: graph_rewrite(ast, PatternMatcher([]), name="View Base AST") if DEBUG >= 5: print(pyrender(ast)) @@ -383,8 +387,11 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: num_params = len([x for x in sink.toposort() if x.op is Ops.PARAM and x.arg.slot != -1]) sink = graph_rewrite(sink, pm_number_params, ctx=[num_params], name="number params with -1", walk=True) + # TODO: delete once migration are done + if ren.casted_consts: sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) + if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") - if SPEC: type_verify(sink, spec_program) + if SPEC: type_verify(sink, spec_program_casted_consts if ren.casted_consts else spec_program) # return the rewritten sink return sink diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 728e6148d3ddc..ec42643ade5ec 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -72,6 +72,8 @@ class Renderer: tensor_cores: list[TensorCore] = [] extra_matcher: PatternMatcher|None = None code_for_op: dict[Ops, Callable] = {} + # migration: this renderer consumes every literal as a casted const CAST(dt, CONST(value)) + casted_consts: bool = False compiler: Compiler = Compiler() diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index 0171dabd58c9d..b9b383e021db5 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -214,6 +214,7 @@ def compile(self, src:str) -> bytes: return base64.b64decode(src) class PythonRenderer(Renderer): code_for_op = python_alu compiler = PythonCompiler() + casted_consts: bool = True def __init__(self, target:Target): assert (emu:=getenv("EMULATE", "")) == "", ("EMULATE is deprecated, use DEV=PYTHON::" + diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 6718d4636e03b..2cb516c960b5e 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -81,7 +81,7 @@ def render_marg(ctx,x:UOp): Ops.RECIPROCAL, Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.CONTIGUOUS, Ops.BARRIER, Ops.DETACH} pm_pyrender_extra = PatternMatcher([ (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val}, {x.dtype})"), - (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})"), + (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})" if x.dtype != x.src[0].dtype else None), (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)}, dtype={x.dtype})"), (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: f"UOp.new_buffer({repr(x.arg.device)}, {x.max_numel()}, {x.dtype}, {x.arg.slot})" diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 35fa4bb94f202..fae9daff7ac5a 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -223,6 +223,12 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.int32),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), ])+spec_shared +# migration: on a casted_consts renderer every literal is CAST(dt, CONST(value)) with a weak inner CONST +spec_program_casted_consts = PatternMatcher([ + (UPat(Ops.CONST, dtype=dtypes.weaks, name="x"), lambda x: x.dtype is dtypes.from_py(x.val)), + (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CAST, src=(UPat(Ops.CONST),)))), lambda: True), +])+spec_program + spec_hcq = PatternMatcher([ (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),), name="x"), lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), diff --git a/tinygrad/uop/validate.py b/tinygrad/uop/validate.py index 2dfb9025ea14c..05c8cd3b0157c 100644 --- a/tinygrad/uop/validate.py +++ b/tinygrad/uop/validate.py @@ -52,6 +52,8 @@ def create_bounded(name:str, vmin:int, vmax:int, z3ctx:z3.Context) -> tuple[z3.A create_bounded(f"cast{len(ctx[1])}", x.dtype.min, x.dtype.max, ctx[0])), # A comparison between floats introduces a new bool variable (UPat(GroupOp.Comparison, src=UPat(dtype=dtypes.floats)), lambda ctx: (z3.Bool(f"float_cmp{len(ctx[1])}", ctx=ctx[0]), None)), + # a same-dtype cast states a width, which z3 does not model: identity. must precede the rules below (bool->bool) + (UPat(Ops.CAST, name="x"), lambda x,ctx: (ctx[1][x.src[0]], None) if x.dtype == x.src[0].dtype else None), # casts from bool/int to int/bool (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,),src=(UPat.var("x", dtypes.bool),)), lambda x,ctx: (z3.If(ctx[1][x], 1, 0), None)), (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,), src=(UPat.var("x", dtypes.ints+(dtypes.weakint,)),)), lambda x,ctx: (ctx[1][x], None)), From c31038ff37cb012d0cb9888e39d76e53f913aef4 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 18 Aug 2026 16:06:03 -0700 Subject: [PATCH 089/306] use KernelCountException when kernel count is being compared (#17584) --- test/backend/test_custom_kernel.py | 6 +++--- test/backend/test_multitensor.py | 4 ++-- test/backend/test_pickle.py | 3 ++- test/null/test_mnist_dataset.py | 3 ++- test/unit/test_allreduce.py | 11 ++++++----- test/unit/test_function.py | 4 ++-- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/test/backend/test_custom_kernel.py b/test/backend/test_custom_kernel.py index 1c8be94cdfb13..83aec0835950c 100644 --- a/test/backend/test_custom_kernel.py +++ b/test/backend/test_custom_kernel.py @@ -4,7 +4,7 @@ from tinygrad.dtype import AddrSpace, dtypes, Invalid from tinygrad.uop.ops import KernelInfo, AxisType, Ops from tinygrad.renderer.ptx import PTXRenderer -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, KernelCountException # **** kernels **** @@ -474,7 +474,7 @@ def _test_mop(self, mop_fxn, max_kernels): y.realize() kernel_count = GlobalCounters.kernel_count self.assertEqual(y.tolist(), x.add(1).tolist()) - self.assertLessEqual(kernel_count, max_kernels) + if kernel_count > max_kernels: raise KernelCountException(max_kernels, kernel_count) # same test with @function, input is PARAM from tinygrad import function x0 = Tensor.arange(32).clone("CPU").realize() @@ -487,7 +487,7 @@ def run(a:Tensor) -> Tensor: y = run(x0).realize() kernel_count = GlobalCounters.kernel_count self.assertEqual(y.tolist(), mop_fxn(x0).add(1).tolist()) - self.assertLessEqual(kernel_count, max_kernels) + if kernel_count > max_kernels: raise KernelCountException(max_kernels, kernel_count) def test_reshape(self): self._test_mop(lambda x: x.reshape(16, 2), max_kernels=2) def test_permute(self): self._test_mop(lambda x: x.reshape(4, 8).T, max_kernels=3) diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index 2f60958977bdb..dbe7cce6f7eaa 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -6,7 +6,7 @@ from tinygrad.engine.realize import run_linear, compile_linear, pm_beam, pm_compile import numpy as np from hypothesis import given, strategies as strat, settings -from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count +from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count, KernelCountException settings.register_profile("my_profile", max_examples=200, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False)) settings.load_profile("my_profile") @@ -395,7 +395,7 @@ def _check(self, a_ref:Tensor, a_multi:Tensor, view_fn): linear, var_vals = b_multi.linear_with_vars() if all(not d.startswith(("WEBGPU", "CL")) for d in b_multi.device): compiled = [call for call in linear.src if call.src[0].op is Ops.SINK] - self.assertEqual(len(compiled), 0, f"expected zero compiled kernels, got {len(compiled)}") + if len(compiled) != 0: raise KernelCountException(0, len(compiled)) run_linear(linear, var_vals) np.testing.assert_equal(b_multi.numpy(), b_ref.numpy()) diff --git a/test/backend/test_pickle.py b/test/backend/test_pickle.py index 7033a4dd6b125..75af07e713c3d 100644 --- a/test/backend/test_pickle.py +++ b/test/backend/test_pickle.py @@ -3,6 +3,7 @@ from tinygrad import Tensor, Device, TinyJit, Variable, dtypes from tinygrad.helpers import GlobalCounters, ContextVar, Context, DEV from tinygrad.uop.ops import PatternMatcher, UPat, UOp, deconstruct_function +from test.helpers import KernelCountException class TestPickle(unittest.TestCase): def test_pickle_code_object(self): @@ -41,7 +42,7 @@ def test_pickle_realized_tensor(self): t2:Tensor = pickle.loads(st) np.testing.assert_equal(t_values, t2.numpy()) # expect at most one COPY kernel - self.assertLessEqual(GlobalCounters.kernel_count, 1) + if GlobalCounters.kernel_count > 1: raise KernelCountException(1, GlobalCounters.kernel_count) def test_pickle_realized_tensor_alt(self): print("** init") diff --git a/test/null/test_mnist_dataset.py b/test/null/test_mnist_dataset.py index bc334d1caf19e..4f959b0be18ac 100644 --- a/test/null/test_mnist_dataset.py +++ b/test/null/test_mnist_dataset.py @@ -1,6 +1,7 @@ import unittest from tinygrad.helpers import GlobalCounters from tinygrad.nn.datasets import mnist +from test.helpers import KernelCountException class TestDataset(unittest.TestCase): def test_dataset_is_realized(self): @@ -8,7 +9,7 @@ def test_dataset_is_realized(self): X_train[0].contiguous().realize() GlobalCounters.reset() X_train[0].contiguous().realize() - self.assertLessEqual(GlobalCounters.kernel_count, 1) # 0 if SLICE (zero-copy), 1 otherwise + if GlobalCounters.kernel_count > 1: raise KernelCountException(1, GlobalCounters.kernel_count) # 0 if SLICE (zero-copy), 1 otherwise if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_allreduce.py b/test/unit/test_allreduce.py index 84dce582429d1..59663089cc04c 100644 --- a/test/unit/test_allreduce.py +++ b/test/unit/test_allreduce.py @@ -2,6 +2,7 @@ from tinygrad import Tensor, UOp, dtypes from tinygrad.helpers import Context from tinygrad.uop.ops import Ops +from test.helpers import KernelCountException class TestRingAllReduce(unittest.TestCase): def test_schedule_ring(self): @@ -13,7 +14,7 @@ def test_schedule_ring(self): copies = [si for si in linear.src if si.src[0].op is Ops.COPY] pairs = [(c.src[1].buffer.device, c.src[2].buffer.device) for c in copies] # N*(N-1) scatter reduce, and N*(N-1) allgather - self.assertEqual(len(pairs), N*(N-1)*2) + if len(pairs) != N*(N-1)*2: raise KernelCountException(N*(N-1)*2, len(pairs)) # copy topology forms a ring self.assertEqual(len(set(pairs)), N) @@ -25,8 +26,8 @@ def test_schedule_all2all(self): linear = t.sum(0).mul(2.0).contiguous().linear_with_vars()[0] copies = [si for si in linear.src if si.src[0].op is Ops.COPY] sinks = [si for si in linear.src if si.src[0].op is Ops.SINK] - self.assertEqual(len(copies), 24) - self.assertEqual(len(sinks), 26) + if len(copies) != 24: raise KernelCountException(24, len(copies)) + if len(sinks) != 26: raise KernelCountException(26, len(sinks)) @Context(RING=0, ALL2ALL=0) def test_schedule_naive(self): @@ -39,8 +40,8 @@ def test_schedule_naive(self): sinks = [si for si in linear.src if si.src[0].op is Ops.SINK] pairs = [(c.src[1].buffer.device, c.src[2].buffer.device) for c in copies] - self.assertEqual(len(pairs), N*(N-1)) - self.assertEqual(len(sinks), 2) + if len(pairs) != N*(N-1): raise KernelCountException(N*(N-1), len(pairs)) + if len(sinks) != 2: raise KernelCountException(2, len(sinks)) self.assertTrue(all(dst != src for dst, src in pairs)) def test_symbolic_shape(self): diff --git a/test/unit/test_function.py b/test/unit/test_function.py index 2bc6577e87603..865f55ca5a7d7 100644 --- a/test/unit/test_function.py +++ b/test/unit/test_function.py @@ -4,7 +4,7 @@ from tinygrad import Tensor, GlobalCounters, Device from tinygrad.dtype import Invalid from tinygrad.uop.ops import UOp, Ops, KernelInfo, ProgramInfo -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, KernelCountException class TestFunction(unittest.TestCase): def test_simple(self): @@ -516,7 +516,7 @@ def count_kernels(t:Tensor): Tensor.realize(a) c = f(a) - self.assertEqual(count_kernels(c), 1) + if count_kernels(c) != 1: raise KernelCountException(1, count_kernels(c)) c.sum().backward() Tensor.realize(a.grad) From 2cfb421a81ca11b76efad426f768b725e69570f5 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 18 Aug 2026 16:51:03 -0700 Subject: [PATCH 090/306] ci: cleanup deps (#17583) --- .github/actions/setup-tinygrad/action.yml | 20 ++++++++++++++------ .github/workflows/autogen.yml | 2 +- .github/workflows/test.yml | 14 +++----------- pyproject.toml | 4 ++++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 354bf0001ce01..5e82665da462c 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -42,7 +42,11 @@ inputs: required: false default: 'false' qemu: - description: "Install qemu" + description: "Install qemu?" + required: false + default: 'false' + ninja: + description: "Install ninja?" required: false default: 'false' runs: @@ -130,7 +134,7 @@ runs: # ******************* apt ******************* - name: Setup apt - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') shell: bash run: | sudo mkdir -p /var/cache/apt/archives @@ -158,7 +162,7 @@ runs: echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list - name: Compute Package List + Hash - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') id: apt-pkgs shell: bash run: | @@ -183,25 +187,29 @@ runs: if [[ "${{ inputs.qemu }}" == "true" ]]; then pkgs+=" qemu-user-static" fi + # **** ninja **** + if [[ "${{ inputs.ninja }}" == "true" ]]; then + pkgs+=" ninja-build" + fi echo "pkgs=$pkgs" >> "$GITHUB_OUTPUT" echo "hash=$(echo -n "$pkgs" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - name: Cache apt (PR) - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') && github.event_name == 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name == 'pull_request' uses: actions/cache/restore@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Cache apt - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') && github.event_name != 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name != 'pull_request' uses: actions/cache@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Run apt Update + Install - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') shell: bash run: | sudo apt -qq update || true diff --git a/.github/workflows/autogen.yml b/.github/workflows/autogen.yml index 73174a6cd383b..6b3fb964b79b5 100644 --- a/.github/workflows/autogen.yml +++ b/.github/workflows/autogen.yml @@ -35,7 +35,7 @@ jobs: key: 'autogen' amd: 'true' llvm: 'true' - pydeps: 'pyyaml mako' + deps: 'autogen' - name: Install autogen support packages run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev libdrm-dev liburing-dev - name: Regenerate autogen files diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84c6823c3c7a9..1e87b4d0a51d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,8 +31,7 @@ jobs: - name: Setup Environment uses: ./.github/actions/setup-tinygrad with: - deps: docs - pydeps: "capstone torch" + deps: "docs testing_minimal" - name: Build wheel and show size run: | uv build --wheel @@ -73,10 +72,7 @@ jobs: deps: testing_unit pydeps: "pillow torchvision expecttest" llvm: 'true' - - name: Install ninja - run: | - sudo apt update || true - sudo apt install -y --no-install-recommends ninja-build + ninja: 'true' - name: Test ResNet-18 run: DEBUG=2 python3 extra/torch_backend/example.py - name: Test one op in torch tests @@ -98,12 +94,8 @@ jobs: with: key: torch-backend-pillow-torchvision-et-pt deps: testing_unit - pydeps: "pillow torchvision expecttest" llvm: 'true' - - name: Install ninja - run: | - sudo apt update || true - sudo apt install -y --no-install-recommends ninja-build + ninja: 'true' - name: Test beautiful_mnist in torch with TINY_BACKEND run: STEPS=20 DEV=CPU TARGET_EVAL_ACC_PCT=90.0 MAX_BUFFER_SIZE=0 TINY_BACKEND=1 python3 examples/other_mnist/beautiful_mnist_torch.py diff --git a/pyproject.toml b/pyproject.toml index 5a7533ea8b73c..86618374bd5fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,10 @@ docs = [ "numpy", ] mesa = ["tinymesa==25.2.7.2"] +autogen = [ + "pyyaml", + "mako", +] [tool.mutmut] From e37b44d048f673be20016cde2963dbbf8b2333aa Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 18 Aug 2026 21:07:43 -0400 Subject: [PATCH 091/306] casted CONST migration for LLVM and PTX [pr] (#17585) --- tinygrad/renderer/llvmir.py | 9 +++++---- tinygrad/renderer/ptx.py | 13 +++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 3ce6fc8919b6a..1d0809e128d41 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -81,8 +81,8 @@ def _bf16(dt:DType): return dtypes.ushort if dt is dtypes.bfloat16 else dt (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.AFTER)),), allow_any_len=True, name="x"), lambda ctx,x: f" {ctx[x]} = getelementptr inbounds {ldt(x.dtype)}, {ldt(x.dtype, ptr=True)} {ctx[x.src[0]]}, {ldt(x.src[1].dtype)} {ctx[x.src[1]]}"), # register index - (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.cvar("idx")), name="x"), lambda ctx,buf,idx,x: - f" {ctx[x]} = extractelement {ldt(buf.dtype, buf.max_numel())} {ctx[buf]}, i32 {idx.val}" if buf.addrspace == AddrSpace.ALU else None), + (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.cvar("c").cast()), name="x"), lambda ctx,buf,c,x: + f" {ctx[x]} = extractelement {ldt(buf.dtype, buf.max_numel())} {ctx[buf]}, i32 {c.val}" if buf.addrspace == AddrSpace.ALU else None), # load/store (UPat(Ops.LOAD, src=(UPat.var("idx"), UPat.var("alt"), UPat.var("mask")), name="x"), @@ -146,6 +146,7 @@ def _bf16(dt:DType): return dtypes.ushort if dt is dtypes.bfloat16 else dt ]) class LLVMRenderer(Renderer): + casted_consts = True abi: str | None string_rewrite: PatternMatcher code_for_op = {k:lambda:None for v in lop.values() for k in v.keys()} @@ -165,7 +166,7 @@ def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None) -> tuple[t local_args: list[str] = [] name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op is Ops.AFTER: r[u] = r[u.src[0]] continue @@ -185,7 +186,7 @@ def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None) -> tuple[t kernel.append(f" {r[u]} = addrspacecast [{size} x {ldt(u.dtype)}] addrspace(3)* @{r[u][1:]} to [{size} x {ldt(u.dtype)}]*") else: kernel.append(f" {r[u]} = alloca [{size} x {ldt(u.dtype)}], align 16") - elif u.op is Ops.CONST: r[u] = lconst(u.val, u.dtype) + elif u.op is Ops.CAST and u.src[0].op is Ops.CONST: r[u] = lconst(u.src[0].val, u.dtype) elif u.op is Ops.CAST and ldt(u.dtype) == ldt(u.src[0].dtype): r[u] = r[u.src[0]] # cast from signed to unsigned of the same size is a noop, or pointer cast else: diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index 30c73c2f930a5..a71724615c9cf 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -79,8 +79,8 @@ def modifier(a: DType, b: DType): return '.rzi' if dtypes.is_int(a) and dtypes.i (a.itemsize < b.itemsize or dtypes.is_int(b) or b == dtypes.bool) else '' string_rewrite = PatternMatcher([ - (UPat.cvar("x", dtypes.bool), lambda ctx, x: f"setp.ne.s16 {ctx.r[x]}, {render_val(x.val, x.dtype)}, 0;"), - (UPat.cvar("x"), lambda ctx, x: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {render_val(x.val, x.dtype)};"), + (UPat.cvar("c").cast(dtypes.bool, name="x"), lambda ctx, x, c: f"setp.ne.s16 {ctx.r[x]}, {render_val(c.val, x.dtype)}, 0;"), + (UPat.cvar("c").cast(name="x"), lambda ctx, x, c: f"mov.b{ctx.types[x.dtype][1:]} {ctx.r[x]}, {render_val(c.val, x.dtype)};"), (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: f"mov.u32 %{x.arg}, %{'ctaid' if x.arg[0] == 'g' else 'tid'}.{chr(120+int(x.arg[-1]))};"), (UPat(Ops.PARAM, name="x"), lambda ctx, x: f"ld.param.{ctx.types[dtypes.ulong] if x.addrspace is AddrSpace.GLOBAL else ctx.mem_types[x.dtype]} {ctx.r[x]}, [data{x.arg.slot}+0];"), @@ -136,6 +136,7 @@ def modifier(a: DType, b: DType): return '.rzi' if dtypes.is_int(a) and dtypes.i class PTXRenderer(Renderer): suffix = "PTX" + casted_consts = True global_max, local_max, shared_max = CUDARenderer.global_max, CUDARenderer.local_max, CUDARenderer.shared_max tc_sm80 = [x for x in tc.cuda_sm80 if x.dtype_in in [dtypes.half, dtypes.float]] code_for_op = asm_for_op @@ -186,7 +187,7 @@ def ssa(prefix:str, u:UOp|None=None, dtype:str|None=None) -> str: name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op is Ops.AFTER: self.r[u] = self.r[u.src[0]] continue @@ -201,9 +202,9 @@ def ssa(prefix:str, u:UOp|None=None, dtype:str|None=None) -> str: continue if u.op in {Ops.INDEX, Ops.SHRINK, Ops.LOAD} and u.src[0].addrspace in (AddrSpace.REG, AddrSpace.ALU): # on REG, INDEX/SHRINK pick the register (must be CONST) and LOAD is a noop - if u.op is not Ops.LOAD and u.src[1].op is not Ops.CONST: + if u.op is not Ops.LOAD and not (u.src[1].op is Ops.CAST and u.src[1].src[0].op is Ops.CONST): raise RuntimeError(f"PTX does not support dynamic register indexing: {u}") - r[u] = r[u.src[0]] if u.op is Ops.LOAD else r[u.src[0]][u.src[1].val] + r[u] = r[u.src[0]] if u.op is Ops.LOAD else r[u.src[0]][u.src[1].src[0].val] continue if u.op is Ops.SPECIAL: r[u] = "%" + u.arg elif u.op is Ops.LOAD: @@ -216,7 +217,7 @@ def ssa(prefix:str, u:UOp|None=None, dtype:str|None=None) -> str: [ssa("wmma_acc", dtype="b32") for _ in range(0, len(r[u.src[2]]), 4 // u.dtype.itemsize)]] r[u] = [ssa("wmma", dtype=self.types[u.dtype]) for _ in range(u.max_numel())] prefix, dtype = {Ops.CAST: ("cast", None), Ops.BITCAST: ("cast", None), Ops.END: ("pred", "pred"), Ops.RANGE: ("ridx", None), - Ops.CONST: ("const", None), Ops.BUFFER: ("local", "u64"), Ops.INDEX: ("bidx", "u64"), Ops.SHRINK: ("bidx", "u64"), + Ops.BUFFER: ("local", "u64"), Ops.INDEX: ("bidx", "u64"), Ops.SHRINK: ("bidx", "u64"), Ops.PARAM: ("dat", "u64" if u.addrspace is AddrSpace.GLOBAL else None), **{op: ("alu", None) for op in GroupOp.ALU}}.get(u.op, (None, None)) if u.op is Ops.RANGE and u.dtype == dtypes.void: prefix = None # loop headers don't have a register if prefix: r[u] = ssa(prefix, u, dtype) From df3b114fbc6404a9437cb29605ff18e9d0912c2e Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 18 Aug 2026 18:49:07 -0700 Subject: [PATCH 092/306] ci: standardize all ubuntu runs-on to ubuntu-24.04 (#17586) --- .github/workflows/docs.yml | 2 +- .github/workflows/python-publish.yml | 2 +- .github/workflows/szdiff.yml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c32c1466b1e54..b9149d9dcafe8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ permissions: contents: write jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Configure Git Credentials diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 397467a86fb44..23aef5aca7a53 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Set up Python diff --git a/.github/workflows/szdiff.yml b/.github/workflows/szdiff.yml index 37a985f0c549d..900282151c9e8 100644 --- a/.github/workflows/szdiff.yml +++ b/.github/workflows/szdiff.yml @@ -10,7 +10,7 @@ concurrency: jobs: checkbranch: name: Check PR Branch status - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: branchstat: ${{ steps.brstat.outputs.stat}} steps: @@ -44,7 +44,7 @@ jobs: permissions: contents: read pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: checkbranch if: needs.checkbranch.outputs.branchstat == 'false' steps: @@ -87,7 +87,7 @@ jobs: name: Core Library Line Difference permissions: pull-requests: write - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: checkbranch if: needs.checkbranch.outputs.branchstat == 'true' steps: From ca86a427034ca468c933e65dfa2d2f70bed9e8d0 Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 18 Aug 2026 23:04:07 -0400 Subject: [PATCH 093/306] casted CONST migration for nir [pr] (#17588) --- tinygrad/renderer/nir.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index a2153394f66b2..24e0b616bc956 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -116,6 +116,7 @@ def reg(b, buf): class NIRRenderer(Renderer): suffix = "NIR" + casted_consts = True nir_options: bytes global_max, local_max, shared_max = CUDARenderer.global_max, CUDARenderer.local_max, CUDARenderer.shared_max code_for_op = {**{k:lambda:None for k in u_aop.keys()}, **{k:lambda:None for k in s_aop.keys()}, **{k:lambda:None for k in f_aop.keys()}} @@ -145,7 +146,7 @@ class NIRRenderer(Renderer): ]) def_rewrite = PatternMatcher([ - (UPat(Ops.CONST, name="x"), lambda ctx,x: nimm(ctx.b, x.val, x.dtype)), + (UPat.cvar("c").cast(name="x"), lambda ctx,x,c: nimm(ctx.b, c.val, x.dtype)), (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx.param(ctx.b, x, x.dtype.itemsize if x.addrspace is AddrSpace.ALU else 8)), (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: nchannel(ctx.b, {'g':ngid, 'l':nlid, 'i': nid}[x.arg[0]](ctx.b), int(x.arg[-1]))), (UPat(Ops.STORE, src=(UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"),UPat.var("off")), allow_any_len=True), UPat.var("val"))), @@ -186,16 +187,17 @@ def postrender(self, uops:list[UOp]): pass def render(self, uops:list[UOp]): self.prerender(uops) - for u in [u for u in uops if u.op is Ops.SPECIAL and u.arg[0] == "l"]: self.b.shader.contents.info.workgroup_size[int(u.arg[-1])] = u.src[0].val + for u in [u for u in uops if u.op is Ops.SPECIAL and u.arg[0] == "l"]: + self.b.shader.contents.info.workgroup_size[int(u.arg[-1])] = u.src[0].src[0].val self.r: dict[UOp, Any] = {} self.param_idx = 0 ranges: list[mesa.nir_def|None] = [] for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP} or (u.op is Ops.STACK and len(u.src) == 0): pass + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST} or (u.op is Ops.STACK and len(u.src) == 0): pass elif u.op in {Ops.INDEX, Ops.SHRINK}: # INDEX on a register value picks the element, memory INDEX is handled in the LOAD/STORE patterns - if u.src[0].op not in {Ops.PARAM, Ops.BUFFER, Ops.AFTER}: self.r[u] = nchannel(self.b, self.r[u.src[0]], u.src[1].val) + if u.src[0].op not in {Ops.PARAM, Ops.BUFFER, Ops.AFTER}: self.r[u] = nchannel(self.b, self.r[u.src[0]], u.src[1].src[0].val) elif u.op is Ops.AFTER: self.r[u] = self.r[u.src[0]] elif u.op == Ops.SINK: From 8c2bf02d17c7095835216341755e273718f0eb7f Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:07:30 +0800 Subject: [PATCH 094/306] fused_qkv_rope in UOp (#17591) * llama: 4% faster fused_qkv_rope * prep * add uop kernel, has_hipcc is cached * less --- extra/thunder/amd/fa.py | 37 +++++++++++---- extra/thunder/amd/fused_qkv_rope.cpp | 69 ---------------------------- test/backend/test_asm_gemm.py | 2 + test/backend/test_llama_kernels.py | 21 +++++---- 4 files changed, 41 insertions(+), 88 deletions(-) delete mode 100644 extra/thunder/amd/fused_qkv_rope.cpp diff --git a/extra/thunder/amd/fa.py b/extra/thunder/amd/fa.py index 685a1b8e77860..36cdfad824d81 100644 --- a/extra/thunder/amd/fa.py +++ b/extra/thunder/amd/fa.py @@ -19,16 +19,33 @@ def _sharded_empty(shape:Tensor, ref:Tensor, axis:int|None, dtype:DTypeLike|None @functools.cache def custom_fused_qkv_rope_forward(q:UOp, k:UOp, v:UOp, xqkv:UOp, freqs_cis:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): - code = (pathlib.Path(__file__).parent / "fused_qkv_rope.cpp").read_text() - threads = 256 - thread_idx = UOp.special(threads, "lidx0") - block_idx_x, block_idx_y = UOp.special(B, "gidx0"), UOp.special(N, "gidx1") - sink = UOp.sink(q.base, k.base, v.base, xqkv.base, freqs_cis.base, thread_idx, block_idx_x, block_idx_y, - arg=KernelInfo(name="fused_qkv_rope_forward")) - compile_args = ["-std=c++20", "-ffast-math", f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", - f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DTHREADS_PER_BLOCK={threads}"] - lib = HIPCCCompiler(arch, compile_args).compile_cached(code) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) + group_size = H // H_KV + q, k, v = q.reshape(B, N, H, D), k.reshape(B, N, H_KV, D), v.reshape(B, N, H_KV, D) + xqkv = xqkv.reshape(B, N, H_KV, group_size + 2, D) + b, n = UOp.range(B, 0), UOp.range(N, 1) + pair = UOp.range(D // 2, 2) + even = pair * 2 + c = freqs_cis[0, n, 0, pair, 0].cast(dtypes.float) + s = freqs_cis[0, n, 0, pair, 1].cast(dtypes.float) + ordered:UOp|None = None + for kvh in range(H_KV): + q_out, k_out, v_out = (x.after(ordered) if ordered is not None else x for x in (q, k, v)) + x_in = xqkv.after(ordered) if ordered is not None else xqkv + stores:list[UOp] = [] + for rep in range(group_size): + a = x_in[b, n, kvh, rep, even].cast(dtypes.float) + bb = x_in[b, n, kvh, rep, even + 1].cast(dtypes.float) + h = kvh * group_size + rep + stores += [q_out[b, n, h, even].store((a * c - bb * s).cast(q.dtype)), q_out[b, n, h, even + 1].store((a * s + bb * c).cast(q.dtype))] + a = x_in[b, n, kvh, group_size, even].cast(dtypes.float) + bb = x_in[b, n, kvh, group_size, even + 1].cast(dtypes.float) + stores += [k_out[b, n, kvh, even].store((a * c - bb * s).cast(k.dtype)), + k_out[b, n, kvh, even + 1].store((a * s + bb * c).cast(k.dtype)), + v_out[b, n, kvh, even].store(x_in[b, n, kvh, group_size + 1, even]), + v_out[b, n, kvh, even + 1].store(x_in[b, n, kvh, group_size + 1, even + 1])] + ordered = UOp.group(*stores) + assert ordered is not None + return ordered.end(pair, n, b).sink(arg=KernelInfo(name="fused_qkv_rope_forward")) @functools.cache def custom_fused_qkv_rope_backward(dxqkv:UOp, dq:UOp, dk:UOp, dv:UOp, freqs_cis:UOp, diff --git a/extra/thunder/amd/fused_qkv_rope.cpp b/extra/thunder/amd/fused_qkv_rope.cpp deleted file mode 100644 index fcb7ee6bf9d4d..0000000000000 --- a/extra/thunder/amd/fused_qkv_rope.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -#ifndef ATTN_B -#define ATTN_B 2 -#endif -#ifndef ATTN_N -#define ATTN_N 8192 -#endif -#ifndef ATTN_H -#define ATTN_H 32 -#endif -#ifndef ATTN_H_KV -#define ATTN_H_KV 8 -#endif -#ifndef ATTN_D -#define ATTN_D 128 -#endif -#ifndef THREADS_PER_BLOCK -#define THREADS_PER_BLOCK 256 -#endif - -constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; -constexpr int HALF_D = ATTN_D / 2; -constexpr int PACKED_D = (GROUP_SIZE + 2) * ATTN_D; - -extern "C" __global__ __launch_bounds__(THREADS_PER_BLOCK) void -fused_qkv_rope_forward( - __hip_bfloat16* __restrict__ q, - __hip_bfloat16* __restrict__ k, - __hip_bfloat16* __restrict__ v, - const __hip_bfloat16* __restrict__ xqkv, - const __hip_bfloat16* __restrict__ freqs_cis) { - const int b = blockIdx.x; - const int n = blockIdx.y; - const int bn = b * ATTN_N + n; - const int packed_bn = bn * ATTN_H_KV * PACKED_D; - const int q_bn = bn * ATTN_H * ATTN_D; - const int kv_bn = bn * ATTN_H_KV * ATTN_D; - - if (threadIdx.x < HALF_D) { - const int pair = threadIdx.x; - const int even = pair << 1; - const float c = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 0]); - const float s = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 1]); - - for (int kvh = 0; kvh < ATTN_H_KV; kvh++) { - const int base = packed_bn + kvh * PACKED_D; - - for (int rep = 0; rep < GROUP_SIZE; rep++) { - const int qbase = base + rep * ATTN_D; - const int h = kvh * GROUP_SIZE + rep; - const float a = static_cast(xqkv[qbase + even]); - const float bb = static_cast(xqkv[qbase + even + 1]); - const int out = q_bn + h * ATTN_D + even; - q[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - q[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - } - - const float a = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even]); - const float bb = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even + 1]); - const int out = kv_bn + kvh * ATTN_D + even; - k[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - k[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - v[out] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even]; - v[out + 1] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even + 1]; - } - } -} diff --git a/test/backend/test_asm_gemm.py b/test/backend/test_asm_gemm.py index e18458f808130..6e34b996c0279 100644 --- a/test/backend/test_asm_gemm.py +++ b/test/backend/test_asm_gemm.py @@ -1,4 +1,5 @@ import unittest +import functools from tinygrad import Tensor, Device, dtypes, Context from tinygrad.helpers import getenv, system, DEV from extra.gemm.cdna_asm_gemm import asm_gemm, hk_bf16_atb_gemm @@ -9,6 +10,7 @@ # Use DEV=NULL:HIP:gfx950 to also test the assembly def is_cdna4(): return Device[Device.DEFAULT].renderer.target.arch.startswith("gfx950") +@functools.cache def has_hipcc(): try: system("hipcc --version") except Exception: return False diff --git a/test/backend/test_llama_kernels.py b/test/backend/test_llama_kernels.py index f03a57b393cfd..3f9f1bcad0a8f 100644 --- a/test/backend/test_llama_kernels.py +++ b/test/backend/test_llama_kernels.py @@ -99,22 +99,20 @@ def test_multi_tensor_local_shard_amax(self): assert_kernel_count(2) self.assertEqual(out.tolist(), [[0., 7., 14., 21.], [28., 35., 42., 49.], [120., 135., 150., 165.], [180., 195., 210., 225.]]) -@unittest.skipUnless(has_hipcc() and Device.DEFAULT == "AMD", "requires hipcc to compile and amd device to run") class TestFusedQKVRoPE(unittest.TestCase): SHAPE = (2, 8192, 32, 8, 128) + def setUp(self): + if dtypes.bfloat16 not in Device[Device.DEFAULT].renderer.supported_dtypes(): self.skipTest("test uses bf16 inputs") + def rand_bf16(self, *shape:int) -> Tensor: return (Tensor.randn(*shape) * 0.1).cast(dtypes.bfloat16).contiguous().realize() - def freqs_cis(self) -> Tensor: - _, N, _, _, D = self.SHAPE - return precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() - - def test_llama31_8b_forward(self): + def _test_fused_qkv_fw(self, shape:tuple[int, int, int, int, int]): Tensor.manual_seed(0) - B, N, H, H_KV, D = self.SHAPE + B, N, H, H_KV, D = shape GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = (Tensor.randn(1, N * 2, 1, D // 2, 2) * 0.1).cast(dtypes.bfloat16).contiguous().realize() x = self.rand_bf16(B, N, H_KV * (GROUP + 2) * D) q, k, v = fused_qkv_rope(x, freqs_cis, H, H_KV, D) @@ -131,12 +129,17 @@ def test_llama31_8b_forward(self): self.assertTrue(k.allclose(k_ref, atol=2e-2, rtol=0).item(), "K forward mismatch") self.assertTrue(v.allclose(v_ref, atol=0, rtol=0).item(), "V forward mismatch") + def test_forward(self): self._test_fused_qkv_fw((1, 32, 8, 2, 16)) + + def test_llama31_8b_forward(self): self._test_fused_qkv_fw(self.SHAPE) + + @unittest.skipUnless(has_hipcc(), "backward kernel requires hipcc to compile") def test_llama31_8b_backward(self): Tensor.manual_seed(1) B, N, H, H_KV, D = self.SHAPE PARTIALS = 2 GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() dq = self.rand_bf16(B, N, H, D) dk_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) dv_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) From d539aaf75257a8c33fec84a3ef87655533be9f7d Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:42:55 +0800 Subject: [PATCH 095/306] Revert "fused_qkv_rope in UOp (#17591)" (#17593) This reverts commit 8c2bf02d17c7095835216341755e273718f0eb7f. --- extra/thunder/amd/fa.py | 37 ++++----------- extra/thunder/amd/fused_qkv_rope.cpp | 69 ++++++++++++++++++++++++++++ test/backend/test_asm_gemm.py | 2 - test/backend/test_llama_kernels.py | 21 ++++----- 4 files changed, 88 insertions(+), 41 deletions(-) create mode 100644 extra/thunder/amd/fused_qkv_rope.cpp diff --git a/extra/thunder/amd/fa.py b/extra/thunder/amd/fa.py index 36cdfad824d81..685a1b8e77860 100644 --- a/extra/thunder/amd/fa.py +++ b/extra/thunder/amd/fa.py @@ -19,33 +19,16 @@ def _sharded_empty(shape:Tensor, ref:Tensor, axis:int|None, dtype:DTypeLike|None @functools.cache def custom_fused_qkv_rope_forward(q:UOp, k:UOp, v:UOp, xqkv:UOp, freqs_cis:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): - group_size = H // H_KV - q, k, v = q.reshape(B, N, H, D), k.reshape(B, N, H_KV, D), v.reshape(B, N, H_KV, D) - xqkv = xqkv.reshape(B, N, H_KV, group_size + 2, D) - b, n = UOp.range(B, 0), UOp.range(N, 1) - pair = UOp.range(D // 2, 2) - even = pair * 2 - c = freqs_cis[0, n, 0, pair, 0].cast(dtypes.float) - s = freqs_cis[0, n, 0, pair, 1].cast(dtypes.float) - ordered:UOp|None = None - for kvh in range(H_KV): - q_out, k_out, v_out = (x.after(ordered) if ordered is not None else x for x in (q, k, v)) - x_in = xqkv.after(ordered) if ordered is not None else xqkv - stores:list[UOp] = [] - for rep in range(group_size): - a = x_in[b, n, kvh, rep, even].cast(dtypes.float) - bb = x_in[b, n, kvh, rep, even + 1].cast(dtypes.float) - h = kvh * group_size + rep - stores += [q_out[b, n, h, even].store((a * c - bb * s).cast(q.dtype)), q_out[b, n, h, even + 1].store((a * s + bb * c).cast(q.dtype))] - a = x_in[b, n, kvh, group_size, even].cast(dtypes.float) - bb = x_in[b, n, kvh, group_size, even + 1].cast(dtypes.float) - stores += [k_out[b, n, kvh, even].store((a * c - bb * s).cast(k.dtype)), - k_out[b, n, kvh, even + 1].store((a * s + bb * c).cast(k.dtype)), - v_out[b, n, kvh, even].store(x_in[b, n, kvh, group_size + 1, even]), - v_out[b, n, kvh, even + 1].store(x_in[b, n, kvh, group_size + 1, even + 1])] - ordered = UOp.group(*stores) - assert ordered is not None - return ordered.end(pair, n, b).sink(arg=KernelInfo(name="fused_qkv_rope_forward")) + code = (pathlib.Path(__file__).parent / "fused_qkv_rope.cpp").read_text() + threads = 256 + thread_idx = UOp.special(threads, "lidx0") + block_idx_x, block_idx_y = UOp.special(B, "gidx0"), UOp.special(N, "gidx1") + sink = UOp.sink(q.base, k.base, v.base, xqkv.base, freqs_cis.base, thread_idx, block_idx_x, block_idx_y, + arg=KernelInfo(name="fused_qkv_rope_forward")) + compile_args = ["-std=c++20", "-ffast-math", f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", + f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DTHREADS_PER_BLOCK={threads}"] + lib = HIPCCCompiler(arch, compile_args).compile_cached(code) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) @functools.cache def custom_fused_qkv_rope_backward(dxqkv:UOp, dq:UOp, dk:UOp, dv:UOp, freqs_cis:UOp, diff --git a/extra/thunder/amd/fused_qkv_rope.cpp b/extra/thunder/amd/fused_qkv_rope.cpp new file mode 100644 index 0000000000000..fcb7ee6bf9d4d --- /dev/null +++ b/extra/thunder/amd/fused_qkv_rope.cpp @@ -0,0 +1,69 @@ +#include +#include + +#ifndef ATTN_B +#define ATTN_B 2 +#endif +#ifndef ATTN_N +#define ATTN_N 8192 +#endif +#ifndef ATTN_H +#define ATTN_H 32 +#endif +#ifndef ATTN_H_KV +#define ATTN_H_KV 8 +#endif +#ifndef ATTN_D +#define ATTN_D 128 +#endif +#ifndef THREADS_PER_BLOCK +#define THREADS_PER_BLOCK 256 +#endif + +constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; +constexpr int HALF_D = ATTN_D / 2; +constexpr int PACKED_D = (GROUP_SIZE + 2) * ATTN_D; + +extern "C" __global__ __launch_bounds__(THREADS_PER_BLOCK) void +fused_qkv_rope_forward( + __hip_bfloat16* __restrict__ q, + __hip_bfloat16* __restrict__ k, + __hip_bfloat16* __restrict__ v, + const __hip_bfloat16* __restrict__ xqkv, + const __hip_bfloat16* __restrict__ freqs_cis) { + const int b = blockIdx.x; + const int n = blockIdx.y; + const int bn = b * ATTN_N + n; + const int packed_bn = bn * ATTN_H_KV * PACKED_D; + const int q_bn = bn * ATTN_H * ATTN_D; + const int kv_bn = bn * ATTN_H_KV * ATTN_D; + + if (threadIdx.x < HALF_D) { + const int pair = threadIdx.x; + const int even = pair << 1; + const float c = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 0]); + const float s = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 1]); + + for (int kvh = 0; kvh < ATTN_H_KV; kvh++) { + const int base = packed_bn + kvh * PACKED_D; + + for (int rep = 0; rep < GROUP_SIZE; rep++) { + const int qbase = base + rep * ATTN_D; + const int h = kvh * GROUP_SIZE + rep; + const float a = static_cast(xqkv[qbase + even]); + const float bb = static_cast(xqkv[qbase + even + 1]); + const int out = q_bn + h * ATTN_D + even; + q[out] = static_cast<__hip_bfloat16>(a * c - bb * s); + q[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); + } + + const float a = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even]); + const float bb = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even + 1]); + const int out = kv_bn + kvh * ATTN_D + even; + k[out] = static_cast<__hip_bfloat16>(a * c - bb * s); + k[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); + v[out] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even]; + v[out + 1] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even + 1]; + } + } +} diff --git a/test/backend/test_asm_gemm.py b/test/backend/test_asm_gemm.py index 6e34b996c0279..e18458f808130 100644 --- a/test/backend/test_asm_gemm.py +++ b/test/backend/test_asm_gemm.py @@ -1,5 +1,4 @@ import unittest -import functools from tinygrad import Tensor, Device, dtypes, Context from tinygrad.helpers import getenv, system, DEV from extra.gemm.cdna_asm_gemm import asm_gemm, hk_bf16_atb_gemm @@ -10,7 +9,6 @@ # Use DEV=NULL:HIP:gfx950 to also test the assembly def is_cdna4(): return Device[Device.DEFAULT].renderer.target.arch.startswith("gfx950") -@functools.cache def has_hipcc(): try: system("hipcc --version") except Exception: return False diff --git a/test/backend/test_llama_kernels.py b/test/backend/test_llama_kernels.py index 3f9f1bcad0a8f..f03a57b393cfd 100644 --- a/test/backend/test_llama_kernels.py +++ b/test/backend/test_llama_kernels.py @@ -99,20 +99,22 @@ def test_multi_tensor_local_shard_amax(self): assert_kernel_count(2) self.assertEqual(out.tolist(), [[0., 7., 14., 21.], [28., 35., 42., 49.], [120., 135., 150., 165.], [180., 195., 210., 225.]]) +@unittest.skipUnless(has_hipcc() and Device.DEFAULT == "AMD", "requires hipcc to compile and amd device to run") class TestFusedQKVRoPE(unittest.TestCase): SHAPE = (2, 8192, 32, 8, 128) - def setUp(self): - if dtypes.bfloat16 not in Device[Device.DEFAULT].renderer.supported_dtypes(): self.skipTest("test uses bf16 inputs") - def rand_bf16(self, *shape:int) -> Tensor: return (Tensor.randn(*shape) * 0.1).cast(dtypes.bfloat16).contiguous().realize() - def _test_fused_qkv_fw(self, shape:tuple[int, int, int, int, int]): + def freqs_cis(self) -> Tensor: + _, N, _, _, D = self.SHAPE + return precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() + + def test_llama31_8b_forward(self): Tensor.manual_seed(0) - B, N, H, H_KV, D = shape + B, N, H, H_KV, D = self.SHAPE GROUP = H // H_KV - freqs_cis = (Tensor.randn(1, N * 2, 1, D // 2, 2) * 0.1).cast(dtypes.bfloat16).contiguous().realize() + freqs_cis = self.freqs_cis() x = self.rand_bf16(B, N, H_KV * (GROUP + 2) * D) q, k, v = fused_qkv_rope(x, freqs_cis, H, H_KV, D) @@ -129,17 +131,12 @@ def _test_fused_qkv_fw(self, shape:tuple[int, int, int, int, int]): self.assertTrue(k.allclose(k_ref, atol=2e-2, rtol=0).item(), "K forward mismatch") self.assertTrue(v.allclose(v_ref, atol=0, rtol=0).item(), "V forward mismatch") - def test_forward(self): self._test_fused_qkv_fw((1, 32, 8, 2, 16)) - - def test_llama31_8b_forward(self): self._test_fused_qkv_fw(self.SHAPE) - - @unittest.skipUnless(has_hipcc(), "backward kernel requires hipcc to compile") def test_llama31_8b_backward(self): Tensor.manual_seed(1) B, N, H, H_KV, D = self.SHAPE PARTIALS = 2 GROUP = H // H_KV - freqs_cis = precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() + freqs_cis = self.freqs_cis() dq = self.rand_bf16(B, N, H, D) dk_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) dv_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) From 68b4407fe326379b9c39c161882d0e3c07f1dffd Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 09:01:40 -0400 Subject: [PATCH 096/306] casted CONST migration for cstyle [pr] (#17587) --- test/external/external_benchmark_schedule.py | 6 ++- test/null/test_uops.py | 3 +- tinygrad/renderer/cstyle.py | 45 ++++++++++---------- tinygrad/renderer/wgsl.py | 8 ++-- tinygrad/runtime/ops_qcom.py | 4 +- tinygrad/uop/render.py | 2 + 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/test/external/external_benchmark_schedule.py b/test/external/external_benchmark_schedule.py index 7c230c449586d..b7195cbe7cbe7 100644 --- a/test/external/external_benchmark_schedule.py +++ b/test/external/external_benchmark_schedule.py @@ -4,7 +4,7 @@ from tinygrad.uop.ops import Ops from tinygrad.codegen import full_rewrite_to_sink from tinygrad.codegen.late.linearizer import linearize -from tinygrad.uop.spec import type_verify, spec_program +from tinygrad.uop.spec import type_verify, spec_program, spec_program_casted_consts if __name__ == "__main__": mdl = ResNet50() @@ -41,5 +41,7 @@ for u in rewritten_uops: uops_line.append(linearize(u)) with Timing("***** model verify in "): - for u in uops_line: type_verify(u, spec_program) + # TODO: clean up after migration + spec = spec_program_casted_consts if Device.default.renderer.casted_consts else spec_program + for u in uops_line: type_verify(u, spec) print(sum(len(u) for u in uops_line)) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index c1d6931fd6da1..a64106506cca0 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -348,10 +348,9 @@ def test_fast_idiv_bounded_numerator_zero(self): def test_fast_idiv_remove_powers_of_two(self): ridx = UOp.range(2**20, 0) uops = to_uops_list([ridx//(7*64)], ren=Device[Device.DEFAULT].renderer) - ops = [x.op for x in uops] # this requires shifting out the powers of two before doing fast_idiv # (((ridx0>>6)*18725)>>17) instead of (int)((((long)(ridx0)*1198373)>>29)) - self.assertNotIn(Ops.CAST, ops) + self.assertNotIn(dtypes.long, [x.dtype for x in uops]) @unittest.expectedFailure def test_fast_idiv_overflow(self): diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 0014952bdf7e2..81835cc4c1f77 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -20,19 +20,19 @@ (UPat((Ops.ENDIF, Ops.END)), lambda ctx: "}"), # const - (UPat(Ops.CONST, dtype=dtypes.floats, name="x"), lambda ctx,x: None if math.isfinite(v:=x.val) else \ + (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda ctx,x,c: None if math.isfinite(v:=c.val) else \ f"({ctx.render_cast(x, ctx.nan if math.isnan(v) else ctx.infinity if v > 0 else f'-{ctx.infinity}')})"), - (UPat(Ops.CONST, dtype=dtypes.float, name="x"), lambda ctx,x: f"{x.val}f"), - (UPat(Ops.CONST, dtype=dtypes.int64, name="x"), lambda ctx,x: f"{x.val}l"), - (UPat(Ops.CONST, dtype=dtypes.uint64, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}ul"), - (UPat(Ops.CONST, dtype=dtypes.uint32, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}u"), - (UPat(Ops.CONST, dtype=dtypes.bool, name="x"), lambda ctx,x: "1" if x.val else "0"), + (UPat.cvar("c").cast(dtypes.float), lambda ctx,c: f"{c.val}f"), + (UPat.cvar("c").cast(dtypes.int64), lambda ctx,c: f"{c.val}l"), + (UPat.cvar("c").cast(dtypes.uint64, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}ul"), + (UPat.cvar("c").cast(dtypes.uint32, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}u"), + (UPat.cvar("c").cast(dtypes.bool), lambda ctx,c: "1" if c.val else "0"), # consts are rendered to larger type and casted - (UPat(Ops.CONST, (*dtypes.fp8s, dtypes.bfloat16, dtypes.half), name="x"), lambda ctx,x: f"({ctx.render_cast(x, f'{x.val}f')})"), - (UPat(Ops.CONST, (dtypes.uint8, dtypes.uint16), name="x"), lambda ctx,x: f"({ctx.render_cast(x, f'{x.val}u')})"), - (UPat(Ops.CONST, (dtypes.int8, dtypes.int16), name="x"), lambda ctx,x: f"({ctx.render_cast(x, str(x.val))})"), + (UPat.cvar("c").cast((*dtypes.fp8s, dtypes.bfloat16, dtypes.half), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, f'{c.val}f')})"), + (UPat.cvar("c").cast((dtypes.uint8, dtypes.uint16), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, f'{c.val}u')})"), + (UPat.cvar("c").cast((dtypes.int8, dtypes.int16), name="x"), lambda ctx,x,c: f"({ctx.render_cast(x, str(c.val))})"), # default const render - (UPat(Ops.CONST, name="x"), lambda ctx,x: str(x.val)), + (UPat.cvar("c").cast(), lambda ctx,c: str(c.val)), # casting (UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \ @@ -47,7 +47,7 @@ # SHRINK/INDEX (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var('idx')), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), - (UPat(Ops.SHRINK, src=(UPat.var("buf"), UPat.var('idx'), UPat.cvar()), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), + (UPat(Ops.SHRINK, src=(UPat.var("buf"), UPat.var('idx'), UPat.cvar().cast()), name="x"), lambda ctx,**kwargs: ctx.render_index(**kwargs)), (UPat(Ops.STACK, name="x"), lambda ctx,x: f"{ctx.float4.replace('float4', ctx.render_type(x))}" + \ f"{ctx.float4_style[0]}{','.join([ctx[y] for y in x.src])}{ctx.float4_style[1]}"), @@ -114,6 +114,7 @@ def wmma_args(uops:list[UOp]): for uop in uops if uop.op is Ops.WMMA) class CStyleLanguage(Renderer): + casted_consts = True abi: str = "" kernel_typedef: str = "void" buffer_prefix: str = "" @@ -161,8 +162,8 @@ def render_kernel(self, function_name:str, kernel:list[str], bufs:list[tuple[str def render_index(self, x:UOp, buf:UOp, idx:UOp): if buf.addrspace == AddrSpace.ALU: # this is lane access in C - if idx.op is not Ops.CONST: return f"({self[buf]})[{self[idx]}]" - return self[buf]+(f"[{idx.val}]" if buf.max_numel() > self.gep_arr_threshold else f".{'xyzwabcd'[idx.val]}") + if not (idx.op is Ops.CAST and idx.src[0].op is Ops.CONST): return f"({self[buf]})[{self[idx]}]" + return self[buf]+(f"[{idx.src[0].val}]" if buf.max_numel() > self.gep_arr_threshold else f".{'xyzwabcd'[idx.src[0].val]}") return f"({self[buf]}+{strip_parens(self[idx]) if idx.arg == Ops.ADD else self[idx]})" def render_buffer(self, x:UOp): @@ -208,7 +209,7 @@ def _render(self, uops:list[UOp]) -> tuple[str, list[str], list[tuple[str,tuple[ c: defaultdict[str, int] = defaultdict(int) name = "test" for u in uops: - if u.op in {Ops.NOOP, Ops.GROUP}: continue + if u.op in {Ops.NOOP, Ops.GROUP, Ops.CONST}: continue if u.op == Ops.STACK and len(u.src) == 0: continue if u.op is Ops.AFTER: r[u] = r[u.src[0]] @@ -226,7 +227,7 @@ def _render(self, uops:list[UOp]) -> tuple[str, list[str], list[tuple[str,tuple[ if u.op is Ops.SPECIAL: r[u] = u.arg elif u.op is Ops.RANGE: r[u] = f"{axis_letters[u.arg[-1]]}idx"+range_str(u) else: - prefix = {Ops.WMMA: "wmma", Ops.CONST: "const", Ops.BUFFER: "buf", Ops.CAST: "cast", Ops.BITCAST: "cast", Ops.STACK: "cast", + prefix = {Ops.WMMA: "wmma", Ops.BUFFER: "buf", Ops.CAST: "cast", Ops.BITCAST: "cast", Ops.STACK: "cast", Ops.INDEX: "bidx", Ops.LOAD: "val"}.get(u.op, "alu") r[u] = f"{prefix}{c[prefix]}" @@ -234,7 +235,8 @@ def _render(self, uops:list[UOp]) -> tuple[str, list[str], list[tuple[str,tuple[ assert l is not None, f"failed to render {u.op} {u.dtype} {[(x.op,x.dtype) for x in u.src]} {u.arg}" if u.op in {Ops.ENDIF, Ops.END}: depth -= 1 - if (u.op is not Ops.CAST or u.max_numel() == 1) and (u.op in {Ops.CONST, Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \ + if (u.op is not Ops.CAST or u.max_numel() == 1) and ((u.op is Ops.CAST and u.src[0].op is Ops.CONST) or \ + u.op in {Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \ (u.op is Ops.LOAD and u.src[0].addrspace == AddrSpace.REG and child_count[u] == 1) or \ (u.op is Ops.CAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \ (u.op in {Ops.STACK, *(GroupOp.ALU-{Ops.WHERE}), Ops.CAST, Ops.BITCAST} and child_count[u] == 1 and not getenv("EXPAND_SSA"))): @@ -318,8 +320,7 @@ class OpenCLRenderer(CStyleLanguage): string_rewrite = PatternMatcher([ (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), # bfloat16 constants need to be rendered as their bit pattern since bf16 is stored as ushort - (UPat(Ops.CONST, dtypes.bfloat16, name="x"), - lambda ctx,x: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(x.val)))[0] >> 16)}u"), + (UPat.cvar("c").cast(dtypes.bfloat16), lambda ctx,c: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(c.val)))[0] >> 16)}u"), # load/store image (OpenCL) (UPat.var('buf').index(UPat.var('idx_y'), UPat.var('idx_x')), lambda ctx,buf,idx_y,idx_x: f"IMAGE<{ctx[buf]}, {ctx[idx_y]}, {ctx[idx_x]}>"), (UPat(Ops.LOAD, dtype=dtypes.float, src=(UPat.var('buf').index(UPat.var('idx_y'), UPat.var('idx_x')), UPat.var("var"), UPat.var("gate"))), @@ -493,8 +494,8 @@ def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 = (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}," f" {fp8_index(x.src[0].dtype)}, {fp8_index(x.src[0].dtype)}, 0, 0, 0, 0)" if x.arg[0][2] == 128 else None), (UPat(Ops.WMMA, name="x"), lambda ctx,x: f"__{_wmma_name(x)}({ctx[x.src[0]]}, {ctx[x.src[1]]}, {ctx[x.src[2]]}, 0, 0, 0)"), - (UPat(Ops.CONST, dtypes.fp8s, name="x"), lambda ctx,x: - f"f32_to_fp8({ctx.nan if math.isnan(v:=x.val) else ctx.infinity if v == math.inf else f'-{ctx.infinity}' if v == -math.inf else f'{v}f'}," + (UPat.cvar("c").cast(dtypes.fp8s, name="x"), lambda ctx,x,c: + f"f32_to_fp8({ctx.nan if math.isnan(v:=c.val) else ctx.infinity if v == math.inf else f'-{ctx.infinity}' if v == -math.inf else f'{v}f'}," f" {fp8_index(x.dtype)})"), (UPat(Ops.CAST, dtypes.fp8s, (UPat(dtype=dtypes.float),), name="x",), lambda ctx,x: f"f32_to_fp8({ctx[x.src[0]]}, {fp8_index(x.dtype)})"), @@ -536,7 +537,7 @@ def render_kernel(self, function_name, kernel, bufs, uops, prefix=None) -> str: prefix, ockl = [], [] type_map = { dtypes.bfloat16: "bf16", dtypes.float: "f32", dtypes.half: "f16", dtypes.fp8e4m3: "_fp8_fp8", dtypes.fp8e5m2: "_bf8_bf8" } used_dtypes = uops_to_dtypes(uops) - if any(u.op is Ops.CONST and not math.isfinite(u.val) for u in uops): + if any(u.op is Ops.CAST and u.src[0].op is Ops.CONST and not math.isfinite(u.src[0].val) for u in uops): prefix += ["#define INFINITY (__builtin_inff())", "#define NAN (__builtin_nanf(\"\"))"] if any(u.op is Ops.SPECIAL for u in uops): prefix.append("typedef long unsigned int size_t;") @@ -550,7 +551,7 @@ def render_kernel(self, function_name, kernel, bufs, uops, prefix=None) -> str: if any(dt in dtypes.fp8s for dt, _ in used_dtypes): prefix += ["typedef unsigned char hip_bf8;", "typedef unsigned char hip_fp8;"] if any((u.op is Ops.CAST and u.dtype in dtypes.fp8s and u.src[0].dtype == dtypes.float) or - (u.op is Ops.CONST and u.dtype in dtypes.fp8s) for u in uops): + (u.op is Ops.CAST and u.src[0].op is Ops.CONST and u.dtype in dtypes.fp8s) for u in uops): prefix.append("""static inline __attribute__((device)) unsigned char f32_to_fp8(float v, int is_bf8) { v = (((*(unsigned*)&v)&0x7F800000)!=0x7F800000)?__builtin_amdgcn_fmed3f(v,is_bf8?57344.0f:448.0f,is_bf8?-57344.0f:-448.0f) : v; return (unsigned char)(is_bf8?__builtin_amdgcn_cvt_pk_bf8_f32(v,v,0,false):__builtin_amdgcn_cvt_pk_fp8_f32(v,v,0,false));\n}""") diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 83a9355e6b754..b618db4ddc4fd 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -69,10 +69,10 @@ class WGSLRenderer(CStyleLanguage): string_rewrite = PatternMatcher([ (UPat(Ops.NEG, dtypes.uints, src=(UPat.var('x'))), lambda ctx,x: f"(0-{ctx[x]})"), - (UPat.cvar("x", dtype=dtypes.bool), lambda x: "true" if x.val else "false"), - (UPat(Ops.CONST, dtype=(dtypes.uchar, dtypes.ushort, dtypes.uint32), name="x"), - lambda x: f"bitcast({x.val})" if x.val < 0 else f"{x.val&0xFFFFFFFF}u"), - (UPat(Ops.CONST, dtype=dtypes.int32, name="x"), lambda ctx,x: f"{truncate[x.dtype](x.val)}"), + (UPat.cvar("c").cast(dtypes.bool), lambda c: "true" if c.val else "false"), + (UPat.cvar("c").cast((dtypes.uchar, dtypes.ushort, dtypes.uint32)), + lambda c: f"bitcast({c.val})" if c.val < 0 else f"{c.val&0xFFFFFFFF}u"), + (UPat.cvar("c").cast(dtypes.int32, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}"), (UPat(Ops.BUFFER, name="x"), lambda ctx,x: f"var{'' if x.addrspace == AddrSpace.LOCAL else ''} {ctx[x]}: array<{ctx.buf_map(x)},{_packed_size(x)}>;"), (UPat(Ops.BITCAST, dtype=dtypes.half, name="x", src=(UPat(dtype=(dtypes.short, dtypes.ushort, dtypes.uint32),),)), diff --git a/tinygrad/runtime/ops_qcom.py b/tinygrad/runtime/ops_qcom.py index 3535207b23690..f69a14c5547c0 100644 --- a/tinygrad/runtime/ops_qcom.py +++ b/tinygrad/runtime/ops_qcom.py @@ -23,8 +23,8 @@ def dcache_flush(): buf, n = UOp.param(0, dtypes.uint8, shape=(1,)), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) i = UOp.range(n, 0, dtype=dtypes.int) flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg='__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");') - sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg='__asm__ volatile("dsb sy" ::: "memory");'), arg=KernelInfo(name="dcache_flush")) - prg = to_program(UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(sink.toposort())))), Device["CPU"].renderer) + sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg='__asm__ volatile("dsb sy" ::: "memory");'), arg=KernelInfo(name="dcache_flush"), tag=1) + prg = to_program(sink, Device["CPU"].renderer) return Device["CPU"].runtime(prg.to_elf()) #Parse C-style defines: ___SHIFT and ___MASK from the adreno module into the following format: diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 2cb516c960b5e..b005a6eaf2c3f 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -39,6 +39,8 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: (UPat(Ops.RANGE, dtypes.void, name="x"), lambda x: f"loop{x.arg[0]}"), (UPat(Ops.RANGE, name="x"), lambda x: f"r{range_str(x)}"), (UPat(Ops.CONST, name="x"), lambda x: str(x.val)), + # CAST states the width, the weak CONST carries the value + (UPat.cvar("c", dtypes.weaks+(dtypes.bool,)).cast(), lambda c: str(c.val)), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({str(x.dtype)[7:]})({ctx[x.src[0]]})"), (UPat(Ops.NEG, name="x"), lambda ctx,x: f"(-{ctx[x.src[0]]})"), (UPat(Ops.RECIPROCAL, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), From e8ba214b568c25c8c4d2cadeddf526ae4a738cbf Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 09:38:09 -0400 Subject: [PATCH 097/306] casted CONST migration for x86 [pr] (#17592) * casted CONST migration for x86 [pr] * style --- test/backend/test_isel.py | 6 +- tinygrad/codegen/__init__.py | 2 +- tinygrad/codegen/late/regalloc.py | 8 +-- tinygrad/renderer/isa/x86.py | 96 ++++++++++++++++--------------- tinygrad/uop/ops.py | 3 + 5 files changed, 62 insertions(+), 53 deletions(-) diff --git a/test/backend/test_isel.py b/test/backend/test_isel.py index ed38be7c3b38d..a3b986c619254 100644 --- a/test/backend/test_isel.py +++ b/test/backend/test_isel.py @@ -7,7 +7,7 @@ from tinygrad.renderer.isa import IselContext # INDEX on a register value with a constant index extracts a single element (the old GEP) -def lane(y:UOp, i:int) -> UOp: return y.index(UOp.const(i, dtypes.int), dtype=y.dtype) +def lane(y:UOp, i:int) -> UOp: return y.index(UOp.cconst(i, dtypes.int), dtype=y.dtype) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only x86") class TestIselX86(unittest.TestCase): @@ -46,10 +46,10 @@ def test_vinsertps(self): # complex address is [base + index*scale + displacement] def test_complex_address(self): a = UOp.variable("a", 0, 0, dtypes.int32) - load = UOp.param(0, dtypes.int32, (16,)).index(a + 1).load() + load = UOp.param(0, dtypes.int32, (16,)).index(a + UOp.cconst(1, dtypes.int32)).load() n = self.isel_rewrite(load) # displacement is the constant in "a" scaled to the buffer element size, dtype is int8 when the value fits otherwise int32 - self.assertTrue(n.src[2].op is Ops.CONST and n.src[2].dtype is dtypes.int8 and n.src[2].val == 4) + self.assertTrue(n.src[2].dtype is dtypes.int8 and n.src[2].src[0].op is Ops.CONST and n.src[2].src[0].val == 4) if __name__ == "__main__": unittest.main() diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index fb62365aa4d89..ccfae126b7e9e 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -282,7 +282,7 @@ def add_war_barrier(end:UOp): ]) pm_casted_consts = PatternMatcher([ - (UPat(Ops.CONST, dtypes.all, name="c"), lambda c: UOp(Ops.CAST, c.dtype, src=(UOp.const(c.val),), arg=c.dtype)), + (UPat(Ops.CONST, dtypes.all, name="c"), lambda c: UOp.cconst(c.val, c.dtype)), ]) def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: diff --git a/tinygrad/codegen/late/regalloc.py b/tinygrad/codegen/late/regalloc.py index 0d675e62ee019..b43c934c844aa 100644 --- a/tinygrad/codegen/late/regalloc.py +++ b/tinygrad/codegen/late/regalloc.py @@ -4,7 +4,7 @@ from tinygrad.renderer.isa import ISARenderer, Register, greg from tinygrad.dtype import dtypes -PSEUDO_OPS = {Ops.CONST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} +PSEUDO_OPS = {Ops.CONST, Ops.CAST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} class LinearScanRegallocContext: # returns the uop that defines the virtual register @@ -52,7 +52,7 @@ def fill(v:Register, i:int, cons:tuple[Register, ...]|None=None) -> Register: # the value of a BUFFER is its 64bit address, XMM registers need 16 bytes sz = 16 if v.cons[0].size == 16 else (8 if self.vdef(v).op is Ops.BUFFER else self.vdef(v).dtype.itemsize) offset = self.stack_size + (sz - self.stack_size % sz) % sz - self.spills[v] = UOp.const(offset, dtypes.int32) + self.spills[v] = UOp.cconst(offset, dtypes.int32) self.stack_size = offset + sz r = alloc(cons if cons is not None else v.cons, i) self.insert_before.setdefault(i, []).append((v, r)) @@ -84,7 +84,7 @@ def fill(v:Register, i:int, cons:tuple[Register, ...]|None=None) -> Register: # allocate stack array if u.op is Ops.BUFFER: - self.locals[u] = UOp.const(self.stack_size, dtypes.int32) + self.locals[u] = UOp.cconst(self.stack_size, dtypes.int32) self.stack_size += u.max_numel() * u.dtype.itemsize # loop prologue, avoid loading inside the loop @@ -125,7 +125,7 @@ def regalloc_rewrite(ctx:LinearScanRegallocContext, x:UOp): # alloc/dealloc stack if ctx.stack_size > 0: sp = ctx.ren.stack_pointer() - offset = UOp.const(ctx.stack_size, sp.dtype) + offset = UOp.cconst(ctx.stack_size, sp.dtype) if i == 0: before = [ctx.ren.isel_matcher.rewrite(UOp(Ops.SUB, src=(sp, offset), tag=sp.tag))] + before elif i == len(ctx.uops) - 2: before += [ctx.ren.isel_matcher.rewrite(UOp(Ops.ADD, src=(sp, offset), tag=sp.tag))] diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 85b4ece0d238a..22fb4fc12cd81 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -166,7 +166,7 @@ def scratch_buffer(elem_dt:DType, count:int, slot:int) -> UOp: def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): local = scratch_buffer(addr.src[0].dtype, x.max_numel(), next(ctx)) - local_idx = local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64) + local_idx = local.index(UOp.cconst(0, dtypes.int32), dtype=dtypes.uint64) # the selected address is a 64bit value, the AFTER orders the load after the scratch store and carries the element dtype for the encoder sel = gate.where(addr.replace(dtype=dtypes.uint64), local_idx) ptr = UOp(Ops.AFTER, addr.dtype, (sel, (local_idx if x.max_numel() == 1 else local).store(alt))) @@ -174,7 +174,7 @@ def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): def gated_store(addr:UOp, gate:UOp, val:UOp): local = scratch_buffer(addr.src[0].dtype, val.max_numel(), -1) - sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.const(0, dtypes.int32), dtype=dtypes.uint64)) + sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.cconst(0, dtypes.int32), dtype=dtypes.uint64)) return UOp(Ops.AFTER, addr.dtype, (sel,)).store(val) # legalize the new style graph for isel. NOTE: this runs after the spec is verified, some of these rewrites violate it @@ -195,7 +195,7 @@ def gated_store(addr:UOp, gate:UOp, val:UOp): # if gate in scalar int cmove is not a comparison need to add one to set the flag # NOTE: the 0 is int so the bool gate zero-extends and compares as int (a byte compare renders different kernels) (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), - lambda m,a,b: m.ne(UOp.const(0, dtypes.int)).where(a,b) if m.op not in GroupOp.Comparison else None), + lambda m,a,b: m.ne(UOp.cconst(0, dtypes.int)).where(a,b) if m.op not in GroupOp.Comparison else None), ]) # ***** X86 registers ***** @@ -221,15 +221,15 @@ def gated_store(addr:UOp, gate:UOp, val:UOp): # ***** X86 instruction selection ***** def base(x:UOp, i:int) -> UOp: return s.src[0] if (s:=x.src[i]).op is Ops.INDEX else s -def lane(x:UOp, i:int) -> int: return s.src[1].val if (s:=x.src[i]).op is Ops.INDEX else 0 +def lane(x:UOp, i:int) -> int: return s.src[1].src[0].val if (s:=x.src[i]).op is Ops.INDEX else 0 def to_int(dt:DType): return {dtypes.float16: dtypes.int16, dtypes.float32: dtypes.int32, dtypes.float64: dtypes.int64}[dt] def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, dt, arg=X86Ops.DEFINE, tag=None if reg is None else (reg,)) -def imm(dt:DType, v:int) -> UOp: return UOp.const(truncate[dt](v), dt).rtag() +def imm(dt:DType, v:int) -> UOp: return UOp.cconst(truncate[dt](v), dt).rtag() def to_imm(c:UOp) -> UOp|None: - if c.op is not Ops.CONST: return None - if c.dtype is dtypes.int64: return imm(dtypes.int32, c.val) if not c.overflows(dtypes.int32) else None - if c.dtype is dtypes.uint64: return imm(dtypes.uint32, c.val) if not c.overflows(dtypes.uint32) else None - if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, c.val) + if not (c.op is Ops.CAST and (v:=c.src[0]).op is Ops.CONST): return None + if c.dtype is dtypes.int64: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None + if c.dtype is dtypes.uint64: return imm(dtypes.uint32, v.val) if not v.overflows(dtypes.uint32) else None + if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, v.val) return None def cmp(x:UOp) -> UOp: if x.src[0].dtype is dtypes.float32: return x.ins(X86Ops.VUCOMISS, dtype=dtypes.void) @@ -289,8 +289,9 @@ def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v # buffers are indexed by element, everything else (the stack pointer) by byte scale = base.dtype.itemsize if base.op in {Ops.PARAM, Ops.BUFFER, Ops.AFTER} else 1 sz = imm(dtypes.uint8, base.dtype.itemsize) - if idx.op is Ops.ADD and idx.src[1].op is Ops.CONST: return (base, _cast(idx.src[0]), _disp(idx.src[1].val * scale), sz) - if idx.op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.val * scale), sz) + if idx.op is Ops.ADD and (c:=idx.src[1]).op is Ops.CAST and c.src[0].op is Ops.CONST: + return (base, _cast(idx.src[0]), _disp(c.src[0].val * scale), sz) + if idx.op is Ops.CAST and idx.src[0].op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.src[0].val * scale), sz) return (base, _cast(idx), _disp(0), sz) def abi(ctx:IselContext, x:UOp) -> UOp|None: @@ -353,7 +354,7 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # cast of void is a noop (UPat.var("y").cast(name="x"), lambda y,x: y if y.dtype == dtypes.void else None), # range is lowered to acc, cmp, jmp after regalloc - (UPat(Ops.RANGE, src=(UPat.cvar("c"),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(c.dtype, c.val),) + x.src[1:])), + (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),) + x.src[1:])), (UPat(Ops.RANGE, name="x"), lambda ctx,x: x.replace(tag=(ctx.vreg(WGPR),)) if not isinstance(x.tag, tuple) else None), # really all a backedge END is is an IF with a tag referencing the RANGE start label (UPat(Ops.END, src=(UPat(), UPat(), UPat(GroupOp.Comparison, name="cond")), name="x"), @@ -367,10 +368,10 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # function abi constraints (UPat((Ops.PARAM, Ops.SPECIAL), name="x"), abi), # constants that can't be immediates, move them to registers - (UPat.cvar("x", dtypes.int64s), lambda x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, x.val),)) if not x.tag else None), - (UPat.cvar("x", dtypes.ints+(dtypes.bool,)), lambda x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, x.val),)) if not x.tag else None), - (UPat.cvar("x", dtypes.floats), lambda x: - UOp.const(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, x.val))[0], dt).bitcast(x.dtype) if not x.tag else None), + (UPat.cvar("c").cast(dtypes.int64s, name="x"), lambda c,x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, c.val),)) if not x.tag else None), + (UPat.cvar("c").cast(dtypes.ints+(dtypes.bool,), name="x"), lambda c,x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, c.val),)) if not x.tag else None), + (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda c,x: + UOp.cconst(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, c.val))[0], dt).bitcast(x.dtype) if not x.tag else None), # conditional moves that use masks NOTE: these currently assume a mask producing cmp exists (UPat.var("m").where(UPat.var("a", dtypes.int8s+dtypes.int16s+dtypes.int32s+(dtypes.int64,)), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.VPBLENDVB, src=(b, a, m.replace(dtype=m.src[0].dtype))) if a.max_numel() > 1 else None), @@ -380,7 +381,7 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: a.ins(X86Ops.VBLENDVPD, src=(b, a, m.replace(dtype=m.src[0].dtype)))), # in this case we have a mask producing comparison whose user expects a bool, so we convert to bool (UPat(GroupOp.Comparison, dtypes.bool, (UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: - UOp(Ops.AND, src=(x.replace(dtype=y.dtype).bitcast(dt:=to_int(y.dtype)), UOp.const(1, dt))).f(Ops.NOOP, dtype=dtypes.bool)), + UOp(Ops.AND, src=(x.replace(dtype=y.dtype).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).f(Ops.NOOP, dtype=dtypes.bool)), # conditional moves that use flags (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.sints), UPat()), name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVL, src=(b, a, cmp(m)))), @@ -420,15 +421,15 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(Ops.STACK, dtypes.float32, name="x"), vinsertps), (UPat(Ops.STACK, dtypes.ints+(dtypes.bool,), name="x"), vpins), # INDEX on a vector register value extracts a single element - (UPat.var("y", dtypes.int8s+(dtypes.bool,)).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int8s+(dtypes.bool,)).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRB, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int16s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int16s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRW, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int32s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int32s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRD, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.int64s).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.int64s).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRQ, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), - (UPat.var("y", dtypes.floats).index(UPat.cvar("c"), name="x"), + (UPat.var("y", dtypes.floats).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPSRLDQ, src=(y, imm(dtypes.uint8, c.val * x.dtype.itemsize))) if _is_vec_xmm(y) else None), # packed bitwise ((UPat() & UPat()).named("x"), lambda x: x.ins(X86Ops.VPAND) if x.max_numel() > 1 else None), @@ -453,15 +454,19 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # scalar int binary ((UPat(dtype=dtypes.ints).alu(Ops.CDIV, UPat())).named("x"), idiv), # scalar int binary with immediate - (UPat.var("a", dtypes.ints) << UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SHLi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.uints) >> UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SHRi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.sints) >> UPat.cvar("c"), lambda a,c: a.ins(X86Ops.SARi, src=(a, imm(dtypes.uint8, c.val)))), - (UPat.var("a", dtypes.ints) + UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ADDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints) * UPat.cvar("c"), lambda a,c: a.ins(X86Ops.IMULi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ANDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.cvar("c"), lambda a,c: a.ins(X86Ops.ORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.cvar("c"), lambda a,c: a.ins(X86Ops.XORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), - (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.cvar("c"))), lambda a,c: a.ins(X86Ops.SUBi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints) << UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHLi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.uints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SHRi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.sints) >> UPat.cvar("c").cast(), lambda a,c: a.ins(X86Ops.SARi, src=(a, imm(dtypes.uint8, c.val)))), + (UPat.var("a", dtypes.ints) + UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.ADDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints) * UPat.cvar().cast(name="c"), lambda a,c: a.ins(X86Ops.IMULi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) & UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ANDi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) | UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.ORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat.var("a", dtypes.ints+(dtypes.bool,)) ^ UPat.cvar().cast(name="c"), + lambda a,c: a.ins(X86Ops.XORi, src=(a, i)) if (i:=to_imm(c)) is not None else None), + (UPat(Ops.SUB, dtypes.ints, (UPat.var("a"), UPat.cvar().cast(name="c"))), + lambda a,c: a.ins(X86Ops.SUBi, src=(a, i)) if (i:=to_imm(c)) is not None else None), # scalar int binary with register ((UPat(dtype=dtypes.ints) << UPat()).named("x"), lambda x: shift(x, X86Ops.SHL)), ((UPat(dtype=dtypes.uints) >> UPat()).named("x"), lambda x: shift(x, X86Ops.SHR)), @@ -572,7 +577,7 @@ def lower_range(ctx, x:UOp) -> tuple[UOp, list[UOp]]: if x.dtype is dtypes.void: return (label, [label]) else: acc = x.ins(X86Ops.MOVi, src=(imm(x.dtype, 0),) + x.src[1:]) - cmp = UOp(Ops.INS, arg=X86Ops.CMPi if x.src[0].op is Ops.CONST else X86Ops.CMP, src=(acc, x.src[0])) + cmp = UOp(Ops.INS, arg=X86Ops.CMPi if x.src[0].op is Ops.CAST else X86Ops.CMP, src=(acc, x.src[0])) jump_out = UOp(Ops.INS, arg=X86Ops.JGE, src=(cmp,), tag=f".LOOP_OUT_{loop_label}") ctx.loop_label[acc] = loop_label return (acc, [acc, label, cmp, jump_out]) @@ -591,7 +596,7 @@ def lower_loop(ctx, x:UOp) -> tuple[UOp, list[UOp]]: # final rewrite to match the isa spec post_regalloc_matcher = PatternMatcher([ # rewrite FRAME_INDEX to IMM now that the stack size is known - (UPat(Ops.INS, arg=X86Ops.FRAME_INDEX, name="x"), lambda ctx,x: (nx:=x.const_like(ctx.stack_size + x.tag), [nx])), + (UPat(Ops.INS, arg=X86Ops.FRAME_INDEX, name="x"), lambda ctx,x: (nx:=UOp.cconst(ctx.stack_size + x.tag, x.dtype), [nx])), # expand the cmp here so we can preserve rng src edge to get label from ctx (UPat(Ops.INS, arg=X86Ops.LOOP_CMP, name="x"), lower_loop), # rewrite RANGE to ACC = 0 -> LABEL -> JUMP if ACC >= loop bound @@ -614,7 +619,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No rm = cast(Register, greg(rm_uop)).index idx = cast(Register, greg(idx_uop)).index if idx_uop is not None and greg(idx_uop) is not None else 4 # for a memory operand the rm size is the element size from the address, otherwise it's the size of the value in the register - rm_sz = sz_uop.val if sz_uop is not None else rm_uop.dtype.itemsize + rm_sz = sz_uop.src[0].val if sz_uop is not None else rm_uop.dtype.itemsize reg_sz = reg_uop.dtype.itemsize if reg_uop is not None else 0 sz = reg_sz or rm_sz @@ -647,10 +652,10 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # 0b10 -- signals memory access with 32bit displacement # 0b11 -- signals no memory access if disp_uop is not None: - assert disp_uop.op is Ops.CONST, "displacement must be a constant" + assert disp_uop.op is Ops.CAST, "displacement must be a literal" assert disp_uop.dtype in (dtypes.int8, dtypes.int32), "displacement can only be 1 or 4 byte signed int" # rbp/r13 always require a displacement - if disp_uop.val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 + if disp_uop.src[0].val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 else: mod = 0b00 else: mod = 0b11 # x 0b0 and idx 0b100 means rsp which means no index exists @@ -664,10 +669,10 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # DISP byte if mod == 0b01 or mod == 0b10: assert disp_uop is not None - inst += struct.pack(unwrap(disp_uop.dtype.fmt), disp_uop.val) + inst += struct.pack(unwrap(disp_uop.dtype.fmt), disp_uop.src[0].val) # IMM byte if imm_uop is not None: - if imm_uop.op is Ops.CONST: inst += struct.pack(unwrap(imm_uop.dtype.fmt), imm_uop.val) + if imm_uop.op is Ops.CAST: inst += struct.pack(unwrap(imm_uop.dtype.fmt), imm_uop.src[0].val) elif isinstance(greg(imm_uop), Register): inst += bytes([(greg(imm_uop).index & 0b1111) << 4 | 0b0000]) return inst @@ -677,13 +682,13 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No if x.arg in X86GroupOp.WriteMem: if len(x.src) > 4: address, rest = x.src[:4], x.src[4:] else: address, rest = (x, None, None, None), x.src - imm_uop = rest[:1] if rest and rest[0].op is Ops.CONST else (None,) + imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(rest[0], *address, *(None, *rest[1:])) if reg is None else _encode(None, *address, *(None, *imm_uop)) if x.arg in X86GroupOp.Rm1st: if len(x.src) > 3: address, rest = x.src[:4], x.src[4:] else: address, rest = (x.src[0], None, None, None), x.src[1:] - imm_uop = rest[:1] if rest and rest[0].op is Ops.CONST else (None,) + imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(x, *address, *(None, *imm_uop)) if reg is None else _encode(None, *address, *(x if sel else None, *imm_uop)) if x.arg in X86GroupOp.Rm2nd: @@ -701,7 +706,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No encodings = { # moves X86Ops.MOVABS: lambda x: - bytes([0b0100 << 4 | 0b1 << 3 | 0b00 << 2 | greg(x).index >> 3, 0xB8 + (greg(x).index & 0b111)]) + struct.pack(x.dtype.fmt, x.src[0].val), + bytes([0b0100 << 4 | 0b1 << 3 | 0b00 << 2 | greg(x).index >> 3, 0xB8 + (greg(x).index & 0b111)]) + struct.pack(x.dtype.fmt, x.src[0].src[0].val), X86Ops.MOV: lambda x: encode(x, 0x8B), X86Ops.MOVi: lambda x: encode(x, 0xC7, reg=0), X86Ops.MOVm: lambda x: encode(x, 0x89), X86Ops.LEA: lambda x: encode(x, 0x8D), X86Ops.VMOVSS: lambda x: encode(x, 0x10, pp=2, sel=1), X86Ops.VMOVSSm: lambda x: encode(x, 0x11, pp=2, sel=1), @@ -724,8 +729,8 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No X86Ops.VCVTPS2PD: lambda x: encode(x, 0x5A, pp=0, sel=1), X86Ops.VCVTPD2PS: lambda x: encode(x, 0x5A, pp=1, sel=1), X86Ops.VCVTTPS2DQ: lambda x: encode(x, 0x5B, pp=2, sel=1), X86Ops.VCVTTPD2DQ: lambda x: encode(x, 0xE6, pp=1, sel=1), # the int src is the 2nd src (the rm field), if it was folded into a memory operand its width is the element size of the address - X86Ops.VCVTSI2SS: lambda x: encode(x, 0x2A, pp=2, sel=1, we=(x.src[4].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), - X86Ops.VCVTSI2SD: lambda x: encode(x, 0x2A, pp=3, sel=1, we=(x.src[4].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), + X86Ops.VCVTSI2SS: lambda x: encode(x, 0x2A, pp=2, sel=1, we=(x.src[4].src[0].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), + X86Ops.VCVTSI2SD: lambda x: encode(x, 0x2A, pp=3, sel=1, we=(x.src[4].src[0].val if len(x.src) > 4 else x.src[1].dtype.itemsize) == 8), X86Ops.VCVTTSS2SI: lambda x: encode(x, 0x2C, pp=2, sel=1, we=x.dtype.itemsize == 8), X86Ops.VCVTTSD2SI: lambda x: encode(x, 0x2C, pp=3, sel=1, we=x.dtype.itemsize == 8), # int division @@ -801,6 +806,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No } class X86Renderer(ISARenderer): + casted_consts = True device = "CPU" has_local = False has_threads = bool(getenv("THREADS", 1)) @@ -840,10 +846,10 @@ def asm_str(self, uops:list[UOp], function_name:str) -> str: def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg))[-1] in ('i', 'm') else o[7:]).lower():7s}" def _format_operands(x:UOp) -> str: def _format(src:tuple[UOp, ...]) -> list[str]: - return [str(s.val) if s.op is Ops.CONST else reg_strs[o].get(s.dtype.itemsize, o) if \ + return [str(s.src[0].val) if s.op is Ops.CAST else reg_strs[o].get(s.dtype.itemsize, o) if \ (o:=str(greg(s))) in reg_strs else o for s in src if greg(s) is not None] def _mem_adress(base:UOp, idx:UOp, disp:UOp, sz:UOp) -> list[str]: - return [f"[{greg(base)}" + (f" + {greg(idx)}*{sz.val}" if greg(idx) else "") + (f" + {disp.val}" if disp.val else "") + "]"] + return [f"[{greg(base)}" + (f" + {greg(idx)}*{sz.src[0].val}" if greg(idx) else "") + (f" + {d}" if (d:=disp.src[0].val) else "") + "]"] if len(x.src) > 4 and x.arg in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) elif len(x.src) > 3 and x.arg in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index f1f641fe541f7..49a7429d0a73e 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -615,6 +615,9 @@ def const(b:ConstLike, dtype:DType|None=None): # NOTE: it always has to be STACK now, even if they are all the same if isinstance(b, tuple): return UOp.stack(*[UOp.const(c, dtype) for c in b]) return UOp(Ops.CONST, dtype, arg=dtype.const(b), src=()) + # weak CONST with width on the CAST. TODO: this is the final const + @staticmethod + def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, dtype, src=(UOp.const(b),), arg=dtype) @staticmethod def range(end:sint, axis_id, axis_type=AxisType.WEAK, *arg, dtype=dtypes.weakint, src=(), **kwargs): return UOp(Ops.RANGE, src=(sint_to_uop(end, dtype),)+src, arg=(axis_id, axis_type)+arg, **kwargs) From b3e2f17b24ab84828d3884fb9ba7df41a6180b9c Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 10:26:38 -0400 Subject: [PATCH 098/306] update NULL tests that depends on strong dtype CONST (#17594) --- test/null/test_const_folding.py | 16 +--- test/null/test_graph_rewrite.py | 62 +++++++--------- test/null/test_simplify_valid_idx.py | 40 +++++----- test/null/test_uop_graph.py | 105 +++++++++------------------ 4 files changed, 84 insertions(+), 139 deletions(-) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index a4ac45e350b5c..cfc5454fbcc74 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -51,14 +51,6 @@ def test_weakfloat_math(self): def test_invalid_poison(self): self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) - def test_single_rounding_log10_backward(self): - # log10 backward folds log10(2)/log(2) = 1/log(10) in one rounding, not the double-rounded 1/float32(log(10)) - x = Tensor([1.0, 2.0, 3.0]) - ast = next(s.src[0] for s in x.log10().sum().gradient(x)[0].schedule_linear().src if s.src[0].op is Ops.SINK) - const = next(u.arg for u in full_rewrite(ast).toposort() if u.op is Ops.CONST and u.dtype is dtypes.float32) - # correctly rounded: within half a float32 ulp of the exact value (folding at float32 lands 0.66 ulp off) - self.assertLess(abs(const - 1/math.log(10)), 2**-26) - class TestBinaryOpsConstFolding(unittest.TestCase): def test_add_literal_zero(self): _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + 0) @@ -128,7 +120,7 @@ def test_scalar_bitcast(self): def t(cases: dict[DType, ConstType]): for (from_dt, from_v), (to_dt, to_v) in itertools.product(cases.items(), cases.items()): if not math.isnan(from_v): - r = full_rewrite(UOp.const(from_v, from_dt).bitcast(to_dt).sink()).src[0] + r = UOp.const(from_v, from_dt).bitcast(to_dt).simplify() self.assertEqual(r.op, Ops.CONST, msg:=f"{from_dt} -> {to_dt} ({from_v} -> {to_v})") self.assertEqual(r.dtype, to_dt, msg) np.testing.assert_equal(r.val, to_v, msg) @@ -152,9 +144,9 @@ def t(cases: dict[DType, ConstType]): def test_vec_bitcast(self): with Context(SPEC=0): - srcs = full_rewrite(UOp.const((-1, -2**31, 75), dtypes.int32).bitcast(dtypes.uint32).sink()).src - self.assertTrue(all(r.op is Ops.CONST and r.dtype == dtypes.uint32 for r in srcs)) - self.assertEqual(tuple(x.val for x in srcs), (2**32-1, 2**31, 75)) + result = full_rewrite(UOp.const((-1, -2**31, 75), dtypes.int32).bitcast(dtypes.uint32).sink()) + expected = full_rewrite(UOp.const((2**32-1, 2**31, 75), dtypes.uint32).sink()) + self.assertEqual(result.src, expected.src) # folds advance indexing into basic indexing class TestIndexingConstFolding(unittest.TestCase): diff --git a/test/null/test_graph_rewrite.py b/test/null/test_graph_rewrite.py index 06aa090359010..bd860523cbb56 100644 --- a/test/null/test_graph_rewrite.py +++ b/test/null/test_graph_rewrite.py @@ -11,19 +11,18 @@ def apply_rewrite(expr): return full_rewrite(expr.sink()).src[0] -@Context(SPEC=0) -def apply_rewrite_values(expr): - srcs = full_rewrite(expr.sink()).src - if len(srcs) == 1: - if srcs[0].op is Ops.CONST: return (srcs[0].val,) - if srcs[0].op is Ops.STACK: return tuple(s.val for s in srcs[0].src) - return tuple(s.val for s in srcs) +def const_value(uop:UOp): + if uop.op is Ops.CAST: uop = uop.src[0] + assert uop.op is Ops.CONST + return uop.val def evaluate_uop(uop, variables): if uop.op == Ops.CONST: return uop.val elif uop.op == Ops.PARAM and uop.arg.addrspace is AddrSpace.ALU: return variables[uop.expr] + elif uop.op is Ops.CAST: + return uop.dtype.const(evaluate_uop(uop.src[0], variables)) elif uop.op in GroupOp.ALU: src_values = [evaluate_uop(src, variables) for src in uop.src] return exec_alu(uop.op, uop.dtype, src_values) @@ -33,31 +32,27 @@ def evaluate_uop(uop, variables): class TestArithmeticSimplifications(unittest.TestCase): def test_full_graph_rewrite_division_by_zero(self): optimized_div_uop = apply_rewrite(UOp.const(10.0) / UOp.const(0.0)) - self.assertEqual(optimized_div_uop.op, Ops.CONST) - self.assertTrue(math.isinf(optimized_div_uop.val) or math.isnan(optimized_div_uop.val)) + value = const_value(optimized_div_uop) + self.assertTrue(math.isinf(value) or math.isnan(value)) def test_full_graph_rewrite_redundant_operations(self): optimized_uop = apply_rewrite((UOp.const(10.0) + UOp.const(0.0)) * UOp.const(1.0)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 10.0) + self.assertIs(optimized_uop, apply_rewrite(UOp.const(10.0))) def test_full_graph_rewrite_large_graph(self): prev_uop = UOp.const(0) for i in range(1, 101): prev_uop += UOp.const(i) optimized_uop = apply_rewrite(prev_uop) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, sum(range(1, 101))) + self.assertIs(optimized_uop, apply_rewrite(UOp.const(sum(range(1, 101))))) def test_full_graph_rewrite_division_by_one(self): optimized_uop = apply_rewrite(UOp.const(42.0) / UOp.const(1.0)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 42.0) + self.assertIs(optimized_uop, apply_rewrite(UOp.const(42.0))) def test_full_graph_rewrite_modulo_by_one(self): optimized_uop = apply_rewrite(UOp.const(42) % UOp.const(1)) - self.assertEqual(optimized_uop.op, Ops.CONST) - self.assertEqual(optimized_uop.val, 0) + self.assertIs(optimized_uop, apply_rewrite(UOp.const(0, dtypes.int))) class TestFoldingAndReduction(unittest.TestCase): @@ -109,27 +104,22 @@ def test_full_graph_rewrite_modulo_folding_with_define_var(self): # index dtype because div-mod rules only work on index x_var_uop = UOp.variable('x', 0, 100).cast(dtypes.weakint) optimized_mod_uop = apply_rewrite(((x_var_uop * 4) + 2) % 4) - self.assertEqual(optimized_mod_uop.op, Ops.CONST) - self.assertEqual(optimized_mod_uop.val, 2) + self.assertIs(optimized_mod_uop, apply_rewrite(UOp.const(2, dtypes.int))) def test_full_graph_rewrite_division_folding_with_define_var(self): # index dtype because div-mod rules only work on index n_var_uop = UOp.variable('n', 1, 1000).cast(dtypes.weakint) optimized_div_uop = apply_rewrite((n_var_uop * 6) // 3) - self.assertEqual(optimized_div_uop.op, Ops.MUL) - self.assertEqual(optimized_div_uop.src[1].val, 2) + self.assertIs(optimized_div_uop, apply_rewrite(n_var_uop * 2)) def test_full_graph_rewrite_complex_mod_div_folding(self): # index dtype because div-mod rules only work on index k_var_uop = UOp.variable('k', 0, 50).cast(dtypes.weakint) optimized_div_uop = apply_rewrite(((k_var_uop * 12 + 8) % 6) // 2) - self.assertEqual(optimized_div_uop.op, Ops.CONST) - self.assertEqual(optimized_div_uop.val, 1) + self.assertIs(optimized_div_uop, apply_rewrite(UOp.const(1, dtypes.int))) def test_graph_rewrite_div_folding_bug(self): - lhs = UOp(Ops.ADD, src=( - UOp(Ops.STACK, arg=None, src=(UOp(Ops.SPECIAL, src=(UOp.const(32),), arg='lidx0'),)*4), - UOp.const((0, 256, 512, 768)))) + lhs = UOp.stack(*(UOp.special(32, 'lidx0'),)*4) + UOp.const((0, 256, 512, 768)) rhs = UOp.const((2,)*4) unopt = lhs 0, - f"Expected +inf for reciprocal(0.0), got {optimized_recip_zero.val}") + log2_neg, recip_zero = const_value(optimized_log2_neg), const_value(optimized_recip_zero) + self.assertTrue(math.isnan(log2_neg), f"Expected NaN for log2(-1.0), got {log2_neg}") + self.assertTrue(math.isinf(recip_zero) and recip_zero > 0, f"Expected +inf for reciprocal(0.0), got {recip_zero}") @unittest.skip("broken") def test_full_graph_rewrite_modulo_negative_dividend(self): @@ -183,28 +173,30 @@ class TestGEPAndVectorizeRewrite(unittest.TestCase): def test_gep_single_element_extraction(self): # GEP on a vector dtype to extract a single element base_vector = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(apply_rewrite(base_vector.index(2)).val, 3.0) + self.assertIs(apply_rewrite(base_vector.index(2)), apply_rewrite(base_vector.src[2])) def test_gep_tuple_extraction(self): # GEP on a vector dtype to extract multiple elements as a vector base_vector = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(list(apply_rewrite_values(UOp.stack(*[base_vector.index(i) for i in (2, 3)]))), [3.0, 4.0]) + self.assertIs(apply_rewrite(UOp.stack(*[base_vector.index(i) for i in (2, 3)])), + apply_rewrite(UOp.stack(base_vector.src[2], base_vector.src[3]))) def test_gep_on_const_stack(self): # GEP on a const STACK to extract a single element const_stack = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertEqual(apply_rewrite(const_stack.index(2)).val, 3.0) + self.assertIs(apply_rewrite(const_stack.index(2)), apply_rewrite(const_stack.src[2])) def test_gep_tuple_on_const_stack(self): # GEP on a const STACK using a tuple to extract multiple elements const_stack = UOp.const((7.0, 8.0, 9.0, 10.0)) - self.assertEqual(list(apply_rewrite_values(UOp.stack(*[const_stack.index(i) for i in (1, 3)]))), [8.0, 10.0]) + self.assertIs(apply_rewrite(UOp.stack(*[const_stack.index(i) for i in (1, 3)])), + apply_rewrite(UOp.stack(const_stack.src[1], const_stack.src[3]))) def test_vectorize_multiple_elements(self): # Vectorizing multiple elements using GEP base_vector = UOp.const((5.0, 10.0, 15.0, 20.0)) - vectorized_uop = UOp(Ops.STACK, src=tuple(base_vector.index(i) for i in range(4))) - self.assertEqual(list(apply_rewrite_values(vectorized_uop)), [5.0, 10.0, 15.0, 20.0]) + vectorized_uop = UOp.stack(*(base_vector.index(i) for i in range(4))) + self.assertIs(apply_rewrite(vectorized_uop), apply_rewrite(base_vector)) import inspect diff --git a/test/null/test_simplify_valid_idx.py b/test/null/test_simplify_valid_idx.py index a5dc1d83d2908..2d278eb75fed7 100644 --- a/test/null/test_simplify_valid_idx.py +++ b/test/null/test_simplify_valid_idx.py @@ -15,16 +15,12 @@ def simplify_valid_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move def simplify_image_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move_where_on_load+indexing_simplify, name="simplify_image_idx") def get_gated_load_uop(valid:UOp, idx:UOp): - return UOp(Ops.LOAD, src=( - UOp.param(0, dtypes.float, (1024,)).index(idx.valid(valid)), - )) + return UOp.param(0, dtypes.float, (1024,)).index(idx.valid(valid)).load() def get_load_image_uop(image_shape:tuple[int, ...], valid:UOp, idx:tuple[UOp, UOp]): - return UOp(Ops.LOAD, src=( - UOp.param(0, dtypes.float, image_shape).index(idx[1].valid(valid), idx[0].valid(valid)), - )) + return UOp.param(0, dtypes.float, image_shape).index(idx[1].valid(valid), idx[0].valid(valid)).load() -def Special(expr, nmax): return UOp(Ops.SPECIAL, src=(UOp.const(nmax),), arg=expr) +def Special(expr, nmax): return UOp.special(nmax, expr) def Variable(expr, nmin, nmax): return UOp.variable(expr, nmin, nmax, param=True) def Range(n, nmax): return UOp.range(nmax, n) @@ -512,7 +508,7 @@ def test_drop_true_gate_on_index(self): buf = UOp.param(0, dtypes.int, (1,)) idx = UOp.const(0) true_gate = UOp.const(True) - index_with_gate = UOp(Ops.INDEX, src=(buf, idx.valid(true_gate))) + index_with_gate = buf.index(idx.valid(true_gate)) # apply the optimization result = graph_rewrite(index_with_gate, sym+indexing_simplify) # the True valid should be dropped (INDEX should only have 2 sources) @@ -524,13 +520,17 @@ def get_ranges(self, sink): result = full_rewrite(sink) return [u for u in result.toposort() if u.op is Ops.RANGE] + def assert_range_end(self, ranges:list[UOp], end:int): + self.assertEqual(len(ranges), 1) + with Context(NOOPT=1, SPEC=0): expected = full_rewrite(UOp.const(end, dtypes.int).sink()).src[0] + self.assertIs(ranges[0].src[0], expected) + def test_range_shrink_single_guard(self): # range 0..203 guarded by r < 4 everywhere -> shrink to 0..3 r = Range(0, 204) load = get_gated_load_uop(r < UOp.const(4), r) ranges = self.get_ranges(load.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + self.assert_range_end(ranges, 4) def test_range_shrink_picks_max_guard(self): # two loads guard the same range with r < 4 and r < 8 -> shrink to max(4, 8) = 8 @@ -538,25 +538,22 @@ def test_range_shrink_picks_max_guard(self): load1 = get_gated_load_uop(r < UOp.const(4), r) load2 = get_gated_load_uop(r < UOp.const(8), r) ranges = self.get_ranges(UOp.sink(load1, load2)) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 8) + self.assert_range_end(ranges, 8) def test_range_no_shrink_guard_ge_max(self): # guard r < 300 with range max 204 -> no shrink (guard doesn't constrain) r = Range(0, 204) load = get_gated_load_uop(r < UOp.const(300), r) ranges = self.get_ranges(load.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_no_shrink_when_unguarded_elsewhere(self): # one load guards r < 4, but another load uses r without a gate -> no shrink r = Range(0, 204) load1 = get_gated_load_uop(r < UOp.const(4), r) - load2 = UOp(Ops.LOAD, src=(UOp.param(1, dtypes.float, (204,)).index(r),)) + load2 = UOp.param(1, dtypes.float, (204,)).index(r).load() ranges = self.get_ranges(UOp.sink(load1, load2)) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_no_shrink_when_used_in_reduce(self): # range used in both a gated load AND directly in the reduce expression -> no shrink @@ -564,8 +561,7 @@ def test_range_no_shrink_when_used_in_reduce(self): gated_load = get_gated_load_uop(r < UOp.const(4), r) red = (r.cast(dtypes.float) + gated_load).reduce(r, arg=Ops.ADD) ranges = self.get_ranges(red.sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 204) + self.assert_range_end(ranges, 204) def test_range_shrink_to_single_iteration(self): # guard r < 1 shrinks range to 1 -> single iteration, range eliminated entirely @@ -580,8 +576,7 @@ def test_range_shrink_store_where_invalid(self): r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r < 4).where(x, Invalid)).sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + self.assert_range_end(ranges, 4) def test_range_shrink_store_where_invalid_flipped(self): # above, but flipped @@ -589,8 +584,7 @@ def test_range_shrink_store_where_invalid_flipped(self): r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r >= 4).where(Invalid, x)).sink()) - self.assertEqual(len(ranges), 1) - self.assertEqual(ranges[0].src[0].val, 4) + self.assert_range_end(ranges, 4) if __name__ == '__main__': unittest.main() diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 6e77bcec12ebe..f6f0c54009618 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -1,10 +1,10 @@ import unittest, pytest from tinygrad import dtypes, Variable, Device from tinygrad.dtype import AddrSpace -from tinygrad.helpers import DEBUG, Context +from tinygrad.helpers import DEBUG from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, GroupOp, AxisType, broadcast_axes, KernelInfo from tinygrad.uop.symbolic import sym -from test.helpers import to_uops_list +from test.helpers import full_rewrite, to_uops_list from tinygrad.codegen import full_rewrite_to_sink simple_pm = PatternMatcher([ @@ -46,11 +46,10 @@ def xfail_broken_const_wraparound(fn): return unittest.expectedFailure(fn) class TestModularWraparound(unittest.TestCase): def _test(self, uop:UOp, expected:int): - results = to_uops_list([uop]) - self.assertEqual(len(results), 2) # +1 for SINK - self.assertEqual(results[0].op, Ops.CONST) - self.assertEqual(results[0].dtype, uop.dtype) - self.assertEqual(results[0].val, expected) + result = uop.simplify() + self.assertEqual(result.op, Ops.CONST) + self.assertEqual(result.dtype, uop.dtype) + self.assertEqual(result.val, expected) @xfail_broken_const_wraparound def test_cast(self): @@ -195,11 +194,7 @@ def test_add_constant_fold(self): c1 = UOp.const(1.0, dtypes.float) c2 = UOp.const(2.0, dtypes.float) out = c1+c2 - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 3.0) + self.assertIs(out.simplify(), UOp.const(3.0, dtypes.float)) def test_where_same_fold(self): v = UOp.variable('tmp', 0, 1) @@ -207,47 +202,24 @@ def test_where_same_fold(self): vc = v != c0 c1 = UOp.const(1.0, dtypes.float) out = vc.where(c1, c1) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 1.0) + self.assertIs(out.simplify(), c1) def test_where_const_fold(self): bf = UOp.const(False) c1 = UOp.const(1.0, dtypes.float) c2 = UOp.const(2.0, dtypes.float) out = bf.where(c1, c2) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 2.0) + self.assertIs(out.simplify(), c2) def test_const_cast(self): bf = UOp.const(False) out = bf.cast(dtypes.int) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 0) + self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) def test_const_bitcast(self): bf = UOp.const(1.0, dtypes.float) out = bf.bitcast(dtypes.uint32) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK - out = uops[-2] - self.assertEqual(out.op, Ops.CONST) - self.assertEqual(out.val, 0x3F800000) - - @unittest.expectedFailure - def test_const_shape_change_bitcast(self): - bf = UOp.const(0x3F).cast(dtypes.uint8) - out = bf.bitcast(dtypes.half) - uops = to_uops_list([out]) - self.assertEqual(len(uops), 2) # +1 for SINK + self.assertIs(out.simplify(), UOp.const(0x3F800000, dtypes.uint32)) def test_devectorize_derives_lane_dtype(self): from tinygrad.codegen import do_devectorize @@ -262,10 +234,10 @@ def test_noop_vectorize_fold(self): d0 = UOp.param(0, dtypes.float, (1,)) idx = UOp.const(0) ld = d0.load(idx, dtype=dtypes.float) - vec = UOp(Ops.STACK, dtypes.float, (ld,)) + vec = UOp.stack(ld) x = vec.index(0) - alu = UOp(Ops.SQRT, src=(x, )) - out = UOp(Ops.STORE, src=(d0, idx, alu)) + alu = x.sqrt() + out = d0.index(idx).store(alu) uops = to_uops_list([out]) self.assertEqual(len([x for x in uops if x.op is Ops.STACK]), 0) @@ -276,13 +248,13 @@ def test_gep_vec_fold(self): d2 = UOp.param(2, dtypes.float, (1,)) idx = UOp.const(0) def _test_vec(geps, count=4): - vec = UOp(Ops.STACK, dtypes.float, geps) + vec = UOp.stack(*geps) out = d0.index(idx).store(vec) - uops = to_uops_list([out]) + rewritten = full_rewrite(out.sink()) if DEBUG >= 4: from tinygrad import Device - print(Device[Device.DEFAULT].renderer.render(uops)) - return uops[-2].src[-1] # -2 to skip SINK + print(Device[Device.DEFAULT].renderer.render(rewritten.toposort())) + return rewritten.src[0].src[1] # possible val = d1.index(idx).load(dtype=dtypes.float) @@ -312,11 +284,8 @@ def _test_vec(geps, count=4): def test_gep_vec_const_fold(self): for vec_size in [2, 4, 8]: consts = [UOp.const(float(i), dtypes.float) for i in range(vec_size)] - vec = UOp(Ops.STACK, src=tuple(consts)) - with Context(SPEC=0): - uops = to_uops_list([vec.index(i) for i in range(vec_size)]) - for uop, const in zip(uops, consts): - self.assertEqual(uop, const) + vec = UOp.stack(*consts) + for i, const in enumerate(consts): self.assertIs(vec.index(i), const) def test_cast_alu_fold(self): d0 = UOp.param(0, dtypes.bool, (1,)) @@ -326,7 +295,7 @@ def test_cast_alu_fold(self): alu = (ld<1).cast(dtypes.bool) out = d0.index(idx).store(alu) uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.CAST]), 0) + self.assertEqual(len([x for x in uops if x.op is Ops.CAST and x.src[0].op is not Ops.CONST]), 0) def test_double_cast_fold(self): d0 = UOp.param(0, dtypes.float, (1,)) @@ -336,7 +305,7 @@ def test_double_cast_fold(self): alu = ld.cast(dtypes.float).cast(dtypes.float) out = d0.index(idx).store(alu) uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.CAST]), 1) + self.assertEqual(len([x for x in uops if x.op is Ops.CAST and x.src[0].op is not Ops.CONST]), 1) def test_depth_2_const_fold(self): v = UOp.variable("tmp", 0, 1, dtypes.int, param=True) @@ -344,12 +313,7 @@ def test_depth_2_const_fold(self): c4 = UOp.const(4, dtypes.int) vc = v+c2 out = vc+c4 - uops = to_uops_list([out]) - self.assertEqual(len(uops), 5) # +1 for SINK, +1 for the PARAM shape STACK - out = uops[-2] # -2 to skip SINK - self.assertEqual(out.op, Ops.ADD) - self.assertEqual(out.src[1].op, Ops.CONST) - self.assertEqual(out.src[1].val, 6) + self.assertIs(out.simplify(), (v+UOp.const(6, dtypes.int)).simplify()) def test_bitcast_to_same_dtype_fold(self): for dt in dtypes.ints + dtypes.floats + (dtypes.bool,): @@ -360,9 +324,8 @@ def test_bitcast_to_same_dtype_fold(self): def test_sub_with_cast_folds(self): a = Variable("a", 0, 5) - uops = to_uops_list([a.cast(dtypes.int)+(-a).cast(dtypes.int)]) - assert uops[0] == UOp.const(0, dtypes.int) - assert uops[-1].op == Ops.SINK + out = a.cast(dtypes.int)+(-a).cast(dtypes.int) + self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) def test_where_on_gated_load_fold(self): ridx0 = UOp.range(100, 0) @@ -371,9 +334,10 @@ def test_where_on_gated_load_fold(self): w = (ridx0<50).where(ld, 5) out = UOp.param(1, dtypes.long, (100,)) uops = to_uops_list([out.index(ridx0).store(w)]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: assert u.src[1].val==5 + if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: self.assertIs(u.src[1], expected) def test_where_on_gated_load_folds_swapped_branches(self): ridx0 = UOp.range(100, 0) @@ -381,9 +345,10 @@ def test_where_on_gated_load_folds_swapped_branches(self): ld = d0.index(ridx0.valid((ridx0<50).logical_not())) w = (ridx0<50).where(5, ld) uops = to_uops_list([w]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD: assert u.src[1].val==5 + if u.op is Ops.LOAD: self.assertIs(u.src[1], expected) def test_where_on_gated_load_with_cast(self): ridx0 = UOp.range(100, 0) @@ -393,9 +358,10 @@ def test_where_on_gated_load_with_cast(self): w = (ridx0<50).where(ld, 5.0) out = UOp.param(1, dtypes.float, (100,)) uops = to_uops_list([out.index(ridx0).store(w)]) + expected = full_rewrite(UOp.const(5, dtypes.int).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: assert u.src[1].val == 5 + if u.op is Ops.LOAD and u.src[0].src[0].op is Ops.PARAM: self.assertIs(u.src[1], expected) def test_where_on_casted_gated_load_extra_cond(self): ridx0 = UOp.range(100, 0) @@ -425,9 +391,10 @@ def test_where_in_store_becomes_gate(self): val = (ridx0<50).where(5, ld) st = idx.store(val).end(ridx0) uops = to_uops_list([st]) + expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: assert u.op is not Ops.WHERE - if u.op is Ops.STORE: assert u.src[1].val==5 + if u.op is Ops.STORE: self.assertIs(u.src[1], expected) def test_load_idx_becomes_int(self): # mnist indexing with split reduceop @@ -506,7 +473,7 @@ def test_asserts_bad_gate(self): glbl0 = UOp.param(0, dtypes.int, (1,)) idx = UOp.const(0) bad_gate = UOp.const(1) - with self.assertRaises(AssertionError): to_uops_list([UOp(Ops.STORE, src=(glbl0, idx, UOp.const(42), bad_gate))]) + with self.assertRaises(AssertionError): to_uops_list([glbl0.index(idx).store(UOp.const(42), bad_gate)]) def test_after_end(self): r = UOp.range(10, 0) @@ -575,7 +542,7 @@ def test_const_bufferize_with_ranges(self): from tinygrad.schedule.rangeify import pm_const_buffer_folding, BufferizeOpts c = UOp.const(42.0) r1 = UOp.range(3, 0) - bufferize_with_range = UOp(Ops.STAGE, src=(c, r1), arg=BufferizeOpts(device="CPU")) + bufferize_with_range = c.bufferize(r1, arg=BufferizeOpts(device="CPU")) self.assertEqual(len(bufferize_with_range.src), 2) # const + 1 range result = graph_rewrite(bufferize_with_range, pm_const_buffer_folding, name='test') @@ -590,7 +557,7 @@ def test_const_bufferize_with_multiple_ranges(self): c = UOp.const(3.14) r1 = UOp.range(3, 0) r2 = UOp.range(4, 1) - bufferize_with_ranges = UOp(Ops.STAGE, src=(c, r1, r2), arg=BufferizeOpts(device="CPU")) + bufferize_with_ranges = c.bufferize(r1, r2, arg=BufferizeOpts(device="CPU")) self.assertEqual(len(bufferize_with_ranges.src), 3) # const + 2 ranges result = graph_rewrite(bufferize_with_ranges, pm_const_buffer_folding, name='test') From 95503787045ea303406724e1061b49e89b2f957a Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 10:43:55 -0400 Subject: [PATCH 099/306] finish casted_consts migration [PR] (#17595) --- test/backend/test_linearizer.py | 7 +++---- test/external/external_benchmark_schedule.py | 6 ++---- test/null/test_uops.py | 5 ----- tinygrad/codegen/__init__.py | 9 +++++---- tinygrad/renderer/__init__.py | 2 -- tinygrad/renderer/cstyle.py | 1 - tinygrad/renderer/isa/x86.py | 1 - tinygrad/renderer/llvmir.py | 1 - tinygrad/renderer/nir.py | 1 - tinygrad/renderer/ptx.py | 1 - tinygrad/runtime/ops_python.py | 1 - tinygrad/uop/spec.py | 10 +++------- 12 files changed, 13 insertions(+), 32 deletions(-) diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index ab63e732b5e3f..b21dc7ace6cbd 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -261,7 +261,6 @@ def test_simple_unroll_no_between_phi_dependencies(self): assert end_range < uops.index(u) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") - @unittest.skipIf(Device[Device.DEFAULT].renderer.casted_consts, "reads a literal, which is casted here. TODO: flip this") def test_default_global_reversed(self): # shrink so that the dims do not collapse t = Tensor.ones(5, 6, 7).contiguous().realize().shrink(((0, 4), (0, 5), (0, 6))) @@ -269,9 +268,9 @@ def test_default_global_reversed(self): uops = tuple(to_program(replace_opts(ast, []), renderer=Device[Device.DEFAULT].renderer).src[1].src) idxs = dedup([uop for uop in uops if uop.op is Ops.SPECIAL]) idxs = sorted(idxs, key=lambda uop: uop.arg) - assert (idxs[0].arg, idxs[0].src[0].val) == ('gidx0', 6), idxs[0] - assert (idxs[1].arg, idxs[1].src[0].val) == ('gidx1', 5), idxs[1].arg - assert (idxs[2].arg, idxs[2].src[0].val) == ('gidx2', 4), idxs[2].arg + assert (idxs[0].arg, idxs[0].src[0].src[0].val) == ('gidx0', 6), idxs[0] + assert (idxs[1].arg, idxs[1].src[0].src[0].val) == ('gidx1', 5), idxs[1].arg + assert (idxs[2].arg, idxs[2].src[0].src[0].val) == ('gidx2', 4), idxs[2].arg def test_sum_collapse(self): t = Tensor([2]).reshape(1, 1).expand(256, 256).sum() diff --git a/test/external/external_benchmark_schedule.py b/test/external/external_benchmark_schedule.py index b7195cbe7cbe7..7c230c449586d 100644 --- a/test/external/external_benchmark_schedule.py +++ b/test/external/external_benchmark_schedule.py @@ -4,7 +4,7 @@ from tinygrad.uop.ops import Ops from tinygrad.codegen import full_rewrite_to_sink from tinygrad.codegen.late.linearizer import linearize -from tinygrad.uop.spec import type_verify, spec_program, spec_program_casted_consts +from tinygrad.uop.spec import type_verify, spec_program if __name__ == "__main__": mdl = ResNet50() @@ -41,7 +41,5 @@ for u in rewritten_uops: uops_line.append(linearize(u)) with Timing("***** model verify in "): - # TODO: clean up after migration - spec = spec_program_casted_consts if Device.default.renderer.casted_consts else spec_program - for u in uops_line: type_verify(u, spec) + for u in uops_line: type_verify(u, spec_program) print(sum(len(u) for u in uops_line)) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index a64106506cca0..40115c3e448e4 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -41,11 +41,6 @@ def test_const_default_dtype_is_derived(self): # an explicit (strong) const dtype is legal until the field is removed self.assertEqual(UOp.const(3, dtypes.int32).dtype, dtypes.int32) - def test_weak_dtype_rejected_by_program_spec(self): - for weak, concrete, value in ((dtypes.weakint, dtypes.int32, 1), (dtypes.weakfloat, dtypes.float32, 1.0)): - with self.assertRaises(RuntimeError): type_verify(UOp.const(value, weak).sink(), spec_program) - type_verify(UOp.const(value, concrete).sink(), spec_program) - def test_invalid_stated_dtype(self): # UOp.const normalizes a stated dtype away (const_like/full pass their position's); the core constructor does not, # and the spec is what rejects a non-bool Invalid diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index ccfae126b7e9e..076c0db87a6f3 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -5,7 +5,7 @@ from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak, pm_cast_weak from tinygrad.uop.render import pyrender -from tinygrad.uop.spec import type_verify, spec_tensor, spec_program, spec_program_casted_consts +from tinygrad.uop.spec import type_verify, spec_tensor, spec_program from tinygrad.renderer import Renderer, Estimates from tinygrad.renderer.isa import ISARenderer, IselContext, PreRegAllocContext from tinygrad.dtype import dtypes, AddrSpace @@ -387,11 +387,12 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: num_params = len([x for x in sink.toposort() if x.op is Ops.PARAM and x.arg.slot != -1]) sink = graph_rewrite(sink, pm_number_params, ctx=[num_params], name="number params with -1", walk=True) - # TODO: delete once migration are done - if ren.casted_consts: sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) + # spell every literal as a casted const CAST(dt, CONST(value)) + # TODO: remove once consts are always weak + sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") - if SPEC: type_verify(sink, spec_program_casted_consts if ren.casted_consts else spec_program) + if SPEC: type_verify(sink, spec_program) # return the rewritten sink return sink diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index ec42643ade5ec..728e6148d3ddc 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -72,8 +72,6 @@ class Renderer: tensor_cores: list[TensorCore] = [] extra_matcher: PatternMatcher|None = None code_for_op: dict[Ops, Callable] = {} - # migration: this renderer consumes every literal as a casted const CAST(dt, CONST(value)) - casted_consts: bool = False compiler: Compiler = Compiler() diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 81835cc4c1f77..4bc96020371a7 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -114,7 +114,6 @@ def wmma_args(uops:list[UOp]): for uop in uops if uop.op is Ops.WMMA) class CStyleLanguage(Renderer): - casted_consts = True abi: str = "" kernel_typedef: str = "void" buffer_prefix: str = "" diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 22fb4fc12cd81..3dd4ff6442847 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -806,7 +806,6 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No } class X86Renderer(ISARenderer): - casted_consts = True device = "CPU" has_local = False has_threads = bool(getenv("THREADS", 1)) diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 1d0809e128d41..a6efae940f33e 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -146,7 +146,6 @@ def _bf16(dt:DType): return dtypes.ushort if dt is dtypes.bfloat16 else dt ]) class LLVMRenderer(Renderer): - casted_consts = True abi: str | None string_rewrite: PatternMatcher code_for_op = {k:lambda:None for v in lop.values() for k in v.keys()} diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index 24e0b616bc956..3b254ef2389c1 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -116,7 +116,6 @@ def reg(b, buf): class NIRRenderer(Renderer): suffix = "NIR" - casted_consts = True nir_options: bytes global_max, local_max, shared_max = CUDARenderer.global_max, CUDARenderer.local_max, CUDARenderer.shared_max code_for_op = {**{k:lambda:None for k in u_aop.keys()}, **{k:lambda:None for k in s_aop.keys()}, **{k:lambda:None for k in f_aop.keys()}} diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index a71724615c9cf..d46a68b3a3b6a 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -136,7 +136,6 @@ def modifier(a: DType, b: DType): return '.rzi' if dtypes.is_int(a) and dtypes.i class PTXRenderer(Renderer): suffix = "PTX" - casted_consts = True global_max, local_max, shared_max = CUDARenderer.global_max, CUDARenderer.local_max, CUDARenderer.shared_max tc_sm80 = [x for x in tc.cuda_sm80 if x.dtype_in in [dtypes.half, dtypes.float]] code_for_op = asm_for_op diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index b9b383e021db5..0171dabd58c9d 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -214,7 +214,6 @@ def compile(self, src:str) -> bytes: return base64.b64decode(src) class PythonRenderer(Renderer): code_for_op = python_alu compiler = PythonCompiler() - casted_consts: bool = True def __init__(self, target:Target): assert (emu:=getenv("EMULATE", "")) == "", ("EMULATE is deprecated, use DEV=PYTHON::" + diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index fae9daff7ac5a..95fba930e5bbb 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -200,11 +200,13 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # these ops can exist in programs but not the tensor spec. example: LOAD spec_program = PatternMatcher([ + # a literal is CAST(dt, CONST(value)), so its inner CONST is the one weak node a program may contain + (UPat(Ops.CONST, dtype=dtypes.weaks, name="x"), lambda x: x.dtype is dtypes.from_py(x.val)), # index and weak dtypes are not allowed in programs (UPat(GroupOp.All, (dtypes.weakint, dtypes.weakfloat)), lambda: False), # allow special SHRINK - (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CONST))), lambda: True), + (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CONST).or_casted())), lambda: True), # movement ops are not allowed in programs (UPat(GroupOp.Movement), lambda: False), @@ -223,12 +225,6 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.int32),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), ])+spec_shared -# migration: on a casted_consts renderer every literal is CAST(dt, CONST(value)) with a weak inner CONST -spec_program_casted_consts = PatternMatcher([ - (UPat(Ops.CONST, dtype=dtypes.weaks, name="x"), lambda x: x.dtype is dtypes.from_py(x.val)), - (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CAST, src=(UPat(Ops.CONST),)))), lambda: True), -])+spec_program - spec_hcq = PatternMatcher([ (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),), name="x"), lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), From bd6e70ac1573ec2b397cf5777582f1dd74a75322 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 11:18:05 -0400 Subject: [PATCH 100/306] delete stale tests (#17596) --- test/null/test_const_folding.py | 110 +--------------------- test/null/test_graph_rewrite.py | 159 +------------------------------- test/null/test_uop_graph.py | 78 ---------------- 3 files changed, 2 insertions(+), 345 deletions(-) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index cfc5454fbcc74..7724863b38a2c 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -1,39 +1,10 @@ import unittest, itertools, math -from tinygrad import Tensor, dtypes, Context +from tinygrad import dtypes, Context from tinygrad.dtype import DType, ConstType from tinygrad.uop.ops import Ops, UOp from test.helpers import full_rewrite import numpy as np -def _check_ast_count(desired_count:int, t:Tensor): - # NOTE: this has side effect because everything can be scheduled only once - linear = t.schedule_linear() - asts = [s for s in linear.src if s.src[0].op is Ops.SINK] - len(asts) - # NOT SUPPORTED ANYMORE - #assert len(asts) == desired_count, f"{len(asts)} != {desired_count}" - -class TestUnaryOpsConstFolding(unittest.TestCase): - def test_all_consts_ops(self): - _check_ast_count(0, Tensor.ones(4).exp()) - _check_ast_count(0, Tensor.ones(4).sqrt()) - _check_ast_count(0, Tensor.ones(4) + Tensor.ones(4)) - _check_ast_count(0, Tensor.ones(4) / Tensor.ones(4)) - - def test_cast(self): - _check_ast_count(0, Tensor.ones(4).cast(dtypes.int16)) - _check_ast_count(0, Tensor.full(4, fill_value=-1).cast(dtypes.uint16)) - - def test_neg_folding(self): - _check_ast_count(0, Tensor([1, 2, 3]).mul(-1).neg()) - _check_ast_count(0, Tensor([1, 2, 3]).neg().mul(-1)) - _check_ast_count(0, Tensor([1, 2, 3]).neg().neg()) - - def test_neg_realized_no_fold(self): - x = Tensor.randn(32, 32) - x = x.clip(0, 1).realize() - _check_ast_count(1, x.neg()) - class TestWeakConstFolding(unittest.TestCase): def test_weakint_math(self): out = (UOp.const(2**40) + UOp.const(2**40)).simplify() @@ -51,70 +22,6 @@ def test_weakfloat_math(self): def test_invalid_poison(self): self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) -class TestBinaryOpsConstFolding(unittest.TestCase): - def test_add_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + 0) - def test_add_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + Tensor.zeros(4)) - def test_literal_zero_add(self): - _check_ast_count(0, 0 + Tensor([1.0, 2, 3, 4])) - def test_tensor_zero_add(self): - _check_ast_count(0, Tensor.zeros(4) + Tensor([1.0, 2, 3, 4])) - - def test_sub_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - 0) - def test_sub_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - Tensor.zeros(4)) - - def test_mul_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 0) - def test_mul_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.zeros(4)) - def test_literal_zero_mul(self): - _check_ast_count(0, 0 * Tensor([1.0, 2, 3, 4]) * 0) - def test_tensor_zero_mul(self): - _check_ast_count(0, Tensor.zeros(4) * Tensor([1.0, 2, 3, 4])) - - def test_mul_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 1) - def test_mul_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.ones(4)) - def test_literal_one_mul(self): - _check_ast_count(0, 1 * Tensor([1.0, 2, 3, 4])) - def test_tensor_one_mul(self): - _check_ast_count(0, Tensor.ones(4) * Tensor([1.0, 2, 3, 4])) - - def test_bool_tensor_mul_bool(self): - _check_ast_count(0, Tensor([True, False]) * True) - _check_ast_count(0, Tensor([True, False]) * False) - def test_bool_mul_bool_tensor(self): - _check_ast_count(0, True * Tensor([True, False])) - _check_ast_count(0, False * Tensor([True, False])) - - def test_div_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / 1) - def test_div_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / Tensor.ones(4)) - - def test_floordiv_literal_one(self): - _check_ast_count(0, Tensor([1, 2, 3, 4]) // 1) - def test_floordiv_tensor_one(self): - _check_ast_count(0, Tensor([1, 2, 3, 4]) // Tensor.ones(4, dtype=dtypes.int32)) - - def test_pow_literal_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 0) - def test_pow_tensor_zero(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.zeros(4)) - - def test_pow_literal_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 1) - def test_pow_tensor_one(self): - _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.ones(4)) - def test_literal_one_pow(self): - _check_ast_count(0, 1 ** Tensor([1.0, 2, 3, 4])) - def test_tensor_one_pow(self): - _check_ast_count(0, Tensor.ones(4) ** Tensor([1.0, 2, 3, 4])) - class TestBitcastConstFolding(unittest.TestCase): def test_scalar_bitcast(self): def t(cases: dict[DType, ConstType]): @@ -148,20 +55,5 @@ def test_vec_bitcast(self): expected = full_rewrite(UOp.const((2**32-1, 2**31, 75), dtypes.uint32).sink()) self.assertEqual(result.src, expected.src) -# folds advance indexing into basic indexing -class TestIndexingConstFolding(unittest.TestCase): - def test_scalar_index(self): - t = Tensor.arange(16).float().reshape(1,1,4,4).clone().realize() - _check_ast_count(1, t[:,:,Tensor(1),:]) - _check_ast_count(1, t[:,:,Tensor(1)+2,:]) - _check_ast_count(1, t[:,:,Tensor(1),Tensor(0)]) - - def test_const_tensor_index(self): - # TODO: these can be 0, implement const tensor folded indexing - t = Tensor.arange(16).float().reshape(1,1,4,4).clone().realize() - _check_ast_count(1, t[:,:,Tensor.ones(2,1,dtype=dtypes.int),:]) - _check_ast_count(1, t[:,:,Tensor.ones(1,2,dtype=dtypes.int)+2,:]) - _check_ast_count(1, t[:,:,Tensor.ones(1,1,dtype=dtypes.int),Tensor.zeros(2,1,2,dtype=dtypes.int)]) - if __name__ == '__main__': unittest.main() diff --git a/test/null/test_graph_rewrite.py b/test/null/test_graph_rewrite.py index bd860523cbb56..a1f8745174f0c 100644 --- a/test/null/test_graph_rewrite.py +++ b/test/null/test_graph_rewrite.py @@ -1,8 +1,7 @@ import unittest, math from tinygrad import dtypes -from tinygrad.dtype import AddrSpace from tinygrad.helpers import all_same, Context -from tinygrad.uop.ops import GroupOp, UOp, Ops, exec_alu, PatternMatcher, TrackedPatternMatcher, UPat +from tinygrad.uop.ops import GroupOp, UOp, Ops, PatternMatcher, TrackedPatternMatcher, UPat from test.helpers import full_rewrite from hypothesis import given, strategies as strat @@ -16,108 +15,7 @@ def const_value(uop:UOp): assert uop.op is Ops.CONST return uop.val -def evaluate_uop(uop, variables): - if uop.op == Ops.CONST: - return uop.val - elif uop.op == Ops.PARAM and uop.arg.addrspace is AddrSpace.ALU: - return variables[uop.expr] - elif uop.op is Ops.CAST: - return uop.dtype.const(evaluate_uop(uop.src[0], variables)) - elif uop.op in GroupOp.ALU: - src_values = [evaluate_uop(src, variables) for src in uop.src] - return exec_alu(uop.op, uop.dtype, src_values) - else: - raise NotImplementedError(f"Unsupported UOp {uop.op}") - -class TestArithmeticSimplifications(unittest.TestCase): - def test_full_graph_rewrite_division_by_zero(self): - optimized_div_uop = apply_rewrite(UOp.const(10.0) / UOp.const(0.0)) - value = const_value(optimized_div_uop) - self.assertTrue(math.isinf(value) or math.isnan(value)) - - def test_full_graph_rewrite_redundant_operations(self): - optimized_uop = apply_rewrite((UOp.const(10.0) + UOp.const(0.0)) * UOp.const(1.0)) - self.assertIs(optimized_uop, apply_rewrite(UOp.const(10.0))) - - def test_full_graph_rewrite_large_graph(self): - prev_uop = UOp.const(0) - for i in range(1, 101): - prev_uop += UOp.const(i) - optimized_uop = apply_rewrite(prev_uop) - self.assertIs(optimized_uop, apply_rewrite(UOp.const(sum(range(1, 101))))) - - def test_full_graph_rewrite_division_by_one(self): - optimized_uop = apply_rewrite(UOp.const(42.0) / UOp.const(1.0)) - self.assertIs(optimized_uop, apply_rewrite(UOp.const(42.0))) - - def test_full_graph_rewrite_modulo_by_one(self): - optimized_uop = apply_rewrite(UOp.const(42) % UOp.const(1)) - self.assertIs(optimized_uop, apply_rewrite(UOp.const(0, dtypes.int))) - - -class TestFoldingAndReduction(unittest.TestCase): - @unittest.skip("reduce is removed now") - def test_full_graph_rewrite_constant_reduction_folding(self): - const1 = UOp.const(5) - const2 = UOp.const(10) - const3 = UOp.const(20) - optimized_sink = apply_rewrite((const1 + const2 + const3).reduce(Ops.ADD)) - expected_sum = 5 + 10 + 20 - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("reduce is removed now") - def test_full_graph_rewrite_reduction_with_unused_range(self): - const1 = UOp.const(15) - const2 = UOp.const(25) - rng = UOp.range(10, idx=0) - optimized_sink = apply_rewrite((const1 + const2).reduce(Ops.ADD, rng)) - expected_sum = 10 * (15 + 25) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_range_reduction(self): - simple_range = UOp.range(5, idx=0) - optimized_sink = apply_rewrite(simple_range.reduce(Ops.ADD, simple_range)) - expected_sum = sum(range(5)) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_simple_reduction_folding(self): - simple_range = UOp.range(4, idx=0) - add_uop = simple_range + UOp.const(1) - optimized_sink = apply_rewrite(add_uop.reduce(Ops.ADD, simple_range)) - expected_sum = sum(i + 1 for i in range(4)) - self.assertEqual(optimized_sink.val, expected_sum) - - @unittest.skip("currently failing") - def test_full_graph_rewrite_nested_loop_collapse(self): - outer_range = UOp.range(8, 0) - inner_range = UOp.range(4, 1) - expr = (outer_range * 10) + inner_range - optimized_reduce_uop = apply_rewrite(expr.reduce(Ops.ADD, outer_range, inner_range)) - self.assertEqual(optimized_reduce_uop.op, Ops.CONST) - self.assertEqual(optimized_reduce_uop.val, sum((i * 10) + j for i in range(8) for j in range(4))) - - class TestModuloAndDivisionFolding(unittest.TestCase): - def test_full_graph_rewrite_modulo_folding_with_define_var(self): - # index dtype because div-mod rules only work on index - x_var_uop = UOp.variable('x', 0, 100).cast(dtypes.weakint) - optimized_mod_uop = apply_rewrite(((x_var_uop * 4) + 2) % 4) - self.assertIs(optimized_mod_uop, apply_rewrite(UOp.const(2, dtypes.int))) - - def test_full_graph_rewrite_division_folding_with_define_var(self): - # index dtype because div-mod rules only work on index - n_var_uop = UOp.variable('n', 1, 1000).cast(dtypes.weakint) - optimized_div_uop = apply_rewrite((n_var_uop * 6) // 3) - self.assertIs(optimized_div_uop, apply_rewrite(n_var_uop * 2)) - - def test_full_graph_rewrite_complex_mod_div_folding(self): - # index dtype because div-mod rules only work on index - k_var_uop = UOp.variable('k', 0, 50).cast(dtypes.weakint) - optimized_div_uop = apply_rewrite(((k_var_uop * 12 + 8) % 6) // 2) - self.assertIs(optimized_div_uop, apply_rewrite(UOp.const(1, dtypes.int))) - def test_graph_rewrite_div_folding_bug(self): lhs = UOp.stack(*(UOp.special(32, 'lidx0'),)*4) + UOp.const((0, 256, 512, 768)) rhs = UOp.const((2,)*4) @@ -127,26 +25,6 @@ def test_graph_rewrite_div_folding_bug(self): print(opt) if opt.op is Ops.STACK: self.assertFalse(all_same(opt.src)) - def test_full_graph_rewrite_modulo_large_divisor(self): - # index dtype because div-mod rules only work on index - x_var_uop = UOp.variable('x', 1, 5) - self.assertIs(apply_rewrite(x_var_uop.cast(dtypes.weakint) % 10).render(simplify=False), x_var_uop.render(simplify=False)) - - def test_full_graph_rewrite_division_with_remainder(self): - x_var_uop = UOp.variable('x', 7, 9, param=True) - optimized_sink = apply_rewrite(x_var_uop // 2) - for x_value in range(7, 10): - self.assertEqual(x_value // 2, evaluate_uop(optimized_sink, {'x': x_value})) - - def test_full_graph_rewrite_complex_mod_div_expression(self): - x_var_uop = UOp.variable('x', 1, 10, param=True) - optimized_sink = apply_rewrite(((x_var_uop * 5) % 3) // 2) - for x_value in range(1, 11): - original_result = ((x_value * 5) % 3) // 2 - optimized_result = evaluate_uop(optimized_sink, {'x': x_value}) - self.assertEqual(original_result, optimized_result) - - class TestEdgeCasesAndSpecialOperations(unittest.TestCase): def test_full_graph_rewrite_transcendental_edge_cases(self): optimized_sink = full_rewrite(UOp.const(-1.0).log2().sink(UOp.const(0.0).reciprocal())) @@ -155,20 +33,6 @@ def test_full_graph_rewrite_transcendental_edge_cases(self): self.assertTrue(math.isnan(log2_neg), f"Expected NaN for log2(-1.0), got {log2_neg}") self.assertTrue(math.isinf(recip_zero) and recip_zero > 0, f"Expected +inf for reciprocal(0.0), got {recip_zero}") - @unittest.skip("broken") - def test_full_graph_rewrite_modulo_negative_dividend(self): - x_var_uop = UOp.variable('x', -5, -1) - optimized_sink = full_rewrite((x_var_uop % 3).sink()) - for x_value in range(-5, 0): - self.assertEqual(x_value % 3, evaluate_uop(optimized_sink.src[0], {'x': x_value})) - - @unittest.skip("broken") - def test_full_graph_rewrite_division_negative_divisor(self): - x_var_uop = UOp.variable('x', 1, 5) - optimized_sink = full_rewrite((x_var_uop // -2).sink()) - for x_value in range(1, 6): - self.assertEqual(x_value // -2, evaluate_uop(optimized_sink.src[0], {'x': x_value})) - class TestGEPAndVectorizeRewrite(unittest.TestCase): def test_gep_single_element_extraction(self): # GEP on a vector dtype to extract a single element @@ -181,17 +45,6 @@ def test_gep_tuple_extraction(self): self.assertIs(apply_rewrite(UOp.stack(*[base_vector.index(i) for i in (2, 3)])), apply_rewrite(UOp.stack(base_vector.src[2], base_vector.src[3]))) - def test_gep_on_const_stack(self): - # GEP on a const STACK to extract a single element - const_stack = UOp.const((1.0, 2.0, 3.0, 4.0)) - self.assertIs(apply_rewrite(const_stack.index(2)), apply_rewrite(const_stack.src[2])) - - def test_gep_tuple_on_const_stack(self): - # GEP on a const STACK using a tuple to extract multiple elements - const_stack = UOp.const((7.0, 8.0, 9.0, 10.0)) - self.assertIs(apply_rewrite(UOp.stack(*[const_stack.index(i) for i in (1, 3)])), - apply_rewrite(UOp.stack(const_stack.src[1], const_stack.src[3]))) - def test_vectorize_multiple_elements(self): # Vectorizing multiple elements using GEP base_vector = UOp.const((5.0, 10.0, 15.0, 20.0)) @@ -248,16 +101,6 @@ def test_sin(self): ret = substitute(ret, {a.sin():b}) self.assertIs(ret, b.sin()) - # broken due to infinite recursion - # NOTE: VIZ hangs and doesn't recover if you click this one - @unittest.skip("recursion error no longer raised") - def test_assert_inf_recurse(self): - a = UOp.variable('a', 0, 10) - n1 = a.sin() - ret = n1 - with self.assertRaises(RecursionError): - ret = substitute(ret, {n1:n1.sqrt()}) - def test_sin_to_sqrt(self): a = UOp.variable('a', 0, 10, dtype=dtypes.float) n1 = a.sin() diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index f6f0c54009618..180ee23189bb3 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -1,7 +1,6 @@ import unittest, pytest from tinygrad import dtypes, Variable, Device from tinygrad.dtype import AddrSpace -from tinygrad.helpers import DEBUG from tinygrad.uop.ops import Ops, UOp, UPat, PatternMatcher, graph_rewrite, GroupOp, AxisType, broadcast_axes, KernelInfo from tinygrad.uop.symbolic import sym from test.helpers import full_rewrite, to_uops_list @@ -34,13 +33,6 @@ def test_add_const(self): self.assertEqual(ret.op, Ops.STACK) self.assertEqual(const_values(ret), (5,7,9)) - def test_add_const_lose_v(self): - v1 = UOp.const((0,1,2)) - v2 = UOp.const((2,1,0)) - ret = graph_rewrite(v1+v2, sym) - self.assertEqual(ret.op, Ops.STACK) - self.assertEqual(const_values(ret), (2,2,2)) - def xfail_broken_const_wraparound(fn): fn = pytest.mark.xfail(reason="const folding does not properly implement modular arithmetic")(fn) return unittest.expectedFailure(fn) @@ -190,12 +182,6 @@ def test_consts_go_last(self): self.assertEqual(len([x for x in sink.toposort() if x.op is Ops.CONST]), 1) class TestUOpGraph(unittest.TestCase): - def test_add_constant_fold(self): - c1 = UOp.const(1.0, dtypes.float) - c2 = UOp.const(2.0, dtypes.float) - out = c1+c2 - self.assertIs(out.simplify(), UOp.const(3.0, dtypes.float)) - def test_where_same_fold(self): v = UOp.variable('tmp', 0, 1) c0 = UOp.const(0) @@ -216,11 +202,6 @@ def test_const_cast(self): out = bf.cast(dtypes.int) self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) - def test_const_bitcast(self): - bf = UOp.const(1.0, dtypes.float) - out = bf.bitcast(dtypes.uint32) - self.assertIs(out.simplify(), UOp.const(0x3F800000, dtypes.uint32)) - def test_devectorize_derives_lane_dtype(self): from tinygrad.codegen import do_devectorize # an Invalid lane derives bool while the value lane derives float: the lane rebuild must derive, not inherit @@ -229,58 +210,6 @@ def test_devectorize_derives_lane_dtype(self): invalid_lane_mul = next(u for u in out.src[0].toposort() if u.op is Ops.MUL) self.assertIs(invalid_lane_mul.dtype, dtypes.bool) - @unittest.skip("this test isn't valid uops") - def test_noop_vectorize_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - idx = UOp.const(0) - ld = d0.load(idx, dtype=dtypes.float) - vec = UOp.stack(ld) - x = vec.index(0) - alu = x.sqrt() - out = d0.index(idx).store(alu) - uops = to_uops_list([out]) - self.assertEqual(len([x for x in uops if x.op is Ops.STACK]), 0) - - @unittest.skip("this test isn't valid uops") - def test_gep_vec_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - d1 = UOp.param(1, dtypes.float, (1,)) - d2 = UOp.param(2, dtypes.float, (1,)) - idx = UOp.const(0) - def _test_vec(geps, count=4): - vec = UOp.stack(*geps) - out = d0.index(idx).store(vec) - rewritten = full_rewrite(out.sink()) - if DEBUG >= 4: - from tinygrad import Device - print(Device[Device.DEFAULT].renderer.render(rewritten.toposort())) - return rewritten.src[0].src[1] - - # possible - val = d1.index(idx).load(dtype=dtypes.float) - xyzw = tuple(val.index(i) for i in range(4)) - self.assertIs(_test_vec(xyzw).op, Ops.LOAD) - - # unaligned - val = d1.index(idx).load(dtype=dtypes.float) - wzyx = tuple(val.index(i) for i in reversed(range(4))) - self.assertIs(_test_vec(wzyx).op, Ops.STACK) - - # different_size - val = d1.index(idx).load(dtype=dtypes.float) - xy = tuple(val.index(i) for i in range(2)) - self.assertIs(_test_vec(xy+xy).op, Ops.STACK) - val = d1.index(idx).load(dtype=dtypes.float) - xy = tuple(val.index(i) for i in range(2)) - self.assertIs(_test_vec(xy, count=2).op, Ops.STACK) - - # different vals - val1 = d1.index(idx).load(dtype=dtypes.float) - val2 = d2.index(idx).load(dtype=dtypes.float) - xy1 = tuple(val1.index(i) for i in range(2)) - xy2 = tuple(val2.index(i) for i in range(2)) - self.assertIs(_test_vec(xy1+xy2).op, Ops.STACK) - def test_gep_vec_const_fold(self): for vec_size in [2, 4, 8]: consts = [UOp.const(float(i), dtypes.float) for i in range(vec_size)] @@ -468,13 +397,6 @@ def test_fold_gated_store(self): # only the second store happens self.assertEqual(len([u for u in uops if u.op is Ops.STORE]), 1) - @unittest.skip("this is a uop type error") - def test_asserts_bad_gate(self): - glbl0 = UOp.param(0, dtypes.int, (1,)) - idx = UOp.const(0) - bad_gate = UOp.const(1) - with self.assertRaises(AssertionError): to_uops_list([glbl0.index(idx).store(UOp.const(42), bad_gate)]) - def test_after_end(self): r = UOp.range(10, 0) From c218b4842d26aaa90b2f50696c0d80f0e41f507d Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 14:39:36 -0400 Subject: [PATCH 101/306] fold_bitcast should truncate its input [pr] (#17601) --- test/null/test_const_folding.py | 6 ++++++ tinygrad/uop/symbolic.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index 7724863b38a2c..e173114a57197 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -23,6 +23,12 @@ def test_invalid_poison(self): self.assertTrue(UOp.invalid().alu(Ops.CDIV, UOp.const(0)).simplify().is_invalid) class TestBitcastConstFolding(unittest.TestCase): + def test_out_of_range_source_value(self): + for val, src_dt, dst_dt, bits in ((3000000000, dtypes.int32, dtypes.uint32, 3000000000), + (70000, dtypes.int16, dtypes.uint16, 4464), + (-5, dtypes.uint32, dtypes.int32, -5)): + self.assertEqual(UOp.const(val, src_dt).bitcast(dst_dt).simplify().val, bits) + def test_scalar_bitcast(self): def t(cases: dict[DType, ConstType]): for (from_dt, from_v), (to_dt, to_v) in itertools.product(cases.items(), cases.items()): diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 3f4950d14db1b..4e9e3b4191497 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -2,7 +2,7 @@ import math from collections import defaultdict from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu -from tinygrad.dtype import PyConst, ConstType, dtypes, can_lossless_cast, Invalid, bitcast +from tinygrad.dtype import PyConst, ConstType, dtypes, can_lossless_cast, Invalid, bitcast, truncate from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup @@ -22,7 +22,7 @@ def simplify_pow(x:UOp, c:UOp) -> UOp|None: def fold_bitcast(root:UOp, c:UOp) -> UOp|None: if c.dtype.fmt is None or root.dtype.fmt is None or c.dtype.itemsize != root.dtype.itemsize: return None - return root.const_like(bitcast(c.val, c.dtype, root.dtype)) + return root.const_like(bitcast(truncate[c.dtype](c.val), c.dtype, root.dtype)) # const folding works for CONST, STACK, and casted CONST const_folding_pat = UPat.any(UPat((Ops.CONST, Ops.STACK)), UPat(Ops.CAST, src=(UPat(Ops.CONST),))) From a4fadcf606289c37f92b19225588fd8b932fa2fe Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 14:53:17 -0400 Subject: [PATCH 102/306] fix TestDevCopySpeeds SIZE (#17602) SIZE should be int --- test/external/external_test_usb_asm24.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/external/external_test_usb_asm24.py b/test/external/external_test_usb_asm24.py index 7aec4adacbf38..fe1314d34e85e 100644 --- a/test/external/external_test_usb_asm24.py +++ b/test/external/external_test_usb_asm24.py @@ -6,7 +6,7 @@ class TestDevCopySpeeds(unittest.TestCase): @classmethod def setUpClass(cls): - cls.sz = getenv("SIZE", 2e6) + cls.sz = getenv("SIZE", 2000000) cls.dev = Device["AMD"] if not cls.dev.is_usb(): raise unittest.SkipTest("only test this on USB devices") From 0c5307b4f3c1ec301bae064120df2e27a7b0eb88 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:04:06 +0300 Subject: [PATCH 103/306] realize: fast stat (#17600) * hcq2: fast stat * x * Dx --- tinygrad/engine/realize.py | 163 ++++++++++++++++--------------- tinygrad/runtime/support/hcq2.py | 4 +- 2 files changed, 84 insertions(+), 83 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index b703a95a37778..fb69ab11efc90 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -1,9 +1,9 @@ from __future__ import annotations from typing import cast, Iterator, Any, Sequence -import time, random, itertools, math, contextlib, weakref, array +import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple -from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events +from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.dtype import dtypes @@ -17,6 +17,7 @@ def get_call_arg_uops(call:UOp) -> tuple[UOp, ...]: return tuple(s for s in call def get_call_var_uops(call:UOp, prg:UOp) -> list[UOp]: bound = {s.src[0].expr: s.src[1].src[1] for s in call.src[1:] if s.is_bound_var} return [bound.get(v.expr, v) for v in prg.arg.vars] + def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: ast = call.src[0] if ast.op is Ops.PROGRAM: return tuple(ast.arg.outs), tuple(ast.arg.ins) @@ -24,6 +25,12 @@ def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return (0,), tuple(range(1, len(get_call_arg_uops(call)))) return (), () +def get_call_kernels(call:UOp) -> list[tuple[str, UOp]]: + if (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": return [(d, k) for devs, k, _ in call.arg.aux.kernels for d in devs] + if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return [(to_tuple(ast.device)[0], call)] + if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "validate": return [] + return [(d, call) for d in to_tuple(call.src[1].device)] + def get_call_name(call:UOp, bufs:Sequence[Buffer|UOp], var_vals:dict[str, int]|None=None) -> str: def _uop_sz_to_str(uop:UOp) -> str: return size_to_str(sym_infer(prod(uop.shape) * uop.dtype.itemsize, var_vals or {})) def _dev_str(buf:Buffer|UOp) -> str: return ', '.join(d[:7] for d in to_tuple(buf.device)) @@ -39,49 +46,52 @@ def _dev_str(buf:Buffer|UOp) -> str: return ', '.join(d[:7] for d in to_tuple(bu # **************** Stat **************** def estimate_uop(call:UOp) -> Estimates: - ast = call.src[0] - if ast.op is Ops.PROGRAM: return ast.src[0].arg.estimates or Estimates() + if (ast:=call.src[0]).op is Ops.PROGRAM: return ast.src[0].arg.estimates or Estimates() if ast.op is Ops.COPY or (ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec"): - nbytes = prod(call.src[1].shape) * call.src[1].dtype.itemsize - return Estimates(lds=nbytes, mem=nbytes) + return Estimates(lds=(nbytes:=prod(call.src[1].shape) * call.src[1].dtype.itemsize), mem=nbytes) if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return get_graph_runtime(ast).estimates if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": return call.arg.aux.estimates return Estimates() first_run_cache:set[bytes] = set() -@contextlib.contextmanager -def track_stats(ctx:ExecContext, call:UOp, device:str, bufs:list[Buffer], var_vals:dict[str, int]): - if PROFILE: - outputs, inputs = get_call_outs_ins(call) - cpu_events.append(ProfilePointEvent(device, "exec", len(cpu_events), {"var_vals": var_vals, - "bufs": [b.trace_num for b in bufs], "name": get_call_name(call, bufs, var_vals), "outputs": outputs, "inputs": inputs})) - et: list[float|None] = [None] - if DEBUG >= 2: st = time.perf_counter() - yield et - if not ctx.update_stats: return - - if DEBUG >= 2 and et[0] is None: - Device[device].synchronize() - et[0] = time.perf_counter() - st - - estimates = estimate_uop(call) - GlobalCounters.kernel_count += 1 - GlobalCounters.global_ops += (op_est:=sym_infer(estimates.ops, var_vals)) - GlobalCounters.global_mem += (mem_est:=sym_infer(estimates.mem, var_vals)) - if et[0] is not None: GlobalCounters.time_sum_s += et[0] - if DEBUG >= 2: - display_name = get_call_name(call, bufs, var_vals) - lds_est = sym_infer(estimates.lds, var_vals) - header_color = 'magenta' if ctx.jit else ('green' if call.src[0].key not in first_run_cache else None) - ptm = colored(time_to_str(et[0], w=9), "yellow" if et[0] > 0.01 else None) if et[0] is not None else "" - flops, membw, ldsbw = op_est/(et[0] or 1e-20), mem_est/(et[0] or 1e-20), lds_est/(et[0] or 1e-20) +def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|None]): + if ctx.update_stats: + is_hcq = (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq" + estimates, n = estimate_uop(call), 1 if is_hcq else len(get_call_kernels(call)) + GlobalCounters.kernel_count += len(call.arg.aux.kernels) if is_hcq else n + GlobalCounters.global_ops += n*sym_infer(estimates.ops, ctx.var_vals) + GlobalCounters.global_mem += n*sym_infer(estimates.mem, ctx.var_vals) + GlobalCounters.time_sum_s += sum(et for et in ets if et is not None) + if DEBUG < 2 and not PROFILE: return + + kernels = get_call_kernels(call) # everything below is the per kernel display: exec events for the profiler and DEBUG=2 lines + args = resolve_params(call, ctx.input_uops) if kernels and kernels[0][1] is call else [] + lanes = list(unwrap_multi(call, [args[g] for g in call.src[0].arg.globals] if call.src[0].op is Ops.PROGRAM else args)) if args else [] + for i, (device, kcall) in enumerate(kernels): + et, bufs = ets[i] if i < len(ets) else None, lanes[i][0] if i < len(lanes) else [] + if PROFILE: # backdate the event to the start of the call, the viz matches a device range with the exec event before it + outputs, inputs = get_call_outs_ins(kcall) + cpu_events.append(ProfilePointEvent(device, "exec", len(cpu_events), {"var_vals": ctx.var_vals, + "bufs": [b.trace_num for b in bufs], "name": get_call_name(kcall, bufs, ctx.var_vals), "outputs": outputs, "inputs": inputs}, ts=st)) + if DEBUG < 2 or not ctx.update_stats: continue + if et is None: + Device[device].synchronize() + et, st = float(perf_counter_us() - st)*1e-6, perf_counter_us() + GlobalCounters.time_sum_s += et + + estimates = estimate_uop(kcall) + display_name = get_call_name(kcall, bufs, ctx.var_vals) + op_est, mem_est, lds_est = (sym_infer(x, ctx.var_vals) for x in (estimates.ops, estimates.mem, estimates.lds)) + header_color = 'magenta' if ctx.jit else ('green' if kcall.src[0].key not in first_run_cache else None) + ptm = colored(time_to_str(et, w=9), "yellow" if et > 0.01 else None) if et is not None else "" + flops, membw, ldsbw = op_est/(et or 1e-20), mem_est/(et or 1e-20), lds_est/(et or 1e-20) flops_str = f"{flops*1e-9:7.0f} GFLOPS" if flops < 1e14 else colored(f"{flops*1e-12:7.0f} TFLOPS", 'green') mem_str = f"{membw*1e-9:4.0f}|{ldsbw*1e-9:<6.0f} GB/s" if membw < 1e13 and ldsbw < 1e15 else \ colored(f"{membw*1e-12:4.0f}|{ldsbw*1e-12:<6.0f} TB/s", 'green') print(f"{colored(f'*** {device[:7]:7s} {GlobalCounters.kernel_count:4d}', header_color)}"+ f" {display_name+' '*(46-ansilen(display_name))} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ - ("" if et[0] is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) - first_run_cache.add(call.src[0].key) + ("" if et is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) + first_run_cache.add(kcall.src[0].key) local_size_cache: dict[bytes, tuple[int, ...]] = {} def optimize_local_size(call:UOp, prg:UOp) -> UOp|None: @@ -154,33 +164,31 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d for x in call.src[0].toposort()) for j, per_dev in enumerate(zip(*[cast(MultiBuffer, b).bufs for b in bufs])): yield list(per_dev), {"_device_num": j} if has_dnum else {} -def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): dest, src = bufs[0].ensure_allocated(), bufs[1].ensure_allocated() - with track_stats(ctx, call, dest.device, [dest, src], ctx.var_vals): - if hasattr(dest.allocator,'_transfer') and dest.allocator.supports_transfer and dest.device.split(":")[0] == src.device.split(":")[0]: - dest.allocator._transfer(dest._buf, src._buf, dest.nbytes, src_dev=src.allocator.dev, dest_dev=dest.allocator.dev) - elif src.device.startswith("DISK") and getattr(src.allocator.dev, 'fd', None) is not None \ - and hasattr(dest.allocator, 'copy_from_disk') and src.nbytes >= 4096 and dest.allocator.supports_copy_from_disk: - dest.allocator.copy_from_disk(dest._buf, src._buf, src.nbytes) - elif hasattr(dest.allocator, '_as_buffer'): src.allocator._copyout(dest.as_memoryview(force_zero_copy=True), src._buf) - else: dest.allocator._copyin(dest._buf, src.as_memoryview(allow_zero_copy=True)) - return None - -def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - et = None + if hasattr(dest.allocator,'_transfer') and dest.allocator.supports_transfer and dest.device.split(":")[0] == src.device.split(":")[0]: + dest.allocator._transfer(dest._buf, src._buf, dest.nbytes, src_dev=src.allocator.dev, dest_dev=dest.allocator.dev) + elif src.device.startswith("DISK") and getattr(src.allocator.dev, 'fd', None) is not None \ + and hasattr(dest.allocator, 'copy_from_disk') and src.nbytes >= 4096 and dest.allocator.supports_copy_from_disk: + dest.allocator.copy_from_disk(dest._buf, src._buf, src.nbytes) + elif hasattr(dest.allocator, '_as_buffer'): src.allocator._copyout(dest.as_memoryview(force_zero_copy=True), src._buf) + else: dest.allocator._copyin(dest._buf, src.as_memoryview(allow_zero_copy=True)) + return [] + +def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: + ets:list[float|None] = [] resolved = resolve_params(call, ctx.input_uops) for device, (bufs, device_vars) in zip(to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): var_vals = {**ctx.var_vals, **device_vars} prg_bufs = [b.ensure_allocated() for b in bufs] rt = get_runtime(device, ast, cache=ctx.cache) global_size, local_size = ast.arg.launch_dims(var_vals) - with track_stats(ctx, call, device, prg_bufs, var_vals) as tm: - et = tm[0] = rt(*[b.get_buf(device) for b in prg_bufs], global_size=global_size, local_size=local_size, vals=ast.arg.vals(var_vals), - wait=ctx.wait, timeout=ctx.timeout) - return et + ets.append(rt(*[b.get_buf(device) for b in prg_bufs], global_size=global_size, local_size=local_size, vals=ast.arg.vals(var_vals), + wait=ctx.wait, timeout=ctx.timeout)) + return ets -def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: import numpy as np for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): bufs, dev_bufs = bufs[:len(bufs)//2], bufs[len(bufs)//2:] @@ -189,43 +197,36 @@ def exec_validate(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: global_size, local_size = prg.arg.launch_dims(var_vals) cpu_rt(*[bufs[i].ensure_allocated()._buf for i in prg.arg.globals], global_size=global_size, local_size=local_size, vals=prg.arg.vals(var_vals)) for i in prg.arg.outs: np.testing.assert_allclose(dev_bufs[i].ensure_allocated().numpy(), bufs[i].numpy(), rtol=1e-3, atol=1e-3) - return None + return [] -def exec_encdec(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_encdec(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: bufs = [cast(Buffer, b.buffer).ensure_allocated() for b in resolve_params(call, ctx.input_uops)] shape, pos_var = tuple(s.val for s in ast.src if s.op is Ops.CONST), ast.variables()[0].expr - with track_stats(ctx, call, bufs[0].device, bufs, ctx.var_vals): - bufs[0].allocator._encode_decode(bufs[0]._buf, bufs[1]._buf, bufs[2]._buf, [x._buf for x in bufs[3:]], shape, ctx.var_vals[pos_var]) - return None + bufs[0].allocator._encode_decode(bufs[0]._buf, bufs[1]._buf, bufs[2]._buf, [x._buf for x in bufs[3:]], shape, ctx.var_vals[pos_var]) + return [] -def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: - rt = get_graph_runtime(ast, ctx.input_uops) - with track_stats(ctx, call, rt.device, [], ctx.var_vals) as t: t[0] = rt(ctx.input_uops, ctx.var_vals, wait=ctx.wait) - return t[0] +def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: + return [get_graph_runtime(ast, ctx.input_uops)(ctx.input_uops, ctx.var_vals, wait=ctx.wait)] -def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> float|None: +def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: dev = cast(Any, Device[(info:= call.arg.aux).device[0]]) addrs = [(b.bufs[j] if isinstance(b:=_resolve(ctx.input_uops[k], ctx.input_uops).buffer, MultiBuffer) else b).get_buf(dev_name).va_addr for devs, idxs in info.input_idxs for j, dev_name in enumerate(devs) for k in idxs] dev.rt_buffer._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) - tables = [UOp.from_buffer(dev.rt_buffer.view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) - for devs, idxs in info.input_idxs for j in range(len(devs))] - if info.inputs is not None: call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) - exec_kernel(replace(ctx, update_stats=DEBUG>=3, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer._buf.va_addr + base}), call, ast) - - tms = [] - for devices, stat_call, prof in info.kernels: - for device in devices: - tm = None - if prof: - (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, *prof) - if ctx.wait: - d.synchronize(timeout=ctx.timeout) - st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) - tms.append(tm:=float(en-st)/d.timestamp_divider/1e6) - with track_stats(ctx, stat_call, device, [], ctx.var_vals) as et: et[0] = tm - return max(tms) if tms else None + if info.inputs is not None: + tables = [UOp.from_buffer(dev.rt_buffer.view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) + for devs, idxs in info.input_idxs for j in range(len(devs))] + call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) + exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer._buf.va_addr + base}), call, ast) + + def _prof_tm(device:str, stat_call:UOp, prof:tuple[int, ...]) -> float|None: + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, *prof) + if not ctx.wait: return None + d.synchronize(timeout=ctx.timeout) + st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) + return float(en-st)/d.timestamp_divider/1e6 + return [_prof_tm(device, k, prof) for devices, k, prof in info.kernels if prof for device in devices] if PROFILE or ctx.wait else [] # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src pm_flatten_linear = PatternMatcher([ @@ -280,7 +281,7 @@ def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequenc inputs = list(input_uops) if not jit: linear = link_linear(compile_linear(linear, validate=VALIDATE_WITH_CPU, input_uops=inputs)) ctx = ExecContext(var_vals or {}, tuple(inputs), update_stats, jit, wait or DEBUG>=2) - for call in linear.src: pm_exec.rewrite(call, ctx) + for call in linear.src: track_stats(ctx, call, perf_counter_us(), pm_exec.rewrite(call, ctx)) def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> float: if clear_l2: @@ -290,4 +291,4 @@ def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) ctx = ExecContext(var_vals or {}, update_stats=False, wait=True, timeout=timeout, cache=False) linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True), cache=ctx.cache) - return max(pm_exec.rewrite(c, ctx) or 0.0 for c in linear.src) + return max(et for c in linear.src for et in pm_exec.rewrite(c, ctx) or [0.0]) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 9d195fbf8ee9f..f9886ca63620e 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -252,7 +252,7 @@ def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? devs, queue = get_submit(calls[0]).src[0].arg body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() return make_call(f"submit {queue} ({len(calls)})", body, - replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()))) + replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) def merge_queues(linear:UOp) -> UOp: new_src:list[UOp] = [] @@ -426,7 +426,7 @@ def merge_batch(batch:list[UOp]) -> UOp: cmds = [c.src[0].src[0].call(*[_lane_arg(a.without_after, j, tables + off) for a in c.src[1:]], UOp.variable("_device_num", 0, 1 << 30).bind(j)) for (c, j, _), off in zip(lanes, offs)] - info = HCQInfo((HCQ_RUNTIME_DEV.value,), sum((c.arg.aux.estimates for c in batch), start=Estimates()), + info = HCQInfo((HCQ_RUNTIME_DEV.value,), sum((c.arg.aux.estimates for c in batch), start=Estimates()).simplify(), input_idxs=tuple(x for c in batch for x in c.arg.aux.input_idxs), kernels=tuple(k for c in batch for k in c.arg.aux.kernels)) body = UOp.custom_function("hcq", make_submit(*cmds, devs=HCQ_RUNTIME_DEV.value, queue="SUBMIT:0").sink()) return body.call(*[s for c in batch for s in c.src[1:] if s.without_after.tag != "inputs"], name=f"hcq_submitter ({len(batch)})", aux=info) From 7064e76bc80bc8dba8f451aa1610d16adb0c4197 Mon Sep 17 00:00:00 2001 From: Bennett <95309712+EthanBnntt@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:31:21 -0400 Subject: [PATCH 104/306] fix roll on zero-sized tensors (#17603) Signed-off-by: Bennett Co-authored-by: Bennett --- test/backend/test_ops.py | 4 ++++ tinygrad/mixin/movement.py | 1 + 2 files changed, 5 insertions(+) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 27beab433157c..ca97691358db4 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -2164,6 +2164,10 @@ def test_diagonal(self): def test_roll(self): helper_test_op([(2, 4)], lambda x: x.roll(1)) helper_test_op([(2, 4)], lambda x: x.roll((1,))) + helper_test_op([(0,)], lambda x: x.roll(1, 0)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1, 0)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1, 1)) + helper_test_op([(2, 0, 3)], lambda x: x.roll(1)) self.helper_test_exception([(2, 4)], lambda x: x.roll((1, 2)), expected=RuntimeError) helper_test_op([(2, 4)], lambda x: x.roll(1, 0)) helper_test_op([(2, 4)], lambda x: x.roll(-1, 0)) diff --git a/tinygrad/mixin/movement.py b/tinygrad/mixin/movement.py index c900993019b9d..44ec39155e053 100644 --- a/tinygrad/mixin/movement.py +++ b/tinygrad/mixin/movement.py @@ -550,6 +550,7 @@ def roll(self, shifts:int|tuple[int, ...], dims:int|tuple[int, ...]|None=None) - if dims is None: return self.flatten().roll(shifts, 0).reshape(self.shape) dims, shifts = tuple(self._resolve_dim(d) for d in make_tuple(dims, 1)), make_tuple(shifts, 1) if len(dims) != len(shifts): raise RuntimeError(f"{len(dims)=} != {len(shifts)=}") + if 0 in self.shape: return self shrink_arg: list[tuple[sint, sint]|None] = [None] * self.ndim for d, s in zip(dims, shifts): shrink_arg[d] = (delta:=self.shape[d]-s%self.shape[d], delta+self.shape[d]) return self.repeat(*tuple(2 if i in dims else 1 for i in range(self.ndim))).shrink(tuple(shrink_arg)) From b8cc74ecf8cc5bf0bedaa330a9da9eabd0a49f62 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 15:40:16 -0400 Subject: [PATCH 105/306] no float in tensor shape [pr] (#17605) --- test/null/test_tensor.py | 5 +++++ tinygrad/uop/ops.py | 9 ++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/null/test_tensor.py b/test/null/test_tensor.py index 32ccc255e3fae..560ce3f5e96f9 100644 --- a/test/null/test_tensor.py +++ b/test/null/test_tensor.py @@ -171,6 +171,11 @@ def test_full_like_device_on_multi_raises(self): t = Tensor.ones(8, 4).shard(("NULL:0", "NULL:1"), axis=0) with self.assertRaises(RuntimeError): t.full_like(5, device="NULL") +class TestTensorShape(unittest.TestCase): + def test_float_shape_raises(self): + for dim in (2.0, 2.5): + with self.subTest(dim=dim), self.assertRaisesRegex(RuntimeError, "shape must be int"): Tensor.ones(dim) + class TestTensorDevice(unittest.TestCase): def test_create_from_single_device_tuple(self): (Tensor([1.0], device=(Device.DEFAULT,)) + Tensor([2.0])).realize() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 49a7429d0a73e..9a30dbe3120c2 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -94,11 +94,10 @@ def multirange_str(rngs:Iterable[UOp], color=False, pad=None) -> str: return ret def shape_to_shape_arg(arg:tuple[sint, ...]) -> UOp: - for x in arg: - if isinstance(x, UOp) and not dtypes.is_int(x.dtype): raise RuntimeError(f"shape must be int, got {x.dtype} in {arg}") - if len(arg) == 0: return UOp(Ops.STACK) - elif len(arg) == 1: return UOp.const(arg[0], dtypes.weakint) - else: return UOp(Ops.STACK, src=tuple(UOp.const(x) if isinstance(x, int) else x for x in arg)) + src = tuple(x if isinstance(x, UOp) else UOp.const(x) for x in arg) + for x in src: + if not dtypes.is_int(x.dtype): raise RuntimeError(f"shape must be int, got {x.dtype} in {arg}") + return src[0] if len(src) == 1 else UOp(Ops.STACK, src=src) def consumer_map_from_toposort(lst:Iterable[UOp]): ret: dict[UOp, dict[UOp, None]] = {} From 0a0b6cb596cf6ef3875f05f6d09a6641c8475d41 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 15:54:41 -0400 Subject: [PATCH 106/306] fix TestDevCopySpeeds command (#17607) --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 739f7242c05dc..ea52f06955721 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -541,7 +541,7 @@ jobs: - name: openpilot run_pickle big_driving_supercombo run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_run_pickle RUN_PICKLE=1 PICKLE_OOB=1 PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py - openpilot.pkl - name: Test copy speeds - run: SIZE=64e6 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds + run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds driverbenchmarks: name: PCI Driver Benchmark (DEV=${{ matrix.dev }}) From fc214da4178bd378a659f44bc4acd97af3325b0d Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 15:56:01 -0400 Subject: [PATCH 107/306] test updates for weak const change (#17606) --- test/backend/test_const_folding.py | 8 ++++---- test/backend/test_linearizer.py | 3 --- test/null/test_uop_graph.py | 12 ++---------- test/null/test_uops.py | 2 +- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/test/backend/test_const_folding.py b/test/backend/test_const_folding.py index 0c2c80156063c..fc8ad9f0a84f2 100644 --- a/test/backend/test_const_folding.py +++ b/test/backend/test_const_folding.py @@ -1,7 +1,7 @@ import unittest, math from tinygrad import Tensor, Device, dtypes from tinygrad.dtype import DTYPES_DICT -from tinygrad.uop.ops import Ops, UOp +from tinygrad.uop.ops import Ops, UOp, GroupOp from tinygrad.codegen.decomp.op import threefry2x32 import numpy as np from test.helpers import not_support_multi_device @@ -17,7 +17,7 @@ def _check_ast_count(desired_count:int, t:Tensor): class TestMovedConstFolding(unittest.TestCase): def test_contiguous_deviceless_const(self): t = Tensor(UOp.const(2.0, dtypes.float)).contiguous() - self.assertIs(t.uop.op, Ops.CONST) + self.assertIs(t.uop, UOp.const(2.0, dtypes.float)) self.assertIsNone(t.uop.device) def test_add_shrunk_zero(self): @@ -169,8 +169,8 @@ def test_multi_const_folding_tensor(self): class TestThreefryConstFolding(unittest.TestCase): def test_threefry(self): # THREEFRY(const,const) folds to a const once decomposed - x = threefry2x32(UOp.const(5, dtypes.uint64), UOp.const(10, dtypes.uint64)) - self.assertIs(x.simplify().op, Ops.CONST) + x = threefry2x32(UOp.const(5, dtypes.uint64), UOp.const(10, dtypes.uint64)).simplify() + self.assertEqual([u.op for u in x.toposort() if u.op in GroupOp.ALU], []) class TestTautologicalCompare(unittest.TestCase): # without const folding, these would have triggered -Wtautological-compare in clang diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index b21dc7ace6cbd..2d527140cb60e 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -16,8 +16,6 @@ from test.backend.test_softmax_fusion import single_kernel_softmax MOCKGPU = DEV.interface.startswith("MOCK") -from tinygrad.uop.render import print_uops # noqa: F401 # pylint: disable=unused-import - @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, ISARenderer), "isa backends don't preserve the op spec when lowering") class TestLinearizer(unittest.TestCase): def test_arg_dedup(self): @@ -248,7 +246,6 @@ def test_simple_unroll_no_between_phi_dependencies(self): uops = tuple(to_program(replace_opts(ast, opt), renderer=Device[Device.DEFAULT].renderer).src[1].src) begin_range = [i for i, x in enumerate(uops) if x.op is Ops.RANGE][-1] end_range = [i for i, x in enumerate(uops) if x.op is Ops.END][0] - for i,u in enumerate(uops): print(i, u.op, [uops.index(s) for s in u.src], u.arg, u.dtype) for u in uops: if u.op is Ops.STORE and u.src[0].addrspace is AddrSpace.REG: if uops.index(u) < begin_range: diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 180ee23189bb3..e144da641a037 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -13,25 +13,17 @@ ((UPat.var('x') + UPat.cvar('c1')) + UPat.cvar('c2'), lambda x,c1,c2: x + (c1.val+c2.val)), ]) -def const_values(u:UOp): - if u.op is Ops.CONST: return (u.val,) - if u.op is Ops.STACK: return tuple(x.val for x in u.src) - raise AssertionError(f"expected const-like UOp, got {u.op}") - class TestGraphRewriteConst(unittest.TestCase): def test_gep_const(self): v1 = UOp.const((0,1,2), dtypes.int) v2 = v1.index(1) ret = graph_rewrite(v2, sym) - self.assertEqual(ret.dtype, dtypes.int) - self.assertEqual(ret.val, 1) + self.assertIs(ret, UOp.const(1, dtypes.int)) def test_add_const(self): v1 = UOp.const((0,1,2)) v2 = UOp.const((5,6,7)) - ret = graph_rewrite(v1+v2, sym) - self.assertEqual(ret.op, Ops.STACK) - self.assertEqual(const_values(ret), (5,7,9)) + self.assertIs(graph_rewrite(v1+v2, sym), UOp.const((5,7,9))) def xfail_broken_const_wraparound(fn): fn = pytest.mark.xfail(reason="const folding does not properly implement modular arithmetic")(fn) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index 40115c3e448e4..de18f1a93171d 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -129,7 +129,7 @@ def __eq__(self, other): return other is Invalid self.assertFalse(Invalid != HoldsInvalid()) def test_matchers_agree_on_nan(self): - n = UOp.const(math.nan, dtypes.float32) + n = UOp.const(math.nan) for compiled in (False, True): pm = PatternMatcher([(UPat(Ops.CONST, arg=math.nan), lambda: True)], compiled=compiled) self.assertTrue(pm.rewrite(n), f"{compiled=}") From ab68c587590f54678ed8470f8bd04e255a266c3f Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:02:17 +0300 Subject: [PATCH 108/306] hcq2: speed (#17604) * hcq2: speed * x --- tinygrad/runtime/support/hcq2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f9886ca63620e..be85e52c6c16e 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -591,7 +591,7 @@ def rt_buffer(self) -> Buffer: def new_buffer(self, b:UOp, cache:bool) -> Buffer: if cache or b.tag in HCQ_CACHE_TAGS: - return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=b.tag != "program", cpu_access=True, nolru=True)) + return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=b.tag not in ("program","kernargs"), cpu_access=True,nolru=True)) return self.rt_buffer.view(b.max_numel(), b.dtype, self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) @functools.cache From 2067133732470549d44cfdfb648be6bd896bcb16 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 19 Aug 2026 14:55:08 -0700 Subject: [PATCH 109/306] cpu: link with rt (#17608) --- test/null/test_elf.py | 5 +++-- tinygrad/runtime/ops_cpu.py | 18 +++++++++--------- tinygrad/runtime/support/c.py | 2 +- tinygrad/runtime/support/compiler_cpu.py | 11 ++++------- tinygrad/runtime/support/compiler_llvm.py | 14 ++++---------- tinygrad/runtime/support/elf.py | 10 +++++----- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/test/null/test_elf.py b/test/null/test_elf.py index 0f9a3d48f0678..ac8bf6c41675f 100644 --- a/test/null/test_elf.py +++ b/test/null/test_elf.py @@ -1,6 +1,7 @@ import unittest, subprocess, platform from tinygrad.runtime.support.compiler_cpu import ClangCompiler from tinygrad.runtime.support.elf import elf_loader +from tinygrad.runtime.support.c import DLL class TestElfLoader(unittest.TestCase): def test_load_clang_jit_strtab(self): @@ -23,7 +24,7 @@ def test_clang_jit_compiler_external_raise(self): } ''' with self.assertRaisesRegex(RuntimeError, 'evil_external_function'): - ClangCompiler([{'AMD64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine(), m), "native"]).compile(src) + elf_loader(ClangCompiler([{'AMD64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine(), m), "native"]).compile(src)) def test_link(self): src = ''' float powf(float, float); // from libm @@ -32,7 +33,7 @@ def test_link(self): args = ('-x', 'c', '-c', '-target', f'{platform.machine()}-none-unknown-elf', '-march=native', '-fPIC', '-O2', '-ffreestanding', '-nostdlib') obj = subprocess.check_output(('clang',) + args + ('-', '-o', '-'), input=src.encode()) with self.assertRaisesRegex(RuntimeError, 'powf'): elf_loader(obj) - elf_loader(obj, link_libs=['m']) + elf_loader(obj, link_libs=[DLL('m', 'm')]) if __name__ == '__main__': unittest.main() diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 0aed3949c35fc..28f2165571e9c 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -6,6 +6,7 @@ from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device from tinygrad.runtime.support.hcq import HCQBuffer, MMIOInterface from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_cmdbuf, make_signal +from tinygrad.runtime.support.c import DLL from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer from tinygrad.renderer.nir import LVPRenderer @@ -111,9 +112,9 @@ def encode_queue(q:UOp) -> UOp: MAP_JIT = 0x0800 class CPUProgram(Program['CPUDevice']): - rt_lib = None - try: rt_lib = ctypes.CDLL(ctypes.util.find_library('System' if OSX else 'kernel32') if OSX or WIN else 'libgcc_s.so.1') - except OSError: pass + rt_lib, libm = DLL('rt', 'System' if OSX else 'kernel' if WIN else 'gcc_s'), DLL('m', 'm') + + def _load(self, lib, base=0): return lib if lib[:4] != libc.ELFMAG.encode() else jit_loader(lib, base=base, link_libs=[self.libm, self.rt_lib]) def __init__(self, dev:CPUDevice, obj:TinyELF): self.dev, self.name, self.signature = dev, obj.name, obj.signature @@ -125,10 +126,10 @@ def __init__(self, dev:CPUDevice, obj:TinyELF): ctypes.windll.kernel32.VirtualAlloc.restype = ctypes.c_void_p self.addr = ctypes.windll.kernel32.VirtualAlloc(ctypes.c_void_p(0), ctypes.c_size_t(len(obj.lib)), MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE) - ctypes.memmove(self.addr, obj.lib, len(obj.lib)) + ctypes.memmove(self.addr, (loaded:=self._load(obj.lib, self.addr)), len(loaded)) ctypes.windll.kernel32.GetCurrentProcess.restype = ctypes.c_void_p proc = ctypes.windll.kernel32.GetCurrentProcess() - ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(obj.lib))) + ctypes.windll.kernel32.FlushInstructionCache(ctypes.c_void_p(proc), ctypes.c_void_p(self.addr), ctypes.c_size_t(len(loaded))) self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) else: # On apple silicon with SPRR enabled (it always is in macos) RWX pages are unrepresentable: https://blog.svenpeter.dev/posts/m1_sprr_gxf/ @@ -137,18 +138,17 @@ def __init__(self, dev:CPUDevice, obj:TinyELF): self.addr = mv_address(self.mem) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(False) - lib = jit_loader(obj.lib, base=ctypes.addressof(ctypes.c_void_p.from_buffer(self.mem)), link_libs=['m']) if self.lvp else obj.lib - self.mem.write(lib) + self.mem.write(loaded:=self._load(obj.lib, mv_address(self.mem))) if OSX: unwrap(CPUProgram.rt_lib).pthread_jit_write_protect_np(True) # __clear_cache isn't a normal libc function, but a compiler support routine found in libgcc_s for gcc and compiler-rt for clang. # libgcc_s comes as shared library but compiler-rt is only a bunch of static library archives which we can't directly load, but fortunately # it somehow found its way into libSystem on macos (likely because it used __builtin_clear_cache) and libgcc_s is ~always present on linux # Using ["name"] instead of .name because otherwise name is getting mangled: https://docs.python.org/3.12/reference/expressions.html#index-5 - if CPUProgram.rt_lib is not None: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.addr), ctypes.c_void_p(self.addr + len(lib))) + if 'rt' in DLL._loaded_: CPUProgram.rt_lib["__clear_cache"](ctypes.c_void_p(self.addr), ctypes.c_void_p(self.addr + len(loaded))) else: # msync should be a universal POSIX way to do this - libc.msync(ctypes.c_void_p(self.addr), len(lib), libc.MS_SYNC | libc.MS_INVALIDATE) + libc.msync(ctypes.c_void_p(self.addr), len(loaded), libc.MS_SYNC | libc.MS_INVALIDATE) self.fxn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(self.addr) if self.lvp else ctypes.CFUNCTYPE(None)(self.addr) diff --git a/tinygrad/runtime/support/c.py b/tinygrad/runtime/support/c.py index 009bf60e755e3..aade78160fbe0 100644 --- a/tinygrad/runtime/support/c.py +++ b/tinygrad/runtime/support/c.py @@ -91,7 +91,7 @@ class DLL(ctypes.CDLL): @staticmethod def findlib(nm:str, paths:list[str], extra_paths=[]): - if nm == 'libc' and OSX: return '/usr/lib/libc.dylib' + if nm in ('libc', 'm') and OSX: return f'/usr/lib/lib{nm.removeprefix("lib")}.dylib' if pathlib.Path(path:=getenv(nm.replace('-', '_').upper()+"_PATH", '')).is_file(): return path for p in paths: libpaths = {"posix": [d for d in os.environ.get('LD_LIBRARY_PATH', '').split(os.pathsep) if d] + ["/usr/lib64", "/usr/lib", "/usr/local/lib"], diff --git a/tinygrad/runtime/support/compiler_cpu.py b/tinygrad/runtime/support/compiler_cpu.py index 7bb24f5f3136f..9fb5212c8730b 100644 --- a/tinygrad/runtime/support/compiler_cpu.py +++ b/tinygrad/runtime/support/compiler_cpu.py @@ -1,10 +1,9 @@ import subprocess from tinygrad.device import Compiler -from tinygrad.helpers import getenv, capstone_flatdump -from tinygrad.runtime.support.elf import jit_loader +from tinygrad.helpers import getenv, capstone_flatdump, cpu_objdump class ClangCompiler(Compiler): - def __init__(self, arch:list[str], cachekey="compile_clang_jit"): + def __init__(self, arch:list[str], cachekey="compile_clang_obj"): assert len(arch) >= 2, f"invalid arch string: {','.join(arch)!r}, expected ',,[]' (eg. 'x86_64,znver2')" self.arch, cpu, *feats = arch match self.arch: @@ -16,15 +15,13 @@ def __init__(self, arch:list[str], cachekey="compile_clang_jit"): case _: raise RuntimeError(f"unsupported arch: {self.arch!r}") super().__init__(f"{cachekey}_{'_'.join(arch)}") - def compile_to_obj(self, src:str) -> bytes: + def compile(self, src:str) -> bytes: """Compile C source to ELF object file (before linking).""" # -fno-math-errno is required for __builtin_sqrt to become an instruction instead of a function call return subprocess.check_output([getenv("CC", 'clang'), '-c', '-x', 'c', '-O2', '-fPIC', '-ffreestanding', '-fno-math-errno', '-nostdlib', '-fno-ident', f'--target={self.arch}-none-unknown-elf', *self.args, '-', '-o', '-'], input=src.encode('utf-8')) - def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src)) - - def disassemble(self, lib:bytes): return capstone_flatdump(lib, self.arch) + def disassemble(self, lib: bytes): cpu_objdump(lib) class X86Compiler(Compiler): diff --git a/tinygrad/runtime/support/compiler_llvm.py b/tinygrad/runtime/support/compiler_llvm.py index be012441a26b4..56a874602093e 100644 --- a/tinygrad/runtime/support/compiler_llvm.py +++ b/tinygrad/runtime/support/compiler_llvm.py @@ -1,7 +1,6 @@ import ctypes from tinygrad.device import Compiler, CompileError -from tinygrad.helpers import getenv, capstone_flatdump, amdgpu_disassemble, unwrap, DEBUG -from tinygrad.runtime.support.elf import jit_loader +from tinygrad.helpers import getenv, cpu_objdump, amdgpu_disassemble, unwrap, DEBUG from tinygrad.runtime.autogen import llvm def cerr(): return ctypes.pointer(ctypes.pointer(ctypes.c_char())) @@ -11,7 +10,6 @@ def expect(x, err, ret=None): return ret class LLVMCompiler(Compiler): - jit = True def __init__(self, arch:str, processor:str, feats:str, cache_key=None): for component in ['Target', 'TargetInfo', 'TargetMC', 'AsmParser', 'AsmPrinter']: getattr(llvm, "LLVMInitialize" + {'arm64': 'AArch64', 'x86_64': 'X86', 'riscv64': 'riscv64'}.get(arch, "AMDGPU") + component)() @@ -43,13 +41,13 @@ def handle_diag(diag_ref, _arg): self.diag_msgs.append(msg) self.handle_diag = handle_diag llvm.LLVMContextSetDiagnosticHandler(self.context, handle_diag, None) - super().__init__(cache_key or f"compile_llvm_{processor}_{feats}{'_jit' if self.jit else ''}{'_opt' if opt else ''}") + super().__init__(cache_key or f"compile_llvm_{processor}_{feats}{'_opt' if opt else ''}") def __del__(self): if hasattr(self, 'pbo'): llvm.LLVMDisposePassBuilderOptions(self.pbo) if hasattr(self, 'context'): llvm.LLVMContextDispose(self.context) - def compile_to_obj(self, src:str) -> bytes: + def compile(self, src:str) -> bytes: self.diag_msgs.clear() src_buf = llvm.LLVMCreateMemoryBufferWithMemoryRangeCopy(ctypes.create_string_buffer(src_bytes:=src.encode()), len(src_bytes), b'src') mod = expect(llvm.LLVMParseIRInContext(self.context, src_buf, ctypes.pointer(m:=llvm.LLVMModuleRef()), err:=cerr()), err, m) @@ -64,9 +62,6 @@ def compile_to_obj(self, src:str) -> bytes: if self.diag_msgs: raise RuntimeError("llvm diagnostic: " + "\n".join(self.diag_msgs)) return obj - def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src)) if self.jit else self.compile_to_obj(src) - - class CPULLVMCompiler(LLVMCompiler): def __init__(self, arch:list[str], cache_key=None): assert len(arch) >= 2, f"invalid arch string: {','.join(arch)!r}, expected ',,[]' (eg. 'x86_64,znver2')" @@ -78,10 +73,9 @@ def __init__(self, arch:list[str], cache_key=None): # +reserve-x18 here does the same thing as -ffixed-x18 in ClangCompiler, see comments there for why it's needed on arm osx super().__init__(self.arch, cpu, ('+reserve-x18,' if self.arch == "arm64" else '') + featstr, cache_key) - def disassemble(self, lib:bytes): capstone_flatdump(lib, self.arch) + def disassemble(self, lib: bytes): cpu_objdump(lib) class AMDLLVMCompiler(LLVMCompiler): - jit = False def __init__(self, arch: str): self.arch = arch super().__init__("AMDGPU", self.arch, "+cumode") diff --git a/tinygrad/runtime/support/elf.py b/tinygrad/runtime/support/elf.py index 41917b2cb9afa..5dbaff7b0f612 100644 --- a/tinygrad/runtime/support/elf.py +++ b/tinygrad/runtime/support/elf.py @@ -1,4 +1,4 @@ -import struct, ctypes, ctypes.util +import struct, ctypes from dataclasses import dataclass from tinygrad.helpers import getbits, i2u, unwrap from tinygrad.runtime.autogen import libc @@ -6,13 +6,13 @@ @dataclass(frozen=True) class ElfSection: name:str; header:libc.Elf64_Shdr|libc.Elf32_Shdr; content:bytes # noqa: E702 -def link_sym(sym:str, libs:list[str]) -> int: +def link_sym(sym:str, libs:list[ctypes.CDLL]) -> int: for lib in libs: - try: return unwrap(ctypes.cast(getattr(ctypes.CDLL(ctypes.util.find_library(lib)), sym), ctypes.c_void_p).value) + try: return unwrap(ctypes.cast(getattr(lib, sym), ctypes.c_void_p).value) except (OSError, AttributeError): pass raise RuntimeError(f'Attempting to relocate against an undefined symbol {sym}') -def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[str]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: +def elf_loader(blob:bytes, force_section_align:int=1, link_libs:list[ctypes.CDLL]|None=None) -> tuple[memoryview, list[ElfSection], list[tuple]]: assert blob[:4] == libc.ELFMAG.encode(), "blob is not an ELF, missing magic bytes" ecls = {libc.ELFCLASS32: "Elf32", libc.ELFCLASS64: "Elf64"}[blob[libc.EI_CLASS]] @@ -49,7 +49,7 @@ def _to_carray(sh, ctype): return (ctype * (sh.header.sh_size // sh.header.sh_en return memoryview(image), sections, relocs -def jit_loader(obj: bytes, base:int=0, link_libs:list[str]|None=None) -> bytes: +def jit_loader(obj: bytes, base:int=0, link_libs:list[ctypes.CDLL]|None=None) -> bytes: image_, _, relocs = elf_loader(obj, link_libs=link_libs) image = bytearray(image_) From c89ae6c083a7de0660bee8c8ea647471c8c9a9b8 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Wed, 19 Aug 2026 17:57:18 -0400 Subject: [PATCH 110/306] gptoss: save more (#17613) --- examples/mlperf/models/gpt_oss.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/mlperf/models/gpt_oss.py b/examples/mlperf/models/gpt_oss.py index 14822db6dbb7b..00cc5cad2f32a 100644 --- a/examples/mlperf/models/gpt_oss.py +++ b/examples/mlperf/models/gpt_oss.py @@ -183,10 +183,12 @@ def attention(self, x:Tensor, freqs_cis:Tensor, mask:Tensor, sliding:bool, *, at xq, xk = apply_rotary_emb(xq, xk, freqs_cis) xq, xk, xv = xq.cast(dtypes.bfloat16), xk.cast(dtypes.bfloat16), xv.cast(dtypes.bfloat16) # (B,N,H,D)/(B,N,KV,D) + fa_saves = [] if getenv("HK_FLASH_ATTENTION"): from extra.thunder.amd.fa import flash_attention - attn, *_ = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks, window=self.sliding_window if sliding else 0) + attn, _, l_vec = flash_attention(xq, xk, xv, is_causal=True, write_flat=True, sinks=sinks, window=self.sliding_window if sliding else 0) attn = attn.reshape(bsz, seqlen, self.n_heads * self.head_dim) + fa_saves = [xq, xk, xv, l_vec] elif sliding: attn = self._sliding_attention(xq, xk, xv, sinks) else: @@ -200,7 +202,7 @@ def attention(self, x:Tensor, freqs_cis:Tensor, mask:Tensor, sliding:bool, *, at attn = (w @ xvm).permute(0, 3, 1, 2, 4).reshape(bsz, seqlen, self.n_heads * self.head_dim) out = matmul_mx(attn, wo, wo_scale) + wo_bias - return out, [x_normed, rrms, attn] + return out, [x_normed, rrms, attn] + fa_saves def feed_forward(self, x:Tensor, *, ffn_norm:Tensor, gate:Tensor, gate_bias:Tensor, w_gate_up:Tensor, w_gate_up_scale:Tensor, w_gate_up_bias:Tensor, @@ -221,6 +223,7 @@ def feed_forward(self, x:Tensor, *, ffn_norm:Tensor, gate:Tensor, gate_bias:Tens z = grouped_mx_gemm(_pad_cols(y.cast(dtypes.bfloat16)), (w_down, w_down_scale), r.off)[:, :dim] \ + (onehot @ w_down_bias.float()).cast(dtypes.bfloat16) out = combine(z, r, inp.shape[0], self.experts_per_tok).reshape(bsz, seqlen, dim) + return out, [x_normed, rrms, xg, h, y, z, r.weights, r.dest_row, r.off] else: thresh = logits.topk(self.experts_per_tok)[0][..., -1:] weights = (logits >= thresh).where(logits, -float("inf")).softmax(-1) From c63d94e0590e92cdc001db6136eb22c46110113d Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 19 Aug 2026 18:36:07 -0700 Subject: [PATCH 111/306] benchmarks: split multigpu (#17615) --- .github/workflows/benchmark.yml | 69 +++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ea52f06955721..f02c210cecf83 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -108,10 +108,6 @@ jobs: - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } - - name: Symlink models and datasets - run: | - mkdir -p weights - ln -s /raid/weights/LLaMA-3 weights/LLaMA-3 - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -129,10 +125,6 @@ jobs: # just metal for now if: ${{ matrix.dev == 'METAL' }} run: BENCHMARK_LOG=olmoe JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m olmoe --benchmark --warmup - - name: Run LLaMA-3 8B on 4 GPUs with BEAM - # only run on machines with multiple gpus - if: ${{ matrix.dev != 'METAL' }} - run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 - name: Run process replay tests uses: ./.github/actions/process-replay @@ -182,10 +174,6 @@ jobs: # slow on metal if: ${{ matrix.dev != 'METAL' }} run: time BENCHMARK_LOG=cifar DEFAULT_FLOAT=HALF STEPS=1000 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py - - name: Run full CIFAR training steps w 6 GPUS - # only run on machines with multiple gpus - if: ${{ matrix.dev != 'METAL' }} - run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py - name: Run process replay tests uses: ./.github/actions/process-replay @@ -231,11 +219,6 @@ jobs: run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) run: BENCHMARK_LOG=resnet_10steps DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py - - name: Run 10 MLPerf ResNet50 training steps (6 gpu) - run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py - - name: Run 10 MLPerf Bert training steps (6 gpu) - # TODO: remove BERT_LAYERS once scheduler is fast - run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py - name: Run process replay tests uses: ./.github/actions/process-replay @@ -285,6 +268,58 @@ jobs: - name: Run process replay tests uses: ./.github/actions/process-replay + multigpubenchmark: + name: Multi-GPU Benchmarks (DEV=${{ matrix.dev }}) + runs-on: [self-hosted, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] + strategy: + fail-fast: false + matrix: + dev: ['AMD', 'NV'] + timeout-minutes: 60 + defaults: + run: + shell: bash -e -o pipefail {0} + env: + DEV: ${{ matrix.dev }} + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + if: github.repository_owner == 'tinygrad' + steps: + - name: Checkout Code + uses: actions/checkout@v6 + - name: Setup (AMD) + if: ${{ matrix.dev == 'AMD' }} + run: | + ./extra/amdpci/setup_python_cap.sh + ./extra/hcq/hcq_smi.py amd rmmod + ./extra/hcq/hcq_smi.py amd kill_pids + - name: Setup (NV) + if: ${{ matrix.dev == 'NV' }} + run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + - name: Symlink models and datasets + run: | + mkdir -p weights + mkdir -p extra/datasets + ln -s /raid/weights/LLaMA-3 weights/LLaMA-3 + ln -s /raid/datasets/imagenet extra/datasets/imagenet + - name: setup staging db + if: github.ref == 'refs/heads/update_benchmark_staging' + run: | + echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV + rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: reset process replay + run: python3 test/external/process_replay/reset.py + - name: Run LLaMA-3 8B on 4 GPUs with BEAM + run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 + - name: Run full CIFAR training steps w 6 GPUS + run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py + - name: Run 10 MLPerf ResNet50 training steps (6 gpu) + run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py + - name: Run 10 MLPerf Bert training steps (6 gpu) + # TODO: remove BERT_LAYERS once scheduler is fast + run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py + - name: Run process replay tests + uses: ./.github/actions/process-replay + tests: name: Tests (DEV=${{ matrix.dev }}) runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"] From e6324d1e1cc878e3f66f3729782039418ab9bbe6 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Aug 2026 22:56:25 -0400 Subject: [PATCH 112/306] test updates from weak const branch (#17618) --- test/amd/test_emu2_pcode.py | 57 +++++++++++++------------------------ test/null/test_viz.py | 6 ++-- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/test/amd/test_emu2_pcode.py b/test/amd/test_emu2_pcode.py index 29dcfefdff404..dc20ff2fd121d 100644 --- a/test/amd/test_emu2_pcode.py +++ b/test/amd/test_emu2_pcode.py @@ -67,32 +67,26 @@ class TestParseExpr(unittest.TestCase): def test_integer_literals(self): """Test parsing integer literals.""" - self.assertEqual(parse_expr('0', {}).val, 0) - self.assertEqual(parse_expr('42', {}).val, 42) - self.assertEqual(parse_expr('42U', {}).val, 42) + self.assertIs(parse_expr('0', {}), UOp.const(0, dtypes.uint32)) + self.assertIs(parse_expr('42', {}), UOp.const(42, dtypes.uint32)) + self.assertIs(parse_expr('42U', {}), UOp.const(42, dtypes.uint32)) def test_negative_integers(self): """Test parsing negative integer literals.""" - result = parse_expr('-1', {}) - self.assertEqual(result.val, -1) - self.assertEqual(result.dtype, dtypes.int) + self.assertIs(parse_expr('-1', {}), UOp.const(-1, dtypes.int)) def test_float_literals(self): """Test parsing float literals.""" - result = parse_expr('1.0F', {}) - self.assertEqual(result.val, 1.0) - self.assertEqual(result.dtype, dtypes.float32) + self.assertIs(parse_expr('1.0F', {}), UOp.const(1.0, dtypes.float32)) def test_hex_literals(self): """Test parsing hex literals.""" - result = parse_expr('0xFF', {}) - self.assertEqual(result.val, 255) + self.assertIs(parse_expr('0xFF', {}), UOp.const(255, dtypes.uint32)) def test_variable_lookup(self): """Test variable lookup in parse_expr.""" vrs = {'x': UOp.const(42, dtypes.uint32)} - result = parse_expr('x', vrs) - self.assertEqual(result.val, 42) + self.assertIs(parse_expr('x', vrs), vrs['x']) def test_binary_ops(self): """Test parsing binary operations.""" @@ -103,9 +97,7 @@ def test_binary_ops(self): self.assertEqual(result.op, Ops.ADD) # Subtraction with constant folding - result = parse_expr('10 - 5', {}) - self.assertEqual(result.op, Ops.CONST) - self.assertEqual(result.val, 5) + self.assertIs(parse_expr('10 - 5', {}), UOp.const(5, dtypes.uint32)) def test_ternary(self): """Test parsing ternary expressions.""" @@ -142,15 +134,8 @@ def test_clz_with_zero(self): S0 = UOp.const(0, dtypes.uint32) _vrs, assigns = parse_pcode(pcode, {'S0': S0}) - # Check that the innermost value (default) is -1 (may be wrapped in CAST) - val = assigns[0][1] - # Traverse to innermost WHERE - while val.op == Ops.WHERE: - val = val.src[2] # false branch - # Unwrap CAST if present - while val.op == Ops.CAST: - val = val.src[0] - self.assertEqual(val.val, -1) + # every cond folds (S0 is a const), leaving the default branch: -1 in the destination dtype + self.assertIs(assigns[0][1].simplify(), UOp.const(-1, dtypes.uint32)) def test_ctz_parsing(self): """Test CTZ pcode parsing.""" @@ -262,8 +247,8 @@ def test_ds_store_address_calculation(self): _, assigns = parse_pcode(pcode, srcs) # Check addresses: 100 + 2*4 = 108, 100 + 5*4 = 120 # assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp - self.assertEqual(assigns[0][1][0].simplify().val, 108) # type: ignore[index] - self.assertEqual(assigns[1][1][0].simplify().val, 120) # type: ignore[index] + self.assertIs(assigns[0][1][0].simplify(), UOp.const(108, dtypes.uint32)) # type: ignore[index] + self.assertIs(assigns[1][1][0].simplify(), UOp.const(120, dtypes.uint32)) # type: ignore[index] def test_ds_store_data_values(self): """Test DS_STORE_2ADDR_B32 uses correct data values.""" @@ -280,8 +265,8 @@ def test_ds_store_data_values(self): _, assigns = parse_pcode(pcode, srcs) # assigns[i][1] is (addr, val) tuple for MEM writes; mypy sees UOp # DATA[31:0] should preserve the value - self.assertEqual(assigns[0][1][1].simplify().val, 0xAAAAAAAA) # type: ignore[index] - self.assertEqual(assigns[1][1][1].simplify().val, 0xBBBBBBBB) # type: ignore[index] + self.assertIs(assigns[0][1][1].simplify(), UOp.const(0xAAAAAAAA, dtypes.uint32)) # type: ignore[index] + self.assertIs(assigns[1][1][1].simplify(), UOp.const(0xBBBBBBBB, dtypes.uint32)) # type: ignore[index] class TestConditionalParsing(unittest.TestCase): """Test conditional (if/elsif/else) pcode parsing.""" @@ -306,12 +291,12 @@ class TestConcatWidthParsing(unittest.TestCase): def test_permlanex16_altrow_concat(self): for row, expected in [(0, 1), (1, 0), (2, 3), (3, 2)]: parsed = parse_expr('{ row[1], ~row[0] }', {'row': UOp.const(row, dtypes.uint32)}) - self.assertEqual(parsed.simplify().val, expected) + self.assertIs(parsed.simplify(), UOp.const(expected, dtypes.uint32)) def test_permlane64_altlane_concat(self): for lane, expected in [(0, 32), (1, 33), (31, 63), (32, 0), (63, 31)]: parsed = parse_expr('{ ~lane[5], lane[4:0] }', {'lane': UOp.const(lane, dtypes.uint32)}) - self.assertEqual(parsed.simplify().val, expected) + self.assertIs(parsed.simplify(), UOp.const(expected, dtypes.uint32)) def test_permlane64_wave64_pcode_indices(self): vgpr = UOp.param(0, dtypes.uint32, (256,)) @@ -327,19 +312,17 @@ def test_permlane64_wave64_pcode_indices(self): 'S2': UOp.const(0, dtypes.uint32), } - def load_idx(v: UOp) -> int: + def check_load_idx(v: UOp, expected: int): simp = v.simplify() self.assertEqual(simp.op, Ops.LOAD) self.assertEqual(simp.src[0].op, Ops.INDEX) - idx = simp.src[0].src[1].simplify() - self.assertEqual(idx.op, Ops.CONST) - return idx.val + self.assertIs(simp.src[0].src[1].simplify(), UOp.const(expected, dtypes.uint32)) _, assigns = parse_pcode(PCODE[VOP1Op.V_PERMLANE64_B32_E32], srcs) self.assertEqual(len(assigns), 64) for lane, (dst_idx, src_idx) in {0: (64, 32), 31: (95, 63), 32: (96, 0), 63: (127, 31)}.items(): - self.assertEqual(assigns[lane][1][0].simplify().val, dst_idx) # type: ignore[index] - self.assertEqual(load_idx(assigns[lane][1][1]), src_idx) # type: ignore[index] + self.assertIs(assigns[lane][1][0].simplify(), UOp.const(dst_idx, dtypes.uint32)) # type: ignore[index] + check_load_idx(assigns[lane][1][1], src_idx) # type: ignore[index] class TestAllPcode(unittest.TestCase): """Test that all pcode from all architectures can be parsed.""" diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 49fbff61406c8..87ecf4951a0ad 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -236,8 +236,8 @@ def test_enter_calls_rewrite(self): def test_const_node_visibility(self): with save_viz() as viz: a = UOp.variable("a", 0, 10, dtype=dtypes.int) - z = UOp.const(0, a.dtype) - y = UOp.const(math.pi, dtypes.float) + z = UOp.const(0) + y = UOp.const(math.pi) alu = a*z ret = exec_rewrite(sink:=UOp.sink(alu, y), [sym]) lst = viz.list_items() @@ -249,7 +249,7 @@ def test_const_node_visibility(self): self.assertTrue(graphs[0][id(y)]["exclude"]) self.assertFalse(graphs[0][id(alu)]["exclude"]) self.assertEqual(graphs[0][id(y)]["label"].split("\n")[:2], ["CONST", "3.14159"]) - self.assertEqual(list(graphs[1]), [id(z), id(y), id(ret)]) + self.assertEqual(list(graphs[1]), [id(u) for u in ret.toposort()]) # rewrite graph keys follow the rewritten sink's toposort def test_const_reshape_expand_folded(self): # CONST->EXPAND should be folded into the ALU node, not shown as separate EXPAND nodes From a1263fadf3c3149752b8d286ff35464bd445ff08 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:28:34 +0800 Subject: [PATCH 113/306] fused_qkv_rope in UOp try 2 (#17619) * fused_qkv_rope in UOp try 2 * dont need that * less --- extra/thunder/amd/fa.py | 37 +++++++++++---- extra/thunder/amd/fused_qkv_rope.cpp | 69 ---------------------------- test/backend/test_asm_gemm.py | 2 + test/backend/test_llama_kernels.py | 19 ++++---- 4 files changed, 38 insertions(+), 89 deletions(-) delete mode 100644 extra/thunder/amd/fused_qkv_rope.cpp diff --git a/extra/thunder/amd/fa.py b/extra/thunder/amd/fa.py index 685a1b8e77860..36cdfad824d81 100644 --- a/extra/thunder/amd/fa.py +++ b/extra/thunder/amd/fa.py @@ -19,16 +19,33 @@ def _sharded_empty(shape:Tensor, ref:Tensor, axis:int|None, dtype:DTypeLike|None @functools.cache def custom_fused_qkv_rope_forward(q:UOp, k:UOp, v:UOp, xqkv:UOp, freqs_cis:UOp, device:str, arch:str, B:int, N:int, H:int, H_KV:int, D:int): - code = (pathlib.Path(__file__).parent / "fused_qkv_rope.cpp").read_text() - threads = 256 - thread_idx = UOp.special(threads, "lidx0") - block_idx_x, block_idx_y = UOp.special(B, "gidx0"), UOp.special(N, "gidx1") - sink = UOp.sink(q.base, k.base, v.base, xqkv.base, freqs_cis.base, thread_idx, block_idx_x, block_idx_y, - arg=KernelInfo(name="fused_qkv_rope_forward")) - compile_args = ["-std=c++20", "-ffast-math", f"-DATTN_B={B}", f"-DATTN_N={N}", f"-DATTN_H={H}", - f"-DATTN_H_KV={H_KV}", f"-DATTN_D={D}", f"-DTHREADS_PER_BLOCK={threads}"] - lib = HIPCCCompiler(arch, compile_args).compile_cached(code) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=code), UOp(Ops.BINARY, arg=lib))) + group_size = H // H_KV + q, k, v = q.reshape(B, N, H, D), k.reshape(B, N, H_KV, D), v.reshape(B, N, H_KV, D) + xqkv = xqkv.reshape(B, N, H_KV, group_size + 2, D) + b, n = UOp.range(B, 0), UOp.range(N, 1) + pair = UOp.range(D // 2, 2) + even = pair * 2 + c = freqs_cis[0, n, 0, pair, 0].cast(dtypes.float) + s = freqs_cis[0, n, 0, pair, 1].cast(dtypes.float) + ordered:UOp|None = None + for kvh in range(H_KV): + q_out, k_out, v_out = (x.after(ordered) if ordered is not None else x for x in (q, k, v)) + x_in = xqkv.after(ordered) if ordered is not None else xqkv + stores:list[UOp] = [] + for rep in range(group_size): + a = x_in[b, n, kvh, rep, even].cast(dtypes.float) + bb = x_in[b, n, kvh, rep, even + 1].cast(dtypes.float) + h = kvh * group_size + rep + stores += [q_out[b, n, h, even].store((a * c - bb * s).cast(q.dtype)), q_out[b, n, h, even + 1].store((a * s + bb * c).cast(q.dtype))] + a = x_in[b, n, kvh, group_size, even].cast(dtypes.float) + bb = x_in[b, n, kvh, group_size, even + 1].cast(dtypes.float) + stores += [k_out[b, n, kvh, even].store((a * c - bb * s).cast(k.dtype)), + k_out[b, n, kvh, even + 1].store((a * s + bb * c).cast(k.dtype)), + v_out[b, n, kvh, even].store(x_in[b, n, kvh, group_size + 1, even]), + v_out[b, n, kvh, even + 1].store(x_in[b, n, kvh, group_size + 1, even + 1])] + ordered = UOp.group(*stores) + assert ordered is not None + return ordered.end(pair, n, b).sink(arg=KernelInfo(name="fused_qkv_rope_forward")) @functools.cache def custom_fused_qkv_rope_backward(dxqkv:UOp, dq:UOp, dk:UOp, dv:UOp, freqs_cis:UOp, diff --git a/extra/thunder/amd/fused_qkv_rope.cpp b/extra/thunder/amd/fused_qkv_rope.cpp deleted file mode 100644 index fcb7ee6bf9d4d..0000000000000 --- a/extra/thunder/amd/fused_qkv_rope.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -#ifndef ATTN_B -#define ATTN_B 2 -#endif -#ifndef ATTN_N -#define ATTN_N 8192 -#endif -#ifndef ATTN_H -#define ATTN_H 32 -#endif -#ifndef ATTN_H_KV -#define ATTN_H_KV 8 -#endif -#ifndef ATTN_D -#define ATTN_D 128 -#endif -#ifndef THREADS_PER_BLOCK -#define THREADS_PER_BLOCK 256 -#endif - -constexpr int GROUP_SIZE = ATTN_H / ATTN_H_KV; -constexpr int HALF_D = ATTN_D / 2; -constexpr int PACKED_D = (GROUP_SIZE + 2) * ATTN_D; - -extern "C" __global__ __launch_bounds__(THREADS_PER_BLOCK) void -fused_qkv_rope_forward( - __hip_bfloat16* __restrict__ q, - __hip_bfloat16* __restrict__ k, - __hip_bfloat16* __restrict__ v, - const __hip_bfloat16* __restrict__ xqkv, - const __hip_bfloat16* __restrict__ freqs_cis) { - const int b = blockIdx.x; - const int n = blockIdx.y; - const int bn = b * ATTN_N + n; - const int packed_bn = bn * ATTN_H_KV * PACKED_D; - const int q_bn = bn * ATTN_H * ATTN_D; - const int kv_bn = bn * ATTN_H_KV * ATTN_D; - - if (threadIdx.x < HALF_D) { - const int pair = threadIdx.x; - const int even = pair << 1; - const float c = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 0]); - const float s = static_cast(freqs_cis[((n * HALF_D + pair) * 2) + 1]); - - for (int kvh = 0; kvh < ATTN_H_KV; kvh++) { - const int base = packed_bn + kvh * PACKED_D; - - for (int rep = 0; rep < GROUP_SIZE; rep++) { - const int qbase = base + rep * ATTN_D; - const int h = kvh * GROUP_SIZE + rep; - const float a = static_cast(xqkv[qbase + even]); - const float bb = static_cast(xqkv[qbase + even + 1]); - const int out = q_bn + h * ATTN_D + even; - q[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - q[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - } - - const float a = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even]); - const float bb = static_cast(xqkv[base + GROUP_SIZE * ATTN_D + even + 1]); - const int out = kv_bn + kvh * ATTN_D + even; - k[out] = static_cast<__hip_bfloat16>(a * c - bb * s); - k[out + 1] = static_cast<__hip_bfloat16>(a * s + bb * c); - v[out] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even]; - v[out + 1] = xqkv[base + (GROUP_SIZE + 1) * ATTN_D + even + 1]; - } - } -} diff --git a/test/backend/test_asm_gemm.py b/test/backend/test_asm_gemm.py index e18458f808130..6e34b996c0279 100644 --- a/test/backend/test_asm_gemm.py +++ b/test/backend/test_asm_gemm.py @@ -1,4 +1,5 @@ import unittest +import functools from tinygrad import Tensor, Device, dtypes, Context from tinygrad.helpers import getenv, system, DEV from extra.gemm.cdna_asm_gemm import asm_gemm, hk_bf16_atb_gemm @@ -9,6 +10,7 @@ # Use DEV=NULL:HIP:gfx950 to also test the assembly def is_cdna4(): return Device[Device.DEFAULT].renderer.target.arch.startswith("gfx950") +@functools.cache def has_hipcc(): try: system("hipcc --version") except Exception: return False diff --git a/test/backend/test_llama_kernels.py b/test/backend/test_llama_kernels.py index f03a57b393cfd..ee8538c3214bf 100644 --- a/test/backend/test_llama_kernels.py +++ b/test/backend/test_llama_kernels.py @@ -99,22 +99,20 @@ def test_multi_tensor_local_shard_amax(self): assert_kernel_count(2) self.assertEqual(out.tolist(), [[0., 7., 14., 21.], [28., 35., 42., 49.], [120., 135., 150., 165.], [180., 195., 210., 225.]]) -@unittest.skipUnless(has_hipcc() and Device.DEFAULT == "AMD", "requires hipcc to compile and amd device to run") class TestFusedQKVRoPE(unittest.TestCase): SHAPE = (2, 8192, 32, 8, 128) + def setUp(self): + if dtypes.bfloat16 not in Device[Device.DEFAULT].renderer.supported_dtypes(): self.skipTest("test uses bf16 inputs") + def rand_bf16(self, *shape:int) -> Tensor: return (Tensor.randn(*shape) * 0.1).cast(dtypes.bfloat16).contiguous().realize() - def freqs_cis(self) -> Tensor: - _, N, _, _, D = self.SHAPE - return precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() - - def test_llama31_8b_forward(self): + def test_forward(self): Tensor.manual_seed(0) - B, N, H, H_KV, D = self.SHAPE + B, N, H, H_KV, D = 1, 32, 8, 2, 16 GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = (Tensor.randn(1, N * 2, 1, D // 2, 2) * 0.1).cast(dtypes.bfloat16).contiguous().realize() x = self.rand_bf16(B, N, H_KV * (GROUP + 2) * D) q, k, v = fused_qkv_rope(x, freqs_cis, H, H_KV, D) @@ -131,12 +129,13 @@ def test_llama31_8b_forward(self): self.assertTrue(k.allclose(k_ref, atol=2e-2, rtol=0).item(), "K forward mismatch") self.assertTrue(v.allclose(v_ref, atol=0, rtol=0).item(), "V forward mismatch") - def test_llama31_8b_backward(self): + @unittest.skipUnless(has_hipcc(), "backward kernel requires hipcc to compile") + def test_llama31_8b(self): Tensor.manual_seed(1) B, N, H, H_KV, D = self.SHAPE PARTIALS = 2 GROUP = H // H_KV - freqs_cis = self.freqs_cis() + freqs_cis = precompute_freqs_cis(D, N * 2).cast(dtypes.bfloat16).clone().realize() dq = self.rand_bf16(B, N, H, D) dk_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) dv_partial = self.rand_bf16(B * PARTIALS, N, H_KV, D) From 57d1104a92225412b895f78ef847f101a7de2186 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 19 Aug 2026 23:51:15 -0700 Subject: [PATCH 114/306] bump amd firmware (#17621) --- tinygrad/helpers.py | 2 +- tinygrad/runtime/autogen/am/__init__.py | 2 +- tinygrad/runtime/autogen/am/fw.py | 145 ++++++++++++------------ 3 files changed, 75 insertions(+), 74 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 66014d594cb37..f3dbca7c3c557 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -490,7 +490,7 @@ def fetch_fw(path:str, name:str, sha256:str) -> bytes: if sys.version_info >= (3,14) and (p:=pathlib.Path(f"/lib/firmware/{path}/{name}.zst")).is_file(): from compression.zstd import decompress if hashlib.sha256(b:=decompress(p.read_bytes())).hexdigest() == sha256: return b - return fetch(f"https://gitlab.com/kernel-firmware/linux-firmware/-/raw/1e2c15348485939baf1b6d1f5a7a3b799d80703d/{path}/{name}", + return fetch(f"https://gitlab.com/kernel-firmware/linux-firmware/-/raw/0a6871b19abf5d6e024b5d208b101ae53e7fa0de/{path}/{name}", subdir="fw", sha256=sha256).read_bytes() # *** Exec helpers diff --git a/tinygrad/runtime/autogen/am/__init__.py b/tinygrad/runtime/autogen/am/__init__.py index 54c5f2cc92a68..b413cc6ecc269 100644 --- a/tinygrad/runtime/autogen/am/__init__.py +++ b/tinygrad/runtime/autogen/am/__init__.py @@ -8,7 +8,7 @@ rocm_src="https://github.com/ROCm/rocm-systems/archive/cccc350dc620e61ae2554978b62ab3532dc10bd9.tar.gz" AMD, AMDINC = "{}/drivers/gpu/drm/amd", "{}/drivers/gpu/drm/amd/include" inc, kern_rules = ["-include", "stdint.h"], [(r'le32_to_cpu', ''),] -fw_src="https://gitlab.com/kernel-firmware/linux-firmware/-/archive/1e2c15348485939baf1b6d1f5a7a3b799d80703d/1e2c15348485939baf1b6d1f5a7a3b799d80703d.tar.gz" +fw_src="https://gitlab.com/kernel-firmware/linux-firmware/-/archive/0a6871b19abf5d6e024b5d208b101ae53e7fa0de/0a6871b19abf5d6e024b5d208b101ae53e7fa0de.tar.gz" pmc_src="https://raw.githubusercontent.com/ROCm/rocm-systems/cccc350dc620e61ae2554978b62ab3532dc10bd9/projects/rocprofiler-compute/src/rocprof_compute_soc/profile_configs/counter_defs.yaml" reg_files = { diff --git a/tinygrad/runtime/autogen/am/fw.py b/tinygrad/runtime/autogen/am/fw.py index 4b6d62e146c4f..b4fa3880ab92a 100644 --- a/tinygrad/runtime/autogen/am/fw.py +++ b/tinygrad/runtime/autogen/am/fw.py @@ -1,81 +1,82 @@ hashes = { - 'psp_13_0_0_sos.bin': 'b5592f46885585b935e013f46c949db8ff2f15c0b346caf70e7fcd2776623d13', + 'psp_13_0_0_sos.bin': '4a51299f6d0a15bbba9694419f7891e6accc01dbd2dd67c06add7bfd75a45ac6', 'psp_13_0_10_sos.bin': '0bcaaad9cd8578d3841ae69155a6bd4fc3ceae8f4fb5a6ba4f576e7ace94d1d9', - 'psp_13_0_12_sos.bin': '89da90bf4286b38678b1fd175c78462a426afa3d258d15872cd14072d7098b9b', - 'psp_13_0_14_sos.bin': 'a4f0d5f76d27b77409ec0b71d7cc6a848ddfd29f8c84f3003edf74ad3999fb7d', - 'psp_13_0_6_sos.bin': '27657daa0f91ad8095d3610224a7de748b8b348a4cb211ecb5fccabe47369716', - 'psp_13_0_7_sos.bin': 'ef1af0ecea38abbac6f85cce71789f19848c498d0cb8ef13748dab2d65b23c31', + 'psp_13_0_12_sos.bin': '7113a165c75c232d4cb7193a920b503e0bf082689adde3b45fdc38f58bfd18b3', + 'psp_13_0_14_sos.bin': 'db863768cb25e806b68033e9237e0869f9f3603119df4d369ff4d80418d585d0', + 'psp_13_0_15_sos.bin': '3b28d53e75a88131155e3931378ac8434eca4880ada9211d3b4e8915b6289583', + 'psp_13_0_6_sos.bin': '36cce3a9441a0dcde81badd8fcf0416de8e4c39a7707865eff4d9d75e6bb0466', + 'psp_13_0_7_sos.bin': '94db505fa6482f258c33a0a8d412050f6d843ab4ada368252e988f82f8a26fa8', 'psp_14_0_2_sos.bin': '7b538448b57d4f9dd06b2eea90d4f86a16e65e3027cdecee8db71c2c5f1fa243', - 'psp_14_0_3_sos.bin': '23bea01a0c6f36d00759d0765d46cb4cb4aa87398b2fbccacbf547a890c0bf51', - 'smu_13_0_0.bin': '2ffac37fd8534965eeba19755db0e5ec80278213487dc4af0fbc8453befb64b1', - 'smu_13_0_0_kicker.bin': '7f83656a2a89b7fce1c8a85e96d91cd8265a91fe883a7027f1a0ed18ced501de', - 'smu_13_0_10.bin': 'daedb9cbdf48942be7ffe00d31b7c16bb36e11ff5a9d7495f218e95c07717b71', + 'psp_14_0_3_sos.bin': '28469a0857c813c54a0492423cdf0b0caf757428400036377e19c47e5af62478', + 'smu_13_0_0.bin': '93e46a5526f19dcc3d13bfd9e23f88bc8eee52138bfe9caf0951b4eef5e49914', + 'smu_13_0_0_kicker.bin': 'd0ef51d9ed06d0c17e06667302be21e7aedd86ed7a72be6e2f55b102214131cc', + 'smu_13_0_10.bin': '9376ae64149e6b0b684898ffbc12c2230f8c50a2e9447dc7dafc95c0c16b5227', 'smu_13_0_14.bin': 'a4f36de75fdcecd8000246762e027b4be489b6787afea57675225b0b39d35625', 'smu_13_0_6.bin': 'ad7232264e8c57c2094244fbdd5a55d7a4575ffe9b44d229884bc0b6a44fb0b1', - 'smu_13_0_7.bin': 'ccecc0fd0196b9613c920a51c2fd9436e739ff19dda5bdf74d97562387231732', - 'smu_14_0_2.bin': '6951995d1d606f4dc60c895f19d34ed18aa40e62129f83d8510c45e8aa9ae2fc', - 'smu_14_0_3.bin': 'df230947ddb7bbfd6e77d1280001db886e69adf2b2a448b47fa668a48bc0009f', - 'smu_14_0_3_kicker.bin': '8ddc1da5b4e1619796c2cc81f19f388a35bf7d78bfe476cee559625589cb4dc7', - 'sdma_4_4_2.bin': '456061b814268425843537da6f2191c8861d4e1a18d4c5d90c44ea6be18c78ff', - 'sdma_4_4_4.bin': 'af47a2940e72b932d3e3a7e8f34f7a182624e5e433f7c56dff939ca5549cd33a', - 'sdma_4_4_5.bin': '6127baabea3de7b18db3868c983b02c0fbf2cd75997f7f11241a5b1be27e5134', + 'smu_13_0_7.bin': '68ec18bd605e680085c927ff72c609f8c771aff0718d0cfab58a3744dff8e5b7', + 'smu_14_0_2.bin': '1b2054e3f710d1ab8dbdf6ff35914ad376b51caa6337831260c955add874b2ee', + 'smu_14_0_3.bin': '4e1522d3c96c1028be2961dfcfc5f1ff783fb77724b260a99c4c8b4a901ef3fe', + 'smu_14_0_3_kicker.bin': '9ff142656ae5f57be1b5ecc134e9da8f76650e793fbc5c499acd75094ff24453', + 'sdma_4_4_2.bin': 'ff885711d2d5d75ceed51cf239e93c882584b918cd5d5d1ff58ee5aecc0c50ae', + 'sdma_4_4_4.bin': '06a9d4d02c187844313a78469321d6091e59a334f0ce3b61d770d810c984d70b', + 'sdma_4_4_5.bin': 'e2a30faa9403933fbfba7ce8e9feba460fff6ecdb15304818d24c9f3eeaad0a6', 'sdma_5_2_6.bin': '3a163db00eb7e4752be8adbd61cf7dd8f08d924e59a6f798ced7dfcd89f340ed', 'sdma_5_2_7.bin': '16fe80dc866b323e15a06f51646ef0f036878ad34da66921fcdb8167207d6b2b', - 'sdma_6_0_0.bin': '0f3da6b211f376356335b41be07149f650c10cfa4e23f7e25d53836006ed11f5', - 'sdma_6_0_1.bin': 'ff565d3c215a30737560d4e3df6fc2c637738407e91d212fb200fdfb185b6744', - 'sdma_6_0_2.bin': '398380184bb69113ef4c8964a3b55f6184deb0c1ffd96c9683490a3eec3ba8f3', - 'sdma_6_0_3.bin': '0e8a83513087db865ba926f8b65cfb003fd41098f707e178d7a7ae2941fed0b1', - 'sdma_6_1_0.bin': '22e55d0ad5f0247a7f0fffc67cfd3161b39f24ad6062ff3c91ec7ff38bd7e1e1', - 'sdma_6_1_1.bin': '74533a581b8e3e2743b3c9c803d0666405e80898c4a630acefed82cb6b516ba2', - 'sdma_6_1_2.bin': '4fe04b0286ec739b0414e8aee17e62e85e691f0246d1d9b56bc18a1219072314', - 'sdma_6_1_3.bin': '35c9ed7e3a237c0d4a83b4975c63b62488f72aeafbb648342f384618e103f66b', + 'sdma_6_0_0.bin': '82cd01a24171af12de6c7ac4ee7471aa2dfcf51f9677e7bae9cd4c75e07761ee', + 'sdma_6_0_1.bin': '708c2c2c45262c98ebe8e34e32c3f1ce8eb5b007bab560c9ea9b576a8e4d6768', + 'sdma_6_0_2.bin': '16c374344d2894da751f7028f9ec1f7520035fa9548d8c750d99a00a4afa86c7', + 'sdma_6_0_3.bin': 'd47ac4db523aa35d77b27d57c35d4c04f431229dec6c0d667c36d98b985a6933', + 'sdma_6_1_0.bin': '85f9f3714de68eee74cdf4852d709bc36a5c73a00e943b707bc2ce10d6b7bab4', + 'sdma_6_1_1.bin': 'e7b7a23923ab691665e6ad16bbc8431a92f7c049da4b0b19a82c45fba03d4979', + 'sdma_6_1_2.bin': '5947d78eb308a3f6a62d772c5a6493b21439c73eac139f9c22f080f660b4f4f3', + 'sdma_6_1_3.bin': '8c651f32cbf030b6239ecc44f01bc9f5d5a193f333e21f2103736aff33227361', 'sdma_7_0_0.bin': 'beaafb53993a106edd392392d5896245ae2a957c6d0f495d0002eec72ad8ad38', 'sdma_7_0_1.bin': '73c29e1c1714ebc95d2221ba56e187910902891593010653bf9518937e414a59', - 'gc_10_3_6_pfp.bin': '793d678427887a0e724c79e356440aec33e6d1301f2a4e63543500249ebec064', + 'gc_10_3_6_pfp.bin': '042f5d2d223aac6a62b500a47d0d0bf33984200110da0ffca4fe5df9a96571c0', 'gc_10_3_7_pfp.bin': '3ae29aac3f424f7de97f82ce7158beba69509afb2dcbf1a428dc315df474a524', - 'gc_11_0_0_pfp.bin': 'e175cb0f580a38c961a6f7366142c08e413995f57f78f39795368b15442df8a3', - 'gc_11_0_1_pfp.bin': 'f5bf21dfbd9e72a30b4caf4704282c27854710c1b7c4affbb2a19530466b12a8', - 'gc_11_0_2_pfp.bin': '001c4dec1119e29314d725cc1280fc4f0cd9cabdf61ea5ee2260cfd4e62ec141', - 'gc_11_0_3_pfp.bin': '0488034c85be97125e39e860308d33c3f76a01df8250092a32d4d55acb2526fd', - 'gc_11_0_4_pfp.bin': '5ae8b7bb6316f87ae8b978354c088e3bd8c890959382d72886377cda25b1ffd1', - 'gc_11_5_0_pfp.bin': '0124f540871a7759fa8aaae046d458dfb34aeea12a1183ff962c3f1a33067d5a', - 'gc_11_5_1_pfp.bin': '7794ea46d0d3cf9cb3f7938affbdf09dd7a9970340da5cd02b774cb393436d24', - 'gc_11_5_2_pfp.bin': '55e64741de28c506524959f7f696713a72aafe46f49ccd827781d67a9475b386', - 'gc_11_5_3_pfp.bin': 'ce805040fb347fddbc89b2715e66b446865dda9e2056a9b233269b72bc09c387', - 'gc_12_0_0_pfp.bin': '16bfd64c10fe73b5e760055069a60e5841dba16c0ed4edb56c20d675e23901f6', - 'gc_12_0_1_pfp.bin': '49efb319305c5fffd90ac1eef7d7a0bdec72998ecb5cf4526996311788a53dc3', - 'gc_10_3_6_me.bin': '141b59faad3f2f1be16a2178833b7ca8e97519e1e844c8fda6689572c3767902', + 'gc_11_0_0_pfp.bin': 'b360393c8629144b194f69a3cd961ed509331feff7a5cc1e4eb21c901da2710a', + 'gc_11_0_1_pfp.bin': 'fb1ee527c05c55679c80a8bcf60fbb533724891baeb0eabc2917fc44e63a45dc', + 'gc_11_0_2_pfp.bin': '9020f53788ad881fa01aa656fc082f9f8d3cdfc81f70aaac0bed6e6001491128', + 'gc_11_0_3_pfp.bin': '362db904fa16c1fea2af7ad1295532434df7f85662b4a69332f51ae6c7290b61', + 'gc_11_0_4_pfp.bin': 'aad22ca342c47d857bc1107a9aa9127e5e4ba7f7fd42d432213b1850bda1f4e1', + 'gc_11_5_0_pfp.bin': '82ccf0265d841351183b011a79422799431f0c11f6d11165d64d7dfe404bda31', + 'gc_11_5_1_pfp.bin': '633404d8db1dc03fe997f7d0d0e15ef908069727abaf9de55841be3f3c97348b', + 'gc_11_5_2_pfp.bin': 'baee1456dd1800cdaedd4998c2dd7d76cdc0cf0ec928679fe67b02485905ea2c', + 'gc_11_5_3_pfp.bin': 'fee840b049b5e082215df72a93fad80a64f07ef6f638408a2d56fae97449a2cb', + 'gc_12_0_0_pfp.bin': 'd1b043c60920e509e5c8f9677221fb78ff7985f68b605e8f39a04a57333a9366', + 'gc_12_0_1_pfp.bin': '9d8d6188efeca5ef05482d9299c4f102fab7db3dae51a23e59de9baa34997123', + 'gc_10_3_6_me.bin': '776d2299bc4f3abffd4a7999f5a21a4e38aced8b6b4c199a83610dbabf08176d', 'gc_10_3_7_me.bin': '9eb0b56e9bcc9dad5d53437b162226fcb37e5df102832260f1232832f3658edf', - 'gc_11_0_0_me.bin': 'f8fba8a63dd4293b8fc1e4aab78b6fac630e575d1d62838c7996d9210f82aea1', - 'gc_11_0_1_me.bin': '5030040b00955de94876341ec64ea43b96640413d7a03dc460a83c8386bf76e0', - 'gc_11_0_2_me.bin': '0f21fd43f1dfbc6ccced9a2b3774de25c993c61a689aabab8b45333937b7945e', - 'gc_11_0_3_me.bin': '3acb5061dba342ade81d329d1932f19ec01f0c5bf44e6e3568008a951a351bac', - 'gc_11_0_4_me.bin': 'e4f1f6abcd213d54ad9e885d9f550083b0e2f67d983566015e8a53981e1cb155', - 'gc_11_5_0_me.bin': '8f906b64d0a29503daa662c93ec44d076fcac11b78f70cd50ce0af2b500a05a6', - 'gc_11_5_1_me.bin': '7e42602bcbaf1e511f8b4f6ed2246844ad1f6e351ce2b663d89062a7be263663', - 'gc_11_5_2_me.bin': 'aae26255d8efff81e0e3bbcb727efb8b837d8e25fe85c708545f5328f1077b50', - 'gc_11_5_3_me.bin': '93cd588348b16fe432609fe8da6e6b5da0a52da5c5884882aecf7b1001f72700', - 'gc_12_0_0_me.bin': 'd7eba5197f2580f32b8256b1d9cb68e723e9e644293a34446a7913e3c093cba5', - 'gc_12_0_1_me.bin': '365e7f193b39cbb10d3af44905fefaca0e9844721801755276baebac7b19c1ea', - 'gc_10_3_6_mec.bin': '247943415658159704a21f670dd7b3e7cb2d2fc0c17b000a5098715979c8d95e', + 'gc_11_0_0_me.bin': 'f2f5a793d811c6abad1a18af0fcf7694c443478f224176da86650c22aa71ca7a', + 'gc_11_0_1_me.bin': '476db2ec7e33d1e126b1736649208443e3ccc68aa60e4978574cdbced2b26543', + 'gc_11_0_2_me.bin': 'f5fe48f97acbd3ce13b35929290bfbac01ce522631cc91dcef1fdeb3ff35c8ed', + 'gc_11_0_3_me.bin': 'd02c25070e5bdf0ec0146f5c9d6d2f8b86de43bd2a318a0b67eb5201963bafdc', + 'gc_11_0_4_me.bin': 'f075220f75ffe43eacc5986ff8448946c27405e632764fda83323e7ec8d55566', + 'gc_11_5_0_me.bin': '338019a1fcdab39729e3f492ffc9f5970c2c81b12c8a4f431494ca28cfdadedf', + 'gc_11_5_1_me.bin': '4c4dd30c22d4f7f2c5d3a19c645f505e30cdac115a91c65791e2651b22932175', + 'gc_11_5_2_me.bin': 'cab2999186d26c0e9a3d46b5a43d2854d88be880cb764c096ad2b43038566384', + 'gc_11_5_3_me.bin': '94e2d74e834725b3d51e03e830160e95c56f3a31e93f5d61c852afd8fe8cc779', + 'gc_12_0_0_me.bin': 'fb10cb3535ae4a6a8fb3e78166cf30c5b717341b1f20cde73065c62b642adfed', + 'gc_12_0_1_me.bin': '56a1ae0031aa938f6b61348a56404ab2cee92f1f45630fc82a801aa4d908f98a', + 'gc_10_3_6_mec.bin': '7003c4a77537e9edaf67064104cd9371fac38a84f71f948349140b28d3c210e8', 'gc_10_3_7_mec.bin': 'ee58a523375bcf5b89400b32b801f95e182b632a26bce4f2bed5c07928d486dc', - 'gc_11_0_0_mec.bin': '801a09c9bf06188260db9b51ad8f978f15d84c72ca91b90643a2ef8af4074776', - 'gc_11_0_1_mec.bin': '6afadcb7504bb11bcc9d4a205cdf73f7934a615e28f178fcf7285971df2ccd05', - 'gc_11_0_2_mec.bin': '0da0edee28c73a6fa1191f77853d380ec2503cbf43e0aaae4617f32f1f8a48fa', - 'gc_11_0_3_mec.bin': '323cfa6658b6b5169830f852e2ff0552acae8dfb9e44b42c63de7b2900d3fd9e', - 'gc_11_0_4_mec.bin': '5d89cf6b60354f3746c2cbd1ff0cb1a741556ca20d72745242cb69b553d0985c', - 'gc_11_5_0_mec.bin': 'a01c324ab14ec89792449a621a541829b9af26865019027a411a14b910145dfa', - 'gc_11_5_1_mec.bin': 'eab05719371caa68df09d4f7574e3958a3c4f5044ab3c7b0d2b214add0c6d1c4', - 'gc_11_5_2_mec.bin': 'a374b2335802e24f8b9a3ce40000a1d37a52a14eb87099bebcc6680c27cc93e5', - 'gc_11_5_3_mec.bin': '165025437cba80dd32c19ebbc83b756fa7adac7053ff7780ba4aa2f8089c6a3f', - 'gc_12_0_0_mec.bin': '1931593440b8f9423580d9e2cdc5b34e7c682cdffe1ca4b74b0c2f6a0420236d', - 'gc_12_0_1_mec.bin': 'f57541688a5108730bf210663f1137ffc2121f3acfe614a6de09ec1982c69a2f', - 'gc_9_4_3_mec.bin': '3159176e72301fb88dc416721fb3d0ab82ece484cf93a43c3f37430c7e6673a1', - 'gc_9_4_3_sjt_mec.bin': 'd19468dbb47849640bd0e6cdc8d7e25a3c8442c7ca2ca81357702e0d6baab50f', - 'gc_9_4_4_mec.bin': '5004f73e43db2dd45e77d65942e33d4a69e7157618cfd23944c30f801c77a0f3', - 'gc_9_4_4_sjt_mec.bin': '627a9e98102e70fe3bf0947eb764187f29f5e775d1130c7310e0ba5fc0502dbe', - 'gc_9_5_0_mec.bin': 'c5eca4311a6f6e8f81cf41c2c46941d5dcf90789ee8326901da2dfc86ac14c31', - 'gc_9_5_0_sjt_mec.bin': 'f162e509379288e3f3b1eead541b315c2262d625d433287ecd34ca185614d312', + 'gc_11_0_0_mec.bin': '1dd1de8ecf5455ea4719c502b64b32ac18763d5601128c01b4a4a36211a122c2', + 'gc_11_0_1_mec.bin': '505ae64eccb2e4b4751fe18ec1b584e1f6b4c81d0f5ec089afbcf378cad59711', + 'gc_11_0_2_mec.bin': '19bf080d6e672de5ed3fb86e3fdbdda4d700d8e3bda2dbdcc923101484ad645b', + 'gc_11_0_3_mec.bin': 'a37bc1a4e245300a5c3e26da34ea213842447d7df6c5c81e9fc78887a2fde26f', + 'gc_11_0_4_mec.bin': '850d5302b4fee6022f42f706c2de103531b45b7794a45f2d6dce6015767a1ad6', + 'gc_11_5_0_mec.bin': '5e022bae6638967d82e2b1077e3024f52bc83b3cb850aa31fba51469c7517c4c', + 'gc_11_5_1_mec.bin': 'e49964d5e58686c53e66d98d4e3b9fab70e98fad3b28379c6e60aed03c83ee80', + 'gc_11_5_2_mec.bin': '9691d7bff5d2c933d8eecb7d171635612a76a2dd1441cffcd65a8a02bdb5a2c5', + 'gc_11_5_3_mec.bin': 'd368f3886b9245dd0d21d57fccfd8aa7e872c2564e23f292abe735348121277e', + 'gc_12_0_0_mec.bin': '9c7602d6ebf1f7e6ec7a5d1ceefded18f35fa1c08fbea1e3e1a0d78d519db8e8', + 'gc_12_0_1_mec.bin': 'caf1dbaf72b0ef0c4c973947414033aeec002994f63967bb53e9165195a3c2c3', + 'gc_9_4_3_mec.bin': '99bc12230f00b930cf286105a35cc6110d87461cd48cb4fdf3cb6caff73ac1e7', + 'gc_9_4_3_sjt_mec.bin': '2945dbd098c4158870df7dc4ccb33d40031fd1cce37cdbe5df291d8941d03567', + 'gc_9_4_4_mec.bin': '7f14258f8301d2717e0a707ccfad7b3091af478b0df6d5134adfd56caa7429d8', + 'gc_9_4_4_sjt_mec.bin': '0bbef279bbc07c502098b80765b876f69fcda9834e5ed269a7d8236c85e89e19', + 'gc_9_5_0_mec.bin': '0c39078c53e10e99538901df5fc14e7f1b1f3639ea825b1b3126ae87a28b2464', + 'gc_9_5_0_sjt_mec.bin': 'a769745367567fc6f389695aa5f48c154c07560e21a93052185e19f950205240', 'gc_11_0_0_imu.bin': 'b4f8fc056b45709a6abf48e7885fb1b4ab8d3cc092cbfa2c554a78564a6403bc', 'gc_11_0_1_imu.bin': 'ac71f4eec713fc35b4a1fe27531e3eb04edd81eeac2cef64df01ac50d8510805', 'gc_11_0_2_imu.bin': '9befca62b0b0cfd252c3df4a9edca295526f4d43821cd99a6326454995a6ca2d', @@ -90,17 +91,17 @@ 'gc_10_3_6_rlc.bin': 'acfbac75c0dcfbfe40e222640ef17eb3dc8d206d30bc3863f275f2dd1cb132a5', 'gc_10_3_7_rlc.bin': 'a02585ebe3b36d942e883057119572d9497600c52fc65b8a523487eb65d874f2', 'gc_11_0_0_rlc.bin': 'dabd49039772d02f5fd5e48dc21d35ad52a6b1283b470dabca86ca159c4c7c8e', - 'gc_11_0_1_rlc.bin': '86145719a58e9428562930c6b5ee3b6ced4701d34a80d0b4d84d6026c93134f2', + 'gc_11_0_1_rlc.bin': '5f07dc1f0a75ecd9cb56d805ea869184a50ed9e43d811ebf833b8906534650ef', 'gc_11_0_2_rlc.bin': 'b43eb2fd0600f50a1a5796bc9983d6b39b5c20960234920f5e89cb362193e0b8', - 'gc_11_0_3_rlc.bin': '29b0b456f5b53076ddffa6f09de3bb697219e8e7b33504bf6c197e8b858426dc', - 'gc_11_0_4_rlc.bin': '823573078b608108fbe4dd8176c396ec582632913db9c59a512d82b068f8eba0', - 'gc_11_5_0_rlc.bin': '68cd85567f4f2f8d6b80db294988806d956bf826979c3597daccb71c7ee6aadd', + 'gc_11_0_3_rlc.bin': '890d8e0123efb40c0179dd8ac3e9af073a0b87cbbccfec1db54e5ed2315a8d39', + 'gc_11_0_4_rlc.bin': '257ced82d7bec41249b06592ee0c44fb8f9262de2c6af9c52dc6f6a8a702063e', + 'gc_11_5_0_rlc.bin': '0dc8b6ef5530a4a53938c8baa0d49cd458607d95233237859fa98d44feb3e985', 'gc_11_5_1_rlc.bin': '92731ecabbeb77865fb71787b4268dc738a58779f1190bdc2056482cb88a08f6', - 'gc_11_5_2_rlc.bin': 'ef3a9209d3eccfbe18fce9e972c146ac283719798bb788096c176b796dc9aee5', + 'gc_11_5_2_rlc.bin': 'c9ad70b8ac309257cb8929bb6b4efa6b551ec1e5229d7a419332a9797f31fc9e', 'gc_11_5_3_rlc.bin': '10a68940c6258d5818d9c05fd98eb0ccc8d5aee99b2769fbad30e5abd0d9327e', 'gc_12_0_0_rlc.bin': '6436b582734a413456fff3d3c7195e71cc9e78a7ed31ee21c83ffd6fae1ad186', 'gc_12_0_1_rlc.bin': '6ba4459532246a5c415d3cb33c9b1248294e48f67b827e2accb292a8d1a5c0ec', - 'gc_9_4_3_rlc.bin': '5345d388712d547b0ae16f199ad5ccadb65643584b3efa7817049ddeb3fdcd12', + 'gc_9_4_3_rlc.bin': '54cbd0de3a0ec35d2e58e992babeee2a237f870ccdf37e734652e4daeeba59d5', 'gc_9_4_4_rlc.bin': 'e0c3585c72f8136670ca63e607fba32c1ae4948f493f13e33fc4d466bd6318a8', 'gc_9_5_0_rlc.bin': '9b1268f5751153fe57f527c9acb417bfa53ed42c9bc083c9d3da2ba61fe5fdc4', } \ No newline at end of file From c117da9850a9d6f0040604f17540868155423426 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:00:23 -0700 Subject: [PATCH 115/306] safe changes for mi350p (#17620) * safe changes for mi350p * bump amd firmware --- tinygrad/runtime/ops_amd.py | 2 +- tinygrad/runtime/support/am/amdev.py | 14 ++++++++++++-- tinygrad/runtime/support/am/ip.py | 17 ++++++++++------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 6c0878af25005..524987df448f3 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -842,7 +842,7 @@ def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return ((self.drm_dev_info.cu class PCIIface(PCIIfaceBase): def __init__(self, dev, dev_id): - super().__init__(dev, dev_id, vendor=0x1002, devices=((0xffff, (0x74a1,0x744c,0x7480,0x7550,0x7551,0x7590,0x75a0)),), vram_bar=0, + super().__init__(dev, dev_id, vendor=0x1002, devices=((0xffff, (0x74a1,0x744c,0x7480,0x7550,0x7551,0x7590,0x75a0,0x75a8)),), vram_bar=0, va_start=AMMemoryManager.va_allocator.base, va_size=AMMemoryManager.va_allocator.size, dev_impl_t=AMDev) self._compute_props() diff --git a/tinygrad/runtime/support/am/amdev.py b/tinygrad/runtime/support/am/amdev.py index d4b577123ce71..a854d36e3e6ca 100644 --- a/tinygrad/runtime/support/am/amdev.py +++ b/tinygrad/runtime/support/am/amdev.py @@ -1,6 +1,6 @@ from __future__ import annotations import ctypes, collections, dataclasses, functools, hashlib, array -from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw +from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw, to_mv from tinygrad.runtime.autogen import pci from tinygrad.runtime.autogen.am import am, fw from tinygrad.runtime.support.amd import AMDReg, import_module, import_asic_regs @@ -238,7 +238,8 @@ def recover(self, force=False) -> bool: if DEBUG >= 3: print(f"am {self.devfmt}: Recovery complete") return True - def is_hive(self) -> bool: return self.gmc.xgmi_seg_sz > 0 + # a hive has multiple XGMI regions; single-node parts (like MI350P) may still program LFB_SIZE with region 0 only + def is_hive(self) -> bool: return self.gmc.xgmi_seg_sz > 0 and self.gmc.xgmi_max_region > 0 def paddr2mc(self, paddr:int) -> int: return self.gmc.mc_base + paddr def paddr2xgmi(self, paddr:int) -> int: return self.gmc.paddr_base + paddr @@ -315,6 +316,15 @@ def _run_discovery(self): ip_offset += 8 + (8 if ihdr.base_addr_64_bit else 4) * ip.num_base_address + # HARV(EST) table: harvested instances must be excluded (like amdgpu_discovery_harvest_ip) + # layout: u32 signature, u16 version, u16 size, then 32 entries of {hw_id:u16, inst:u8, rsv:u8} + self.harvested:dict[int, set[int]] = collections.defaultdict(set) + if (harv_off:=self.bhdr.table_list[am.HARVEST_INFO].offset) != 0 and \ + (blob:=to_mv(ctypes.addressof(self.bhdr) + harv_off, 8 + 32*4).cast('I'))[0] == am.HARVEST_TABLE_SIGNATURE: + inv_hw_id = {hw_id: hw_ip for hw_ip, hw_id in am.hw_id_map.items()} + for ent in blob[2:]: + if (ip_:=inv_hw_id.get(ent & 0xffff)) is not None: self.harvested[ip_].add((ent >> 16) & 0xff) + gc_info = am.struct_gc_info_v1_0.from_address(gc_addr:=ctypes.addressof(self.bhdr) + self.bhdr.table_list[am.GC].offset) self.gc_info = getattr(am, f"struct_gc_info_v{gc_info.header.version_major}_{gc_info.header.version_minor}").from_address(gc_addr) self.reserved_vram_size = (384 << 20) if self.ip_ver[am.GC_HWIP][:2] in {(9,4), (9,5)} else (64 << 20) diff --git a/tinygrad/runtime/support/am/ip.py b/tinygrad/runtime/support/am/ip.py index fcebddd30ac1d..99234d348d029 100644 --- a/tinygrad/runtime/support/am/ip.py +++ b/tinygrad/runtime/support/am/ip.py @@ -29,7 +29,9 @@ def _ih_srcs(pref:str, hwip:int) -> dict[int, str]: def init_hw(self): if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}: - self.adev.regXCC_DOORBELL_FENCE.write(0x0) + # fence doorbells for harvested xccs (0xff & ~xcc_mask in the kernel); a fully-unharvested chip keeps the previous 0x0 + live_xccs = sum(1 << i for i in self.adev.regs_offset[am.GC_HWIP] if i not in self.adev.harvested[am.GC_HWIP] and i < 8) + self.adev.regXCC_DOORBELL_FENCE.write(0xff & ~live_xccs) for aid in range(1, self.adev.gmc.vmhubs): self.adev.indirect_wreg_pcie(self.adev.regXCC_DOORBELL_FENCE.addr[0], self.adev.regXCC_DOORBELL_FENCE.encode(shub_slv_mode=1), aid=aid) self.adev.regBIFC_GFX_INT_MONITOR_MASK.write(0x7ff) @@ -52,7 +54,8 @@ def init_sw(self): self.vmhubs = len(self.adev.regs_offset[am.MMHUB_HWIP]) # XGMI (for supported systems) - self.xgmi_phys_id = self.adev.regMMMC_VM_XGMI_LFB_CNTL.read_bitfields()['pf_lfb_region'] if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_CNTL') else 0 + xgmi_lfb_cntl = self.adev.regMMMC_VM_XGMI_LFB_CNTL.read_bitfields() if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_CNTL') else {} + self.xgmi_phys_id, self.xgmi_max_region = xgmi_lfb_cntl.get('pf_lfb_region', 0), xgmi_lfb_cntl.get('pf_max_region', 0) self.xgmi_seg_sz = self.adev.regMMMC_VM_XGMI_LFB_SIZE.read_bitfields()['pf_lfb_size']<<24 if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_SIZE') else 0 self.paddr_base = self.xgmi_phys_id * self.xgmi_seg_sz @@ -189,13 +192,13 @@ def mode1_reset(self): if DEBUG >= 2: print(f"am {self.adev.devfmt}: mode1 reset") if self.adev.ip_ver[am.MP0_HWIP] >= (14,0,0) or self.adev.ip_ver[am.MP0_HWIP] in {(13,0,0), (13,0,7), (13,0,10)}: self._send_msg(__DEBUGSMC_MSG_Mode1Reset:=2, 0, debug=True) - elif self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6), (13,0,12)}: self._send_msg(self.smu_mod.PPSMC_MSG_GfxDriverReset, 1) + elif self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6), (13,0,12), (13,0,15)}: self._send_msg(self.smu_mod.PPSMC_MSG_GfxDriverReset, 1) else: self._send_msg(self.smu_mod.PPSMC_MSG_Mode1Reset, 0) if not self.adev.is_hive(): time.sleep(0.5) # 500ms def read_table(self, table_t, arg): - if self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6),(13,0,12)}: self._send_msg(self.smu_mod.PPSMC_MSG_GetMetricsTable, arg) + if self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6),(13,0,12),(13,0,15)}: self._send_msg(self.smu_mod.PPSMC_MSG_GetMetricsTable, arg) else: self._send_msg(self.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, arg) return table_t.from_buffer(bytearray(self.adev.vram.view(self.driver_table_paddr, ctypes.sizeof(table_t))[:])) @@ -206,7 +209,7 @@ def read_clocks(self, clk_list:tuple[int]) -> dict[int, list[int]]: def set_clocks(self, level:int|None): clks = tuple([self.smu_mod.PPCLK_UCLK, self.smu_mod.PPCLK_FCLK, self.smu_mod.PPCLK_SOCCLK]) - if self.adev.ip_ver[am.MP0_HWIP] not in {(13,0,6), (13,0,12)}: clks += (self.smu_mod.PPCLK_GFXCLK,) + if self.adev.ip_ver[am.MP0_HWIP] not in {(13,0,6), (13,0,12), (13,0,15)}: clks += (self.smu_mod.PPCLK_GFXCLK,) if level is None: for clck in clks: @@ -246,7 +249,7 @@ def _send_msg(self, msg:int, param:int, read_back_arg=False, timeout=10000, debu class AM_GFX(AM_IP): def init_sw(self): - self.xccs = len(self.adev.regs_offset[am.GC_HWIP]) + self.xccs = sum(1 for i in self.adev.regs_offset[am.GC_HWIP] if i not in self.adev.harvested[am.GC_HWIP]) self.mqd_paddr = [self.adev.mm.palloc(0x1000 * self.xccs, zero=False, boot=True) for i in range(2)] self.mqd_mc = [self.adev.paddr2mc(mqd_paddr) for mqd_paddr in self.mqd_paddr] @@ -514,7 +517,7 @@ def init_hw(self): **({'utc_l1_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP] <= (5,2,0) else {}), inst=inst) if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}: - for aid_id in range(4): + for aid_id in range(self.adev.gmc.vmhubs): for dev_inst, (port, awid, offset, awaddr) in enumerate([(1, 0xe, 0xe, 0x1), (2, 0x8, 0x8, 0x2), (5, 0x9, 0x9, 0x8), (6, 0xa, 0xa, 0x9)]): entry = dev_inst + 1 + 4 * aid_id self.adev.reg(f"regDOORBELL0_CTRL_ENTRY_{entry}").write(**{f"bif_doorbell{entry}_range_size_entry": 20, From 8c3cb00d36759b3fdb1ab4f801bb6c1de9dbf51f Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:42:48 +0300 Subject: [PATCH 116/306] hcq2: staging (#17622) --- test/device/test_hcq2.py | 11 +++++++++++ tinygrad/runtime/ops_npy.py | 2 ++ tinygrad/runtime/support/hcq2.py | 29 ++++++++++++++++++++++------- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index a8ffe60a29604..e312e980d3f74 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -1,6 +1,8 @@ import unittest, numpy as np from unittest.mock import patch from tinygrad import Device, Tensor +from tinygrad.device import Buffer +from tinygrad.dtype import dtypes from tinygrad.helpers import getenv from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in @@ -10,6 +12,15 @@ def test_copy_without_copy_queue(self): with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + @unittest.skipIf(Device.DEFAULT == "CPU", "staged copies need a non-CPU hcq2 device") + def test_staged_copy_slot_reuse(self): + # chunks of a staged copy rotate through the staging buffer slots, many rotations must stay bit-exact in both directions + import tinygrad.runtime.support.hcq2 as hcq2 + buf = Buffer("CPU", 1 << 20, dtypes.uint8, preallocate=True) + data = np.random.default_rng(42).integers(0, 256, (5 << 20) + 123, dtype=np.uint8) + with patch.object(hcq2, "STAGING_SIZE", 1 << 20), patch.object(hcq2, "STAGING_SLOTS", 4), patch.object(hcq2, "_staging", lambda: buf): + np.testing.assert_equal(Tensor(data).to(Device.DEFAULT).realize().numpy(), data) + def test_overlapping_device_tuples(self): # an op on a wide device tuple followed by an op on an overlapping smaller tuple used to MMU-fault the smaller one d4, d2 = tuple(f"{Device.DEFAULT}:{i}" for i in range(4)), tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) diff --git a/tinygrad/runtime/ops_npy.py b/tinygrad/runtime/ops_npy.py index b7ef4e47fbe17..2e5196ed67fc4 100644 --- a/tinygrad/runtime/ops_npy.py +++ b/tinygrad/runtime/ops_npy.py @@ -6,6 +6,8 @@ class NpyAllocator(Allocator['NpyDevice']): def _alloc(self, size:int, options=None) -> np.ndarray: return np.empty(size, dtype=np.uint8) def _as_buffer(self, src:np.ndarray) -> memoryview: return flat_mv(np.require(src, requirements='C').data) def _copyout(self, dest:memoryview, src:np.ndarray): dest[:] = self._as_buffer(src) + def _offset(self, buf:np.ndarray, size:int, offset:int) -> np.ndarray: + return np.require(buf, requirements='C').reshape(-1).view(np.uint8)[offset:offset+size] class NpyDevice(Compiled): def __init__(self, device:str): super().__init__(device, NpyAllocator(self), [], None) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index be85e52c6c16e..f54b33f052d15 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -100,20 +100,33 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| # ***************** # 1.1. prep: staging copies +STAGING_SIZE, STAGING_SLOTS = 128 << 20, 2 + +@functools.cache +def _staging() -> Buffer: return Buffer("CPU", STAGING_SIZE, dtypes.uint8, preallocate=True) + def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_DEVS) +def stage_copy(dst:UOp, src:UOp) -> UOp|None: + if not (_need_staging(src, dst) or _need_staging(dst, src)): return None + + assert src.dtype.itemsize == dst.dtype.itemsize, "staged copies must be dtype-size matched" + base, it, copies = UOp.from_buffer(_staging()), src.dtype.itemsize, [] + chunk = (STAGING_SIZE // STAGING_SLOTS) // it + for i, off in enumerate(range(0, src.max_numel(), chunk)): + stage = base[(so:=(i % STAGING_SLOTS) * chunk * it):so + (n:=min(chunk, src.max_numel() - off)) * it] + copies += [src[off:off+n].copy_to_device("CPU").call(stage, src[off:off+n]), stage.copy_to_device(dst.device).call(dst[off:off+n], stage)] + return UOp(Ops.LINEAR, src=tuple(copies)) + +# ***************** +# 1.2. prep: kernel copies + def _get_enqueue_devs(call:UOp) -> Any|None: if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None if call.src[0].op is Ops.COPY: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device return devs if all_devices_in(devs, HCQ_DEVS) else None -def stage_copy(dst:UOp, src:UOp) -> UOp|None: - if not (_need_staging(src, dst) or _need_staging(dst, src)): return None - - stage = UOp.new_buffer("CPU", src.max_numel() * src.dtype.itemsize, dtypes.uint8) - return UOp(Ops.LINEAR, src=(src.copy_to_device("CPU").call(stage, src), stage.copy_to_device(dst.device).call(dst, stage))) - def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: if (devs:=_get_enqueue_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None d, s = (UOp.param(i, dst.dtype, (n:=dst.max_numel(),), device=devs) for i in range(2)) @@ -336,7 +349,9 @@ def split_patches(call:UOp) -> UOp|None: runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) tables = [make_addr_table(call, gs, n) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec - gathers = make_gather_loop(ipathces, tables[0][0], tables[0][3], lt_patches) if (ipathces:=[p for p in rt_patches if p.tag == "inputs"]) else {} + + ipatches = [p for p in rt_patches if p.tag == "inputs" and all(v in tables[0][3] for v in p.src[1].src)] # only getaddrs go to the table + gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches) if ipatches else {} body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}) lt_srcs = collections.defaultdict(list) From e68aa16e3fe764f2eeeae020a93626d9c2b0ccb5 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:21:53 +0300 Subject: [PATCH 117/306] hcq2: faster beam (#17624) --- tinygrad/codegen/opt/search.py | 4 ++-- tinygrad/engine/realize.py | 21 +++++++++++---------- tinygrad/runtime/support/hcq2.py | 9 +++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index c8c7a3680a803..f4f8f601719ea 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -42,9 +42,9 @@ def _time_program(prg:UOp, var_vals:dict[str, int], rawbufs:list[Buffer], early_ global_size, factor = get_test_global_size(prg.arg.global_size, max_global_size, var_vals) prg = prg.replace(arg=replace(prg.arg, global_size=tuple(global_size))) call = prg.call(*[UOp.from_buffer(b) for b in rawbufs]) - tms = [] + tms, timer = [], time_call(call, var_vals, timeout=timeout, clear_l2=clear_l2) for _ in range(cnt): - try: tms.append(time_call(call, var_vals, timeout=timeout, clear_l2=clear_l2) * factor) + try: tms.append(next(timer) * factor) except AssertionError: return [math.inf] * cnt if early_stop is not None and early_stop < min(tms): break return tms diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index fb69ab11efc90..17d94cff20e31 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -212,13 +212,13 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: dev = cast(Any, Device[(info:= call.arg.aux).device[0]]) addrs = [(b.bufs[j] if isinstance(b:=_resolve(ctx.input_uops[k], ctx.input_uops).buffer, MultiBuffer) else b).get_buf(dev_name).va_addr for devs, idxs in info.input_idxs for j, dev_name in enumerate(devs) for k in idxs] - dev.rt_buffer._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) + dev.rt_buffer()._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) if info.inputs is not None: - tables = [UOp.from_buffer(dev.rt_buffer.view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) + tables = [UOp.from_buffer(dev.rt_buffer().view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) for devs, idxs in info.input_idxs for j in range(len(devs))] call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) - exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer._buf.va_addr + base}), call, ast) + exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer()._buf.va_addr + base}), call, ast) def _prof_tm(device:str, stat_call:UOp, prof:tuple[int, ...]) -> float|None: (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, *prof) @@ -283,12 +283,13 @@ def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequenc ctx = ExecContext(var_vals or {}, tuple(inputs), update_stats, jit, wait or DEBUG>=2) for call in linear.src: track_stats(ctx, call, perf_counter_us(), pm_exec.rewrite(call, ctx)) -def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> float: - if clear_l2: - if hasattr(dev:=Device[call.src[1].device], 'invalidate_caches'): dev.invalidate_caches() - else: - from tinygrad.tensor import Tensor - with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) +def time_call(call:UOp, var_vals:dict[str, int]|None=None, timeout:int|None=None, clear_l2:bool=False) -> Iterator[float]: ctx = ExecContext(var_vals or {}, update_stats=False, wait=True, timeout=timeout, cache=False) linear = link_linear(compile_linear(UOp(Ops.LINEAR, src=(call,)), beam=0, profile=True), cache=ctx.cache) - return max(et for c in linear.src for et in pm_exec.rewrite(c, ctx) or [0.0]) + while True: + if clear_l2: + if hasattr(dev:=Device[call.src[1].device], 'invalidate_caches'): dev.invalidate_caches() + else: + from tinygrad.tensor import Tensor + with Context(DEBUG=0, BEAM=0, CAPTURING=0, TRACK_MATCH_STATS=0): Tensor.ones(1024, 1024).contiguous().realize(do_update_stats=False) + yield max(et for c in linear.src for et in pm_exec.rewrite(c, ctx) or [0.0]) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f54b33f052d15..ef4ff904b31d3 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -600,14 +600,15 @@ def _at_profile_finalize(self): tdiffs.append((st+perf_counter_us())/2 - gpu) Compiled.profile_events.append(ProfileDeviceEvent(self.device, statistics.median(tdiffs), self.device_props())) - @functools.cached_property - def rt_buffer(self) -> Buffer: - return Buffer(self.device, self.rt_allocator.size, dtypes.uint8, options=BufferSpec(uncached=True, cpu_access=True), preallocate=True) + @functools.cache + def rt_buffer(self, uncached:bool=True) -> Buffer: + return Buffer(self.device, self.rt_allocator.size, dtypes.uint8, options=BufferSpec(uncached=uncached, cpu_access=True), preallocate=True) def new_buffer(self, b:UOp, cache:bool) -> Buffer: if cache or b.tag in HCQ_CACHE_TAGS: return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=b.tag not in ("program","kernargs"), cpu_access=True,nolru=True)) - return self.rt_buffer.view(b.max_numel(), b.dtype, self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) + return self.rt_buffer(uncached=b.tag!="kernargs").view(b.max_numel(), b.dtype, + self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) @functools.cache def signal(self, name:str|int, init_value:int=0) -> Buffer: From 6732d05157afad4f4de77ff7e3bdb16bf889ecf7 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 10:47:23 -0400 Subject: [PATCH 118/306] fix dtype_from_uop for invalid ALU [PR] (#17626) invalid is bool, and ALU(invalid) is invalid which is bool --- tinygrad/uop/ops.py | 9 +++++---- tinygrad/uop/spec.py | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 9a30dbe3120c2..a0c0e72d6e563 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -142,7 +142,7 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: case Ops.CMPLT | Ops.CMPNE | Ops.CMPEQ: return dtypes.bool case Ops.SIN | Ops.LOG2 | Ops.EXP2 | Ops.SQRT | Ops.RECIPROCAL: - return least_upper_float(src[0].dtype) + return dtypes.bool if src[0].base.is_invalid else least_upper_float(src[0].dtype) case Ops.WHERE: if src[0].dtype != dtypes.bool: raise RuntimeError(f"where cond must be bool, got {src[0].dtype}") return promo_dtype(src[1:]) @@ -159,7 +159,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: case Ops.GETADDR: return dtypes.uint64 case Ops.SHL | Ops.SHR: - if not all(dtypes.is_int(x.dtype) for x in src): raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") + if not all(dtypes.is_int(x.dtype) or x.base.is_invalid for x in src): + raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") return src[0].dtype case Ops.BUFFER | Ops.PARAM: assert isinstance(arg, ParamArg), "BUFFER/PARAM must have ParamArg" @@ -191,7 +192,7 @@ def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py # an INDEX presents its access dtype, which a still-weak source matches up to weakness if SPEC == 2 and op is not Ops.CONST and \ - not any(s.base.is_invalid for s in src) and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype and \ + (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype and \ not (op is Ops.INDEX and weak_dtype(expected_dtype) == weak_dtype(dtype)): raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret @@ -1747,7 +1748,7 @@ def graph_rewrite(sink:UOp, pm:PatternMatcher, ctx=None, bottom_up=False, name=N def _rebuild_dtype(n:UOp, new_src:tuple[UOp,...]) -> DType: # TODO: delete this once the dtype field is removed, every rebuild will re-derive - if all(a.dtype is b.dtype or b.base.is_invalid for a,b in zip(n.src, new_src)): return n.dtype + if all(a.dtype is b.dtype for a,b in zip(n.src, new_src)): return n.dtype return dtype_from_uop(n.op, new_src, n.arg) or n.dtype def sint_to_uop(x:sint, dtype=dtypes.weakint) -> UOp: return UOp.const(x, dtype) diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 95fba930e5bbb..36d72390d619e 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -68,8 +68,10 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base (UPat(GroupOp.Comparison, dtype=dtypes.bool, src=(UPat.var("x"), UPat.var("y"))), lambda x,y: matches_dtype(x, y.dtype) or matches_dtype(y, x.dtype) or x.dtype in dtypes.weaks or y.dtype in dtypes.weaks), (UPat((Ops.AND, Ops.OR, Ops.XOR, Ops.SHL, Ops.SHR), name="x"), lambda x: False if any(dtypes.is_float(s.dtype) for s in x.src) else None), - (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat(dtype=dtypes.uint)), name="a"), lambda a,x: matches_dtype(x, a.dtype) or None), - (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), lambda x: None if dtypes.is_int(x.dtype) else False), + (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: (matches_dtype(x, a.dtype) or x.dtype is dtypes.weakint) + and (matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid)), + (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), + lambda x: None if dtypes.is_int(x.dtype) or any(s.base.is_invalid for s in x.src) else False), (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype in dtypes.weaks for y in x.src)), # CAST @@ -134,7 +136,8 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # these ops can exist in tensor but not programs. example: movement spec_tensor = PatternMatcher([ - (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), lambda u: dtypes.is_float(u.dtype)), + (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), + lambda u: dtypes.is_float(u.dtype) or u.src[0].base.is_invalid), # BUFFER (UPat(Ops.BUFFER, src=(UPat(),), name="buf"), lambda buf: From f55c1a37d2e480f25b3605db0b8bf5d208e3bb97 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 15:18:15 -0400 Subject: [PATCH 119/306] clean up some index(dtype=) [PR] (#17634) --- tinygrad/codegen/late/coalesce.py | 10 +++++----- tinygrad/codegen/late/gater.py | 4 ++-- tinygrad/schedule/rangeify.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index bac011e4ec1c1..f3bfcbeec87ec 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -51,8 +51,8 @@ def simplify_valid_image_load(buf:UOp, idx_y:UOp, idx_x:UOp, valid:UOp) -> UOp|N if not drop_stmt and idx is start_idx: return None new_valid = UOp.uprod(*ss) if (ss:=[s for s in valid.split_uop(Ops.AND) if s not in drop_stmt]) else None idx_y, idx_x = idx.index(1), idx.index(0) - if new_valid is not None: return buf.index(idx_y.valid(new_valid), idx_x.valid(new_valid), dtype=dtypes.float) - return buf.index(idx_y, idx_x, dtype=dtypes.float) + if new_valid is not None: return buf.index(idx_y.valid(new_valid), idx_x.valid(new_valid)) + return buf.index(idx_y, idx_x) indexing_simplify = PatternMatcher([ # image load valid idx simplification @@ -88,9 +88,9 @@ def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: buf = buf.replace(src=(shape_to_shape_arg((h, w, 4)),)) shapes[buf.arg.slot] = (h, w) if valid.op is not Ops.CONST or valid.val is not True: - return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid), dtype=dtypes.float) + return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid)) else: - return buf.index(cidx.src[1], cidx.src[0], dtype=dtypes.float) + return buf.index(cidx.src[1], cidx.src[0]) pm_simplify_add_image = PatternMatcher([ (UPat(Ops.SHRINK, src=(UPat(Ops.PARAM, name="buf"), UPat(name="x"), UPat(arg=4))), transform_to_image), @@ -149,7 +149,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: grp = full_grp[:length] # NOTE: we apply the valid again after we determine the length offset = offset.valid(valid) if valid is not None else offset - idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(len(grp)))) if len(grp) > 1 else buf.index(offset, dtype=offsets[grp[0]][0].src[0].dtype) + idx = UOp(Ops.SHRINK, src=(buf, offset, UOp.const(len(grp)))) if len(grp) > 1 else buf.index(offset) if op == Ops.STORE: datas = [] for i,g in enumerate(grp): diff --git a/tinygrad/codegen/late/gater.py b/tinygrad/codegen/late/gater.py index e9b46ecfd52b2..319a60c320cb1 100644 --- a/tinygrad/codegen/late/gater.py +++ b/tinygrad/codegen/late/gater.py @@ -10,10 +10,10 @@ def move_where_load(gate, l, a, w): # for image idx (must be first) (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).load(name="l"), - lambda buf,gate,idx_y,idx_x,l: buf.index(idx_y, idx_x, dtype=dtypes.float).load(l.vconst_like(0), gate)), + lambda buf,gate,idx_y,idx_x,l: buf.index(idx_y, idx_x).load(l.vconst_like(0), gate)), (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).store(UPat.var("data")), - lambda buf,gate,idx_y,idx_x,data: buf.index(idx_y, idx_x, dtype=dtypes.float).store(data, gate)), + lambda buf,gate,idx_y,idx_x,data: buf.index(idx_y, idx_x).store(data, gate)), # here we create the alt value for load to be 0s and remove the where Invalid (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(), UPat.var("gate").where(UPat.var("idx"), UPat(arg=Invalid)),), name="mop", allow_any_len=True) \ diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 430fb52a6a900..26ed0d880dde7 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -39,12 +39,12 @@ def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): def _mop_index(r:UOp, idx:UOp): idxs = idx.src[1:] if len(idxs) == len(r.shape): - return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), dtype=idx.dtype, arg=idx.arg) + return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), arg=idx.arg) if r.op is Ops.RESHAPE: src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: if src_prefix == 0: return r.src[0] if r.src[0].dtype == idx.dtype else None - ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), dtype=idx.dtype, arg=idx.arg) + ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), arg=idx.arg) return ret if ret.shape == idx.shape else None pm_mops = PatternMatcher([ From a8ecb7336351703dd953abda147e49b766d081f0 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 15:30:17 -0400 Subject: [PATCH 120/306] x64 imm uint64 (#17636) * x64 imm uint64 * fix --- test/backend/test_ops.py | 3 +++ tinygrad/renderer/isa/x86.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index ca97691358db4..5cfadfb6fca8e 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -823,6 +823,9 @@ def test_and(self): helper_test_op(None, lambda x: (1 < x) & (x < 2), forward_only=True, vals=[[1.2, 1.2, 1.2, 3.2]]) + if not COMPILE_ONLY: + np.testing.assert_equal((Tensor(2**64-1, dtype=dtypes.uint64) & 0xFFFFFFFF).numpy(), 0xFFFFFFFF) + def test_or(self): data = [[1,-8,1],[32,1,6]] tor = torch.tensor(data, dtype=torch.int) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 3dd4ff6442847..b8c8c3594ec43 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -227,8 +227,7 @@ def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, dt, ar def imm(dt:DType, v:int) -> UOp: return UOp.cconst(truncate[dt](v), dt).rtag() def to_imm(c:UOp) -> UOp|None: if not (c.op is Ops.CAST and (v:=c.src[0]).op is Ops.CONST): return None - if c.dtype is dtypes.int64: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None - if c.dtype is dtypes.uint64: return imm(dtypes.uint32, v.val) if not v.overflows(dtypes.uint32) else None + if c.dtype in dtypes.int64s: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, v.val) return None def cmp(x:UOp) -> UOp: From eba5b7e750f6d2405d143ac7fb9dd20f1df54a8a Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 20 Aug 2026 13:12:14 -0700 Subject: [PATCH 121/306] benchmarks: cleanups (#17631) --- .github/workflows/benchmark.yml | 10 +++++----- extra/hcq/hcq_smi.py | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f02c210cecf83..a94fe047cfbd0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -215,8 +215,6 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: reset process replay run: test/external/process_replay/reset.py - - name: Run MLPerf resnet eval on training data - run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) run: BENCHMARK_LOG=resnet_10steps DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py - name: Run process replay tests @@ -312,6 +310,8 @@ jobs: run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 - name: Run full CIFAR training steps w 6 GPUS run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py + - name: Run MLPerf resnet eval on training data + run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py - name: Run 10 MLPerf ResNet50 training steps (6 gpu) run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py - name: Run 10 MLPerf Bert training steps (6 gpu) @@ -445,7 +445,7 @@ jobs: - name: UsbGPU tiny tests run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py - name: UsbGPU copy speeds - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds + run: sudo -E PYTHONDONTWRITEBYTECODE=1 SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds #- name: UsbGPU openpilot test # run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx - name: UsbGPU (USB4/TB) install script @@ -598,8 +598,8 @@ jobs: - name: Setup run: | ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} rmmod - ./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} kill_pids + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} rmmod + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} kill_pids mkdir -p extra/datasets ln -s /raid/datasets/imagenet extra/datasets/imagenet - name: setup staging db diff --git a/extra/hcq/hcq_smi.py b/extra/hcq/hcq_smi.py index edb465235ed61..2482038b71a8f 100755 --- a/extra/hcq/hcq_smi.py +++ b/extra/hcq/hcq_smi.py @@ -97,11 +97,13 @@ def add_common_commands(parent_subparsers): parser = argparse.ArgumentParser() backend_subparsers = parser.add_subparsers(dest="backend", required=True, metavar="{nv,amd}", help="Hardware backend to target") - nv_parser = backend_subparsers.add_parser("nv", help="NVIDIA GPUs") + nv_parser = backend_subparsers.add_parser("nv", aliases=["NV"], help="NVIDIA GPUs") + nv_parser.set_defaults(backend="nv") nv_commands = nv_parser.add_subparsers(dest="command", required=True) add_common_commands(nv_commands) - amd_parser = backend_subparsers.add_parser("amd", help="AMD GPUs") + amd_parser = backend_subparsers.add_parser("amd", aliases=["AMD"], help="AMD GPUs") + amd_parser.set_defaults(backend="amd") amd_commands = amd_parser.add_subparsers(dest="command", required=True) add_common_commands(amd_commands) From 1707dca3b474f792343b8c67da1e83a54b5d3e3a Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:28:25 -0700 Subject: [PATCH 122/306] remove kernel_cnt, names are no longer unique (#17633) * remove kernel_cnt, names are no longer unique * check uops, not names * fix hcq2 --- extra/hcq2/ops_amd2.py | 5 +++-- test/backend/test_multitensor.py | 4 ++-- tinygrad/codegen/opt/postrange.py | 9 ++------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index be8872c65cbcd..e603df20a42eb 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -261,10 +261,11 @@ class AMDProgramData: private_segment_size:int; kernargs_segment_size:int; kernargs_alloc_size:int enable_dispatch_ptr:int; enable_private_segment_sgpr:int -_amd_program_cache:dict[tuple[bytes,str], tuple[AMDProgramData,bytes]] = {} +_amd_program_cache:dict[tuple[bytes, tuple[str, ...]], UOp] = {} def amd_build_program(prg:UOp) -> UOp: dev = Device[to_tuple(prg.device)[0]] # TODO: rm this - if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[3].arg, dev.device))) is None: + # key on the full device tuple: the same lib can be built for different device sets, each needs its own program buffer + if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[3].arg, to_tuple(prg.device)))) is None: image, sections, relocs = elf_loader(lib) rodata = next(sh.header.sh_addr for sh in sections if sh.name == ".rodata") for off, sym, typ, addent in relocs: diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index dbe7cce6f7eaa..d76874ae92917 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -72,9 +72,9 @@ def test_shard_no_recompile(self): X.shard_(devices_2, 0) out = (X + X) linear = compile_linear(out.schedule_linear()) - names = [call.src[0].src[0].arg.name for call in linear.src if call.src[0].op is Ops.PROGRAM] + uops = [call.src[0].src[0] for call in linear.src if call.src[0].op is Ops.PROGRAM] run_linear(linear) - self.assertEqual(len(set(names)), 1, "function was relinearized") + self.assertEqual(len(set(uops)), 1, "function was relinearized") def test_shard_beam(self): cpu_2 = ("CPU:1", "CPU:2") diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index f5f682c2703e2..aa2e562043c22 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -1,12 +1,11 @@ from __future__ import annotations import math, itertools -from collections import defaultdict -from typing import cast, Final +from typing import cast from tinygrad.uop.ops import Ops, UOp, KernelInfo, graph_rewrite, AxisType, ssimplify, remove_all_tags from tinygrad.uop.ops import axis_letters, axis_colors, axis_to_pos from tinygrad.device import Buffer from tinygrad.dtype import dtypes, Invalid -from tinygrad.helpers import colored, getenv, DEBUG, to_function_name, NOOPT, argsort, round_up, prod, merge_dicts, get_single_element, flatten +from tinygrad.helpers import colored, getenv, DEBUG, NOOPT, argsort, round_up, prod, merge_dicts, get_single_element, flatten from tinygrad.helpers import ALLOW_TF32, count, Context from tinygrad.codegen.opt import Opt, OptOps, KernelOptError, check from tinygrad.codegen.simplify import pm_flatten_range @@ -48,7 +47,6 @@ def copy(self) -> Scheduler: if hasattr(self, 'tensor_core'): ret.tensor_core = self.tensor_core return ret - kernel_cnt: Final[defaultdict[str, int]] = defaultdict(int) def get_optimized_ast(self, name_override:str|None=None) -> UOp: if name_override is not None: name = name_override else: @@ -56,9 +54,6 @@ def get_optimized_ast(self, name_override:str|None=None) -> UOp: special_uops = sorted([x for x in self.ast.toposort() if x.op is Ops.SPECIAL], key=lambda x: x.arg) special_ops = [colored(str(x.vmax+1), "blue" if x.arg[0] == "g" else "cyan") for x in special_uops] name = k_type + colored('_', 'BLACK').join(['']+special_ops+[colored(x.src[0].render(), color) for x,color in zip(self.rngs, self.colors())]) - Scheduler.kernel_cnt[(function_name := to_function_name(name))] += 1 - num = f"n{Scheduler.kernel_cnt[function_name]-1}" if Scheduler.kernel_cnt[function_name] > 1 else "" - name += colored(num, 'BLACK') self.ast = graph_rewrite(self.ast, pm_flatten_range, name="flatten range") return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts), dont_use_locals=self.dont_use_locals), tag=1) From a57188ea6d9a7a7604ea96585e32fbb88b2d4fda Mon Sep 17 00:00:00 2001 From: George Hotz Date: Thu, 20 Aug 2026 13:39:48 -0700 Subject: [PATCH 123/306] hotfix: disable HCQ2 --- .github/workflows/benchmark.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a94fe047cfbd0..1b199252f649c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,7 +94,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: "0" if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -141,7 +141,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: "0" if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -190,7 +190,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: "0" if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -233,7 +233,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: "0" if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -279,7 +279,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: "0" if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code From 0e7ab863a0d522643acbd21cd8c942a581d3f7e8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 16:49:04 -0400 Subject: [PATCH 124/306] x86 REX issue (#17580) * failing test * fix --- test/backend/test_ops.py | 1 + tinygrad/renderer/isa/x86.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 5cfadfb6fca8e..d3bb8e20ee373 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -822,6 +822,7 @@ def test_and(self): helper_test_op([], lambda: tor0&tor1, lambda: ten0&ten1, forward_only=True) helper_test_op(None, lambda x: (1 < x) & (x < 2), forward_only=True, vals=[[1.2, 1.2, 1.2, 3.2]]) + helper_test_op([(3000,)]*10, lambda *xs: (sum(xs[1:], xs[0]) > 5) & (xs[0] < 0.9), forward_only=True) if not COMPILE_ONLY: np.testing.assert_equal((Tensor(2**64-1, dtype=dtypes.uint64) & 0xFFFFFFFF).numpy(), 0xFFFFFFFF) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index b8c8c3594ec43..0510fa99499d3 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -637,10 +637,12 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No if sz == 2: inst += bytes([0x66]) # bit signaling 64 bit variant of instruction w = sz == 8 - # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed - if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2): inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) # legacy 8bit opcode is 1 less than 16-64bit variants - if (rm_sz == 1 or reg_sz == 1) and x.arg not in X86GroupOp.ReadFlags | {X86Ops.LEA}: opc -= 1 + demote = (rm_sz == 1 or reg_sz == 1) and x.arg not in X86GroupOp.ReadFlags | {X86Ops.LEA} + # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed + if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2) | (demote and disp_uop is None and rm >= 4): + inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) + if demote: opc -= 1 # OPCODE byte inst += opc.to_bytes((opc.bit_length() + 7) // 8, 'big') # MODRM byte From c773891e3f461c248248b6f3e23f7bce87c44dfc Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 20 Aug 2026 14:26:26 -0700 Subject: [PATCH 125/306] skip INT_MIN % -1 on X86 and LLVM (#17637) --- test/backend/test_dtype_alu.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/backend/test_dtype_alu.py b/test/backend/test_dtype_alu.py index 6a182616c880e..d55e51807e2b5 100644 --- a/test/backend/test_dtype_alu.py +++ b/test/backend/test_dtype_alu.py @@ -6,6 +6,8 @@ from tinygrad.runtime.ops_python import from_storage_scalar from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.nir import NIRRenderer +from tinygrad.renderer.llvmir import CPULLVMRenderer +from tinygrad.renderer.isa.x86 import X86Renderer from tinygrad.uop import Ops import numpy as np import pytest @@ -64,6 +66,8 @@ class ht: def universal_test(a, b, dtype, op): if not isinstance(op, tuple): op = (op, op) if op[0] == operator.mod and b == 0: return + # TODO: throws floating point exception + if isinstance(Device[Device.DEFAULT].renderer, (X86Renderer, CPULLVMRenderer)) and op[0] == operator.mod and a == dtype.min and b == -1: return # lt and max with nan is undefined in tinygrad if op[0] in (operator.lt, Tensor.maximum) and (math.isnan(a) or math.isnan(b)): return ta, tb = Tensor([a], dtype=dtype), Tensor([b], dtype=dtype) From d80254c1d9fcb2911d8410d4aade4255e0432f64 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Thu, 20 Aug 2026 17:26:44 -0400 Subject: [PATCH 126/306] fa: remove swa recompute (#17635) --- extra/thunder/amd/fa.py | 47 ----------------------------- extra/thunder/amd/fa_fwd_causal.cpp | 20 ++++++++++++ 2 files changed, 20 insertions(+), 47 deletions(-) diff --git a/extra/thunder/amd/fa.py b/extra/thunder/amd/fa.py index 36cdfad824d81..4498b30e7bfa0 100644 --- a/extra/thunder/amd/fa.py +++ b/extra/thunder/amd/fa.py @@ -126,49 +126,6 @@ def fused_qkv_rope(xqkv:Tensor, freqs_cis:Tensor, n_heads:int, n_kv_heads:int, h def _sharded_empty_like(ref:Tensor, axis:int|None=None) -> Tensor: return _sharded_empty(ref.shape, ref, axis) -@functools.cache -def _windowed_lse(xq:Tensor, xk:Tensor, sinks, W:int) -> Tensor: - B, N, H, hd = xq.shape - H_KV = xk.shape[2]; R = H // H_KV; nb = N // W; sm = hd ** -0.5 - q = xq.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() - k = xk.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() - k_prev = k.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] - sc_d = (q @ k.transpose(-1, -2)) * sm - sc_p = (q @ k_prev.transpose(-1, -2)) * sm - li, lj = Tensor.arange(W).reshape(W, 1), Tensor.arange(W).reshape(1, W) - pv = (Tensor.arange(nb).reshape(nb, 1, 1) >= 1) - sc_d = (lj <= li).where(sc_d, -float("inf")) - sc_p = ((li < lj) & pv).where(sc_p, -float("inf")) - m = sc_d.max(-1, keepdim=True).maximum(sc_p.max(-1, keepdim=True)) - if sinks is not None: m = m.maximum(sinks.reshape(1, H_KV, R, 1, 1, 1).float()) - denom = (sc_d - m).exp().sum(-1, keepdim=True) + (sc_p - m).exp().sum(-1, keepdim=True) - if sinks is not None: denom = denom + (sinks.reshape(1, H_KV, R, 1, 1, 1).float() - m).exp() - return (m + denom.log()).reshape(B, H, N).unsqueeze(2) # (B, H, 1, N), matches saved l_vec - -def _windowed_delta(xq:Tensor, xk:Tensor, xv:Tensor, do:Tensor, sinks, W:int) -> Tensor: - B, N, H, hd = xq.shape - H_KV = xk.shape[2]; R = H // H_KV; nb = N // W; sm = hd ** -0.5 - q = xq.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() - k = xk.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() - v = xv.permute(0, 2, 1, 3).reshape(B, H_KV, 1, nb, W, hd).float() - dob = do.reshape(B, N, H_KV, R, hd).permute(0, 2, 3, 1, 4).reshape(B, H_KV, R, nb, W, hd).float() - k_prev = k.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] - v_prev = v.pad((None, None, None, (1, 0), None, None))[:, :, :, :nb] - sc_d = (q @ k.transpose(-1, -2)) * sm - sc_p = (q @ k_prev.transpose(-1, -2)) * sm - li, lj = Tensor.arange(W).reshape(W, 1), Tensor.arange(W).reshape(1, W) - pv = (Tensor.arange(nb).reshape(nb, 1, 1) >= 1) - sc_d = (lj <= li).where(sc_d, -float("inf")) - sc_p = ((li < lj) & pv).where(sc_p, -float("inf")) - m = sc_d.max(-1, keepdim=True).maximum(sc_p.max(-1, keepdim=True)) - if sinks is not None: m = m.maximum(sinks.reshape(1, H_KV, R, 1, 1, 1).float()) - e_d, e_p = (sc_d - m).exp(), (sc_p - m).exp() - denom = e_d.sum(-1, keepdim=True) + e_p.sum(-1, keepdim=True) - if sinks is not None: denom = denom + (sinks.reshape(1, H_KV, R, 1, 1, 1).float() - m).exp() - o = ((e_d / denom) @ v) + ((e_p / denom) @ v_prev) - delta = (dob * o).sum(-1) - return delta.reshape(B, H, N).unsqueeze(2) - def _fa_grad_fxn(B, H, N, D, H_local, H_KV_local, H_KV, B_local, shard_axis, shard_axis_t, single_device, arch, has_sink, window=0): def grad(dou:UOp, ker:UOp) -> tuple: do = Tensor(dou, device=dou.device) @@ -177,8 +134,6 @@ def grad(dou:UOp, ker:UOp) -> tuple: xq = Tensor(ker.src[3], device=ker.src[3].device) xk = Tensor(ker.src[4], device=ker.src[4].device) xv = Tensor(ker.src[5], device=ker.src[5].device) - if window: - l_vec = _windowed_lse(xq, xk, Tensor(ker.src[6], device=ker.src[6].device) if has_sink else None, window) dq = _sharded_empty((B, H, N, D), xq, axis=shard_axis_t) GROUP_SIZE = H_local // H_KV_local @@ -189,8 +144,6 @@ def grad(dou:UOp, ker:UOp) -> tuple: # delta_vec = (do * attn).sum(-1, dtype=dtypes.float32).transpose(1, 2).unsqueeze(-2).detach() delta_vec = _sharded_empty((B, H, 1, N), xq, dtype=dtypes.float32, axis=shard_axis_t) delta_vec, dq = Tensor.custom_kernel(delta_vec, dq, attn, do, fxn=functools.partial(custom_fa_backward_pre, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D))[:2] - if window: - delta_vec = _windowed_delta(xq, xk, xv, do, Tensor(ker.src[6], device=ker.src[6].device) if has_sink else None, window) dq, dk_partial, dv_partial = Tensor.custom_kernel(dq, dk_partial, dv_partial, do, xq, xk, xv, l_vec, delta_vec, fxn=functools.partial(custom_fa_backward, device=single_device, arch=arch, B=B_local, N=N, H=H_local, H_KV=H_KV_local, D=D, window=window))[:3] diff --git a/extra/thunder/amd/fa_fwd_causal.cpp b/extra/thunder/amd/fa_fwd_causal.cpp index 3deb93bf46c11..2dfb10cf9c977 100644 --- a/extra/thunder/amd/fa_fwd_causal.cpp +++ b/extra/thunder/amd/fa_fwd_causal.cpp @@ -269,7 +269,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p qo_tile q_reg_fl; load<1, qo_tile, _gl_QKVO>(q_reg_fl, g.Qg, {batch_idx, tile_idx, head_idx, 0}); + #if !WINDOW mul(q_reg_fl, q_reg_fl, TEMPERATURE_SCALE); // Use sqrtf for clarity + #endif copy(q_reg, q_reg_fl); transpose(q_reg_transposed, q_reg); @@ -288,6 +290,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif __builtin_amdgcn_sched_barrier(0); if constexpr (causal) { const int kv_end_pos = (min_tile + 1) * KV_BLOCK_SIZE; @@ -337,6 +342,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif #if WINDOW // window masks interior tiles that causal skips mask_kv_tile(att_block[1], tile_idx, j - 2, neg_inf_v, lane); @@ -401,6 +409,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif // Finish softmax for QK1 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -469,6 +480,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif // Finish softmax for QK2 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -535,6 +549,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[0]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[0], k_reg_transposed, q_reg_transposed, att_block[0]); + #if WINDOW + mul(att_block[0], att_block[0], TEMPERATURE_SCALE); + #endif // Finish softmax for QK3 exp2(att_block[1].tiles[1][0], att_block[1].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); @@ -597,6 +614,9 @@ __global__ void attend_ker(bf16 *O_ptr, float *L_vec_ptr, bf16 *Q_ptr, bf16 *K_p zero(att_block[1]); transpose(k_reg_transposed, k_reg); mma_AtB(att_block[1], k_reg_transposed, q_reg_transposed, att_block[1]); + #if WINDOW + mul(att_block[1], att_block[1], TEMPERATURE_SCALE); + #endif // Finish softmax for QK4 exp2(att_block[0].tiles[1][0], att_block[0].tiles[1][0]); mul(norm_vec, norm_vec, scale_vec); From 3715006a21f055e365607955a025507b98269aa8 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 20 Aug 2026 15:14:30 -0700 Subject: [PATCH 127/306] fix float_to_bf16 on non-float32 inputs (#17638) --- test/null/test_dtype_spec.py | 3 ++- tinygrad/dtype.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/null/test_dtype_spec.py b/test/null/test_dtype_spec.py index e963ffb6c0622..8460d9c86db88 100644 --- a/test/null/test_dtype_spec.py +++ b/test/null/test_dtype_spec.py @@ -106,7 +106,8 @@ def test_float_to_fp16(self): def test_float_to_bf16(self): max_bf16 = torch.finfo(torch.bfloat16).max - for a in [1, 1.1, 1234, 23456, -777.777, max_bf16, max_bf16 * 1.00001, -max_bf16, -max_bf16 * 1.00001, math.inf, -math.inf]: + for a in [1, 1.1, 1234, 23456, -777.777, max_bf16, max_bf16 * 1.00001, -max_bf16, -max_bf16 * 1.00001, + max_bf16 * 2, -max_bf16 * 2, math.inf, -math.inf]: self.assertEqual(float_to_bf16(a), torch.tensor([a], dtype=torch.bfloat16).item()) self.assertTrue(math.isnan(float_to_bf16(math.nan))) diff --git a/tinygrad/dtype.py b/tinygrad/dtype.py index e160cb6cddcb0..31891c5e85adf 100644 --- a/tinygrad/dtype.py +++ b/tinygrad/dtype.py @@ -221,7 +221,7 @@ def float_to_fp16(x): def float_to_bf16(x): if not math.isfinite(x): return x - u = struct.unpack('I', struct.pack('f', x))[0] + u = struct.unpack('I', struct.pack('f', truncate[dtypes.float](x)))[0] u = (u + 0x7FFF + ((u >> 16) & 1)) & 0xFFFF0000 return struct.unpack('f', struct.pack('I', u))[0] From 592e3f8363d0472f9ef8d6a7e0425eb48ed723c8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 18:49:34 -0400 Subject: [PATCH 128/306] update const selector where folding [pr] (#17640) folding a strong dtype WHERE to a weak const branch keeps the strong dtype --- test/null/test_uop_symbolic.py | 8 ++++++++ tinygrad/uop/symbolic.py | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 369ea785feca6..db3bd00ed8131 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -1020,6 +1020,14 @@ def test_where_cast(self): self.assertIs(graph_rewrite(cond.where(a, uconst(2)).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), UOp.const(2, dtypes.half))) self.assertIs(graph_rewrite(cond.where(a, UOp.invalid()).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), UOp.invalid())) + def test_where_const_gate_keeps_stated_width(self): + a = Variable("a", 0, 3, dtypes.half) + self.assertIs(graph_rewrite(UOp.const(True, dtypes.bool).where(uconst(0.0), a), sym), UOp.const(0.0, dtypes.half)) + self.assertIs(graph_rewrite(UOp.const(True, dtypes.bool).where(uconst(0), Variable("i", 0, 3, dtypes.int)), sym), UOp.const(0, dtypes.int)) + self.assertIs(graph_rewrite(UOp.const(False, dtypes.bool).where(uconst(0.0), a), sym), a) + self.assertIs(graph_rewrite(UOp.const(False, dtypes.bool).where(uconst(0.0), UOp.invalid()), sym), UOp.invalid()) + self.assertIs(graph_rewrite(UOp.const(True, dtypes.bool).where(uconst(0.0), uconst(1)), sym), uconst(0.0)) + def test_where_merge_branches(self): cond1 = Variable("s", 0, 10) < 6 cond2 = Variable("s", 0, 10) > 2 diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 4e9e3b4191497..d581f86c98455 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -101,6 +101,11 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: if any(x.is_invalid for x in s.src) else None), ]) +def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: + # folding a strong dtype WHERE to a weak const branch keeps the strong dtype + ret = c0 if gate.val else c1 + return commit_weak(ret, w.dtype) if ret.op is Ops.CONST and ret.dtype in dtypes.weaks and w.dtype not in dtypes.weaks else ret + symbolic_simple = pm_data_invalid + PatternMatcher([ # ** self folding ** (UPat.var("x") + 0, lambda x: x), # x+0 -> x @@ -175,7 +180,7 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: # ** simple where folding ** # a conditional with the same results either way is a noop, also fold const conditionals (UPat.var().where(UPat.var("val"), UPat.var("val")), lambda val: val), - (UPat.cvar("gate").where(UPat.var("c0"), UPat.var("c1")), lambda gate, c0, c1: c0 if gate.val else c1), + (UPat.cvar("gate").where(UPat.var("c0"), UPat.var("c1")).named("w"), fold_const_where), # a.where(b.where(c, d), d) -> (a & b).where(c, d) (UPat.var("a").where(UPat.var("b").where(UPat.var("c"), UPat.var("d")), UPat.var("d")), lambda a,b,c,d: (a&b).where(c,d)), # a.where(c, b.where(c, d)) -> (a | b).where(c, d) From c655aaf3a221caaf2a3ea29a33854c06927d7759 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 20 Aug 2026 15:54:54 -0700 Subject: [PATCH 129/306] ci: venv in /opt/venv (#17632) default python is now 3.14, not 3.12 --- .github/actions/setup-tinygrad/action.yml | 21 ++++++++++++--------- .github/workflows/platform.yml | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 5e82665da462c..4a7e1ecad1f31 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -4,7 +4,7 @@ inputs: python-version: description: 'Python version to use' required: false - default: '' # if you don't set a version, the native python version will be used + default: '3.14' key: description: 'Key for the python cache' required: false @@ -59,6 +59,11 @@ runs: echo "OMP_NUM_THREADS=1" >> "$GITHUB_ENV" # no buffers should be over 300MB in CI echo "MAX_BUFFER_SIZE=300000000" >> "$GITHUB_ENV" + if [[ "$RUNNER_OS" == "Linux" ]]; then + echo "VIRTUAL_ENV=/opt/venv/${{ inputs.python-version }}" >> "$GITHUB_ENV" + else + echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" + fi - name: Set up uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b @@ -67,7 +72,6 @@ runs: - name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v6 - if: inputs.python-version != '' with: python-version: ${{ inputs.python-version }} @@ -109,15 +113,15 @@ runs: if: inputs.deps != '' shell: bash run: | - uv venv .venv + uv venv --allow-existing --python ${{ inputs.python-version }} "$VIRTUAL_ENV" DEPS="${{ inputs.deps }}" - uv pip install --python .venv -e ".[${DEPS// /,}]" ${{ inputs.pydeps }} --torch-backend cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ + uv pip install --python "$VIRTUAL_ENV" -e ".[${DEPS// /,}]" ${{ inputs.pydeps }} --torch-backend cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/ - name: Install dependencies in venv (without extra) if: inputs.deps == '' shell: bash run: | - uv venv .venv - uv pip install --python .venv -e . ${{ inputs.pydeps }} + uv venv --allow-existing --python ${{ inputs.python-version }} "$VIRTUAL_ENV" + uv pip install --python "$VIRTUAL_ENV" -e . ${{ inputs.pydeps }} - name: Prune uv cache if: github.event_name != 'pull_request' shell: bash @@ -125,11 +129,10 @@ runs: - name: Configure venv shell: bash run: | - echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" if [[ "$RUNNER_OS" == "Windows" ]]; then - echo "${{ github.workspace }}/.venv/Scripts" >> "$GITHUB_PATH" + echo "$VIRTUAL_ENV/Scripts" >> "$GITHUB_PATH" else - echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + echo "$VIRTUAL_ENV/bin" >> "$GITHUB_PATH" fi # ******************* apt ******************* diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index b48e8081b327e..cdd00999e9c91 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -166,7 +166,7 @@ jobs: uses: ./.github/actions/setup-tinygrad with: key: windows-${{ matrix.dev }}-minimal - deps: testing_unit + deps: testing_minimal pydeps: ${{ matrix.dev == 'WEBGPU' && 'dawn-python' || '' }} - name: Set env shell: bash From 0ccef542e0c56aba79521b839a10deb8010fa92c Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:20:16 -0700 Subject: [PATCH 130/306] add extract support to fetch (#17641) --- tinygrad/helpers.py | 15 ++++++++++++++- tinygrad/runtime/support/compiler_qcom.py | 7 ++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index f3dbca7c3c557..d291c7eae2dfc 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -460,14 +460,16 @@ def _ensure_downloads_dir() -> pathlib.Path: return pathlib.Path(cache_dir) / "downloads" def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip:bool=False, allow_caching=not getenv("DISABLE_HTTP_CACHE"), - headers:dict[str, str]={}, sha256:str|None=None) -> pathlib.Path: + headers:dict[str, str]={}, sha256:str|None=None, extract:bool=False) -> pathlib.Path: import urllib.request if url.startswith(("/", ".")): return pathlib.Path(url) if name is not None and (isinstance(name, pathlib.Path) or '/' in name): fp = pathlib.Path(name) else: hh = "_"+hashlib.md5(("\n".join(f"{k.strip()}:{v.strip()}" for k,v in sorted(headers.items()))).encode("utf-8")).hexdigest() if headers else "" fp = _ensure_downloads_dir() / (subdir or "") / ((name or hashlib.md5(url.encode('utf-8')).hexdigest()) + hh + (".gunzip" if gunzip else "")) + extract_dir = fp.parent / f"{fp.name}.extract" if not fp.is_file() or not allow_caching or (sha256 and hashlib.sha256(fp.read_bytes()).hexdigest() != sha256): + if extract: shutil.rmtree(extract_dir, ignore_errors=True) (_dir := fp.parent).mkdir(parents=True, exist_ok=True) with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.13.0", **headers}), timeout=10) as r: assert r.status in {200, 206}, r.status @@ -484,6 +486,17 @@ def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip pathlib.Path(f.name).rename(fp) progress_bar.update(close=True) if length and (file_size:=os.stat(fp).st_size) < length: raise RuntimeError(f"fetch size incomplete, {file_size} < {length}") + if extract: + if not extract_dir.is_dir(): + import tarfile + tmpdir = tempfile.mkdtemp(dir=fp.parent) + try: + with tarfile.open(fp) as t: t.extractall(tmpdir, filter="data") + try: os.rename(tmpdir, extract_dir) # rename is atomic, so concurrent fetches can't see a partial extraction + except OSError: + if not extract_dir.is_dir(): raise + finally: shutil.rmtree(tmpdir, ignore_errors=True) + return extract_dir return fp def fetch_fw(path:str, name:str, sha256:str) -> bytes: diff --git a/tinygrad/runtime/support/compiler_qcom.py b/tinygrad/runtime/support/compiler_qcom.py index 64a7cba8d6960..62693ea5ed7ed 100644 --- a/tinygrad/runtime/support/compiler_qcom.py +++ b/tinygrad/runtime/support/compiler_qcom.py @@ -1,4 +1,4 @@ -import ctypes, struct, platform, pathlib, shutil, tarfile, tempfile +import ctypes, struct, platform, pathlib, shutil from tinygrad.device import Compiler from tinygrad.helpers import DEBUG, system, fetch from tinygrad.runtime.support.compiler_mesa import disas_adreno @@ -12,8 +12,9 @@ def __init__(self, arch:str): assert arch.split(',')[0] == "a630", "only a630 supported" if platform.machine() == "aarch64": self.arch, self.chip_id, self.llvm_inst = arch, 0x6030001, llvm_qcom.cl_compiler_create_llvm_instance() else: - self.arch, self.chip_id, self.fs, root = arch, 0x6030001, tempfile.TemporaryDirectory(), pathlib.Path(__file__).parents[3] - with tarfile.open(fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz')) as t: t.extractall(fs:=self.fs.name) + # extract once into the download cache, all processes share the rootfs (extract=True) + self.arch, self.chip_id = arch, 0x6030001 + fs, root = fetch('https://git.tinygrad.win/sirhcm/images/releases/download/v2/qcomcl.tar.gz', extract=True), pathlib.Path(__file__).parents[3] self.compiler_process = self.server(f"{qemu} -cpu max,pauth=off -L {fs} {fs}/usr/bin/python3" if (qemu:=shutil.which("qemu-aarch64-static")) else (f"docker run --rm -i --platform linux/aarch64 -v {fs}/usr:/usr -v {root}:{root} " f"-e PYTHONPATH={root} -e QEMU_CPU=max,pauth=off gcr.io/distroless/static python3"), arch) From 3fdbb82bfebfb4dc8add74508ead469b89917240 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:26:02 -0700 Subject: [PATCH 131/306] add ansipad and PARALLEL contextvar (#17642) --- test/backend/test_schedule.py | 2 +- test/null/test_viz.py | 2 +- tinygrad/engine/realize.py | 4 ++-- tinygrad/helpers.py | 8 ++++++-- tinygrad/renderer/cstyle.py | 3 ++- tinygrad/renderer/isa/x86.py | 3 ++- tinygrad/renderer/llvmir.py | 3 ++- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/test/backend/test_schedule.py b/test/backend/test_schedule.py index dd2b9e5d71fda..4f4bffa10aebc 100644 --- a/test/backend/test_schedule.py +++ b/test/backend/test_schedule.py @@ -176,7 +176,7 @@ def test_limit_bufs_arange_condition(self): def test_limit_bufs_linear_scaling(self): def sched_time(n): - with Context(TRACK_MATCH_STATS=0, DEBUG=0): + with Context(TRACK_MATCH_STATS=0, DEBUG=0, PARALLEL=0): bufs = [Tensor.ones(16).contiguous().realize() for _ in range(4)] root = bufs[0] for i in range(n): root = root + bufs[i % 4] diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 87ecf4951a0ad..5bfb663baee3b 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -43,7 +43,7 @@ def save_viz(): Buffer.profile_events.clear() cpu_events.clear() viz = VizTrace() - with Context(VIZ=-1, TRACK_MATCH_STATS=2, PROFILE=1): + with Context(VIZ=-1, TRACK_MATCH_STATS=2, PROFILE=1, PARALLEL=0): yield viz viz.set_data() diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 17d94cff20e31..ce48d256f3bc3 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -2,7 +2,7 @@ from typing import cast, Iterator, Any, Sequence import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansilen, all_int, prod, flatten, Context, getenv, to_tuple +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry @@ -89,7 +89,7 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No mem_str = f"{membw*1e-9:4.0f}|{ldsbw*1e-9:<6.0f} GB/s" if membw < 1e13 and ldsbw < 1e15 else \ colored(f"{membw*1e-12:4.0f}|{ldsbw*1e-12:<6.0f} TB/s", 'green') print(f"{colored(f'*** {device[:7]:7s} {GlobalCounters.kernel_count:4d}', header_color)}"+ - f" {display_name+' '*(46-ansilen(display_name))} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ + f" {ansipad(display_name, 46)} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ ("" if et is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) first_run_cache.add(kcall.src[0].key) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index d291c7eae2dfc..36f153261fc03 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -44,6 +44,7 @@ def time_to_str(t:float, w=8) -> str: return next((f"{t * d:{w}.2f}{pr}" for d,p def size_to_str(s:int) -> str: return next((f"{s / d:.2f} {pr}" for d,pr in [(1<<30, "GB"),(1<<20, "MB"),(1<<10, "KB")] if s >= d), f"{s} B") def ansistrip(s:str): return re.sub('\x1b\\[(K|.*?m)', '', s) def ansilen(s:str): return len(ansistrip(s)) +def ansipad(s:str, w:int): return s+' '*max(w-ansilen(s), 0) def make_tuple(x:int|Sequence[int], cnt:int) -> tuple[int, ...]: return (x,)*cnt if isinstance(x, int) else tuple(x) def to_tuple(x:T|tuple[T, ...]) -> tuple[T, ...]: return x if isinstance(x, tuple) else (x,) def flatten(l:Iterable[Iterable[T]]): return [item for sublist in l for item in sublist] @@ -263,6 +264,9 @@ def _get_cpu_count() -> int: NULL_ALLOW_COPYOUT = ContextVar("NULL_ALLOW_COPYOUT", 0) # VIZ implies PROFILE, but you can run PROFILE without VIZ VIZ = ContextVar("VIZ", 0) +# this PARALLEL is for BEAM and compilation, it's currently disabled if you are using VIZ +# pytest-xdist workers share the CPU budget, explicit PARALLEL still overrides this default +PARALLEL = ContextVar("PARALLEL", NUM_CPU_THREADS.value // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) PROFILE = ContextVar("PROFILE", abs(VIZ.value)) SPEC = ContextVar("SPEC", 1) # TODO: disable by default due to speed @@ -598,9 +602,9 @@ def SI(x): est_text = f'<{HMS(elapsed/prog-elapsed) if self.n else "?"}' if self.t else '' it_text = (SI(self.n/elapsed) if self.unit_scale else f"{self.n/elapsed:5.2f}") if self.n else "?" suf = f'{prog_text} [{HMS(elapsed)}{est_text}, {it_text}{self.unit}/s]' - sz = max(ncols-len(self.desc)-3-2-2-len(suf), 1) + sz = max(ncols-ansilen(self.desc)-3-2-2-len(suf), 1) bar = '\r' + self.desc + (f'{100*prog:3.0f}%|{("█"*int(num:=sz*prog)+" ▏▎▍▌▋▊▉"[int(8*num)%8].strip()).ljust(sz," ")}| ' if self.t else '') + suf - print(bar[:ncols+1], flush=True, end='\n'*close, file=sys.stderr) + print(bar, flush=True, end='\n'*close, file=sys.stderr) @classmethod def write(cls, s:str): print(f"\r\033[K{s}", flush=True, file=sys.stderr) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 4bc96020371a7..8cd584f63e776 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -258,7 +258,8 @@ class ClangRenderer(CStyleLanguage): gep_arr_threshold = 0 has_local = False has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + @property + def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] infinity = "__builtin_inff()" nan = '__builtin_nanf("")' diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 0510fa99499d3..e4d2e7471bec2 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -810,7 +810,8 @@ class X86Renderer(ISARenderer): device = "CPU" has_local = False has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + @property + def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] extra_matcher = extra_matcher pre_isel_matcher = pre_isel_matcher isel_matcher = isel_matcher diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index a6efae940f33e..2b47bb8ea990a 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -204,7 +204,8 @@ def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None) -> tuple[t class CPULLVMRenderer(LLVMRenderer): has_local = False has_threads = bool(getenv("THREADS", 1)) - global_max = (NUM_CPU_THREADS.value, 0, 0) + @property + def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] abi = 'win64cc' if sys.platform == 'win32' else None string_rewrite = base_rewrite def render(self, uops: list[UOp]) -> str: return "\n".join((k:=self._render_kernel(uops))[0] + (k[1], self._render_footer(uops))) From 9aa9e11301faf291781a78dda99b2bd5ae36decf Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:41:45 -0700 Subject: [PATCH 132/306] compile kernels in parallel (#17629) * compile kernels in parallel (slop) * cleanups * fixes * hand cleanups * no PARALLEL with VIZ * ugh, we need to remove the n from names * no hcq2 in that test * main for abstractions3 * fix compile server to be fork safe * fix num cpu threads in context * just use a lock * fix __main__ on spawn * compileonce * python3 is double * xdist sizing * parallel compile * fetch supports extract * fix issues * revert compiler server to master, drop compileonce * normal workflow * PARALLEL=0 for QCOMCL compiletest --- .github/workflows/test.yml | 3 +- test/backend/test_multitensor.py | 4 +-- tinygrad/codegen/__init__.py | 13 +++++--- tinygrad/codegen/opt/search.py | 23 +++++--------- tinygrad/engine/realize.py | 53 ++++++++++++++++++++++++++------ tinygrad/engine/worker.py | 49 +++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 32 deletions(-) create mode 100644 tinygrad/engine/worker.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e87b4d0a51d6..3dc93d9e89d86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -679,4 +679,5 @@ jobs: run: | python -c "from tinygrad import Device; assert Device.DEFAULT == 'NULL'" DEBUG=4 python3 test/backend/test_ops.py TestOps.test_add - python -m pytest -n=auto test/backend/test_ops.py --durations=20 + # QCOMCL compiles in qemu, too slow for parallel workers + ${{ contains(matrix.dev, 'QCOMCL') && 'PARALLEL=0' || '' }} python -m pytest -n=auto test/backend/test_ops.py --durations=20 diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index d76874ae92917..dc0deb0d04237 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -3,7 +3,7 @@ from tinygrad.uop.ops import Ops, UOp, AxisType, graph_rewrite from tinygrad.helpers import getenv, prod, Context from tinygrad.nn.state import get_parameters -from tinygrad.engine.realize import run_linear, compile_linear, pm_beam, pm_compile +from tinygrad.engine.realize import run_linear, compile_linear, lower_and_compile, pm_beam import numpy as np from hypothesis import given, strategies as strat, settings from test.helpers import not_support_multi_device, needs_second_gpu, slow, call_is_graph, check_schedule, assert_kernel_count, KernelCountException @@ -80,7 +80,7 @@ def test_shard_beam(self): cpu_2 = ("CPU:1", "CPU:2") src = Tensor.ones(16).shard(cpu_2, 0).realize() lin = UOp(Ops.LINEAR, src=(src.to(cpu_2[::-1]).schedule_linear().src[0],)) - with Context(BEAM=1, IGNORE_BEAM_CACHE=1): call = graph_rewrite(graph_rewrite(lin, pm_beam, ctx=1, walk=True), pm_compile, walk=True).src[0] + with Context(BEAM=1, IGNORE_BEAM_CACHE=1): call = lower_and_compile(graph_rewrite(lin, pm_beam, ctx=1, walk=True)).src[0] self.assertNotEqual(call.src[0].src[0].arg.applied_opts, ()) def test_shard_same_device(self): diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 076c0db87a6f3..277430f1fe321 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -1,7 +1,7 @@ from dataclasses import replace, dataclass import itertools, functools from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC -from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TracingKey, Context, panic +from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak, pm_cast_weak from tinygrad.uop.render import pyrender @@ -488,9 +488,14 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: if VIZ: graph_rewrite(prg, PatternMatcher([]), name="View Program") return prg +# config affects generated programs and cache keys; context also carries compile-only behavior to workers +to_program_config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, + DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT) +to_program_context = (*to_program_config, SPEC, DEBUG) +def to_program_key(ast:UOp, renderer:Renderer) -> tuple: + return (ast.key, type(renderer), renderer.target, *[x.value for x in to_program_config]) + to_program_cache: dict[tuple, UOp] = {} def to_program(ast:UOp, renderer:Renderer) -> UOp: - config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT) - key = (ast.key, type(renderer), renderer.target, *[x.value for x in config]) - if (prg:=to_program_cache.get(key)) is None: to_program_cache[key] = prg = do_to_program(ast, renderer) + if (prg:=to_program_cache.get(key:=to_program_key(ast, renderer))) is None: to_program_cache[key] = prg = do_to_program(ast, renderer) return prg diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index f4f8f601719ea..cd7bc59284a5b 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -1,12 +1,13 @@ -import math, time, multiprocessing, traceback, signal, atexit +import math, time, traceback, signal from dataclasses import replace from tinygrad.uop.ops import sym_infer, AxisType, UOp, Ops from tinygrad.uop.render import pyrender from tinygrad.device import Device, Buffer -from tinygrad.helpers import prod, flatten, DEBUG, CACHELEVEL, diskcache_get, diskcache_put, getenv, Context, colored, time_to_str +from tinygrad.helpers import prod, flatten, DEBUG, CACHELEVEL, diskcache_get, diskcache_put, getenv, colored, time_to_str from tinygrad.helpers import IGNORE_BEAM_CACHE from tinygrad.codegen.opt import Opt, OptOps, KernelOptError from tinygrad.engine.realize import time_call +from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool from tinygrad.codegen import to_program from tinygrad.codegen.opt.postrange import Scheduler @@ -78,11 +79,6 @@ def _try_compile(x:tuple[int,Scheduler]) -> tuple[int, tuple[UOp, float]|None]: if hasattr(signal, "alarm"): signal.alarm(0) return x[0], ret -# workers should not open devices and should ignore ctrl c and should not launch VIZ -def _init_worker(): - Context(ALLOW_DEVICE_USAGE=0, VIZ=0, TRACK_MATCH_STATS=0).__enter__() - signal.signal(signal.SIGINT, signal.SIG_IGN) - def _ensure_buffer_alloc(bufs:list[Buffer]) -> list[Buffer]: return [buf.ensure_allocated() if buf is not None else buf for buf in bufs] # *** external API *** @@ -111,9 +107,8 @@ def get_kernel_actions(s:Scheduler, include_0=True, max_up:int|None=None) -> dic except KernelOptError: pass return acted -beam_pool, BEAM_DEBUG = None, getenv("BEAM_DEBUG") +BEAM_DEBUG = getenv("BEAM_DEBUG") def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:int, allow_test_size=True, disable_cache=IGNORE_BEAM_CACHE.value): - global beam_pool key = {"ast": s.ast.key, "amt": amt, "allow_test_size": allow_test_size, "device": s.ren.target.device, "suffix": s.ren.suffix} if not disable_cache and CACHELEVEL >= 1 and (val:=diskcache_get("beam_search", key)) is not None: ret = s.copy() @@ -123,11 +118,7 @@ def beam_search(s:Scheduler, rawbufs:list[Buffer], var_vals:dict[str,int], amt:i beam: list[tuple[Scheduler, float]] = [(s, float("inf"))] seen_libs = set() - default_parallel = multiprocessing.cpu_count() if s.ren.target.device in {"CUDA", "AMD", "NV", "METAL", "HIP"} else 0 - if beam_pool is None and (workers := getenv("PARALLEL", default_parallel)): - beam_pool = multiprocessing.get_context("spawn").Pool(workers, _init_worker, (), getenv("BEAM_MAX_TASKS_PER_CHILD", 16)) - @atexit.register - def close_pool(): beam_pool.close() + pool = get_worker_pool() min_progress = getenv("BEAM_MIN_PROGRESS", 0.01)/1e6 if BEAM_DEBUG: @@ -143,7 +134,7 @@ def close_pool(): beam_pool.close() candidates: list[Scheduler] = flatten([get_kernel_actions(si, include_0=False).values() for si,_ in beam]) timed: list[tuple[Scheduler, float]] = [] least_compute_ops = math.inf - for i, proc in ((map if beam_pool is None else beam_pool.imap_unordered)(_try_compile, enumerate(candidates))): + for i, proc in ((map if pool is None else pool.imap_unordered)(_try_compile, enumerate(candidates))): if proc is None: continue prg, compile_et = proc if (lib:=prg.src[3].arg) in seen_libs: continue @@ -179,7 +170,7 @@ def close_pool(): beam_pool.close() print(f"\r{time.perf_counter() - st:7.2f}s:", colored(time_to_str(beam[0][1], w=12), "green" if exiting else None), f"from {len(candidates):3d} -> {len(opts):3d} actions\033[K", beam[0][0].colored_shape()) except KeyboardInterrupt as e: - if beam_pool is not None: beam_pool.terminate() + terminate_worker_pool() raise e if CACHELEVEL >= 1: diskcache_put("beam_search", key, beam[0][0].applied_opts) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index ce48d256f3bc3..fb3284b1eec45 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -2,14 +2,15 @@ from typing import cast, Iterator, Any, Sequence import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple, tqdm from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us -from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite +from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.dtype import dtypes -from tinygrad.renderer import Estimates -from tinygrad.codegen import to_program +from tinygrad.renderer import Estimates, Renderer +from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context from tinygrad.codegen.opt.postrange import args_from_ast +from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool # **************** Helpers **************** @@ -247,10 +248,44 @@ def _validate(call:UOp, sink:UOp) -> UOp: lambda ctx,call,sink: call.replace(src=(sink.replace(arg=replace(sink.arg, beam=ctx)), *call.src[1:])) if sink.arg.beam == 0 else None), ]) -pm_compile = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat((Ops.SINK, Ops.PROGRAM), name="ast"),), name="call", allow_any_len=True), lambda call,ast: - call.replace(src=(to_program(ast, Device[call.device if isinstance(call.device, str) else call.device[0]].renderer), *call.src[1:]))), -]) +# **************** parallel lowering + compilation **************** + +def _compile_kernel(x:tuple[int, tuple[UOp, Renderer], dict]) -> tuple[int, UOp]: + with Context(**x[2]): return x[0], to_program(*x[1]) + +def _needs_compile(c:UOp) -> bool: + if c.op is not Ops.CALL: return False + if c.src[0].op is Ops.SINK: return True + # a PROGRAM with a ProgramInfo and a BINARY is already compiled + return c.src[0].op is Ops.PROGRAM and not (isinstance(c.src[0].arg, ProgramInfo) and c.src[0].src[-1].op is Ops.BINARY) + +def lower_and_compile(linear:UOp) -> UOp: + # collect the kernels to lower and compile, deduped by their compile cache key + calls = [c for c in linear.toposort() if _needs_compile(c)] + rens = {c: Device[c.device if isinstance(c.device, str) else c.device[0]].renderer for c in calls} + keys = {c: to_program_key(c.src[0], rens[c]) for c in calls} + if not len(calls): return linear + + # lower and compile what's not cached, in parallel if there's a worker pool + todo = list({keys[c]: (c.src[0], rens[c]) for c in calls if keys[c] not in to_program_cache}.items()) + if len(todo): + # kernels that beam search must compile in the parent, beam needs device access to time candidates + + pool = None if len(todo) == 1 or any(getattr(c.src[0].arg, "beam", 0) for c in calls) else get_worker_pool() + ctx = {v.key: v.value for v in to_program_context} + tasks = ((i, ast_ren, ctx) for i, (_, ast_ren) in enumerate(todo)) + try: + with tqdm(total=len(todo), desc="compiling", disable=DEBUG<1) as pbar: + for i, prg in (map if pool is None else pool.imap_unordered)(_compile_kernel, tasks): + pbar.set_description(f"compiling {ansipad(prg.src[0].arg.name, 40)}") + to_program_cache[todo[i][0]] = prg + pbar.update(1) + except KeyboardInterrupt: + if pool is not None: terminate_worker_pool() + raise + + # swap the compiled PROGRAMs into the calls + return linear.substitute({c: c.replace(src=(to_program_cache[keys[c]], *c.src[1:])) for c in calls}, name="precompile kernels") pm_optimize_local_size = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), optimize_local_size), @@ -270,7 +305,7 @@ def _validate(call:UOp, sink:UOp) -> UOp: def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:list[UOp]|None=None, profile:bool|None=None) -> UOp: if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) - linear = graph_rewrite(linear, pm_compile, name="precompile kernels", walk=True) + linear = lower_and_compile(linear) linear = graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) return linear diff --git a/tinygrad/engine/worker.py b/tinygrad/engine/worker.py new file mode 100644 index 0000000000000..17fb7f92a585b --- /dev/null +++ b/tinygrad/engine/worker.py @@ -0,0 +1,49 @@ +import multiprocessing, atexit, signal, sys, threading, contextlib +from multiprocessing.context import SpawnContext, SpawnProcess +from tinygrad.helpers import Context, getenv, PARALLEL + +# generic pool of worker processes for parallel compilation, shared by kernel lowering and BEAM search + +# workers should not open devices and should ignore ctrl c and should not launch VIZ +def _init_worker(): + Context(ALLOW_DEVICE_USAGE=0, VIZ=0, TRACK_MATCH_STATS=0).__enter__() + signal.signal(signal.SIGINT, signal.SIG_IGN) + +# spawn normally reimports the user's __main__ before _init_worker. This replays top-level code and can recursively create pools. There is no public +# multiprocessing switch to skip that import, so hide the two attributes used to locate __main__ while each worker (including replacements) starts. +_spawn_lock, _missing = threading.Lock(), object() +@contextlib.contextmanager +def _without_main(): + main = sys.modules.get("__main__") + if main is None: + yield + return + with _spawn_lock: + saved = {name:getattr(main, name, _missing) for name in ("__file__", "__spec__")} + try: + for name in saved: setattr(main, name, None) + yield + finally: + for name,value in saved.items(): delattr(main, name) if value is _missing else setattr(main, name, value) + +class _WorkerProcess(SpawnProcess): + @staticmethod + def _Popen(process_obj): + with _without_main(): return SpawnProcess._Popen(process_obj) + +class _WorkerContext(SpawnContext): Process = _WorkerProcess + +worker_pool = None +def get_worker_pool(): + global worker_pool + if multiprocessing.current_process().daemon or PARALLEL == 0: return None + if worker_pool is None: + worker_pool = _WorkerContext().Pool(PARALLEL.value, _init_worker, (), getenv("BEAM_MAX_TASKS_PER_CHILD", 16)) + @atexit.register + def close_pool(pool=worker_pool): pool.close() + return worker_pool + +def terminate_worker_pool(): + global worker_pool + if worker_pool is not None: worker_pool.terminate() + worker_pool = None From 9216aa494cd9c420bf3532b0000261ae70d6d273 Mon Sep 17 00:00:00 2001 From: b1tg <33436708+b1tg@users.noreply.github.com> Date: Fri, 21 Aug 2026 09:58:34 +0800 Subject: [PATCH 133/306] llm prefill failing test (#17630) --- test/unit/test_llm_server.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/unit/test_llm_server.py b/test/unit/test_llm_server.py index e6b2df7cd4029..92b68cacca07b 100644 --- a/test/unit/test_llm_server.py +++ b/test/unit/test_llm_server.py @@ -1,6 +1,8 @@ import unittest +import numpy as np from unittest.mock import patch from tinygrad import Tensor, UOp +from tinygrad.nn.state import get_state_dict from tinygrad.schedule import schedule_cache from tinygrad.llm.model import Transformer, TransformerConfig from tinygrad.llm.serve import StreamRouter @@ -152,6 +154,23 @@ def mock_call(self, tokens, start_pos, temperature, **kwargs): # 4 tokens, chunk_size=4 -> 1 prefill chunk self.assertEqual(get_prefill_flags(list(range(4)), 4), [True, False, False]) + @unittest.expectedFailure + def test_chunked_prefill_kv_cache_matches_single_chunk(self): + config = TransformerConfig(num_blocks=1, dim=8, hidden_dim=16, n_heads=1, n_kv_heads=1, norm_eps=1e-5, + vocab_size=32, head_dim=4, rope_theta=1000000, rope_dim=4, qk_norm=4, v_head_dim=4, max_context=16) + def model(): + m = Transformer(config) + rng = np.random.RandomState(1234) + for t in get_state_dict(m).values(): + t.assign(Tensor(rng.uniform(-1, 1, t.shape).astype(np.float32))).realize() + return m + def prefill(m, chunk_size): + gen = m.generate(list(range(1, 9)), chunk_size=chunk_size, temperature=0.0) + next(gen) + return [b.cache_kv.numpy() for b in m.blk] + for g, r in zip(prefill(model(), 4), prefill(model(), 8)): + np.testing.assert_allclose(g[:, :, :, :8, :], r[:, :, :, :8, :], atol=1e-5) + def test_kv_cache_resume_matches_fresh(self): model = Transformer(TEST_CONFIG) From d716d0d9276a76380cf41bd9a7399f178fb0b1da Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:38:45 +0800 Subject: [PATCH 134/306] hotfix: qkv bw kernel requires hipcc and cdna4 (#17645) --- test/backend/test_llama_kernels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/backend/test_llama_kernels.py b/test/backend/test_llama_kernels.py index ee8538c3214bf..81f98eb7e6868 100644 --- a/test/backend/test_llama_kernels.py +++ b/test/backend/test_llama_kernels.py @@ -9,7 +9,7 @@ from extra.models.llama import apply_rotary_emb, precompute_freqs_cis from extra.thunder.amd.fa import custom_fused_qkv_rope_backward, fused_qkv_rope from test.helpers import needs_second_gpu, assert_kernel_count -from test.backend.test_asm_gemm import has_hipcc +from test.backend.test_asm_gemm import has_hipcc, is_cdna4 def run_fused_ce(bs:int, seqlen:int, vocab:int, label_smoothing:float=0.0) -> None: Tensor.manual_seed(0) @@ -129,7 +129,7 @@ def test_forward(self): self.assertTrue(k.allclose(k_ref, atol=2e-2, rtol=0).item(), "K forward mismatch") self.assertTrue(v.allclose(v_ref, atol=0, rtol=0).item(), "V forward mismatch") - @unittest.skipUnless(has_hipcc(), "backward kernel requires hipcc to compile") + @unittest.skipUnless(has_hipcc() and is_cdna4(), "backward kernel requires hipcc to compile") def test_llama31_8b(self): Tensor.manual_seed(1) B, N, H, H_KV, D = self.SHAPE From 1bcb6bdc6242207c822d9cc6d8ea30edbbbc5e5c Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 20 Aug 2026 23:56:06 -0400 Subject: [PATCH 135/306] no-op weak.py reorder [PR] (#17646) --- tinygrad/uop/weak.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 8b237e5ca5f55..ed8eb5d4a2b15 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -11,34 +11,39 @@ def commit_weak(s:UOp, dt:DType) -> UOp: # a CONST commits directly at dt (the value stays mathematical, emission truncates), a non-const src takes the cast return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) -def commit_weak_srcs(u:UOp) -> UOp|None: - if not any(s.dtype in dtypes.weaks for s in u.src): return None - if (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None +def commit_srcs_at(u:UOp, dt:DType) -> UOp: # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) -# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer -pm_commit_weak = PatternMatcher([ - (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), - # demand from the destination: a STORE's weak value commits at the destination's dtype - (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), - lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), -]) +def commit_weak_srcs(u:UOp) -> UOp|None: + if not any(s.dtype in dtypes.weaks for s in u.src) or (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None + return commit_srcs_at(u, dt) # a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - dt = least_upper_dtype(c.dtype, default_dtype(u)) - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)).cast(c.dtype) + return commit_srcs_at(u, least_upper_dtype(c.dtype, default_dtype(u))).cast(c.dtype) pm_cast_weak = PatternMatcher([ (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), (UPat(Ops.CAST, name="c", src=(UPat(Ops.CONST, dtype=dtypes.weaks, name="u"),)), lambda c,u: commit_weak(u, c.dtype)), ]) +# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer +pm_commit_weak = PatternMatcher([ + (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), + # demand from the destination: a STORE's weak value commits at the destination's dtype + (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), + lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), +]) + +# A weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition. +_lower_weak_ops = GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL} def lower_weak_node(u:UOp) -> UOp|None: - start, src = (1 if u.op is Ops.WHERE else 0), tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) + src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) + start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None + # Binary can widen from the bounds, all other nodes derive from the lowered sources. dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary else unwrap(dtype_from_uop(u.op, src, u.arg))) return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else commit_weak(s, dt) for s in src[start:])).cast(u.dtype) @@ -49,11 +54,9 @@ def lower_weak_node(u:UOp) -> UOp|None: # a SINGLE weak cast is never rewritten here, each consumer absorbs it on its own edge (see lower_weak_srcs) (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), lambda u,x: x.cast(default_dtype(u.src[0])).cast(default_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), - # Binary can widen from the bounds, all other nodes derive from the lowered sources. - # a weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition - (UPat(GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL}, name="u"), lower_weak_node), (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), + (UPat(_lower_weak_ops, name="u"), lower_weak_node), ]) def lower_weak_srcs(ctx:dict[UOp, UOp]|None, u:UOp) -> UOp|None: From 1cb06000869f7d19f91dda723a87b3a417ce53b5 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Aug 2026 21:31:43 -0700 Subject: [PATCH 136/306] fix llm vars regression (kimi) (#17644) * fix llm regression (kimi) * unneeded * more principled --- test/unit/test_call.py | 15 +++++++++++++++ test/unit/test_llm_server.py | 1 - tinygrad/schedule/__init__.py | 14 ++++++++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/test/unit/test_call.py b/test/unit/test_call.py index 12e55f12d7286..238b52fa1cc4a 100644 --- a/test/unit/test_call.py +++ b/test/unit/test_call.py @@ -224,6 +224,21 @@ def f(x:Tensor, scale:UOp) -> Tensor: return x * scale np.testing.assert_equal(x.numpy(), [2, 2, 2]) np.testing.assert_equal(y.numpy(), [3, 3, 3]) + def test_precompile_nested_scope_collision(self): + # a precompiled function body gets its own positional p{slot} params; they must not be renumbered when the call is + # scheduled inside an enclosing realize with a different slot ordering. the store must use this call's Variable + cache = Tensor.zeros(16) + @function(precompile=True, allow_implicit=True) + def store(x:Tensor, sp:UOp) -> Tensor: + # update a cache at a symbolic offset, like an attention KV cache update + return Tensor(cache.uop.after(cache[sp:sp+x.shape[0]].uop.store(x.uop)))[:sp+x.shape[0]].sum() + sp_v, nt_v = UOp.variable("sp", 0, 8), UOp.variable("nt", 1, 8) + t = Tensor.arange(16).float().realize() + sp, nt = sp_v.bind(0), nt_v.bind(8) + store(t[sp:sp+nt].clone().realize(), sp).realize() + np.testing.assert_equal(cache.numpy()[:8], t[:8].numpy()) + np.testing.assert_equal(cache.numpy()[8:], np.zeros(8)) + def test_precompile_schedule_cache_hit(self): """two instances of the same @function should produce identical function body keys (schedule cache hit)""" @function(precompile=True) diff --git a/test/unit/test_llm_server.py b/test/unit/test_llm_server.py index 92b68cacca07b..a4b17b7da1d34 100644 --- a/test/unit/test_llm_server.py +++ b/test/unit/test_llm_server.py @@ -154,7 +154,6 @@ def mock_call(self, tokens, start_pos, temperature, **kwargs): # 4 tokens, chunk_size=4 -> 1 prefill chunk self.assertEqual(get_prefill_flags(list(range(4)), 4), [True, False, False]) - @unittest.expectedFailure def test_chunked_prefill_kv_cache_matches_single_chunk(self): config = TransformerConfig(num_blocks=1, dim=8, hidden_dim=16, n_heads=1, n_kv_heads=1, norm_eps=1e-5, vocab_size=32, head_dim=4, rope_theta=1000000, rope_dim=4, qk_norm=4, v_head_dim=4, max_context=16) diff --git a/tinygrad/schedule/__init__.py b/tinygrad/schedule/__init__.py index 2a87a48fb8f41..f00ec3f7c158e 100644 --- a/tinygrad/schedule/__init__.py +++ b/tinygrad/schedule/__init__.py @@ -97,11 +97,17 @@ def create_new_buffer(ctx:tuple[dict[UOp, UOp], tuple[UOp, ...]], b:UOp): create_new_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), ]) -def resolve_linear_call(linear_call:UOp): +def resolve_linear_call(linear_call:UOp, outer_binds:dict[str, UOp]|None=None): linear = graph_rewrite(linear_call.src[0], pm_post_sched_cache, ctx=({}, linear_call.src[1:]), walk=True, name="params to buffers") - # map the call body params back to the original Variables stored in the call args - binds = {f"p{i}":x.src[0].replace(op=Ops.PARAM) for i,x in enumerate(linear_call.src[1:]) if x.is_bound_var} - return linear.substitute({v:binds[v.expr] for v in linear.variables() if v.expr in binds}, enter_calls=True, name="resolve scalar params") + # nested LINEAR calls are lexical scopes: their positional params shadow the enclosing scope, while calls without + # scalar args (e.g. precompiled allreduce) inherit it + binds = {**(outer_binds or {}), + **{f"p{i}":x.src[0].replace(op=Ops.PARAM) for i,x in enumerate(linear_call.src[1:]) if x.is_bound_var}} + def apply_binds(si:UOp) -> UOp: + if si.op is Ops.CALL and si.src[0].op is Ops.LINEAR: return resolve_linear_call(si, binds) + subs = {v:binds[v.expr] for v in si.variables() if v.expr in binds} + return si.replace(src=tuple(s.substitute(subs, name="resolve scalar params") for s in si.src)) + return linear.replace(src=tuple(apply_binds(si) for si in linear.src)) pm_resolve_linear_call = PatternMatcher([ # call LINEAR is resolved here From 80bf60d782a01fae111c497c24614a72f8dac8b6 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:52:35 +0800 Subject: [PATCH 137/306] profiler failing test for non unique kernel names (#17647) * err * expected --- test/null/test_viz.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 5bfb663baee3b..37c6524217eaf 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -516,6 +516,19 @@ def custom_binary(X:UOp): src_render = get_render(viz.data, steps[src_idx]["query"])["src"] self.assertEqual(src, src_render) + @unittest.expectedFailure + def test_profiler_duplicate_name(self): + kernel_name = "duplicate_name" + def one(A:UOp): return A[0].store(UOp.const(1.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) + def zero(A:UOp): return A[0].store(UOp.const(0.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) + with save_viz() as viz: + Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=one)[0].realize() + Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=zero)[0].realize() + ctx_refs = [i for i,c in enumerate(viz.list_items()) if c["name"] == kernel_name] + profile = decode_profile(unwrap(get_profile(viz.data, cpu_events))) + events = [e for e in profile["layout"]["NULL"]["events"] if e["name"] == kernel_name] + self.assertEqual([e["ref"] for e in events], ctx_refs) + from tinygrad.device import ProfileDeviceEvent, ProfileGraphEvent, ProfileGraphEntry from tinygrad.viz.serve import get_profile from tinygrad.viz.cli import decode_profile From dcc2d021e78a8d3731e1e1273b79fd63992474be Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:18:25 +0800 Subject: [PATCH 138/306] prereq viz cleanups for unique profile keys (#17649) * cleaner * just use VIZ=-2 * better --- .github/workflows/test.yml | 2 +- test/null/test_viz.py | 12 ++++++++---- tinygrad/codegen/__init__.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3dc93d9e89d86..70b474c17701e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -504,7 +504,7 @@ jobs: - name: Run AMD renderer tests (AMD:LLVM) run: DEV=MOCKKFD+AMD:LLVM python -m pytest -n=auto test/amd/ --durations 20 - name: Run SQTT profiling tests - run: PROFILE=1 SQTT=1 python3 -m pytest -n=auto test/amd/test_sqtt_profiler.py + run: VIZ=-2 python3 -m pytest -n=auto test/amd/test_sqtt_profiler.py - name: Run AMD emulated tests on NULL backend env: AMD: 0 diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 37c6524217eaf..766c346ef0b6c 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -522,12 +522,16 @@ def test_profiler_duplicate_name(self): def one(A:UOp): return A[0].store(UOp.const(1.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) def zero(A:UOp): return A[0].store(UOp.const(0.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) with save_viz() as viz: - Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=one)[0].realize() - Tensor.custom_kernel(Tensor.empty(4, device="NULL"), fxn=zero)[0].realize() - ctx_refs = [i for i,c in enumerate(viz.list_items()) if c["name"] == kernel_name] + @TinyJit + def f(a:Tensor, b:Tensor): return Tensor.custom_kernel(a, fxn=one)[0], Tensor.custom_kernel(b, fxn=zero)[0] + a, b = Tensor.empty(4, device="NULL"), Tensor.empty(4, device="NULL") + # warmup + for _ in range(2): Tensor.realize(*f(a, b)) + Tensor.realize(*f(a, b)) + kernels = {i for i,c in enumerate(viz.list_items()) if c["name"] == kernel_name} profile = decode_profile(unwrap(get_profile(viz.data, cpu_events))) events = [e for e in profile["layout"]["NULL"]["events"] if e["name"] == kernel_name] - self.assertEqual([e["ref"] for e in events], ctx_refs) + self.assertEqual({e["ref"] for e in events}, kernels) from tinygrad.device import ProfileDeviceEvent, ProfileGraphEvent, ProfileGraphEntry from tinygrad.viz.serve import get_profile diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 277430f1fe321..1189199714f0a 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -459,7 +459,7 @@ def do_compile(ctx:Renderer, prg:UOp, source:UOp) -> UOp|None: (UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.LINEAR), UPat(Ops.SOURCE, name="source")), name="prg"), do_compile), ]) -@rewrite_group(name=lambda ast,renderer,ret,**kwargs: TracingKey(ret.src[0].arg.name,(ret.src[0].arg.function_name, ast), ret=renderer), replay=True) +@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ast),ret=renderer), replay=True) @Context(ALLOW_DEVICE_USAGE=0) def do_to_program(ast:UOp, renderer:Renderer) -> UOp: """ From e8a8d99b992febb2d86dd9be07771fb77161e1d4 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:06:01 +0800 Subject: [PATCH 139/306] profiler tracing using unique keys (#17651) * profiler tracing using unique keys * cleaner + comment --- extra/sqtt/roc.py | 2 +- test/null/test_viz.py | 1 - tinygrad/codegen/__init__.py | 2 +- tinygrad/device.py | 5 +++-- tinygrad/engine/realize.py | 2 +- tinygrad/helpers.py | 7 ++++--- tinygrad/runtime/graph/hcq.py | 3 ++- tinygrad/runtime/graph/metal.py | 2 +- tinygrad/runtime/ops_metal.py | 6 ++++-- tinygrad/runtime/ops_null.py | 11 ++++++----- tinygrad/runtime/ops_nv.py | 4 ++-- tinygrad/runtime/support/hcq.py | 15 +++++++++------ tinygrad/uop/ops.py | 2 +- tinygrad/viz/serve.py | 19 ++++++++++--------- 14 files changed, 45 insertions(+), 36 deletions(-) diff --git a/extra/sqtt/roc.py b/extra/sqtt/roc.py index 8ed0e22a89a02..dc25f9beddda2 100755 --- a/extra/sqtt/roc.py +++ b/extra/sqtt/roc.py @@ -192,7 +192,7 @@ def unpack_insts(viz_data, i:int, j:int, data:dict) -> dict: prev_instr = max(prev_instr, e.time + e.dur) summary = [{"label":"Total Cycles", "value":w.end_time-w.begin_time}, {"label":"SE", "value":w.se}, {"label":"CU", "value":w.cu}, {"label":"SIMD", "value":w.simd}, {"label":"Wave ID", "value":w.wave_id}, {"label":"Run number", "value":data["run_number"]}] - return {"rows":[tuple(v.values()) for v in rows.values()], "cols":columns, "metadata":[summary], "ref":viz_data.ref_map.get(data["prg"].name)} + return {"rows":[tuple(v.values()) for v in rows.values()], "cols":columns, "metadata":[summary],"ref":viz_data.ref_map.get(data["prg"].profile_key)} def print_data(data:dict) -> None: from tabulate import tabulate diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 766c346ef0b6c..614f731b1f4d6 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -516,7 +516,6 @@ def custom_binary(X:UOp): src_render = get_render(viz.data, steps[src_idx]["query"])["src"] self.assertEqual(src, src_render) - @unittest.expectedFailure def test_profiler_duplicate_name(self): kernel_name = "duplicate_name" def one(A:UOp): return A[0].store(UOp.const(1.0, dtypes.float)).sink(arg=KernelInfo(kernel_name)) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 1189199714f0a..16afcdefdbb7f 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -459,7 +459,7 @@ def do_compile(ctx:Renderer, prg:UOp, source:UOp) -> UOp|None: (UPat(Ops.PROGRAM, src=(UPat(), UPat(Ops.LINEAR), UPat(Ops.SOURCE, name="source")), name="prg"), do_compile), ]) -@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ast),ret=renderer), replay=True) +@rewrite_group(name=lambda ast,renderer,ret,**_: TracingKey((k:=ret.src[0].arg).name,(k.function_name, ast, ret.key),ret=renderer), replay=True) @Context(ALLOW_DEVICE_USAGE=0) def do_to_program(ast:UOp, renderer:Renderer) -> UOp: """ diff --git a/tinygrad/device.py b/tinygrad/device.py index 85742adf3f0cf..66b4a72c02f51 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -66,10 +66,10 @@ def canonicalize_device(device:str|tuple|list|None) -> str|tuple[str, ...]: class ProfileDeviceEvent(ProfileEvent): device:str; tdiff:decimal.Decimal=decimal.Decimal(0); props:dict[str,Any]|None=None # noqa: E702 @dataclass(frozen=True) -class ProfileProgramEvent(ProfileEvent): device:str; name:str; lib:bytes|None; base:int|None; tag:int|None=None # noqa: E702 +class ProfileProgramEvent(ProfileEvent): device:str; name:str; lib:bytes|None; base:int|None; tag:int|None=None; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) -class ProfileGraphEntry: device:str; name:str|TracingKey; st_id:int; en_id:int # noqa: E702 +class ProfileGraphEntry: device:str; name:str|TracingKey; st_id:int; en_id:int; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) class ProfileGraphEvent(ProfileEvent): ents:list[ProfileGraphEntry]; deps:list[list[int]]; sigs:list[decimal.Decimal] # noqa: E702 @@ -326,6 +326,7 @@ class TinyELF: target: Target # tuple of (name, slot, dtype, shape) signature: tuple[tuple[str|None, int, DType, tuple], ...] + profile_key: bytes|None = None @staticmethod def iter_sig(signature:tuple[tuple[str|None, int, DType, tuple], ...], offset:int=0) -> Generator[tuple[int, DType], None, None]: diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index fb3284b1eec45..3fadd43037811 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -222,7 +222,7 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer()._buf.va_addr + base}), call, ast) def _prof_tm(device:str, stat_call:UOp, prof:tuple[int, ...]) -> float|None: - (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, *prof) + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, prof[0], prof[1], stat_call.key) if not ctx.wait: return None d.synchronize(timeout=ctx.timeout) st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 36f153261fc03..51fbf5dc39864 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -364,7 +364,8 @@ class TracingKey: class ProfileEvent: pass @dataclass -class ProfileRangeEvent(ProfileEvent): device:str; name:str|TracingKey; st:decimal.Decimal; en:decimal.Decimal|None=None # noqa: E702 +class ProfileRangeEvent(ProfileEvent): + device:str; name:str|TracingKey; st:decimal.Decimal; en:decimal.Decimal|None=None; profile_key:bytes|None=None # noqa: E702 @dataclass(frozen=True) class ProfilePointEvent(ProfileEvent): @@ -372,8 +373,8 @@ class ProfilePointEvent(ProfileEvent): cpu_events:list[ProfileEvent] = [] @contextlib.contextmanager -def cpu_profile(name:str|TracingKey, device="TINY", display=True) -> Generator[ProfileRangeEvent, None, None]: - res = ProfileRangeEvent(device, name, perf_counter_us()) +def cpu_profile(name:str|TracingKey, device="TINY", display=True, profile_key:bytes|None=None) -> Generator[ProfileRangeEvent, None, None]: + res = ProfileRangeEvent(device, name, perf_counter_us(), profile_key=profile_key) try: yield res finally: res.en = perf_counter_us() diff --git a/tinygrad/runtime/graph/hcq.py b/tinygrad/runtime/graph/hcq.py index e9d5e73f6275d..39e68a4430405 100644 --- a/tinygrad/runtime/graph/hcq.py +++ b/tinygrad/runtime/graph/hcq.py @@ -139,7 +139,8 @@ def __init__(self, *args, **kwargs): prof_ji_desc = runtime.name if runtime is not None else TracingKey(f"{bufs[1].device} -> {bufs[0].device}", ret=bufs[0].nbytes) prof_name = enqueue_dev.device if runtime is not None else f"{enqueue_dev.device}:SDMA:{queue_idx}" - self.prof_graph_entries.append(ProfileGraphEntry(prof_name, prof_ji_desc, sig_st, j * 2 + 1)) + self.prof_graph_entries.append(ProfileGraphEntry(prof_name, prof_ji_desc, sig_st, j * 2 + 1, + runtime.profile_key if runtime is not None else None)) self.prof_graph_deps.append([d - 1 for _, d in rdeps]) self.last_j[enqueue_queue] = j diff --git a/tinygrad/runtime/graph/metal.py b/tinygrad/runtime/graph/metal.py index d38e43043277d..39e33e8096069 100644 --- a/tinygrad/runtime/graph/metal.py +++ b/tinygrad/runtime/graph/metal.py @@ -102,7 +102,7 @@ def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=Fal def collect_timestamps(self): # create a graph event and evenly space each program st, en = decimal.Decimal(self.command_buffer.GPUStartTime()) * 1000000, decimal.Decimal(self.command_buffer.GPUEndTime()) * 1000000 - ents = [ProfileGraphEntry(self.device, rt.name, i, i+1) for i, rt in enumerate(self.runtimes) if rt is not None] + ents = [ProfileGraphEntry(self.device, rt.name, i, i+1, rt.profile_key) for i, rt in enumerate(self.runtimes) if rt is not None] self.dev.profile_events += [ProfileGraphEvent(ents, [], [st + (en-st)/len(ents)*i for i in range(len(ents)+1)])] def __del__(self): diff --git a/tinygrad/runtime/ops_metal.py b/tinygrad/runtime/ops_metal.py index 3a0fe42d2cb62..2c00be72c7439 100644 --- a/tinygrad/runtime/ops_metal.py +++ b/tinygrad/runtime/ops_metal.py @@ -34,6 +34,7 @@ def __init__(self, device:str): self.mtl_queue = self.sysdevice.newCommandQueueWithMaxCommandBufferCount(1024) if self.mtl_queue is None: raise RuntimeError("Cannot allocate a new command queue") self.mtl_buffers_in_flight: list[metal.MTLCommandBuffer] = [] + self.mtl_profile_keys: dict[int, bytes] = {} self.timeline_signal = self.sysdevice.newSharedEvent() self.timeline_value = 0 @@ -55,7 +56,7 @@ def synchronize(self): st, en = decimal.Decimal(cbuf.GPUStartTime()) * 1000000, decimal.Decimal(cbuf.GPUEndTime()) * 1000000 # NOTE: command buffers from MetalGraph are not profiled here if PROFILE and (lb:=cmdbuf_label(cbuf)) is not None and not lb.startswith("batched"): - Compiled.profile_events += [ProfileRangeEvent(self.device, lb, st, en)] + Compiled.profile_events += [ProfileRangeEvent(self.device, lb, st, en, self.mtl_profile_keys.pop(id(cbuf), None))] self.mtl_buffers_in_flight.clear() class MetalCompiler(Compiler): @@ -113,7 +114,7 @@ def disassemble(self, lib:bytes): class MetalProgram(Program[MetalDevice]): def __init__(self, dev:MetalDevice, obj:TinyELF): - self.dev, self.name, self.lib, self.signature = dev, obj.name, obj.lib, obj.signature + self.dev, self.name, self.lib, self.signature, self.profile_key = dev, obj.name, obj.lib, obj.signature, obj.profile_key data = objc.dispatch_data_create(obj.lib, len(obj.lib), None, None) self.library = self.dev.sysdevice.newLibraryWithData_error(data, ctypes.byref(error_lib:=metal.NSError().retained())).retained() error_check(error_lib) @@ -145,6 +146,7 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup command_buffer.setLabel(to_ns_str(self.name)) # TODO: is this always needed? command_buffer.commit() self.dev.mtl_buffers_in_flight.append(command_buffer) + if PROFILE and self.profile_key is not None: self.dev.mtl_profile_keys[id(command_buffer)] = self.profile_key if wait: wait_check(command_buffer) return command_buffer.GPUEndTime() - command_buffer.GPUStartTime() diff --git a/tinygrad/runtime/ops_null.py b/tinygrad/runtime/ops_null.py index 228ef917dbc3c..e7c8d97c4c0d4 100644 --- a/tinygrad/runtime/ops_null.py +++ b/tinygrad/runtime/ops_null.py @@ -17,9 +17,9 @@ def asm(self, prg: UOp, lin: UOp) -> bytes: return assemble_linear(prg, lin, self.target.arch) class NullProgram(Program['NullDevice']): - def __init__(self, dev:'NullDevice', obj:TinyELF): self.device, self.name = dev.device, obj.name + def __init__(self, dev:'NullDevice', obj:TinyELF): self.device, self.name, self.profile_key = dev.device, obj.name, obj.profile_key def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw): - with cpu_profile(self.name, self.device): return 1e-3 + with cpu_profile(self.name, self.device, profile_key=self.profile_key): return 1e-3 class NullAllocator(Allocator['NullDevice']): def _alloc(self, size, options): pass @@ -38,13 +38,14 @@ def __call__(self, input_uops:tuple[UOp, ...], var_vals:dict[str, int], wait=Fal for (_,_,bufs,_),runtime in zip(self.calls, self.runtimes): # description based on command, copied from HCQ graph device = runtime.device if runtime is not None else f"{bufs[1].device}:SDMA:0" - descs.append((device, runtime.name if runtime is not None else f"{bufs[1].device} -> {bufs[0].device}", count:=event_count.get(device, 0))) + descs.append((device, runtime.name if runtime is not None else f"{bufs[1].device} -> {bufs[0].device}", + runtime.profile_key if runtime is not None else None, count:=event_count.get(device, 0))) event_count[device] = count+1 # pack events evenly per device dur, sigs, ents = max(1, math.ceil((perf_counter_us()-st)/max(event_count.values()))), [], [] - for i,(device,name,count) in enumerate(descs): + for i,(device,name,profile_key,count) in enumerate(descs): sigs += [st+count*dur, st+(count+1)*dur] - ents.append(ProfileGraphEntry(device, name, 2*i, 2*i+1)) + ents.append(ProfileGraphEntry(device, name, 2*i, 2*i+1, profile_key)) cpu_events.append(ProfileGraphEvent(ents, [], sigs)) return 1e-1 diff --git a/tinygrad/runtime/ops_nv.py b/tinygrad/runtime/ops_nv.py index 59d6db6b2047a..5b67afe0ab22c 100644 --- a/tinygrad/runtime/ops_nv.py +++ b/tinygrad/runtime/ops_nv.py @@ -22,7 +22,7 @@ PMA = ContextVar("PMA", abs(VIZ.value)>=2) @dataclass(frozen=True) -class ProfilePMAEvent(ProfileEvent): device:str; kern:str; blob:bytes; exec_tag:int # noqa: E702 +class ProfilePMAEvent(ProfileEvent): device:str; kern:str; blob:bytes; exec_tag:int; profile_key:bytes|None=None # noqa: E702 class NVSignal(HCQSignal): def _sleep(self, time_spent_since_last_sleep_ms:int): @@ -335,7 +335,7 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup if self.dev.pma_enabled: self.dev.synchronize() if pma_blob:=self.dev._prof_readback(): - Compiled.profile_events += [ProfilePMAEvent(self.dev.device, self.name, pma_blob, self.dev.prof_exec_counter)] + Compiled.profile_events += [ProfilePMAEvent(self.dev.device, self.name, pma_blob, self.dev.prof_exec_counter, self.profile_key)] return res class NVAllocator(HCQAllocator['NVDevice']): diff --git a/tinygrad/runtime/support/hcq.py b/tinygrad/runtime/support/hcq.py index ec68b106feb76..1481fe7e0cdbd 100644 --- a/tinygrad/runtime/support/hcq.py +++ b/tinygrad/runtime/support/hcq.py @@ -295,7 +295,8 @@ def wait(self, value:int, timeout:int|None=None): if not_passed and self.value < value: raise RuntimeError(f"Wait timeout: {timeout} ms! (the signal is not set to {value}, but {self.value})") @contextlib.contextmanager -def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue]|None=None, queue:HWQueue|None=None, dev_suff:str|None=None): +def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue]|None=None, queue:HWQueue|None=None, dev_suff:str|None=None, + profile_key:bytes|None=None): st, en = (dev.new_signal(), dev.new_signal()) if enabled else (None, None) assert queue is not None or queue_type is not None, "Either queue or queue_type must be provided" @@ -309,7 +310,8 @@ def hcq_profile(dev:HCQCompiled, enabled, desc, queue_type:Callable[[], HWQueue] elif enabled and queue_type is not None: queue_type().wait(dev.timeline_signal, dev.timeline_value - 1).timestamp(en).signal(dev.timeline_signal, dev.next_timeline()).submit(dev) - if enabled and PROFILE: dev.sig_prof_records.append((unwrap(st), unwrap(en), desc, f"{dev.device}:{dev_suff}" if dev_suff else dev.device)) + if enabled and PROFILE: dev.sig_prof_records.append((unwrap(st), unwrap(en), desc, f"{dev.device}:{dev_suff}" if dev_suff else dev.device, + profile_key)) class HCQArgsState(Generic[ProgramType]): def __init__(self, buf:HCQBuffer, prg:ProgramType, bufs:tuple[HCQBuffer, ...], vals:tuple[sint|None, ...]=()): @@ -332,8 +334,9 @@ def __init__(self, buf:HCQBuffer, prg:ProgramType, bufs:tuple[HCQBuffer, ...], v class HCQProgram(Program[HCQDeviceType]): def __init__(self, args_state_t:Type[HCQArgsState], dev:HCQDeviceType, obj:TinyELF, kernargs_alloc_size:int, base:int|None=None): self.args_state_t, self.dev, self.name, self.signature, self.kernargs_alloc_size = args_state_t, dev, obj.name, obj.signature, kernargs_alloc_size + self.profile_key = obj.profile_key self.prof_prg_counter = next(self.dev.prof_prg_counter) - if PROFILE: Compiled.profile_events += [ProfileProgramEvent(dev.device, obj.name, obj.lib, base, self.prof_prg_counter)] + if PROFILE: Compiled.profile_events += [ProfileProgramEvent(dev.device, obj.name, obj.lib, base, self.prof_prg_counter, self.profile_key)] @staticmethod def _fini(dev, buf, spec): dev.allocator.free(buf, buf.size, spec) @@ -372,7 +375,7 @@ def __call__(self, *bufs:HCQBuffer, global_size:tuple[int,int,int]=(1,1,1), loca q = unwrap(self.dev.hw_compute_queue_t)().wait(self.dev.timeline_signal, self.dev.timeline_value - 1).memory_barrier() self.dev.prof_exec_counter += 1 - with hcq_profile(self.dev, queue=q, desc=self.name, enabled=wait or PROFILE) as (sig_st, sig_en): + with hcq_profile(self.dev, queue=q, desc=self.name, enabled=wait or PROFILE, profile_key=self.profile_key) as (sig_st, sig_en): q.exec(self, kernargs, global_size, local_size) q.signal(self.dev.timeline_signal, self.dev.next_timeline()).submit(self.dev) @@ -401,7 +404,7 @@ def __init__(self, device:str, allocator:HCQAllocatorBase, compilers:list[type[R self.signal_t, self.hw_compute_queue_t, self.hw_copy_queue_t = signal_t, comp_queue_t, copy_queue_t self.timeline_value:int = 1 - self.sig_prof_records:list[tuple[HCQSignal, HCQSignal, str|TracingKey, str]] = [] + self.sig_prof_records:list[tuple[HCQSignal, HCQSignal, str|TracingKey, str, bytes|None]] = [] self.prof_exec_counter:int = 0 self.prof_prg_counter = itertools.count(0) @@ -437,7 +440,7 @@ def synchronize(self, timeout:int|None=None): if self.timeline_value > (1 << 31): self._wrap_timeline_signal() if PROFILE: - Compiled.profile_events += [ProfileRangeEvent(dev, name, st.timestamp, en.timestamp) for st,en,name,dev in self.sig_prof_records] + Compiled.profile_events += [ProfileRangeEvent(dev, name, st.timestamp, en.timestamp, pk) for st,en,name,dev,pk in self.sig_prof_records] self.sig_prof_records = [] def next_timeline(self): diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index a0c0e72d6e563..75a8e26a3e86f 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1198,7 +1198,7 @@ def to_elf(self) -> TinyELF: assert self.op is Ops.PROGRAM and isinstance(self.arg, ProgramInfo), "to_elf should only be called on a PROGRAM ast" sig = tuple((u.arg.name, u.arg.slot, u.dtype, u._shape) for u in tuple(filter(lambda u: u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU, self.src[1].src)) + self.arg.vars) - return TinyELF(self.src[3].arg, self.arg.function_name, self.arg.target, sig) + return TinyELF(self.src[3].arg, self.arg.function_name, self.arg.target, sig, self.key) @dataclass(frozen=True) class KernelInfo: diff --git a/tinygrad/viz/serve.py b/tinygrad/viz/serve.py index 86ba657d7b1d9..cb7f54b63cf8a 100755 --- a/tinygrad/viz/serve.py +++ b/tinygrad/viz/serve.py @@ -231,10 +231,11 @@ def timeline_layout(data:VizData, dev_events:list[tuple[int, int, float, DevEven ei:ProfilePointEvent|None = None for st,et,dur,e in dev_events: if isinstance(e, ProfilePointEvent) and e.name == "exec": ei = e - if dur == 0: continue + # only visualize range events with an end timestamp + if dur == 0 or isinstance(e, ProfilePointEvent): continue name, key = e.name, None fmt:dict = {} - if (ref:=data.ref_map.get(name)) is not None and ref < len(data.ctxs): + if (ref:=data.ref_map.get(e.profile_key)) is not None and ref < len(data.ctxs): name = data.ctxs[ref]["name"] if (ki:=data.ctxs[ref].get("ki")) is not None and ki.estimates is not None and ei is not None: for est_key,est_val in (("FLOPS", ki.estimates.ops), ("B/s mem", ki.estimates.mem), ("B/s lds", ki.estimates.lds)): @@ -333,14 +334,14 @@ def unpack_pmc(e) -> dict: def load_amd_counters(data:VizData, profile:list) -> None: counter_events:dict[tuple[int, int], dict] = {} - durations:dict[str, list[float]] = {} + durations:dict[bytes|str, list[float]] = {} prg_events:dict[int, ProfileProgramEvent] = {} arch = "" for e in profile: if type(e).__name__ in {"ProfilePMCEvent", "ProfileSQTTEvent"}: counter_events.setdefault((e.kern, e.exec_tag), {}).setdefault(type(e).__name__, []).append(e) - if isinstance(e, ProfileRangeEvent) and e.device.startswith("AMD") and e.en is not None: - durations.setdefault(str(e.name), []).append(float(e.en-e.st)) + if isinstance(e, ProfileRangeEvent) and e.device.startswith("AMD") and e.en is not None and e.profile_key is not None: + durations.setdefault(e.profile_key, []).append(float(e.en-e.st)) if isinstance(e, ProfileProgramEvent) and e.device.startswith("AMD") and e.tag is not None: prg_events[e.tag] = e if isinstance(e, ProfileDeviceEvent) and e.device.startswith("AMD"): arch = f"gfx{unwrap(e.props)['gfx_target_version']//1000}" if len(counter_events) == 0: return None @@ -348,12 +349,12 @@ def load_amd_counters(data:VizData, profile:list) -> None: run_number = {n:0 for n,_ in counter_events} for (k, tag),v in counter_events.items(): # use the colored name if it exists - name = data.ctxs[r]["ki"].name if (r:=data.ref_map.get(pname:=prg_events[k].name)) is not None else pname + name = data.ctxs[r]["ki"].name if (r:=data.ref_map.get(unwrap(prg_events[k].profile_key))) is not None else prg_events[k].name run_number[k] += 1 steps:list[dict] = [] if (pmc:=v.get("ProfilePMCEvent")): steps.append(create_step("PMC", ("/prg-pmc", len(data.ctxs), len(steps)), pmc[0])) - all_counters[(name, run_number[k], pname)] = pmc[0] + all_counters[(name, run_number[k], unwrap(prg_events[k].profile_key))] = pmc[0] # to decode a SQTT trace, we need the raw stream, program binary and device properties if (sqtt:=v.get("ProfileSQTTEvent")): for e in sqtt: @@ -496,10 +497,10 @@ def get_profile(data:VizData, profile:list[ProfileEvent], sort_fn:Callable[[str] def load_nv_counters(data:VizData, profile:list) -> None: steps:list[dict] = [] sm_version = {e.device:e.props.get("sm_version", 0x800) for e in profile if isinstance(e, ProfileDeviceEvent) and e.props is not None} - run_number:dict[str, int] = {} + run_number:dict[bytes, int] = {} for e in profile: if type(e).__name__ == "ProfilePMAEvent": - run_number[e.kern] = run_num = run_number.get(e.kern, 0)+1 + run_number[profile_key] = run_num = run_number.get(profile_key:=unwrap(e.profile_key), 0)+1 steps.append(create_step(f"PMA {e.kern}"+(f"n{run_num}" if run_num>1 else ""), ("/prg-pma-pkts", len(data.ctxs), len(steps)), data=(e.blob, sm_version[e.device]))) if steps: data.ctxs.append({"name":"All Counters", "steps":steps}) From 1c3c9e96f63e7f344afa35ba8bce0212780eefda Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:29:11 +0800 Subject: [PATCH 140/306] remove setUp from test_sqtt_profiler (#17652) * remove setUp from test_sqtt_profiler * remove that * cleaner * do not need that --- test/amd/test_sqtt_profiler.py | 12 +++++------- test/null/test_viz.py | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/test/amd/test_sqtt_profiler.py b/test/amd/test_sqtt_profiler.py index 3738bcfcc5b23..45c23664f60ed 100644 --- a/test/amd/test_sqtt_profiler.py +++ b/test/amd/test_sqtt_profiler.py @@ -1,30 +1,28 @@ import unittest, contextlib from tinygrad import Device, Tensor, Context, TinyJit -from tinygrad.device import Compiled, ProfileProgramEvent, ProfileDeviceEvent +from tinygrad.device import Compiled, ProfileProgramEvent from tinygrad.engine.realize import run_linear from tinygrad.codegen import to_program from tinygrad.viz.serve import load_amd_counters, VizData @contextlib.contextmanager def save_sqtt(): + Device[Device.DEFAULT].synchronize() + profile_start = len(Compiled.profile_events) data = VizData() yield data.ctxs Device[Device.DEFAULT].synchronize() Device[Device.DEFAULT]._at_profile_finalize() - load_amd_counters(data, Compiled.profile_events) + load_amd_counters(data, [e for e in Compiled.profile_events[:profile_start] if isinstance(e, ProfileProgramEvent)] + + Compiled.profile_events[profile_start:]) data.ctxs[:] = [r for r in data.ctxs if r["name"].startswith("SQTT")] @unittest.skipUnless(Device.DEFAULT == "AMD", "only runs on AMD") class TestSQTTProfiler(unittest.TestCase): - # TODO: can we enable SQTT profiling in context? @classmethod def setUpClass(cls): if not Device[Device.DEFAULT].sqtt_enabled: raise unittest.SkipTest("device must be in SQTT profiling mode") - def setUp(self): - Device[Device.DEFAULT].synchronize() - Compiled.profile_events[:] = [e for e in Compiled.profile_events if isinstance(e, (ProfileProgramEvent, ProfileDeviceEvent))] - def test_simple(self): t = Tensor.empty(1) + 1 with save_sqtt() as sqtt: diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 614f731b1f4d6..9812cefda2b89 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -1,5 +1,5 @@ -import unittest, decimal, sys, json, contextlib, tempfile, pickle, io, math -from pathlib import Path +import unittest +import decimal, sys, json, contextlib, tempfile, pickle, io, math, pathlib from dataclasses import dataclass from typing import Generator @@ -835,8 +835,6 @@ def test_dedup_users(self): @needs_tracked_pm class TestCfg(unittest.TestCase): - def setUp(self): self.arch = "gfx1100" - def get_cfg(self, name:str, k:Kernel): insts = k.finalize() def fxn(out:UOp) -> UOp: @@ -845,7 +843,7 @@ def fxn(out:UOp) -> UOp: sink = UOp.sink(out.base, lidx, gidx, arg=KernelInfo(name=name)) return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) with save_viz() as viz: - with Context(DEV=f"NULL::{self.arch}"): + with Context(DEV="NULL::gfx1100"): out = Tensor.custom_kernel(Tensor.empty(1), fxn=fxn)[0] _ = do_to_program(out.schedule_linear().src[-1].src[0], Device[out.device].renderer) codegen_rewrites = next(s for s in viz.list_items() if s["name"] == name) @@ -1027,8 +1025,8 @@ def run_cli(*cli_args) -> list[dict]: @contextlib.contextmanager def write_files(viz) -> list[str]: with tempfile.TemporaryDirectory() as tmpdir: - (r:=Path(tmpdir)/"rewrites.pkl").write_bytes(pickle.dumps(viz.data.trace)) - (p:=Path(tmpdir)/"profile.pkl").write_bytes(pickle.dumps(cpu_events)) + (r:=pathlib.Path(tmpdir)/"rewrites.pkl").write_bytes(pickle.dumps(viz.data.trace)) + (p:=pathlib.Path(tmpdir)/"profile.pkl").write_bytes(pickle.dumps(cpu_events)) yield ["--rewrites-path", str(r), "--profile-path", str(p)] class TestCLI(unittest.TestCase): From 176377ff6eb98032e65efdb7da04d11d68de0aa9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 09:33:43 -0400 Subject: [PATCH 141/306] weak 1 for FDIV in get_late_rewrite_patterns [PR] (#17655) --- tinygrad/codegen/__init__.py | 8 ++++---- tinygrad/codegen/decomp/op.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 16afcdefdbb7f..0a177904c0d70 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -377,6 +377,10 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: pm_final_rewrite = pm_commit_weak+pm_cast_weak+pm_decomp+extra_matcher+pm_split_ends sink = graph_rewrite(sink, pm_final_rewrite+pm_remove_invalid, ctx=ren, name="final rewrite") + # spell every literal as a casted const CAST(dt, CONST(value)) + # TODO: remove once consts are always weak + sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) + # add implicit barriers (stores/loads through LOCAL memory ordered by AFTER or across loop iterations need workgroup barriers) sink = graph_rewrite(sink, pm_implicit_barriers, name="add implicit barriers") @@ -387,10 +391,6 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: num_params = len([x for x in sink.toposort() if x.op is Ops.PARAM and x.arg.slot != -1]) sink = graph_rewrite(sink, pm_number_params, ctx=[num_params], name="number params with -1", walk=True) - # spell every literal as a casted const CAST(dt, CONST(value)) - # TODO: remove once consts are always weak - sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) - if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") if SPEC: type_verify(sink, spec_program) diff --git a/tinygrad/codegen/decomp/op.py b/tinygrad/codegen/decomp/op.py index 6a48cdca53cc0..a23142809f7dc 100644 --- a/tinygrad/codegen/decomp/op.py +++ b/tinygrad/codegen/decomp/op.py @@ -128,6 +128,6 @@ def get_late_rewrite_patterns(ops:tuple[Ops, ...], disable_fast_idiv:bool) -> Pa if Ops.SHL in ops: pat += [(UPat.var('x').alu(Ops.SHL, UPat.cvar('n'))+UPat.var('c'), lambda x,n,c: x.alu(Ops.MULACC, x.const_like(1< a/b if Ops.FDIV in ops: - pat += [(UPat.var("x").reciprocal(), lambda x: x.const_like(1).alu(Ops.FDIV, x))] - pat += [(UPat.var("a", dtypes.floats) * UPat(Ops.FDIV, dtypes.floats, src=(UPat.const(1), UPat.var("b"))), lambda a,b: a.alu(Ops.FDIV, b))] + pat += [(UPat.var("x").reciprocal(), lambda x: UOp.const(1.0).alu(Ops.FDIV, x))] + pat += [(UPat.var("a") * UPat(Ops.FDIV, dtypes.floats, src=(UPat.const(1), UPat.var("b"))), lambda a,b: a.alu(Ops.FDIV, b))] return PatternMatcher(pat) From 554d078ac47e331df95013472ca87b545afaf5aa Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 10:03:22 -0400 Subject: [PATCH 142/306] fix render_marg [pr] (#17656) --- test/null/test_uops.py | 10 +++++++++- tinygrad/uop/render.py | 11 ++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index de18f1a93171d..a21fae30a8d9f 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -5,7 +5,7 @@ from tinygrad.helpers import Timing, Context, cdiv from tinygrad.dtype import dtypes, AddrSpace, ConstFloat, Invalid # noqa: F401 from tinygrad.device import Device -from tinygrad.uop.ops import Ops, ParamArg, PatternMatcher, UOp, UPat, dtype_from_uop, exec_alu, graph_rewrite # noqa: F401 # ParamArg used by eval(str(uop)) roundtrip tests +from tinygrad.uop.ops import Ops, AxisType, ParamArg, PatternMatcher, UOp, UPat, dtype_from_uop, exec_alu, graph_rewrite # noqa: F401 # ParamArg used by eval(str(uop)) roundtrip tests from tinygrad.uop.weak import pm_lower_index_dtype from tinygrad.uop.spec import spec_program, spec_shared, type_verify from tinygrad.uop.symbolic import sym, pm_remove_invalid @@ -457,6 +457,14 @@ def test_nested(self): self.assertEqual(a.device, Device.DEFAULT) class TestUOpRender(unittest.TestCase): + def test_render_ssimplified_marg_outside_toposort(self): + r = UOp.range(UOp.const(16, dtypes.int), 2, AxisType.WEAK, dtype=dtypes.int) + offset = (r * 2) + (r * 2) + shrink = UOp(Ops.SHRINK, src=(UOp.param(0, dtypes.uint, (32,)), offset, UOp.const(2, dtypes.int))) + self.assertIsNot(shrink.src[1], shrink.marg[0][0]) + self.assertEqual(shrink.render(simplify=False), "p0.shrink((((r2*4), 2),))") + self.assertEqual(UOp.range(1, 0, src=(shrink,), dtype=dtypes.int).render(simplify=False), "r0") + def test_render_vectorize_empty(self): u = UOp(Ops.STACK, dtype=dtypes.void, src=()) self.assertEqual(u.render(simplify=False), "{}") diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index b005a6eaf2c3f..1028f524c4238 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -1,6 +1,6 @@ from tinygrad.dtype import AddrSpace, dtypes from tinygrad.uop import Ops, GroupOp -from tinygrad.uop.ops import ParamArg, UOp, PatternMatcher, UPat, multirange_str, range_str, consumer_map_from_toposort +from tinygrad.uop.ops import ParamArg, UOp, PatternMatcher, UPat, multirange_str, range_str, consumer_map_from_toposort, sint from tinygrad.helpers import strip_parens def pretty_print(x:UOp, cache=None, d=0)->str: @@ -69,14 +69,15 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: # *** pyrender *** def srcs(ctx, src): return f"({ctx[src[0]]},)" if len(src) == 1 else f"({', '.join([ctx[x] for x in src])})" +# marg is ssimplify'd, so a bound can be a node this graph never contained +def marg_str(ctx, a:sint) -> str: return str(a) if not isinstance(a, UOp) else ctx[a] if a in ctx else a.render() + def render_marg(ctx,x:UOp): if x.op is Ops.PERMUTE: return str(x.marg) if x.op is Ops.FLIP: return str(tuple([i for i,x in enumerate(x.marg) if x])) pieces = [] - if x.op in {Ops.RESHAPE, Ops.EXPAND}: - pieces = [f"{ctx[a] if isinstance(a, UOp) else str(a)}" for a in x.marg] - if x.op in {Ops.PAD, Ops.SHRINK}: - pieces = [f"({ctx[a[0]] if isinstance(a[0], UOp) else str(a[0])}, {ctx[a[1]] if isinstance(a[1], UOp) else str(a[1])})" for a in x.marg] + if x.op in {Ops.RESHAPE, Ops.EXPAND}: pieces = [marg_str(ctx, a) for a in x.marg] + if x.op in {Ops.PAD, Ops.SHRINK}: pieces = [f"({marg_str(ctx, a[0])}, {marg_str(ctx, a[1])})" for a in x.marg] return f"({','.join(pieces)})" if len(pieces) != 1 else f"({pieces[0]},)" sugar = {Ops.SINK, Ops.END, Ops.STORE, Ops.LOAD, Ops.SQRT, Ops.INDEX, Ops.REDUCE, Ops.AFTER, Ops.THREEFRY, From 77f698e55b1d835f67b16079b3a5980fbdb43483 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:59:04 +0800 Subject: [PATCH 143/306] cleanup the mxfp4 gemm (#17660) --- extra/gemm/gemm_mxfp4.py | 679 --------------------------------------- 1 file changed, 679 deletions(-) diff --git a/extra/gemm/gemm_mxfp4.py b/extra/gemm/gemm_mxfp4.py index 04d8195eaa33e..678e1ffd5d9bf 100644 --- a/extra/gemm/gemm_mxfp4.py +++ b/extra/gemm/gemm_mxfp4.py @@ -24,14 +24,11 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_and_b32(s[1], s[1], LIT, 65535)) k.emit(s_mov_b32(s[47], s[2])) k.emit(s_mov_b32(s[48], s[3])) - k.emit(s_mov_b32(s[64], s[4])) k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) k.emit(s_mov_b32(s[8], 0)) k.emit(s_mov_b32(s[9], 0)) k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[41], 1.0)) - k.emit(s_mov_b32(s[42], 0)) k.emit(s_mov_b32(s[36], N)) k.emit(s_mov_b32(s[37], K)) k.emit(s_mov_b32(s[38], K)) @@ -42,7 +39,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) k.emit(s_mov_b32(s[39], scale_k)) k.emit(s_mov_b32(s[40], scale_k)) - k.emit(s_mov_b32(s[65], 0)) k.emit(v_lshrrev_b32_e32(v[1], 10)) k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) @@ -68,17 +64,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_or_b32(s[5 + i * 8], s[5 + i * 8], LIT, 262144)) k.emit(s_or_b32(s[9 + i * 12], s[9 + i * 12], LIT, 262144)) k.emit(s_or_b32(s[17 + i * 8], s[17 + i * 8], LIT, 262144)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(9), target='L0_0194') - k.emit(s_lshr_b32(s[66], s[45], s[65])) - k.emit(s_add_u32(s[66], s[66], LIT, 255)) - k.emit(s_lshr_b32(s[66], s[66], 8)) - k.emit(s_lshl_b32(s[66], s[66], 8)) - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_sub_i32(s[62], s[45], s[63])) - k.emit(s_cmp_lt_i32(s[62], s[66])) - k.emit(s_cselect_b32(s[45], s[62], s[66])) - k.label('L0_0194') k.emit(s_lshr_b32(s[37], s[37], 1)) k.emit(s_mul_i32(s[62], s[48], LIT, 128)) k.emit(s_mul_hi_u32(s[63], s[37], s[62])) @@ -92,14 +77,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[37], s[62])) k.emit(s_mov_b32(s[14], s[63])) k.emit(s_mov_b32(s[15], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(5), target='L0_01F4') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_lshr_b32(s[62], s[63], 1)) - k.emit(s_add_u32(s[12], s[12], s[62])) - k.emit(s_addc_u32(s[13], s[13], 0)) - k.emit(s_sub_u32(s[14], s[14], s[62])) - k.label('L0_01F4') k.emit(v_lshrrev_b32_e32(v[4], 3)) k.emit(v_lshrrev_b32_e32(v[5], 2, v[4])) k.emit(v_lshlrev_b32_e32(v[5], 4, v[5])) @@ -161,13 +138,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[39], s[62])) k.emit(s_mov_b32(s[22], s[63])) k.emit(s_mov_b32(s[23], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(4), target='L0_0334') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_add_u32(s[20], s[20], s[63])) - k.emit(s_addc_u32(s[21], s[21], 0)) - k.emit(s_sub_u32(s[22], s[22], s[63])) - k.label('L0_0334') k.emit(v_lshlrev_b32_e32(v[218], 2)) k.emit(s_mul_i32(s[63], s[46], 32)) k.emit(s_mul_i32(s[63], s[63], s[39])) @@ -189,15 +159,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[38], s[62])) k.emit(s_mov_b32(s[18], s[63])) k.emit(s_mov_b32(s[19], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(6), target='L0_03BC') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_lshr_b32(s[62], s[63], 1)) - k.emit(s_mul_i32(s[62], s[62], 16)) - k.emit(s_add_u32(s[16], s[16], s[62])) - k.emit(s_addc_u32(s[17], s[17], 0)) - k.emit(s_sub_u32(s[18], s[18], s[62])) - k.label('L0_03BC') k.emit(v_lshlrev_b32_e32(v[220], 4)) k.emit(s_mul_i32(s[63], s[46], LIT, 128)) k.emit(s_mul_i32(s[62], s[63], s[38])) @@ -223,13 +184,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[63], s[40], s[62])) k.emit(s_mov_b32(s[26], s[63])) k.emit(s_mov_b32(s[27], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(4), target='L0_047C') - k.emit(s_mul_i32(s[63], s[66], s[64])) - k.emit(s_add_u32(s[24], s[24], s[63])) - k.emit(s_addc_u32(s[25], s[25], 0)) - k.emit(s_sub_u32(s[26], s[26], s[63])) - k.label('L0_047C') k.emit(v_lshlrev_b32_e32(v[236], 2)) k.emit(s_mul_i32(s[63], s[46], LIT, 128)) k.emit(s_mul_i32(s[63], s[63], s[40])) @@ -342,17 +296,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_sub_i32(s[62], s[62], s[63])) k.emit(s_mov_b32(s[6], s[62])) k.emit(s_mov_b32(s[7], LIT, 131072)) - k.emit(s_cmp_gt_i32(s[65], 0)) - k.emit(s_cbranch_scc0(11), target='L0_0F54') - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(s_mul_i32(s[62], s[46], LIT, 256)) - k.emit(v_add_u32_e32(v[240], s[62], v[4])) - k.emit(v_add_u32_e32(v[241], LIT, v[240], 128)) - k.emit(s_mul_i32(s[62], s[36], 64)) - k.emit(v_add_u32_e32(v[242], s[62], v[240])) - k.emit(v_add_u32_e32(v[243], s[62], v[241])) - k.emit(s_branch(22), target='L0_0FAC') - k.label('L0_0F54') k.emit(v_and_b32_e64(v[4], v[0], 15)) k.emit(v_mul_lo_u32(v[240], s[36], v[4])) k.emit(v_lshrrev_b32_e32(v[4], 5)) @@ -1018,242 +961,14 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.label('L0_37CC') k.emit(s_waitcnt()) k.emit(s_barrier()) - k.emit(s_cmp_eq_u32(s[65], 0)) - k.emit(s_cbranch_scc1(2149), target='L0_5970') - k.emit(v_lshrrev_b32_e32(v[4], 4)) - k.emit(v_mul_i32_i24_e64(v[4], v[4], 8)) - k.emit(v_and_b32_e64(v[5], v[0], 15)) - k.emit(v_lshlrev_b32_e32(v[5], 8, v[5])) - k.emit(v_add_i32(v[4], v[4], v[5])) - k.emit(s_mul_i32(s[62], s[46], LIT, 16384)) - k.emit(s_add_i32(s[62], s[62], 0)) - k.emit(v_add_i32(v[4], v[4], s[62])) - for i in range(2): - for j9 in range(4): - k.emit(v_accvgpr_read(v[8 + j9 * 1 + i * 4], v[0 + j9 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j9 * 1 + i * 4], s[41], v[8 + j9 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17])) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32)) - for i in range(2): - for j10 in range(4): - k.emit(v_accvgpr_read(v[8 + j10 * 1 + i * 4], v[32 + j10 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j10 * 1 + i * 4], s[41], v[8 + j10 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 96)) - for i in range(3): - for j11 in range(2): - k.emit(v_accvgpr_read(v[8], v[4 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[5 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[6 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[7 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[20 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[21 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[22 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[23 + j11 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 0 + j11 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32 + j11 * 64, 16 + i * 16)) - for i in range(2): - for j12 in range(2): - k.emit(v_accvgpr_read(v[8 + j12 * 4], v[64 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j12 * 4], s[41], v[8 + j12 * 4])) - k.emit(v_accvgpr_read(v[9 + j12 * 4], v[65 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j12 * 4], s[41], v[9 + j12 * 4])) - k.emit(v_accvgpr_read(v[10 + j12 * 4], v[66 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j12 * 4], s[41], v[10 + j12 * 4])) - k.emit(v_accvgpr_read(v[11 + j12 * 4], v[67 + j12 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j12 * 4], s[41], v[11 + j12 * 4])) - for j13 in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + j13 * 1], v[8 + j13 * 2], v[9 + j13 * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + i * 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + i * 64)) - for i in range(3): - for j14 in range(2): - k.emit(v_accvgpr_read(v[8], v[68 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[69 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[70 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[71 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[84 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[85 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[86 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[87 + j14 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + j14 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + j14 * 64, 16 + i * 16)) - k.emit(s_waitcnt(49279)) - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(v_add_i32(v[4], v[4], s[62])) - k.emit(s_mul_i32(s[63], s[36], 0)) - k.emit(v_add_u32_e32(v[244], s[63], v[240])) - k.emit(ds_read_b32(v[16], v[4])) - for i in range(15): - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 1 + i * 4)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 2 + i * 4)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 3 + i * 4)) - for j15 in range(4): - k.emit(s_waitcnt(50047 + j15 * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + j15 * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_mul_i32(s[63], s[36], 4 + i * 4)) - k.emit(v_add_u32_e32(v[244], s[63], v[240])) - k.emit(ds_read_b32(v[16], v[4], v[0], v[0], 0, 0, 4 + i * 4)) - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 61)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 62)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 63)) - for i in range(4): - k.emit(s_waitcnt(50047 + i * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(v_lshrrev_b32_e32(v[4], 4)) - k.emit(v_mul_i32_i24_e64(v[4], v[4], 8)) - k.emit(v_and_b32_e64(v[5], v[0], 15)) - k.emit(v_lshlrev_b32_e32(v[5], 8, v[5])) - k.emit(v_add_i32(v[4], v[4], v[5])) - k.emit(s_mul_i32(s[62], s[46], LIT, 16384)) - k.emit(s_add_i32(s[62], s[62], 0)) - k.emit(v_add_i32(v[4], v[4], s[62])) - for i in range(2): - for j16 in range(4): - k.emit(v_accvgpr_read(v[8 + j16 * 1 + i * 4], v[128 + j16 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j16 * 1 + i * 4], s[41], v[8 + j16 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17])) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32)) - for i in range(2): - for j17 in range(4): - k.emit(v_accvgpr_read(v[8 + j17 * 1 + i * 4], v[160 + j17 * 1 + i * 16])) - k.emit(v_mul_f32_e32(v[8 + j17 * 1 + i * 4], s[41], v[8 + j17 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 96)) - for i in range(3): - for j18 in range(2): - k.emit(v_accvgpr_read(v[8], v[132 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[133 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[134 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[135 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[148 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[149 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[150 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[151 + j18 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 0 + j18 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 32 + j18 * 64, 16 + i * 16)) - for i in range(2): - for j19 in range(2): - k.emit(v_accvgpr_read(v[8 + j19 * 4], v[192 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j19 * 4], s[41], v[8 + j19 * 4])) - k.emit(v_accvgpr_read(v[9 + j19 * 4], v[193 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j19 * 4], s[41], v[9 + j19 * 4])) - k.emit(v_accvgpr_read(v[10 + j19 * 4], v[194 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j19 * 4], s[41], v[10 + j19 * 4])) - k.emit(v_accvgpr_read(v[11 + j19 * 4], v[195 + j19 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j19 * 4], s[41], v[11 + j19 * 4])) - for j20 in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + j20 * 1], v[8 + j20 * 2], v[9 + j20 * 2])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + i * 64)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + i * 64)) - for i in range(3): - for j21 in range(2): - k.emit(v_accvgpr_read(v[8], v[196 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) - k.emit(v_accvgpr_read(v[9], v[197 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) - k.emit(v_accvgpr_read(v[10], v[198 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) - k.emit(v_accvgpr_read(v[11], v[199 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) - k.emit(v_accvgpr_read(v[12], v[212 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) - k.emit(v_accvgpr_read(v[13], v[213 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) - k.emit(v_accvgpr_read(v[14], v[214 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) - k.emit(v_accvgpr_read(v[15], v[215 + j21 * 32 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(ds_write_b64(v[0], v[4], v[16:17], v[0], 0, 128 + j21 * 64, 16 + i * 16)) - k.emit(ds_write_b64(v[0], v[4], v[18:19], v[0], 0, 160 + j21 * 64, 16 + i * 16)) - k.emit(s_waitcnt(49279)) - k.emit(v_mul_i32_i24_e64(v[4], v[0], 4)) - k.emit(v_add_i32(v[4], v[4], s[62])) - k.emit(s_mul_i32(s[63], s[36], 0)) - k.emit(v_add_u32_e32(v[244], s[63], v[242])) - k.emit(ds_read_b32(v[16], v[4])) - for i in range(15): - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 1 + i * 4)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 2 + i * 4)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 3 + i * 4)) - for j22 in range(4): - k.emit(s_waitcnt(50047 + j22 * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + j22 * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_mul_i32(s[63], s[36], 4 + i * 4)) - k.emit(v_add_u32_e32(v[244], s[63], v[242])) - k.emit(ds_read_b32(v[16], v[4], v[0], v[0], 0, 0, 4 + i * 4)) - k.emit(ds_read_b32(v[17], v[4], v[0], v[0], 0, 0, 61)) - k.emit(ds_read_b32(v[18], v[4], v[0], v[0], 0, 0, 62)) - k.emit(ds_read_b32(v[19], v[4], v[0], v[0], 0, 0, 63)) - for i in range(4): - k.emit(s_waitcnt(50047 + i * -256)) - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[244], s[4:7], 0, 0, 1)) - k.emit(v_add_u32_e64(v[244], v[244], s[36])) - k.emit(s_branch(1344), target='L0_6E70') - k.label('L0_5970') k.emit(s_mul_i32(s[62], s[36], 0)) k.emit(v_add_u32_e32(v[244], s[62], v[240])) for i in range(2): for j23 in range(2): k.emit(v_accvgpr_read(v[8 + j23 * 4], v[0 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j23 * 4], s[41], v[8 + j23 * 4])) k.emit(v_accvgpr_read(v[9 + j23 * 4], v[1 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j23 * 4], s[41], v[9 + j23 * 4])) k.emit(v_accvgpr_read(v[10 + j23 * 4], v[2 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j23 * 4], s[41], v[10 + j23 * 4])) k.emit(v_accvgpr_read(v[11 + j23 * 4], v[3 + j23 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j23 * 4], s[41], v[11 + j23 * 4])) for j24 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j24 * 1], v[8 + j24 * 2], v[9 + j24 * 2])) k.emit(s_nop(1)) @@ -1268,13 +983,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j25 in range(2): k.emit(v_accvgpr_read(v[8 + j25 * 4], v[4 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j25 * 4], s[41], v[8 + j25 * 4])) k.emit(v_accvgpr_read(v[9 + j25 * 4], v[5 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j25 * 4], s[41], v[9 + j25 * 4])) k.emit(v_accvgpr_read(v[10 + j25 * 4], v[6 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j25 * 4], s[41], v[10 + j25 * 4])) k.emit(v_accvgpr_read(v[11 + j25 * 4], v[7 + j25 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j25 * 4], s[41], v[11 + j25 * 4])) for j26 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j26 * 1], v[8 + j26 * 2], v[9 + j26 * 2])) k.emit(s_nop(1)) @@ -1289,13 +1000,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j27 in range(2): k.emit(v_accvgpr_read(v[8 + j27 * 4], v[8 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j27 * 4], s[41], v[8 + j27 * 4])) k.emit(v_accvgpr_read(v[9 + j27 * 4], v[9 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j27 * 4], s[41], v[9 + j27 * 4])) k.emit(v_accvgpr_read(v[10 + j27 * 4], v[10 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j27 * 4], s[41], v[10 + j27 * 4])) k.emit(v_accvgpr_read(v[11 + j27 * 4], v[11 + j27 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j27 * 4], s[41], v[11 + j27 * 4])) for j28 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j28 * 1], v[8 + j28 * 2], v[9 + j28 * 2])) k.emit(s_nop(1)) @@ -1310,13 +1017,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j29 in range(2): k.emit(v_accvgpr_read(v[8 + j29 * 4], v[12 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j29 * 4], s[41], v[8 + j29 * 4])) k.emit(v_accvgpr_read(v[9 + j29 * 4], v[13 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j29 * 4], s[41], v[9 + j29 * 4])) k.emit(v_accvgpr_read(v[10 + j29 * 4], v[14 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j29 * 4], s[41], v[10 + j29 * 4])) k.emit(v_accvgpr_read(v[11 + j29 * 4], v[15 + j29 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j29 * 4], s[41], v[11 + j29 * 4])) for j30 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j30 * 1], v[8 + j30 * 2], v[9 + j30 * 2])) k.emit(s_nop(1)) @@ -1331,13 +1034,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j31 in range(2): k.emit(v_accvgpr_read(v[8 + j31 * 4], v[64 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j31 * 4], s[41], v[8 + j31 * 4])) k.emit(v_accvgpr_read(v[9 + j31 * 4], v[65 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j31 * 4], s[41], v[9 + j31 * 4])) k.emit(v_accvgpr_read(v[10 + j31 * 4], v[66 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j31 * 4], s[41], v[10 + j31 * 4])) k.emit(v_accvgpr_read(v[11 + j31 * 4], v[67 + j31 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j31 * 4], s[41], v[11 + j31 * 4])) for j32 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j32 * 1], v[8 + j32 * 2], v[9 + j32 * 2])) k.emit(s_nop(1)) @@ -1352,13 +1051,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j33 in range(2): k.emit(v_accvgpr_read(v[8 + j33 * 4], v[68 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j33 * 4], s[41], v[8 + j33 * 4])) k.emit(v_accvgpr_read(v[9 + j33 * 4], v[69 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j33 * 4], s[41], v[9 + j33 * 4])) k.emit(v_accvgpr_read(v[10 + j33 * 4], v[70 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j33 * 4], s[41], v[10 + j33 * 4])) k.emit(v_accvgpr_read(v[11 + j33 * 4], v[71 + j33 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j33 * 4], s[41], v[11 + j33 * 4])) for j34 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j34 * 1], v[8 + j34 * 2], v[9 + j34 * 2])) k.emit(s_nop(1)) @@ -1373,13 +1068,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j35 in range(2): k.emit(v_accvgpr_read(v[8 + j35 * 4], v[72 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j35 * 4], s[41], v[8 + j35 * 4])) k.emit(v_accvgpr_read(v[9 + j35 * 4], v[73 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j35 * 4], s[41], v[9 + j35 * 4])) k.emit(v_accvgpr_read(v[10 + j35 * 4], v[74 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j35 * 4], s[41], v[10 + j35 * 4])) k.emit(v_accvgpr_read(v[11 + j35 * 4], v[75 + j35 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j35 * 4], s[41], v[11 + j35 * 4])) for j36 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j36 * 1], v[8 + j36 * 2], v[9 + j36 * 2])) k.emit(s_nop(1)) @@ -1394,13 +1085,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j37 in range(2): k.emit(v_accvgpr_read(v[8 + j37 * 4], v[76 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j37 * 4], s[41], v[8 + j37 * 4])) k.emit(v_accvgpr_read(v[9 + j37 * 4], v[77 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j37 * 4], s[41], v[9 + j37 * 4])) k.emit(v_accvgpr_read(v[10 + j37 * 4], v[78 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j37 * 4], s[41], v[10 + j37 * 4])) k.emit(v_accvgpr_read(v[11 + j37 * 4], v[79 + j37 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j37 * 4], s[41], v[11 + j37 * 4])) for j38 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j38 * 1], v[8 + j38 * 2], v[9 + j38 * 2])) k.emit(s_nop(1)) @@ -1415,13 +1102,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j39 in range(2): k.emit(v_accvgpr_read(v[8 + j39 * 4], v[128 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j39 * 4], s[41], v[8 + j39 * 4])) k.emit(v_accvgpr_read(v[9 + j39 * 4], v[129 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j39 * 4], s[41], v[9 + j39 * 4])) k.emit(v_accvgpr_read(v[10 + j39 * 4], v[130 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j39 * 4], s[41], v[10 + j39 * 4])) k.emit(v_accvgpr_read(v[11 + j39 * 4], v[131 + j39 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j39 * 4], s[41], v[11 + j39 * 4])) for j40 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j40 * 1], v[8 + j40 * 2], v[9 + j40 * 2])) k.emit(s_nop(1)) @@ -1436,13 +1119,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j41 in range(2): k.emit(v_accvgpr_read(v[8 + j41 * 4], v[132 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j41 * 4], s[41], v[8 + j41 * 4])) k.emit(v_accvgpr_read(v[9 + j41 * 4], v[133 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j41 * 4], s[41], v[9 + j41 * 4])) k.emit(v_accvgpr_read(v[10 + j41 * 4], v[134 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j41 * 4], s[41], v[10 + j41 * 4])) k.emit(v_accvgpr_read(v[11 + j41 * 4], v[135 + j41 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j41 * 4], s[41], v[11 + j41 * 4])) for j42 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j42 * 1], v[8 + j42 * 2], v[9 + j42 * 2])) k.emit(s_nop(1)) @@ -1457,21 +1136,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[244], s[62], v[242 + i * 1])) for j43 in range(2): k.emit(v_accvgpr_read(v[8], v[136 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) k.emit(v_accvgpr_read(v[9], v[137 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) k.emit(v_accvgpr_read(v[10], v[138 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) k.emit(v_accvgpr_read(v[11], v[139 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) k.emit(v_accvgpr_read(v[12], v[152 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) k.emit(v_accvgpr_read(v[13], v[153 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) k.emit(v_accvgpr_read(v[14], v[154 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) k.emit(v_accvgpr_read(v[15], v[155 + j43 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) @@ -1487,21 +1158,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[244], s[62], v[242 + i * 1])) for j44 in range(2): k.emit(v_accvgpr_read(v[8], v[140 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[8], s[41], v[8])) k.emit(v_accvgpr_read(v[9], v[141 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[9], s[41], v[9])) k.emit(v_accvgpr_read(v[10], v[142 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[10], s[41], v[10])) k.emit(v_accvgpr_read(v[11], v[143 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[11], s[41], v[11])) k.emit(v_accvgpr_read(v[12], v[156 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[12], s[41], v[12])) k.emit(v_accvgpr_read(v[13], v[157 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[13], s[41], v[13])) k.emit(v_accvgpr_read(v[14], v[158 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[14], s[41], v[14])) k.emit(v_accvgpr_read(v[15], v[159 + j44 * 32 + i * 56])) - k.emit(v_mul_f32_e32(v[15], s[41], v[15])) k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) @@ -1518,13 +1181,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j45 in range(2): k.emit(v_accvgpr_read(v[8 + j45 * 4], v[200 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j45 * 4], s[41], v[8 + j45 * 4])) k.emit(v_accvgpr_read(v[9 + j45 * 4], v[201 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j45 * 4], s[41], v[9 + j45 * 4])) k.emit(v_accvgpr_read(v[10 + j45 * 4], v[202 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j45 * 4], s[41], v[10 + j45 * 4])) k.emit(v_accvgpr_read(v[11 + j45 * 4], v[203 + j45 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j45 * 4], s[41], v[11 + j45 * 4])) for j46 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j46 * 1], v[8 + j46 * 2], v[9 + j46 * 2])) k.emit(s_nop(1)) @@ -1539,13 +1198,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j47 in range(2): k.emit(v_accvgpr_read(v[8 + j47 * 4], v[204 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j47 * 4], s[41], v[8 + j47 * 4])) k.emit(v_accvgpr_read(v[9 + j47 * 4], v[205 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[9 + j47 * 4], s[41], v[9 + j47 * 4])) k.emit(v_accvgpr_read(v[10 + j47 * 4], v[206 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[10 + j47 * 4], s[41], v[10 + j47 * 4])) k.emit(v_accvgpr_read(v[11 + j47 * 4], v[207 + j47 * 16 + i * 32])) - k.emit(v_mul_f32_e32(v[11 + j47 * 4], s[41], v[11 + j47 * 4])) for j48 in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + j48 * 1], v[8 + j48 * 2], v[9 + j48 * 2])) k.emit(s_nop(1)) @@ -1555,7 +1210,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_nop(1)) k.emit(buffer_store_dwordx4(v[16:19], v[244], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[244], v[244], 64)) - k.label('L0_6E70') k.emit(s_waitcnt()) k.emit(s_endpgm()) elif (tile_m, tile_n) == (192, 256): @@ -1565,8 +1219,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mov_b32(s[9], 0)) k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[41], 1.0)) - k.emit(s_mov_b32(s[42], 0)) k.emit(s_mov_b32(s[36], N)) k.emit(s_mov_b32(s[37], K)) k.emit(s_mov_b32(s[38], K)) @@ -2583,14 +2235,11 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_endpgm()) elif (tile_m, tile_n) == (256, 256): k.emit(s_and_b32(s[1], s[1], LIT, 65535)) - k.emit(s_mov_b32(s[56], s[4])) k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) k.emit(s_mov_b32(s[8], 0)) k.emit(s_mov_b32(s[9], 0)) k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[38], 1.0)) - k.emit(s_mov_b32(s[39], 0)) k.emit(s_mov_b32(s[40], N)) k.emit(s_mov_b32(s[41], K)) k.emit(s_mov_b32(s[42], K)) @@ -2601,7 +2250,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) k.emit(s_mov_b32(s[36], scale_k)) k.emit(s_mov_b32(s[37], scale_k)) - k.emit(s_mov_b32(s[57], 0)) k.emit(v_lshrrev_b32_e32(v[1], 10)) k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) @@ -2680,28 +2328,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_or_b32(s[9], s[9], LIT, 262144)) k.emit(s_or_b32(s[17], s[17], LIT, 262144)) k.emit(s_or_b32(s[13], s[13], LIT, 262144)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(9), target='L2_0234') - k.emit(s_lshr_b32(s[58], s[45], s[57])) - k.emit(s_add_u32(s[58], s[58], LIT, 255)) - k.emit(s_lshr_b32(s[58], s[58], 8)) - k.emit(s_lshl_b32(s[58], s[58], 8)) - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_sub_i32(s[52], s[45], s[53])) - k.emit(s_cmp_lt_i32(s[52], s[58])) - k.emit(s_cselect_b32(s[45], s[52], s[58])) - k.label('L2_0234') k.emit(s_lshr_b32(s[41], s[41], 1)) k.emit(s_mul_i32(s[52], s[41], s[43])) k.emit(s_mov_b32(s[14], s[52])) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(5), target='L2_025C') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_lshr_b32(s[52], s[53], 1)) - k.emit(s_add_u32(s[12], s[12], s[52])) - k.emit(s_addc_u32(s[13], s[13], 0)) - k.emit(s_sub_u32(s[14], s[14], s[52])) - k.label('L2_025C') k.emit(s_lshr_b32(s[42], s[42], 1)) k.emit(s_mul_i32(s[52], s[42], s[44])) k.emit(s_mov_b32(s[18], s[52])) @@ -2765,13 +2394,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mov_b32(s[52], LIT, 4096)) k.emit(v_add_u32_e64(v[220], v[220], s[52])) k.emit(v_add_u32_e32(v[221], LIT, v[220], 33792)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(4), target='L2_03C4') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_add_u32(s[20], s[20], s[53])) - k.emit(s_addc_u32(s[21], s[21], 0)) - k.emit(s_sub_u32(s[22], s[22], s[53])) - k.label('L2_03C4') k.emit(v_lshlrev_b32_e32(v[222], 2)) k.emit(s_mul_i32(s[52], s[47], LIT, 256)) k.emit(s_mul_i32(s[53], s[49], 32)) @@ -2784,15 +2406,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_add_i32(s[60], s[60], 0)) k.emit(v_lshlrev_b32_e32(v[224], 2)) k.emit(v_add_u32_e32(v[224], 0, v[224])) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(6), target='L2_0420') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_lshr_b32(s[52], s[53], 1)) - k.emit(s_mul_i32(s[52], s[52], 16)) - k.emit(s_add_u32(s[16], s[16], s[52])) - k.emit(s_addc_u32(s[17], s[17], 0)) - k.emit(s_sub_u32(s[18], s[18], s[52])) - k.label('L2_0420') k.emit(v_lshlrev_b32_e32(v[225], 4)) k.emit(s_mul_i32(s[52], s[46], LIT, 256)) k.emit(s_mul_i32(s[53], s[49], 64)) @@ -2805,13 +2418,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[228], s[52], v[227])) for i in range(4): k.emit(v_add_u32_e32(v[229 + i * 1], LIT, v[225 + i * 1], 1024)) - k.emit(s_cmp_gt_u32(s[57], 0)) - k.emit(s_cbranch_scc0(4), target='L2_0484') - k.emit(s_mul_i32(s[53], s[58], s[56])) - k.emit(s_add_u32(s[24], s[24], s[53])) - k.emit(s_addc_u32(s[25], s[25], 0)) - k.emit(s_sub_u32(s[26], s[26], s[53])) - k.label('L2_0484') k.emit(v_lshlrev_b32_e32(v[233], 2)) k.emit(s_mul_i32(s[52], s[46], LIT, 256)) k.emit(s_mul_i32(s[53], s[49], 64)) @@ -3619,270 +3225,9 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.label('L2_3B10') k.emit(s_waitcnt(112)) k.emit(s_barrier()) - k.emit(s_cmp_eq_u32(s[57], 0)) - k.emit(s_cbranch_scc1(1505), target='L2_52A4') - for i in range(2): - for j77 in range(4): - k.emit(v_accvgpr_read(v[8 + j77 * 1 + i * 4], v[0 + j77 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j77 * 1 + i * 4], s[38], v[8 + j77 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[235], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[235], v[235], 64)) - for i in range(2): - for j78 in range(4): - k.emit(v_accvgpr_read(v[8 + j78 * 1 + i * 4], v[64 + j78 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j78 * 1 + i * 4], s[38], v[8 + j78 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[235], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[235], v[235], 64)) - for i in range(2): - for j79 in range(4): - k.emit(v_accvgpr_read(v[8 + j79 * 1 + i * 4], v[4 + j79 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j79 * 1 + i * 4], s[38], v[8 + j79 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[236], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[236], v[236], 64)) - for i in range(2): - for j80 in range(4): - k.emit(v_accvgpr_read(v[8 + j80 * 1 + i * 4], v[68 + j80 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j80 * 1 + i * 4], s[38], v[8 + j80 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[236], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[236], v[236], 64)) - for i in range(2): - for j81 in range(4): - k.emit(v_accvgpr_read(v[8 + j81 * 1 + i * 4], v[8 + j81 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j81 * 1 + i * 4], s[38], v[8 + j81 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[237], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[237], v[237], 64)) - for i in range(2): - for j82 in range(4): - k.emit(v_accvgpr_read(v[8 + j82 * 1 + i * 4], v[72 + j82 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j82 * 1 + i * 4], s[38], v[8 + j82 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[237], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[237], v[237], 64)) - for i in range(2): - for j83 in range(4): - k.emit(v_accvgpr_read(v[8 + j83 * 1 + i * 4], v[12 + j83 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j83 * 1 + i * 4], s[38], v[8 + j83 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[238], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[238], v[238], 64)) - for i in range(2): - for j84 in range(4): - k.emit(v_accvgpr_read(v[8 + j84 * 1 + i * 4], v[76 + j84 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j84 * 1 + i * 4], s[38], v[8 + j84 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[238], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[238], v[238], 64)) - for i in range(2): - for j85 in range(4): - k.emit(v_accvgpr_read(v[8 + j85 * 1 + i * 4], v[16 + j85 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j85 * 1 + i * 4], s[38], v[8 + j85 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[239], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[239], v[239], 64)) - for i in range(2): - for j86 in range(4): - k.emit(v_accvgpr_read(v[8 + j86 * 1 + i * 4], v[80 + j86 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j86 * 1 + i * 4], s[38], v[8 + j86 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[239], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[239], v[239], 64)) - for i in range(2): - for j87 in range(4): - k.emit(v_accvgpr_read(v[8 + j87 * 1 + i * 4], v[20 + j87 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j87 * 1 + i * 4], s[38], v[8 + j87 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[240], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[240], v[240], 64)) - for i in range(2): - for j88 in range(4): - k.emit(v_accvgpr_read(v[8 + j88 * 1 + i * 4], v[84 + j88 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j88 * 1 + i * 4], s[38], v[8 + j88 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[240], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[240], v[240], 64)) - for i in range(2): - for j89 in range(4): - k.emit(v_accvgpr_read(v[8 + j89 * 1 + i * 4], v[24 + j89 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j89 * 1 + i * 4], s[38], v[8 + j89 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[241], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[241], v[241], 64)) - for i in range(2): - for j90 in range(4): - k.emit(v_accvgpr_read(v[8 + j90 * 1 + i * 4], v[88 + j90 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j90 * 1 + i * 4], s[38], v[8 + j90 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[241], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[241], v[241], 64)) - for i in range(2): - for j91 in range(4): - k.emit(v_accvgpr_read(v[8 + j91 * 1 + i * 4], v[28 + j91 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j91 * 1 + i * 4], s[38], v[8 + j91 * 1 + i * 4])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[242], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[242], v[242], 64)) - for i in range(2): - for j92 in range(4): - k.emit(v_accvgpr_read(v[8 + j92 * 1 + i * 4], v[92 + j92 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j92 * 1 + i * 4], s[38], v[8 + j92 * 1 + i * 4])) - for i in range(8): - for j93 in range(2): - k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) - k.emit(v_cvt_pk_bf16_f32(v[17], v[10], v[11])) - k.emit(v_cvt_pk_bf16_f32(v[18], v[12], v[13])) - k.emit(v_cvt_pk_bf16_f32(v[19], v[14], v[15])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - k.emit(buffer_atomic_pk_add_bf16(v[16], v[242 + j93 * 1 + i * 1], s[4:7], 0, 0, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[17], v[242 + j93 * 1 + i * 1], s[4:7], 0, 4, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[18], v[242 + j93 * 1 + i * 1], s[4:7], 0, 8, 1)) - k.emit(buffer_atomic_pk_add_bf16(v[19], v[242 + j93 * 1 + i * 1], s[4:7], 0, 12, 1)) - k.emit(v_add_i32(v[242 + j93 * 1 + i * 1], v[242 + j93 * 1 + i * 1], 64)) - k.emit(v_accvgpr_read(v[8], v[128 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[38], v[8])) - k.emit(v_accvgpr_read(v[9], v[129 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[38], v[9])) - k.emit(v_accvgpr_read(v[10], v[130 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[38], v[10])) - k.emit(v_accvgpr_read(v[11], v[131 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[38], v[11])) - k.emit(v_accvgpr_read(v[12], v[160 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[38], v[12])) - k.emit(v_accvgpr_read(v[13], v[161 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[38], v[13])) - k.emit(v_accvgpr_read(v[14], v[162 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[38], v[14])) - k.emit(v_accvgpr_read(v[15], v[163 + j93 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[38], v[15])) - for i in range(4): - k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[16], v[18])) - k.emit(s_nop(1)) - k.emit(v_permlane16_swap_b32_e32(v[17], v[19])) - k.emit(s_nop(1)) - for i in range(4): - k.emit(buffer_atomic_pk_add_bf16(v[16 + i * 1], v[250], s[4:7], 0, 0 + i * 4, 1)) - k.emit(v_add_i32(v[250], v[250], 64)) - k.emit(s_branch(1312), target='L2_6724') - k.label('L2_52A4') for i in range(2): for j94 in range(4): k.emit(v_accvgpr_read(v[8 + j94 * 1 + i * 4], v[0 + j94 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j94 * 1 + i * 4], s[38], v[8 + j94 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3895,7 +3240,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j95 in range(4): k.emit(v_accvgpr_read(v[8 + j95 * 1 + i * 4], v[64 + j95 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j95 * 1 + i * 4], s[38], v[8 + j95 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3908,7 +3252,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j96 in range(4): k.emit(v_accvgpr_read(v[8 + j96 * 1 + i * 4], v[4 + j96 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j96 * 1 + i * 4], s[38], v[8 + j96 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3921,7 +3264,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j97 in range(4): k.emit(v_accvgpr_read(v[8 + j97 * 1 + i * 4], v[68 + j97 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j97 * 1 + i * 4], s[38], v[8 + j97 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3934,7 +3276,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j98 in range(4): k.emit(v_accvgpr_read(v[8 + j98 * 1 + i * 4], v[8 + j98 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j98 * 1 + i * 4], s[38], v[8 + j98 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3947,7 +3288,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j99 in range(4): k.emit(v_accvgpr_read(v[8 + j99 * 1 + i * 4], v[72 + j99 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j99 * 1 + i * 4], s[38], v[8 + j99 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3960,7 +3300,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j100 in range(4): k.emit(v_accvgpr_read(v[8 + j100 * 1 + i * 4], v[12 + j100 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j100 * 1 + i * 4], s[38], v[8 + j100 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3973,7 +3312,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j101 in range(4): k.emit(v_accvgpr_read(v[8 + j101 * 1 + i * 4], v[76 + j101 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j101 * 1 + i * 4], s[38], v[8 + j101 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3986,7 +3324,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j102 in range(4): k.emit(v_accvgpr_read(v[8 + j102 * 1 + i * 4], v[16 + j102 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j102 * 1 + i * 4], s[38], v[8 + j102 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -3999,7 +3336,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j103 in range(4): k.emit(v_accvgpr_read(v[8 + j103 * 1 + i * 4], v[80 + j103 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j103 * 1 + i * 4], s[38], v[8 + j103 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4012,7 +3348,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j104 in range(4): k.emit(v_accvgpr_read(v[8 + j104 * 1 + i * 4], v[20 + j104 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j104 * 1 + i * 4], s[38], v[8 + j104 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4025,7 +3360,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j105 in range(4): k.emit(v_accvgpr_read(v[8 + j105 * 1 + i * 4], v[84 + j105 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j105 * 1 + i * 4], s[38], v[8 + j105 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4038,7 +3372,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j106 in range(4): k.emit(v_accvgpr_read(v[8 + j106 * 1 + i * 4], v[24 + j106 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j106 * 1 + i * 4], s[38], v[8 + j106 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4051,7 +3384,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j107 in range(4): k.emit(v_accvgpr_read(v[8 + j107 * 1 + i * 4], v[88 + j107 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j107 * 1 + i * 4], s[38], v[8 + j107 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4064,7 +3396,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j108 in range(4): k.emit(v_accvgpr_read(v[8 + j108 * 1 + i * 4], v[28 + j108 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j108 * 1 + i * 4], s[38], v[8 + j108 * 1 + i * 4])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4077,7 +3408,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): for i in range(2): for j109 in range(4): k.emit(v_accvgpr_read(v[8 + j109 * 1 + i * 4], v[92 + j109 * 1 + i * 32])) - k.emit(v_mul_f32_e32(v[8 + j109 * 1 + i * 4], s[38], v[8 + j109 * 1 + i * 4])) for i in range(8): for j110 in range(2): k.emit(v_cvt_pk_bf16_f32(v[16], v[8], v[9])) @@ -4092,21 +3422,13 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(buffer_store_dwordx4(v[16:19], v[242 + j110 * 1 + i * 1], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[242 + j110 * 1 + i * 1], v[242 + j110 * 1 + i * 1], 64)) k.emit(v_accvgpr_read(v[8], v[128 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[8], s[38], v[8])) k.emit(v_accvgpr_read(v[9], v[129 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[9], s[38], v[9])) k.emit(v_accvgpr_read(v[10], v[130 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[10], s[38], v[10])) k.emit(v_accvgpr_read(v[11], v[131 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[11], s[38], v[11])) k.emit(v_accvgpr_read(v[12], v[160 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[12], s[38], v[12])) k.emit(v_accvgpr_read(v[13], v[161 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[13], s[38], v[13])) k.emit(v_accvgpr_read(v[14], v[162 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[14], s[38], v[14])) k.emit(v_accvgpr_read(v[15], v[163 + j110 * 64 + i * 4])) - k.emit(v_mul_f32_e32(v[15], s[38], v[15])) for i in range(4): k.emit(v_cvt_pk_bf16_f32(v[16 + i * 1], v[8 + i * 2], v[9 + i * 2])) k.emit(s_nop(1)) @@ -4116,7 +3438,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_nop(1)) k.emit(buffer_store_dwordx4(v[16:19], v[250], s[4:7], 0, 0, 1)) k.emit(v_add_i32(v[250], v[250], 64)) - k.label('L2_6724') k.emit(s_waitcnt()) k.emit(s_endpgm()) else: From cc32aa18db3a8ff277f2649a5f93540df1e4c0ee Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 12:16:07 -0400 Subject: [PATCH 144/306] don't match casted const in const_folding_pat [pr] (#17658) --- tinygrad/uop/symbolic.py | 7 +++++-- tinygrad/uop/weak.py | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index d581f86c98455..3e4c1fd0ce07d 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -24,8 +24,8 @@ def fold_bitcast(root:UOp, c:UOp) -> UOp|None: if c.dtype.fmt is None or root.dtype.fmt is None or c.dtype.itemsize != root.dtype.itemsize: return None return root.const_like(bitcast(truncate[c.dtype](c.val), c.dtype, root.dtype)) -# const folding works for CONST, STACK, and casted CONST -const_folding_pat = UPat.any(UPat((Ops.CONST, Ops.STACK)), UPat(Ops.CAST, src=(UPat(Ops.CONST),))) +# const folding works for CONST and STACK +const_folding_pat = UPat((Ops.CONST, Ops.STACK)) def const_arg(u:UOp) -> ConstType|tuple[ConstType, ...]|None: if u.op is Ops.CONST: return u.val @@ -142,6 +142,9 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) # ** constant folding ** + # a CAST to a concrete dtype over a CONST is a value conversion: evaluate it once, at the width the CAST states + # TODO: delete this once CONST has no dtype + (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val)), (UPat(GroupOp.Unary, src=(const_folding_pat,), name="a"), fold_const_alu), # NOTE: THREEFRY(const,const) folds via its decomposition (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(const_folding_pat,)*2, name="a"), fold_const_alu), diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index ed8eb5d4a2b15..b59cd79967321 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -72,9 +72,6 @@ def lower(s:UOp) -> UOp: return None if ret is u else ret pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ - # a CAST between two concrete dtypes over a CONST is a value conversion: evaluate it once, at the width the CAST states - # TODO: delete this once CONST has no dtype - (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c", dtypes.all),)), lambda root, c: root.const_like(c.val)), (UPat(GroupOp.All, name="u"), lambda ctx,u: lower_weak_srcs(ctx, u) if u.dtype not in dtypes.weaks and any(s.dtype in dtypes.weaks for s in u.src) else None), # a valid index into an n-element buffer lives in [0,n): a gated long index narrows when n-1 fits int32 (out-of-gate wraps, discarded) From 756e82e05563cc6601cdfde73e7c1c64741e9998 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:38:50 -0700 Subject: [PATCH 145/306] usb amd: pipelined copyin over the 0xF2 engine (2.6x faster) (#17628) Stream 240KB chunks into two alternating 256KB SRAM bounce windows; each chunk ends in a unique 512B sentinel that a prebuilt SDMA ring polls before copying the chunk to VRAM, followed by an in-order drain fence that the host waits on before re-arming a window. No timing assumptions in either direction: the sentinel is in-stream proof of data landing, the fence proves a full drain. Adds a small pooled async bulk-OUT layer to USB3 so staging the next chunk overlaps the wire, and a slot_start parameter to scsi_write for the second window. 107 -> 276 MB/s copyin on tinyc8 (Kryo-3XX host). Co-authored-by: tiny --- test/mockgpu/usb.py | 17 +++++++--- tinygrad/runtime/ops_amd.py | 60 ++++++++++++++++++++++++++++++--- tinygrad/runtime/support/usb.py | 36 ++++++++++++++++---- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/test/mockgpu/usb.py b/test/mockgpu/usb.py index 746c951d518cc..adb84e4cb107c 100644 --- a/test/mockgpu/usb.py +++ b/test/mockgpu/usb.py @@ -160,7 +160,7 @@ def control_write(self, request:int, value:int=0, index:int=0, data:bytes=b'', t elif request == 0xE5: self.state._xram_write_byte(value, index) elif request == 0xF2: - op = ("sram_read" if value & 0x8000 else "sram_write", 0xF000, (value & 0x7FFF) * 512) + op = ("sram_read" if value & 0x8000 else "sram_write", 0xF000 + (index & 0xFF) * 0x4000, (value & 0x7FFF) * 512) if value & 0x8000: self._bulk_read_op = op else: self._bulk_write_op = op elif request == 0xF0: @@ -193,19 +193,26 @@ def bulk_write(self, data:bytes, timeout:int=1000): op, address, size = self._bulk_write_op assert len(data) == size if op == "sram_write": - host_addr, region_size = self.state._dma_regions[address] - ctypes.memmove(host_addr, data, min(len(data), region_size)) + ctrl, (host_addr, region_size) = next((ca, r) for ca, r in self.state._dma_regions.items() if ca <= address < ca + r[1]) + ctypes.memmove(host_addr + (address - ctrl), data, min(len(data), region_size - (address - ctrl))) + self.state.driver._emulate_execute() # landed data may un-stall a ring polling on it (e.g. copyin sentinels) elif op == "pcie_write": self.state._pcie_write(address, data) else: raise RuntimeError(f"cannot bulk write for {op}") self._bulk_write_op = None + def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: # the mock completes transfers synchronously + self.bulk_write(bytes(payload), timeout) + return 0 + + def bulk_wait(self, tag:int): pass + def bulk_read(self, length:int, timeout:int=1000) -> memoryview: assert self._bulk_read_op is not None op, address, size = self._bulk_read_op assert length == size if op == "sram_read": - host_addr, region_size = self.state._dma_regions[address] - data = bytes((ctypes.c_ubyte * min(length, region_size)).from_address(host_addr)) + ctrl, (host_addr, region_size) = next((ca, r) for ca, r in self.state._dma_regions.items() if ca <= address < ca + r[1]) + data = bytes((ctypes.c_ubyte * min(length, region_size - (address - ctrl))).from_address(host_addr + (address - ctrl))) elif op == "pcie_read": data = self.state._pcie_read(address, length) else: raise RuntimeError(f"cannot bulk read for {op}") self._bulk_read_op = None diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 524987df448f3..b36008d29e56b 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import cast -import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit +import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit, time assert sys.platform != 'win32' from dataclasses import dataclass from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, CLikeArgsState, HCQSignal, HCQProgram, FileIOInterface @@ -649,6 +649,56 @@ def _do_free(self, opaque, options:BufferSpec): self.dev.iface.free(opaque) def _do_map(self, buf:HCQBuffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _copyin(self, dest:HCQBuffer, src:memoryview): + if not self.dev.is_usb(): return super()._copyin(dest, src) + from tinygrad.runtime.support.usb import alloc_cbuffer + # Pipelined copyin over the 0xF2 engine. 240KB chunks stream into two alternating 256KB SRAM bounce windows; the + # engine can't signal data landing, so each chunk ends in a 512B sentinel sector tagged with its sequence number. + # A prebuilt SDMA ring polls each chunk's sentinel before copying it to VRAM, then bumps a drain fence; the host + # waits on that fence before re-arming a window. No timing is assumed in either direction. + dev, usb, ts, sdma = self.dev, self.dev.iface.pci_dev.usb, self.dev.timeline_signal, self.dev.sdma + CHUNK, src_mv = 0x3C000, src.cast('B') # 15 16KB slots: the wire image must end mid-window (full windows corrupt) + nchunks = ceildiv(src.nbytes, CHUNK) + FENCE = 0xA800 # drain fence: the GPU writes it via sys_buf (PCIe 0x820800), the host reads it here (xdata) + if not hasattr(self, '_usb_seq'): # one-time: clear the fence and zero both windows so garbage can't match a sentinel + self._usb_seq, self._usb_stage = 0, [alloc_cbuffer(0x40000) for _ in range(2)] # (backing array, memoryview) pairs + self._usb_wins = (self.b[0].offset(0, 0x40000), self.b[0].offset(0x40000, 0x40000)) # two windows, engine slots 0/16 + usb.write(FENCE, bytes(8)) + for bi in range(2): usb.scsi_write(bytes(0x40000), slot_start=bi * 16) + + def wait_drain(count): # spin until the drain fence reaches count, i.e. chunks 0..count-1 are fully in VRAM + t0 = time.monotonic() + while int.from_bytes(usb.read(FENCE, 8), 'little') < count: + if time.monotonic() - t0 > 10: raise RuntimeError(f"GPU failed to drain USB copyin chunk {count - 1} (10s, hung GPU?)") + + # build the whole ring upfront: per chunk, poll the sentinel, copy SRAM->VRAM, bump the fence; then one doorbell + POLL_EQ = sdma.SDMA_OP_POLL_REGMEM | sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) + POLL_DW5 = sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) + q = dev.hw_copy_queue_t().wait(ts, dev.timeline_value - 1) + for c in range(nchunks): + seq, size = self._usb_seq + c, min(CHUNK, src.nbytes - c * CHUNK) + q.q(POLL_EQ, *data64_le(self._usb_wins[seq & 1].va_addr + round_up(size, 512)), 0x51000000 | (seq & 0xFFFFFF), 0xFFFFFFFF, POLL_DW5) + q.copy(dest.offset(c * CHUNK), self._usb_wins[seq & 1], size) + q.write(dev.iface.sys_buf.offset(0x800, 8), seq + 1, b64=True) + q.signal(ts, dev.next_timeline()).submit(dev) + + # stream the chunks: stage the wire image [payload][sentinel], arm the window, send. A window is reusable once + # its previous occupant (seq-2) is both fully sent (tag reaped) and fully drained to VRAM (the fence). + inflight = [None, None] + for c in range(nchunks): + seq, size = self._usb_seq + c, min(CHUNK, src.nbytes - c * CHUNK) + if inflight[seq & 1] is not None: usb.usb.bulk_wait(inflight[seq & 1]) + buf = self._usb_stage[seq & 1][1] + buf[:size] = src_mv[c * CHUNK : c * CHUNK + size] + struct.pack_into(' HCQBuffer: # usb allocates uncached and cpu_access in vram. vram writes are faster than sram writes - if host and self.sys_next_off + size < self.sys_buf.size: - self.sys_next_off += size - return self.sys_buf.offset(self.sys_next_off - size, size) + # NOTE: host allocs deliberately do NOT use sys_buf (the 0x820000 NVMe SQ region): the GPU's signal writes there + # collide with the 0xF2 engine mid-stream. Signals in VRAM are read back via 0xF0 streaming reads instead. # force devmem return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access, contiguous=contiguous, force_devmem=True, **kwargs) @@ -1048,7 +1097,8 @@ def sdma_queue(self, idx:int): if getenv("AMD_DISABLE_SDMA"): return None if idx in self.sdma_queues: return self.sdma_queues[idx] with contextlib.suppress(OSError): - self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx) + # USB: a copyin submits its whole ring at once (3 packets per 240KB chunk), so it needs more than the 0x200 default + self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, (1 << 20) if self.is_usb() else (16 << 20), idx=idx) return self.sdma_queues.get(idx, None) def _ensure_has_local_memory(self, private_segment_size): diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index b7e8b8186ecaf..e4ef70f2a70a9 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -1,6 +1,6 @@ import ctypes, struct, time, functools, itertools from tinygrad.runtime.autogen import libusb -from tinygrad.helpers import DEBUG, DEV, to_mv, round_up, ceildiv +from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv from tinygrad.runtime.support.hcq import MMIOInterface from tinygrad.runtime.support import c @@ -35,6 +35,11 @@ def __init__(self, dev:c.POINTER[libusb.struct_libusb_device], *args, **kwargs): self._tags, self._transferred = itertools.count(1), ctypes.c_int(0) self._bulk_buf, self._bulk_mv = alloc_cbuffer(4 << 20) self._ctrl_buf, self._ctrl_mv = alloc_cbuffer(0x1000) + # async bulk OUT state: tag -> (pooled transfer, keepalive payload mv); transfer errors latch into _async_err + self._async_seq, self._async_err = itertools.count(1), 0 + self._async_pending: dict = {} + self._async_pool: list = [] + self._async_cb = libusb.libusb_transfer_cb_fn(self._on_bulk_done) self.handle = c.init_c_var(c.POINTER[libusb.struct_libusb_device_handle], lambda x: checked(libusb.libusb_open)(dev, x)) @@ -73,6 +78,26 @@ def bulk_write(self, payload:bytes, timeout:int=1000): (self.handle, 0x02, self._bulk_buf, len(payload), self._transferred, timeout) assert self._transferred.value == len(payload), f"bulk OUT short write: {self._transferred.value}/{len(payload)} bytes" + def _on_bulk_done(self, xfer): # runs in libusb event handling; latch errors (exceptions here are unraisable) + if xfer.contents.status != 0 or xfer.contents.actual_length != xfer.contents.length: self._async_err = xfer.contents.status or -1 + self._async_pool.append(self._async_pending.pop(int(xfer.contents.user_data or 0))[0]) + + def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: + """Queue a bulk OUT transfer without blocking; payload is kept alive until bulk_wait(tag).""" + tr = self._async_pool.pop() if self._async_pool else libusb.libusb_alloc_transfer(0) + tr.contents.dev_handle, tr.contents.endpoint, tr.contents.type = self.handle, 0x02, libusb.LIBUSB_TRANSFER_TYPE_BULK + tr.contents.timeout, tr.contents.length = timeout, len(payload) + tr.contents.buffer = ctypes.cast(from_mv(payload, ctypes.c_ubyte), ctypes.POINTER(ctypes.c_ubyte)) + tr.contents.callback, tr.contents.user_data = self._async_cb, (tag := next(self._async_seq)) + self._async_pending[tag] = (tr, payload) + checked(libusb.libusb_submit_transfer, "async bulk OUT submit failed")(tr) + return tag + + def bulk_wait(self, tag:int): + """Block until the tagged transfer completes; raises if any async transfer failed.""" + while tag in self._async_pending: checked(libusb.libusb_handle_events)(None) + if self._async_err: raise RuntimeError(f"async bulk OUT failed: status={self._async_err}") + def bulk_read(self, length:int, timeout:int=1000) -> memoryview: if length > len(self._bulk_mv): self._bulk_buf, self._bulk_mv = alloc_cbuffer(length) checked(libusb.libusb_bulk_transfer, "bulk IN 0x81 failed")(self.handle, 0x81, self._bulk_buf, length, self._transferred, timeout) @@ -160,13 +185,10 @@ def write(self, base_addr:int, data:bytes): """Write to chip XDATA via vendor control OUT (bRequest=0xE5). wValue=addr, wIndex=val.""" for off, val in enumerate(data): self.usb.control_write(0xE5, value=base_addr + off, index=val) - def scsi_write(self, buf:bytes): + def scsi_write(self, buf:bytes, slot_start:int=0): """Write to SRAM via 0xF2 vendor command + bulk OUT.""" buf_padded = buf + b'\x00' * (round_up(len(buf), 512) - len(buf)) - sectors = len(buf_padded) // 512 - num_slots = ceildiv(len(buf_padded), 0x4000) # 16KB per slot - windex = (num_slots & 0xFF) << 8 - self.usb.control_write(0xF2, value=sectors, index=windex) + self.usb.control_write(0xF2, value=len(buf_padded) // 512, index=(slot_start & 0xFF) | (ceildiv(len(buf_padded), 0x4000) << 8)) self.usb.bulk_write(buf_padded) def scsi_read_arm(self, size:int): @@ -189,7 +211,7 @@ def __getitem__(self, index): assert sz % 4 == 0 and off % 4 == 0, f"pcie_mem_read requires 4-byte aligned access, got off={off}, sz={sz}" data = self.usb.pcie_mem_read(self.addr + off, sz) else: data = self.usb.scsi_read(sz) if self.addr == 0xf000 else self.usb.read(self.addr + off, sz) - return int.from_bytes(data, "little") if sz == self.el_sz else data + return data if isinstance(index, slice) else int.from_bytes(data, "little") def __setitem__(self, index, data): off, _ = self._off_from_index(index) From 3919ce8427c1f063ae8fb3b2c79d8dcca3ef9506 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 14:02:20 -0400 Subject: [PATCH 146/306] ceildiv in wgsl _packed_size [pr] (#17662) --- test/backend/test_uops.py | 21 +++++++++++---------- tinygrad/renderer/wgsl.py | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/backend/test_uops.py b/test/backend/test_uops.py index d2b51ae7ceae5..ea6f2ec960527 100644 --- a/test/backend/test_uops.py +++ b/test/backend/test_uops.py @@ -2,7 +2,7 @@ import unittest, math import numpy as np from tinygrad.tensor import Tensor, _to_np_dtype -from tinygrad.helpers import Context +from tinygrad.helpers import Context, ceildiv from tinygrad.dtype import dtypes, DType, AddrSpace, ConstFloat # noqa: F401 from tinygrad.device import Buffer, Device from tinygrad.uop.ops import Ops, UOp, KernelInfo, AxisType, buffers @@ -193,15 +193,16 @@ def test_local_packed(self): @unittest.skipUnless(Device.DEFAULT == "WEBGPU", "Test local memory size for packed data types") def test_packed_smem_size(self): _dtypes = [dtypes.char, dtypes.uchar, dtypes.short, dtypes.ushort, dtypes.half] - size = 16 - for dtype in _dtypes: - temp = UOp.placeholder((size,), dtype, slot=0, addrspace=AddrSpace.LOCAL) - uops = to_uops_list([temp], ren=Device[Device.DEFAULT].renderer) - out = Device[Device.DEFAULT].renderer.render(uops) - # half is supported in wgsl, so it doesn't have to be packed - corrected_size = size//(4//dtype.itemsize) if dtype != dtypes.half else size - # temp0: array<{Device[Device.DEFAULT].renderer.buf_map(dtype)},{corrected_size}>; - self.assertIn(f",{corrected_size}>;", out) + # a partial word still needs a whole word, so sizes that don't fill one must round up + for size in (16, 5): + for dtype in _dtypes: + temp = UOp.placeholder((size,), dtype, slot=0, addrspace=AddrSpace.LOCAL) + uops = to_uops_list([temp], ren=Device[Device.DEFAULT].renderer) + out = Device[Device.DEFAULT].renderer.render(uops) + # half is supported in wgsl, so it doesn't have to be packed + corrected_size = ceildiv(size, 4//dtype.itemsize) if dtype != dtypes.half else size + # temp0: array<{Device[Device.DEFAULT].renderer.buf_map(dtype)},{corrected_size}>; + self.assertIn(f",{corrected_size}>;", out) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared memory") @unittest.skip("tinygrad doesn't support this behavior") diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index b618db4ddc4fd..4170f7bd921bb 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -1,7 +1,7 @@ from tinygrad.dtype import DType, dtypes, truncate, AddrSpace from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat from tinygrad.renderer.cstyle import CStyleLanguage, base_rewrite -from tinygrad.helpers import strip_parens +from tinygrad.helpers import strip_parens, ceildiv def _mask(dt:DType): return 0xFF if dt.itemsize == 1 else 0xFFFF @@ -34,7 +34,7 @@ def is_packed(x:UOp): elif x.op is Ops.STORE: dt, addrspace = x.src[1].dtype, x.src[0].addrspace else: dt, addrspace = x.dtype, x.addrspace return dt.itemsize < 4 and dt != dtypes.half and addrspace != AddrSpace.REG -def _packed_size(u:UOp): return u.max_numel() // (4//u.dtype.itemsize) if is_packed(u) else u.max_numel() +def _packed_size(u:UOp): return ceildiv(u.max_numel(), 4//u.dtype.itemsize) if is_packed(u) else u.max_numel() def is_nan(a): bs, (exp, mant) = a.dtype.bitsize, dtypes.finfo(a.dtype) return (a.bitcast(getattr(dtypes, f"uint{bs}")) & ((1 << (bs - 1)) - 1)) > (((1 << exp) - 1) << mant) From 1cf8a2c7fe03ed9692960c6036b1e085eb6525fe Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:24:34 +0300 Subject: [PATCH 147/306] hcq2: use shrink.bitcast (#17653) * hcq2: shrink.bitcast * x * x * x * s * x * Dx * Revert "hotfix: disable HCQ2" This reverts commit a57188ea6d9a7a7604ea96585e32fbb88b2d4fda. * x --- .github/workflows/benchmark.yml | 10 +++--- test/backend/test_uops.py | 29 +++++++++++++++++ tinygrad/codegen/__init__.py | 5 +-- tinygrad/renderer/cstyle.py | 14 +++++--- tinygrad/runtime/ops_python.py | 5 +-- tinygrad/runtime/support/hcq2.py | 55 ++++++++++++++++++-------------- 6 files changed, 80 insertions(+), 38 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1b199252f649c..a94fe047cfbd0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,7 +94,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: "0" + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -141,7 +141,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: "0" + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -190,7 +190,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: "0" + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -233,7 +233,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: "0" + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -279,7 +279,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: "0" + HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code diff --git a/test/backend/test_uops.py b/test/backend/test_uops.py index ea6f2ec960527..e1c0c324432f0 100644 --- a/test/backend/test_uops.py +++ b/test/backend/test_uops.py @@ -57,6 +57,35 @@ def _test_uops_result(output_dtype, uops, res): run_uops([out], [buf]) return np.frombuffer(buf.as_memoryview(), _to_np_dtype(output_dtype))[0] +@unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage) and + dtypes.uint64 in Device[Device.DEFAULT].renderer.supported_dtypes(), "requires C-style pointer bitcast and 64-bit ints") +class TestBitcastBufferView(unittest.TestCase): + @Context(SPEC=2) + def test_render(self): + buf = UOp.param(0, dtypes.uint32, (4,)) + uops = to_uops_list([buf.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0).store(1)], ren=Device[Device.DEFAULT].renderer) + idx = next(u for u in uops if u.op is Ops.INDEX and u.src[0].op is Ops.BITCAST) + self.assertEqual(idx.src[0].src[0].op, Ops.SHRINK) + Device[Device.DEFAULT].renderer.render(uops) + + @Context(SPEC=2) + def test_load(self): + val = 0x1122334455667788 + src, out = UOp.param(0, dtypes.uint32, (4,)), UOp.param(1, dtypes.uint64, (1,)) + ibuf = Buffer(Device.DEFAULT, 4, dtypes.uint32, initial_value=np.array([0, 0x55667788, 0x11223344, 0], dtype=np.uint32).tobytes()) + obuf = Buffer(Device.DEFAULT, 1, dtypes.uint64).allocate() + run_uops([out.index(0).store(src.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0))], [ibuf, obuf]) + self.assertEqual(np.frombuffer(obuf.as_memoryview(), dtype=np.uint64)[0], val) + + @Context(SPEC=2) + def test_store(self): + val = 0x1122334455667788 + dst = UOp.param(0, dtypes.uint32, (6,)) + buf = Buffer(Device.DEFAULT, 6, dtypes.uint32, initial_value=bytes(24)) + view = dst.shrink(((1, 5),)).bitcast(dtypes.uint64) # two stores through one view: it must inline, not get a declared vector-pointer + run_uops([view.index(0).store(val ^ 0xff), view.index(1).store(val)], [buf]) + self.assertEqual(np.frombuffer(buf.as_memoryview(), dtype=np.uint64, count=2, offset=4).tolist(), [val ^ 0xff, val]) + class TestUOps(unittest.TestCase): def _equal(self, v1, v2): assert isinstance(v2, (float, int, bool)) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 0a177904c0d70..9b3f7ad18e51c 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -233,10 +233,11 @@ def expand_horizontal_reduce(r:UOp): (UPat(Ops.SINK, name="sink"), merge_reduce_ends), ])+pm_clean_up_group_sink +def is_shape_changing_bitcast(u:UOp): return u.op is Ops.BITCAST and u.shape != u.src[0].shape def maybe_load(u:UOp): return u.load() if u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL, AddrSpace.REG) else u pm_add_loads = PatternMatcher([ - # BITCAST? - (UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), lambda x: x.replace(src=tuple([maybe_load(u) for u in x.src]))), + (UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), + lambda x: None if is_shape_changing_bitcast(x) else x.replace(src=tuple(map(maybe_load, x.src)))), (UPat(Ops.STORE, name="x"), lambda x: x.replace(src=(x.src[0], maybe_load(x.src[1]))+x.src[2:])), ]) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 8cd584f63e776..3035395f96beb 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -38,7 +38,8 @@ (UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \ if x.max_numel() > 1 and x.addrspace is AddrSpace.REG else None), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx[x.src[0]])})"), - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: ctx[x.src[0]] if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"(({ctx._render_dtype(x.dtype, addrspace=x.addrspace)})({ctx[x.src[0]]}))" + if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"__builtin_bit_cast({ctx.render_type(x)}, ({ctx.render_type(x.src[0])})({ctx[x.src[0]]}))"), # GPU stuff @@ -237,7 +238,7 @@ def _render(self, uops:list[UOp]) -> tuple[str, list[str], list[tuple[str,tuple[ if (u.op is not Ops.CAST or u.max_numel() == 1) and ((u.op is Ops.CAST and u.src[0].op is Ops.CONST) or \ u.op in {Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \ (u.op is Ops.LOAD and u.src[0].addrspace == AddrSpace.REG and child_count[u] == 1) or \ - (u.op is Ops.CAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \ + (u.op in {Ops.CAST, Ops.BITCAST} and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \ (u.op in {Ops.STACK, *(GroupOp.ALU-{Ops.WHERE}), Ops.CAST, Ops.BITCAST} and child_count[u] == 1 and not getenv("EXPAND_SSA"))): r[u] = l else: @@ -318,7 +319,8 @@ class OpenCLRenderer(CStyleLanguage): extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), # bfloat16 constants need to be rendered as their bit pattern since bf16 is stored as ushort (UPat.cvar("c").cast(dtypes.bfloat16), lambda ctx,c: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(c.val)))[0] >> 16)}u"), # load/store image (OpenCL) @@ -369,7 +371,8 @@ def __init__(self, target:Target): ]) + pm_manual_bf16_cast string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), ]) + base_rewrite def render_kernel(self, function_name, kernel, bufs, uops, prefix=None): @@ -425,7 +428,8 @@ def __init__(self, target:Target, use_nvcc=False): (UPat(Ops.CAST, dtypes.fp8s, UPat.var("x", dtypes.fp8s), name='y'), lambda x,y: x.cast(dtypes.float).cast(y.dtype) if x.dtype!=y.dtype else None), ]) string_rewrite = PatternMatcher([ - (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"), + (UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))" + if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None), ]) + base_rewrite def render_vector_prefix(self, dt:DType, count:int) -> str: diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index 0171dabd58c9d..9a7ea339af9dc 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -89,7 +89,7 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup if g: _store(m, o+j, v, src_dtypes[1]) i += 1 continue - if u.op is Ops.AFTER: values[u] = src_values[0] + if u.op is Ops.AFTER or (u.op is Ops.BITCAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)): values[u] = src_values[0] elif u.op is Ops.PARAM and u.addrspace is AddrSpace.ALU: values[u] = [pvals.pop(0)] * warp_size elif u.op in {Ops.PARAM, Ops.BUFFER}: storage_fmt = storage_fmt_for_dtype(u.dtype) @@ -114,7 +114,8 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup if ox < 0 or ox >= u.src[0]._shape[1] or oy < 0 or oy >= u.src[0]._shape[0]: ret.append((m, None)) else: ret.append((m, ox*4 + oy*u.src[0]._shape[1]*4)) else: - for m,o in zip(src_values[0], src_values[1]): ret.append((m,o)) + scale = u.src[0].dtype.itemsize // u.src[0].src[0].dtype.itemsize if u.src[0].op is Ops.BITCAST else 1 + for m,o in zip(src_values[0], src_values[1]): ret.append((m[0], m[1]+o*scale) if isinstance(m, tuple) else (m, o*scale)) values[u] = ret elif u.op is Ops.RANGE: if u not in values: values[u] = [0] * warp_size diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index ef4ff904b31d3..f4204c2f06ad0 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -8,7 +8,7 @@ from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp from tinygrad.uop.symbolic import symbolic -from tinygrad.dtype import dtypes, truncate +from tinygrad.dtype import dtypes, truncate, DType from tinygrad.runtime.support.hcq import MMIOInterface, HCQBuffer from tinygrad.runtime.support.memory import BumpAllocator from tinygrad.renderer import Renderer, Estimates @@ -40,6 +40,9 @@ def unwrap_mstack(u): if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s)) return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,) +def unwrap_view(v:UOp) -> tuple[UOp, int]: + return unwrap_view(v.src[0]) if v.op is Ops.BITCAST else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) + def is_value_known_at_link(val:UOp) -> bool: runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)] addressed_bufs = [b for g in val.toposort() if g.op is Ops.GETADDR for b in unwrap_mstack(g.buf_uop)] @@ -48,16 +51,17 @@ def is_value_known_at_link(val:UOp) -> bool: return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs) def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]: - def _mk_store(ps:list[tuple[sint, UOp]], tag:str|None) -> UOp: - offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const(off // buf.dtype.itemsize, dtypes.int) for off,_ in ps)) - vals = UOp(Ops.STACK, ps[0][1].dtype, tuple(val for _,val in ps)) - return buf.index(offs, dtype=vals.dtype).store(vals).rtag(tag) - - patches = [(off, val.cast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val) for off, val in patches] - link, runtime = partition(patches, lambda p: is_value_known_at_link(p[1])) - inputs, runtime = partition(runtime, lambda p: p[1].op is Ops.GETADDR) - return tuple(_mk_store(list(ps), tag) for cls, tag in ((link, "link"), (inputs, "inputs"), (runtime, None)) - for _, ps in itertools.groupby(sorted(cls, key=lambda p: p[1].dtype), key=lambda p: p[1].dtype)) + groups:dict[tuple[str|None, DType, sint], list[tuple[sint, UOp]]] = collections.defaultdict(list) + for off, val in patches: + tag = "link" if is_value_known_at_link(val) else "inputs" if val.op is Ops.GETADDR else None + groups[(tag, (v:=(val.bitcast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val)).dtype, off % v.dtype.itemsize)].append((off, v)) + + ret, bit = [], buf.dtype.itemsize + for (tag, dt, r), ps in groups.items(): + view = buf.shrink(((r // bit, (max(off for off,_ in ps) + dt.itemsize) // bit),)).bitcast(dt) + offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const((off - r) // dt.itemsize, dtypes.int) for off,_ in ps)) + ret.append(view.index(offs).store(UOp(Ops.STACK, dt, tuple(val for _,val in ps))).rtag(tag)) + return tuple(ret) def make_binary_patch(buf:UOp, blob:bytes) -> UOp: data = UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype) @@ -328,14 +332,16 @@ def make_addr_table(call:UOp, gaddrs:list[UOp], name:str) -> tuple[UOp, dict[UOp return table, reads, fills, {g:slots[bare[g]] for g in gaddrs} def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp]) -> dict[UOp, UOp]: - (dst,), words = dedup(p.buf_uop for p in patches), [(off.val, slots[val]) for p in patches for off, val in zip(p.src[0].src[1].src, p.src[1].src)] + (dst,), words = dedup(p.buf_uop for p in patches), [(unwrap_view(p.src[0].src[0])[1] + off.val*(val.dtype.itemsize//p.buf_uop.dtype.itemsize), + slots[val]) for p in patches for off,val in zip(p.src[0].src[1].src, p.src[1].src)] # build a runtime loop that writes every input address pairs = UOp.placeholder((2*len(words),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words)))) r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst)) off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()))) - return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: dst.index(off, dtype=table.dtype).store(table.index(slot).load()).end(r)} + patch = dst.shrink(((off, off+table.dtype.itemsize//dst.dtype.itemsize),)).bitcast(table.dtype).index(0).store(table.index(slot).load()).end(r) + return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: patch} def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) @@ -391,14 +397,13 @@ def replace_params(call:UOp) -> UOp|None: def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp: base = bv.src[0].after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()) - if bv.op is Ops.BITCAST: return UOp(Ops.GETADDR, src=(base,), arg=g.arg) - itemsize = bv.src[0].dtype.itemsize if bv.src[0].without_after.op in (Ops.BUFFER, Ops.MSTACK, Ops.MSELECT) else bv.dtype.itemsize - return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64) + addr = UOp(Ops.GETADDR, src=(base,), arg=g.arg) + return addr if bv.op is Ops.BITCAST else addr + UOp.const(bv.src[1].val * bv.dtype.itemsize, dtypes.uint64) pm_early_simplify = PatternMatcher([ (UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv").or_after(),), name="g"), resolve_getaddr_view), - (UPat(Ops.INDEX, src=(UPat(Ops.SHRINK, name="bv"),), allow_any_len=True, name="x"), - lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))), + (UPat(Ops.SHRINK, src=(UPat(Ops.SHRINK, name="bv"), UPat(), UPat()), name="x"), + lambda bv,x: bv.src[0].shrink(((start:=bv.src[1]+x.src[1], start+x.src[2]),))), ]) # ***************** @@ -507,11 +512,13 @@ def fold_binary(buf:UOp, blob:UOp) -> UOp: b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[:len(blob.arg)] = blob.arg return UOp(Ops.NOOP) -def fold_const_store(buf:UOp, off:UOp, val:UOp) -> UOp: +def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp: + buf, start = unwrap_view(view) for off,val in zip(off.src, val.src): for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype]((v.src[0] if v.op is Ops.CAST else v).val)) - b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[(bo:=off.val*buf.dtype.itemsize):bo+len(data)] = data + bo = start*buf.dtype.itemsize + off.val*val.dtype.itemsize + b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[bo:bo+len(data)] = data return UOp(Ops.NOOP) def resolve_getaddr(buf:UOp, g:UOp) -> UOp: @@ -532,10 +539,10 @@ def resolve_getaddr(buf:UOp, g:UOp) -> UOp: (UPat(Ops.GETADDR, src=(UPat(name="buf"),), name="g"), resolve_getaddr), # folders - (UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True) - .store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()).index(UPat(Ops.RANGE), allow_any_len=True).load()) - .end(UPat(Ops.RANGE)), fold_binary), - (UPat({Ops.BUFFER, Ops.MSTACK}, name="buf").index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), + (UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True).store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()) + .index(UPat(Ops.RANGE), allow_any_len=True).load()).end(UPat(Ops.RANGE)), fold_binary), + (UPat((Ops.BITCAST, Ops.SHRINK, Ops.BUFFER, Ops.MSTACK), name="view") + .index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), ]) pm_assert_no_afters = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: panic(RuntimeError, f"AFTER left at hcq_link: {a.src[0].op}"))]) From 8f59041ee5232490a8ef77160a292f2788ef9832 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 14:34:25 -0400 Subject: [PATCH 148/306] more wgsl pack cleanups [PR] (#17664) --- tinygrad/renderer/wgsl.py | 45 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 4170f7bd921bb..f032cf435ac2a 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -3,37 +3,36 @@ from tinygrad.renderer.cstyle import CStyleLanguage, base_rewrite from tinygrad.helpers import strip_parens, ceildiv -def _mask(dt:DType): return 0xFF if dt.itemsize == 1 else 0xFFFF - def sign_extend(val:UOp, sext_am:int): - return (UOp.where((val >> (sext_am - 1)) > 0, UOp.const(0xffffffff << sext_am, dtypes.uint32), UOp.const(0, dtypes.uint32)) \ + return (((val >> (sext_am - 1)) > 0).where(UOp.const(0xffffffff << sext_am, dtypes.uint32), UOp.const(0, dtypes.uint32)) \ | val.bitcast(dtypes.uint32)).bitcast(dtypes.int) +# a packed field of dt: the word it lives in, its offset in that word, and its mask. width is 8*itemsize, bool is one bit in a byte +def packed_field(bidx:UOp, dt:DType) -> tuple[UOp, UOp, int]: + elems, width = 4//dt.itemsize, 8*dt.itemsize + return bidx.src[0].index(bidx.src[1] // elems), (bidx.src[1].cast(dtypes.uint32) % elems) * width, (1 << width)-1 + # store for char: buf[idx/4] <- (var << (idx%4)*8)) -def packed_store(bidx:UOp, var:UOp, gate:UOp|None=None): - elems, mask = 4//var.dtype.itemsize, _mask(var.dtype) - shift_am, div_idx = (bidx.src[1].cast(dtypes.uint32) % elems) * (8*var.dtype.itemsize), bidx.src[1] // elems +def packed_store(s:UOp): + bidx, var, *gate = s.src + idx, shift_am, mask = packed_field(bidx, var.dtype) # bool does its mask math at int32: renderer rewrites run after weak dtypes are lowered, and bool & 0xFF would create a weakint const if var.dtype == dtypes.bool: var = var.cast(dtypes.int32) new_v, wmask = (var & mask).cast(dtypes.uint32) << shift_am, ((mask << shift_am) ^ 0xFFFFFFFF).cast(dtypes.uint32) - idx = UOp(Ops.INDEX, src=(bidx.src[0], div_idx)) - buf = UOp.load(idx, *((UOp.const(0, dtypes.uint32), gate) if gate is not None else ()), dtype=dtypes.uint32) - return UOp.store(idx, (buf & wmask) | new_v, *((gate,) if gate is not None else ())) + buf = idx.load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ()), dtype=dtypes.uint32) + return idx.store((buf & wmask) | new_v, *gate) # load for char: sign_extend(buf[idx/4] >> ((idx%4)*8)) -def packed_load(root:UOp, bidx:UOp, dtype:DType, var:UOp|None=None, gate:UOp|None=None): - elems, mask = 4//dtype.itemsize, _mask(dtype) - shift_am, div_idx = (bidx.src[1].cast(dtypes.uint32) % elems) * (8*dtype.itemsize), bidx.src[1] // elems - idx = UOp(Ops.INDEX, src=(bidx.src[0], div_idx)) - load = UOp.load(idx, *((var, gate) if var is not None and gate is not None else root.src[1:]), dtype=dtypes.uint32, arg=root.arg) - val = (load.cast(dtypes.uint32) >> shift_am) & mask +def packed_load(root:UOp): + bidx, *alt = root.src + idx, shift_am, mask = packed_field(bidx, dtype:=root.dtype) + load = idx.load(*((alt[0].cast(dtypes.uint32), *alt[1:]) if alt else ()), dtype=dtypes.uint32, arg=root.arg) + val = (load >> shift_am) & mask return sign_extend(val, 8*dtype.itemsize).cast(dtype) if dtype in [dtypes.char, dtypes.short] else val.cast(dtype) def is_packed(x:UOp): - if x.op is Ops.LOAD: dt, addrspace = x.dtype, x.src[0].addrspace - elif x.op is Ops.STORE: dt, addrspace = x.src[1].dtype, x.src[0].addrspace - else: dt, addrspace = x.dtype, x.addrspace - return dt.itemsize < 4 and dt != dtypes.half and addrspace != AddrSpace.REG + dt = x.src[1].dtype if x.op is Ops.STORE else x.dtype + return dt.itemsize < 4 and dt != dtypes.half and x.buf_uop.addrspace != AddrSpace.REG def _packed_size(u:UOp): return ceildiv(u.max_numel(), 4//u.dtype.itemsize) if is_packed(u) else u.max_numel() def is_nan(a): bs, (exp, mant) = a.dtype.bitsize, dtypes.finfo(a.dtype) @@ -42,12 +41,8 @@ def is_nan(a): wgsl_matcher = PatternMatcher([ (UPat((Ops.CMPLT, Ops.XOR), src=(UPat(name="a", dtype=dtypes.bool), UPat.var("b")), name="c"), lambda a,b,c: a.cast(dtypes.int).alu(c.op, b.cast(dtypes.int)).cast(dtypes.bool)), - (UPat.load(UPat.var("b"), UPat.var("c"), UPat.var("gate"), name="l"), - lambda l,b,c,gate: packed_load(l,b,l.dtype,c.cast(dtypes.uint32),gate) if is_packed(l) else None), - (UPat.load(UPat.var("b"), name='l'), lambda l,b: packed_load(l,b,l.dtype) if is_packed(l) else None), - (UPat.store(UPat.var("b"), UPat.var("var"), UPat.var("gate"), name="s"), - lambda b,var,gate,s: packed_store(b,var,gate) if is_packed(s) else None), - (UPat.store(UPat.var("b"), UPat.var("var"), name="s"), lambda b,var,s: packed_store(b,var) if is_packed(s) else None), + (UPat(Ops.LOAD, name="l"), lambda l: packed_load(l) if is_packed(l) else None), + (UPat(Ops.STORE, name="s"), lambda s: packed_store(s) if is_packed(s) else None), (UPat.var("a") << UPat.var("b"),lambda a,b:(a.bitcast(dtypes.uint32)<> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), # fix nan check: 'a != a -> is_nan()'. the decomp rewrites (a != a).logical_not() to CMPEQ, so match both forms From 8f9cbdf0cc082a18b24cbcebce0d8f56fda3e598 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 15:21:28 -0400 Subject: [PATCH 149/306] few more self folding [pr] (#17657) --- test/backend/test_renderer_failures.py | 8 ++++++++ test/null/test_uop_symbolic.py | 7 +++++++ tinygrad/renderer/wgsl.py | 11 +++++++---- tinygrad/uop/symbolic.py | 4 ++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/test/backend/test_renderer_failures.py b/test/backend/test_renderer_failures.py index d1f7a145faa55..403a938cdc5de 100644 --- a/test/backend/test_renderer_failures.py +++ b/test/backend/test_renderer_failures.py @@ -77,6 +77,14 @@ def test_repeat_sub(self): self._test_src_strip_paren(Ops.SUB, should_strip_pare @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, WGSLRenderer), "tests for wgsl renderer") class TestWGSLFailures(unittest.TestCase): + def test_folded_packed_store(self): + b = UOp.param(0, dtypes.char, (4,)) + idx = b.index(UOp.const(0).cast(dtypes.int)) + store = UOp.store(idx, UOp.load(idx, dtype=dtypes.uint32) & UOp.const(0xffffff00).cast(dtypes.uint32)) + src = Device[Device.DEFAULT].renderer.render(UOp.sink(store, arg=KernelInfo()).toposort()) + self.assertIn("atomicAnd(&data0_4[0],4294967040u);", src) + self.assertNotIn("atomicAdd", src) + def test_multiply_infinity(self): # multiplying a positive constant by infinity should return infinity # WGSL pipelines do not handle this reliably, some of which return zero, unless infinity always comes from a read on a dynamic buffer diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index db3bd00ed8131..0f27eb0e166a5 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -149,6 +149,13 @@ def test_neg(self): def test_xor_0(self): self.helper_test_variable(Variable("a", 0, 8, dtypes.int) ^ 0, 0, 8, "a", test_z3=False) + def test_or_0(self): + self.helper_test_variable(Variable("a", 0, 8, dtypes.int) | 0, 0, 8, "a", test_z3=False) + + def test_shift_0(self): + self.helper_test_variable(Variable("a", 0, 8, dtypes.int) << 0, 0, 8, "a") + self.helper_test_variable(Variable("a", 0, 8, dtypes.int) >> 0, 0, 8, "a") + def test_xor_self_inverse(self): self.helper_test_variable((Variable("a", 0, 8, dtypes.int) ^ 5) ^ 5, 0, 8, "a", test_z3=False) diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index f032cf435ac2a..6d34cc70c927d 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -38,6 +38,9 @@ def is_nan(a): bs, (exp, mant) = a.dtype.bitsize, dtypes.finfo(a.dtype) return (a.bitcast(getattr(dtypes, f"uint{bs}")) & ((1 << (bs - 1)) - 1)) > (((1 << exp) - 1) << mant) +# the read-modify-write packed_store emits: a load of the very index being stored to, masked (a gated store loads with 3 srcs) +packed_rmw = UPat(Ops.LOAD, src=(UPat.var("b"),), allow_any_len=True) & UPat.var("wmask") + wgsl_matcher = PatternMatcher([ (UPat((Ops.CMPLT, Ops.XOR), src=(UPat(name="a", dtype=dtypes.bool), UPat.var("b")), name="c"), lambda a,b,c: a.cast(dtypes.int).alu(c.op, b.cast(dtypes.int)).cast(dtypes.bool)), @@ -82,10 +85,10 @@ class WGSLRenderer(CStyleLanguage): (UPat.load(UPat.var("b"), UPat.var("v"), UPat.var("gate")), lambda ctx,b,v,gate: f"select({ctx[v]}, {ctx.render_load(ctx[b], b.src[0])}, {ctx[gate]})"), (UPat.load(UPat.var("b")), lambda ctx, b: ctx.render_load(ctx[b], b)), - (UPat.store(UPat.var("b"), UPat.var("v")), lambda ctx,b,v:\ - # (load & mask) | var -> mask = v.src[0].src[1], var = v.src[1] - f"atomicAnd(&{ctx[b]},{ctx[v.src[0].src[1]]});\n atomicAdd(&{ctx[b]},{ctx[v.src[1]]});" if is_packed(b) \ - else f"{ctx[b]} = {ctx[v]};"), + # packed_store writes (load & wmask) | new_v: atomicAnd clears the field, atomicAdd sets it. new_v is gone when it is 0 + (UPat.store(UPat.var("b"), UPat.any(packed_rmw, packed_rmw | UPat.var("nv"))), lambda ctx,b,wmask,nv=None: + f"atomicAnd(&{ctx[b]},{ctx[wmask]});"+(f"\n atomicAdd(&{ctx[b]},{ctx[nv]});" if nv is not None else "") if is_packed(b) else None), + (UPat.store(UPat.var("b"), UPat.var("v")), lambda ctx,b,v: f"{ctx[b]} = {ctx[v]};"), (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.var("idx"))), lambda ctx,b,idx: f"{ctx[b]}[{strip_parens(ctx[idx]) if idx.arg is Ops.ADD else ctx[idx]}]"), ]) + base_rewrite diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 3e4c1fd0ce07d..7f48a18ce2abf 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -108,9 +108,9 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: symbolic_simple = pm_data_invalid + PatternMatcher([ # ** self folding ** - (UPat.var("x") + 0, lambda x: x), # x+0 -> x + (UPat({Ops.ADD, Ops.XOR, Ops.OR}, src=[UPat.var("x"), UPat.const(0)]), lambda x: x), # x+0 / x^0 / x|0 -> x + (UPat({Ops.SHL, Ops.SHR}, src=(UPat.var("x"), UPat.const(0))), lambda x: x), # x<<0 / x>>0 -> x (UPat.var("x") * 1, lambda x: x), # x*1 -> x - (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) ^ 0, lambda x: x), # x^0 -> x (UPat.var("x") // UPat.var("x"), lambda x: x.const_like(1)), # x//x -> 1 (UPat.var("x") // 1, lambda x: x), # x//1 -> x (UPat.var("x") // -1, lambda x: -x), # x//-1 -> -x From 402bea7ddd2f0546e0d4c60c3d186289a8f62ebe Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 15:55:45 -0400 Subject: [PATCH 150/306] l2i and sign_extend cleanups [pr] (#17668) towards good threefry decomp --- tinygrad/codegen/decomp/dtype.py | 6 +++--- tinygrad/renderer/wgsl.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index db85526c83d37..00f7e8a059b27 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -25,10 +25,10 @@ def l2i(op: Ops, dt: DType, *uops:UOp): match op: case Ops.NEG: return l2i(Ops.SUB, dt, zero, zero, *uops) case Ops.CAST if dt in (dtypes.long, dtypes.ulong) and uops[0].dtype not in dtypes.floats: - # the high word is the sign extension; bool has no sign, test the already-cast low word instead (bool < 0 would promote to weakint) + # the high word is the sign extension, and unsigned and bool sources zero extend x, lo = uops[0], uops[0].cast(l2i_dt[dt]) - sign = lo if x.dtype is dtypes.bool else x - return lo, (sign < sign.const_like(0)).where(lo.const_like(-1), lo.const_like(0)) + if x.dtype is dtypes.bool or x.dtype in dtypes.uints: return lo, lo.const_like(0) + return lo, (x < x.const_like(0)).where(lo.const_like(-1), lo.const_like(0)) case Ops.CAST if dt in (dtypes.long, dtypes.ulong): return (lo:=uops[0].cast(l2i_dt[dt])), (uops[0] / 2**32).cast(l2i_dt[dt]) - ((uops[0] < 0) & lo.ne(0)) case Ops.CAST if dt in dtypes.floats: diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 6d34cc70c927d..1a3db1e177ae1 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -3,9 +3,8 @@ from tinygrad.renderer.cstyle import CStyleLanguage, base_rewrite from tinygrad.helpers import strip_parens, ceildiv -def sign_extend(val:UOp, sext_am:int): - return (((val >> (sext_am - 1)) > 0).where(UOp.const(0xffffffff << sext_am, dtypes.uint32), UOp.const(0, dtypes.uint32)) \ - | val.bitcast(dtypes.uint32)).bitcast(dtypes.int) +# a field of `width` bits sitting in the low bits of val: shift it up to the sign bit, then let the arithmetic shift fill +def sign_extend(val:UOp, width:int): return (val << (32-width)).bitcast(dtypes.int) >> (32-width) # a packed field of dt: the word it lives in, its offset in that word, and its mask. width is 8*itemsize, bool is one bit in a byte def packed_field(bidx:UOp, dt:DType) -> tuple[UOp, UOp, int]: From 3082956a17ed3046f1daebd34e2d11e7135f38f2 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:12:03 -0700 Subject: [PATCH 151/306] usb copyin: async arm and drain, 323 MB/s on comma (#17663) * usb copyin: can safely arm before drain * perf counter * 294 MB/s for comma * free speed with async transfers --- .gitignore | 1 + test/mockgpu/usb.py | 7 +++++++ tinygrad/runtime/ops_amd.py | 23 +++++++++++++---------- tinygrad/runtime/support/usb.py | 26 ++++++++++++++++++++------ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 12333d702b923..97d11025789b8 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ mutants dagre/ graphlib/ uv.lock +pi_session_window0.jsonl diff --git a/test/mockgpu/usb.py b/test/mockgpu/usb.py index adb84e4cb107c..f18b8cfdba1b5 100644 --- a/test/mockgpu/usb.py +++ b/test/mockgpu/usb.py @@ -204,6 +204,13 @@ def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: # the self.bulk_write(bytes(payload), timeout) return 0 + def control_write_async(self, request:int, value:int=0, index:int=0, data:bytes=b"", timeout:int=1000) -> int: + self.control_write(request, value, index, data, timeout) + return 0 + + def control_read_async(self, request:int, length:int, value:int=0, index:int=0, timeout:int=1000) -> tuple[int, memoryview]: + return 0, self.control_read(request, length, value, index, timeout) + def bulk_wait(self, tag:int): pass def bulk_read(self, length:int, timeout:int=1000) -> memoryview: diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index b36008d29e56b..58a56e3a972fc 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -652,12 +652,12 @@ def _do_map(self, buf:HCQBuffer): return self.dev.iface.map(buf._base if buf._ba def _copyin(self, dest:HCQBuffer, src:memoryview): if not self.dev.is_usb(): return super()._copyin(dest, src) from tinygrad.runtime.support.usb import alloc_cbuffer - # Pipelined copyin over the 0xF2 engine. 240KB chunks stream into two alternating 256KB SRAM bounce windows; the - # engine can't signal data landing, so each chunk ends in a 512B sentinel sector tagged with its sequence number. + # Pipelined copyin over the 0xF2 engine. ~256KB chunks stream into two alternating 256KB SRAM bounce windows; the + # engine can't signal data landing, so each chunk's wire image ends in a 4B sentinel tagged with its sequence number. # A prebuilt SDMA ring polls each chunk's sentinel before copying it to VRAM, then bumps a drain fence; the host # waits on that fence before re-arming a window. No timing is assumed in either direction. dev, usb, ts, sdma = self.dev, self.dev.iface.pci_dev.usb, self.dev.timeline_signal, self.dev.sdma - CHUNK, src_mv = 0x3C000, src.cast('B') # 15 16KB slots: the wire image must end mid-window (full windows corrupt) + CHUNK, src_mv = 0x40000 - 4, src.cast('B') # payload per chunk: the 256KB window minus the 4B trailing sentinel nchunks = ceildiv(src.nbytes, CHUNK) FENCE = 0xA800 # drain fence: the GPU writes it via sys_buf (PCIe 0x820800), the host reads it here (xdata) if not hasattr(self, '_usb_seq'): # one-time: clear the fence and zero both windows so garbage can't match a sentinel @@ -667,9 +667,9 @@ def _copyin(self, dest:HCQBuffer, src:memoryview): for bi in range(2): usb.scsi_write(bytes(0x40000), slot_start=bi * 16) def wait_drain(count): # spin until the drain fence reaches count, i.e. chunks 0..count-1 are fully in VRAM - t0 = time.monotonic() + t0 = time.perf_counter() while int.from_bytes(usb.read(FENCE, 8), 'little') < count: - if time.monotonic() - t0 > 10: raise RuntimeError(f"GPU failed to drain USB copyin chunk {count - 1} (10s, hung GPU?)") + if time.perf_counter() - t0 > 10: raise RuntimeError(f"GPU failed to drain USB copyin chunk {count - 1} (10s, hung GPU?)") # build the whole ring upfront: per chunk, poll the sentinel, copy SRAM->VRAM, bump the fence; then one doorbell POLL_EQ = sdma.SDMA_OP_POLL_REGMEM | sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) @@ -677,7 +677,7 @@ def wait_drain(count): # spin until the drain fence reaches count, i.e. chunks q = dev.hw_copy_queue_t().wait(ts, dev.timeline_value - 1) for c in range(nchunks): seq, size = self._usb_seq + c, min(CHUNK, src.nbytes - c * CHUNK) - q.q(POLL_EQ, *data64_le(self._usb_wins[seq & 1].va_addr + round_up(size, 512)), 0x51000000 | (seq & 0xFFFFFF), 0xFFFFFFFF, POLL_DW5) + q.q(POLL_EQ, *data64_le(self._usb_wins[seq & 1].va_addr + round_up(size + 4, 512) - 4), 0x51000000 | (seq & 0xFFFFFF), 0xFFFFFFFF, POLL_DW5) q.copy(dest.offset(c * CHUNK), self._usb_wins[seq & 1], size) q.write(dev.iface.sys_buf.offset(0x800, 8), seq + 1, b64=True) q.signal(ts, dev.next_timeline()).submit(dev) @@ -690,10 +690,13 @@ def wait_drain(count): # spin until the drain fence reaches count, i.e. chunks if inflight[seq & 1] is not None: usb.usb.bulk_wait(inflight[seq & 1]) buf = self._usb_stage[seq & 1][1] buf[:size] = src_mv[c * CHUNK : c * CHUNK + size] - struct.pack_into(' int: - """Queue a bulk OUT transfer without blocking; payload is kept alive until bulk_wait(tag).""" + def _submit_async(self, endpoint:int, xtype:int, payload:bytes|bytearray|memoryview, timeout:int) -> int: # payload kept alive till bulk_wait tr = self._async_pool.pop() if self._async_pool else libusb.libusb_alloc_transfer(0) - tr.contents.dev_handle, tr.contents.endpoint, tr.contents.type = self.handle, 0x02, libusb.LIBUSB_TRANSFER_TYPE_BULK + tr.contents.dev_handle, tr.contents.endpoint, tr.contents.type = self.handle, endpoint, xtype tr.contents.timeout, tr.contents.length = timeout, len(payload) - tr.contents.buffer = ctypes.cast(from_mv(payload, ctypes.c_ubyte), ctypes.POINTER(ctypes.c_ubyte)) + tr.contents.buffer = ctypes.cast(from_mv(memoryview(payload), ctypes.c_ubyte), ctypes.POINTER(ctypes.c_ubyte)) tr.contents.callback, tr.contents.user_data = self._async_cb, (tag := next(self._async_seq)) self._async_pending[tag] = (tr, payload) - checked(libusb.libusb_submit_transfer, "async bulk OUT submit failed")(tr) + checked(libusb.libusb_submit_transfer, "async submit failed")(tr) return tag + def bulk_write_async(self, payload:memoryview, timeout:int=10000) -> int: + """Queue a bulk OUT transfer without blocking; payload is kept alive until bulk_wait(tag).""" + return self._submit_async(0x02, libusb.LIBUSB_TRANSFER_TYPE_BULK, payload, timeout) + + def control_write_async(self, request:int, value:int=0, index:int=0, data:bytes=b"", timeout:int=1000) -> int: + """Queue a vendor control OUT without blocking; completes via bulk_wait(tag) like bulk_write_async.""" + setup = bytearray(struct.pack(' tuple[int, memoryview]: + """Queue a vendor control IN without blocking; the data lands in the returned buffer by bulk_wait(tag).""" + buf = bytearray(struct.pack(' Date: Fri, 21 Aug 2026 13:19:47 -0700 Subject: [PATCH 152/306] ci: remove setup-python (#17665) --- .github/actions/setup-tinygrad/action.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 4a7e1ecad1f31..d55221f688724 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -61,6 +61,7 @@ runs: echo "MAX_BUFFER_SIZE=300000000" >> "$GITHUB_ENV" if [[ "$RUNNER_OS" == "Linux" ]]; then echo "VIRTUAL_ENV=/opt/venv/${{ inputs.python-version }}" >> "$GITHUB_ENV" + echo "UV_PYTHON_INSTALL_DIR=/opt/python" >> "$GITHUB_ENV" else echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV" fi @@ -70,11 +71,6 @@ runs: with: enable-cache: 'false' # see below for manual caching - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ inputs.python-version }} - # **** Caching packages **** - name: Cache Python packages (PR) From 12f889aaadfc33b1b99dc4e1a52651194c80746e Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:27:21 +0300 Subject: [PATCH 153/306] hcq2: parallel compile (#17667) * hcq2: parallel compile * Dx --- tinygrad/engine/realize.py | 22 +++++++++++----------- tinygrad/runtime/support/hcq2.py | 19 +++++-------------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 3fadd43037811..3f1ae94f6e33a 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -253,25 +253,24 @@ def _validate(call:UOp, sink:UOp) -> UOp: def _compile_kernel(x:tuple[int, tuple[UOp, Renderer], dict]) -> tuple[int, UOp]: with Context(**x[2]): return x[0], to_program(*x[1]) -def _needs_compile(c:UOp) -> bool: - if c.op is not Ops.CALL: return False - if c.src[0].op is Ops.SINK: return True +def _get_call_to_compile(c:UOp) -> tuple[UOp, Renderer]|None: + ast = a0.src[0] if (a0:=c.src[0]).op is Ops.CUSTOM_FUNCTION and a0.arg == "hcq" else a0 # a PROGRAM with a ProgramInfo and a BINARY is already compiled - return c.src[0].op is Ops.PROGRAM and not (isinstance(c.src[0].arg, ProgramInfo) and c.src[0].src[-1].op is Ops.BINARY) + if ast.op is Ops.SINK or (ast.op is Ops.PROGRAM and not (isinstance(ast.arg, ProgramInfo) and ast.src[-1].op is Ops.BINARY)): + return ast, Device[c.device if isinstance(c.device, str) else c.device[0]].renderer + return None def lower_and_compile(linear:UOp) -> UOp: # collect the kernels to lower and compile, deduped by their compile cache key - calls = [c for c in linear.toposort() if _needs_compile(c)] - rens = {c: Device[c.device if isinstance(c.device, str) else c.device[0]].renderer for c in calls} - keys = {c: to_program_key(c.src[0], rens[c]) for c in calls} - if not len(calls): return linear + if not len(ar:={c: a for c in linear.toposort() if c.op is Ops.CALL and (a:=_get_call_to_compile(c)) is not None}): return linear # lower and compile what's not cached, in parallel if there's a worker pool - todo = list({keys[c]: (c.src[0], rens[c]) for c in calls if keys[c] not in to_program_cache}.items()) + keys = {c: to_program_key(*a) for c, a in ar.items()} + todo = list({keys[c]: a for c, a in ar.items() if keys[c] not in to_program_cache}.items()) if len(todo): # kernels that beam search must compile in the parent, beam needs device access to time candidates - pool = None if len(todo) == 1 or any(getattr(c.src[0].arg, "beam", 0) for c in calls) else get_worker_pool() + pool = None if len(todo) == 1 or any(getattr(c.src[0].arg, "beam", 0) for c in ar) else get_worker_pool() ctx = {v.key: v.value for v in to_program_context} tasks = ((i, ast_ren, ctx) for i, (_, ast_ren) in enumerate(todo)) try: @@ -285,7 +284,8 @@ def lower_and_compile(linear:UOp) -> UOp: raise # swap the compiled PROGRAMs into the calls - return linear.substitute({c: c.replace(src=(to_program_cache[keys[c]], *c.src[1:])) for c in calls}, name="precompile kernels") + return linear.substitute({c: c.replace(src=(c.src[0].substitute({a[0]: to_program_cache[keys[c]]}), *c.src[1:])) for c, a in ar.items()}, + name="precompile kernels") pm_optimize_local_size = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), optimize_local_size), diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f4204c2f06ad0..d985becfc65ab 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -13,7 +13,7 @@ from tinygrad.runtime.support.memory import BumpAllocator from tinygrad.renderer import Renderer, Estimates from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop -from tinygrad.engine.realize import pm_flatten_linear +from tinygrad.engine.realize import pm_flatten_linear, lower_and_compile # ***************** # 0. helpers @@ -389,9 +389,10 @@ def replace_params(call:UOp) -> UOp|None: sub = {(b:=u.without_after): UOp.param(i, u.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=b.op is Ops.PARAM and b.arg.volatile) for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} | _rank_ranges(tops) info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args + refhold) if u.without_after.tag == "inputs"), None)) - return call.replace(src=(body.substitute(sub).replace(arg="hcq_args"), *c_args, *refhold), arg=replace(call.arg, aux=info)) + prg_sink = body.src[0].substitute(sub).replace(arg=KernelInfo("hcq_submit"), tag=1) + return call.replace(src=(body.replace(src=(prg_sink,)), *c_args, *refhold), arg=replace(call.arg, aux=info)) pm_replace_params = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), replace_params)]) + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), replace_params)]) # ***************** @@ -425,15 +426,6 @@ def pack_hcq_placeholders(call:UOp) -> UOp|None: pm_pack_placeholders = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), pack_hcq_placeholders)]) -# ***************** -# 8. callify hcq programs - -def callify_hcq(call:UOp, cf:UOp) -> UOp: - prg = to_program(cf.src[0].replace(arg=KernelInfo("hcq_submit"), tag=1), Device[HCQ_RUNTIME_DEV.value].renderer) - return call.replace(src=(cf.replace(src=(prg,), arg="hcq"), *call.src[1:])) -pm_callify_hcq = PatternMatcher([(UPat(Ops.CALL, src=( - UPat(Ops.CUSTOM_FUNCTION, arg="hcq_args", src=(UPat(Ops.SINK),), name="cf"),), name="call", allow_any_len=True), callify_hcq)]) - # ***************** # 9. merge submitters @@ -469,8 +461,7 @@ def hcq_lower(linear:UOp, pm_encode:PatternMatcher) -> UOp: linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") # and compile it - linear = graph_rewrite(linear, pm_replace_params, name="replace params") - return graph_rewrite(linear, pm_callify_hcq, name="callify hcq", enter_calls=True) + return lower_and_compile(graph_rewrite(linear, pm_replace_params, walk=True, name="replace params")) @rewrite_group(lambda linear,input_uops,profile,ret: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: From daa154aa226b5767976765b4eec374e7e9ce53e0 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 16:28:04 -0400 Subject: [PATCH 154/306] FLOORDIV to SHR for powers of 2 [pr] (#17669) --- test/null/test_uops.py | 7 ++++--- tinygrad/codegen/decomp/op.py | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index a21fae30a8d9f..43cd44de4f430 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -301,9 +301,9 @@ def test_floormod_power_of_two(self): self.assertNotIn(Ops.CMOD, ops, f"For dtype={dt} FLOORMOD by pow2 left a MOD") self.assertNotIn(Ops.FLOORMOD, ops, f"For dtype={dt} FLOORMOD survived past late rewrite") - def test_floordiv_power_of_two_uint(self): - # uint FLOORDIV by a power of two lowers to a shift, leaving no IDIV/FLOORDIV in the kernel - for dt in (dtypes.uint32, dtypes.uint64): + def test_floordiv_power_of_two(self): + # FLOORDIV by a power of two lowers to a shift, with no round toward zero correction (a shift is exactly floor division) + for dt in (dtypes.int32, dtypes.uint32, dtypes.int64, dtypes.uint64): g = UOp.param(0, dt, (3,)) c = UOp.const(2).cast(dt) a = UOp(Ops.FLOORDIV, dt, (g.index(c), c)) @@ -311,6 +311,7 @@ def test_floordiv_power_of_two_uint(self): ops = [x.op for x in uops] self.assertIn(Ops.SHR, ops, f"For dtype={dt} FLOORDIV by power of two did not simplify to shift") self.assertNotIn(Ops.CDIV, ops, f"For dtype={dt} FLOORDIV by power of two did not simplify to shift") + self.assertNotIn(Ops.CMOD, ops, f"For dtype={dt} FLOORDIV by pow2 kept the round toward zero correction") self.assertNotIn(Ops.FLOORDIV, ops, f"For dtype={dt} FLOORDIV survived past late rewrite") @Context(DISABLE_FAST_IDIV=0) diff --git a/tinygrad/codegen/decomp/op.py b/tinygrad/codegen/decomp/op.py index a23142809f7dc..4e1d5f81c4505 100644 --- a/tinygrad/codegen/decomp/op.py +++ b/tinygrad/codegen/decomp/op.py @@ -75,7 +75,11 @@ def floormod_to_mod(a:UOp, b:UOp) -> UOp: @functools.cache def get_simplifying_rewrite_patterns(ops:tuple[Ops, ...]) -> PatternMatcher: # these are rewrites that make things simpler - pat: list[tuple[UPat, Callable]] = [(UPat.var("a")//UPat.var("b"), floordiv_to_idiv)] + pat: list[tuple[UPat, Callable]] = [] + # FLOORDIV by 2**y -> x >> y (an arithmetic shift is exactly floor division for any sign); fires before floordiv_to_idiv + if Ops.SHR in ops: pat.append((UPat.var("x", dtypes.ints)//UPat.cvar("c"), + lambda x,c: x >> v if (v:=powers_of_two.get(c.val, 0)) else None)) + pat.append((UPat.var("a")//UPat.var("b"), floordiv_to_idiv)) # FLOORMOD by 2**y -> x & (2**y-1) (correct floor mod for any sign in two's complement); fires before floormod_to_mod if Ops.AND in ops: pat.append((UPat.var("x", dtypes.ints)%UPat.cvar("c"), lambda x,c: x & (c.val-1) if c.val in powers_of_two else None)) pat.append((UPat.var("a")%UPat.var("b"), floormod_to_mod)) From 4fd4eafb23fbdbfe512a8274c130ea18c20a3a4b Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 21 Aug 2026 23:37:09 +0300 Subject: [PATCH 155/306] nv: hevc (#17661) * nv: hevc * x * nv: zero the nvdec scratch buffers --- .github/workflows/benchmark.yml | 3 +++ tinygrad/device.py | 3 ++- tinygrad/runtime/ops_amd.py | 4 ++-- tinygrad/runtime/ops_nv.py | 21 +++++++++++---------- tinygrad/runtime/support/memory.py | 4 ++-- tinygrad/runtime/support/nv/ip.py | 23 +++++++++++++++++------ tinygrad/runtime/support/system.py | 4 ++-- 7 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a94fe047cfbd0..ecefd8f432e48 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -634,6 +634,9 @@ jobs: run: | GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit + - name: HEVC Decode Benchmark + if: ${{ matrix.dev == 'NV' }} + run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) if: ${{ matrix.dev == 'NV' }} run: BENCHMARK_LOG=resnet_10steps MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py diff --git a/tinygrad/device.py b/tinygrad/device.py index 66b4a72c02f51..c21f7e275b395 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -83,6 +83,7 @@ class BufferSpec: cpu_access: bool = False host: bool = False nolru: bool = False + zero: bool = False external_ptr: int|None = None class MultiBuffer: @@ -265,7 +266,7 @@ def free_cache(self): for opaque in opaques: super().free(opaque, sz, options) opaques.clear() def free(self, opaque:Any, size:int, options:BufferSpec|None=None): - if LRU and (options is None or (not options.nolru and options.external_ptr is None)): self.cache[(size, options)].append(opaque) + if LRU and (options is None or (not (options.nolru or options.zero) and options.external_ptr is None)): self.cache[(size, options)].append(opaque) else: super().free(opaque, size, options) class DepsTracker: diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 58a56e3a972fc..e5f968133d0f8 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -977,13 +977,13 @@ def _dma_region(self, ctrl_addr, sys_addr, size): region = self.dev_impl.mm.map_range(vaddr:=self.dev_impl.mm.alloc_vaddr(size=size), size, [(sys_addr, size)], aspace=AddrSpace.SYS, uncached=True) return HCQBuffer(vaddr, size, meta=PCIAllocationMeta(region, has_cpu_mapping=False), view=self.pci_dev.dma_view(ctrl_addr, size), owner=self.dev) - def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, **kwargs) -> HCQBuffer: + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, zero=False, **kwargs) -> HCQBuffer: # usb allocates uncached and cpu_access in vram. vram writes are faster than sram writes # NOTE: host allocs deliberately do NOT use sys_buf (the 0x820000 NVMe SQ region): the GPU's signal writes there # collide with the 0xF2 engine mid-stream. Signals in VRAM are read back via 0xF0 streaming reads instead. # force devmem - return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access, contiguous=contiguous, force_devmem=True, **kwargs) + return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access, contiguous=contiguous, force_devmem=True, zero=zero, **kwargs) def sleep(self, timeout): pass diff --git a/tinygrad/runtime/ops_nv.py b/tinygrad/runtime/ops_nv.py index 5b67afe0ab22c..f814a752b6df6 100644 --- a/tinygrad/runtime/ops_nv.py +++ b/tinygrad/runtime/ops_nv.py @@ -340,7 +340,7 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup class NVAllocator(HCQAllocator['NVDevice']): def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: - return self.dev.iface.alloc(size, cpu_access=options.cpu_access, host=options.host) + return self.dev.iface.alloc(size, cpu_access=options.cpu_access, host=options.host, zero=options.zero) def _do_free(self, opaque:HCQBuffer, options:BufferSpec): self.dev.iface.free(opaque) @@ -565,7 +565,7 @@ def __init__(self, dev, dev_id): # Setup classes for the GPU self.gpfifo_class, self.compute_class, self.dma_class = (gsp:=self.dev_impl.gsp).gpfifo_class, gsp.compute_class, gsp.dma_class - self.viddec_class = None + self.viddec_class = gsp.viddec_class def setup_usermode(self): return 0xce000000, self.pci_dev.map_bar(bar=0, fmt='I', off=0xbb0000, size=0x10000) def setup_vm(self, vaspace): pass @@ -603,7 +603,7 @@ def __init__(self, device:str=""): vaspace_params = nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS(vaBase=0x1000, vaSize=0x1fffffb000000, flags=nv_gpu.NV_VASPACE_ALLOCATION_FLAGS_ENABLE_PAGE_FAULTING | nv_gpu.NV_VASPACE_ALLOCATION_FLAGS_IS_EXTERNALLY_OWNED) - vaspace = self.iface.rm_alloc(self.nvdevice, nv_gpu.FERMI_VASPACE_A, vaspace_params) + self.vaspace = vaspace = self.iface.rm_alloc(self.nvdevice, nv_gpu.FERMI_VASPACE_A, vaspace_params) self.iface.setup_vm(vaspace) @@ -643,7 +643,8 @@ def _new_gpu_fifo(self, gpfifo_area, ctxshare, channel_group, offset=0, entries= notifier = self.iface.alloc(48 << 20, uncached=True) params = nv_gpu.NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS(gpFifoOffset=gpfifo_area.va_addr+offset, gpFifoEntries=entries, hContextShare=ctxshare, hObjectError=notifier.meta.hMemory, hObjectBuffer=self.virtmem if video else gpfifo_area.meta.hMemory, - hUserdMemory=(ctypes.c_uint32*8)(gpfifo_area.meta.hMemory), userdOffset=(ctypes.c_uint64*8)(entries*8+offset), engineType=19 if video else 0) + hUserdMemory=(ctypes.c_uint32*8)(gpfifo_area.meta.hMemory), userdOffset=(ctypes.c_uint64*8)(entries*8+offset), engineType=19 if video else 0, + hVASpace=self.vaspace if video and self.is_nvd() else 0) # gsp has no default vaspace, rm maps the decoder ctx into its own gpfifo = self.iface.rm_alloc(channel_group, self.iface.gpfifo_class, params) if compute: @@ -709,22 +710,22 @@ def _ensure_has_local_memory(self, required): def _ensure_has_vid_hw(self, w, h): if self.iface.viddec_class is None: raise RuntimeError(f"{self.device} Video decoder class not available.") - coloc_size = round_up((round_up(h, 64) * round_up(h, 64)) + (round_up(w, 64) * round_up(h, 64) // 16), 2 << 20) + coloc_sz = round_up((round_up(h, 64) * round_up(h, 64)) + (round_up(w, 64) * round_up(h, 64) // 16), 2 << 20) self.intra_top_off = round_up(h, 64) * (608 + 4864 + 152 + 2000) intra_unk_size = ((2 << 20) if self.iface.viddec_class >= nv_gpu.NVCFB0_VIDEO_DECODER else 0) self.intra_unk_off = (round_up(self.intra_top_off, 0x10000) + (64 << 10)) if intra_unk_size > 0 else None - filter_size = round_up(round_up(self.intra_top_off, 0x10000) + (64 << 10) + intra_unk_size, 2 << 20) + filter_sz = round_up(round_up(self.intra_top_off, 0x10000) + (64 << 10) + intra_unk_size, 2 << 20) if not hasattr(self, 'vid_gpfifo'): self.vid_gpfifo = self._new_gpu_fifo(self.gpfifo_area, 0, self.nvdevice, offset=0x200000, entries=2048, compute=False, video=True) - self.vid_coloc_buf, self.vid_filter_buf = self.allocator.alloc(coloc_size), self.allocator.alloc(filter_size) - self.vid_stat_buf = self.allocator.alloc(0x1000) + self.vid_coloc_buf, self.vid_filter_buf = (self.allocator.alloc(sz, BufferSpec(zero=True)) for sz in [coloc_sz, filter_sz]) + self.vid_stat_buf = self.allocator.alloc(0x1000, BufferSpec(zero=True)) NVVideoQueue().wait(self.timeline_signal, self.timeline_value - 1) \ .setup(copy_class=self.iface.viddec_class) \ .signal(self.timeline_signal, self.next_timeline()).submit(self) else: - if coloc_size > self.vid_coloc_buf.size: self.vid_coloc_buf, _ = self._realloc(self.vid_coloc_buf, coloc_size, force=True) - if filter_size > self.vid_filter_buf.size: self.vid_filter_buf, _ = self._realloc(self.vid_filter_buf, filter_size, force=True) + if coloc_sz > self.vid_coloc_buf.size: self.vid_coloc_buf,_= self._realloc(self.vid_coloc_buf, coloc_sz, BufferSpec(zero=True), force=True) + if filter_sz > self.vid_filter_buf.size: self.vid_filter_buf,_= self._realloc(self.vid_filter_buf, filter_sz, BufferSpec(zero=True), force=True) def hw_copy_queues(self): return super().hw_copy_queues() + ([("NVDEC:0", NVVideoQueue)] if hasattr(self, 'vid_gpfifo') else []) diff --git a/tinygrad/runtime/support/memory.py b/tinygrad/runtime/support/memory.py index ac7239b769657..30d0fbc5eb84f 100644 --- a/tinygrad/runtime/support/memory.py +++ b/tinygrad/runtime/support/memory.py @@ -236,7 +236,7 @@ def identity_va(self, uncached:bool) -> int: self.map_range(va:=self.alloc_vaddr(self.vram_size, self.vram_size), self.vram_size, [(0, self.vram_size)], AddrSpace.PHYS, uncached=uncached) return va - def valloc(self, size:int, align=0x1000, uncached=False, contiguous=False) -> VirtMapping: + def valloc(self, size:int, align=0x1000, uncached=False, contiguous=False, zero=False) -> VirtMapping: if not getenv("GMMU", 1): paddr = self.palloc(size:=round_up(size, 0x1000), align, zero=False) return VirtMapping(self.identity_va(uncached) + paddr, size, [(paddr, size)], aspace=AddrSpace.PHYS, uncached=uncached) @@ -251,7 +251,7 @@ def valloc(self, size:int, align=0x1000, uncached=False, contiguous=False) -> Vi while rem_size > 0: while self.palloc_ranges[nxt_range][0] > rem_size: nxt_range += 1 - try: paddrs += [(self.palloc(try_sz:=self.palloc_ranges[nxt_range][0], self.palloc_ranges[nxt_range][1], zero=False), try_sz)] + try: paddrs += [(self.palloc(try_sz:=self.palloc_ranges[nxt_range][0], self.palloc_ranges[nxt_range][1], zero=zero), try_sz)] except MemoryError: # Move to a smaller size and try again. nxt_range += 1 diff --git a/tinygrad/runtime/support/nv/ip.py b/tinygrad/runtime/support/nv/ip.py index 75a208d05bcae..58885ac304875 100644 --- a/tinygrad/runtime/support/nv/ip.py +++ b/tinygrad/runtime/support/nv/ip.py @@ -345,7 +345,7 @@ def kfsp_send_msg(self, nvmd:int, buf:bytes): class NV_GSP(NV_IP): def init_sw(self): - self.handle_gen = itertools.count(0xcf000000) + self.handle_gen, self.chan_runlists = itertools.count(0xcf000000), {} self.init_rm_args() self.init_libos_args() self.init_wpr_meta() @@ -355,6 +355,7 @@ def init_sw(self): self.rpc_set_registry_table() self.gpfifo_class, self.compute_class, self.dma_class = nv_gpu.AMPERE_CHANNEL_GPFIFO_A, nv_gpu.AMPERE_COMPUTE_B, nv_gpu.AMPERE_DMA_COPY_B + self.viddec_class = {"AD":nv_gpu.NVC9B0_VIDEO_DECODER, "GB":nv_gpu.NVCFB0_VIDEO_DECODER}.get(self.nvdev.chip_name[:2]) # nvdec: ada and blackwell match self.nvdev.chip_name[:2]: case "AD": self.compute_class = nv_gpu.ADA_COMPUTE_A case "GB": @@ -453,8 +454,8 @@ def init_wpr_meta(self): self.wpr_meta, _, wpr_meta_addrs = self.nvdev._alloc_boot_mem(ctypes.sizeof(type(m)), data=bytes(m)) self.wpr_meta_sysmem = wpr_meta_addrs[0] - def promote_ctx(self, client:int, subdevice:int, obj:int, ctxbufs:dict[int, GRBufDesc], bufs=None, virt=None, phys=None): - res, prom = {}, nv_gpu.NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS(entryCount=len(ctxbufs), engineType=0x1, hChanClient=client, hObject=obj) + def promote_ctx(self, client:int, subdevice:int, obj:int, ctxbufs:dict[int, GRBufDesc], bufs=None, virt=None, phys=None, engine=0x1): + res, prom = {}, nv_gpu.NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS(entryCount=len(ctxbufs), engineType=engine, hChanClient=client, hObject=obj) for i,(buf,desc) in enumerate(ctxbufs.items()): use_v, use_p = (desc.virt if virt is None else virt), (desc.phys if phys is None else phys) x = (bufs or {}).get(buf, self.nvdev.mm.valloc(desc.size, contiguous=True)) # allocate buffers @@ -470,6 +471,9 @@ def init_golden_image(self): subdev = self.rpc_rm_alloc(hParent=dev, hClass=nv_gpu.NV20_SUBDEVICE_0, params=nv_gpu.NV2080_ALLOC_PARAMETERS()) vaspace = self.rpc_rm_alloc(hParent=dev, hClass=nv_gpu.FERMI_VASPACE_A, params=nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS()) + di = self.rpc_rm_control(subdev, nv_gpu.NV2080_CTRL_CMD_FIFO_GET_DEVICE_INFO_TABLE, nv_gpu.NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS()) + self.runlists = {di.entries[i].engineData[2]: di.entries[i].engineData[3] for i in range(di.numEntries)} + # reserve 512MB for the reserved PDES res_va = self.nvdev.mm.alloc_vaddr(res_sz:=(512 << 20)) @@ -549,10 +553,16 @@ def rpc_rm_alloc(self, hParent:int, hClass:int, params:Any, client=None) -> int: self.cmd_q.send_rpc(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, bytes(alloc_args) + (bytes(params) if params is not None else b'')) self.stat_q.wait_resp(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC) + if hClass == self.gpfifo_class: + self.chan_runlists[obj] = self.runlists.get((e:=params.engineType) + 10*(e >= nv_gpu.NV2080_ENGINE_TYPE_NVDEC0), 0) if hClass == nv_gpu.FERMI_VASPACE_A and client != self.priv_root: self.rpc_set_page_directory(device=hParent, hVASpace=obj, pdir_paddr=self.nvdev.mm.root_page_table.paddr, client=client) if hClass == nv_gpu.NV01_DEVICE_0 and client != self.priv_root: self.device = obj # save user device handle if hClass == nv_gpu.NV20_SUBDEVICE_0: self.subdevice = obj # save subdevice handle + if hClass == self.viddec_class and client != self.priv_root: + ctx, eng = {0: GRBufDesc(0x1000, phys=True, virt=True)}, nv_gpu.NV2080_ENGINE_TYPE_NVDEC0 + bufs = self.promote_ctx(client, self.subdevice, hParent, ctx, virt=False, engine=eng) + self.promote_ctx(client, self.subdevice, hParent, ctx, bufs, phys=False, engine=eng) if hClass == self.compute_class and client != self.priv_root: phys_gr_ctx = self.promote_ctx(client, self.subdevice, hParent, {k:v for k,v in self.grctx_bufs.items() if k in [0, 1, 2]}, virt=False) self.promote_ctx(client, self.subdevice, hParent, {k:v for k,v in self.grctx_bufs.items() if k in [0, 1, 2]}, phys_gr_ctx, phys=False) @@ -575,9 +585,10 @@ def rpc_rm_control(self, hObject:int, cmd:int, params:Any, client=None, extra=No res = self.stat_q.wait_resp(nv.NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL) st = type(params).from_buffer_copy(res[len(bytes(control_args)):]) if params is not None else None - # NOTE: gb20x requires the enable bit for token submission. Patch workSubmitToken here to maintain userspace compatibility. - if self.nvdev.chip_name.startswith("GB2") and cmd == nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN: - cast(nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS, st).workSubmitToken |= (1 << 30) + # NOTE: gsp only fills in the channel id, the runlist id (and, on gb20x, the doorbell enable bit) are added by the driver. + if cmd == nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN: + cast(nv_gpu.NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN_PARAMS, st).workSubmitToken |= (self.chan_runlists[hObject] << 16) | \ + ((1 << 30) if self.nvdev.chip_name.startswith("GB2") else 0) return st def rpc_set_page_directory(self, device:int, hVASpace:int, pdir_paddr:int, client=None, pasid=0xffffffff): diff --git a/tinygrad/runtime/support/system.py b/tinygrad/runtime/support/system.py index cf6839503038a..1f11403d30ef0 100644 --- a/tinygrad/runtime/support/system.py +++ b/tinygrad/runtime/support/system.py @@ -262,7 +262,7 @@ def __init__(self, dev, dev_id, vendor, devices:tuple[tuple[int, tuple[int, ...] self.dev_impl = dev_impl_t(self.pci_dev) self.dev, self.vram_bar, self.count = dev, vram_bar, len(hcq_filter_visible_devices(System.list_devices(vendor, devices, base_class), dn)) - def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, **kwargs) -> HCQBuffer: + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, zero=False, **kwargs) -> HCQBuffer: should_use_sysmem = host or ((cpu_access if self.is_bar_small() else (uncached and cpu_access)) and not force_devmem) # Align size to huge pages for large allocations, otherwise the unaligned tail falls back to 4KB pages, increasing TLB pressure. @@ -274,7 +274,7 @@ def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguo mapping = self.dev_impl.mm.map_range(vaddr, size, [(paddr, 0x1000) for paddr in paddrs], aspace=AddrSpace.SYS, snooped=True, uncached=True) return HCQBuffer(vaddr, size, meta=PCIAllocationMeta(mapping, has_cpu_mapping=True, hMemory=paddrs[0]), view=memview, owner=self.dev) - mapping = self.dev_impl.mm.valloc(size:=round_up(size, 0x1000), uncached=uncached, contiguous=cpu_access) + mapping = self.dev_impl.mm.valloc(size:=round_up(size, 0x1000), uncached=uncached, contiguous=cpu_access, zero=zero) barview = self.pci_dev.map_bar(bar=self.vram_bar, off=mapping.paddrs[0][0], size=mapping.size) if cpu_access else None return HCQBuffer(mapping.va_addr, size, view=barview, meta=PCIAllocationMeta(mapping, cpu_access, hMemory=mapping.paddrs[0][0]), owner=self.dev) From f986829461abee5f1bc8bb9ddc4b6cf248f704b6 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:12:39 -0700 Subject: [PATCH 156/306] keep IndexingContext scoped in indexing (#17670) --- tinygrad/schedule/indexing.py | 6 ++---- tinygrad/schedule/rangeify.py | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index 430542a932cb1..549e3fdae3724 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -12,8 +12,6 @@ class IndexingContext: realize_map: dict[UOp, None|list[int]] = field(default_factory=dict) non_removable: dict[UOp, None] = field(default_factory=dict) range_map: dict[UOp, tuple[tuple[UOp, ...], tuple[UOp, ...]]] = field(default_factory=dict) - # loads reachable from each UOp memoized across matches - buf_cache: dict[UOp, frozenset[UOp]] = field(default_factory=dict) # create ranges range_idx: Iterator[int] = field(default_factory=itertools.count) @@ -187,7 +185,7 @@ def apply_movement_op(op:Ops, in_shape:tuple[sint,...], arg:tuple, rngs:tuple[UO return rngs @rewrite_group(new_ctx=False) -def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: +def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: if debug: print("**************************") rctx = IndexingContext() @@ -322,7 +320,7 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]: tsink = graph_rewrite(tsink, pm_apply_rangeify, ctx=rctx, bottom_up=True, name="apply rangeify") # if a deviceless value must materialize, place it on the sink device tsink = graph_rewrite(tsink, pm_fix_deviceless, ctx=tsink.device, name="add device to deviceless") - return tsink, rctx + return tsink def render_ranges(*rngs_list, realized) -> str: disp = [] diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 26ed0d880dde7..979cef04c0fa4 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -10,7 +10,7 @@ from tinygrad.helpers import PCONTIG, FLOAT16, OPENPILOT_HACKS, argsort, partition, get_single_element from tinygrad.codegen.simplify import pm_flatten_range, pm_reduce_simplify from tinygrad.codegen.opt import Opt -from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, IndexingContext, apply_movement_op +from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, apply_movement_op from tinygrad.schedule.multi import multi_pm from tinygrad.schedule.allreduce import create_allreduce_function @@ -352,7 +352,12 @@ def no_indexing_calls(u:UOp): ]) DEVICE_MAX_BUFS = {"METAL": 31, "WEBGPU": 8, "CPU": 31} # TODO: get from device? -def limit_bufs(ctx:IndexingContext, root:UOp): +@dataclass +class LimitBufsContext: + buf_cache: dict[UOp, frozenset[UOp]] = field(default_factory=dict) + range_idx: itertools.count = field(default_factory=itertools.count) + +def _limit_bufs(ctx:LimitBufsContext, root:UOp): if (device:=root.device) is None: return None # no device, index related calculations device = device if isinstance(device, str) else device[0].split(":")[0] if not (MAX_BUFS:=MAX_KERNEL_BUFFERS.value or DEVICE_MAX_BUFS.get(device, 0)): return None @@ -374,7 +379,7 @@ def visitor(u:UOp) -> frozenset[UOp]: s = s.substitute(dict(zip(orig_ranges, end_ranges))).bufferize(*end_ranges, arg=BufferizeOpts(device=s.device)).index(*orig_ranges) srcs.append(s) return root.replace(src=tuple(srcs)) -pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), limit_bufs)]) +pm_limit_bufs = PatternMatcher([(UPat(set.union(GroupOp.Binary, GroupOp.Ternary), name="root"), _limit_bufs)]) # ***************** # 4. put in buffers for bufferize @@ -578,20 +583,21 @@ def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): @rewrite_group(new_ctx=False) def get_kernel_graph(sink:UOp) -> UOp: + # prepare for rangeify tsink = graph_rewrite(sink, multi_pm, name="multi_pm") if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") - tsink = graph_rewrite(tsink, pm_copy_to_store, ctx=itertools.count(0), bottom_up=True, name="convert copy to store") # convert movement ops to ranges - tsink, rctx = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) + tsink = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) + # cleanups for speed and runability tsink = graph_rewrite(tsink, symbolic+pm_reduce_simplify+pm_const_buffer_folding+pm_remove_bufferize, name="symbolic+reduce_collapse+debuf") - tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=rctx, name="limit buffers") - + next_range = max((x.arg[0] for x in tsink.toposort() if x.op is Ops.RANGE), default=-1) + 1 + tsink = graph_rewrite(tsink, pm_limit_bufs, ctx=LimitBufsContext(range_idx=itertools.count(next_range)), name="limit buffers") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Rangeify") # bufferize -> store From 07cce78cecc2c98ea23eddb929755fbde41c817d Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Fri, 21 Aug 2026 16:18:32 -0700 Subject: [PATCH 157/306] compile3: log printed timings (#17671) --- examples/openpilot/compile3.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/examples/openpilot/compile3.py b/examples/openpilot/compile3.py index 7bf253fd8d882..20c98cd463809 100644 --- a/examples/openpilot/compile3.py +++ b/examples/openpilot/compile3.py @@ -107,14 +107,21 @@ def run_onnx_jit(**kwargs): return next(iter(run_onnx({k:v.to(Device.DEFAULT) fo return inputs, test_val def test_vs_compile(run, inputs, test_val=None): + if (log:=bool(getenv("BENCHMARK_LOG", ""))): from extra.bench_log import WallTimeEvent, BenchEvent # run 20 times step_times = [] for _ in range(20): st = time.perf_counter() - out = run(**inputs) - mt = time.perf_counter() - val = out.numpy() + if log: + with WallTimeEvent(BenchEvent.STEP): + out = run(**inputs) + mt = time.perf_counter() + val = out.numpy() + else: + out = run(**inputs) + mt = time.perf_counter() + val = out.numpy() et = time.perf_counter() step_times.append((et-st)*1e3) print(f"enqueue {(mt-st)*1e3:6.2f} ms -- total run {step_times[-1]:6.2f} ms") @@ -160,12 +167,6 @@ def test_vs_onnx(new_inputs, test_val, onnx_file, tol): print("test vs onnx passed") return timings -def bench(run, inputs): - from extra.bench_log import WallTimeEvent, BenchEvent - for _ in range(10): - with WallTimeEvent(BenchEvent.STEP): - run(**inputs).numpy() - if __name__ == "__main__": if getenv("RUN_PICKLE"): with open(OUTPUT, "rb") as f: pickle_loaded = load_pickle(f) @@ -181,6 +182,3 @@ def bench(run, inputs): test_vs_compile(pickle_loaded, inputs, outputs) if getenv("SELFTEST"): test_vs_onnx(inputs, outputs, onnx_file, 1e-4) - - if getenv("BENCHMARK_LOG", ""): - bench(pickle_loaded, inputs) From 52596dbf38d0a917198c9a50a3d7959537cf63e6 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Fri, 21 Aug 2026 19:19:28 -0400 Subject: [PATCH 158/306] gptoss: fused ce (#17672) --- examples/mlperf/model_train.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index 19ba390c6e675..367962bd21c6e 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -1773,8 +1773,13 @@ def _scale_key(n): def minibatch(tokens:Tensor): if is_dp: tokens = tokens.to(None).shard(device, 0) if not is_sharding: tokens = tokens.to(None) + logits:Tensor = model(tokens[:, :-1], save=True) - loss = logits.sparse_categorical_crossentropy(tokens[:, 1:]) + if getenv("FUSED_CE", 0): + from extra.llama_kernels.fused_ce import fused_ce_loss + loss = fused_ce_loss(logits.cast(dtypes.bfloat16), tokens[:, 1:], label_smoothing=0.0) + else: + loss = logits.sparse_categorical_crossentropy(tokens[:, 1:]) for g, new_g in zip(grads, loss.gradient(*optim.params)): apply_grad(g, new_g.uop) From af242819d8ae5300d77a37af8534af5903809894 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:00:58 -0700 Subject: [PATCH 159/306] refactor the AMD emulator slop (kimi) (#17673) * refactor the AMD emulator slop (kimi) * mypy --- test/mockgpu/amd/emu.py | 1128 ++++++++++++---------------------- test/mockgpu/amd/pcode.py | 70 ++- test/mockgpu/amd/sqtt_enc.py | 100 +++ 3 files changed, 541 insertions(+), 757 deletions(-) create mode 100644 test/mockgpu/amd/sqtt_enc.py diff --git a/test/mockgpu/amd/emu.py b/test/mockgpu/amd/emu.py index 7c6fe9038d21d..5874b8b8f335a 100644 --- a/test/mockgpu/amd/emu.py +++ b/test/mockgpu/amd/emu.py @@ -6,7 +6,7 @@ # arg=3: lds - local data share # arg=4: scratch - per-lane scratch memory from __future__ import annotations -import ctypes, functools, re, platform, subprocess, tempfile +import ctypes, functools, itertools, re, platform, subprocess, tempfile from typing import Callable # Set/restore DAZ+FTZ (denormals-are-zero + flush-to-zero) to match RDNA3 default float mode @@ -66,132 +66,17 @@ def __exit__(self, *args): from tinygrad.runtime.autogen.amd.rdna3 import ins as ir3 from tinygrad.runtime.autogen.amd.rdna4 import ins as ir4 from tinygrad.runtime.autogen.amd.cdna import ins as irc -from tinygrad.renderer.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp +from tinygrad.renderer.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp, Inst from tinygrad.runtime.autogen.amd.common import Fmt, OpType from test.amd.helpers import decode_dpp16 -from test.mockgpu.amd.pcode import parse_block, _FUNCS, _set_bits, _to_bool, _val_to_bits +from test.mockgpu.amd.pcode import parse_pcode, _FUNCS, _set_bits, _to_bool, _to_u32, _val_to_bits MASK32 = 0xFFFFFFFF -# ═══════════════════════════════════════════════════════════════════════════════ -# SQTT TRACE COLLECTION -# ═══════════════════════════════════════════════════════════════════════════════ - -# Global trace storage: populated by run_asm as raw SQTT blobs, consumed by amdgpu.py +# SQTT encoder lives in sqtt_enc.py; traces are consumed by amdgpu.py +from test.mockgpu.amd.sqtt_enc import make_encoder as _make_sqtt_encoder sqtt_traces: list[bytes] = [] -# Encoder primitives -from tinygrad.renderer.amd.sqtt import _build_decode_tables, PACKET_TYPES_RDNA3, LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST, InstOp - -_NIB_COUNTS: dict = {cls: nc for _, (cls, nc, *_) in _build_decode_tables(PACKET_TYPES_RDNA3)[0].items()} - -def _encode_raw(pkt_cls, **kwargs) -> tuple[int, int]: - raw = pkt_cls.encoding.default - for k, v in kwargs.items(): raw = pkt_cls.__dict__[k].set(raw, v) - return raw, _NIB_COUNTS[pkt_cls] - -def _emit_nibbles(nibbles: list[int], pkt_cls, **kwargs): - raw, nc = _encode_raw(pkt_cls, **kwargs) - for i in range(nc): nibbles.append((raw >> (i * 4)) & 0xF) - -def _nibbles_to_bytes(nibbles: list[int]) -> bytes: - result = bytearray() - for i in range(0, len(nibbles), 2): result.append(nibbles[i] | ((nibbles[i + 1] if i + 1 < len(nibbles) else 0) << 4)) - return bytes(result) - -def _init_sqtt_encoder(): - """Initialize and return SQTT encoder state. Called once per dispatch with tracing enabled.""" - from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp as SOPPOp3 - from tinygrad.runtime.autogen.amd.rdna4.enum import SOPPOp as SOPPOp4 - import re - - _SOPP = (ir3.SOPP, ir4.SOPP, irc.SOPP) - _SMEM = (ir3.SMEM, ir4.SMEM, irc.SMEM) - _VALU = (ir3.VOP1, ir3.VOP2, ir3.VOP3, ir3.VOP3P, ir3.VOPC, ir3.VOPD, ir3.VOP3SD, ir3.VOP3_SDST, ir3.VOP1_SDST, - ir4.VOP1, ir4.VOP2, ir4.VOP3, ir4.VOP3P, ir4.VOPC, ir4.VOPD, ir4.VOP3SD, ir4.VOP3_SDST, ir4.VOP1_SDST, - irc.VOP1, irc.VOP2, irc.VOP3, irc.VOP3P, irc.VOP3PX2, irc.VOPC, irc.VOP3SD, irc.VOP3_SDST) - _DS = (ir3.DS, ir4.DS, irc.DS) - _GLOBAL = (ir3.GLOBAL, ir4.VGLOBAL, irc.GLOBAL) - _FLAT = (ir3.FLAT, ir4.VFLAT, irc.FLAT) - _SCRATCH = (ir3.SCRATCH, ir4.VSCRATCH, irc.SCRATCH) - - # SOPP classification sets - _SOPP_SKIP = {SOPPOp3.S_ENDPGM.value, SOPPOp3.S_ENDPGM_SAVED.value, SOPPOp3.S_ENDPGM_ORDERED_PS_DONE.value, - SOPPOp3.S_DELAY_ALU.value} - _SOPP_IMMEDIATE = {SOPPOp3.S_NOP.value, SOPPOp3.S_CLAUSE.value, SOPPOp3.S_WAITCNT.value, SOPPOp3.S_WAITCNT_DEPCTR.value, - SOPPOp3.S_WAIT_IDLE.value, SOPPOp3.S_WAIT_EVENT.value, SOPPOp3.S_SLEEP.value, - SOPPOp3.S_SET_INST_PREFETCH_DISTANCE.value} - for _op in (SOPPOp4.S_WAIT_ALU, SOPPOp4.S_WAIT_LOADCNT, SOPPOp4.S_WAIT_STORECNT, SOPPOp4.S_WAIT_SAMPLECNT, - SOPPOp4.S_WAIT_BVHCNT, SOPPOp4.S_WAIT_EXPCNT, SOPPOp4.S_WAIT_DSCNT, SOPPOp4.S_WAIT_KMCNT, - SOPPOp4.S_WAIT_LOADCNT_DSCNT, SOPPOp4.S_WAIT_STORECNT_DSCNT): - _SOPP_IMMEDIATE.add(_op.value) - _SOPP_BARRIER = {SOPPOp3.S_BARRIER.value} - if hasattr(SOPPOp4, 'S_BARRIER_WAIT'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_WAIT.value) - if hasattr(SOPPOp4, 'S_BARRIER_LEAVE'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_LEAVE.value) - _SOPP_BRANCH = {SOPPOp3.S_BRANCH.value, SOPPOp3.S_CBRANCH_SCC0.value, SOPPOp3.S_CBRANCH_SCC1.value, - SOPPOp3.S_CBRANCH_VCCZ.value, SOPPOp3.S_CBRANCH_VCCNZ.value, - SOPPOp3.S_CBRANCH_EXECZ.value, SOPPOp3.S_CBRANCH_EXECNZ.value} - - # VALU sub-classification patterns - _VALUT_4_RE = re.compile(r'V_(EXP|LOG|RCP|RSQ|SQRT|SIN|COS|CEIL|FLOOR|TRUNC|RNDNE|FRACT|FREXP)_') - _VALUB_2_RE = re.compile(r'V_(LSHLREV|LSHRREV|ASHRREV)_(B|I)64') - _VALUB_4_RE = re.compile(r'V_MAD_(U|I)64') - _VALUB_16_RE = re.compile(r'V_\w+_F64') - - def _valu_op(op_name: str) -> InstOp|None: - if 'CMPX' in op_name: return InstOp.VALU1_WR_EXEC - if _VALUB_2_RE.search(op_name): return InstOp.VALUB_2 - if _VALUB_4_RE.search(op_name): return InstOp.VALUB_4 - if _VALUB_16_RE.search(op_name): return InstOp.VALUB_16 - if _VALUT_4_RE.search(op_name): return InstOp.VALUT_4 - return None - - def _mem_op(t, op_name: str) -> InstOp: - is_store = "STORE" in op_name - if issubclass(t, _DS): return InstOp.LDS_WR_2 if is_store else InstOp.LDS_RD - if issubclass(t, _GLOBAL): return InstOp.SGMEM_WR_2 if is_store else InstOp.SGMEM_RD_1 - if issubclass(t, _FLAT): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 - if issubclass(t, _SCRATCH): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 - return InstOp.SALU - - nibbles: list[int] = [] - started: set[int] = set() - _emit_nibbles(nibbles, LAYOUT_HEADER, layout=3, sel_a=6) - - def emit(wave_id: int, inst, branch_taken: bool|None): - """Emit an SQTT packet for one executed instruction.""" - w = wave_id & 0x1F - if wave_id not in started: - _emit_nibbles(nibbles, WAVESTART, delta=1, simd=0, wgp=0, wave=w, id7=wave_id) - started.add(wave_id) - inst_type, inst_op, op_name = type(inst), inst.op.value if hasattr(inst, 'op') else 0, inst.op.name if hasattr(inst, 'op') else "" - if issubclass(inst_type, _SOPP): - if inst_op in _SOPP_SKIP: return - elif inst_op in _SOPP_IMMEDIATE: _emit_nibbles(nibbles, IMMEDIATE, delta=1, wave=w) - elif inst_op in _SOPP_BARRIER: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.BARRIER) - elif inst_op in _SOPP_BRANCH: - _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.JUMP if branch_taken else InstOp.JUMP_NO) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SALU) - elif issubclass(inst_type, _VALU): - op = _valu_op(op_name) - if op is None: _emit_nibbles(nibbles, VALUINST, delta=1, wave=w) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=op) - elif issubclass(inst_type, _SMEM): _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SMEM_RD) - else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=_mem_op(inst_type, op_name)) - - def finish(wave_id: int): - """Emit WAVEEND for a completed wave.""" - if wave_id in started: _emit_nibbles(nibbles, WAVEEND, delta=1, simd=0, wgp=0, wave=wave_id & 0x1F) - - def finalize() -> bytes: - """Pad and return the encoded SQTT blob.""" - while len(nibbles) % 2 != 0: nibbles.append(0) - nibbles.extend([0] * 32) - while len(nibbles) % 64 != 0: nibbles.append(0) - return _nibbles_to_bytes(nibbles) - - return emit, finish, finalize - def _c(val, dtype=dtypes.uint32): return UOp.const(val, dtype) def _u64(lo: UOp, hi: UOp) -> UOp: @@ -214,27 +99,25 @@ def _apply_src_mods(val: UOp, mod_bit: int, abs_bits: int, neg_bits: int, bits: if neg_bits & (1 << mod_bit): fv = fv.neg() return fv.bitcast(ut).cast(dtypes.uint32) if bits == 16 else fv.bitcast(ut) -# Map VOPD ops to VOP2 ops for pcode lookup (both RDNA3 and RDNA4) -VOPD_TO_VOP2 = { - ir3.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir3.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32, - ir3.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir3.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32, - ir3.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir3.VOPDOp.V_DUAL_MAX_F32: ir3.VOP2Op.V_MAX_F32_E32, - ir3.VOPDOp.V_DUAL_MIN_F32: ir3.VOP2Op.V_MIN_F32_E32, ir3.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32, - ir3.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir3.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32, - ir3.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir3.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32, - ir3.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32, - ir3.VOPDOp.V_DUAL_DOT2ACC_F32_F16: ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, - # RDNA4 mappings (same VOP1/VOP2 targets, RDNA4 uses _NUM_ suffix for min/max) - ir4.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir4.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32, - ir4.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir4.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32, - ir4.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir4.VOPDOp.V_DUAL_MAX_NUM_F32: ir3.VOP2Op.V_MAX_F32_E32, - ir4.VOPDOp.V_DUAL_MIN_NUM_F32: ir3.VOP2Op.V_MIN_F32_E32, ir4.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32, - ir4.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir4.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32, - ir4.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir4.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32, - ir4.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir4.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32, - ir4.VOPDOp.V_DUAL_DOT2ACC_F32_F16: ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, +# Map VOPD ops to VOP2/VOP1 ops for pcode lookup (both RDNA3 and RDNA4 share these targets) +_VOPD_TARGETS = { + 'V_DUAL_FMAC_F32': ir3.VOP2Op.V_FMAC_F32_E32, 'V_DUAL_MUL_F32': ir3.VOP2Op.V_MUL_F32_E32, + 'V_DUAL_ADD_F32': ir3.VOP2Op.V_ADD_F32_E32, 'V_DUAL_SUB_F32': ir3.VOP2Op.V_SUB_F32_E32, + 'V_DUAL_SUBREV_F32': ir3.VOP2Op.V_SUBREV_F32_E32, 'V_DUAL_MAX_F32': ir3.VOP2Op.V_MAX_F32_E32, + 'V_DUAL_MIN_F32': ir3.VOP2Op.V_MIN_F32_E32, 'V_DUAL_ADD_NC_U32': ir3.VOP2Op.V_ADD_NC_U32_E32, + 'V_DUAL_LSHLREV_B32': ir3.VOP2Op.V_LSHLREV_B32_E32, 'V_DUAL_AND_B32': ir3.VOP2Op.V_AND_B32_E32, + 'V_DUAL_MOV_B32': ir3.VOP1Op.V_MOV_B32_E32, 'V_DUAL_CNDMASK_B32': ir3.VOP2Op.V_CNDMASK_B32_E32, + 'V_DUAL_FMAAK_F32': ir3.VOP2Op.V_FMAAK_F32_E32, 'V_DUAL_FMAMK_F32': ir3.VOP2Op.V_FMAMK_F32_E32, + 'V_DUAL_DOT2ACC_F32_F16': ir3.VOP2Op.V_DOT2ACC_F32_F16_E32, } +# RDNA4 uses a _NUM_ suffix for min/max +VOPD_TO_VOP2 = {getattr(ir3.VOPDOp, n): t for n, t in _VOPD_TARGETS.items()} +VOPD_TO_VOP2.update({getattr(ir4.VOPDOp, n.replace('_MAX_', '_MAX_NUM_').replace('_MIN_', '_MIN_NUM_')): t for n, t in _VOPD_TARGETS.items()}) def _wave_size(arch: str) -> int: return 64 if arch.startswith("cdna") else 32 +def _iattr(inst, name: str, default: int = 0) -> int: + """Optional integer attribute of a decoded instruction (None/missing -> default).""" + v = getattr(inst, name, default) + return default if v is None else v # Special registers stored after inline constants (256-259) PC_LO_IDX, PC_HI_IDX, SCRATCH_STRIDE_IDX = 256, 257, 259 # SGPR buffer: 0-127 = SGPRs, 128-255 = inline constants, 256-259 = special registers @@ -246,10 +129,6 @@ def _op_name(inst) -> str: if hasattr(inst, 'opx'): return f"{inst.opx.name}_{inst.opy.name}" # VOPD has opx/opy not op return inst.op.name if hasattr(inst.op, 'name') else str(inst.op) -def _to_u32(val: UOp) -> UOp: - if val.dtype == dtypes.uint32: return val - if val.dtype.itemsize == 4: return val.bitcast(dtypes.uint32) # same size: bitcast (float32->uint32) - return val.cast(dtypes.uint32) # different size: cast (bool, int16, etc) def _lane_active(exec_mask: UOp, lane: UOp) -> UOp: if exec_mask.dtype == dtypes.uint64: return ((exec_mask >> lane.cast(dtypes.uint64)) & UOp.const(1, dtypes.uint64)).ne(UOp.const(0, dtypes.uint64)) return ((exec_mask >> lane.cast(dtypes.uint32)) & _c(1)).ne(_c(0)) @@ -284,7 +163,7 @@ def _get_pcode_dict(op) -> dict: """Return the PCODE dictionary for the given opcode based on its architecture.""" return PCODE_CDNA if 'cdna' in type(op).__module__ else PCODE_RDNA4 if 'rdna4' in type(op).__module__ else PCODE_RDNA3 -# Pcode parser +# Pcode lookup with hardware errata fixes (the AMD-pdf pcode for these ops is subtly wrong) @functools.cache def get_pcode(op) -> str: op_name = op.name @@ -293,60 +172,41 @@ def get_pcode(op) -> str: # VOP3 ops ending in _E64 may share pcode with VOP1 _E32 equivalents import importlib enum_mod = importlib.import_module(type(op).__module__) - vop1_cls = getattr(enum_mod, 'VOP1Op', None) - e32_name = op_name.replace('_E64', '_E32') + vop1_cls, e32_name = getattr(enum_mod, 'VOP1Op', None), op_name.replace('_E64', '_E32') if vop1_cls and hasattr(vop1_cls, e32_name): op = vop1_cls[e32_name] pcode = pcode_dict[op] fix_name = op_name.replace('_E64', '').replace('_E32', '') if fix_name in _pcode_fixes: pcode = pcode.replace(*_pcode_fixes[fix_name]) - if 'V_DIV_SCALE' in op_name: - dt, exp_lim, ldexp_val = ('f32', '23', '64') if 'F32' in op_name else ('f64', '52', '128') - for old, new in [(f'S2.{dt} / S1.{dt} == DENORM.{dt}', f'divWouldBeDenorm(S2.{dt}, S1.{dt})'), (f"1.0 / 64'F(S1.{dt}) == DENORM.f64", '0'), - (f'1.0 / S1.{dt} == DENORM.{dt}', '0'), (f'S1.{dt} == DENORM.{dt}', f'isDENORM(S1.{dt})'), - (f'D0.{dt} = NAN.{dt}', f'VCC = 0x1LL;\nD0.{dt} = NAN.{dt}'), - (f'elsif isDENORM(S1.{dt}) then\nD0.{dt} = ldexp(S0.{dt}, {ldexp_val})', f'elsif 1 == 0 then\nD0.{dt} = S0.{dt}'), - (f'elsif exponent(S2.{dt}) <= {exp_lim} then\n// Numerator is tiny\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})', - f'elsif exponent(S2.{dt}) <= {exp_lim} then\nVCC = 0x1LL;\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})'), - (f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\nVCC = 0x1LL;\n' - f'if S0.{dt} == S2.{dt} then\n// Only scale the numerator\n' - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif', - f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\n' - f'VCC = 0x1LL;\nD0.{dt} = S0.{dt}'), - (f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif\nelsif', - f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nelse\n' - f'D0.{dt} = S0.{dt}\nendif\nelsif')]: - pcode = pcode.replace(old, new) - lines = pcode.rstrip().split('\n') - for i in range(len(lines) - 1, -1, -1): - if lines[i].strip() == 'endif': - lines.insert(i, f'else\nD0.{dt} = S0.{dt}') - break - pcode = '\n'.join(lines) + f';\nif isDENORM(S1.{dt}) then\nD0.{dt} = NAN.{dt}\nendif' - pcode = pcode.replace('VCC = 0x0LL', 'VCC.u64[laneId] = 0').replace('VCC = 0x1LL', 'VCC.u64[laneId] = 1') - return pcode - -def parse_pcode(pcode: str, srcs: dict[str, UOp | int] | None = None) -> tuple[dict, list]: - env: dict = srcs.copy() if srcs else {} - assigns: list[tuple[str, UOp]] = [] - raw_lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')] - # TODO: pcode.py should tokenize full pcode string instead of line-by-line, then this hack can be removed - lines: list[str] = [] - for l in raw_lines: - if lines and re.search(r'(&&|\|\||[&|+\-*/^])\s*$', lines[-1]): lines[-1] = lines[-1] + ' ' + l - else: lines.append(l) - _, final, _ = parse_block(lines, 0, env, assigns=assigns) - sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d) - for var, val in final.items(): - if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA'] and isinstance(val, UOp): - if var in sliced and not any(re.match(rf'{var}\.\w+\s*=', l) for l in lines): continue - for l in lines: - if (m := re.match(rf'{var}\.(\w+(?:\[\w+\])?)', l)): - assigns.append((f'{var}.{m.group(1)}', val)) - break - else: assigns.append((var, val)) - return env, assigns + return _fix_div_scale(pcode, 'f32' if 'F32' in op_name else 'f64') if 'V_DIV_SCALE' in op_name else pcode + +def _fix_div_scale(pcode: str, dt: str) -> str: + """V_DIV_SCALE fixes: only the divWouldBeDenorm/exponent-overflow path may return the scaled value; + all other paths must return S0 unchanged, and VCC is set exactly when scaling happened.""" + exp_lim, ldexp_val = ('23', '64') if dt == 'f32' else ('52', '128') + for old, new in [(f'S2.{dt} / S1.{dt} == DENORM.{dt}', f'divWouldBeDenorm(S2.{dt}, S1.{dt})'), (f"1.0 / 64'F(S1.{dt}) == DENORM.f64", '0'), + (f'1.0 / S1.{dt} == DENORM.{dt}', '0'), (f'S1.{dt} == DENORM.{dt}', f'isDENORM(S1.{dt})'), + (f'D0.{dt} = NAN.{dt}', f'VCC = 0x1LL;\nD0.{dt} = NAN.{dt}'), + (f'elsif isDENORM(S1.{dt}) then\nD0.{dt} = ldexp(S0.{dt}, {ldexp_val})', f'elsif 1 == 0 then\nD0.{dt} = S0.{dt}'), + (f'elsif exponent(S2.{dt}) <= {exp_lim} then\n// Numerator is tiny\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})', + f'elsif exponent(S2.{dt}) <= {exp_lim} then\nVCC = 0x1LL;\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})'), + (f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\nVCC = 0x1LL;\n' + f'if S0.{dt} == S2.{dt} then\n// Only scale the numerator\n' + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif', + f'elsif divWouldBeDenorm(S2.{dt}, S1.{dt}) then\n' + f'VCC = 0x1LL;\nD0.{dt} = S0.{dt}'), + (f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nendif\nelsif', + f'D0.{dt} = ldexp(S0.{dt}, {ldexp_val})\nelse\n' + f'D0.{dt} = S0.{dt}\nendif\nelsif')]: + pcode = pcode.replace(old, new) + lines = pcode.rstrip().split('\n') + for i in range(len(lines) - 1, -1, -1): + if lines[i].strip() == 'endif': + lines.insert(i, f'else\nD0.{dt} = S0.{dt}') + break + pcode = '\n'.join(lines) + f';\nif isDENORM(S1.{dt}) then\nD0.{dt} = NAN.{dt}\nendif' + return pcode.replace('VCC = 0x0LL', 'VCC.u64[laneId] = 0').replace('VCC = 0x1LL', 'VCC.u64[laneId] = 1') def _write_64bit(val: UOp, wfn, reg_or_addr, is_mem: bool, *args) -> list[UOp]: """Write a 64-bit value as two 32-bit writes. args passed to wfn after reg/addr and lo/hi value.""" @@ -407,6 +267,29 @@ def _collect_data_slices(assigns: list[tuple[str, UOp]], data_prefix: str, pcode # INSTRUCTION COMPILER - converts decoded instruction to UOp SINK # ═══════════════════════════════════════════════════════════════════════════════ +def _int_clamp(op_name: str, srcs: dict) -> UOp | None: + """Integer clamp for V_*ADD/SUB/MAD* ops: compute in wide arithmetic and saturate to the narrow range. None if not applicable. + NOTE: MUL_LO ops don't saturate - they always return the low bits.""" + s0, s1, s2 = srcs.get('S0'), srcs.get('S1'), srcs.get('S2') + if not isinstance(s0, UOp) or not isinstance(s1, UOp): return None + is_signed, is_16bit = '_I' in op_name and '_U' not in op_name, '16' in op_name + if any(p in op_name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')): + if is_16bit and is_signed: return None # skip 16-bit signed ops due to codegen issues + narrow_dt = dtypes.uint16 if is_16bit else (dtypes.int32 if is_signed else dtypes.uint32) + wide_dt = dtypes.int32 if is_16bit else dtypes.int64 + narrow_max, narrow_min = (0xFFFF, 0) if is_16bit else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0)) + def to_wide(x: UOp) -> UOp: return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt) + full = (to_wide(s0) * to_wide(s1) + to_wide(s2)) if 'MAD' in op_name and isinstance(s2, UOp) else \ + (to_wide(s1) - to_wide(s0)) if 'SUBREV' in op_name else \ + (to_wide(s0) - to_wide(s1)) if 'SUB' in op_name else (to_wide(s0) + to_wide(s1)) + return full.clamp(narrow_min, narrow_max).cast(narrow_dt) + # V_SUB_U32 / V_ADD_U32 with clamp: unsigned saturate (SUB underflow->0, ADD overflow->0xFFFFFFFF) + if any(p in op_name for p in ('_SUB_U32', '_ADD_U32', '_SUB_U16', '_ADD_U16')): + a, b = (s1.cast(dtypes.uint32), s0.cast(dtypes.uint32)) if 'SUBREV' in op_name else (s0.cast(dtypes.uint32), s1.cast(dtypes.uint32)) + if 'SUB' in op_name: return (a < b).where(_c(0), a - b) # underflow -> 0 + return (a + b < a).where(_c(0xFFFFFFFF), a + b) # overflow -> MAX + return None + class _Ctx: """Context for instruction compilation - holds buffers and helpers.""" __slots__ = ('inst_size', 'dyn_fields', '_axis_id', 'wave_size', 'vgpr', 'accvgpr') @@ -472,6 +355,10 @@ def inst_field(self, field) -> UOp: hi_part = self.inst_word(dword_idx + 1) & UOp.const(hi_mask, dtypes.uint32) return lo_part | (hi_part << UOp.const(lo_bits, dtypes.uint32)) + def optional_field(self, inst: Inst, name: str) -> UOp | None: + """Extract a field that only exists on some instruction formats (e.g. 'literal', 'saddr'). None if absent.""" + return self.inst_field(getattr(type(inst), name)) if hasattr(type(inst), name) else None + def inst_field_signed(self, field) -> UOp: """Extract field and sign-extend based on field width.""" val = self.inst_field(field) @@ -627,8 +514,7 @@ def scalar_stores(self, assigns: list[tuple[str, UOp]], sdst_reg: UOp, sdst_size def compile_sop_pcode(self, op, srcs: dict[str, UOp | int], sdst_reg: UOp, sdst_size: int) -> UOp: """Compile a scalar instruction with dynamic destination register.""" pcode = get_pcode(op) - srcs.update({'VCC': self.rmask(_c(VCC_LO.offset)), 'EXEC': self.rexec(), 'SCC': self.rsgpr_dyn(_c(SCC.offset)), - '_wave_size': self.wave_size}) + srcs.update(self.base_srcs(self.rexec()), VCC=self.rmask(_c(VCC_LO.offset))) if 'D0' not in srcs: srcs['D0'] = self.rsgpr_dyn(sdst_reg) # D0 is current dest value for read-modify-write ops _, assigns = parse_pcode(pcode, srcs) return UOp.sink(*self.scalar_stores(assigns, sdst_reg, sdst_size), *self.inc_pc()) @@ -639,8 +525,7 @@ def compile_lane_pcode(self, op, inst) -> UOp: op_name = op.name if hasattr(op, 'name') else str(op) src0_off, vdst_off = self.inst_field(type(inst).src0), self.inst_field(type(inst).vdst) src0_reg = (src0_off >= _c(256)).where(src0_off - _c(256), _c(0)) # VGPR index or 0 - src1_off = self.inst_field(type(inst).src1) if hasattr(type(inst), 'src1') else None - src2_off = self.inst_field(type(inst).src2) if hasattr(type(inst), 'src2') else None + src1_off, src2_off = self.optional_field(inst, 'src1'), self.optional_field(inst, 'src2') src1_reg = (src1_off >= _c(256)).where(src1_off - _c(256), src1_off) if src1_off is not None else _c(0) src2_reg = (src2_off >= _c(256)).where(src2_off - _c(256), src2_off) if src2_off is not None else _c(0) exec_val = self.rexec() @@ -659,6 +544,16 @@ def compile_lane_pcode(self, op, inst) -> UOp: elif dest.startswith('VGPR['): stores.append(self.vgpr.index(val[0]).store(val[1].cast(dtypes.uint32))) return UOp.sink(*stores, *self.inc_pc()) + def base_srcs(self, exec_mask: UOp, lane: UOp | None = None) -> dict[str, UOp | int]: + """Pcode environment entries shared by all instructions: EXEC/SCC, rounding mode (emulator always rounds-nearest-even), + and CDNA SDWA byte/word select defaults (E32 encodings always use BYTE0/WORD0).""" + srcs: dict[str, UOp | int] = {'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), '_vgpr': self.vgpr, '_wave_size': self.wave_size, + 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), 'ROUND_NEAREST_EVEN': _c(0), + 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), + 'WORD0': _c(0), 'WORD1': _c(1)} + if lane is not None: srcs['laneId'] = lane + return srcs + def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: UOp, exec_mask: UOp, opsel_dst_hi: bool | UOp = False, sdst_reg: int | None = None, clmp: int = 0, src0_off: UOp | None = None) -> UOp: @@ -666,82 +561,36 @@ def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: pcode = get_pcode(op) vcc_reg = sdst_reg if sdst_reg is not None else VCC_LO.offset if 'VCC' not in srcs: srcs['VCC'] = self.rmask(_c(vcc_reg)) - srcs.update({'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane, 'VDST': vdst_reg, - 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), 'ROUND_NEAREST_EVEN': _c(0), '_vgpr': self.vgpr, '_wave_size': self.wave_size, - 'MAX_FLOAT_F32': UOp.const(3.4028234663852886e38, dtypes.float32), - # CDNA SDWA byte/word select constants (E32 always uses BYTE0/WORD0 defaults) - 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), - 'WORD0': _c(0), 'WORD1': _c(1)}) # rounding mode and SDWA constants + srcs.update(self.base_srcs(exec_mask, lane), VDST=vdst_reg, MAX_FLOAT_F32=UOp.const(3.4028234663852886e38, dtypes.float32)) _, assigns = parse_pcode(pcode, srcs) - # For integer ops with clamp, compute overflow using wide arithmetic - # NOTE: MUL_LO ops don't saturate - they always return the low bits - int_saturate = None - if clmp and any(p in op.name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')): - is_signed, is_16bit = '_I' in op.name and '_U' not in op.name, '16' in op.name - if not (is_16bit and is_signed): # Skip 16-bit signed ops due to codegen issues - s0, s1, s2 = srcs.get('S0'), srcs.get('S1'), srcs.get('S2') - if s0 is not None and s1 is not None: - narrow_dt = dtypes.uint16 if is_16bit else (dtypes.int32 if is_signed else dtypes.uint32) - wide_dt = dtypes.int32 if is_16bit else dtypes.int64 - narrow_max, narrow_min = (0xFFFF, 0) if is_16bit else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0)) - def to_wide(x): return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt) - is_sub, is_mad = 'SUB' in op.name, 'MAD' in op.name - full = (to_wide(s0) * to_wide(s1) + to_wide(s2)) if is_mad and s2 is not None else \ - (to_wide(s1) - to_wide(s0)) if is_sub and 'SUBREV' in op.name else \ - (to_wide(s0) - to_wide(s1)) if is_sub else (to_wide(s0) + to_wide(s1)) - int_saturate = full.clamp(narrow_min, narrow_max).cast(narrow_dt) - # V_SUB_U32 / V_ADD_U32 with clamp: unsigned saturate (SUB underflow->0, ADD overflow->0xFFFFFFFF) - if clmp and int_saturate is None and any(p in op.name for p in ('_SUB_U32', '_ADD_U32', '_SUB_U16', '_ADD_U16')): - s0, s1 = srcs.get('S0'), srcs.get('S1') - if s0 is not None and s1 is not None: - assert isinstance(s0, UOp) and isinstance(s1, UOp) - a, b = (s1.cast(dtypes.uint32), s0.cast(dtypes.uint32)) if 'SUBREV' in op.name else (s0.cast(dtypes.uint32), s1.cast(dtypes.uint32)) - if 'SUB' in op.name: - int_saturate = (a < b).where(_c(0), a - b) # underflow -> 0 - else: - raw_sum = a + b - int_saturate = (raw_sum < a).where(_c(0xFFFFFFFF), raw_sum) # overflow -> MAX - - raw_stores: list = [] + # For integer ops with clamp, pre-compute the saturated result; floats clamp to [0,1] at write time + int_saturate = _int_clamp(op.name, srcs) if clmp else None + + lane_stores, scalar_stores, slice_stores = [], [], [] vcc_val, exec_val = None, None for dest, val in assigns: - # VGPR bit-slice assignment: VGPR[lane][reg][hi:lo] = (vgpr_idx, rhs_val, hi, lo[, cond]) -> read-modify-write + # VGPR bit-slice assignment: VGPR[lane][reg][hi:lo] -> read-modify-write with optional condition if dest.startswith('VGPR[') and re.search(r'\[\d+:\d+\]', dest): - # VGPR bit-slice: (vgpr_idx, rhs_val, hi_bit, lo_bit) - hi/lo are UOp constants hi_bit, lo_bit = int(val[2].val), int(val[3].val) - width = hi_bit - lo_bit + 1 - old = self.vgpr.index(val[0]).load() - new_val = _set_bits(old, _val_to_bits(val[1]), width, lo_bit).cast(dtypes.uint32) - active = _lane_active(exec_mask, lane) - if len(val) > 4: active = active & _to_bool(val[4]) - raw_stores.append(('vgpr_direct', self.vgpr.index(val[0].valid(active)).store(new_val))) - continue - if 'D0' in dest and '[laneId]' in dest: - raw_stores.extend([('vcc', s) for s in self.wmask_lane_bit(_c(VCC_LO.offset), lane, val, exec_mask)]) + new_val = _set_bits(self.vgpr.index(val[0]).load(), _val_to_bits(val[1]), hi_bit - lo_bit + 1, lo_bit).cast(dtypes.uint32) + active = _lane_active(exec_mask, lane) & _to_bool(val[4]) if len(val) > 4 else _lane_active(exec_mask, lane) + lane_stores.append(self.vgpr.index(val[0].valid(active)).store(new_val)) + elif 'D0' in dest and '[laneId]' in dest: continue # per-lane mask bits are written via VCC/EXEC assigns instead elif dest.startswith('D0'): - dest_suffix = re.match(r'D0\.(\w+)', dest) - if dest_suffix is not None: + if (dest_suffix := re.match(r'D0\.(\w+)', dest)) is not None: target_dt = {'u16': dtypes.uint16, 'i16': dtypes.int16, 'f16': dtypes.half}.get(dest_suffix.group(1)) if target_dt is not None and val.dtype != target_dt: val = val.cast(target_dt) - if (slice_match := re.match(r'D0\[(\d+)\s*:\s*(\d+)\]', dest)): - d0_hi_bit, d0_lo_bit = int(slice_match.group(1)), int(slice_match.group(2)) - if d0_hi_bit != 31 or d0_lo_bit != 0: - d0_width, slice_mask = d0_hi_bit - d0_lo_bit + 1, (1 << (d0_hi_bit - d0_lo_bit + 1)) - 1 - val_bits = val.bitcast(dtypes.uint16).cast(dtypes.uint32) if val.dtype == dtypes.half else \ - val.cast(dtypes.uint32) if val.dtype in (dtypes.uint16, dtypes.int16) else \ - val.cast(dtypes.uint32) & UOp.const(slice_mask, dtypes.uint32) - raw_stores.append(('vgpr_slice', (d0_lo_bit, d0_width, val_bits))) - continue - # For integer ops with clamp, use pre-computed saturated value; for floats, clamp to [0,1] + if (slice_match := re.match(r'D0\[(\d+)\s*:\s*(\d+)\]', dest)) and (int(slice_match.group(1)), int(slice_match.group(2))) != (31, 0): + slice_stores.append((int(slice_match.group(2)), int(slice_match.group(1)) - int(slice_match.group(2)) + 1, _val_to_bits(val))) + continue if int_saturate is not None: val = int_saturate elif clmp and val.dtype in (dtypes.float32, dtypes.half, dtypes.float64): clamped = val.maximum(UOp.const(0.0, val.dtype)).minimum(UOp.const(1.0, val.dtype)) val = _FUNCS['isNAN'](val).where(UOp.const(0.0, val.dtype), clamped) if val.dtype in (dtypes.uint64, dtypes.int64, dtypes.float64): lo, hi = _split64(val) - raw_stores.extend([('vgpr', self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask)), - ('vgpr', self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask))]) + lane_stores.extend([self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask), self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask)]) elif val.dtype in (dtypes.half, dtypes.uint16, dtypes.int16): result, old_val = _val_to_u32(val), self.rvgpr_dyn(vdst_reg, lane) hi_result = (old_val & UOp.const(0xFFFF, dtypes.uint32)) | (result << UOp.const(16, dtypes.uint32)) @@ -749,34 +598,27 @@ def to_wide(x): return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.it lo_result = (result & UOp.const(0xFFFF, dtypes.uint32)) if self.wave_size == 64 else \ (old_val & UOp.const(0xFFFF0000, dtypes.uint32)) | (result & UOp.const(0xFFFF, dtypes.uint32)) result = opsel_dst_hi.where(hi_result, lo_result) if isinstance(opsel_dst_hi, UOp) else hi_result if opsel_dst_hi else lo_result - raw_stores.append(('vgpr', self.wvgpr_dyn(vdst_reg, lane, result, exec_mask))) - else: raw_stores.append(('vgpr', self.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask))) + lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) + else: lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask)) elif dest.startswith('S0') and src0_off is not None: # Write back to src0 VGPR (e.g. v_swap_b32). src0_off is raw encoding (256+ = VGPR) - src0_vgpr = src0_off - _c(256) - raw_stores.append(('vgpr_s0', self.wvgpr_dyn(src0_vgpr, lane, _val_to_u32(val), exec_mask))) + lane_stores.append(self.wvgpr_dyn(src0_off - _c(256), lane, _val_to_u32(val), exec_mask)) elif dest.startswith('VCC'): vcc_val = val elif dest.startswith('EXEC'): exec_val = val - elif dest.startswith('SCC'): raw_stores.append(('scc', self.wsgpr_dyn(_c(SCC.offset), _to_u32(val)))) + elif dest.startswith('SCC'): scalar_stores.append(self.wsgpr_dyn(_c(SCC.offset), _to_u32(val))) - lane_stores = [s for t, s in raw_stores if t in ('vgpr', 'vgpr_s0', 'vgpr_direct')] - stores, scalar_stores = [], [s for t, s in raw_stores if t == 'scc'] - slice_stores = [s for t, s in raw_stores if t == 'vgpr_slice'] - if slice_stores: - result = self.rvgpr_dyn(vdst_reg, lane) - for lo_bit, width, val_bits in slice_stores: - mask = UOp.const(((1 << width) - 1) << lo_bit, dtypes.uint32) - result = (result & (mask ^ UOp.const(0xFFFFFFFF, dtypes.uint32))) | (val_bits << UOp.const(lo_bit, dtypes.uint32)) - lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) # VCC/EXEC mask writes must be computed BEFORE VGPR stores to avoid reading modified VGPRs. # When vdst overlaps with src operands (e.g. v_add_co_u32 v[0], vcc, s[8], v[0]), the carry # computation reads the original source values only if its range loop runs before the VGPR write loop. - mask_stores: list[UOp] = [] + stores: list[UOp] = [] for mask_val, reg in [(vcc_val, vcc_reg), (exec_val, EXEC_LO.offset)]: if mask_val is None: continue - def get_bit(l, v=mask_val): return (_to_u32(v.substitute({lane: l})) & _c(1)).cast(dtypes.uint32) - mask_stores.extend(self.wmask(_c(reg), self.unroll_lanes(get_bit, exec_mask, apply_exec=False))) - stores.extend(mask_stores) + stores.extend(self.wmask(_c(reg), self.unroll_lanes(lambda l, v=mask_val: (_to_u32(v.substitute({lane: l})) & _c(1)).cast(dtypes.uint32), + exec_mask, apply_exec=False))) + if slice_stores: # merge D0[hi:lo] slices into one read-modify-write of the destination VGPR + result = self.rvgpr_dyn(vdst_reg, lane) + for lo_bit, width, val_bits in slice_stores: result = _set_bits(result, val_bits, width, lo_bit) + lane_stores.append(self.wvgpr_dyn(vdst_reg, lane, result, exec_mask)) if lane_stores: stores.append(UOp.sink(*lane_stores).end(lane)) stores.extend(scalar_stores) return UOp.sink(*stores, *self.inc_pc()) @@ -791,9 +633,7 @@ def _compile_sopp(inst: ir3.SOPP | ir4.SOPP, ctx: _Ctx) -> UOp: return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32)), ctx.wsgpr_dyn(_c(PC_HI_IDX), UOp.const(0xFFFFFFFF, dtypes.uint32))) # S_BARRIER: advance PC past the barrier instruction. The execution loop detects barriers before executing and handles synchronization. - barrier_ops = {ir3.SOPPOp.S_BARRIER, irc.SOPPOp.S_BARRIER} - if hasattr(ir4.SOPPOp, 'S_BARRIER_WAIT'): barrier_ops.add(ir4.SOPPOp.S_BARRIER_WAIT) - if inst.op in barrier_ops: return UOp.sink(*ctx.inc_pc()) + if inst.op in _BARRIER_OPS: return UOp.sink(*ctx.inc_pc()) # S_NOP and S_WAITCNT are no-ops in emulator (no pipeline/cache to wait on) if inst.op in (ir3.SOPPOp.S_NOP, ir4.SOPPOp.S_NOP, irc.SOPPOp.S_NOP, irc.SOPPOp.S_WAITCNT): return UOp.sink(*ctx.inc_pc()) # NOTE: we ignore SOPPs without PCODE @@ -843,7 +683,7 @@ def _compile_smem(inst: ir3.SMEM | ir4.SMEM, ctx: _Ctx) -> UOp: def _compile_sop(inst: ir3.SOP1|ir3.SOP2|ir3.SOPC|ir3.SOPK|ir4.SOP1|ir4.SOP2|ir4.SOPC|ir4.SOPK|irc.SOP1|irc.SOP2|irc.SOPC|irc.SOPK, ctx: _Ctx) -> UOp: bits = inst.canonical_op_bits - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') if isinstance(inst, (ir3.SOPK, ir4.SOPK, irc.SOPK)): sdst_off = ctx.inst_field(type(inst).sdst) @@ -961,11 +801,11 @@ def _dpp16_ctrl(lane: UOp, dpp: int, row_mask: int, bank_mask: int, wave_size: i def _load_dpp16_src0(ctx: _Ctx, inst, lane: UOp, fallback: UOp) -> UOp: """Load a DPP16-swizzled src0 value from vsrc0.""" - src_lane, enabled, valid = _dpp16_ctrl(lane, getattr(inst, 'dpp', 0) or 0, getattr(inst, 'row_mask', 0xf) or 0xf, - getattr(inst, 'bank_mask', 0xf) or 0xf, ctx.wave_size) + src_lane, enabled, valid = _dpp16_ctrl(lane, _iattr(inst, 'dpp'), _iattr(inst, 'row_mask', 0xf), + _iattr(inst, 'bank_mask', 0xf), ctx.wave_size) safe_src_lane = (enabled & valid).where(src_lane, _c(0, dtypes.int)) swizzled = ctx.rvgpr_dyn(ctx.inst_field(type(inst).vsrc0), safe_src_lane) - invalid = UOp.const(0, fallback.dtype) if getattr(inst, 'bc', 0) else fallback + invalid = UOp.const(0, fallback.dtype) if _iattr(inst, 'bc') else fallback return enabled.where(valid.where(swizzled, invalid), fallback) def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc.VOPC_SDWA_SDST, ctx: _Ctx) -> UOp: @@ -974,7 +814,7 @@ def _compile_sdwa(inst: irc.VOP1_SDWA | irc.VOP2_SDWA | irc.VOP2_SDWA_SDST | irc exec_mask = ctx.rexec() # sd=1 means use sdst register, sd=0 means use VCC (for VOPC_SDWA_SDST and VOP2_SDWA_SDST) if isinstance(inst, (irc.VOP2_SDWA_SDST, irc.VOPC_SDWA_SDST)): - sdst_off = _c(inst.sdst.offset) if getattr(inst, 'sd', False) else _c(VCC_LO.offset) + sdst_off = _c(inst.sdst.offset) if _iattr(inst, 'sd') else _c(VCC_LO.offset) else: sdst_off = _c(VCC_LO.offset) # Read SDWA fields (these are dynamic but shared across lanes) @@ -1019,11 +859,7 @@ def get_cmp_bit(lane) -> UOp: if has_dst_sel: dst_sel = ctx.inst_field(type(inst).dst_sel) # type: ignore[union-attr] dst_unused = ctx.inst_field(type(inst).dst_unused) # type: ignore[union-attr] - srcs.update({'VCC': ctx.rmask(_c(VCC_LO.offset)), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), - 'laneId': lane, 'VDST': vdst_reg, 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0), - 'ROUND_NEAREST_EVEN': _c(0), '_vgpr': ctx.vgpr, '_wave_size': ctx.wave_size, - 'SDWA_SRC0_SEL': _c(0), 'BYTE0': _c(0), 'BYTE1': _c(1), 'BYTE2': _c(2), 'BYTE3': _c(3), - 'WORD0': _c(0), 'WORD1': _c(1)}) + srcs.update(ctx.base_srcs(exec_mask, lane), VCC=ctx.rmask(_c(VCC_LO.offset)), VDST=vdst_reg) _, assigns = parse_pcode(pcode, srcs) stores = [] vcc_val = None @@ -1049,6 +885,23 @@ def get_cmp_bit(lane) -> UOp: return UOp.sink(UOp.sink(*stores).end(lane), *ctx.inc_pc()) return UOp.sink(*ctx.inc_pc()) +def _load_vsrc0(ctx: _Ctx, inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | ir3.VOP2_DPP16 | + ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP1_DPP16 | ir4.VOP2 | ir4.VOP2_DPP16 | + irc.VOP1 | irc.VOP1_DPP16 | irc.VOP2 | irc.VOP2_DPP16, + lane: UOp, bits: dict, literal: UOp | None, is_f64: bool, is_float: bool, fallback: UOp) -> tuple[UOp, UOp | None]: + """Load VOP src0: DPP16 swizzle (with abs/neg mods for floats), 16-bit VGPR hi-half (src0 >= 384), or plain operand. + Returns (value, src0_offset) - offset is None for DPP16.""" + if hasattr(type(inst), 'dpp') and hasattr(type(inst), 'vsrc0'): + s0 = _load_dpp16_src0(ctx, inst, lane, fallback) + if is_float: s0 = _apply_src_mods(s0, 0, 1 if _iattr(inst, 'src0_abs') else 0, 1 if _iattr(inst, 'src0_neg') else 0, bits['s0']) + return s0, None + src0_off = ctx.inst_field(type(inst).src0) + s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) + if bits['s0'] == 16: # src0 >= 384 means hi half of v[src0-384]. Guard index against OOB access. + src0_hi = src0_off >= _c(384) + s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_hi.where(src0_off - _c(384), _c(0)), lane)), s0) + return s0, src0_off + def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | ir3.VOP2_DPP16 | ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP1_DPP16 | ir4.VOP2 | ir4.VOP2_DPP16 | irc.VOP1 | irc.VOP1_DPP16 | irc.VOP2 | irc.VOP2_DPP16, ctx: _Ctx) -> UOp: @@ -1062,7 +915,7 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | val = ctx.raccvgpr_dyn(acc_src0_off - _c(256), lane) return UOp.sink(ctx.waccvgpr_dyn(vdst_reg, lane, val, exec_mask).end(lane), *ctx.inc_pc()) lane, exec_mask, bits = ctx.range(), ctx.rexec(), inst.canonical_op_bits - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') is_f64 = 'F64' in op_name and 'B64' not in op_name is_float = any(x in op_name for x in ('F16', 'F32', 'F64')) is_dpp16 = hasattr(type(inst), 'dpp') and hasattr(type(inst), 'vsrc0') @@ -1070,50 +923,25 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP1_DPP16 | ir3.VOP2 | write_hi_half = bits['d'] == 16 and (vdst_reg >= _c(128)) if isinstance(write_hi_half, UOp): vdst_reg = write_hi_half.where(vdst_reg - _c(128), vdst_reg) elif write_hi_half: vdst_reg -= 128 - src0_off: UOp | None = None if isinstance(inst, (ir3.VOP1, ir4.VOP1, irc.VOP1)): - # Handle VOP1 hi-half source operand (src0 >= v[128] for 16-bit ops) d0 = _cond_hi16(write_hi_half, ctx.rvgpr_dyn(vdst_reg, lane)) - if is_dpp16: - s0 = _load_dpp16_src0(ctx, inst, lane, d0) - else: - src0_off = ctx.inst_field(type(inst).src0) - s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) - if bits['s0'] == 16 and not is_dpp16: - src0_hi = src0_off >= _c(384) - # Only compute hi-half when src0_off >= 384, use guarded index to prevent OOB access - src0_reg = src0_hi.where(src0_off - _c(384), _c(0)) - s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0) - if is_dpp16 and is_float: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - srcs:dict[str, UOp | int] = {'S0': s0, 'D0': d0} + s0, src0_off = _load_vsrc0(ctx, inst, lane, bits, literal, is_f64, is_float, d0) + srcs: dict[str, UOp | int] = {'S0': s0, 'D0': d0} else: vsrc1_reg = ctx.inst_field(type(inst).vsrc1) vsrc1_hi = bits['s0'] == 16 and (vsrc1_reg >= _c(128)) - vsrc1_actual = _cond(vsrc1_hi, vsrc1_reg - _c(128), vsrc1_reg) if bits['s1'] == 64: s1 = _u64(ctx.rvgpr_dyn(vsrc1_reg, lane), ctx.rvgpr_dyn(vsrc1_reg + _c(1), lane)) d0 = _u64(ctx.rvgpr_dyn(vdst_reg, lane), ctx.rvgpr_dyn(vdst_reg + _c(1), lane)) else: - s1 = _cond_hi16(vsrc1_hi, ctx.rvgpr_dyn(vsrc1_actual, lane)) + s1 = _cond_hi16(vsrc1_hi, ctx.rvgpr_dyn(_cond(vsrc1_hi, vsrc1_reg - _c(128), vsrc1_reg), lane)) d0 = _cond_hi16(write_hi_half, ctx.rvgpr_dyn(vdst_reg, lane)) # FMAC/FMAMK hi-half dest needs hi-half accumulator - # Handle VOP2 hi-half src0 operand (src0 >= v[128] for 16-bit ops) - if is_dpp16: - s0 = _load_dpp16_src0(ctx, inst, lane, d0) - else: - src0_off = ctx.inst_field(type(inst).src0) - s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal, is_f64) - if bits['s0'] == 16 and not is_dpp16: - src0_hi = src0_off >= _c(384) - # Only compute hi-half when src0_off >= 384, use guarded index to prevent OOB access - src0_reg = src0_hi.where(src0_off - _c(384), _c(0)) - s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0) + s0, src0_off = _load_vsrc0(ctx, inst, lane, bits, literal, is_f64, is_float, d0) if is_dpp16 and is_float: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - s1 = _apply_src_mods(s1, 0, 1 if getattr(inst, 'src1_abs', 0) else 0, 1 if getattr(inst, 'src1_neg', 0) else 0, bits['s1']) + s1 = _apply_src_mods(s1, 0, 1 if _iattr(inst, 'src1_abs') else 0, 1 if _iattr(inst, 'src1_neg') else 0, bits['s1']) srcs = {'S0': s0, 'S1': s1, 'D0': d0} # FMAAK_(DTYPE)_E32 series - if 'V_FMAA' in _op_name(inst) or 'V_FMAM' in _op_name(inst): + if 'V_FMAA' in op_name or 'V_FMAM' in op_name: assert literal is not None srcs['SIMM32'] = literal return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=write_hi_half, src0_off=src0_off) @@ -1140,7 +968,7 @@ def _compile_vopc(inst: ir3.VOPC|ir3.VOPC_DPP16|ir3.VOP3|ir4.VOPC|ir4.VOPC_DPP16 src1_off = ctx.inst_field(type(inst).src1) # type: ignore[union-attr] dst_off = ctx.inst_field(type(inst).vdst) # type: ignore[union-attr] vsrc1_hi = False - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') is_float, is_f64, pcode = any(x in op_name for x in ('_F32', '_F64', '_F16')), '_F64' in op_name, get_pcode(inst.op) def get_cmp_bit(lane) -> UOp: @@ -1154,8 +982,8 @@ def get_cmp_bit(lane) -> UOp: if bits['s0'] == 16 and opsel: s0, s1 = _apply_opsel(s0, 0, opsel), _apply_opsel(s1, 1, opsel) if is_float: if is_dpp16: - s0 = _apply_src_mods(s0, 0, 1 if getattr(inst, 'src0_abs', 0) else 0, 1 if getattr(inst, 'src0_neg', 0) else 0, bits['s0']) - s1 = _apply_src_mods(s1, 0, 1 if getattr(inst, 'src1_abs', 0) else 0, 1 if getattr(inst, 'src1_neg', 0) else 0, bits['s1']) + s0 = _apply_src_mods(s0, 0, 1 if _iattr(inst, 'src0_abs') else 0, 1 if _iattr(inst, 'src0_neg') else 0, bits['s0']) + s1 = _apply_src_mods(s1, 0, 1 if _iattr(inst, 'src1_abs') else 0, 1 if _iattr(inst, 'src1_neg') else 0, bits['s1']) s0 = _apply_src_mods(s0, 0, abs_bits, neg_bits, bits['s0']) s1 = _apply_src_mods(s1, 1, abs_bits, neg_bits, bits['s1']) for dest, val in parse_pcode(pcode, {'S0': s0, 'S1': s1, 'laneId': lc, 'D0': UOp.const(0, dtypes.uint64)})[1]: @@ -1184,7 +1012,7 @@ def _compile_bitop3(inst, ctx: _Ctx, exec_mask: UOp, bits: dict, op_name: str) - src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, bits['s1'], None, 's1' in ops and ops['s1'][0] == Fmt.FMT_NUM_F64) src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, bits['s2'], None, 's2' in ops and ops['s2'][0] == Fmt.FMT_NUM_F64) # Truth table: TTBL = { omod[1:0], abs[2:0], neg[2:0] } = 8-bit LUT - ttbl = ((getattr(inst, 'omod', 0) or 0) << 6) | ((getattr(inst, 'abs', 0) or 0) << 3) | (getattr(inst, 'neg', 0) or 0) + ttbl = ((_iattr(inst, 'omod')) << 6) | ((_iattr(inst, 'abs')) << 3) | (_iattr(inst, 'neg')) is_16 = 'B16' in op_name dt, mask = (dtypes.uint16, 0xFFFF) if is_16 else (dtypes.uint32, 0xFFFFFFFF) s0, s1, s2 = src0.cast(dt), src1.cast(dt), src2.cast(dt) @@ -1198,7 +1026,7 @@ def bnot(v): return v ^ UOp.const(mask, dt) def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: exec_mask = ctx.rexec() bits = inst.canonical_op_bits - opsel, op_name = getattr(inst, 'opsel', 0) or 0, _op_name(inst) + opsel, op_name = _iattr(inst, 'opsel'), _op_name(inst) # Lane operations if op_name in ('V_READLANE_B32', 'V_READFIRSTLANE_B32', 'V_READFIRSTLANE_B32_E64', 'V_WRITELANE_B32'): @@ -1210,7 +1038,7 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # VOP3 VOPC (v_cmp_*_e64) - delegate to unified VOPC handler if 'V_CMP' in op_name or 'V_CMPX' in op_name: - return _compile_vopc(inst, ctx, opsel=opsel, abs_bits=getattr(inst, 'abs', 0) or 0, neg_bits=getattr(inst, 'neg', 0) or 0) + return _compile_vopc(inst, ctx, opsel=opsel, abs_bits=_iattr(inst, 'abs'), neg_bits=_iattr(inst, 'neg')) # BITOP3: abs/neg/omod encode truth table, not source modifiers if 'BITOP3' in op_name: @@ -1218,14 +1046,13 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # VOP3 specific fields vdst_reg = ctx.inst_field(type(inst).vdst) - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] - abs_bits, neg_bits = getattr(inst, 'abs', 0) or 0, getattr(inst, 'neg', 0) or 0 + literal = ctx.optional_field(inst, 'literal') + abs_bits, neg_bits = _iattr(inst, 'abs'), _iattr(inst, 'neg') # VOP3_SDST: v_s_* instructions goes to SGPR if 'V_S_' in op_name: src0 = _apply_src_mods(ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), _c(0, dtypes.int), bits['s0'], literal), 0, abs_bits, neg_bits, bits['s0']) - srcs = {'S0': src0, 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': _c(0, dtypes.int), - 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0)} + srcs = {**ctx.base_srcs(exec_mask, _c(0, dtypes.int)), 'S0': src0} _, assigns = parse_pcode(get_pcode(inst.op), srcs) stores = [ctx.wsgpr_dyn(vdst_reg, _val_to_u32(val)) for dest, val in assigns if dest.startswith('D0')] return UOp.sink(*stores, *ctx.inc_pc()) @@ -1248,7 +1075,7 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3 | irc.VOP3, ctx: _Ctx) -> UOp: # FMAC instructions need D0 (accumulator) from destination register if 'FMAC' in op_name: srcs['D0'] = ctx.rvgpr_dyn(vdst_reg, lane) opsel_dst_hi = bool(opsel & 0b1000) and bits['d'] == 16 - return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=opsel_dst_hi, clmp=getattr(inst, 'clmp', 0)) + return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=opsel_dst_hi, clmp=_iattr(inst, 'clmp')) def _compile_vinterp(inst: ir3.VINTERP | ir4.VINTERP, ctx: _Ctx) -> UOp: lane, exec_mask = ctx.range(), ctx.rexec() @@ -1270,13 +1097,13 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD | irc.VOP3SD, ctx: _Ctx) -> UO # Read operands dynamically from instruction encoding vdst_reg, sdst_off = ctx.inst_field(type(inst).vdst), ctx.inst_field(type(inst).sdst) src0_off, src1_off, src2_off = ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2) - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') has_carry_in = 's2' in ops and ops['s2'][2] == OpType.OPR_SREG vcc_in_off = src2_off if has_carry_in else sdst_off def load_srcs(lane_uop): - ret = {'VCC': ctx.rmask(vcc_in_off), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane_uop} + ret = {**ctx.base_srcs(exec_mask, lane_uop), 'VCC': ctx.rmask(vcc_in_off)} ret['S0'] = ctx.rsrc_dyn(src0_off, lane_uop, bits['s0'], literal, ops['s0'][0] == Fmt.FMT_NUM_F64) ret['S1'] = ctx.rsrc_dyn(src1_off, lane_uop, bits['s1'], literal, ops['s1'][0] == Fmt.FMT_NUM_F64) if 's2' in ops: ret['S2'] = ctx.rsrc_dyn(src2_off, lane_uop, bits['s2'], literal, ops['s2'][0] == Fmt.FMT_NUM_F64) @@ -1287,7 +1114,7 @@ def load_srcs(lane_uop): _, assigns = parse_pcode(pcode, srcs) has_per_lane_vcc = any('[laneId]' in dest for dest, _ in assigns if dest.startswith('VCC') or dest.startswith('D0.u64')) - clmp = getattr(inst, 'clmp', 0) + clmp = _iattr(inst, 'clmp') if has_per_lane_vcc: # VCC computation: RANGE+REDUCE gets axis ID first (lower ID = runs first) # This ensures VCC reads source values BEFORE VGPR stores modify them @@ -1322,195 +1149,107 @@ def get_vcc_bit(lane_uop) -> UOp: else: return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, sdst_reg=inst.sdst.offset) +# MFMA shape -> (lanes per group, lane groups, output regs per lane) +_MFMA_SHAPES = {(16, 16): (16, 4, 4), (32, 32): (32, 2, 16), (4, 4): (4, 16, 4)} + def _compile_mfma(inst: irc.VOP3P|irc.VOP3PX2, ctx: _Ctx) -> UOp: - """CDNA MFMA matrix multiply-accumulate emulation. - - Uses local temp arrays to cache inputs, avoiding aliasing issues when vdst overlaps src0/src1. - Phase 1: Read all input f32 values from VGPRs into temp arrays (range loop over 64 lanes). - Phase 2: Compute 256 output values using temp arrays and write to VGPRs (range loop over 64 lanes) - - Register layout (wave64): - - 16x16: 4 groups of 16 lanes. Each lane in group holds k_per_grp elements. 4 output ACCVGPRs per lane. - - 32x32: 2 groups of 32 lanes. lanes%16 = M/N index within block, lanes//16 selects block. 16 output ACCVGPRs per lane. - - 4x4: 16 groups of 4 lanes. 4 output ACCVGPRs per lane. - """ - import re as _re - op_name = _op_name(inst) - exec_mask = ctx.rexec() + """CDNA MFMA matrix multiply-accumulate. Inputs are unpacked/converted into a local temp array (uint32 bit patterns, + avoiding aliasing when vdst overlaps src0/src1), then phase 2 computes the dot products and writes outputs. + wave64 register layout: 16x16 = 4 groups of 16 lanes (K split across groups), 32x32 = 2 groups of 32 lanes, + 4x4 = 16 independent groups of 4 lanes (K not split).""" + op_name, exec_mask = _op_name(inst), ctx.rexec() vdst_reg = ctx.inst_field(type(inst).vdst) - src0_off = ctx.inst_field(type(inst).src0) - src1_off = ctx.inst_field(type(inst).src1) - src0_r = src0_off - _c(256) # VGPR-relative index (only valid when src is VGPR) - src1_r = src1_off - _c(256) - src2_off = ctx.inst_field(type(inst).src2) - use_acc = bool(getattr(inst, 'acc_cd', 0)) - # Check if sources are VGPRs (offset >= 256) vs inline constants/SGPRs - src0_is_vgpr = src0_off >= _c(256) - src1_is_vgpr = src1_off >= _c(256) + src0_off, src1_off, src2_off = ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2) + use_acc = bool(_iattr(inst, 'acc_cd')) scaled = isinstance(inst, irc.VOP3PX2) if scaled: assert isinstance(inst, irc.VOP3PX2) # F8F6F4 input formats: 0=FP8(E4M3), 1=BF8(E5M2). FP6/FP4 (2-4) not emulated. - src0_fmt, src1_fmt = int(inst.cbsz), int(inst.blgp) - if src0_fmt > 1 or src1_fmt > 1: raise RuntimeError(f"unsupported scaled MFMA formats cbsz={src0_fmt} blgp={src1_fmt}") + if int(inst.cbsz) > 1 or int(inst.blgp) > 1: raise RuntimeError(f"unsupported scaled MFMA formats cbsz={inst.cbsz} blgp={inst.blgp}") # scale_src0/scale_src1 are source operands pointing at 32-bit registers holding 4 packed E8M0 scale exponents. # The 2-bit opsel/opsel_hi select which byte applies to A/B for this instruction. - scale0_off = ctx.inst_field(type(inst).scale_src0) - scale1_off = ctx.inst_field(type(inst).scale_src1) - sel0, sel1 = int(inst.opsel) & 3, int(inst.opsel_hi) & 3 - def _scale_exp(off: UOp, sel: int, lane: UOp) -> UOp: - sv = ctx.rsrc_dyn(off, lane, 32) - byte = (sv >> UOp.const(sel * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) + scale0 = ctx.inst_field(type(inst).scale_src0), _iattr(inst, 'opsel') & 3 + scale1 = ctx.inst_field(type(inst).scale_src1), _iattr(inst, 'opsel_hi') & 3 + def _scale_exp(off_sel: tuple[UOp, int], lane: UOp) -> UOp: + byte = (ctx.rsrc_dyn(off_sel[0], lane, 32) >> UOp.const(off_sel[1] * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) return byte.cast(dtypes.int32) - UOp.const(127, dtypes.int32) - # combined A*B scale for this lane: 2^(ea-127) * 2^(eb-127) - def scale_factor(lane: UOp) -> UOp: - return UOp.exp2((_scale_exp(scale0_off, sel0, lane) + _scale_exp(scale1_off, sel1, lane)).cast(dtypes.float32)) + def scale_factor(lane: UOp) -> UOp: # combined A*B scale for this lane: 2^(ea-127) * 2^(eb-127) + return UOp.exp2((_scale_exp(scale0, lane) + _scale_exp(scale1, lane)).cast(dtypes.float32)) - m = _re.search(r'(\d+)X(\d+)X(\d+)', op_name) - if m is None: raise ValueError(f"could not parse MFMA dimensions from {op_name}") + if (m := re.search(r'(\d+)X(\d+)X(\d+)', op_name)) is None: raise ValueError(f"could not parse MFMA dimensions from {op_name}") M, N, K = int(m.group(1)), int(m.group(2)), int(m.group(3)) + if (M, N) not in _MFMA_SHAPES: raise RuntimeError(f"unsupported MFMA shape {M}x{N}x{K}") + grp_size, n_grps, out_per_lane = _MFMA_SHAPES[(M, N)] # lanes per group, lane groups, output regs per lane - is_bf16 = 'BF16' in op_name - is_fp8 = 'FP8' in op_name or 'F8' in op_name - is_i8 = 'I8' in op_name # Source type is the LAST type in the name: V_MFMA_F32_16X16X32_**F16** -> source is F16, not F32 - src_type = op_name.rsplit('_', 1)[-1] # e.g. "F16", "BF16", "F32", "I8" - is_f32_src = src_type == 'F32' + src_type = op_name.rsplit('_', 1)[-1] + is_bf16, is_fp8 = 'BF16' in op_name, 'FP8' in op_name or 'F8' in op_name + is_i8, is_f32_src = 'I8' in op_name, src_type == 'F32' is_int_out = 'I32' in op_name.split('_')[2] # V_MFMA_I32_... - - # Determine elements per VGPR and conversion function - if is_i8: vpg = 4 - elif is_f32_src: vpg = 1 - elif is_fp8: vpg = 4 - else: vpg = 2 - - # For 16x16: grp_size=16, n_grps=4, out_per_lane=4 - # For 32x32: grp_size=32, n_grps=2, out_per_lane=16 - # For 4x4: grp_size=4, n_grps=16, out_per_lane=4 - if M == 16 and N == 16: - grp_size, n_grps, out_per_lane = 16, 4, 4 - elif M == 32 and N == 32: - grp_size, n_grps, out_per_lane = 32, 2, 16 - elif M == 4 and N == 4: - grp_size, n_grps, out_per_lane = 4, 16, 4 - else: - raise RuntimeError(f"unsupported MFMA shape {M}x{N}x{K}") - - # For 4x4: each group independently computes a 4x4 block. K is NOT split across groups. - # For 16x16/32x32: K IS split across groups (each group has K/n_grps elements). - k_per_grp = K if M == 4 else K // n_grps - # Temp array size: for 4x4, store all 16 independent blocks; for others, store shared MxK/NxK - n_a_elems = n_grps * M * K if M == 4 else M * K - n_b_elems = n_grps * N * K if M == 4 else N * K - - # src2 can be VGPR (>=256) or inline constant/SGPR (<256) - src2_is_vgpr = src2_off >= _c(256) - src2_r = src2_off - _c(256) - if is_int_out: - acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)).cast(dtypes.int32) - else: - acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)).bitcast(dtypes.float32) - - # Phase 1: Read all A and B values from VGPRs into temp arrays. - # Layout: tmp[0..n_a_elems-1] = A[m][k], tmp[n_a_elems..n_a_elems+n_b_elems-1] = B[n][k] - # Within each group of lanes, lane%grp_sub gives M/N index, lane//grp_sub gives sub-block - grp_sub = min(M, 16) # lanes within group mapped to M/N dimension - b_off = UOp.const(n_a_elems, dtypes.int) + vpg = 4 if is_i8 else 1 if is_f32_src else 4 if is_fp8 else 2 # elements per VGPR acc_dt = dtypes.int32 if is_int_out else dtypes.float32 - # Use uint32 temp array to prevent optimizer from eliminating f16→f32 bitcast chains. - # The optimizer folds bitcast(uint32→float32) stores to float32 arrays, losing the conversion. - tmp = UOp.placeholder((n_a_elems + n_b_elems,), dtypes.uint32, slot=0, addrspace=AddrSpace.LOCAL) # Per-operand fp8 format ("fp8"=E4M3, "bf8"=E5M2) for A and B if 'F8F6F4' in op_name: assert isinstance(inst, (irc.VOP3P_MFMA, irc.VOP3PX2)) - _fmts = {0: "fp8", 1: "bf8"} - a_fmt, b_fmt = _fmts.get(int(inst.cbsz), "fp8"), _fmts.get(int(inst.blgp), "fp8") - elif is_fp8: - # A/B formats from name suffix, e.g. V_MFMA_F32_16X16X32_BF8_FP8 - suffixes = op_name.rsplit('_', 2)[-2:] - a_fmt, b_fmt = ("bf8" if sfx == "BF8" else "fp8" for sfx in suffixes) + a_fmt, b_fmt = {0: "fp8", 1: "bf8"}.get(int(inst.cbsz), "fp8"), {0: "fp8", 1: "bf8"}.get(int(inst.blgp), "fp8") + elif is_fp8: # A/B formats from name suffix, e.g. V_MFMA_F32_16X16X32_BF8_FP8 + a_fmt, b_fmt = ("bf8" if sfx == "BF8" else "fp8" for sfx in op_name.rsplit('_', 2)[-2:]) else: a_fmt = b_fmt = "fp8" - def cvt_elem(raw: UOp, sub_idx: int, fp8_fmt: str = "fp8") -> UOp: - if is_i8: - # Extract i8, sign-extend to i32 - byte_val = (raw >> UOp.const(sub_idx * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) - return (byte_val.cast(dtypes.int32) ^ UOp.const(0x80, dtypes.int32)) - UOp.const(0x80, dtypes.int32) - elif is_f32_src: - return raw # already uint32 (f32 bit pattern) - elif is_fp8: - return _FUNCS[f"{fp8_fmt}_to_f32"](raw >> UOp.const(sub_idx * 8, dtypes.uint32)).bitcast(dtypes.uint32) - elif is_bf16: - # bf16→f32 bits: just shift left by 16 (bf16 is upper 16 bits of f32) - return ((raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32)) << UOp.const(16, dtypes.uint32) - else: - # f16→f32 conversion using float arithmetic to avoid UOp optimizer eliminating the conversion. - # The optimizer folds bitcast(uint32→float32) chains, so we compute the float value directly. - h = (raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) - sign = (h >> UOp.const(15, dtypes.uint32)) & UOp.const(1, dtypes.uint32) - exp = (h >> UOp.const(10, dtypes.uint32)) & UOp.const(0x1F, dtypes.uint32) - mant = h & UOp.const(0x3FF, dtypes.uint32) - # Use bf16 path: shift left by 16 to create bf16 bits, then shift mantissa and adjust exponent in float domain - # bf16 bits = (sign << 15) | (exp_bf16 << 7) | mant_bf16 -- but f16 and bf16 have different formats - # Instead: construct f32 bits properly, use a local uint32 array to force materialization - f32_bits = (sign << UOp.const(31, dtypes.uint32)) | \ - ((exp + UOp.const(112, dtypes.uint32)) << UOp.const(23, dtypes.uint32)) | \ - (mant << UOp.const(13, dtypes.uint32)) - is_zero = exp.eq(UOp.const(0, dtypes.uint32)) - # Return uint32 (f32 bit pattern) — stored directly to uint32 temp array, bitcast to float on read - return is_zero.where(UOp.const(0, dtypes.uint32), f32_bits) + # Phase 1: read all A and B values into tmp. Layout: tmp[0:n_a] = A[m][k], tmp[n_a:] = B[n][k]. + # For 4x4 each group is an independent block (K not split), for 16x16/32x32 groups share MxK/NxK and split K. + k_per_grp = K if M == 4 else K // n_grps + n_a_elems = n_grps * M * K if M == 4 else M * K + n_b_elems = n_grps * N * K if M == 4 else N * K + # Use a uint32 temp array: the optimizer folds bitcast(uint32->f32) chains on float arrays, losing conversions. + tmp = UOp.placeholder((n_a_elems + n_b_elems,), dtypes.uint32, slot=0, addrspace=AddrSpace.LOCAL) + + def cvt_elem(raw: UOp, sub_idx: int, fp8_fmt: str) -> UOp: + if is_i8: # extract i8, sign-extend to i32 + byte = (raw >> UOp.const(sub_idx * 8, dtypes.uint32)) & UOp.const(0xFF, dtypes.uint32) + return (byte.cast(dtypes.int32) ^ UOp.const(0x80, dtypes.int32)) - UOp.const(0x80, dtypes.int32) + if is_f32_src: return raw # already uint32 (f32 bit pattern) + if is_fp8: return _FUNCS[f"{fp8_fmt}_to_f32"](raw >> UOp.const(sub_idx * 8, dtypes.uint32)).bitcast(dtypes.uint32) + h = (raw >> UOp.const(sub_idx * 16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) + if is_bf16: return h << UOp.const(16, dtypes.uint32) # bf16 is the upper 16 bits of f32 + # f16 -> f32 bit pattern, done in integer domain so the optimizer can't fold away the conversion + sign, exp, mant = (h >> _c(15)) & _c(1), (h >> _c(10)) & _c(0x1F), h & _c(0x3FF) + f32_bits = (sign << _c(31)) | ((exp + _c(112)) << _c(23)) | (mant << _c(13)) + return exp.eq(_c(0)).where(_c(0), f32_bits) + + def mn_idx(lane: UOp) -> UOp: # M/N matrix index held by a lane + if M == 32: # (lane%32)/16 selects the 16-wide block, (lane%32)%16 the index within it + return ((lane // UOp.const(16, dtypes.int)) % UOp.const(2, dtypes.int)) * UOp.const(16, dtypes.int) + lane % UOp.const(16, dtypes.int) + return lane % UOp.const(grp_size, dtypes.int) + + def grp_idx(lane: UOp) -> UOp: return lane // UOp.const(grp_size, dtypes.int) read_lane = ctx.range() - # For 32x32: lane%16 = M/N index within 16-wide block, lane//16 = which of 4 quarter-waves - # Groups: lanes 0-31 = group 0, lanes 32-63 = group 1 - # Within group: (lane%32)%16 = M/N[0-15], (lane%32)//16 selects M/N[0-15] or [16-31] - lane_in_grp = read_lane % UOp.const(grp_size, dtypes.int) - grp_idx = read_lane // UOp.const(grp_size, dtypes.int) - - if M == 32: - # 32x32: lane_in_grp%16 = sub-row/col (0-15), lane_in_grp//16 = block (0=rows 0-15, 1=rows 16-31) - sub_mn = lane_in_grp % UOp.const(16, dtypes.int) - block_mn = lane_in_grp // UOp.const(16, dtypes.int) - mn_idx = block_mn * UOp.const(16, dtypes.int) + sub_mn # actual M/N index (0-31) - else: - mn_idx = lane_in_grp # for 16x16 and 4x4 + mn, grp = mn_idx(read_lane), grp_idx(read_lane) + def mat_idx(base: int, dim: int, kl: int) -> UOp: # tmp index of element (mn, kl) of this lane's group + if M == 4: return UOp.const(base + kl, dtypes.int) + grp * UOp.const(dim * K, dtypes.int) + mn * UOp.const(K, dtypes.int) + return UOp.const(base + kl, dtypes.int) + mn * UOp.const(K, dtypes.int) + grp * UOp.const(k_per_grp, dtypes.int) read_stores = [] for kl in range(k_per_grp): reg_idx, sub_idx = kl // vpg, kl % vpg - # Read A/B sources. Use rsrc_dyn for inline constants/SGPRs (src_off < 256), rvgpr_dyn for VGPRs (src_off >= 256). - a_raw = src0_is_vgpr.where(ctx.rvgpr_dyn(src0_r + _c(reg_idx), read_lane), - ctx.rsrc_dyn(src0_off, _c(0, dtypes.int), 32)) - a_val = cvt_elem(a_raw, sub_idx, a_fmt) - if M == 4: - a_idx = grp_idx * UOp.const(M * K, dtypes.int) + mn_idx * UOp.const(K, dtypes.int) + UOp.const(kl, dtypes.int) - else: - a_idx = mn_idx * UOp.const(K, dtypes.int) + grp_idx * UOp.const(k_per_grp, dtypes.int) + UOp.const(kl, dtypes.int) - read_stores.append(tmp.index(a_idx).store(a_val)) - - b_raw = src1_is_vgpr.where(ctx.rvgpr_dyn(src1_r + _c(reg_idx), read_lane), - ctx.rsrc_dyn(src1_off, _c(0, dtypes.int), 32)) - b_val = cvt_elem(b_raw, sub_idx, b_fmt) - if M == 4: - b_idx = b_off + grp_idx * UOp.const(N * K, dtypes.int) + mn_idx * UOp.const(K, dtypes.int) + UOp.const(kl, dtypes.int) - else: - b_idx = b_off + mn_idx * UOp.const(K, dtypes.int) + grp_idx * UOp.const(k_per_grp, dtypes.int) + UOp.const(kl, dtypes.int) - read_stores.append(tmp.index(b_idx).store(b_val)) - + # src_off >= 256 means VGPR operand, otherwise inline constant/SGPR + for off, is_vgpr, fmt, base, dim in ((src0_off, src0_off >= _c(256), a_fmt, 0, M), (src1_off, src1_off >= _c(256), b_fmt, n_a_elems, N)): + raw = is_vgpr.where(ctx.rvgpr_dyn(off - _c(256 - reg_idx), read_lane), ctx.rsrc_dyn(off, _c(0, dtypes.int), 32)) + read_stores.append(tmp.index(mat_idx(base, dim, kl)).store(cvt_elem(raw, sub_idx, fmt))) read_phase = UOp.group(*read_stores).end(read_lane) - # Phase 2: Compute dot products and write outputs. - # For 16x16: each lane computes 4 outputs. n_idx = lane%16, grp selects which 4 rows. - # For 32x32: each lane computes 16 outputs. Layout: lane%16 selects n within block, lane//16 selects column block. - # Output mapping: out_reg r at lane l -> D[m][n] where - # n = (l%32)%16 + ((l%32)//16)*16, m = (l//32)*4 + r (for r in 0..3), with 4 groups of 4 rows -> 16 outputs total - # Actually: 16 ACCVGPRs per lane, organized as 4 groups (l//32 gives half, each half has 2 sub-groups) of 4 rows + # Phase 2: dot products and accumulate. acc reads src2 (VGPR, or scalar inline constant/SGPR broadcast). + src2_is_vgpr = src2_off >= _c(256) + acc_scalar = ctx.rsgpr_dyn(src2_off, src2_is_vgpr.ne(True)) + acc_scalar = acc_scalar.cast(dtypes.int32) if is_int_out else acc_scalar.bitcast(dtypes.float32) + acc_reader, acc_writer = (ctx.raccvgpr_dyn, ctx.waccvgpr_dyn) if use_acc else (ctx.rvgpr_dyn, ctx.wvgpr_dyn) tmp2 = tmp.after(read_phase) def _dot_accum(acc: UOp, a_row: UOp, b_row: UOp, lane: UOp) -> UOp: - """acc += sum_k A[a_row+k] * B[b_row+k]. For scaled MFMA, only the dot product is scaled: D = dot*scale + C.""" + """acc += sum_k A[a_row+k] * B[b_row+k] in order (FP-associativity matters). For scaled MFMA only the dot is scaled: D = dot*scale + C.""" def prod(k: int) -> UOp: return tmp2.index(a_row + UOp.const(k, dtypes.int)).bitcast(acc_dt) * tmp2.index(b_row + UOp.const(k, dtypes.int)).bitcast(acc_dt) if not scaled: @@ -1521,135 +1260,89 @@ def prod(k: int) -> UOp: return acc + dot * scale_factor(lane) compute_lane = ctx.range() - compute_stores = [] - - if M == 32 and N == 32: - # 32x32: each lane has 16 output ACCVGPRs - # Lane mapping: n = (lane%32)%16 + ((lane%32)//16)*16, gives column 0-31 - # Row groups: 4 groups of 4, covering rows 0-31. Group g (0-3): rows g*4 .. g*4+3 - # group assignment: lane//16 gives quarter (0-3), each quarter maps to 4 rows - c_lane_in_32 = compute_lane % UOp.const(32, dtypes.int) - c_sub = c_lane_in_32 % UOp.const(16, dtypes.int) - c_block = c_lane_in_32 // UOp.const(16, dtypes.int) - n_idx = c_block * UOp.const(16, dtypes.int) + c_sub - c_half = compute_lane // UOp.const(32, dtypes.int) # 0 or 1 - - for out_reg in range(16): - # Each half covers 8 rows. out_reg 0-3: rows 0-3 (half0) or 16-19 (half1) - # out_reg 4-7: rows 4-7 (half0) or 20-23 (half1), etc. - # Actually: for 32x32, the output layout per lane is: - # acc[0:3] -> rows 0-3 (half 0) or rows 0-3 (half 1)? - # Let me use the ISA doc: for 32x32, D has 16 dwords per lane. The mapping is: - # acc[r] at lane l -> D[m][n] where n = (l%32)%16 + ((l%32)//16)*16 - # m = (l//32)*16 + (r//4)*4 + (r%4) ... giving rows in blocks of 4 - # So: m_base = half * 16 + (out_reg // 4) * 4 + (out_reg % 4) - m_base = c_half * UOp.const(16, dtypes.int) + UOp.const((out_reg // 4) * 4 + (out_reg % 4), dtypes.int) - - acc_v = (ctx.raccvgpr_dyn if use_acc else ctx.rvgpr_dyn)(src2_r + _c(out_reg), compute_lane, src2_is_vgpr) - if is_int_out: acc_v = acc_v.cast(dtypes.int32) - else: acc_v = acc_v.bitcast(dtypes.float32) - acc = src2_is_vgpr.where(acc_v, acc_scalar) - - acc = _dot_accum(acc, m_base * UOp.const(K, dtypes.int), b_off + n_idx * UOp.const(K, dtypes.int), compute_lane) - - if is_int_out: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.cast(dtypes.uint32), exec_mask)) - else: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.bitcast(dtypes.uint32), exec_mask)) - else: - # 16x16 and 4x4: each lane computes out_per_lane outputs - n_idx = compute_lane % UOp.const(grp_sub, dtypes.int) - c_grp = compute_lane // UOp.const(grp_sub, dtypes.int) - - for out_reg in range(out_per_lane): - acc_v = (ctx.raccvgpr_dyn if use_acc else ctx.rvgpr_dyn)(src2_r + _c(out_reg), compute_lane, src2_is_vgpr) - if is_int_out: acc_v = acc_v.cast(dtypes.int32) - else: acc_v = acc_v.bitcast(dtypes.float32) - acc = src2_is_vgpr.where(acc_v, acc_scalar) - - if M == 4: - # 4x4: each group is independent. A/B indexed per-group. - m_base = c_grp * UOp.const(M * K, dtypes.int) + UOp.const(out_reg * K, dtypes.int) - b_base = b_off + c_grp * UOp.const(N * K, dtypes.int) + n_idx * UOp.const(K, dtypes.int) - else: - # 16x16: K is split across groups. Shared MxK/NxK arrays. - m_base = c_grp * UOp.const(out_per_lane, dtypes.int) + UOp.const(out_reg, dtypes.int) - b_base = b_off + n_idx * UOp.const(K, dtypes.int) - - acc = _dot_accum(acc, m_base if M == 4 else m_base * UOp.const(K, dtypes.int), b_base, compute_lane) - - if is_int_out: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.cast(dtypes.uint32), exec_mask)) - else: - compute_stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)( - vdst_reg + _c(out_reg), compute_lane, acc.bitcast(dtypes.uint32), exec_mask)) + c_mn, c_grp = mn_idx(compute_lane), grp_idx(compute_lane) + b_off = UOp.const(n_a_elems, dtypes.int) + def out_ab(out_reg: int) -> tuple[UOp, UOp]: # A/B tmp base indices for one output element of this lane + if M == 32: # 16 outputs per lane: rows (lane//32)*16 + blocks of 4 within the wave + m_base = c_grp * UOp.const(16, dtypes.int) + UOp.const((out_reg // 4) * 4 + out_reg % 4, dtypes.int) + return m_base * UOp.const(K, dtypes.int), b_off + c_mn * UOp.const(K, dtypes.int) + if M == 4: # each group of 4 lanes computes an independent 4x4 block + a_base = c_grp * UOp.const(M * K, dtypes.int) + UOp.const(out_reg * K, dtypes.int) + return a_base, b_off + (c_grp * UOp.const(N, dtypes.int) + c_mn) * UOp.const(K, dtypes.int) + m_base = c_grp * UOp.const(out_per_lane, dtypes.int) + UOp.const(out_reg, dtypes.int) # 16x16: 4 outputs, one row each + return m_base * UOp.const(K, dtypes.int), b_off + c_mn * UOp.const(K, dtypes.int) + compute_stores = [] + for out_reg in range(out_per_lane): + acc_v = acc_reader(src2_off - _c(256 - out_reg), compute_lane, src2_is_vgpr) + acc_v = acc_v.cast(dtypes.int32) if is_int_out else acc_v.bitcast(dtypes.float32) + a_base, b_base = out_ab(out_reg) + acc = _dot_accum(src2_is_vgpr.where(acc_v, acc_scalar), a_base, b_base, compute_lane) + out_bits = acc.cast(dtypes.uint32) if is_int_out else acc.bitcast(dtypes.uint32) + compute_stores.append(acc_writer(vdst_reg + _c(out_reg), compute_lane, out_bits, exec_mask)) compute_phase = UOp.group(*compute_stores).end(compute_lane) return UOp.sink(read_phase, compute_phase, *ctx.inc_pc()) def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P, ctx: _Ctx) -> UOp: - op_name = _op_name(inst) - exec_mask = ctx.rexec() + """RDNA3/4 WMMA: D = A@B + C on 16x16 tiles. A/B are unpacked to flat f32/i32 arrays, then all 256 outputs are + computed directly with scalar ops (no lane loop - the wave32 lane structure is baked into the index maps).""" + op_name, exec_mask = _op_name(inst), ctx.rexec() vdst_reg = ctx.inst_field(type(inst).vdst) - src0_r = ctx.inst_field(type(inst).src0) - _c(256) - src1_r = ctx.inst_field(type(inst).src1) - _c(256) + src0_r, src1_r = ctx.inst_field(type(inst).src0) - _c(256), ctx.inst_field(type(inst).src1) - _c(256) src2_r = ctx.inst_field(type(inst).src2) src2_r = (src2_r >= 256).where(src2_r - _c(256), src2_r) output_type = op_name.split("WMMA_", 1)[1].split("_", 1)[0] - is_bf16 = 'BF16' in op_name - cvt = _FUNCS['bf16_to_f32'] if is_bf16 else _FUNCS['f16_to_f32'] - is_rdna4 = isinstance(inst, ir4.VOP3P) - sz = 8 if "8" in op_name else 16 - # read matrix from VGPRs → flat f32/i32 array[row*16+k] + is_bf16, is_rdna4 = 'BF16' in op_name, isinstance(inst, ir4.VOP3P) + cvt = _FUNCS['bf16_to_f32' if is_bf16 else 'f16_to_f32'] + sz = 8 if any(t in op_name for t in ('IU8', 'FP8', 'BF8')) else 16 # input element size + + # read a source element from VGPRs: (src, lane, vgpr, element-in-vgpr) -> f32/i32 def gval(src, lane, vgpr, ridx): v = ctx.rvgpr_dyn(src + _c(vgpr), UOp.const(lane, dtypes.int)) pkd = v >> UOp.const(ridx * sz, dtypes.uint32) if ridx > 0 else v pkd = pkd & UOp.const((1 << sz) - 1, dtypes.uint32) if "F" in output_type: return cvt(pkd) - return (pkd << _c(24, dtypes.uint)).bitcast(dtypes.int32) >> _c(24, dtypes.int32) # sign extend + return (pkd << _c(24, dtypes.uint)).bitcast(dtypes.int32) >> _c(24, dtypes.int32) # sign extend - # RDNA3 f16/bf16: 16 lanes × 8 VGPRs × 2 halves, k maps linearly - # RDNA3 iu8: 16 lanes × 4 VGPRs × 4 quarters, k maps linearly - # RDNA4: 32 lanes x 4 VGPRS x 2 halves, k bits are scrambled (k[2] goes to lane bit 4) + # RDNA3 f16/bf16: 16 lanes x 8 VGPRs x 2 halves, k maps linearly + # RDNA3 iu8: 16 lanes x 4 VGPRs x 4 quarters, k maps linearly + # RDNA4: 32 lanes x 4 VGPRs x 2 halves, k bits are scrambled (k[2] goes to lane bit 4) def read_mat(src): - n = 32 // sz # values per vgpr - # (row, k) → (lane, vgpr, row index) - def ab_map(i, k): + n = 32 // sz # values per vgpr + def ab_map(i, k): # (row, k) -> (lane, vgpr, element-in-vgpr) elem, lane = ((k & 3) | ((k >> 1) & 4), i + ((k >> 2) & 1) * 16) if is_rdna4 else (k, i) return lane, elem // n, elem % n return [gval(src, *ab_map(row, k)) for row in range(16) for k in range(16)] mat_a, mat_b = read_mat(src0_r), read_mat(src1_r) - # (row, col) -> (lane, vgpr) - def d_map(m, n): + def d_map(m, n): # output (row, col) -> (lane, vgpr) lane_bit, vgpr = (m >> 3, m & 7) if is_rdna4 else (m & 1, m >> 1) return n + lane_bit * 16, vgpr - if output_type in ["F16", "BF16"]: - # read accumulator C with f16 layout: for RDNA4, pairs of f32 vgprs pack into one f16 vgpr - # for RDNA3, same layout as f32 but only lo 16 bits used + + # Accumulator C. RDNA4 f16/bf16 packs two f32 accumulator VGPRs into one f16 VGPR; RDNA3 uses the lo half of each. + if output_type in ("F16", "BF16"): mat_c = [gval(src2_r, *((lane, vgpr // 2, vgpr % 2) if is_rdna4 else (lane, vgpr, 0))) for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] - mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] - def f32_to_f16_bits(v: UOp) -> UOp: return v.cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32) - def f32_to_bf16_bits(v: UOp) -> UOp: return (v.bitcast(dtypes.uint32) >> UOp.const(16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32) - out_cvt = f32_to_bf16_bits if is_bf16 else f32_to_f16_bits - if is_rdna4: # pack 2 f16 per VGPR: adjacent m values share (lane, vgpr) since vgpr=m&7, half=m&1 - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1] // 2), UOp.const(d_map(m, n)[0], dtypes.int), - out_cvt(mat_d[m*16+n]) | (out_cvt(mat_d[(m+1)*16+n]) << UOp.const(16, dtypes.uint32)), exec_mask) - for n in range(16) for m in range(0, 16, 2)] - else: # (rdna3) 1 f16 per VGPR (lo half only) - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int), out_cvt(mat_d[m*16+n]), exec_mask) - for m in range(16) for n in range(16)] - else: # f32/i32 + else: out_dt = dtypes.float32 if output_type == "F32" else dtypes.int32 - mat_c = [ctx.rvgpr_dyn(src2_r + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int)).bitcast(out_dt) - for m in range(16) for n in range(16)] - mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] - stores = [ctx.wvgpr_dyn(vdst_reg + _c(d_map(m, n)[1]), UOp.const(d_map(m, n)[0], dtypes.int), mat_d[m*16+n].bitcast(dtypes.uint32), exec_mask) - for m in range(16) for n in range(16)] + mat_c = [ctx.rvgpr_dyn(src2_r + _c(vgpr), UOp.const(lane, dtypes.int)).bitcast(out_dt) + for m in range(16) for n in range(16) for lane, vgpr in [d_map(m, n)]] + mat_d = [sum(mat_a[r*16+k] * mat_b[c*16+k] for k in range(16)) + mat_c[r*16+c] for r in range(16) for c in range(16)] + + def w_store(m: int, n: int, val: UOp, vgpr_off: int) -> UOp: # store one output element to its (lane, vgpr) slot + lane_i, _ = d_map(m, n) + return ctx.wvgpr_dyn(vdst_reg + _c(vgpr_off), UOp.const(lane_i, dtypes.int), val, exec_mask) + if output_type in ("F16", "BF16"): + def to_bits(v: UOp) -> UOp: # f32 result -> 16 output bits + return ((v.bitcast(dtypes.uint32) >> UOp.const(16, dtypes.uint32)) & UOp.const(0xFFFF, dtypes.uint32)) if is_bf16 \ + else v.cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32) + if is_rdna4: # pack 2 outputs per VGPR (adjacent m values share a VGPR) + stores = [w_store(m, n, to_bits(mat_d[m*16+n]) | (to_bits(mat_d[(m+1)*16+n]) << UOp.const(16, dtypes.uint32)), d_map(m, n)[1] // 2) + for n in range(16) for m in range(0, 16, 2)] + else: # one output per VGPR (lo half) + stores = [w_store(m, n, to_bits(mat_d[m*16+n]), d_map(m, n)[1]) for m in range(16) for n in range(16)] + else: # f32/i32 + stores = [w_store(m, n, mat_d[m*16+n].bitcast(dtypes.uint32), d_map(m, n)[1]) for m in range(16) for n in range(16)] return UOp.sink(*stores, *ctx.inc_pc()) def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _Ctx) -> UOp: @@ -1689,56 +1382,40 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P | irc.VOP3P | irc.VOP3PX2, ctx: _ is_pk_f32 = 'PK' in op_name and 'F32' in op_name and 'MOV' not in op_name # CDNA packed F32 ops is_pk_mov_b32 = 'PK_MOV_B32' in op_name # CDNA packed MOV needs special handling do_cast = any(x in op_name for x in ('F16', 'F32', 'BF16')) and 'IU' not in op_name and not is_pk_f32 - literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None # type: ignore[union-attr] + literal = ctx.optional_field(inst, 'literal') src0 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), lane, 16, literal=literal, do_cast=do_cast) src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, 16, literal=literal, do_cast=do_cast) src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, 16, literal=literal, do_cast=do_cast) - opsel, opsel_hi = getattr(inst, 'opsel', 0) or 0, getattr(inst, 'opsel_hi', 3) if getattr(inst, 'opsel_hi', 3) is not None else 3 - opsel_hi2 = getattr(inst, 'opsel_hi2', 1) if getattr(inst, 'opsel_hi2', 1) is not None else 1 - neg, neg_hi = getattr(inst, 'neg', 0) or 0, getattr(inst, 'neg_hi', 0) or 0 + opsel, opsel_hi = _iattr(inst, 'opsel'), _iattr(inst, 'opsel_hi', 3) + opsel_hi2 = _iattr(inst, 'opsel_hi2', 1) + neg, neg_hi = _iattr(inst, 'neg'), _iattr(inst, 'neg_hi') + + def _pk_sel(src_lo: UOp, src_off: UOp, sel: int) -> UOp: + """Lo (sel=0) / hi (sel!=0) half of a packed source: VGPR/SGPR pairs select from the two registers of the pair + (off>=256 is a VGPR pair, off<128 an SGPR pair), inline constants (128<=off<256) broadcast to both halves.""" + if not sel: return (src_off >= _c(256)).where(ctx.rvgpr_dyn(src_off - _c(256), lane), src_lo) + is_sgpr_pair = src_off < _c(128) + return (src_off >= _c(256)).where(ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane), + is_sgpr_pair.where(ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair), src_lo)) if is_pk_mov_b32: # v_pk_mov_b32: D[lo] = src0[opsel_bit0 ? hi : lo], D[hi] = src1[opsel_bit1 ? hi : lo] - src_offs = [ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1)] - def _pk_mov_sel(src_lo: UOp, src_off: UOp, sel_bit: int) -> UOp: - is_vgpr = src_off >= _c(256) - vgpr_lo = ctx.rvgpr_dyn(src_off - _c(256), lane) if lane is not None else _c(0) - vgpr_hi = ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane) if lane is not None else _c(0) - is_sgpr_pair = src_off < _c(128) - sgpr_hi = ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair) - scalar_sel = is_sgpr_pair.where(sgpr_hi, src_lo) if sel_bit else src_lo - return is_vgpr.where(vgpr_hi if sel_bit else vgpr_lo, scalar_sel) - lo_val = _pk_mov_sel(src0, src_offs[0], opsel & 1) - hi_val = _pk_mov_sel(src1, src_offs[1], opsel & 2) - result = _u64(lo_val, hi_val) - lo_out, hi_out = _split64(result) + lo_val = _pk_sel(src0, ctx.inst_field(type(inst).src0), opsel & 1) + hi_val = _pk_sel(src1, ctx.inst_field(type(inst).src1), opsel & 2) + lo_out, hi_out = _split64(_u64(lo_val, hi_val)) stores = [ctx.wvgpr_dyn(vdst_reg, lane, lo_out, exec_mask), ctx.wvgpr_dyn(vdst_reg + _c(1), lane, hi_out, exec_mask)] return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) srcs: dict[str, UOp | int] = {} if is_pk_f32: - # CDNA packed F32: read 32-bit sources, build 64-bit packed values using opsel. - # For VGPRs: opsel selects between v[reg] (0) and v[reg+1] (1) for each half. - # For SGPR pairs (off < 128): s[N] = lo float32, s[N+1] = hi float32. - # For inline constants (128 <= off < 256): broadcast same value to both halves. + # CDNA packed F32: read 32-bit sources, build 64-bit packed values per opsel, with per-half negation src_offs = [ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2)] - def build_pk_f32(src_lo: UOp, src_off: UOp, opsel_lo: int, opsel_hi_bit: int, neg_lo: int, neg_hi_bit: int) -> UOp: - is_vgpr = src_off >= _c(256) - vgpr_lo = ctx.rvgpr_dyn(src_off - _c(256), lane) if lane is not None else _c(0) - vgpr_hi = ctx.rvgpr_dyn(src_off - _c(256) + _c(1), lane) if lane is not None else _c(0) - # For SGPR pairs, opsel selects between s[N] (0) and s[N+1] (1); inline constants always broadcast. - is_sgpr_pair = src_off < _c(128) - sgpr_hi = ctx.rsgpr_dyn(src_off + _c(1), is_sgpr_pair) - scalar_lo_sel = src_lo if not opsel_lo else is_sgpr_pair.where(sgpr_hi, src_lo) - scalar_hi_sel = src_lo if not opsel_hi_bit else is_sgpr_pair.where(sgpr_hi, src_lo) - lo = is_vgpr.where(vgpr_hi if opsel_lo else vgpr_lo, scalar_lo_sel) - hi = is_vgpr.where(vgpr_hi if opsel_hi_bit else vgpr_lo, scalar_hi_sel) - if neg_lo: lo = lo ^ UOp.const(0x80000000, dtypes.uint32) - if neg_hi_bit: hi = hi ^ UOp.const(0x80000000, dtypes.uint32) - return _u64(lo, hi) - srcs = {'S0': build_pk_f32(src0, src_offs[0], opsel & 1, opsel_hi & 1, neg & 1, neg_hi & 1), - 'S1': build_pk_f32(src1, src_offs[1], opsel & 2, opsel_hi & 2, neg & 2, neg_hi & 2), - 'S2': build_pk_f32(src2, src_offs[2], opsel & 4, 1 if opsel_hi2 else 0, neg & 4, neg_hi & 4)} + hi_bits = (opsel_hi & 1, opsel_hi & 2, 1 if opsel_hi2 else 0) + for i, (bit, s0) in enumerate(zip((1, 2, 4), (src0, src1, src2))): + lo, hi = _pk_sel(s0, src_offs[i], opsel & bit), _pk_sel(s0, src_offs[i], hi_bits[i]) + if neg & bit: lo = lo ^ UOp.const(0x80000000, dtypes.uint32) + if neg_hi & bit: hi = hi ^ UOp.const(0x80000000, dtypes.uint32) + srcs[f'S{i}'] = _u64(lo, hi) elif 'FMA_MIX' in op_name or 'MAD_MIX' in op_name: combined_opsel_hi = (opsel_hi & 0x3) | ((opsel_hi2 & 0x1) << 2) # For FMA_MIX: neg_hi is ABS (not neg!), neg is actual negation @@ -1801,12 +1478,12 @@ def _compile_vopd(inst: ir3.VOPD | ir4.VOPD, ctx: _Ctx) -> UOp: if label == 'Y': srcs = {'S0': srcy0, 'S1': srcy1, 'D0': ctx.rvgpr_dyn(vdst_reg, lane)} else: srcs = {'S0': ctx.rsrc_dyn(src0_off, lane, literal=literal), 'S1': ctx.rvgpr_dyn(vsrc1_reg, lane), 'D0': ctx.rvgpr_dyn(vdst_reg, lane)} # VOP2_FMAAK/FMAMK_(DTYPE)_E32 - if vop in (ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32, ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32): + if vop in (ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32): assert literal is not None srcs['SIMM32'] = literal if op in (ir3.VOPDOp.V_DUAL_CNDMASK_B32, ir4.VOPDOp.V_DUAL_CNDMASK_B32): srcs['VCC'] = ctx.rmask(_c(VCC_LO.offset)) pcode = get_pcode(vop) - srcs.update({'VCC': ctx.rmask(_c(VCC_LO.offset)), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane}) + srcs.update(ctx.base_srcs(exec_mask, lane), VCC=ctx.rmask(_c(VCC_LO.offset))) for dest, val in parse_pcode(pcode, srcs)[1]: if dest.startswith('D0'): all_stores.append(ctx.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask, after=srcy1)) return UOp.sink(UOp.group(*all_stores).end(lane), *ctx.inc_pc()) @@ -1824,37 +1501,26 @@ def _compile_mem_op(inst: ir3.DS|ir3.FLAT|ir3.GLOBAL|ir3.SCRATCH|ir4.DS|ir4.VFLA is_lds = isinstance(inst, (ir3.DS, ir4.DS, irc.DS)) is_scratch = isinstance(inst, (ir3.SCRATCH, ir4.VSCRATCH, irc.SCRATCH)) # CDNA acc bit: when set, VGPR operands (vdst/vdata) target ACCVGPR file instead of VGPR - use_acc = bool(getattr(inst, 'acc', 0)) + use_acc = bool(_iattr(inst, 'acc')) mem = ctx.lds if is_lds else ctx.scratch if is_scratch else ctx.vmem addr_shift = UOp.const(2, dtypes.uint32 if is_lds else dtypes.uint64) - # Extract register info - all dynamic for deduplication + # Field names differ per format: DS (addr/data0/offset0+offset1), RDNA4 V* (vaddr/vsrc/ioffset), RDNA3+CDNA (addr/data/offset) + if is_lds: addr_field, data_field = 'addr', 'data0' + elif isinstance(inst, (ir4.VGLOBAL, ir4.VSCRATCH, ir4.VFLAT)): addr_field, data_field = 'vaddr', 'vsrc' + else: addr_field, data_field = 'addr', 'data' + addr_reg = ctx.inst_field(getattr(type(inst), addr_field)) + vdata_reg, vdst_reg = ctx.inst_field(getattr(type(inst), data_field)), ctx.inst_field(type(inst).vdst) if is_lds: - addr_reg = ctx.inst_field(type(inst).addr) # type: ignore[union-attr] - vdata_reg = ctx.inst_field(type(inst).data0) # type: ignore[union-attr] - vdst_reg = ctx.inst_field(type(inst).vdst) - offset0 = ctx.inst_field(type(inst).offset0) # type: ignore[union-attr] - offset1 = ctx.inst_field(type(inst).offset1) # type: ignore[union-attr] - offset = (offset1 << _c(8)) | offset0 # DS offset is 16-bit: (offset1 << 8) | offset0 - saddr_reg = None - elif isinstance(inst, (ir4.VGLOBAL, ir4.VSCRATCH, ir4.VFLAT)): # RDNA4: vaddr, vsrc, ioffset - addr_reg = ctx.inst_field(type(inst).vaddr) - vdata_reg = ctx.inst_field(type(inst).vsrc) - vdst_reg = ctx.inst_field(type(inst).vdst) - offset = ctx.inst_field_signed(type(inst).ioffset) - offset0, offset1 = _c(0), _c(0) - saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None - else: # RDNA3: addr, data, offset - addr_reg = ctx.inst_field(type(inst).addr) - vdata_reg = ctx.inst_field(type(inst).data) # type: ignore[union-attr] - vdst_reg = ctx.inst_field(type(inst).vdst) - offset = ctx.inst_field_signed(type(inst).offset) # type: ignore[union-attr] - offset0, offset1 = _c(0), _c(0) - saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None # type: ignore[union-attr] + offset0, offset1 = ctx.inst_field(type(inst).offset0), ctx.inst_field(type(inst).offset1) # type: ignore[union-attr] + offset, saddr_reg = (offset1 << _c(8)) | offset0, None # DS offset is 16-bit: (offset1 << 8) | offset0 + else: + offset0, offset1, saddr_reg = _c(0), _c(0), ctx.optional_field(inst, 'saddr') + offset = ctx.inst_field_signed(getattr(type(inst), 'ioffset' if hasattr(type(inst), 'ioffset') else 'offset')) # Data width from canonical_op_bits (32/64/96/128), default to 32 for untyped ops data_bits_mem = inst.canonical_op_bits.get('data', 32) - is_atomic, glc = 'ATOMIC' in op_name, getattr(inst, 'glc', 0) + is_atomic, glc = 'ATOMIC' in op_name, _iattr(inst, 'glc') has_data1 = is_lds and hasattr(inst, 'data1') and inst.data1 is not None data1_reg = ctx.inst_field(type(inst).data1) if is_lds else _c(0) # type: ignore[union-attr] @@ -1881,7 +1547,7 @@ def make_addr(lane: UOp) -> UOp: scratch_stride = ctx.rsgpr_dyn(_c(SCRATCH_STRIDE_IDX)).cast(dtypes.uint64) base = lane.cast(dtypes.uint64) * scratch_stride # SVE (Scratch VGPR Enable): when SVE=1, VADDR is used as offset; when SVE=0, VADDR is ignored - sve = getattr(inst, 'sve', 0) + sve = _iattr(inst, 'sve') vaddr = ctx.rvgpr_dyn(addr_reg, lane).cast(dtypes.uint64) addr_offset = vaddr if sve == 1 else UOp.const(0, dtypes.uint64) # Add saddr value only if use_saddr is true (saddr < 124) @@ -1912,17 +1578,13 @@ def wmem(addr: UOp, val: UOp, active: UOp, data_bits: int = 32) -> UOp: def make_srcs(lane: UOp) -> dict: addr = make_addr(lane) if is_lds: - if data_bits_mem == 128: - data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA1': ctx.rvgpr_dyn(vdata_reg + _c(1), lane), - 'DATA2': ctx.rvgpr_dyn(vdata_reg + _c(2), lane), 'DATA3': ctx.rvgpr_dyn(vdata_reg + _c(3), lane)} - elif data_bits_mem == 96: - data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA1': ctx.rvgpr_dyn(vdata_reg + _c(1), lane), - 'DATA2': ctx.rvgpr_dyn(vdata_reg + _c(2), lane)} - elif data_bits_mem <= 32: + if data_bits_mem <= 32: data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), 'DATA2': ctx.rvgpr_dyn(data1_reg, lane) if has_data1 else UOp.const(0, dtypes.uint32)} - else: + elif data_bits_mem == 64: # DATA/DATA2 are the 64-bit input registers, formed from VGPR pairs data = {'DATA': _u64(ctx.rvgpr_dyn(vdata_reg, lane), ctx.rvgpr_dyn(vdata_reg + _c(1), lane)), 'DATA2': _u64(ctx.rvgpr_dyn(data1_reg, lane), ctx.rvgpr_dyn(data1_reg + _c(1), lane)) if has_data1 else UOp.const(0, dtypes.uint64)} + else: # 96/128-bit: one VGPR per dword + data = {'DATA': ctx.rvgpr_dyn(vdata_reg, lane), **{f'DATA{i}': ctx.rvgpr_dyn(vdata_reg + _c(i), lane) for i in range(1, data_bits_mem // 32)}} # RDNA3 uses ADDR/OFFSET, RDNA4 uses vgpr_a/offset (lowercase) + CalcDsAddr function return {'ADDR': addr, 'ADDR_BASE': addr, 'OFFSET': offset, 'OFFSET0': offset0, 'OFFSET1': offset1, '_lds': mem, 'laneId': lane, 'vgpr_a': ctx.rvgpr_dyn(addr_reg, lane), 'offset': offset, 'offset0': offset0, 'offset1': offset1, **data} @@ -1965,7 +1627,8 @@ def make_stores(dest: str, val: UOp, lane: UOp, active: UOp, writes_return_data: if is_scratch: return _mem_store_bytes(mem, val[0], val[1], active, data_bits) return _mem_store(mem, val[0], val[1], active, 64, data_bits) if dest.startswith('RETURN_DATA') and writes_return_data: - _wdata = (lambda r, v, l, e: ctx.waccvgpr_dyn(r, l, v, e)) if use_acc else (lambda r, v, l, e: ctx.wvgpr_dyn(r, l, v, e)) + write_gpr = ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn + _wdata = lambda r, v, l, e: write_gpr(r, l, v, e) # noqa: E731 (arg order: reg, val, lane, exec) if (m := re.match(r'RETURN_DATA\[(\d+)\s*:\s*(\d+)\]', dest)): bit_width, dword_idx = int(m.group(1)) - int(m.group(2)) + 1, int(m.group(2)) // 32 return _write_val(bit_width, val, _wdata, vdst_reg + _c(dword_idx), lane, exec_mask) @@ -2019,7 +1682,7 @@ def make_stores(dest: str, val: UOp, lane: UOp, active: UOp, writes_return_data: def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: """CDNA MUBUF: linear buffer address = base + soffset + (stride * index) + vgpr_offset + inst_offset""" exec_mask, op_name = ctx.rexec(), _op_name(inst) - use_acc, is_store, is_lds = bool(getattr(inst, 'acc', 0)), 'STORE' in op_name, bool(getattr(inst, 'lds', 0)) + use_acc, is_store, is_lds = bool(_iattr(inst, 'acc')), 'STORE' in op_name, bool(_iattr(inst, 'lds')) n_dwords = 4 if 'X4' in op_name else 2 if 'X2' in op_name else 1 # instruction fields @@ -2072,28 +1735,26 @@ def _compile_mubuf(inst: irc.MUBUF, ctx: _Ctx) -> UOp: stores.append((ctx.waccvgpr_dyn if use_acc else ctx.wvgpr_dyn)(vdata + _c(i), lane, val, exec_mask)) return UOp.sink(UOp.group(*stores).end(lane), *ctx.inc_pc()) -# Dispatch table: instruction type -> handler function -_INST_HANDLERS: dict[type, Callable[..., UOp]] = { - ir3.SOPP: _compile_sopp, ir3.SMEM: _compile_smem, ir3.SOP1: _compile_sop, ir3.SOP2: _compile_sop, ir3.SOPC: _compile_sop, ir3.SOPK: _compile_sop, - ir3.VOP1: _compile_vop12, ir3.VOP1_SDST: _compile_vop12, ir3.VOP1_DPP16: _compile_vop12, ir3.VOP2: _compile_vop12, ir3.VOP2_DPP16: _compile_vop12, - ir3.VOPC: _compile_vopc, ir3.VOPC_DPP16: _compile_vopc, ir3.VOP3: _compile_vop3, ir3.VINTERP: _compile_vinterp, - ir3.VOP3_SDST: _compile_vop3, ir3.VOP3SD: _compile_vop3sd, ir3.VOP3P: _compile_vop3p, ir3.VOPD: _compile_vopd, - ir3.DS: _compile_mem_op, ir3.FLAT: _compile_mem_op, ir3.GLOBAL: _compile_mem_op, ir3.SCRATCH: _compile_mem_op, - # RDNA4 instruction classes - ir4.SOPP: _compile_sopp, ir4.SMEM: _compile_smem, ir4.SOP1: _compile_sop, ir4.SOP2: _compile_sop, ir4.SOPC: _compile_sop, ir4.SOPK: _compile_sop, - ir4.VOP1: _compile_vop12, ir4.VOP1_SDST: _compile_vop12, ir4.VOP1_DPP16: _compile_vop12, ir4.VOP2: _compile_vop12, ir4.VOP2_DPP16: _compile_vop12, - ir4.VOPC: _compile_vopc, ir4.VOPC_DPP16: _compile_vopc, ir4.VOP3: _compile_vop3, ir4.VINTERP: _compile_vinterp, - ir4.VOP3_SDST: _compile_vop3, ir4.VOP3SD: _compile_vop3sd, ir4.VOP3P: _compile_vop3p, ir4.VOPD: _compile_vopd, - ir4.DS: _compile_mem_op, ir4.VFLAT: _compile_mem_op, ir4.VGLOBAL: _compile_mem_op, ir4.VSCRATCH: _compile_mem_op, - # CDNA instruction classes - irc.SOPP: _compile_sopp, irc.SMEM: _compile_smem, irc.SOP1: _compile_sop, irc.SOP2: _compile_sop, irc.SOPC: _compile_sop, irc.SOPK: _compile_sop, - irc.VOP1: _compile_vop12, irc.VOP1_DPP16: _compile_vop12, irc.VOP2: _compile_vop12, irc.VOP2_DPP16: _compile_vop12, - irc.VOPC: _compile_vopc, irc.VOP3: _compile_vop3, - irc.VOP3_SDST: _compile_vop3, irc.VOP3SD: _compile_vop3sd, irc.VOP3P: _compile_vop3p, irc.VOP3PX2: _compile_vop3p, - irc.VOP1_SDWA: _compile_sdwa, irc.VOP2_SDWA: _compile_sdwa, irc.VOP2_SDWA_SDST: _compile_sdwa, irc.VOPC_SDWA_SDST: _compile_sdwa, - irc.DS: _compile_mem_op, irc.FLAT: _compile_mem_op, irc.GLOBAL: _compile_mem_op, irc.SCRATCH: _compile_mem_op, - irc.MUBUF: _compile_mubuf, -} +# Dispatch table: instruction type -> handler function. Classes are looked up by name across all three archs. +def _inst_kinds(*names: str) -> tuple[type, ...]: + return tuple(getattr(m, n) for m in (ir3, ir4, irc) for n in names if hasattr(m, n)) + +_COMMON_HANDLERS: list[tuple[Callable[..., UOp], tuple[str, ...]]] = [ + (_compile_sopp, ('SOPP',)), + (_compile_smem, ('SMEM',)), + (_compile_sop, ('SOP1', 'SOP2', 'SOPC', 'SOPK')), + (_compile_vop12, ('VOP1', 'VOP1_SDST', 'VOP1_DPP16', 'VOP2', 'VOP2_DPP16')), + (_compile_vopc, ('VOPC', 'VOPC_DPP16')), + (_compile_vop3, ('VOP3', 'VOP3_SDST')), + (_compile_vinterp, ('VINTERP',)), + (_compile_vop3sd, ('VOP3SD',)), + (_compile_vop3p, ('VOP3P', 'VOP3PX2')), + (_compile_vopd, ('VOPD',)), + (_compile_sdwa, ('VOP1_SDWA', 'VOP2_SDWA', 'VOP2_SDWA_SDST', 'VOPC_SDWA_SDST')), + (_compile_mem_op, ('DS', 'FLAT', 'GLOBAL', 'SCRATCH', 'VFLAT', 'VGLOBAL', 'VSCRATCH')), +] +_INST_HANDLERS: dict[type, Callable[..., UOp]] = {t: h for h, names in _COMMON_HANDLERS for t in _inst_kinds(*names)} +_INST_HANDLERS[irc.MUBUF] = _compile_mubuf # CDNA only (rdna3 also has a MUBUF class, intentionally unhandled) # ═══════════════════════════════════════════════════════════════════════════════ # PROGRAM DECODE AND COMPILATION @@ -2245,7 +1906,6 @@ def _init_wave(lib: int, wave_start: int, total_threads: int, lx: int, ly: int, def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c, scratch_size: int = 0, arch: str = "rdna3", user_data: list[int]|None = None) -> int: """Execute AMD assembly program. scratch_size is private_segment_fixed_size from kernel descriptor (per-lane).""" - from tinygrad.renderer.amd.dsl import Inst program: dict[int, tuple[Callable, list[int], bool, Inst]] = {} # pc -> (fxn, globals, is_barrier, inst) lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT) * 512 total_threads = lx * ly * lz @@ -2258,7 +1918,7 @@ def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, # Initialize SQTT encoder — emits packets inline as instructions execute (only when profiling) if PROFILE: - sqtt_emit, sqtt_finish, sqtt_finalize = _init_sqtt_encoder() + sqtt_emit, sqtt_finish, sqtt_finalize = _make_sqtt_encoder() def _ensure_compiled(pc: int) -> tuple[Callable, list[int], bool, Inst]: if pc not in program: @@ -2272,53 +1932,47 @@ def _ensure_compiled(pc: int) -> tuple[Callable, list[int], bool, Inst]: print(colored(msg, 'green') if len(_canonical_runner_cache) > prev_len else msg) return program[pc] + def _run_workgroup(gidx: int, gidy: int, gidz: int, tracing: bool): + """Initialize all wavefronts for one workgroup and execute them with barrier synchronization. + Each wave runs until it hits s_barrier or s_endpgm. When all waves have stopped, release barrier waves.""" + waves: list[tuple[WaveState, list]] = [] + for wave_start in range(0, total_threads, wave_size): + st = _init_wave(lib, wave_start, total_threads, lx, ly, lz, args_ptr, rsrc2, scratch_size, arch, gidx, gidy, gidz, user_data, wave_size) + waves.append((st, [ctypes.c_uint64(st.sgpr_buf._buf.va_addr), ctypes.c_uint64(st.vgpr_buf._buf.va_addr), + ctypes.c_uint64(vmem_buf._buf.va_addr), ctypes.c_uint64(lds_buf._buf.va_addr), + ctypes.c_uint64(scratch_buf._buf.va_addr if scratch_buf else 0), + ctypes.c_uint64(st.accvgpr_buf._buf.va_addr)])) + done = [False] * len(waves) + for _ in range(10_000_000): + if all(done): return + for wi, (st, c_bufs) in enumerate(waves): + if done[wi]: continue + # Run this wave until barrier or endpgm + for _ in range(1_000_000): + pc = st.pc + if pc == ENDPGM_PC: + done[wi] = True + if tracing: sqtt_finish(wi) + break + fxn, globals_list, is_barrier, inst = _ensure_compiled(pc) + if DEBUG >= 5: print(f" exec gid=({gidx},{gidy},{gidz}) w={wi} PC={pc - lib}: {inst!r}", flush=True) + fxn(*[c_bufs[g] for g in globals_list]) + if tracing: + inst_op = inst.op.value if hasattr(inst, 'op') else 0 + sqtt_emit(wi, inst, (st.pc != ENDPGM_PC and st.pc != pc + inst.size()) if inst_op in _BRANCH_OPS else None) + if is_barrier: break # s_barrier hit: PC already advanced past it, pause this wave + else: raise RuntimeError("exceeded 1M instructions in single wave, likely infinite loop") + # All waves have either hit barrier or endpgm — release barrier waves for next round + raise RuntimeError("exceeded 10M total scheduling rounds") + # Set DAZ+FTZ during emulator execution, restore afterward to avoid breaking hypothesis tests # Only trace the first workgroup (like real HW traces one CU/SIMD), subsequent workgroups run but don't add to trace tracing = bool(PROFILE) - with _MXCSRContext(): - for gidz in range(gz): - for gidy in range(gy): - for gidx in range(gx): - # Initialize all wavefronts for this workgroup - waves: list[tuple[WaveState, list]] = [] - for wave_start in range(0, total_threads, wave_size): - st = _init_wave(lib, wave_start, total_threads, lx, ly, lz, args_ptr, rsrc2, scratch_size, arch, gidx, gidy, gidz, user_data, - wave_size) - c_bufs = [ctypes.c_uint64(st.sgpr_buf._buf.va_addr), ctypes.c_uint64(st.vgpr_buf._buf.va_addr), - ctypes.c_uint64(vmem_buf._buf.va_addr), ctypes.c_uint64(lds_buf._buf.va_addr), - ctypes.c_uint64(scratch_buf._buf.va_addr if scratch_buf else 0), - ctypes.c_uint64(st.accvgpr_buf._buf.va_addr)] - waves.append((st, c_bufs)) - - # Execute wavefronts with barrier synchronization - # Each wave runs until it hits s_barrier or s_endpgm. When all waves have stopped, release barrier waves. - done = [False] * len(waves) - for total_inst in range(10_000_000): - if all(done): break - for wi, (st, c_bufs) in enumerate(waves): - if done[wi]: continue - # Run this wave until barrier or endpgm - for _ in range(1_000_000): - pc = st.pc - if pc == ENDPGM_PC: - done[wi] = True - if tracing: sqtt_finish(wi) - break - fxn, globals_list, is_barrier, inst = _ensure_compiled(pc) - if DEBUG >= 5: print(f" exec gid=({gidx},{gidy},{gidz}) w={wi} PC={pc - lib}: {inst!r}", flush=True) - fxn(*[c_bufs[g] for g in globals_list]) - if tracing: - inst_op = inst.op.value if hasattr(inst, 'op') else 0 - sqtt_emit(wi, inst, (st.pc != ENDPGM_PC and st.pc != pc + inst.size()) if inst_op in _BRANCH_OPS else None) - if is_barrier: break # s_barrier hit: PC already advanced past it, pause this wave - else: raise RuntimeError("exceeded 1M instructions in single wave, likely infinite loop") - # All waves have either hit barrier or endpgm — release barrier waves for next round - else: raise RuntimeError("exceeded 10M total scheduling rounds") - tracing = False # only trace the first workgroup - - # Reset LDS for next workgroup - if lds_size > 0: ctypes.memset(lds_buf._buf.va_addr, 0, max(lds_size, 4)) + for gidz, gidy, gidx in itertools.product(range(gz), range(gy), range(gx)): + _run_workgroup(gidx, gidy, gidz, tracing) + tracing = False # only trace the first workgroup + if lds_size > 0: ctypes.memset(lds_buf._buf.va_addr, 0, max(lds_size, 4)) # reset LDS for next workgroup if PROFILE: sqtt_traces.append(sqtt_finalize()) return 0 diff --git a/test/mockgpu/amd/pcode.py b/test/mockgpu/amd/pcode.py index bb66ffe03f1f2..88584252dee16 100644 --- a/test/mockgpu/amd/pcode.py +++ b/test/mockgpu/amd/pcode.py @@ -1,5 +1,20 @@ # Tokenizer-based expression parser for AMD pcode +import ast, itertools, operator, re from typing import Any, Callable + +_BINOPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, ast.LShift: operator.lshift, ast.RShift: operator.rshift, + ast.BitAnd: operator.and_, ast.BitOr: operator.or_, ast.BitXor: operator.xor} +def _const_int(expr: str) -> int: + """Evaluate a compile-time integer expression (integer literals and basic arithmetic only).""" + def ev(node: ast.AST) -> int: + if isinstance(node, ast.Expression): return ev(node.body) + if isinstance(node, ast.Constant) and isinstance(node.value, int): return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + return (-1 if isinstance(node.op, ast.USub) else 1) * ev(node.operand) + if isinstance(node, ast.BinOp) and type(node.op) in _BINOPS: return _BINOPS[type(node.op)](ev(node.left), ev(node.right)) + raise ValueError(f"not a constant integer expression: {expr!r}") + return ev(ast.parse(expr.strip(), mode='eval')) from tinygrad.dtype import dtypes from tinygrad.uop.ops import Ops, UOp from tinygrad.codegen.decomp.dtype import f2f @@ -360,22 +375,13 @@ def _sad_u8(a: UOp, b: UOp, acc: UOp, masked: bool = False) -> UOp: 'fp8_to_f32': _fp8_to_f32, 'bf8_to_f32': _bf8_to_f32, 'f32_to_fp8': _f32_to_fp8, 'f32_to_bf8': _f32_to_bf8, 'f32_to_bf16': _f32_to_bf16, 'f32_to_bf16_SR': _f32_to_bf16_sr, 'f32_to_bf16_sr': _f32_to_bf16_sr, } -for is_max, name in [(False, 'min'), (True, 'max')]: - for dt, sfx in [(dtypes.float32, 'f32'), (dtypes.int, 'i32'), (dtypes.uint32, 'u32'), (dtypes.int16, 'i16'), (dtypes.uint16, 'u16')]: - _FUNCS[f'v_{name}_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a) - _FUNCS[f'v_{name}3_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a) -# f16 min/max/min3/max3/med3 -for is_max, name in [(False, 'min'), (True, 'max')]: - _FUNCS[f'v_{name}_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}3_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}3_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}imum_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}imum_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) - _FUNCS[f'v_{name}imum3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a]) - _FUNCS[f'v_{name}imum3_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a) +# min/max family: min/max + 3-input (x3), IEEE num variants (f16/f32 only), and long names minimum/maximum (f16/f32 only) +for is_max, name, full in [(False, 'min', 'minimum'), (True, 'max', 'maximum')]: + for dt, sfx, pre in [(dtypes.float32, 'f32', None), (dtypes.int, 'i32', None), (dtypes.uint32, 'u32', None), + (dtypes.int16, 'i16', None), (dtypes.uint16, 'u16', None), (dtypes.half, 'f16', _f16_extract)]: + def mm(*a, im=is_max, d=dt, p=pre): return _minmax_reduce(im, d, *(a if p is None else [p(x) for x in a])) + extra = (f'v_{name}_num_{sfx}', f'v_{name}3_num_{sfx}', f'v_{full}_{sfx}', f'v_{full}3_{sfx}') if dt in (dtypes.float32, dtypes.half) else () + for fn in (f'v_{name}_{sfx}', f'v_{name}3_{sfx}', *extra): _FUNCS[fn] = mm # ═══════════════════════════════════════════════════════════════════════════════ # TOKENIZER/PARSER @@ -890,6 +896,8 @@ def _cmp_nan(self, l: UOp, r: UOp, fn) -> UOp: return result & _isnan(l).logical_not() & _isnan(r).logical_not() return result +_break_var_ids = itertools.count() # unique names for per-loop break-tracking variables + def _match_bracket(toks: list[Token], start: int) -> tuple[int, list[Token]]: """Match brackets from start, return (end_idx, inner_tokens).""" j, depth = start + 1, 1 @@ -987,7 +995,7 @@ def parse_bound(): i += 1 # Execute loop with break support has_break = any('break' in bl.lower() for bl in body_lines) - found_var = f'_found_{id(body_lines)}' if has_break else None + found_var = f'_found_{next(_break_var_ids)}' if has_break else None if found_var: env[found_var] = block_assigns[found_var] = _const(dtypes.bool, False) for loop_i in range(start_val, end_val + 1): subst_lines = [_subst_loop_var(bl, loop_var, loop_i) for bl in body_lines if not (has_break and bl.strip().lower() == 'break')] @@ -1087,7 +1095,7 @@ def parse_bound(): j, slice_toks = _match_bracket(toks, j) slice_str = _tok_str(slice_toks) hi_str, lo_str = slice_str.split(':') - hi_val, lo_val = int(eval(hi_str.strip())), int(eval(lo_str.strip())) + hi_val, lo_val = _const_int(hi_str), _const_int(lo_str) if j < len(toks) and toks[j].type == 'DOT': j += 2 # skip .type suffix if j < len(toks) and toks[j].type == 'EQUALS': j += 1 ln = parse_tokens(lane_toks, env, funcs) @@ -1145,7 +1153,7 @@ def parse_bound(): hi_str = ' '.join(t.val for t in toks[bracket_start:colon_pos] if t.type != 'EOF') lo_str = ' '.join(t.val for t in toks[colon_pos+1:j] if t.type != 'EOF') try: - hi_val, lo_val = int(eval(hi_str)), int(eval(lo_str)) + hi_val, lo_val = _const_int(hi_str), _const_int(lo_str) hi, lo = max(hi_val, lo_val), min(hi_val, lo_val) j += 1 if j < len(toks) and toks[j].type == 'DOT': j += 2 @@ -1159,7 +1167,7 @@ def parse_bound(): block_assigns[var] = env[var] = _set_bits(old, _val_to_bits(val), hi - lo + 1, lo) i += 1 continue - except Exception: pass + except (ValueError, SyntaxError): pass # non-constant slice bounds - fall through to other statement forms elif toks[1].type == 'LBRACKET': # bit index: var[expr] (only for var[...], not var.type[...]) existing = block_assigns.get(var, env.get(var)) if existing is not None and isinstance(existing, UOp) and \ @@ -1360,3 +1368,25 @@ def _cond_side_effect(cnd, dest, val): def parse_expr(expr: str, env: dict[str, VarVal], funcs: dict | None = None) -> UOp: return parse_tokens(tokenize(expr.strip().rstrip(';')), env, funcs) + +def parse_pcode(pcode: str, srcs: dict[str, UOp | int] | None = None) -> tuple[dict, list]: + env: dict = srcs.copy() if srcs else {} + assigns: list[tuple[str, UOp]] = [] + raw_lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')] + # TODO: pcode.py should tokenize full pcode string instead of line-by-line, then this hack can be removed + lines: list[str] = [] + for l in raw_lines: + if lines and re.search(r'(&&|\|\||[&|+\-*/^])\s*$', lines[-1]): lines[-1] = lines[-1] + ' ' + l + else: lines.append(l) + _, final, _ = parse_block(lines, 0, env, assigns=assigns) + sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d) + for var, val in final.items(): + if var in ['D0', 'S0', 'SCC', 'VCC', 'EXEC', 'PC', 'RETURN_DATA', 'VDATA'] and isinstance(val, UOp): + if var in sliced and not any(re.match(rf'{var}\.\w+\s*=', l) for l in lines): continue + for l in lines: + if (m := re.match(rf'{var}\.(\w+(?:\[\w+\])?)', l)): + assigns.append((f'{var}.{m.group(1)}', val)) + break + else: assigns.append((var, val)) + return env, assigns + diff --git a/test/mockgpu/amd/sqtt_enc.py b/test/mockgpu/amd/sqtt_enc.py new file mode 100644 index 0000000000000..924d7ac29274c --- /dev/null +++ b/test/mockgpu/amd/sqtt_enc.py @@ -0,0 +1,100 @@ +# SQTT trace encoder for the emulator (the decoder lives in tinygrad/renderer/amd/sqtt.py). +# run_asm emits packets inline as instructions execute; finished traces end up in emu.sqtt_traces. +from __future__ import annotations +from tinygrad.renderer.amd.dsl import Inst +from tinygrad.renderer.amd.sqtt import (_build_decode_tables, PACKET_TYPES_RDNA3, PacketType, InstOp, + LAYOUT_HEADER, WAVESTART, WAVEEND, INST, IMMEDIATE, VALUINST) + +_NIB_COUNTS = {cls: nc for _, (cls, nc, *_) in _build_decode_tables(PACKET_TYPES_RDNA3)[0].items()} + +def _emit_nibbles(nibbles: list[int], pkt_cls: type[PacketType], **kwargs): + raw = pkt_cls.encoding.default + for k, v in kwargs.items(): raw = pkt_cls.__dict__[k].set(raw, v) + nibbles.extend((raw >> (i * 4)) & 0xF for i in range(_NIB_COUNTS[pkt_cls])) + +def make_encoder(): + """Build an SQTT trace encoder for the emulator. Returns (emit, finish, finalize).""" + from tinygrad.runtime.autogen.amd.rdna3.enum import SOPPOp as SOPPOp3 + from tinygrad.runtime.autogen.amd.rdna4.enum import SOPPOp as SOPPOp4 + from tinygrad.runtime.autogen.amd.rdna3 import ins as ir3 + from tinygrad.runtime.autogen.amd.rdna4 import ins as ir4 + from tinygrad.runtime.autogen.amd.cdna import ins as irc + import re + + def _kinds(*names: str) -> tuple[type[Inst], ...]: + return tuple(getattr(m, n) for m in (ir3, ir4, irc) for n in names if hasattr(m, n)) + _SOPP, _SMEM, _DS = _kinds('SOPP'), _kinds('SMEM'), _kinds('DS') + _GLOBAL, _FLAT, _SCRATCH = _kinds('GLOBAL', 'VGLOBAL'), _kinds('FLAT', 'VFLAT'), _kinds('SCRATCH', 'VSCRATCH') + _VALU = _kinds('VOP1', 'VOP2', 'VOP3', 'VOP3P', 'VOP3PX2', 'VOPC', 'VOPD', 'VOP3SD', 'VOP3_SDST', 'VOP1_SDST') + + # SOPP classification sets + _SOPP_SKIP = {SOPPOp3.S_ENDPGM.value, SOPPOp3.S_ENDPGM_SAVED.value, SOPPOp3.S_ENDPGM_ORDERED_PS_DONE.value, SOPPOp3.S_DELAY_ALU.value} + _SOPP_IMMEDIATE = {SOPPOp3.S_NOP.value, SOPPOp3.S_CLAUSE.value, SOPPOp3.S_WAITCNT.value, SOPPOp3.S_WAITCNT_DEPCTR.value, + SOPPOp3.S_WAIT_IDLE.value, SOPPOp3.S_WAIT_EVENT.value, SOPPOp3.S_SLEEP.value, SOPPOp3.S_SET_INST_PREFETCH_DISTANCE.value} + for _op in (SOPPOp4.S_WAIT_ALU, SOPPOp4.S_WAIT_LOADCNT, SOPPOp4.S_WAIT_STORECNT, SOPPOp4.S_WAIT_SAMPLECNT, + SOPPOp4.S_WAIT_BVHCNT, SOPPOp4.S_WAIT_EXPCNT, SOPPOp4.S_WAIT_DSCNT, SOPPOp4.S_WAIT_KMCNT, + SOPPOp4.S_WAIT_LOADCNT_DSCNT, SOPPOp4.S_WAIT_STORECNT_DSCNT): + _SOPP_IMMEDIATE.add(_op.value) + _SOPP_BARRIER = {SOPPOp3.S_BARRIER.value} + if hasattr(SOPPOp4, 'S_BARRIER_WAIT'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_WAIT.value) + if hasattr(SOPPOp4, 'S_BARRIER_LEAVE'): _SOPP_BARRIER.add(SOPPOp4.S_BARRIER_LEAVE.value) + _SOPP_BRANCH = {SOPPOp3.S_BRANCH.value, SOPPOp3.S_CBRANCH_SCC0.value, SOPPOp3.S_CBRANCH_SCC1.value, + SOPPOp3.S_CBRANCH_VCCZ.value, SOPPOp3.S_CBRANCH_VCCNZ.value, + SOPPOp3.S_CBRANCH_EXECZ.value, SOPPOp3.S_CBRANCH_EXECNZ.value} + + # VALU sub-classification patterns + _VALUT_4_RE = re.compile(r'V_(EXP|LOG|RCP|RSQ|SQRT|SIN|COS|CEIL|FLOOR|TRUNC|RNDNE|FRACT|FREXP)_') + _VALUB_2_RE = re.compile(r'V_(LSHLREV|LSHRREV|ASHRREV)_(B|I)64') + _VALUB_4_RE = re.compile(r'V_MAD_(U|I)64') + _VALUB_16_RE = re.compile(r'V_\w+_F64') + + def _valu_op(op_name: str) -> InstOp|None: + if 'CMPX' in op_name: return InstOp.VALU1_WR_EXEC + if _VALUB_2_RE.search(op_name): return InstOp.VALUB_2 + if _VALUB_4_RE.search(op_name): return InstOp.VALUB_4 + if _VALUB_16_RE.search(op_name): return InstOp.VALUB_16 + if _VALUT_4_RE.search(op_name): return InstOp.VALUT_4 + return None + + def _mem_op(t: type[Inst], op_name: str) -> InstOp: + is_store = "STORE" in op_name + if issubclass(t, _DS): return InstOp.LDS_WR_2 if is_store else InstOp.LDS_RD + if issubclass(t, _GLOBAL): return InstOp.SGMEM_WR_2 if is_store else InstOp.SGMEM_RD_1 + if issubclass(t, _FLAT) or issubclass(t, _SCRATCH): return InstOp.FLAT_WR_3 if is_store else InstOp.FLAT_RD_2 + return InstOp.SALU + + nibbles: list[int] = [] + started: set[int] = set() + _emit_nibbles(nibbles, LAYOUT_HEADER, layout=3, sel_a=6) + + def emit(wave_id: int, inst: Inst, branch_taken: bool|None): + """Emit an SQTT packet for one executed instruction.""" + w = wave_id & 0x1F + if wave_id not in started: + _emit_nibbles(nibbles, WAVESTART, delta=1, simd=0, wgp=0, wave=w, id7=wave_id) + started.add(wave_id) + inst_type, inst_op, op_name = type(inst), inst.op.value if hasattr(inst, 'op') else 0, inst.op.name if hasattr(inst, 'op') else "" + if issubclass(inst_type, _SOPP): + if inst_op in _SOPP_SKIP: return + if inst_op in _SOPP_IMMEDIATE: _emit_nibbles(nibbles, IMMEDIATE, delta=1, wave=w) + elif inst_op in _SOPP_BARRIER: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.BARRIER) + elif inst_op in _SOPP_BRANCH: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.JUMP if branch_taken else InstOp.JUMP_NO) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SALU) + elif issubclass(inst_type, _VALU): + if (op := _valu_op(op_name)) is None: _emit_nibbles(nibbles, VALUINST, delta=1, wave=w) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=op) + elif issubclass(inst_type, _SMEM): _emit_nibbles(nibbles, INST, delta=1, wave=w, op=InstOp.SMEM_RD) + else: _emit_nibbles(nibbles, INST, delta=1, wave=w, op=_mem_op(inst_type, op_name)) + + def finish(wave_id: int): + """Emit WAVEEND for a completed wave.""" + if wave_id in started: _emit_nibbles(nibbles, WAVEEND, delta=1, simd=0, wgp=0, wave=wave_id & 0x1F) + + def finalize() -> bytes: + """Pad and return the encoded SQTT blob.""" + while len(nibbles) % 2 != 0: nibbles.append(0) + nibbles.extend([0] * 32) + while len(nibbles) % 64 != 0: nibbles.append(0) + return bytes(nibbles[i] | ((nibbles[i + 1] if i + 1 < len(nibbles) else 0) << 4) for i in range(0, len(nibbles), 2)) + + return emit, finish, finalize From 7204d467866cd338be5018cf0c3444fb7802b05d Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 21:08:06 -0400 Subject: [PATCH 160/306] delete dtype_from_uop INDEX exempt (#17674) --- test/unit/test_dtype_weak.py | 7 ------- tinygrad/uop/ops.py | 7 ++----- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 59cc8f40f3bd3..7851310666759 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -114,13 +114,6 @@ def test_uop_scalar_const_lifts_kind(self): self.assertIsInstance((x + 2).src[1].val, float) self.assertIs(x + UOp.const(2), x + 2) - def test_index_dtype_ignores_weakness(self): - with Context(SPEC=2): - idx = UOp.const(0).cast(dtypes.int32) - weak = UOp.const(1.0).expand((1,)) - self.assertEqual(UOp(Ops.INDEX, dtypes.float32, (weak, idx)).dtype, dtypes.float32) - with self.assertRaisesRegex(RuntimeError, "bad dtype"): UOp(Ops.INDEX, dtypes.int32, (weak, idx)) - def test_store_weak_value_uses_destination_dtype(self): with Context(DEFAULT_FLOAT=dtypes.float16): dst = UOp.param(0, dtypes.bfloat16, (1,)).index(UOp.const(0).cast(dtypes.int32)) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 75a8e26a3e86f..9102f2af5b977 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -5,7 +5,7 @@ from enum import Enum, auto from tinygrad.uop import Ops, GroupOp from tinygrad.dtype import ConstType, dtypes, DType, DTypeLike, truncate, least_upper_dtype, least_upper_float, Invalid, AddrSpace, strong_dtype -from tinygrad.dtype import PyConst, InvalidType, weak_dtype, bitcast +from tinygrad.dtype import PyConst, InvalidType, bitcast from tinygrad.device import Buffer, MultiBuffer, canonicalize_device, TinyELF from tinygrad.helpers import ContextVar, all_int, prod, getenv, all_same, Context, partition, temp, unwrap, T, argfix, Metadata, flatten, TRACEMETA from tinygrad.helpers import PROFILE, dedup, cdiv, cmod, floordiv, floormod, diskcache_put, to_function_name, cpu_profile, TracingKey @@ -190,10 +190,7 @@ def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg if dtype is None: dtype = dtype_from_uop(op, src, arg) or dtypes.void # CONST derives its dtype by value only when the constructor omits one # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py - # an INDEX presents its access dtype, which a still-weak source matches up to weakness - if SPEC == 2 and op is not Ops.CONST and \ - (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype and \ - not (op is Ops.INDEX and weak_dtype(expected_dtype) == weak_dtype(dtype)): + if SPEC == 2 and op is not Ops.CONST and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype: raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(*key)) From 356f66537713ac20ddea532cbe56c3b73388f0f9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 21:50:29 -0400 Subject: [PATCH 161/306] test update for weak const (#17675) --- test/mockgpu/amd/emu.py | 2 +- test/null/test_const_folding.py | 7 ++----- test/unit/test_dtype_weak.py | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/mockgpu/amd/emu.py b/test/mockgpu/amd/emu.py index 5874b8b8f335a..d96841694b467 100644 --- a/test/mockgpu/amd/emu.py +++ b/test/mockgpu/amd/emu.py @@ -572,7 +572,7 @@ def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: for dest, val in assigns: # VGPR bit-slice assignment: VGPR[lane][reg][hi:lo] -> read-modify-write with optional condition if dest.startswith('VGPR[') and re.search(r'\[\d+:\d+\]', dest): - hi_bit, lo_bit = int(val[2].val), int(val[3].val) + hi_bit, lo_bit = int(val[2]), int(val[3]) new_val = _set_bits(self.vgpr.index(val[0]).load(), _val_to_bits(val[1]), hi_bit - lo_bit + 1, lo_bit).cast(dtypes.uint32) active = _lane_active(exec_mask, lane) & _to_bool(val[4]) if len(val) > 4 else _lane_active(exec_mask, lane) lane_stores.append(self.vgpr.index(val[0].valid(active)).store(new_val)) diff --git a/test/null/test_const_folding.py b/test/null/test_const_folding.py index e173114a57197..e7072dae05c38 100644 --- a/test/null/test_const_folding.py +++ b/test/null/test_const_folding.py @@ -3,7 +3,6 @@ from tinygrad.dtype import DType, ConstType from tinygrad.uop.ops import Ops, UOp from test.helpers import full_rewrite -import numpy as np class TestWeakConstFolding(unittest.TestCase): def test_weakint_math(self): @@ -27,16 +26,14 @@ def test_out_of_range_source_value(self): for val, src_dt, dst_dt, bits in ((3000000000, dtypes.int32, dtypes.uint32, 3000000000), (70000, dtypes.int16, dtypes.uint16, 4464), (-5, dtypes.uint32, dtypes.int32, -5)): - self.assertEqual(UOp.const(val, src_dt).bitcast(dst_dt).simplify().val, bits) + self.assertIs(UOp.const(val, src_dt).bitcast(dst_dt).simplify(), UOp.const(bits, dst_dt)) def test_scalar_bitcast(self): def t(cases: dict[DType, ConstType]): for (from_dt, from_v), (to_dt, to_v) in itertools.product(cases.items(), cases.items()): if not math.isnan(from_v): r = UOp.const(from_v, from_dt).bitcast(to_dt).simplify() - self.assertEqual(r.op, Ops.CONST, msg:=f"{from_dt} -> {to_dt} ({from_v} -> {to_v})") - self.assertEqual(r.dtype, to_dt, msg) - np.testing.assert_equal(r.val, to_v, msg) + self.assertIs(r, UOp.const(to_v, to_dt), f"{from_dt} -> {to_dt} ({from_v} -> {to_v})") t({dtypes.int8: 0, dtypes.uint8: 0, dtypes.bool: False}) t({dtypes.int8: 1, dtypes.uint8: 1, dtypes.bool: True}) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 7851310666759..92eb70234b655 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -118,13 +118,13 @@ def test_store_weak_value_uses_destination_dtype(self): with Context(DEFAULT_FLOAT=dtypes.float16): dst = UOp.param(0, dtypes.bfloat16, (1,)).index(UOp.const(0).cast(dtypes.int32)) gate = UOp.const(True) - out = graph_rewrite(dst.store(UOp.const(5.0), gate), pm_lower_index_dtype, ctx={}) + out = graph_rewrite(dst.store(UOp.const(5.0), gate), pm_commit_weak) # a bare weak CONST commits directly: the pass runs without symbolic, so a CAST here would survive it self.assertEqual((out.src[1], out.src[2]), (UOp.const(5.0, dtypes.bfloat16), gate)) def test_weak_srcs_commit_only_at_a_concrete_lub(self): weak_lub = UOp(Ops.ADD, src=(UOp.const(1), UOp.const(1.0))) - self.assertIs(graph_rewrite(weak_lub, pm_lower_index_dtype, ctx={}), weak_lub) + self.assertIs(graph_rewrite(weak_lub, pm_commit_weak), weak_lub) concrete = UOp.const(2.0).cast(dtypes.float16) where = graph_rewrite(UOp(Ops.WHERE, src=(UOp.const(True), concrete, UOp.const(1.0))), pm_lower_index_dtype, ctx={}) self.assertEqual(tuple(x.dtype for x in where.src), (dtypes.bool, dtypes.float16, dtypes.float16)) From 8950942e75fe861a615b53696db6facbfd0f7ca7 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 21 Aug 2026 22:27:36 -0400 Subject: [PATCH 162/306] remove explicit dtype for NOOP and decomp [PR] (#17678) --- tinygrad/codegen/decomp/dtype.py | 4 ++-- tinygrad/engine/jit.py | 2 +- tinygrad/schedule/rangeify.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 00f7e8a059b27..7dcdf4dd04894 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -139,7 +139,7 @@ def f2f_store(st, idx, val, fr:DType, to:DType): (UPat(GroupOp.Defines, src=(UPat.var("sz"),), name="x"), lambda x,sz: x.replace(dtype=l2i_dt[x.dtype], arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(sz*2,)) if x.dtype in l2i_dt else None), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: - reindex(x, x.tag[0]).replace(dtype=x.tag[1], tag=None) if x.tag is not None else None), + reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: st.replace(src=(idx.rtag((0, dt:=l2i_dt[idx.dtype])), val.rtag((0, dt)))).group( st.replace(src=(idx.rtag((1, dt)), val.rtag((1, dt))))) if val.tag is None else None), @@ -160,7 +160,7 @@ def f2f_store(st, idx, val, fr:DType, to:DType): split_l2i(ctx, x.op, l2i_dt[x.dtype], *flatten((a.rtag((0, l2i_dt[x.dtype])), a.rtag((1, l2i_dt[x.dtype]))) for a in x.src))[x.tag[0]] if x.tag is not None else None), (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda x,idx: - x.replace(dtype=l2i_dt[x.dtype], src=(reindex(idx, x.tag[0]).replace(dtype=l2i_dt[x.dtype], tag=None),), tag=None) if x.tag is not None else None), + x.replace(dtype=l2i_dt[x.dtype], src=(reindex(idx, x.tag[0]).replace(tag=None),), tag=None) if x.tag is not None else None), (UPat(Ops.CONST, tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), lambda x: UOp.const(truncate[x.tag[1]]((x.val >> 32) if x.tag[0] == 1 else (x.val & 0xFFFFFFFF)), x.tag[1])) ]) diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index a543623072fa5..221889f7d677b 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -211,7 +211,7 @@ def get_input_uops() -> list[UOp]: return flatten([[t.uop.src[0]] if t.uop.op is # collect buffer UOps (including MultiBuffer) input_buf_uops: list[UOp] = [u.base for u in input_uops if u.base.realized is not None] if len(set(input_buf_uops)) != len(input_buf_uops): raise JitError("duplicate inputs to JIT") - inputs = [(*(u.substitute({u.base:UOp(Ops.NOOP, u.base.dtype)}, extra_pm=mop_cleanup).unbind_all()), u.dtype, u.device) for u in input_uops] + inputs = [(*(u.substitute({u.base:UOp(Ops.NOOP)}, extra_pm=mop_cleanup).unbind_all()), u.dtype, u.device) for u in input_uops] _var_vals = merge_dicts([x[1] for x in inputs] + [dict(v.unbind() for v in (args + tuple(kwargs.values())) if isinstance(v, UOp))]) var_vals = {k.expr:v for k,v in _var_vals.items()} expected_input_info = [(x[0], tuple(sorted(x[1].keys(), key=lambda v: v.expr)), x[2], x[3]) for x in inputs] diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 979cef04c0fa4..4751e44212e47 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -79,7 +79,7 @@ def split_reduceop(reduce:UOp, x:UOp): # get expanded by rangeifying the UOp x indexed = x.index(*[UOp.range(s, i) if resolve(s>1) else 0 for i,s in enumerate(x.shape)]) - range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP, x.base.dtype)}, extra_pm=pm_mops).ranges] + range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP)}, extra_pm=pm_mops).ranges] is_expanded = [i not in range_nums for i in range(len(x.shape))] if not (split_candidates:=[(i,d) for i in range(reduce.arg[1]) From a9069c177a9da9cca18593edf55acd2e6073cca6 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 22 Aug 2026 08:01:15 -0400 Subject: [PATCH 163/306] make decomp pass SPEC=2 [PR] (#17681) --- tinygrad/codegen/decomp/dtype.py | 46 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 7dcdf4dd04894..64b46fa758a3d 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -1,8 +1,8 @@ from dataclasses import replace from tinygrad.dtype import dtypes, DType, truncate -from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES, Context, SPEC +from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp -from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite, ParamArg +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite from tinygrad.renderer import Renderer from tinygrad.codegen.decomp.transcendental import exponent_bias, shl, shr @@ -127,17 +127,18 @@ def f2f_clamp(val:UOp, dt:DType, sat=True) -> UOp: return val.ne(val).where(val, (val < -mx).where(-sat, (mx < val).where(sat, val))) def f2f_load(x: UOp, fr:DType, to:DType) -> UOp: - if (n:=x.max_numel()) == 1: return f2f(x.replace(dtype=f2f_dt[fr]), fr, to) - return UOp(Ops.STACK, src=tuple(f2f(x.replace(dtype=f2f_dt[fr], src=(reindex(x.src[0], i, 1),)), fr, to) for i in range(n))) + storage_idx = graph_rewrite(x.src[0], pm_float_decomp, ctx=(fr, to), bottom_up=True) + if (n:=x.max_numel()) == 1: return f2f(storage_idx.load(*x.src[1:]), fr, to) + return UOp(Ops.STACK, src=tuple(f2f(reindex(storage_idx, i, 1).load(*x.src[1:]), fr, to) for i in range(n))) def f2f_store(st, idx, val, fr:DType, to:DType): if (n:=val.max_numel()) == 1: return st.replace(src=(idx, f2f(val.bitcast(f2f_dt[to]), to, fr))) return UOp.group(*(st.replace(src=(reindex(idx, i, 1), f2f(val.index(i).bitcast(f2f_dt[to]), to, fr))) for i in range(n))) # tag is the 32-bit word this node becomes - (0 for the low word, 1 for the high, the dtype the consumer wants) -pm_long_decomp = PatternMatcher([ +pm_long_decomp: PatternMatcher = PatternMatcher([ (UPat(GroupOp.Defines, src=(UPat.var("sz"),), name="x"), lambda x,sz: - x.replace(dtype=l2i_dt[x.dtype], arg=replace(x.arg, dtype=l2i_dt[x.dtype]), src=(sz*2,)) if x.dtype in l2i_dt else None), + UOp(x.op, src=(sz*2,), arg=replace(x.arg, dtype=l2i_dt[x.dtype]), tag=x.tag) if x.dtype in l2i_dt else None), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: @@ -159,21 +160,23 @@ def f2f_store(st, idx, val, fr:DType, to:DType): (UPat((*(GroupOp.ALU - GroupOp.Comparison - {Ops.SHL, Ops.SHR, Ops.WHERE}), Ops.BITCAST), tuple(l2i_dt.keys()), name="x"), lambda ctx,x: split_l2i(ctx, x.op, l2i_dt[x.dtype], *flatten((a.rtag((0, l2i_dt[x.dtype])), a.rtag((1, l2i_dt[x.dtype]))) for a in x.src))[x.tag[0]] if x.tag is not None else None), - (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda x,idx: - x.replace(dtype=l2i_dt[x.dtype], src=(reindex(idx, x.tag[0]).replace(tag=None),), tag=None) if x.tag is not None else None), + (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda ctx,x,idx: + reindex(graph_rewrite(idx, pm_long_decomp, ctx=ctx, bottom_up=True), x.tag[0]).replace(tag=None).load() if x.tag is not None else None), (UPat(Ops.CONST, tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), lambda x: UOp.const(truncate[x.tag[1]]((x.val >> 32) if x.tag[0] == 1 else (x.val & 0xFFFFFFFF)), x.tag[1])) ]) # float decomposition patterns - ctx is (fr, to) tuple -pm_float_decomp = PatternMatcher([ - (UPat((*GroupOp.Defines, Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: - x.replace(dtype=f2f_dt[ctx[0]], arg=replace(x.arg, dtype=f2f_dt[ctx[0]]) if isinstance(x.arg, ParamArg) else x.arg, tag=ctx[0]) - if x.dtype == ctx[0] and (x.op is not Ops.INDEX or x.src[0].op not in {Ops.LOAD, Ops.STACK}) else None), +pm_float_decomp: PatternMatcher = PatternMatcher([ + (UPat(GroupOp.Defines, name="x"), lambda ctx,x: + UOp(x.op, src=x.src, arg=replace(x.arg, dtype=f2f_dt[ctx[0]]), tag=ctx[0]) if x.dtype == ctx[0] else None), + (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: + UOp(x.op, src=(graph_rewrite(x.src[0], pm_float_decomp, ctx=ctx, bottom_up=True), *x.src[1:]), arg=x.arg, tag=ctx[0]) if + x.dtype == ctx[0] and (x.op is not Ops.INDEX or x.src[0].op not in {Ops.LOAD, Ops.STACK}) else None), (UPat(Ops.LOAD, dtypes.floats, name="x"), lambda ctx,x: f2f_load(x, *ctx) if x.dtype == ctx[0] else None), # bitcasted load should just replace load (UPat(Ops.BITCAST, src=(UPat(Ops.LOAD, name="ld"),), name="bc"), lambda ctx,bc,ld: - ld.replace(dtype=f2f_dt[ctx[0]]).bitcast(bc.dtype) if ld.dtype == ctx[0] else None), + graph_rewrite(ld.src[0], pm_float_decomp, ctx=ctx, bottom_up=True).load(*ld.src[1:]).bitcast(bc.dtype) if ld.dtype == ctx[0] else None), # bitcast from (UPat(Ops.BITCAST, src=(UPat.var("x", dtypes.floats),), name="bc"), lambda ctx,bc,x: bc.replace(src=(f2f(x.bitcast(f2f_dt[ctx[1]]), ctx[1], ctx[0]),)) if x.dtype == ctx[1] and bc.dtype.bitsize == ctx[0].bitsize else None), @@ -185,23 +188,20 @@ def f2f_store(st, idx, val, fr:DType, to:DType): # a CONST has no srcs to cast, it restates its value at the emulating dtype (UPat(Ops.CONST, dtypes.floats, name="x"), lambda ctx,x: UOp.const(x.val, ctx[1]) if x.dtype == ctx[0] else None), (UPat(GroupOp.All-GroupOp.Defines-{Ops.CAST, Ops.BITCAST, Ops.CONST}, dtypes.floats, name="x"), lambda ctx,x: - x.replace(dtype=ctx[1], src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src)) - if x.dtype == ctx[0] else None), + UOp(x.op, src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src), arg=x.arg, tag=x.tag) if x.dtype == ctx[0] else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat(Ops.BITCAST, dtypes.floats, name="val")), name='st'), lambda ctx,st,idx,val: - st.replace(src=(idx, val.replace(dtype=f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), + st.replace(src=(idx, val.src[0].bitcast(f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and (idx:=idx.src[0] if idx.op == Ops.CAST else idx).tag == ctx[0] else None), ]) def do_dtype_decomps(sink:UOp, ctx:tuple[set[DType], Renderer]) -> UOp: def _should_emulate(dt): return dt in EMULATED_DTYPES.tolist(dtypes) or dt not in ctx[1].supported_dtypes() - # NOTE: dtype decomp creates intermediate UOps that don't follow the spec (e.g. half LOAD on ushort BUFFER) - with Context(SPEC=min(SPEC.value, 1)): - for fr in sorted(filter(_should_emulate, ctx[0])): - to = dtypes.int if fr == dtypes.long else dtypes.half if not _should_emulate(dtypes.half) and fr in dtypes.fp8s else dtypes.float - if DEBUG >= 2: print(f"emulating {fr} as {to}") - pm = pm_float_decomp if fr in dtypes.floats else pm_long_decomp - sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx={} if pm is pm_long_decomp else (fr, to), bottom_up=True) + for fr in sorted(filter(_should_emulate, ctx[0])): + to = dtypes.int if fr == dtypes.long else dtypes.half if not _should_emulate(dtypes.half) and fr in dtypes.fp8s else dtypes.float + if DEBUG >= 2: print(f"emulating {fr} as {to}") + pm = pm_float_decomp if fr in dtypes.floats else pm_long_decomp + sink = graph_rewrite(sink, pm, name=f"decomp {fr} -> {to}", ctx={} if pm is pm_long_decomp else (fr, to), bottom_up=True) ctx[0].clear() return sink From a2e64e16aa1845b06991d194c81a67f211b89790 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:22:39 +0300 Subject: [PATCH 164/306] hcq2: early usb (#17683) * hcq2: usb interface and submit * x * x * x * x * r * x --- extra/hcq2/ops_amd2.py | 85 ++++++++++++++++++----- tinygrad/runtime/ops_cpu.py | 8 +-- tinygrad/runtime/support/hcq2.py | 75 +++++++++++++-------- tinygrad/runtime/support/usb.py | 112 ++++++++++++++++++++++++++++++- tinygrad/uop/ops.py | 4 +- 5 files changed, 231 insertions(+), 53 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index e603df20a42eb..c4f7b6bc3b3d0 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -19,7 +19,7 @@ from tinygrad.runtime.support.am.amdev import AMDev, AMMemoryManager from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_soc, import_pmc from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, USBPCIDevice, MAP_FIXED, MAP_NORESERVE -from tinygrad.runtime.support.usb import USB3 +from tinygrad.runtime.support.usb import USB3, usb_ib, usb_push, usb_arm_bytes, pm_usb_stage, pm_usb_hostio, pm_usb_bufferize from tinygrad.runtime.support.memory import AddrSpace, BumpAllocator from tinygrad.runtime.ops_amd import SQTT, SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE, PMC from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_EQ, WAIT_REG_MEM_FUNCTION_NEQ, WAIT_REG_MEM_FUNCTION_GEQ @@ -146,11 +146,14 @@ def pm4_program(ctx, call, prg): (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), ]) +def queue_ptrs(devs, qname:str, q:AMDQueueDesc) -> tuple[UOp, ...]: + return tuple(UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"{qname}_{n}") + for n, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) + def pm4_submit(ctx, lin): # ensure compute queues are allocated for d in (devs:=ctx.devs): q = Device[d].compute_queue - ring, wptr, doorbell, put_ptr = (UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"COMPUTE:0_{name}") - for name, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) + ring, wptr, doorbell, put_ptr = queue_ptrs(devs, "COMPUTE:0", q) # the host fence at the start of the batch guarantees the ib is free to reuse size_dw = sum(len(ins.src) for ins in lin.src) @@ -216,8 +219,7 @@ def sdma_submit(cmdbuf, devs): # the sdma queue's ring and its host-side ring/write/put pointers for d in devs: q = Device[d].sdma_queue(0) - ring, wptr, doorbell, put_ptr = (UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"COPY:0_{name}") - for name, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) + ring, wptr, doorbell, put_ptr = queue_ptrs(devs, "COPY:0", q) # sdma needs the cmdbuf contiguous: if it won't fit before the ring end, restart at 0 and zero the tail put_b = put_ptr.index(zero) @@ -244,15 +246,32 @@ def sdma_submit(cmdbuf, devs): pm_sdma_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), lambda ctx, lin: sdma_submit(make_cmdbuf(lin, ctx.devs), ctx.devs))]) +# ***************** +# USB submit + +def amd_usb_submit(ctx, lin): + for d in ctx.devs: q = Device[d].compute_queue if (comp:=ctx.qname.startswith("COMPUTE")) else Device[d].sdma_queue(0) + + if nb:=usb_arm_bytes(ctx.pre, Device[ctx.devs[0]].iface.usb_sram): + poke = (ctx.sdma.SDMA_OP_WRITE, *data64_le(Device[ctx.devs[0]].iface.cq_buf.va_addr + 12), 0, 0) + lin = lin.replace(src=lin.src + (UOp(Ops.INS, arg="poke", src=tuple(UOp.const(x, dtypes.uint32) for x in poke)),)) + + ib_host, ib_gpu, pkt_dw = usb_ib(ctx.devs, lin, 32 if comp else 0x100, nb) + pkt = (ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER,2),*data64_le(ib_gpu.getaddr(ctx.devs)),pkt_dw|ctx.pm4.INDIRECT_BUFFER_VALID) if comp else () + return usb_push(ctx.devs, *queue_ptrs(ctx.devs, ctx.qname, q), ib_host, ib_gpu, pkt, 4 if comp else 1) + +pm_usb_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), amd_usb_submit)]) + @dataclass(frozen=True) class AMDEncodeCtx: # encode-time constants for one queue: devs (every cmdbuf address resolves into these) + gfx version + packet/ip modules devs: tuple[str, ...]; target: tuple[int, ...]; pm4: Any; sdma: Any; soc: Any # noqa: E702 - gc: AMDIP; nbio: AMDIP; xccs: int; max_copy_size: int; tmpring_size: Callable # noqa: E702 + gc: AMDIP; nbio: AMDIP; xccs: int; max_copy_size: int; tmpring_size: Callable; qname: str; pre: UOp # pre: the queue before opsel def encode_queue(q:UOp) -> UOp|None: d = Device[(devs:=to_tuple(q.arg[0]))[0]] - ctx = AMDEncodeCtx(devs, d.target, d.pm4, d.sdma, d.soc, d.gc, d.nbio, d.xccs, d.max_copy_size, d.tmpring_size) - opsel, submit = (pm_pm4_opsel, pm_pm4_submit) if q.arg[1].startswith("COMPUTE") else (pm_sdma_opsel, pm_sdma_submit) + ctx = AMDEncodeCtx(devs, d.target, d.pm4, d.sdma, d.soc, d.gc, d.nbio, d.xccs, d.max_copy_size, d.tmpring_size, q.arg[1], q) + opsel = pm_pm4_opsel if (comp:=q.arg[1].startswith("COMPUTE")) else pm_sdma_opsel + submit = d.pm_submit if d.pm_submit is not None else (pm_pm4_submit if comp else pm_sdma_submit) return submit.rewrite(graph_rewrite(q, opsel + pm_flatten_linear, walk=True, ctx=ctx, name=f"{q.arg[1]} opsel"), ctx) @dataclass(frozen=True) @@ -282,13 +301,14 @@ def amd_build_program(prg:UOp) -> UOp: wave32=bool(desc.kernel_code_properties & 0x400), private_segment_size=desc.private_segment_fixed_size, kernargs_segment_size=desc.kernarg_size, kernargs_alloc_size=desc.kernarg_size + (ctypes.sizeof(hsa.hsa_kernel_dispatch_packet_t) if edp else 0), enable_dispatch_ptr=edp, enable_private_segment_sgpr=desc.kernel_code_properties & hsa.AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER) + image = bytes(image).ljust(round_up(len(image), 4), b"\x00") # the program is uploaded as whole dwords buf = UOp.placeholder((len(image),), dtypes.uint8, next(UOp.unique_num), device=prg.device).rtag("program") - cached = _amd_program_cache[key] = prg.replace(src=(buf.after(make_binary_patch(buf, bytes(image))),), arg=(data, prg.arg)) + cached = _amd_program_cache[key] = prg.replace(src=(buf.after(make_binary_patch(buf, image)),), arg=(data, prg.arg)) return cached class AMDAllocator(HCQAllocator['AMDDevice']): def __init__(self, dev:AMDDevice): - super().__init__(dev, supports_copy_from_disk=dev.has_copy_queue, supports_transfer=dev.has_copy_queue and not dev.is_usb()) + super().__init__(dev, supports_copy_from_disk=dev.has_copy_queue, supports_transfer=dev.has_copy_queue and not dev.is_usb) def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer: return self.dev.iface.alloc(size, host=options.host, uncached=options.uncached, cpu_access=options.cpu_access or not self.dev.has_copy_queue) @@ -524,8 +544,7 @@ def _collect_interrupts(self, reset=False, drain_only=False): cq = d.compute_queue for b in (cq.put_value, cq.read_ptr, cq.write_ptr): b._buf.view.view(fmt='Q')[0] = 0 d.iface.dev_impl.gfx.setup_ring(*cq.params) - d.signal('timeline')._buf.cpu_view().mv.cast('Q')[0] = \ - d.signal('value', 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] - 1 + d.signal('timeline')._buf.cpu_view().view(fmt='Q')[0] = d.signal('value', 1, device="CPU")._buf.cpu_view().view(fmt='Q')[0] - 1 def sleep(self, timeout): if hasattr(self.pci_dev, 'irq_poller') and self.pci_dev.irq_poller is not None and (events_cnt:=len(self.pci_dev.irq_poller.poll(timeout))): @@ -539,6 +558,32 @@ def on_device_hang(self): def device_fini(self): self.dev_impl.fini() +class USBIface(PCIIface): + def __init__(self, dev, dev_id): # pylint: disable=super-init-not-called + if dev_id >= len(visible:=hcq_filter_visible_devices(USB3.list_devices(0xADD1, 0x0001) + USB3.list_devices(0x3801, 0x0001), "AMD")): + raise RuntimeError(f"AMD:{dev_id} does not exist ({pluralize('device', len(visible))} available)") + self.dev, self.pci_dev, self.vram_bar, self.count = dev, USBPCIDevice("AM", *visible[dev_id]), 0, len(visible) + self.dev_impl = AMDev(self.pci_dev) + self._compute_props() + self.sram = self._dma_region(ctrl_addr=0xf000, sys_addr=0x200000, size=0x80000) + self.cq_buf = self._dma_region(ctrl_addr=0xb800, sys_addr=0x822000, size=0x1000) # +12 is the dword that releases an armed read + self.usb_handle = unwrap(ctypes.cast(self.pci_dev.usb.usb.handle, ctypes.c_void_p).value) + + def _dma_region(self, ctrl_addr, sys_addr, size): + region = self.dev_impl.mm.map_range(vaddr:=self.dev_impl.mm.alloc_vaddr(size=size), size, [(sys_addr, size)], aspace=AddrSpace.SYS, uncached=True) + return HCQBuffer(vaddr, size, meta=PCIAllocationMeta(region, has_cpu_mapping=False), view=self.pci_dev.dma_view(ctrl_addr, size), owner=self.dev) + + def alloc(self, size:int, host=False, uncached=False, cpu_access=False, contiguous=False, force_devmem=False, **kwargs) -> HCQBuffer: + # everything, even host-style signals, lives in vram: gpu writes into the bridge's own memory collide with an armed 0xF2 read stream + return super().alloc(size, host=False, uncached=uncached, cpu_access=cpu_access or host, contiguous=contiguous, force_devmem=True, **kwargs) + + def sleep(self, timeout): pass + + # we don't own the sram region, so the buffer never frees it + @functools.cached_property + def usb_sram(self) -> Buffer: + return Buffer(self.dev.device, (b:=self.sram).size, dtypes.uint8, options=BufferSpec(external_ptr=b.va_addr, nolru=True)).allocate(opaque=b) + def _mock(iface, name=None): return type(name or f"MOCK{iface.__name__}", (iface,), {}) class AMDDevice(HCQ2Compiled): @@ -549,19 +594,21 @@ class AMDDevice(HCQ2Compiled): # encoding of cmdbuf (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue), ]) + pm_submit: PatternMatcher|None = None timestamp_divider = 100.0 # AMD GPU clock: ticks/us max_scratch_psize = 0 - ifaces = [KFDIface, PCIIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface)] + ifaces = [KFDIface, PCIIface, USBIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface), _mock(USBIface)] def device_props(self): return self.iface.props def is_am(self) -> bool: return isinstance(self.iface, (PCIIface,)) - def is_usb(self) -> bool: return False def __init__(self, device:str=""): self.iface = self._select_iface(device) + self.is_usb = isinstance(self.iface, USBIface) + if self.is_usb: self.rt_nbytes = 4 << 20 self.target:tuple[int, ...] = ((trgt:=self.iface.props['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100) self.arch = "gfx%d%x%x" % self.target @@ -586,7 +633,7 @@ def __init__(self, device:str=""): self.is_aql = getenv("AMD_AQL", int(self.xccs > 1)) if self.is_aql: - self.pm4_ibs = self.iface.alloc(0x2000 if self.is_usb() else (16 << 20), uncached=True, cpu_access=True) + self.pm4_ibs = self.iface.alloc(0x2000 if self.is_usb else (16 << 20), uncached=True, cpu_access=True) self.pm4_ib_alloc = BumpAllocator(self.pm4_ibs.size, wrap=True) self.max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000 @@ -599,6 +646,10 @@ def __init__(self, device:str=""): self.max_private_segment_size = 0 self.pm_bufferize = PatternMatcher([(UPat(Ops.PARAM, tag="scratch", name="b"), lambda ctx, b: ctx[0].scratch_buffer(b.max_numel()))]) + self.pm_bufferize + if self.is_usb: + self.pm_bufferize = pm_usb_bufferize + self.pm_bufferize + self.pm_stage_copy, self.pm_host_lower, self.pm_submit = pm_usb_stage, pm_usb_hostio, pm_usb_submit + self.pmc_enabled:bool = PROFILE > 0 and PMC > 0 if self.pmc_enabled: self.iface.require_profile_mode() @@ -659,7 +710,7 @@ def compute_queue(self) -> AMDQueueDesc: wg_data_size = round_up((vgpr_size_per_cu + sgrp_size_per_cu + lds_size_per_cu + hwreg_size_per_cu) * self.cu_cnt, mmap.PAGESIZE) ctl_stack_size = round_up((12 if self.target[0] != 9 else 8) * self.wave_cnt + 8 + 40, mmap.PAGESIZE) return self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_COMPUTE_AQL if self.is_aql else kfd.KFD_IOC_QUEUE_TYPE_COMPUTE, - 0x2000 if self.is_usb() else (16 << 20), eop_buffer_size=0x1000, + 0x2000 if self.is_usb else (16 << 20), eop_buffer_size=0x1000, ctx_save_restore_size=0 if self.is_am() else wg_data_size + ctl_stack_size, ctl_stack_size=ctl_stack_size, debug_memory_size=round_up(self.wave_cnt * 32, 64)) @@ -667,7 +718,7 @@ def sdma_queue(self, idx:int): if getenv("AMD_DISABLE_SDMA"): return None if idx in self.sdma_queues: return self.sdma_queues[idx] with contextlib.suppress(OSError): - self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx) + self.sdma_queues[idx] = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x2000 if self.is_usb else (16 << 20), idx=idx) return self.sdma_queues.get(idx, None) def tmpring_size(self, private_segment_size): diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 28f2165571e9c..1e1293733f1ab 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -5,7 +5,7 @@ from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, to_tuple from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device from tinygrad.runtime.support.hcq import HCQBuffer, MMIOInterface -from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_cmdbuf, make_signal +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_cmdbuf, make_buf from tinygrad.runtime.support.c import DLL from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer @@ -80,7 +80,7 @@ def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), lambda ctx, dst, val: cpu_cmd(ctx, signal_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),)), - lambda ctx, dst: cpu_cmd(ctx, timestamp_prog, dst.getaddr(ctx), *(() if WIN else (make_signal(ctx, tag="func:clock_gettime").getaddr(ctx),)))), + lambda ctx, dst: cpu_cmd(ctx, timestamp_prog, dst.getaddr(ctx), *(() if WIN else (make_buf(ctx, tag="func:clock_gettime").getaddr(ctx),)))), ]) def encode_queue(q:UOp) -> UOp: @@ -91,7 +91,7 @@ def encode_queue(q:UOp) -> UOp: assert cnt < RING_SLOTS, f"submit of {cnt} entries doesn't fit the ring" cmdbuf = make_cmdbuf(lin, devs, buf=UOp.placeholder((cnt*CMD_SIZE,), dtypes.uint64, next(UOp.unique_num), device=devs).rtag("cmdbuf")) ring = UOp.placeholder((ring_words:=RING_SLOTS*CMD_SIZE,), dtypes.uint64, 0, device=devs, volatile=True).rtag(f"{queue}_ring") - put, done, sem, sysbuf = (make_signal(devs, tag=f"{queue}_{name}") for name in ("put", "done", "sem", "sys")) + put, done, sem, sysbuf = (make_buf(devs, tag=f"{queue}_{name}") for name in ("put", "done", "sem", "sys")) # submits are serialized on the submitter, so they can bump put without atomics ran = done.after(l:=UOp.loop(next(UOp.unique_num))).index(0).load() @@ -104,7 +104,7 @@ def encode_queue(q:UOp) -> UOp: if WIN: return sysbuf.after(bumped).index(0).store(put.after(bumped).index(0).load()) e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(bumped,)) - return make_signal(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) + return make_buf(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) # ***************** diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index d985becfc65ab..4decf65da5a90 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -68,16 +68,15 @@ def make_binary_patch(buf:UOp, blob:bytes) -> UOp: r = UOp.range(len(blob) // buf.dtype.itemsize, 0, dtype=dtypes.int, src=(buf, data)) return buf.index(r).store(data.index(r).load()).end(r).rtag("link") -def make_cmdbuf(lin, devs, buf:UOp|None=None): +def make_buf(devs, slot:int=0, tag:str="signal") -> UOp: return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True, tag=tag) + +def make_cmdbuf(lin, devs, buf:UOp|None=None, dep:tuple[UOp, ...]=()): blob, patches = bytearray(), [] for s in (s for ins in lin.src for s in ins.src): if s.op is not Ops.CONST: patches.append((len(blob), s)) blob.extend(struct.pack(f'<{s.dtype.fmt}', s.val if s.op is Ops.CONST else 0x0)) cmdbuf = buf if buf is not None else UOp.placeholder((len(blob) // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("cmdbuf") - return cmdbuf.after(make_binary_patch(cmdbuf, bytes(blob)), *make_patches(cmdbuf, patches)) - -def make_signal(devs, slot:int=0, tag:str="signal") -> UOp: - return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True).rtag(tag) + return cmdbuf.after(*dep, make_binary_patch(cmdbuf, bytes(blob)), *make_patches(cmdbuf, patches)) def make_submit(*cmds, devs:str|tuple[str, ...], queue:str) -> UOp: return UOp.custom_function("submit_cmdbuf", UOp(Ops.LINEAR, src=tuple(cmds), arg=(to_tuple(devs), queue))) @@ -101,6 +100,17 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| return call.replace(src=call.src[:1] + tuple(s if s.op is Ops.PARAM or s.is_bound_var else s.param_like(slots[s]) for s in call.src[1:])) pm_replace_buffers = PatternMatcher([(UPat(Ops.CALL, name="call"), replace_call_buffers)]) +# ***************** + +def stage_copy_ext(call:UOp) -> UOp|None: + if (d:=next((d for b in call.src[1:] for d in to_tuple(b.device) if not d.startswith("CPU")), None)) is None: return None + return pm.rewrite(call) if (pm:=getattr(Device[d], "pm_stage_copy", None)) is not None else None + +def encode_host_call(call:UOp) -> UOp|None: + if (pm:=getattr(Device[call.arg.aux.device[0]], "pm_host_lower", None)) is None: return None + body = graph_rewrite(call.src[0], pm, name="lower host access", enter_calls=True) + return None if body is call.src[0] else call.replace(src=(body, *call.src[1:])) + # ***************** # 1.1. prep: staging copies @@ -126,6 +136,8 @@ def stage_copy(dst:UOp, src:UOp) -> UOp|None: # 1.2. prep: kernel copies def _get_enqueue_devs(call:UOp) -> Any|None: + if (call.arg.name or "").startswith("hcq_"): return None # host exec is not any device + if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None if call.src[0].op is Ops.COPY: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device @@ -138,6 +150,7 @@ def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: return call.replace(src=(to_program(ast, Device[dev].renderer), dst, src)) pm_insert_copy_staging = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), stage_copy_ext), (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy), (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src")), name="call"), kernel_copy) ]) @@ -174,7 +187,7 @@ def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int] waits = [] for (ddevs, dqueue, dtag), by_lane in deps.items(): for ls in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): - s = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) for dl, d in zip(ls, devices)]) + s = UOp.mstack(*[make_buf(d, tag="sentinel_signal") if dl is None else make_buf(ddevs[dl], slots[dqueue]) for dl, d in zip(ls, devices)]) waits.append(UOp(Ops.INS, arg="wait", src=(s, UOp.const(dtag + 1, dtypes.uint64)))) return waits, {dtag for _, _, dtag in deps} @@ -196,19 +209,20 @@ def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[t signal_tags |= cur_signal_tags # wait the syncs and signal the device epoch, then bump the timeline on the host - tl_signal, tl_value = make_signal(devs, tag="timeline_signal"), make_signal(devs, tag="timeline_value") + tl_signal, tl_value = make_buf(devs, tag="timeline_signal"), make_buf(devs, tag="timeline_value") fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() # fence once per device group on this schedule's previous epoch qs = dedup([qn for bdevs, qn in batch_info if set(bdevs) & set(devs)]) - sched_epoch = make_signal(devs, next(UOp.unique_num)) + sched_epoch = make_buf(devs, next(UOp.unique_num), tag="epoch") wait_device_epoch = (done:=tl_signal.after(loop:=UOp.loop(0)).index(0).load()).end(loop, done < sched_epoch.index(0).load()) fences.append(make_call("hcq_fence", UOp.sink(wait_device_epoch), HCQInfo(devs))) - # queues of other groups wait on these signals, so reset them only after every group reached its epoch - if qs: resets.append(make_call("hcq_reset", UOp.sink(*[make_signal(devs, slots[q]).index(0).store(0) for q in qs]), HCQInfo(devs))) + # queues of other groups wait on these signals, reset them after every group reached its epoch + rst = functools.reduce(lambda a,q: a+(make_buf(devs, slots[q]).after(*a[-1:]).index(0).store(0),), qs, cast(tuple[UOp, ...], ())) + if rst: resets.append(make_call("hcq_reset", UOp.sink(*rst), HCQInfo(devs))) fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) return fences + resets, fins, signal_tags @@ -234,19 +248,19 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li for tag, ((call, _), (devices, queue), q) in enumerate(zip(batch, batch_info, call_waits)): # first queue use, sync prior device work with the device timeline if batch_info.index((devices, queue)) == tag: - epoch = make_signal(devices, tag="timeline_value").index(0) - 1 - q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_signal(devices, tag="timeline_signal"), epoch))] + q + epoch = make_buf(devices, tag="timeline_value").index(0) - 1 + q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_buf(devices, tag="timeline_signal"), epoch))] + q # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] kerns.append((devices, make_call(name, call.src[0], info), tuple(ts_ids))) - ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_signal(devices, s),)) for s in ts_ids] + ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_buf(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] # signal the queue if someone waits for us - if tag in signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_signal(devices, slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] + if tag in signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_buf(devices, slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] src.append(make_call(f"submit {name}", make_submit(*q, devs=devices, queue=queue).sink(), info)) # append batch timestamps to finalizers @@ -304,7 +318,8 @@ def encode_cmdbuf(submit:UOp, lin:UOp) -> UOp|None: if (pm:=Device.get_class(lin.arg[0][0]).pm_lower) is None: return None return graph_rewrite(submit, pm, name=f"encode {lin.arg[0]}", enter_calls=True) pm_encode_cmdbufs = PatternMatcher([ - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), encode_cmdbuf)]) + (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), encode_cmdbuf), + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), encode_host_call)]) # ***************** @@ -350,20 +365,20 @@ def split_patches(call:UOp) -> UOp|None: lt_patches:list[UOp] = [] body = graph_rewrite(call.src[0], pm_trim_link_patches, ctx=(rt_patches, lt_patches), name=f"trim link-time patches ({call.arg.name})") - # split patches - inputs, internals = partition(dedup(g for p in rt_patches for g in get_getaddrs(p)), is_input_addr) + # split patches. addresses read in the body go through the tables too + inputs, internals = partition(dedup([g for p in rt_patches for g in get_getaddrs(p)] + get_getaddrs(body)), is_input_addr) runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) tables = [make_addr_table(call, gs, n) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec ipatches = [p for p in rt_patches if p.tag == "inputs" and all(v in tables[0][3] for v in p.src[1].src)] # only getaddrs go to the table gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches) if ipatches else {} - body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}) + body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}).substitute(reads) lt_srcs = collections.defaultdict(list) for p in lt_patches: lt_srcs[p.buf_uop].append(p) return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), - arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=((call.arg.aux.device, + arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=((to_tuple(inputs[0].arg), tuple(sorted(dedup(b.arg.slot for g in inputs for b in unwrap_mstack(g.buf_uop))))),) if inputs else call.arg.aux.input_idxs))) pm_split_patches = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), split_patches)]) @@ -500,7 +515,7 @@ def push_stack(op, s): return UOp(Ops.STACK, def fold_binary(buf:UOp, blob:UOp) -> UOp: for b in (m.bufs if isinstance(m:=buf.buffer, MultiBuffer) else (m,)): - b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[:len(blob.arg)] = blob.arg + b.ensure_allocated()._buf.cpu_view().view(fmt='B')[:len(blob.arg)] = blob.arg return UOp(Ops.NOOP) def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp: @@ -509,7 +524,7 @@ def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp: for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype]((v.src[0] if v.op is Ops.CAST else v).val)) bo = start*buf.dtype.itemsize + off.val*val.dtype.itemsize - b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[bo:bo+len(data)] = data + b.ensure_allocated()._buf.cpu_view().view(fmt='B')[bo:bo+len(data)] = data return UOp(Ops.NOOP) def resolve_getaddr(buf:UOp, g:UOp) -> UOp: @@ -561,6 +576,7 @@ def hcq_link(linear:UOp, cache=True) -> UOp: class HCQ2Compiled(Compiled): timestamp_divider: float = 1000.0 wait_timeout_ms: float = 30000.0 + rt_nbytes: int = 64 << 20 # scratch that single-run placeholders are carved out of def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Renderer]], runtime, can_recover:bool=False, arch=None): self.can_recover = can_recover @@ -568,14 +584,15 @@ def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Rende self.pm_bufferize = PatternMatcher([ (UPat(Ops.PARAM, tag="sentinel_signal"), lambda ctx: ctx[0].signal("sentinel", (1 << 64) - 1)), (UPat(Ops.PARAM, tag="timeline_signal"), lambda ctx: ctx[0].signal("timeline")), - (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1)), + (UPat(Ops.PARAM, tag="timeline_value"), lambda ctx: ctx[0].signal("value", 1, device="CPU")), + (UPat(Ops.PARAM, tag="epoch", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot, device="CPU")), (UPat(Ops.PARAM, tag="signal", name="b"), lambda ctx, b: ctx[0].signal(b.arg.slot)), (UPat(Ops.PARAM, name="b"), lambda ctx, b: None if b.tag is None else ctx[0].new_buffer(b, cache=ctx[1])) ]) super().__init__(device, allocator, compilers, runtime, None, arch=arch) - self.rt_allocator = BumpAllocator(64 << 20) + self.rt_allocator = BumpAllocator(self.rt_nbytes) self.prof_ents:dict[int, ProfileGraphEntry] = {} def collect_prof(self): @@ -609,12 +626,12 @@ def new_buffer(self, b:UOp, cache:bool) -> Buffer: self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) @functools.cache - def signal(self, name:str|int, init_value:int=0) -> Buffer: - buf = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True) - buf.as_memoryview(force_zero_copy=True, no_sync=True).cast('Q')[0] = init_value + def signal(self, name:str|int, init_value:int=0, device:str|None=None) -> Buffer: + buf = Buffer(device or self.device, 1, dtypes.uint64, options=BufferSpec(host=True, uncached=True, cpu_access=True), preallocate=True) + buf._buf.cpu_view().view(fmt='Q')[0] = init_value return buf - def _wait_signal(self, sig:memoryview, value:int, timeout:int|None=None): + def _wait_signal(self, sig:MMIOInterface|memoryview, value:int, timeout:int|None=None): timeout = timeout if timeout is not None and self.can_recover else None st, done = time.perf_counter(), sig[0] while done < value: @@ -624,8 +641,8 @@ def _wait_signal(self, sig:memoryview, value:int, timeout:int|None=None): def synchronize(self, timeout:int|None=None): if HCQ_RUNTIME_DEV.value != self.device: Device[HCQ_RUNTIME_DEV.value].synchronize() - sig = self.signal("timeline").as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') - tl = self.signal("value", 1).as_memoryview(force_zero_copy=True, no_sync=True).cast('Q') + sig = self.signal("timeline")._buf.cpu_view().view(fmt='Q') + tl = self.signal("value", 1, device="CPU")._buf.cpu_view().view(fmt='Q') self._wait_signal(sig, tl[0] - 1, timeout) if self.prof_ents: self.collect_prof() diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index 2229f4a5b8380..18ce7903097ad 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -1,6 +1,11 @@ import ctypes, struct, time, functools, itertools +from typing import Any, cast from tinygrad.runtime.autogen import libusb -from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv +from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv, unwrap, dedup, to_tuple +from tinygrad.dtype import dtypes +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher +from tinygrad.device import Buffer, BufferSpec, Device +from tinygrad.runtime.support.hcq2 import HCQInfo, make_buf, make_cmdbuf, make_submit, HCQ_RUNTIME_DEV from tinygrad.runtime.support.hcq import MMIOInterface from tinygrad.runtime.support import c @@ -220,6 +225,7 @@ def _off_from_index(self, index): return (index * self.el_sz, self.el_sz) def __getitem__(self, index): + Device[HCQ_RUNTIME_DEV.value].synchronize() # one driver on the link: drain the compiled submits before python touches it off, sz = self._off_from_index(index) if self.pcimem: assert sz % 4 == 0 and off % 4 == 0, f"pcie_mem_read requires 4-byte aligned access, got off={off}, sz={sz}" @@ -228,12 +234,114 @@ def __getitem__(self, index): return data if isinstance(index, slice) else int.from_bytes(data, "little") def __setitem__(self, index, data): + Device[HCQ_RUNTIME_DEV.value].synchronize() off, _ = self._off_from_index(index) data = struct.pack(self.fmt, data) if isinstance(data, int) else bytes(data) if not self.pcimem: self.usb.scsi_write(data) if self.addr == 0xf000 else self.usb.write(self.addr + off, data) - else: self.usb.pcie_mem_write(self.addr+off, data) + else: + # writes are whole dwords + assert len(data) % 4 == 0 and off % 4 == 0, f"pcie_mem_write requires 4-byte aligned access, got off={off}, sz={len(data)}" + self.usb.pcie_mem_write(self.addr+off, data) def view(self, offset:int=0, size:int|None=None, fmt=None): return USBMMIOInterface(self.usb, self.addr+offset, self.nbytes-offset if size is None else size, fmt=fmt or self.fmt, pcimem=self.pcimem) +# ***************** + +def _libusb(devs, dep:tuple[UOp, ...], fn:str, *args) -> UOp: + return make_buf(devs, tag=f"func:{fn}").after(*dep).index(0).load().call(make_buf(devs, tag="usb_handle").index(0).load(), + *[UOp.const(a, dtypes.int) if isinstance(a, int) else a for a in args], ret_dtype=dtypes.void) + +def usb_bulk(devs, dep, endpoint:int, data:UOp, length, timeout:int=1000) -> UOp: # NULL actual_length out param + return _libusb(devs, dep, "libusb_bulk_transfer", endpoint, data, length, UOp.const(0, dtypes.uint64), timeout) + +def usb_stream(devs, dep:tuple[UOp, ...], addr:UOp, data:UOp, nbytes:int, write:bool) -> UOp: + hdr = UOp.placeholder((2,), dtypes.uint64, device=devs, tag="usb_scratch").after(*dep) + arm = _libusb(devs, (hdr.index(0).store(addr), hdr.index(1).store(UOp.const(nbytes // 4, dtypes.uint64))), "libusb_control_transfer", + 0x40, 0xF0, (0x60 if write else 0x20) | (0x0F << 8), 1 if write else 2, hdr.index(0), 12, 5000) + return usb_bulk(devs, (arm,), 0x02 if write else 0x81, data, nbytes) + +def usb_writes(devs, ws:list[tuple[UOp, UOp, int]]) -> tuple[UOp, ...]: + return functools.reduce(lambda dep, w: (usb_stream(devs, dep, w[0], w[1], w[2], True),), ws, ()) + +def usb_load(b:UOp, idx:UOp, dt) -> UOp: + got = UOp.placeholder((1,), dt, device=(devs:=to_tuple(b.device)), tag="usb_scratch") + addr = b.getaddr((HCQ_RUNTIME_DEV.value,)) + (idx*dt.itemsize).cast(dtypes.uint64) + return got.after(usb_stream(devs, b.src[1:] if b.op is Ops.AFTER else (), addr, got.index(0), dt.itemsize, False)).index(0).load() + +def usb_write(b:UOp, idx:UOp, v:UOp) -> UOp: + val = (s:=UOp.placeholder((1,), v.dtype, device=(devs:=to_tuple(b.device)), tag="usb_scratch")).after(s.index(0).store(v)) + addr = b.getaddr((HCQ_RUNTIME_DEV.value,)) + (idx*v.dtype.itemsize).cast(dtypes.uint64) + return usb_stream(devs, b.src[1:] if b.op is Ops.AFTER else (), addr, val.index(0), v.dtype.itemsize, True) + +def usb_idle(devs) -> UOp: + v = usb_load(make_buf(devs, tag="timeline_signal").after(loop:=UOp.loop(0)), UOp.const(0, dtypes.int), dtypes.uint64) + return v.end(loop, v + 1 < make_buf(devs, tag="timeline_value").index(0).load()) + +def usb_scsi(devs, read:bool, nbytes:int) -> UOp: + return _libusb(devs, (usb_idle(devs),), "libusb_control_transfer", 0x40, 0xF2, ceildiv(nbytes, 512) | (0x8000 if read else 0), + (ceildiv(nbytes, 0x4000) & 0xFF) << 8, UOp.const(0, dtypes.uint64), 0, 1000) + +def usb_stage_copy(dst:UOp, src:UOp) -> UOp|None: + if (cin:=to_tuple(src.device)[0].startswith("CPU")) == to_tuple(dst.device)[0].startswith("CPU"): return None + + total, ops, win = dst.nbytes(), [], cast(Any, Device[(devs:=to_tuple((dst if cin else src).device))[0]]).iface.usb_sram + for off in range(0, total, win.size): # off and nb are bytes, the two ends of the copy can have different dtypes + sram = UOp.from_buffer(win)[0:(nb:=min(win.size, total - off))] + s, d = src[off // src.dtype.itemsize:(off + nb) // src.dtype.itemsize], dst[off // dst.dtype.itemsize:(off + nb) // dst.dtype.itemsize] + if cin: + push = usb_bulk(devs, (usb_scsi(devs, False, nb),), 0x02, s.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) + ops += [UOp.custom_function("hcq", push.sink()).call(sram, s, name="hcq_copyin", aux=HCQInfo(devs)), + sram.copy_to_device(d.device).call(d, sram)] + else: + pad = UOp.new_buffer("CPU", round_up(nb, 512), dtypes.uint8)[0:nb] + submit = make_submit(UOp(Ops.CALL, dtypes.void, (UOp(Ops.COPY, dtypes.void, ()), sram, s)), devs=devs, queue="COPY:0") + pull = usb_bulk(devs, (submit,), 0x81, pad.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) + ops += [UOp.custom_function("hcq", pull.sink()).call(pad, sram, s, name="hcq_copyout", aux=HCQInfo(devs)), + pad.copy_to_device("CPU").call(d, pad)] + return UOp(Ops.LINEAR, src=tuple(ops)) +pm_usb_stage = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), usb_stage_copy)]) + +def usb_arm_bytes(lin:UOp, sram:Buffer) -> int: + dsts = [c.src[1] for c in lin.src if c.op is Ops.CALL and c.src[0].op is Ops.COPY] # the rest of a linear is INS, some with no srcs + return next((d.nbytes() for d in dsts if d.base.op is Ops.BUFFER and d.base.buffer is sram), 0) + +def usb_ib(devs, lin:UOp, align:int, arm:int=0) -> tuple[UOp, UOp, int]: + pkt_dw = sum(s.dtype.itemsize for ins in lin.src for s in ins.src) // 4 # by bytes: sdma packs 64-bit addresses as single srcs + kargs = dedup([b for b in lin.toposort() if b.op is Ops.PARAM and b.tag == "kernargs"]) + offs, up_dw = {}, round_up(pkt_dw, align) + for k in kargs: offs[k], up_dw = up_dw, round_up(up_dw + k.max_numel(), 32) + ib_gpu = UOp.placeholder((up_dw,), dtypes.uint32, device=devs, tag="cmdbuf") + ib_host = UOp.placeholder((up_dw,), dtypes.uint32, device=devs, tag="usb_scratch") + gsubs = {g: g.replace(src=(d if a.op is not Ops.AFTER else d.after(*a.src[1:]),)) for g in lin.toposort() if g.op is Ops.GETADDR + for a in [g.src[0]] if (k:=a.src[0] if a.op is Ops.AFTER else a) in offs for d in [ib_gpu[offs[k]:offs[k] + k.max_numel()]]} + lin = lin.substitute(gsubs, walk=True).substitute({k: ib_host[offs[k]:offs[k] + k.max_numel()] for k in kargs}, walk=True) + return make_cmdbuf(lin, devs, buf=ib_host, dep=(usb_scsi(devs, True, arm),) if arm else ()), ib_gpu, pkt_dw + +def usb_push(devs, ring:UOp, wptr:UOp, doorbell:UOp, put_ptr:UOp, ib_host:UOp, ib_gpu:UOp, pkt:tuple, unit:int) -> UOp: + stage = UOp.placeholder(((n:=round_up(len(pkt), 4)) + 2,), dtypes.uint32, device=devs, tag="usb_scratch") + put, step = put_ptr.index(zero:=UOp.const(0, dtypes.int)), (n * 4 if pkt else ib_host.nbytes()) // unit + st = stage.after(*[stage.index(i).store(UOp.const(v, dtypes.uint32)) for i, v in enumerate(pkt)], + *[stage.index(n + i).store((((put + step) >> (32 * i)) & 0xffffffff).cast(dtypes.uint32)) for i in (0, 1)]) + + writes = [(ib_gpu.getaddr((HCQ_RUNTIME_DEV.value,)), ib_host.index(zero), ib_gpu.nbytes())] if pkt else [] + writes += [(ring.getaddr((HCQ_RUNTIME_DEV.value,)) + ((put % (ring.nbytes() // unit)) * unit).cast(dtypes.uint64), + (st if pkt else ib_host).index(zero), step * unit)] + writes += [(p.getaddr((HCQ_RUNTIME_DEV.value,)), st.index(n), 8) for p in (wptr, doorbell)] + return put_ptr.after(*usb_writes(devs, writes)).index(zero).store(put + step) + +USB_HOST_TAGS = {"signal", "timeline_signal"} +pm_usb_hostio = PatternMatcher([ + (UPat(Ops.LOAD, src=(UPat(Ops.INDEX, src=(UPat(Ops.PARAM, tag=USB_HOST_TAGS).or_after(name="b"), UPat(name="idx"))),), + name="ld"), lambda b, idx, ld: usb_load(b, idx, ld.dtype)), + (UPat(Ops.STORE, src=(UPat(Ops.INDEX, src=(UPat(Ops.PARAM, tag=USB_HOST_TAGS).or_after(name="b"), UPat(name="idx"))), UPat(name="v"))), usb_write)]) + +pm_usb_bufferize = PatternMatcher([ + (UPat(Ops.PARAM, tag={"systems", "runtime", "inputs", "usb_scratch"}, name="b"), + lambda ctx, b: Buffer("CPU", b.max_numel(), b.dtype, options=BufferSpec(nolru=True), preallocate=True)), + (UPat(Ops.PARAM, tag="usb_handle", name="b"), lambda ctx, b: ctx[0].signal(b.tag, ctx[0].iface.usb_handle, device="CPU")), + (UPat(Ops.PARAM, name="b"), lambda ctx, b: None if not isinstance(b.tag, str) or not b.tag.startswith("func:") else + ctx[0].signal(b.tag, unwrap(ctypes.cast(getattr(libusb.dll, b.tag[5:]), ctypes.c_void_p).value), device="CPU")), +]) + if DEV.interface.startswith("MOCK"): from test.mockgpu.usb import MockUSB3 as USB3 # type: ignore # noqa: F811 diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 9102f2af5b977..ae40f94eb26a1 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1135,14 +1135,16 @@ def pyrender(self): # *** uop high level syntactic sugar *** @staticmethod - def placeholder(shape:tuple[int, ...], dtype:DType, slot:int, addrspace=AddrSpace.GLOBAL, device=None, volatile=False): + def placeholder(shape:tuple[int, ...], dtype:DType, slot:int|None=None, addrspace=AddrSpace.GLOBAL, device=None, volatile=False, tag=None): dtype = strong_dtype(dtype) # storage is never weak: a placeholder commits the width of what's put in it + if slot is None: slot = next(UOp.unique_num) if addrspace is AddrSpace.GLOBAL: ret = UOp(Ops.PARAM, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace, device=device,volatile=volatile)) else: assert addrspace in (AddrSpace.LOCAL, AddrSpace.REG) assert device is None, "LOCAL and REG placeholders cannot have a device" ret = UOp(Ops.BUFFER, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace)) + if tag is not None: ret = ret.rtag(tag) if len(shape) > 1: ret = ret.reshape(shape) return ret def placeholder_like(self, slot:int, addrspace=AddrSpace.GLOBAL): From b0a12853303ed8ef625b599752ce01523b5b7e5f Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 22 Aug 2026 18:11:14 -0400 Subject: [PATCH 165/306] lil decomp cleanup [PR] (#17682) --- tinygrad/codegen/decomp/dtype.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 64b46fa758a3d..6daf24e459275 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -137,8 +137,8 @@ def f2f_store(st, idx, val, fr:DType, to:DType): # tag is the 32-bit word this node becomes - (0 for the low word, 1 for the high, the dtype the consumer wants) pm_long_decomp: PatternMatcher = PatternMatcher([ - (UPat(GroupOp.Defines, src=(UPat.var("sz"),), name="x"), lambda x,sz: - UOp(x.op, src=(sz*2,), arg=replace(x.arg, dtype=l2i_dt[x.dtype]), tag=x.tag) if x.dtype in l2i_dt else None), + (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), src=(UPat.var("sz"),), name="x"), lambda x,sz: + UOp(x.op, src=(sz*2,), arg=replace(x.arg, dtype=l2i_dt[x.dtype]), tag=x.tag)), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: @@ -170,9 +170,10 @@ def f2f_store(st, idx, val, fr:DType, to:DType): pm_float_decomp: PatternMatcher = PatternMatcher([ (UPat(GroupOp.Defines, name="x"), lambda ctx,x: UOp(x.op, src=x.src, arg=replace(x.arg, dtype=f2f_dt[ctx[0]]), tag=ctx[0]) if x.dtype == ctx[0] else None), - (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda ctx,x: - UOp(x.op, src=(graph_rewrite(x.src[0], pm_float_decomp, ctx=ctx, bottom_up=True), *x.src[1:]), arg=x.arg, tag=ctx[0]) if - x.dtype == ctx[0] and (x.op is not Ops.INDEX or x.src[0].op not in {Ops.LOAD, Ops.STACK}) else None), + # INDEX into a LOAD/STACK selects a lane of an already converted value, the load rules below own those + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat(GroupOp.All-{Ops.LOAD, Ops.STACK}),), allow_any_len=True, name="x"), lambda ctx,x: + UOp(x.op, src=(graph_rewrite(x.src[0], pm_float_decomp, ctx=ctx, bottom_up=True), *x.src[1:]), arg=x.arg, tag=ctx[0]) + if x.dtype == ctx[0] else None), (UPat(Ops.LOAD, dtypes.floats, name="x"), lambda ctx,x: f2f_load(x, *ctx) if x.dtype == ctx[0] else None), # bitcasted load should just replace load (UPat(Ops.BITCAST, src=(UPat(Ops.LOAD, name="ld"),), name="bc"), lambda ctx,bc,ld: @@ -191,8 +192,8 @@ def f2f_store(st, idx, val, fr:DType, to:DType): UOp(x.op, src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src), arg=x.arg, tag=x.tag) if x.dtype == ctx[0] else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat(Ops.BITCAST, dtypes.floats, name="val")), name='st'), lambda ctx,st,idx,val: st.replace(src=(idx, val.src[0].bitcast(f2f_dt[ctx[0]]))) if val.dtype == ctx[0] and idx.tag == ctx[0] else None), - (UPat(Ops.STORE, src=(UPat.var("idx"), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: - f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and (idx:=idx.src[0] if idx.op == Ops.CAST else idx).tag == ctx[0] else None), + (UPat(Ops.STORE, src=(UPat.var("idx").or_casted(), UPat.var("val", dtypes.floats)), name='st'), lambda ctx,st,idx,val: + f2f_store(st, idx, val, *ctx) if val.dtype == ctx[1] and idx.tag == ctx[0] else None), ]) def do_dtype_decomps(sink:UOp, ctx:tuple[set[DType], Renderer]) -> UOp: From 5b60a09ab0a7f20c3426e505251c48cac020d1fa Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Sat, 22 Aug 2026 22:48:07 -0700 Subject: [PATCH 166/306] some fixes for the AMD emulator (#17684) * some fixes for the AMD emulator * simpler * revert * min --- test/amd/hw/test_sop.py | 33 ++++++++++++++ test/amd/hw/test_vop1.py | 61 ++++++++++++++++++++++++++ test/amd/hw/test_vop2.py | 47 ++++++++++++++++++++ test/amd/hw/test_vop3.py | 90 +++++++++++++++++++++++++++++++++++++++ test/amd/hw/test_vopc.py | 65 ++++++++++++++++++++++++++++ test/mockgpu/amd/emu.py | 69 ++++++++++++++++++++++-------- test/mockgpu/amd/pcode.py | 64 ++++++++++++++++++++++++---- 7 files changed, 402 insertions(+), 27 deletions(-) diff --git a/test/amd/hw/test_sop.py b/test/amd/hw/test_sop.py index ba9ea6691ad37..f6a5006f25d2f 100644 --- a/test/amd/hw/test_sop.py +++ b/test/amd/hw/test_sop.py @@ -1002,6 +1002,39 @@ def test_barrier_two_phases(self): for tid in range(64): self.assertEqual(st.vgpr[tid][0], tid + 100 + 1000, f"tid={tid}") +class TestSMaxMinSCCRegressions(unittest.TestCase): + """Regression test: S_MAX sets SCC only on strict inequality (equal operands -> SCC=0).""" + + def test_s_max_i32_equal_scc(self): + st = run_program([s_mov_b32(s[4], 64), s_mov_b32(s[5], 64), s_max_i32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 0) + self.assertEqual(st.sgpr[6], 64) + st = run_program([s_mov_b32(s[4], 65), s_mov_b32(s[5], 64), s_max_i32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 1) # still set when strictly greater + + def test_s_max_u32_equal_scc(self): + st = run_program([s_mov_b32(s[4], 64), s_mov_b32(s[5], 64), s_max_u32(s[6], s[4], s[5])], n_lanes=1) + self.assertEqual(st.scc, 0) + +class TestAbsdiffOverflowRegressions(unittest.TestCase): + """Regression test: S_ABSDIFF_I32 computes abs on the WRAPPED 32-bit difference (found by random difftest vs hardware).""" + + def test_s_absdiff_wrapped(self): + # |45 - (-2147483647)| overflows int32; hardware takes abs of the wrapped 32-bit difference + instructions = [s_mov_b32(s[4], 45), s_mov_b32(s[5], 0x80000001), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0x7FFFFFD4) + self.assertEqual(st.scc, 1) + # INT_MIN - 1 wraps to +2147483647, already positive + instructions = [s_mov_b32(s[4], 0x80000000), s_mov_b32(s[5], 1), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0x7FFFFFFF) + # equality -> 0 and SCC=0 + instructions = [s_mov_b32(s[4], 7), s_mov_b32(s[5], 7), s_absdiff_i32(s[6], s[4], s[5])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.sgpr[6], 0) + self.assertEqual(st.scc, 0) + if __name__ == '__main__': unittest.main() diff --git a/test/amd/hw/test_vop1.py b/test/amd/hw/test_vop1.py index d1f9b655717bf..ced25512e69c3 100644 --- a/test/amd/hw/test_vop1.py +++ b/test/amd/hw/test_vop1.py @@ -1629,5 +1629,66 @@ def test_v_swap_b32_chain(self): self.assertEqual(st.vgpr[0][1], 0x55555555) +class TestCvtFrexpRegressions(unittest.TestCase): + """Regression tests for float<->int conversion and FREXP corner cases (found by random difftest vs hardware).""" + + def test_cvt_i32_f32_nan_is_zero(self): + """v_cvt_i32_f32 of NaN is 0, not INT_MIN (x86 cvttss2si returns INT_MIN).""" + for nan in (0x7FC00000, 0xFFC00000, 0x7F800001): + st = run_program([v_mov_b32_e32(v[0], nan), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0, f"nan=0x{nan:08x}") + + def test_cvt_i32_f32_positive_overflow(self): + """v_cvt_i32_f32 saturates positive overflow/inf to INT_MAX, not INT_MIN.""" + for bits in (0x7F800000, 0x4F000000, 0x4F800000): # +inf, 2^31, ~2^32 + st = run_program([v_mov_b32_e32(v[0], bits), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0x7FFFFFFF, f"bits=0x{bits:08x}") + + def test_cvt_i32_f32_negative_overflow(self): + """v_cvt_i32_f32 saturates negative overflow/-inf to INT_MIN.""" + for bits in (0xFF800000, 0xCF000001): # -inf, below -2^31 + st = run_program([v_mov_b32_e32(v[0], bits), v_cvt_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0x80000000, f"bits=0x{bits:08x}") + + def test_cvt_u32_f32_nan_is_zero(self): + """v_cvt_u32_f32 of NaN is 0, not UINT_MAX.""" + for nan in (0x7FC00000, 0xFFC00000, 0x7F800001): + st = run_program([v_mov_b32_e32(v[0], nan), v_cvt_u32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1], 0, f"nan=0x{nan:08x}") + + def test_cvt_i32_f64_nan_and_overflow(self): + """v_cvt_i32_f64: NaN -> 0, positive overflow/+inf -> INT_MAX.""" + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x7FF80000), v_cvt_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0) + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x41F00000), v_cvt_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x7FFFFFFF) # 2^32 -> INT_MAX + + def test_frexp_f32_denormal(self): + """v_frexp_exp/mant_f32 of denormal/zero inputs is (0, signed zero) on hardware.""" + for bits in (0x00000001, 0x007FFFFF, 0x00000000): + st = run_program([v_mov_b32_e32(v[0], bits), v_frexp_exp_i32_f32_e32(v[1], v[0]), v_frexp_mant_f32_e32(v[2], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1] & 0xFFFFFFFF, 0, f"exp bits=0x{bits:08x}") + self.assertEqual(st.vgpr[0][2], bits & 0x80000000, f"mant bits=0x{bits:08x}") + # negative denormal: mant is -0.0 + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_frexp_mant_f32_e32(v[2], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + + def test_frexp_f64_denormal(self): + """v_frexp_exp_f64 of a denormal returns the normalized exponent (-1073 for min-denormal); zero -> 0.""" + st = run_program([v_mov_b32_e32(v[0], 1), v_mov_b32_e32(v[1], 0), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2] & 0xFFFFFFFF, 0xFFFFFBCF) # -1073 + st = run_program([v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0) + + def test_frexp_exp_inf_nan(self): + """v_frexp_exp of +/-inf and NaN is 0 on hardware (host frexp gives 129/1024), for both f32 and f64.""" + for bits in (0x7F800000, 0xFF800000, 0x7FC00000): + st = run_program([v_mov_b32_e32(v[0], bits), v_frexp_exp_i32_f32_e32(v[1], v[0])], n_lanes=1) + self.assertEqual(st.vgpr[0][1] & 0xFFFFFFFF, 0, f"f32 bits=0x{bits:08x}") + for lo, hi in ((0, 0x7FF00000), (0, 0xFFF00000), (0, 0x7FF80000), (1, 0x7FF00000)): + st = run_program([v_mov_b32_e32(v[0], lo), v_mov_b32_e32(v[1], hi), v_frexp_exp_i32_f64_e32(v[2], v[0:1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2] & 0xFFFFFFFF, 0, f"f64 bits=0x{hi:08x}{lo:08x}") + + if __name__ == '__main__': unittest.main() diff --git a/test/amd/hw/test_vop2.py b/test/amd/hw/test_vop2.py index a341cb47ba8b3..c69b96fc45221 100644 --- a/test/amd/hw/test_vop2.py +++ b/test/amd/hw/test_vop2.py @@ -989,6 +989,53 @@ def test_v_add_co_ci_u32_vop3sd_null_sdst(self): self.assertEqual(st.vgpr[0][0], 0) # 0xFFFFFFFF + 1 + 0 = 0 (overflow) self.assertEqual(st.vcc, 0xDEADBEEF) # VCC unchanged - carry was discarded +class TestSelectFlushRegressions(unittest.TestCase): + """Regression tests: f32 MIN/MAX flush denormal inputs to signed zero (select-style ops propagate inputs bitwise).""" + + def test_v_min_f32_denormal_flush(self): + """min(denormal, 1.0) is +0, min(-denormal, -1.0) is -0.""" + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x3F800000), v_min_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + # flush(-denormal) = -0.0 > -1.0, so the result is -1.0 (both operand orders) + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0xBF800000), v_min_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0xBF800000) + st = run_program([v_mov_b32_e32(v[1], 0xBF800000), v_mov_b32_e32(v[2], 0x80000001), v_min_f32_e32(v[3], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0xBF800000) + + def test_v_max_f32_denormal_flush(self): + """max(-denormal, -1.0) is -0; max(+denormal, -0) is +0.""" + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0xBF800000), v_max_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x80000000), v_max_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + + +class TestCarryExecRegressions(unittest.TestCase): + """Regression tests: per-lane VCC writes (carry ops) zero inactive lane bits - VCC = mask & EXEC, never preserved.""" + + def test_co_ci_e32_vcc_masked_by_exec(self): + """v_sub_co_ci_u32_e32 with EXEC=0xFFFF0000: hw clears inactive VCC bits instead of preserving them.""" + instructions = [ + s_mov_b32(EXEC_LO, 0xFFFF0000), + s_mov_b32(VCC_LO, 0xFFFFFFFF), # preset all bits + v_mov_b32_e32(v[0], 0xFFFFFFFE), v_mov_b32_e32(v[1], 0x80000000), + v_sub_co_ci_u32_e32(v[2], v[0], v[1]), # active lanes: no borrow + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.vcc, 0x00000000) + + def test_co_ci_e32_vcc_masked_by_exec_ones(self): + """Same with all-ones carry: VCC = borrow_mask & EXEC.""" + instructions = [ + s_mov_b32(EXEC_LO, 0x0F0F0F0F), + s_mov_b32(VCC_LO, 0), + v_mov_b32_e32(v[0], 0xFFFFFFFF), v_mov_b32_e32(v[1], 1), + v_add_co_ci_u32_e32(v[2], v[0], v[1]), # all lanes would carry if active + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.vcc, 0x0F0F0F0F) + self.assertEqual(st.vgpr[31][2], 0) # 0xFFFFFFFF + 1 wraps to 0 in active lanes + if __name__ == '__main__': unittest.main() diff --git a/test/amd/hw/test_vop3.py b/test/amd/hw/test_vop3.py index e828e35743e89..ff9217e370661 100644 --- a/test/amd/hw/test_vop3.py +++ b/test/amd/hw/test_vop3.py @@ -3264,6 +3264,23 @@ def test_v_mad_u16_no_clamp(self): # 0xFFFF * 2 = 0x1FFFE, low 16 bits = 0xFFFE self.assertEqual(st.vgpr[0][3] & 0xFFFF, 0xFFFE, f"expected 0xFFFE, got 0x{st.vgpr[0][3] & 0xFFFF:04x}") +class TestMadNarrowClampRegressions(unittest.TestCase): + """Regression tests: mad i16/i24 with clamp saturate to narrow output range (found by random difftest vs hardware).""" + + def test_mad_i16_clamp_sat_max(self): + # neg/src-floggled 16-bit mul operands are sign-extended after toggling bit15; sum > INT_MAX saturates + instructions = [s_mov_b32(s[4], 1232348160), v_mov_b32_e32(v[3], 0x80000000), + v_mov_b32_e32(v[1], 0x7F7FFFFF), v_mad_i32_i16(v[0], s[4], v[3], v[1], 0, 3, 5, 1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][0], 0x7FFFFFFF) + + def test_mad_i24_clamp_sat_min(self): + # sext24(-6344704) * sext24(+4210688) << -2^31 saturates to INT_MIN + instructions = [s_mov_b32(s[7], 4290772992), v_mov_b32_e32(v[1], 1077936128), + v_mad_i32_i24(v[0], s[7], v[1], v[1], 1, 0, 0, 1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][0], 0x80000000) + class TestCvtPkF16(unittest.TestCase): """Tests for V_CVT_PK_RTZ_F16_F32 - pack two f32 to f16 with round toward zero.""" @@ -3651,6 +3668,79 @@ def test_v_permlanex16_b32_identity(self): self.assertEqual(st.vgpr[21][1], 5) self.assertEqual(st.vgpr[31][1], 15) +class TestClampLdExpRegressions(unittest.TestCase): + """Regression tests for f32 clamp (-0 -> +0) and ldexp input passthrough.""" + + def test_clamp_negative_zero(self): + """clmp=1 maps -0.0 to +0.0 (found by random difftest vs hardware).""" + instructions = [ + v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 0x80000000), + v_add_f32_e64(v[2], v[0], v[1], clmp=1), # -0 + -0 = -0, clamp -> +0 + ] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + instructions = [ + v_mov_b32_e32(v[0], 0x3F800000), v_mov_b32_e32(v[1], 0x80000000), + v_min_f32_e64(v[2], v[0], v[1], clmp=1), # min(1.0, -0) = -0, clamp -> +0 + ] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x00000000) + + def test_ldexp_special_inputs(self): + """v_ldexp_f32 of 0/-0/inf/NaN propagates the input instead of computing val * 2**exp (0*inf = NaN on host).""" + # -0.0 * 2^INT_MIN = -0.0 (src1 as integer exponent; huge negative) + instructions = [v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 0x80000000), v_ldexp_f32(v[2], v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x80000000) + # inf stays inf even with negative exponent + instructions = [v_mov_b32_e32(v[0], 0x7F800000), v_mov_b32_e32(v[1], 0xFFFFFF80), v_ldexp_f32(v[2], v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x7F800000) + + def test_ldexp_denormal_flush(self): + """v_ldexp_f32/f64 flush denormal inputs to signed zero (found by random difftest vs hardware).""" + # ldexp(+denorm, 1) = +0, ldexp(-denorm, 250) = -0 + for src, exp_val, want in [(0x00000001, 1, 0x00000000), (0x80000001, 250, 0x80000000)]: + st = run_program([v_mov_b32_e32(v[0], src), v_mov_b32_e32(v[1], exp_val), v_ldexp_f32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], want) + + def test_v_mul_neg_modifier_nan_sign(self): + """neg modifier is a pure sign-bit toggle on a NaN operand; result keeps that sign (found by random difftest).""" + # mul(normal, NEG(ABS(qNaN))): NaN payload negated in the operand stays negative qNaN + instructions = [v_mov_b32_e32(v[0], 0xC96CF47F), v_mov_b32_e32(v[1], 0x7FC00000), + v_mul_f32_e64(v[2], v[0], v[1], s[0], 0, 7, 6)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0xFFC00000) + # plain neg modifier still applies to non-NaN values: mul(-1.0, NEG(2.0)) = +2.0 + st = run_program([v_mov_b32_e32(v[0], 0xBF800000), v_mov_b32_e32(v[1], 0x40000000), + v_mul_f32_e64(v[2], v[0], v[1], s[0], 0, 2, 0)], n_lanes=1) + self.assertEqual(st.vgpr[0][2], 0x40000000) + + +class TestNaNPropagationRegressions(unittest.TestCase): + """Regression tests: float arithmetic propagates a NaN from the FIRST NaN operand, quieted with its own sign/payload.""" + + def test_mul_nan_priority(self): + # first NaN operand wins (sign+payload), not x86's second-source propagation + for a, b, want in [(0x7FC00001, 0x7F800003, 0x7FC00001), (0xFFC00005, 0x7F800003, 0xFFC00005), + (0x7F800001, 0xFFC00005, 0x7FC00001), (0xFF9F1800, 0x7F800001, 0xFFDF1800)]: + st = run_program([v_mov_b32_e32(v[0], a), v_mov_b32_e32(v[1], b), + v_mul_f32_e32(v[2], v[0], v[1])], n_lanes=1) + self.assertEqual(st.vgpr[0][2], want, f"mul({a:#x}, {b:#x})") + +class TestMinMaxFlushE64Regressions(unittest.TestCase): + """Regression tests: f32 min/max/median flush denormal inputs to signed zero (e64 forms).""" + + def test_v_min3_f32_denormal_flush(self): + st = run_program([v_mov_b32_e32(v[0], 0x00000001), v_mov_b32_e32(v[1], 0x3F800000), v_mov_b32_e32(v[2], 0x40000000), + v_min3_f32(v[3], v[0], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0x00000000) # min(+denorm, 1, 2) = +0 + + def test_v_med3_f32_denormal_flush(self): + st = run_program([v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x3F800000), v_mov_b32_e32(v[2], 0x40000000), + v_med3_f32(v[3], v[0], v[1], v[2])], n_lanes=1) + self.assertEqual(st.vgpr[0][3], 0x3F800000) # med(-0, 1, 2) = 1 + if __name__ == '__main__': unittest.main() diff --git a/test/amd/hw/test_vopc.py b/test/amd/hw/test_vopc.py index 5a7ed5e938865..7e0f1d4435790 100644 --- a/test/amd/hw/test_vopc.py +++ b/test/amd/hw/test_vopc.py @@ -973,6 +973,71 @@ def test_v_cmpx_e64_partial_wave(self): self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x4, "Only lane 2 should be active after v_cmpx_eq_u32_e64") +class TestClassDenormalRegressions(unittest.TestCase): + """Regression tests: V_CMP_CLASS classifies denormals as DENORMAL (raw bits), not as zero class.""" + + def test_class_pos_denormal(self): + for bits in (0x00000001, 0x007FFFFF): + instructions = [v_mov_b32_e32(v[0], bits), v_mov_b32_e32(v[1], 0x80), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1, f"bits=0x{bits:08x}") # n_lanes=1 + # ...and it is not the zero class + instructions = [v_mov_b32_e32(v[0], bits), v_mov_b32_e32(v[1], 0x40), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0, f"bits=0x{bits:08x}") + + def test_class_neg_denormal(self): + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x10), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1) # n_lanes=1 + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 0x20), v_cmp_class_f32_e64(VCC_LO, v[0], v[1])] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) # not the negative-zero class + + +class TestIntCmpModRegressions(unittest.TestCase): + """Regression tests: int compares (i32/u32) honor abs/neg as bit-level sign clear/flip (not integer abs/negate).""" + + def test_cmp_i32_abs_neg_bit_level(self): + # abs(0x80000001) = 1 -> 1 > 1 is false (integer abs would give 2147483647 > 1) + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 1), v_cmp_gt_i32_e64(VCC_LO, v[0], v[1], abs=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) + # neg(0x80000001) flips the sign bit -> 1 > 2 is false (integer negate would give 2147483647 > 2) + instructions = [v_mov_b32_e32(v[0], 0x80000001), v_mov_b32_e32(v[1], 2), v_cmp_gt_i32_e64(VCC_LO, v[0], v[1], neg=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 0) + + def test_cmp_u32_abs_bit_level(self): + # abs(0x80000000) = 0 -> 0 < 1 is true + instructions = [v_mov_b32_e32(v[0], 0x80000000), v_mov_b32_e32(v[1], 1), v_cmp_lt_u32_e64(VCC_LO, v[0], v[1], abs=1)] + st = run_program(instructions, n_lanes=1) + self.assertEqual(st.vcc, 1) # n_lanes=1 + + +class TestCmpxSdstRegressions(unittest.TestCase): + """Regression tests: V_CMPX_*_E64 writes EXEC only, never SDST (hardware verified).""" + + def test_cmpx_e64_no_sdst(self): + instructions = [ + s_mov_b32(VCC_LO, 0), # preset VCC to 0 + v_mov_b32_e32(v[0], 0x3F800000), v_mov_b32_e32(v[1], 0x40000000), + v_cmpx_lt_f32_e64(VCC_LO, v[0], v[1]), # 1.0 < 2.0 + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.sgpr[EXEC_LO.offset], 0xFFFFFFFF) # EXEC updated + self.assertEqual(st.vcc, 0) # but VCC untouched + + def test_cmpx_e64_partial_exec(self): + instructions = [ + s_mov_b32(EXEC_LO, 0x0F0F0F0F), + s_mov_b32(VCC_LO, 0xFFFFFFFF), + v_mov_b32_e32(v[0], 0), v_mov_b32_e32(v[1], 0x3F800000), + v_cmpx_lt_f32_e64(VCC_LO, v[0], v[1]), + ] + st = run_program(instructions, n_lanes=32) + self.assertEqual(st.sgpr[EXEC_LO.offset], 0x0F0F0F0F) # EXEC = computed & old EXEC + if __name__ == '__main__': unittest.main() diff --git a/test/mockgpu/amd/emu.py b/test/mockgpu/amd/emu.py index d96841694b467..6c8b3380f4a8f 100644 --- a/test/mockgpu/amd/emu.py +++ b/test/mockgpu/amd/emu.py @@ -69,7 +69,7 @@ def __exit__(self, *args): from tinygrad.renderer.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp, Inst from tinygrad.runtime.autogen.amd.common import Fmt, OpType from test.amd.helpers import decode_dpp16 -from test.mockgpu.amd.pcode import parse_pcode, _FUNCS, _set_bits, _to_bool, _to_u32, _val_to_bits +from test.mockgpu.amd.pcode import parse_pcode, _FUNCS, _set_bits, _to_bool, _to_u32, _val_to_bits, _ftz_f32 MASK32 = 0xFFFFFFFF @@ -96,7 +96,8 @@ def _apply_src_mods(val: UOp, mod_bit: int, abs_bits: int, neg_bits: int, bits: ut, ft, mask = _SRC_MOD_TYPES[bits] fv = val.cast(ut).bitcast(ft) if bits == 16 else val.bitcast(ft) if val.dtype == ut else val if abs_bits & (1 << mod_bit): fv = (fv.bitcast(ut) & UOp.const(mask, ut)).bitcast(ft) - if neg_bits & (1 << mod_bit): fv = fv.neg() + # neg modifier is a pure sign-bit toggle (preserves NaN payloads), not an arithmetic negate + if neg_bits & (1 << mod_bit): fv = (fv.bitcast(ut) ^ UOp.const((mask + 1) & (1 << (bits - 1)), ut)).bitcast(ft) return fv.bitcast(ut).cast(dtypes.uint32) if bits == 16 else fv.bitcast(ut) # Map VOPD ops to VOP2/VOP1 ops for pcode lookup (both RDNA3 and RDNA4 share these targets) @@ -157,6 +158,23 @@ def _val_to_u32(val: UOp) -> UOp: 'V_DIV_FIXUP_F64': ('D0.f64 = sign_out ? -abs(S0.f64) : abs(S0.f64)', 'D0.f64 = isNAN(S0.f64) ? (sign_out ? -INF : +INF) : (sign_out ? -abs(S0.f64) : abs(S0.f64))'), 'V_TRIG_PREOP_F64': ("result = 64'F((1201'B(2.0 / PI)[1200 : 0] << shift.u32) & 1201'0x1fffffffffffff)", "result = trig_preop_result(shift)"), + # exponent() returns 0 for denormals; frexp_exp handles them per hardware (f32: 0, f64: normalized) + 'V_FREXP_EXP_I32_F32': ('D0.i32 = exponent(S0.f32) - 127 + 1', 'D0.i32 = frexp_exp(S0.f32)'), + 'V_FREXP_EXP_I32_F64': ('D0.i32 = exponent(S0.f64) - 1023 + 1', 'D0.i32 = frexp_exp(S0.f64)'), + # route through ldexp() which propagates 0/inf/NaN inputs instead of computing val * 2**exp (0*inf = NaN on the host) + 'V_LDEXP_F32': ('D0.f32 = S0.f32 * 2.0F ** S1.i32', 'D0.f32 = ldexp(S0.f32, S1.i32)'), + 'V_LDEXP_F64': ('D0.f64 = S0.f64 * 2.0 ** S1.i32', 'D0.f64 = ldexp(S0.f64, S1.i32)'), + # hardware sets SCC only on STRICT inequality for S_MAX (equal operands -> SCC=0) + 'S_MAX_I32': ('SCC = S0.i32 >= S1.i32', 'SCC = S0.i32 > S1.i32'), + 'S_MAX_U32': ('SCC = S0.u32 >= S1.u32', 'SCC = S0.u32 > S1.u32'), + # hardware computes abs on the WRAPPED 32-bit difference; the i32 pcode overflows into UB on the host (e.g. |45 - -2147483647|), + # so compute in u32 with a UB-free two's-complement negate + 'S_ABSDIFF_I32': ('D0.i32 = S0.i32 - S1.i32;\nif D0.i32 < 0 then\nD0.i32 = -D0.i32\nendif', + 'D0.u32 = S0.u32 - S1.u32;\nif D0.i32 < 0 then\nD0.u32 = -D0.u32\nendif'), + # CLASS denormal test uses abs(x) > 0.0, which the host's DAZ flushes; use bit-domain test instead + 'V_CMP_CLASS_F32': ('64\'F(abs(S0.f32)) > 0.0', '(64\'U(S0.u32 & 0x7FFFFFFF) != 0)'), + 'V_CMP_CLASS_F16': ('64\'F(abs(S0.f16)) > 0.0', '(64\'U(S0.u32 & 0x7FFF) != 0)'), + 'V_CMP_CLASS_F64': ('64\'F(abs(S0.f64)) > 0.0', '(64\'U(S0.u64 & 0x7FFFFFFFFFFFFFFF) != 0)'), } def _get_pcode_dict(op) -> dict: @@ -274,14 +292,24 @@ def _int_clamp(op_name: str, srcs: dict) -> UOp | None: if not isinstance(s0, UOp) or not isinstance(s1, UOp): return None is_signed, is_16bit = '_I' in op_name and '_U' not in op_name, '16' in op_name if any(p in op_name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')): - if is_16bit and is_signed: return None # skip 16-bit signed ops due to codegen issues - narrow_dt = dtypes.uint16 if is_16bit else (dtypes.int32 if is_signed else dtypes.uint32) - wide_dt = dtypes.int32 if is_16bit else dtypes.int64 - narrow_max, narrow_min = (0xFFFF, 0) if is_16bit else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0)) + op_bits = 16 if '16' in op_name else (24 if '24' in op_name else 32) + # D0 range: 16 for the *_U16/*_I16 result-narrow ops, else 32 (mad*32* D0 is u32/i32; mul operands have op-fmt width) + narrow_dt = dtypes.uint16 if is_16bit and '32' not in op_name else (dtypes.int32 if is_signed else dtypes.uint32) + wide_dt = dtypes.int64 + narrow_max, narrow_min = ((0xFFFF, 0) if narrow_dt == dtypes.uint16 else + ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0))) + def to_mulin(x: UOp) -> UOp: # mul-source: extract the op-fmt-width suboperand with sext for signed + mask = (1 << op_bits) - 1 + if op_bits == 32: return x.bitcast(narrow_dt) if x.dtype.itemsize == 4 else x.cast(narrow_dt) + m = (x & _c(mask)).cast(dtypes.int) + if not is_signed: return m.cast(wide_dt) + sign = (m >> _c(op_bits - 1)) & _c(1) + return sign.ne(_c(0)).where(m - _c(1 << op_bits), m).cast(wide_dt) def to_wide(x: UOp) -> UOp: return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt) - full = (to_wide(s0) * to_wide(s1) + to_wide(s2)) if 'MAD' in op_name and isinstance(s2, UOp) else \ - (to_wide(s1) - to_wide(s0)) if 'SUBREV' in op_name else \ - (to_wide(s0) - to_wide(s1)) if 'SUB' in op_name else (to_wide(s0) + to_wide(s1)) + if isinstance(s2, UOp) and 'MAD' in op_name: full = to_mulin(s0) * to_mulin(s1) + to_wide(s2) + elif 'SUBREV' in op_name: full = to_wide(s1) - to_wide(s0) + elif 'SUB' in op_name: full = to_wide(s0) - to_wide(s1) + else: full = to_wide(s0) + to_wide(s1) return full.clamp(narrow_min, narrow_max).cast(narrow_dt) # V_SUB_U32 / V_ADD_U32 with clamp: unsigned saturate (SUB underflow->0, ADD overflow->0xFFFFFFFF) if any(p in op_name for p in ('_SUB_U32', '_ADD_U32', '_SUB_U16', '_ADD_U16')): @@ -562,6 +590,10 @@ def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: vcc_reg = sdst_reg if sdst_reg is not None else VCC_LO.offset if 'VCC' not in srcs: srcs['VCC'] = self.rmask(_c(vcc_reg)) srcs.update(self.base_srcs(exec_mask, lane), VDST=vdst_reg, MAX_FLOAT_F32=UOp.const(3.4028234663852886e38, dtypes.float32)) + # f32 min/max/median ops flush denormal inputs to signed zero (select-style ops: results propagate inputs bitwise) + # (RDNA4 calls them _NUM_: V_MIN_NUM_F32 etc.) + if any(p in op.name for p in ('MIN_F32', 'MAX_F32', 'MIN3_F32', 'MAX3_F32', 'MED3_F32', 'MIN_NUM_F32', 'MAX_NUM_F32')): + srcs = {k: _ftz_f32(v) if k in ('S0', 'S1', 'S2') and isinstance(v, UOp) else v for k, v in srcs.items()} _, assigns = parse_pcode(pcode, srcs) # For integer ops with clamp, pre-compute the saturated result; floats clamp to [0,1] at write time @@ -586,8 +618,8 @@ def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: continue if int_saturate is not None: val = int_saturate elif clmp and val.dtype in (dtypes.float32, dtypes.half, dtypes.float64): - clamped = val.maximum(UOp.const(0.0, val.dtype)).minimum(UOp.const(1.0, val.dtype)) - val = _FUNCS['isNAN'](val).where(UOp.const(0.0, val.dtype), clamped) + # hardware clamp: -0 becomes +0 and NaN becomes 0 (hardware verified) + val = (val > UOp.const(0.0, val.dtype)).where(val.minimum(UOp.const(1.0, val.dtype)), UOp.const(0.0, val.dtype)) if val.dtype in (dtypes.uint64, dtypes.int64, dtypes.float64): lo, hi = _split64(val) lane_stores.extend([self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask), self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask)]) @@ -613,8 +645,9 @@ def compile_vop_pcode(self, op, srcs: dict[str, UOp | int], lane: UOp, vdst_reg: stores: list[UOp] = [] for mask_val, reg in [(vcc_val, vcc_reg), (exec_val, EXEC_LO.offset)]: if mask_val is None: continue + # hardware zeroes the inactive lane bits of per-lane VCC writes (VCC = mask & EXEC), it never preserves them stores.extend(self.wmask(_c(reg), self.unroll_lanes(lambda l, v=mask_val: (_to_u32(v.substitute({lane: l})) & _c(1)).cast(dtypes.uint32), - exec_mask, apply_exec=False))) + exec_mask, apply_exec=reg != EXEC_LO.offset))) if slice_stores: # merge D0[hi:lo] slices into one read-modify-write of the destination VGPR result = self.rvgpr_dyn(vdst_reg, lane) for lo_bit, width, val_bits in slice_stores: result = _set_bits(result, val_bits, width, lo_bit) @@ -986,6 +1019,9 @@ def get_cmp_bit(lane) -> UOp: s1 = _apply_src_mods(s1, 0, 1 if _iattr(inst, 'src1_abs') else 0, 1 if _iattr(inst, 'src1_neg') else 0, bits['s1']) s0 = _apply_src_mods(s0, 0, abs_bits, neg_bits, bits['s0']) s1 = _apply_src_mods(s1, 1, abs_bits, neg_bits, bits['s1']) + elif abs_bits or neg_bits: # int compares also honor abs/neg, as bit-level sign clear/flip (not integer abs/negate) + s0 = _apply_src_mods(s0, 0, abs_bits, neg_bits, bits['s0']) + s1 = _apply_src_mods(s1, 1, abs_bits, neg_bits, bits['s1']) for dest, val in parse_pcode(pcode, {'S0': s0, 'S1': s1, 'laneId': lc, 'D0': UOp.const(0, dtypes.uint64)})[1]: if '[laneId]' in dest and ('D0' in dest or 'EXEC' in dest): return val.cast(dtypes.uint32) return _c(0) @@ -994,12 +1030,9 @@ def get_cmp_bit(lane) -> UOp: # Both VOPC and VOP3 clear inactive lane bits (hardware verified) new_result = new_bits & exec_mask - # CMPX e32: writes EXEC only; CMPX e64: writes both EXEC and SDST; non-CMPX: writes dst only - if is_cmpx: - stores = ctx.wmask(_c(EXEC_LO.offset), new_result) - if not is_vopc: stores.extend(ctx.wmask(dst_off, new_result)) - else: - stores = ctx.wmask(dst_off, new_result) if not is_vopc else ctx.wmask(_c(VCC_LO.offset), new_result) + # CMPX writes EXEC only (hardware verified: e64 CMPX does not write SDST); non-CMPX writes SDST/VCC + if is_cmpx: stores = ctx.wmask(_c(EXEC_LO.offset), new_result) + else: stores = ctx.wmask(dst_off, new_result) if not is_vopc else ctx.wmask(_c(VCC_LO.offset), new_result) return UOp.sink(*stores, *ctx.inc_pc()) diff --git a/test/mockgpu/amd/pcode.py b/test/mockgpu/amd/pcode.py index 88584252dee16..89608c109b8c9 100644 --- a/test/mockgpu/amd/pcode.py +++ b/test/mockgpu/amd/pcode.py @@ -200,7 +200,20 @@ def _abs(val: UOp) -> UOp: def _f_to_u(f, dt): clamped = (f < _const(f.dtype, 0.0)).where(_const(f.dtype, 0.0), f) truncated = UOp(Ops.TRUNC, src=(clamped,)) - return (truncated >= _const(f.dtype, 2**(dt.itemsize*8))).where(_const(dt, dt.max), truncated.cast(dt)) + res = (truncated >= _const(f.dtype, 2**(dt.itemsize*8))).where(_const(dt, dt.max), truncated.cast(dt)) + return _isnan(f).where(_const(dt, 0), res) # float->uint conversion of NaN is 0 on hardware + +def _f_to_i32(a: UOp) -> UOp: + """v_cvt_i32_f32/f64: truncate toward zero, saturate to [INT_MIN, INT_MAX], NaN -> 0. + (x86 cvttss2si returns 0x80000000 for all of these, which matches hardware only for negative overflow.)""" + res = (a >= _const(a.dtype, 2147483648.0)).where(_const(dtypes.int, 0x7FFFFFFF), UOp(Ops.TRUNC, src=(a,)).cast(dtypes.int)) + return _isnan(a).where(_const(dtypes.int, 0), res) + +def _ftz_f32(v: UOp) -> UOp: + """Flush f32 denormals to signed zero (RDNA default float mode flushes denormal f32 inputs on select-style ops).""" + bits = v.bitcast(dtypes.uint32) if v.dtype == dtypes.float32 else v + return ((bits & _u32(0x7FFFFFFF)) < _u32(0x00800000)).where((bits & _u32(0x80000000)).bitcast(dtypes.float32), + v if v.dtype == dtypes.float32 else v.bitcast(dtypes.float32)) def _cvt_quiet(val: UOp) -> UOp: bits, _, _, qb, _ = _float_info(val) @@ -245,18 +258,51 @@ def _ldexp(val: UOp, exp: UOp) -> UOp: if val.dtype == dtypes.uint32: val = val.bitcast(dtypes.float32) elif val.dtype == dtypes.uint64: val = val.bitcast(dtypes.float64) if exp.dtype in (dtypes.uint32, dtypes.uint64): exp = exp.cast(dtypes.int if exp.dtype == dtypes.uint32 else dtypes.int64) - return val * UOp(Ops.EXP2, src=(exp.cast(val.dtype),)) + bits = val.bitcast(dtypes.uint32) if val.dtype == dtypes.float32 else val.bitcast(dtypes.uint64) + abs_max = _const(bits.dtype, 0x7F800000 if val.dtype == dtypes.float32 else 0x7FF0000000000000) + sign_mask = _const(bits.dtype, 0x80000000 if val.dtype == dtypes.float32 else 0x8000000000000000) + # hardware flushes denormal inputs to signed zero + magn_mask = _const(bits.dtype, 0x7FFFFFFF if val.dtype == dtypes.float32 else 0x7FFFFFFFFFFFFFFF) + is_denorm = ((bits & abs_max).eq(_const(bits.dtype, 0))) & ((bits & magn_mask).ne(_const(bits.dtype, 0))) + val = is_denorm.where((bits & sign_mask).bitcast(val.dtype), val) + # hardware propagates 0/+-inf/NaN unchanged (avoids 0*inf = NaN on the host) + res = val * UOp(Ops.EXP2, src=(exp.cast(val.dtype),)) + is_special = (bits & abs_max).eq(_const(bits.dtype, 0)) | ((bits & abs_max) >= abs_max) + return is_special.where(val, res) def _frexp_mant(val: UOp) -> UOp: val = val.bitcast(dtypes.float32) if val.dtype == dtypes.uint32 else val.bitcast(dtypes.float64) if val.dtype == dtypes.uint64 else val - if val.dtype == dtypes.float32: return ((val.bitcast(dtypes.uint32) & _u32(0x807FFFFF)) | _u32(0x3f000000)).bitcast(dtypes.float32) - return ((val.bitcast(dtypes.uint64) & _const(dtypes.uint64, 0x800FFFFFFFFFFFFF)) | - _const(dtypes.uint64, 0x3fe0000000000000)).bitcast(dtypes.float64) + if val.dtype == dtypes.float32: + bits = val.bitcast(dtypes.uint32) + # denormal/zero inputs (exponent field == 0) return signed zero on hardware + return ((bits & _u32(0x7F800000)).ne(_u32(0))).where(((bits & _u32(0x807FFFFF)) | _u32(0x3F000000)).bitcast(dtypes.float32), + (bits & _u32(0x80000000)).bitcast(dtypes.float32)) + bits = val.bitcast(dtypes.uint64) + return ((bits & _const(dtypes.uint64, 0x7FF0000000000000)).ne(_const(dtypes.uint64, 0))).where( + ((bits & _const(dtypes.uint64, 0x800FFFFFFFFFFFFF)) | _const(dtypes.uint64, 0x3fe0000000000000)).bitcast(dtypes.float64), + (bits & _const(dtypes.uint64, 0x8000000000000000)).bitcast(dtypes.float64)) + +def _msb(val: UOp, bits: int) -> UOp: + """Index of the highest set bit, or -1 if val == 0.""" + dt = dtypes.uint64 if bits > 32 else dtypes.uint32 + val = val.cast(dt) if val.dtype != dt else val + result = _const(dtypes.int, -1) + for i in range(bits - 1, -1, -1): + cond = ((val >> _const(dt, i)) & _const(dt, 1)).ne(_const(dt, 0)) & result.eq(_const(dtypes.int, -1)) + result = cond.where(_const(dtypes.int, i), result) + return result def _frexp_exp(val: UOp) -> UOp: val = val.bitcast(dtypes.float32) if val.dtype == dtypes.uint32 else val.bitcast(dtypes.float64) if val.dtype == dtypes.uint64 else val - if val.dtype == dtypes.float32: return ((val.bitcast(dtypes.uint32) >> _u32(23)) & _u32(0xFF)).cast(dtypes.int) - _const(dtypes.int, 126) - return ((val.bitcast(dtypes.uint64) >> _const(dtypes.uint64, 52)) & _const(dtypes.uint64, 0x7FF)).cast(dtypes.int) - _const(dtypes.int, 1022) + if val.dtype == dtypes.float32: + e = (val.bitcast(dtypes.uint32) >> _u32(23)) & _u32(0xFF) + return e.ne(_u32(0)).where(e.cast(dtypes.int) - _const(dtypes.int, 126), _const(dtypes.int, 0)) # f32 denormals -> 0 (hardware verified) + bits = val.bitcast(dtypes.uint64) + e = (bits >> _const(dtypes.uint64, 52)) & _const(dtypes.uint64, 0x7FF) + mant = bits & _const(dtypes.uint64, 0xFFFFFFFFFFFFF) + # f64 denormals: normalized exponent = highest set mantissa bit - 1073, zero -> 0 (hardware verified) + denorm = mant.ne(_const(dtypes.uint64, 0)).where(_msb(mant, 52) - _const(dtypes.int, 1073), _const(dtypes.int, 0)) + return e.ne(_const(dtypes.uint64, 0)).where(e.cast(dtypes.int) - _const(dtypes.int, 1022), denorm) TWO_OVER_PI = int( "0145f306dc9c882a53f84eafa3ea69bb81b6c52b3278872083fca2c757bd778ac36e48dc74849ba5c00c925dd413a32439fc3bd" @@ -314,9 +360,9 @@ def _sad_u8(a: UOp, b: UOp, acc: UOp, masked: bool = False) -> UOp: 'fma': lambda a, b, c: a * b + c, 'i32_to_f32': lambda a: a.cast(dtypes.int).cast(dtypes.float32), 'u32_to_f32': lambda a: a.cast(dtypes.uint32).cast(dtypes.float32), - 'f32_to_i32': lambda a: UOp(Ops.TRUNC, src=(a.bitcast(dtypes.float32),)).cast(dtypes.int), + 'f32_to_i32': lambda a: _f_to_i32(a.bitcast(dtypes.float32)), 'f32_to_u32': lambda a: _f_to_u(a.bitcast(dtypes.float32), dtypes.uint32), - 'f64_to_i32': lambda a: UOp(Ops.TRUNC, src=(a.bitcast(dtypes.float64),)).cast(dtypes.int), + 'f64_to_i32': lambda a: _f_to_i32(a.bitcast(dtypes.float64)), 'f64_to_u32': lambda a: _f_to_u(a.bitcast(dtypes.float64), dtypes.uint32), 'f16_to_f32': lambda a: _f16_extract(a).cast(dtypes.float32), 'f32_to_f16': lambda a: a.cast(dtypes.half), From 93865e2c66e0f798ebf0f18a10709d5daed91906 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:20:40 +0300 Subject: [PATCH 167/306] hcq2: sunday housekeeping (#17686) * hcq2: sunday housekeeping * x --- tinygrad/runtime/support/hcq2.py | 112 ++++++++++++++----------------- 1 file changed, 50 insertions(+), 62 deletions(-) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 4decf65da5a90..0ce66f98ac224 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -36,13 +36,15 @@ class HCQInfo: def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c -def unwrap_mstack(u): +def unwrap_mstack(u:UOp) -> tuple[UOp, ...]: if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s)) return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,) def unwrap_view(v:UOp) -> tuple[UOp, int]: return unwrap_view(v.src[0]) if v.op is Ops.BITCAST else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) +# patches + def is_value_known_at_link(val:UOp) -> bool: runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)] addressed_bufs = [b for g in val.toposort() if g.op is Ops.GETADDR for b in unwrap_mstack(g.buf_uop)] @@ -51,6 +53,7 @@ def is_value_known_at_link(val:UOp) -> bool: return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs) def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]: + # group patches into stacks: (tag, type, offset). offset is used for shrink later groups:dict[tuple[str|None, DType, sint], list[tuple[sint, UOp]]] = collections.defaultdict(list) for off, val in patches: tag = "link" if is_value_known_at_link(val) else "inputs" if val.op is Ops.GETADDR else None @@ -63,12 +66,7 @@ def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...] ret.append(view.index(offs).store(UOp(Ops.STACK, dt, tuple(val for _,val in ps))).rtag(tag)) return tuple(ret) -def make_binary_patch(buf:UOp, blob:bytes) -> UOp: - data = UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype) - r = UOp.range(len(blob) // buf.dtype.itemsize, 0, dtype=dtypes.int, src=(buf, data)) - return buf.index(r).store(data.index(r).load()).end(r).rtag("link") - -def make_buf(devs, slot:int=0, tag:str="signal") -> UOp: return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True, tag=tag) +def make_binary_patch(buf:UOp, blob:bytes) -> UOp: return buf.store(UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype)).rtag("link") def make_cmdbuf(lin, devs, buf:UOp|None=None, dep:tuple[UOp, ...]=()): blob, patches = bytearray(), [] @@ -90,6 +88,8 @@ def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: words = [get_call_arg_uops(call)[gi].getaddr(devs) for gi in info.globals] + list(info.vars) return buf.after(*make_patches(buf, list(zip(itertools.accumulate((w.dtype.itemsize for w in words), initial=0), words)))) +def make_buf(devs, slot:int=0, tag:str="signal") -> UOp: return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True, tag=tag) + # ***************** # 0.1. prep: replace buffers with params @@ -100,17 +100,6 @@ def replace_call_buffers(ctx:tuple[list[UOp], dict[UOp, int]], call:UOp) -> UOp| return call.replace(src=call.src[:1] + tuple(s if s.op is Ops.PARAM or s.is_bound_var else s.param_like(slots[s]) for s in call.src[1:])) pm_replace_buffers = PatternMatcher([(UPat(Ops.CALL, name="call"), replace_call_buffers)]) -# ***************** - -def stage_copy_ext(call:UOp) -> UOp|None: - if (d:=next((d for b in call.src[1:] for d in to_tuple(b.device) if not d.startswith("CPU")), None)) is None: return None - return pm.rewrite(call) if (pm:=getattr(Device[d], "pm_stage_copy", None)) is not None else None - -def encode_host_call(call:UOp) -> UOp|None: - if (pm:=getattr(Device[call.arg.aux.device[0]], "pm_host_lower", None)) is None: return None - body = graph_rewrite(call.src[0], pm, name="lower host access", enter_calls=True) - return None if body is call.src[0] else call.replace(src=(body, *call.src[1:])) - # ***************** # 1.1. prep: staging copies @@ -121,6 +110,10 @@ def _staging() -> Buffer: return Buffer("CPU", STAGING_SIZE, dtypes.uint8, preal def _need_staging(a, b): return all_devices_in(a.device, HCQ_DEVS - {"CPU"}) and not all_devices_in(b.device, HCQ_DEVS) +def stage_copy_ext(call:UOp) -> UOp|None: + if (d:=next((d for b in call.src[1:] for d in to_tuple(b.device) if not d.startswith("CPU")), None)) is None: return None + return pm.rewrite(call) if (pm:=getattr(Device[d], "pm_stage_copy", None)) is not None else None + def stage_copy(dst:UOp, src:UOp) -> UOp|None: if not (_need_staging(src, dst) or _need_staging(dst, src)): return None @@ -136,14 +129,13 @@ def stage_copy(dst:UOp, src:UOp) -> UOp|None: # 1.2. prep: kernel copies def _get_enqueue_devs(call:UOp) -> Any|None: - if (call.arg.name or "").startswith("hcq_"): return None # host exec is not any device - + if call.src[0].op not in (Ops.PROGRAM, Ops.COPY): return None # only these bodies can be enqueued if not (bufs:=call.src[1:]) or not all(all_devices_in(b.device, HCQ_DEVS) for b in bufs): return None if call.src[0].op is Ops.COPY: bufs = bufs[::-1] # copies push from the src device: p2p writes are faster than reads devs = min(bufs, key=lambda b: to_tuple(b.device)[0].startswith("CPU")).device # prio to enqueue on not CPU device return devs if all_devices_in(devs, HCQ_DEVS) else None -def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: +def copy_with_kernel(call:UOp, dst:UOp, src:UOp) -> UOp|None: if (devs:=_get_enqueue_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None d, s = (UOp.param(i, dst.dtype, (n:=dst.max_numel(),), device=devs) for i in range(2)) ast = d.index(r:=UOp.range(n, 0)).store(s.index(r).load()).end(r).sink(arg=KernelInfo(name="copy"), tag=1) @@ -152,7 +144,7 @@ def kernel_copy(call:UOp, dst:UOp, src:UOp) -> UOp|None: pm_insert_copy_staging = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), stage_copy_ext), (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), stage_copy), - (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src")), name="call"), kernel_copy) + (UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src")), name="call"), copy_with_kernel) ]) # ***************** @@ -174,7 +166,7 @@ def _get_deps(ctx:DepsTracker, bufs_by_lane:list[list[Any]], write, key:tuple[tu dep_lanes += [(dep, dlane, lane) for dep, dlane in ctx.access_resources(bufs, written, (key, lane))] return dep_lanes -def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int]], devices:tuple[str, ...], queue:str) -> tuple[list[UOp], set[int]]: +def _build_wait_ins(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int]], devices:tuple[str, ...], queue:str) -> tuple[list[UOp], set[int]]: # opt1: same-queue ops are fifo-ordered if devices[0].split(":")[0] in {"AMD", "QCOM", "CPU"} or queue.startswith("COPY"): dep_lanes = [(dep, dlane, lane) for dep, dlane, lane in dep_lanes if (dep[0][dlane], dep[1]) != (devices[lane], queue)] @@ -205,7 +197,7 @@ def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[t # to finalize the batch, sync all accesses from other devices to buffers that belong to this device fin_deps = [dl for dl in _get_deps(tracker, [list(dev_bufs[d].values()) for d in devs], None, key=(devs, "COMPUTE:0", n)) if dl[0][2] < n] - waits, cur_signal_tags = _build_wait_cmds(slots, fin_deps, devs, "COMPUTE:0") + waits, cur_signal_tags = _build_wait_ins(slots, fin_deps, devs, "COMPUTE:0") signal_tags |= cur_signal_tags # wait the syncs and signal the device epoch, then bump the timeline on the host @@ -226,6 +218,31 @@ def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[t fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) return fences + resets, fins, signal_tags +def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? + if len(calls) == 1: return calls[0] + devs, queue = get_submit(calls[0]).src[0].arg + body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() + return make_call(f"submit {queue} ({len(calls)})", body, + replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) + +def _merge_queues(submits:list[UOp]) -> list[UOp]: + new_src:list[UOp] = [] + opened_qs:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> list of hcq calls, kept in submit order + limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) + + for call in submits: + devs, queue = get_submit(call).src[0].arg + if (old:=opened_qs.pop(key:=(devs, queue), None)) is not None: + if limits[key] and len(old) >= limits[key]: new_src, old, limits[key] = new_src + [_merged_hcq_call(old)], [], limits[key] * 2 + new_rec = old + [call] + else: + # no such queue opened: close every open submit on this queue that shares a device, so submit order is kept + closing = [k for k in opened_qs if k[1] == queue and set(k[0]) & set(devs)] + new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in closing] + new_rec = [call] + opened_qs[(devs, queue)] = new_rec + return new_src + [_merged_hcq_call(c) for c in opened_qs.values()] + def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: batch_info = [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch] @@ -236,7 +253,7 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li call_waits:list[list[UOp]] = [] for tag, ((call, _), (devices, queue)) in enumerate(zip(batch, batch_info)): deps = _get_deps(deps_tracker, _get_call_bufs_by_lane(call, devices), get_call_outs_ins(call)[0], key=(devices, queue, tag)) - cmds, cur_signal_tags = _build_wait_cmds(slots, deps, devices, queue) + cmds, cur_signal_tags = _build_wait_ins(slots, deps, devices, queue) call_waits.append(cmds) signal_tags |= cur_signal_tags @@ -265,7 +282,7 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li # append batch timestamps to finalizers fins = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, kernels=tuple(x for x in kerns if set(x[0]) & set(a.device))))) for f in fins] - return fences + src + fins + return fences + _merge_queues(src) + fins def sched_hcq_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] @@ -275,45 +292,17 @@ def sched_hcq_batches(l:UOp, profile:bool) -> UOp: else: srcs, batch = srcs + _finalize_batch(batch, profile) + [call], [] return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) -# ***************** -# 3. merge into queues - -def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? - if len(calls) == 1: return calls[0] - devs, queue = get_submit(calls[0]).src[0].arg - body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() - return make_call(f"submit {queue} ({len(calls)})", body, - replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) - -def merge_queues(linear:UOp) -> UOp: - new_src:list[UOp] = [] - opened_qs:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> list of hcq calls, kept in submit order - limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) - - for call in linear.src: - # non-hcq call, fence or finalizer: close all open queues - if not isinstance(call.arg.aux, HCQInfo) or (call.arg.name or "").startswith("hcq_"): - new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in list(opened_qs)] + [call] - continue - - devs, queue = get_submit(call).src[0].arg - if (old:=opened_qs.pop(key:=(devs, queue), None)) is not None: - if limits[key] and len(old) >= limits[key]: new_src, old, limits[key] = new_src + [_merged_hcq_call(old)], [], limits[key] * 2 - new_rec = old + [call] - else: - # no such queue opened: close every open submit on this queue that shares a device, so submit order is kept - closing = [k for k in opened_qs if k[1] == queue and set(k[0]) & set(devs)] - new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in closing] - new_rec = [call] - opened_qs[(devs, queue)] = new_rec - return linear.replace(src=tuple(new_src + [_merged_hcq_call(c) for c in opened_qs.values()])) - pm_schedule_and_merge = PatternMatcher([(UPat(Ops.LINEAR, name="l"), - lambda ctx, l: merge_queues(sched_hcq_batches(l, ctx[1]).substitute(ctx[0], walk=True, enter_calls=True)))]) + lambda ctx, l: sched_hcq_batches(l, ctx[1]).substitute(ctx[0], walk=True, enter_calls=True))]) # ***************** # 4.2. hcq lowering: ops to ir +def encode_host_call(call:UOp) -> UOp|None: + if (pm:=getattr(Device[call.arg.aux.device[0]], "pm_host_lower", None)) is None: return None + body = graph_rewrite(call.src[0], pm, name="lower host access", enter_calls=True) + return None if body is call.src[0] else call.replace(src=(body, *call.src[1:])) + def encode_cmdbuf(submit:UOp, lin:UOp) -> UOp|None: if (pm:=Device.get_class(lin.arg[0][0]).pm_lower) is None: return None return graph_rewrite(submit, pm, name=f"encode {lin.arg[0]}", enter_calls=True) @@ -545,8 +534,7 @@ def resolve_getaddr(buf:UOp, g:UOp) -> UOp: (UPat(Ops.GETADDR, src=(UPat(name="buf"),), name="g"), resolve_getaddr), # folders - (UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True).store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()) - .index(UPat(Ops.RANGE), allow_any_len=True).load()).end(UPat(Ops.RANGE)), fold_binary), + (UPat(name="buf").store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast())), fold_binary), (UPat((Ops.BITCAST, Ops.SHRINK, Ops.BUFFER, Ops.MSTACK), name="view") .index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), ]) From bb0e99acbf1e8ed3a4f08388666f0aab1b0fbca0 Mon Sep 17 00:00:00 2001 From: George Hotz Date: Sun, 23 Aug 2026 08:40:46 -0700 Subject: [PATCH 168/306] hotfix: skip that nan test on mac --- test/amd/hw/test_vop3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/amd/hw/test_vop3.py b/test/amd/hw/test_vop3.py index ff9217e370661..52ed3c73e50b6 100644 --- a/test/amd/hw/test_vop3.py +++ b/test/amd/hw/test_vop3.py @@ -4,6 +4,7 @@ v_alignbit_b32, v_bfe_i32, v_mad_u64_u32, v_readlane_b32, v_writelane_b32 """ import unittest +from tinygrad.helpers import OSX from test.amd.hw.helpers import * class TestFMA(unittest.TestCase): @@ -3720,6 +3721,7 @@ def test_v_mul_neg_modifier_nan_sign(self): class TestNaNPropagationRegressions(unittest.TestCase): """Regression tests: float arithmetic propagates a NaN from the FIRST NaN operand, quieted with its own sign/payload.""" + @unittest.skipIf(OSX, "broken on mac, TODO: why?") def test_mul_nan_priority(self): # first NaN operand wins (sign+payload), not x86's second-source propagation for a, b, want in [(0x7FC00001, 0x7F800003, 0x7FC00001), (0xFFC00005, 0x7F800003, 0xFFC00005), From 477b5738079afe8bab4841680e7c65af09075990 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Sun, 23 Aug 2026 14:02:54 -0700 Subject: [PATCH 169/306] update llm kv cache to be half (#17690) * update llm kv cache to be half / chunk_size to always be 32 * just dtype --- tinygrad/llm/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index 0d167f43133ef..e9b68860181f1 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -200,7 +200,7 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: def _init_state(self, x:Tensor): if not hasattr(self, "cache_kv"): self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, - dtype=dtypes.default_float, device=x.device) + dtype=dtypes.half, device=x.device) self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device) class MLATransformerBlock(FFNBlock): From 8b164aefeae34be71e15b75ae2a2a6dda9f2945c Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 23 Aug 2026 17:30:16 -0400 Subject: [PATCH 170/306] test update from weak const (#17689) --- test/mockgpu/amd/pcode.py | 4 +--- test/null/test_simplify_valid_idx.py | 4 ++-- test/null/test_uop_graph.py | 8 ++++---- test/null/test_uop_symbolic.py | 3 +-- test/null/test_uops.py | 12 ++++++------ 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/test/mockgpu/amd/pcode.py b/test/mockgpu/amd/pcode.py index 89608c109b8c9..fa5f214715197 100644 --- a/test/mockgpu/amd/pcode.py +++ b/test/mockgpu/amd/pcode.py @@ -1022,9 +1022,7 @@ def parse_bound(): p.eat('NUM') p.eat('QUOTE') if p.at('NUM'): return int(p.eat('NUM').val.rstrip('UuLl')) - expr = p.parse().simplify() - assert expr.op == Ops.CONST, f"loop bound must be constant, got {expr}" - return int(expr.val) + return int(p.parse()) start_val = parse_bound() p.eat('COLON') end_val = parse_bound() diff --git a/test/null/test_simplify_valid_idx.py b/test/null/test_simplify_valid_idx.py index 2d278eb75fed7..2166e69e8a07a 100644 --- a/test/null/test_simplify_valid_idx.py +++ b/test/null/test_simplify_valid_idx.py @@ -3,7 +3,7 @@ from tinygrad.codegen.late.coalesce import indexing_simplify from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops, graph_rewrite -from tinygrad.uop.weak import pm_lower_index_dtype +from tinygrad.uop.weak import pm_commit_weak from tinygrad.uop.symbolic import simplify_valid, sym, pm_move_where_on_load from tinygrad.helpers import Context from test.helpers import full_rewrite @@ -496,7 +496,7 @@ def test_drop_gate_committed_in_the_index_pass(self): idx_y = (f + UOp.const(1.0)).cast(dtypes.int) load = get_load_image_uop((10, 10, 4), (UOp.const(-1) < idx_y) & (idx_y < UOp.const(10)), (Special("gidx0", 10), idx_y)) - off = graph_rewrite(load.sink(), pm_lower_index_dtype+indexing_simplify, ctx={}).src[0].src[0] + off = graph_rewrite(load.sink(), pm_commit_weak+indexing_simplify).src[0].src[0] self.assertEqual(off.src[1].get_valid(), UOp.const(True)) class TestDropTrueGate(unittest.TestCase): diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index e144da641a037..6f9464b8c915d 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -230,11 +230,11 @@ def test_double_cast_fold(self): def test_depth_2_const_fold(self): v = UOp.variable("tmp", 0, 1, dtypes.int, param=True) - c2 = UOp.const(2, dtypes.int) - c4 = UOp.const(4, dtypes.int) + c2 = UOp.const(2) + c4 = UOp.const(4) vc = v+c2 out = vc+c4 - self.assertIs(out.simplify(), (v+UOp.const(6, dtypes.int)).simplify()) + self.assertIs(out.simplify(), (v+UOp.const(6)).simplify()) def test_bitcast_to_same_dtype_fold(self): for dt in dtypes.ints + dtypes.floats + (dtypes.bool,): @@ -245,7 +245,7 @@ def test_bitcast_to_same_dtype_fold(self): def test_sub_with_cast_folds(self): a = Variable("a", 0, 5) - out = a.cast(dtypes.int)+(-a).cast(dtypes.int) + out = a+(-a) self.assertIs(full_rewrite(out.sink()).src[0], full_rewrite(UOp.const(0, dtypes.int).sink()).src[0]) def test_where_on_gated_load_fold(self): diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 0f27eb0e166a5..08f5dc2f2bdba 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -6,7 +6,6 @@ from tinygrad.uop.ops import UOp, Ops, graph_rewrite, sym_infer from tinygrad.uop.spec import spec_shared, type_verify from tinygrad.uop.symbolic import sym, commutative, pm_simplify_valid, pm_move_where_on_load -from tinygrad.uop.weak import pm_cast_weak from tinygrad.uop.validate import uops_to_z3 def check_uop_against_string(self, v:UOp, s:str): @@ -36,7 +35,7 @@ def check_equal_z3(self, expr1, expr2): self.assertEqual(solver.check(expr1 != expr2), z3.unsat, "simplified expression not equal to original") def helper_test_variable(self, v, n, m, s, test_z3:bool=True): - v_simplified = graph_rewrite(v, sym+pm_cast_weak, name="simplify symbolic uop") + v_simplified = graph_rewrite(v, sym, name="simplify symbolic uop") if test_z3: self.check_equal_z3(v, v_simplified) nmin, nmax = v_simplified.vmin, v_simplified.vmax check_uop_against_string(self, v_simplified, s) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index 43cd44de4f430..2128234644f82 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -280,7 +280,7 @@ class TestFastIdiv(unittest.TestCase): def test_division_power_of_two(self): for dt in (dtypes.int32, dtypes.uint32): g = UOp.param(0, dt, (3,)) - c = UOp.const(2).cast(dt) + c = UOp.const(2) l = g.index(c) a = UOp(Ops.CDIV, dt, (l, c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) @@ -293,7 +293,7 @@ def test_floormod_power_of_two(self): # FLOORMOD by a power of two lowers to AND (correct floor mod for any sign in two's complement) for dt in (dtypes.int32, dtypes.uint32): g = UOp.param(0, dt, (9,)) - c = UOp.const(8).cast(dt) + c = UOp.const(8) a = UOp(Ops.FLOORMOD, dt, (g.index(c), c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) ops = [x.op for x in uops] @@ -305,7 +305,7 @@ def test_floordiv_power_of_two(self): # FLOORDIV by a power of two lowers to a shift, with no round toward zero correction (a shift is exactly floor division) for dt in (dtypes.int32, dtypes.uint32, dtypes.int64, dtypes.uint64): g = UOp.param(0, dt, (3,)) - c = UOp.const(2).cast(dt) + c = UOp.const(2) a = UOp(Ops.FLOORDIV, dt, (g.index(c), c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) ops = [x.op for x in uops] @@ -318,7 +318,7 @@ def test_floordiv_power_of_two(self): @unittest.skipIf(Device.DEFAULT == "WEBGPU", "WEBGPU doesn't support long") def test_fast_idiv_and_mod(self): g = UOp.param(0, dtypes.uint32, (4,)) - c = UOp.const(3).cast(dtypes.uint) + c = UOp.const(3) l = g.index(c) a = UOp(Ops.CDIV, src=(l, c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) @@ -338,7 +338,7 @@ def test_fast_idiv_and_mod(self): def test_fast_idiv_bounded_numerator_zero(self): x = UOp.variable("x", 0, 1, dtype=dtypes.int32) for val in range(2): - self.assertEqual(eval_uop(x.alu(Ops.CDIV, UOp.const(3).cast(x.dtype)), vals=(val,)), cdiv(val, 3)) + self.assertEqual(eval_uop(x.alu(Ops.CDIV, UOp.const(3)), vals=(val,)), cdiv(val, 3)) @Context(DISABLE_FAST_IDIV=0) def test_fast_idiv_remove_powers_of_two(self): @@ -363,7 +363,7 @@ def test_fast_idiv_overflow(self): def test_disable_fast_idiv(self): g = UOp.param(0, dtypes.uint32, (4,)) - c = UOp.const(3).cast(dtypes.uint) + c = UOp.const(3) l = g.index(c) a = UOp(Ops.CDIV, src=(l, c)) with Context(DISABLE_FAST_IDIV=1): From a31aca9e52a6859cbfc6e7d974d22231657cd2f1 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 23 Aug 2026 22:00:41 -0400 Subject: [PATCH 171/306] don't hardcode dtype.int const in decode_hevc_frame (#17696) --- tinygrad/tensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index d23c6ce33ac76..28d0dfb3065ce 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -744,7 +744,7 @@ def decode_hevc_frame(self, frame_pos:Variable, shape:tuple[int,...], state:Tens ref_frames = [x.contiguous() for x in ref_frames or []] assert frame_pos.is_bound_var, "frame_pos must be a bound Variable" srcs = (out:=Tensor.empty(*shape, device=self.device, dtype=self.dtype), self.contiguous(), state.contiguous(), *ref_frames) - fn = UOp(Ops.CUSTOM_FUNCTION, src=(frame_pos.src[0], *[UOp.const(s, dtypes.int) for s in shape]), arg="encdec") + fn = UOp(Ops.CUSTOM_FUNCTION, src=(frame_pos.src[0], *[UOp.const(s) for s in shape]), arg="encdec") return Tensor(out.uop.after(fn.call(*[s.uop for s in srcs], frame_pos))) P = ParamSpec("P") From 11edcc144ca3587ed116d7d156ff0ffe2cda01aa Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Sun, 23 Aug 2026 19:18:47 -0700 Subject: [PATCH 172/306] disk cache: thread-local db conn (#17694) --- tinygrad/helpers.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 51fbf5dc39864..8e6e7af03a9b3 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -1,7 +1,7 @@ from __future__ import annotations import time START_TIME = time.perf_counter() -import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc +import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc, threading from collections import defaultdict import shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib from dataclasses import dataclass, field, replace @@ -398,18 +398,17 @@ def my_gc_callback(phase, info): CACHEDB: str = getenv("CACHEDB", os.path.abspath(os.path.join(cache_dir, "cache.db"))) VERSION = 22 -_db_connection = None +_db_connection = threading.local() def db_connection(): - global _db_connection - if _db_connection is None: + if (conn:=getattr(_db_connection, "conn", None)) is None: os.makedirs(CACHEDB.rsplit(os.sep, 1)[0], exist_ok=True) - _db_connection = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") + conn = _db_connection.conn = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") # another connection has set it already or is in the process of setting it # that connection will lock the database - with contextlib.suppress(sqlite3.OperationalError): _db_connection.execute("PRAGMA journal_mode=WAL").fetchone() - _db_connection.execute("PRAGMA synchronous=NORMAL") - if DEBUG >= 8: _db_connection.set_trace_callback(print) - return _db_connection + with contextlib.suppress(sqlite3.OperationalError): conn.execute("PRAGMA journal_mode=WAL").fetchone() + conn.execute("PRAGMA synchronous=NORMAL") + if DEBUG >= 8: conn.set_trace_callback(print) + return conn def diskcache_clear(): cur = db_connection().cursor() From 9d0cd0ebcb49ea473ff24d24f87fa4c847f11d52 Mon Sep 17 00:00:00 2001 From: George Hotz Date: Sun, 23 Aug 2026 20:34:30 -0700 Subject: [PATCH 173/306] hotfix: switch benchmark to qwen3.8 --- .github/workflows/benchmark.yml | 6 +++--- .github/workflows/test.yml | 2 +- tinygrad/llm/cli.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ecefd8f432e48..33023859c7ee2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -117,10 +117,10 @@ jobs: run: python3 test/external/process_replay/reset.py - name: Run llama3.2 run: BENCHMARK_LOG=llama32_3b-f16 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m llama3.2:3b-f16 --benchmark --warmup - - name: Run qwen3.6 - # qwen3.6:35b-a3b doesn't fit on mac + - name: Run qwen3.8 + # qwen3.8:27b doesn't fit on mac if: ${{ matrix.dev != 'METAL' }} - run: BENCHMARK_LOG=qwen36_35b-a3b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.6:35b-a3b --benchmark --warmup + run: BENCHMARK_LOG=qwen38_27b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.8:27b --benchmark --warmup - name: Run olmoe # just metal for now if: ${{ matrix.dev == 'METAL' }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70b474c17701e..38b87c21cd13b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,7 @@ jobs: - name: Run process replay tests uses: ./.github/actions/process-replay - name: Repo line count <= 26000 lines - run: MAX_LINE_COUNT=26000 python sz.py + run: MAX_LINE_COUNT=26500 python sz.py spec: strategy: diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index d0865e26ba21f..5d37b5172eecc 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -88,6 +88,8 @@ def is_end(self, token_id:int) -> bool: return token_id in (self.eos_id, self.eo "qwen3.5:9b": "https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf", "qwen3.6:27b": "https://huggingface.co/unsloth/Qwen3.6-27B-GGUF/resolve/main/Qwen3.6-27B-Q4_K_M.gguf", "qwen3.6:35b-a3b": "https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + # pinned to the last revision with the plain IQ4_XS quant: the UD replacement uses Q3_K tensors the loader doesn't support + "qwen3.8:27b": "https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/b62a80264f8b0c1bb849ee1c9c487415ebeca194/Qwen3.8-27B-IQ4_XS.gguf", "olmoe": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct-GGUF/resolve/main/olmoe-1b-7b-0924-instruct-q4_k_m.gguf", "moonlight": "https://huggingface.co/gabriellarson/Moonlight-16B-A3B-Instruct-GGUF/resolve/main/Moonlight-16B-A3B-Instruct-Q4_K_M.gguf", "glm-4.7-flash": "https://huggingface.co/unsloth/GLM-4.7-Flash-GGUF/resolve/main/GLM-4.7-Flash-Q4_K_M.gguf", From 4ee114b42a1ef8aa415141cc91741556e7135575 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:49:55 +0800 Subject: [PATCH 174/306] llama rope freqs in fp32 (#17698) --- extra/models/llama.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/models/llama.py b/extra/models/llama.py index 4bd2d0d06f9f2..314e663a7996b 100644 --- a/extra/models/llama.py +++ b/extra/models/llama.py @@ -5,9 +5,9 @@ # https://github.com/facebookresearch/llama/blob/1076b9c51c77ad06e9d7ba8a4c6df775741732bd/llama/model.py#L47 def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0) -> Tensor: - freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2)[:(dim // 2)] / dim)) - freqs = Tensor.arange(end).unsqueeze(dim=1) * freqs.unsqueeze(dim=0) - return Tensor.stack(freqs.cos(), freqs.sin(), dim=-1).reshape(1, end, 1, dim//2, 2) + freqs = 1.0 / (theta ** (Tensor.arange(0, dim, 2, dtype=dtypes.float32)[:(dim // 2)] / dim)) + freqs = Tensor.arange(end, dtype=dtypes.float32).unsqueeze(dim=1) * freqs.unsqueeze(dim=0) + return Tensor.stack(freqs.cos(), freqs.sin(), dim=-1).cast(dtypes.default_float).reshape(1, end, 1, dim//2, 2) # matches meta, non hugging face weights # (a+i*b) * (c+i*d) = (ac-bd) + i*(ad+bc) From df528499ced865a714baff5a4ab2ad52ff2652ed Mon Sep 17 00:00:00 2001 From: George Hotz Date: Sun, 23 Aug 2026 21:13:29 -0700 Subject: [PATCH 175/306] Revert "disk cache: thread-local db conn (#17694)" This reverts commit 11edcc144ca3587ed116d7d156ff0ffe2cda01aa. --- tinygrad/helpers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 8e6e7af03a9b3..51fbf5dc39864 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -1,7 +1,7 @@ from __future__ import annotations import time START_TIME = time.perf_counter() -import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc, threading +import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc from collections import defaultdict import shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib from dataclasses import dataclass, field, replace @@ -398,17 +398,18 @@ def my_gc_callback(phase, info): CACHEDB: str = getenv("CACHEDB", os.path.abspath(os.path.join(cache_dir, "cache.db"))) VERSION = 22 -_db_connection = threading.local() +_db_connection = None def db_connection(): - if (conn:=getattr(_db_connection, "conn", None)) is None: + global _db_connection + if _db_connection is None: os.makedirs(CACHEDB.rsplit(os.sep, 1)[0], exist_ok=True) - conn = _db_connection.conn = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") + _db_connection = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") # another connection has set it already or is in the process of setting it # that connection will lock the database - with contextlib.suppress(sqlite3.OperationalError): conn.execute("PRAGMA journal_mode=WAL").fetchone() - conn.execute("PRAGMA synchronous=NORMAL") - if DEBUG >= 8: conn.set_trace_callback(print) - return conn + with contextlib.suppress(sqlite3.OperationalError): _db_connection.execute("PRAGMA journal_mode=WAL").fetchone() + _db_connection.execute("PRAGMA synchronous=NORMAL") + if DEBUG >= 8: _db_connection.set_trace_callback(print) + return _db_connection def diskcache_clear(): cur = db_connection().cursor() From a5ea95d8b815090b39fd6ce90d0ec4bae479953e Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Sun, 23 Aug 2026 21:30:00 -0700 Subject: [PATCH 176/306] usb: don't fail async transfers on signal interruption (#17692) --- tinygrad/runtime/support/usb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index 18ce7903097ad..dde7a97201355 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -113,8 +113,10 @@ def control_read_async(self, request:int, length:int, value:int=0, index:int=0, return self._submit_async(0, libusb.LIBUSB_TRANSFER_TYPE_CONTROL, buf, timeout), memoryview(buf)[8:] def bulk_wait(self, tag:int): - """Block until the tagged transfer completes; raises if any async transfer failed.""" - while tag in self._async_pending: checked(libusb.libusb_handle_events)(None) + """Block until the tagged transfer completes; raises if any async transfer failed. LIBUSB_ERROR_INTERRUPTED is retried.""" + while tag in self._async_pending: + if (rc:=libusb.libusb_handle_events(None)) < 0 and rc != libusb.LIBUSB_ERROR_INTERRUPTED: + raise RuntimeError(f"libusb_handle_events: {ctypes.string_at(libusb.libusb_strerror(rc)).decode()}") if self._async_err: raise RuntimeError(f"async bulk OUT failed: status={self._async_err}") def bulk_read(self, length:int, timeout:int=1000) -> memoryview: From 7e561fcb9734fb0e47226840b170c557a4204342 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:46:35 -0700 Subject: [PATCH 177/306] mergable fast RDNA3 Qwen 3.8 (#17512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * mergable fast RDNA3 Qwen 3.6 * AMD * quant 256 multiple * cleanup cast * llm kernels: adapt to Ops.BIND removal Variables are 0-d ALU BUFFERs in the tensor graph and take the ALU PARAM form inside kernels (UOp.variable(param=True)). Add kernel_var helper for the conversion, and keep start_pos in bound form at the graph level so function implicit-input collection and the schedule's binds rename-back line up. * adaptive prefill chunk sizes for recurrent models + iq4xs model entry one TinyJit per static prefill chunk size: capture 128 and 32 at warmup, generate picks the largest that fits the remaining prompt. long prompts prefill 2x faster (555 tok/s on Qwen3.6-27B IQ4_XS) without pushing short prompts through token-by-token decode. * minimize diff: early-return custom attention path, keep master state init * minimize: master _attention with gated fused-scan swap, kernels/amd only, single chunk size - GatedDeltaNetBlock._attention keeps master's symbolic-padding structure; the recurrent scan is swapped for the fused gated_delta_prefill kernel only on RDNA3 with static shapes (fast_scan), everything else uses the old path - all AMD kernel code lives in tinygrad/llm/kernels/amd.py (drop kernels/__init__.py, drop the generic fallback kernel - the old scan covers non-RDNA3) - single prefill chunk size 32; non-RDNA3 recurrent keeps master's chunk_size=1 - the conv+normalize miscompile doesn't trigger with master's window-buffer conv, so the contiguous workaround is dropped * warmup: single code path for fast and old recurrent * drop fast_scan/fast_recurrent flags, inline the RDNA3 gate (cached) * generate: chunk size is always 32, no device gating static chunks for recurrent models everywhere: the fused kernel path on RDNA3, the old scan elsewhere (which is also faster chunked than token-by-token) * warmup: drop redundant _init_state loop (lazy init in the eager step covers it) * symbolic-length prefill with the custom kernels the prefill path is fully symbolic again (master's generate, one prefill graph for every chunk size, no static-tail decode): padded steps are exact no-ops in the scan (beta=0, alpha=exp(0)=1), flash attention positions queries at start_pos instead of valid_kv_len-M, and quant linears pad to the chunk bucket prefill 401 tok: 284 -> 348 tok/s on Qwen3.8-27B IQ4_XS (tail chunks no longer decode token-by-token), decode unchanged at 45 tok/s * cli: default qwen3.6:27b to the fast IQ4_XS quant, add qwen3.8:27b Q4_K_M falls back to slow inline dequant with the custom kernels, IQ4_XS is the fast path. qwen3.8 quants use unsloth's UD (dynamic) naming * warmup: back to master's two-liner plus a cache reset with symbolic prefill, generate([0])'s 1-token chunk captures the symbolic prefill graph that serves every chunk size, and JIT batching on capture measurably doesn't matter with the fused kernels (347.7 tok/s either way) * cli: pin qwen3.8:27b to the pre-UD revision the UD-IQ4_XS replacement mixes in Q3_K tensors (ggml type 11) the loader doesn't support; the pinned revision is byte-identical to the known-good file * warmup: identical to master the leftover cache is self-consistent: get_start_pos only reuses a full strict-prefix match, everything else restarts with a state reset * model: hoist the quantized_attention import to the top level * hoist the GDN query scale out of the branch, restore master dtype.py the scale is the same op in both paths, apply it once after the transpose. the dtype.py diff was a stale pre-SPEC=2 copy, not intentional work * gated_delta_prefill: don't pass the bound start_pos as a call src device-less param buffers in call srcs crash hcq2's _get_enqueue_devs. the var already reaches the graph through the state AFTER chain (conv state store), same as the flash kernels' valid_end * llm: half KV cache with custom flash kernels, drop the int8 quantized cache matches master's new half cache default: no scales, no packing, one less buffer. the store casts to half explicitly (buffer-only half usage misses the renderer's half define). 45.5 tok/s decode, 348.7 tok/s prefill — same as int8 * llm: zero-init the KV cache the int8 path was accidentally protected from uninitialized memory by its zero-initialized scale buffer; with a plain half cache the flash prefill kernel's P*V wmma computes 0*NaN=NaN on masked lanes past the valid region (manifested as garbage tokens at 32k context where the allocator reuses dirty VRAM) * gate that * llm/kernels/amd: reorganize by kernel family, drop the clutter sections: shared helpers, quant linear, flash attention, gated delta prefill. no AxisType.WEAK (default), no ALLOW_DEVICE_USAGE override (unneeded), magic numbers become names (QUANT_SIZES, Q5_K/Q6_K/IQ4_XS), merged wrapper layers (flash_attention_causal_cached folded into flash_attention), one _unbind helper for the bound-var dance * test: universal recurrent reuse assertion, fix lambda lint * 1-token chunks have a static shape: they are decode steps a 1-token chunk routes to the decode graph via the existing dispatch, so warmup and decode-only workloads never build the big symbolic prefill graph: CI benchmark command 12m50s -> 5m29s (master: 6m48s), 220 -> 123 compile jobs also restores the ALLOW_DEVICE_USAGE override in amd_custom_kernels_supported: Device[] asserts inside @function contexts (ALLOW_DEVICE_USAGE=0), and the first gate call can happen there depending on test order * generate: back to plain symbolic binding, the static-1 rule wasn't worth it * custom kernels: Q4_K support (ggml type 12) Q4_K is Q5_K without the high-bit array: same d/dmin/scales layout (so _q5_scales works unchanged), 144-byte blocks, qs at word 4. both the dp4a decode kernel and the WMMA prefill kernel take a ggml_type branch now. Qwen3-8B Q4_K_M: decode 16.5 -> 114.8 tok/s, prefill 69 -> 536 tok/s * raise line count to 26500 (qwen did it) * benchmark qwen3.8 * little updates --- test/unit/test_attention.py | 28 ++ test/unit/test_llm_amd.py | 107 ++++++++ test/unit/test_llm_server.py | 5 +- tinygrad/llm/kernels/amd.py | 499 +++++++++++++++++++++++++++++++++++ tinygrad/llm/model.py | 48 ++-- 5 files changed, 669 insertions(+), 18 deletions(-) create mode 100644 test/unit/test_llm_amd.py create mode 100644 tinygrad/llm/kernels/amd.py diff --git a/test/unit/test_attention.py b/test/unit/test_attention.py index 8203caf0d9564..2efa3afe5059e 100644 --- a/test/unit/test_attention.py +++ b/test/unit/test_attention.py @@ -5,6 +5,8 @@ GatedDeltaNetBlock, SSMConfig, TransformerBlock, TransformerConfig, apply_rope as apply_rope_new, precompute_freqs_cis, pairwise_topk, ) +from tinygrad.llm.kernels.amd import Linear, gated_delta_prefill, amd_custom_kernels_supported +from tinygrad.llm.gguf import ggml_data_to_tensor def apply_rope(x:Tensor, start_pos:int): B, H, T, Hd = x.shape @@ -12,6 +14,15 @@ def apply_rope(x:Tensor, start_pos:int): freqs_cis = precompute_freqs_cis(Hd, start_pos+T)[start_pos:start_pos+T] return apply_rope_new(x, freqs_cis) +class TestLinear(unittest.TestCase): + def test_recovers_packed_ggml_weight(self): + for ggml_type,packed_size,words in ((13, 176, 44), (14, 210, 210), (23, 136, 34)): + packed = Tensor.empty(packed_size+4, dtype=dtypes.uint8, device="CPU")[4:] + decoded = ggml_data_to_tensor(packed, 256, ggml_type).reshape(1, 256) + linear = Linear(256, 1, bias=False) + linear.set_quantized(decoded) + self.assertEqual((linear.ggml_type, linear.weight.numel()), (ggml_type, words)) + class TestAttention(unittest.TestCase): def test_apply_rope(self): x = Tensor.randn(1, 2, 4, 8, dtype=dtypes.float32) @@ -41,6 +52,23 @@ def test_partial_rope_in_attention(self): np.testing.assert_allclose(block.cache_kv[0, :, :, :seqlen, :].numpy(), expected.numpy(), rtol=1e-5, atol=1e-5) class TestGatedDeltaNetBlock(unittest.TestCase): + def test_gated_delta_rectangular_state_and_row_decay(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + q, k = (rng.normal(size=(1, 1, 3, 32)).astype(np.float32) for _ in range(2)) + v, beta = rng.normal(size=(1, 1, 3, 4)).astype(np.float32), rng.uniform(size=(1, 1, 3)).astype(np.float32) + alpha, initial = rng.uniform(0.8, 1, size=(1, 1, 3, 4)).astype(np.float32), rng.normal(size=(1, 1, 4, 32)).astype(np.float32) + expected_state, expected_out = initial.copy(), np.empty_like(v) + for t in range(3): + previous, av = expected_state.copy(), alpha[:, :, t, :, None] + delta = (v[:, :, t] - (previous*k[:, :, t, None]).sum(-1)*alpha[:, :, t]) * beta[:, :, t, None] + expected_state = previous*av + delta[..., None]*k[:, :, t, None, :] + expected_out[:, :, t] = (previous*q[:, :, t, None]).sum(-1)*alpha[:, :, t] + delta*(q[:, :, t]*k[:, :, t]).sum(-1) + state = Tensor(initial).contiguous().realize() + out = gated_delta_prefill(Tensor(q), Tensor(k), Tensor(v), Tensor(beta), Tensor(alpha), state).realize() + np.testing.assert_allclose(out.numpy(), expected_out, rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(state.numpy(), expected_state, rtol=1e-4, atol=1e-4) + def _tensor_linspace(self, start:float, stop:float, shape:tuple[int, ...]) -> Tensor: return Tensor.linspace(start, stop, int(np.prod(shape)), dtype=dtypes.float32).reshape(*shape) diff --git a/test/unit/test_llm_amd.py b/test/unit/test_llm_amd.py new file mode 100644 index 0000000000000..eaf08ab233178 --- /dev/null +++ b/test/unit/test_llm_amd.py @@ -0,0 +1,107 @@ +import unittest +import numpy as np +from tinygrad import Tensor, UOp, dtypes, nn +from tinygrad.llm.kernels.amd import Linear, amd_custom_kernels_supported, q8_quantize, flash_attention +from tinygrad.llm.gguf import ggml_data_to_tensor + +class TestQ8Quantize(unittest.TestCase): + def test_word_quant_weights_use_typed_buffer_view(self): + for ggml_type, type_size in ((13, 176), (23, 136)): + with self.subTest(ggml_type=ggml_type): + raw = Tensor(np.zeros(type_size + 4, dtype=np.uint8), device="CPU").contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 256, ggml_type).reshape(1, 256) + linear = Linear(256, 1, bias=False) + linear.set_quantized(decoded) + self.assertEqual(linear.ggml_type, ggml_type) + self.assertEqual(linear.weight.dtype, dtypes.uint32) + self.assertEqual(linear.weight.nbytes(), type_size) + self.assertEqual(linear.weight.uop.buf_uop.buffer.offset, 4) + + def test_values_and_scales(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + x = np.linspace(-3.1, 2.7, 64, dtype=np.float32).reshape(2, 32) + quant, scale = q8_quantize(Tensor(x), 2, 32) + scale_np = np.maximum(np.max(np.abs(x), axis=-1, keepdims=True) / 127, 1e-8) + expected = np.clip(np.rint(x / scale_np), -127, 127).astype(np.int8) + np.testing.assert_array_equal(quant.bitcast(dtypes.int8).reshape(2, 32).numpy(), expected) + np.testing.assert_allclose(scale.numpy(), scale_np, rtol=1e-6) + + def test_q6_linear_compiles(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + packed = rng.integers(0, 256, 210, dtype=np.uint8) + packed[-2:] = np.array([0.01], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 256, 14).reshape(1, 256) + linear = Linear(256, 1, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + self.assertTrue(np.isfinite(linear(Tensor.randn(1, 256)).realize().item())) + self.assertEqual(linear.weight.uop.buf_uop.buffer.offset, 4) + + def test_q4_k_linear(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + in_features, blocks = 2048, 16*2048//256 + packed = rng.integers(0, 256, blocks*144, dtype=np.uint8) + for i in range(blocks): packed[i*144:i*144+4] = np.array([0.01, 0.002], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 16*in_features, 12).reshape(16, in_features) + weight = decoded.numpy() + linear = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + x = rng.normal(size=(3, in_features)).astype(np.float32) + scale = np.maximum(np.abs(x).reshape(3, in_features//32, 32).max(-1, keepdims=True) / 127, 1e-8) + xq = np.clip(np.rint(x.reshape(3, in_features//32, 32) / scale), -127, 127) * scale + np.testing.assert_allclose(linear(Tensor(x)).numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertEqual(linear.ggml_type, 12) + + def test_q6_linear_multiple_tokens(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + in_features, blocks = 2048, 16*2048//256 + packed = rng.integers(0, 256, blocks*210, dtype=np.uint8) + for i in range(blocks): packed[i*210+208:i*210+210] = np.array([0.01], dtype=np.float16).view(np.uint8) + raw = Tensor(np.pad(packed, (4, 0))).contiguous().realize()[4:] + decoded = ggml_data_to_tensor(raw, 16*in_features, 14).reshape(16, in_features) + weight = decoded.numpy() + linear = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) + x = rng.normal(size=(3, in_features)).astype(np.float32) + scale = np.maximum(np.abs(x).reshape(3, in_features//32, 32).max(-1, keepdims=True) / 127, 1e-8) + xq = np.clip(np.rint(x.reshape(3, in_features//32, 32) / scale), -127, 127) * scale + np.testing.assert_allclose(linear(Tensor(x)).numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertEqual(linear.ggml_type, 14) + + # symbolic token counts take the padded kernel path and give the same results + generic = Linear(in_features, 16, bias=False) + nn.state.load_state_dict(generic, {"weight":decoded}, verbose=False, realize=False) + sym = Tensor(np.concatenate([x, np.zeros((1, in_features), np.float32)])).contiguous()[:UOp.variable("tokens", 1, 4).bind(3)] + np.testing.assert_allclose(generic(sym)[:3].numpy(), xq.reshape(3, in_features) @ weight.T, rtol=2e-3, atol=2e-2) + self.assertTrue(generic.use_custom_quant) + self.assertEqual(generic.ggml_type, 14) + + def test_attention_uses_physical_cache_length(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + q, k, v = Tensor.zeros(1, 2, 1, 32), Tensor.randn(1, 1, 1, 32), Tensor.randn(1, 1, 1, 32) + cache = Tensor.empty(2, 1, 1, 256, 32, dtype=dtypes.half).contiguous() + assigned = Tensor(cache.uop.after(cache[:, :, :, 0:1, :].uop.store(Tensor.stack(k, v).cast(dtypes.half).uop))) + out = flash_attention(q, assigned, 1).realize() + np.testing.assert_allclose(out.numpy(), v.expand(1, 2, 1, 32).numpy(), rtol=2e-2, atol=2e-2) + + def test_prefill_attention_unaligned_start(self): + if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") + rng = np.random.default_rng(42) + start_pos = 1718 + q = Tensor.zeros(1, 8, 32, 128) + old_kv = rng.normal(size=(2, 1, 1, start_pos, 128)).astype(np.float32) + new_kv = rng.normal(size=(2, 1, 1, 32, 128)).astype(np.float32) + cache = Tensor.zeros(2, 1, 1, 2048, 128, dtype=dtypes.half).contiguous() + Tensor.realize(cache[:, :, :, :start_pos].assign(Tensor(old_kv).cast(dtypes.half))) + sp = UOp.variable("start_pos", 0, 2047).bind(start_pos) + assigned = Tensor(cache.uop.after(cache[:, :, :, sp:sp+32, :].uop.store(Tensor(new_kv).cast(dtypes.half).uop))) + out = flash_attention(q, assigned, sp+32).realize() + values = np.concatenate([old_kv[1, 0, 0], new_kv[1, 0, 0]]).astype(np.float16).astype(np.float32) + expected = np.stack([values[:start_pos+i+1].mean(0) for i in range(32)])[None, None].repeat(8, axis=1) + np.testing.assert_allclose(out.numpy(), expected, rtol=2e-3, atol=2e-3) + +if __name__ == "__main__": unittest.main() diff --git a/test/unit/test_llm_server.py b/test/unit/test_llm_server.py index a4b17b7da1d34..e8ade873fabc1 100644 --- a/test/unit/test_llm_server.py +++ b/test/unit/test_llm_server.py @@ -44,7 +44,10 @@ def mock_call(self, tokens, start_pos, temperature, **kwargs): return Tensor([[42]]) with patch.object(Transformer, '__call__', mock_call): next(model.generate([1, 2, 3, 4, 5, 42, 10])) - self.assertEqual(calls, [((1, 1), V_START_POS.bind(5)), ((1, 1), V_START_POS.bind(6))]) + # resumes from the reused state at position 5 and consumes the 2 new tokens (one chunk or two decode steps) + self.assertEqual(calls[0][1], V_START_POS.bind(5)) + def ntok(shape): return shape[1] if isinstance(shape[1], int) else shape[1].unbind()[1] + self.assertEqual(sum(ntok(c[0]) for c in calls), 2) def test_recurrent_divergent_prompt_restarts(self): model, calls = Transformer(TEST_CONFIG), [] diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py new file mode 100644 index 0000000000000..65c0b68158e9f --- /dev/null +++ b/tinygrad/llm/kernels/amd.py @@ -0,0 +1,499 @@ +from __future__ import annotations +import functools, math +from typing import Callable, cast +from tinygrad import Tensor, UOp, nn, Device, Context +from tinygrad.device import Buffer +from tinygrad.dtype import AddrSpace, dtypes +from tinygrad.helpers import prod +from tinygrad.uop.ops import AxisType, KernelInfo, Ops, resolve + +BLOCK_M, BLOCK_N, DECODE_HEAD_TILE, WARP_SIZE = 32, 32, 8, 32 +WMMA_M, WMMA_N, WMMA_K = 16, 16, 16 +WAVES_M, WAVES_N, LANES_PER_WAVE_M, LANES_PER_WAVE_N = 2, 2, 2, 16 +WMMA_ACC, THREADS_PER_BLOCK = WMMA_M // LANES_PER_WAVE_M, WARP_SIZE * WAVES_M * WAVES_N +LDS_PAD, WMMA_ARG, LOG2E = 4, ((WMMA_M, WMMA_N, WMMA_K), 'AMD', 32), math.log2(math.e) +Q4_K, Q5_K, Q6_K, IQ4_XS, GGML_BLOCK_SIZE, Q8_GROUP_SIZE, Q4_WORDS, Q5_WORDS, Q6_BYTES, IQ4_WORDS = 12, 13, 14, 23, 256, 32, 36, 44, 210, 34 +QUANT_SIZES = {Q4_K: Q4_WORDS*4, Q5_K: Q5_WORDS*4, Q6_K: Q6_BYTES, IQ4_XS: IQ4_WORDS*4} # bytes per 256-weight block + +def kernel_var(x:UOp) -> UOp: + # a Variable is a 0-d ALU BUFFER in the tensor graph; inside kernels it takes the ALU PARAM form (same name keeps the value binding) + return x.substitute({v: UOp.variable(v.expr, v.vmin, v.vmax, dtype=v.dtype, multiple_of=v.arg.multiple_of, param=True) + for v in x.toposort() if v.is_variable}) + +def _unbind(v:int|UOp) -> int|UOp: return kernel_var(v.unbind_all()[0]) if isinstance(v, UOp) else v + +@functools.cache +def amd_custom_kernels_supported(device:str|tuple[str, ...]|None) -> bool: + # the custom kernels are tuned for RDNA3 (gfx11): the WMMA register layouts don't match gfx12 (RDNA4) + # or CDNA (MFMA-only, wave64), and the dp4a builtins and 32-lane wave ops aren't portable either. + if isinstance(device, tuple): device = device[0] + if device is None or device.split(":")[0] != "AMD": return False + # @function contexts set ALLOW_DEVICE_USAGE=0 (scheduling must not open devices); the device is always open here + with Context(ALLOW_DEVICE_USAGE=1): + return (t:=getattr(Device[device], "target", None)) is not None and t[0] == 11 + +def warp_reduce(val:UOp, maximum:bool=False, full_wave:bool=False) -> UOp: + for offset in ((16, 8, 4, 2, 1) if full_wave else (8, 4, 2, 1)): + if val.op is Ops.INDEX and val.addrspace == AddrSpace.REG: val = val.load() + other = UOp(Ops.CUSTOM, dtypes.float, (val,), arg= + f"__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {{0}}), {0x1f | offset<<10}))") + val = val.maximum(other) if maximum else val + other + return val + +def _reg(shape:tuple[int, ...], slot:int, value:float, dep:UOp|None=None) -> UOp: + ret = UOp.placeholder(shape, dtypes.float, slot=slot, addrspace=AddrSpace.REG) + return ret.after((ret if dep is None else ret.after(dep)).store(ret.const_like(value))) + +# ******** quant linear: q8-activation kernels over packed ggml weights (Q4_K/Q5_K/Q6_K/IQ4_XS) ******** + +class Linear(nn.Linear): + ggml_type:int|None = None + use_custom_quant = True + def __init__(self, in_features:int, out_features:int, bias=True): + super().__init__(in_features, out_features, bias) + self.in_features, self.out_features = in_features, out_features + def set_quantized(self, decoded:Tensor): + packed_sizes = {decoded.numel() // 256 * type_size:typ for typ,type_size in QUANT_SIZES.items()} + raw = next((u for u in decoded.uop.toposort() if u.op is Ops.SHRINK and u.dtype == dtypes.uint8 and prod(u.shape) in packed_sizes), None) + if raw is None: return + raw_offset = raw.contiguous_view_offset() + assert raw_offset is not None and raw_offset % 4 == 0 and raw.buf_uop.dtype == dtypes.uint8 + self.ggml_type = packed_sizes[prod(raw.shape)] + # store a typed buffer view: a lazy BITCAST is decomposed into byte-combining ALU before custom-kernel + # scheduling and would copy the entire packed weight on every JIT graph + packed_dtype = dtypes.uint8 if self.ggml_type == Q6_K else dtypes.uint32 + self.weight = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer) + .view(raw.max_numel() * raw.dtype.itemsize // packed_dtype.itemsize, packed_dtype, raw_offset))) + def __call__(self, x:Tensor) -> Tensor: + supported = self.use_custom_quant and amd_custom_kernels_supported(self.weight.device) + if self.ggml_type is None and supported: + self.set_quantized(self.weight) + if self.ggml_type is None: self.use_custom_quant = supported = False # not a supported quant format + if self.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) and supported: + if isinstance(x.numel(), int): return q8_linear(self, x) + # symbolic token count: pad to the max chunk size so the kernels see static shapes, garbage rows are sliced off + out = q8_linear(self, x.pad_to(x.max_shape)) + return out.shrink(tuple((0, s) for s in (*x.shape[:-1], self.out_features))) + return super().__call__(x) + +def _amd_dp4a(a:UOp, b:UOp, c:UOp) -> UOp: + return UOp(Ops.CUSTOMI, dtypes.int32, (a.int(), b.int(), c), arg="__builtin_amdgcn_sudot4(true, {}, true, {}, {}, false)") + +def _amd_byte_perm(a:UOp, b:UOp, selectors:UOp) -> UOp: + return UOp(Ops.CUSTOMI, dtypes.uint32, tuple(x.cast(dtypes.uint32) for x in (a, b, selectors)), arg="__builtin_amdgcn_perm({}, {}, {})") + +def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: + assert ptr.op is Ops.INDEX + if lanes is None: return UOp(Ops.CUSTOMI, ptr.dtype, (ptr,), arg="__builtin_nontemporal_load({0})") + buf, coords = ptr.src[0], ptr.src[1:] + idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0, dtypes.weakint)) + return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes, dtypes.weakint))).load(dtype=ptr.dtype) + +def _load_byte(raw:UOp, base:UOp, offset:UOp) -> UOp: return (raw[base + offset//4] >> ((offset&3)*8).cast(dtypes.uint32)) & 255 +def _half(value:UOp) -> UOp: return value.cast(dtypes.uint16).bitcast(dtypes.float16).float() + +def _iq4_bytes(packed:UOp, shift:int) -> UOp: + selectors = (packed >> shift) & 0x0f0f0f0f + low = _amd_byte_perm(UOp.const(0xf6eaddcf, dtypes.uint32), UOp.const(0xbfad9881, dtypes.uint32), selectors) + high = _amd_byte_perm(UOp.const(0x71594535, dtypes.uint32), UOp.const(0x26190d01, dtypes.uint32), selectors & 0x07070707) + return _amd_byte_perm(high, low, 0x03020100 | ((selectors & 0x08080808) >> 1)) + +def _q5_scales(raw:UOp, base:UOp, subgroup:UOp) -> tuple[UOp, UOp, UOp, UOp]: + scale = (subgroup < 4).where(_load_byte(raw, base, 4 + subgroup) & 63, + (_load_byte(raw, base, 8 + subgroup) & 15) | ((_load_byte(raw, base, subgroup) >> 6) << 4)) + minimum = (subgroup < 4).where(_load_byte(raw, base, 8 + subgroup) & 63, + (_load_byte(raw, base, 8 + subgroup) >> 4) | ((_load_byte(raw, base, 4 + subgroup) >> 6) << 4)) + d, dmin = (raw[base] & 0xffff).cast(dtypes.uint16), (raw[base] >> 16).cast(dtypes.uint16) + return _half(d), _half(dmin), scale.float(), minimum.float() + +def _iq4_scales(raw:UOp, base:UOp, subgroup:UOp) -> tuple[UOp, UOp]: + low = _load_byte(raw, base, 4 + subgroup//2) + scale = ((low >> (4*(subgroup%2)).cast(dtypes.uint32)) & 15) | ((((raw[base] >> 16) >> (2*subgroup).cast(dtypes.uint32)) & 3) << 4) + return _half(raw[base] & 0xffff), (scale.cast(dtypes.uint8).bitcast(dtypes.int8)-32).float() + +@functools.cache +def iq4_half_lut(device:str) -> Tensor: + from tinygrad.runtime.autogen.ggml_common import kvalues_iq4nl + return Tensor([x for j in range(16) for i in range(16) for x in (kvalues_iq4nl[i], kvalues_iq4nl[j])], + dtype=dtypes.float16, device=device).bitcast(dtypes.uint32).contiguous() + +@functools.cache +def _q8_quantize_kernel(q:UOp, scale:UOp, x:UOp, tokens:int, in_features:int) -> UOp: + groups = in_features//Q8_GROUP_SIZE + token_group, lane = UOp.range(tokens*groups, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) + token, group = token_group//groups, token_group%groups + x = x.reshape(tokens, groups, 32) + group_scale = (warp_reduce(x[token, group, lane].float().abs(), maximum=True, full_wave=True) / 127).maximum(1e-8) + word_lane = lane.minimum(7) + xs = tuple(x[token, group, word_lane*4+i].float() for i in range(4)) + word = sum(((v/group_scale).round().clip(-127, 127).cast(dtypes.int8).cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) + for i,v in enumerate(xs)), UOp.const(0, dtypes.uint32)) + stores = (q[token, group, lane.valid(lane < 8)].store(word), scale[token, group.valid(lane.eq(0))].store(group_scale)) + return UOp.group(*stores).end(token_group, lane).sink(arg=KernelInfo(name="q8_quantize", opts_to_apply=())) + +def q8_quantize(x:Tensor, tokens:int, in_features:int) -> tuple[Tensor, Tensor]: + groups = in_features//Q8_GROUP_SIZE + q = Tensor.empty(tokens, groups, 8, dtype=dtypes.uint32, device=x.device) + scale = Tensor.empty(tokens, groups, dtype=dtypes.float32, device=x.device) + q, scale = Tensor.custom_kernel(q, scale, x, fxn=functools.partial(_q8_quantize_kernel, tokens=tokens, in_features=in_features))[:2] + return q, scale + +def _decode_linear(out:UOp, out_features:int, group_count:int, group_dot, name:str) -> UOp: + chunks = (group_count+31)//32 + token_output_chunk, lane = UOp.range(out.shape[0]*out_features*chunks, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) + token, output, chunk = token_output_chunk // (out_features*chunks), (token_output_chunk//chunks) % out_features, token_output_chunk % chunks + group = lane+chunk*32 + value = group_dot(token, output, group) if group_count % 32 == 0 else \ + (group < group_count).where(group_dot(token, output, group.minimum(group_count-1)), UOp.const(0, dtypes.float32)) + total = warp_reduce(value, full_wave=True) + return out[token, output, chunk.valid(lane.eq(0))].store(total.cast(out.dtype)).end(token_output_chunk, lane).sink( + arg=KernelInfo(name=name, opts_to_apply=())) + +@functools.cache +def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: + group_count = in_features // Q8_GROUP_SIZE + def group_dot(token:UOp, output:UOp, group:UOp) -> UOp: + block, subgroup = group // 8, group % 8 + xwords = _amd_load(xq[token, group, 0], 8) + if ggml_type in (Q4_K, Q5_K): + base = (output * in_features//GGML_BLOCK_SIZE + block) * (Q4_WORDS if ggml_type == Q4_K else Q5_WORDS) + qs_base, dot, qsum = base + (4 if ggml_type == Q4_K else 12) + (subgroup//2)*8, UOp.const(0, dtypes.int32), UOp.const(0, dtypes.int32) + for word_idx in range(8): + word = (raw[qs_base+word_idx] >> ((subgroup&1)*4).cast(dtypes.uint32)) & 0x0f0f0f0f + if ggml_type == Q5_K: word |= ((raw[base+4+word_idx] >> subgroup.cast(dtypes.uint32)) & 0x01010101) << 4 + dot, qsum = _amd_dp4a(word, xwords[word_idx], dot), _amd_dp4a(UOp.const(0x01010101, dtypes.uint32), xwords[word_idx], qsum) + d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) + return (dot.float()*d*scale - qsum.float()*dmin*minimum) * xd[token, group] + if ggml_type == IQ4_XS: + base = (output * in_features//GGML_BLOCK_SIZE + block) * IQ4_WORDS + dot = UOp.const(0, dtypes.int32) + for word_idx in range(8): + packed = _amd_load(raw[base + 2 + subgroup*4 + word_idx%4]) + dot = _amd_dp4a(_iq4_bytes(packed, 4*(word_idx//4)), xwords[word_idx], dot) + d, scale = _iq4_scales(raw, base, subgroup) + return dot.float() * xd[token, group] * d * scale + base = (output*in_features//GGML_BLOCK_SIZE+block)*Q6_BYTES + dots = [UOp.const(0, dtypes.int32)] * 2 + for word_idx in range(8): + pos, within = subgroup*32 + word_idx*4, (subgroup*32 + word_idx*4)%128 + low = _amd_load(raw[base + (pos//128)*64 + within%64], 4) >> ((within//64)*4).cast(dtypes.uint8) + high = _amd_load(raw[base + 128 + (pos//128)*32 + within%32], 4) >> ((within//32)*2).cast(dtypes.uint8) + quant = ((low & 15) | ((high & 3) << 4)).bitcast(dtypes.int8) - 32 + word = sum((quant[i].cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) for i in range(4)), UOp.const(0, dtypes.uint32)) + dots[word_idx//4] = _amd_dp4a(word, xwords[word_idx], dots[word_idx//4]) + scales = [raw[base + 192 + subgroup*2+i].cast(dtypes.uint8).bitcast(dtypes.int8).float() for i in range(2)] + dbits = raw[base+208].cast(dtypes.uint16) | (raw[base+209].cast(dtypes.uint16) << 8) + return (dots[0].float()*scales[0] + dots[1].float()*scales[1]) * xd[token, group] * _half(dbits) + names = {Q4_K: "linear_q4_k", Q5_K: "linear_q5_k", IQ4_XS: "linear_iq4_xs", Q6_K: "linear_q6"} + return _decode_linear(out, out_features, group_count, group_dot, names[ggml_type]) + +def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): + output_waves = 2 if out_features % (32*output_tiles) == 0 else 1 + token_block, output_block = UOp.range(out.shape[0]//token_tile, 0), UOp.range(out_features//(16*output_tiles*output_waves), 1) + lane, wave = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) + hw_lane = UOp(Ops.CUSTOM, dtypes.int32, (lane.int(),), arg="__builtin_amdgcn_mbcnt_lo(-1, 0)").cast(dtypes.weakint) + col, half = hw_lane % 16, hw_lane // 16 + outputs = tuple((output_block*output_waves+wave)*(16*output_tiles) + tile*16 + col for tile in range(output_tiles)) + inputs = tuple(token_block*token_tile + tile*16 + col for tile in range(token_tile//16)) + tokens = tuple(tuple(token_block*token_tile + tile*16 + half*8 + i for i in range(8)) for tile in range(token_tile//16)) + return output_waves, token_block, output_block, lane, wave, half, outputs, inputs, tokens + +def _wmma_stores(out, outputs, tokens, accs, update, half): + def values(acc:UOp) -> tuple[UOp, ...]: + vals = tuple(acc.after(update)[i].load() for i in range(8)) + swapped = tuple(UOp(Ops.CUSTOM, dtypes.float32, (value,), + arg="__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {0}), 50688))") for value in vals) + low = half.eq(0) + return tuple(low.where(vals[i], swapped[i+4]) if j == 0 else low.where(swapped[i], vals[i+4]) for i in range(4) for j in range(2)) + return [out[token, output].store(value) for output,output_accs in zip(outputs, accs) + for tile_tokens,acc in zip(tokens, output_accs) for token,value in zip(tile_tokens, values(acc))] + +def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, dequant, name): + x = x.reshape(out.shape[0], in_features) + _, token_block, output_block, lane, wave, physical_half, outputs, input_tokens, tokens = layout + token_tile, output_tiles = len(tokens)*16, len(outputs) + output_words = in_features // GGML_BLOCK_SIZE * type_words + accs = tuple(tuple(UOp.placeholder((8,), dtypes.float32, slot=ot*(token_tile//16)+tile, addrspace=AddrSpace.REG) + for tile in range(token_tile // 16)) for ot in range(output_tiles)) + accs = tuple(tuple(acc.after(acc.store(acc.const_like(0))) for acc in output_accs) for output_accs in accs) + group = UOp.range(in_features // Q8_GROUP_SIZE, 4, AxisType.REDUCE) + block, subgroup = group // 8, group % 8 + wmma_accs = [list(output_accs) for output_accs in accs] + for half in range(2): + afrags = tuple(UOp.stack(*(x[input_token, group*32 + half*16 + i].cast(dtypes.float16) for i in range(16))) + for input_token in input_tokens) + for output_tile,output in enumerate(outputs): + bfrag = UOp.stack(*dequant(output*output_words + block*type_words, subgroup, half)) + for tile,afrag in enumerate(afrags): + previous = accs[output_tile][tile].after(group) if half == 0 else wmma_accs[output_tile][tile] + wmma_accs[output_tile][tile] = UOp.wmma(afrag, bfrag, previous, *WMMA_ARG) + update = UOp.group(*(acc.store(value) for output_accs,output_values in zip(accs, wmma_accs) + for acc,value in zip(output_accs, output_values))).end(group) + return UOp.group(*_wmma_stores(out, outputs, tokens, accs, update, physical_half)).end(token_block, output_block, lane, wave).sink( + arg=KernelInfo(name=name, opts_to_apply=())) + +@functools.cache +def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: + token_tile, output_tiles = (64, 1) if out_features <= 1024 and out.shape[0] % 64 == 0 else \ + (64, 2) if out.shape[0] % 64 == 0 else (32 if out.shape[0] % 32 == 0 else 16, 2) + def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: + d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) + qs_base = base + (4 if ggml_type == Q4_K else 12) + (subgroup // 2)*8 + half*4 + words = tuple((raw[qs_base+i] >> ((subgroup&1)*4).cast(dtypes.uint32) & 0x0f0f0f0f) | + (((raw[base+4+half*4+i] >> subgroup.cast(dtypes.uint32) & 0x01010101) << 4) if ggml_type == Q5_K else 0) for i in range(4)) + return tuple(((word >> (byte*8) & 255).float()*d*scale-dmin*minimum).cast(dtypes.float16) for word in words for byte in range(4)) + return _quant_linear_wmma(out, x, out_features, in_features, Q4_WORDS if ggml_type == Q4_K else Q5_WORDS, + _wmma_layout(out, out_features, token_tile, output_tiles), dequant, + f"linear_q{4 if ggml_type == Q4_K else 5}_k_f16_wmma") + +@functools.cache +def _iq4_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, lut:UOp, out_features:int, in_features:int) -> UOp: + token_tile = 32 if out_features <= 1024 and out.shape[0] % 32 == 0 else 64 if out.shape[0] % 64 == 0 and \ + (out_features <= 6144 or out_features == 5120 and in_features > 8192) else 128 if out.shape[0] % 128 == 0 else \ + 32 if out.shape[0] % 32 == 0 else 16 + output_tiles = 1 if out_features <= 1024 else 2 if out_features <= 6144 else 1 if out_features < 8192 else 2 + layout = _wmma_layout(out, out_features, token_tile, output_tiles) + output_waves, _, _, lane, wave, _, _, _, _ = layout + local_lut = UOp.placeholder((256,), dtypes.uint32, slot=32, addrspace=AddrSpace.LOCAL) + tid, lut_items = wave*32+lane, 256//(32*output_waves) + lut = local_lut.after(UOp.group(*(local_lut[tid*lut_items+i].store(lut[tid*lut_items+i]) for i in range(lut_items))).barrier()) + def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: + d, scale = _iq4_scales(raw, base, subgroup) + scale = scale * d + if out_features <= 6144: + pairs = tuple(lut[((raw[base + 2 + subgroup*4 + word] >> (byte*8)) & 255).cast(dtypes.weakint)] + for word in range(4) for byte in range(4)) + return tuple((_half((pair >> (half*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in pairs) + def nibble(packed:UOp, index:int): return (packed >> (8*index+4*half)) & 15 + lut_pairs = (lut[(nibble(packed, i) | nibble(packed, i+1)<<4).cast(dtypes.weakint)] + for packed in (raw[base+2+subgroup*4+i] for i in range(4)) for i in (0, 2)) + return tuple((_half((pair >> (i*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in lut_pairs for i in range(2)) + return _quant_linear_wmma(out, x, out_features, in_features, IQ4_WORDS, layout, dequant, "linear_iq4_xs_f16_wmma") + +def q8_linear(layer:Linear, x:Tensor) -> Tensor: + assert layer.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) + tokens = int(x.numel()) // layer.in_features + raw, out_features, in_features = layer.weight.uop.buf_uop, layer.out_features, layer.in_features + def run(fxn:Callable[..., UOp], out:UOp, *srcs:UOp) -> Tensor: + all_srcs = (out,)+srcs + params = tuple(UOp.placeholder_like(src, slot=i) for i,src in enumerate(all_srcs)) + kernel = fxn(*params, out_features=out_features, in_features=in_features).call(*all_srcs) + result = Tensor(out.after(kernel)) + if len(result.shape) == 3: result = result.sum(-1) + result = result.reshape(*x.shape[:-1], out_features) + return result if layer.bias is None else result + layer.bias + out = Tensor.empty(tokens, out_features, dtype=dtypes.float32, device=x.device).uop + if tokens % 16 == 0 and out_features % 16 == 0 and layer.ggml_type in (Q4_K, Q5_K, IQ4_XS): + fxn = _iq4_linear_f16_wmma_kernel if layer.ggml_type == IQ4_XS else functools.partial(_q5_linear_f16_wmma_kernel, ggml_type=layer.ggml_type) + extra = (iq4_half_lut(str(x.device)).uop,) if layer.ggml_type == IQ4_XS else () + return run(fxn, out, raw, x.cast(dtypes.float16).contiguous().uop, *extra) + xq, xd = q8_quantize(x, tokens, in_features) + decode = functools.partial(_quant_decode_kernel, ggml_type=layer.ggml_type) + out = Tensor.empty(tokens, out_features, (in_features+1023)//1024, dtype=dtypes.float32, device=x.device).uop + return run(decode, out, raw, xq.uop, xd.uop) + +# ******** flash attention on the KV cache ******** + +@functools.cache +def _amd_flash_attention_decode_partial(out, stats, q, cache_kv, valid_kv_len, max_kv_len, block_n): + valid_kv_len = _unbind(valid_kv_len) + _, B, H_KV, N, D = cast(tuple[int, int, int, int, int], cache_kv.shape) + _, H, M, _ = cast(tuple[int, int, int, int], q.shape) + assert M == 1 and H % H_KV == 0 and D % WARP_SIZE == 0 and max_kv_len <= N and max_kv_len % block_n == 0 + G, CHUNK, DV, heads_per_wave = H // H_KV, block_n, D // WARP_SIZE, 2 + head_tile = min(DECODE_HEAD_TILE, G) # share each KV stream across two GQA heads per wave + assert G % head_tile == 0 and head_tile % heads_per_wave == 0 + decode_waves, decode_group = head_tile // heads_per_wave, 4 + block_bhkv = UOp.range(B*H_KV*(G//head_tile), 0, AxisType.GLOBAL) + valid_chunks = (valid_kv_len+CHUNK-1)//CHUNK + group_count = min(valid_chunks, out.shape[2]) if isinstance(valid_chunks, int) else valid_chunks.minimum(out.shape[2]) + block_n, lane = UOp.range(group_count, 1, AxisType.GLOBAL), UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL) + wave = UOp.range(decode_waves, 3, axis_type=AxisType.LOCAL) + head_group, bhkv = block_bhkv % (G//head_tile), block_bhkv // (G//head_tile) + b, kv_head = bhkv // H_KV, bhkv % H_KV + dims = tuple(lane + i*WARP_SIZE for i in range(DV)) + acc, row_max, row_sum = _reg((heads_per_wave, DV), 0, 0), _reg((heads_per_wave,), 1, -math.inf), _reg((heads_per_wave,), 2, 0) + groups_per_chunk, offset = CHUNK // decode_group, UOp.range(((valid_chunks+group_count-1)//group_count)*(CHUNK//decode_group), 100, AxisType.REDUCE) + chunk = block_n + (offset // groups_per_chunk) * group_count + keys = tuple(chunk*CHUNK + (offset % groups_per_chunk)*decode_group + i for i in range(decode_group)) + valid = tuple(key < valid_kv_len for key in keys) + kvals, vvals = (tuple(tuple(is_valid.where(cache_kv[kv, b, kv_head, key, d].float(), UOp.const(0, dtypes.float)) for d in dims) + for key,is_valid in zip(keys, valid)) for kv in range(2)) + q_heads = tuple(kv_head*G + head_group*head_tile + wave*heads_per_wave + head for head in range(heads_per_wave)) + updates:list[UOp] = [] + for head,q_head in enumerate(q_heads): + scores = tuple(warp_reduce(sum((q[b, q_head, 0, d].float()*k for d,k in zip(dims, key_kvals)), + UOp.const(0, dtypes.float)), full_wave=True) / math.sqrt(D) for key_kvals in kvals) + prev_acc, prev_max, prev_sum = acc.after(offset)[head], row_max.after(offset)[head], row_sum.after(offset)[head] + new_max = functools.reduce(lambda a,vs:a.maximum(vs[0].where(vs[1], UOp.const(-math.inf, dtypes.float))), zip(valid, scores), prev_max) + alpha = ((prev_max-new_max)*LOG2E).exp2() + betas = tuple(is_valid.where(((score-new_max)*LOG2E).exp2(), UOp.const(0, dtypes.float)) for is_valid,score in zip(valid, scores)) + updates += [acc[head].store(prev_acc*alpha + sum((UOp.stack(*value)*beta for value,beta in zip(vvals, betas)), acc[head].const_like(0))), + row_sum[head].store(prev_sum*alpha + sum(betas, UOp.const(0, dtypes.float))), row_max[head].store(new_max)] + update = UOp.group(*updates).end(offset) + acc, row_max, row_sum = acc.after(update), row_max.after(update), row_sum.after(update) + stores = [out[b, q_head, block_n, d].store(acc[head, i]) for head,q_head in enumerate(q_heads) for i,d in enumerate(dims)] + \ + [stats[b, q_head.valid(lane.eq(0)), block_n, i].store(x[head]) for head,q_head in enumerate(q_heads) for i,x in enumerate((row_max, row_sum))] + return UOp.group(*stores).end(lane, wave, block_n, block_bhkv).sink(arg=KernelInfo(name="flash_decode_partial", opts_to_apply=())) + +def amd_flash_attention_decode(q:Tensor, cache_kv:Tensor, valid_kv_len:int|UOp, max_kv_len:int) -> Tensor: + B, H, D = cache_kv.shape[1], q.shape[1], cache_kv.shape[4] + chunks = min(64, max_kv_len // 128) + partial = Tensor.empty(B, H, chunks, D, dtype="float32", device=q.device) + stats = Tensor.empty(B, H, chunks, 2, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_attention_decode_partial, valid_kv_len=valid_kv_len, max_kv_len=max_kv_len, block_n=128) + partial, stats = Tensor.custom_kernel(partial, stats, q, cache_kv, fxn=fxn)[:2] + live = (valid_kv_len+127)//128 + live = min(live, chunks) if isinstance(live, int) else live.minimum(chunks) + partial, stats = partial[:, :, :live], stats[:, :, :live] + weights = ((stats[..., 0]-stats[..., 0].max(2, keepdim=True))*LOG2E).exp2() + return ((partial*weights.unsqueeze(-1)).sum(2) / (stats[..., 1]*weights).sum(2, keepdim=True)).unsqueeze(2) + +@functools.cache +def _amd_flash_attention(o:UOp, q:UOp, cache:UOp, valid_kv_len:int|UOp, q_start:int|UOp|None=None) -> UOp: + valid_kv_len, q_start = _unbind(valid_kv_len), _unbind(q_start) if q_start is not None else None + BH, M, D = q.shape + _, B, H_KV, physical_n, cache_dim = cache.shape + k, v = cache[0].reshape(B*H_KV, physical_n, cache_dim), cache[1].reshape(B*H_KV, physical_n, cache_dim) + assert k.shape == v.shape and BH % k.shape[0] == 0 and k.shape[2] == D + gqa_group = BH // k.shape[0] + if isinstance(M, int) and isinstance(valid_kv_len, int): assert M % BLOCK_M == 0 and valid_kv_len % BLOCK_N == 0 + assert isinstance(D, int) and D % WMMA_K == 0 and D % LANES_PER_WAVE_N == 0 + TM, TN, TD, SCALE = BLOCK_M//(WAVES_M*LANES_PER_WAVE_M), BLOCK_N//LANES_PER_WAVE_N, D//(WAVES_N*LANES_PER_WAVE_N), 1/math.sqrt(D) + # query row 0 sits at sequence position q_base (the queries may be padded beyond valid_kv_len - q_base rows) + q_base = valid_kv_len - M if q_start is None else q_start + block_bh, block_m = UOp.range(BH, 0, AxisType.GLOBAL), UOp.range(M // BLOCK_M, 1, AxisType.GLOBAL) + kv_head = block_bh // gqa_group + q, o = (x.reshape(BH, M//BLOCK_M, BLOCK_M, D)[block_bh, block_m] for x in (q, o)) + k, v = k[kv_head], v[kv_head] + wave_m, wave_n, lane = UOp.range(WAVES_M, 2, AxisType.LOCAL), UOp.range(WAVES_N, 3, AxisType.LOCAL), UOp.range(WARP_SIZE, -1, AxisType.WARP) + tid, lane_m, lane_n = (wave_m * WAVES_N + wave_n) * WARP_SIZE + lane, lane // LANES_PER_WAVE_N, lane % LANES_PER_WAVE_N + Q_ELEMS_PER_THREAD, KV_ELEMS_PER_THREAD = BLOCK_M * D // THREADS_PER_BLOCK, BLOCK_N * D // THREADS_PER_BLOCK + QP_lds = UOp.placeholder((BLOCK_M, D + LDS_PAD), dtypes.half, slot=0, addrspace=AddrSpace.LOCAL) + KV_lds = UOp.placeholder((BLOCK_N, D + LDS_PAD), dtypes.half, slot=1, addrspace=AddrSpace.LOCAL)[:, :D] + acc, m_i, l_i = _reg((TM, TD), 2, 0), _reg((TM,), 3, -math.inf), _reg((TM,), 4, 0) + n_tile = UOp.range((q_base + (block_m + 1) * BLOCK_M + BLOCK_N - 1) // BLOCK_N, 100, AxisType.REDUCE) + Q_lds = QP_lds[:, :D] + Q_store = Q_lds.after(n_tile).reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid].store(q.reshape(THREADS_PER_BLOCK, Q_ELEMS_PER_THREAD)[tid]) + load_k = UOp.range(KV_ELEMS_PER_THREAD, 90) + kval = k.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_k].float() + K_store = KV_lds.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_k].store(kval).end(load_k) + qk_load_barrier = UOp.barrier(UOp.group(Q_store, K_store)) + Q_lds, KV_lds_k = Q_lds.after(qk_load_barrier), KV_lds.after(qk_load_barrier) + S_reg = _reg((TM, TN), 6, 0, n_tile) + k_qk, tm1, tn1 = UOp.range(D//WMMA_K, 101, AxisType.REDUCE), UOp.range(TM//WMMA_ACC, 200), UOp.range(TN, 201) + S_frag = S_reg.reshape(TM // WMMA_ACC, WMMA_ACC, TN).permute(0, 2, 1)[tm1, tn1] + q_frag = Q_lds.reshape(WAVES_M, TM // WMMA_ACC, WMMA_M, D // WMMA_K, WMMA_K)[wave_m, tm1, lane_n, k_qk] + k_frag = KV_lds_k.reshape(TN, WMMA_N, D // WMMA_K, WMMA_K)[tn1, lane_n, k_qk] + qk_done = S_frag.store(UOp.wmma(q_frag, k_frag, S_frag.after(k_qk), *WMMA_ARG)).end(tm1, tn1).end(k_qk) + S_reg = S_reg.after(qk_done, S_reg.store(S_reg * SCALE)) + rm, rn = UOp.range(TM, 250), UOp.range(TN, 251) + q_idx = q_base + block_m * BLOCK_M + wave_m * WMMA_M + rm * LANES_PER_WAVE_M + lane_m + k_idx = n_tile * BLOCK_N + rn * LANES_PER_WAVE_N + lane_n + S_reg = S_reg.after(S_reg[rm, rn].store((k_idx <= q_idx).where(S_reg[rm, rn], S_reg[rm, rn].const_like(-math.inf))).end(rm, rn)) + m_ij, rm2 = _reg((TM,), 7, -math.inf, n_tile), UOp.range(TN, 261, AxisType.REDUCE) + m_ij = m_ij.after(m_ij.store(m_ij.after(rm2).maximum(S_reg[:, rm2])).end(rm2)) + ri_w = UOp.range(TM, 270) + m_ij = m_ij.after(m_ij[ri_w].store(warp_reduce(m_ij[ri_w], maximum=True)).end(ri_w)) + tile_max = m_ij.reshape(TM, 1).expand(TM, TN).maximum(-1e30) + S_reg = S_reg.after(S_reg.store(((S_reg - tile_max) * LOG2E).exp2())) + p_local, ri_ws = _reg((TM,), 8, 0, n_tile), UOp.range(TM, 295) + p_sum = p_local.after(p_local[ri_ws].store(sum((warp_reduce(S_reg[ri_ws, rn]) for rn in range(TN)), S_reg.const_like(0))).end(ri_ws)) + P_lds = QP_lds.flatten()[:WAVES_N * BLOCK_M * BLOCK_N].reshape(WAVES_N, BLOCK_M, BLOCK_N) + P_write = P_lds.reshape(WAVES_N, WAVES_M, TM, LANES_PER_WAVE_M, 1, TN, LANES_PER_WAVE_N, 1).permute((1, 0, 3, 6, 2, 4, 5, 7)) \ + .reshape(THREADS_PER_BLOCK, TM, TN) + P_store = P_write[tid].store(S_reg.cast(dtypes.half)) + beta_i, ri4, rj4 = UOp.placeholder((TM,), dtypes.float, slot=9, addrspace=AddrSpace.REG), UOp.range(TM, 330), UOp.range(TD, 331) + m_new = m_i[ri4].maximum(m_ij[ri4]) + alpha_val, beta_val = ((m_i[ri4] - m_new) * LOG2E).exp2(), ((m_ij[ri4] - m_new) * LOG2E).exp2() + correction = UOp.group(acc[ri4, rj4].store(alpha_val * acc[ri4, rj4]).end(rj4), + l_i[ri4].store(alpha_val * l_i[ri4] + beta_val * p_sum[ri4]), + m_i[ri4].store(m_new), beta_i[ri4].store(beta_val)).end(ri4) + acc, l_i, m_i, beta_i = acc.after(correction), l_i.after(correction), m_i.after(correction), beta_i.after(correction) + V_lds = UOp.placeholder((D, BLOCK_N + LDS_PAD), dtypes.half, slot=1, addrspace=AddrSpace.LOCAL)[:, :BLOCK_N] + V_copy, load_v = V_lds.after(qk_done).permute(1, 0), UOp.range(KV_ELEMS_PER_THREAD, 390) + vval = v.reshape(physical_n*D)[n_tile*BLOCK_N*D + tid*KV_ELEMS_PER_THREAD + load_v].float() + V_store = V_copy.reshape(THREADS_PER_BLOCK, KV_ELEMS_PER_THREAD)[tid, load_v].store(vval).end(load_v) + pv_barrier = UOp.barrier(UOp.group(P_store, V_store)) + P_lds, V_lds = P_lds.after(pv_barrier), V_lds.after(pv_barrier) + pv_acc = _reg((TM, TD), 10, 0, n_tile).after(pv_barrier) + k_pv, tm2, tn2 = UOp.range(BLOCK_N//WMMA_K, 400, AxisType.REDUCE), UOp.range(TM//WMMA_ACC, 401), UOp.range(TD, 402) + pv_frag = pv_acc.reshape(TM // WMMA_ACC, WMMA_ACC, TD).permute(0, 2, 1)[tm2, tn2] + p_frag = P_lds[wave_n].reshape(WAVES_M, TM // WMMA_ACC, WMMA_M, BLOCK_N // WMMA_K, WMMA_K)[wave_m, tm2, lane_n, k_pv] + v_frag = V_lds.reshape(WAVES_N, TD, WMMA_N, BLOCK_N // WMMA_K, WMMA_K)[wave_n, tn2, lane_n, k_pv] + pv_done = pv_frag.store(UOp.wmma(p_frag, v_frag, pv_frag.after(k_pv), *WMMA_ARG)).end(tm2, tn2).end(k_pv) + pv_acc = pv_acc.after(pv_done) + ri5, rj5 = UOp.range(TM, 410), UOp.range(TD, 411) + n_tile_end = acc[ri5, rj5].store(acc[ri5, rj5] + beta_i[ri5] * pv_acc[ri5, rj5]).end(ri5, rj5).barrier().end(n_tile) + acc, l_i, m_i = acc.after(n_tile_end), l_i.after(n_tile_end), m_i.after(n_tile_end) + acc = acc.after(acc.store(acc * (1 / l_i).reshape(TM, 1).expand(TM, TD))) + o = o.reshape(WAVES_M, TM, LANES_PER_WAVE_M, 1, WAVES_N, TD, LANES_PER_WAVE_N, 1) \ + .permute((0, 4, 2, 6, 1, 3, 5, 7)).reshape(THREADS_PER_BLOCK, TM, TD) + return o[tid].store(acc).end(wave_m, wave_n, lane).end(block_m, block_bh).sink(arg=KernelInfo(opts_to_apply=())) + +def flash_attention(q:Tensor, assigned_kv:Tensor, valid_end:int|UOp) -> Tensor: + # cached flash attention on the half KV cache (already written through assigned_kv); valid_end stays bound at the graph level + T_real, q_start = q.shape[2], None + if resolve(T_real == 1): return amd_flash_attention_decode(q.half(), assigned_kv, valid_end, cast(int, assigned_kv.shape[3])) + if isinstance(T_real, UOp): + # symbolic chunk: pad the queries to the static tile; garbage rows are sliced off + T_pad = q.max_shape[2] + assert T_pad % BLOCK_M == 0, "chunk_size must be a multiple of 32" + q, q_start = q.pad_to((*q.shape[:2], T_pad, q.shape[3])), valid_end - T_real + B, H, T, D = q.shape + out = Tensor.empty(B*H, T, D, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_attention, valid_kv_len=valid_end, q_start=q_start) + out = Tensor.custom_kernel(out, q.half().reshape(B*H, T, D), assigned_kv, fxn=fxn)[0].reshape(B, H, T, D) + return out if q_start is None else out[:, :, :T_real] + +# ******** gated delta net: fused recurrent scan ******** + +@functools.cache +def _gated_delta_prefill_kernel(core:UOp, q:UOp, k:UOp, v:UOp, beta:UOp, alpha:UOp, state:UOp, kq:UOp, start_pos:UOp|None=None) -> UOp: + batch, heads, tokens, value_dim, row_tile = *core.shape, 4 + key_dim, alpha_dim = q.shape[-1], alpha.shape[-1] if len(alpha.shape) == 4 else 1 + assert all(isinstance(x, int) for x in (batch, heads, tokens, value_dim, key_dim)) and key_dim % 32 == 0 and value_dim % row_tile == 0 + batch, heads, tokens, value_dim, key_dim = cast(tuple[int, int, int, int, int], (batch, heads, tokens, value_dim, key_dim)) + core, v = (x.reshape(batch*heads, tokens, value_dim) for x in (core, v)) + q, k = (x.reshape(batch*heads, tokens, key_dim) for x in (q, k)) + beta, kq = (x.reshape(batch*heads, tokens) for x in (beta, kq)) + alpha, state = alpha.reshape(batch*heads, tokens, alpha_dim), state.reshape(batch*heads, value_dim, key_dim) + bh_row, lane = UOp.range(batch*heads*value_dim//row_tile, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) + bh, row_base = bh_row // (value_dim//row_tile), (bh_row % (value_dim//row_tile))*row_tile + rows, cols = tuple(row_base+i for i in range(row_tile)), tuple(lane + i*32 for i in range(key_dim//32)) + current = UOp.placeholder((row_tile*key_dim//32,), dtypes.float32, slot=0, addrspace=AddrSpace.REG) + initial = None if start_pos is None else start_pos.eq(0) + current = current.after(current.store(UOp.stack(*(state[bh, row, col].float() if initial is None else + initial.where(0, state[bh, row, col].float()) for row in rows for col in cols)))) + token = UOp.range(tokens, 2, AxisType.REDUCE) + keys = tuple(k[bh, token, col].load() for col in cols) + queries = tuple(q[bh, token, col].load() for col in cols) + updates, stores = [], [] + for row_idx,row in enumerate(rows): + previous = tuple(current.after(token)[row_idx*key_dim//32+i].load() for i in range(key_dim//32)) + av, bv = alpha[bh, token, row if alpha_dim > 1 else 0].load(), beta[bh, token].load() + state_k = warp_reduce(sum((x*y for x,y in zip(previous, keys)), UOp.const(0, dtypes.float32)), full_wave=True) + state_q = warp_reduce(sum((x*y for x,y in zip(previous, queries)), UOp.const(0, dtypes.float32)), full_wave=True) + delta = (v[bh, token, row].load() - state_k*av) * bv + updates += [x*av + delta*y for x,y in zip(previous, keys)] + stores.append(core[bh, token, row.valid(lane.eq(0))].store(state_q*av + delta*kq[bh, token])) + step = UOp.group(*stores, current.store(UOp.stack(*updates))).end(token) + state_stores = (state[bh, row, col].store(current.after(step)[row_idx*key_dim//32+i].load().cast(state.dtype)) + for row_idx,row in enumerate(rows) for i,col in enumerate(cols)) + return UOp.group(*state_stores).end(lane, bh_row).sink(arg=KernelInfo(name="gated_delta_prefill", opts_to_apply=())) + +def gated_delta_prefill(q:Tensor, k:Tensor, v:Tensor, beta:Tensor, alpha:Tensor, state:Tensor, start_pos:Tensor|None=None) -> Tensor: + batch, heads, tokens, key_dim = q.shape + value_dim = v.shape[-1] + assert q.shape == k.shape and v.shape[:3] == beta.shape == (batch, heads, tokens) and state.shape == (batch, heads, value_dim, key_dim) + assert alpha.shape[:3] == (batch, heads, tokens) and (len(alpha.shape) == 3 or alpha.shape[-1] in (1, value_dim)) + assert key_dim % 32 == 0 and value_dim % 4 == 0 + core, kq = Tensor.empty_like(v), (q*k).sum(-1).contiguous() + srcs = (core, q.contiguous(), k.contiguous(), v.contiguous(), beta.contiguous(), alpha.contiguous(), state, kq) + if start_pos is None: return Tensor.custom_kernel(*srcs, fxn=_gated_delta_prefill_kernel)[0] + contig = tuple(x.uop if x.uop.op is Ops.AFTER else x.uop.contiguous() for x in srcs) + params = tuple(UOp.placeholder_like(x, slot=i) for i,x in enumerate(contig)) + assert start_pos.uop.is_bound_var + # the bound start_pos reaches the graph through the state AFTER chain, like the flash kernels' valid_end + call = _gated_delta_prefill_kernel(*params, kernel_var(start_pos.uop.src[0])).call(*contig) + return Tensor(contig[0].after(call)) diff --git a/tinygrad/llm/model.py b/tinygrad/llm/model.py index e9b68860181f1..5231e8e2a4994 100644 --- a/tinygrad/llm/model.py +++ b/tinygrad/llm/model.py @@ -2,7 +2,7 @@ import enum, functools, itertools, pathlib from dataclasses import dataclass, replace from tinygrad import Tensor, nn, UOp, TinyJit, getenv, function, dtypes -from tinygrad.nn import Linear +from tinygrad.llm.kernels.amd import Linear, gated_delta_prefill, flash_attention, amd_custom_kernels_supported from tinygrad.llm.gguf import gguf_load from tinygrad.uop.ops import resolve @@ -181,7 +181,13 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: k = apply_rope(k[..., :self.config.rope_dim], self.freqs_cis[start_pos:start_pos+T]).cat(k[..., self.config.rope_dim:], dim=-1) # NOTE: we don't want to change self.cache_kv, the function API doesn't support this well - assigned_kv = Tensor(self.cache_kv.uop.after(self.cache_kv[:, :, :, start_pos:start_pos+T, :].uop.store(Tensor.stack(k, v).uop))) + store = self.cache_kv[:, :, :, start_pos:start_pos+T, :].uop.store(Tensor.stack(k, v).cast(dtypes.half).uop) + assigned_kv = Tensor(self.cache_kv.uop.after(store)) + # on RDNA3, hybrid models use custom flash attention kernels on the KV cache + if amd_custom_kernels_supported(x.device) and self.config.ssm is not None: + attn = flash_attention(q, assigned_kv, start_pos+T) + attn = attn.transpose(1, 2).reshape(B, T, -1) # back to (B,T,D) + return self.attn_output(attn if not self.config.attn_output_gate else (attn * gate.sigmoid())) k = assigned_kv[0, :, :, 0:start_pos+T, :] v = assigned_kv[1, :, :, 0:start_pos+T, :] @@ -199,7 +205,8 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: def _init_state(self, x:Tensor): if not hasattr(self, "cache_kv"): - self.cache_kv = Tensor.empty(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, + # zeroed so the flash kernels can safely read whole tiles past the valid region (masked lanes multiply by 0) + self.cache_kv = Tensor.zeros(2, x.shape[0], self.config.n_kv_heads, self.config.max_context, self.config.head_dim, dtype=dtypes.half, device=x.device) self.freqs_cis = precompute_freqs_cis(self.config.rope_dim, self.config.max_context, self.config.rope_theta, device=x.device) @@ -311,21 +318,28 @@ def _attention(self, x:Tensor, start_pos:int|UOp) -> Tensor: v = v.reshape(B, T_pad, self.num_v_heads, self.head_v_dim) # layout the per-step operands to broadcast against the (B, H, V, K) state q, k, v, beta = (z.transpose(1, 2).float() for z in (q, k, v, beta)) - q, k, v, beta = q.unsqueeze(-2) * self.head_k_dim**-0.5, k.unsqueeze(-2), v.unsqueeze(-1), beta.unsqueeze(-1).unsqueeze(-1) - alpha = log_alpha.transpose(1, 2).exp().unsqueeze(-1) # per-channel decay for kda, per-head otherwise (B, H, T, V|1, 1) + q = q * self.head_k_dim**-0.5 + alpha = log_alpha.transpose(1, 2).exp() # per-channel decay for kda, per-head otherwise (B, H, T, V|1) # recurrent: scan over the (padded) tokens, updating the recurrent state. collect the per-step outputs - state = Tensor(self.recurrent_state.uop.after(conv_state_store)).float() # carry the conv write into this graph - state = initial.where(0, state) - outs = [] - for t in range(T_pad): - s1 = state * alpha[:, :, t] # decay the state - delta = (v[:, :, t] - (s1*k[:, :, t]).sum(-1, keepdim=True)) * beta[:, :, t] # the delta rule update - state = s1 + delta * k[:, :, t] - outs.append((state * q[:, :, t]).sum(-1)) - - # store the updated recurrent state in place, then read the stacked outputs after the write - core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop))) + state = Tensor(self.recurrent_state.uop.after(conv_state_store)) # carry the conv write into this graph + if self.head_k_dim % 32 == 0 and self.head_v_dim % 4 == 0 and amd_custom_kernels_supported(x.device): + # one fused kernel for the whole scan; it resets and updates the recurrent state in place (RDNA3) + core = gated_delta_prefill(q, k, v, beta, alpha, state, Tensor(start_pos)).transpose(1, 2) + else: + q, k, v, beta = q.unsqueeze(-2), k.unsqueeze(-2), v.unsqueeze(-1), beta.unsqueeze(-1).unsqueeze(-1) + alpha = alpha.unsqueeze(-1) + state = initial.where(0, state.float()) + outs = [] + for t in range(T_pad): + s1 = state * alpha[:, :, t] # decay the state + delta = (v[:, :, t] - (s1*k[:, :, t]).sum(-1, keepdim=True)) * beta[:, :, t] # the delta rule update + state = s1 + delta * k[:, :, t] + outs.append((state * q[:, :, t]).sum(-1)) + + # store the updated recurrent state in place, then read the stacked outputs after the write + state_store = self.recurrent_state.uop.store(state.cast(self.recurrent_state.dtype).uop) + core = Tensor(outs[0].stack(*outs[1:], dim=1).contiguous().uop.after(state_store)) # output; undo the padding before the output projection z = (self.ssm_norm(core) * (out_gate.sigmoid() if is_kda else out_gate.silu())).cast(x.dtype).contiguous() @@ -462,7 +476,7 @@ def get_start_pos(self, tokens:list[int]) -> int: return min(block._reusable_prefix_len(prefix_len, len(self._cached_tokens)) for block in self.blk) def generate(self, tokens:list[int], chunk_size:int=32, temperature:float=0.0): - if self.has_recurrent_block: chunk_size = 1 + if self.has_recurrent_block and not amd_custom_kernels_supported(self.token_embd.weight.device): chunk_size = 1 v_start_pos = UOp.variable("start_pos", 0, self.max_context-1) v_toks = UOp.variable("toks", 1, chunk_size) # TODO: use UOp.variable for temperature once float variables are supported From 9b0c65688d1f6f6b474f21cd419fea66dbc6211d Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:10:02 +0800 Subject: [PATCH 178/306] llama script changes from the speedups branch (#17701) --- .../llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh | 4 ++-- .../llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh | 2 +- .../implementations/tinybox_8xMI350X/run_and_time.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh index 570d299ee45a2..ede63df0a5a84 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh @@ -35,7 +35,7 @@ export BASEDIR="/raid/datasets/c4-8b/" export SMALL=1 export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"} export EVAL_TARGET=3.3 EVAL_FREQ=12288 -export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000 +export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=2048 MAX_STEPS=1200000 export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS)) export SAMPLES=$((MAX_STEPS * GBS)) export SEQLEN=${SEQLEN:-8192} @@ -46,7 +46,7 @@ export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 -export FAKEDATA=${FAKEDATA:-$([[ "$DEV" == NULL:* ]] && echo 1 || echo 0)} BENCHMARK=${BENCHMARK:-10} +export FAKEDATA=${FAKEDATA:-1} BENCHMARK=${BENCHMARK:-10} if [ -z "$FULL_LAYERS" ]; then export LLAMA_LAYERS=${LLAMA_LAYERS:-2} fi diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index 6ec564f12db30..4ee7eacff844c 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -35,7 +35,7 @@ export BASEDIR="/raid/datasets/c4-8b/" export SMALL=1 export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"} export EVAL_TARGET=3.3 EVAL_FREQ=12288 -export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=4096 MAX_STEPS=1200000 +export LR="1e-3" END_LR="1e-4" WARMUP_SAMPLES=2048 MAX_STEPS=1200000 export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS)) export SAMPLES=$((MAX_STEPS * GBS)) export SEQLEN=${SEQLEN:-8192} diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh index e1d5a86734b75..6dc5ce1a14816 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh @@ -17,7 +17,7 @@ export USE_ATOMICS=1 export ASM_GEMM=1 export WQKV=1 export MASTER_WEIGHTS=1 -export FP8=1 +export MXFP4=1 export ALLREDUCE_CAST=1 export FAST_CE=1 export FUSED_INPUT_QUANTIZE=1 @@ -26,7 +26,7 @@ export FUSED_ADD_NORM_MUL_QUANTIZE=1 export FUSED_SILU_W13=1 export SPLIT_W13=0 -export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16" +export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="float32" export DP=8 MP=1 BS=16 EVAL_BS=8 GRADIENT_ACC_STEPS=2 export GBS=$((BS * GRADIENT_ACC_STEPS)) From 9b481a4893a8c2c0f97d4004b66ee9a360bc10b9 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 24 Aug 2026 11:05:00 -0700 Subject: [PATCH 179/306] dsp: cleanup temp linker script (#17704) --- tinygrad/runtime/ops_dsp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tinygrad/runtime/ops_dsp.py b/tinygrad/runtime/ops_dsp.py index e8ffa2ef8375f..0f001ed79cefd 100644 --- a/tinygrad/runtime/ops_dsp.py +++ b/tinygrad/runtime/ops_dsp.py @@ -105,7 +105,7 @@ def _offset(self, buf, size:int, offset:int): return DSPBuffer(buf.va_addr+offse class DSPCompiler(Compiler): def __init__(self, mock:bool=False): - compiler_args = "--target=hexagon -mcpu=hexagonv65 -fuse-ld=lld -nostdlib -mhvx=v65 -mhvx-length=128b" + self.mock, compiler_args = mock, "--target=hexagon -mcpu=hexagonv65 -fuse-ld=lld -nostdlib -mhvx=v65 -mhvx-length=128b" if mock: self.args = f"-static {compiler_args}" else: # Generate link script to pass into clang. Aligning all used sections to 4k fixes invoke problem. @@ -120,6 +120,9 @@ def __init__(self, mock:bool=False): super().__init__(None if mock else "compile_dsp") + def __del__(self): + if not self.mock: os.unlink(self.link_ld.name) + def compile(self, src:str) -> bytes: # TODO: remove file write. sadly clang doesn't like the use of /dev/stdout here with tempfile.NamedTemporaryFile(delete=True) as f: From 7b21ffac006e5e3000cbbc4dd322250dd6face60 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 14:37:15 -0400 Subject: [PATCH 180/306] PYLITERAL has dtypes.void (#17705) also deleted Ops.WIAT, it currently does not have entry in dtype_from_uop --- tinygrad/uop/__init__.py | 2 +- tinygrad/uop/ops.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tinygrad/uop/__init__.py b/tinygrad/uop/__init__.py index d3a516e32a7fe..39b73f3a8025d 100644 --- a/tinygrad/uop/__init__.py +++ b/tinygrad/uop/__init__.py @@ -73,7 +73,7 @@ class Ops(FastEnum): # ** 5 -- control flow / consts / custom ** # control flow ops - BARRIER = auto(); RANGE = auto(); IF = auto(); END = auto(); ENDIF = auto(); WAIT = auto() + BARRIER = auto(); RANGE = auto(); IF = auto(); END = auto(); ENDIF = auto() # const. CONST = auto() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index ae40f94eb26a1..4ad771539d7d2 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -116,13 +116,13 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: match op: case Ops.STORE | Ops.LINEAR | Ops.SINK | Ops.PROGRAM | Ops.SOURCE | \ Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | \ - Ops.TUPLE | Ops.FUNCTION | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR: + Ops.TUPLE | Ops.FUNCTION | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: # always void return dtypes.void case Ops.CALL: # a CALL of an opaque body is void, a CALL of an address can return a value return dtypes.void if src[0].dtype is dtypes.void else None - case Ops.CUSTOM | Ops.CUSTOMI | Ops.PYLITERAL: + case Ops.CUSTOM | Ops.CUSTOMI: return None case Ops.INS: return None From 7a887e84b238f9d03a1f23dbddb3c95ae4e7f6c4 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:42:35 +0300 Subject: [PATCH 181/306] am: use gc regs for xgmi (#17707) --- tinygrad/runtime/support/am/ip.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tinygrad/runtime/support/am/ip.py b/tinygrad/runtime/support/am/ip.py index 99234d348d029..d0008537c521c 100644 --- a/tinygrad/runtime/support/am/ip.py +++ b/tinygrad/runtime/support/am/ip.py @@ -53,10 +53,9 @@ class AM_GMC(AM_IP): def init_sw(self): self.vmhubs = len(self.adev.regs_offset[am.MMHUB_HWIP]) - # XGMI (for supported systems) - xgmi_lfb_cntl = self.adev.regMMMC_VM_XGMI_LFB_CNTL.read_bitfields() if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_CNTL') else {} + xgmi_lfb_cntl = self.adev.regGCMC_VM_XGMI_LFB_CNTL.read_bitfields() if hasattr(self.adev, 'regGCMC_VM_XGMI_LFB_CNTL') else {} self.xgmi_phys_id, self.xgmi_max_region = xgmi_lfb_cntl.get('pf_lfb_region', 0), xgmi_lfb_cntl.get('pf_max_region', 0) - self.xgmi_seg_sz = self.adev.regMMMC_VM_XGMI_LFB_SIZE.read_bitfields()['pf_lfb_size']<<24 if hasattr(self.adev, 'regMMMC_VM_XGMI_LFB_SIZE') else 0 + self.xgmi_seg_sz = self.adev.regGCMC_VM_XGMI_LFB_SIZE.read_bitfields()['pf_lfb_size']<<24 if hasattr(self.adev, 'regGCMC_VM_XGMI_LFB_CNTL') else 0 self.paddr_base = self.xgmi_phys_id * self.xgmi_seg_sz From 5aabbb199181531d60095ad2e810dc154b3fbd79 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:33:10 -0700 Subject: [PATCH 182/306] fix Muon weight decay being a no-op (#17709) * fix Muon weight decay being a no-op LARS._step computed the post-momentum weight decayed param but only used it for a dtype cast, so the decay was never applied. Fold the decay into the update instead, matching torch's param.mul_(1 - lr*wd). test_muon_wd passed anyway since lr*wd=1e-5 is far below atol, so also bump the test's weight_decay to 10 to actually exercise it. * muon: apply weight decay after lr scaling keeps the decoupled decay independent of the LARS trust ratio r, matching torch's param.mul_(1 - lr*wd). no behavior change today since r is always 1.0 on the pre_wd=False path (Muon has tcoef=0). --- test/backend/test_optim.py | 3 ++- tinygrad/nn/optim.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/backend/test_optim.py b/test/backend/test_optim.py index 299ec7a44dcbd..74ce1d1f251f5 100644 --- a/test/backend/test_optim.py +++ b/test/backend/test_optim.py @@ -87,7 +87,8 @@ def test_multistep_sgd_high_lr_nesterov_momentum_wd(self): def test_muon(self): self._test_muon(1, {'lr': 0.001}, 1e-3, 0) # TODO: disabled due to big atol # def test_muon_high_lr(self): self._test_muon(1, {'lr': 10}, 1e-6, 3e-4) - def test_muon_wd(self): self._test_muon(1, {'lr': 0.001, 'weight_decay': 0.01}, 1e-3, 3e-4) + # NOTE: big weight_decay so a missing wd would be way over atol + def test_muon_wd(self): self._test_muon(1, {'lr': 0.001, 'weight_decay': 10}, 1e-3, 3e-4) # TODO: disabled due to big atol # def test_muon_high_lr_wd(self): self._test_muon(1, {'lr': 10, 'weight_decay': 0.01}, 1e-6, 5e-4) diff --git a/tinygrad/nn/optim.py b/tinygrad/nn/optim.py index b07fd75f1388c..2eca8b3e6ed35 100644 --- a/tinygrad/nn/optim.py +++ b/tinygrad/nn/optim.py @@ -121,10 +121,10 @@ def _step(self, params:list[Tensor], grads:list[Tensor]) -> tuple[list[Tensor], self.b[i].assign(self.momentum * self.b[i] + g) # NOTE: self.b[i] is zero on the first run, no if required g = (g + self.momentum * self.b[i]) if self.nesterov else self.b[i] if self.ns_coefficients: g = g.reshape(g.shape[0], -1).newton_schulz(self.ns_steps, self.ns_coefficients).reshape(g.shape) - # muon does post momentum weight decay - if not self.pre_wd and self.wd > 0: t = t.detach() * (1.0 - self.wd * self.lr) # popular momentum does pre learning rate update if not self.classic: g = g * r * self.lr + # muon does post momentum weight decay + if not self.pre_wd and self.wd > 0: g = g + self.wd * self.lr * t.detach() ret.append(g.cast(t.dtype)) return ret, self.b From 08fbd25f1a829f54aaa85251c61623d8a2b9b193 Mon Sep 17 00:00:00 2001 From: George Hotz Date: Mon, 24 Aug 2026 13:34:42 -0700 Subject: [PATCH 183/306] AGENTS: no amend commits --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 962039e68aed2..00c02b4ab7cea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,3 +4,4 @@ - Run `python -m mypy tinygrad/` to typecheck - Run `python -m ruff check .` to lint - Read `./tinygrad/viz/README.md` for profiling and debugging rewrite rules +- Do not do amend commits. Always do a new commit if a force push to origin would be required. From 76bf6b7eec5ea713194f2cb74b3a5daa99a91d3b Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:09:34 -0700 Subject: [PATCH 184/306] llm: add DEBUG=2 for weights loading (#17710) * llm: add DEBUG=2 for weights loading * 00 --- tinygrad/llm/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index 5d37b5172eecc..4a17a9f6d02b8 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -141,7 +141,8 @@ def main(): args = parser.parse_args() # load the model - model, kv = Transformer.from_gguf(fetch(models.get(args.model, args.model)), args.max_context) + with Context(DEBUG=max(DEBUG.value, 2 if args.serve else 0)): + model, kv = Transformer.from_gguf(fetch(models.get(args.model, args.model)), args.max_context) model_name = kv.get('general.name') or kv.get('general.basename') or args.model file_sizes = [y.nbytes() for y in UOp.sink(*[x.uop for x in nn.state.get_parameters(model)]).toposort() if y.op is Ops.BUFFER] print(f"using model \"{model_name}\" with {sum(file_sizes):,} bytes and {sum(x.numel() for x in nn.state.get_parameters(model)):,} params, " From f4cc28824c7c7727290139c1f6ec1e41041484b3 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 17:25:28 -0400 Subject: [PATCH 185/306] delete GETTUPLE on UNSHARD rule [PR] (#17711) not used --- tinygrad/schedule/multi.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index 0d415d56cbc09..e928130d9b640 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -301,9 +301,6 @@ def param_to_multi(p:UOp): # resolve TUPLE+GETTUPLE (needed in multi) (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - # GETTUPLE on UNSHARD: passthrough UNSHARD (e.g. when FUNCTION was replaced by UNSHARD(GETTUPLE(...))) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.UNSHARD, name="multi"),), name="g"), - lambda g, multi: multi.src[0].gettuple(g.arg).unshard(multi.arg, multi.src[1:]) if multi.src[0].op in {Ops.FUNCTION, Ops.TUPLE} else multi), # rewrite into FUNCTION calls explicitly for UNSHARD (value-producing) (UPat(Ops.FUNCTION, name="call"), rewrite_into_function), (UPat((Ops.CALL, Ops.FUNCTION, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), From ed110993d3ed88404f2d6114897bc30934f4160a Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:32:16 -0700 Subject: [PATCH 186/306] mi350p: aqua vanjaram bringup for the raw PCI driver (#17693) * mi350p: aqua vanjaram bringup for the raw PCI driver Minimum stable bringup, all root-caused: - disable ASPM on the PCI path: link L1 across retimers makes GPU reads oscillate to 0xffffffff (no-op on non-sysfs backends) - probe live AIDs via MMHUB FB_LOCATION; dead ones read 0xffffffff and indirect writes to them corrupt the fabric - aqua reset semantics: full boot over a live state can kill the fabric, partial boot + reset_mec is the deepest safe reset - tolerate the HQD dequeue timeout like the kernel (wedged waves can survive RESET_WAVES) * am/aspm: simplify to GPU-endpoint-only clearing via pci config abstractions * fix AID derivation for non-dense sdma instance keys + bound the aspm capability walk for 0xff reads --- tinygrad/runtime/support/am/amdev.py | 19 +++++++++++++++++++ tinygrad/runtime/support/am/ip.py | 24 +++++++++++++++--------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/tinygrad/runtime/support/am/amdev.py b/tinygrad/runtime/support/am/amdev.py index a854d36e3e6ca..e28190aaf9238 100644 --- a/tinygrad/runtime/support/am/amdev.py +++ b/tinygrad/runtime/support/am/amdev.py @@ -145,8 +145,18 @@ def on_range_mapped(self): class AMDev: Version = 0xA0000008 + def _disable_aspm(self): + # L1 across retimers makes reads oscillate to 0xffffffff; power on defaults it enabled. Clearing the GPU endpoint + # alone suffices: L1 only engages when both ends of the link enable it. + cap, seen = self.pci_dev.read_config(0x34, 1) & 0xfc, set() # bound the walk: a dead link can return 0xff pointers forever + while cap and cap not in seen and self.pci_dev.read_config(cap, 1) != 0x10: + seen.add(cap) + cap = self.pci_dev.read_config(cap + 1, 1) & 0xfc + if cap and cap not in seen: self.pci_dev.write_config_flush(cap + 0x10, self.pci_dev.read_config(cap + 0x10, 2) & ~3, 2) # PCIe cap lnkctl + def __init__(self, pci_dev:PCIDevice, reset_mode=False): self.pci_dev, self.devfmt = pci_dev, pci_dev.pcibus + self._disable_aspm() self.vram, self.doorbell64, self.mmio = self.pci_dev.map_bar(0), self.pci_dev.map_bar(2, fmt='Q'), self.pci_dev.map_bar(5, fmt='I') self._run_discovery() @@ -170,6 +180,9 @@ def __init__(self, pci_dev:PCIDevice, reset_mode=False): if DEBUG >= 2: print(f"am {self.devfmt}: Malformed state. Issuing a full reset.") self.partial_boot = False + # aqua (gc 9.5.0): full boot over live state can kill the fabric (power cycle recovers); partial boot+reset_mec is the deepest safe reset + if self.ip_ver[am.GC_HWIP] == (9,5,0) and self.reg("regSCRATCH_REG7").read() == AMDev.Version: self.partial_boot = True + # Init hw for IP blocks where it is needed if not self.partial_boot: if self.psp.is_sos_alive() and self.smu.is_smu_alive(): @@ -342,3 +355,9 @@ def _build_regs(self): for prefix, hwip in mods: self.__dict__.update(import_asic_regs(prefix, self.ip_ver[hwip], cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[hwip]))) self.__dict__.update(import_asic_regs('mp', (11, 0, 0), cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[am.MP1_HWIP]))) + + # Live AIDs like the kernel: 4 SDMAs per AID; the AID lives iff its group's alive-mask is 0xf/0x3/0xc. + # Dead AIDs must never be touched via the indirect window: writes poison the whole fabric. + live_sdma = {k for k in self.regs_offset[am.SDMA0_HWIP] if k not in self.harvested[am.SDMA0_HWIP]} + max_aid = max((k >> 2 for k in self.regs_offset[am.SDMA0_HWIP]), default=0) + self.aids = [0] + [aid for aid in range(1, max_aid + 1) if sum(1 << (i & 3) for i in live_sdma if i >> 2 == aid) in {0xf, 0x3, 0xc}] diff --git a/tinygrad/runtime/support/am/ip.py b/tinygrad/runtime/support/am/ip.py index d0008537c521c..4f9a585449171 100644 --- a/tinygrad/runtime/support/am/ip.py +++ b/tinygrad/runtime/support/am/ip.py @@ -1,5 +1,5 @@ import ctypes, time, contextlib, functools -from typing import Literal +from typing import Iterable, Literal from tinygrad.helpers import to_mv, data64, lo32, hi32, DEBUG, wait_cond, pad_bytes, getbits from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.amd import import_soc @@ -32,7 +32,7 @@ def init_hw(self): # fence doorbells for harvested xccs (0xff & ~xcc_mask in the kernel); a fully-unharvested chip keeps the previous 0x0 live_xccs = sum(1 << i for i in self.adev.regs_offset[am.GC_HWIP] if i not in self.adev.harvested[am.GC_HWIP] and i < 8) self.adev.regXCC_DOORBELL_FENCE.write(0xff & ~live_xccs) - for aid in range(1, self.adev.gmc.vmhubs): + for aid in self.adev.aids[1:]: self.adev.indirect_wreg_pcie(self.adev.regXCC_DOORBELL_FENCE.addr[0], self.adev.regXCC_DOORBELL_FENCE.encode(shub_slv_mode=1), aid=aid) self.adev.regBIFC_GFX_INT_MONITOR_MASK.write(0x7ff) self.adev.regBIFC_DOORBELL_ACCESS_EN_PF.write(0xfffff) @@ -80,9 +80,11 @@ def init_sw(self): # MM hub is inited before any tlb flushes and is still valid during partial_boot, so set it to true self.hub_initted = {"MM": True, "GC": False} + self.mm_insts = self.adev.aids if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)} else list(range(self.vmhubs)) # dead mmhubs hang us + self.pf_status_reg = lambda ip: f"reg{ip}VM_L2_PROTECTION_FAULT_STATUS{'_LO32' if self.adev.ip_ver[am.GC_HWIP] >= (12,0,0) else ''}" - def init_hw(self): self.init_hub("MM", inst_cnt=self.vmhubs) + def init_hw(self): self.init_hub("MM", insts=self.mm_insts) def flush_hdp(self): self.adev.wreg(self.adev.reg("regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL").read() // 4, 0x0) def flush_tlb(self, ip:Literal["MM", "GC"], vmid, flush_type=0): @@ -91,7 +93,7 @@ def flush_tlb(self, ip:Literal["MM", "GC"], vmid, flush_type=0): # Can't issue TLB invalidation if the hub isn't initialized. if not self.hub_initted[ip]: return - for inst in range(self.adev.gmc.vmhubs if ip == "MM" else self.adev.gfx.xccs): + for inst in (self.adev.gmc.mm_insts if ip == "MM" else range(self.adev.gfx.xccs)): if ip == "MM": wait_cond(lambda: self.adev.regMMVM_INVALIDATE_ENG17_SEM.read(inst=inst) & 0x1, value=1, msg="mm flush_tlb timeout") self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").write(flush_type=flush_type, per_vmid_invalidate_req=(1 << vmid), invalidate_l2_ptes=1, @@ -116,9 +118,9 @@ def enable_vm_addressing(self, page_table, ip:Literal["MM", "GC"], vmid, inst): self.adev.reg(f"reg{ip}VM_CONTEXT{vmid}_CNTL").write(0x1800000, **fault_flags, **en_def_flags, enable_context=1, page_table_depth=((2 if self.trans_futher else 3) - page_table.lv), page_table_block_size=9 if self.trans_futher else 0, inst=inst) - def init_hub(self, ip:Literal["MM", "GC"], inst_cnt:int): + def init_hub(self, ip:Literal["MM", "GC"], insts:Iterable[int]): # Init system apertures - for inst in range(inst_cnt): + for inst in insts: self.adev.reg(f"reg{ip}MC_VM_AGP_BASE").write(0, inst=inst) self.adev.reg(f"reg{ip}MC_VM_AGP_BOT").write(0xffffffffffff >> 24, inst=inst) # disable AGP self.adev.reg(f"reg{ip}MC_VM_AGP_TOP").write(0, inst=inst) @@ -257,7 +259,7 @@ def init_hw(self): wait_cond(lambda: self.adev.regCP_STAT.read() == 0 or self.adev.regRLC_RLCS_BOOTLOAD_STATUS.read_bitfields()['bootload_complete'] == 0, value=True, msg="RLC autoload timeout") - self.adev.gmc.init_hub("GC", inst_cnt=self.xccs) + self.adev.gmc.init_hub("GC", insts=range(self.xccs)) if self.adev.partial_boot: return self.reset_mec() self._config_mec() @@ -405,7 +407,11 @@ def _dequeue_hqds(self): if self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1: self.adev.regCP_HQD_DEQUEUE_REQUEST.write(0x2, inst=xcc) # 1 - DRAIN_PIPE; 2 - RESET_WAVES self.adev.regSPI_COMPUTE_QUEUE_RESET.write(0x1, inst=xcc) - if not self.adev.is_err_state: wait_cond(lambda: self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1, value=0, msg="HQD dequeue timeout") + if not self.adev.is_err_state: + try: wait_cond(lambda: self.adev.regCP_HQD_ACTIVE.read(inst=xcc) & 1, value=0, msg="HQD dequeue timeout") + # kernel tolerates this too; a wedged wave can survive RESET_WAVES + except TimeoutError: + if DEBUG >= 2: print(f"am {self.adev.devfmt}: HQD dequeue timeout xcc{xcc} q{q}, continuing") self._grbm_select() class AM_IH(AM_IP): @@ -516,7 +522,7 @@ def init_hw(self): **({'utc_l1_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP] <= (5,2,0) else {}), inst=inst) if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}: - for aid_id in range(self.adev.gmc.vmhubs): + for aid_id in self.adev.aids: for dev_inst, (port, awid, offset, awaddr) in enumerate([(1, 0xe, 0xe, 0x1), (2, 0x8, 0x8, 0x2), (5, 0x9, 0x9, 0x8), (6, 0xa, 0xa, 0x9)]): entry = dev_inst + 1 + 4 * aid_id self.adev.reg(f"regDOORBELL0_CTRL_ENTRY_{entry}").write(**{f"bif_doorbell{entry}_range_size_entry": 20, From 8df3dac0ec4263d2538485a30a4a53468d0289b3 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 18:00:02 -0400 Subject: [PATCH 187/306] remove redundant UOp dtype [PR] (#17713) --- tinygrad/renderer/isa/x86.py | 2 +- tinygrad/runtime/ops_cpu.py | 8 ++++---- tinygrad/runtime/support/hcq2.py | 4 ++-- tinygrad/uop/ops.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index e4d2e7471bec2..9903fb7a2c15a 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -836,7 +836,7 @@ def spill(self, disp:UOp, x:UOp) -> UOp: if x.op is Ops.BUFFER: x = x.replace(dtype=dtypes.uint64) is_xmm = isinstance(x.tag, tuple) and x.tag[0].cons[0].size == 16 op = X86Ops.VMOVUPSm if is_xmm else X86Ops.MOVm - return UOp(Ops.INS, dtypes.void, fold_address(self.stack_pointer().index(disp)) + (x,), op, x.tag) + return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=op, tag=x.tag) def fill(self, disp:UOp, x:UOp, reg:Register) -> UOp: is_xmm = reg.cons[0].size == 16 diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 1e1293733f1ab..1ce6b428ea299 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -61,20 +61,20 @@ class CPUWorker: ring:Buffer; put:Buffer; sem:Buffer; sys:Buffer; done:Buffer; t def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: progs = [get_runtime(d, prog) if isinstance(prog, UOp) else cast(CPUDevice, Device[d]).prgs[prog] for d in devs] addrs = tuple(UOp.const(p.addr, dtypes.uint64) for p in progs) - words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, dtypes.uint64, addrs)),) + args - return UOp(Ops.INS, dtypes.void, words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg="cmd") + words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs)),) + args + return UOp(Ops.INS, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg="cmd") def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(ctx, prg, *args) la = [cpu_cmd(ctx,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t, dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] - return UOp(Ops.LINEAR, dtypes.void, tuple(la)) + return UOp(Ops.LINEAR, src=tuple(la)) pm_cpu_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), - (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP, dtypes.void, ())), + (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP)), (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: cpu_cmd(ctx, wait_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 0ce66f98ac224..b4fa7330845ab 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -62,8 +62,8 @@ def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...] ret, bit = [], buf.dtype.itemsize for (tag, dt, r), ps in groups.items(): view = buf.shrink(((r // bit, (max(off for off,_ in ps) + dt.itemsize) // bit),)).bitcast(dt) - offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const((off - r) // dt.itemsize, dtypes.int) for off,_ in ps)) - ret.append(view.index(offs).store(UOp(Ops.STACK, dt, tuple(val for _,val in ps))).rtag(tag)) + offs = UOp(Ops.STACK, src=tuple(UOp.const((off - r) // dt.itemsize, dtypes.int) for off,_ in ps)) + ret.append(view.index(offs).store(UOp(Ops.STACK, src=tuple(val for _,val in ps))).rtag(tag)) return tuple(ret) def make_binary_patch(buf:UOp, blob:bytes) -> UOp: return buf.store(UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype)).rtag("link") diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 4ad771539d7d2..d2feea591b612 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -614,7 +614,7 @@ def const(b:ConstLike, dtype:DType|None=None): return UOp(Ops.CONST, dtype, arg=dtype.const(b), src=()) # weak CONST with width on the CAST. TODO: this is the final const @staticmethod - def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, dtype, src=(UOp.const(b),), arg=dtype) + def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @staticmethod def range(end:sint, axis_id, axis_type=AxisType.WEAK, *arg, dtype=dtypes.weakint, src=(), **kwargs): return UOp(Ops.RANGE, src=(sint_to_uop(end, dtype),)+src, arg=(axis_id, axis_type)+arg, **kwargs) @@ -797,7 +797,7 @@ def _mop(self, op:Ops, arg) -> UOp: case Ops.STACK: srcs = (self,)+tuple(arg) dtype = cast(DType, dtype_from_uop(Ops.STACK, srcs, None)) - return UOp(Ops.STACK, dtype, tuple(u if u.base.is_invalid else UOp.const(u.val, dtype) if u.op is Ops.CONST else u.cast(dtype) for u in srcs)) + return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else UOp.const(u.val, dtype) if u.op is Ops.CONST else u.cast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] if len(usrcs) == 0: return UOp(op, src=(self,), arg=arg) From 2e7790f16c76d85fffdae80c3772831e3eb53003 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 24 Aug 2026 15:09:01 -0700 Subject: [PATCH 188/306] benchmarks: cleanup dsp benchmark (#17706) --- .github/workflows/benchmark.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 33023859c7ee2..b8a0db12d8d36 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -524,15 +524,6 @@ jobs: shell: bash -e -o pipefail {0} if: github.repository_owner == 'tinygrad' steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: setup staging db - if: github.ref == 'refs/heads/update_benchmark_staging' - run: | - echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV - rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - - name: reset process replay - run: test/external/process_replay/reset.py - name: Checkout Code uses: actions/checkout@v6 - name: setup staging db From 021c015eb41de0e68da917aef22b5eb7c39b5246 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 24 Aug 2026 15:32:36 -0700 Subject: [PATCH 189/306] benchmarks: more aggressive timeouts (#17715) --- .github/workflows/benchmark.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b8a0db12d8d36..691618c62414f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -88,7 +88,7 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 30 defaults: run: shell: bash -e -o pipefail {0} @@ -135,7 +135,7 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -184,7 +184,7 @@ jobs: fail-fast: false matrix: dev: ['AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} @@ -227,7 +227,7 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 15 defaults: run: shell: bash -e -o pipefail {0} @@ -273,7 +273,7 @@ jobs: fail-fast: false matrix: dev: ['AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 20 defaults: run: shell: bash -e -o pipefail {0} @@ -327,7 +327,7 @@ jobs: fail-fast: false matrix: dev: ['METAL', 'AMD', 'NV'] - timeout-minutes: 60 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -422,7 +422,7 @@ jobs: testusbgpu: name: UsbGPU Benchmark runs-on: [self-hosted, macOS] - timeout-minutes: 10 + timeout-minutes: 3 defaults: run: shell: bash -e -o pipefail {0} @@ -547,7 +547,7 @@ jobs: testcommausbgpubenchmark: name: UsbGPU Benchmark (comma) runs-on: [self-hosted, Linux, comma4] - timeout-minutes: 20 + timeout-minutes: 10 defaults: run: shell: bash -e -o pipefail {0} @@ -576,7 +576,7 @@ jobs: fail-fast: false matrix: dev: ['AMD', 'NV'] - timeout-minutes: 20 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} @@ -650,7 +650,7 @@ jobs: llvmspeed: name: LLVM Speed runs-on: [self-hosted, Linux, tinyboxrandom] - timeout-minutes: 20 + timeout-minutes: 5 if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code From 6f87158d77f66a36d5f8bbe915170b24e2acabe8 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Mon, 24 Aug 2026 18:40:15 -0400 Subject: [PATCH 190/306] feat: bump version to 0.14.0 (#17716) --- pyproject.toml | 2 +- tinygrad/helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86618374bd5fa..18939ec5a4afe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tinygrad" -version = "0.13.0" +version = "0.14.0" description = "You like pytorch? You like micrograd? You love tinygrad! <3" authors = [{ name = "George Hotz" }] diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 51fbf5dc39864..7390709b05d12 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -476,7 +476,7 @@ def fetch(url:str, name:pathlib.Path|str|None=None, subdir:str|None=None, gunzip if not fp.is_file() or not allow_caching or (sha256 and hashlib.sha256(fp.read_bytes()).hexdigest() != sha256): if extract: shutil.rmtree(extract_dir, ignore_errors=True) (_dir := fp.parent).mkdir(parents=True, exist_ok=True) - with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.13.0", **headers}), timeout=10) as r: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "tinygrad 0.14.0", **headers}), timeout=10) as r: assert r.status in {200, 206}, r.status length = int(r.headers.get('content-length', 0)) if not gunzip else None readfile = gzip.GzipFile(fileobj=r) if gunzip else r From d9004cff2245629aec1b3ddc0f545bd3fb419991 Mon Sep 17 00:00:00 2001 From: Raine Date: Mon, 24 Aug 2026 20:11:46 -0400 Subject: [PATCH 191/306] use native shifts in Payne Hayek instead of pow2 mul/divs (#17717) * use native shifts instead of pow2 mul/divs * wow --- tinygrad/codegen/decomp/transcendental.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinygrad/codegen/decomp/transcendental.py b/tinygrad/codegen/decomp/transcendental.py index e4e66fbd5a5b1..62c319175e46d 100644 --- a/tinygrad/codegen/decomp/transcendental.py +++ b/tinygrad/codegen/decomp/transcendental.py @@ -90,8 +90,8 @@ def _take(an:UOp, offset:int, count:int=0) -> UOp: if count+offset < len(two_over_pi_f) - 1: an = i.ne(count).where(_take(an, offset, count=count+1), an.const_like(two_over_pi_f[count+offset])) return an - def _shl_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) * pow2if(y, d.dtype).cast(dtypes.uint64)).cast(dtypes.uint32) - def _shr_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) // pow2if(y, d.dtype).cast(dtypes.uint64)).cast(dtypes.uint32) + def _shl_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) << y.cast(dtypes.uint64)).cast(dtypes.uint32) + def _shr_lazy(x:UOp, y:UOp): return (x.cast(dtypes.uint64) >> y.cast(dtypes.uint64)).cast(dtypes.uint32) a = [_take(UOp.const(0, dtypes.uint32), i) for i in range(4)] # (two_over_pi_f[Int(i) + n] << e) | (two_over_pi_f[Int(i) + n+1] >> (nbits - e)) From a7df1a1ace5527a74959ce671c1e6e66fce57265 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 20:32:57 -0400 Subject: [PATCH 192/306] const are weak 3 [pr] (#17695) --- test/mockgpu/amd/pcode.py | 30 +++++---- test/null/test_uop_symbolic.py | 2 +- test/null/test_uops.py | 12 ++-- test/unit/test_dtype_weak.py | 25 ++++++-- tinygrad/codegen/__init__.py | 21 +++---- tinygrad/codegen/decomp/dtype.py | 12 ++-- tinygrad/renderer/cstyle.py | 5 +- tinygrad/renderer/isa/x86.py | 8 +-- tinygrad/renderer/nir.py | 5 +- tinygrad/renderer/wgsl.py | 3 +- tinygrad/runtime/support/hcq2.py | 10 +-- tinygrad/schedule/rangeify.py | 10 +-- tinygrad/uop/ops.py | 15 +++-- tinygrad/uop/spec.py | 10 +-- tinygrad/uop/symbolic.py | 51 ++++++++------- tinygrad/uop/weak.py | 104 +++++++++++++++++++------------ 16 files changed, 186 insertions(+), 137 deletions(-) diff --git a/test/mockgpu/amd/pcode.py b/test/mockgpu/amd/pcode.py index fa5f214715197..8d49b6b42847c 100644 --- a/test/mockgpu/amd/pcode.py +++ b/test/mockgpu/amd/pcode.py @@ -23,6 +23,7 @@ def ev(node: ast.AST) -> int: VarVal = UOp | tuple[str, list[str], str] def _const(dt, v): return UOp.const(v, dt) +def _single_value(v: UOp): return v.vmin if v.vmin == v.vmax else None def _u32(v): return _const(dtypes.uint32, v) def _u64(v): return _const(dtypes.uint64, v) def _to_u32(v): return v if v.dtype == dtypes.uint32 else v.bitcast(dtypes.uint32) if v.dtype.itemsize == 4 else v.cast(dtypes.uint32) @@ -70,8 +71,8 @@ def _expr_bits(v: UOp) -> int: if v.op in (Ops.AND, Ops.XOR): widths: list[int] = [] for src in v.src: - if src.op == Ops.CONST and isinstance(src.val, int) and src.val > 0 and (src.val & (src.val + 1)) == 0: - widths.append(src.val.bit_length()) + if isinstance(sv:=_single_value(src), int) and sv > 0 and (sv & (sv + 1)) == 0: + widths.append(sv.bit_length()) if widths: return max(widths) return v.dtype.bitsize @@ -159,9 +160,9 @@ def minmax(a: UOp, b: UOp) -> UOp: def _find_two_pi_mul(x): if x.op != Ops.MUL or len(x.src) != 2: return None for i, s in enumerate(x.src): - if s.op == Ops.CONST and abs(s.val - 6.283185307179586) < 1e-5: return (x.src[1-i], 6.283185307179586) + if (sv:=_single_value(s)) is not None and abs(sv - 6.283185307179586) < 1e-5: return (x.src[1-i], 6.283185307179586) if s.op == Ops.MUL and len(s.src) == 2: - vals = [ss.val for ss in s.src if ss.op == Ops.CONST] + [ss.src[0].val for ss in s.src if ss.op == Ops.CAST and ss.src[0].op == Ops.CONST] + vals = [sv for ss in s.src if (sv:=_single_value(ss)) is not None] if len(vals) == 2 and abs(vals[0] * vals[1] - 6.283185307179586) < 1e-5: return (x.src[1-i], vals[0] * vals[1]) return None @@ -178,7 +179,7 @@ def _trig_reduce(x, phase=0.0): def _signext(val: UOp) -> UOp: for bits, mask, ext in [(4, 0xF, 0xFFFFFFF0), (8, 0xFF, 0xFFFFFF00), (16, 0xFFFF, 0xFFFF0000)]: - if (val.op == Ops.AND and len(val.src) == 2 and val.src[1].op == Ops.CONST and val.src[1].val == mask) or val.dtype.itemsize == bits // 8: + if (val.op == Ops.AND and len(val.src) == 2 and _single_value(val.src[1]) == mask) or val.dtype.itemsize == bits // 8: v32 = val.cast(dtypes.uint32) if val.dtype != dtypes.uint32 else val sb = (v32 >> _u32(bits - 1)) & _u32(1) return sb.ne(_u32(0)).where(v32 | _u32(ext), v32).cast(dtypes.int) @@ -549,7 +550,7 @@ def _apply_binop(self, left, right, op): if not dtypes.is_int(right.dtype): right = right.cast(dtypes.uint32) return (left >> right) if op == '>>' else (left << right) case '+' | '-': - if op == '-' and left.op == Ops.CONST and right.op == Ops.CONST: return _const(left.dtype, left.val - right.val) + if op == '-' and (lv:=_single_value(left)) is not None and (rv:=_single_value(right)) is not None: return _const(left.dtype, lv - rv) return (left + right) if op == '+' else (left - right) case '*' | '/': # Integer promotion: promote 16-bit integers to 32-bit before multiply to avoid overflow @@ -559,7 +560,7 @@ def _apply_binop(self, left, right, op): left, right = left.cast(pdt), right.cast(pdt) if op == '*': return left * right return (left // right) if dtypes.is_int(left.dtype) else (left / right) - case '**': return UOp(Ops.EXP2, src=(right.cast(left.dtype),)) if left.op == Ops.CONST and left.val == 2.0 else left + case '**': return UOp(Ops.EXP2, src=(right.cast(left.dtype),)) if _single_value(left) == 2.0 else left _PREC = [('||',), ('&&',), ('|',), ('^',), ('&',), ('==', '!=', '<>'), ('>=', '<=', '>', '<'), ('>>', '<<'), ('+', '-'), ('*', '/'), ('**',)] @@ -581,8 +582,8 @@ def unary(self) -> UOp: return inner.eq(_const(inner.dtype, 0)) if self.try_eat_val('-', 'OP'): inner = self.unary() - if inner.op == Ops.CONST: - return _const(dtypes.int if inner.dtype == dtypes.uint32 else inner.dtype, -inner.val) + if (v:=_single_value(inner)) is not None: + return _const(dtypes.int if inner.dtype == dtypes.uint32 else inner.dtype, -v) return inner.neg() if self.try_eat_val('+', 'OP'): return self.unary() return self.postfix() @@ -721,15 +722,13 @@ def _handle_bracket_rest(self, first: UOp, base: UOp, var_name: str | None = Non self.eat('OP') width = self.parse() self.eat('RBRACKET') - if width.op == Ops.CONST: - w = int(width.val) + if isinstance(w:=_single_value(width), int): return (base >> _to_u32(first)) & _const(base.dtype, (1 << w) - 1) return base if self.try_eat('COLON'): second = self.parse() self.eat('RBRACKET') - if first.op == Ops.CONST and second.op == Ops.CONST: - a, b = int(first.val), int(second.val) + if isinstance(a:=_single_value(first), int) and isinstance(b:=_single_value(second), int): if a < b: return _bitreverse(base, b - a + 1) hi, lo = a, b if lo >= base.dtype.itemsize * 8: @@ -750,8 +749,7 @@ def _handle_bracket_rest(self, first: UOp, base: UOp, var_name: str | None = Non dt_suffix = DTYPES.get(self.eat('IDENT').val, dtypes.uint32) if var_name is None: var_name = self._find_var_name(base) - if first.op == Ops.CONST: - idx = int(first.val) + if isinstance(idx:=_single_value(first), int): # Check for array element (var@idx) if var_name and f'{var_name}@{idx}' in self.vars: v = self.vars[f'{var_name}@{idx}'] @@ -924,7 +922,7 @@ def mindex(idx:UOp): return mem.index(idx.valid(active) if active is not None el def _coerce_cmp(self, l: UOp, r: UOp) -> tuple[UOp, UOp]: if l.dtype != r.dtype: - if r.dtype == dtypes.int and r.op == Ops.CONST and r.val < 0: l = l.cast(dtypes.int) + if r.dtype == dtypes.int and isinstance(rv:=_single_value(r), int) and rv < 0: l = l.cast(dtypes.int) else: r = r.cast(l.dtype) return l, r diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 08f5dc2f2bdba..b127a89dbc9e1 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -1023,7 +1023,7 @@ def test_where_cast(self): cond = Variable("s", 0, 3, dtypes.int) < 2 a = Variable("a", 0, 3, dtypes.int) self.assertIs(graph_rewrite(cond.where(a, a+1).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), (a+1).cast(dtypes.half))) - self.assertIs(graph_rewrite(cond.where(a, uconst(2)).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), UOp.const(2, dtypes.half))) + self.assertIs(graph_rewrite(cond.where(a, uconst(2)).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), uconst(2.0))) self.assertIs(graph_rewrite(cond.where(a, UOp.invalid()).cast(dtypes.half), sym), cond.where(a.cast(dtypes.half), UOp.invalid())) def test_where_const_gate_keeps_stated_width(self): diff --git a/test/null/test_uops.py b/test/null/test_uops.py index 2128234644f82..aaded43fdf08c 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -6,7 +6,7 @@ from tinygrad.dtype import dtypes, AddrSpace, ConstFloat, Invalid # noqa: F401 from tinygrad.device import Device from tinygrad.uop.ops import Ops, AxisType, ParamArg, PatternMatcher, UOp, UPat, dtype_from_uop, exec_alu, graph_rewrite # noqa: F401 # ParamArg used by eval(str(uop)) roundtrip tests -from tinygrad.uop.weak import pm_lower_index_dtype +from tinygrad.uop.weak import pm_lower_weak from tinygrad.uop.spec import spec_program, spec_shared, type_verify from tinygrad.uop.symbolic import sym, pm_remove_invalid from test.helpers import eval_uop, to_uops_list @@ -76,16 +76,18 @@ def test_gated_shrink_lowers_to_selected_width(self): buf = UOp.param(0, dtypes.float, (2**31+64,)) i = UOp.variable("i", 0, 2**28) shrink = UOp(Ops.SHRINK, src=(buf, (i*24).valid(i < 2**28), UOp.const(4))) - lowered = graph_rewrite(shrink.sink(), pm_lower_index_dtype) - self.assertTrue(all(u.dtype != dtypes.weakint for u in lowered.backward_slice_with_self), "lowering must resolve all weakint") + lowered = graph_rewrite(shrink.sink(), pm_lower_weak) + self.assertTrue(all(u.op is Ops.CONST for u in lowered.backward_slice_with_self if u.dtype in dtypes.weaks), + "lowering must resolve every weak width, except a typed literal's value half") sh = next(u for u in lowered.backward_slice_with_self if u.op is Ops.SHRINK) self.assertEqual(sh.src[1].dtype, dtypes.long) def test_reg_buffer_size_lowers(self): reg = UOp.placeholder((4,), dtypes.float, 0, addrspace=AddrSpace.REG) self.assertEqual(reg.src[0].dtype, dtypes.weakint) - lowered = graph_rewrite(reg.sink(), pm_lower_index_dtype) - self.assertTrue(all(u.dtype != dtypes.weakint for u in lowered.backward_slice_with_self), "lowering must resolve all weakint") + lowered = graph_rewrite(reg.sink(), pm_lower_weak) + self.assertTrue(all(u.op is Ops.CONST for u in lowered.backward_slice_with_self if u.dtype in dtypes.weaks), + "lowering must resolve every weak width, except a typed literal's value half") self.assertEqual(next(u for u in lowered.backward_slice_with_self if u.op is Ops.BUFFER).src[0].dtype, dtypes.int) class TestSafeCast(unittest.TestCase): diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 92eb70234b655..84e9de85d4459 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -4,7 +4,7 @@ from tinygrad.helpers import Context from tinygrad.dtype import least_upper_float from tinygrad.uop.ops import UOp, Ops, GroupOp, dtype_from_uop, graph_rewrite -from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak +from tinygrad.uop.weak import pm_commit_weak from tinygrad.uop.symbolic import symbolic_simple from tinygrad.uop.spec import spec_shared, type_verify from tinygrad.engine.jit import JitError @@ -74,7 +74,7 @@ def test_weak_expression_anchors_at_strong_lub(self): recips = [u for u in (x / y)._uop.toposort() if u.op is Ops.RECIPROCAL] self.assertEqual([(u.dtype, u.src[0].dtype) for u in recips], [(dtypes.float32, dtypes.float32)]) with Context(DEFAULT_FLOAT=dtypes.float16): - committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_lower_index_dtype, ctx={}) + committed = graph_rewrite((UOp.const(1).cast(dtypes.int32) + UOp.const(1.0)).cast(dtypes.float32), pm_commit_weak) self.assertEqual([u.dtype for u in committed.toposort() if u.op is Ops.ADD], [dtypes.float32]) def test_div_sub_operand_kept_weak(self): @@ -85,7 +85,7 @@ def test_div_sub_operand_kept_weak(self): def test_cast_weak_expression_commits_at_cast_floor(self): # the floor never narrows: a cast BELOW the default does not pull the compute width down with it with Context(DEFAULT_FLOAT=dtypes.float32): - narrowed = graph_rewrite((UOp.const(1.0) + UOp.const(2.0)).cast(dtypes.float16), pm_lower_index_dtype, ctx={}) + narrowed = graph_rewrite((UOp.const(1.0) + UOp.const(2.0)).cast(dtypes.float16), pm_commit_weak) self.assertEqual((narrowed.dtype, narrowed.src[0].dtype), (dtypes.float16, dtypes.float32)) def test_cast_weak_expression_value_uses_cast_floor(self): @@ -126,8 +126,23 @@ def test_weak_srcs_commit_only_at_a_concrete_lub(self): weak_lub = UOp(Ops.ADD, src=(UOp.const(1), UOp.const(1.0))) self.assertIs(graph_rewrite(weak_lub, pm_commit_weak), weak_lub) concrete = UOp.const(2.0).cast(dtypes.float16) - where = graph_rewrite(UOp(Ops.WHERE, src=(UOp.const(True), concrete, UOp.const(1.0))), pm_lower_index_dtype, ctx={}) - self.assertEqual(tuple(x.dtype for x in where.src), (dtypes.bool, dtypes.float16, dtypes.float16)) + # the weak arm stays bare: its sibling states the width, so the WHERE already derives float16 for it + where = graph_rewrite(UOp(Ops.WHERE, src=(UOp.const(True), concrete, UOp.const(1.0))), pm_commit_weak) + self.assertEqual((where.dtype, tuple(x.dtype for x in where.src)), (dtypes.float16, (dtypes.bool, dtypes.float16, dtypes.weakfloat))) + + def test_derivable_const_rounds_at_the_derived_width(self): + # re-rounds a derivable const in place (still bare) so value-keyed folds (x*1 -> x, x*-1 -> NEG) still fire + x = UOp.param(0, dtypes.float32, (1,)).index(UOp.const(0).cast(dtypes.int32)).load() + mul = graph_rewrite(x * UOp.const(-0.9999999893980771), symbolic_simple+pm_commit_weak) + self.assertIs(mul.src[1], UOp.const(-1.0)) + self.assertIs(graph_rewrite(x * UOp.const(1.0000000106), symbolic_simple+pm_commit_weak), x) + + def test_committed_const_conversion_folds_for_native_format(self): + folded = graph_rewrite(UOp.const(16256, dtypes.ushort).cast(dtypes.uint), symbolic_simple) + self.assertIs(folded, UOp.const(16256, dtypes.uint)) + # fmt-less targets are lowered by renderer rewrites, where collapsing this pair would cycle with float-intermediate insertion. + emulated = UOp.const(1.0, dtypes.float).cast(dtypes.bfloat16) + self.assertIs(graph_rewrite(emulated, symbolic_simple), emulated) def test_weak_shift_lhs_commits_the_node(self): # a shift derives its lhs's dtype, so committing the lhs restates the root (WGSL's packed store writes `mask << shift_am`) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 9b3f7ad18e51c..15e9edbd9fb98 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -3,7 +3,7 @@ from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType -from tinygrad.uop.weak import pm_lower_index_dtype, pm_commit_weak, pm_cast_weak +from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const from tinygrad.uop.render import pyrender from tinygrad.uop.spec import type_verify, spec_tensor, spec_program from tinygrad.renderer import Renderer, Estimates @@ -282,10 +282,6 @@ def add_war_barrier(end:UOp): (UPat(Ops.END, name="end"), add_war_barrier), ]) -pm_casted_consts = PatternMatcher([ - (UPat(Ops.CONST, dtypes.all, name="c"), lambda c: UOp.cconst(c.val, c.dtype)), -]) - def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: if VIZ: graph_rewrite(ast, PatternMatcher([]), name="View Base AST") if DEBUG >= 5: print(pyrender(ast)) @@ -347,11 +343,13 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # extra symbolic before decomp. crashes without this? # NOTE: also run indexing_simplify here, while the index is still weakint and (x+y)*c -> x*c+y*c applies - sink = graph_rewrite(sink, sym+indexing_simplify, name="extra symbolic") + # commit widths minted in this fixpoint before lowering inspects INDEX shapes + sink = graph_rewrite(sink, sym+indexing_simplify+pm_commit_weak, name="extra symbolic") - # lower index dtype + # the boundary: required compute dtypes settle here; derivable const edges may stay bare # NOTE: we need indexing_simplify to remove the cast to long using the Invalid - sink = graph_rewrite(sink, symbolic_simple+pm_lower_index_dtype+indexing_simplify, ctx={}, name="lower all index dtypes") + # NOTE: symbolic must NOT be composed here -- pm_data_invalid pushes the weak result CAST into a gated WHERE, remaking the weak node, and it cycles + sink = graph_rewrite(sink, pm_lower_weak+indexing_simplify, name="lower all index dtypes") # final symbolic before decomp sink = graph_rewrite(sink, symbolic, name="final symbolic") @@ -375,12 +373,11 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: # final rules for the renderer (without sym) extra_matcher = ren.extra_matcher if ren.extra_matcher is not None else PatternMatcher([]) - pm_final_rewrite = pm_commit_weak+pm_cast_weak+pm_decomp+extra_matcher+pm_split_ends + pm_final_rewrite = pm_commit_weak+pm_decomp+extra_matcher+pm_split_ends sink = graph_rewrite(sink, pm_final_rewrite+pm_remove_invalid, ctx=ren, name="final rewrite") - # spell every literal as a casted const CAST(dt, CONST(value)) - # TODO: remove once consts are always weak - sink = graph_rewrite(sink, pm_casted_consts, name="casted consts", walk=True) + # commit every const still bare so no renderer reads one + sink = graph_rewrite(sink, pm_cast_const, name="cast consts") # add implicit barriers (stores/loads through LOCAL memory ordered by AFTER or across loop iterations need workgroup barriers) sink = graph_rewrite(sink, pm_implicit_barriers, name="add implicit barriers") diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 6daf24e459275..61ea1195cea64 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -3,6 +3,7 @@ from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite +from tinygrad.uop.weak import commit_weak_consts from tinygrad.renderer import Renderer from tinygrad.codegen.decomp.transcendental import exponent_bias, shl, shr @@ -137,6 +138,8 @@ def f2f_store(st, idx, val, fr:DType, to:DType): # tag is the 32-bit word this node becomes - (0 for the low word, 1 for the high, the dtype the consumer wants) pm_long_decomp: PatternMatcher = PatternMatcher([ + # the decomp's own bottom-up rewrite can mint bare consts mid-flight: word splitting commits them at the long sibling's dtype + (UPat(GroupOp.All, name='x'), lambda x: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in l2i_dt), None))), (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), src=(UPat.var("sz"),), name="x"), lambda x,sz: UOp(x.op, src=(sz*2,), arg=replace(x.arg, dtype=l2i_dt[x.dtype]), tag=x.tag)), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: @@ -148,6 +151,9 @@ def f2f_store(st, idx, val, fr:DType, to:DType): split_l2i(ctx, x.op, dt:=l2i_dt[a.dtype], *flatten((s.rtag((0, dt)), s.rtag((1, dt))) for s in x.src))), (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: split_l2i(ctx, Ops.BITCAST, l2i_dt[x.dtype], a.rtag((0, dt:=l2i_dt[a.dtype])), a.rtag((1, dt)))[x.tag[0]]), + # a const splits by value; the general CAST arm below would drop its high word + (UPat(Ops.CAST, src=(UPat(Ops.CONST, name='c'),), tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), + lambda x,c: UOp.const(truncate[x.tag[1]](c.val >> (32*x.tag[0])), x.tag[1])), (UPat(Ops.CAST, tuple(l2i_dt.keys()), src=(UPat.var('a'),), name="x"), lambda ctx,a,x: split_l2i(ctx, x.op, x.dtype, a)[x.tag[0]] if x.tag is not None else None), (UPat(Ops.CAST, src=(UPat.var('a', tuple(l2i_dt.keys())),), name="x"), lambda ctx,a,x: @@ -161,9 +167,7 @@ def f2f_store(st, idx, val, fr:DType, to:DType): split_l2i(ctx, x.op, l2i_dt[x.dtype], *flatten((a.rtag((0, l2i_dt[x.dtype])), a.rtag((1, l2i_dt[x.dtype]))) for a in x.src))[x.tag[0]] if x.tag is not None else None), (UPat(Ops.LOAD, tuple(l2i_dt.keys()), src=(UPat.var('idx'),), name='x'), lambda ctx,x,idx: - reindex(graph_rewrite(idx, pm_long_decomp, ctx=ctx, bottom_up=True), x.tag[0]).replace(tag=None).load() if x.tag is not None else None), - (UPat(Ops.CONST, tag={(w, dt) for w in (0, 1) for dt in l2i_dt.values()}, name='x'), lambda x: - UOp.const(truncate[x.tag[1]]((x.val >> 32) if x.tag[0] == 1 else (x.val & 0xFFFFFFFF)), x.tag[1])) + reindex(graph_rewrite(idx, pm_long_decomp, ctx=ctx, bottom_up=True), x.tag[0]).replace(tag=None).load() if x.tag is not None else None) ]) # float decomposition patterns - ctx is (fr, to) tuple @@ -186,8 +190,6 @@ def f2f_store(st, idx, val, fr:DType, to:DType): f2f(x.bitcast(f2f_dt[ctx[0]]), ctx[0], ctx[1]) if bc.dtype == ctx[0] else None), (UPat(Ops.CAST, dtypes.floats, src=(UPat.var("val"),), name="x"), lambda ctx,x,val: f2f_clamp(val.cast(ctx[1]), ctx[0]) if x.dtype == ctx[0] else None), - # a CONST has no srcs to cast, it restates its value at the emulating dtype - (UPat(Ops.CONST, dtypes.floats, name="x"), lambda ctx,x: UOp.const(x.val, ctx[1]) if x.dtype == ctx[0] else None), (UPat(GroupOp.All-GroupOp.Defines-{Ops.CAST, Ops.BITCAST, Ops.CONST}, dtypes.floats, name="x"), lambda ctx,x: UOp(x.op, src=tuple(s.cast(ctx[1]) if s.dtype == ctx[0] else s for s in x.src), arg=x.arg, tag=x.tag) if x.dtype == ctx[0] else None), (UPat(Ops.STORE, src=(UPat.var("idx"), UPat(Ops.BITCAST, dtypes.floats, name="val")), name='st'), lambda ctx,st,idx,val: diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 3035395f96beb..ac9f2eda85792 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -3,6 +3,7 @@ from collections import defaultdict, Counter from tinygrad.codegen.opt import tc from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str, axis_letters +from tinygrad.uop.weak import commit_weak_consts from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, NUM_CPU_THREADS, IMAGE, FLOAT16, is_image_shape from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, float_to_bf16 from tinygrad.renderer import Renderer @@ -75,6 +76,8 @@ def create_non_native_float_pats(dts:tuple[DType, ...], casting:bool=True): patterns = PatternMatcher([ + # a weak CONST states no width and cannot be restated: commit it at the emulated dtype a sibling src states + (UPat(GroupOp.ALU, name="x"), lambda x, dts=dts: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in dts), None))), (UPat(Ops.WHERE, dtype=dts, src=(UPat.var("b"), UPat.var("x"), UPat.var("y")), name="w"), lambda w,b,x,y: b.where(x.cast(dtypes.float), y.cast(dtypes.float)).cast(w.dtype)), (UPat(GroupOp.ALU-{Ops.WHERE}, dtype=dts, name="x"), @@ -524,8 +527,6 @@ def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 = (UPat(Ops.WMMA, name="x", dtype=dtypes.float), lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) if x.src[0].max_numel() == 8 and x.src[0].dtype in dtypes.fp8_ocp else None), - # bfloat16 constant casting - (UPat.cvar('x', dtypes.bfloat16), lambda x: cast_float_to_bf16(UOp.const(x.val, dtypes.float))), ]) def asm(self, prg:UOp, lin:UOp) -> bytes: diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 9903fb7a2c15a..60b90290d8df4 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -150,9 +150,9 @@ class X86GroupOp: # no cmpne for packed ints, y != x => !(y==x) (UPat(Ops.CMPNE, src=(UPat.var("y", dtypes.ints), UPat.var("x")), name="cmp"), lambda y,x,cmp: UOp(Ops.CMPEQ, src=(y,x))^True if y.max_numel() > 1 else None), - # float where expects a mask - (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats), UPat.var("b")), - lambda m,a,b: m.cast(a.dtype).ne(0).where(a, b) if m.src[0].dtype not in dtypes.floats else None), + # float WHERE needs a mask unless its comparison already has a float operand + (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats+(dtypes.weakfloat,)), UPat.var("b")).named("w"), + lambda m,a,b,w: m.cast(w.dtype).ne(0).where(a, b) if w.dtype in dtypes.floats and not dtypes.is_float(m.src[0].dtype) else None), # rewrite -x -> 0 - x (UPat(Ops.NEG, name="x"), lambda x: UOp(Ops.SUB, src=(x.const_like(0),) + x.src)), # TODO: add support for mod, requires support for accessing the 2nd+ reg of a multi output instruction @@ -653,7 +653,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # 0b10 -- signals memory access with 32bit displacement # 0b11 -- signals no memory access if disp_uop is not None: - assert disp_uop.op is Ops.CAST, "displacement must be a literal" + assert disp_uop.op is Ops.CAST, "displacement must be a const" assert disp_uop.dtype in (dtypes.int8, dtypes.int32), "displacement can only be 1 or 4 byte signed int" # rbp/r13 always require a displacement if disp_uop.src[0].val != 0 or rm == 0b101: mod = 0b01 if disp_uop.dtype.itemsize == 1 else 0b10 diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index 3b254ef2389c1..cbfa3e46db45d 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -121,8 +121,6 @@ class NIRRenderer(Renderer): code_for_op = {**{k:lambda:None for k in u_aop.keys()}, **{k:lambda:None for k in s_aop.keys()}, **{k:lambda:None for k in f_aop.keys()}} extra_matcher = PatternMatcher([ - # handle negative unsigned CONST - (UPat.cvar("x", dtypes.uints), lambda x: UOp.const(x.dtype.max+x.val+1, x.dtype) if x.val < 0 else None), # from ptx (UPat.var('x', dtype=dtypes.bool) uint8 @@ -136,9 +134,10 @@ class NIRRenderer(Renderer): # ref: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpConvertFToU (UPat(Ops.CAST, (dtypes.uchar, dtypes.ushort), src=(UPat.var("x", dtypes.floats),), name="c"), lambda x,c: x.cast(dtypes.int32).cast(c.dtype)), # load/store use pointer arithmetic, and the cast does nothing. NOTE: this doesn't apply to image indexing cause it's 1-D + # nor to REG/ALU register picks, which keep their own index dtype (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), lambda x,buf,off: x.replace( src=(buf,UOp.const(off.val, dtypes.long) if off.op is Ops.CONST else off.cast(dtypes.long))+x.src[2:]) - if buf.addrspace != AddrSpace.REG and not is_image_shape(buf._shape) else None), + if buf.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) and not is_image_shape(buf._shape) else None), # images need index to be int for nir (coordinates only: the INDEX keeps its access dtype) (UPat.var("buf").index(UPat.var("idx_y"), UPat.var("idx_x"), name="x"), lambda x,buf,idx_y,idx_x: x.replace(src=(buf, idx_y.cast(dtypes.int), idx_x.cast(dtypes.int)))), diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 1a3db1e177ae1..9c7fd26e94d46 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -69,7 +69,8 @@ class WGSLRenderer(CStyleLanguage): (UPat.cvar("c").cast(dtypes.bool), lambda c: "true" if c.val else "false"), (UPat.cvar("c").cast((dtypes.uchar, dtypes.ushort, dtypes.uint32)), lambda c: f"bitcast({c.val})" if c.val < 0 else f"{c.val&0xFFFFFFFF}u"), - (UPat.cvar("c").cast(dtypes.int32, name="x"), lambda ctx,x,c: f"{truncate[x.dtype](c.val)}"), + # a negative const must state its type: contextual conversion of a bare abstract int rejects it in a u32 position + (UPat.cvar("c").cast(dtypes.int32, name="x"), lambda ctx,x,c: f"i32({v})" if (v:=truncate[x.dtype](c.val)) < 0 else f"{v}"), (UPat(Ops.BUFFER, name="x"), lambda ctx,x: f"var{'' if x.addrspace == AddrSpace.LOCAL else ''} {ctx[x]}: array<{ctx.buf_map(x)},{_packed_size(x)}>;"), (UPat(Ops.BITCAST, dtype=dtypes.half, name="x", src=(UPat(dtype=(dtypes.short, dtypes.ushort, dtypes.uint32),),)), diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index b4fa7330845ab..fd3ee107e7e25 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -71,8 +71,8 @@ def make_binary_patch(buf:UOp, blob:bytes) -> UOp: return buf.store(UOp(Ops.BINA def make_cmdbuf(lin, devs, buf:UOp|None=None, dep:tuple[UOp, ...]=()): blob, patches = bytearray(), [] for s in (s for ins in lin.src for s in ins.src): - if s.op is not Ops.CONST: patches.append((len(blob), s)) - blob.extend(struct.pack(f'<{s.dtype.fmt}', s.val if s.op is Ops.CONST else 0x0)) + if not (is_const:=(s.op is Ops.CAST and s.src[0].op is Ops.CONST)): patches.append((len(blob), s)) + blob.extend(struct.pack(f'<{s.dtype.fmt}', s.val if is_const else 0x0)) cmdbuf = buf if buf is not None else UOp.placeholder((len(blob) // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("cmdbuf") return cmdbuf.after(*dep, make_binary_patch(cmdbuf, bytes(blob)), *make_patches(cmdbuf, patches)) @@ -344,7 +344,9 @@ def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patc lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words)))) r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst)) off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()))) - patch = dst.shrink(((off, off+table.dtype.itemsize//dst.dtype.itemsize),)).bitcast(table.dtype).index(0).store(table.index(slot).load()).end(r) + # SHRINK(offset, length): a const length keeps the end bound from becoming an expression the program spec rejects + patch = UOp(Ops.SHRINK, src=(dst, off, off.const_like(table.dtype.itemsize//dst.dtype.itemsize))).bitcast(table.dtype).index(0) \ + .store(table.index(slot).load()).end(r) return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: patch} def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) @@ -511,7 +513,7 @@ def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp: buf, start = unwrap_view(view) for off,val in zip(off.src, val.src): for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): - data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype]((v.src[0] if v.op is Ops.CAST else v).val)) + data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype](v.val)) bo = start*buf.dtype.itemsize + off.val*val.dtype.itemsize b.ensure_allocated()._buf.cpu_view().view(fmt='B')[bo:bo+len(data)] = data return UOp(Ops.NOOP) diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 4751e44212e47..89fc1b622c4ef 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -304,15 +304,15 @@ def after_all_invalid(after:UOp): (UPat(Ops.INDEX, name="idx").f(Ops.STAGE, allow_any_len=True, name="b2"), remove_noop_bufferize), (UPat(Ops.INDEX, src=(UPat(Ops.STAGE),), allow_any_len=True, name="idx").f(Ops.NOOP).f(Ops.STAGE, allow_any_len=True, name="b2"), remove_noop_bufferize), - # no buffers for const (ranges don't matter for const - it's the same value everywhere) - (UPat(Ops.CONST, name='c').f(Ops.STAGE, allow_any_len=True, name="b"), lambda c,b: b.const_like(c.val)), - # indexing a const is a const - (UPat(Ops.INDEX, src=(UPat(Ops.CONST, name="c"),),), lambda c: c), + # no buffers for a const, in either spelling + (UPat.cvar('c').or_casted().f(Ops.STAGE, allow_any_len=True, name="b"), lambda c,b: b.const_like(c.val)), + # indexing a const is the const + (UPat(Ops.INDEX, src=(UPat.cvar().or_casted("c"),),), lambda c: c), # indexing an after with all fully invalid stores is invalid (UPat(Ops.INDEX, src=(UPat(Ops.AFTER, name="after"),), allow_any_len=True, name="idx"), lambda idx,after: idx.const_like(Invalid) if after_all_invalid(after) else None), # hack if a noop turned to a const - (UPat(Ops.NOOP, src=(UPat.cvar("c"),)), lambda c: c), + (UPat(Ops.NOOP, src=(UPat.cvar().or_casted("c"),)), lambda c: c), # a deviceless MSTACK src is the same value on every device, so indexing the stack is just indexing that value (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True, name="idx"), lambda s,idx: idx.replace(src=(s,)+idx.src[1:]) if s.device is None else None), diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index d2feea591b612..f66e8a32356c3 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -188,9 +188,8 @@ class UOpMetaClass(type): def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, metadata:tuple[Metadata,...]|None=None, _buffer:Buffer|None=None): if dtype is None: dtype = dtype_from_uop(op, src, arg) or dtypes.void - # CONST derives its dtype by value only when the constructor omits one # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py - if SPEC == 2 and op is not Ops.CONST and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype: + if SPEC == 2 and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype: raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(*key)) @@ -256,8 +255,10 @@ def replace(self, **kwargs) -> UOp: def rtag(self, tag=True): return self.replace(tag=tag) @property def val(self): - assert self.op is Ops.CONST, f"val is only for CONST, got {self.op}" - return self.arg + if self.op is Ops.CONST: return self.arg + # a casted const CAST(dt, CONST(v)) is one const: .val reads the value through the CAST + assert self.op is Ops.CAST and self.src[0].op is Ops.CONST, f"val is only for consts, got {self.op}" + return self.src[0].val @property def is_invalid(self) -> bool: return self.op is Ops.CONST and self.val is Invalid @recursive_property @@ -611,7 +612,8 @@ def const(b:ConstLike, dtype:DType|None=None): if isinstance(b, UOp): return b.cast(dtype) # NOTE: it always has to be STACK now, even if they are all the same if isinstance(b, tuple): return UOp.stack(*[UOp.const(c, dtype) for c in b]) - return UOp(Ops.CONST, dtype, arg=dtype.const(b), src=()) + # .cast folds away at exactly the dtypes a CONST derives (bool/weakint/weakfloat): bare there, the pair everywhere else + return UOp(Ops.CONST, arg=dtype.const(b), src=()).cast(dtype) # weak CONST with width on the CAST. TODO: this is the final const @staticmethod def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @@ -987,7 +989,8 @@ def expr(self) -> str: return unwrap(self.arg.name) def bind(self, val:int|UOp): assert self.is_variable, f"op is {self.op}, need Variable" - uval = self.const_like(val) if isinstance(val, int) else val + # the Variable states the width, so the bound value stays bare: is_bound_var tests for a CONST there, unbind reads .val + uval = UOp.const(val) if isinstance(val, int) else val assert self.vmin <= uval.vmin and uval.vmax <= self.vmax, f"bind {val} not in range [{self.vmin}, {self.vmax}]" assert uval.divides(self.arg.multiple_of) is not None, f"bind {val} not divisible by {self.arg.multiple_of}" return self.after(self.store(uval)) diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 36d72390d619e..09b9c17c9060a 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -203,10 +203,9 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # these ops can exist in programs but not the tensor spec. example: LOAD spec_program = PatternMatcher([ - # a literal is CAST(dt, CONST(value)), so its inner CONST is the one weak node a program may contain - (UPat(Ops.CONST, dtype=dtypes.weaks, name="x"), lambda x: x.dtype is dtypes.from_py(x.val)), - # index and weak dtypes are not allowed in programs - (UPat(GroupOp.All, (dtypes.weakint, dtypes.weakfloat)), lambda: False), + # every width in a program is stated: a CONST appears only under the CAST stating its width, and is the only weak node + (UPat(GroupOp.All, name="x"), lambda x: False if x.op is not Ops.CAST and any(s.op is Ops.CONST for s in x.src) else None), + (UPat(GroupOp.All-{Ops.CONST}, dtypes.weaks), lambda: False), # allow special SHRINK (UPat(Ops.SHRINK, src=(UPat((Ops.PARAM, Ops.BUFFER, Ops.AFTER)), UPat(), UPat(Ops.CONST).or_casted())), lambda: True), @@ -254,8 +253,9 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.SINK, dtypes.void), lambda: True), # the store of a bound Variable binds it: AFTER(BUFFER, STORE(BUFFER, CONST)) in call args (UPat(Ops.STORE, dtypes.void, (UPat(Ops.BUFFER, name="b"), UPat(Ops.CONST))), lambda b: b.is_variable), - # const + stack to make vconsts and shape args + # const + stack to make vconsts and shape args. a 0-size/bound reduce keeps its const casted (UPat(Ops.CONST, src=()), lambda: True), + (UPat(Ops.CAST, src=(UPat(Ops.CONST, src=()),)), lambda: True), (UPat(Ops.STACK, name="s"), lambda s: all(x.op in (Ops.CONST, Ops.PARAM) or x.is_variable or x.is_bound_var for x in s.src) or None), # linear for more kernels (TODO: we should enter non sink calls) #(UPat(Ops.LINEAR), lambda: True), diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 7f48a18ce2abf..81868f4ff20a6 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -1,12 +1,12 @@ # all of symbolic lives here now import math from collections import defaultdict -from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu -from tinygrad.dtype import PyConst, ConstType, dtypes, can_lossless_cast, Invalid, bitcast, truncate +from tinygrad.uop.ops import Ops, PatternMatcher, UPat, UOp, GroupOp, exec_alu, promo_dtype +from tinygrad.dtype import PyConst, dtypes, can_lossless_cast, Invalid, bitcast, truncate from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup -from tinygrad.uop.weak import commit_weak +from tinygrad.uop.weak import pm_uncast_const, commit_weak # TODO: symbolic shouldn't be importing from codegen from tinygrad.codegen.decomp.transcendental import xpow @@ -22,20 +22,11 @@ def simplify_pow(x:UOp, c:UOp) -> UOp|None: def fold_bitcast(root:UOp, c:UOp) -> UOp|None: if c.dtype.fmt is None or root.dtype.fmt is None or c.dtype.itemsize != root.dtype.itemsize: return None + # the value is mathematical and may not fit: reading it as bits is the emission that pins it to the stated width return root.const_like(bitcast(truncate[c.dtype](c.val), c.dtype, root.dtype)) -# const folding works for CONST and STACK -const_folding_pat = UPat((Ops.CONST, Ops.STACK)) - -def const_arg(u:UOp) -> ConstType|tuple[ConstType, ...]|None: - if u.op is Ops.CONST: return u.val - if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return u.dtype.const(u.src[0].val) - if u.op is Ops.STACK and all(s.op is Ops.CONST for s in u.src): return tuple(s.val for s in u.src) - return None - -def fold_const_alu(a:UOp) -> UOp|None: - vals = [const_arg(s) for s in a.src] - return None if any(v is None for v in vals) else a.const_like(exec_alu(a.op, a.dtype, vals, False)) +# no truncate: ints stay mathematical past the fold (emission truncates); floats re-round in the mint +def fold_const_alu(a:UOp) -> UOp: return a.const_like(exec_alu(a.op, a.dtype, [const_arg(s) for s in a.src], False)) def _quotient_base(q:UOp, base:UOp, div:int) -> UOp|None: # the B with q == B//div and B%div == base%div, or None. only such congruence is needed to recombine, and canonicalization @@ -71,6 +62,12 @@ def fold_add_divmod_recombine(x:UOp) -> UOp|None: # this needs to be before symbolic so that 0*something_that_might_be_invalid doesnt become 0 invalid_pat = UPat(Ops.CONST, arg=Invalid, name="i") invalid_gate = UPat.var("cond").where(UPat.var("x"), invalid_pat) + +# the two const spellings: Invalid carries no width, so it rides bare inside either +bare_const = UPat.any(UPat(Ops.CONST), UPat(Ops.STACK, src=UPat(Ops.CONST))) +casted_const = UPat.any(p:=UPat(Ops.CAST, src=(UPat(Ops.CONST),)), UPat(Ops.STACK, src=UPat.any(p, UPat(Ops.CONST, arg=Invalid)))) +def const_arg(u:UOp): + return tuple(const_arg(s) for s in u.src) if u.op is Ops.STACK else u.val pm_data_invalid = PatternMatcher([ (invalid_pat.broadcast(), lambda i: i), (UPat(GroupOp.Unary|{Ops.CAST, Ops.BITCAST}, src=(invalid_pat,)), lambda i: i), @@ -142,13 +139,19 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) # ** constant folding ** - # a CAST to a concrete dtype over a CONST is a value conversion: evaluate it once, at the width the CAST states + # a CAST to a concrete dtype over a CONST is a value conversion: evaluate it once, at the CAST's dtype # TODO: delete this once CONST has no dtype (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val)), - (UPat(GroupOp.Unary, src=(const_folding_pat,), name="a"), fold_const_alu), + # collapse committed const conversions when the target has a native constant format. fmt-less targets are emulated and would re-expand this pair. + (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat(Ops.CAST, dtypes.all, src=(UPat(Ops.CONST, name="c"),)),)), + lambda root,c: root.const_like(c.val) if root.dtype.fmt is not None else None), + # one rule per spelling: bare has no width, a pair evaluates at its stated width, mixed commits to the promotion # NOTE: THREEFRY(const,const) folds via its decomposition - (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(const_folding_pat,)*2, name="a"), fold_const_alu), - (UPat(GroupOp.Ternary, src=(const_folding_pat,)*3, name="a"), fold_const_alu), + (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=bare_const, name="a"), fold_const_alu), + (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=casted_const, name="a"), fold_const_alu), + (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=[casted_const, bare_const], name="a"), lambda a: + a.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in a.src)) + if (dt:=promo_dtype(a.src)) not in dtypes.weaks else None), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), (UPat.var('x', dtype=dtypes.bool) + UPat.var('y', dtype=dtypes.bool), lambda x,y: x|y), @@ -166,7 +169,9 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: and isinstance(x.val, float) and (math.isnan(x.val) or math.isinf(x.val)) else 0)), # *** cast/bitcast *** (UPat((Ops.CAST, Ops.BITCAST), name="root"), lambda root: root.src[0] if root.dtype == root.src[0].dtype else None), - (UPat(Ops.BITCAST, name="root", src=(UPat.cvar("c"),)), fold_bitcast), + # a BITCAST reads its operand at the width it states, so a weak const is nonsense here: the bare arm is bool only + (UPat(Ops.BITCAST, name="root", src=(UPat.any(UPat(Ops.CONST, dtypes.bool, name="c"), UPat(Ops.CAST, src=(UPat(Ops.CONST),), name="c")),)), + fold_bitcast), # b.cast(a).cast(b) -> b if a preserves all values in b (UPat.var('x').cast(name="a").cast(name="b"), lambda x,a,b: x if x.dtype == b.dtype and can_lossless_cast(b.dtype, a.dtype) else None), # bitcast twice @@ -292,8 +297,9 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: # cast/long folding # if the intermediate cast doesnt narrow we can do it in one cast (UPat.var('x').cast(name="a").cast(name="b"), lambda x,a,b: x.cast(b.dtype) if can_lossless_cast(x.dtype, a.dtype) else None), + # commit_weak, not .cast: a weak b.dtype is not a const spelling, and a CAST(weakfloat, CONST) reaches no commit round (UPat.var('x', dtypes.ints+(dtypes.weakint,)).cast(dtypes.ints+(dtypes.weakint,), name="a").cast(name="b"), - lambda x,a,b: x.cast(b.dtype) if a.dtype.min<=x.vmin and x.vmax<=a.dtype.max else None), + lambda x,a,b: commit_weak(x, b.dtype) if a.dtype.min<=x.vmin and x.vmax<=a.dtype.max else None), # try to do math in int instead of long, keep weak const weak (UPat(GroupOp.Binary, src=(UPat.var("x", (dtypes.long, dtypes.weakint)), UPat.var("y", (dtypes.long, dtypes.weakint))), name="u"), lambda u,x,y: (UOp.const(x.val) if x.op is Ops.CONST else x.cast(dtypes.int)).alu(u.op, @@ -306,7 +312,8 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: else y.src for y in x.src[1:]]))))), # after/end with 1 src is just src[0] (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), -])+div_and_mod_symbolic + # the rules above key on bare CONSTs, so a redundantly committed const has to be uncast in the same fixpoint +])+div_and_mod_symbolic+pm_uncast_const # ******** we take a small aside to "simplify_valid" to rewrite valids ******** diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index b59cd79967321..ccf51c644d7df 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -1,19 +1,32 @@ from dataclasses import replace from tinygrad.dtype import dtypes, DType, AddrSpace, Invalid, least_upper_dtype, strong_dtype, weak_dtype from tinygrad.helpers import unwrap -from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, graph_rewrite, dtype_from_uop +from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, dtype_from_uop, promo_dtype def default_dtype(u:UOp): if u.dtype is dtypes.weakfloat: return dtypes.default_float return dtypes.long if u.overflows(dtypes.int32) else dtypes.int def commit_weak(s:UOp, dt:DType) -> UOp: - # a CONST commits directly at dt (the value stays mathematical, emission truncates), a non-const src takes the cast + # a CONST re-mints, never takes a cast: at bool/weakint/weakfloat a CAST would be a second spelling of one const return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) -def commit_srcs_at(u:UOp, dt:DType) -> UOp: +# the decomps and float emulation commit bare consts at a dtype another src already states +def commit_weak_consts(u:UOp, dt:DType|None) -> UOp|None: + return None if dt is None else u.replace(src=tuple(commit_weak(s, dt) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in u.src)) + +# the concrete dtypes u commits its srcs at: the operands' meet and u's own derived dtype, None if either is weak +def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: + if u.op not in GroupOp.Broadcastable or (meet:=promo_dtype(src)) in dtypes.weaks \ + or (result:=unwrap(dtype_from_uop(u.op, src, u.arg))) in dtypes.weaks: return None + return meet, result + +def commit_srcs_at(u:UOp, dt:DType) -> UOp|None: # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too - return u.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) + dts = derived_dtypes(u, u.src) + ret = u.replace(dtype=None, src=tuple(UOp.const(dt.const(s.val)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is not None else + commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) + return None if ret is u else ret def commit_weak_srcs(u:UOp) -> UOp|None: if not any(s.dtype in dtypes.weaks for s in u.src) or (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None @@ -21,62 +34,71 @@ def commit_weak_srcs(u:UOp) -> UOp|None: # a concrete CAST over a weak node states the width the value will live at. that width is a floor, never a narrowing def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: + # only within the kind: an int cast of a weakfloat node is a value conversion, not a statement about the node's width if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - return commit_srcs_at(u, least_upper_dtype(c.dtype, default_dtype(u))).cast(c.dtype) + return None if (ret:=commit_srcs_at(u, least_upper_dtype(c.dtype, default_dtype(u)))) is None else ret.cast(c.dtype) -pm_cast_weak = PatternMatcher([ - (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), - (UPat(Ops.CAST, name="c", src=(UPat(Ops.CONST, dtype=dtypes.weaks, name="u"),)), lambda c,u: commit_weak(u, c.dtype)), -]) - -# runs in index lowering and in the decomps: a rule that mints a weak const commits it in the same rewrite, so none reaches the renderer +# rides every round that can mint a weak const, and must reach fixpoint before pm_lower_weak below defaults one pm_commit_weak = PatternMatcher([ (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), - # demand from the destination: a STORE's weak value commits at the destination's dtype (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), + # no CONST arm: a concrete CAST over a weak CONST is already committed, minted that way by UOp.const + (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), ]) -# A weakfloat Unary (sin/exp2/...) must resolve here, before the transcendental decomposition. +# consumers absorb the weak CAST off their srcs and default underivable consts; dtype-producing ops settle here. +# a weakfloat Unary (sin/exp2/...) must resolve before the transcendental decomposition. _lower_weak_ops = GroupOp.Binary|GroupOp.Unary|{Ops.WHERE, Ops.RANGE, Ops.STACK, Ops.SPECIAL} def lower_weak_node(u:UOp) -> UOp|None: + if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const, not a consumer src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) + dts = derived_dtypes(u, src) + src = tuple(commit_weak(s, default_dtype(s)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is None else s + for s in src) start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification - if src == u.src or any(s.dtype in dtypes.weaks for s in src[start:]): return None - # Binary can widen from the bounds, all other nodes derive from the lowered sources. - dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary - else unwrap(dtype_from_uop(u.op, src, u.arg))) - return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid else commit_weak(s, dt) for s in src[start:])).cast(u.dtype) + # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait + if u.op in _lower_weak_ops and src != u.src and not any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): + dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary + else unwrap(dtype_from_uop(u.op, src, u.arg))) + return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) + for s in src[start:])).cast(u.dtype) + return None if src == u.src else u.replace(dtype=None, src=src) pm_lower_weak = PatternMatcher([ - (UPat(Ops.CONST, dtype=dtypes.weaks, name="u"), lambda u: UOp.const(u.val, default_dtype(u)).cast(u.dtype)), + # a gated long index into a small buffer narrows; its out-of-gate value is discarded + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("gate").where(UPat.var("idx", dtypes.long), UPat(Ops.CONST, arg=Invalid))), + allow_any_len=True, name="u"), + lambda u,buf,gate,idx: u.replace(src=(buf, idx.cast(dtypes.int).valid(gate))+u.src[2:]) if buf.max_numel()-1 <= dtypes.int32.max else None), # two stacked weak casts are two kind conversions: each resolves at its own kind's default - # a SINGLE weak cast is never rewritten here, each consumer absorbs it on its own edge (see lower_weak_srcs) (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), lambda u,x: x.cast(default_dtype(u.src[0])).cast(default_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), - (UPat(_lower_weak_ops, name="u"), lower_weak_node), + (UPat(GroupOp.All, name="u"), lower_weak_node), ]) -def lower_weak_srcs(ctx:dict[UOp, UOp]|None, u:UOp) -> UOp|None: - if ctx is None: ctx = {} - def lower(s:UOp) -> UOp: - if (r:=ctx.get(s)) is None: - r = graph_rewrite(s, pm_lower_weak) - # the consumer absorbs the cast on its own edge - ctx[s] = r = r.src[0] if r.op is Ops.CAST and r.dtype in dtypes.weaks else r - return r - # a comparison demands a common operand width: lower it whole so the Binary rule unifies its operands - ret = lower(u) if u.op in GroupOp.Comparison else u.replace(src=tuple(lower(s) if s.dtype in dtypes.weaks else s for s in u.src)) - return None if ret is u else ret +# drop the CAST off a committed const where the consumer re-derives it anyway, so bare-CONST rules keep matching. +# the drop must change nothing the consumer derives: neither the operands' meet nor the node's own dtype +def uncast_const(u:UOp) -> UOp|None: + # a weak CAST over a const is not a commit, it is still resolving + src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype not in dtypes.weaks and s.src[0].op is Ops.CONST + and s.src[0].dtype in dtypes.weaks else s for s in u.src) + if src == u.src or (dts:=derived_dtypes(u, src)) is None or dts[0] != promo_dtype(u.src) or dts[1] is not u.dtype: return None + return u.replace(src=src) -pm_lower_index_dtype = pm_commit_weak+pm_cast_weak+PatternMatcher([ - (UPat(GroupOp.All, name="u"), - lambda ctx,u: lower_weak_srcs(ctx, u) if u.dtype not in dtypes.weaks and any(s.dtype in dtypes.weaks for s in u.src) else None), - # a valid index into an n-element buffer lives in [0,n): a gated long index narrows when n-1 fits int32 (out-of-gate wraps, discarded) - # TODO: more generic - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("gate").where(UPat.var("idx", dtypes.long), UPat(Ops.CONST, arg=Invalid))), - allow_any_len=True, name="u"), - lambda u,buf,gate,idx: u.replace(src=(buf, idx.cast(dtypes.int).valid(gate))+u.src[2:]) if buf.max_numel()-1 <= dtypes.int32.max else None), -]) +pm_uncast_const = PatternMatcher([(UPat(GroupOp.Broadcastable, name="u"), uncast_const)]) + +def cast_const(u:UOp, s:UOp) -> UOp: + if s.op is not Ops.CONST or s.is_invalid: return s # Invalid never commits + # bool is the one strong bare dtype: cconst, since .cast(bool) would fold at construction + if s.dtype is dtypes.bool: return UOp.cconst(s.val, s.dtype) + # commit at the dtype its consumer derives; where nothing does, commit_weak is the identity and spec_program rejects it + return commit_weak(s, dts[0]) if (dts:=derived_dtypes(u, u.src)) is not None else s + +# commit every remaining bare const, keyed on the consumer: "bare" is a property of the edge +def cast_consts(u:UOp) -> UOp|None: + if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const's CONST is its value, not an edge + return None if (src:=tuple(cast_const(u, s) for s in u.src)) == u.src else u.replace(src=src) + +pm_cast_const = PatternMatcher([(UPat(GroupOp.All, name="u"), cast_consts)]) From 65ca68567e4500cbdf6108f57aed5c38573f2ee6 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 21:37:14 -0400 Subject: [PATCH 193/306] const cleanups [PR] (#17719) --- tinygrad/codegen/opt/postrange.py | 2 +- tinygrad/llm/kernels/amd.py | 4 ++-- tinygrad/runtime/ops_dsp.py | 8 -------- tinygrad/uop/render.py | 8 ++++---- tinygrad/uop/validate.py | 2 +- 5 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index aa2e562043c22..cda176f7e73a8 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -193,7 +193,7 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): for b in self.bufs: if rng in (i:=b.src[1].get_idx()).backward_slice_with_self: nb = b.replace(src=(b.src[0], i.valid(valid&b.src[1].get_valid()))) - replaces[b] = nb if b in store_targets else valid.where(nb, UOp.const(Invalid, b.dtype)) + replaces[b] = nb if b in store_targets else valid.where(nb, UOp.const(Invalid)) self.ast = self.ast.substitute(replaces, f"padto {rng.arg[:-1]} {opt.arg}") elif opt.op is OptOps.SWAP: try: diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py index 65c0b68158e9f..b206c3415fa5d 100644 --- a/tinygrad/llm/kernels/amd.py +++ b/tinygrad/llm/kernels/amd.py @@ -86,8 +86,8 @@ def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: assert ptr.op is Ops.INDEX if lanes is None: return UOp(Ops.CUSTOMI, ptr.dtype, (ptr,), arg="__builtin_nontemporal_load({0})") buf, coords = ptr.src[0], ptr.src[1:] - idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0, dtypes.weakint)) - return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes, dtypes.weakint))).load(dtype=ptr.dtype) + idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0)) + return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load(dtype=ptr.dtype) def _load_byte(raw:UOp, base:UOp, offset:UOp) -> UOp: return (raw[base + offset//4] >> ((offset&3)*8).cast(dtypes.uint32)) & 255 def _half(value:UOp) -> UOp: return value.cast(dtypes.uint16).bitcast(dtypes.float16).float() diff --git a/tinygrad/runtime/ops_dsp.py b/tinygrad/runtime/ops_dsp.py index 0f001ed79cefd..e487f666e7ea6 100644 --- a/tinygrad/runtime/ops_dsp.py +++ b/tinygrad/runtime/ops_dsp.py @@ -9,18 +9,10 @@ from tinygrad.runtime.autogen import libc, qcom_dsp if getenv("IOCTL"): import extra.dsp.run # noqa: F401 # pylint: disable=unused-import -from tinygrad.uop.ops import PatternMatcher, UPat - -# NOTE: this just increases readability of the generated code -dsp_string = PatternMatcher([ - (UPat(Ops.CONST, (dtypes.int8, dtypes.uint8), name="x"), lambda ctx,x: str(x.val)), -]) - class DSPRenderer(ClangRenderer): has_threads = False buffer_suffix = " restrict __attribute__((align_value(128)))" kernel_typedef = "__attribute__((noinline)) void" - string_rewrite = dsp_string+ClangRenderer.string_rewrite type_map = { **ClangRenderer.type_map, dtypes.uint64: "unsigned long long", dtypes.int64: "long long" } code_for_op = {k:v for k,v in ClangRenderer.code_for_op.items() if k != Ops.SQRT} diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 1028f524c4238..7f070a06a7251 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -40,7 +40,7 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: (UPat(Ops.RANGE, name="x"), lambda x: f"r{range_str(x)}"), (UPat(Ops.CONST, name="x"), lambda x: str(x.val)), # CAST states the width, the weak CONST carries the value - (UPat.cvar("c", dtypes.weaks+(dtypes.bool,)).cast(), lambda c: str(c.val)), + (UPat.cvar("c").cast(), lambda c: str(c.val)), (UPat(Ops.CAST, name="x"), lambda ctx,x: f"({str(x.dtype)[7:]})({ctx[x.src[0]]})"), (UPat(Ops.NEG, name="x"), lambda ctx,x: f"(-{ctx[x.src[0]]})"), (UPat(Ops.RECIPROCAL, name="x"), lambda ctx,x: f"(1/{ctx[x.src[0]]})"), @@ -83,9 +83,9 @@ def render_marg(ctx,x:UOp): sugar = {Ops.SINK, Ops.END, Ops.STORE, Ops.LOAD, Ops.SQRT, Ops.INDEX, Ops.REDUCE, Ops.AFTER, Ops.THREEFRY, Ops.RECIPROCAL, Ops.EXP2, Ops.LOG2, Ops.SIN, Ops.CONTIGUOUS, Ops.BARRIER, Ops.DETACH} pm_pyrender_extra = PatternMatcher([ - (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val}, {x.dtype})"), + (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val})"), (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})" if x.dtype != x.src[0].dtype else None), - (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)}, dtype={x.dtype})"), + (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)})"), (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: f"UOp.new_buffer({repr(x.arg.device)}, {x.max_numel()}, {x.dtype}, {x.arg.slot})" if isinstance(x.arg, ParamArg) and x.addrspace is AddrSpace.GLOBAL else None), @@ -95,7 +95,7 @@ def render_marg(ctx,x:UOp): # NOTE: range has srcs sometimes after control flow (UPat(Ops.RANGE, src=(UPat(Ops.CONST, name="c"),), allow_any_len=True, name="x"), lambda ctx,x,c: "UOp.range("+', '.join([str(c.val)] + [repr(y) for y in x.arg])+ - (f', src={srcs(ctx, x.src[1:])}' if len(x.src) > 1 else '')+(', dtype='+str(x.dtype) if x.dtype is not dtypes.weakint else '')+")"), + (f', src={srcs(ctx, x.src[1:])}' if len(x.src) > 1 else '')+")"), # TODO: index shouldn't mismatch dtype (UPat(Ops.INDEX, src=(UPat(), UPat()), allow_any_len=True, name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.index({ctx[x.src[1]]}, "+''.join([f"{ctx[xx]}, " for xx in x.src[2:]])+ diff --git a/tinygrad/uop/validate.py b/tinygrad/uop/validate.py index 05c8cd3b0157c..a408c28e4c402 100644 --- a/tinygrad/uop/validate.py +++ b/tinygrad/uop/validate.py @@ -45,7 +45,7 @@ def create_bounded(name:str, vmin:int, vmax:int, z3ctx:z3.Context) -> tuple[z3.A (UPat((Ops.LOAD, Ops.INDEX), dtypes.bool), lambda ctx: (z3.Bool(f"load{len(ctx[1])}", ctx=ctx[0]), None)), # constants (UPat(Ops.CONST, arg=Invalid), lambda ctx: (z3.Int("Invalid", ctx=ctx[0]), None)), - (UPat(Ops.CONST, dtypes.ints+(dtypes.weakint,), name="x"), lambda x,ctx: (z3.IntVal(x.val, ctx=ctx[0]), None)), + (UPat(Ops.CONST, dtypes.weakint, name="x"), lambda x,ctx: (z3.IntVal(x.val, ctx=ctx[0]), None)), (UPat(Ops.CONST, dtypes.bool, name="x"), lambda x,ctx: (z3.BoolVal(x.val, ctx=ctx[0]), None)), # casts from floats create new variables (UPat(Ops.CAST, dtypes.ints+(dtypes.weakint,), src=(UPat(dtype=dtypes.floats),), name="x"), lambda x,ctx: From 1d694dd7008d4d4ac60d54b59bf89374a31ac512 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 24 Aug 2026 22:03:22 -0400 Subject: [PATCH 194/306] remove more explicitly set dtype [PR] (#17721) --- tinygrad/codegen/opt/tc.py | 1 - tinygrad/renderer/isa/x86.py | 4 ++-- tinygrad/renderer/ptx.py | 4 ++-- tinygrad/renderer/wgsl.py | 2 +- tinygrad/runtime/support/usb.py | 2 +- tinygrad/schedule/multi.py | 4 ++-- tinygrad/uop/ops.py | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tinygrad/codegen/opt/tc.py b/tinygrad/codegen/opt/tc.py index 6d13624a1dae2..507a619a3fffc 100644 --- a/tinygrad/codegen/opt/tc.py +++ b/tinygrad/codegen/opt/tc.py @@ -153,7 +153,6 @@ def get_amd(arch): return {"gfx942": amd_cdna3, "gfx950": amd_cdna4, "gfx1200": pm_validate_wmma_rdna4 = PatternMatcher([ (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( - dtype=dtypes.uint16, src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), (UPat(Ops.WMMA, name="x", dtype=dtypes.float), diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 60b90290d8df4..f90c8d5686919 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -143,8 +143,8 @@ class X86GroupOp: (UPat.var("m").where(UPat.var("a", (dtypes.bool,)+dtypes.int8s), UPat.var("b")), lambda m,a,b: m.where(a.cast(dtypes.int16), b.cast(dtypes.int16)).cast(a.dtype) if a.max_numel() == 1 else None), # float16 alus are done in float32 - (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, dtypes.float, - tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), + (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, + src=tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), (UPat(GroupOp.Comparison, src=(UPat.var("a", dtypes.float16), UPat.var("b")), name="x"), lambda x,a,b: UOp(x.op, src=(a.cast(dtypes.float32), b.cast(dtypes.float32))).cast(x.dtype)), # no cmpne for packed ints, y != x => !(y==x) diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index d46a68b3a3b6a..120fa39021453 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -52,8 +52,8 @@ def render_val(x, dtype): (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), lambda x,idx: UOp(x.op, src=(x.src[0], x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), # ptx shr and shl instructions require y to be uint - (UPat.var("x") << UPat.var("y"), lambda x,y: UOp(Ops.SHL, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), - (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + (UPat.var("x") << UPat.var("y"), lambda x,y: UOp(Ops.SHL, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), ]) def mem_type(x:UOp) -> str: return 'shared' if x.addrspace == AddrSpace.LOCAL else 'global' diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index 9c7fd26e94d46..b1b26d4666edd 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -46,7 +46,7 @@ def is_nan(a): (UPat(Ops.LOAD, name="l"), lambda l: packed_load(l) if is_packed(l) else None), (UPat(Ops.STORE, name="s"), lambda s: packed_store(s) if is_packed(s) else None), (UPat.var("a") << UPat.var("b"),lambda a,b:(a.bitcast(dtypes.uint32)<> UPat.var("y"), lambda x,y: UOp(Ops.SHR, x.dtype, (x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), + (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), # fix nan check: 'a != a -> is_nan()'. the decomp rewrites (a != a).logical_not() to CMPEQ, so match both forms (UPat.var("a", dtypes.floats) != UPat.var("a"), is_nan), (UPat.var("a", dtypes.floats).alu(Ops.CMPEQ, UPat.var("a")), lambda a: is_nan(a).ne(True)), diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index dde7a97201355..b2330470aa6dc 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -297,7 +297,7 @@ def usb_stage_copy(dst:UOp, src:UOp) -> UOp|None: sram.copy_to_device(d.device).call(d, sram)] else: pad = UOp.new_buffer("CPU", round_up(nb, 512), dtypes.uint8)[0:nb] - submit = make_submit(UOp(Ops.CALL, dtypes.void, (UOp(Ops.COPY, dtypes.void, ()), sram, s)), devs=devs, queue="COPY:0") + submit = make_submit(UOp(Ops.CALL, src=(UOp(Ops.COPY, dtypes.void, ()), sram, s)), devs=devs, queue="COPY:0") pull = usb_bulk(devs, (submit,), 0x81, pad.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) ops += [UOp.custom_function("hcq", pull.sink()).call(pad, sram, s, name="hcq_copyout", aux=HCQInfo(devs)), pad.copy_to_device("CPU").call(d, pad)] diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index e928130d9b640..45d29d260b12f 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -259,7 +259,7 @@ def store_dest_multi(root:UOp, multi:UOp): # (scalars arrive EXPANDed to the full shape by UOp.store's const_like, so they sub-view like everything else) srcs = [multi.src[0]] + [x.src[0] if x.op is Ops.UNSHARD else shard_subview(x, multi) if tuple(x.shape) == tuple(multi.shape) else x for x in root.src[1:]] - return UOp(root.op, root.dtype, tuple(srcs), root.arg) + return UOp(root.op, src=tuple(srcs), arg=root.arg) def passthrough_multi(root:UOp, multi:UOp): new_src = (multi.src[0],)+tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src[1:]) @@ -306,7 +306,7 @@ def param_to_multi(p:UOp): (UPat((Ops.CALL, Ops.FUNCTION, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — FUNCTION is handled by rewrite_into_function (UPat(Ops.CALL, dtype=dtypes.void, name="root", custom_early_reject=set([Ops.UNSHARD])), lambda root: - UOp(root.op, root.dtype, tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), root.arg)), + UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg)), (UPat((Ops.CAST, Ops.BITCAST, Ops.CONTIGUOUS, Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), passthrough_multi), # STORE of a sharded value into an unsharded dest (e.g. a fragment into a full output tile) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index f66e8a32356c3..41358fa5cb705 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -623,7 +623,7 @@ def range(end:sint, axis_id, axis_type=AxisType.WEAK, *arg, dtype=dtypes.weakint @staticmethod def loop(axis_id:int, *arg): return UOp(Ops.RANGE, src=(UOp(Ops.NOOP),), arg=(axis_id, AxisType.WEAK)+arg) @staticmethod - def special(end:sint, name:str, dtype=dtypes.weakint): return UOp(Ops.SPECIAL, src=(sint_to_uop(end, dtype),), arg=name) + def special(end:sint, name:str): return UOp(Ops.SPECIAL, src=(sint_to_uop(end),), arg=name) @staticmethod def wmma(a:UOp, b:UOp, acc:UOp, dims:tuple[int, int, int], device:str, threads:int, tc_upcast_axes=None): # dtype_in is stored in the arg (not derived from src[0].dtype) because bitcast rewrites change src dtypes From a5678317c2a7676c1136557d2a0080f0efd4a286 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:12:13 -0700 Subject: [PATCH 195/306] split rangeify to prepare.py (#17722) --- test/null/test_uop_graph.py | 4 +- tinygrad/codegen/__init__.py | 2 +- tinygrad/schedule/__init__.py | 5 +- tinygrad/schedule/prepare.py | 205 +++++++++++++++++++++++++++++++++ tinygrad/schedule/rangeify.py | 207 +--------------------------------- tinygrad/uop/ops.py | 2 +- 6 files changed, 218 insertions(+), 207 deletions(-) create mode 100644 tinygrad/schedule/prepare.py diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 6f9464b8c915d..0c8616bce66c9 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -429,7 +429,7 @@ def test_reduce_shapeless_const_unroll(self): class TestMovementOps(unittest.TestCase): def test_pm_mops_partial_reshape_index_removes_reshape(self): - from tinygrad.schedule.rangeify import pm_mops + from tinygrad.schedule.prepare import pm_mops src = UOp.param(0, dtypes.float, shape=(32, 4)) r0, r1 = UOp.range(4, 0), UOp.range(8, 1) result = graph_rewrite(src.reshape((4, 8, 4)).index(r0, r1), pm_mops, name="test") @@ -439,7 +439,7 @@ def test_pm_mops_partial_reshape_index_removes_reshape(self): self.assertNotIn(Ops.RESHAPE, [u.op for u in result.toposort()]) def test_pm_mops_partial_reshape_index_suffix_mismatch_does_nothing(self): - from tinygrad.schedule.rangeify import pm_mops + from tinygrad.schedule.prepare import pm_mops src = UOp.param(0, dtypes.float, shape=(2, 6)) result = graph_rewrite(src.reshape((2, 3, 2)).index(UOp.range(2, 0)), pm_mops, name="test") self.assertEqual(result.op, Ops.INDEX) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 15e9edbd9fb98..5ffbcf2e846ca 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -22,7 +22,7 @@ from tinygrad.codegen.late.gater import pm_move_gates_from_index from tinygrad.codegen.simplify import pm_simplify_ranges, pm_flatten_range, pm_split_ranges, pm_load_collapse, pm_reduce_unparented from tinygrad.schedule.multi import multi_pm -from tinygrad.schedule.rangeify import pm_mops +from tinygrad.schedule.prepare import pm_mops from tinygrad.codegen.late.linearizer import CFGContext, pm_split_ends, pm_add_control_flow, linearize from tinygrad.codegen.late.regalloc import LinearScanRegallocContext, pm_regalloc_rewrite from tinygrad.codegen.late.coalesce import memory_coalescing, pm_simplify_add_image diff --git a/tinygrad/schedule/__init__.py b/tinygrad/schedule/__init__.py index f00ec3f7c158e..9608e032fa4f9 100644 --- a/tinygrad/schedule/__init__.py +++ b/tinygrad/schedule/__init__.py @@ -80,6 +80,7 @@ def create_schedule(sched_sink:UOp) -> UOp: from tinygrad.schedule.memory import memory_plan_rewrite from tinygrad.engine.realize import capturing, pm_flatten_linear +from tinygrad.schedule.prepare import prepare_rangeify from tinygrad.schedule.rangeify import get_kernel_graph from tinygrad.helpers import CAPTURING from tinygrad.uop.ops import PatternMatcher, UPat, ParamArg @@ -123,7 +124,7 @@ def lower_sink_to_linear(function:UOp) -> UOp|None: if not SCACHE or (sc_ret:=schedule_cache.get(cache_key, None)) is None: if SPEC: type_verify(function, spec_tensor) # support recursive CALLs - linear = create_schedule(get_kernel_graph(function)) + linear = create_schedule(get_kernel_graph(prepare_rangeify(function))) if SCACHE: schedule_cache[cache_key] = linear else: # schedule cache hit @@ -156,7 +157,7 @@ def simplify_copy_kernel(call:UOp, ast:UOp, dst:UOp, src:UOp): # NOTE: this is a codegen for SDMA devices if dst.device == src.device and not (isinstance(dst.device, str) and dst.device.startswith("DISK")): return None from tinygrad.codegen.simplify import pm_flatten_range, pm_simplify_ranges - from tinygrad.schedule.rangeify import pm_mops + from tinygrad.schedule.prepare import pm_mops from tinygrad.uop.symbolic import sym sink = graph_rewrite(ast, sym+pm_mops+pm_flatten_range+pm_simplify_ranges, ctx={}, name="simplify ranges in copy") return call.replace(src=(sink,) + call.src[1:]) diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py new file mode 100644 index 0000000000000..7fac55277d8da --- /dev/null +++ b/tinygrad/schedule/prepare.py @@ -0,0 +1,205 @@ +import itertools +from tinygrad.dtype import dtypes, to_dtype +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp +from tinygrad.uop.ops import graph_rewrite, rewrite_group, shape_to_shape_arg, ParamArg, identity_element +from tinygrad.uop.movement import mop_cleanup +from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort +from tinygrad.schedule.indexing import apply_movement_op +from tinygrad.schedule.allreduce import create_allreduce_function +from tinygrad.schedule.multi import multi_pm + +def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): + if (x:=src).op is Ops.CAST and x.dtype == dtypes.half and FLOAT16: x, after = x.src[0], after.cast(dtypes.float) + while True: + if x.op is Ops.PERMUTE: x, after = x.src[0], after.permute(argsort(x.marg)) + elif x.op is Ops.RESHAPE: x, after = x.src[0], after.reshape(x.src[0].shape) + elif x.op is Ops.WHERE and x.src[2].base.is_invalid and x.src[1].op is Ops.PAD: + x, after = x.src[1].src[0], after.shrink(tuple((o, s+o) for (o,_),s in zip(x.src[1].marg, x.src[1].src[0].shape))) + else: break + ctx[x] = after + +# *** fold moved AFTERs (hack for openpilot) *** +pm_fold_moved_after = PatternMatcher([ + (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(), UPat((*GroupOp.Movement,Ops.CAST,Ops.WHERE), name="src")))), name="after"), found_after), + # replace ALU sources with AFTER versions found above + (UPat(GroupOp.ALU, name="alu"), lambda ctx,alu: alu.replace(src=new_src) if (new_src:=tuple(ctx.get(s, s) for s in alu.src)) != alu.src else None), +]) + +# movement op on INDEX as a PatternMatcher +def _mop_index(r:UOp, idx:UOp): + idxs = idx.src[1:] + if len(idxs) == len(r.shape): + return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), arg=idx.arg) + if r.op is Ops.RESHAPE: + src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) + if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: + if src_prefix == 0: return r.src[0] if r.src[0].dtype == idx.dtype else None + ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), arg=idx.arg) + return ret if ret.shape == idx.shape else None + +pm_mops = PatternMatcher([ + # handle movement ops on INDEX + (UPat(GroupOp.Movement, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), _mop_index), + # move movement ops and INDEX after AFTER + (UPat(GroupOp.Movement|{Ops.INDEX}, name="r").after(name="a", allow_any_len=True), + lambda r,a: UOp(r.op, src=(a.replace(src=(r.src[0],)+a.src[1:]),)+r.src[1:], arg=r.arg)), + (UPat(GroupOp.Movement, name="r").end(name="a", allow_any_len=True), lambda r,a: a.replace(src=(r.src[0],)+a.src[1:])), +]) + +# ***************** +# 0. do some cleanup rewrites, mostly copied from the old stuff + +def fix_store_hazard(target:UOp, src:UOp): + if (base:=target.base) not in src.toposort(enter_calls=False): return None + # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk + unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) + reaches_base: dict[UOp, bool] = {} + for s in src.toposort(gate=lambda s: s.op is not Ops.CONTIGUOUS): + reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) + if reaches_base[s] and s.op in unsafe and not (s is target and s.op is Ops.SHRINK): return target.store(src.contiguous()) + +def split_reduceop(reduce:UOp, x:UOp): + if prod(reduce.shape) == 0: return None + if not SPLIT_REDUCEOP or not all_int(x.shape) or (prod(x.shape)//prod(reduce.shape))1) else 0 for i,s in enumerate(x.shape)]) + range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP)}, extra_pm=pm_mops).ranges] + is_expanded = [i not in range_nums for i in range(len(x.shape))] + + if not (split_candidates:=[(i,d) for i in range(reduce.arg[1]) + for d in range(min(256,2**getenv("REDUCEOP_SPLIT_SIZE",22)//prod(reduce.shape)),8-1,-1) + if x.shape[i]%d==0 and not is_expanded[i]]): return None + dim_to_split, divisor = split_candidates[0] + splitted_shape = x.shape[:dim_to_split]+(divisor,)+(x.shape[dim_to_split]//divisor,)+x.shape[dim_to_split+1:] + splitted = x.reshape(splitted_shape).permute(tuple([d for d in range(len(splitted_shape)) if d!=dim_to_split]+[dim_to_split])) + if DEBUG >= 3: print(f"split {divisor}: {x.shape} -> {splitted.shape} -> {reduce.shape}") + # reduce original axes, then split + return splitted._rop(reduce.arg[0], tuple(range(reduce.arg[1]))).contiguous()._rop(reduce.arg[0], (len(reduce.shape),)).reshape(reduce.shape) + +pm_gather_params = PatternMatcher([ (UPat(Ops.PARAM, name="p"), lambda ctx, p: ctx.append(p) if p.arg.slot >= 0 else None), ]) +def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: + if c.arg.precompile: return None + params: list[UOp] = [] + graph_rewrite(c.src[0], pm_gather_params, bottom_up=True, ctx=params, name="gather params") + params = sorted(params, key=lambda x: x.arg.slot) + args = c.src[1:] + + # NOTE: this isn't really needed. it's okay if there's unused args in the function + if not allow_param_mismatch: + if [x.arg.slot for x in params] != list(range(len(params))): raise RuntimeError(f"params not in order: {[x.arg.slot for x in params]}") + if len(params) != len(args): raise TypeError(f"expected {len(params)} args, got {len(args)}") + + dict_map = {x:args[x.arg.slot] for x in params} + for i, (p, a) in enumerate(dict_map.items()): + if p.axis != a.axis: raise TypeError(f"arg {i} axis mismatch: expected {p.axis}, got {a.axis}") + if p.max_shape != a.max_shape: raise TypeError(f"arg {i} shape mismatch: expected {p.shape}, got {a.shape}") + if p.dtype != a.dtype: raise TypeError(f"arg {i} dtype mismatch: expected {p.dtype}, got {a.dtype}") + return c.src[0].substitute(dict_map, walk=True) + +# shape-changing bitcast +def expand_bitcast(bc:UOp) -> UOp|None: + x = bc.src[0] + if (ns:=bc.dtype.itemsize) == (os:=x.dtype.itemsize) or (isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS"))): return None + new_uint, tmp = to_dtype(f"uint{8*ns}"), x.bitcast(to_dtype(f"uint{8*os}")) + if ns > os: + tmp = tmp.reshape(x.shape[:-1] + (x.shape[-1]//(rate := ns//os), rate)) + parts = [tmp.shrink((None,)*(len(tmp.shape)-1) + ((i, i+1),)).cast(new_uint)<<8*i*os for i in range(rate)] + return parts[0].usum(*parts[1:]).squeeze(-1).bitcast(bc.dtype) + parts = [tmp>>8*i*ns for i in range(os//ns)] + return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) + +earliest_rewrites = mop_cleanup+PatternMatcher([ + # resolve FUNCTION calls (inline the body) + (UPat(Ops.FUNCTION, name="c"), resolve_function), + + # resolve TUPLE+GETTUPLE + (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), + + # resolve allreduce (must be bottom up) + (UPat(Ops.ALLREDUCE, src=(UPat.var("buf"),), name="red"), create_allreduce_function), + + # split_reduceop + (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), split_reduceop), + + # remove DETACH/CONTIGUOUS_BACKWARD (TODO: this is copied in allocations) + (UPat((Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), name="x"), lambda x: x.src[0]), + + # SINK only ever references the base + (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=tuple(y.unsharded_base for y in x.src))), + + # ** copy rules ** + + # COPY transfers a contiguous range, so materialize a source that's resized (shrink/pad/expand) or reordered (permute/flip) + (UPat(Ops.COPY, src=(UPat(GroupOp.Movement, name="r"),), name="c"), + lambda c,r: c.replace(src=(r.contiguous(),)) if resolve(r.numel() != r.base.numel(), False) or r.contiguous_view_offset() is None else None), + + # copy to same device is a no-op + (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), + + # copy on reshape is reshape on copy + (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), + + # reshaping on STORE can be a NOOP + (UPat(Ops.STORE, src=(UPat(Ops.RESHAPE, src=(UPat.var("dst",),), allow_any_len=True), + UPat(Ops.RESHAPE, src=(UPat.var("src",),), allow_any_len=True))), + lambda dst,src: dst.store(src) if dst.shape == src.shape else None), + + # ** store rules ** + + # fix store hazard (dest is in used in src) by adding contiguous: TestAssign.test_post_flipped_assignment + (UPat(Ops.STORE, src=(UPat(name="target"), UPat(name="src"))), fix_store_hazard), + + # remove two STOREs that store the same thing to the same place: TestSchedule.test_dedup_Assign + (UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1").after(UPat.var("a1").store(UPat.var("src"))), lambda buf,src,a1:a1), + + # store a buffer's own current contents back into itself: TestAssign.test_nested_after_contiguous_store_no_init + (UPat.var("buf").after(UPat.var("buf").store(UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1"))), lambda buf,src,a1:a1), + + # move bitcast from store dest to source: TestAssign.test_assign_bitcast + (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, src=(UPat(name="target"),)), UPat(name="src"))), + lambda target, src: target.store(src.bitcast(target.dtype))), + + (UPat(Ops.BITCAST, name="bc"), expand_bitcast), + + # ** size 0 ** + + # reduce of size 0 is the identity element + (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), + lambda reduce,x: reduce.const_like(identity_element(reduce.arg[0], reduce.dtype)) if 0 in x.shape and 0 not in reduce.shape else None), + # handle size 0 + (UPat(GroupOp.All-{Ops.SINK}, name="x"), lambda x: x.const_like(0).rtag(x.tag) if x._shape is not None and 0 in x.shape else None), +]) + +def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): + input_src = copy.src[0] + if not input_src.has_buffer_identity(after_ok=True): input_src = input_src.contiguous() + input_src = input_src.flatten() + if existing_buf is not None: + # if the existing buffer is not a full buffer, we can't use it + if not existing_buf.has_buffer_identity(after_ok=True): return None + # if there's already a buffer, we just use it + return existing_buf.flatten().store(input_src) + # create the output buffer + buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg(input_src.max_shape),), arg=ParamArg(next(ctx), copy.dtype, device=copy.device)) + # reshape back to input + return buf.after(buf.store(input_src)).reshape(copy.shape) + +pm_copy_to_store = PatternMatcher([ + (UPat(name="existing_buf").store(UPat(Ops.COPY, name="copy")), convert_copy_to_store), + (UPat(Ops.COPY, name="copy"), convert_copy_to_store), +]) + +@rewrite_group(new_ctx=False) +def prepare_rangeify(sink:UOp) -> UOp: + # prepare for rangeify + tsink = graph_rewrite(sink, multi_pm, name="multi_pm") + if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") + tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") + tsink = graph_rewrite(tsink, pm_copy_to_store, ctx=itertools.count(0), bottom_up=True, name="convert copy to store") + return tsink diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 89fc1b622c4ef..0b8b152d11005 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -1,191 +1,21 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools -from tinygrad.dtype import dtypes, AddrSpace, Invalid, to_dtype, strong_dtype +from tinygrad.dtype import dtypes, AddrSpace, Invalid, strong_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg, shape_to_shape_arg -from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group, identity_element +from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic -from tinygrad.uop.movement import mop_cleanup -from tinygrad.helpers import prod, getenv, dedup, all_int, DEBUG, SPLIT_REDUCEOP, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC -from tinygrad.helpers import PCONTIG, FLOAT16, OPENPILOT_HACKS, argsort, partition, get_single_element +from tinygrad.helpers import prod, dedup, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC +from tinygrad.helpers import PCONTIG, partition, get_single_element from tinygrad.codegen.simplify import pm_flatten_range, pm_reduce_simplify from tinygrad.codegen.opt import Opt from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, apply_movement_op -from tinygrad.schedule.multi import multi_pm -from tinygrad.schedule.allreduce import create_allreduce_function +from tinygrad.schedule.prepare import pm_mops # creation can recurse a lot import sys sys.setrecursionlimit(10000) -def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): - if (x:=src).op is Ops.CAST and x.dtype == dtypes.half and FLOAT16: x, after = x.src[0], after.cast(dtypes.float) - while True: - if x.op is Ops.PERMUTE: x, after = x.src[0], after.permute(argsort(x.marg)) - elif x.op is Ops.RESHAPE: x, after = x.src[0], after.reshape(x.src[0].shape) - elif x.op is Ops.WHERE and x.src[2].base.is_invalid and x.src[1].op is Ops.PAD: - x, after = x.src[1].src[0], after.shrink(tuple((o, s+o) for (o,_),s in zip(x.src[1].marg, x.src[1].src[0].shape))) - else: break - ctx[x] = after - -# *** fold moved AFTERs (hack for openpilot) *** -pm_fold_moved_after = PatternMatcher([ - (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE, src=(UPat(), UPat((*GroupOp.Movement,Ops.CAST,Ops.WHERE), name="src")))), name="after"), found_after), - # replace ALU sources with AFTER versions found above - (UPat(GroupOp.ALU, name="alu"), lambda ctx,alu: alu.replace(src=new_src) if (new_src:=tuple(ctx.get(s, s) for s in alu.src)) != alu.src else None), -]) - -# movement op on INDEX as a PatternMatcher -def _mop_index(r:UOp, idx:UOp): - idxs = idx.src[1:] - if len(idxs) == len(r.shape): - return r.src[0].index(*apply_movement_op(r.op, r.src[0].shape, r.marg, idxs), arg=idx.arg) - if r.op is Ops.RESHAPE: - src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) - if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: - if src_prefix == 0: return r.src[0] if r.src[0].dtype == idx.dtype else None - ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), arg=idx.arg) - return ret if ret.shape == idx.shape else None - -pm_mops = PatternMatcher([ - # handle movement ops on INDEX - (UPat(GroupOp.Movement, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), _mop_index), - # move movement ops and INDEX after AFTER - (UPat(GroupOp.Movement|{Ops.INDEX}, name="r").after(name="a", allow_any_len=True), - lambda r,a: UOp(r.op, src=(a.replace(src=(r.src[0],)+a.src[1:]),)+r.src[1:], arg=r.arg)), - (UPat(GroupOp.Movement, name="r").end(name="a", allow_any_len=True), lambda r,a: a.replace(src=(r.src[0],)+a.src[1:])), -]) - -# ***************** -# 0. do some cleanup rewrites, mostly copied from the old stuff - -def fix_store_hazard(target:UOp, src:UOp): - if (base:=target.base) not in src.toposort(enter_calls=False): return None - # PERMUTE and FLIP reorder indices, SHRINK can have overlapping regions when dest is also shrunk - unsafe = {Ops.PERMUTE, Ops.FLIP} | ({Ops.SHRINK} if target.op_in_backward_slice_with_self(Ops.SHRINK) else set()) - reaches_base: dict[UOp, bool] = {} - for s in src.toposort(gate=lambda s: s.op is not Ops.CONTIGUOUS): - reaches_base[s] = s is base or any(reaches_base.get(c) for c in s.src) - if reaches_base[s] and s.op in unsafe and not (s is target and s.op is Ops.SHRINK): return target.store(src.contiguous()) - -def split_reduceop(reduce:UOp, x:UOp): - if prod(reduce.shape) == 0: return None - if not SPLIT_REDUCEOP or not all_int(x.shape) or (prod(x.shape)//prod(reduce.shape))1) else 0 for i,s in enumerate(x.shape)]) - range_nums = [y.arg[0] for y in indexed.substitute({x.base:UOp(Ops.NOOP)}, extra_pm=pm_mops).ranges] - is_expanded = [i not in range_nums for i in range(len(x.shape))] - - if not (split_candidates:=[(i,d) for i in range(reduce.arg[1]) - for d in range(min(256,2**getenv("REDUCEOP_SPLIT_SIZE",22)//prod(reduce.shape)),8-1,-1) - if x.shape[i]%d==0 and not is_expanded[i]]): return None - dim_to_split, divisor = split_candidates[0] - splitted_shape = x.shape[:dim_to_split]+(divisor,)+(x.shape[dim_to_split]//divisor,)+x.shape[dim_to_split+1:] - splitted = x.reshape(splitted_shape).permute(tuple([d for d in range(len(splitted_shape)) if d!=dim_to_split]+[dim_to_split])) - if DEBUG >= 3: print(f"split {divisor}: {x.shape} -> {splitted.shape} -> {reduce.shape}") - # reduce original axes, then split - return splitted._rop(reduce.arg[0], tuple(range(reduce.arg[1]))).contiguous()._rop(reduce.arg[0], (len(reduce.shape),)).reshape(reduce.shape) - -pm_gather_params = PatternMatcher([ (UPat(Ops.PARAM, name="p"), lambda ctx, p: ctx.append(p) if p.arg.slot >= 0 else None), ]) -def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: - if c.arg.precompile: return None - params: list[UOp] = [] - graph_rewrite(c.src[0], pm_gather_params, bottom_up=True, ctx=params, name="gather params") - params = sorted(params, key=lambda x: x.arg.slot) - args = c.src[1:] - - # NOTE: this isn't really needed. it's okay if there's unused args in the function - if not allow_param_mismatch: - if [x.arg.slot for x in params] != list(range(len(params))): raise RuntimeError(f"params not in order: {[x.arg.slot for x in params]}") - if len(params) != len(args): raise TypeError(f"expected {len(params)} args, got {len(args)}") - - dict_map = {x:args[x.arg.slot] for x in params} - for i, (p, a) in enumerate(dict_map.items()): - if p.axis != a.axis: raise TypeError(f"arg {i} axis mismatch: expected {p.axis}, got {a.axis}") - if p.max_shape != a.max_shape: raise TypeError(f"arg {i} shape mismatch: expected {p.shape}, got {a.shape}") - if p.dtype != a.dtype: raise TypeError(f"arg {i} dtype mismatch: expected {p.dtype}, got {a.dtype}") - return c.src[0].substitute(dict_map, walk=True) - -# shape-changing bitcast -def expand_bitcast(bc:UOp) -> UOp|None: - x = bc.src[0] - if (ns:=bc.dtype.itemsize) == (os:=x.dtype.itemsize) or (isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS"))): return None - new_uint, tmp = to_dtype(f"uint{8*ns}"), x.bitcast(to_dtype(f"uint{8*os}")) - if ns > os: - tmp = tmp.reshape(x.shape[:-1] + (x.shape[-1]//(rate := ns//os), rate)) - parts = [tmp.shrink((None,)*(len(tmp.shape)-1) + ((i, i+1),)).cast(new_uint)<<8*i*os for i in range(rate)] - return parts[0].usum(*parts[1:]).squeeze(-1).bitcast(bc.dtype) - parts = [tmp>>8*i*ns for i in range(os//ns)] - return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) - -earliest_rewrites = mop_cleanup+PatternMatcher([ - # resolve FUNCTION calls (inline the body) - (UPat(Ops.FUNCTION, name="c"), resolve_function), - - # resolve TUPLE+GETTUPLE - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - - # resolve allreduce (must be bottom up) - (UPat(Ops.ALLREDUCE, src=(UPat.var("buf"),), name="red"), create_allreduce_function), - - # split_reduceop - (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), split_reduceop), - - # remove DETACH/CONTIGUOUS_BACKWARD (TODO: this is copied in allocations) - (UPat((Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), name="x"), lambda x: x.src[0]), - - # SINK only ever references the base - (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=tuple(y.unsharded_base for y in x.src))), - - # ** copy rules ** - - # COPY transfers a contiguous range, so materialize a source that's resized (shrink/pad/expand) or reordered (permute/flip) - (UPat(Ops.COPY, src=(UPat(GroupOp.Movement, name="r"),), name="c"), - lambda c,r: c.replace(src=(r.contiguous(),)) if resolve(r.numel() != r.base.numel(), False) or r.contiguous_view_offset() is None else None), - - # copy to same device is a no-op - (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), - - # copy on reshape is reshape on copy - (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), - - # reshaping on STORE can be a NOOP - (UPat(Ops.STORE, src=(UPat(Ops.RESHAPE, src=(UPat.var("dst",),), allow_any_len=True), - UPat(Ops.RESHAPE, src=(UPat.var("src",),), allow_any_len=True))), - lambda dst,src: dst.store(src) if dst.shape == src.shape else None), - - # ** store rules ** - - # fix store hazard (dest is in used in src) by adding contiguous: TestAssign.test_post_flipped_assignment - (UPat(Ops.STORE, src=(UPat(name="target"), UPat(name="src"))), fix_store_hazard), - - # remove two STOREs that store the same thing to the same place: TestSchedule.test_dedup_assign - (UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1").after(UPat.var("a1").store(UPat.var("src"))), lambda buf,src,a1:a1), - - # store a buffer's own current contents back into itself: TestAssign.test_nested_after_contiguous_store_no_init - (UPat.var("buf").after(UPat.var("buf").store(UPat.var("buf").after(UPat.var("buf").store(UPat.var("src")), name="a1"))), lambda buf,src,a1:a1), - - # move bitcast from store dest to source: TestAssign.test_assign_bitcast - (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, src=(UPat(name="target"),)), UPat(name="src"))), - lambda target, src: target.store(src.bitcast(target.dtype))), - - (UPat(Ops.BITCAST, name="bc"), expand_bitcast), - - # ** size 0 ** - - # reduce of size 0 is the identity element - (UPat(Ops.REDUCE, name="reduce", src=(UPat.var("x"),)), - lambda reduce,x: reduce.const_like(identity_element(reduce.arg[0], reduce.dtype)) if 0 in x.shape and 0 not in reduce.shape else None), - # handle size 0 - (UPat(GroupOp.All-{Ops.SINK}, name="x"), lambda x: x.const_like(0).rtag(x.tag) if x._shape is not None and 0 in x.shape else None), -]) - # ***************** # 3.5 cleanups @@ -562,33 +392,8 @@ def split_store(x:UOp) -> UOp|None: (UPat((Ops.STORE, Ops.END), name="x"), split_store), ]) -def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): - input_src = copy.src[0] - if not input_src.has_buffer_identity(after_ok=True): input_src = input_src.contiguous() - input_src = input_src.flatten() - if existing_buf is not None: - # if the existing buffer is not a full buffer, we can't use it - if not existing_buf.has_buffer_identity(after_ok=True): return None - # if there's already a buffer, we just use it - return existing_buf.flatten().store(input_src) - # create the output buffer - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg(input_src.max_shape),), arg=ParamArg(next(ctx), copy.dtype, device=copy.device)) - # reshape back to input - return buf.after(buf.store(input_src)).reshape(copy.shape) - -pm_copy_to_store = PatternMatcher([ - (UPat(name="existing_buf").store(UPat(Ops.COPY, name="copy")), convert_copy_to_store), - (UPat(Ops.COPY, name="copy"), convert_copy_to_store), -]) - @rewrite_group(new_ctx=False) -def get_kernel_graph(sink:UOp) -> UOp: - # prepare for rangeify - tsink = graph_rewrite(sink, multi_pm, name="multi_pm") - if OPENPILOT_HACKS: tsink = graph_rewrite(tsink, pm_fold_moved_after, ctx={}, name="fold moved afters") - tsink = graph_rewrite(tsink, pm_mops+earliest_rewrites, bottom_up=True, name="earliest rewrites") - tsink = graph_rewrite(tsink, pm_copy_to_store, ctx=itertools.count(0), bottom_up=True, name="convert copy to store") - +def get_kernel_graph(tsink:UOp) -> UOp: # convert movement ops to ranges tsink = run_rangeify(tsink, bool(DEBUG_RANGEIFY)) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 41358fa5cb705..12bce3a8af9f7 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -890,7 +890,7 @@ def buf_uop(self) -> UOp: return s def contiguous_view(self) -> tuple[UOp, int]|None: - from tinygrad.schedule.rangeify import pm_mops + from tinygrad.schedule.prepare import pm_mops from tinygrad.uop.symbolic import symbolic # WEBGPU and CL do not support views. From ab798796130f301f97a882a18a2c5f6add82daa0 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:00:18 +0800 Subject: [PATCH 196/306] bring back sqtt examples tests (#17725) --- extra/sqtt/examples/generate_examples.py | 5 ++++- .../examples/gfx1100/profile_empty_run_0.pkl | Bin 66725 -> 37390 bytes .../examples/gfx1100/profile_empty_run_1.pkl | Bin 66740 -> 37390 bytes .../examples/gfx1100/profile_gemm_run_0.pkl | Bin 70989 -> 47881 bytes .../examples/gfx1100/profile_gemm_run_1.pkl | Bin 71008 -> 47881 bytes .../examples/gfx1100/profile_plus_run_0.pkl | Bin 83106 -> 37640 bytes .../examples/gfx1100/profile_plus_run_1.pkl | Bin 82917 -> 37448 bytes .../examples/gfx1100/profile_sync_run_0.pkl | Bin 137026 -> 70555 bytes .../examples/gfx1100/profile_sync_run_1.pkl | Bin 137010 -> 70555 bytes .../examples/gfx1200/profile_empty_run_0.pkl | Bin 45072 -> 18100 bytes .../examples/gfx1200/profile_empty_run_1.pkl | Bin 45065 -> 18100 bytes .../examples/gfx1200/profile_gemm_run_0.pkl | Bin 51709 -> 30159 bytes .../examples/gfx1200/profile_gemm_run_1.pkl | Bin 51739 -> 30191 bytes .../gfx1200/profile_handwritten_run_0.pkl | Bin 56301 -> 24148 bytes .../gfx1200/profile_handwritten_run_1.pkl | Bin 56173 -> 24020 bytes .../examples/gfx1200/profile_plus_run_0.pkl | Bin 60241 -> 18166 bytes .../examples/gfx1200/profile_plus_run_1.pkl | Bin 60227 -> 18166 bytes .../examples/gfx1200/profile_sync_run_0.pkl | Bin 97578 -> 37030 bytes .../examples/gfx1200/profile_sync_run_1.pkl | Bin 97696 -> 37158 bytes .../examples/gfx950/profile_empty_run_0.pkl | Bin 99408 -> 37448 bytes .../examples/gfx950/profile_empty_run_1.pkl | Bin 99294 -> 37320 bytes .../examples/gfx950/profile_gemm_run_0.pkl | Bin 130767 -> 85196 bytes .../examples/gfx950/profile_gemm_run_1.pkl | Bin 130773 -> 85196 bytes .../examples/gfx950/profile_plus_run_0.pkl | Bin 130351 -> 38626 bytes .../examples/gfx950/profile_plus_run_1.pkl | Bin 130345 -> 38626 bytes test/amd/test_sqtt_examples.py | 1 - 26 files changed, 4 insertions(+), 2 deletions(-) diff --git a/extra/sqtt/examples/generate_examples.py b/extra/sqtt/examples/generate_examples.py index a7014fbccb03a..465861527a92a 100644 --- a/extra/sqtt/examples/generate_examples.py +++ b/extra/sqtt/examples/generate_examples.py @@ -1,4 +1,4 @@ -import os, subprocess, sys, shlex +import os, subprocess, sys, shlex, pickle from pathlib import Path from tinygrad.helpers import temp, getenv @@ -23,5 +23,8 @@ # AM_RESET=1 gets a clear trace, does not work on mi300 machines subprocess.run([sys.executable, *shlex.split(test)], cwd=EXAMPLES_DIR.parent.parent.parent, env={**os.environ, "DEV":"AMD", "AM_RESET":"1" if not arch.startswith("gfx9") else "0", "VIZ":"-2", "PYTHONPATH":"."}) + with open(PROFILE_PATH, "rb") as f: events = pickle.load(f) + with open(PROFILE_PATH, "wb") as f: + pickle.dump([e for e in events if type(e).__name__ in {"ProfilePMCEvent", "ProfileSQTTEvent", "ProfileProgramEvent"}], f) PROFILE_PATH.rename(dest:=EXAMPLES_DIR/arch/f"profile_{name}_run_{i}.pkl") print(f"saved SQTT trace to {dest}") diff --git a/extra/sqtt/examples/gfx1100/profile_empty_run_0.pkl b/extra/sqtt/examples/gfx1100/profile_empty_run_0.pkl index f1048a1c6472838253d704120d170360b6538001..8fc25f57068e1bbf1c63771169214defaf3f4031 100644 GIT binary patch delta 787 zcmZ45$O^(ZgSonOB)!l$fHIl3JFToH}LlPhYu-wp;?tj=nBa zdRX!jb5p1EFuG3Jtm@|~FnM#FxkLtg4@-7xQQj19h91`9N|Q3g zH5;eY0!`uQ;SCOq_w)-6362kT^a-7U-y{ZRB$FmLB(Q)S(3v1h$N?-!+SrW24w%f6 zs6ap)0~?Yy4--^(^ScK*`Nq44dd53Chj@m$f?Uo55TR-?#~J!>voz0|0?`=WGA~ literal 66725 zcmeHw3vgW5dEUJsNP-j}5M|4fpV#z53z}Lk7LQ#pq6)r5(I6-+AFegBx>zh0*ad;z z)$W21$#P+4>?)Z}!ql4?O($(&P7pxP=IHrMzEb-_t>?3~7movBqj&z?Sd z3b%Y}E>p{z)vaersdO$sJ(9Up&%JFjS1!(`E+sivbz^3(Sh<8k=Q5YdwVvvRQl?Tn zS?#Dy)Iy=^-hJ7^TxQ>MspnI(`_hHU%sy$FBaq3L3#D-JQf>5Cc2v8P7fPuj8eOYK zFXSp&D_xvV7FFXz_4b6O^5G;mA(`yiYwf0%@9x=OtDrxRH6fZQSN2V$(&sYy$$b?( zB-8WdN?|T}E>p^9W~mV)Gu5q}qC56L$~s`3QElX?P;U>D`S8B^a%taeZem~YQYBl+ z_l0|-`{whx3YFfMPL(sVZ<7~-ft@?hvg!xil6-GUwJ*m=k zd9Sr+&$$beW!3zTG;Mx)K3C!Pk364!e{~xX43#qF`Pqu1_~DT=*$=itC;Q>Mt)cqS zrY%MH@);JNk=uSvtu}v8bE#aO-_yfpwF`C8Tf+D92=iU_81J9@e@YP|c5))cDP}}Q za>V@{-aT=Drz)%K3aLB~j(z|H9-Avn&d+8ZM<$V!Ftva)RciAS)sAwdR^2?AN$2KL zvj`p0p)b^`-2;7ty|KvP&_MstV0f@ER{LBHpKC`ZU&}u3NR7v$;pkwTQ|BkL8`Ni> z-IU#&?aFS|Pur{88$`dB-Px1vKAF8g6w20hv!I5*>cU7Nsd^Zi0LaDvh z)W!UP+0@*`WXekJx6VFQ43~5Hb0J9Sff4=^kLRZfp`M?a02Th3^I; z(E2y?c8T!ae+1b_{iQn_8GYEy_O4o9v;DzhnvDfXXZA-L1Z^l13lH`Vv7luSH6{&3 z!?8h?)Irta)+pSA{TM~#gS|e&J=i}K3CE%;sX@4@=AHTZxrt24s$?_P ze4$u&*E`S~3CDHe#*Ze}hmWQ=KHwd=sowWyQl;5TR;856m!}G)Im}6+p3hjMvACKZi$^2jp{Od_SZh4e3vLbh%sJHb zZKVt}#&pJ-n4faa`BdPfg)f=6f+aX8+)GH9bLAA8n#xYSnum-u*tje%38s>8x%^ z))m{COqi5s(kpERJ9IAWRIDv1h&jxO0r1UGa zM{DU4QSWGI4|<_X!CIIsBa*pEe)T~r~L-;U8C2DArB)PxEs{H2S;Y1Io+2E>$Ym$v90{vd&4^z7;?-H zjuD9Fhk#*xS9yM-T*+1DD?A)xc=3#ggAp3Q95vgDe>^^j$zve8(h#;81Ffprq12DO zulmj;OzYW8^o`SZt!5Am10QQhJfa0wI|w5o8uzU>9_tMc25IrzNV5oR z>i__jfq+T}ZL7i4*7uh$%}o?$bLo$jpoyh3$#h|MHdQQVJ~o-b0O?{$Uli)OD=XZm{9j++P~VVU)LkpM zH4s(e=wZn&)k?BBDg`LFg%Xfw2g*PVj`spe@Du0$0raJ>H^A!H4wxEjP{^+PBk3QA z#>0Jm4zoi~vav`%xD$@Y18P-O~K?=1tGHTWSW;(@gpN8!2QTYALuVE!)WWM^2*}{cPiQ7oZ$}Fg+plSE30JLZq z$&j0Wp(!Zim0TgOj1r))FrKiS@px@yLv;tNogx-QCZP;f(%D*d%VTr7GE5;P6X5u* z(QkK>rNKii^!rNb(tff|K$i=IU;^#6(v^!!qxsfo_r~hh;vB3DUE<7SX3DBZC?m@kM-!|1bTj&A%C%X=JN!GDQvko*8>}XYYJV2(MQv*gVwVf8KIlXq4_q!Lol{D zV{wj_6sT@8w?g^Mg=D#uPUiO?(8*GUlfWIxu2vtuH9Ehsx}7CZGS;eRw0)zgCDEAh zh1Bdh$q^zW5*1BZ6srtWXI`M)&ep2zn$b5ptM8z$%QBC8Bbe7%%_4K@qQ5z)B;_Zn zyXjrdW*~kpk*HgMwW<~%HfB+0L;pf`GZ=47WLR`pd8QFc2_Nihte` zh!|Y$%-*cNN1J&rW6){G?j-&Bpj5rrf;|qKe6~z-y<%_d_O{jO3w5Tpt!IA#`JPj! zF-Z;%4TU2<;4PA3y@TQY03Fpam+exYB54lt6*;CdH8kv-`6}j;YG2c-$XkzVCYpJT zDp~WqGxM7?pKX}=E!<}Q4gSdVGV=|px0RV?=9`rBt({S3eurqR=Oi-oI=A{QojGQH zk9nxa+vfAc%pX$T_B^mA)U^$(O!~TAw3mD9)Hx3TPhayAaSg)V?+#g5MfHvUW=2G?kVycvyD^tb(CNx&?VE@Ow zSWql75FP3d_s0XUpaA3tMvpbzunz{=R~j=9WSp>)Z@ ziW@IY)K}KkoVcil!P8CW#Kt;kKC~xUGJu#uQd0=Y^xIZ}%WGI@Y9q=kxvH&$2gE(7 z0uS`X0bWJ?#;1Y)NFPQ;|53!D0>9(5Mq9A3su3w(s-F4D4#Tils{v)aLPqJ^zv|q! zaT(!gFBz@ii~9fL2hIuz++~IWKJZu=?livZxYg~9v>$>4v32nD8K+v)7a4@x=b#@P z&PnheiuVN}?aF^>u5dmx2k^>rxh7G0Am8DLb}My&_?$F|MV^5GU!3i;h14WqE47&6 zZZHgbG}z-c#I?p^aGDONb(_NPc@d@ozUPudSiFh&RvL?g?SUXWIqBw7vz9EyxLZ3k z5DyQ;915vY%|J8O3-)SN)uS}-Eaw10VqHtEMCta{%SPdK8W=6 z)NBg=ekCV^PLmz`C&r0017V0D3*jRdKwxAL1GDtTdpx z@!kO}E=E<7r@393i^Wua5)LtrH5@860M@~+%e^E?NrOKkhm!QO+C#Nb}iF3Di5PlYYTHG5s8;{4ZIux|#bRV^Mx;Q_X zn{$gSR+M1R^n2EY&?N=4_iMnX{@>dMswFWi0>#8@mM)jCt3{u^8lHBkk@T2Amdo?oy=UcCM%bU zjjPmDM+~cMkv`ApNJ&HScz9?aKzJV@;ng+!%Wo_afqfT^`Ft2i>c#6D45)ht{O$q1 z7s?nTux{PR5m2QpNcR(aa;I{F639n78=`^@m;KlS~*%Y3z zK2nS%3>}i>ZmmPhemAWvVh0k{&Q6a<-}B5+WbQrBL_ln3)jW!w8M3R193#UtvsO1? zeGi+aLfAVao3W6F_@|Kih1$pt-n+5GPUMfIR^52y3_cbr`0+P!jTqI=^M%}G?c|7g zvVz?nJv0Ao13=8J^<5J7&McrJowO&VR^6Jqm@6mMSJzYBIf<88=O<1w>CcP%>krDHXi1N*_P4%P_0+Lb2S?#yr#$6oA$2#N^!6Am zwC<&-{Y+jpFN8>Jj0r2kSRSUe*Pgv(Yy>Pee5$j$Z8r7%C3u|hs&|b&QAsVwh*Y}t zOz*Rv_3UqVwzNK>`uTb5>?2 zhXA>i)q1_E`J3<_cR2ah;0K>Wt%5m)i-lq+(i7%dH2}G5^u1;qu5QECVHyn;D2W|< zcfI2Yn#o{aKWqy>Z4?x@Z2TN0Z6HK(ic*Dmdb8*%tY1Ah51Tx9>e*wwd$r^ zUI*u~6Bl}nY!AeYI1q`&VaE9A6b;)0vv1Uad*S@VB}{103^V!E1RNI+Ky!TJ_=%&e_qF7@DZK+blc~8$ z?%CgMz{L1tN(q}K0|8(F&KuyaiZ5b1sTM-_>XpCq*gF}b)}%_M)FpQ=Y~_AFgRR@v zhpgu_r2>2)(*oW=0mL3NSeIZkHojQd6c$4Zd2DN(MZQ$25+)tCrBfpme^_<5st+ZT zq@1Ty`9eMium}y9Ou{vOu2zMvNj{s;q~TgeTjkQEWy5h3kMKsv20Q5X721i0MNMqm zEQf&O;my+yFBM9vE&HLVM_!l!B>N&Qcaw~enU{Rw;){=^Asg=)PN}$rriWBl_gZAog^o1Z99o|*K`fv{)1E){ zO>I)!YdryfLgkpKt&y28QN=${J7ngUsR^_XZXK!0)wyUrjqlzX(cynoDEeCS*bnrA0m00>Dut zd#D(eodbNr5N0j%zb}SSHO|l}hEwzK$=8`_AN7Hv6s0Hjf+q^%gwMOR5y6uL!$OzT z2>%RhJRApvc?;|K&(LuHESl92IiVxfEOuC-H4FtzZu`Vi_SrGgZO1M22PP!`&w z{;B|{_NecBAXR(Rf2&9RZDeBg#qejSPuV`aZlj0%zQHK1UD->oGY4q2ro$@8rqW2Y z+o+nk1q~+KHS>8=WHw;@svWYr4db64J@G_`X%3?XzBU-k`uI5YJgjbpwM_*M;x=r= zjV<`wZR{{M;hD+~;~!na{I`DpJ{+y-l6#kWWEw(KM>X7rH>yXugBV{_9VZcCx*k+F z``)3S1j-3Zq)n~di5C5fYks-!|4tK-yS%f_DDxGhS-8W#J~npNFn;uxVM=Dp zDcBx#gkJfl|J>O13t`%obLH#d=~7|7m@H?e$>k(DmAeSLjLbvh*Eu_K*@>{)rt_U& z^~7`be;tN5>-0P(^jDuMX7YzmSciYbj!K?~-G=qrf8JrdhSSt8Cdp|cd7)4`r^$n$ z`3rpH$`_CRX1G|&o#(BJ-d24%T%ImsR~Ol*#;>S{GIj~h>Zn)3YQ_2IkA}}pz80oU zZ8(XCNny(-ca>=UY8b0+<>cfg*qXUC9Ws(W_suVd&pYep2WQ=1!G5~)nW+*j)={6W zuZQbh00cru(iBQ$jb@Br(k!olftkv0;AuLYe-tqXm%liA0_m0uSxPJpAO}ZgBHKc)Fy+~T7Kaok8Iv$-OUY|Kq2hf>DNvKyDbqEjKAOrjX~M9wraY%GozNdw%Xw41+U(>nx0hkHT{>+-pS#`uqE>XlyVZO#y!% zjAZ)yGJUah|5RT(J(!xD8cIQLiuLtP#iug8J;n(vm1j!(2_3dhoH(MJVTJb17H}>_ zDOpTavV;&O%Qzk6s)Q+%S!kKc@;wOa)2Uq{%{A zaN+Vk;I<7d=Z94^*Jhx9K#iII@TNQU9US3}|3Ek zQh`=l4J;U6tZ;6cw-KG0Efw&>Cnxc`jqpUdY=o8C6Q-#gszz>Zeh!fkSyFHeQV%Mr z3EZ=&SB2Ca^Or;KP&&}jFpdgLrcf19DM&hj3Id5HO-VhoGI2hgMiK{Ey(y>PaxK5NIWkbi7GU<8NjEt4%&*YJ(PaYgSeptVOyStHGlCCJ)Mv#6fRNTV`m8CIB z%G>j<&N!|M(l|x5NDrY)#;=_S|BpA{EJ%47r{pf;Ch1b1E5V@Q8oo!|X@ z3gz8#L{8{a*m=40P9u@x+-jwiA959tN4XP!+$<(~*S64K*j+j!$tlgPx?8sVUPPSm z4nh0+o}P$*+>TKJp)1js5%4Qt(i8ocB%{8&%afN0*1BpxzR4H|BdqF%N@}_`VmOyz zozMb1v}*W*b`cCX{4F(n!O_67quT4V`jEXotdUe+P=oz~v#|x#7(MI@XwYHvr99eG z-N=4maCr&M{3$-Z>Xxz5!>3bp7>Ls?a>waJ^3d7SPbUvQefY%D(^zDl0Cao~M`xc- z!1d;0I$awvtDSV(CIvds^N~%srD`KMhyB8Iu{P3K-B7@?H2GtaaaJ2$FsoZind!13 zR^5d#sEi#w^~B+)&YnDT^i%|=dC&Y68l?~1?y7D_hT|tspE;fU#K9A1YcqdMEv6>S z{4GKw=FH!z+kd6orkVdnw~m>=r(5UDZQMX~<{$L^4(>iwdz(-_y+*3WhLxyBJyds{ zP8?32IC469{KzMh2M?b){)wY&>Tu^ur29Oi-zlUI9XvUbJp9zjCr%tcd}b}wjZG_2 z@ApvOdF<4o(d4nS#~XXyNzuM9+Eg#vR4>}pFM2am9_gCl_xuh>7uVZVzPgT`h#pQp zd0hJ1NzaWmtr-?BUz#r|nxls4AWAhhCAx))jvhaKT61`{RJ#<9n^7g9&R|HIEdak1 ztj#rqt?DuAH#o1VhIZyQ2ihEHbD+(EHV1zAIPg2a?`K{7@bO`#9&2+iFq_K^_T+Qh zW$p}Of4k%5*{$ZdpCw=W-sV7?18okpInd_7{|g+bRdLcpCY@wd(sZJdP268;$q!pB z)Gt6eHijto*8%zgMcVd&ZFIHjmG$4^;n~_DjnP{?uqmX{e2<6nf=zYc$ibN)FIcob zgbf$8VPpRRbxxb#ZVh;ehIKZa5wCY_Xg}A+G{fQVfE_kc>N z3$#b^*}c{@5S!F2uS^}N%mbATIbvv!Go9?=Os+?;7j6AfohVIbTB~i&>_+^0ml0VKX;Y`Oe?-^Db}XWsr0 z?g^vAyz_?x-w7{m=t!6yiBA6C71-!!eV8{l?KVN;v%KDDMh_07lAVZHparajr9VMX zSxHY{R;y&$Eti#9mDq~3`?zU{Q=Y09w_M)N?Ly2B3D7DVykNInR?CwJD0eSp1_xBq z103Kg->HJ#sQp2M4&iup%jI`*Aa_|UFeBn&NqIL161$OGu_^Hog1fnnOP7f`hVjyt z%O94&M^$6spAs!aS4n_;DiKd#g=(|QUXtLAh8{C6}ds-p^jd@ zXSZ}jbwnTE6aU0`4z9%zP{X`JjFKmch%7@6NWP+Z5{*^M*WxYIvgPud2KmI8cb-Qj zsgER5dlGo07>o9J)ziP|P9GmbF3=~Lb_ui{v#}%b>nmk=27Ea5+f`Kd>Xwefzy0E0 zL@MT;FHx81%RBVjul@C3{;duo&%#TT2?6zC9^xR!5hS4VQP7S{Jc3?qGtT3CIF4_? zLRun(D3b7R+&F&u0fQVE6CXm9lCz}`AP9Nx=r9r%g7k58q3r!!c4-7r!-z6AA16Vw zK$T=%e;kh@xEZ7j6zJmnC+V}Ki0{)!Ntd}ygq|~=?(E`c-#<&#pwDoUx%eFJ^eV{x zz`Xu7eieUg8YD>+P2)i7qwh0soY4{VN>d-DmuU_k@Wdek)f7pfNPe#W4l%o_{Fo7P zcoJbyHB%)`F7qaR3oM`CqWU{n4)7O*ARmnZpL)#OhyMINkksSfogKf7N<8Bq@s-9u zy5oCRjMn2HRf>;Py?P|7caMPo_}4|5)jLKWB^?N8@t`4fh4Bya@~UV2W7)Wu#=p&I z<3Yo)m|t)I_y-<)#=oAX-`_Q=A?i9oAiF_{1=`E_J66)e@Fj?S1s^=_}85D@BBT{SKV{H|Aa69 zmwfxbiuZr>>zY2gxM*H~-&ej9Hm{LNL(B#%7tJx62IvWbyl4(f8j3BNx8(`ayJ#+x zPDDAOoi3USlD3JS#wBetJzal<=yXdCbam0ZA~~QQE}DjSvcIoIjrme+#?=ajh zn$aI2IyN`puIK2%yXeX0DW>d2G)G@%wGBRt;7bDaNE+QuOVY6qW6`{mcBgks z`gJAF*bqcdp# z64L8j_K`1JCRUWMqP(k3RI^LIzDsm$S!MaUCU2AF%a&D^FGJE;zU-VVUkj4P@^wR= zlzjaa(Ye+jUv_?$uZXmg<;(siSiWpJE6dm46P?>q7t5C=Pb^={^5l{)TYgx+?rgl~ zlCK4MKP+F-hurCllCI=yk2~Fd|4P0d3ruJEiazB|@0R*mzDg-~`mm(4d|fZQ)2~T7 zD7t$6uXyPdhy7dn&Nom=UH_v&+ofnjSyTGolCpo% zRMH7u`X3aKApH-SUHTs~-=+TdmfF8>i2ch(ud;s=#3N<@a;)3_-TazI|3kkI6SGJG zKEeUS9F@Qm5;!J-CncclUzBRy{zc|M{qLgKzdap&H;4w?zgqv(_HSMPTRwn+jO>d$ z|C>aksr|B);h6wcF0<{2PlBq?u`I_BOnvmvQhWX^u}88Ef1l2Z!|Aj*0r{nwhc^Nu zx(hUYSN$-FfsPpqwcX1&dWCTGCz|2t<>xAuSf4^F{2X z!TzaAs!~>mFSapqKIWs1k*_~Sj>G7}y$TBFvl8kU`4G-Zurcxk#v<*8zlrxPG{E6z z6F9tM*t~HOBd4IdG@v%QOpEMy9Gc~K$$SX zLFrThr4C?Xf)S%)2=U`QP%Tw};2@qK?-CC8hl6sffWu!FnDV79mwUKX2<+j&>!J=Y zW16W0i8>tq+B(7E;HXju5K!s>I7%uYGxCH4MkR1k0!kfN0@k+@9KMva0dRAX#K{SV zTYsU|fi=S6%PYX)H{TvOeCd53<9f$!@uVbhXeQCW-Q!L%V&T9%<)g5avKU?@Q1=Jk|HM*e$xjEUkHH0 zsr71orByim=8z^=2imEUCYL!b=1_EPaQO0q2M+g+fApqh-2tDU2M))#sa`!2?*xiXQsxi>Z?!oD651Tv zW{-cta5#;BL2&p&0~~(c1BV+G2nWGoorYev2b$^(3Lt&TSM_Xs=-+_=lfgBlZvK^c6DRMMUfL1-fp5 z{>?fZj{e{IS)#AH$Ng8xG}BjTJN19b(SHYh-Bw~n>-}GD=zsKgiN3BAOn3BOqunn4 z%hCn4O5k_|Tc^Ls^OwB;>;JQ{|N8Y({~h%G`>$6$*iQZaA%=iEKR~cs zBLF@wKpQeHn%x49chb|cfP>Ii0YyvBP5dNjo9W4h1sR&W6Cye`FwFp^or3|&ZYj+G z<)S<>Kxvm_fYMH5fbzyiAn-QyeLc$kkhJptS4q0VuCY?-&5TEl&(h zUVn_}+!Q(tP~MarjR0li_h{O1UjQ7wEO1`{96m1T0dTk_=>c%KT|NK~x6=dQa2xIp zfWz(b=(oV(HgwA}7ZHi#5a|`lzSlX85 ziKWewCzduljiqf&(pcKA$P-Ij$p;Q!lpHK=cAI_R@Eekor7a?@<>i*5Jh8Mz84kC( z!7_JSO0%@xko=9(X4>y301m$*@(}=s+v#hB!|n0`aQK4I4}imqk{$qu8!zhl6Y?c+ z_;Q@a0PRB4+$+Mg$OIz&a*c5K?GJh2aDV-;nL7m3jdjug*uz5Ue~E6d{>ScMt8j-{ zjs5#lu>EW6f6SO>?qGHM_XiAzH`D*TaJWPN8&mdg9S&c33?{tD z8p;9EKsi84I|q6s5DbK`=okK)2M7n_4s*o_9Ff3L2|U4pi(4)qlb9zZa9jck2uE=j z5ROuTKscNiRs_PwZRp$U7vBBnS`pa29)97wUtnirhhKO%11aJ+e#_$*zDR%; z7ydmP2){vuR`3f?d>GCC4Dtc9E;fMSSH&;f1BNg9fZ>qL5I_-;jqd5@1YQG?R{KL5A z8vl^A;`oPBuJI40g2ul@eIZbQaIYamcmoiA%L9aW9|iuSfangxyn38~ zP=zyh0mjrG*xL})IJbeL!4to)N+75)(JqSBpfSa2(A>b-+x#+z>p(Q<@4f)~QYX!!+>c4IZ^?&L2>iu`n*NXI6w1W76Qc4uu{QlLNM3eXL zpueW~zkEQ;AA^#+eSq*Wf!Ode-Jak!7zb;7vFQHsHuWcr$3y<^w~M_JRO|l2;iBx4F>_ znr!?YT|oF%k6*YUKp+E^yC(s-J8Lj+ieEPal@lJn@NOy3K=@UUU-%6v&p_oZk6-wh zlxLuF!s8cimuH~zs>d&US?Wit@!r`l+y-q~=7velpb^jq2)`{d(MhFkP?@F8esh~B z&3?x$ZC8DO@I{dVmbPViVrjFbhJo;A(q^YI5I(^`c;n*Bg4E2?)-63?AbeP!SlX^g zP6onl`C=e^L2|ORO)wDND04P9SlX^heislvEbotj@LL|g@Q9=<`SSRM+w!O6%i|Z` zC||cce&M$Kuzc5lNg)c1q zqQm~3aQlU;ebJEfruzb+fGE`;0p$n*B?SV83o$z-pj3RswBQ%MI!6e9`*)r6zt=_o z^9REH^}koe5dz&k#0&sS4htuO>|bO)CV8HefXn_}mHtN>^@{ed0|@7Btk4GSeSzaY zf@zcSmDTx$!}mJ~2wx&UDh-6+IEP+3JdL=|jz0=0|FOIE8MoR#`@(dNsb+CVM1?mcZPU9%0oW@(hictB>tHjD* zx;w1=4bc@GPLu0`m4ivGV@1dugq4GTL0HlESJV^|6TwJ10jT0MzDBJ4#x1-sfj><4v$~zcW-AKPR=>?+leCQjXS$l_xBeRf=?LXQ(Va ziUOPAiCFh#{-pRaf8WN+Z?6(7zv7RT54#=5Z~I{7<3U)tLEt0lnr{(SK1R6dU0~(L z>rSWf-8%_u+X?PAY?^U08sA~C(>RZR7FvXWvXzj1MOz7()gIav#=l^vA&0HB^blIV z8%;0_Su1?$?#92@W&HEEl{i*8jko4Bgzg5}O5oOtwi1dftQ?uiV~m--;`kSgl+*a9 zk#e`Cgz?W<0V}`qPS{ckE8hUDG(lMD?{Y*JkZui|Z$nVS=7x#bf3Hd)sA1zS3M$xi z8wC|?+`?~0nm6POjZV-ouYQK)S3%K4U&H1^U&F>kUpKImf*RMRpvH9()V}Vm&|iMd z+5cN#rv9t$asL%;&GeNkx2ykp*?{}6^HNh<@Bg)i{!jb~^wAVgI4-~v!VZ2HvtRv@4w^y>kQ0)-Bu#%c>fAEZ-xGq=P)>t{Nc7}Dy+QG zjdxs7IJ_v}m4a;gpqDnjx54%j!qL+4?eX#OI^wNG}^s>=`*AFYdDLrmPFL}hV|CkGcRWg?8#ma50 zFaRsJvB&_dyeRbtVC6Pe8Gw~rQl7EQ-5#vm#!>^Ya{G)HbXl-+o98TZVSCF*bFj48X)JBsB43P^+ZZ-W+nA*JNSngS8)eR>1C4Caw4@C_ zPUE)ZGFCn=IT$A8W+Q$EAIcFTu*Yhwvaa!wOF0!=nGK6)WHCbQ*VK<@yX!(*M@YX&m~C!pfn) zV9>$|i-@_)GgSP2C4%+8m$zJ|uVU@KVyyIe&QMAGW4yLus<)ZbxL5xJC!U~Eq&6Ru zz>^YCSUK{vZvP@PSuKX~y_N0XR9*iYCS%2cltcf!DNf_JTENQdPUA}-11IoF;PRfK zBIo5A!=^Qh98Iut^J>=(XcWH{lxLl=^x-y^-p0}k$u^eW#?q|=7UAZn=JR=+?oY?s zzeQMj-%ww;-*HBH9ZT;Y7!s5nzk}M`@4QoC>EjM8ozHZjwaAU|JO3b$udQL}Z2qr^ zHKIcQ%QQac#L}ZKzw;K)BgfPQwG|iULTIci;J@z9Fgx^8m4|c*E~T*a1XhVxI*%9m zU;V*j=_~0H#%rtbI|u(()Fr@LrAr{Q!jcekLijf-fs+zYy2R3U7^aaM6xq^w&R&Y#Ddt{=?KbHPR>sWf-mU?wfen;R$kSzs{2H8?5bvNVR(zA7I>Wa&n zG9y($MA}UV8^p=)JaJMYFg|ww7aG5S{dF2Rg5~-r2}@Q;Xcr>nP!9^qdEA|b_Qvis z5i93mBUYXIv5jKoJZ?B>Hox@mISBnbpQH9EB%0_;ofOn{QuPWoCi)sR?xNT_{luT@ z9HwByY1bnapep^;xSlVt%T`X<3 z^suzukTjMybpVT9n%8f}C5I1|enr~M(q_|HQQFi2EVrdJ%bX!kEN#p3)VOkEUJ?GY ze0k3!Z%R5c6aGZ0hL7Qe^6T>Rk#pP?lK6hJC^S7rAX9$ zDOTfmjt0|r%#Lb39C)eO0i^!EWUcv9fYcGD08Sj0z!MTUCV?jj`@;Q+_sr zkUZG(V>I~R1(sfXcT+4~{k92FTHP5gfw}}M6$(p7p7p}gSGz6`J%Kc4$GZH>qDvsD z1zjQlOaBCBwqdHb8J4~_T>|RC8nE=lHcam;VChTEe%o{5F0u40x4;YmUVX6i zJ8Q+#S3JX|?w8K)-2V($YL2&vr7!#Vr3e1DCxeq@>$LLQp0`_fNa7}A{JS!O5U=&D?F<*=z+bEp zOXr0@*zUOSMF(N&_-tLa6f!GY3Q08nt%qOI-P=+xt#15#+iWRtOW9K3RuGnsBo~&B zQp%QUdHf6dZKs>nAdcFU8S;hVSl?Oi4*YApm*84BA0v4+Q&IOj+Krq9XHsrV>SZ;&Mn+(90 zHfXq+p6pr}xKv>IqQHtb@| z?Ux&XmD}k7SoyHjAAps2OL_oSZu`BX+k%x}5k~r8<=v9T!e%oWgCGvdmozMaEjB$7 zNMYrc1YqU%`)B#`V&!8}J^(9^NO}NPZl?!e<(8BWz{>6N0a*Ehl!tr? zR(@?2SoxypcWcDTi#;B!d}$|iUX7Jwpw}BbJ-!la)$gEaMye(RTma;+1V-u7F^&#vA(XTbc-80K5L z^nvHE0OK9ZRNzv~KE%b+bO=9rTHb4&PnDAAQ?m$NNaZRA=#Nz>Sx1f@I(y7|{D2j8 z{48qw)b#eL>Dfo`!Bf*?I5j;Q=i|}&)bzRodn_Jvfa?DUA4TIYh#p2v_VMcWzJXY8 zbSM%H$D;88u4v{Ml3uKZs`sbolf_IaSQZf_tGbmR(}nqbr8aV-dLKVh zw)A{$q@#KtXH4q!ZPj6`pD?P{gK*?ixIuy=un@75i_rDGK~3&O!)r-grv2C diff --git a/extra/sqtt/examples/gfx1100/profile_empty_run_1.pkl b/extra/sqtt/examples/gfx1100/profile_empty_run_1.pkl index 07a57e03f9bbc89ae225d4f019ca8775387539a4..5a166f28fd3eeefbb9c83bf1c927c83c972a8732 100644 GIT binary patch delta 873 zcmdne$O^(ZgSonOB)!l$fHIl3JFToI0gPIG`v$Ei)$-NCCxj zC+bTGFgyCXOzC0COUwnzGrCUMtmo${FnNERxkLtg4@-7xQQj19h91`9V2@CFCQd-?^31jmOt`h-r&knUj$4xEx9>&=)U2gLH;Y#9pP zOc{#aEE!6^4U8GeAVQ@_s3l2{Dv?v<2J+y zYzW+OLcxK~@jfoW@t!Ub@s7?Ro?)(pT*#84EQia9#t*@7b7UR*fW&9IWklxzfP32Lke>acON6?cn{AIkY`vx9GD3T zzyL=!p#x}wF9R^ZCnqFH+9Mgk2{yvlGdP&A85{^RL^r=qQW9X~o6K7w%a_5PA?(cn zw2>jh(}`hn!ZMM~5f#4#3ylUgxH?9-x{Ur0g7O)G9XORTWq59mTc#$Y um=V~CQxP-JJO{A73-%h|SIPoZngCY%=1Ks5rL1sMBi@I literal 66740 zcmeHw4RBl8b>4dvC6W52xa;+H~Z{1UV={9km(Enp_6z5R z$8pK0<}%f+Ztgl?N~Lr8>5dhABGQBUSUP;aNrVEpqUez>PAd@c_O2Ohnb@cQ5%#P&MQmTkXSIxew zxk}bZ7w3~j(fB~EJ%OowFv(3wCVLJUPf^RC>N!%apg+$vAet#xdM~BYmoxdv-U@D# z>G^V{FqgcXDdjV>)QFK8a~G$`jy;kxjvD7h8#yWv?qM?T>zyx`dS`Q&dW#E{Y#|>B zhWmQw^SKI@?oFr48Sc|)o4G%kPR-7qNAze0IVY>;CZ-BUd)^g^F>ukk4=j zvhNywr40i)pRbxmIWs#Ibd?Vf1yL0@1qHs)hLPiJ$z;`h5)Wa%loy3jPvBsxG+jPq z96WgW>SS3o|NRY{U!KoZxcwuqWS=zm5WzqxQ=Xr#2#W6;IiLN$X6R(!cds>=A86Q8 zbT6M__8GbB)l{?bbDB%#^8B2h(9I6iMNbLu%LjSfMUV0PssH1G5Dh0MQk)o!&`6F( zKZj>eqrYF2m30L~p2tSN7X+S}D@@MMW}ZbR#VJ8*0cWaI=P#LUFq#*3JJWUKFA_OX%6*>{>Bw!JqFN`cfNW9nM| z=xl24(qzg=9x*OFUksLW`O5)t>Cq8>6Hn)-3W1)Uo|i%w>16|lkG5ss-NbtX7HIyR zdAnHn-d}?3VQ=ovhDM*zv*AtiYc}f7rr7~M?#%upC!=t(=t(hN!9Wn0He<`3at=Z{}b))EA8QiDIrA!(sGbz`I7<()b9Z z<#eW8Hp0g7`Kc*9&3qUOWcGJ{v?|v~^Kb(pQ%c{xc>KNK02VJk({6SqYqIT3CP?x# ziI$d(9k?8{%hm>D#2l8zawU~dX9AQx3Ayf&QLdDFjLd8~V@y5uv8nv42H3&)n6vP! zPaQHY2T3Lu;z#%BZr$uIr?VN7KawO_C$FSPGZ?>=n@-Z|d&szyNkdvMC+AW{l6e%T zB`XD%ECbd$eek7gL3EA@Uo&!fgC(LvJx|Y;D;Ikd#n(xQ7(p-2j~_cRJUV>-nDGoo z6pc;N&Bz+xO-{&QGrVIH>=2``EKWhsKb9;^i(5m?^)$sXz z+G`%~9{mCh$P>th(e>-zVi)U@#=52bW+GjSmF-`@LUEvttgpNiJEO za|%?kP<#-w&p@1CWk-dva3tCn931qK1rLxcm;?TR#qw$iNR4a$h=rq}a4;r|J8SL> zN8`b0q+U3N1l(cn#)6DW#p3Fah+}=>0kl>Ybk`Y=2IFy`Iv>Z9S)QMR6h(psqy-2T z)0y1Vf@?VXXndnF4~ah3hgJvUVIKnfct-QBwhu!Y8t|+(I*6pGw>${36{3Z2TVv5+ z#JjCgJk`jcPo+!JDVz&Gfz)uFhRP9Bp!-beB~VTNPi!V zxetfzoz2bVDkN~%uP(7je-SI(kkpS_Lw>cg%suY?_#*unl8~3ufm#P*L9nFf znq_v2R+83HDL}d{lz=|lk%m|#7J}R!^J;K392r282iisKycbY4hC!ga?w2GQ?gvmd z;DKMnqTv`u$gftRJPM%7OL;``IMCp@XU%;ffF^!5cj5JuSai}uvBQipfGMcYYsQH7 zh2p_}*&DaWVUr6UsT7I@D2^3_g=7}}ZQ!5T--Ke5$xr9JbSFoEgGH*UQTezAjaT`fknFZApMD3^uK#X>g47kTHFa>G6 zk}KqeRssYT+7q@ko~@2-GxtH=DPmJ(62ed=ovoUk&&=h@(1nmppySs^ztK*r22ZgN z@GGT-Bcz^yE*A(u2RdY=E7ycX^YziL?dGoH9MlY1;{0T0%BV#MEy=ic1d3FTarjvR zK0nT|zfyYU^8|m0pX0A|LFtF{$n-jI|b+ITei+S)| z!$r|q2Q7%!DoYv=Q%Fh*0k!_NWZ?1^wwhXy@<#S*_rVO}VU&U6aqRQNye6prP#6=` z#``>vo|AA3wpJx1#hcYLKiQ^f)_ygxjL(o(`u6WTx0GClI9hu~8yKYiy?nt*0tsU? z!21R7BRdtJJ)^x0wI7ESv2pC&dAnQ_!{+cHQ2M@Toh1$@4TPXo``Gt%3nQYr!j;S% z&@01bpG4*35!zm^gRVtV9ClBBQ2T5lH3{5G>}I$d9EX)pT%z7KL5yqdj|2yNaBFjq z0PuMi#)02+Npuw>;od$&QnZgZ$$g}|F0$0krDhGa7vpMev_BjQ_DRDdXKSMak-=bp z+~=X~FXsS4VrNV2MY$Us9q8|a-EJK-P=n)r*x`uy)Vp_THU+bCxpU@fZGS)bC|!_X(pjGe>KAER|EERyPE+>Xq(Vk$ogqZr#Z z4w@4SV@ZqD8;o)i+0@BrSy$nK)a0ZwTL9APC604(Fp=5?ayB&cOe$RhHuE4pa2F+lxI~Jg_>;H?Eu19luS<-8cb~Nn_&;NSdq80c`u?U8vh}LY)JB zeY77glIot>W4v`^0364b?_}nhFc=Y)^?7cO5bw&0 zo$PBXPp_(NFl;HXpO?lpb?x2<+k1fTg)*iH>|48r1IC5XAXHY5IR`dBFbDFRdH1n7 zT^i?3Fz>>$-}zRC>I^_8x58`|R7CgV1*8*C5OEeVj8DcORgTh~r1;hp;;r_l*FcS6Q zwmrOD!fX?JIu0`nE{QGTXw1fA;v@wp(Yz7yuGFSazl-)2aSDlQXRF8IcfS-5&At1j z5QyzJ^))0>(Ucf%Mw{QmmbNaug?-K3Mb8;n78`UvQ!Gs%F(gl&D`wKPWmw~bK&lL5 z+>?BE?j>=A?jkm9iK{auhCycj6`s?`A~v1HSr;7pNd_*Ar7D2{I3RfL^x2U^28G7P zpFcHzY!nuD!cvYlQlG+|JR*aQ)4WG@1A8a@G z%%)yhfW-;#dRN&Lm85daNTr3B!WZlIv%k^a)c%C%=cml=Jmpu-VcsYg8_lI@+RemJ z1;%+;sij~-?CRstmfGIttUR0?0_0W}`}LycTf!7aK}=QH!sk${x(^l$#XzVh$hArU zvU~JBdJC>@!PP;U4Fx8NQ+oG&>Iq^=WDwhyv6#nFjU{Rz0;@Y~HPNkOw_}A|zT!M~ zo84hP0HM z%eE4kI}7ub;ykAFi&b++E-!=gIE@QAMja8vvIom1s0?13oa2aK_RSh@eD8+>TXyYU zrve^6P_n;2HV9o*if}e%Y9I{Vf<-A0q1?$h#rwxNt&6(PKwKm<(+K`))d^KqhuP#nK6gQLT#(H=FS$WJ8Ep5Pf0bIA1KnQnELj zewhz(9cVLmLVPMN0Gzf>8eCiTx7mq@CeD3s_ zVHW+Gvffm^!*C{3bCcY&zubnk@yCT6HcL7JzyP=%;I1lj#B@^Zg&vZ7e^+pXGC-|K zl}f1vcP^ahekFqw+{Sy2S2Cpn%plVWynzgeb7oL5G0ZsoVq{a;4=vD@3NPbNt|Pp9&Qd=6j{8ZeoJb^Kh_gt$p&n=hqdT}Q{|(j;iZb`!TS zNXH>Ni1!sbjfQPaoZc)40OaAx(=HoPS82zac!V^)&h-R5p?3{fYa0C1jA+u`zSW*{dlNrlu~n)wW=d8gp4VgXGLh^`(oNT&;& zbgCUvq%VOjAV$-NYb@EgWpaQq^m%kN}vymMhHMNg(it3_w`Z_Sr9l z(jVv2ShoqBkW*<5$#LONj2G!vKb~J=li%jr~K~xP_I>lgW9%lJ6Go7VAT2w{p zjt$|7g0jQs6WfU3MS_u`1u?_F1T_y{fv|32C;ugy?k}QQj>txcV718|~a-+kOiTVvQi><9e#D8no@50)ee(KsGZkdLnDWe>h zq3z;UT|tb0R2?S~VfsBLF4nb8-m9yv92Zdy2*`Z$T9mh%N_Ukj%J|}-$f&NWnCzye z+DYV_p{M97vg3bRndEU%5dld-UR8e8PDM9?a%xCao0_>2E&4Yo$IE^FHA16OF+r^YU5+K>Dal*w#41?_`;>6L%_kF@Te3)0b?>t7B|mkRU6WH~cUW+%z1 z+%;%sq#$Zv;_S#}B|>kT&bNQDE}pahd=M6`)ALx;Z$4kl9{%-Uv6Q>Q=N0Q)_1R!~x`?A)q@ik`7dK@b z6`Yk(p9zXh=YKgIygd0vkWRM2B@dIrxlQgW(fZ{ew%p3e$pvVexiq;NNngJ8+29p> z-F#!O`*S!{cO^4bq8&Txv+?C%tqTA_$T3Z!MEYn(`$akA6%1ge@@u%8PUjEfLjUkA z4j-PLx)z26sh*7*Azi&7cz_;1dhFfX@Ra5IrJg%x5ADDK#u_W(s+p>Y?<&VNUPt?3KC0m#y|V>4rzj=RM+Y-IW<9WI>vzh!)8q^i%WKeuV$mTW=Lqc{NX|t712muF7*I z`11t*QQazC&1>rFF0b0JuJoRfOry%Hc~o81Jc=y*qdwSwAH)sCsq#whQr97SW`y1M zy@;do);zEDB^6;+j7;=%HJ~3+l$H9xKhx3G=E}8vVTq#bKFUs;tpSn&D z*!FXg*G7@%D6jM{NyweweLs%!?l>Z+=u_DKbLX8#BE`AIN>x7YDxhxFmBzFC=viSEJ;?&q7X#Kt|7vi6=VnjgERnbcbc$F{7h5i>*MtOCYCsPyb zc2$33hcTm9TX*1UI7tL3x6S?FeisFhGg+N!Z$a+}b3T za4PSrL4|>zY=vu#p76vp$nktBk0zPh*&Yn0FM*kF;r%l^$3{<_OOaC$yLsfkbBW~f z3+G-;o_O)Znc;KTX}$#f_%hsRpG&|5=UO^l9nsBpa=S@^HgtYu2QI1V2z;_%oi0{K z+RbeR>`jwhCh2F@(M8?tEM=z4f|%KjFsO_Tk3Vq>l7E6zSu~ z&W3=y_2Rf!_4JL^EE+pRMTxlsm6vxJDKR{>2v4g z2yd2ZhZy5VREeoGn3ZM=fG`!@=9xHHn<*~@iT=*>jOA-L8ms392M@`ypC+ZR5U2F!B@OIwjqsOooTj1&Ch`F zlH*JRtWN{1F=cX^5)dh$~;infGvj3I+J@3dvYy;O=)Yl z!m~8_wid^o*_QS*Fiec`84qEli`Ed~np5RN@JgMl9Hl?x68q%v@e8MnXO9|vwqE_7 zF!~H{Sc{3@M^LCqFD{9lvTWy)(5u#VA#FD|4ROL+6@k6nE(G>*;9BPr?X+R2M#K{;2RZO&=hD-H%05(mh=U0lqrl-aE1dYMZ2~<#%ERF zc@8A@g0DiW6ppur`I_x)>yr@?z9#;$_A<u@zM#%_>7xOp=IgT&^S%iYG zQ$7k%S-b>S@IIWzJ77?@0HTP)zp@$!gOd?s;=PCxe3mc}1V6X6X=@aum!m_AzD+MB{q!QS@5|bY?c&AdNcW#a z3N`%@$LeEOaivGmkE`B2!jIyc`t@@%n&yGj2fjx)zF$VrBTapP9;Uu{v@Q-2sOCB` z6!FjaN9h4HoS$@qjGh!Th?=RA20wkBUID7VJ4N-kF(2SB2!TJG0grn0;>-eG5jFpf ze>r&6KL4%#C!~piruok?&^%d*p~m2gM~^pAJbT25`40id{MSKGN<3p^7Tkdt$sSG2 zf8dv!P0oM0P0W9@c_ zEXhl=wyux=KJmw>ia>htCUQ|aPoH=G1nG%ya0I<|YyW|RR2i`nRRbPcyDdr}nD{-B zmty~;$V)*iyFwa3!G=G5<#V!qOuu_~FWPrU)QN%#StkW$okVnvGqk(tziH7=e5+~t z%fDi$Ki)_5MfKGG#Mb(++R86nP{c}xGeu} zO26|r?(`K^{ym)D9&*)hrGF2l-+0uWuBrN;r1bIkxznxo!EO@mi@MX-6@7FD=_iq1 z;|ve}vPPNt>#iC#5B{1^X*;N9>sjrjyBlYSj-hgR@Yh}SY?;5-6^Af?S#>dgS^UEM zW$_F1SN8>?!!n8we`%@>8}e63`q@+>C9i_EADiQKJ!=isyn?)@gMWo zov*vot@7yAx_;x2k*<#o;-)zh)OdM3%hQ3Qq>Gs$UVU6-8GjI z{cSS_YI~T0^a=s#&ou(l%P&_d$+;^&K>F2Ur3CMr#gD8*H=&5zOwm#CcqJ zI5IbdhqF9@%3p6J53JSj@V7@E;Nhyn!{1PNI6C)$QFJ^m;se@dC<&eq%g5u@N4=2Bc0R z;rbnd#u>;17*9=&kkVe z0xM0_OZ^4K!XhvGL%AhvStL`Vew?&wn;N9GP8sIICFl5tCBpqYvFw^WW>8OMmO&~L2|T=+`H%Jkr9M<7n8ksIKU%{Z z*99J)u;bxf3NL5}uj@DdocQN=S>K`U6A6v)K~O^DnvU4t;SB8*6wqh~f@=as-$Sti zn)N=|>3AFT#Tp(=^bY_}TdUER&^XbT&@|H*8JNDrZi)Ukxm8SGV)q&#)W$!3^$T)* zxc_$sLEqkgQN4@)+Q0YgzksHRzT6sUrvJA60QA3mg8DCp*FFAXYk>MM2gud`M6Lho zSvArBMMwX~f6PT+An&f{FB{18<=z3cs@8wGjnPd1hxW+vXKeDu&x5|+^A- z%Qak=5y};nww=<36ehQW?)X(8lyE2AS+y`W+5H8gGp0&20=}y5JP>dz=N_u-F0Bbv zGh{`yNR?xR^0rE2?EA*A5gkM2z#3^oP8p#rsx~tMeqH5c1l+ zuM!Y@%}#{U`9A1luVCOC;Ne{=-3JdZs&pSbd`PAH;Ni;)lD-cfZqfI_!wprx z4<0_D($Q~kJlx4`))+FkSqLz5+imr9nA^tG^VmT(TO7jNCh%}4&rPV(%xzX0bDPCA zjE7rXz}%*(avt2aF&@66N;9`Dsy2IYTS%obw{@vI<~D(cuc|l?JiMsVoZP0bs^{y2 zhlkYj@xjBbbRRt2O83FT?<)GtU)^RCx>3Js{ z>WhZ)F+}NkX9Fy}O<>^$R>C^l@NGl->Pwg=-!3ft_NPB$lLg#ZI2oxy7NB39ZMY2! zXIX$UVR^b2EW8;k2vlw=EC_){SdhOg@OtMGEqYjmI+sWYZNnwT4T7FnxP^Or zV&NqGII(b>ZTP0KaGPy-oEXW#AP|l&fzrbWIe*Nqs-OR|$~HW%Y{O~(^TWa?9+Oxj*i2&K4%_h1DDYxZZQ22x z>fL`uw&Mco?LdUI0^dVH0d;C{5x(}@q67uM&cO}@6>d(k!lc;S$soV1>W_YxKF(y( zAAcN}o`l-n^aa$*^c5Bjq6&*eP+_r~qTf}sjYj`(e8@>(LjBZ#Sv}ED*yz{#FHOq# zQ2*svY3N(ypT6qo|BWfox3F+jq_AiNz5B2BBG4+e7lB~&czeeBQ z|Jw6U)cP+?%BlYnmT%_y4?QdS50L1xF0gRJfrZ~yXdW8c*c#kI`gTyxqACaZ57;q@ zNyKM1EZoY$=%kf{(Mb#MW^~fRG51hiAyqS@lgsLkvG7$7EPP1iU@Uw^-7z|OUA2qR zNvkf#!Yw?K(Mbz!WOVYbD#y5_z{2mUni&gUQ8}FGq>~@@6@~Tr*oH5wbRR6-+Dr4n z!mav!u<+}uejhB{O83FShg5kVEPP1q^`YMi3m;M=!#o#qaGRCJ+-8k3ggl!aKV#t` zRm%=avp8fY-3ctbOXXmmTU1Ynx$U;PV{ThlT;qX-8!9Jrn^lgv?aq0!MZT^|^Ztv~ z66Q85zXuk+qH1PtvwF_lW~jE?u<+X|&I1cyQE5(Y)7RDW@xj71^?ZD=@Ksgb2Mf3O z!v_n$t;#ch)!T+!={{Jv)qWo={I04W{H3t)XjQ^|i2>6*vFHzofe^M?9wqGk)>dp3~u zUn&3b%WucR*T02{tVt}~W*biS-x3R7%Q7zyfTyKxxD5*r%}Q3^$;o)hoIin;|MiZy(Z)4`IG+7AwDf7@0T35}gyU?M|Qg)vaRX3~WDq zepDWy-M9@a_m&kNKCJw^Zlt^pDOyX_kn&|x8f-{8?0y&T5i8fegDREW>xGqX;76r7 zyYUUN^0kjb(HAJWup7VW;Ya13u=1zTOM#WgrR{LjSoxZPve3*L`%!6(m48%W<=?gJ z#+$;*lOQdz@-YWie&-|7k4ha@u6bbP*N@sBvrWCxyNIPo)@;UZeCUXGEr|8`)y~mw z5i4K(R2^2nMkd?h+>D3aI5ZrwDIw=SY8c}D?1muF`}t8psmKrZf5Z5_co88cT zoBv+#T-xkj;oFOqV`w+jm8|*CUsIy_&&O&A^WW`ytlVKYzWN>uE8V8)x=lfet0E|2^F0(4ux%6Hj?}eZwbQ@xBw^J8itPYVX(3JY zC2Y*}MFZQ3l8wHAjfuX(bu~a-jeq*sm+bWK3=>vd>%WApx&Ok1+|_@%ZP4NBzs%bV zeMkSh=Y0Dwp=kg9OW1mo{a<(Vf4oZcYyFqo7OwGMvps+LtZ2{$Y|hWW`Th^R48W2w zY+tP0LhbONmTR6y7`4TM9SoIdMatn*}!OE{ItkMT7x3GNlTVdskYFs_AawoUlRyjSm z&3YcpZ9|F+b`l*gto*vl;juGg?XWObzN~UGw^_XBu`{zVR=%$4+7K(ZxQDrINcG%? zmEToyjFoHZj=60`-8s2Ux3Fj*tlWA&K3KWo;IDc=Dj}8bgOyuYJ7eYbcH=9Gf3}E~ zTkQjXDXjeVCb06=d&J7Gf3_i3-oTHFv=yWIz>@zUk~Hr}Wz+KCX6?rReq!YqV|wO1 z>dni4zFtIG|Mi#uCN_bU+wI0l9mFevUp-c?eCEP#d?Tz}?@p5Ymft{aZ)rPG8ys13 zt@6)(!qSIYSb7UfFC<%7dJ9W8jv9oUkI(1xaPg0itOgzsmX5Em2P1=QUGBotV_|q_ zX%b6s7g+j)7nZ(#NMHRvOq9)H>B)g+OOEL^jj{U4=I+O zXvPN%a3YeCw{CatjiuMHB<&4lcMgfkZ{HnK!7#T1d7e{&Q!4Ob6*#Q|AK}2JWQjEx z)w=A?>(TU!7Jja~(R6)GVd=NFW_Rwy(vR2qB=2(Aogc6BNq+kRI=$gu(S|-)^jqY3 zCDGU?d9zr0LilQdCW&(k^*+fJmd@w!1X}9hlbiu`C{8$2nW$mu$_I^GV+I7Z#QbFC@4AFJbA@mK#X0@@s-1nXu8x zTOC1xnzkb-ouqV7tn?pFvC1boAkM}={rcza^xNJ~SayxRs*{?ce3Bz5GBAB%;Z5|V zE;0|a1$-S%WzuYxo`qF>mpg2Hm^=$_LKI^w)nD3}RvF zOYDQiX?Jd6e2k^9D$LHq61b?+b`YLfR45`;c()IhHuk|Xp>i;~xvK6MOV`vLqnlP; zo5a$W*au5U)rI|LoAvUBcIP409}l~8Czjsk;e%yam3E?=&iB!Ubsn$tNp4~7jCHov z`6RcnM8?wFz61WTQ>fV;GuC;l&L_Erg)`Q1{8&eR|(k>C9iZ>U@%0dpXQsOLacUE&gEsTB`F&Zq?8Hb-d08 zi9@!yxwqZ< z=6vu$NWs=1eh3I#3Wy|ND45G2M)+Aoz-3Fp@H{qO@Jn<-Fc&`fUiX#%kg~ON*RO3x z|9w-*f2{O2Ja^s52OkFD1a}SnKB)r3D)5{NoZ zCt6s03u|v-?QQr9Ys+r^ZNl0I`oikdpDwI@U~r&mf9Km9*sa4=7LEX6%af2ivYWd{ zyL2sT$I>_ReNVYBPwl|+2C)@CEM|~c@k5MDmOu>25)a&;`2)m~yk!X%z3Tj#-&9yS zhQtp`$B;Z|St79q%X?^*#@&Bs-u8XZMN&#^mg)M~hoA8;$NU3$VmSs`dEEIn}sQ9~HI z+V?%Z?b74!&rW{Z!qSTeQGA2%d)j@tXbvphvP-Y?X}Q#?yBOfkBVkqLDX6(}M z#_MWEM5CX+qI9V(+NBqBFWY>$@b^6%>r!}WH)EHM?9J;^ygey&DK?IG`7`&oOY;7{ zvA++i*P76!UTYGCQg`10 zU5Y_q()IcKo({Y8uHU5j3n)&zwq4V2v=f%9{E2V3{DL4&`32FpYtmY~or21T2!hIo z2=Yqb+l@G1r0JoW4;9eA^EA;HP<;mxm3~LsRgTCA<;qjNxl;}&pFw|*J`s!IxhFbr1gMb4IEcbF)ep`Ws3}3D)hz@*d1D?0j z-GoZp!FQ_MPP(&zP6jT8!*UB&WU&0M%D+jl+^X5v@I0hy#v-^}Gl6IL(rPmso=>Qp z9`L1=6Ei5B&IIJRF>vYSuzXpKMgy>1FDk6k$L?HH>5OI8f4|eR>-DiaUsUB8%e+&E zm3OIh#xm=_-?^gF8OyByerHIf``Dc?t8|Q(vOBj1(gQ0u)Sxl5t*bj`wsrMznAt3B zc?Z>Os5E9aVRvpZ6ti4Whi$KudAFMSoxw#V`dZn$|pz$p;rhivCVTjGn>^0 z53Jmx$0(GHlf%a^oHCw0YV_GYjoNaY-g2Ct zJ^au(PT$ubiG`#1FEkh*2o8AJug9YU3T?+XLPJ?7QP<{O q17PktHhS{Nxs#*Ej)bZ+av=7|frthB2jjJIn7O$_)8;Q_g8v^93Auj& diff --git a/extra/sqtt/examples/gfx1100/profile_gemm_run_0.pkl b/extra/sqtt/examples/gfx1100/profile_gemm_run_0.pkl index 7379d89fc6699fcb50a65b3f63bee010e85c7cbc..acba35dd13055c7e102e247d9609d316bd0d6472 100644 GIT binary patch delta 805 zcmX@Rily@#lR^W_)PK7ez#w*tMh|~UW?p4_QDTZ-N@`hVa_W@Hg?4fiZMmeG9erJ< z^swY5=B7^R;VFtYHUc3-vv{K^o9EhX6`DLR&|D&ey@w?`wJ2|jH$x9=adJj#$`qiX z8GMsJ2TpLz5Z7#+QVZmA^za4;#(Vk&hXlumIr@Z7$&l`03J#o-A?wYUAqT|r-fS5P z-b@*a-YgkPz732S${<2za$txso8gqwqzv)N|AHjc@oQsXM$+bG1l9(3fKYItbG(mB zaJ;8WM7*POh-a89Ay=><8OLS}amD*!1wyW1LueCc0y;c5M5-R>a2EF+KLVO^2 z$WmUV<(FiQDLIB*JBbxmO&T|0WJSI9rrljz+wT1|ZMm^{8@Jx3b+z8zZti|`Z?$gI zrcdAB|C~94!2lpB*>c=OkcM;KuQ_vG-}65+Gtl=+4N*NObeQ=tMgEY_|E4>=)p?Vko+AbTFFL9sRpD@5F5)G9Jyw z6uoM9GBP+iG13)1kaJx#G@43`MGl0yE4@5Ao=6{nr@hewscf@emW-yeJ-V76K*-)T zvB~l1n$JW&8yQ{NnheXcKdgi>(21Q$w&fe&g#L3N7FI)U?LMv zh_rih$?*(N_`*B|;c&Cp-9!m*YF?X7BU0zbQ|Tm9{fTUs0?qj3P-ZOpiLActXk#$!O`)^7*v}K)w5Z>UTbL!gtYdK*0v7K7tmU=k7rRdYIGtS zyCr9<`8!&DnqT9#nSofDDAHI(tTN_|RT;Y4*z#CibF986w!#y0A)B$4mz%rTwOvZ= zx>%#gLF~r7!~_Cuz79M+uwQFsQq*Jd7hsPu7@yu8_oc&Oe9PIG>w2(-b zEy7Vx#T7J1Vs+?%426%Du9F&Oob z^Dz1yuNy6YvpYJLin@oJJ~uq^1ve^#`*Zo8ztH4$@AXl4)lB^b8sut4cclhnQR*nd z)a!)zMX1x)4wC(BbT1l5A63Lb3Eex z?$HUiClwtV_BP)=mP$X^EMt70x_~a^V)xGV8#iy?ynDU-<0w%iHcXmL>vymBjYQL) zW@OVnF`0JvOio1Cx{=3Ba>6}4;aMF?j-dJ4B3C~OnT9ZCxCj1N)$n|iB(k+!ev+GNv^X3hb z`TDiB_|Z!jYd1PtwN_uKqeQ#mN@WI8>Ctp1&EG&&XZuTt{xskV6@O(yEg@fv(Rtb5 zY|ATCiFaFsEC&IobX_;=4PlIQ#tu-OqIx8nEL6L@7aI6bU8fNbwbjOm2Q|1xiT$b` zEE=B2Qgyv<^1iOgNKlikHdn4X#(w=uR>RnDboIr)b;YVzu5r}-s6J>+fqR5pXqH8-yuZ4(q!`{xHT377o!ueoaGPkDL_mg&NTu zq%z~^A*j(rONh>SBsw~LAg|M>P+D4j?X4mdYvGbro8RXTl!yQ=T`CqyV5(vn28aA@ z^h8=(iWF|3EdZ-viNQ&nVFr~KRG_WB4MT-?(aq*1Ezll-mDW-Gxfcnly)78pZ-ryuBOY4!VBTZ&ZIARa#kP3^@WnO-|KIzF1Faachw5Q0~v zu*G=Vd4&=TbhP0mXfHBP;e$p#6yvk*f|?Jn7)G}is}sSXzoo+$43r4fB~nRa(exw+ zJ(Ed$X$Sowyg5dy@)L~0L62t44eZ^PUb8ovoQRIOF~=WG=a+4;m4>pxP+>P43bwVi zw)wPDf!6ZS+#+aOlND@jZ3$wcXL(;`?GCnuf~bnctDBglbcxhjCMg!r9RXkA;2CM{ z2p~dZmQhdu^=JvhBBf!=^uc^Rg6*MJpYaal7i?QcFo;2U@fOxH9$1;4OiW_UBkks6 zF+Re~&9*(dwuJF>bYf(5BDzKn=J;lz8r(;1XiPl zXtiMwvrDXJcw#SZ|FlD|8H;@O0NN04pXEbB8uDVnDtX|6z=Ix)t-bES^nNjPd2#z6 zIcUVo`$CK};fRltkuDx`f%E!uy(%$|(XQcR_fT}$oih-Fb@%?Y7|%Dm@A-rq_s{2z zy)zTE-oV}Y664K2BP7p$Og=Es_gRwmV7XqGnHU`&MLx|4H(=1u9VW(UY)@g+u-`Kg zeK?#-4u&Vzt~0EqA8v)xlg{}n7YZv=uGjJkDjjC^-NrcEh%Jo7_#Tdo?UgP(G(|x` z!)`Ja6D6x#TWW=KK1q6U`&oy64P{-f#)(IV0SF&{(Xf4g@o9;uNl)ud^iap5sGhbe zlbeZTb)$muNfpA(%bzDV9m5lobzF{VrJhmkXW2lEklw$o$ zyT1j);>Fummw5ElGs>rCX<9{9Toa!{+orcH*fbV~V`~}Xk5VA-s~wC#K_gDH?Ta6x z5NQ&;NQ)OgLJpVG*u|&G<&|yZ;?GbVA7r~0&+=4X(Z($PE4&PiCSc*FEBlryxv-e)m-0r{?HwJVP|z3f7nyKjrrM67dTWWb z;}wZWGBTc`fQ!#s+FOz6U|W%yb+Ek+AqV{>#?~t_vz{E=7d1Lk%Yt_<*MSh`Y=L6q z>-M$|zz1X2Y#(1+#=*6!Gg+ z#!OOyLG(9iAo$@EhpXdCKCAe!mTyDtK_6N?zL1bVXKxP$uuc*xUZM0_zIwAE0-m&f z*KTXOjg`0d_7=Y{RJeM>t+2f750+ZJ5wLQ6a$j^D6BM_NgTg-=Px5a`uvmVeWxba6 zHmtc8T@#7ljd6-)1ryE#e!?Hd_61AU$*MH~WLe@}!gAPTWC)WS>sp>A5gkD^{`T_Z zfr8*_#vTNF+ppJ%bs)a*n70hXeYR;>1}<33B&$$6A!|__1%cL&PK>5UBUrLB)(P{U zQwMrE^n#&+l~)AZ8Vt4p4wZOLb!bBd_lB{8JrWHE!a-ZOnjd>Aw89<`;dWrWfTAtF z5~SCqMllh_a-dk$wHHKd5KFotW8*{qBg1xxVCnI|4XN1ZaN6zhdz;;XHq4zTGYQQ6 zX;C&W=|MmEJnC0VVUfTp)X`RY;jwmjERqJO5-GQZ8fXi(0jw7c39!J6b?SCsOR&TW zZ#|a7uoQ*$uwqXTzB}5mu_aV6jtK>W8m*IRq2lj@?u_nFL?(uSAT3CPe6`S;t!2eg z3~ON1*4BYt99oI0wjwe#uUi}HSFlLa zgB9;q?CWSXmb?p^o|RN_jgX7iT?9a{Y_XT3HN=}W&yAxg~2FFV%k@1&p^JEBOY|$*XIcMq7E1 zpArF^%R6h5R1`)7VI~XdLcm7HkQZ>fpI#5GRG(49M-12_gOy7I`pQ7z6Ziy;Bu|np z&=6QfO9^~p8P|@}>N>VVQs{h(N%F=AI{f3|8z1nG-}pdF>v&60C>Js82#g2X3S0)p zbH2$ED`$F5GTaiR|A992fR?SLScv!0$VD&18acLydh{B(xsx`)znfkN3TYUl-FI`zA++vOQf&Pa1nI;|*nudE$-u+to%v#NAL@O&f}u zuH>dS!maM?O()G*x@x=JdO0dl1e-pUVQdcViQi07kVU+i!d{I(GIk!ovh_nl$z2KT z&!UVd6mplcojL63bMiQY@ZQ|TdJ2;7=GHtip4aW}GTIu0_NZuXZ{yBCMk%tCL2FUd z4%4)`%PnGl9BGWNgm2xse*b97DCI5(xqKpPq;4bZNn=gxFHwv<+D%Y(squRaN0eRD zv8k{oZWm96mjVz;UPnrc{!W{|mGq*(l)OZQ$bu)Lhg2f8T*@fj$RLYvfugIznkO@1 zro!;OK-_AHxGkCBX^Bu;^2@9bMrKTfia2?BA{(dth)7H~@h%FD{68t5z~fo*#Hz$( zb=$=7Bx;85TWQvtBneIgt)$QqeFUk)*PR^%~kMK)4BmDcX!( zOLxL{d=CM(WiLZG*z8REMtU*>GF`^Ty`R-|cuBKBy>;nS)Y!+uq3C|SirP%R&oh>v=RXp*lQ#h<}*>zcyG^0CPH zz)%Dm%fp`j-bmWxL3`)hwXLVi>n2t2&K+BKuHTNmT4K)?p|>XJgf~K%UNTzCqZi5Z z!0-Y8gFX>4fFjYlW-p&A1Djq^Tp;rGtX|GXOj&(1N2vluCP#>Op=dlic`;JTQ0c&h zqe&Bha;LobxC$$hDM5Ti$iIW0#IhYRi!HmO2vx4g`4W=}kH6W+L-j~tc9ZgRuzF1E z0enTYc+cN~siHqOEl^BRjo!)8iFCfGMyx_FTfco1?a_8b_eTfO$zmUCwrjm!25QNL zIh^CsbOgIMBY6Ft5?8`TchD2bLOi4keO#9sMHmlW+0lbuIhjc(GH8h&M2j8KD@G>_ zork9&11S-VYo#~^7XE@Q*uBCfn}XL}+623u z)k7A>qjxN!QAe|ZJ_~zYGKmB>&8&$Heui%wU!m%i*eH@XFglUdk8(%=E4e*E?9f0< z>UDo66U{_!iGc&?OVO%FCn5vb&$14!{+4asn|YhPq>Vf>cPP$qWPFI{>_Qpjpu)7Q1u-1` zuxMGwsF%m=*k|NfZb2Cg3;5$TV8=elZg?nJkz_J*!0rm0)IS?-#%7Y+-JgvnC$aTq zP%dSlRmT>0Ot(ca26vHU+D9GHl4QC>LGk|Atv80l)WZ%&CMG9F zv2=n2427|Ebv&!11ECGy4-8^Q1?|)yq^=U%d!WUBQS9$W2c4#k@ZdNQ8%cSvaE2;C z8_b0&ETp?U!*_jd0IRfL@EOHPGn4S=_J83n$ zR5CrIVzZIx$CB|cAtZh4AR1mI-6OKk|pk75id;>RiApQph@{6)&q2R@RB&rqth5(h+s@&LwdvB@Dclb{X{Q0wv_l9gvTU>J%< zbEDWIIkb}x$YyZ_V(Ox~S_JsHb5 z4WqnUCk2e3ydyIHBY)+js^nV6uCv{dJ^qt-i$5M@m0`a+|DG_kJVT_m`AZo75^IG+ zd6(XdY?}U5SS%$z8Lupto&O^}#LH!g>6G#I!JkoNGD%CiQ(Sa~D8WVUUp36-T|P4^ z{zI^HJLLm7z#gVlI7N%MzdkZH zIS?89rf+h1I2BE^Z}3sD^V>db<;BckFP37zBb*px=|uu?zTW`q7Y*RId;;I5q9e3N zAUr&}AERa(ym!9JbBv6Jr6tDHBNNV73e3rQ<8Sz|y=o+bF7TxtiRi?}ZuiFTnnvM$ z7*zu=eMx2C^NmOLhl%foAD&F^HKIodNGuJ@XRRRM-}w^B(S2Ohpdf+l+rHFD0=uqh zBH{Y3(4?^4e@qy$Z~6o)d)XSIZ|~6ee6;@@(%U=%5WXOOO?gm$(}!iwRCwqB5;8hS za;bxRe+Q4xjHc?lLSv2QWo&)l7adO0Dk+8H{w-fFW0=2^6!T<~W)M;K9iuL#QLNGQ zZ$mdSIB^edbfJ>nbI-`|{y@O*mz7SIn_|>hRLmkjdS6^+sJ%wJF=o6#)V!@I!y9D0 zM4g7;TgaSMG5NmB_l@rIx`DgxWSy&>-0Zohj-ER6Y2VlX=R*Up$oGp9MNN|Q&%Idx zSWBK+HFyjx9FC19!h@O7q3|%#;Al%^D4-1m@y8m*WvDw1KaJAw3aKc5e}uh!_N&N* zQ#J&}o&D=ERh)l=#QwDIarH5@gnxr(A6?b`f6U63;dyQOV^-V2T*{NLe@%6?eyzgU zN@|6D{7(+5D(rEuP+8?a9e({wWFBXKQUUWyhof_)`sC|n&@<`pR2THv4 zt#Xt(#WlsUU$~on17ZBrk6@wJi+hE-oIUb_316G{o2**BUT1-%#Uw^6#HOQLMlI?=UacUsGdVtiNVHoY@g&J`7ke)M37niHt86NyV7) zm(0^i=1WczsbM}^5#+9Ecmw0;SaK2%d}s)doB0M(DdrQyULTEkFismC&y2$mRhCAF zj0RNc$N=s!gez3yp5M7!25o#uly8f(dBPDyP-sd6tpZal$kLFsz={vm_6-i^d*G&g zbeS4VlF(W(4wBKq3?HuMS>ZJ_io+eLZbV>qPdjpCB?HrZa*!R&9}dv8+um+hjwdkp{5 zUivqT+4WFR*wS9Mqw;R|FVipY^uE(LQ~Hmm)_(!+nleIiD-zr*H1YUQ2E zM?QON5T}ifJ!CkfdP-T6&Zr=ecIdx=L|NQ z*wu=sFtg0+b#fhT`^KVcXmqY(aso-x%lTFb0{HR!@&4({dbe-f6(Nag>rSENyZXW# z`geUQyzx^TyEpFwyB)yd_+Fe`xT_Dl1@;dPVt1JCprf-$p(68L6}UyRU8-L8@JJ%t z<ms;rrKj_h;jSlwwLkJPH-|rwxnS2;C~;F}kVoQMx(eaoiATe6MjI!@WqXfdyjS zb(yiUvc+Nz7R2h>)weO+y=hl?+on&3*Kgdt?f%V|m0`zXQMVLC-5{ghu)e1&ym3d* zmhNpEcVABItYWd)TMJ^Z+q!eZ_VCvJZTY!&ka5umt%wJ$hzG4054w`a+~th(=X?!n z7Y}#A`sLi(9o!hccbm+sgY@}vy5qcZ6^-+dh;utrERj-(3r4z(N4kC6u3bh6UuvvQ zQN|0!N|ic_RtXE1*$uQTyvpcn{*l^H{BAY$HyHQj#{KUM`vyZV?@mMSHttQv{bt+! zCPRO{ahKsMhW?+yVQT*(5H?2X5(%TN4mHBm->t?Txq_TKWT4xzwDcw=9Rh|-Mu*#7p0b) z@)_7!-mSIk#EDPEgdngR7AJN}aS1~Z!r0H7Z)Xwo z`*{3Cj44Vx2o^$=nacCf7i8l=dupq_o)5J^=v9tAFL(1AZ2FxGK z?R@3y?)Aos{jR>Woqv_bPgUh(Vd=>1fdD37e;0zREOb&(EuDgHP zK`{$}iC8^@LhSH@0^5J_gJNYa$Of+Lt4HY9GCBp|`DF*Mw(Q_Wsn{eHV&(^{EzT>lcDxD3tfuH5A&ry$B?7M_*)hAGBmL^OvY+Is1Kfd-tc|3h`5& zwpD1;q+IQLbrA>3nO~;l{~MIXhTqi@2_E16Oi0cg~EI+>BPg$S+=khmqZFcn8R?&jTYDdxX7yb{i`AS091V?JyGctj`T4 z>N9rh&EEAKRta2MwBsXH9Ixb$0`Fo(O{AYUap&(~HEil_Pff+t+g>NGr#l)|+tMhpU?cg24qC%)($62HuSn&}RD61>JU- z!ekZ(v(V2$-&ivMiy>TbTw}0+-lM}{76!90@D7rzDE-+B-8f8PG7E!Q=x3pK*6e}B zFs>M`2?PwiM~A^I3}#{Ajo=!gAV#5>gdq%OVK57QqYi`aDOfy#>kzJKGK9ee7|g<8 z76#r&P%ckW5Qm{T3PTvo!eAD9j}C+GgRpoE*HgH3Dh(K1fWa&bW?|rc6hS;rK^%eR zX&Ay_76!A>&qD91ISGrem1+FHUBMsM?b!AOkFJx-^h5a5bLxa^6(duJzwwCj_Cs$UQI06VRq+37 z@L$hxd#GBS#{b(@{DJeYBL!bqPMv6+KBdf16J3i9sg%NM)`)=FIp_3=BdmwgafI#c zbihnGS3i9-n?CC5T*Y23Q+iyTt@YLO?rP-?*Pf5L={o9)e9T>~9(ApU`!`&wyZ+T_ zSL8>|H(cv~*!Tl7eS=aseMFf)dPMob5#{L1O&c6G{pP#S8hsJDLz{D+`kz{+oOue(E#>px=@a`Y*Y{3;Qr+_=%KZqt_0O5hU3G+= zII+V6(@6O*4V5$3p5@i+(a=>Z52Da+TU&qPM0mp#JJIAWZ|kggs_t?;;ahRl;jQa& z4ZP`eF?ao6_TVp@aj~kNx``d(hi;~VuU5y^`_IBp)9QL>nL2%<>9+D6uG`8{4Q|C% zheBpOO}J3lNMX-J)6kpsJ=LfttKEO;QmX1sgsa((*H*IW@8AKW82^Lfy1mME5M?^q z*0}=VzUiEHHEH;xB1Dx!L=D{eC_+^cs(Ox1*WIk*vATwqQ@De7xMzOgQjz6bt63O{ zo!)Wc1fn=~3YM#?T#vE)J6+D{9j;a7l+0C|u|M18V0Ei15D=baeO;4-Y6;AGNc&Ke zTXX)P>5jGaJ*X;muChOEbSY~ofU?#8zeK>Oy*t7v;A+-{$SAI99$C|)?EbTqfpXMt z9@p)gA7P!HC=^$j-(7bCH&$0Y@5GJbgIN!h)yfZ^Wj!d_hg|B~dQ?)!T4ybaYpokm zI@ZFTM>o9#MRtnEMmcL*Ro+AqmH(-`&Q)G@YZw9SxLdu4zgts|YV}10!8IO*@0IF~ z6Gs$R+1mOS8D5C#9Vql_QME2eao2|#m&!fMo_Gwj%k(bO_lXCrn#8rV4A|T8m2i}!%C)AGwnL2EoTko zOna0aKf<&r*8dpODwuYVX*$zxVhztSt&wSmSi@1)U(XtDW!f#Q;Yp@lgK(6FGDSP2 zG~CFvW2~WC)oRp+uQ9C_;VTWxnDzu~c$#TnW!m$s;V?o|8jdR3^Gd@Cre#?}1Jj;i zS`}+J&a`RP@GDGv3h^opD;13?4cDof11_Of8*XOW^-NQhh8K}GrJ+H&9-}x7uXn?> zOsiKKzQVLGAs9uwNo_cyXe(6hSw(A98fw*u`%Uc{rQrpp-N3XHtl?%w`!Z{ILD6ne z8g5mz=UBs66zyw@mPL9M?Mq6-VMV)E)m%!$&8nuuF%kk3Budq4743x5P^W0i;aJt4 zQnark>xx#dHe9c0(@I0NqE#Vvs`j*^In{q$JgR8ds9KfUP^W5-phy($MWw${ zZFok}ZczGFwc&9*6GeMW(QZ}ypMqA=D%AcdrQuf;ty0mRP_;6(-=Q`8Z@58yC97)3)&A$zpS+;Xds#qb zFG2c(p3dLM+JAxTEsxLPs|@FJj{{A&>5Cu>siP1F_#_*bg6hmao z{P86@e;hT&+c2=rANz9i$NAj+anwG4e16%%4`}{4Pm@M5f8?`D`~0!;dwl-r<^^Dz zKcWCi%^y(`w)rC*+vbmOW1Bz14Gm6sT-1MaMZx^B!pMvd^P_Y?lTYF!+zdA=&fkXR;Rdeg8K%pjk`bA9E#z zH~*IKg*wzXFMN>lhr%r>|3tWja{f<+TVCNrLQ4EqW?n9{#Q$zPr7su%d@lYzBTNcD z7yta5x%jbGf_&6aQvPho@%NHYbT0n`6u$_b&)P)ryl6%6l%@G|!UrjH`Jev)@_$Cp z&%d?&bNM$O5Q&6Y;_nj=i1I)Gqb27*@qb$LAN;rU{B!y5E1v&1a{2!wu5?nQmlZ$z zEhBzZkQwF7e?|PtFbE=IZKTe zr3)=7av)uxRi05UN*&hZW|Y%1MHpy9C+)C0XGszItQlpGbcyD6MhX5i#p9MGg?$Vt zH|YX$t~1J6sl$@T43cB~l+&L>Aye8Qb}*wPU0;}Lm3DN`+26qJZ@1f@mHx4q3jeKk zd;jN+@NXjfGn?)9rvIC{{rz_PPU#x1W;s5PLv*@3-5V{*f8jABMfHewn3K zq<%43qKnqAWAa&5Qar(DDIT+~^7@sKE{fEzdTGb&ml+nXU%k?f*RP9G$Lm+{s}xUu z{W5DhuV3}@x%2vUOzL?3IxEBC^~6_OtGj|KtFSmR?qJI50!pW;& zd!#+DUuXW%Zhtw>RVG^($EaMO*rMrGH+( zg16c2&H9U6omYZE*ej<)&=eW$Su<{^mCkOc`1(&NA9<+?81 zH1ooRf)-#I`~H&o!TkLwu1@mvp^pDz1>-*ycER{>VSXUSfAjxfAOD@6Lq|#V>&*W{ zRbxRuH9yRI0%ZR@UpvPuf!}+)%Mr{%;eO^tsUp?#D&cv=+{ddortJoTA$Iy}Cw1Ob0Ikxt?$Svu>_0)Rh~bUS{M z9>-Ok#N#O07d&em_)n9Ri&6;Ch}p6p3ve0%fyF=6kbrvqhS0m5O_%!nO{90B!3K2ynY} zEY>05M)X5iXq6nhU>%~_Scm8fqw_I1wwuaF>Y&6}7*&~%b1si#cfN;EEKjy?a%A`S${?)wx<;|g(Xg4SX%)=`>c^Lk2i6XeMa4<$J^XG5Lj zuCjwFk{kVbQI6eV=h#0rId;OzvHR}i9J`S?cDEBKa5=Cb1Mr02sl#9v1`=d@N`mZe z6|Nk(s&LhiR~XE}K$7f6BH7+@;L2*;YheLBk?af5FHEv$SHqP7P8|lNQFh?@Lzt|T1lry;2<0|hcj0QLfMDS5#I*(2y|{WPg)o?dfdtyln)S%b zRwQH_Tzic@kN)lkB;}8_%!G;U^dV-cy<{fZ6eu~>?feqAkYBGAka^ zU?1i18Pxm`XfxO{)2?L=0M%g80M*BtNvXfe2xJ2`10DmB15tx?gCc|b-oObvnDnnO zV%6Z*fY6}ifYy&9cyM3V4{i;B4TcUBeS=DT9Bde<8@Lxp9e^3A8?f2MK{cp0V0Nua zgu7g#;^#QeehPmHR1=+V;H(;?TvrI3gGPhP1Azl+17m}8gJItQ)Q#v=g4v+gRiMip zW&^|nc)L`vcw+Rx=V0_;>R{Nw;ZsDowMMo73C_Yn%2gF?8&q4v4grVS{}{1q4SXML z`w^udTL_*~`+>U4kY-hTQt1bhzZPs;{c*#W@pRP8A@!BRs+Lt>c~0#=u4>P#_cpws zVm4zS?b934GSYar5_dw{tO(LRyJV!TTwH~479AfeQ>8fed74Ym`2I3Ewtw}@OXJvb z!zHFB1sod_m{RMQa1Q!{AKbj(bxb~g#JI{C@UT9CDQVk-fs|{YG>|I89rUK~}KTnBgS^iPn z-=+DdjQ`zFApZFua07~8hDk}0934vGz>y!nfpjT;+WwCqml8kAacty2Il7em&o7pL z11T>h{`$Q9pLv$zmmC=x5ty8pfARbqZmB2?x0jax^S@~=|KMv&$-nja8+9c={!4iN zGqpzisD|K-?;w7~q=%d=GD*+i5ZqVluq-l@qs`@{Gf7+p>GnuUhnC*r(n-4D1&Svj zU2w8!5<*U{FVaE`RibKge@wcpAzi1`VXe(huFp$AwWWu9x;DIQGTQ8tqdFj(tq}FU7H?J@Q+eWBX}6h@LdBZDtwr+ICKsIj?Q!y{4W1G*f1jqJE|Ga(`a%__$m*CiD z_$4@YuMD5puXpS0oH`-xOK|KdX^%>DIUKu}h5&qh_wqP)Z+QX7<_qG69NY)Tu@@Ns zVIWhmkC8)+C3Eb)+&)Ij`0r9U_My@o`z^__`Q}9-2UlVr?Yj$=cD zMU1Xkr;?5THk!t(4BPhECX(X5%CN5rXs7XMEF5!iBfd77L)AfiIlZGO#BK?gqT4ux zbYY0C3W)7S?^_^@UDm0bCT_P>h<%Po6*UqkFHacT&#c1O12$o7nz@w{#_lU3jE%+* zLmm9kKw^|56nTpf+{y;A;cFp?{k+^vB_Z}p1&a`tPUtcr_Mw7K2+>~ZW~#oNFgDV- zo)-Wzw?US~MyV*(3BmDJ=|*%yD9BQs5FD54gy73qgvdc`xtXe9GkqT|LKqO+PvcRk zgA&sxbP`m*-nl%8y(bT1H}%;>OqBXOi2Y6h#O|e+0r4*ku@@J{K6!A z#OPPRRcrlJvs~| z#rBqwt8&;@PrV&jHiZe4m1WPu^g=B;X@ZkgxK`ux!awx0(0g!|$EAJh*(Yfqo8prw#+FSax?aywt3L;cd9?!Ucm_=o@P~;dKkHdvW!^ z5c)2YsUs11XX1jZzqUfp5oQIEV`dQXrc!cK0xR z$8b%+9C{~VY^M$rt3Y=5C}n99h7nvNxL_~`ePhiO{62x}5Uy!hK;NjtU=9XSAlv&S zxjc+i9EIT{xL_~~{RQZqH3#AKFCXlFl62Qzj08qH-C4?&>Qwfd)$^&UiK#-eQ|1rW!{kO1w@F)N! zpem5AuM+wpwsMdJDm6eSpe;ycR;rnL;3N_?B*DT8E=dS!N5R?8(LDE+Os+vCXfse!wX}E#N0WEg&imwTLSL zW`SM-A%XhcL6|wae=N?NQqA{D=?@RSU>RB%?gn&Kqp`*up_V_FetFB zDGsS_Rr(=x2_6Lo1vm!*N$@D(EYKw2FJP!cTo|)T(Lk%7R{DXzfWm-}0Kq_?z^}lj z>Qx9`HUL9`UjaRVgMnN@Y7&UarS=0!fkZ*f6ZGjAhg6Wn1RR5iCD>HC3V4-yLWKZk z|1IiqaIHhC_7x>VvYBdzgDCAe_08j)LiOLEz9cZzp@!#GrFRRjSUD6`zbJ}2SyZ}G znOTL97VY1ZbEP2EMVgRO|J-zW!ju;kD})*#VM^;FgxJJL9nD8tn3DIaAEm>1ia;og z$x4m?Fp{GR#mB~rj{o3z(M5=b#(xEe@yzESRN*4TR33zShlDBfAXNQYti95jf`dWo z_~ebSHyDc(1_ooQfzse13|}go|3l#eN(rn;DP$mPNGW9Sihopt7nJ;?^yY;55P#Da zN}q&Kh)FOO9={PL#XpyeU&u8&DN+f+pp*~{3or}+6pJt=q=l@t<`49YG`Om*k6fc#( zCI6P^KQAFve)%sw|MlyP`Ulo9qcr_D#Gexigp~y!;b8*#$c1oBYwY5ad!$_j*_liE zm882U7u+$%m{A5egff@wm*u(O5Go%FB&6>ea_N>jjs?z19mfJDyuz#*@o)%bx<@-~ zfmTJZz*!j<#{wom;8>tn=8i+Ci&Dpdz)23FO!piM%t#jtLMU^QpF@Iy0tjV71P%!X z3WO=oNl3&Y!9am9Wv5)E=aArJfiR_8+H**7vOt*9Odp2??-U4A&Pe|p61-C=OeyWr z4$LU;aA8WbWO$u3aUQR2epza~ww;%{BDKwY9u*X_SwkvG*TjV>ne@%;+#dOCd2KUm z4X)EX4Vj1+XgsrCpKQHo%1` z&q^0XAk;BwmtWfy^ZE1ob+SO1@|b)+yndZ55T-PrKd)c!6be&H|Ga*^Qy@%f)(>94 z-YF2KG{fihtEoVk(uAPAel-;cQ=0x!zs@U7T$u7)2pYMu-*R#WOzl7AoH_pc*kuV* z?)lqO79o}a5~j3{|8fwjVEi|qJ4^$^jZ&MCV1+B|iW4d>W;x~*fQpU_mVm0!3aGl55gn+5t%9o(7Yyd04`6nLQ;myZsz%)b zcU8D*KorV|K$JsKji_p22m=zfBtYfVVc;xAs4lD`0I?d*yzqKEt~+quO<~;wLm0dd zLe*#|RC8-68J&n>3$A-{fl$psUqGllM5w&maP2@6EQE^7Sr#Q!9(@y>-iNCj-ZtXe zOxftVR6^yn5~{H%g|Qc5j3eMdTriM?%2~4qv4BWf5fzuTybyycFe`avu#+S$$0lKD zAygNjFG{F9M5wHimJ1RpftTD%L)A3_)lUln6(&VQr*_w~zLfx1E1bJQtX?HXbuVU38I=zBJPt@z$yrkkI}Sl9 zu&HG%!>JS~)0+-10R`+cMa+uM4F$FWfa1`p|Df`UL&-qy2d8*C;THlr{Nzq>CrC^| zfC?-NOsNJig&*n(?gI>USm_6S0(yeb6sQ+q7i6ev)&4JW&;)#w1)&1;;bK%kOptj3 zVgld6fuKN4KtZL6ctD@(Bo?d4-!jcUZ zuZt6^^GhRCr%Mqke>VbuAB5^$^o&JJ%1)?^9ZEECD_EiEE3!g?fnY7|rYOBb$w^kW z6$8nB2dF#W#pK}rE@oTL=6 zqK1?LRxyUL=fXe5EsuM2AP%KzJH>B=Ns$UzL-88`isGOD{apM4*6{cZnnLj#G-nC% zv+r5+KkzUS8WB9@-+*;IegVtu`On2KX&xnT>G?mM7ymoInxB7Jrj!&TOv>J(`8R0V z67qlS-{$3iuH^GKgtvJ4|6Z>A-|zE3`x#&Ys(%(jRqsKiAv4YlOzwh&f=S7haLx=& zJ1)3luD@51OOpsxl8#A#oE8)#RHh5g4BT>MzX&rh?J!rKQOtFI&J6O!qy(WdQ-XGQ zVM29U#>$z2U#7VTGce1jB%wMhedjZS{P$5YSMW;^swqhTN)W0&(!K4MibGtHb(nK|c#$_#6Xgvu0O zEJCPeWVz+nHpP7YB?wigJWjI&p}HuaFRx!G3ka3zzXYK&kDDz)sHSB2B?uLh=_^5~ zOncNXNvK!{wO__3mKGsYZgc#1d4%fpXJ57xs`)xZF2tnfm$pL*Lprha@v|ELU5Op) zhvW|Rw~B~KS=Rsfs-PJE<<|c$ZHE%dZwqf%QGQ#w0w#-&|KLW9|KO|C`0vMZ{jcQs zkFWog5R%9E@9ZC{Ht6fgbqTL7{kpnks=S)pYqOM z6Epn}rQCc#1grBZ!FmSIx$Ejtq3P7Rp~-M2F%(Hh!>M#6ZGxkJ<=?@~9Tlpz2Yuo| zP#eH%Ygytsp$-AC_E-Rx`BIld#OgM@!b=sgO56>{?*m{>ecKAKF6Dcd#sUTgakM5u z6X2zt6AH%|3<<{=2jK@341w_DK`yez0?uCg838QbYhCL1E|IwnJabTNY`zvVIsaLTrF0keU=Cp=Xe?_%Sjxp_jJ>G?yUg!h<^!z4?_C~S zc9#t{E7su##DkF4mbd) zk^)w<&^PKZkmRbnoPw$@Bv-QlR~`aa-K*ezH7;-}DO%+uaOKo(+!k;hKgi4CbKs=rFL# zS9uRp7)N3I2rQn&1%p}WxqQ{wWAOeIE**vkkpbu%br{HFLP@@=`v?W`Gz=fb1p_Hr zfH$)fh+zE z#+{5Bl~p{5wH zl)72p(z63En3{QGraWx!(75VxXl?-8y7MFNw8=vNj0W*xW{iDdCl(g}10o&{+% zpfk`b&@rGch*$y2-N5CmfW`o`K(8Q5g%kZisere@vjDsxIE7Ex0aQV<3ZP0H4hSJF zAg5}MqQI@5AQVMk$phrV=kFAi1g#*0^_bcZhDFj^{JVBfDj6^=NL)e03K$Cv>oA@2 zhqL}N$JOJUSm8^C#yNje>v`SE30JxAHw*|Tpqew=+kvdFLk(!j^C7-QUJ?5 z=jC$G`8)H91z_>-)$w7Dan8%q7BDbcS#*mnIw;Y_K_U7$___k$tFvseeShctJueq9 zzAqOrE@AvvWB~)&72`h?zy~t^LxA0=JPQJ>rHudh_s&ZoEXDu(pa28|5YPD{->bX$ z7AC%A?*Q#k&NLIi5vaxq1LJdWHKa7o&QqWisLDY~fl8cE8i(7}kWzjxt(@m?fj?Fe zE5sk%P4NpKQF&mw|M+Br>*%q=zr8&IKVP%0k3U->6_!y zV^YU!Td&k{fMwQF4zR>If2Z#-a)7PI7GW=5+n7uPuWe?|i_|tV=e)L=E_rPe->W+> z!{T+$FLk`OotC=%+NPYB<<9|DvOvUYM%wfGRrtL+^ZD}nRpgvM>7UoH!td2xl=i%S z6@IVItbe?I6@IVItUtVd6@IVIw8tZm=lo5D=y@y+VC|tH0AHWEymS7}{m2TilD6;F zeKY~qO9ktH7Jy|L|H*UyFp^t^Qo?z1rN0vdD|(N)n`Wt?1(f37tK-yafeq|M#(x<0 zZRFuWA;y0gz-^W$Tcl#ERNN~S+oa+?u7EGw26hBk3Scc|{5MbQevgbdeA*1@W zlym+@%#k9AQI!<2!d`DNpzKsS$B?Im0M;1}uwF0artg_jHoUCZ2L!nGTm@WL0ha-^ zeog^bPDHD{rN!6U%4M-Q;mV<}kl$~huSJg$EH4z&G8q>;Z!`}WKjEjmUxsyTT>-tmy(QN$c*j-1u=vV`eG_gGAuQz5)`!7F^kFEZMf)(g5q%hZE!u~{ z@eW?JI0s8IEGdRna7>K#TX%d(&C0Oq-zUR5`ySFE$74mlb=R9Gg!OL0F@g1;hW|y^ z_!bw!5)A88D2*$_u=q!2#X_F^)*YS6Ypl|3Gj^GOs0?e0hG3=;)*aYqO?aRJ6u=2Y zup9^i2E?%DpuYgUSHV?DC}J6SNgdb$45SR!1?UT;ujW)3>JGT7!c_xbWw^kw#(=QA zwJ?KT3SW6h_{!UaYZb25xV*50zEOw41sGUmug31C`0jz>?YQnhDq&FiLwEX}ph_?B zkKK9T-iOd{!*v%0(v0-1fgube$8zd080&M$6af{Q0`9l|wD zB>@Ac4il>cR`-(>#$h-)3X4Z@!GIW6=}+A`^~d1kDO@_dJ&Ncbr>q=-=4lwhzzVS% zN&3qBbzI-VH497V`KRvYU|>BS5Y4gp%w6Ap3&5(+16XJK@D7*&tBB;)51p*9A@69w ze^2xZAE`PHXjM+!Dg$Wc;Nn-HRF&+_3PY$$6OvYt!vc7MC{`7HtqLEh0L5{A#Z-AnJu>i0bBG6up0ECsqmqF8`gKx!Ojfk5H> zL0~EHDF}N3OF?o9&I!5E-?)Ng z)^kcfgsngYx!@H6tXtH6z%Nd$@I@{_1wCusFlAjQ{$u@Qh=k2|mg5XfgZ{ zn?Vxdzkq6Ppo0g+r_YyW8YtdVSO9Rvw~rjgJJoJp~z$E$|a(zVNe=JzbQ}( z$s-3T4cdq;v7MW-@rLeGBtP)nHt}fT0;E2dHH|$Wr|-QLMKIP zlqr>eF8|i}4c?a@|0U!eW@{6_Io{02_5KmT0*&2N>#ZSMISIFYii^wMPK zvLV$%*uaDWl?+Sw=HfmF2E`ziTQ0L#arY7p6u}0a(hiehi@XfS2IumCf%q5|lkPb- zxF~fT8_dX9IX0M)b{tZf7@!EGGC^KRNOe|*oR1CiDxK0R853s+h2Nqw?MpDKg!Es6 zQMn}}D8Z;sOaCPpmC3+LFsh5ve+fo4BkhrG$*7LWg5iwHPixy$0Xd`Uly-RS?O#4K zA5J;h^-7l&q%*lvCFw5pTU37e*m-3$7E-j2|DzD1*N*BDcnbn?Gwkerr zUfIq{m%Or>IWJP#7H3pundVnE<-9Ea5{$~MA0-%-`TR>Ts+qj{RmiCFg{%r0m05pz z{VMzxm6`q$jLNLPs6>)c9lHcZbxe-$-n~3Vwdc7dGOEiQ-(jRtYJ7*$Na;OFe2{5d z-zz=7TbNND`mh*PU+!B}FY!Z`+ z{AZ6pRUv}nAH{m#=M7$!){0++SwG(}%Yv*rT75;oiPh1n`TT-%0cMdDO7kPxFJwKj zia33`H~zoy2cOrfY0=vK?Hz6HzJMlN$NxXrz@ffkFcVHhli}3p_)s{KOhyi5yPSFz z*A7l*CeqoivwAhxlC!}~woBEkxx=twU!z|$9@!sGjZLOg;nZLGalYQIGF8Hbl1r6 z{%|^y9EqmG`=ZIz=;TE9zSG}PSjLury=wjTO>1{;+P;3RKP$_j)+h(fr=jFH9OMNN ze-r~n1aQV)fH9|D&c9xljnf$yT`W#VZBV(D52X(zqS?3?dR0& z&Vvkvg6+XTFyw1D%Di|4@#he?2wjn($Db!N3OfD*RKTtCtjXOS|6} zY;Th%Sr#i6$8a)=?DJBQ>9%DbWgKb`_)rE51Qh?anSGhYBEfV7e689NgOSguAQ&wa N3iw)DWVWKd{~uRSWxfCa diff --git a/extra/sqtt/examples/gfx1100/profile_gemm_run_1.pkl b/extra/sqtt/examples/gfx1100/profile_gemm_run_1.pkl index d7e1cbd2a4e1fa882d6433f52138697e71a56f84..c1c3985e950ce279e97c50105658a2b9a4b76d75 100644 GIT binary patch delta 810 zcmaE`ily@#6JG<%)PK7ez#w*t#$-bi;mLJ&vJ-8&q?sLkU8eM~xkgigtj?qLcJoRT5y&6ptv#PZ&384BJ^8H(O48A`qlj2X%x zLS^#1An_=yrUVl)cYC6#V0M{TlXa7*Y5Z551DWypn;*%SKWz`8dg$3r6 z`@!N&KrO$66$ofyV1sFi4H2(Lay`F$kdtq`d#Go;qjQL7m@6n0SU_UP;lKzC2S#8x zFoME?Jww@>BSWP}1n2>r(XqLs|01 literal 71008 zcmeIb33wdWec)MD0LUf*5@1u1A<2?j5G6M$sM!smanP0nUZN;~2S`Cv`bZ$?qQn+SQB8%uXvDE&$A)Fui9Jqc$7bZrH=bFJMQ9@16CdI3X0rZv`EBT# zB;(0uR{Q(Es#jfo0U&kwVv~iccOUiY)&ITU`@fnhKlSG;O+$VUW`gOO)ZobB1LN_& z&_H5nEHOTjNw3;HKH5Jxl-Ls=Igr@;aAG8tc|NoD(acBizN9bl@L+EulP>SsyLAtz zk@#>TGhn8d-!~rb9UM8(nK+#FUDY=@F*X!G98bp)j1DK(e<1$B_|W>^(Z0lboiooxVq{`;JT!JVv*Rl()4tdv2eH`NhCsb!yngM5Oo~K(@0m!Alk4|pI!z42qkWS@ ziT7sG%ic3QI5D9dQkf?+ll#-wL@JYB+L!1Z9F7lhwZ*x5K9jDAHnl|B+FP2NTN_(K zjZM+a6B(+;N{nO%ZpykwBhAgB_U2aQI=O$K%&yXbB?C(bd;`lJ(~5zW1J!E>YIY8+ z3J&-w=D_MJE#1Jio#wzh2I}k*4BU{LnU>a2q{+_AKz)u=Q)62w($=;>KGVx%vEg`X zcycHf%e*aFZfcH(8e8pra$8zURNb~_ zb#HR3p(WZL2}Rm%r<`ioPt4ULlf(NH+FC5$*5;;YsJXdF z3zm4_nTU@M9S)?%<0BLOqvOMYl%07x!Gg@UHb>h+%`L^+Tw6;x9BK(S6}CA^WJUkr zNPKAUgNZuoKK^aM?I(A+9%Yp1rFTI?>Qph~L76LhD&iNOBJ{`~r&18NC{ zqjt#LNa=K

S)Fo=38}Ox?lFh1qOxX$*(j!o>1?=jvy4FUT8wSmOYL?X~%|Dpboj|6BAfe+;d{z!d8;9y8z zSZn2tL6=-cO=-#nbW<=%f_)?b*0_>yEAW zZ4A7J8YQzaF>Kj*-^S2^L@KzJVg^P=Q-PhMBZ&^N#8#{ITF+53a+!C=#aL^7BskyQaGV_o%W5!Cdc~XsYHzGO-NJMk)}|jk$3z^Dsd=}sdIV|D z`A=bcYjcYmUOk1?j&u+Whnpjz#^O)5=aE^;vu#JL#Q+*z=Z)#w7_!dLVQEv+j_gj0 zW_QPYqK~w7k$I%Ic4Qt}aFsdm`{~A_@dZ6iw;>?G@2Yn)RIi)OTh|;1{_w4Ah68`p z**)-ux4wC`x3sr~+M=F*)7u8xLdD+u(h>WRrBZ{jvEkg;MYNGdMQ2;2xkj5J?d>>~ zc!REzH{D=jB1Kp5$oOC?k=I(IO^q#$p~l7{@6c#866Qf@Eb)4)q`M(fyX(k#xCj_+ z4Yfsz$I-)IYNDWRbIWM|sw3SRp zT1zBAUpFxjA7i-U6@>{C35TL>jfDbAOG_ge=4D+p!%2ftR8CYaEp6>Y)m}uZZfR|7 z6LF<|u~D_QHql!|ib!QGZDG2vwkA8n`7ht}O4Uu2s647JN?R&+cMzW&n{X;2B~;1l zq0}9bkxg^4{**|Y5EPtoMnPVy$9aqPCStv#LqqYgiM%`08g6T94>gC2h%~K{a7!eF z$WydB(>F~V9^OAXG}wDbUIKUYCFsp^8$b&pS8J%f-M%$B4XQQLijdjnI2B}n#n9mJ zU`m9s!Z!gy8(TwB=Z5FT+8A!7Qi=^%l<27BF*W!uX#A~>;Wk9aV$GtpG1Amb3o8`X z2P9U%olB8OQ=>>eqvQFt-N=K(_^L>YYh`4MY}sC-#Z?_ljE^LS0u1#BQ@M59+T4cd zi9D|EQ*JG^@HB)X;S!OqP_YHqWNtNN83X!j?`Wda}7OW%+77B$Vh@tyd_#(<{-_Mc4UU)U9CqlSe+Uj8)g0@ z6;M){k~OmDgl9$Psin+jhO+|Y7*f4Uh6XL$UM1LS_VT>s3@Zi1T-|CPtFVUhABZW#as zM-q?3CdPYXBOC5=oOKxGMD1zsYQ~Gonyg5#P%Tt@%%tygByK0Q7?}w@5+6FKeFSB& zLPY%mv9*UxUcY&{wY{t1(kFMEEl*z~Mb|TP_N_w{QG%EV@P~>IQ0$iUY`R`<>QI8_ znb%~pW0FkTX<%x2MVvY13uXsq+%et8wREe^(k^*F0UT-4xd+r|8?#P4jk*h4JL|jLF9&k~h>3CZ80^)9w3` zMR+ah;)KODI}m zswur{EIu9|o{)Hpk6cKSj({79=n`oO(`^jsz?JY9kUa)Chv)35_F}v;z5*4pVzQZyM z7g-=fe@^7Uw;w%R6)V-C;;o&>^>nbzdbAboU}ch_EnK`krB|p4oK11K(#E~_c_u{K z8^h72Ryx;0OlLc_%B)6lLXh^`*m8LE;lwb*lRz#JmGIG)CZ-sp`R^KW;;~@fxkTTZ zyaO>sO=PYVR-yd2M=4<5OG_MDnKEW>yTl~ZilNbXA0wQC**rTB?UcODuC(m)X~zm? zH*aAYtw7-!bXnJUb?Pp$2MoI<=z znl0f7@AM|8o(m>|s_8;{55}0sK9Gn-VvV`+wuIZG1TKK85^prZ6j5u*xzdS2hQZ7T z+EcrE6+xz$Zfq);%2)9=G110kYVq2=eqvy-KNSdu8`cIQEsUH;C&w7~%e-t})>~Sb z%WWsIkOGz9n`iJ5vfR&x|yjWX>qqjE}X>3doHZsW@Ek0+Su3<8ai6~~n za_*pdd?inoo9@D5GgTv>`NRPi;wg{BD0hG#K%lF&k*FWNQ8|oU>FS) z3tGggcwb*&Xq4H6A_AAh3}paY6aXG^YHDhuc1q22uNX&Ad01$6;q7W~YGyu|I?U!* zR}J%WZ&R$ULzwM4~1S2lJm5+M_Z#rS!#{J>UHY^b}hvFCIUfO z-nBNbdYLMxq|WlF+_+oXT9K4XO+Qz%BrVa)gf@fsyf~S-&XYEsDgbUZQRk)b*0pN# zGQji?9I7vIDA1QWJeEVxsDyAU{EYrO{{?|l1SvP_$|3~~VzsnfC+6J7tCo`45wtZy z<_e#RNTj)yE>HJN1#z!ri5fFZqZ9OQ@U)x?i7-zR2LBYPVomnp+G;kGB>v%lqS0N0^mor^N*D|;Zi)c#+}?*KQ1+q+e1ibvZX_Nn{b2` zJuG!f&Nx@tVu)F}TWExZ9ei%gk@ z4~gWGE@Or~^E^v1tMoEYSqOvuE?l3_bgoouC|A0U>IuxGE4JLnlPCf2eVt#YkuHCD zbg(b8v(wy}%IwJ`Ys(BHn5^S(XNyFU3~=eStu1Q1TG`%!-?VRcYTQkwb46z$T|py? zv;5QSXPM~ErBCU(k|9a}bMl5YxVB=6>;bBVW1ZpPdTC-gjNTP4!=6hh@CE_unV zvu>>1lnO7B66DEDQp#f|;`&K;N^BDTDSZc?$k=ymxjk0hKGHu*i*ptd`_kns_@LD; zk8?j`_6uxny1H+4Bo&*SNW_#xGLd;YT_r091U4Z|$+GMfau#T(RuZs4dl~Vt>{-^2 z?3~=M^W`ky`$dh07c~e>H!q%vI?GuU5g%cE?k}ApKov z@AjRY4FS2@y=T|9JsWqhX3JiCCE%?dlO0R|W`kI5P|4fb3u5^|_`#5!FgQ_I-P#5< zQU;p7rLe#**YoKLB`{^uTNOyPLFD)WpaF41+$r+6fDx4U1@u%{ts!v@pAvt05 zUOVQOd6RgSC1zN>J4md0LM}8mIu;DC4XIdz8kfD({MDa3M)qecj_lu5u>3^{1#PU= zVLiMS)=XDvWB2IbNGexpV_-?TY~zkCvRb=5aVXKtOP2Mlna+*rGMJ@O>+lUHQgK#s z#(DgG8dl6KN*39!H-GpBK-tFuCwqx|EiD-;bOyUQzVZJT%ZAgZPM2K!lSbyM05 ztEkcy$e>7fyhF0eFX`np%?*3E?AW*=Obhi$t(CH9Rn}6dwkwNacNjHj#RSvsi|Ewp zHh5>T(q(dNj3qPc2YNrCmXEKp(o0z;GIn@yB$Ga_7y(Xtae}=*gPyb@@PWz1WJ0|* z?S&Osh6Wx{L1T^^tDh4Sy(|#Wnoq%gwbE8H{UOz#Ngxh1reqyC(C{wSYJ^7iALd<3 zubvo*?`Jj3UG(~QZ|~Zw`s_t5!u!u)Xn-(UC!>PRKxCj3unY znKUmDS@``xFY75}#eT26Dp}lvkyWFt^XCPfk_GW#927_k$h#=%Z%ek5|5IMuH>4|7W+Zi~+MolNRSlQQW^mA6YGl91 z#*-(BDZQgWL@Enz$rT)&HB95Q4_=I-dO+C_+E^>!5V)H~B=%|)N2Exe zl8Aqugo)%Qr9^MKB$1qvXnzCb2Q`1fj6ZV@5f=2?uSM0vdt8`MYcHG-zO2r;{RZib zlm9@q?pS3d|52*DmpUuJf%Sl>uD5o7kVA;5^|fcH@g+Yx-& ze(V_$@E|*n?wHfbBlX*_o#0qVFJDxer}UDt$Zsq}?C9Z)h2oPe{Bt~u`p{itI;fb` zQaedOuk=$&N?52)NsT2A+dawy2yO$Tee{!HhXS@zhVBJ>gW8%N_T1x#uy*s0t+4dEv&KaAO;6kNT{GImL9tSvV{v zJ)N$ump%U_)u-xZk@?i=b`#F2GM%MGoh2SfBeM|=zv8`6R`=IzWu`kL!#D>QG^QPqst=6#zE)r_-s}D-#QLfOle_~zziTWpvbig;`75qf>|x|;_|i$s_(o_r zekdk{H}=Tr_(3OmVj#03n9q7qz`qWSjSoJoR1FF;XnZj=abS#9*D{pwf5kRTu-t#h zwlY2!vSHbmy)lLk_I)EH>(9~TRvAG1LI}2$!TjS86P**WzQbf>uvauxdk?&wheJ_;g25+}p2? zt?B!FGW+xDC#=Wm3ICpMpI6oW-!saValh{Vp0T3c@S9J4>r+;F^QV^hn&sLw-t+xw z%fg-W63bZn&rg5rq}V5o?=QiAX?c0aYU`n;H+{a2-!Z1XX&Cqa$TU7|n#L2~ zGK@VxDl81 zlGc=w`N$o{XNlvVe~5!sW1p8;D~(4#TK?3xj8BhVif2U=$Mf%809s(WP^x<62|7IsIR3oVv7Vvz9<|KIDQ5~R&>ccL-& z>3~jr?g?Rm`6S+d0jI+buoVXX4xsKC97v_cCN`{Jk6~bPe~9^-^?TxdiO~`Do@9K* z_5kaLh8`Zilea~Zxu^gyjXPQ46N&^Hqs`6DfkqZYG{#x+(-KZJH6@x_dz<^4dV5>q zef{lmcHLl=bbniaBC^)#0_G&fH^>}lpsQ<(lZ`-d{m>}N(c`hPcxpi2vat!a1Q_5Q zJU%u&A+u&NXW|ndy$9nc{T;wP5flB_156V3kH&ao#ux*~Mkn;_=frwuCv$SH4408z zw*re_*w5I{Ue9;ESy>d?+wYB^y{_-44%g}Ucf?<9ZS*_#b~>)L@nOe4Yhz!1yoCM# zQo_DQBDRo%*V*Cq-|ur#?0x+=*cW?WOJ`o}eQku;y(3|S5Lgg(jL?CR$lt1&_{C#Zajc!PCfXnd3#-q*+NHbVO+CXA3R_J%~_K~5VSo*c%KCM(jR zAp%t@zMt~|@!D7RIrru65!#eUl+c)bw{>IMIjAU%0 zg~0rAjULD7RCIr(%@S7nKpkH9i~79J@hIJ!q@~Agy1%T(?jK)&eN>0n{h~exo$%US zhj*-v>$s4(wY%;|^*KMh&c8m(^Dc6x4zK%deb)UpUequDd43nR>QQvuQlaXJq{9H2h(!4J6_l0avecl2Bcq-u&y`VA2=PA9xv^Q2U%i*GVyDB8IpOvA z{e%vPANl1aZ|uM>C%hi-JL%{9&o}QW5~rl2)0b=SFF*VN9K|l|^~ALf-<}sh8}(W0 zN8fvI6fZe1_BO|(v{NaPu0P#=4`HY4&Kv%xRkrCFuO-(yU2Q$hMX~VXD{LRv^WHH zf>$q~Vq}>~SL=DS9h-~Jp~)VL<0E7#U7;3B2*OX^&-0gFwtL6sy>Zc~_O2CLxwkvE zsb}x|W1HWTsF~?3yZ>cF5wi^hz=RcX`-dLxg>okKc8dPXU5~|_kTO?reoIU8y)j?&iR@J z&ee|lD(C$7PQ0~_d8KpK{{E}uuFaK>yFUNiabN1#Uz>N<}7kk#>^3T7i;TKzO?ayiMS5Lp78c6pb zee%23Nx#(jKF2-joVEEojydI=w>szic(nig@Y+0?7q7NoD4ce8f?x4T5A#Z$o!6#2 z6t`!8}5#0s*+YTO$3wx1o)Iz~PR3}i-$)r!b`MMJYIuklG zj_w577ODA_C+xMpamlFKy-}9yCrhpPr5Hgp8D#C&hP&)7pNgqLupE{hJGH`uqYB|H z=*@NWsQSGp`AH;<^mLA@0E=wZj{lNkcFX_q_#>iZ^vR=b7~#e-G*iQfA{>Q=vVMuvg_f?} zz+9MPo4ZYtcvQ*g=E)!ZkVvYrm=d9Us{60F6d-9B+ox@5`p~lJWy+#^Ic}9IYXi%s zZ8(FYEt6woFW2S59tCh=&#K|aUg3oe%YXTCu{s-Nt$kTT?AIEy55VV_O}|539G#W6U2t2!T9HoZn)+@>#X*B8PY4TCs?+Gef3kh(RD z?<||XL#5I!71NNtywgs5CAJ~^@_sHmB#iNmWz!qg1%=zBE-3eAZL>vR*dssu+^uc4 z>5F^R1);WUn|t+zy+ndgUD{@c_PkTu>{1uqD`;2YvvR%Gim3CUWmcnO(WK7Z4;dd| zF&0gd)+hC&O=2-~EB2~=E&I`xw6F6&BbH^;uN&7$Y_HZ5N&L3%__}V4V$<<|(9T@< zlkYBY@4*-8OPkJX(5_8|)&2E_JUl=>Z2ERulJsvOk<-hp&ENia|M_54J$akHJ@$dS z|C@7(pQ$6_%!U16Zc>-ZE|9g8OJygsU)}Gb6jjC}JP++W1L8Ip#L90xYJBVLb{hC6 z9>q%9sol=H16BxVc9ja--GyrhR>sop(vSj9So`6ayA#)?YJs>smL}fb zCy%9E{9GPN`I%v9z8^5&UoH>s+-*jcG4+#R)sm^7H264HPW@yS3tt+GSuAF;XfUUK z(zR54EyGZaB^I+-%wn!fW6@>dP|mTOqe_BfaS@AIEM~E2D4Y7pPzAoLDlx3U5{p?Z zW-*_`Tvt_(!y1lt91S>N4yLh~#bOqVhFdvq6Q6gG;GI}vF^h#SjRhYS!Zv`H+c|<9 zAzU&0(pb!5F^fgREgbKXIM#Bp9!o4{v6#hNm&T&21Bbgg?%~)emRMZGVit>8EQU4` zXVo@*ZpX(iEU}oyJc~J)#-eKz4qG_x6`x(WU~v(PSuAETH*6-Ft&)pQ3_GyIVit>8 z%(Iw-Rr_)1<4AH0NiMLsh{Y@xvsg6rawH`FgTytAB^I+-%woQX*%$f7b?-Kes(twD z=NRA^p{g+l(^$-6F^fe*oa2DxWsr+eEU}msi!>I&G&T)W98Yo_;W&mX=8KqTG0$Qi zdX(cS%KbFQaqKY9Vm^o2m&T%Nnt+dUJj0QesIfSQ#Vi)HSTsB)C3!+}aTL?D*kUt_ z#VqDo%)zQp6VGRG`~v4$GJyFa=2^_Mm>d3pOni3gC*9kO$mt_2R^gel0tRoA-+5?H zfR!_q#?D*$^G*MZd8~fM^xwkjS>xE5k%wNauQ!f~(F*&H8V^16&{5;4(I|GmZyswd zH#j|1X&&SMttNl`x3cVufc{UL#~$L(&KJ-4*BD}39Em`f>g<2rBO%3#(CedGe?b`l8>Xto{n-bxc!^TiF{M}cl}$+{rfAA z*Z2OA7`fiECh+xFu<`d+zT)4!Cj9lk{t{_!F6X-A>qOC4dEDPu=}%O?@^|(8;o?(Y z{_@|+Z|+`tqu-%)^f~A4Qkwo<^s!RTdybv?xs>{zWAC%}y+*wsHE#Z~;SVf7YMeQ< zD~N5p;QkatzoOaeqD{P%sO`F{!bX!jFyg7zG}|INDJKa7_nN!YMnQZRo`fFTm5}2CEniK0yE$BTNL@`N+U*Q zkL@~hh9q8m5yv&l{f`?j?((mxAd4*>exJ2wE9P2YJb^Z)%nd0zGJ)mW$VI!OT@I^T`?-S(x@jfN$Z$Oto|{hzP^H% zt5zSEdoZA9N;MC**`V@7m|5uG-oPa4rFF3jjnM)Yx` z_PEiLHljz2=yId>DWWr@$Bf$NjcAQg`3 zmRY;hiY_s0%gpGbMzqq3UTf4=o6(a-?Gr}y6XeW{zCcFJ+Ew_sYHN+?b4K)(5q-v} z{XH^j*4|)5KV?L(H=<`q)Qp}sYTsc+KW5b4Xhg3wqVF_oKWRiyo3$S`q8X$1+v5Y!u=O!{>MQg0u zQ)cuzGg@cXzJsu4?P}68Ygbs&XUymsBDA8%Fq+ZjX7oA|vTAQIqiOPBMps(3%gpHY zX7m%JZbf}o?Qy(O4`xrDRa$*O(Sj2^L~WmeDcnLSTiJtxds)9QJ`?5VJNo-%u0Fr$xK(I>2)>#S2Z zSnsRNSd*u$Q>U%y_15G^t?u_3l#Rv3=Dt=MhBOcvP{nV*(G1`3Scq&Kzg?ta^ZZ>N zJJbKZJb4($oS%7CMm7t|$L5I*c++CGos^2l+$}>0k?&9TlhmB08oV?fo7J$|H=n^om=kvD#%CQ6MT@tik)#B-_fBc4l-AKRVr zLjM%vGuvXD_r_xIGLikhhinIU0}(|z6!!R6e4vi-{C_iex8;fJAz~nNr*X5`ZQ&ozMVl)iJE)O_A810KwVbJ<&SK?i%9>%AA8f^^WSpI z??j(h{<&=VyZ_QnU*}bFwfOQ+<(B`(&c&2Jn||^7zl`*E|4B~y?;-u}1?zvI^e^N7 z2maJapSwL{zB*3&<_Vb+lABFc&X^rR$3#P(F?VY>g3Xwhw29U`V_uLsB?$*_o-y}n zw1a3JRx{?9HqqVAm?w{^ zbhI7o7-q7iL(kO{GgBTjWXJiLyFW}NOWqjC5zikGk9L!D*8O^MfAvrD-8*!6CR1_$ zxB2cDKIFvzPUZf+eD|;pf1|kje_GNVHC*cRf8@Idba>Pra9`e)mwz{Y3PbsBz&%^b zs(tOAr+r<}HLBXzd3`@r`*QEq5{cPuJxj&(>KaMM(4kfPa$AjRU#?q`_T|Q<+E<58 zRkbfS52}5=dZ(mwA*X%q*8Wxda@%o{_VtqXsoK|Wo!cVq%ik*LT+%*N`*LGe?d!aD z%WGdJ^!-=u%il#jIqeH&SiAkH_O<81eD_N_ylP*2M)KWfw7Y6wCx5r3yJ}x2PvpCw z*Wp$BdiD47-QE1D_Vwyl^WEL>lbG$>r`j+S5%#0iI-;RmnodVokg{NOU>KLik4{zFJvQ2xWSE&t)UK>kyiD8>(dndQIkf5C0*kXtjK z{6}%~WboTW^E=)s+LQ)mH)85UjuWCpHHPKdPv;)EdK$ZG3#$-UKv$ zw+BsrooaU!xRxlTw;Ygd&I7Wo4s+}f&#dOM@+i;qd&406!d;gMvgrqrg!s)B0ogbr z?(m~;EdC-=y(K~oKRnwY8_$R`{EAORd|ppP3_2hii_#$bmt!I#3$iKP+dmOOqBh88 zrd5ON(i0K;c>9zB*&S~XWY7GJ`>KViA|QL}Z35ZvqaM5<`}0LWcDJl1QPBGGLAIjS z3xe$KTd4~fxX{)ETq1QNSgA|Ien9qiXOeC%3$oj%HyTEO01}pABfrIg><%x;CbSE( z>#BSJ!IgkPgJX$MNGvqTZdeK_TqeM=8cUM{3nAI(FxORC1TE)S&QXN}=3p9&b69AS z-B2OHE9K7$Y|A*X5tKcPxvO5{Swr}B91U1vaSn4`8VfJVZn#5y-ihU{9Jg^`p-6UJ zRe-R!a|Ag;xL}^eT$jeei?SQm3LRaKi)!i%ySwu;YAYIO$=n>n!1D0^tX zIQ9`d$uWcj7Qr+Y=dkdi?1qC< zT*Q11vqsqspA}DEglGRtBXYvW3#EJ2uLRHDCqsr`OP<}`N&n>V?5QqQ${e1JyzmBy zwqWdIS*nfTaG6BA-lo|t0=C}}(H4Ad6;kb+-&CUQcWJh;?ee#jX!~<%b_;-8==fEl zeZ@q(JIbTr672zA+LPeyGUF7ew!-LvXalm#jh>Z?R6j1LxyKJ!7AOtV{s&(cW%pDX zlhY;yy9a_@1<_Tw8)OZdhF628;nI+8aI|UmK&Qdr&}kqx1RJ;wmwv$zE)CfRO#_*s z(?DkMG8`D_{j|c%5O0w9O^Qor%v#X&^@gBnxH_;JJPzvyWdo=o*uZMwGcXyl4c3M^ zU$5Zv4VI8+7&UAerVXzKR)fI>M8nB}(iU(#3N8j#3xqeLplld8uo_?v^ahw82c8S= z7A}31_!YhexkHa3;oxhiICL7a4YEF}P&SAg=naI2V8f+vQrH}PU1tfw25}ogts&I{ zu_4$~3Vp-0E!aMATR3*~DWm5x6BZ414p;`r8WJ9@FneycdX`!}5OUaft=03WDFoZJ zqBlXd&7P$SVn3zmH=w)1I`xDC;76?2GQe{y`hxYfQ`Y2Z>mNYceS);7n4Q1WO!p`) zWGQDs+8srZ_JzeD?ZtxZeWgJ5dFiR&0LTu%zW`(_GV3gaUfe_`!xdX}R^!J2IpU)D z_|YdrOB-b4XHg)V;iA0|T8$sy{)x=*%nz~^(=GtnWZD+{DP`XHkqX#C6{$+vsxL}S zWD>m8L?-db%~$b=RH80+>WjsXA3YP9-T$0FkvaD73PARp7i7P5zyS&625UV|epW#D zD{7)r7U(*hv`Q{*rfjgttNTAH59M4sOxGv2Hq)(=OPeVd-%v>vQ2#MceANZ1r0+j3 zAUT`9j+6AeZBDMzw>_xz9q1~F+Du#0pIdzT=d&PJ%KuN2{&%zGx1-N1|J*sZ{M`=p zmqfGaUq<=Ia?1be8bF<{Qz^7lr@7^KGN{U5H2paT`WxK)`G00ub~hK0sqX)tU6Q`1 z{C4!J{C4pcb^kA;{`akP(x({$B2%Pq24sXTc_RDsqiYxIDYUpx!?Yz5ZboA|I{BF_ znD&Y3(Z4@vHB zl@w0f^u;~;B2WIqa~{YhRG$2YpUaT{Vj}dZg^Y6r3mGp9WcRAKF8RLD4XiE+vQZG@ z<&3uuGn4!?y2h&zJMY6yqQ-p{VqewJPVsRp_BptL4^CEdoy7t}=FI|x6e-a&ZIdk5jU)H?{z`EwA|sH2Qsl3~wz7Z*kAe6k3ytH6}vdb_7%g} z=bj|VBSho?Y+n_`x(i~h3HCV;!LG|E*ypCDIFA$j8ICj=e2n7>DaujF#%>4;Jz3=1_%=n}Z9N~t!2sGwJHE2VQO`jo<_6vIM?5+((2dO_(m zLZEyK7=e(WPB1KZ6U+!rO7Iq{oUkes9HDGrDJUD%3H$_Uf@Gof2t0yJ0h@qaCj>$X z_ChTZ4h3KWtD!cEx+KU6N`;c>lP1chLW@w8d_r_L9UV;=li%tArNOgMj=WA3Is@P* zpcT9c>U6CYMI94GNhlQ>mSswn6W(-<1&tE*N`MsVl@&xN%9MhoUa&r2t8^;Caz|>P zu})>I=+o9~r>vgS)@xv>2Q-EXx2PsIKZaUZv(gN&A)W=_Z|2!j49cB@xI$&hj^|#^ zSEekK|K#1Icl|)dD6LG1Y?7x;iT7$kE3wah#o?zEgIbLI_x3ZWAIv`ou>gaj0`lZP zD&Q@W|A;wH{_D=JVJx{BkCOjN%|T2RGN>G7%9rF@Gd6?Zn{Ov)g~8Z}$!Ay^F;#Kt z;1Gi^70&&yb_lt2cmgzA!LP`kDb{RYv)R>+lUn1e$8TO@OGPE0`V9lUh9}I|RvXn|u0Yk^EsKA&uDJtO3 z@-tSR$*MduV1`rV@Bo`Zb?DG^pq`vCik>lNbiNc7n9)8J6>#&c)Z2DzpNc`%1bMLJ zx%Dup8tp?ds1w?hO9gVDM{`Wi(U)LQuKN`-s2UxA2?jNz-AgbicaFaVgL1={U{G%U zDL0H$F(|jrRh!FIrVQ&^qw7L7R+|_^d+v0R*5=;1rQ*|OP#5&wRBdjzHmTO;-dok$ z0@_Wrwkd5=tu3rgsAK zvzkwZ@cgkE)zfcO(K7IkfmOkv-gfw^qUA5uHo{jGE!V9(_(%Q8pN(gUg72e-V#4E&aQaOc@4IQtZvyQL#taFQb%An(fD9qRT#s+VQQ z;KE4?2~+^7oR<_bg8h;bPxQ8kLiPnxkbPlKQaCRu=-;JZ)ICjE3#v7yWY|wt7xgn6cTv}Ma#>DR5UYt?(nDXj=?#KmUC;eik4kI=Ko<* z?vBCdu<$}EUlo*UC=XKk(pU^x1TE)S&QS#k!CV?rb*&)YGL8z4N)F7vs(PYXL-2JR z4On8nh}oB~W;t|+_`VY#w{qMDufRg7Sq=pV3X}3`Szg4vXhv0_WO=SbaLV1Ja}UQ( zqTVPdWg8dUAu_wL#6l}s)}^r++Jxg4j(fT8B5}+YG3O(ypdhMZ+XdHE?I-F!6dGT; zt_ggU#M_Y61J+8_vLvaR?IoE6$3c!^$>?H9)ewwoXp~TKjsuby7Im^+a7jj`_X`#! z%b+l-hNn4>b33pQMs*JJIn2Idj4GIZoRps7ND~Soi>QV^Ert0EzP`XQ ziz61nG#2@Y%GR=MF(QFg^$5cEdJvV7gQ#lu&;bde+9ZhTr-guu;gHa&RE;s`2VAZ4 zrF=&BpQt`7O3>F(g$cX{pqg~Gq{?K+&r{{1B_(JIbtyoUU(}^ep(_P~f-?c76uXK( zYn%dHrDdznQ@|%QnrxjYG@u|)U?(V3ho7_Lg5O01&9J)q45QX`j}!=fGCg^bP7@hc7jNu z-2{|^X+2}BR4Gygh;kLKz*^{5LCauffHCOQ2}QZkfPzbj29$zW;Hzq@2c0hH6dGKx zs87gVoggkz)`Cs}w4hc%EHt}7Q;;pt6kMv@>bYJiRjYxO*-6szD; zkSYKbXzGZ??x5@@dg>;t=LHKyHF<;e57%2g&zQ{cqbt>u$wI2R4uMpsQHcn*JMXon zoJCiv=6=UQR$DmRf1bU|V`o}Lw*NGDFAh?b-0?FYB7h7cr>|_s w-QAJyKcKqBV zJAQiQzw9K%6+$ZBOF8nNXU9*R-Yi9_E1sl4f|S{@c^ArlM?)%AF$D`{J@Vh%vQU<4 zy8`*I^b)0crYK~QQVyhgS$6!)QIwkTiTuW#Lpemnoc?2R{}0N-hefx|psKiZ_5n3a zY`o_GVPMi|`zt5)c02Q%y zfYRlqANWR2`F#SZ?C7QZ4lk1OJNsa&@_W*^d5tQ+Q&*(#>{+^)@}JL1-)a&_lr6u* zfK>h43@kr=Px&3CCMmxI)D}_yyZ=>A`5%<@v*mYqUEckl%ie#7mtFb&zw`lTl3D?S zO-fP91uzOyxs&H)G=GoJvwA|l2sSvO-RR+GvIt56gK4EG~D)&P{s+V-ls5Y0o zqpVwpMOvGCcT{V0@7xlJ`Gmf6OU3lE+VRu9)2huazO~KhxKwKkXp?GfXSGSSwwKk8 zpSh5#L;EO#MBO|WX>DV=R*JMXOYQi1LHkf`F04(ewYhG&t<9X#_gA$qYk?g6leW>;|U9jWln2ukyua^rTmFuqB*UJSve(uxZX(QEKiE?qn%QrN54cqSV+B*_;PbnJ+&|i|qb4rTn}}5Ub-V#Cnbw zR_E2-LQ@lW^^L|R$NJ)_L~J4zPr2;qSNv0$+1)~0+oL%UE31Ol))EOdDY~_wqE)%g zu=aTvmeTPM1;~mH^Xx6^xn3N@I{OA0*31{Y3@cyJ%JbFB0*0ktIKL!{R-Uhx%+=A~ zF|En3w6gD!MZCj!&q>`DUudBwE(*R{lAXhN3mMjBy~E~KFu)0)m12~nSo41MGH{#2 zu>4oXuxkD*NxKZI@T-^m{zk^Yr4_BhJ%nG7(atT%u=2lpS!}b=H=<}IVM{Zt?GD3I zb8SJn8%B4405-!aqG+|~uU^ihXcds=s}-%f?XOZ_N&bU9v@b5ubd$_NUxD*D+=fmlI-SPg49)^Rit3Ugf=i;Gxz_X;h5 zSQmjQ=rq7DJ#b?|z{di=nKbMp+HJ z2)>bH8^?CZ3Ksb&>)a-(sxA1qm!pgLH&Y>7B`*uHtcE1Mha}$p5=S55vGM9#bpfrE zzEwWO3JS$)=;s*V7$FqqI?=Z}hlN+)$|tH;U87jWISz;~EEL8Xnj-9z97i~g;fnbp z=8_a^_R%cGx_F$rm?q%k9M5p135>Zw(W>hSEJe|(s|dvMRef4u*k^G40>>iVp-qc5^o=t?8<4N)NL9`{hJa1O=Vrzw_FwEBjkSa2&*vl4u@$~SNg)ao5MdRAXl zeUeO<#BYCXj@|v>R%l(>d;D3(YuG5H#YgNWSBk}Twh@|7?z(A7-}UD?W128v9gA_YPceJY?5IH_Fij>pC<*R%! zCiJhK(O;>v_ws{G**pEgs=!|8R-xzx1%pGO5(bKbPl2M~eS9eoS_ct>`tgZ6OR+HY zuRw2X|0mj0idbE5_MpCnMi!{c)}exhvDqJR3Wx^eV$VJ_sX%pTQi19+vgIEtRkf!? zp(}cwB3z)cC5mvd;~~t8-4CBKCoA~2oz=~~K5fbO>#h}m^?CFurrt}#@Ql37eMJD) z!dg~Dve?c3W=bhq?UP}wS_pY%4C}%VzT;t7>Z^4ghSiWc(Ot+YE~f zdk%w(Qgaw;TEw2im@A@aUwX5@D`r?n7QwK7P=sL>m;dJOr&V-FvFH102^FwK{7Bv8 z&0$b&+qBa?`XX-*gHVfSSRVPW^eo1M*@d3=0dXi3k1FHYR)L|CvTs(TMc@|3%Vw zzJw>~J8{aT6K7ufHm#BLogMa6`ueMN3#VW6m7MZd3&Zi0-;=(xfu5wKzuJ~lew|l5 zxL=K1IQB|onj1yU#)XD+bf}HHKiGm zVhn{}t#iMMrx=63fML1e6=Nv;YF*7oXc%(86=Nve>~Bn`uNXt&SL;q_cM8pYFY?v8 zT!yt<*V<*awp@l~D_XhtPqn!zeQ#B3GjuwtwFR`BYHe;SDAL+oH`UsTZT7cY$E8|Z zSf`_i>5Mk1);6Pks@CR~T(!1++NWx5#lBkCp?xTZbyk~lTbt?DziMBFU#)ZRUnxZ^ zeg9PZD*S4lq1{U;TAkI5Nwu%SuhzMYOtr7VuhxZi{HlEwezh*3-DzKXv%eE<81$UT zTfEtyvGmKHHSFrEb;adBng8Qa5}~EEqCN6Uo}xY87nJ`9^_G0KuJruh=|#-{9ntfD zKhW}@68Z|(FsbS)*zEZm1)Dun%FWcDs=;3>`Oov!;=;}Tc7K$POpPxVW>|9`MXUL~ zTDMMh$karKX$*1ljHIM~yg&1#Fs!+f46FO(e4!2FO=4X8t}?ExjLTtKzov{UOS0Od zEulyeJ*`N%kzEFD4Xy;4C#ghF3MHfpb z)+}bx#qy=G=$gjiagJv=(qf5)FO7{?8EfbXDf&^oJd49)99Ya^p2b`z9Bb$^#PtP^ zSt7@L5wlkpYv{88EVgb_AG_mQ&e@#;&p*iH=>lvKoOMiFbZQp9Rz>%gxn z*%)r6Al52_p}2gsYKhTf%6@@-v$pn`{iz5qPe zJG%)Aj)I2)qEO{R9m{4ps8@l$fL@?4;0*`|^ab%k3k$3T9D;j6wg6P1FPnIw?*#p- zgE=X_g?Wh84gJt_Cmzm4T)pWU`f@5GjFPV5%v#aiFbOg(ek13z~&`73dC31^VLq zR7XLEg1g{JOi9W|sNhxTS>0^)K+%9$%M^<`ZuX$W1t&Yr2e_;rz!=c$1$dqX#1gFa zjKHv-o1kAxqYR*Rnk@qbeMQyYglr-xpSLQrdY%$}Et*d4N3FS-aI7g_jF*~KGn;?6 zz+6Qvf1x7QVi{J+4_1wd2=E3Nmhms%gb!9BiR7)rMe4ETKb&&pzj^hr=Cbmi5<-h@ z7LDHW=|#$adJY5sYp5l)*jD5lbs_U_>Ov;Z)J0HVtkoCt0`+{bK+RUza~Q=LR(1_j z5r##Tyt(op5!&(}bzDmRo6E|7rPna+%QCEj4;BOp8CK3_fm2^Bx95nkgHhP(&FWPB&-cePvU)(IzvrSbEE^H|BvBiY zNcuWXNy*OOJn5r;$h7x-2|I4)RQ7{Wq<`{SVK|plekZR|6?(HoE*GEv`L8p_WHTAk z_iq%2l`X&i2$q!JiBnQ4T7Gx4Kq^Rouxh~?FgtR~e^AmdT7FOcJKAZg{G_$@t0s!Gnzq^U{sg1JE`i;08i+OQB7{QZcxB{ z_@j1HjqSWP(Fx`g(J}3&8e1_&WoRF&$&KmTzD(kB-HI@(DebdLd-x7@KyuLpr7}ZPKy#%8=q1{U`Dz|-< zU{r4UB^cEu9e)W%<)%;j(u`{AZDdsYKH=4~nuk&8HA)B*HltF)7)+F3p?rr;o_vQi zvfvseWaul?v+}G_@2(?`dX=p=d!Hqbe5Wq$HR?*#yP^+LQ6*b-{oSoEw&{y|)CGRF zYnyxZg}m}qt>?*ic;2a<7C*n&J>LhZE@(ZgMQjE*m({cKe2}VpKn;~!$yogMAII1I zSln#sslyBmVZ9)uDy?TV_xyaJ8HSxaDvaxiQob^$jxJF(W~eV?z2RL4uPSUMufnWf zZKo| zrshzTpYopke{jNQdR6aaY%DPzn;0DKi^a#s>ZsPNo6|ErYn_Ey!B3I zI<0i2@)&d6tJ2pD#}CCOhDK8pv5DUCc&c|mc?_ie;_yiP;lzZSNT#zUeQh>1+aWP> zU~nWMQLh~8Ba*=nCSrplv4i_FogL|=Ba_3iL%qG3PBZO4(0?eFijN;iq+$;z#wP|x zM>6-m^d-}n1m}4xAiaF!jx8JZZrQPMLpYOP5sjg?_BKyNBp+j3LIg61%BjHtU%Enl z#4eMR0dA*}l>IiO-YWW1hsP3`WCP~3HSxjB4x@ZB`4^a|aCPt68zW6^;YeGky=J=OVoy(#k1~-+PbW$8(U~8P4h*a zF7F>tPfcEWU=TY+qldpFGp{neC^1DZCABOwId#fpj&iw)wp>EYj=nBa zdRX!jb5p1EFuTSZPuXlz9xE{UZo9ce273=nc4|@H6mNze*5c%h)RZYevoiQ58+A-@ z%n;XXoKg$qa`f;92gZB)1&0L3hdKI$PRWq&VG0hMk|FEOm>~zm^4@G23f@c^iry?4 zO1=$@8Ok6+WwK(YI8%oBWZO;&b^KZwm=Ripy^N-mCLvia6ddRr@8c32@97c|@8}%j z8RkmJ1uRI$u^EGno2=KRK*$wr$l6RG+SYVQ)dL;R;vVGWi|k`YppO}WK4t{@m_0+; znv%%FdW_WI1zQ;{SJ|nOLry?exqW@RjgcLIZJ8>#v2HKba nwlVj80Dh$`K&1^JrQ2U~Fc#of%L+4gd$ST_fDkv_38{JjlvVQ9 literal 83106 zcmeHw32+?Od1f~NlAs8ZASK(9Z7FS9mIf5mocmyj3V4a8hoCSdWlLe?;b73f%n+Cv z%|Q}Ul!ci%k+YqISx*urn**CIM@ppLr;^JH- zGo$B>`La=|Hq2d9mC|f}-Z+&h&KXB97{yxS(~YrDG+svb%`?V@eAZ}~L(iN(dJ30f zX2EFWRP&xQl}t8YoJ$&u&DXO8d7l@^SBFJxZK%XKy=Kc3IyX2cK>S@-Cbb4%D+e4k-Gq%4`Yxr`zlW$PZ^2L0OdiJ5r{Cv8`8mXSW zXk^Y;j9DH3d{tw9cI>JbaRjpX6u`%%BKREf~03JiV*f2F3 zqTW*8M=kKGSXXinPF@?J38Q9c!XCmStXGP>FzWH`%~a;92YxBBBZ(PkJEvOXlzs z3#FO*yzyki+$rfpr*>JMTBAN~4peImbIXj8%`ar;5jrA5pKh3=kx(=ciw6UVNHC;F z!qLVj8+gkHjAA1vjO*c8j8S6lh%*Uf7>MvBZ!$Euq|=$18LU-zjvV{cwU*`iZltbA zO!r4!sVg20=s`hvov2H4Absk3N4M^~aqWsH{CXf7cU!ySiI5)h^CWNT%KcoQs`^yb zvLhw}Q9TiNrK&_Ap@&25RAmmGIezlFMsC>L_0ha>Q3x5@F$hn&2CIIjVEuyra44Zi zIB&=HTU$yrqWgo4njIwj-%#()VtrxSsA#pEq18*}YFn-Gke&#)zX`!WEEv@Ng4JxR z_WeetGQX(RDw$$+wp3XFcTDo129Z_vTr#bwDIC=!ep}bAq>w)v*AoekzHgt+7c=wu z7Y&Vs8s2kT(Rf^sMR-vM0}>3yf?+-EkGRjsNX3Bgn>DoQ`mC6aW~H$(_MHST>8vyy zjN%!J$J;g^TiN3~+}=$*^{5_mql#cS9Kc!=Zl{VI9$b@MH}8YspEasgEu}Hp2|KS1nOoCMIlW-$qz)kA*HRhy&g*tn zfd+{+U(FY*wM;Q<_$YfCQs%f;tyRV}W4>x=vwL2eExxRQB(#^Dgy*>C_v|GdgJTU}((*-($!FtZkIh$W&yUF#UnNl|iBX(6b@1@f$)jfuYENhZ4V_KX z&5?s=4(fA8twkQ!gd{!dkfBMMmi4}0HMdj@Myd_-qmT~Ma$LT>T$P~9h~_|jdoc=8 z3d)a}Qo3HA$<&NArngo-&~ne3GC$xZ24_pfnsKQHEd*0&q|8lu5{Db+du7M##jK0? z57}>{Cge0GB#rkA#M&^Q6y1P2q~_xkJg!}W@`g>AZ;T$C&NGkcSVrmGKla^<8fnEdB z(^PYrvLOj$qvrRrYHqHT=JTX7c%Qx7G!pcYr&)7%jp+UXVq?fu zt45`E0@WXVp`MxdK`CZcIj)_>ZZoc7ogCK^x}R&p`Xc&WBSba%f)Muhu-gV14hI8z zEatxRU{S75S8MrNy~g%P+HhL--f$!q)Puo9lWnjX4aZ{vusW_=uWe7;!e1{`Y*R%g zA2i>UhT3?3krsSfxL)pJ{?Ce90?}$xZk!!TS5+&=!71O2VB=^b9ey&Pd;5*~Ofm-q3CJ9CMt*kDIUOXpM$u^(4P^)u1^u^3 zYHMHz+u%H_jr)UoRP4;oeh0#ds2=xvY$GIfxc3#45sB!Dpo^3r3Pk*nK|>z2@KAMe zVH#{^_OS}2w5*ZNmgeU(<*M=6jDgLvWl9L0f<(^uerMmK5Y&NNc~Hpq`TRn@M)G(& zuMvvIpdxS;$wRS#A2c3!0UAj2v7L$Jr`#W3EC3)aDliIXUt{ESxsYl0;c`2vBqYk z2i=ZQoXZ!DeNxfDbOF1~omni9q*M@!mWCPb#7ggR>BycjZjw=m4uGeuoA=F^E*cfq zN8-`v(P{=_ewYU!VzxBtYn?yeEEE#8e5uG4GXTe-K!TCVla1t{xdR%XGR$0NAOqC0 zxrVv*iG_R>N**M`x})$QNP6=7hs=@r%!`XykXYshlWxh}0cJ6k#j}CueNZZlYuVZ* z#?13le?CNN8ny>PIf8uqNwtFxvzFXSjBYaTDK9`vBMP0FF=n-91XnU?m-a&sHKslO zq=xGckR1xEn7UIh5)8$S1_=1u_$% z>PcheD;gKm)k-#9+M#D01!#pGO2cp3=x};ys%%7Jz ze57DQU9%d+ay6elFU%S29p>4akLvo6c^}PzH2>j`6-)pVOGqJc$z89;$?eC@J@ky{ z4bX54T{Tr{4O37d(+Z+a#}Iu@eIg#6d=aG3C+}%F=M)S@C8!7#PFm;cfisml*XQsV z{FR}?Lz45R-nZ!1>}tWCLIOP~+ZflN;e?iRzKW9Fhh=rVFOc3TP3>v5FOY7C3jDI# z7f1yjuzqoJy)}?MnUn2<3=q@9VHXh~9P)?adQ@P5t<0K@N7>C+N8)_tdS+u$_-ehc z5Ce~3hB~qs?e(;97=`^*$(s3z!U5t_1`AK(Quto{v9S$>gH$!C!n-pIg(H-=M?Rr& zj52RHe^B@cHFJkqg2E*0`$qEtg()_loaO4W>laQ_-u|w>K2)-=X0F^-NMm2k+{YN# zZZ_4vtHV&uug3$fyLtdBn1IKw-n6@DN?E&M+GCQ0DC_XDXS6 zDpl;N&m@vcVQ(FibC43OdVCrkAA9@FV>uX^tsLE;K9iwSLEt*f z(|#B(+ZkprJZ26{GzTf>5Tu=I1HSvofuv%fhQ04Q8wxFCOXWq4{{q2r7C;Ny6OU_C z&z<@3(T4nKUr=bYFp@l~-acQ-%mB6H7BDRx2FCB#gP~Sq2wp4s zGJ5&vjBsNJX1Qaxh2SM)A&*&z`B7;g!N~b2|GcY9)_B$=Tg^5Z8UT!uqnHhCZ zn{o1233xoSoz*;`SQtZb^Ci*WW?6b_iBxQ}Lj?9R#!yx%_NR^ZCcez;uz} zzunn~%EO!s2FpTtcRaL!-=B!X2uQvQHm7%H1T;IJ!C?%-jLSI<0Y6kHun1{2+(3g! zLam+;Z4|~DFqD9`hU@$W0`QxH$CHg33K(mYs#twsM%gkWaWa`qxOO=nBg43W$4k6} zIeujg%to=+x;6yju}B!bb$fU)bPZww^{AG+7lLEv#y!qgE`f=Jl9NAo#~cI$0oazt z;vOV1O0-5US(VbfHj8KJoP}T@h(#wNR@jz`;SvZD)vIC)H{DmEsc2awP;nHloNX4r zpokER9bqIG_0ZUkFhfZ=8NzDDZuQGc!f+G2Mf;n9N@4!z7iPPyZ=Fq{N*;A&{6n=f z|GcB`JDcz?q>1qXixgc5@etw|FO=J94jxU1aO#OXWr#V5(N1m-!oUjW82NA-jXWO- zc3y~o8mXUdBzLfLQFd5~?A2+Qn~t1;rxq3b$Xoa&6?5o9DL>OVnN&~KaFR;l?2w}P z3LnScWR$8X%pz!yeV1$Oy(~6^yyoy!tzxy3+@93TO?dbj_-CoJ4K!28QWKO=C{f*I z{E@A*3br1cnW>yE!!eZxqEgLCH96SYyxgg4H(2Yx^V^(867Mso z6~%uXQLUAAC^q0$%qnjM3XW<&+2F&uhih|=7iUYDVZkng(Wf(f8sg|3p{Y|( zA3Jq$68>M@iHoci_u!5fBMZ=RirUYb=iT!m@~r=Poi_|iOfqmB8*h-E$lGc=!p8V% za}(R~8|G1FlFE%aD|2KbNFKAN*pyN)L5wiG#X(nM%i?CGY${nHl0dmhZGqMNC6jpt zNPMYbzTe2BR(Y?NOJ$#bOlP%PzM_5SG&mvjtp?U(c+$58lqc-n0d3oQZ2|8#mp&ub(|n>rGPO&AiAXd6cgRS@ zMepJuu{(JLRsVd&yXt&Jxa~%wLFiJ$E;<;N1OqCIK+K&@aaWuIyc>)+9Tm(O>~@Uf z@YFL9ifc3ZS@>{Hn5v&GXXYkI>5!u#y)@S~^B%wz`%fR4JhTD$mxk8 zCUlI$q%ulpikgWYLbZ-oXj-J0q zf@SgncvR`d=|xC!kRgm>W*VMo2OwiSb^OFpCL#3Xizi2i=}c!9X4uI7ZV)G>eu_0K zPPf5%$zrI|K`bi{%^4RqV(iEn>P@Cn$tLK9OtDnVLr;$m%%tJd z4G0U83Hj2U&BD8h&VS01JO&p$+`^3-r$#{vtdTMZ=5IKWtm*@nfG1DK7V%J;ACc>u zee@+dQ|Dz}%%V=S_|m19AH_(&p9yKRn;;v$&)mj3L#@++$2gm~dEl{EH+^~G5Xc0i zA1ZtSQl|Oi@NL5h*Z>MDzRY}f|A8Z1Fm;HD&+FDx8`mC9^RuXo%A;c%H1^QeW_3Zz zPznIesX~enDf8(nX2ORPs>D<>$j7&5|DG{Oq0J+!&>T*h<|BLu#L&VY&|KVc8MN?+ zRBd1Kv^xg&z0fx3Ahy2b`7clMw9Mk!VF}?d&Pk606SV<_418q?bVR2B>DS8SBu9wH zDWtIlP3{^foWu35cn~w-vOrMf=%~&${eDi4HH_~YE9?W_=rDp zo`^@=;>9#EAdH#e{?cO_n39FeBEgvKAU`dmf$J3P3HY)yl)doj2T#JX*eYN zh7;QFW4)%vebf!`HI=Nblds{#_bB5#SUzZ-eWV3#I$DpzYdv2(K!3<<{>afo&m7a9 zJfH<_mHY+YqXI^rSPq}G>gnxFM#%Pkm1$pPy3P|s`y1E4yp79;k74{$m+viDTURs$ z*Qk(fQ?qtE7%4==Zuc)_Pb++uHr-Y4XN7B2tiPLeI$^lKn;l}Oko5t;J^%=IlYd~# z^;=v0uLlvcdim34XW z8vEuRWGOnZysV>Yd>5{)XX-eXOk}2`_z#q2QMzMCfKdzb5Hfb)KO&j}+hXB!r0hEj z9|?To4q9Tk(w)u1g@V4%=ZKsZkz8kdSRDCwikZ4Vk;^{N;6vS{!YB3o0e-Ptw=8kx z?LSwmpuRfR`LV(c{aQi)P_X`?V9oDt_P-)XKpxq}@&%b`#n@g!aZGWR^ZeeWrH&O_ zdLoD~ZAIF@Toeu?B?fPT7-s|O(}j~HpWQ34YhgRu3SLY1wr6Rh8N|-jI20dMlz(;f z#8U&RI)PaB8&qy8IHl40O^8r@ga;1dGNHtkt@t~t>`*r2p2|+(AN?lS-{$o@qB40M z7U>i%+Z0jG3sp=%c|~98N98cuX8gP920wWn;J1pr+RCx2WCwgApSYIAts>J~%9TZY zInFc6t1KplsbEsrkDBkLJ=v~o_SFBV&9Hfq5kAp^xXS#ppR(N)%E_q`C3?D0Tl6n3 z=BxE;ig~qx&K2}8MF;{=;Zp0^R)(PifdH)+civae!R4}~W z3P!tD@Hu_1QmU8JRb!4kB-69`OMvjnl0f-gHew9aN(8_^R~-JGws^{Z|F7vdB4@4+ zZuIr1q0T>iLOcA16_vgKh#yWs|7JjW6Cb+0lqN^K^uu-ys6WXwD>3)YYs;? zv*DsW^|y4G^jFg}i!e>dXX!&r+4En*!?J20e1qTEYyT`x!@Xe4R>&iO#;tu$Zw><% zGxTY$QiY5c4COZj<KfAoIr z9t5?D_2G!fMQg~e!OhBbwwN^6D_Z7uDfcMy{g&?+UFE;`B0NOP-&SRS;=b|Ue{@^? z>%aQhU)!@cF5f5FEx7hN)>GW_AEsR4)2`q%;`wqqThGs=6T#`3SafqcQH?Ss3yS>Kg`D`awEk)071n}&9skV%XY}Q2wQ_a;zJ0jK)u#cS zF6=v%nK4R5woXFW#iy0_&Cg$0cpU3T0cKGemXXKd(5(lwU z<;W5%UB%~=a#({a<%KF4aHWO45^`kEXK;4>9MX+6ec53yorjlH8fz$hh%H^JO8dL& zKESVzp0k@(bk~|d_W-R!it_EZT2uM5*B<{tk)PHl`_IpAzWtVDKV;e=5ijrM^=XfI z!z2D>kNDrE_yM#k35(@Zl%K@aZTqDf(QouETUK zr-;84U#lp3r9@{`C^~%@kmaTg57EXvVEQt67(8x8pRQIFowGeU?Sa^*^9%I_M1o|A zJ5Xq6t!1Wh&7oc%;#aon+cs^^LeN%#FOH@&sE*~J8`@vx4J`kN-Hp5?l>6LJ6Nt+z_Dyj;Jmt0*tiWqA>; zJb(bzEz{+CC$Ex>WxDLYynYYL(z_?q<@zeGvOKb|AAi5)ouWL7AjQe@l8=$siDp(7 zYQ5iwSSl~qf5{(N5wc=dT4a`CTqF2LWv%r?c34c5lvCt3Aflysq&$hT6er7Gu-i7%|;~X$7gQftYkZW)s}+K>{zFA znD<*I`a-?VY?>hxG|Zh+-!XaERo7t-kr;|DnVXp7CLJ+U_)*ZTxpivt@aYVF8^i9; zyyJ8#edw9f&!rDPclgB7)3EcOh6UF7hBGL7Vb zIe2ld+(-_YgOF#b$srPW8k0+^xwT@5m>f@l2!8pdM2f3O|ETQZ4)}zr0G9&4r(%%Ru4h=`vLKx459u z!Y_#HFXHObYM?`_r`Oo3GT5!vU|XxZPp1y2PaHX&K7Qoc^ufbtj(_y%ng%@7t?5u( z)9;c^A3At4nLhmV$)`>nKYV5_tt*?mwH|J3edn=LhbGg_qTz`oqWNScmA|*``)t#O2!NE4I+;?^c+_uL%+d{v# z{tWbIpg#ls8Q6#!c+W*C>~2Ku|9tA)?J*Q7a_#a+m(enxyl=fqm2PXiyqEi|tViC< z^!~L!1N|B3&p>|$`ZMs|Gy@G2#{wJKG*e8fcEx0evpEP2#3>haLjR> z%wguNiBrcL=9N2t%0}Di8nQFulnv%l+VPl;0yB-y48Y93;AI+(-urOo1)0|DKfph# zR!-tdV_tafkyo*F3bLlDS9@r_YaT!p?^@ zrUzU+;yzJj|a zU)`XS1o}rbY|zB^JzXv8UH{R3>#yOiIT}quX8|036>>2bj^NyFKfWB;G#6$d*rXjU^0V|IvTL+Xc0a~1Ja0RK?l&zN^Y6Bdt zJnD@w!YP=}-G1oMYk`%)fs{Is8e;#q1U9*eqUzP3%d2SO6AXV=gP)j0B|8zZM1bbf z%GVL(;PLUx95u~tz06V7m3xr3jdcxi+@cwQ?W}JIaE1S+t(OUP#Wy*&ULKVJE`%e7 zOX~>S%aT}?@8Ut~#Jd$bDEsxTm*2wzi|M%R9ofqs3h~DRAz4qw-hsRAL?(W-)v2v`aa}B`$W?%1Z~A^8c2Pa4I^(ZSsK^8}P9g&ZMRvhKS zOYk#z9};*6G|J{f6jAsWuOA12lcDGe(J801l!hSad0;?cL3%lcmE)Ipv9hTXNS#2G zvgJ4tk|xTEm*a7yZeb<|^jfOO_AI@Y6!;Fkl=M??J%+z8D9;V?7neo0{!u5WH(M|N zAWKjy-^MJd=U!rQZ!x=+%p-rPPJC2E&{B|j2W%;sN19I> z(Jnvr>cfcb`h3;v`;pBeliPzQon+)vGmlcw(JMmrgK|MI1MUIu@}t!s#49A%$J?Lz zz1IS%=K7f8>m!y*cwkRgL$OlLitwZ6LtcM+6+}<1l6BG=L;dJqU2} zVO4SxNWoybb$!eWdV0NWePndkwm#;?`uOVGzdmkxldX>$+ZS5aM@+yGnl;)oyw*qL z;p-#vc&(4f)3QDyv-kR#5bI-3u8)ZUzCNzd`gmL!L67A6czlTNDB}7M9zNf?m7D6d z{|Ne6nUoPoKlh8sMd?`P6!rF>QThstptmyhS1d@C5h)P|fOmN11}}kN>Mwa-iv2G< zF9k(j_QIpA_fJ)>ivF?oEB{FC|2eM{rBb3!3jQ^Vr7BbI?SIW`KlKfsw}wyzV^`mthdKX8S4rfT(@@ z{5QwH();*tI>vvki1t^S;}_M_^IzfBx6J>_U+}r2@i*IF#rQ`**QC#7_1e#%ef0*h zJVb1$by+>fb^zRgKU`L)WEzSst5@Y6sApNdMa%(}gB{|sT9#>>>F%6N+d_B2U#50O zWe$+~vU)}40QckI-bojv_{m)y|ugLWGvGl)fP1j`l`&s(m zx2E5a=?}8>!7sG5e^vJHA(nn`Yr0>Shm9@z_rccmb21%+L3#-3>XMupMqhpheOdF$ z=xbD_G5WHeEu$|h=Vq$eqKPeZckKykrxSfy&)tQ-taccES@go_%c2QJU%{i)&J72B zElH|j^mR+#G5WIJ2}WO5JB+@jU5 z?;QUJCWZ5V;9q*;|6X;-$B-vpZ{zVt7lRM*1-QU`B?COY_R}0Xz_oyzKMrV z^i{dM8D?F8t`%@oOocqQ;l$4B2 z^bcF3tn_kpC_f;SV3P>G7H&H^)IwSU(O^iA z$`8#HhQZ)Fz9Q~`PE-z^On@=-D`wSMjJUB zYuE~%q#&-Z4>|$=;!q7TUI}6ben=wLkOeq&(u-;k`Cor~S;MbM){rrQ96E7l4O4d> zI>97y)(|{mSLg(pIctc_96DLq1~!aCC*G>Tv;~%M4Tv2&x%r!rYKgwq3Z0NHNQyZw z(20K+>TjmEK_|=4;en(5F3`yrx?~X0$+MV^)j=msIHd`bxIrfjC-H<%5>og5L5!8u z1<-_j4}y?`J)n~$QYR>%8+4L-9H}1A$+s-%5N2UjnX9)j)_LtS0epYOo^5LI&c3^4fi>Y5xAoHs?D4L#L1XluC zZapI+umCcU<%%cBa-|oLMO}Uv6?J|7s(-D}%B`fRf348UpVqcx1_boXpcR{FE!Iz0 z(VF4JP!-~b_x7EIRyI_$w$@L!X!Zs@uTiwdx_3;@C>O1*^%H@X^%I%9i`FC7`soQh zuh9A_MC%FCq}fGlWdIMKRHul)$)T0hq>R8Dv|Oh35$Qu%DAmee5wyXP1A;__mVX4n z6%L{(fTCCan3td+M>&QOlxld2<%o#8KRKHp`A5CdgjUdg@SmvtKV{WZ`vSS3_65p9 z?WcH#Aqol@fEwi;9HF3q0TgKCJO9*kza{2}jsNzCp!!|;6RUmO_yx+;I({)%)D@00 zQ2X@Su_^D~{^%R__ODH%eSsXcj9;KUZ2S`1!AJ!Rw&w9Gj`3IO)V}Tc^XX;tZ+rfH zf@uEj?Q=wk%|Az^dU^h%+r{`nhERR~EZSEsg*K+o1}Pw;4I;p{y{z7liendLH+vQ6 zF<>iLR<%b#2y{10ca>#o=bS9fz!Xbq&6M4mOk-fm6?w;$-J|l(McIATq2|8*I<;ff z#gyHO>@Lfi!H?Sd?4sg`1 z6)W8XS{aq9XAfxQmaN|cTCwV9s_)=XtLl15miK^GuF7-|XvJzD^<}~7m zh%#KL%>ok`wOJIxsBJ>lvYBeOC}a!WacJe1e432rmgF6yw(IhaQQIv^HH_M|w|Hf!WA)Mkx`QJZyVgI1h0H!8=^sI4sToYbaXm(RxoT2UnZ zF#2kTR;=gm0j*fi*8^I)BHQ0tpcN;5wL>e`_&`JwS~(XdiW25pU7!_(SOAi5 z)(EYXfBTy@`I$j0jLqSuhwJD6z@#|j=V4+Ny-aHuO9_I(k1?Tcrib02m6iK3b$eJH z(zNE4x68D~2Ccj)p%o1G5bF&Bhgkqqazw@)m4T;Z;Ft`w@P8=P!vCSdJI?=|qnF># z|4sZgwoRh19-x&}RB;^#Z2I4@!GUZ1yV*UQUEd3=(nL>U;iyDUa!2qP z^=3*OpBW8U$J%$sRR-E|6{e~s<;&oNx_lOUbFa9{%@8uvzP(0V<;Hv4TvBX&p)k22 zR)tKi=?88Ef(d>M1nKIMg6y0x1YZL|13$#94p({AVRC~SUhffCxkG&6S0t{|RWI1& z3zat+U&x2s!WUu!y!b*)k{4fyQeJ!^N_m;w5U1S@S4m}>xC#>r?YN3|6(TLs*BWt^ zt5E(hz7WzP^oDINDK~eceklW~l{MljE4|?=snxrr*l?9s$xQ{Lad$}}Mwlzdj|;a0 zW_Co#Vbdig)hn*D(!^B~mk%n6hE~a)MF>rgBgk=;M5`ynw=7)c=3{8Pn@b8iwb+TN zs8cVYU`Mn>c~<0LPu1>{QtkyUL1FPzZ#;|;=&LhsrY^rzxQYMki`y|59=Hi9ki~=O zh|^j=Aii*}d#3e#K>dWJ_h6%U*MR?BtSy z3E=A^#@)Nzj8c4kM5!d}7h<~0%}w0d8#g&ek}_aPEMWdtx!FG`^wLOm!%fPE2kaf>?F z8KaR3jE92F@vowN|L@q_51v8$7H))=n&Y={8??kbI7Cz7KK~N8>OTG(j`nZAMC~`n zZ)?Bn_-)UB7305oK+qpHE{U5sodhfmE(Qa)6RDrYwrq3g=&H%KYGFETj+G!}*uFLXXxQR?>dhOe7 zE-6-d58Px~miNF-6r0O{#7&mu+%cMSqI8-}W7NjE54JkCQJXcto9Rh2g(c177P{lO z$qo558O@dD9iz5U+0H$b|E8oGMr~JQPDX83%`Vi&T~eGhXO(8uc2m~HsLi4wMr~JR z8lyI=M~vFe$^O}J6Kkec;gVuK9}nE*rhI-LxQRu79=M71d_8cJvaH_&H<^&>9=OS< zO!sg}@ym3)R}we5z6#vrD)9m&zpN2A8J%f!NwLYtDflr8{!jXbgZ*f8Nd_m-g!6w0wD5nJwqtVIT5uC& z_L7g0#N}fY=XQsPd79PJNj@HvF3C3jk1aBG+(i2Z7BZr*UT~9HwiBS?n<==Rc)6sI z+=zt^*(Ox~JT4@PVc8=)HT-q_IIi-k^}4I{aTR#`^l_Cwt^$t5d{UeNm(5gbzC(Rn zrH`xN6u)`;?ENJB$hnVT1s>$!bxQ8wtx>ur4FhT2{(iT zBLXda<98&g(xx?(2ic^io2Uv?JJ?VaC=$!-#8sBM;3`+U;3`+2!w|dDcdti%JqwQ2 z+g~L`PGg6@j0Tq&hC-9ujO*gBLX5H}uCjW6mG+~)7$&nCTtyj2|GVKTx0rnIJnGBe zimQx%6gAnn`wmBaEo}_0a%HW!$_*h`ul1;}sc*O8Dt5WLdDPbi%GG$$?;NhulRe_A zTQ8GNh@F5)1i}J785olRj;nmF=k>Aa&({;Kg5`|M)tG>%SeuCHDpw;jJ#)6v^u9iN zd-DNVK|6%j_H`}+*KZlrlY5(dALa)zs+}D^tiW66vVmkmR z6oj&@UYBYoR$a!c0lfva4R=~xFOu42s$qq!;CH)sU}z-K!kjZvG0 zBy^>=t8(rcwYmADlw}S^ZPv45xQZgv7`0gxHNtu%(-^f`)WfLF?WnJFGQSJ1vLtDU zQJd9{i}zDm=Ct7|PMT9>j!xcBs`Y$4a24yQHV<57Nj^UhT*a#216MgG%X{D|R{b8h ziba1OxXM*oKZr=;Dz{dFt4M$3wc;wuzwU^uH2J?yb~Rv0T5uIGGA;5k@{rx(#`vSS z<0>0?6t>N-#&*_7GPgLJ~!?|bEG{PZ^t@%+rOW!(VGxxQmdAAl*P`v6QIfJx?O zwKJy<9zIIGEZP%VK6k{kN6X+mB80QSpX(U;|QqAycVwxKU8JJGuEcIMnA?caC7hgQ7N7kBRR z4tceFt!M3GNAv~k;bF!Ifra$qE|KS$%+u9s2bsHS7gx4J@bzD(I=%VJ=bQKo6Jc%m zi#qiN1Y@$=S)*6WL>Kf$xfk^ldkI@Vc)hfFwcO|leOcMC53Sg|TC9(%h&XJ(PDI8V zeOdK~R%~7^$Awpm4Snf!TG)3iua=%Zv~q*V_)het+={+vt?0|mk74ZAQ?q-u^mGOo z3X7k5?qP(wqHAmSYMG!{gz8`8X<*COU&Jfq@M^K4FRS%x!K?4()q=OVtMttr2?YR0 zi?qjS5U<|-==27j28N039ewF#eY796Jw>WFyH(>-JNn`{4eZ=a3;n=(Si%2E=nKag zhLxe_hgOEHkEf`J<$C@h1O?79OhNHc6v`kzilRU%-ub8Ayk>9z_AY4uWu0hWxJQgo zSLBCA5X(DA?F;`IYF`||H9|r8p^@(GYt7ThF#c-^YX9vV|7i0xE}H)gwa>8&HveKy zo#Pik5F5Yz&H=l^Q+LoaCm_A7+G*xKjrL2UlT$8%_UZO^~izU}$TkHYlu z{Iy>MfspQ-X~OwwU=t2~_=ME*GJUwEImfo#dK#FeG2TpROHvEHh3@>a790BF)Q4Mo zZKe-*p3ihe*2VPU>?NZcruT%t1n+A-EzD}4X~%;Pd#0n+Wi=Q^ zdh;VSjOH{4wN1%cW7Kv_-nmel^*k80SroE`+Hv!0v7RkMU#w^4g1%hsKy5cL?UgY^x#%)M-MUvZI-yP_|zORpBNYaAcIBpsHsbwmb^%D_`BfXv5aOjr3B z6>|9)rMOoMVou89!mFi&d~A8OkbErcZ_4ya+xfrit3Y2;O|O;}`YtD43H&;JXhl`# zk)td6lJZ1fR<3j^ttfZMD`u$=%Je~*J}6Vz2ac!j71IZ0at}_vIFS2*sj&~5r2C-E z27@xeXhM(KKpE?_v0x|^lKwJXahXjWyke|xqmkAZn(C!>;x6ZIcKS+-;47)6PF^oy zY#=b>2gU}nG7q#^^&pA#iZ-tol!AD|V<4H}XVWX%KH&VSPCz=U%4Ky7mVABuf!y^@Ri7Xob?MyACUnsy&^KV@Rg~N?zl^;(^pzX*Xi}5 zb;Dh{c)eWd33o~LcAD5qXRntg#BHE|cT zADkfELwM5kG=9-nXZr#I>O6Fos@`;r|Mr}7`~rfa`ImD_E&U09Vl)*3BcWLN6&l3C z85poYFYPZo#((W6seSIgL-Q{nAlAP7^S8C%eEzof)hS{IIsz+}0+*?}Z%B16Q+3~z zs$n2#II>BmZKgY>W1^b3(4AGwFx|O*X2q(Dsk%pH%}_nsz9kPnaapxg*9=&()YDAW z?fm2g2Ue{7Ox3N(Ub+A)S7fVQ9Stm%|GTM2{4*=JWND`AzAD@40IaAJQm@SP-tAs3 zewprpt5|w-4_w94OM7^=jLP~wa22b69$qcyWci-FS~#wdK`@8tQ-tivCgEO4F-F;ws=g?(}J4sItwgnK79jb~Uf}DF1Gt zSIZjsJMeB@`8&+SF}ABB^V`PXt$e(Rt4z@=ZO2t^t>P;ycCQvXT$*FpJqgtP@ zX$LeqdhJwwgK?K|D53k~%G9J%n^PYo~NA{mSGI?;n zzmXfx-6ImVo7<3}J^rM2=J?6y8o8ZgxzXfEZkNwjNJDCwj0_aMx*6E&m5cdWPRnIq zNS7<6+5Ei0KVLIrTo~70s2g=7U0W<02N>*zNIiEkoh_9Y(;{tLn=z`{17|9kY`!>` zG!}icdtS0Kqd-1uyexaMXI#TMbq7$QrWdCdF`;AQno-P5&l@ubrcOQm)bSHX#~Yx! z!YsO&{3CM3e`v`5)1glI72ZU9SEtSMv)qY4&0@ zlEdab>^57f7i*2=b#oiLrEJ-HBRODhV;S*wv-A=3zJ<)CbalQ|tEQ{jN~V_0v5cCz zn-VT&E*Mp-kw$XVe0Q@oo?sN`@3lJLe!7vIFt-%z3+YSQY$K_f zyXR&vrE8hWoKZ_(Fe=r2so40))!$H+ddv77E3TY}?P4B+dMxCRuvMB(MBy7y`JjR< z=1fkYxq0fjGar8XB$jJ)6I;?6g&fk8N+FL6&E2LMS~MDk3Ur(1K=s8&agBN0%L);$?1w#>#zU6t^dF-2d?jb6U_(OUk?q0KaUfZA(bP|K?AA9U^e$MqP?16Agl+YJys)0#ig2HSPz8wn^T`Q G^#2b*JmX{l diff --git a/extra/sqtt/examples/gfx1100/profile_plus_run_1.pkl b/extra/sqtt/examples/gfx1100/profile_plus_run_1.pkl index e2986ff47c5de6b441e44ad379c558c232e26eaf..44e0c594df6f18325828000d21fed55d86091acd 100644 GIT binary patch delta 837 zcmaFb&U#`AlR^W_RNF}mU=TY+qldpFGp{neC^1DZCABOwId#fpwsN_Nwp>EYj=nBa zdRX!jb5p1EFuTSZPuXl#9xE{UcDuPm273=nc4|@H6mNze*5c%h)RZYevoiQ58+1%? z%n;XXoKg$qa`f;92gZB)1&0L3hdKI$PRWq&VG0hMk|FEOm>~zm^4@G23f@c^iry?4 zO1=$@8Ok6+WwLyycobGsf(e_#zzj8|N2n+@9cX`mYml3>f2d!GYmm{D(xeRW$s0Rm z)d@I-1?CiiE^#KH7M(5y0$Lc@U|RYhT2^&Q)dOu|aSw9xMfNfy(94WKFEfI?%$}j_ z&5@zfBLZ|QNFlOo4ZyC2hb4!*kCAh{hi3>V7+F9Zn5hcDfJ8R618Ax*11KPCx+U$A z0+JJKgs*3CFkv$|pk{3T)NL-H!`CAaP?VpRnUfkE7!u-ImYP>GW%3lL+t@PLGlabv zfUaQ3@N{CBeBi3c=7z<;1Z6V3y%~Wa?u((0F~f86zN=!J=j;&>lFtb2z^RZa!*lb# zt7<}u8G)TR6)^*CbpYGS^WFfzQWl`n1d!70xg3lEAiHrJ%bMZ2{k$Thn-Di#RjM8U DFDm}w literal 82917 zcmeHweRLbwb>|F7Nu>2f$xdP?u{)6y8^77_vX%wXU*dm&3vWyvD)B=YA>Mp)@k!%Hf`3d-lxwTKaE2^ zHE-53x^>stQYxLz&yJf{>ZNx~XUoO8)RiQcvig$AOsbqoCQZ8 zM!Hx{7RC7e^&$CZ@_{6eBAFZSo1HP{`P@HfWXndrP~ka<>EA!`ybrgbny*;~ zO)=0^-%mY=rg$is`zK!U(SlJov|tb57FJ7nQ5o&{_oqs;_Me0cn9=AE7NlX-ZvEm#k=9Vw=lH<|dxSKWqcwZBjE zscfF_(|x+thqmZ0ffgR-br&-RP0;*Li;ieHIg`@FVnoKXnKxUzMHShEUu4}k{&B1C z-SdU%>YVvl&DyQ#BS8IfnM$oXW%6*ChpTS z(OXHWnNcqS%^Yf=wa7>$5RMEpC1#E_WyJ!cQIX|MhSs)ZGBrJoz3SGHW0$%%vOV8{ z)D;bd1Cfv`b;ZJ=z-T1Sgx8L`6bI6!uJ7op`%c`uVzFRgBqT_vY43`~B7xD^TGW;K zxh_?8sjBIX7z1xMI^s@MBf&s4-bz(g@7WWlo~dQ}tUd3|nwO-IVLSohDO2O-cPrK} z6beTI@}&S_Y27;m14e{$}cXz%zKV_DTO2#y*g<`p-)!}#`H0tse4@X16Kxo+I zEl$n8-At9{t{9b4Dqo%{l;*)5ll*5uWK|=#nbvSL5)Fh#+&1$_JQ~A3A9f%6&Y5gJ zHJ5$fG)SlsBez?P#D}AS;i#zE_z*`UqoaY55f?s~hQ6a@LikOa##D7iE{EM{G#bWx zXIRv1YIHOb!78`j!wfar&$sVr>o8-%*hpYxc*JFeV?koqBO>DyJ1W3Oiq@ukO+jVPiIh`>{E=rP?AbBxGijLD$+1VtzxiDl*nQ3Sq%E|dukt8rh zZOKZ3cP13Y#_wBm1tz zQZ1QDJV=E!7Gqk<(6?`>Y2E6@vZrd+xE!ZYw}=5@bSaRZIbSGUxriMa*E|g_GhMk- zGy_wl<(ZkM$AI)S$|91TPuaREU66M<{UQ71Y;C=_S(HdbblYp;UV&( z;`_$$gQuRstyC-glB73I)5|x8cRK1jn%<@O6fn>u0e7y!@lM+& z9xt@(P(_X3Z@noAwej2)+VE+UmkQsS^+-D{GQEgMeN4QRtv;UEq37O#7o-(-ZPM>} zwG0{5!l8X~O#5#4)BAQk_YjTjHohrxgXmy9k=sxGd?()nxdSwUJN*@qi_$2+t6l)P zQ5tf0^S=e!)lI__cc6|V2DBg$8*#`Tw!9dNMTdb?xyxLA^T2nq$>MyIpcs#1TaQFs z#F&xc;gLXWIO^Ws9ugF@fJwkbUn&7mYJQcB0I&$eC1}wga)pBnS$uA;#ScUbc(v~0*PCav!^4od zf^m;VA1Gg$p8}Jaexw8$Eo~;#g}J#@v1~pvZQ@ngv?Sq?a6AwVx==_s4uRah9nFHS zwR0{zpRJHI-pXj8YAhNE$6W;Sa6At2IW+3dYFIm2AV2MX`{MCY5SG+!G_ErmjeyQQ zTHPx~Nh*m-0b*~V1eD#0i-hAN;H*a7bc>N-7|MlM)PwYPTrf*{bIt%VpRF`D8VL?V z9X{%&rip~ZqfiTl-5brlfpH4bW{*;3xy6~r%8&@CLIbcdqa)K!=u2$T-(J19rI|{!4BVDDHIFPpH&Rj6R`Hf z!5Uk!4s<(aem0vo4=6jL&#yVKYrNvR+eEj26Jj+Nf$GLU_P!XzU%>;q3(weFcK zTrx{MM&i-u&}#}}epH-5#BA!)-@JVO87L$w*+O0@W&n;ufdnI!$7yF(0AnA!8=(TptrJlcnEr|=)Sh^*12bjf_uAB=!>xWWd z$VgW%Gczx0{ZTKeY4|M!xHZ&^W+nw=noRlf~@qK94|O#Zq~3 zp3Fq3dD2+<^X8>wxs*=k4<3@)st#viaaD1)C=p9g-D2%z5>{nv)>)}PkR2v5q`;-r z+yzy_PYO1)HDge!Q1dyybxj+(pkOE}K}Dc&(mIz9oh?;` zK1ZzJFZbpiP@Ff7zDc*{R15Bw66j&o$B+RHC$yY%Wz<}ITK2%Z0_m;N)Sl710_km0 zfiLP^fmGrFn-?cH+XC5>HL`t?;dn3*iM#SB@n9GTl{@d_;8FI9-H~`-wVydy6tP<$ z$i=}USdq3YMr%7w97gUSHL_uTB6o=Rl%CvUIOM($e;jN>?l3h?s_?a$h1@YJ+aaHj zdx8qzcK#su9_r>6vjn*b9{cU)19FqRo`U6Cf_06G@T{Uy*jzSu{YUVD+ zxN);7m$!O63XM|8{jE;+xp5EOscm;rmwJTC$<}EA+#3!^&0vFuED{+3tQeBY)fQdJ zo?@z$nlDqkt{S~a6o%|jwYX^Xpxz0>_)c1VwzPaVEb$9-7fmUpx!43@Xd|#Rin*vd z(Kc-ULT-kAjm9>+1lDiBM2L+j>iDGKaF)fJ=}cP=r?Cy|yaDr=F6e3d5Up*ql-9d%qwd>i<8LGXodHFjxls}$SPp_-K;(ELV$5eRQhBPk;l;e`*3T)Fv1 z!=u4?1Qs(P<=&7>rxWl?s z+-lCkYuwGXl$XdMp#lpoX*b=!r6@oih6!v8vdbu~VX^jE90~jApo0koCF=<*3*!9D_ zF)r;Q8-^ST!~W2FbGVymSGL?pzV`USAT^B7Wq$~T;T;zkmRT*gZxA7+IjfO|*4$2!Uci z$6oG*12$%QphQd2v`L^gh&VXv(r`G2P1*ad?O-!WI2giWMw+G9%{B7q(gY!vZd%z$K&qu@=(wGyq9P2j za~QqItsWRxA%>A3k!VzGB-nT%{VC*rtTw)jW1{S`Gx^b}SzC^sg_jmpe9s&BjcZo# z#X@$vc4}NdRY4S$+__#&^XEQ*zvIKyL~aJB&eS)#rryn|DU`MPCMzYom+_tBhP4GZ zKLy_`eWr$Pa%t*6zML(~g&Xgsl0Q}xW$4(_ zwZUzbP*X149GyC@pQmdva$Q#cNhCEl)~1@zt(slmKFKvS`{A0H&Rs&4b0R-ezzR!d z8Eihq@2OdLr7+nfIrCBMtKHLud?i^en@Q$I%C(PJJIHFAjPmf_ry@J)NVbn`&5b}= z{b&bad&s)|RCP*>^v9HE?srsIv>u(f746*2R2ts1mFZ>cE_%*kgU)WJ?=F^R4;pe$ zIa4&#WRO?i2mEjY9Wst_z_N29ZrQW2i=1=xkorRJxmQwW-r^glhB zs`&kQ)&plwoEjf8=xp-zQ%{^eJOSS?;lM?fiu-USs*&~Q5G5Vt?#1x@NIVyOHXu5N z6($)s4i2%s;b1~y}K(zOFr z(e-YDs%^7&fOi~5!y23rT!)>?9S6e+zFUcT)u}ZUTi@4Sv9$%7xE@k%J+KDP*`m(! zgZJ{8%^Iy}csLFw8*XsSy*Dtu4dM0J4z_MKu-OQEUBCRz#h$!l)YXE_DTw265I${E z59Acz5mZQaa>lNH29Qs&yB@ii2m4O&mTL1S;Z? z>$g*=1#);|yo5qHOlSiXPt+RiK!gBe3TX+hYSm&9K5qvy>E}2i(HwZMbz7$ zW(zB1DaRqf@q%w3GM=lNRg(oa;rfKcK;{xxP0Bn3)G^CxIPa+N$Nm|P4?%dO8(lTJqkTekEa5FTTRH>A@(p(B*kDfON;p6q7@w{0oz$q=Q z)VvTE5HtyTC@eDti^`;6G*`$YT-O}RrAn0mNjn`ybr$QxLY`sWn@o}zmrmsi`7HGG z7{GKA9^C-2AeoRK-MKV8n<(~En&dIK-{BHY)QA`bDX>DyAlSYk8d=#7ECF|(LKca$ zqzI4epLzI$6i?@cfLz4@t>OnSzwj_-`U5Pa&1`{e_%>@h4~BZDV8@6{Tn~2aHB5h2 zIs>wR^dq_d04dY@N%*xP`Za)pl0P+`eiSR7$pBY=JCtW!OQVpa{e!shGY za>jQMlaov09W=4WmphN+9m%NV_$~P*Ypdd>-qnuxAjSh+$3=(o^|aMXRcjX1vgJxZ zG~*`*sWBWLy%#Sfi34G7MmS3k8sJLiQ&$Mb%Pv~#TV|r&W>;tvMvPJ zg#f{CiqC7gerc=w?I7Zo&tj3v(t2P$ud`9ut;+EHjjyvXtbGH~&Bxyq6$7a%f|AL?6o~&&QB|cY4gf|ys1Rfv?jH$tiS0`6 z^Q7)O2OkN1;0}6Xc+#Cq!-ayr&exDQB@>0#_^1r{cAAY{pvvXXGx*UqDe*}`e@Gmh z_ALuSefv+ZT(h3HzZg$Bv}2co84~+5|l@lv0_7JwHUuED2+8|8PBg> zHY;S~A^ZnFFo*Y!lwFYyGjh0OOLLmsDU#3bmgu##9&LuNm3!O!=|$6nH&^Y*$bhE( z@bQz6`*eK_sr>8FZfc0pX#d9K*~kuYau|m(ZA9zG-vMoxwiVY@cMSjNH^zVK$Jb?J z>ewgqDOvR?lNt}yF#XgKW2GO}!))v6Z>9_U)X^s{HFb2<gW`-uP{cpv-G zPY2-OfiN`XWSNZhe`R*AFqNA7WS}rJQ#LEwCm59P`?UZfe8N8A0&MR;C5lLBe~kwE zHrv3!nhktDFk32Ai^;M%OD>Yhne1hN_+(36(F{1o*0M^73@KWir?F9PI;8_>V>X|Lilx0jRTikG}pC|!`gN&KzX)^V60@$(Dzw!QAUK|IgzA&G9X-;egzE$E=<1~pn$aaBpG*y zc+EUr^r!!N07m`g$q8V>ph{G&L~KakX;kIv3q4uyh2MPh1|bV;8eL)RZY zAA5U1t#W@jCQH#CvZrUOc8xca`hG>*+#c;NO+DZ0`J$ux_g0+u()QP{`6%scfBoaz zNB-zLKwq5A8Knikylk01FW=V_YeeLq=)Ueqx4hsBp(m$vOKv< zWNYfR^!NYojRGb4Wxi}in^YBLdHX<4{At?%4EYEf;a~v&%>rojXDXFq`QU*AxX4te zfSt}CIGvg{3whorA?)J2N(biVF3vxS{UZm%CH2GBI)#SYIbhuLHneo{&7q4XKHWTC7q z@X7~(zc!4VFV-c#KK%WKV()7YZvN;Ewz6c%$w_6KLmCIT{uss2K2jV@Qov+R# z5hP38fktoEN@@zn4B8cE;>f$c}0oy-fH&hD0k1wXv zoXr=^CzR7A3jNQim>#{cxqZ~6ceb5gGSgLNGbn{qFHE<64<(O1bL#NKiKFtiZ2Ca- zN<5@`PkQm4sw1_psB5ZAdzGrM_AhljFALH(M13iGR{Pp+r2Wq~-YBU0YX7p2vcAe! z^<}blA5N%km9O?YbyQ@m^40j&@q190o;{VX_E&XO^-+X>;`vrrn)WbGDNWT^e2hAd z)r<03^YsCwQhl}mEB?q%Pz`f!Sy-Lp7{foRYwsUwz;dCKoT6R>GFgd7+GD6oX{vrv z)>m@Y65U|OuX+9x$ZM!SA_}Ye&E=b~PosEq8nvM6i`MFYo6F83i_*k?SudyR(FWw+5#Fkte#407Cz8UBY3CEfx`p{{lzF^$$+QP1z6k;g%r5hf#{x|XZah-nD9pT*g@p8Hqg@Dd%i=6;S2zTD5#p*Q!h zaX_cJehY5&j-K5}@7mV2dXKjBzWa&OM<$X_Jbj{Z*1eSMdZTr5 zqjhnkb$O$=anIv@Ilj;LfV#Ndw)EBa#L4i{q%#O2!Oi=xj7O?Pxk zEw=5npF5p6apsI%;q~^~C)T)KuSC=)b|om5YDet%Ov5I&zKj33bA9EzrEl#DxV;{4 zZ4cenx+~CKf$j=)SKxN6z(2&5uzNdN|EJUDUXQ6Lk!z1fzDic*)OGVwsdSsu)wO!h zs&>@1%I_Y#E6`nm?h15Qpt}O!O)F5d5E$4@Cs{G6I~9{{Tzm)(M4Sr>(jh+4D5G-- z$JLn6>SJ$BL>;eLOSeAGMoWARH5eIZgI!A72HME6X>`g5Gy9yEX*5RfN7M^4tvPr| zd{fCSWCj^k%VKa;lwmWZ@pEII!zuLL=yjMqMzLgeJ%LTD>zCpS%k-IN5tWe5yt}Th`^f|ghU)t-*v^7` zCtkWuu>`t78gA3Vc3oX<>fL;Bzs0d*pTa1<8OI_cg>Oo@AOx{uq)Eg*XVrI7vf{4-xrn%5k4k9pgjlcOVTgtH)zGw6(x+l3OY!@H1qo35oz?Slgjkv zucMXz#W%EjsI?=$5A2Dg-jzu0PvDYL7Cwdc?R?+<=KQ7ip%nUWPv0w_Tcd!!7(vS` z=dc<_zGR`vm-~H3zf?Xw_Cn%ePW#&(5C0fHgh#%F+=Uk@8z-8~rc+KMy^btLrwwU% z2!0mN!wEbC1{L#TVnpFzymkTvPKKgPr-IHB2692qKA*;?^l{=T=zd*0D--CJrXHf3sTU8mq#*&#tUidpL_fZ#=mxZ< zAANNcg%vS~mZ_08KYf6n1*XsUQ~N%q1N=ot*a#a>T-wo#ANir}=u+-~zAt+x9B& ze+I9Y*#9u^qqJDGf_L8jcS55#n0}MU^*QZCuXvwndhvn}B+0L4?WTVHtDuj>gi4@? z>}OGd@_GAQ{08O!At%sNm-tgYrN-!0O3i?VR&IzIbo$4lES<`-boytaEE62+djItE zp9XQ-{U5pK_Zs?FZK46yW@G<}KdJZsU-)zf3M9VPasOk#-q8P1&;Ao~PSm_({&KBo zRQ3K>{=DP(kIH_*SnL@W)_1X^7duK~IOAusD}>qu9qwyfV& zB|*K8JlGj7>ys)Eyy~)EBz}qN;VrPNANeZvW48rTM|o8O?4hGAn9$)$<>A$|tY1@C zSeIpeOkM4yt0i@X?Gyd{HT83Ym^SJgoTQ^X-k8h!swxkj5~EaO#>VOxGq_9CQ+ar8 zp&xaHH|ny!_}{6YYbp(P-dHzv^%lBXR#$jsFJn0Jr!RdFi%esK4ePRgear9fiKsj} z>W0dPp9}KuZO&g%`ER5A#Ya5z-_H4uHRs<{_3x+rBZ=nxWtIN`=TA51Usd_Ap+)^m z&H48HFd5L-Pawa(%1mEDVl!w3n7$0PPE23+iZXrORQHPM>%1ztm0G^4^0x6fYSfRW zN-%w$S0$Lf?EB92Wz!JTS5cK@`kGW%Okd0Dis@_diYS=;8VZk zkbjT->0dPPf3y)dG1_2bTKGS3G=l#_=4SZ66%wfhi_c7Gll*%{$_qCC_fr32JNfsD zlz(6RcFDgh@5T7Xh&Z=ljE6PNVBT2pe^{iWl+}*^LzyR3nJ3kWkbkkXP4X`apHyW? zM&Xt>ApfpBOB|oz|D^m|=l_iF!A8dN#s4MJXe+-YW_TaOKyI!;s)RyMYIWMWp6#)Ow5=2GFrQnL6r)DDv!4V zRm#s*D#`ha9-zvlVx@!+CQ&fYE@09Prbgj^5S*n8m~;V?yx9$+!uu*BL{#TC5k#f) z*r0$#MnYkH4A$}GPa8@Zfupd3QsAJ|x6O9Y*~(5$4cLe-un_|cb_+J*8`G~|#%olm z`swZ3=BFD5a#{DtAW7SIw@W*Euo`vnhAr3#6dGRG2s3g`*a+AaNFey(q)9gj-iFk_ zoF%Xklp)<9r`%R-NDPUPP|_9~c}-y>42Y~38^J8Sctb47aqb*t9#~ zzea;f&T+v;mOhL2?fh13P3n{saD>(w9lx?q8G;$>Je^42iF!(ETXWi~m3Ki~5ZruC`r!@~7VK zTj>4b$}ek>;Z}YW)dZH3dgxE|t3-vAfZdN#l=Chq$~7m7qF;ZK5tuqyhMLZwzC;AM z=Ka^l4x*SMBL}LY56~mRvdzY@lk@-k#WvVU0!q~N$j*3Vh3t$+wrPJv7VVFHv~_x6 zCwHdo{MrWgM;Gh_^Y)URv07gHBWiUnJENT@*?9#wcTMb=_D3l@%l)w~I}hZc5+uza z?w!O=R-D+$)j4bvl;5lA0381tVJH8Y69{UPYWy8IO$cZ}(~fZyTRPs z|Mc^pZ|HxV`ghFVz5iP{|AmJ6`<`!{zl5e}{%TEWdiD7$oQqQH^H(@W2kUQq!8w25 zuTuX48ln525YR~T{r>~|{;PXM{mZq&={owK+$;MB8AA8Ig8ubJZT1*3Ktv%ZxU6$H z6+th!3aUKTW*5~J)Gj~^h=Zb%*zFyt1naS{|2g%upi1;p2}9LpZFW%Qu{L{EUG1Qf z=hYQ>$z}b9y26Hu@u@4uNv?l``YEb3)@IxDVQuzR)hk}ajxmGZbj*&?5?k-hXbBq@ z(Ck=`?OWw_Q*E&}+g_@{baTOxjkRuY?q194+&TU9H{*vWvpT9ew2Fm__!X{I@w z9+}$gJQwW5F3HqpQx8+yHAPEIZATb8x!Hi7ET|GL)OJJ7z=hf-Rh}C*$JmKYQA~68 z4BBIJYW)R$wO}XbRlWyyQdId&U#-~5g39;6PHg(~z)r5J`X1Pc-M$BQGN$T-h!l2W zbZ$bUuWk}MS^oMjJLF@=PFM_-wu3@G1}(SR&Pe$fS;W#c+0Jyr|DlYK9#H+(+0LxA z+77R%+XZ7UytMe%5Iez=Ji)`kl02zS1pkK=FYE-hNHXF3 zumS#WCE14m3qFXO&+<~9u`9HAMCd8z@%{>})|LK$d7M~Ia zUq}MF#7)R}oqk|KAei8naPcVt^8)DuKb%1Dzz@YC;o^rB(hKs*Eyqm~u0ACjvZ=v< z2YH+?^)K${lh+hq=)g^wMQrC&^73u-DZv0q7T|>>r7)j(@r8-HUeFUaS#jVd(xzH? zn4oF>JlU)=U$`;cMEfhWq7>RLxJhsgpOWDFP=8Imd8a-l-w%$o2{)0zh?h@^8*bw2 zQ&PuGPAJ@@)u+UPna9lXT#pou!Hv80dgo|J&TN5|N zY6-a*g}vlrlxf=kQ1~gP7*bxWg^Te1=jBtfl!LlYim`6E$k_MF{9Y*0^&@{zxQI~u z_aQ;5-FMKbPeZk5UChv4c*9irzg?ZNrnGFaJC20yIAE-}g@F zcU7BB{R@``>VM_;>itU$gF2Nud^%OQ4DB#&fBI@;|7Xy@bN&J%;Q5Q@o93@D6za4- ze+f}^GXJY}pAM}5^$#`9U&2n*ziN|ua?D?17)|q+7?!s7{BJhQ|N2$xUo_A6UqDpc ze|`Qd_51HWf9YP*$^4fON&0i(BDVe&Sq|R_OlG<6H^G`}8?RtZ^?9Y)-AW~GeKHg` z%XQR)b<_eEnN%8NFoZxfRPDRyYE|i)85e2PRNM8yD>|?Y)>Jp@s0A((RQ<4~+U|#O zktNkDYpQLXHsc~URXx^J7gb5dMQ*Aq2QD(8(%NaN8y^SVwwd?9MOL4exP%8TV*7k} z;3DUh&AkUMazo{N;3Bq9fd?*fP1X0nMeOlm-U=64Q0v4r*NDOeRofZww zvFm%_BKG(_aFMI3ebAS}MXrwELdgscX0q|o<176JF$K*3ZOo_SX84zyagmkX=v?9= zcoBN>e{FFQup~nM#fgxA!IF^t`(0=k^OcU}Ux!c0tIDSY3vh&Q6;6(-lVj?niT}eQ zJ)yEX$3@Vx;Qvtll&ao2|Ceys#kBYok8RASg#1Rld`igQMByUAIgHmKT=D1~e*|vw zkuGkMPE`se@qJ?o;??yx!8aW^%GV#~eg~|a6*9B@(M^1JIf+jRBI?z2A?d_P4mDkO zag%haT=5_2;wD|(1Y!2(=)3e2{1tLP!3N?~JS6u|yh09e4>*JS5ys#WUT*W@wKm_v zofn{MB!*Ajjk*Qs8i~b2fno8b-!=VBdIWAV2IjlP-=ud;ANW3gm$r$(Re!N2HXMaZ zw4ty5j1xC`g<^3z^o9=Hq$L)YD=z*}o_24FkALCN~COCEbCCEhVhd4(s7I36n6~j?4 zdB);eb-_)Ryl@jc7T32<;Rdw#W!Ho2$4$oQSz!9ym_JGJy$D@q$Knb%#p06tA0B~L zdqRmDBTMXm6ow7Y-I}o&un{)lCaBe7PsqZ%M=XYyT5uDl4bNDF*wX01W}`t;XYvW@ zgZP9LHhesv@|BL|=EVE3lw(xsHrf+nwbtDKu*OaMpLZm}DQ{~+!j0#o zUK*3Sseh|A7S~P?hrlg*uz&p{;U+>Y-UqFFLck(B=v1iCHN8h$c}?OLNR>KvN)-^s z4xCB|L44Sqmecd6JNy~Y|0|CWZc*=FwMqRey?tZ<662vxCB{KRk!3YVWbJ?Y62)q? z=l{z06K?RIIDzIbJ!ibr8?YpY!} z*;iFb)?_#0Ca<8ujW4(5%B^KAD%4fZHsU;TIs>)}*cBv&6*EN;TdhKva zEH1l!)@z4bVsQmkeaw4V4|6Q8CABh4b7S=0u&HfKs@hC#tLn-HH?jL+YO`nVg3^^3 zH(6CPW||ACE5=RiHEpCe-ClpjOQGOqjP^^v;++RywOpsh5t(ofR$?E|Ge#H-1)x^*$sm2>wy1bp|{0u zrY`@w*v(+NM=@RImXE2ECfo!o@r25Hl21_Ugi0ZZjH|E7|Dl%P|Guf@Ur+vTB~|DD zSXgmHVg>(q)4)bX^2ICb^Cx)(w-1j5el7ka2^Du8{v-=);3oRF`u-9e-51fC?Z(fo z%0IQASr*y_jU7`q4X})L9aFlFDUE$W?x_yo*!y}PE zbkyYwC?oM$G!Pfi%bEa7uK-xiy8$d)#`L9k;O*I|W6J+qBa)HxZR~QcYoBleEb9sA zB^8VivRnWba5!E73z(?&0W2@6kDYL^S9JGsi-OVJFk5}>1U%zAdG0vbfP(S0junh` z2Ga##$#^rESgpr-fGE?%U?Qc7!9?LT8BEj?3??bi*9_?8_}EEHK(7S_ux#{WCx3;B z*#L{n$4(jmmezn?H#-7YIt=LL_OTNOz*7IP$%6f{6PZnSr{TjU@NXf?x6`O@Q$R15 zkDaXPn4$m{7snK?D*%?nqv*`zV<)e>2lS!|xcm}3(2@jLmKp#S`(r09jwv@?0G4Gh zfQ1Q)1eDwKv6CD0EKvK`HwLgQjX439mXDomRKUgqDE2=*t-E>Mk#egH6JKiC|J(&^ z@BI%gkm88p6w>qX2`S3>q)EVj&1L^{7qA@w%d5&T5epzhKrFzT0yYZY+Wl{3r@jAq zi`eagT`xBPEUh0qx$#v9-~wPk$gag+Qn1S)SWwTa4_M&T4j6?~6)*|~getvH$EolJ z*+Hl3V;9hSxBuy5ztPZtq_KbdV;}W4(Y)04_tL5Q*hgdk>SHHs_iu1OMs|$B{Etkb ze+duu(WKSKMsSKpiOex+dIHL!{ypZeJ~py;|HZmv2=)I_&;BX)p6uT-e@FlF2GRPj zbUy#9SDowcyGHA;0;XYlj`_=bO3l~jFQF{D|MtgTP_6U%FZ}`tgeJzyZV|A{R^++> zER8B}jUIlgjfGy;7u3~Oy0X>B+vv(x2eYzUIHuUD=zc0;sM-v$3@F7ktGutOD;Jgb zHI>IIZ`+{-yXLYk98)G$308UA{j90-w#UpW?`1VU7l1_s?6O;CmG@OuzES1f_&n;i z0@=ec#a3Vo$AK1r<*HIZdpM>HsDMTujw#EkJ}bZ%TL2cjeO7=UX#rU5`oeLb1z>S1 zz?Eall3E$2xkfl|Qsv>XSZ?~5&Vb5eYCEs4wo*wO_}E5Q!Z9VNN-)hWDJo!Uv)f{7 zYXn&AwwT(Asvf&RHBy^!OlbsI7F2l`IPa<&5kxn~m>FQ1RC!Eo3+jrgO*p1Bf_nDs znA&XWY@{~bzJDH$DR#b~uNHvihN2G-#}u1BJseX8RDD5TEdWbVkPw)l{Mo<7ONNJLPk*@h0uNdZ2^|VQoGukc8eTa&UbN_F7DFB zU2+G&@mP;Z2Zb)~(#2i)qaw*J?sA)P7r-kDU1y0L?LWs_DH zZ~V>TD%YNG!BtxMO6sY_am9hFfD3{MgCBUFH62$_M&K%_MTR?E3z7+b+@w?;dl~Ma z*vl3ceE4JxY_Llr;@^P|qlaX79_y5lMfL|YEl zUtd?iFZPAHjkrqDma8|0tE?XW;q926*GJ^r$klTHqkDuWdIN#5QLD*tkwtZ}|E*Be ziH;2y-S7u^S`Vskt z51~wcx))Ce#N=LW3viVkq}La)NgooV+I$C{s(@Qtl($5$j#Qyy@1s)zvFyO99dL_? ztL;yBdbL^>KawL{ zK{VeqfBQo)XkK{hG|fLz?_Z6zgZ>>Kf}#F@u4(>qO=Lp12$F@K4H(W-2y|FJr* zg#LZ6QU8wl%OGY=_rKo1!k}pV>-S$mEHr(cEwh_|lg?u--Be0qR%I8JvUjVdE!V$C zvW>26H7_f%-6DFes!m$#lbfo(2X125_rOhVsQMnb ziS2mcft%QlHJG--O>8n@++>pY6}r(rDq%0V3vSX#Y?Eqvnb?*U@ob}31a4yA4ko$t zss!UEb{-R(rt+BBY=UB9n^bu_sb#w#CN}Y*lSbUcF3-5hs;ccmZ1#wl*n(<4p18>k zRl>zFWkKaN5}R({e-GTm&iBGi)cyCsP3-&cftxI-`X0E+HI?syn=Gq*58TAg2N5aU zWO*I9$$6W<+Zb+g^Ix>ZP3i%?ZoOm5y7{|~J`&&2hfZF)h5Q{$y|;+J1E=-`2!T1~ zC)LTCxXEVuJF**;0bOrAh+dtz$?`e^daXDYxFb12~up16kN zkHAefkL0zGNORYpTW;3^^Zbg52ddImpKd!gpfY_;w>JaO#c znPU@&4+d+QzRX=RW2dzp8OEcJ8D~$NdZw1yJ(w97-;vql_vexjktbrl+?TfkJ6*bz ztz?W$`nhDWRG7)mnc@pF)8@q?Jriqu4iO0I5BM8kInP>I%Zm4GtM*J~cIGP9K^){nX

mhCv)K_QWAqqVgB4ovCti-kdL# zu3XHTmugnr8OBaF?5`K_(|}T?QtFCW0Xe`y<<6Mab5R1D$DPe2(dmp?@(%_YI>R+% z(PANJ0RvO1vKg3}ugKAsGpVAfnto|lrn}u7q#K{Pf8u#x<{g&2rQPl3w!*m4a3mfL z#Nw^)bJ1`VUwo>6*J%>ZquOzDGth|S&LSl$PsE@)VLDdSS=qFJ^(sf`aZf$A3jaap0yJ#7jSCUN*RbNi z#(70~A#z`X&Ib*ov1n=qt*w*KoPF}CQ`oPqExf1Iav9{0Yq=~Aw02vjY1gRbN>I96 zzVh?6@t$g~icGBeKrAvG8o`>6hQpyiG_GC>4SnSP2K{FGa;rFq#&y+r?pu^bE$99R z&Rkl?yCwpBKQt^~H;paR{cmX5*J;Zjs$e`c8W@$YxTbYs%4%r2XB{nvg7J|+FzP;L zu^LwIOv%Km@?xv?Xk6B4G(3#eMQl#-Sgbk3EgcULEyss3#NdeQvdSB_9a_OgYET2* z&G$VTj|C&c!-4iUr{0PhpK3)=4_XNgM~3m#y04eK$Mt5^ooH<)9*qXVp|-14Z$^=d POEd7-jSQ=1%)tK#T}mVg diff --git a/extra/sqtt/examples/gfx1100/profile_sync_run_0.pkl b/extra/sqtt/examples/gfx1100/profile_sync_run_0.pkl index ccf76d979105d478602e44eb21939efc5131dbd2..ab7ae141819c7a2861597edb3cfcd79a12c4caf5 100644 GIT binary patch delta 1121 zcmai!&ui0Q7{}kYP188{qiZLb&TidSW_MAkLQf*ytX<49+osz=kf2@l0oN+4Llni% z)07C4WO&$4U7a4pOOYJrNzkK+e}D%MqTZ$luRd?rb^+TpkU-$`eBbZ$A>{oVVf`i7 zUF6mxloEDbHbchFs?P{*_@ABW7<5LJbEeMaujI0(-ji7>+`gI97Um0D;ch--R@9CA zE@Pk+=?AyY-N-KHO_}JfLI$$)CMH3*ajUc%1#x`Ye1NZl-lZlrX+}+}+FW#c)`S7w zp(ai6%M=Fj9+G(o$PO5mIT%rj6h_%2XlR?_rKTxr`xN4=rRd(p>~%ap5lfw)yf{0P zj-|rp(p3r z*$tzz071PEyE4$;_GGnf!=(_P4o_-RQkrSx*h@V%fFkXwE10SfCi2~j!)I+G4^$%* zNmbit2(^q#cq`(e-9~QP-wAH;$plX$AXQ4+pL>sjEK_u<)LQZlf+BI}%Qf#1Bv)EO z4upRE{N9Vu=N7nBQUG8CK9RHbNpFFy= z*6-0Rwl!1JoI-!r1%rOvpF)9F|7+SX*gOjjTJ8^f3Y$|*q4*UQhSn0^>d%4f3Tlp{ OOAB3&c@H+I?D2n=`B26H literal 137026 zcmeIb3wT@CbtZZaMM>1lqUqFeoKAA0_yH<11QL8fja=wuS)nMClGZgKR9*k`{V>+HSuzt&!RzWv8H8m2+dvkRe0XE~F* zc($0F3a3*ug;cS$P`P`wn4iwfq{frEv#BHJQn~WNrxvb}D@S{~!;|@3IdgU{KUWHMhl{D?Oy>DisGFh01-IX=|3

8WvvU2&dbt0cjabwf> zk38RoPM^yyRIE~JW;*OiAEFXeR$LV%_=Ps~GDk}!7AkMUfXx+iDlzg2?M@cYmiAb? zcRzQ2s-&v_VBPAM<}zij|M2taw^g=NfuUlmG&fULB|b2GD*cXT%1J+PvpG~As#{Yu zFPCCc8@}OJ)k^(wnoVYMJWg*hD;>y-MhV~6!`$zx#UK`H|F|lMx|0LRPjyCQI77Xk z!Pryp?^J1ZUW!`M_YQxw((&%u{M6h`>QTfILJ3m^I8J$C?o6evR9>iTo=Qz-W|K2; z9oDW-EmS)DW3hpuSgdboC^{7G?-^M5_yWG{wp4B*{b)5b)*I;!_xBBP=-ip~1~nt3 zH>EeHJJMVA+m6cCDrqmIcXp>cPo(b-h0I&u zVAz}gPF^pvy?MeBz`i%yiG1X*AszNSf)vAc{=?fSb6Y1L5e9BChm7cTnF@ zcyOR0-EEe&pha}vIr$cl%uUa>cO=mbM#Hflf1(>42=@;*COW-zw`RH;MxaG>;O1!U z^~`k9NKY6`Eia>K! zOetJQ7R#CBOh`u)PI!-%EKa8PGrK~r?(Wl(XNk9MExmW((i@Eq+I*dhAp0Y|k%91F zZ%+f3FI~8mOKyt<>us-Y!K`}6ks#=-uP5Bor|8T}f_;7AULO*yw4XY5;+chXN9FFP zGO6==+ie{MAdp^gO}YJ%K`iD&ebL$}7eTT9aCATgIX5-gjetmy`Oe(j?3q;2DyLJ{ zT)t59WLQkx> zCmb12nYnwYm%d+FQ2d&(V>**#e8D2Lz|(-$jHYyf+F6>0ZEFd+RQYKd_Cv%gj5JgV_UD;!m%EfLgHB(Ai)4M(} zoqNFoZCD?0Cw^hq9_zUVBO^yn9kAYwPDN!C^m6#XsRQA&sj>%$vNVD;?ZS?Td6azbG1H%}JOwl) zA?*q6)|i@jR2vxDjoND-9MIlWJ~3CAN|sXzbZfb^-}BBzRNmnS`w(c6x=;q{#@OCi zWn+fmpM}c(QeiMXK5!RsjW1J2yamzFt3l1WclZTxR@;(psISEi-GOTn&3Xe-zr`Zj zGZ+c?#R6v;e1UUkO65#>uFR7nCKvB|5ykWu9vtlTn;82Cd&5J$flHBN4%>_wS7q+N z*M{F#`Mw0u@R^JBh0|B9))Pzu@2Urm)67GtI4!3d6$epoFtdj-BUXBSvkW=rl06pX zuz#1X(=Pdr->CCLb{Op*9?Kr5=C{6nNC^m;t_Ax4L2{ze-v0h@Eb9M#4n@Mz!NBjc z(lHAuMkZ01^{`ljF)(NYQ3s19h>MtefYH5S@4@l%Xgw zMZGEqH)9B7s~7y6YKtFZxDCrQ@@3x8Q)lo}k#MZvuODK)y|Hj_oow&JT3VW$1!qN^ z2G|F18fR0P>5DaGM0>-rKL0ZM00;L5SN$%Y=z?o@7qcjXV2tkGL!Erf=Qc9B_{JTGs!|J^{%NDrcQ6)#D*Y>>Gxrv z2fzWx!bAN^O5-=IJ7zMonKE(ZwF?kZN2B4sh)V5h0)qNt;T|32n=y0dn$qf@(X4xq*;cqsTA+5r(X`a?-> zJ>L&f<0FCe%P-@eNCbPB;EdJAbC@N zW4(jnh~KP&Vlj}>!N9y7oWwold?AlbXW3%jns*8gK(W%_#6BpMJDbU+_6l>({Bu`j z=OktpV#M{PWTDbmj||^5;`Ut|PHGGbl9)4!ut~zhPpHrI{$W5`?g3iLD3&R^KcVTx_fY!ql*r)PjdZDuA-LshzHbe*pbeWBV z9~k-D_R3~G6Ju*oE?(S68yXbNX+qdF?Xf1y7nlc9DF!xHwiafwN8#yvp>k>}HEmTr z)Yi(nun)VhZtIapE&P6vCHAsYn#+-3ONoXHv!pD*IfX7@t4K0@cX1!x$O5U^S9oJ{2QDXi8}Eeb+6Np#j9P!0OYjg=itkk$N8={=Gz zFo7zC&nIV|lNce|RwA!yi+t7Q))^NvYiEU3^lGR7d3)te)CZ#4p#}&`A+u++>$>1# zQ0(2LBAm;WT{N^aDNv(_LaH0dg^H#p-U}m7R}(|kT{QYgNLn~wd8f%s(IU3z(3%0Q z65>+Yf2uf_a*)I~+p`CVxqPeMGkSKGcJ;xAZ_;e|0V!jTg?&Bt_A@1-_lC`~E8gPD zwVFz`cUyaMf4=^X+mjFU0E!Gp^MQu01o#zbWq-Am}&hBiS10 z85-z=_FFLKd=GR<^E2mCn)7tX_#BOo4FYBI+u{xl0h$WV-LX@A(@DLNgN#RSd6nSo z_b@5if-|*6_XTMv|;7w0JIVYG#KgOmt+0VdDH?ZKaj7} z=5F^EQ@JUKKwU^2!0ae#O!#1{kX>Pt?bShkd2eR`5dwH@u01F@5(6ZJnnUd@H3aRD z9!SOl(O}0c^>%`MZ~}x1VEMI2GYt$skSTSPL$uYP0WSwlNbDUZEsx3u;4nDH62cJ$ zIUj>C_&_9%C~3@6`mU}~y(vH%qgs?X?Tq$!lmzuIB z(*(4c2OY&CLN-dp_Bo2(P3|^T(1_8-B~><{qZevR+rXu98@R7VtPPdT=Tph&sEL04 z^=?jZ5Jhwk&#U}Cjvk-MOkIe$^Z!Ar7e& zH=ReH=`#(`0;(aIt;q|SQi2Zl&_Sl1Q~6vuF;_|@X7Yf1N(-N;Y$pi?p>X*E&V6)O zcF-@umMpozsi!TF>fkU2iF8iPovH1g1KeC94;+*_Fml-4M5dr^c9f8luzt|??yfv! zQz+ybC@&RG&Xx0LQ@JosHie5*xnvYal0fk7Cr6Hq*ln(~Lk4nw;geeoqy4XOQtVZu zvN>N!oXMaqbPQ}`=?u;xEeyxug1$wPT$@Yz>FH9cyfAFdc^WiL1Hetmo~3SXKl#My zs9jig27aI1Xc(XPnvtD_`-O$drc!2h7OdQGCuHib&Z)`ksV7Ez-Fdy|lF=5?IfW|i zfH)(l6h6f%oPvR4<2iWsG*QpC!zbN2Q!V>mF{^ne@riMk)C*Kfwdo(AMyn6oQRg}M zD!Q_p{cCo!gPZz-JK1|a*4F+WsYUi92*fOKN&S43f2#GUsR9o{2fFQ0;`p&s-UhyA zy!62jd$Vqs=MTH<_sI^!c;yvvh8yghe#BYHYVG`cNl^)^*yFC=dw!#><3o0qu69?m ztZA^f^V&CWZS&70+pwYDye=8rZBkp;Kx{gC{E34Hj=S6W$xWc0Kijm?uBGa|O1t{l zK*A+OdGR{U_93-vHKc~QR()PSERKIFT5#V}W_~RUY^ASxF_o5gvy|#WWh?Oum>F5s z>)nOo*?pE~Pfiw6lOzVH@)MzC3F>)=`TXl?bqw}dNMndcN);8n|A$25^N=geBxlb| zC2<}v5qfeoSq_E3HiS5n6Gz7nj6m&1>CTXRWEb94GLj(eA+LR0ysBOZ zp3tHXtAZi>A(2Y=9+r{>;)R#mE8Az1&tHVFnPpoGtfxjNI3b=WUOe6Nte?p0Z`+$g z8M(Ee!U{}cxrNFR7KbT0Q}HZ`C(vP`aqe8IFV;rH!;wmnV5#!~)T!*k-ngU$b6L*$ z>lK#8+gNSo`%@X@D(~S!z7UFZhdEb$7_w{Rc54r=g)CdTfj4!)_m07rWM757PlR+F z{P!r3=R5W&rhSTjG!KT>eTqK4DLa9g0_x@MsS7C_9Nz#{-i6@@D;pr4V*aWFhq#kG z*LPPsMB;~#N;`ce3(qbL8_9G>_9^57#%!)!m;+<>EZC+@PCMsttRBh^)dRdk(E(_$ zK;yjz1alwYO@I1Up5|z&z51!n-EpLiBN)CAWDNeXz5(cw_11!?Zn~*#0?G+RFNaJG z6W~oA`u-5Krh!9DW8T*~;Hr>N&mOC%H|A?wp>@!VPTJiuB*ri_=x8V!>5D-3*#q!U zNZ%l|qK5`+5p)lQZA&Duzl0X#EX*O?%312eAhR7Hv(r7jgU^E29_eWj(H%uZyaX#! z>FW(ghP-Nqu}HLUpq~yx4b-Xvc&KPo8ETjS0&XP%&4Yl#nx8GDisj=d;K=*ua5Od< zR>g(D-h;yIH>Hr%GDUGmnWA7a8o;5~o*Jc14vG!LpgbEWjLU9{D(-IJOs#2CD^U@2kme2-3xHZ;i?> za_f)uKx8XYKqhE!|y{TTe8GV>nJ(uATyr z+rR)cUt_g=tmGE0$*niytAbB;>y7k7(_J5c^l7O>xZrbiuotsP_AlD+APH!j));IZ z{Bs-xROrmw4))xx^GlDLmCXsI2e43iRO>e}RC16t_V-v9s;2~5Af+OAhjBc3u9!+d zr4&Mja%##3J#JT3pEk2!r5)z5t|c#Uy6!Mrmc&`o)NH{`Ve~2hWXH$ekDN-cf3%ach(@qv)|@0!L4AW^9Q6uX>8oo@_61tIT&Fs- zAA(ow2q&LX0}t&v((QZ1dyOFN?V5`oH?x03b5U!Ozr`j3T9Z6&&Trl%+s)^2$4&D6 zy*)UG*sBi&xVA|B&`*MTXz;nX?7yzrI6yHoiZg&d8?OF=7z_pAxNy*m9uHtJ^>nC;w~C*b|xb4$v3vgE)1LP$>=6!K|&? z&`@6w7~(;nIR)d4!^_g6enV4H23-VBk zNxRf((wjAW&^k_f^EL}w$4N(8W(U@8EU<+n**A!YJ(3_3hg~BralyUAEpfpu6&JYe zr^g~#l6zs5so#I?ryXe>!LVbsF8Hk0E!z)}m#x*>P|Nm%ZRDtBBZ%*Y3SJ-XFx~u? z*rz4iZ)xqV>+Uswv5((kjT7NRT2;%l?eE3qJAPW_vQht1%|;!-13&~H92Q(b6j+vA z<>Lp#yzI%1k`Sn)?=$n%4VVP<)oL}~rEb6BlN;gJI*B(oDDB09bE~o?6+a^ztYUWm z<~vBmtYu_leMUA~3r=gnX)QRnaKQm7tLwC<@uZ zc2EvNJ<>XG@{|X7?;C_7H$(w`8Pf%wu{uVByVR93v-xwWS)A~-$drj2h=(}82Z**G z*gY@^2@Jx6t38uX!WM|OTkXwwC;}<1PVHU*2ZnHFV<=c&P}#05H1qMXnhHba1mn>@ zk{p~^gL&nM&wXinnXYuw&8fJb%o5w%o;pL3!0#?Lzzn#gjqT(FkKs-jsDOcw>5Iw; zxu@Dl57~?BLQ5^lMAnc8;s{G%H{VlAXQsa*qwq*m*$IAmdo7SREpno=M`SVr})^lW`PMQV4E* zXd9N>AV`bU5mt4=P&JH1!&0?2r0&fdVnQ5%@XZ@SRWN6P=Qb)GsSAaq$Jh~3L0>OP z_iBtCQ4o&P!XkMc1(Ca9ph5MnkJUHM0YcAkzqT9a`eHD(Kzxa5B#=gMSdBh%U=L(V z-qRD9p88-0Po;Ke3^tU^$*E5AqmFG<>$8Qqsm!dGq``SZN7ip*f@xM~^~asWMr6F1mK;Cht6?$`#a2r1l2ScGv`ceZ^{-hzfk z%rW7~hX|o5>`K*>;}(Od)CF~sx2IeBhG?@8(T9S#x21r7=w^`J+Q4ex zLnN*9_NQu?y@$wu$VX?n4@zDT!`9IjsoaI@o?-VPU&17aD;eA!(+{m0$Tod*?k565 zvjQu=L*?w-A^oUx6najM9v6u;U|Sq{!WpK(jPK%WsV2#&My`yDs$Gj0lH)rb@cUx= zp?!cYTd;CfMcsl1A%?QU?#>t%hH1m+BT$Bp1m8^;+!vih*K~bUF+XEXgJj&59su={ zRkT`NxIP474~zpyLEVx4pWb~Yl3+&ab*d2R&c(rgQWnEU6qs8(*l%aM>Gr!`Xb<^x zC1 zjwGVJi9RusW}&)lG|Fk)DrVcZa{9j6*Z2F_H%!??=#R1s#Fn!LIfzY!J%*rB;cD=L>w_!Gj_D{9pE&8o7nbyboggm+LW?wdKHVl(&Fxr6IQei*s2g!b#*9_eK z)hv#}YSPC&#jx!%DxiD#!>WAWioCO$%X^WFZmT1)Hhktye4HXX>VJaX$LSAM#+i*W zycSRgM?r4Ga@7Df^5!u$<;{3SOym@0lE!~=1s$28D`wU>Kg?1Ma zpUr1KG=*_M2*+<~wlQw%x|NsCx2&+-VOCg(3b4<`@fxNACx*pA01>oqfAI9vB9QY_ z`cmzGc;`Gp1(4rk5%MLFJ}3i3HGDmHrn1wvfvt?EE>yZ;FIT7jr&ZF(XWdm5s104~Yu4nCT6&GFP_3Hut`hher1>~ z^dsZlybg{So$0$np-QX?bG5MgO6I!vmgWit+|Rf-J^6mVX}+@!=0hg)g^QWoLgk-}@7xfEL{SC{+2Gn_upNfSB@r|DCrNnB5toWwPU!>Nm*X}$)U zD*XXvIRlAQr_q)3Faxq5mb1gTGZ(QD$6@?bE_ntwChmtl+sBR_Kf=~%o4PYnT8HjT zkRkHI#f6c-*pTk4e1ePBCQkCjux_ece>kbFkLI$zm~i(Hjfb**~A^#w~{L zw4P5D^SE|#Qm}Z~oXyXsfZL&)aZR_CPT~rcd~W679 zwhzCx0(*JLPTR|?fO%jAJ?`P4YKAiE^2+qR!;>&Fohm8s)}@ZB1hGRZtfrXg|;Rc|C^ujZa#-QX7j;^%ps@fi$Fdbb_mE=P{9FF z2ygXC&2>+1^X)XbWsidPMmpQFkK^}_oS?G9RKuY*%r<+O&}e21m~rs&B_IvkW8tQT zv-%ECHclNu>JKV1h1GC;1)A&&db)&VaP@J93Sqf*HlaRvPE`ma>pj-)0`6nLO|CjJ zEd7@D7bGdY@vW&;g52Ahnt|2;Jor23s$5in;c1u*PUolaf#IU2)040$OqMKNo@cbD znurc5o8VL9Y@r*oLtzG2g@lkdtqbQeld1jc$7z&%{!YLXFSTdCO_@;64D+5Ac8(aj{*#Vlm$=V_D$of76FO#e- zm}fibL)?JLY~kR0anAn3M~*)Rp=HD8Cx9t=pAAMcDZQ!gw0C`94lc8yTWI*Y7_K5N zw=sV0Yjkf5->|}d>WUP)_=fH^p=(L#IuSs8xcY-4;JOaur&|A8eeedR0Vee3|L7?I zQeb{={2kq+_S$Q~8T{+`#}G(hIX~fY>O$EF7xQ#`f)Os2i{<1Q17;KWKd#kL4|&j| ze`yx0_dxxgqsjyo*Tvbt^%5g&C;s~tF|YG^bA$AB@dI7Bg?oV}P!U`NS5BUV(t?v& zch1PH=Aq~P!GR%Y5y8?&6-#r3z~GCmHB-6IMoKh8N|)i$27RHjQSnS`<=T|;M!^6xr1mN@w2$!8LWo;h^<$VtF0XP}Ds z+(M=8>pJ}PBbV+SG}J8|;V$;49!jz77O zU7;FN6|#Q>*RYxWWA*bb`q`BI6Z&b(ew%*Uv#;O>g=YU$|NeXY`jpyJr___{EEN}P zH7K>Wrqr&JW4Q6_@X5rn!%rs;96ELEsUz!ZaC@Voqcuf;pA>!Yz=`3+p(jo}cKq0( zQ|l?+*wm=>zM9f^9vwe8k~sS0F?Xxm$=i3JO*PP_8fa4w^kyy@_a5>%zX{aE`PQYc zj-$tW4<#PQHEW8r+Ueb0rj=#l@-6d4Rpy9cItry6>lV6&3mrLj^5i;8?NEJOuT&!H z6lSHFJUBn2v%&n;MDSnQb8{1sTTyO#58QXlb8KbU>VY>-4+zKk#;I|urmY^R_P`(i zJK3C6{cfi#sI(ye+gX=-dCN{u;%pN~z^}{$LG6(Ig2J%}sQXAd$)|4QR$!|KT0PL} zfmRQ+df*Q4fdx86o0?28Fljo0$vP@hP!b_!Bhr*|sYod;g1)v&2P**KhTDb8(9t*LG6M1WD3M$p$GCp!OlC9+B$OzWu>^OQeRNqfO(MY^A%T@5XkA zsq3s*`2TJce95KQ1NnVu_vs5^DvtGxE|7XrF?k;ErDyk8XMxxxXA+eC{_-49*^tAB z?jXnIQm){t2P-2~UoF-}IFZk#)HUQ>q0d8qXMh!*l%4}>Lj~`#tw;A;y^dD>kvgo?lgf}TM5vmyncH8WXnu!T8vF8H>3djN0SXyG=F#pQ zZb{MaXeqjul@AU^$@;Xz^nwUe24M{VmxahFbtU}h)MAs9W2a8}%1yGrjST}%pE3q) zWM9ME!KJ$m5W>eg9s0(9d20UEv4_}i^TCPlt|%u$ z=U=2?s$JF3c!~UIr>_0T$??cRpC-sZi7(+8sNW*55WI-Ce>HyW(mlq<=8=*pR8iQ< z+u;Nyx3w8#7M%21bQm#w6&76P_}0HbM-L+)j#p#zF(M>O=b%20l}8Y|*_cOg44__)R|aE%S1(_MlHa`OrTx zZx9vQqcZ*gYUCUh1nOrjYe!RqYtE+-)NsJer9niqOStmxfD+7WME#TjdI=@|PJiooz#QgTkI_9^{FZ29nao>36H#9eB zenXcuoZk>f&2PvjXnsSS#`9Zuo1WjE#fNarpcQKDFUZ+WPW70pKMsdlT8&ln`e#8O zD|I`4g3beE&}s zmp@qK5d4_)U&K008B_V@^{)4#B$YkYV(i;0PjWiSSJkBQSN@12bgaAl)h|2CAHP85 z`}V>LJ|$`YAF3KtORwAIk5$_*ebq$!<3Hyr|2yvXuQ=NO@2llI+V3b|bdJug9v1_FG@k<)gGkbNnjGH?Pr}NS%$Q zESjCPs?Zx2(nZsfFeFo`+&Wt8mebOV}oQ8e=i{^yHz`BT> zr4Ha37tMKj!^G++2i&70COC~nvmi0CVxpY?Lgmk4^Ed@Rh1sE&TjYtN&T zscvtfSXaLPnNNL2!sypDB>a8~A3x#^UzPB;Qg~O+8{YW=o&Wc9_)mDlMe$=`w;G-FT+J&_INOTElaO4ec5B{Ltk+j z)y-UwQ7XqxU!xL(>1$Qqn7(cZg)n_BN?2q1nx=9lBsJ4lK~gh)*>z_6vZ;XS%cex8 zFZ+98`kIiIEBg8%m2*SF5Z+Jw~EBZ1{ zd(uxxIMdhlkT=|ta7ABZ-f){f6n#DK4L2k`)7SXtyy15KXw|CO^?L|63qyDj-mi8~ z;A+1?n;zPKbp2mQx&r=hO92Ljo>_8mP{uYrH{|Co$i8CUp)czJh zjrqTr=KscMe;dUAO>D;eOZ0W+=Rq3v_qSsyUI|c@)fL+G;gi5KmVxs%gZ4AYhrW~C z{hhJh5-tAfLS<71#_V96@x`o(scd-tkoeNw)q{0?ZP^awb|-`3Wdg$=s|SXc-d`>! zX3qtH;pYqGVmE+jz{UT&F7XzSOHdHshC&_c*b|LRzLj@;)|e$IzhrVhX7m5|p{vB-{NB;}x4POEM*VO(AcmJORc^4{lUUkv5!AFMKS0B`M2iFP^`rA~F9sd@>>`T&C20+LE8~2;|)q~mHAERl5+5w=u z4N3-p?hygigFyEW>iH!AbiXX&0igS;fSv)M`=XQ|0J>X}KiVyzyA885%~>vLvuTK_ z&5r2v zmh=IjyPZGiOF;K)YXIF>--Co;C)W$QPdtK^-3_`k-mR6&Abr(J7sBX+*-+Xmyjcdl z;elU}(glKC`)`NR1^5>r3{bp5oEoJIu7n44=lwT!EicvEe>*_;+o*KGte^tj8H!g* zWwgt?6i1tk&EH&?(gnKYC?%&JrQHQP6#s`H8fx|#7Dr>yeLEJyX7=9-bYCH*GMCat zC+Q;)eKiBRPx7jZ%B`-@rZfO_r!d7(#?6K7eoSCD4Y*HqwE*|w7U13j+|&1uJWmEW zmAy~4-Pr=%`A7nbw+KL2vWCcT1_mK_vhPg*?y=r}TuEGOrw6xu!5nt9Nx;2L0q(2d zu>^2u^;Oz+Z-|?tr!l2g0rv^KH3X0l$M-QK|MjEJ1>Xa>Yb{GkS_8Q21t05yTJR|n zRwb~Nc(7pI9N<3YqeaeAc4U6FX2EA2-SudZzu1J7{hRdxALoMqq%8Qx%S~vJqqmMo zZ#^cCpama2sup~t@+|lW8j<8)DSH)g@77x67etHP*0P+i0e1#z752^<8fNEv^@6`n zz&$d9j06>})fh47k7HLZ;XYdAth>{YyjMQgK#P23?ONnxwq|)1T=&%???(Lt0r!|F zkoanm2LkS6wOZsXR8rEYwF2&@;nE_%L4-toPB=Shk&k+@?0KR>dsO_fY{Y;v3=}I| zM+D5}DR?&=Fmt&{Kzo_^UdnErTI9>wsd|7mkfn8Lk>exupWm9@(D==7vZ1+JGru_m zinnn?fu{!@Tth{PEWaPsZ9aemt8bB zAu*WR?D$M=m!-}?A{VQI7N#~!!U6&9Rf)kgXSbl9_{6lwKLF6aEaMXZXxr%n0PVN{ z&H;e-qU6u?b)`m&+)f_=XkU}`0f4qmzo0Jxw3pTZXkR9{6a3$L0qxZv`VGv7L0aS_ z*}X-8Hdqo66f|eRzj#g&!sm6?B7gA~YLPqnzo*b$akTQ*X_2FE6rc^q?cCo60osvG znB<7QngO(@cx6TNS667$rvdFWoK7ur(l8f5+j<{jQ!u7I(h|a7*P{UMPu#4A`D#nU z+-L#bi5B4fj}Y+w@l#tch2i-g+y`o_n)mfYhD4#9Z+mov4C1y&+o(!?Rr3uB`8L3Q z)ga&Pant%iG}A`D7n68vh%_ON?_*B<+mAUp1F!uQ97+afKu=T1H~qn*75PkoSY-c2+l@0Fi!gnVOUysG9J z#tyQd!X}FjeuFAnRVyc^#-nP!g5z?x9{E0wOyg8`1N$kWf%3FypnTW1pF(O2+J4G} z3;CWPWRdz@knd}~+C%eaO=)pc<&2e_J$PD`yO4r+!vQlpox2}1*nWyd>@a1wPUJiC zz8Np_?NsG~O@kohn6TKA2<~9jmS!s2R@$p4L)jqdo^%=6S>|P&2RejtXGi-^IPYq z(Up3mvg)JCJ#m%hCv0Te0aKdetE9^P+w9Q+4-KqtC#M2o4ReDr_FF0ioPv^)Q)%^X zC#Q~U05GcmnAWcXVzSG>a)gj)L0(akMw3y#26m}@8+nGWAiu8iwN|&$MERY+>@5HK z&l2)1$iJOEw_lKd_Ab9muzh3NjYiDLHSY_?F;@jC|X9F%}t@ zupJcBMgSR$v@KzPEA&_8Hl!TJBCp9CW0997XT~D!8Zj0*FEJVUcB7C=mHURIW-M}9 z;@3sKm)xqhHOO~d!UK@+MF|f;zB^ygE|J82Pp&FQ&Os86BoJn`)TaIwg#$?V7y# zAm4VF1Npw@LcR-tf zkNNO_=zxPfT;RzLiQ_Ok5OhR*9uvn=aj5+*;&_m6kPK%BM96(G)Bi{IHVsfJ9)H*?aRdA6k;HwHg+lmjYp+QN+X+jmqPZhk5K}i~Wb<|~@s^BlJ&9(_obu|VhtMl91 zRKd|DK^vkq&2Najp83sTP}~cNo2xEByjB(b#yJ}hg>{kZHwX|{aApVKOtF_iPBCcR zF4$?s+5mduyT)t7YEkfmgl>qT-`^8?0r+fudbNh8&snKFk8_wp-zvgQH6^j6A-KSLO zpKwE{{nDpYQnmfkSLCYc0SicmQ$Rq6C&sD6BrLV=zRyByS8+E(<7!f%e)qu|aH1`u>2p9fKiC zOLAt2(vX}PqP#3`K7jbDgn=OhME{_!tpgArl^6_B+T}1rIWMJRIdy>R3{e^q#t`MA zyfH*+!`-Zw>u1}ed<*UzYFyK-_QxmNf>&jSqppqVZ+Gva80R z_>!a#Pz8@mcmN=NO~L~J@lFXxy9E$mlAc7>93mE`wzz~bwb{MQ)Mi7#Ol=bqa}(uK zkTKrO8r+|xIO*>fVd5q2LR&s_$&IVQ3bc@F8~m?0rdbt+;Y&D z0OD&=1viLSrTxb`4T`UR{OiQYIstL!ZRR1vH0Ej1xNQm%;Cj>^0BlQ*LWG0=18e5E zFblhat2O&u|D%Ax2E|`omnwL}{jDD$j>aA2ITMaU>_EE}AP%1+67-ljj!GN_h$Bw( z`&&d-{2z)~!~V9)|J6X^YXZdS+$+9rG+>o)kqQ9BNkd9Si<=Wo0pep(-bG<*^aA3b z4O@-eT>UU2h=+OQ+knz<5nyh$0P_}L-U7@QT7Y@Dm@3W9lyT=N*-06n%jIyZEnVk& zTWqKF^$qlg`xGqh0nGb*W8r}&HOM#C0n9O1!`74rPt0pcOrZ_YDZ~kSOye8%0CPeC z?i>yBTW323>ku)@&1sN-vuVJ5%m*;X+U{W}u_0ztQyp+JUSKxsP`??!-m)pp7l z0ZJ~weEB8;^UIAi$QuCW^UVR~jc(vvc^duT0nAy_fAAY?(ID3tJ2sLIt#bMv-L_L4 zfVtRCVYr*x0)B3uRW!&Q#>&lk7ht~Z1+BtAKpK+sQv2yyCdn6Ao(DHt%X zFA#jO34!2CO#tTp8bsV(-V-vLQRxFL$3fqy`3tH!XY{NE$MUTdX$+OO@AP@w{t!+EpqH>w)k)c-vMSPNi}>~)_~RjTdRy6V(^ z0r$zN!S%JZ-*E#emH%l1?5pFitFoQSueRS&e#7>gqdXa7gU#rqvi4n1Y1fTWUK6~J zF-jZF++-MwRnrfQQCfoVbx@eH_B}7D84>T47(N)KjU6&ZIV$N`MSMx#7^7Tvp_Ai2 z*1qkU_$ck#uHsBfnh}-qYG~*A5Tf~(R!qY}E;?ijhh~JPmOoHC?2=;LJ5Sudg zZ4U>N+=P$`6PrymOl+${G}~O|FtK$?Iv-;51H=mwp8;{3pnQmJUScw_*|lY2D@e;7 zfcUETF(AGysp}D2Lk;n34FK_F2@e3omn1v@5FeHBb!v!Tmh=IDctOHJL;{E}I+Y|dFwt7q&RRa5*O7_^}ME7-9kTnof`j2-YfDh|cp!AJ3Th(kloY+Vq4 zhdArt@5bn|Eqz9?=7#kG85SUcOB1$HR(y?pucSdvF54)un<9Ytd{cmUu!i_6f`8NP zOV&KPM?4fDo_rABZJnnDn5WK#lQT2<$z(ZS3@0Y@g^T&Qa;SB?=Rzp^hgidh|0w$( z@l)wZWf=E7mnLembNBA&&QF!3I$ypkeNW{rhz3Y^I_Hc-u~n>lhhIp)xw0(@@Rw_%Z=Jk(mkwoc@8seKFO7GRF6I$u}7d?4Bvj`-c~**7p04adaTczT0e z?OEA+VC3+=lZQtR?29adl`1XzzeYMnZE;+ci?&hB*Qh|%^&xkKRunBw8z ztZdfv*FxnX7Et)iY_sR~XiNC^-Nc;uE0rC|QerkWn=f8Gmr0#ps0`|liF1;Pu^J(C z=3+UOEEba&l`ten@1^c;`O=;?vtOl|W7wHZ0)@vaW3@oh+mz>z|;Hco^T|sqq7cj*J{Rb-;SJ)nnz0R)Ss*A2@X&d^S}M zxxY_KCWnB}zH8*Xnf)>BR4Q+ERv8x-CsDt0ekOw&N)?)$N9Hk)gw;rtN~vP`II=kM z{<-8#C~0T72lbk^-&Fpssb=^Fu58#&Lbs5bQB=o7M&4qkdnylepT3S$%}`%&PoytA zR4bc9NN*%oweHl^R6}zN%{7>E^i&gjOLeO0Y|B25@452NXWy~Bj zlR1+(TbSEpWoDCS)yJ!NiU?AFkfWewXR#6==1QC{gjH>xC03)HnN5WXVOq+|nS3sh zOU|bBWKakrYpm>raB}VfDy||E1F^p#N$CyW|FlIVV40>R+DgroQslucVfN-?c9|xr z(`rS>?I2)MPES6Uy7(-rUQeV{pv@} zhpOH4**~Jf5}_wYljTt8i9#uS^4N*tJr=pp%SXo#jDYQ-mhRtG%H*Egg}0gH?3t;g zHCeu}hrIS#nAs9&G9=L6jqC9cNlE+TrriUq?zG!g%hsBH?{I5Pcdf_| z44+EBqw+RQwmd~(a&m60Wb^gNowcUNB9XqHp&?nh9r`vD(mOa1?z8o6zJf(hb`aSK z%!uiZ>>yC-?8^u^I0~fr!$*!k)@GV522->hG&MGx^C>;k;!Sn!==0|?p^t=2z z(s2^0KuE{azXkmYBhc#cq$_Cu0%5#(zr4Sl+FyQU%r+y<|3Kjue!&eM9evU;KKjMv>{Pmx z3_q7D=D@Q2*ncp(emYF}#E*UKi{a$d6d2!9qErCG@iS*<@@JAWpAYA!r%S1_@zdcF zpp%ZD3j?D8fB0M`H}&%>ir#tsa`yEcGY^!NF=j!EQw_l8YITz}_r?N6%qcN<&n z?@Q>Le&s(zl13XQg!T?APJZ*>{h2TQ`nlA%-r0V2Oz7EJCCq#FBl#Js$Mp|7-bf_U znL=W6E;E&wo<=kKho+LfJ<-(QnL#^su!Gt|&yxQB)tlenFD-o*-W~EG`9*nM{o4N; z#l8AjBl=0FBFBzL6A2W=|72}7$_OVS{-_yRWqwOP`+J-J|Y9IRO zrcFbe|LzlCGmLY8rY62?aQ{^6{~NFmWX*V7(<21Y{6KerAN_e6uj+7!t@#P--@p0$ z*Yf1Ns>AhOz@g(_#zn)J!pt0iT+}a2w$>rIHa=Y)_ch zUvLVU*|}MGf@Fz5F=$~gC(qzFjeM0${mPQ#U>g)4qj`Yb+-xF=OeG$@(<1M9KLgAH z%#Xv9S%Y(vlM;#FnRh8o76~Mg0JND{2rdy>5EJ1HDyAmqm;utB64qdSOdNjZ#DS4x zhwNcnw}a@5cuS$bAbwN|@J=(7mgX=?FY}uGPUwU*uTfdDZp(Z}O0ECzYp>-cz07O! zE95ERl3r(P(Dz1pOSsH~^6O16^_SnbBQ5bOl3wOr`IUJWQTXwGj}OC_{3N~2UwQZ} zmD(`k;?Tc)@k{CLc|rD7;vpGJI!^`qby}O}k_PLJlKqRUE7E_$Ha&{;$?bf+anB-<{36asa7+4_CxN`mFZGYN ziz%yX*s#+xq$3iY(ps#WWeELVfS>dooc8lwy>Nfh_E8QgSG+I5;g`Os8H4XgME&bc zzmWYE)bx{^jCpi}mHjGShRwM%Rq28ghDiTF`q7HIZ35Uue;+sjja|4=4TM03VWS%V z4w<@8jqGo8G1=e2Pxg1|&ujQ|P%r~em0#D_&&dG2uBEmLlO7O$Ro+{_tsZFgK&uB@ zJ@ESUz)xKA<7R5hZZ1}Bx?An@y{bphIwky-;4(q&m3Tqn9=5qAnzUPZKY6c-*z#!g zK&uB@J<#fbRu8-ZdSF3a&)epNAl9Z!-C0#|>}$(*V4<#DzV!-@HHwX*(ybL7p%Phc zM#C}L2CbGXtFu6l&+mvxVRn*a>ihNwt3*>u9Sus)ESiUUo>ISdhdFfP-i?AUIn`-M zM>4eg^o1}LXF(=nU9d7atC&2G_tLX_EG^(Cx%B?>TmgEb4j6)b}87k)Bszm{jF0(ujeVESfnQsjShx;6)f%o%n=fcIso&4i<`~tw+J`aX!#;e zip#~#_GLh3+_lD!eIDmDl>u{e`NIg}6Q&>h@E1O_f}2iFzRtG2#<9`fOq#}jyXrW$ z-1V*(k+OPhxt>`X$Hk^NE*oT)2FGqmGQy{ANu<<4*E9#&apx>?&K-}#j_1`inWb?Y zTVB`2rY{CxY{h4l)jbuCW6SKUF-!BZ99u>Q9OM?kaflu0fWzW*L>!Na!YobIj;9&7kTni5w`=WEbNNR>D@0V+pCQtyOH`({52@+gFf13q z;^tU#_n2vs0MYFS4|&YRU2rndrW@uzcCFF%%0;+!Xroyu zaGtMXrb_5wLTl{nnRUDxKM?t+Aw)Z-?R z3HmYPMD(DK^_tzf@u7d!fr7Q2;_+YqTfh0uLK!t?865K)I$sSv9O|A5e3o}{o#Eq| z-{7O>H^foY0Ut#k{`1>QGQS0zWfEW?X8XO_X=?3Ux=K7CIG zQSH^5`3=og^BbDm{QQQ1RXw;ERS&v-a0Pd+prYHUaJdIUis7>S{+Nr^dnizw>u0F^YBR@l)v5ey zGwpjAP^g&os@9dNBKL4KF8{{QJKI10Ih3z$S%J#zuA}nh9tikW+rJ|BV8FN9{xy_u z{2OQaUH`Yc{mx-_lrLt@P^)VB_B|x<^$xRN*R!8l__w-zye^v8ANZeN_>5_r{6Y8O z1_@eEXk^>yE%LB_L$XD4N!~!7j$0g{%d=>@%_7;xh_Dvm!sIBGV@Yaswc{oMu%IYa z!T_%eK$p1;)5MT3*^nmEYf8OW3x!3 zQX@8tWWRx3L^&s@9J{BWBIRg7U9(7?Vm64)Zgu6651MK}W?aIdW?xC2Fm`qYx5 zw~p!SvV=kJYtbej!@ME=&-68(M?Te1%((XWGkuMJA~2lkYy5NG@MX!L>1+Jgyy2H6 zoat-)+urbN63+BB{#V}ch|ssDum9z#e?iitRjZ&cgqx$Jvn*KV8Qzf1lhWM0HXCgrw%kJoK0`6mH_hLH zby55svaI3nR%jFLch6#5oAFYU_fXuT{p&ZIHoK+op}^-_2ix;nT0Y=iylnByJ_+S}oi zz%!O%TBb8c$%np^-2I)g>OB;%DzjHF-mFYx@$NU2OruOPjmPTAG+Jh_{(+jkYRN0G zuCz=M@wcCBnIa0M-+*Qd?&^=l2Kq=|5$lbHqkMU`M_hpjRikIw`r-;xDfch=i7VRT z<|3Uu6ZFa~Xdy*?ypYyizsFpix=NV|oaYf&5JnMXMc>QrC4qf{-)MO4EZ80wocF77 z_}n@x`VMi$ErRp>*geMZYObBF-V#z}_s}@S?xAmj*geE)$nGId5S)jc=}-`hggort z$~GV!ktGU@we|3CdP`fc_K5PY!^)~@S%#b4>#PX7_r8YW1oLw3KE=GTA2|l@UChp1 z2yRkbvGS!x;)%*{6rIeA`F&ItVLWgW}(&CyJCC7PHdhbtCPDx^erH?F*tVV zfMHmG+X#}z<`|E@1q7bTqqTY7)s%&RpTB8-|7uO}O_@>Wx)-k_1#1(peBdUG5 zMqGiO_@2&h-FkkzAoE*yTkZT7u`p)N?QHy_3Bm+W2LII>afM+negpwZoM0N}mA`~I zW9;|YgT#nhvap?;a!ZJ5jQyTUVZv7-+a2VTTS5>)L;nWm*7}cmxhl>;`CX3?oG-U@ zAR{F$@RV=gQh`FX=mF)+jUwb!$iDwMOjm`>YuCV@+wH$T>n`8Xel6KBD0|gG?bK*} zqY3#|+b@09MEeWw_J5Je*L{jk*OCotzqE-;l3Pmb2;Bc#!bG)dzWt7C#nJ!Q|2v5@ z9R05(OWgmec}=xn>e@v6m$vHu2blsce;9DJxh&8q$Y;^C0a--!3NUQomjxIFNw5-$v;4xVVAGrX0+4U0@IN60pg5BDIcl)#To8fL@eoZrZ$@rS)8#X z{may5kIp8_*`^Stws9YE#)PEyp*BOpe8gpzg!xdL5@#&B#2Imk&(yXmZ%l2Y(mJNL zf`s|pA`+LFOl?Y>VN(>-+zm<1)HW~iJ>rac8J_@ghCRLk;tYFy1H>5x8Q%bLMnu8` z#2Im^Uw}AcRKf$q8J8u0(3gla3boq=C(SzD=BVcX%q!~?XUtD+ts~AL894U07K*Il z{~Y_<3+v+lz!0rXoDqCWoKs!p&8RXl-G;FiaR!)>~9fSi8I!^zl~cc%E|wAc4852R-ECu zMFcm6XmLi(Eh1x%TSRV=IAf_L&S+hG?Y`B6PjIB~xLZ70;tViGEpbLmoH0{YU+5cI zoYB+M+v5~x*m91Zo>)W&%$;c<&1h57j8%s;gF#nQUUD1MOp9b@<}- zm1dyZj-&q=QT1?p=vuA3Qk7;1+E*jZSeYRH&HxZ$n3jdVPHyj-NHf-Z_@aR{BSMUM zlhTZ_FE@~8jNQJ&7sR4^4qvQP4`0yD=AWg8yc4n1uA%F z@lK51{NIo5Md?bY;XZ@0;yQz2UM`@j*oB+d=OCdnJNxh|k_LTbX0@&i3i0)0F24gV z-VJ=Lv9v4hJc3bQD^K#Dw|BEYM)|Kp$kwmk~!v6Hxy3K|<3-n&K{B=ZWlf1*v>nnu53@O@ObKyfjh%a`kWq zwf_Q16O^!~gBq=xNA1^w6>9&=?`UZboVu>$tC4>=*Ufy`o@6lWmv^GGRVTT7u zGwkpHX~s2~e*&Z#%Mu6*8Q@Jk z+ALsF8fvp3)!L*Pa`^To(Pp_S+AI!f#;w=Zdqw!aTO`du-*^sZpeG*Vc@>VM;&@yf zG{S3?W+0!y{q5TLzsox@tYX| zzdq;RU3ZqR-yqd$OWM(rcC@4&tve@M_eekA1`}`kR&bb#n>hs|c_ZqId%A&HIhCN{ z{*hmxQ)JED_xQl@^S*aDH!rKW`5f+;J146`jc8BIDeaJJ8+v-8L%4uRZme%0?%1Gj zW&pD!wx(=xM|<2fevp??ZDDHh=ENP#@Av|(#{q{oR;q_LMBD*p#{aDD`o$e!UX-|l zB`Cpgps<5ig&hQ6c)#%)9#(u;9pYdI*P`6=qQHU3vm5JIFX`jdBb7 z-l?{xUTu7IBUu%8#4WVjA?yGmeq$}RrsDgM-UFQ1KLfBPTT?3y#U0Op(exbN(5Ewk zY)$E4Z0w;v)OP3_^?N1SZtM)uGEzFBg{v{#%lZ=i3Q7k41?I+XXm+i-qcam;TM z7Kqh7zfHK{7_Y4<9Nu^U+hHYc*nqkjJ)S?kM z1sW$`h0@o*+?l^L^YSm-;trI5vIHLzRr_MQWb0`r-&O6SDmBSPQDtN zH-J*sf6N?Q)BZ7>(B!(vrv!qBtd;f>CRK`K-e7I(O<1KH*H zh&vXg4lM4l4}IXs+oGw&9itM10qA9UV*q+yN@a1!WeM}K4rIq<0Q!ccbBH@^h?)Ut zyEohbwEKH7m&MjjfVks^ga?Q_RwX=0+#%rs;tu13di(>#9itM?aCEJ>!%iO{?pT!a z(QXlUEJ@#?Xs@laQ3+#lhuzCeZJiS4Bkr(AZZns|hd1o_Omp)xwp%EMC3*P>*K7)5 zYKu!c7I!Sk8&jJ)ys`A4rd+1Eh{RxOv&&&>D@dt6)HWeuEbg#lI>a4zzcJ0(t*b|E z=4I*s0C9&sJ^|tmd;9|qZ`kw~B<_&%1H>J17k$;*qMDcR0C9)?{a^@0+_Ag{afd}r z0joo;Pu$V@o*G+J4*t(^cw?RXA9y=Z6&^Sgiw}p(IKkJJ;Ik%MR99tx3y0d@`inc( zwZDBanEzu=udX^DoeYoDb(%i>9~ihp(i4Zp(UAW`D&p|D{~Gdth`kxdQI(#$D~@WVzHj{80`aoSVkgEqkmBxVk(!gBw;t-Ql@>A&eVP;NKli zUk1?yB{(+cW zW8LuhN4t`TSPpr}n5?lIM>0kvjU&e{M{Si z4i&!$2jPYO$3K{94X|AbHt{Vi26;*NMSpon6UMBN18s9y`VkU`M;`LZTJb{UQ#EFV zzB$Mlqq)uVi%9jjq`Db?u_@c7j=MKHEtKlu7|r-U)BIvH$3J{FI*sn$Fm|Cp-tRQN zd!w;2tJ`?|10SI0_(%P_H^wX^Q1TFVs?Cy*F)O)ygK1a^1M3;H`mT>WjI^_$)p05CjG7hY>{c zoBw{;aktP*fp>2pq?!FNYV`mL;z^94A9fJx!;W>$Z*`Bi(SG=9jWH|KqWSp^ebaD$ zLx#t>5r{k@4$ppgWe4`2ktGV1<88I-e~$Uhun^NZzb&8Fcq9B*l`$*V@sAsSiTO{- zL)sya*iP~QZJ-I#1byaUJ2@d*Bb!)eyGTCT$tibJRO6Db+)V-Cu>NDlt9MhN{H_ry zUm^b;C{&+g@RYAbGv4y`JsK$Az5=9i`HMdb{=C}$e@60%YWw9#301|n{rb#B2Q}JJ zzK&}&E`Q0@{_#KXv|k&jqVnZ#3^Y$wklU{$KUBW9V6~m9B6njnF8{{QIop5zF32Ms zLq#*SNnTb4PUV4UUuAcO5SzF z53%Dj%}vPYFtyqA$kev{p#BQ@_{TMg=|gSv66TPHxQ~ByN(`npyBwxAyUr{RvFqIA z@ejKnuYUpZkjqlOrZ4Z^ z9EQ+efIK83=`nyJ59wTkJmhALSy?;k9skh#TMLz|cl;L&i0AkRd_Z7$z@*%A{_iE( z-(D5|Z{7Rb7uT}Cb;v{5sf;xfUUEs{5X{MW(%1siL2 z@P82tW9H=l=I_$`+a}~8D~`K4aKuB7f24VrG>D9A?&dJ#&%v8EQJD7pj(;e5$jUDT z#aa18PzdtFbBp99R!d&eIzG}mKGHfq(mFoUIzBQtQ?~Y7bXI13E|*Ic!^OE=hy_&S zr$mC0;Y_-(vXgxgOlNFLT3UGT@JG{wm7R0BOqt?`XOo#62bCA*&ZK`J{b;2sFBur> z4-XCa$x8cbFfv-&mYn_Ge>j%^3n zqVU8(esU#29HcUrHDRIhPJVLvitv+7T>cT+htzI}-JE~4#t$Cyn1y%3*`&Orp}DM; zrq7SOL}x{)6250eNH}?rkCNEuNBFA?JU?uapMBEUy4Kg!CeNA#^h`Jy(Vu8qId1IA*yN?*0R=0aZBnCs& zOY-K1s9oQix%h&{iv#2#5eW~Fhg_EM0C~t|nO_3rA(o^MkcTWwdd913&xhFQ1LPrg z{n2cZhuB??rZ1XyM>4V5WWvN|%SxEoI;BII*zDohIy zGVDxjtMcYUY|9enkcYTQt{^d39%7fn@{ns%BOhX$moO$a`^|Aa#QyY{4%^ zWBz}A=R=n6*x0Q z5C{jK1fG=|b6DNNAI8mz8N_HT5BZItIAfm*3gNQBafZo?*C&HwE&TZJVGX7yyB{;@ zCvNuA^p@bnXbDbQf|Hiuq$N0M2~J)|!O75IG#v32oWy#2!%;E3mEO?(k%bo)Lg`JF ztp`R9?>l*T-Y(s$c$fMS&V<(|*C zun(#{qz+N3!)y6sI5C+oT+GjvL$l}hXnSxt=*IG^ET{WQ_S2MZu$U^(6>}?N5sd>;oB>3noV9vlxFhf zQld0jOqM6p9MP(DQNa1+xm3w6WMR0oa&NUX6_CoE&E!&)^<6Vl$OPwL6PaA%xibsH z@yh1h+-%~)2iF2u9DU;7Fyyx1_o5mbO($4m~CRgc)9l9TS z!;${}e(q52i0s3FVvs=xFd|n6raLMd`RMIJc00m`jqDEmP`_=QDqk$57P299bEU2H z{KD}3T=qT$qPsi$W3hpuSZpXd0F#S@1A|nX4senJ>+-KSo;X<;_IJd>MB_obz!*_1?;wk`MiV7=QMhz?Pi%}&ipRBlQw<`?GY z#7sIb=9jZ6TI9D@>JJ8e&ELkg@hWI%`XuF@wsqE&V#Vj(dyln{e~Q9%rijSffnQ1U;MG98@p z3w))PCvUHKu_Zea4@N7E5i~~77}1Pbpi8hqw=%eU`5v^EOIb%E{UPx#ulHcJn|}lk zIUw?kLW9YeI$PE0! z7s6>I;}8q{EAAV)t~R*HpW3`S;8o{M(&Okn1DY%PTV*aR4)lvY3iawroJ`t!q2UF` z6d1yI;G#aSts>uy2h##h;Dd}!S>Jo6o}|cXlN;wToin3-I_NI=M%g^(b4I&E`_Sy zmE7E=bEWiDIGdR%W=iFy>TP4C!gOvXGoGG1mpOhuGgn#qMyfs45?-)h9GP7fx{%QmNkk)?TXl-roC{DrnD81EQI7R<(d)Z~1*QkYFWl_||-W~c%qAFS@?n3<_^s(fi~a;Z9;whmcmR28|Yp}y$Qeru|7 zshBy$f6L+ATqPFm4Nn&4D!Fs>h52%*H(bi3XL3(xLcLt2dk)T*O9y9iXAc%HRkDS- zShz2GaDFaVA!-LF)8!1ef3%~zCpDR#nK^^UqZy=}TB_ceo3>^-9O|`lWoxcbu}&4{ zGThkgU87HTpws8)ma0}cGcz4_$@xtayb1E~+3GGXl z&XxCD`}RF`VXCaEe{aL;m*;a8uK&o>*|%1A5y4O?Q=XrxD2n%toXOtZ3Z3jd*IPpM zzJ@hL^X4+lY9rVDs#D!^$fOY>)|9p%bWb^BCi zGB=x^LFkwceR8SV-JeJdB;tM1Xy4F4IGTtr{m2r&?2gRbQuYByY=3_wJlNOIvGZrM zThxe<-Im>+?aJ=dce|@QYplJL-P4=xK9#*K6w3DCgU#N)+3yt1{9yJ?H|7HaSC4jN z-_pXjg4f>qck)`X?e#N$_JBV}XCtF;HnV*j=Fx0CkTtV|K^&R=p?c;T8Vtvyin+2! z>Z1~gaDQ{Y+b(lKo9WzR@(o~_>mF@yMy4Bx#=}GYOg9j%@neI*Zo8hk+asMv5NI9*so8{rxJ+S$uvgAQBY5XKsG> zY^G#YvKec>P%OKP#-b@48}uu>KNcGb$NRmbxfJ~anRIF9l2s|C=gQNC(kvE(1a4RW zV?0ghkM+mHeR^uB@1g#|!AN+ZZ@}}B3Z324xjDucEJ6!h4d@>nf($gEa@Q9;&=-rM zZG(XY?g?o4K^v-8vLjv45SC&KZB%Iqq)A8m;xysuO$yn|opQEz1j&`cES$0q{= z(V=i6xKPIe^>)BplbLea>a&i_Pfz3J&h=pu%|C!Q{v%6zy;?oc02tDU*+DSGLGT1G z^S{wq-I1!{QTa@mFfYP`Y+Nk#RM?4!S-@&?<=kAklAfE)geZ9m_{x5(Tq*ThnVE9N zn%?`t>A7btaEJB5`pnPl-ETb=CLB#c1wEr3X0@k0navOokRmiJbv{jK-uT(vxfIAe zwck3MnFPXCPR*u^ggpr3r78tRctVdpQVdhd(7q=whS4}Ce9_9yS&U5V?|tV?x$SMq|QqrE#K8>o$gIWVa`%rr=g#+B1Dx~I%Q|U@3g>J2s54rA{MD=bz;D=y~%*6_D zH%9l8)vY-KfR?IvNrfTy_yAr&Hoin1@n$4LuLd>ij*({|Tiueb2m9Ir(M>ZO?GYF( z9vO;;^|0p}xdt#I#`}XuuAcJz*>WXUnXmAmh_OYFi?>SQ1A~J@VTbT<=YxRvOVBs^D=kF`aX&e+YZfYuT&g%*FJHy_;*mbAu6*RHSl=K-q`_FQe03WMh&gBj zFlAmSL2u#`vSN^H2LaeSge)0nV$lIa1&$`wTQD!9R2B5ycJ`BC)}6pXT1WDGgEm z(Qv=k`S^ADZWhMJ>)xcJ;DNqy|DdXtyWqiCIMJZSJ*uwIds>wOWZpsvdSs_kAQnsX z^@n4=)sEu|1D*)(59poIA4*;8{%+#&ek{#=KgxJ~us=MQ2rgKyHb+TUh#J=Fa8jG6zK-XOX$5x@Qt&3d!Dj9kNu7HxfSf`c!0ZuhPut z@9Drqe6jk5nZkukiR(yG%?zlfA(Y2e00Os*WXRoxA%-h+g*mlufwC0qD;O#~ur#uz zdMg%0MVLBFL29Z@W|yiv-Zh&mV-198;FbAdhykPjy|W6OpGRP<1}ddX_tP2%q&ZIr z3#I+mWaT1@J}Sq+*6PmUES4oY)0wHvv{iecR#Vo+`?28awGKRB;rBzVrkBOh{2Xbs zls>pPOBMoLQs^R9horc#C-r4wp zwX?!1d9Bm`ytDcS>I1RoP%j8mAWLPm(7ISBm)HwP#WzWv3J#tPMYtw z@laEs3SAE9G_Y;{LJjvvTxbCm^%#!~h?oT}uH=alG4^Z0n zJaIDvx^2+rH8VcD3FnEy$N)4yzVpOjbRgWPqkPsxHHAq{imP9PMgb|ru|9UdA+=d( zuzxTdX@uXvJR&_?rn3Fl<^2(?LkG2fy?*(O)!YzJm|&>+cG!>>X3l4{@ad!5!vs0l z2lc&Q&-9~VSU(3A@0dZ~a9S_spxV*PUZpSl0}PC|zDz9>zM+NO*IxV8mYFL`f7Tug z|JNN0ql1w|3}3w8Xa#eGL^v`O_{CSdZ3*{aDKj?(&1W4NhXJV%BjJF;lYu1oLn zO4Tjs=p`?;EliEuzSCn6{C88m zD537|V{+e%{^4d*#&eF0m8JY`NL1aarnVDv(*^XI-pl~=9}mmyOkd2EQ?#Rpb};Rk zD$G?<^W{uxrU2Ndy!46cF49pD5?3fCxk(cAKl6P=j1p`mr5`(fK#nBKERT-CiiB&gRe-+W)n+d=?v!mPUpVf|*5{ zLfgxQ>FIK&vNU4NyBai2FMykpKS$l%dHRvDF`HO^2G4$Mt6_ZNt44kn;b)er+se7w zSwQn6-O#1ib&jX3#~vAr)|d63MWZ94OA1vx0dq!BDSV1k*#3gR*7JDO*Frrzj-9SA znQ9sTwplAXjZch=q+TE@wWfcN8m&HTN1f;KRPmMF>|eK=9o*Ds>a)G);~kyvky_+G zib#wCi|XfN{8OukrwaTEy3lP$QzuWHaX0W4P2jO=;R|u4xg-V=Z|dz@BHbut#&O{?^Qe1#|9QIE6y8k&}<)8yXs*zjJ4|X`cVn| zLot5)o(l6D5n!wF)>Tzn+s&G)i`AVZFJNS33$J$-OXu#lw0Ls5n3*J1Kuw+qrOU9& zJH{JdpHTZ>pM*w+WTZ?{J>X@lv5j$I_Kh2x3F{^odg=`z;EM zjX(0x_~B7l+$ggd(vj@Noyta9r2X{hekQM~7s5lB&xaLZ=zmD9(z~BEC4rdXVrO;N zO#10dP&l)GYl+R%XzM0a6s1c~^gZdPw)*d#ts#xv+D~EzCe_?h^*F1ZOn3-tNe)W_pP{RvotY3$PppVsqDbsJz#7G{X6}b@PdvKV2r>t_?PAG0NClp{KvL7B8bRf~% zFd5z_2w0B<8gAyd!L-W>mm|kOA{vhR81o`30W(k;51s6)_j=^_cuRwoF)T;DLuh~B z0DQ892ZK#v^LKhm8yJkm!vosL)j1&d_YKARV6U#tzy0cVx2LqB7yz@(!*wTdrBDsEL9)SHckwc9AT^d z{no|W=0H|Rsl>fu>N`lN4BrEU6j#68ugQT zm_@r*w`;a1_DwowyV*oXOt?`@LxVA|TAXHqL6`_>t+vm62NR)yEuC6RjUwW(GKxCI z21nEYR?w}O+2o(0xvOqNXa2)@)ZV?xd(?(E(mNc(DyMGu?#zooILJ;>i+ z&j9T~p4R5q?~CpFOStI<`Di3MI2gv_f#vzSMGZ#i!6;m$1lkehf6lXTh{Gi&cG`J& zHuguM4H(4gDR3Wb{>z@yC=;wX9g7;hQMVsEQ{fFIa3@3lH#}t}2C=cy&&I1i76CjG z?i&i+C7S=5yEJ6Mv}B~!CDYXkHB_Tw-nP;bw*gsXD?`fECDAza1a^X>dUa*s4p$ikA22{ zdRIAeUV58txV=EvFw5|#!5&${H4UenH1Q=$h!3y0|!FU7IPH;N(P_LHXiBk z2k4@|Xdl_kuQc)`HQ-;`z!0I(!GnNLO9$bQug)KUa5MP`Nra=priuA~DoiIB5;@qV zuJ(Bev4J>tcQ>e-U6@TDGAA5V`q-u8tK6R_^ad|$d zA2zZ10b3vjVm_TXgzf#{mSD$b`Ty*0_5kJ|_<8Wzd*43Qn`RS{K7h$XKDuRmgHf1G z^y&BRSK1y=Y0)TnI^s7LpfvPGJQ6&M-RUkZ8Yh<^kRAM5N*H*5@V23v*?Io$RIa^h zSKCQ@)OOPA^?cCYPkQ}M3)=fhN85e}Htry>4I|m-hKNIwAh(7+qitQm9V2aB!41_F zxV@%BT!;<87j2)9bc`r?Y=VtrkX+)Jt&2Xhb=&y^jKVkSZm8}2!S-_0b`n$}b!+ha zaFZG4w{<>k#eUmjZ&N3(`RjaqCTrR$KIlJL$Jv)=;AEZuqGzEV=?5Y)81(_45?I7T z-w_Qi?FDyfi6}hj18?$yn-K-H7alo+1d04_`jm#2M*Uj~=n`;GU}-h6r0VBkgYC=i z+I|z+m$f}?Y|g_*d%|f?IPD4N22MDk@7Xzpu=OJ6zJpq~R;T5K$6x5ZphXIrKB~%- z@)6iZT8B@csn>%fpn8I`-4`m4K?REAYXTLQ)gE<_%xvL&W)|CfE%IYh55&jgv;)Ua zCkO2&IpYpCs;=&yDWu^DL_4l_7o33hwom7-?~ncnXu_L#V1MjVE}D7on7hLL5jY{o zv80^YRW6S6fl@Yb(yw;Yd8s&+%o69@t~x`joPYw|PXvIX4Z5*ln{b?SnS)bVxHMNj z$lcW*f+p9tCgf_{ij|=R6pXQ8vx3{p+1zx+3Ptw!T5)T>NH(>uilaZ^E-dEV1TX-- z7CCL#FW5(&Qgs#|i(2fhq^@|BF%ap8k8!W=2T|~djeZG!m%FED(m0=3yL@*QjJn~Y z@o{qott*^|H#`fd8=k7+B^s`(wI_9V*)YU`zizK|2+cJ%9)uGOoW6HuE*8@sPe;fY zM`sRd&9amE1|kEn+7U!QMO0x9lwSJoE}l4MBQU<4__>iRsL5=74A zT|I$qAfzSsvh_e@kPab$$};d%yoG9gt~fuHo2_SQ$_q=BsE_y!OAGWAkhegUYW4Pg z`z+NT>8Y|6qBFgFU9}#H;1C3e_?{0P#e)}Zg{dD;-vXBi8Jt8|f_GPUv9R{Vo)n)? zrEg#WI^u!g9WK>7*s*{GYkQnwFqOHe&hK{h3vIihqY8W~-j^U(?2V2ExSg0<7w*s3 zC4@#qmHi7RQ#Qc(D0c zb+;an0J85e1;Er4udh>ptf!Bdy}{FSyYqcQe&=y^QoP)*8sB9Qp#tQg{UZu z%Anj2{6 zuRCI>HEh`$7!dvs*FqE98|l0Q-?2Fc(E+R;?AR^e3HiIO-7%P|L89;*G!cc<kWqcq$f6#l=OM zh3c})D5qzugzeeN`SeY`Pv6K-!&FR!{wS|N>^EzWqu4>%gNXVOcvDdBzq$^?tC#lR zHp1@E{-JieMZX@0X>l$meBXGzDmkl^172ea{P2SVoPT=mU74Z;!CjE$u z40|pk0=*+2QS^N)^7dLO??oxPt&Ysv?U_6CNlNUf|0%j3r$1B~XEDm{T0k8fgzDYO z)gsu%o6FIV!{d;2E|Ik>`6EmD)712jl1mJ7mH}UBaMbo*dX|45{(d6=IEUYlzo+s~ z&|mSO_4;#Sq@3Pd%&(#u)f?rCDF00=?R)ElXk+e#@_$M-zTv)v^8bm7eI0rbYU4)q z;@Ue9>A074x$Rm2+g{GJ=-Ss`QSm**jUc1pbjOI^2w1o3g9#!wZPbjv^Ih0Ere{9! z7L~sR&+@zQXQYzfjUP;l`8|{_-_3XV9{j*JlnLKZ)!1g_u>2cT2}U}92Z!eKcjC9^ zMrwe}@##EZ?*@)~UoE^1w`LQ0{jPx8#x*z8xaNEC83+j02XlSkV?JO_9P18pBH`V{ zQ_oyC@~HW?2bP=61Is`>LHmdNaB7MI-V)avz?=*6HiN)VsGqO7@4ro+#09jGQbSaA z^C(3DRna>(=g(I6*bcCj=hUTY4?Mh1o7{?o@s58oeI%62(Wkz42_WjfQ3Ja_XnIBIlfZwrO0 zi57g-;#HEb?t|s|Vi5;49?VXDfKQk2?tt%*$wKi`Zf>di6MRq=qU3Bmbxt6fx!?N0 zd}cnw?#b2Zes~zpUf`@L9fz&-nersgr%O-boWqgKrO-4V|4fzspz`8@Osdo9n0fdB zIRv-a;kmPyunxyg{LEbXERIY(1ZTDno;Z1&UD395Xr#0b-I*dUaK0h2vW~D)Na*9NssL^A{({@h{Gc#4S$9g$H)H8?J#d@(dlw%>LY|7e9FMnRlS2KTOrmpMul&k+)RgD-XqKUwKu? z4s4*uJseccP)?m*nZ09V5+0^Al}B;n!Mo|uN)B@m_uj9(^`=XcS@Ozwc>JN`XHsV# zNgW4rm9G% zCvsBl<>s!@U+id6?!W#K@A^x)Y1SSrboJvLXzUUN67AUN#M?eq%wQty)e3EBa9il4 z!7cv~cyF}3BmXddZ^{8GKSDJe>cD7okQt3e#()tAA72XEu>BT}X*j13@np}`5#;`m zVpF&b$5)`)zNm*wxCK`qXQ&vKV`o$9gXdhuFpA!9?JMFi1{~$86T{7K`A|`^(j6b0 zN)+Va*31m72JnEtbFRuIg%>^nf5F+p6h1JV()7e6+zFEx%eseW^+Pof9aRp&r^eYm zH%5oz49*D&p=_EL&gUjGht!YLBlp7104FYX=KnoGLT}b1+KuyM-i99c?R)CNlsZ7F z?Y#W^!+E)k@;K^^JX+Kg>@qRHD;5l1`(??UAE2R|Tpf~(tPewQ3(4AjdA^%I#4Q-i zmX5p^+w4Db{N#hsSvI{s14zkzZ82KO>CSbt-Rr}0aFPujLBnUoa20W~jq&SWr9)fz zcolx8&PSnBZ|G1HI+KLX5&^=8b3Z5n&gw8e-~Qk3gV!++Fr&BrM-Kr|0`qn4f6^gp zue=gmz|ReT9Fc^T^A)aSE>?_isX!hdjc~bAs-({v@S4E?ai)&C=mA~&mqxKh7tHTD zsSHrVnwr;J+V<1zpaYn&hWbALz6#911jnir^HuO8OjZ7Mw_gb3|q<4?VoY z+z@6raKz#n_$h|Gx`+{5^-dcpaV{;fKwqlvk?COcsP9xz?Id3ks1jC>{dnr=<3~>(KMlC$EKCufTB>%OPU19-i<6U3 zVpKb^EXH|VX0-#2AK8Wp)1_MA@QxUXZU zQzwqSFLn6nnG=s4-&BJ;n~{!rNWW1?A31z#Bz5$WQxBdzarDe)s2kgwQIC75?|Eqa z$Y|=JM^DtZx|1IJzGzduXj8ptQ@`l#Oc{qB@_T**xQoke$X{I#os1q$J&ZHf6l-6Q@sal4_Ug<3?18sWTXrW(tt}jP54mR}0Z! zu*c>W61S7w@E*AHhL_kbu-yZ%pB@m2^Yv5Xc1_zoQ0sw@{=6(sYR_KHP!P4C=dWg6 z8kH?8JxOzm1Ok3#90+QMlou3_H9*5l@<}-j6SpJVJ<#rfb`P|BpxpyEc@HdAv5Ph{ znPOnlbOMtNtO#HvLdHg9DOG18rK||z9n~&20K)OMOV!Ia-q^`wC*j&g6)>b3_&NG#u3@dc1a(MT_2~;vSQ``yFhrc&aemi z`_R58E{2I5+ZkOX^P*Dv0`ALC?zheXu}ROQsQ5#bd7!c(#}hh$94AZFMb{p%F+%Ou zVq1h$g}ID6gPbe$1=#Nlu%VN(b3ko~@P12}sAI>EJo=FJz#%K@ zXw^5>UY)*7j%*=9wXCfi{Q|Z1P3CFrOSfikXJZ8zWCXcKyJMt%PO_iv>dj_Fm59gS zLjwnbc(p0<7*0)u({YZ0qJgy^-CA#QI(g=dui_;Cdss1G_bG$GM*bDt9a*`}Fn-d! zoTVXu?LK2ihw&oxPs4RHz}oZ1j>WgKR?58gD58=^N3ydcY4TrhV5=-VbSdV_t{9^=3?PVAz7S)>Q;*LeNxJm9r+$~HR&vtR5?2g6V z61bHEh$6KYwvOyrRQNCgXqbA^!vUnZodaCt8&$9yrQczY*6;Zpi*Mq z%@T7L2iEo?wd#iCoe1vbGIEP==ig^{EZ!r5dnIt61l}QmkOX=qu#W@Z-m&;jMdel$ zy`O_g>Z1d-r(qR*3{QtS(+fKm4|4#`JtBdl5;(>IwEMU`c~AlmN#J1#oRGkKBrqa@ zlM)z}z$pnl!hz&&@J(ce!o3}R{CjanM^rzE@$cG`#s`cY1}P!e{?rcQ4wq^DCH{h` z%qs4_>cOz+2 zrcN$5;ritJ(GN#{tBS&&-_ddOxBjok7G6%igP(0bGVxs0x#1?Nmuflc$bfN z(pPInLeJd+6wGVH{Zt${55rl9e?Zs9`(v&=_CuYZqsO=LreaE*;-@EZ``wKk2S=={4;~P4_HNK%IHZi^-R}14?LXU4*_xP6l z2lx<97&Jri_;#XGO;+?~O#>enzG<3Tjdk;-4DrV)c|hNarQ>;ce2HV|lUn;02f5I1 ztMvH#%<)Ak;1BshCmtmKNaa9q?cb{$^z=Wd9Q?teu;9zIzKD64>PPg=@t3K1|5|{R zsToxKn#w^$zsbQadaA0qi-Lcu(zZZ<;+LHCU%IsW?$55Rsxna_M=$8&UGyF8PyRd3 z#`Ifne|KH`U-~p<>ahWpj*X6}rPjx@=BJ9I`E&*W?b| zvuql2x3w-MB)?_zD#=4s%67WDa@QWviAZ`d{IXe;6cGO$W$mQ9YxnBhn0l7Y6-kM& zW7)LiZg*Wi44-Jv0MWTBxiOM1n+uX2!|AfQDtC|>QA?>223ALnAU&cVBo2~>;|{~^ zve`3Ebh@Rd?x1IO>tJ@}=qXIh%V>2u~U`&}`AElb=sDtSWQ<950mf6Hy4V@V3;FZ;bSe_fOG%wKly zru?-o`Ix_Y28qsU1OBqpGk@82X8u~1vY5Z@mNS2?O5GKIo#yt`@t55?=C2h=@8Yj+ zI_THtuklZ!oI3tmmiMRl>uc`#qU2Zn^*`M45y`Lk>tK(oe4Bq2e@(jM>-nqaxxjeE zUw`0^pAh~;tJcjP<957{g(2LC?B`t*)V1HBMGvn(?n2fz$bTD>0!^dxX`&_T+n?B5 zFaKfC)`IloT9Ag&5-k5AQptZ1NnG+DqJrcS8l8Atc!Q#yHMKAjD}YT4S%o^8eaZDrIMOGAAp8mC{{|nK%#*gJ22rk z(mh{ZG#XMSaJ}$FhUf zJ@aslOdS@UP3Pt~%09k9?-xtN;}L8NQyw+h!$K{h4-Q$y+iO07@krbu@Fs*hNZVA9 z_5|cAfwUQ4qjh)3usLxNqo*La<^o}cO(7(t@x4sz%^Am(zvzOr*Rb|pQ$CVC?}2}X>0%^0F=*Vv|q^+Q8ka9rU_Op8U+6`&5bfXZ@6JALBgrJ!7`gS6Lbkai*%(*BkWX#-E+Wi0&t{to=N@PF^Z->-e$ z=&+0TPdJL7&1U-BL^StntQ9<59!-)W?lS7Ys8(FP<2qAHRgVuc}JEVr=IJ zKWG~C&DgB-qUncLB-yyZs6W`Rj(&5ggv)pW^)9&I}Q;&YMlS5%5N`tZ#kY+T^iBeg`h^1 z4HH3?pp$|MeeFVUO`-3*D5&9K17maJ%e?xpfNc>Zqv+oX$XUuHqPk3~ia@^g^ff$9 zRnbZU3bsx^B9JfBfAJ>>@>UuJYQIL5UG3KixcxQy(pMX6|N1XF+y7!WAm^k&{&oGY z5p{R}>$F7F(SF@mE%g7`zjo5^xI&P7?fpC2uZw5;-%xLih}P)KyQ2PYoqqQoP2VJ_ z@<<0k$HM~cfqjIfzLn@)B^i!* zk3mpFQb4u_U@dVBg4!t=WFM2b-ITIh@-YZ%NF2lK9rqKR33sRe>xS5`EDFxvxt+L+%})aYO1063>umhX-=EA$_!a+3aA*eO>yTd9I$@ zEQw=ovy~W4x$UyNILZbff+*5CZpZBo{f{D;12y8cF*6Vk`;k-&%ql>CP@qw?gG z1RmjlkNoE`DN8nz|Hgg_<GT)yh0FsvWB-O zYk0fBuX$H^21^5;rm-f39we474zK(9#x} zS8;sfLCv#8CfUe@2&0*6d_%jNu4j;@>G*~;uJsIPsr8Iwd=os|vz~F_*vd&w|V;)>e@ekAK}@x_Dh?HzJm6-{eR%se!-rp z{kpGOX#dqOIop5b3oiP#{?~A|tN+(jRjB`+^jm-bkwY{;fcDXsVO$loZY#wt2;R1h?jjP$2;()mV{GvXbhHAU-Hrax`{3E#66a$yzAAByEsisuZI{dl%3_kS|F20Vd>Ql3($c2hX%Y3%oq0{mf7sQsDHHPb|N1G6pMqy!ahZ^lVprqYB1l8n z@Yplp+4bPr$lgf)dtT(fU=wnOJ;Q6WzTJfUhgKft7l(ja-y(2ao;)alha^BRmY2t` zy8K7$TV+Dd>)SeehJ}BMO3?bY1rzeM49~>Kuuf*;*t1TT2hUDA@NDb-NI?s!?fAv! zT*~0Q6yE)b>oqT5Z=07JZM-|x#=HL)#=Adtd+bZ&ARv0qIMCZ${xrIoLUDj=dYLvw?5pG2Ar;ny5qk zE_341j8iZOHZ&(-h%U@#kW6tUfvGn4or2_bif+cl74VHHv=A2p3ixK-IV4G3YkN?- zOVKU9Q=Y%6fFTF)T?2-U7sSvUjXT0ELck>$0KVCTV1Qf%V{*4OHFSJgVDDg9C~afi zZ04ve&Y1xIn34a6zt;hLUw$w03xw(wa!unltfwU22l!sUThVWfqb7f<3DBDK0lpo+ zQ)(c$FYwK_=`^b7bQH9Gr>v0-)D-x3_)cN88|xoHoE*dg48Zq^MP>eB_P7kq1^#B? zSA87GFh~aYPWi44eAf)k%_1?14SWw1*HNGMyTkf{7SGHG$*J~RX#{*%>|`4F zj!YwK)A7h0b}oL*VBmX!tUIW<%>v&~9&QAD*9^Hmz&8&@SZL$Ir+YmMAAGSlZsFsz zk>**CAcND_uByYw*Ge-p8z~|8e0tWrC%mPZiVZ1;ep0rcTrGb;D(9d@2MOV z)F3T_0+LfugR}u^sTQt+kfSU2zYk1-C~+v-0g7*>yXAN2eB0>m>V5hSgTBMp2&6pt2w~uRSyC_*X#<&T zS{so#h9Xzxj)Ct5xnn5O5~>VE8WP7KqyoO}^bAGX=^2W&!9<24Z8{7^PDsfNd@su# z=19Q$(hm$m_Au~0CMg(-v`xM1p~(6-VqW{O?*9PbdsX5CfbXjk9{_yY0zK10} z^PJ6(KHOFmu3>IlmpkURWx4ZFvnb%Z-jMsUq-Sokx!Q-@?DjCX6{WV!ZL4y}-1gGA z*LTX8q+o7am%DmyGp|Yi2Lay_9{_x>O1$DP5AZ!E{1E_rMvlJ&0eYW{Av{s zN8bd=3+NIhFCa~jynr-K%#OzJs)#iraKa73|_9fl~54W*z4WVP5=)l9_prtt;&SSBqs;#+WlZJRWK2-nl&0z?RG^FqQ$}m35mhkZ4^-$8+m-r3Z*!p#!@of`$ zI9ll%-_QYSJ%p&B^$^mi^$^kojc-UZijL%2Wi#WO!rY*_G^oF zYCnkz(tZbhh0-&94){>wGT zGHCxxYeZi)uZ!r5FZ00mi!X5`*wp(sJ_q)oj>Vh<16m))CN}Pdj#@TttbBubc(>qn zj7=&$ydI&nQ|u%NiD>IK6v`e=- zgRyfnx2;NgA3WTSV?5mE9>&A%mNU00L-2YX%$T&U5x2SYEnY)#`~3&t;R}K{EB^A} z;WmE+;NcU(KLL1nMB){Hc@4oOJ^&BDCf^5`2=nWPeTiSz^516h@Zz6+8r<6g9=vJs1LZ4@S%TLut=j3DP%3&ha2oFc!9HTS@0tCr_ zNb|6y36lSihL$Z{PLTYUycI)}THgj+d^zMlHzIDX|1%J16%RkBL>zE38GCCyypb<) zGxB~E-y9F`4@&d0Bd>Y&BaXt%D}Mzf{RZLXRvRyGhfA-4{dmni%1|`w z^iy8*s4_G(7#>tWdQDk(s&YKhj^y&as;qOQcqm+1ZF=7s46N5FXtj5fZ}3M%Vj^lQO7Aa2v7QU z%Xre7El)JEA4g9#l_$`VP2~xsX)aIf^2ifeIh)RQl>5Uq3v!R6^0EJdmqGHxtK>N4 zaudfX-FGYcjXAujqjIu*_2A{o6FE65Zxt{1wI8pYy?H`gm#{RYu+kG^Izositvu12 z7J`9zxdSN|7v@W3JR3hk*Y06yenl`U;M9(^p30OkZQWL|++g@1k0@PTz6%CDH#g!t1q7 zdMoW0Y?d96$~58C{_WRg5+(okJLtFG{%gNrCX{jMljYrlDoxZVYb+rS$`lswpvn-XosuC+8?bZ$;@#4IhA8b64FGX-SR9uG;Ne%r zu|xnKep%v`{jLWOw;fvq;NfGEKL8K69cKjK;dcFk@Ng*~Z$sf9>Qur{N-EwBOF-ggIcs;W%OWm2-6dry}o-xbW?Pq4Q)7LYb*)4c+fPJ{le*t*7 z?WjEf55FewHvkWJIo|O&`nK~kf4$_f50A+A5rBse3w`jH;Nf+91HF4jk zFHa=Iop`-@d4B|c#?^tJ^JlYLU5ITFQl1Z0J=}RF_F-Q01-d_36-e=~2jmC`;XE7-d zoz9d(z2Q1yxSvdyG3aJ+$m>+)QZW-gn=WU<)3X&_+j2Ht%t+Cnp-*@OU*VbY!$*&g z9zS!~dY9E_6-ri$ZjK#3b2xl1Qwi07ot6v@0Uv$O=mj(XlUS!z-{Pz?4k}KgewD&Z z4mFf2v^I(?U=#_fH&HHUN|lo+;`j&V(=(y8UEqGyYudh3<+lf#k!v`tVHbI93aRl# zbxdUR&1SamHQ&e*i}dxy;!s0l42?Aya`aG>cyn#2>F&rs zjPJSn6ErJY*~t&2ilxGI4)dn^J~H@(KQNz}&!p%}IV2Cm*$XVaq;wo+&rEsp&>0-u zo|`*2lDQO`-YbMZXlI(roX<^Wp4m%mSUFTKO@{S{yx+>qrO(b}rVfpbKl0#-lgIbd z^p+n%%|?eha7M&IyA0PbgF~1(Y9@C!b*?zS-^$IV}BNfZ&}{tzcY%g$jQKE{=J zq8L`Sd6EQ;N^Uk2Du!tyujC4Isk!uQMh^zXFp9>!UJR$_FQVcqF^La{ijtM?@cmC) zL;;gDP0&_mrktS%+!7XUF6CEgka|MR=QtMxBFYn!Ph~DWiE7n7JgXm~n-F!hFk8eM znNt45z@x>P^d#zAI!E13U3y6UsPRy>dm;Z#B9;n0I+m`4LXQ;7;nOEhjqJB5gkFAV z{O~A*9%|{Kz2)58Q+shYlb$_0m9{1;7x&Yn`z?%YDKvQw!zdm;5qVPg8afB@IBlL= z_fCK@BU|$O@smG*KlJXf8`T*7@16OBxUb$-DP6jsUOF{2U75^=<_pFBM29~$RpU?Q z|CDNcoYV3fQ!k81Yt?olo0=(H$docA%>5Tixr$~~jOhA>VbsUWQ0w&So_e^0kMLx<-?~_nL&JDUVKs^Nh9N}H zmoh2+`DLc;PJ6X#*`Cwy7-`Sxbu;okBWJRASKq4HmWK!oPR?bOEWHlgWOI67Y+xXk zkeQoT4|+~V%wS(M92?^Ei(qZ@WlVaqg2+!`M9g;O2Z2fFUqHl>G2p}>IezlN4$~aQ zH_X=-V;$Pw+BK|$W6I3MIE?#Yoc*x_f4hxajcvH6yu$l6FTveAOD4LdTI(1 zaXD2kLg4rp=Vl6L(=%TP7pAAnnTqk#;WB`euAdD9p@Dq(RBmqS=T#ED^7@tVxl&=i zm?~$^&1U8*sp;Is%v6dp8ULE;fs7pq?Bm>A*Oxudna0nAXVP=$<{{ZV|41=2cl4xn z^h@@m)cH)Q45{r(hw(yqHhnQQJ(I4aE)+^nY4RXw{dAZloqusW6^o|gzZNc*a_7^P zj7{B9wVw@_&lO9MZXxM)eo5VwA%D%NN5&Vz;P2*zO<>!X)QywC7sD6Q=QGo#0=kS^Z~aQR))+|V zlXxSAQfjs^l`(!^zr6~&GE@0g+?|`8JAfbhkJozOz`5y*eSMLLyk_YuoQ)B95r6h< z0k`Py^I@Hm*!#9E+l*oToe%5hWH)k~vD1+6OZc09v;WRtV%$qvwsM!oTL%h(!+-p zjfC;G^ARy&~Kq%^J&NmyVuHAwD0X44u^(*dJ zDBp2;pZDtL^8Wc2e-iq+bNr-7N;~=2uh`=!r!$OypwsJH`{zQy^D6<*|A?MW9{J+6Z{`cw6EBJFn zmg24&zoxe@yV&(LCEg~+pGdnK+C9+jfp!nHd*HS2fvpKYVa8jJ)>3)%-RLLZt9l5m z;%<+KR|3np>J}9`7TV<)yp^DzR_dXC>Gt?z7ZCYl~xQK(;YBs$+I;J%=rBFhZJWgNu_@O zp%BhpV=9x9~u+CG~*Y$A@)3|qm@S@Y2hD;3BGyGKH)oa77jR#G za=)eZ`=pjWRGBZrOw{p&3<$BcWrl%j9bJ3C29mW~WgkhFlFEjVEhN7HL)-zj5)}Kt z5E0&Q)i#1$Naret=nt7-A3J{J(TA)D4p~u0tG>_nki1&mKgS-TSRZ1rxyZ)L1CDGq^WP%aT!ZDya7o}fve_K2o8#AhhPQ^&TGKRU z@RxU&e&#c=&#u8ojmZbuc6tI^?X{%2dhtWFXH*|))$`Y%N6tNXus|w@g+?xE8oF@D zZroeBq6m;)gio&^g;K*U%lPzQkrfIDZdqQVBPvc9WKEj9=5|T5LcXzysm874W=v&H+>bKEkCHyZca_wK z_0RA?F4K5Bf>i#MSkv;?ezwWM%xmvQYaItseCi`hKbusJwcc>4`!9451U&~c(+6Z9 zMB#9(Wgpf|hf~0>+^6xGCB(57wqCPSrLV4#V=Xcv^FNq*%|b%O{Bkh!+DSYbCRO2f zajfM%g2F9@Ar@U7L#FjweP!~1LO63tvCgFT^PzVv<&P2ZbL?@QK-@(r2Y#OqZ4CS7hB|MucG z5Z2?{_<=p~&n6w?8~p0SC4&VT($V;a{9Q88tH*cca>;-vYJ5XLygjT-V;~Pk*@eO_4`uLV~ zxMcWi_xRTR=cv3mS}m7ga1`d9-zNGjh?0Tu z{ElvckU@U`%bchkfdTsCe@EX(jlO*Z1&GQK5D3Od9L?#EeW|Yf zr@GO8IRXP^ZlL`R`fG9oMRWSY_3i&>pf7I1&|{ANmo`!JR0TUJSbNQzqkoMyHf#Qa zC#;u2-(-V3dIkc^=5PoVp}P*cvt9cj+p=l9O9J;ed}hItXW6vf3&BjteL!>1Ew_Qr z*gJH|_@Ghpefka}C`y(%pev|@#6judpaZ^Kt~py`mq^``58@w=)c8T7bM;Y2Z>I%^b`!F9EVV|OQfq(FIeR)qdNLB zi{fG*R$|NMOOK-*iKAcBl6ZEB^wJ02@wSV9c8S#UId}Z3xLjhFNIfg=_;rbAms>sm z*&TmX;@?X38}ICK)z7X!tke*HpF94l)E|uje?5$Nb5;6+`D;vijrnU>;+Vhe9%cSA zB#!xOLhhKqY*$#@iB8WGL?q zWx4a=ugg*+=C4&r+4KmC@sAK4`)l*zuXQQe&0lha1oPLGpFugqS8|!x%gsgIsYv z@||P-n(PK;KbO(3+4>d2qguay-DmyEGVlT86f4k+o^$Rph zV}1Ej7_8A46iH6YOe}uQiPMrri_%8FXwAIeR%FDYP*)5pMMm%u zAzEKyD&zhYhrWW(@}+Y>JBH26v;o3CC1iPq=G10@A=3C>ruF#`Ipn<#m%f5P676Eu zBk%d?D;mgqh*EeSK&Lu9?<>QIy3QkKUvqs$LwOH9eZ%lP^o`P2pxc7<6-YD6Z`a22 z5YTvD3oi86QY7TDzGBUzuNeDnE$NTki!JCY{$&$b zJoz{zNteDtD<4Q-p@Se;(^v3^b>z3uNMC(LO@~1~W0k(*gjZjY(17$B9pgdkf3(R$ z5Q|r=Pu3`?nBPZVaboeXVOYrEIRXR^Z5aK10;9idADLa#SF8o;E7l$aWnbU4D4H&F zk@OXW(ADcJ8f%$UUd~_C?{7o;iWghgSMcE`*P*ZI-t)9WUvb2xuh8S0g{FAMwYe|tlK_B~o2XgEK1r@Yz zmezo$)v}#9MgMP=q3Z$RZW!BRd-2*+0O?Q{FTp>lwdVE+`9BT5j|$8Wbl-=R;T_CHxq-`W4Q z_9sQ30!mu)vMsyY=J?2Qqi6gARxF+>tgxM~c5oWtRJ|~FSS{}xh zk3@-%C1tS&V^!|z5oY=F)URnDSVF;?ZY}37*aRj3XX>#$|~|dzZ~c)@Rs#&OEn3@{Fx7v-_O6 z&F*F9wutno54YJIvYqI3us)-n=j;?d`Z7y6#7AFNlsM+Lb-81GS%;54!{#C8wqdC= zbDPZ*%x%}CEFXQwy5wVh*&^#RZ0!#7+=QfHZZo78>$%Mw{s=mmdYAcY(WB2WBwq2C zN1suzFYEYkZheNW4^jN((Pz~2*Ab6C!>H#kk3PfJmoR@Vdh{6!k{?X8ZZ5JuV|4@i zjIj}9r1e3^=JXj0&wma5EBcH@w?3n0uhK~VgWRFyKS&}$@*mPD5g%#l*0*0>L6pZ{ z#ZRB%u~+ev{~Y#so+Ct_do_;KcIq?K5tvQnKQ#9kUK*RS9GAd@5_pIMNb|5f30mJG z%}GgfJ?mSC{I}+@S6RP=?~nQGU~_%O6o09x)A}0Os$s~*rB6w$=`+@_RKy33sA2O4 z=`&W^`i%CG*Y!tw@D7gbO?QMxTb}{JsIAXv>oaC5>I;2c>oekg1L1ykuzCIVi&%dI z4hh9wQRD3w9ZH?C?o?;or>Dp*!{+i6m?>>_#?_~AM|huimxcYLw%r10>n?Nn-A;9e zd;1099ziF@`KdE(aSx&h))Tn!1PfOcaEW_JgH?m^m0_g0PIX3-&XZOr#x)Z6p1nSC zj|IE^Y_qad0-Add;Pq@TAL2qt3Yen_Ap!W&6dlk2+)J0nn@m%c&OnGA9U`C2_Kq?HA3|8GhR@ny51zTPhOzYjrs7e>8S!(7cE4JescV+hE*Q9Ciz zrU-MP;ag)8{Kp*A@2~OjW5PSUgVF}JUksBDkA^otdpsh_rh2ssxk6^-WAhGmhG+YQ z8s9ABYP6ii*Vwu`1Bo}boPAEVmjy3p9pl@LS7&^^v3VX3b;>+Xk8i}aF9dJDzyQ^B zIg3tG;~NqOEoYG?XnaF1rLLt(XanjDue#RyGrW7f`ds&HzkoVpT*FPBz@E)3v!qT? z+KMhb(8&HS3My>{Aa^7A`&?otf|8b=iaG;N6|(Q|Rk4nP&6V1T-=N>~qKm%LmJofd zEnxbhPOvN3MO9I|Dt1v-wCcrx=-v1-BfpF^)IX~IcMzOkYroc3Fnu}k9h7|AuXj~! zkiJ8mLG6Ev)ETOI-2b{w)P7MXA?RqoJ{^72?O&;>b3p&4uhzF;YpY!CPyRve{cF_+ zb-F$sor0pyYTo~=!yT&s!G6zQQrIxJDxGZQLKn-Rs*X+ixL~C=nk8xa2=PQR)mX zQi=F}>Wr4$3tX@K_gt`h0haXC`qtl^r8W5v&2`Ct=$i-8#k^#FNCHa!L*gL$54nQm zKjc#KUvh)%TaP+pbpzWo)~0#pLIc)m3x=pO+}ktOuv1^u84EK=O)ISJ`1RQVUw2~p z<~>rax78hObw^v>(LQsseU9|g9q{p{PX&jsxS2EXk~gXjxTh1CRZwXf?i&3?+DF#P zd5`yuJnegibL+~A>o4J^J99D~8-TjQ?{o&l;M@i|u|D5e-LXX<%>bEOyiM7=H#&#S ziFfl9s?dG&I(Kh$|E)*e!J>{92jRBCrS5_){wCmi@*bk{K6&UM3q=(Zqn5K&Fl9Y~|Z!IuRL3|4lWt=Trag+UnjE@2?%^H4@0%x8?C52nt7Y@+r6Q;XSjB6uoo$MQGV(Uzh}vJ z;?*k5+sE5fV|B-xg}iDll+a5dvJhWX`?^Kl;n=+aO@^pD5{q53+|-m1D|`0Yf$nYPs&-mJP~nRxv=)E(=`KI~R^_%C-o z>W+rXU3_4zs&gIQrX1rNSHk6upr!GxkvjLe*G8QSmb+Qh9p9DlE!dY{y5aaX{O91W z2I>yuUJW;QLN{Pu`5@E{TAR^@2O9a`g`n1a7$$;W=Mp;+)W~}mJypp2E_$j^`o@Pl zqh1()jo$#SsMHOh-?K{e1@1>d3QZ^aTARZ3|4og)!2L*|RU<@S)Ga_M8(-$|uRG~q zxrfvZs(5a{ZWGa0XuhlcYYO@2_UpP*S7_Cc(VV`u;-uemiPR0cOd{H>{kqec{-)Z$ z{8cCYm%c&mSK1tIzk|M`{k8sgv|sAFvHrh2kI|prKSR$--BAqTrO`76pl#q46x7KpzV~~D5VYN0*P^`Lhx~my5p+kW&k?!4o#KS9m5jG0Ccy+F#x?RaSTA) zz%{Eox+Tt`?zk+^7=X6v_}J60N=gQx$Aqeny2A$P9qNt=Nzdwz006rFdoa7jdOkqi zu`atG0@NM$?hge={}Z=2DSLNCfV#sjKS14K@4jF-y2rD79)@A>{|CJ8b?5P={;R9q zncakVlR4BTy-{`l&DSti2C6&Y7SUx_geB~aR(C*HY-#sKOX`jrCjUWL+{YdL?Z)OV z3JH?hm(OdTQAvDCo&>CKvpYJX zE0iYD5#!%vZGG$6y8-#C`)PcC#9u9Vqgr$9-k{|mJ_%fF9(4y<%jw-4-Lpv1K;2>X z93=(Z?*yf}LFy2rr|r#Zq<#3td=ck%9cM{-}hG%unO&mi^X699^EC!4WH= z-tc^(7{(DOc-|Y%%$4U$89JpNr>s;mQz7ZJS8I>T`^-FfpKb5{Xz%{`zS#XS6p4jn zj@=(MZ&pKtk#JP`xoYaos#EDhEQdaXAzGSaw+@@Dck>*p^&#ssxN8bXB8~554!`jk zhrH<7{!#O0CFU^@ayFt5fl#CLA;_m7VLYibk8y1OaNva-Gml{`IdJ>Ob2}FA;nqL9 z5xfw&l)U)$mgU9PPTxRJD0vY*(NtbU8YM3x*L%1Tc+$E)WNkNOv&af%4Ah7?z0-TV zy+nRj%Zr=c{?WvnRX4pEl1E$7hx}ZVO-^e+jMlomS^4Wjc&i8XijGDjn|QN=kQ==H z(ED+V?^2bzS=O4d9dY`jD2rS&28YCfh%{H>(d>y-t0I zb$r3C4{_>yYwO{*K1BIpZee>{llAb1#y2Fsp7pR}d<)iy)K1?>{y(G~=>(3ZFLU@W z5Yo$6zwIBZe`D)I;LmCN6Qqw&+Keus`FbD6E(Gn<8Sqr0NFU+aI(AVlBM{tfj1*ylBWVuq6K9}9x5`shO@B(9Uv z+lVe23QSrM+HG}l+v#pX;_z|%>q9I_!3gz&pwNs^Uz0n0Sq@WwMyMl_FHj${BDsC^ zA?uRcXZy!xiSyBi*!|$phm1-3MhJEN_h_y((TBM3YL7RoqT~)et(Xs%+FM;Ll*6o8K0k<%ap>U`MD4)sOXte2}VY8*?4si zKSea{u_O(OlZ z!$0hZc%(nl>h6)P4YVcLr%66~w5lXRlkuiiCCFalq0czv0Xw#qOH%tV-p>iOo6$f4PRDt}vD5ya8 z+Gi9PGOe1kD$wuwuS8!dW4egGQpPZST_zQ;yjgV;eXZG`piWCcr8o$rT8l6jzEs!# zVbT`V+P@~wsz6knWg+Nje~rFY98mjhXHhjm0U`79uQ=(Ce+u-qGK$(SWm3?${knqO z|C?(6wO?}5f9e0MZ@=^X*XZj#Mb!UNR|?kXYsDG0*<7H>oEXfZ4zXcZ2B?c9p->9c zWZ8ANhYb!hKy9boMk#Grcstz*ET;FtkwyDV0R*bp-SLq}Px4w;aA3{YQ_ zI|is%Ny4JqGBmBcVJ%1s2B_`y3{YQ{da*iWSyKA=Cz_Cy3{V>(O~;`Qu_OhnL+q04 z0cvEcU8e0UDqwfWus91+XthThvMlid+K_c|W)+|fS(W$zZHOV|2WUesOMHMfWI~)J zq0OQVu{(xYZdf{!nJpq+$joNPvF^=&am;KBl5!iBd|6)HcDhsAkYP!|EN8!3W;UA+ zGh4Uh_F*=gc>=W|^(<%8VP@-=I{0WqR)i`uo82DPhS+`N(1zH$A!a$d1t(Z@3>e8Gs>H-_Fta$m%b>JLIcegio5aPX@)Pa?C4_Vs}YzAc3V%{UL@$y7GrG zl78ZP4^3~YO^mkMq^&k-t4-Q!leXIARacwXn@MoSNPjpoP}7^#VxoQh;kb%{vSmx} zN0**i3T3xdcOD)+cK_*PqlfR0EM>d0J9WhF>Me+{4m@C;IdSUorR<*GZ1AdL66>JqeQFDp+PYRKg;SG-;-$iTB{X|}zxD=)ZEmc+%5%Oi=RZyP21}XB zd}&T4vGZhR3#CivbD0aFvzIEFbg7iSq*P#YbYJf6m7Q_z9c*`uEM(tQ4Yzl&eLw7A z>mQ8whX?$Imj0nAcCg9Nk{`mSC!c_v<;haIGMVLsR<(yBE~L+A$~KXuk?!gpHEJp%Gj}dGm!YC>otZ)**aw@+ z&840?yEHOf-99%zo4PnTxin%{d(KT?OjXjQbD2u&e5O>+73P-SbM@yV&r$@hdOTSROM1Jvy=~^ zqpKa|rdu5`FGJXDmi}$A#ce8 z(8kB%%rVla_iV+xA zjG(Y$%~1BP=KzM42r#U`x{*T50M&F3cON6?cn{AIh_OHp%vc3r6d)VR0W=m81z=-A z+Hf1oX$&^k*E2Ynu)%ClgE#L@G#Aj~>k$Yj%1_J8NevDR32`k;%`2HQc?vXo*)rHO zguNMnPGiXM+^ki8fkP(4+nW)X0DLi2FalNlX?eyWpAp!BQxOwT(W^;dMV&YmF#{Fd PSir`~oe>CCl&S{+md);0 literal 45072 zcmeHQd2Afld7mXIiPS}ka%|I1ow4G?OPLz(eW<2OiK1nCMM^wm#8uSsaCerwtG%?d zE0I20nQ77_n6z%Z1^P#erbSb?4U#BHlO|1zxJd&9DAGho{xHy>a0}#*wrQKTXknm0 z+~4<(|(^idDA{ zoT-%3*}S=66j#jYOJ=cJf4M&TV*LeV-^k+*B%?slnUnfMdMi` zKb|b5%yHSYrU$cVl`2|!tv>s?{dT{8xnh*j=(-)aoULZmWVxo7dE>ns?eV6In$DV_ z>!V}pA!_-d(WCV$`t#@}m6}#{eA!4|FpH`2Dqi$t&8n6P`USI6H1pJmM9$vLQbflN z8tR0)#M{WCy#7&E%|qigt1_O?E{~Vjs+m$TsQClqwPLnPrN@(oWwJia_Sl1Z(#YqR z5It)mXR2=Bn@y_)_Us*1vzA&cRn@st(PSOS+&BAd4+gSUtlO$(=F?hRc`wz#L9wQw zz~_4~ax9y!*X;)|3AIX*7X}>fVWYBQjj4wZU$~sIc=O-0Y4fdGw#wR{csBEZy_YKV zR!pmwuksonN-Sj_+)h22hi(>v{oYMmitZInwtNylc514<`8*YjY>~~=h{x^+ESe?k zmxtN7^B!aVss9VSA{tJXNO62He2FZLein02qd&;Y3RqsA=kD2ep@K&XrBp3%K88$k zrD)UwmZ@5=E!#a-wQlc9naOOy$m8jRczU^R4+X-pP|P3qhhpJCP>VqD zZotgli3^!~>?fKwPD@Q*V@yr27ANvXVL4@}`cd`l>9S^Jix<3DrV|Oa6=#a+l6Q1; z^qlWJ?KUj&*`Cb(+t@f*1KZy;Z`B&U`CX7X;=H^wzS$8^#(&)lHWTW;PBYPN%QN%y ztt%}a^lQFYjIFfH@s_l>4{H?bb*ojlOX7QX{iQWRE}gjg7Ob9|#=>{x`iuL*AuSy4 zvi`zRE$HuDe|F!}%-qxUOuv2LsjPWfobBpKkR6$N+X9b={Xs42i#RUua4?_+!#u48 z%@XXvVy&=jR@7?7RBNTO)d@Kq(4x_>1LUwT6bfnnpucnL5af57MkT+dRx3u)N|!1H z98N?)sUY||Isi}+pXT>*P^~*O?2BQ9LJ@~^f~wh<&K8Y)_E}RU;-#(mC>!;o^So>u zT)!_81Xkz*cW=c6`It1-{6v#=I8&I50Ec{$O34nf8%Qs7^tSk7FT> zV=XXxS}r9(&kIW=tYTbRxC&O9rtD3(MIRolfI1 zb=gdUe7E$1Q6?ftt~k9~Vx*?Gac&j3^qb)60otOUJYov)V=ytAdzKx1dHMXeHctdhA^llP%V|{S9umosG2&4w`xtay|b$92c0O=t9L)gBkJ(l_IUJ5eL5RnA$+kfVN8$m^AMOJ6UL2oRtpJLOhzw8;ATn0W zYAZ;&Egp+Fu9T=RPRvy|xQrHG=rWN}@CRT!oZm7BpQ-2EQAnv8plNvjey-ioaD8BRAk@ZKa5-?tu8Gpw8w@%z zn&D_97RQe20+;icT&Cs7V}_$qzh4VB+)3? zBBVPqnN1|a!D^{o0)JUm84b-igC<#_9o`1tV-{DkMRQyd;fzimwg;2ICPZ)xCZukM zHX}C-B&iJE;^zI55Fur?AMlSi!vDDe<~0F~WVrMGo65Ywrg)RE82I1v;sk%+&3HM+;dCtP+w*M){rD@AVNU!Z0uB|7vCJ zC~+^Smlg1WMH*9+)m2XHd}nrOhrPR802d-kETzn}x)H&dD0THH_^eU&$YUfYeV(Za zBu`W;lJtzrqsxU+@CXc&cNI(_oy1Lbo!`NPdg&~h->C37m@$^ITp%?B!6tUhTQo20 zRwbzyk4}hWS%xKHbYym}vG~qxZHK*&twEWwZZ8S`N7SaHG1_G#e?jK(5|ar)X_aC* z1KXSz67uHO^6X-uU+uH+qOMET5T6kocZ>ofc52m`UF0j}WA-7M%e;x@*R~`!IK;Xw z77$~00kfrl-VHVxGbY!tBx~h6Au7K3z<)#QhQ&$Z6l-FsQZwZ;_=CR8ckTNH8`#Dy zHuJfIL|Pw{pvP43>)_e*7Om?od(&vIOKD%;7*vad*xY&y+``DP>%Lhhrc!;B+`@!S`2n%Kcj52H>vZD&#L%LL?xPoCUY=|t z(Rc(pMZe>)#6n@1j0AbFHaQ^o7tBJbvZg{u$Fvw5T6umreuCq}#Z3>#Rv2_JHcngO zg;))`sKG1GVV4@VZa^pH%AL2ebh`!C6L%st3>&ZzzCnL?%B7Uf>x{*)Z_NY+0WELRJQrirh7=Umfti?J_ zrg$1u6#;)2y~nH>E!7wt1eO^alo7ll<(AjFF{ zX@6K+3~A7bIa6dqWbd0b@~TwVw6{1&N|LCUqBhXMU?8Buiq{$Rpp}Ji5=tNoT|>|j zKt2y8fzVxbfDVwcS)fCM!axTis8*Pfw$r9|zqwjAiYZvsG$|cfu?UuGSj>9c_`+5U z3L9bc)dBK=kxHrg5+t!sggS>E2ch0+$-&a1K4=@eDFydcK;B*=%Pi8}b+j24AYI|c zgE1}c@3P$XP@7lEwN$pyu^vKk7!w8r>(Dk5Wa9?AlP-wj;Nin69}FX9sa|r?7;SGe zRNPSZ32LGPdXx+#yFlLq&k{2UJssq29Z;d0gv5@o({`v}V$kL;Yxr)auB_TW}TOWUkmUuJO|qLN+cYIO=h$U zVBje*w%srW*?~%Xf)2Q32Z~m+vTd^%iA8~0yX})9T6SbRS1IMyG^Vr@G&Q-?Zj;R) z2>>G94w9jUaCW>x1JZ#xV3FX%(r{{Z3@9)TGpjD6v6mgJum*)LsKssrt%^Z$jib+T zkdcN36YK)ENkDxcX(ZwAMDUq8<@EjM;=aQD=X|Jc(DqEj4@CM1Jmmd>}}s6Cj%lL0Q9lKlm#9dSO8 zi`YI2Nj=N#GTtY>Vi&L)+JW3)2~<{9d(p(>y~4 z#w^6+^L1THDbv;!L_IDc;{_@68*}uEB7H>C5ZP#+7Jz;HJvjIsf?%P5Afvwx!qL z{e0Qp!H$5sJT$j+(HTs4z87~ly|PekorsKS{Oa6jBp z4cNaMK+yftumCA`AMKI)c`CcR=`6B$m1@;;4G8^w-QJlkipLr}e?iqqcSLGwfJg+t zlbcD4J7VUw4N>#WJ0Q9?^G+TIcc2_I6!$^h5_BSYL!^lT&o4~lT1ej3rBl9jOUL!A zxWoJ^0{##r8jwrG;vH%;mKO5`wLnC2&Gz61kt?n>7zSZQcOP&9suMF#ojD6qxSGnQ zVO#r{?eQ^Y#@+!c9Mm%?(hU!4dpFSN(Zv(9$Bz1d5u51tJ>&&JHhK)tl6S?dGDObQ zc6=KoA}B3Tz_5_4mCG=-9M2>#GAFSEJ@zh8QRTHP?gID?rgKA*G z9BrTx&7!dkw#+|e?=YOYQAn|#{iqj5BifB)iXZ zNtox6J6@9LZ5V&z6-MrG;scdmC2u!qL*XZCc_Hz`%##-xf71C?0p9e(&n-ia`vNJ< zX^jxWr$4v)!o%q4`xy~_why%b{q`Q#7^3s2G$loRA;=*+q=naYHpWNjY|PC8b6P7ZKs)f&sueLip#=td_`S z5y{W7DK5!L@s)?eh57J%0h`DQA{i!l&@4^4;^O`55gyg!NwaWby=$K4@?n6Y*^a6*~kH55EQGR9m z)Z;xK&m&R~afp{GH0NB&wKxck`eT_K0^GHkD&VWCOe+p9st1Rgvj!<;o(y z9OD^fD2p}5Oq8SQw^N_2S5~|5!)l6+i;VD!8bm1b%XZ3o=TS}$iEPt$LaK}YVq(17 z!@m&~$WYGNE|mE!(o}@tvQM5lt0*7%3>eCcWq<>r3;e9#{0C+DlN$N=yz~`qrBbSu zb<12KTNOQ>T?KbWOrr87mL0hoiQwH*3)aM#e z`Xz90(B*!-NBJ`Da$D8O!a~1Xs$3A&qe9Oo+2t<3KmAp$T*+QyPLG|f`jlp^l;Iyn zT&nUp{$j!3FfXEB(YQAH52v*YsV{5f9*66HSXFTUWL>4YzMw(-X6dOlaGcpBU1yTK z@byn=mzrU-muA?Xg=gL+GhHF&9raoLinh@O2tw%cn^J{%Qd9Y~81gCx(5(Ii-d2*u zBM9gZb8+OzN_y2F@cHH3$Px0$4T1@D_|dcv_hOdC{*v*`??pwqqV)Ao()+)Bu}AsPpMT)_;O{K_AHkU0ak3U2#M zmvkUt9o(Q^vZPGb8i&kuJfFW*I0CepgOHECs3{nurHxo99ZaW^%c;;Zt~v`v&E-@$ zwH%HH4AUPlf}!;CC_CFAC&Y=P3dAW|yn_1)yqRjXY#kjR$4jQRtU(SpzF?%xQjwiZ z;L>n|P)qeHkoOT>xS$2pKp4iP*V7U?g3ckEj~-xQNiT?OgiCJ)9#=G6i)$(xPBI*P zinda$X&J-HDB?fGS1XEEDUse%fdY25%5u}WhiEfjDPihTDNLNAEnAjC!$q`!LMLg} zSVouu2>!%FWou-wpLlxi*zC-d^Qc^ZER7e@Hu7}rMh{8I?aQ3X(jJoK8|Q?`_>3q> z`-}dWH`x6ZTxG0A4d|EZ{Q_K-^+*wU;(480bx;=`oMm& z!(yN${VegC9KRuv{V{|TC(D;bc}f4jL=)WfYpY(FMOsUF$uEeC+Vi)+FQ7cdwUaTj z{&-sf`6|P6DBC_nh*GW&(0@f)<$(g$kL451YOKE`;$`1ORu*^#;;%JF;#t-!qYHR& zD*wBp{Fuxr!uIlH;{Xjy{i8dTX8>Had#lDuJ)tx!{+d}?e&r0~i~T;_KpOJF1hUDN zkc&!15*hFdQK@}zLp3QxcubcAq8ZmgOHO)r%8`qv`_(H&G|Jw=jFDh<;m!R3JIEe5 zGds0t&|MbIwt4#(=k?=f7oXOro}M~2y$Jp0GGvw)a9i}^Jgh%flgWC*WB1WzlJwAn z?kD;X81)3M9>2U&rq|vQG@@h_NE|_ZR`J*a6?4VnmD$61MlENi7apHFeRghXdclVa ztaCp^qZ%fH`;cK~ZgFW*f9lw&v-R9RQ;Vqyxqrn|!jtC>sde#2Ywg*p)<8$C!;AA%`l%C(`pk*<>Bpv)W}ce9p$_+MRdukV z>UYShj~|;$=u@ZX9zQiRwRA(Z_ia^ssH675lMBaZ^^<33T6^6`(T)@C;S=rQ6Ya_I ziQd6#PW0#4Jl}=Y#o%tbzWPs|3QXxw%*ej>(R*v1YK|=~$2$9YowUd{DkX@URyx2c zot;@+6eE1SweD;kW7gHST4_<6z)SfOgl19@)rQBe>-B}@cC>e0#of&A2)HBQj=)cC z1U~(=B%gn3O}H%;zumF&+r9s{Wl_06-4Sp{z#Rd11l$pDN8o2-1nP7Tf|=BrXwuUx zn%uez%yDqX5hkHIWUsoeO6HVPM&s*5pz9C@NlWv80E^qnEyR zL)C43?xtc~CO5hBW9kYdHb$P=P)t;7kji?SV#v{!Zk=dO-iTo4n;WnELS(vRUZ1>7@sDz63XR;O>Pce^+jO$dYi8&W80N(|imGoyp?-(vqXn zB=?V`NKm){SjpW33BvJLp?x~gQ_l1c9USmnnPg|f!1@C%N+pl-4aSzvD?RglJ=gGm zr^CawJ}0FY0rb2CV=(+~wETQl8CZW)xr;zea%G-;C$Ng=_4)0!Z~me~yBl3X2j&-) zfu7^vumSXmfu5;v{NxvRU%hq@i!*B0)p`Di)GP0!WU5>w(<6(cmWVidY{b1Wu)bHh zgmE2N->0ZZd2L{Q7{TD=$ofI$3ZAa*MmN295RcGD_w?{sQjPYbzaRBxXicseFB5fC&zLE8Q7LKkDh<9#T;(7fD%X(3Hx{q{^*S?7&zs#aMSC6xh zhL_a1*Stu2lWF5U>SQB{&BRr%A8nZlmZRg}wO`$0*gXpAtQA%l!EXD_f2`+PFM*iv zb$)2e-~KA?>^#0lSkl1yi%hvavwnpXrB^&xpV~_aTMgVu+YFAZlL{9K|8+7Mps;nG zM%pHLRLs%H`rQiMFEYA*4+}@u`BaarkBImCSQr=kaAy6Gfd6h5j;=o>-rpm__ll4Y z!um~R!@W(`WM1-2CZ&KvLqQY-5RvJLv++Bqz^2N>9EK0LmuMDg+lUATekzHqzThjSw! zY7#fj55C&J;fELQ$*F*LCv$>T;shVxj1#mjR;zm9QWs9}av7gW1s4XMsfizQxxD62 zpYl6wTrQ7!NV;6!t>^M$0o)7N{7LE!gE@TR1q@)r$!s3zNZ69?WEOZW9G!=Abgwsa zbkB@2Z0T}z7>2hINB1Bo*~wR5R6joT?Yri7d0+elf0)0b%s+^8lQVlSsxPzTd0m-* zUIF{)Im7t}I#_Rx`*S%y&eFRapUd$%iY^#d?*8w#b96o3>!uWs%khn@yFwBnpV^j> zq-8n^vaBwTLK<@O%1`OP})W2{S1L(jV09+Rrv25v7z-;HSc zhDj5*2ZbL#;2_+ia+`B|J+0iH7y7$vuQKtNrv(+$Y5ocN^5r(+fFNN`Lpv2IJ_Yzx(K|(SLXpSAR!(Lzgsds1CP9n%=bj zu6S$KZx{5c+;(nn($(L&+?>?5Z)?l6EY&T!xo5VkzxyRdpb+pT{hj9-ZYDrt$6YNI zixkVZpvw)ub=+VqjPKMt z>;C9l$wBVw2X62eU=ZJG={bcMe2r}wpPY|J{r+IY7x($JKvaAME_V{?FW{qH2a+|t zj9YxIY$2ty4};Yc_*5l(Wml=!@m<_K?3J=5Yjnfm9+r{ct4`VPqKn(DdPj~TFo>O^(ZgSonOB)!l$fHIl3JFToI0gPIG`v$Ei)$-NCCxj zC+bTGFgyCXOzC0COUwnzGrCUMJkM_{-{gg{<`Nm~JuKO&MR`-a8G2ZYlQU9NrT|UN z;G6s{b^=p|_+-X-Vb?H6pU^28(mhPUfm1SMy%{s)fLPv}B}2iRDMJxREBP`oW+;OQ zl^&s@)b!xM_yE@+H)sD)zYy0T!zranV3Q;q2pGbI+Ylp&A?5M1^#lxI#%+i(*bvRe zDYZZ^a`cD>2Rg_5xCF;LIr@3WJNx^&`FJ{q5b~)a3vLTcAg(M(kkSCUlEpp9$rm}q z7=a8W+D0_2cs7yYaAQ_n<4tErXyN{7`yoV>)9V{RY%mf8sm?N9O0W<*; z<|4o_2WvwLbpwb~5)wr;!A`+v5F6B>%{vm!1sM4ze=L*b$zabA_GSQD$&lf>S*`p6 zhfIdIHzP16eW6ZegR5W!s`%0Jj6*&numh(eCZM8clfa5PaVla4D!Q_Ojgvbg5UMCu F4**$Z-*x~1 literal 45065 zcmeHwdyE|Sec$Zek-X!ZM_JZMt~{pgc#k^H?d*G5Q9bf_qUq(Gcn=Cu7&Zj^FX#~Hh_y0C(vW>&3EG34xD zYnr)IZ6j-K^>QD~m+VHx+%i}$XOC5Fw6;*_y0v9@rk&BI)#@xcBdzsLIPA>K7VA}O z_9gRWvof2j=dD@UHP?exv+GT*vDI1p=3!^txY;xt=yb=4-Ym6>YOc{X8ocx2UVp-c znr5&r7{>IBdV+d>V)}Hah4DPSL!*}6nq4<@*R5K9wuKiX*S1^rs&UUE_UZQDJ?WHS@XIw%weql-6e(TdiWf7Sr_TY`a!!QR&&7XB z@Tyo>P~cZaFmo)MVRW1)u?X#EjTc5e;ge=_!=6!3p1giDZ}aYde#h?H?NW>NKl^g= zN#`In7;aj2yVBw<|6A>g>EuRkKuM>ogT| z#!(lo63)vP*u3)*WBqCTE4(3^PL@b^063u=IYfrO(gqIa{g?XMw7e9>#p03vO z?TYmbGRd8yQ4d(AR;Ru0jM%M?voCMuN>#Ihr}N_JO`PFqA{|dhqX_h5LQ5r+ouBLA zypCA4PVpIcaw4A5lF0;1Zm$ub>fh83fk%VfWlK7=AeSUn-K; z5=sC4m(q1Dni$-F&e+<;r58HIap&0cCF`a@?&<~59mS4kgJ<+)MvG*kej7Zar&Hau zz70-wAE~vg>sC{36)m-0Z`cDhMv1h1<9geD`2XhMs}I2sukz@b8^W>!isTPjg6p6;V;EThGv z9LIR;P3bf=AN)hzKiITDKjtiTybZ8IfOW0uK$yip{JD<6na-&l$WB3z z&tfCYVlOa)wuSmif8p0o%&6BjqS84*GP!slCnlO$~!%4R5NYn#yRTndzr0cI(=-Z1Eou zQIW+c)>h8WFDx#somHR4jH0s!y_`S0c2?W4S{|BM71V5@4>S@*E9?DK$l2Gl*kSWc@lS#?Px6WfM#VmaJWx`PW@1_@G2~cgK*%NhkEJtS zg*>HXG^$?WDy~N|S|XG2p*q2AB_djK7;SirXu}dD6F{0bn~>oQKoRsNqNV)_LePU5 zEfX0enL!tiI|qQ8Q7uQa{d96N5d|Zt`x4^GL@bg7+!{i+JqGk?x2vG2h{yot03u_< zDiyZ8lY(k8T1xjN;}S7YxmX8Z@aG(6ux*Em6ETp%{uL)Ox|WXn(}y5sL23-`Z!{Ix z5`zEm;DSUVlhUG*A=N%1xRatAu2dQGR#v`(wn3 zFvtt~zt!A2P23CGWd*`uk!I9fYm*Z@-(5Vq$JyVgf(sEP*78g*d|xW^`A#pE8AWUYKBNX0iF_;0A)usKPbVxL)Swk^30{@GaZ zd(IPr4fHUJZa()2k=AEr)iWyib@1#JoA&h`N7LS0E=g5W3IB=LPe+$^eiS3At2J7+QqVl(?lDoz{<=-ZZ zHd_821m%B&e-gozf0ydr4Q?rahjKnxoKpUMYU6&Qk@6p~UO!OWQT`8X4h6*B8B>)1 zi1G%Q;I8Ddfy^cEYpoIJok>WSGPFX;y8jcnC|K%0z&q1l#YaJFQ2e5DyO#6&;5u zO1*N!611eRN-jnH&ZK-4QkR z*Fk8pi))D!Vl5b=g|NiJ0WECbg+?lnI^W6A?LOE}JUp~;Pw1D{Gf7D2dJ^Iu{7Of4 z2&8>Qp{|qu$f}UKKmjYHQ%t2@{<|ZJ($#81r12Ipryss{I^cDgq$AKO-NBIc$Mk&b z$!JP*TL-z0eiZWlVT8gVCVM{z>tpro>RO+8JRZyFWbu;_`Y1O6!qVb8%w+sY1ZR>P z9aZZ$tSZDjT4Qn?{zi|}G0SM-y9S};ZfvTekvcnrBOVMSh?Vx%Sbo{_Pk6W7!vzCV` zja$n@#gee-f{~ox{KitCqoc#N@i8->S1WahV0~$Hs+&}1L$*bXtkZ^3(axbJDBBxk zl0{0pfj*}aNiF5x#o|O$n;BwfhA5^EP?t9v?R=>!wu8GM)UJZn@vK^0hl)`bs-Jwqo?~2Pr~|V*g77DQ9p$AbsMk)DsS(+B%rI+(r$?F>UJ`%Yms5< z;={}?F9(+v0A@H1cn(W~4uqdg66OkNDd2-4dT4F|DlG{H>IJI_80mWmq24A#f+67k zDD6D5nrqf8Y5}V{ST!(7+I?1fb3w+jT672}>m2QhWoMf-Ap&&tPE0ZaIkAtjJRXl= zf)g0fkjXg65G%|;p#7ewiB0EgPKZn#eJK4j`MdWZdk)4^2 zv*-L8&JGp)();*j6=&>5y_D}PWkX9XIF(G7|JEp^p8I>g7|1IeN5-8o@_p$z`_0Xg zZSX@l?HtKtMUA#?84SMKov%6v$?$^ojld7&9CnC8(#tY~47iC+pdL87keSL-dwq#D z{bgxU`%&>8yCS)r`0mx-!e)kPy*Hixw9XKIF%!|J8_kW=s@PSl4J$`l#@;>%n>NgQ z&olqLtK3cR8g#t0v8^VPn3Vqo)+u`ndTQ=X1J@!WeC4v)3Wu=+wAG7C*%_6dF0VXy zVdd;1?De?a9qC?A;7vH;KsP%>QKwn+ynA6p!u>|$4MU?$>e}fU;!B2TCw_X&Iao1Y z-h#ynQWo6UJo*<#Kxr(3H{2gx0Bc)LtSlO3Qt9;iQ^+q@{t+85##By@+ zQ=ve<9muyef`(jt2~WU#_p1V42Hi&Z#0R$$J^^lpEt6y*LS5(H%k@&N)$iO}-0O^< zT|7_J&6u@m<$xIA0?^5(oKfh&nfvcJ98paezMBBh0_--X$kPBv6z%} zJ4bp%uDH-(5`-1qdcX;&LC83N`3gwkR=!k#QSCEMD8gW>vjJO*dUtzuC*N?z1XW)~zPC@oOHFpX?C8nCgPE#_Wg-eJc^ zoPD688e1jY0`T)p-G(G-?ZyrMK$p#^FSV_<#fWfjsfS3dc#{=1L>io%EIS8_GkKds zxWIfisMD`7YKRKq54S*&n;6U@&cOIjtF3PVwSh*oYUVnu9?yV=eD>m{1x5kx%I;H+ z4%2Cv)jS*7UyK4pewCAB6=D@2C%_E{Y^c(ZF=ucM>Wt79-hiKGn0jM2o931`7hK0* zw%{tRepY?iYSv*unUm5Tkb-bv1G9-~hD(iFG@(PS*Wf-`K|ZtD(ukVf)jbt_ei#w2 zI*%C!k?A?JR=QW#7cM05wex zg%}(6fSXaDg)t(xY7~4)`Cn7TA1Oyt{$=U{IoaJuKBWAsROx+ZLdw5R)dsAG=Q5Pw zN(X`{#Aa{?mpNE52ztL*kE41N)D~QsB5*r`?!x8h&`U~2Agzx`e0!I;F*8NDvAhJ( zwRmi#{2aoE62FyKsE6qhAgkFfj)d&l4`Bgz8<5b?sE}oCxZz&}rC(*Gfw@UaA%JN^ zU|R@tIBX9$H0gN2ZZ!hlBJ=%*hN^LIPeU`?FzFYW$xZuALl&hsW<*adNE?3U42}qf zCYXq5i^KS<;P!9@5il3j|5pj%UqiS0BG*ME$7Azcl9T5<56Fc%?}t%0krhNToZ*36 znsO_E`yWX76u^DoORxgC{~^KsSR@|PQ@G89tEHJDiG`Xbv^7->CAg0_*DTZ%#aYZ$E!n zLNKSKl?nVmq8w88;+@LQ;g5cE?0+wOPgEwuagk2ZvP}`y7tHP9uQF7Q@^>o7lBi+& zJ<5Y_I3nI<=qksG$PR=>J`py=t0L1q(j?D2sLHOq8SMchjD1S2nxy=dC=O z7a0*2Er?L&m;IFOE~A{B64|HSgwz)OrNw-C!+$3lkfB_&eJJw{q^StOW?#5`MNxkC zm%&jMZ4)dA-Q8#Z!hcdGzpjyI&l|s~Z8YochGAP9WUOKoN}FKMh)Yy{on=R^ZX%es zjoR3^2I5)vf2F|?cB2if{x6?vShe{}>ijpmQN|6hZ_wrby%FWxxW;YMAQKDYX1#e` zG>-;DUuW04{L_Ws)*8*y4d(7R*sEXE?2QII!-!2)zR6!~cp6qj)LRFKRd3b+Z?D-QR#u-VLkJB<&rI zS^Z6|Hv~vR=-QillbBLV`6V&sEli-*`YpU|z;~)TxcaChpYH<=V&@3dsY4 z1@!sRvQLg;mBsmz>kjCl92><>ljl>ErR{o5*{{fRA+`;JCG<>xkK^xkMY*MnjnC2h zfBV*m^5x(7*;ix#XyyM={q#rmQ<~q8^>{&{8CMLWSZd&|$Wq?OnE7aAJyOuEL?l;; z_?7$samVhI_sHBW%0iv}vG~3HyNdEpg1?XH9CUGp*v|*Vo_@$GdL$iBS?l>kem#+jnwB0lWAVcJG{bEW6awR@ z0&t2RZ{TKvaIw{D*r#V_@ltHBYY@cEu9$hNUSpUEY#Qzk>T6yF@IIB);#yRVCZf^! z2kD8NLI04g#|WUXq!&asf~EHZjw>2&#I+O+m<)(d(Kc#rt!UasMf|7uR#VZMby8d^ z(705h_))tI+iY<$HFPvx@{{oT|^5ggh^ZGI>I8V;7>eM?hNdW^Diu&UA#E& zKPw+Rmgb9S8;Ko<(L=Ik?>48hbcSU4E}jq>UlaxDTv1zuH|ZzO+Yvnf<@@i~WqEn7 zWGI(crpxlI68xLOAJr|>&pooRpBGnu0kIS(%S$>z zhI6b`@1OU57D-gT3zsB)FC)4Yi#j|VNQkmD2KJK!7851uX9?Hj{2dk9pFv1*vV237 zm-PP|w7{;Pr+H-&X?^7-zaSd&=J&p@pghHS$r#yw##2DP%J3@6dZ!3c%G(q4Ur{!B zpn&yb`9!z6`!9!hId+ki1z`dB+wMp_%XVdS6%T&p|FI}PBQuK7Tb_&@pknF#?Y+v2 zs9beMTjoY5tGG3PZdR6=R!YWLPbDL`xh3hc`s+*TicxW`^J87zq>$k;We$jD+ypH- z>BV_pE}E`ZZ`RN$XAiSRg4sp5{3kd;&cx-#`Bjr{v2Yvb9bR2F&Rto3!I*zx{?fuK z6rAf2T3*LJ(W}ca|JcmsI@ypjM%PHv!w80-9YbJtvbc5p=0<~FN9$0Dl2ssa1f4}C zTwYjtcK*35OKS@&5!_u}{&RGyYa@6F87?lZuB{r+pS^UYQ~s~i zW9mZrzvC$zD*pu!{s#qP<^M^+Ncn&9w;v)vqviiC!vBNNr&WE2R#)$~)sZ{38XahL za&>v$xO9HixOo02jkEJ>7oT6as{xPRsp;52)1Qz{pF6vhHRhjNdiK)A`L(-hee6!H z#|K(Ja$)7%qH*EM#r{!`QMB(uhxkH=_(F%ue4+QSmb2q!w$6`Wcd>eR++X7tE=A{! zFJ6>m9i#XDHq|oQT)u6N^EPRd^%|ubcWiWmHM)3lbydvp2W@q4{~WWSdRnDTZ2>M- z>JXYqL6irMeW2H`SYELAfrB!}jzzr`G^tnJX zxrZxP$9d!ayB7?&70AvA_X0R1w08C{;P#j12yV(N53|2S@53_~IX2AIttl^T(wSb$3qBqoV?-t-0T%%lS zouPkphx7S`b5|~?&zw=Cu2KClw=~qP60;$e?tZ)b{0*G*0*_Z1&jvR--ed}82@010E9Fl@gmCU1sGp9FG>YR# zk57bd%`xOKvHfJ9Rw>l@_lz%HRz{Y`MsDN(UZ02CV}4pMsxb7mf5?*Vw*2j)GO_)> z@(5LRnQQapJL_-g*7EN9xBQY%zq>=i0G3yjiIH=^=b+jzPK?a|-j9B2|J%1e$>Pkm z{q{0{MCz@dq-1JcB-0~{qn?O3dhEu%H?e(Cxq*3|+CHSHNcq9U_9TKMb5q;Lm0Nha zy&uC2<3T(^Aw4p}V@WqUj*`b%$>qn8aSl<+zNziA3dNzaPf&!id=P2-*z!$nkF#)k zdqTW((~{8FPO+>nDKCtX^6~cXqsZT2QK2`^v5=;h^tiXfNP3^C<3q|^H;Jvp+gw4~ zw-PMJz`xsn^A6J~bpr?yFl3yuFa*$hzFzfM!01)dgbG`0PxLN|;|Z-0`7Q`>y0 zr?#iW`{OLkh;w*x`-G_fvn-t6{+xLKya<0ngnSyO#4eiJhWQMBQ`;v+_>`!BMuewC zsEPLGL_TaP{HC}0ZkXDh7xCvsxFEu3MR-AkUlid*5q?R8SrJ|mA?fI;Uu4UST{elS zpQESle*UnUp4xtn+Etdnq`ahDRM<9P5$=DXTYP-TG3J*e^h2BN+F?iGw(Y-dyF1$5 zn9sA^zdlJU>Vc;#|Aj#g@3~q2dT?a6=hzZ3IE<$cYu`TXM}|#`{1>`>hx7ApoM|)e zNFUt3iVgQ9tM+yTkMAofyu3$OZs5Kz+%NZ)@)T;y-aSM3&UnugFELV9QQO1J3bu$9 ze0C>R(0-}aGO9O*u!1)m_)03+FfdInhA3e3++RKAx7Y-19`lh5*t`e8=4E2|dZFuH zqprnVIugO>on!Jw$?`J5kuWCP%S`b4IJyw$=-zd6bT7^@Xc=&Hn1+uLNB0>}vUBge zuD<%-Uw&d~-}2{`<(~n%BoXqNT?tA0Hq-;=_(GlX6_7&r%t;4>`6nyMUW}aNOLj-RkfD zIwMeM-@Ej8p%=NC0LdNq$ur>&Ik)%Kd)4g)+#Xb;0k;=$djYri;oKY?)6CKCj zFCh6fe6{OXu5C1Mi?3a(<_-2`uuc}=s${S1D)kONjC+8+Qnp;1Za6%^GV*)XDf=UI zal2iqn4cca6Jn{G~*H^+Cr^l$$-q_m*`bx%T{TX)%SGBC4I{;@hX O8Rhg|1$S0j+W!YjG9okp diff --git a/extra/sqtt/examples/gfx1200/profile_gemm_run_0.pkl b/extra/sqtt/examples/gfx1200/profile_gemm_run_0.pkl index 96202b24ae407ed861a74de6a1ebf566d9b0150b..998115a4e4d7c96b9777b1297b13b7c7bb896058 100644 GIT binary patch delta 761 zcmex6nfd%{MuP^HsYgl~z#w*tMh|~UW?p4_QDTZ-N@`hVa_W>G;eew2w9K4TAO#f9 zov1G%&FtvwGNp$lFEJM=&r=j{Yy?7vX7NT-HXGQx3Qj&8U@not-ouieT9h}%o1urb zI5{IVWeU)O48F+{ffJZA#3!!}lu*x*?qLcJoRT5y&6ptv#PZ%O84BJ^8Hzw!$(MmK zLm5P}oirG$}(|vvEo-&=igy-r&G^Pru-h;P_xipU^1;OkzSZY4U_17LWsG z2FVg~05g&{He;{@CW{0s5Yoniq|L(w)!qE=K~BE$?xCLXj?N*TVXh#Tvw*~qLxB+( z3XH%|U<8E%YlgBn2QU!c2TRr?1ptS;kCAh{hi3@H1Rw`yf&wrckxk$Lng9;R9+9Hd zbg;Gn*C01%|4_dW*B}E__i=*deLaJNv6w54W-uGn;LW!}%muXgdIX?pB{(o7#I-Cn zuVl*PDbN^W%V5tC_GSP&jUmHx^YM%c;xZZD-i*LF^~F%Z2vl*Yd6~F;Mqmd{MNB|N aKj)SkRSwt zGyqysPLetejwp(Pk2n^c_HCMWizsn=Y}&rW?y=qDv9jniZQ8U-_Pt)O%T2QT+P>_* zZN0z$H!~mr4kX#Jozy1C!8hN1U%%h~`!HGa^#A^^jDH@@JJjavNb1yZIy&f#$CBf* zbSAHEIGi3E8cD{EL{r1D2cC?jviU3dom2Tw!Fk|tL_Z!)?im;xjO{T}R`L={WyaFZ@l*MOFRfQCkrV0YIMSR~Jts!8ad}{TA~LR} z-LaA!$56@{;VFnjcDBnqDB(MH-k;ARQp*Q3*)&r9p?tRh&FI+RL^AfFyjuUk(UDAs zTV(U+@)P~4n91hVb%U{ikC>t@B%Ez~r zSo__I-{}i@x%EVUT+#|PUKd{%x5TZwt}(to-n28`+#BEEh_@i4@zyt+zxbwZA%0i9 zP0K@kYei}TL8n(wO?*eWo!1+5dc2<7q*S#=BBRml=tMFS$-gs+_9}|g<n~w@dUoquIPODGvG3CcIjzR;4)La=JseOL6?6cPOPnx6|jUlG32p=?<<=sakXF zK<~%$af`a);gQ%0`55n_@F5`r!N>>-TEk+#-Wi^G&%C|m^=_oCM%O4QrteL&ug0eP|)i_ z*8Jw-Hl$-1UQUHtadn-FGk>pdsu*Tj0~d z7i-k|$jW?>h&gG>k@w0a^T2Uu>2xHI49O!H;xpN3Y9Qty=Ml_D?J{PBopLOhiOEAd zJ~5Q~l#FH}f1=#;r*^c<$DK5d?WB1G!?aCMTQURj7)>A%nvEh)MrkrV(mygBK{UtP z<^I?JX1EMS%yA^F*y1ADu?RH^NBGdVlUzD>J#x~C%<+g%$|EV+k%=XT+IQZU%w!+k zX~g&(O$^=0#jzuuT@M_5;8>^pL6j&G8zIg9&SRa<;aJwO6WNqgV_CU(EET(7Mjj{9 zDS0U6*cD9=XWHdmyN;irHzVA;1$A9I#v5y1eKp2*nm29K z{6v2yJCdEq@{WiWUD+2Em*0)CPcfUg6ju;^%I9{Q_QjHZvkskH3)@&9pyhSnquL`_ z1|?5X!=lEdjbLbd_uXn$L=B#nLo|Hr%ON!51|j~7s>d`uM@jo_m&yCy9eRRx7;We6 zYm@O`emmP@{8zjC;{W08Z)0djnDKpn^Wih(!t&5>-V9U=R$1AR$oOc*H?+&2=-WHBh5NRMP=t6Htk?IQoCGl&vWyqH0%On5d7JqT7C zM#?*y;&-{JubZaWuXr&P`m`nVs&zmchKJ0=C}s>AL@_dA+8&OL44tYh6u;tgJH0_I zn2JL2D*=zwYJ41pVm1`dY0#M1R1I*c7+<9GzeSW95PQ0|D#= z6pvXm@CV(B+v)e1ti7?2SP87z*md}WewTvbgO45+2@7J#f>o6|TF-}#YGvpTxx82p z`pl}y9}0zhNMx1quz_ZcYE`3zLT;zqZx(F8<#M}FZB>G8N@IE*h(!j*lF8_Jrm~6x z?f_PLu7Fvk2izf4QP5pwy|XQIYP5eWIWllxMHKf9#xTZJw1t4ni-oNDE+pXbyF5<6 zQl%F+CPzj`vb0(=nIz!X9l*A$q$-QGjI8;v^D%E?Y9n8-@eO;iGn9H&g4lwXG?0MT zR=+hCW9e08?sfSU%xh+whJfPrU_dk5-caBT$7AVKEGYva z7|B+aFuJJU8PEr=@BGmWA)=9oCu@OM4roYH$nW&%9ilSaAciGxu-ahB zi?~}$v)T405C{fP6dtoxJnRs!;;RyNGX~3umg0!n#sgzKH7b{Bp9zG}{R8^up|Uti zRu8S&vGFm$Cs~;<-}s_*Wz+6Zl{;8+$5O*1sn{N4{l}MREo##M26|fS>FeLT>bup- zV~0sXcI?y;nZ#!>SawgS8pZ0JWdkSF05x;uy7i!doWc1^wFtBj@JYPl9DjC>U`qU%tM;%y> zx61?BliEV(>cMZ;&?1>nnpm2`5nos%T|DGf&YNph>-Z=Z$hwbXgRvob#Xwu%%O~&0 z7GS5m`$IC+PnIp$CsG7jaCgq}(Vb2`B*#hYSFks5mL%=bTD55+H8M1Ud=?|zhkXKf zI6g|-0ScRT3XW9lL?n|Qh@|e{t6Lj>xD`szaLxfU6xKwo+Q{3e;V`cr)0gIYY!M{J zc_NxTZn$vJt_A@O$z-bStxDZiq*imz_f=O9zEY!Zq^ujjo>p~Osqn=bt@KZt?_{)= zbWzM6*HTrA=LC z@3Hhm%xE>gR1;rNx9U5TihWG!7OaU@;hjd%?J~BK*iI%h)b4Man__y`7Ud=T+FNY= z0)bFS!AjO_l^ejo8vrO~KJS){t}R;S(^P5HTgl+6HO4)ONsMl)_m+d>>ToK?#>8m~ ztWJ~0Es^8Y(Eb%Na zLwyjq_289wp4_etPHzgudUqfO7=ZjPY+R}gPC?+&KJ&q81&>^rn<|#siYw&vI9}l-qZtai%4R0$!_L6v({@}HOKt3>15U3FgRBa>6<{)g*?eaAQ*J|{i~+~*a12Plw1w=YfO$s2eI)n0sVMF2&FXP zPIxv8{5l9e#u-$!?5t`cme|ShXlf9wW_cg+hFoBWP**ER zZLt!F$q6~DVW}IUgM)H%42VPdYOF~k3A{mgM?sVXT+9b39&|pB-pf~|wK0tq%#%co zuhd$JUpD~jKxj4Oi5MZ*5_Gz%jcPVZ@$mS>;K*oY-}8Aex)kLI%1hSg^?@A;1*

x;kJkAu$j`nWVCixhwB0eud_yfG%rS-bnlkNHjE(DrAo)i(_z}tHKSTvO*?n zKa_J2wOpOsX{MG(@(by)q&$RL)q*Z|VGwnGFtuh=HMlm&uDPlW7|lgcXiu8TqjKQz zhulaQM$pyO7xG|;f^>n-x2-DW22N{0AP1OdIkoI1ub}FXZPQ>CY#o&0&J1joq?Z zi`KyM|nz%j>D@f<0+U)B5uiBae1J-nzxahjmAk-J;{-OKIYc@?^AA09umw(rbzS z=PL9*(rX1KlqDia7CaHX^rnTjk}^s+GRP7S;-$N;G*88Z6<>7Uw}@M;5_d@^cv_;A zmhv*|gHbW2)h`M1axR~s{AiIB-6XmxH1dDmsDZP2t;Vd{=Jr5pXbde*mtnT3HIODj ztF=Z^pAl_B+o?7Uj-|4ZiA*fQ*D9I(Q)&arAP`lBr54GAHv~>z;&sdonuOKNy94d&Chw9L+i$81I}% z5A=4o%k*~m$f1XhbRL9Mm?qstwA~KqG;c(Iwv*BQJbEp84j4Y-deo^!3^J0KzMbuS zYYocvZ6zACd|gp%`2s4hKEP>MjmAt56IX_!aU9{rNGU_31E(BI7s*rZloua=1CE)| z*kUc@-^WZ+BK??$tndgzH84Eq_}I9^wbRK%br@vZ-NL&8cg6-7YgRe}V6s}ly1-9! zzb?p)C52EtGje!rB$X{cGZL>;rOt!RB`R-0t0_Vw3l~_h&*(k(C zqZj}z2H6Q4wT9{>k8>*BnB}^~7{Yk;?Oi{p>&7Os;}aN*9>tg)RqIAly3WB-gm&zWlZx>j;Is&X5lJl&*_GVxr0bT$t?>%2WCdoHxL3 zHV}eXQoywu5#n}4K6s23yxO#~7Sj(PwO~Y~Q%c&%x4Ox79S&o~mK_`!g80@pRdDe| ztXhkuC>D8`N>+~Fs8+Pk`;YEF*m=JT&9r3kY9r}b63E~Un51qGvSysiaj2m?7~3&= zVCsUn%*6OOKjCL9b`?$PmQGV>Lt!Dz{-$t(1biNptj3TOvEN)eD>6& zGhpb9pWt32x*cS#W0?U+85pOp979~BmlFL6KCF=^JUF#QQtRLy_d?*qnd(1}X!a zJVIjTj|@O4gU$yG(2NNQA!s3$icxpw6kW8q4$5H zAJFcnoO*H61|{-|lb?D&a{3FD?8Lv&?7daB@x)N7WDhhP4|c^V{VIA&H`iwCF2~#M2b; zyJ=04xJo&C%NG=h&r`7Pfn`K9#4~Xu3nvIYI5P!ke+5A(u?hjfaj%whenZ3YZ*QgH zIPp&?*n=Cy#6MF3|1=J%iT^_>42p^WODWt_9GL<8nT?JZC?Y^d+huHhhD$q}OAFwW zTF1xM09{${yyN;XkLABM@i@kvh7=?}|B7*vWCh2a2p#8s#+}g#$Pem{>A2F~all^gGcz{ z9D^A@#O6TQG@sF2wO3P0*W;6GFDlO3dTxzltSb=3!ddXX7Px+9h1q9;fRHc?;D%?;J4Gb;WI zu#=%;Bk|DTFk`2$Va*)RM6tG_QxKWUH(ARUoFoK%<;*oFm;}f`Wg-~@a({U^Io2Od ze$hELG?a;D*%$ay*z#*mhy-JMa2&Agmoz8(g8DWASbp39nr|DxZ#y-vHxnBs*@DQ> z$Vn`nX$9Z$TRg|eXvDC@!g@Gm`OQ`4sdD~HBosjeQm>!qSm}$@^=*b9i!I2YV>En9z2my(uwRxeG6!=}|czWbXeq>=)0@<%QGsEMM-lpAz z^(9S{!Lfm)X2ibe)JU{1mxky(KKL~!$+6=IIZpu2aS+)}c@Tcx3CJ@O89arAj116` z*n#7}kLoLphHFZr`7%W4pNtKq2{ENmd^ge zBXeeDMt?W@8>z;#g}d!w9lI>t?3vu9YhV2Z=fD5E$NGQM=wFm58k!9M+)L?KY00xv z4XS}fBJq*&$iT$NU}VS>3ix8)=%8B}^x&&Bj5qP#q5Ekw{9;bH)u)p+b(}#aEXF`^ zE64v9>^j*$BAGwRoE6V7nf)V%Z%kDW|0$E|QC|=KDQoz@iXQJiL07lU3zn6Ub9;<_3H;rn(iFZ(}w znXx~DdHz#Q_65BEzwf|OY}VAee?{El1e+q2y(paH>h$CpQft`cuL|V1hSgn0*qBS} zFrnA|9=+RGovepNZuk`6q`TVycde|>rN_bTYs zkL>DwVe(~({T9;y6+ADlJ@2tHFxD*J*2fbYH4J==@yJ+?UC>|jWf;?!VeFs&_4P4w zAJpxQVd-(hBUzSwdO+sH()SbiDfjg3mvOrXf~M|Wd( zO8`FO8hOy`_YXybz9H|>;6VSNuixzo`U0{3L1nNXr$nPMw`WTX91;|M}~)6}wgxJ+}qNjb9r zW4)aR4|EyD@TOHNKU!-Wi<-)Hy0I=dA~4#&p(nr8rwso_KQq+Fb%)elC@y0kV)VZz znEmtXua6o2jecgR4&A?DZ}``Z*=D>@*oM8)FAcTQzma}J-44GrHyZwpallZGaR5&E zhw@PRc|SB{X86bKqkSsYdAcdof!UqsP9{i{b;k{QFmVvJv&(GUoSAjf2I3M{So*lGb}{P zK8F8Lg$F}xsE@!`vFuAia5ZR}le!tLzsa4Ef1~{kz|6?I?vyLPqWhmHHN?9SuVFlj z7c>9&=y*n#;ZRpA{S#jcE+e0-V~-S_wOMin4|_a`}TyQ0vpN;fxlU+Jbk)a`=HG4|W|2_a1xThzpk!B_c>v zQJkg`4i5AlJ$5wmaA!|ApBSJNQxXy}yg_^abN>`ld9bGz7mt730@=*YfFT$2_0M$mZ*=u9b@e~#>L2RrU+Ajg z|AwyrMuq*7uD`CUhW>x)`v0k`e^;UY4_$AlH+B8@b@gv`_3w4nNT-qhzt+u-aKEMN zjrd>DNBH-ws!v|8yhl}8UtzpgRlVBZv&#FQ$Lx>Kr}Yvr)X(VpYO3L`np(}DVP4Ij zVQ#pqrW)?5sfNFA==u1%t{VDEf5t-FP>scGHPvueO|9n7Ft6s%FgM&)Qw?|3RKuTP zZ!F{ueWj`w&ziDU0&69(Rsw4!uvP+VC9qZk@A?wRt2lli8;EddQYeKcZy|R|3>XCJ zwVygDIR>D~ohl-v-btPxM@RDN+*>cuze57S3qK@(Xs_%k$=si4f|u^t z2+7epjFU`vb{{?t1^pX8JZ6Yo2HYMxb6#?NQW42_dwlq18T}d@fZQB;Z`LvUqkYGY znS%C-N6=<)f|di2EO8mSeUlJM+8~a{EzPa@x9X?2m6004D}TthMSV={s}Yy* zU1##LRAcOoDhEHob(CAZS^rBztbY1+wh`e9KO|{TR@8Hi@WUf_mcG6xOwwJ<5;D+t zgw>1t{y;^rAFda>{@{_sGrzlZ7dLz6&*y)(PkV*!cX7tLaAv%^S;0!7W>;vA}5!9xI*(ID;N%T65Q8=(Tjpr<$89WzZ0lh_q0ZNCyVAMW|=Nz63cxK@Q z`U3Q5E%XJGo+;Rz$MY0oxrA7tUxdB@JqiaFtP6#M$AV{l7Ye5f7BE2R&=-u_1w8BU z)Z=M_6X*-jqqWc%OnOAv)Znppq4#v5_jIB6z!mf;4p>NdYVp|cG{ORU^d9CAq1o{H>Xf5;wlb+MCIg4iov0Owf&@Vz?fF8vGi%C4^@La$%3k&Gcd*}-$ zC=VD<;W>}zDLj|p1o{H>Xf5;wlc_nas{1xD_t!oP$&9685*W0yj(Zu)HM1OJ)(x!J zDs)US)|GLKv<9(G&1s=x@|$Y@3m!q= zB%7LBdA(75UE0kjLm=*j1sTa!2OD#9PV|g_v?_gg)Ex{-Vi+epmoA_#raHDPZ zRcIWuxqElCef|2SYnQAdLvh<+ykxunVosDTU$@>X&c14sdcT{y@ttPty*n&lzhp%K z)*a#{95Hz{w_`{1*BgwePsB?-O6e z^WEHQHvC_*e$)2iJh}L$4gZ&}N;%l$`R)(CoP*U5@P+?ZfAHlmfB6SL(7&e+|0OTy zr41-&?Uick@Lv@x?Z(RM;hQU_E3b#YTQr?3zV@NvN;drMm0G25ELdA=zj5duUU7R- zf!1FauProPMg3S?70bex*r6*7j=u6*vrVuG3r)KPYjdql7`vAH%4=NwUJy0d+ZSGgZCZE&58J*#kc@_*1LUsj@9=GuIx%Y^{g{AGknu1PB}q7Mp~kd97`sX#r*0Dq<|ah&D?v3k!#+e427|9{D2l z7z1d~pchh6F|g!@*JwCINww|8;Bja>BEDu5XYYFr3Ez%tn$69NxyxvN3?Mmn@`Xn~Kb(lzv0KD^F*g5SM{lv!pM_M&tA62iISaeaw3&D;pi zHnmFVw=>8JioNz#o3wy1W*COG94ct$24Y8QFao!--F$?zc42I1*DfJzHnwn&WWC~Q z<|T$|m~9odqwKCNU=W)@@gmSy7g{houCQe(~~&n{fKK^+f$()gaW zy@uv<4TU$~;c;ABxU`VF@d3xRsX48huVTQOV{VaMU>JeLKcYx#f~|pZJYSGv92> zU1rbkdrd|?xPo8CZ1W{CH_xtNpnHAcqtjjD%YP;1uCV!U+FoCJh}mnzkJ@X+Z`Dak zjifM15hdjeQ!LU{^MB_JGh8bv4U&DGghbC1sM?o8U~aPch{p!eMqvP@ZE-1GAq-P=dV{ z-^{*_DRYQIP^<`+Dd*upv_Hj^X{LyR-NuxSlD(cOnzEhD7_*OqmtztxRcT3KQ&~Wl9ZG>hLYtHzQurz7<-*UN0zh zqWyDBxrZsLXy1->ikM^V&oX5awxY6tTnLKD>`w{G-AK8hTojZx!G2aynwheV**6GE z3vw+e8<93a*~IKum{Kc*w=nxvrpz@PBYXMKVPDi)cTRvRP851?3(|*(ND>N#PeH`(;VlA}N>93`Ipi zE+nN@3ZFrG&??37IkX&6`5dYP;Yi_m$-Z4uo|BZTXj-CjMN$?dMH0i4lKr%(oRgGi z5x%IL6_sg8Ig4;4`*}e*Cn}$n!Y@k7^OAC244;=KrUYe53ZIrP*gq%rwNmf=n#6>| zPTJJ+brr`Tz5*fn5ppH@ zKRv|-fDcShk|64&z%Fvp)!P9Bn=25C&C?_zOTYn%WRqHsLu{@9Fm~z~#OCQujBfVZ zIeiybH&1I&#^&kGx_%2+LpsiMV0woh{`_OR#Q40x~f&g=4nkr zdGqvrdid?Sx?5MBdVC$azfP|1oZhGFyL5HGu0Eiv59;bey82;VJ)o-}(baBU?a|eP zx~hRyOD$+S9b{C*Yu>4-H%}j;cv;^^*c0pka2~Udz_fh9MU9yYq2trk?u#9uGK{4s z%$seoMOS*L&>QWy!nmk?tL%ljy*$jy{?YdZWbM|N*;?0lZ*~pdUT)hznkA7O-WK0p zdISw^8xM2I#k*4gnjhtHJsNK5#a*Kp5?27qcfWZ)ZSt;_dt4MTK|*Rj%}iQW)Of51 zZ^fZyp2%h+qfb`h&`ylwho(VZfo3Y9kbWxpvBWP?b*#~2#wCw$?w6)ZNU&f4mtvcp zbiqyi_+6WjaVb*bw@~o`jkqjD4 z2(UsK|7B^YLVy(v{>#!(LV&e{@3N>m1GWeTY>^BabO!9D=Pp>lw3P5*1<+tqH#E|& zV6b79hAMPe;VB9oY?!41^p`=0wS)b#s4yTtY!Q6eA{jK8@L`2AI?U2gg%2wjbQsD9 z(dc|wJLoTqsuN<1AjB5QpuvO?E0pnJC?5o3LWmU%J`Cjp7dj!<4*tuc!hjgDMKEHE zWYA#3h!x5RF%$`i9}`BbU=U&`61dPAv33w(7FDOj7VAMEk%1WrRw(1ZP$US%gaj)X z92klOE_4#C9h{d%)mgAbuwaX1&|t!X70O636v@7uFMaJ^?8q8SxUd4Ku&Kiur?U)3 z4CRA3bxN!q446gLIkCm35GNTlm~di+GD-~P1OCW_6D!mcjRZ;zu7biOEn&kh6xlGHV}mq=L5vZ{Vor=T7s)ON zUAzv6ubdW})2(~qzev~gS=VSWAnf>m%V@DVofdPWoL302xs}&9K#MJXTIa&1ZfN}2 z?ln&AJ-~_mgt@S(!#X9m#)-YHoS4Riy|XOXOCoztfHcQzwy7IhEmv}4eFupXdm4;W zt@t#^FeZgTjDa4TW?}GPAQmQx^&$odl-D%pvMgY$Bv3R&g@G;uKL(mBPox^H zhH1=F4QInZI)Nl<06&9iiqk1z$-sbtHv^>xS`35|_%YB;;Kx8T-39i8^GL0T7i1A< z$-sJnUjoYn76k+v7&)r|;)@6|Fe(;-7%)(6V74;LyOufoB7k1-=W6-z88|a13IHlW8EVK=joLM1_Iu0`~=u?t-WhCm{6f zoZ2ETD<=>Iw@CoEHg#Ad6cvzd&okoH@Rj z_!|M_oU%>`Zv;h0{8ku@SF>mbAEtu!Vc`oB@nLg9_->I1HV}I+2+FgJcruWY&k5mq z1TKNW0tW_GEc^`T)pBBZ4g{5`fa?NJ23{=(`ihvaF!;6wfjGJ6IUfcd3tU$lC(J;J zf$Pd6ULpLPX#XsVQw%ptc2H=bzU~I+gy=X=wowYtO7?retAUr|JlPh`!?j3Z@LQnD zz=1ta+*lYy=2j6LSa^;TVBosyh~)~4B9V({81Y^p&%kwo4+F2Z1)MAL$O$m;qoA+O zh)OLBKPTBgOH5dJEAd`-@L`k0frUZ7f+0I6hNmSv$S}~W;K|Ml#D_`5lZBs0VkG6X z6rK{pQ_{p)N%@=aPK7n$JF2$!N%ZY3keu@keeHL6UL^t)jw8f-HMA!e67xUf(>u#v)H^1oj=V$h;lB%~Y9?Py7y`{2!Yp z`}`VB#(#KVjV8-x_709kCdP4vBQD^~Mza}%o%%O^ifn}@^Wyin%YHAl#LA89{)5Kv z^xlRh6E&LbaR>p?i7^LGj4i#w32mWl4uM8^{3m>3Y=pDx)lZDgy;Ud1mL7h?C&pGz zk+C?PB9l>I`YAGcEuI+rFF(B}#&%TDWbL5E44RCNj1_6JcFJTLw>-G@;2B%V@G>6_#xA)+|{8=fsF5TLw#(G&v@=SawP*S%nr` zyfrOW&}p${&|*mwR&2416-!nj#a`l6!ELCq0=P1w#+E^iB~6&I#WH3rS%nx|yfraa z(222S5MxObUTm?97fV*5#TIW(ixt3=5iPb1S}ZAoTmlPLdNgdYj1@~(A;lJNO^Ox3 zk`XDk3{os35q%V5EhCM4Km83~rG!htPT=fK>- zEuiU$2`hjaBPMJaOjy!{2wN;8!je^Zu*F;RU!nVg&jM?eq$49k3owY=Ph&`>%$wH>EPAU7L4xN zDJfxcABId#G!cMU>G7wfzh`iMOop#XCNxiyGqzF2QCOM30?KentZxs2e1KM-uH*0? zgJe1IHh^#hcM%8&Fb_zcMN$Cr0nY=H*8r$Q$KD8U1F|#?`E?Ei0^S1-0|*fE-zrD= zAg2xp5K`bk-5@Oum=Ne15H(<9fNpXOc%B^yAK*7gkVEQy8_+PoW*iq3B@$SNC_do8 zDMoM&#Os0E&2iKX7$hJ)2&n@@1M&kT66g@XG=PNw;ea&)ma7v8O`H@-BwZ5100jXD z0z_m4YLe7Da6v$fK$(CYKEv#Q6(N-lEDo3=1myt-0=ENXXyrg4&_qCqb%;&`;z$Dh zz#5x5G`ES7M75R6onIidD!f2K_Q384I0QNeG_egC7s3Gaws3qA&?98Bw-WkC681m` z0sV0hQ2|~!$7S(>2SVZ;AR-_^V1Pg$HwlCd0tN&;_?)0T4PkneFc)(NvIh})fPV8_ z6dtJHvnWqMp+HYXLh67S0`~(>sKJB4^dKKUCqm|%q}zc9LJl5SUyfwlNdg}5*E1ZK zQ#l?8Idvd}ZIGDfQu3RHFfhR9M98%hc({=y>q*|iYRlA6O!b;q`b(b;q6>L9{AxV_!WWdk#s&F!4}jD8Ux9`E0Bx_ns-SGZ1X3n1ZcIypyE; z8diZr0dD}SxGhX^JBjzyTho3`8#fG%_Wk493y-f!ysyD{Z>VeVCU@KZ@fF-mg3s+G z-XG%t0l6)aco!bm4$jc*Kqu8#eKeTwsUKGw{&D^z)5h&mYtryPRkua0NyG2jb^OF& z5pR5R((o&nMd4@O15W?SF9es^cs+{K_|=EofP9jEEmSGUw|x!dd&3}~4(QRycFSv_ zetJQ^B{PtZ?u8;*cmnwr4Ow^s`Py|^cmnwr4Ow^s`ATKsdkk54!uX1pLG_q`e9L7Z zUr!Z~Z`lC(k|mPxJ%%JaVSL5Apn6O|zU4BIucr#gw`_oX$r91`9zc3r^qnxi;#E*R zCLrH(8OYaD1>{>cB;S)Ig6}lP_fwAvytfQ^&)vBLh~uJh|5HyDjBnY%`1(HaMxnjUcLeP%mqL5r)9$P%Xzzx0 zfLsGymRgy8l07|5mqsWxA|Yr1)Bt+{P&>n=?0}#FzyfktBSC~OydFYP9Pb0h2YEbA zmJc9ctrXS-`GA7e;lc?>WpUIGLU(}VEWp}0(g*MkD3|~wj^ASej1LleK>Z*u2jM-4 z*a43NkO$Nc=w2HK5CMb%83ghLL=OlcfIonP5c30u2>cEx9^gL!g+S8)4nkUP8>6c! z0O0`_dw~)B2S5-A;5rBp@_Q_%ae)mNqys<+RB@gogOC^mFb8B1&>_G~AqFD$7VFZc?>~Bs8m$AZIr8%2`X18fJx?F9n+2=W82m&cV7Tuczy+1(;tYXJlhnA$cW3~@lfaGHc4 zMDBp@Z6_H%y72j=q)K41Z1L@guRK zx;{CGK=7Le_(}B0@&0^whq^8`F&fc-Q?_Mz=p=rnB0U_-M)cn^&wu3i{zPCCxHrGF z5L7FEp5p$a`ww>B@5(343PW*vJ;9a2NOa|+D@#D9bxZrsN$ za=h$n2eYTfWBEio^!Rn#kK;#dYbFwZ2ptu?M+y3ZUT+Ah%jNV1w1PK}A#n_mYrzXf z@QH`XjKWU*JG@m1JLK^@gLWy0H%c diff --git a/extra/sqtt/examples/gfx1200/profile_gemm_run_1.pkl b/extra/sqtt/examples/gfx1200/profile_gemm_run_1.pkl index 98e382ecb8e5ac9e6f0d1a7d318a84819eb446b8..6a8f76943badc12258d0949b1218632b80358675 100644 GIT binary patch delta 723 zcmbO|h57w!Mui5JsZUB7z#w*tMh|~UW?p4_QDTZ-N@`hVa_W@H|Lo)@+Hy%VJNmjz z>0!xB%uSur!&4M*Yy?7vX7NT-Hf!0t3Qpb|U@not-ouieT9h}%o1urbI5{IVWeU*H z48F-iffF1v#5EhI)B?F2J-oqz@t%IcA;Ixsjy|DNGNgN$f&-^y$a*to$N{muH%o?s zH&ccpkXG_#V9Zbk5h{}zgT$FK#3#!JNvIRh!i3Nw>}52iGzrOaq2NI0cpsPGcu$vz zct__D&oEa)E?`D7j?EZs++@CB1wyW1LDpsh(bf|zRS$GLi+hlhFS3srfj(vg`j`>q zW7Z61Z;lL=9+9HdbdbXM0M{TlXa7*Y5Z53BRNrv8`xrULdw7O`{J;X@z)V#Dh9k16 z96(bc;TRVpX^#{joM0n-J%fV@o52P(WAl{|a{)%a$vqjeJQ?g6!rlx(+ZZxDHy_NH zATE>P?ac@bb6;qLv%ys`0#%%8UM4P|5!iuK5ff0+=Xq@cKt-K66)^)9{oK?hzztWF Gss{j&B+usn literal 51739 zcmeHwdvqJuede7Rk^%uzph&@{B+D8zFHoi+0K|}h6J<$Jwxf_DMM>~B$F*RH1RoGc zA^>PfNxUU#$(Fz|D3OTm*cP1TY??l3+D@8o+8#gJ?rC#+xRNM0X}Z~?^pD-{IlDZG z|7d%%d$#p{zdJJ^01hO@vC||biNT%wzVn;={J!74n5y`4-Cr~Q_ej>M)}=5ka*cON0kT$A`NkXVya342~w_vGAD?52aQ{#^R|nh;%Y?CYfzkr9>o^?N!Cp z38dVzcVuELviI}hFN9-z2POt1dySG6+(gEc6A4%RO!m-gn^jBbR3aQlnX{_*)M#o% z9*9qd;#%3AYsGO6jk`j;1ffuKi`+y7Z)(0Tn?k0RPbE_cl={=zZUL6DiNVQO0YPDywc9j0}v9g=4ropx-S(US7rT^SAo^es5dAhRQ zMvRPSNA6bZ^4_iPfEV7}ck;xDq{(!oYGl)hWyGr6YDYGY*qcY{dPlZ6N9ysMBMtBM z@kZ|G7DnzIY1AHJWJhsfJRZg6_VdDwG{I}_FF< z^Lg=N70s`xX`DdX_VLNF6On|R8i~l06Y=D_lzqN7m)GCAmhxJgQ@l#M%V#2EO8Blw zI1xJ|rxM}u&9;aIG+ z@NK?UpUdqv6-!$?9?a9KJ=i+2)FdM4Tn8fZiOHdLGHGjXYj?TbrZQ>ww);@r^(}>D zQq5mvSH7fc0%-Sn+Ff47Ov>%P*7kBKuJvqc14fpCNHQsVcC!913nIXqe86?tqlaxh>OM=d7oS`0GxCcjzjY3kUWaMJ(&uR4@8_4 zJcNO$MaEFjEJtF=h&>VPecYVv?bA5#!;|*Z40F)LR2H1!M?bQLOORpeAOg++U?mu+@u}=9@kSGcpBFll!W1X(yNXpra z$CSq>QgZLac;r4A&p4SFmxsokyTgg$WQ)9e_sLUqXQX?Vpl(V;cumc!4<#nYL&kGC zcWVvXWjL%osB<@-LXU;fb#Njy86OO%A|Z%3mE2ci=UJ)u?43*|_Qpm}?2VsEjZBQU zx;$PC`Qxd`=@eY)y>|Djm80Y1G`4Ry3O+eLP+0WAe8Im-0@;ROkZ!rh%MXPqjOOcl zCb#197WPbqmEMOAyi5zMPfnglrbbhfDc%TCq1Uy;cE#s)dD=|7jds5mZM01rgG-xt zL9^L}2Ck)TObk%*x_7FM5T-w|GgPmrE@_cFY~o0?ey{ zYQg*}H5!VKl?)sKHw_N$g|#+6)?#9z_+6g%G6T{U8umsb$rSp8Q;E@3q_oxsJZ`Vw zg{jM0U%!^N*Y69sJOOPSD6O@s4LuK%)q0WQb{6mk+-{em6pSEiskZt&ei!D)=7Ulr zdWYoX7zPaLL(wr}#2$`}4xK3#N~_PSxP0weGR5<)K&wyjxY~8Ubp=P)mmCSlv6!KG z@>;==SevU&pQRP02>As3e)yU94pAmBR+L!4?Q6rNyUnc)y(Nh$ZnqyjjmfMV!?Iao zimy$OA5G^cC19&%e|z{yx%AzL`3#uexS4$yeFeS44}8d+gHlW9cpOMDEDf- zG(ive%A~4b@?PfU+Y=aE2O^(_5_V$~5BI*yz}3iY997#|Vn6w7Rf< zG<%VOc0cB?9_=NstF*Ozvc?zc>ly>n^}9Vt*R+BJ{EFMO))EbZcccS=ms0bOWtz=4Ecj(g2OdykDzB zZyC}`94c6J7)A4=&b3k}Z(i(nbeZVYOp`8{J~X5z;uFA4QZk>u@k#00dflliw=w08 zj1P~FNA?=?KR!jPSM3An>1nQ~&wsP3?`BhvCX<3RHEU2z^taGkc2BBXV-u$$30_8; z)5MT#7?VJsc7q9CNtVu12|9-`PfU$YjO+8|JGv!xGv?oMAXkq1(zwVBT4LvL5mB$nc@id<+w0J;t%Y$dJ6|pv~{))AwNs&@Auyv<&l0Mbq`kal$KjI9GhE*`=rC zJdO1V)&{PEqCHZn+9$_Hheq+7`3(1CoxlUe$7ne~Y12xhhttD%7p?=mw&vy*f-)_-u`4`lroTGFnY~T5Y128jGNMmQT&{rrW>zKkNn4PBK zokr3vGM189PR5c{?{E1u`SNbe$_v)DY4VA`w zE9hLc%D6W=h2BkVy=mvTKAn=jG5Q20@^kfs(I;t6n(zIh=P5;6Ip3nsi_TKO+v)D2 z84CHqc5~57l*jwovqiJK)E{Vf7JZh>Q11k8-g!lzr?4BH(|ba(ek~va>YNm;vvu&d zc&!M(_JGS<9+uMQV{2nm@ig1-$EpHgykfTG@BrxdxB`m#s#e_+4=2K7NlMy$Ipa}4 zgSp%uv-KDz*~k}poA#<|1J>XZvBx6%fNFx5Lo8SkVTE4+P2?A+NIgK4ZPUaHCd8Y^ zB4ZPYGcwR~4l(BW6{pr+e^qOKM{n9)7pFm8b-w?n*&xp$r03u?Muv^~K=M5R#vd?B zS5SWxKcPpw{_qx8;ME3&Pm*E9fYwJ;?>3L**$;c^==$#^Cd6omS>2J`C@E6Zk zY)mlu^5Pc;`}(OXZ7=9%3u}~z&`{IPO4Wd!{b7(%okx!q_1^8^P<@!2dQEt6?iT=> ztRu~(HC3(G7|*eZ$0B1`tjMKoDiUaengSF9)?<|92XGMRpLq`(y%&>~LckLd+Nnha z+SlV3EyOn==JU1|5a3bDhym*Q^6GDLry>)x0h z9m>GT5U}szNXQfNmed8sqk!NBO0?E#<%}kWIdSWsTNPs}x($cpB)N=u*Tr2%r{_R+q z0pBXqK!#%B6c{h9?4`;2J!RA`PA_+DR`TDK)UI4H*^063UV z#!#eIQu8rm9LNuVwyRaw0a{n_Y7>~lJVxBO$%{i=a0I~D&C2X+L$Buzm@l1G8&o(z`Q;*?A|SF!Uzv0WhW_+HJ8d63iwpn;$1=Y()%F0CV-)?R7nd15e=EA8a`W^%IMyixEdr%A7&$Eu zrq0Am$T+~qes`;aT4mBHp_cpc3c<>h5eijiJ#F*)Oev6}hzY;d?r|$#sJyh78%s7r z8xEk| zq0b7Gr3kntj=_4aOSi2JYH#lKXi&Sy14;vwn7*c2*Hr4L=8*zJVj?CF;jNY=jhE*G zw-+er=8T$$q={g}fNuT@>+-PB-UJDxr1PNY0gw-7a{*Awyyoanw64@OoIHX`4p7fx zcDbFRD$rXn;bF+ngF(HmO#3dF437~u4~0xhpX;>k_kaI0^u)vMj! z!ZF|(^R%rC=^0y#O`bsA)S5)7)k_~w8%D|kE}HA85)gJMs5;deL%ozVZs6Vr z--T@VX0E!qIq%7vWLB*_a12e8B7XK={B<+6;<1U*!EA50(3^s?XVf7v=8QJtySt5& zh|0Jvbaj}zHgMM+2&-BSrxN*Gx@)^-wGve+47F2X2e%8Mh_xT%H6^tR#93$3(wPw=*``%cU|#i?#(mf&w5CfNEZkFlj+mm!-Id4 zN>m^NpQ&pg@7mlgYdN1o8KVt|ty(%yk0x~~cUMrzC$oC#y5LWi!m|E5%8_S#0C(MD z^fP)Odfa^A{7qruj`Cu(Py$+!_t9;E{o^HeAK5j52}OnQlLs$EFWqUWt(A<*jR$1W z2XNC}RrpN#g5_`Y*f+^rER%P^L-4YMsVv1Z>xWZ5r}Z~cit=POO3$NZk`EK@rqn3@ zQ^p%Om(|{wRa@a693PrMjnnm)Eouc6N>FR9VZ6_fHlj7F_Q8qqRA@383GwMlGJ8R- zAuR+Vt1#Unb?{m;iS*;z2~Z5LMmkWpCjFG&$rDDobWOp3t2Oj(Egepk?*cw~Sms}jS`Urhnk|7 z@ZP$opxPb_kDV9{L$y8R3?2@roKEz2uA>KgyIW+sJA927xM zRGkLhc9-xgjXdWUUOrzd_9>W7+`QQ0pg`3jzbF&(yG0I9jE<*@cZ-ZvsZ!^m1Ed~a z5jhGDuMLm`$;TgkyIG^qG7aui@|onN3Eb&lEn^{ZVYnWVgzYC^1)6Y z)J+qUsrV$iqDRnWht;alaoy(RC78rU6-;vTx?$~}R=3iMrKH*PzIaE+@8l8Y_kTx+ zK9R)klhy!AgC!21NEX7Em|$Xw4fU7&Yb&?)D+U$hjhn8**i1fVIV z&xTd|+DuH}c~p=7khUjj{oeW-*L66E`C4jlbO;JucdLS%C#6wrY z^S+}84t3t=MlCIvy4pxRmJ~90{Ux>AL#z%va-3@5HhOmS9T>QvE;AXAL+@tq$iU~h zvicTL-2`2e_?gl1tUAZ(1$Y@64w{k&y3-c<^OKRu2p>B&?F=}&Mo#glA>9wU){*1@ zv(@TZ^BJb2F5-x1lBCU1sj{BhS;Tk`21_LU(_Q-hn1XN=7q3eI(V9))$ zNWGwSuQ3|xv1oSb&unP141_f1s70qq9>dlV zClxE4NQBRnhJtGR7b4BjNBM~Sg-Buo+A{+N*nzGciuG7|LEg~4mPf+ijwZ&jr6Y!L z;Y7+sqtR_`Ez<-Fs(32g@su)E`&HUj3C}q(8fB_mW zp&KrwfaJgbSM>TyVZ7@qj=aG{7-~c*b6y2#kYKu%$ zGv|Tg*@LbKl|cQ#=tvtK@NAg5K{LNaegK*%no5g4uZVt`68?3XQA97(6TR=#iRf1; z*$=}c0_Xr>5W~u(7$d1iqv-d@Kyml=a3N*SmR4{Vavy(!RJ6m26yNk8&^+2KlBaIPh+XFDcGw zeagd_zKu)_qNfB)@i28TkDypZjwf_St>5g@6fFmj@ToaEGroh(iL_}vqp@nAW)#lG zr#AW-O@CAy8gH^}`Ox()!x^(&& z{Wbz&yG1>PmC#(W`{gZ+{nq_G4~T-$QK4;|XA=8++TD@=>(K8Wt<`RNVRSG*tH!sT z)v_wsDQ*WY`s?7|Yvxb�=A-`;)Vgr|wi_bWpPI|XWJZy0%cQ*M@>79L%olZc!!+I*iT^zOWSCe;_1 zm|W2|h~%Xktp4jR(gI#MyW|3s041noC`mZ(?+nK#PK0CMa7_#iB_k>Jbv_ri{DBKP z!B`)h1S-y#9a&znHqEfe@-mqzy{Bg3R%5E>dijkz;T;XD3_KQSIOWO!m` zJv?st_BwY8-uQbisAmmNVif${zIbH3t4Hp7E$fpGbqUD0PTq#IQi#zeTCU@OJO#zK!^UZ z$WVd+Q%XhtW7pb)VHHa|s3#J%j)<_A^}3XTSR<)Ffo*tTd=Ct|P|5b}86Gtyzhvv>%L(GlD%@UMbjC;LYf^QXyk;#nrM ze?<3w)jH+V2VV!c%X)Bv$xds5noRbR@FX{9rp}UC!KQvsptu#R>Jrk%SXzYvz3R{C-o~nAJuM2uhj_al zZVSRSuqwBn2luZmPUE-rG^lJ8m#2^TXhc;4`5``=s`@b=S8oebZ%XWsQ2tkOy|i(C z$Yfy5S$?4RCpM}W_%r$=V>)(TztN{*OrM6afBM&VCn&sM_cywwM~#T&S@72bE+>+> zkI+xKr{{nSB@^rxm7RNI6WDj12*tyx5gJ=VNgO*cf{`~7A4?Km7SbV4gcvv(#-71p z_(wvtopu=T;Lt<}<75bH-_S(Tz&(?DflKa<#U2~mgV8Mt{ES=V!Pd66p|IaK)H*ac zaAMGRg48?OBPRxx!4puX3r9TOaI0_VL^GdT0fNJHLDt7Ny~M-Vbn6^RrQ*r^_U?sc zWb%Xy5ZB%#;laqnIGJ>+SxQnMM3YbrJH@3oWyCkynPEPv2c+ggc^T^vqy5?8_RsIWJ7L5(+L>WG_4tOr z5np#^+i*i^8~#SSG|bZYM)?hM7vj>`Xv8=A0mC%<0R-WH^bCc6pMZtjjQALRw7-&h zp6<#zj2qqD3zOm-(TsT`e<^UEqQVval@x&HXB5{6mS1=39htHIGrB87ewc^wKTIkz|DAeJt#MRSC2ng=!knGU&DC}H)ip7>Uc(%5l}Zv;}c&C zG9#O=VvizmS(Q@ZVX&x$=x!mj!ZrE~pay$-yrffGaIQk37Y?$>ABv$i=IOvf+5~Hz ztZFxqokLxws16-Fm>5T))Jo3d6S*9H2rWphK76R_XqYw&7pk#rKH4AJA3XZGP}k?W zdhS08PVoc~+mkrg?`S`m?9&4SP+d|h=pZS&5%Kul6)?itZc&v^4adnUVdYL)R?tE& zdx!~Ybs{pH)G|}+agSUM-+$zRuD)RJvHOp>aXL{nghJ)DX=)MRVDHgmM?(*F_5`!h z0V**SAsWFQl&6L?W0;JJ=m;5NbX2oNVIb4!N!^UWG|AO-lU$F!*IdP0$-Dp(-$phFIv!Ev=G%^bS2NZ+Y;sPc`IrcPxq$v%kp54w=49S zgT}K~ki9rhIm#=SX`U7>PpV{VnNo^3&9s_ldg$QMqq>BD)?BNKWy~M7Bv-1`5%f!d z(6ajpsQh$T>$X0nn;zYC>84LN_vq$s-MmXTKdze})6KdPvq`sqR5xw9>CjChU!xrD zy1P>1-=^D*bnemZPTjn>#5DX3cOzW09&VRzzNPo@A6io%yx#hdnzBB__^_J#YyZ$1 z?*FaLg8zdq0mJ+)-CoW#!j&`2#WUQ?#WUQEaOF%RTshN-_lEu)uj{5^FO6qRv<=gk z%$75aaOKQ$@eKEJ@eFq(TshMSSI#uz8UDsZ-msUNx_CB>4GC;WU_$~M64;Qyh6FYw z@M|xDEbYLL41_o|DHKAJH`xb33>Y-(wVydC*ax5~o+={L-btY!TSv0$#rI#Mf1M`3 zQ5apLAA0~&H&^M`^iRW7P<}8WDUnCZX`$jVoY1c)ZKC`3X=fOjDd$5)6Ngb6YW%u_ zzOH0i>{qZ0?Niaqgg(v`DX>HFlb?D|Ij$Xhp9r6#?LUvS$hvkDp{M&&lR#*l1unEd zjm|#cS~9x#ngiEjU9)PZPty69e5Zq^!Tt^W2F)b)VzhCkeoaXhC8EsN4qt4+>6N3Y zee^{~XCAnJfAB&1)B9v^!NdKHrg`a!jFKX))7Z&$dk5lUQ_!#Y-Z8tba~bg9ynCFn zuwLsMK5f{V?>CN)(UA8#9kV~$f9#kkXditTbp{(~IRMF`mtfmJ1*N1dVtm9>*HA4? zb?|ra|E$;sd70G_4TI~9ZhojH@>B^nl?U}!u|5(3v z3*V{VAaBZD_oiF!Ca<~CEq9~R{j_?fhMj_3x6RZt^7%paOg$srrfo9~?CdmBvf?o{ za+2SIkAj8|za?tUf|y%)%>G6M?0^&7v~8x7ksA_okOS+lf!`*sO1f=E(+z8$sYbR8 z(KSs6;hjA^=;ztzDv0~){|KvL9XBjBRUJ2CmW~@en>%hS!eJ5iW!P6?j|p(!)N!Mw z8kZgJu&=_t3i~qbJtEvIa9I(y2JWyg!@dZ+MTJ9)gsT#l4OcBZV7I7nu&8jcsBmhT z!F3i_8kdS7urI=HQQ=@w;nMR2e9qyTMJ`Vx7uc6!UxeMF!l7jf*OR!;<2rxp=c1uiSD8U%rT5q1kkM|{DhM}ki!E*o;GMJ})}!@dYR z!~u^PTxW5majEct-J-$)@}Ms`wLF3A9IjbhPa_EIi?CZ%IK1}iSHOf%zk20J7uc6!UxXd9fyWcL&f%KH^)x(Sx2SM{aOew8 zEmOFj#C0B520>t7gdLTIzTna`4WFlQT|h2#$OZOg*cV}kIN-s$AP!s>T${TfjxKn> z0rH?PIJF44HsPwqWk(R$7hy+bp)a`fi14YvW$i-a=|bb_LgPUw*dY#hNVqC-*>Kgu z19mhX`hpAO0p}T9XK|%*sR#o5BJ8Lv^aYomC*X4q*DP{*8o9u}4Et-}`R#i!m4NHC z`)yuVR^i(+h3iRN=W%6_7VKzA^aU3P2hP*Dp2Bqj*BpYtz6d)i3w^<5Wt@4ahe0jW zQvNBF1+Avh{p^&b(*#OG@I`VZaKkBC=s z{V09ihR>Y!ZQD!p2r>PRCNS9XnY%2d5f0ane)38hL4JZSe6IfFl~-Q*$xrm}>GE+3R>~ z+h4^S*(ulryG_XL5v+BUHen){OTT)(k-tUz%R;)Ya#omFvc7g*M)kCbyK+0l`YnoF z%XT8w5CXp@?vpZhR8G9SEtLoMv&*b+mf0lhH$`g)D)TPuMbUb(N$!WHQ0B6jvCm$w z>%(OedhPY>drm8>oE5E1wp8?4za{qN(ic(VFJAXwv7>_cXWwqLTN}RDh-d-@I{dWM zn`V`Lvr@>~$$IO_Fc!*Os+c&hst#Q#~LekvOXK) zSOvMUxi5`KX+ieiyKDMYO0un&Sf<)}Q za#p6Gds^z-DP-WtY=Tv)#3ysycFoymTXFWyiuICgX-3NtE>j9QygdrVmeNeI9EWgo zed+zE__GWmO3#b*iuHcf6TQ4=S!{}9nokAac>}e6thl@HQp|>xzMDh{c zb&&jg7er?huhgq~18<@KMHhc{7Ee7Z&F>d6+|1$jZD)F(z47}< z;@xj{iEn=GZ5ukc1@@!qS2ba*xR?jKe2bpVZpOZfvBr+FFH0! ziYO>m@Ddb3P%@%Jkdzw9AuwgLq?~6CNm8bm;~WcGm@>_jI=C=JWsX`&*^G2>!<2gF zuuFK!Eu2pc336Hw4gl4 z9F-`BsHB;rK~lED%9J~pV-r)d%u&r0K~!o4$J2tsB!?sf?`Do!rd(tW8&hgUr4ivp z<(!~wXO5jr$uP$*G*D4FFF5YR1B#9>qs)TCf$|8BO@i{2piGI5udvA%nd1^uo6_lK)IHcfRl4BMxOA20;91DVSP72;3Ikrf_tmOEL zsMMnj2rnu-#Nak5xD(GM1z(gLcZy1bsBD*%OUOuaJS9yoh{~5GNb)bUpkd$3g@HxqGSqkQ)$!EpWj;~0;m!!$(CC9VUcO5TCf~;wx<}rv@i|uUX z+$~@Q;j1MvXimKLx1x$G5+%_uncu7hHN7*op9$`JxqtP{G*EKM0p8iL;**x=6-TpD%{J3tCYBiO^shilX!0o$r^Ikpu7Tw&Vn=U=S4n15a zH=Aen>-H|)JfNHR>*fQx`JiroMmG=Y=4W-YTQ_@j^N?=#a&x7A9bWT`%6i*OA1^}x zXW8f3LEt=Qe}b$%Cqh)1g%JM!Bw=rSk-=|D66sHvSKEAzE(u?1HR^AvyJ3I7{DqEB z@gz&zNB`IHXg9~qHk!tVvuW`9A{^G*ecAT$*1rlA@Rs=Y%EPE=ck?t?+`Kup!}8NC zjz_~Oy*O(0d~^YzeD{0j)28lRJIF;5qt7F?UuGsPC2Bm@12^N)lAlkdLSv7W;m}US z@k`SnuRt>uP)NU&{8;pB^g1@^G2@KK_x4lMYuj0H@HKvywcynB0xu6vAlHwG#wL(I zg5dQ{AXR+pOc50(XjIr&3#qV2TX-|wpu!-A4-XZ#a?ctSW~tG+uw`&z%izKmVP6Cn zW~tH1upW?KJs`gpVP6Cp21LK(#&ldKT;LHaV8vQMf>~6Z7Fz}_woDE+CbZaM5i4e? zDMO1b8myS5rhpb}0SjhPbzW>4yx1~1)R^#Mi$%1UrKSuowrJ2|mYM=ytOYcfMb(M1 zWe{V_1L<-2T7I0q{RcFJN!Gx;z}{mf+}L6fH3s=0MH6mp(V)g4AB51ku@+Ea7F8$5mO+j! zlS7ROIks5DjX^$0#DpAMG`KOy2O)HFtOZ<{MTG;gW6NO2mdT;UgdJNfBF7*fkVhu$ z*kUztNFc{hV-P}T$67#!p%H2H*fQv`Wpb!7p~n`B*fEF%iI~u1iv~LeksyRlkF|gR zLnG4ov1Kr0%j8gFLWwOFF=7x25;38~7B5hOV8jfHZKaOA_8sD9`=fM#hV@sJ6Ek?d z4~`rA&N^;vpoklD=LxatSM#_19zMA)<;L6^!?W^nq_D=0`EG+9TOm-L2t$E!YQ+jZ zjA$}&B%A^46hzPqv!bs993hig(-YVIK)$;`EB4ynW}G4eP38ty!8kuQD_ybehc9Q! z{N!|%5N+X_MRUodW$_C5G4V&U;$?u=R#9$jf;BUDgE+A{;PoP#BUUT}J9sXHv^Ies zvx0#DNw)Gye`cpfih&o?I5B|KbMs)x=F&}I+`y2LKM`Row)Dym-E@x+r_6{D1Kx}e zQD*+MPKBk{D6?09Hsf;@R2XQnm*@*0vT}Y56q!M&eaB$Qh#I>kOBP?F#@<=Gjjyp} z+gPn+!@tgwNe$BD;2knHwhx5a_HW^Z5J6xEJ@GPVvo~_qSBbwOTESq=mWVV1qcSV( z%3bEH*@}B7QtVp->uG(>E8D~+J8p?P+XJ2iL{x^jGiw)HvV-xU4=5EdXm-#%ADv;d z9rT{Tb6LOX^>3GCR`IlCudi49V0<#TdaYo-`ab$maCz73z~9;NlGx!V$h|T%-vRc> z+E59ui@BZPb?^aQgBVunS|gaXZ(VOBwg_wvt}Aw=6-qZ$B7|I5L89md*#$!F5|L>j z)sTw2Rz&%URii+MuYk${aSl>Vl0f$$RAV!8xFqTXt_?I32zdLC8;Fc&^MWi_;DLdi zgF$O*JPcN-*D8X-01rj6<--!_ury%NW-wAWc1p+X@P3(?N=mW!bqRzPVz}Lq2p$GB z7&QWWJ;)H^%iu#{`ylZjPrswRS>o0*Z}g?R@FuT;nioK$;FAF>1$NC2cFjI39w*U2 z`tXL};e`wI7%sF$jUFSS4Iaogm)?(Dt~K^@rmYVYPzG!oOM}jv6~?W^vt8jl8#&UG z(H8`L{0)eU+%Afo`Z?Yg+9ATe1cBMa-j`94@E#!C9YStisBBnqg8>TV`iNsiJp-9T zv?JZ23hhGGv4bK)MpTI)!380+1QzYG1ojP;P+9^DFTJ6$ZL@D%W}$`vQcMEjVzGf9 zLfSb{l=Qm1asw>%q#{%da=BexK7!)Tf%?K5;Dps4Y6}YzUReVo?V6Qx)-G^*Aag+> zr6t9mX<)h8>+U|Q^f7n!5_Mj0^x+#E*@9T#A?_r8nJE0ZcR_tYj%*%+vk5PY_Ib$1 zwt^DQihc8t`2yW=&Wj2)D}o>7CJ3)wZfSM}%_x&wn!ol*c@NQhSHZxc=y(q=QDaPt zij0<;zUZz4Q`fKp7CW~z9oN{XS>ZL2j%Bzk&C(+aFOUFpR*+;;?j<7IX5XgNcM4an z;K*#^*E%Wb%qu&@3~2C6T0C@XM4W*N%e-N|{z>Pov~)EGK8J@5*7<8r6dY<9 zwl}WMieKv@U5;0lX2q9@8(aGQw}~y=54P;O?J^3!#+Ct11$zLt4D`~sMQIYm82F?q z_T?GyTOznIqQN$U6$2TDW+XVKSx_Z135*)iFd)pphkGM{bBqW! zFk#@mo)CyI17!tn3=7}&5XD%v>gy>puQ|35pSUGK)iwR0*eOn32fg> zAaa=FSt7s`&}PK6iNv~riUaSp4atI~(>N+d{MH?Ua$Y3<5A>TxwtWRzaHa|j9LPLS zYjt==(0Cx97-%X*L`hI`Ml={Gq6;EXeOrVeXfqILVA{@tY$FyBVw)g* zNx6&jWFWo3Y=QRzrF0%N5ph=_x|FmSw1R>X6cAlGK>^DJCKAM0qZqshwo)P8FUJ5?N zxiN!G`>w{O2^U>DY@(~{Zct^K%I*eL_E}yrw#{r%W$y>o%Y|w3wLz7E9o}G(XWm}| z`?gqQUK>APs!Yg6{{&F}&rOqkb%Q3m1kI4{4Vo;K+&4H8nvCNJM;ySJ3a63=JM}BR zi>!bP^D6l5Sd$+Q33&XiUi{vv@!P_;pvgpi#~74=XvdfnJH}St;)FJ5zY9kr;jO>A zJx07RQo;Sty<=>g1AygsjIsA?$Jok4AJ~qu@BZ@c7;7q_$yz{*88jJf8OzgTEuh5= znvC|0%CKY0H)qEdv1^Rju~o2R zF`XQH?YZ;79BQyptVQ2Cwp>J!#mX>b%Qt7p7O{1V7_wC`WHA$hY`KUaibTTP~uJx#maDF%Qxr77IkiH z6`YtE8@61;hQ-Q|Vaqos!xq7g5gE1$GAw4og)J9xVKGo)uYKnpC@I#M5Ms+WC&U(Y zLTnX;Sj>bETQ1_mVrA&CtZ=o$&pV?Z+d@}353%$FxV@jCPR5(dD7r0pAR;ZPKUJ3Mn_)pN`65p)Oa4GQgCbM#?z&{{xfPa9K0sI4c2iOc~9N<78e?ad5@@*v*chXe9L(_@}3I!Yv=nhwa z2iyfn513yg3IiCA!+y|{-_8Ic5$Sv>M<^Tn8RdB9K3zM^Li|IQKk<009#M z6a<0@lxs#LMS8$^!1;g+0{;Vg1mJBR!U5FBK_)D>awbwL5j z_#%)tpmada1ZcTKzg<_12MP#~57^r-Ay|jRMF${5z<9vkfCvJjhtfOnzj;H^{TX0# zBI)pN5upYT9e7fAhra#12rYNzIbej)iYKU#FhbyeKmmaTUIbPr2I~lNBQm-tkU~h zV=}p0nIU!dmGUPA2+UKId`TKB7v3L@Vu5XF#FHlewwVo$_!&(Z|F*D zmdf&y2@1Gt0|opLqJS%AC?FjiMauGo0xlcM@`M7mRBOuegaR%b%JPH)7And27)tU4 z0_M+->M=n9SBp@A3-0rN*k^_Zy0uNL8dJ!SB}RRjNv6)4E} z0PW)n@&y0oPmby_!T(l^@V}k{{O>gmkJJ<#AGKOk@jZq{Jb{3V#&JQNC#R|0HJ)KR2Ng6uw1OhGs1ms6X^_ZZ5t3@ba zPZ<<&)j$Db1sd@^hDJPrfQtYDv55jlN|~U5t3@baPZ<<&)j$DPjxx_h!rfOs@w1|U zJ8lOB?EeX&fX8HDhf}4~NVu;26X6}di(~foF$Vy@8IIdkh`|~$$Z!+{P#F%q0q_G( zmVwe65HhZ`2gD8t86Y|!dC=H{wjYo=po7o`OYI5dvid zK)9WQozU$AJPF_ph$cWs0G7b{ax4g>a~Dqj004%-9K8d`mj%>HCv6bi2)#g{gHZJY z)OQDj4<$<=TR=|*Qm})n9~AXo1_46(PaxQDfx~zJ9ibNpCBM4>%W(z4Z35~Be&B@^1nc6?+j(`mB7U^&f==K3t1jt8fe+1w`JCG1cDE0y8J4+xXfs0T& zd{LyMEP(t0NCx-^l|g`lP+o+>AI=d0hzR%)upgj4r~?A?YvgBRKzk239{@#Qet`70 z(J2co?d`7SOzg|be@Q3=hM)^QV z83>>te~A~#u>bU@ux_A$LoPZY9=`xRF%V8oBwWVv`Ob|qGd7NtPi0g1)y9%vhxIG; z>#*i$X0*|n85%+e81NsUAX(?cguWV<`mP2-beA^lfp>xYL<<993)!;w@-|Hbp{XaDSP1U6YB1=Wflu(IV=@mGuj2RlTiZRYZm-|(YWL+eH%)Sg9z*6@@`90k^dWMi zw4?tXcV*J{c|5LwyVbyhZ=SaJv!$(g+gxt{&C*sYh7u7-fD6Vbd8q_i-EJQafAP2G zQERhOj^-s`l+sA2PM0LoQQV5FRr`?z^H&vJOU8IlMl!|=X?1%PH-6;a-=+&K;`;xw Ch0=Ne diff --git a/extra/sqtt/examples/gfx1200/profile_handwritten_run_0.pkl b/extra/sqtt/examples/gfx1200/profile_handwritten_run_0.pkl index 592c7a657215d5f1351fcf2191655fda2c981c08..267839c7237aa52a4f0d6cec3c625f6dccafaf2e 100644 GIT binary patch delta 752 zcmaF6o%zZhMuP^Hsh)8RU=TY+qldpFGp{neC^1DZCABOwIdw{pa6nOhT4qivkOGS5 zPSls+V|Mg)nbO0OmzWEb7n;0LM|kssm_MwOpC*|zWw1|HOcr6v;G1lfJb@`geDe8Z ziTVub9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdvd>I%sltF|_k5Ex+dT?NTfNPMOvwx^x zh-;AHl+vUOam~glwNo@QIC^-2vYvjyA;Iy%jy|DN2$;(RHFxs-6mgJKcBjZX5YWPm z+bPCir%X0WRUqUa7TkuIp!$d3J;=#7-aXVa-qAV4Gt3p_BNmVta#%3}!-^3YR*ayq zV$D$YX6DFHnJkzlS&tMl9PU0w&hZ|eArK>g9GDRbz~DwUf&*v-B)CO@!41}i6yOG^ z4&(&O`+5ckV=-48&0sdD!JGf2*|Rh9P5xLc%ag&LA?(cnbOJ+$=VrB%4_q=C-rkJB yNcDwADjQq{BT&Ve7B>a?jKB_@ikN_k&aGO;4^-5NQxP*zk--5rA#S*$R6PKOKHLWY literal 56301 zcmeHweRNyLb>{;JN+3mw6y*<5c4SzVZBV8me31aOrbCI8EeRqmij-@wB0qt^BLN6s z#)m}8c0$5V6EiQYY9ndZr^{v1ZjS9_T{Y=Bo6|p3^`y;F(^L`LH2z53s%f{`_H6gK zr<-o}G$-EQoq6x!n*=3UrDf0-=Dm6I?%cU^=gyrwckY`peBmcQ64>AIoLO#9hho!{ zNngMclp+Z!naatV4kzOip@?+E7n_v!oswed+{?MH+1zs|zAhl03i+j+Y&bf)?+B8Z zFDm7NI=Oi)>GOwTlU`{$U%Dj_N+lw`X_1wZ8>MI>J&l)6O4F%am)ww)(z#(-pB~Sd z&2smyU_2`AdfIo!7un^H2c%spY6TZkEEP{$64SY%b?arLcsl7z0O_1;KOIU3P5wki zOmO7Qd3?+hF^kAh5XG);({{pqd)Mw<8tri}Q>m0n@A5?hyVAH2{h3rc9u-eY$(R%& z1iTkPJ9-lI#R8|3p>$e`<>Wm+Q=e&!W64@;wps1YZc`vVosjz2Z^{yir5*Mzi$5Mq zhbA-eOv>D4NlLy*=!|6UVrcEyl}ROcMMC4d64U8mJm#?2?7K2Cssu>+eJP32U`Q{w ziGE)sGKTCS2_*wLc}r-*6lK@uE>kFFipA5W;do494GwM{I->{SGr-l9k|Gn9lK0J2 z0|&)W!3*EegVL;+DCXpC=(J2S#$N^;^G;uKGSzL`x%1@dK#HS(@3QEpGNClX-+Lyw zO>UtI%}FVhiKKarJG^7TyKAW@xZ_$O$oDOaDVi6P7`u7jtAwh&d_P5fp&09@PMvH7 zEb1kUulpI@d5gg<)czy9A|fYyNO_zXR&R)?A41;~_1pNn3M|eo!P~q)kc|&T> zdKhI?MzIhAtV}wW8JG2`bWUC$ko=*jFM_K<)1+uCEXTb=A-W<1!yMM7|0aDC7iY*y}C<>owJ=Ynlr!S>zl~L{ONc+lCmUx$#lpUF)PIi zbdqk9FX<2Vv3Y~8y1I^8kCT#7mx60g3VIw4OtD~VnN1+%8b44Q7<&J;GvKYt%rWiBt*99-Gx->)`{X2w``x-DRZAERLRXk^~9w)`?JzY1AgtkBhN(V>+?)*b6A3>Np;&+hy^19OVO_ zV74SBtY-X@X*@GgLI{t|>ag_oRv?6@#|E;ytAK1`oimI%!gIMTHixSs ziUn%eW&#?Tm^Rr=dovRg=)#x{5-RvFpUx>Nnf%}~iyviGxC?#03*FAt*f$MwlbBy~ zg(VBE>uABFt^Ld=Ern}76V@W3R4A58`(l2{OvOd4AG%Gcbh67NMN*P!V*4`_vFA+S z0@E|aFF&`v+jP=GYelXKcX)p)w?{%T@Qz5!J@J%}ma9j`Lz5!vIN5C)m;6{lro^Z( zL5j7?U1B=UmM`X`k0mTr(!BH7lm#tgwN9BrF%wg}ySpBUq|(Q`lyaz*MB0!98PpER#~YWSM1B7Dy9COCwq&sm=IVFd-j-t}pV&n0G1* z#?rQ#cLH%mt--=I5EnCvfG;hHAZI$&S8~sw%Xe2=(2zzhO{KA(WGmMZxiLfwo1DB= z#T64tg@qiH_VuK+ccK^wTCGL5dB2OM*4+H!vb$Usr>lZ6_4K%*f4izk zsg6`;Je3ZmGifIKAWTZdpR3nqaofu#F1OWcb=WMup2||XKwPbdOyzJ3mW%)0=|G`wO=dp!Wg6nYYA4^Ro9Jxoeapi05_ zI4w?3nPP^fczQtyXBE0`WMZr;P^?6Id-X9LJ6@~; z(gj&vMbvLgO-IM$k&yp^q!h^bCD9*`M0|;q^guv@NGv9X%LU=?a8%$5x7`jVb5&9Q zTO*-pC{1c~=~Uvjy6w>S@jC0JB3fX6!5T%2nfJ;6H11J z+wFm!)yCPh7^cVOu=F^q5S6e0yljJ_eEkPpi9c=160!s?CV!`guZPNTZO2ek_C3V1 z^%+S9G8tFuE|1%3!Q!Xla4VSkH>cx?I2M0tQ_3Hd0!)=JFr~<@8dfRnO<2Eosg`o4 z=iVl_`5_}nVJ66Uo|B!+=?&W}pt8M-Zxw~#rpYuQ-x7(RmXZuhQb@SY4!HaRT5?I1 z=2G&Rnbjr~kMVT|EcaLpVdw1OoVP*Vhy_dndv5_KJZXP0CpSG54W+PV!DF)7{A|e3 zD8tmrWGVFbbaHw(EjUmwd%=uVO}ELPp5kiD&xQsY<>o{bs|e+dv4AvT%4hI(k7;T* zmPTEs2Ol;eeU|MOsmqv5j5gW$wnQRI8&wRFc?yd-nozxPzeXYX1SU<6X zi74$x5o}sPnPbvvF_rX-vEAG@Re%#c0(I2leB z2>z>lhq87nS>b8eLNWdaD8|@m^Z>2l=ZG&tm!H&pgcB@6YedQWxd>?%JsZ*RlM4>GGm36zh1P0 z^4RS?*q*Cmtk{IzSzk0oU@IIFgDO2WG2O;R>*{FYE%cO`AK z$o;OqYjaIbg=8U)3v0V*AQ$^iX-7AI0x+^Kg0@&{>v|ZU?o`IpXf<4F% z3o2}!Jyq;MEzHmcA7Lz8i><_X5xdqNSGh3=BFqVMq_Z-n1>$B0HZ@^YffFx5XZy3| zE-lcz<&Fy31?;aK{lLOD3MwWdE*|PC)I;jEl zHhN8R1AKsUvM!VZyzpgum9oR{;41th{HWEi*PHeP0?E+?JP!zC9l`bLh6Dp4(&ZI~ z%P?wSRdcYykf}jle_HaLq$(@n-^WV-BWmsP&TwF#1_dIaz?3yt1pKFj9u-uY|G(7x zF5t_spcg00Uu1MLuzKFcJ7%xm3m1({EESrJNdc2T=)-2TZf^lKP$2wo1pXKJQK|9Q zbgft0uV?L-8zO;}f@V4w{tk-B&3vXh5SxgDVv1*~u`WkEO*- zN)jV+7*tZZ7v&bR+>=Qwo`Cmmm)uGzF)6d?!+*UA+Z%AHrM-<|*)T3)RCtBVcoF3o z0roNi5Tn5Neb3NfF>zUu7OjZ6kj477>@N8}jaOm)N5zuGmr2JbrI-cI>Xu|6=5xZs z9IS5`eeAGJYrky3PQmHiT;pej+4F+DKAsTALue3r^&3(F_y&V58&fdJS#gz-*NbV) zED|~B8$=jr%iDBk1GX24R!mggDyk&1>s9Ip*vU|W*NFK z2-{af-Tl=;S3vjcHKBW`I?OV3+ZQ`lL*2vGLG#(@@xmFE%|-`j^yW{h6CEZ}K7B%4 z#%)V0Ozx53AtAzrm)8@Rum*mm5!^HcWN>MU_0ULM7|a2FK`N$((95 zKl{rcw%oDXpw*?!*}O{DFq&1_g&lP&NEB85iRx9qTs$KloYn8ts8-ah;)yU&9pZ16 zK+JCKuR^HsBBNIks=?}jf4dm)oN*^e)B3T8tB@xADnU1eUt^V2B1@!tU8Q8n>bq+p zOS(G5GP2w`xC*jNRR>%~mTr(`<6V2Nnk>&&ud9?S&+9FKYTt*fv$kKk#!K)_AZ)$bD zf`O*2jqapNz5-$Lxrt<(I662wRwCXSZxLSjVT&f)3gu|AzWk$A33m zQcq3oajvGGQ)o}@f4oytPfh*v&}!<*qCKC~ZP->!x0>2BxSDzv(4H^$JX2CnP3<|f zntEPGd%imIwUT;jYR}ke>iG`ZbNM2_r z>;6zTTf0qiD6$M}zBq=td7wn-H0oxAKgd_Y4Zsylh(n25pqGlY#@~AX%NlffT`Clf z+In-|_7pSy%7#cBJ3Y+KJd|Dr{;3iP)+h+)G_VC$X;#bvEmH$p# z;c{;cZqgK02O9;&UefW~?%bF9($=!YVT=|e1%&wJDO{TNu!Lffc79{cU= z8EQLb+l>>WUESn0T*dY7GX}XO;yW{qvlPs^JI4-GQV<<@?k1;?*^ZaFZ5tZblDFQ8 zM!BWXtvV+kFZSwIV+@%P3&{jmVsa73QN3kdyVZAgo0z{h*hPC!@xXqqaqYPDorRF_ z{gf+Sa1(9wp4(wo*Z+~5^GAep z@}nR)Q7jxv?^gYNaMpl>Qm`?r)t7bXhXKnEvXKKwucgPzj~S0t_I?0K5R6!)`iU_fomz*ar6njo}S|f<5$1I+0;c0qa6Bew9HuI~?dz z!)Cc1HvA!aTLnkEqOe)P#Py7oRX>kk>6q^t^XzhYdT|WX0;j3cvyL9{GtMj6lt?g$ z-PWc#eW(E#JCrDmT(>D@bGZS+9=uD7!}o zhxY8YB9csjJJtp|K~KlR$XkUm=#X(eg>#cVHE3>%f6A^@CXvATzFk58(=3k2UOl`^ z{PDze2vG=sjUAT2BlVmWKWc_%cAK8gNEwOw=I}FO$g~7cvsXps8Ny;nDL*DpHTtA^ zf*t!Oq@Q8u&gdolL)1zND1zy_4_ccgHa-pE4@FLj`NnbHuJ7=X#~wLwXdlz#+6i<3xB(#|gDL2PD{wr9xH=J(LqI4C=}|Ss z)#UVHy7gW17Ez=#i+*1$9t%Ou1p)yPN4KLn8OkpmH9Y2zq_}S$#Y(_=L)_xTG`x-y zl0QxB1T+m%gHq;DlzN^5kMJvzM=3N<-2cotqF6phC-RvO6H)P*spsxTOFzL>>HDz^ z^KO+lFl4B93V*1cQezaZhNd~BoPNB``&IY@N$I2T_xm`Vdt}#)!d<)hp+t&^g$9l7 zIr8YfF>&lManF%~{l^CRdSN@be*19?l~E&R9n{Os?4;c&2iw9h;c&TbwRPx}9$^DF zaTIBlz=|rN8Y!A)UBRGUAA{rM*zze;sjDOD67-qvO0Qk+PFYqY7d-XH z$Gs}qCNz?43!A`tLwoh%yOFL*`4rwk7!K$mY<4j|BN+!yt0ZtR_Qc6VrrQJ;#YuiN zlrJ!j;O%EQm-5xR!u1J7T4K4&$EG=9QI9OMD7wnQ--HFAu@Fwc{}6|PmBJL0sV||v zN_Xt|E!Ci&{$*jGkimvITkFK+W|BAn!J-l!?s_;)+Xmi{#ewY*vS*4nx!Pi=Cfo)NaaI3v7zp1g?3-Iv^RXNA|!liw>jsFFiz zR(Sn9d1{hNCAmO8FMRDh`4*BBAUW;5AbkBidBc&r8u^r|{$AvWMXpn-lM{I$W#KcV zdiRj)O%~2Fs#^>BoMho8qWW_XJmy>h(ve35b78x?7~~ z=BT?Zv>T$sW(C_)Ap5`YU+VwrG$`N!Fx_hZ6cJ=p-b??2&Jw-~bgu|Wh{p}gKA?=%F$4l}tZ=i7)5Hml#e@Cg`>y4l@m%GNa{w4*nN%Q&O5t@yg^5T%(r5`;I&^@YvDe zv3*CZaOw+x6-aUaF$#E#0tbdi$3{gM{g39tuM@_ELip>r^6J9>jVEtVVhH~ZCHn9= ze)qdbP-*xZO8QNt6>7CDQ|su})~a8rR(n~k9it-y;-SG&@xb5{;+}!A1CQ^!ss=Z# zRJEh5>RZ*S_wE_?iUW@gKXT~6z}QvQZdj>yXIbrSj~>}OBtCldKyj-LlwGl-b-bf> zyrXqt-qDS$W}L8N{d^0!i@{x%zl@I_vJZ&+52$T5(0y^8rZ8i!igg-!oy27MN(th! zl{T?ThYpO6DimIAt?P1bMkratT?KF~byvD!JfBeKXLiljMz6jy*QU3cQ zbz=P}6QSPr4=n{gHC(LSDQe}1wmr2XUag(g@@s|rVJWA!Q%!TSkr_ytb`ICKymr2f zQ5{<*wDrb6NLH$s(^_+>1TTz99)DpeZm@$5>q{h zVaC25?F;VitG+2nMt-Fj!saR)gNiY}_!+O6nbw~|#aR_q_PsZlx!Ly*sJi?I5D*)H z%-zSkO_NZBd=UgVJtOs{Gg$1H3v#ew$^s)4KF((_Q-A)J$J?ZcpWHi+A@qCUr--ml z?D`?9Da`BGej}9dk1#?g8`bNA{GvkXzQKKak3MR8xX)zg&AI~qEDvw@8AILp4u?Y8 zsHx=oQ2&s_wR~rA%{Y!I)($(1L{=CPNPA%O^jfs9?r^h3yJGMd9T^*|Xz>Vt1tZcc z@YS0TehYVduQUt7CVe7kY~S3Zo9SnBSW|YJYGY#ey0Z^5Rd+; zaCRv7_bTaqN_xMNnw7LmNp~vg0}A|ZC4EpyElPd;O1V8O?aJ;|?gx}~P)YYG=_5+| zsFLni(gR9*P)WT?dPqr!lysP-OZS8H{Yuu2t$M_PsvqN6JSsdbGzkX$SLsi8IL2(C z)yn9s6tGATu9`p3{TZvaDf_-~O9cdUXAd&O)aOf|s=)r*$iO%liL6xMF`-Gn_uDcm z+ux)g`1bt2iEl66&hnmn)jK-EFY$Ed33^O8DUa!rRKN$rU)p#hS98%oo?4 z;7m;P??LvG8TGs`XeJ!p*?#3E-haA96^L?w`J3$diu|FQ;iFCMXr1u!BEamX{6}|o zy~=+}4b<}^4f=`~<*%|FUAy7`LKdS@vqkbzCvYOHL{t)vg+1Pxtsoo=`mnQvA(j=6 zV1v%=M^rd=l=1qFlFOZnWYk?Dh8HkC4hSt(#G{})_naah4;U&H(p}icW}K`x89pztZ&dv% z6OZg)fqxT&%kyvhe^uqHHp`Y@EI~DW0z9~lwRy>k%l8ErZhoGV*M;zrK!jENZ1^1r zTJO~xIj%J4JqITh+rdZ2yzwbAZ#=S`c_a07IxR*|RWWazPT+HpFfCyHUpHM`^P#}H z-ylp^n!k1ZOYiVTFE@VN@Gre<7785YvM8~ISwwR6fz1lnZq0)u+zHXII5+S^TSZn1 z9k){Sn+mKHC%Tnct!|~LTPZrTAFP3-Zl$PODVVqs3ak`McVjWu|I2PY{>|WDw*Qy2 zix)p1ef}N+_L14eKjD`%xcqMZ@=w*vk;BkU`OeRK z!|dO=CL{%SluNfxomQa6tU19d%n2r;b=avFhUWl-{X5r$9ucsOd0^+7kZGltP=^8#1uo6u0bYJ;gK(+&vT*6C<}1SdJ;FS$=R1_k zvrY5O=Y-D*^C;JGwrK;N@~ruVg_8TN#gAXPEG*0c%EH3S!rW!yC0za%Pyp=ZF9Ck; zoUri9*(TIF$M5D>xHY1dh1tdLGjc2qpzAKHT{nATDS!Fr`OCk^U;aS7TYwW`8?PVKXFc(|77fC0d$-1unJEw z{^9Hbsy9<9;*NRzYi1cGywhdepgv!Tqfq@*_}d^Xq;L>sDF`%HOv}S7Pkj!cAk5J8 zVMK^UmGjg~VEN5-L*rncbxJ}16kO7PM^|b5UT~*MKINf$JA2|y)@cRZrM;x|4|NIe zApC0mv#@LjZEO|<2$1uYWQRs{g`qjagBG~QYq+%<(I+NjE9KBr$|Z`hr^v!JiHtk) z^tlLzhqxZo2@4v_b7E<9gZu#UjD)3^Ois*yj-(xC1&9iw&O)A&DzWBEh*#-$l|-98 z@fM^1mW~1m+=_;Rdat0fua1CBrqo0|G?^srXUV$N1I@%ZzyEomTn6NMYg zBrV_?as|J0$XO`5#b}%FF`#QIl$pD1oV#G;ziisYotlDxq<3VwYB6j25hI%M703^$ z8sEyYd5L82OVHc+-5liS9G`S(vbltR1-I(_!%Fd|YH7*>RJFt~nsS*6<81T{d2^ID4w?o?%2dAQsy=)yFWGAa=P7C@?E68-*-pvVkdx*>z?Lqwpwgszn`v!=ghB&`c*OkL`P zN|WmT`B~EyMjGQ2x6TMXF9^DsgNn_*vWE^m)Ln3}GIcK~6+JQTb2ZOJ{Am;MTo#n3s;A>wlg(-MrskTUuluV49b3F8nb(zh)1xepB58tnL9;KfH#BdvP(7pzz~jD(y?B`bzG( zhaY+Wdq=`LB=T0}d}96Z8lq6+0OZb{Cr<}bj84IuC`h>G=w)*|5X(Cs^jLdfciSvh zHHPO39$mGK3C6iu$zLPvU6(Bu5=7-rE_XaqaWBgwTtz0YGvot61ARFyK;5s)U4NdPAySC zr$&c|JF{!-5bsCD0(SY!Oxj)bIW^&r>*v(gAphPp7O-=_UEBP=bc`vm#LNZf)an+n zHL85xEEX``i664)7^N0gRfSxld1U$X`~MzC%?Z7m-N?E`M*{lM(6wx2)xD=Hvasq* z`+zZ>`{uzIxP|q_YdvGT*jBf&)-9}c3v1oNdR^_KH=~8s;dWab&I%S*CoHVJ+P68D zKV#d#Evy1!glHDlyO@FsqkZw8&RVyyX76D%>B{b4X=nCcCB08c>yf~+s|!MQ9naX- zEv)%SV08>nLtq?94u*++X7qelu8Db%~S`$#b?iQtahlVMMB%@B0;I*~8Ks73p&1J0Ak` zJt(5Pu)fHGU)e|cnl1Q3N8N&7x8T<;_;m|@-GWaaB8vJ#F&3yajc*XL&zx$m&^_>x zcDXB9@Oxc778f_|;}f8p8cU;O-in;y!*Z+5>FBZBZ7vJqYW6TV;Z8hyE@zh8{D}-c zSCo`e_|3%dSyDl6PKBZY^>#*X!)J)ZabN7DRzxSa_N99F6*>#M8c0s0CkD!bgVi!B%g(yfG3; zu`dpZp_q7bJm>9~H%^vh3G(__CMr&zPSN8w0wPW%ULeH$87g%PVVjD?(?HIj^rihl_P``}OioOR zX*lysY4MblOoigH+`&KmfexRLJtf5g@gyjb8`8JSP55q8$S<+m!x?!4eLE?Q(kw&x zw!AJ7Cmtb!wFi<>F^RWPJ-(b2Pv*Qc^7>#1%>*ziK+R7DqCSzg19Z^Kw`!SFlhIKA z2@YLoPoB<58HL~~1=h)}^w^&e`9u0d?d{ZBUO@CmPKp!$cr=ko<7>4in$SbQJR%a; zurJsWM@^j88b1_ik(>R#gl{|)38klV2WP%A5R^CMbL@-<1JZbAQcNV{iQLeejdB}n zO(B>s&dW`iSO`c_qvx93-~aW$-1^}C-&Z9l;=$vvT6#V9 zg25BM6Q{-i;4MSRsKfVn8`(#_a^WB28NQ|SE)p7Rje+!ZLdu2Tg$Ecu`g;c7_BCX} ze~+gaq;2@9vCZvv*sLCp#o^+gHLe2RFp-o1rwDv89FMEF*KV`eTpl*u@l{jH{|AoN B2QvTw diff --git a/extra/sqtt/examples/gfx1200/profile_handwritten_run_1.pkl b/extra/sqtt/examples/gfx1200/profile_handwritten_run_1.pkl index e837c4a607b7d4fb42421294595eccec9c79cad1..b3a7aca797ba749ca76bb7092a160edaabca3eee 100644 GIT binary patch delta 767 zcmaF6jrq!MMuP^HsV8F@z#w*tMh|~UW?p4_QDTZ-N@`hVa_W>G;eew2w9K4TAO#f9 zov1IN$n5CrGNp$lFEJM=FO*zbT#}y~pOKiCQeKof2d!GYmniT(xeP=&BiIUQ#3L-dU%1do_@h0!STV4KA}?xn9Bq; zck=ucagbAXr^q@G(87${DaK%@Og2kZAmkqw+=iH-`iI{=$jLX}J=8Pa(K*C3%oXG# z7LXWnSTO>_iV+xAjG(Y$%~1Ac=EzW)ESM%)j}$T-?mkA&@gANb5F>ybm=OxV;6^rr z184*!xJ7`$4c3Mf;0CA;>n1G7;&IN+3mv6#Xa4j{GdkHYn3i0Eiz_YpRqe*^(gAqDZ-RW%&sN9tl7o z0OLmz{gaTeli18F*R_$K_33gIcTbM(WF0lhIh)g-s>;c3kGGrSO1s_0NjJM{w{6;< z?H=cJlg*xH-sv znM(SDN=S<)v}7u$Z#7+K3%602aNiCfl(aq_J zoXw{9>CV&PbVv>) zGHQY&Z!6$qn~W(cLqS!$d*rQz`PS~8xis3-w?w5{D!s!$9o&&dL=9w8>G-sIQcK3P zC?Vi`2ehM{kUthYoeZbbS}dpU_R9nEILDH;*5PK%r z+Afqa7)2ojSebM#GohPP>72ecs0G5){wQL5MeJ3);&i#3c6-0Q&*f42Jnr15a~Rxa zaCB%_KGWfF_af8FGBXpQCN2{~YeH*7)=;Z>TCcYj7&{kg?+$g0gf`l2p-v3h&`no5 zQqjnFLbqL`@dc(HHivGnW2m6l>mMf9i)*i*??Vq%XXuc9xXToBte8bZu9}<~^44O= z&_63-uD*W7>E_H8+Ebk4bU2khd+%~=w^oSI2G3PWK?mlKePFhAo3a&mW=R0p*j@cfuMp`a`T{|{L-Qx2GjclVkEJH#$!X|8q}$6_iEqdIh9U!%UU$0$&*{3nT$OrgAL?oN?(3%YmaTktucBpCIMhU@tWtXG<8{ z(MJ*rm9%X?c1A(VSgmK|a7<>ZcTe~I(Ny|)x2SITAJ#x;^`QylM|KbG8{Rj*TYeDa z1g0uI?A<-STba_*WlJqtERrTmmPfQuGMe$zP(nWfeP88`v27QN#`3n=wgYjY*+kTabcD0^nm^}DJqYDg*9&ZM!XWUJRvy*W&ao1DJcz!h^zmBk$N_O+z9 zccB;vTB}93`u-Rb);fRbLbfr*thy=2X&T0SqQI!Xe<`C6 z$r-Zt(CjFzJ)utD--iCL?yL&VRwOujWO{pGtH5XLM_3p{B$FihMk6K!jiF8kFEX-x zC@Z8@H`KlMKCj~L=bB8(`o`;WyA_v{Yg83O-8v0hTv$y^7pqJTd#|$>OLVa)DguWb zmlH$Z!^OQRJ|@dc7{i>!{K2O_(H8nQ7=rq&+|-fc7w@L4hg<55>&stG*ix_;puRi z)adf5#9?=P>~5vMw@UxHJx;gctudA8>zO8fq-v+RJ-zg1#Ug@gb30)Ms6mWQjv;LG zayky3HJ-$NN5OK1*S*d@#Vy8CNylMz;>1*2gW1+&YXk(~I|^k(!qaPaVGUAcNO*dk zu!f0sOGVG|C7_ROQdBMh9au+!mE6LXR*Gvc>~9`n`K^H4fuSb$KGcc@8c7C>e~%dd zWne)9kE=Ql6t7g$@kAWUzqFhRgtQ>j<%`TIy0?yH3VRbK@EwM^oGH4u>g@pt3DTGe zGV*h}YbnKHYZX+scJuwB$PZ{X4eB>XtI}3ZKO3{$gyS*3 z(17_Kt08Qk?aKL@^bJ_SB(MP&gvOH&gmQYzgVW&@7A?poyUkCBEzLU2olKZQflnuA zcG8Li^|BXiSl0B&f%F-!xBPT?Z?oQ-n8q?fyfGfsCgnl`U--ypc4BSREkCeJM))k- zGBVaNnHcS~@x6(}H0@R~NVYRr#nGHvRM&4cGkfo380hCK+=B%aE0~z3Eh&OcODJ1R zJFTXY0X4Rh+oy_fqAXCyD9(4UemeYSv%a2jU;+5JSip$dR3M|A_D4?|C2X|Z1E5Kn zQu&IrxGc6yim~Oz1)+ap(Ql@v8{3M!M=*miU5eI2XG*k*{1T6I%Cq`yf)mQubva#d z4O^80P)Va~h4m~*Vlb|umKx6_kT_hdlCez-jwC}zYG6E>(Tu*oWC{IDzg;Ze%9eO} zY@r$d9W-NX!?C@+8aU^%YUsl!PZwy{So&7is8^XjjM>j^?^m!GEnoUlPA~Mf8cW}z zX{wVCH&Op^&LdPXLjBq_PcJUwxLuyz?O% zLL0V^>*kTL3T3_5i{)OGwXetBk3BZA4OX$z=``eeLAfek?%DgjZmdnQ%&#(u(Au=G zzs5RpBX(&0(cn#M$&MPjJyNsXN&wt{QBAnMC3oSgra0COMVy0 zy7h26nFr`=_Q`q^+<T!a zDkz%&KaJ;Z;LEU}7pE#;WOTBydVYX+Y(BFO9vYcgDm)d_f^s0_$6mB)PZ2dxAo5QH z{ulT$sPW$^ZZ_I)X6@ITqQR6vGo6e45sK)oe4;uKn~Z~E!Z+1goo8D8XTm9!JhaIx zy*(I@rPWMIQ=@U1Q&PE?^fofwlR+z|q2TMuGeL?%}9Bx0i{OBtPtV15GmtPRe58}h)nyi@}?Z-IkaP+X0*b~RN{f1@tc+iJsf0QC}b zw4L4Xg&G(~+(d^`bUSK;u0VHA`g~pJ-dr1I1-f(6owW#D4c$$(L03Wd&AQONqc+S6 zbl0D4UkP>P+Mp}Y%}QUW3*BwCVOF4JHWhU4iZk(wFK& zcW-T&73jVoZCwd<_tyqp1>LXJh3=u+Fe}jQINP}r>K?8Qn$Jd$7SE_`Haa+Kwtdo= z=rED;=@Z%_pHWL@q47GvmrZsDvzs2=Wq^--kxdnmFHtb4H=jannCb>gDtT;~O4gta zy<>aJ=2WZg*p!up25BO15Og{6Caa_xS)#S; zDkn?U+*1!((zPL0kmZiODYu9(@$Z0s$ueP!G+P&z-^#;0SCtqQfNw$b*Vobf3wglHHh=xmUG+5o zo!Ss9u)Gt^uLsNTiG?{=h5Gdl+{yLdt582=M>r#`|BT;)l*mhJh+m2vS`mIP3yLl(ygxc>|IGci)ham z+|QKNQ&)Qqt)!l}(4Mc1ezmNgy4o|ol6w9K?K%JH+wUx;TV3sWVkPzbC$#6?FZ;^s zsjEG~CF-eI8=JmknycR?IUHRAHeVdW+&ok!bec`G(jOKo;RfI;CdA=HJh$QXlDmtG5<*+;aL9TxD&EB-aPE_DL~2RqMXI~%Kw67U;m9X;FxgoT=GOg} zaG8$!rze7b+<5XCa(jhSGCP@TgGpq-R=|o^x|iKsbc_|`cNmqOOD)PNKAKvTjAQf1 zQ3>s$0(oq=vpm#x%&`*(M!S2+XSjyr-DfR&Th#yb431JT*X|rUPf7Raz;8D>bIftP z!fD&myqbLVPBiOn#ZJ{Z{dlQQw{gvojO&nOI1-bCI8N#<>DX<&vr}f?-e8yfd&2+v z&F0nP(DyQggzqPL@BF;i|A=rlEcM{#9-ckqW-t2OaNkZO>4-8p_#1~m;J!_?E%;6$-VIK9(w_rXa64oYBSGpap>p+e_`2Oiyh=m1VB?^A#M!1#W3 z=@{FZQHL9&Yuvl)IRru+qVhvV>fD80Zgv?g*ZmW;R@!lIqg0&oJsY?bqE)0?o< z%{IcVxI4{{k>kfczZJI&;D!PC06d4ibhz!Ma>ua=?hjcb>i`8i=$UjP!(0N^gkwCG z!F_f(&9zH6(Cc%#J@7g=(5rE4Rw!{TugYqi#jke6chz}y;m$QWhFdXKoZeoS!-X@6 z#<@AYJ@4^hT)@~VZfWF-O*xnAO_20(#7PIklQ^)sOE=jW1M1E2tAg9pNU;N!-U?5Y zonw24cki_0BAFsztaWsHo=$|3s|sV#Ve4AD%}wvGLv0hzDLYb`L;^?qc7y^?u{%We znBiIyh$m*kxP|bavvU&2GLBgBQ)XypkNi|d%V^9ohaV3^q7pjIUR6aNuEfw%0nD7n z?US}icI2Oseuf=6qnGdxw^q_+BABZOpt32ki5WPCdu)V_ zKbiE;l$XL?B~NQmF6F!Br?q4p>O{csABLKWn@*qrzy}DoF~}i5+<@aTT&fdAIot=O zkP=nby_%dpOtHRG-=wN^TruE}#baUUxIiGN;@I|dPKV}8rwoq;qABi}NB1P)s39J4 zS{g1#2`!MOMFN_JD}z$D@HF*2T{^;JD!--BHhJGO6S&3lIXaEc9GHlT&zyPgKD6`+ zrbpk0HJER+zK$V7wbS*7#sM`(;aX_g!s6iLt-i0o8AwYXg|pwU(UC_MXB6(<$OS5S|YHb;;KfvO|zk0P_K_e@505WOwY#!mdjidxE@~eIH>J$;NB*sNM6z; z%hKr(w(xPkV$miwlW2>`;Jo2I=Ez+LSEYE0Y$FVZ%#bxZ7@LuZ1EW=wHyC>AR3g(O z!##0|9}4A*iz9gZS(J_8|t*|BpggK!kaY7_=aWo{3 zOCMnXAS1XP0CmvpK$3re2mG)#Z~xczb)*4`Q|&qZ9!`Y_Fp^3oRD;aOS94&=gEVZTz$zkcTRfqJo&tm zb1FHLW~H~zlb0sBQ8P*uOQR6nmgJW#CVTWooIvwaBERhJd^vsx!jD zrCvr}JN{LzLWtqpmU*8nX;}K@=GRCkesJ6Qgx-|)PqCvVg&XCIXN3HLH9PK;veW%t z;`%Oj431Abl+ou@@)tPhoAUM&p_tN9Fj}VLu6tuL8Xl@T5z*%yH1OhnI zq+93$REn6mdGYuUvqvd`M&KG)1QkzQg%JfdeQOn_dM;;n_ z>lx{LywF+bm+j)_!ZS|S*~_hMeXemAK5dkK78~*X{#-iT(zT3 zyrWILqfHUs(ao%89Is>jd^5OlO=Uyt?$cMjCv6eKIZD26L@#m1np#+T;tnwf3=NmQIQP-WkHgPogw`+%w|eEa|x z1CYJ@c#k{)3uHl<Xxju_SsYS?EX{6Qn9j`>9m3!-Hab3D)#4F( z9V5~w@zt9Yc@Ix}F0@M0Msp%$?YODMG&{)Vu$Jr=!^p&*O@kh$>W)h0QH%K!e%Dlq zT(Z>ChfOE?{|5nT$*o~1PU8=XcLUyyZWxfQKD+rqNCvg;TsUD-DA zyk3MGL`Y^TN$Sc1E()NMR>n}-y^~YM5u`R21U8uEbPwi5zj*++$+L;B78`M z4~uZW2oH$xpa^{;JS4(l5st9%()}R)ph&vDRgdmM)sJv29+sYxS|khptM+Gl|7T!T zsb(3|;lH^${=9gBRojyNK)RX5N`Fk}?q`S@`IkObh5hp*1LI&cx?F)rq!#m@@9L;$ ze~WqOy9@uM{@~JWEbX~Bd}E_Lirm@9D4Wm`*%W1Im>rEYissXP-h!dxhrzWgyIx8l z=e?F}r$jdAuIx?H3$u8u6@=$(ifsIu&c<2TLW99#aTYe3_uJ@<&SC5 zYVg;kChFPICUezD<*O{k^uFVN5d)XYKa>0%%U_OuB>akdv@2Uh_!V_mS6PQIDg4mi zUD=PR@awE#>+2+|`iHNd+``&?$&Tm;k{1s@&*^Kz_y8d8Onfo& zLkL>ml^ZcGv=&?gr-W_b<72+~6qzp`TFQKpdMce(r%%-|Uz|?hQ;#qmV13^(Oc)rM;cZ@N+_>%^ayKjxZhS{rDgKOEKBLz|ABhs@pdwum{la;HpUEn* zLYTM}LdHavw?drg5wlvu3em7abY(wW14zRP@o}|6T)GP@uEAgSnDK8G|FVO>oI88? z|NZUrcT1!9OLHeid2|-hBZcUg5xw+lP(Zo;V(!E^OT9FRvPX>qqsPhhu2mYmi+-gM z_U~K^f}$r%rAL=3U!cyUdC4x#OR}_n#ATL7<^hBKJJ*8X-O{9PX#-_0PywD}N9l9j zco#_oWJ3YSxfUeasW4y*Kq&C;JTma|lk22+TQ5rQKG}L%TDV(UKzyN7L|<-MXgw!= zPFk3gA7xZP`OcSH)}a*7T3B2xd){1{{n|xoaUReX7hjd;FG{Z<`a3`cbu1kP`1y0v z;%hIrp!RwGw6M&h6}2qR$-mC~Yh0R>A2YgcPJX-){f$EOSU!3=AAQk`Ca^w!xdpA? zN-e!;U1+_4e>_BhC-H!e{<-<06~V&J3oQ7yS(2^)g7eA3Cu6TlpxQ#GU3!dh4`&w8 zzLiQ5Z!F+nD@!QjoNntnBY!!5LiJDLZ=JN5Hc6WnzHJ8GQLXc`1e#EPf#Qqk;q&Vx zM5%ul->w;D?Pfi@fHLA~k-B_Ek9jf`kL*OF_7~%TKV5u%RRovW$YSfV?$H zsDnh*TkjUANJQiRXmrZf-z+LJQkBve37mkpgKQ?r3F^IL29fqJEWVe$U}fw;G~9`_ zcNP}sFBX!{??=JK#hVz7LD3Yi89 zjNpn_RJlOC1fIW%9%vveuuduJpQ0#@_;b?vJi`|~8G=t_F7+t-hq{7y41V+dvM4%c z?(&nY>)zX1DMB{1N)qJ9`BKr*1tWfGd?F-o_Y`?XbK}F;X{Ufh!Z1;y_Vckc`tB-D3>o z!sJ9_^g6KD++{!O3@CGg2WxTN)vy^4|77>RK z!S6g$7E2!?1{d5Gt{;|_nZIbAzhLFRY{tct5rTj;A!fN&@q17?pj>=A%Vr{yx3Acx zxw8|D0FUw1dGLPG6h$*q(W5aNu~Lx6MWrbU=thYVV0n>EgI3aizsWS=H?j~GXerC1 zb@WgcF6FW_9{6Ue`64=-76cz88Dyc&nkaAjUx%_QjV za5a_or&9xE&)kEL9Ql2t5fcJ^vp8SaIDCdnq;Y_8`}UKkgDFO*&<%9Mcixf9)#p_l zRSuuI-F-!ua4zHFW#%4UvpA zo%^yOUn`H7j>tA~1L|d{26+SOolHN4O}_Nc)G(lS zW$$J*>CSFrVORDZ5#B4p`$X8ds4H8agU&TPBHJ*a7B1>)7*MgzQ{y7Rh5?n`3RrSW zVdJ8%U*+oS`drl2Frc!@`9_<(8wS*diy$#}SHpnXl^quPH@|@Pnz{%!45+J?#cl`# zs>%K-ana^n$wk(t_gC*lm^9jMbnE=3@BPZH^Oqj2$Hn==FG>vb)m@yg+CYDKW5Yn- zFwi#)^bG@j!$42p0h;!QW9%N$G(G`X@|jV0zujB)5=)QWZdbhA_ZeT{+z7E^}zCSAIZLB^c zVZIsP-;S6>s16O$0ZI)ce*HHo)TJ-%EN zPv(5H`r1$!%>*ziK+R7Dr~N8#2k2ndZ^V6T@##b+jjzR?Xh9DF^QcN(!@eL(95s15-}vEZo8B7mC;SuP zXgEESJ2?CLP)J`_NO3YA3~CdZDK(LdCvwBzZ`RvcYl^{qVL@-n#KJ&|8a?0A_b0#g zUp7B*-=B!i7vH)q$23rcX{Kz{lzd7oQaXD29E!u?bry}D$Q?NL1qkn1%Elc&fZNQz zs+EiUSLESiHa|i@gRME3o=IrA$dBOSg|F^DjgRYEGLgSQE{18l)8%s79nLF<=%02;#L9^~YU z99)dR;9>*@7b7URSTmHpIWklxzv+~;Lke>acON6?cn{AIkY`vx9GD3TzyL=!fdgm) z*f*0CyCm(AjNk+t;p-V3OxO%Igc+in-*hPnF!D{7StQGo!JZ-P%>cBKA;WWX;Gzv2 zG8x|9jKEm*g+>4yTm>Ug1^=ctPWg<$4xEaZfQnWgI>rH1)QM9OGf+|UrDq)6a7C$l E06YKS<^TWy literal 60241 zcmeHwYj7M_c3#f_Bmq7_>WS4#?&Yqe0R=W@rsv5LWq=?hQA2>lgNhWI+iJ}8V5T`Q z&VvM`RYGD{b~e@PFxJ@KO=4S>)UM)k*(+@-&aRUxDZA@hr&6gBEB=vGrBbR?@++w= z|8N{7cA|XeKIVlU03=9>vU-Zd?Y@t5&%J#g=bU@axz`4N{n`Jnu;0aIz!|CMORLLO zGo$6KLdB}qn$E#<)p9mpu;$IuvUTc;RjN1NY>r)TzK-lWGuD-S+G;w3=NC@RBPf|g ztC{mTyBDiwI$v5&S*z{bdo%f3rC_caESEE67&)_+GYpiwWUbbkW6nU;syAmHe|@PL z2sq;txpL8(xNN>^7ADf=j5Q&#>v*tAwQ^OftTt!f-4Bpgt7ZjwH=Xd+d_AY8D-EN< z@ej073S>)~!LTrlv2pben7^~_l6OCGRqL5#jsI1m=<&v(2!V?XkOdu!HX3b(fGUIm+ z7-_U%5z#Xia%P&&!}+XQWY2*yHD6OpZtpRiFa@xp);IAT_pYvbyXBbTmbY8?H?H$=bI$k!SEsaJDfa`sY% zfvQz&6zaUjCsT{LuWY5B+>_gd;5@M*rf6QtVuLUBXKtuE8-J&wnJ=;LH0pDP0E@m7 z#=?`V-+7BMCaC@MydvsOmPm2DGxStG_n>ot7ZD%=JZUub+s@F_#d4-mu%2l;2juwB zs9u(*-fS#6{#w21?8;c_e97;mi6H`ANO+*sms2-0d;t4GpPBhW6aB_jGNI|w z_2bGJTs%Gdax*vN9DFfvT@~zx`W)D(T(h@=L>QZEP zUEhF2NCy=wK=o9Uj6{Q4BpG6jTi1Yn*?h??|0K+M>P?Q$75PD8g==G zi6FkM7ESWB-Y#fU@y`8_2&AoAP1V&CjcgWD0<{zZC1pS2QN? z!vw|yeVsT zD%31Bd-%0%>2(!DLw&8g@au=i)k_))9yq&nXl9SkIaEvMEMk}p5+4{>Op<`iFXfjF zRC8%uU9!>;Nz{y@Ss~U;jyj`WW(;uP{Mm{|IRi&7tY~N)t9V7tmsG~MjgLK5sMRly z$r`^+98e0aSe&1lJ~eY{aY}t!4XHrZpqG0C-t*3?a=zroAE(Qu zy0ub=c!Ehf=A0dQ;**=s7bN11Qrbl>0G9A>b;trb#K0WGU~M|j2s9uksR#H79#>Z& z)nOeL7^4rT@yufymXk)+GBB0OX>>+LFxPhv)wItUM7NtwCnfOYau|*furo3ItYW#k zdIbwM-dQtPo=knUVrfex{>c_;8jt{|mNP4s92j@2f7g7@&U(3!Ct1QHooG`_fFn<9 z_1zC>+>at3CQ-jpJeqnK{JPHCVAg6@wSETRPhD=9g#e^wEs*2tg{T%BSL5_2sRg-+ z%;VheMb0W74{C|98v%jMClu1+3HQwh>vChMR?pWPb+%Q~p3}1lCmxdkht%=IkR>&!@Q?*b{`oy}l9H z5yJ8_wGG%wLfP8_?ttiZ+ou=zpzs5$bTFyK^nR;!LIcy|&VmgUaRQV#D#hM~UynrN zS|sYat3)C}@V&7<+}A;39`iU$0Y!baic_a`n@S`CSxrzJZ1k{TF+C9lPn~o}cMs@x zfaNrJHzS>hfl=0CabB_qYCIGUYhgDDBQbUlgSM_UieROQ0|8qB4rJNNXIFbW1z?gb zhJRfZ*eUx~v7c>q{Q-xgAxIfq8OV4t5<$Dd?me=X_>g}1#v;L>78ae^i*FKDhN6AI zK2}>TE`jt+KUD?imbQ#^xlk}GHS4L2g{_l!%!oY6N+d8+Knl1}2}wO1jB1fcpTV-P zkT2%z#HhPaA1E6RX+d#7){TEsj{^T_Y`yR$N1t>bLUzr_Fl58C99y~JU?IW1&ePP z(c-SsoFo>2IOG=YWS;#oHjf@06JgMWvAElKOX6EX;w2W7UV()xuxJ#sBI34GB$J_F zOw+^t1VF5%DsOi0GLgiQR6+}fJ65&UfJ-LBSe|tE1`DBEig#v*(CvU6ugRpY05rA%;$b-9dNI%lFmGl`_O_b(M(|n2@A{ zc6Op$5CK`G<$TGSkn#v7GC1TMNMr3J*@KYBG@ZyswDw^aL=KN}l@2@R2hrJZ?k|+D zT2+Q45%C4oYC;+i;Sb|tC@ z*H;*|&rALLL6YXM4GJ<8i!`WRaLYN~8EoQ8&+K%9|D0Sr+#8qSJ z(PvbIUn4yh2AI%kl;{A9D;rdbV~~w7NP!h0ROm+V;tqCno6V#3tvXLYti_o3N|Dq| z2s{b40wwFJQLCnn(y`+rS(af*=vTwe$0TV^*!PIwI;DpP1dBQ)>ljxd2!$}TP(#Vho7F#-n2K5p9NBrj{JXtX(>9sv~8d?0|raiFWzB$fz4THHs-cd%ks&0>u} z_Yw3ZlL=_=g;Mle z2&p+VH2K+r)X*Fo=+MyQg2A_iVDOXIxAf$UOtX^jnx1Biv_th4eor}aL;K-7pj&hA zDMu!tuz~aEmW8rIA<)16(OZ9H6{M=4wU~1+h*>ntAD^|Mc3_#1hnR?tA zz$p*2+aH1fxXF}^$DAQ~wu2OB5L{2K2~Pkie@d|c;m9KwbiJ4^S5{U23x&to18Pv8 zKB}I3dGYzPv+5%kf<@R4Loc_|fDQA-W=b)go%xb{^O+4;_2b-p(h(=)aXlGJz`h|Y z)gtUP_9kgJY-jP38?u)X*g5=3Jz?h&!0CW}3E`MsU~hH_0iipP^4%)B_GMNUnA$b| z1T5_a9@{3Rz%m|!ZF4f=E@c+l8z-Q0txhd0b}^#yXc(#nkY2m6b1W^c!)TpwxdU1|M!QciH z>se5O8w*9QHp@cU4_(d9_!FdrwdBjk%AZ$jBb zDEjnb3{e3ze>-hj+MN{F^^g|s)5!;Fc^uI~dx~p2t;0DE)iMYoT)Q})z;uoD{d~1t zK3i7?aLNSczO`ZU%kBneIBTV zxTvh5PH4o-e=td)vc%E0)d^LQs7 zg~n9`zAIcOhz@PAsFsYmQ@hSV*l+WWb(Pc*>{vJgH8)?^L2 zEfFb{t;0E%KuZM`-U~Mg6*AQ8N})S@2N+^3E_bH8ntvQpS35*7aCiizVF#8Y=a(ao##{7&;}EpfygIX|Jo` zJ{rpccE90_??^lDdi6IzkB2Zj6JkwS2Yo->Wk9>5T*JZyh0NZ5!3@NLy4m5^lf<(wb`ov1!63m&gE(c6?_5npw>^ z;aq61bbUm|X^20I)Hj={{mdWB{#GKJMonkO$wgR#QNb5J#xJEfgICJ=OmjBno2|ox zi+y2GQ3Cc?@i!GC5Ox-iov{zN#@@$bP2_ck&ef}}T2lK`slcABc7 z1iMVIEBGU8X8?w$GMVZ^1;($`5FbHK`BDSD=nb(dS5SH^rOd9wKEje&U971y>HwJ= zVo7mrY?5Wa&Y%r&Xm5a4wO>YzGc>i5uZfOJ4N}fuZt^@7uB%#Qu&O92I<7#E9q+Hw zJICB7ihm1Hy}&jo=65UBD&KmryTHEDxhm0I(g&R)_3CyhQ(1e9eTg`_yi^uyt$YZ!3gGl;b2XB$f#(_fa> zzn`jX=~krZPGsvkszRNgsi>|vyJ?<7r=6L$K3%CUA5+DuvQV+oq|Mh}2LdoP9am2> zZ)q2})3l3Fs-%U@s&WC)4>2@S*Pv8dFpEnW6CRt4!1;4#JrKZL*A`CCrp8rzIyZmz zx%sIXSe9|~Fj8|oj5l74R7b}t>KLn@qZdHrh2TYvR}2+S(u^D%XUdMBsLBaz5|$W@(Ab5CEwOEJeN@(!tPnA)To>17kbew&`VL#? zCcFdW0aYGPX_ay%5FFDO)Rrx(XZ8kr22a^wJNhb~5}|bMcLzjs+f^dG_fB=dcNA+IMe0ZkEzkNRSmZ5-p{lC-=6Esdr!a}kLh5`aJspkWs8R)J_5_;rt;G@$nK=y zE6=s#J?7dmj$sgnfKk^wmPyHevowh8f>M<09B4DW!U^#qkl*B~n3LJ-4@IWWod;`N z&*ZbP3Y~O(K{gVd9pKf$O@oJNuWin5oNpXkI5{(QEQo`Y4(@&rxpN?!DR#s{ZX#xs z5prniB)7mwfIJ1y1Z%fOr2@0LiCp?J^L=#C5C1fD%94lU-M_~)vydchk2u{6>U~_j z+^`xJW81j(6e6|URaVpxX>i?R)zYy4ksg=>S>}U+kbaHP%~S~g;klWdAVID9I1IA1 z($Xq;IdBqI$y|aV+i`Fj&z?SWit!0sGUb!4Lw6cxF~eH+KL+5Y^_#3(;X?=RnnhEk zeOX$#<{1}?Wz4Y+VPjUS=4x*)I9Phs8iSeGm(^FTY8iHIX(<8*(*XBO5J;h$$t_jR zgt}e11P5US0`%Fa`v=~S-hyVzS_9@q`^I2S=A@URLJuozfEgmC_^-WgP$6B36WAT zYUqgo{9uW$WT!)c!^aMfffsE%#zJ*)bj@4%DKMk&ir$T&Z60{41GpEnV zwq}2y_T5|F((E5lF>gcL$eXdZp`BpP&Ah`v?=TP)C%@&(^};Xj0wbcA7tu*Ip+|5l z8yXJ$phnvv7+Tw~!d}QUjW^ z<0>?cmW5v{#wrC-`fIE-q~)YTBBYoVA$EkC4VSqGDw_1)#;*PYLq+bRD<+o#;zAYM4M|{wSPoHzY8#~z)gFQ7Q+hfLK?OgbWLDa zJVMjRzN+J*t`9xR|uZm3XDOVBkl?2ZyLs_iVXQCWczm@uAy|UW*|5nei zevuIYQG*C&eu<~7cMj!bS4o_<5>j3CKOy?7H*`e24COc5g)&!=rXqw!!*l1(E6UgZ z0EhRvnu*Ji$jNi<@Be4z(04UD$$Rq~f1ttQ1CDHJMvYYWzqec{FPVk6wQ@FFv+Bxs z+2Q=q?`d!r3O$8O(8T|Jo<$t-ZvtRwy8uQu1@I$nxms>ijGDDfmX$^}zk zkhK^MZ6)HUf4MaD&)3CM_J@C0gNL2v2I$d$dKOas=`-r|yR9hW3J&{W9Q12`<%hU& zcf}y1J>zP*dP!6d2m{|`S6uxQSbzLWty0ZjVIGgy+{Ed70Hj^Y{7@s`wzv$9RR{0SthuPq_s=w__}7fgDwGuRX}Z`ced)*eSS{!S z@A8*U=--E1-78kMN)`gtZuLi6yBW}(p{u~kRZ?THl;0O4r;bjv>VJf{<#g#N0{V|3 zb@b?Rb|n-JhU5^Fz2p<)6T{H;NAu&*07k1=A6^otYA?kryA?UV z<@};h{{I4=2Pc&gEP{TD`%nM%SN0_S&=bTbyrgk|nc!-qwVNwNO)FK}S;8%P`y6WOP{@t!iJ{5hgN zQdH&7QT(GdMF}d(&)%c7oyv}%Qyckj$IoTE6y^Q*d^`L*2e0k?xuRSX>DS&HQ2y{e zMR^DRf5-d(8R!GKXZ(p64|~Lyl;cTE4`oF>2r@vC+tUC1e;=2rNI;|uFv>YuP{5PH z8JLh*)nlXur=B@;QiW<~Pz$Poi9#6{QdEtKS1)k?8O3RH&B7ebEo zB@?dSmyvE6bXUW&k%z64ft8D{TQkZvssC1+z}e?Sp>U;m6zh!*g(~)ghfp-S$dBcw4Fl0u0jJ#+&;a;w zindg%Db!uGkt(za)y*Y@Ie_3#JfsD3D~qIY^5xm7nbXtVO~0)->Mx>g9%FA_0 z?%S;h30OktK_Dv1QXAM$A}l&e($8{v64BVq`R^G76er79M0rX7-=GiN@$0ExnL%1t z`2^1_%lGE*eV<2mio*un^^@i4nyk*Be3jt^B=&X@<$q_Gms;V00@jb^lkJz|FO7KF zc9E3@0kQBmIwSEc>y^<(Jh+wrEm1xsGm5acJh_sAHf!_m?o_S;xathl&1G2Hb_%sS zSy^t7F7%ajILCnZfKZ3szvP)IDTa%BrB?{&;g%Y>5ZyApSx}oz=YZs-XQo}bXlIZZ z93bWFU?zg(_s0Gej96#*+|2ZXN!K@YTF&lYm@`hCUwGM=etG)LsRihgFG0oUQq%D- z%)umkC7p%^volEd(b9t-%})&?Fq_q zn3`TZ{o<+HYVg2jRqN}jen3`zVrn*JOrM>7_RQ(&#oMZVaI@MY>uNvn-290de`uf0KlBb(b85(D-}!!wE(UkQ@ip|^neepn{AtCEYc1<}K|TI?xci{^>ePSLZF-OB5w zS9+JdTPgk7&+E29efC@BwZQ9v&qojZ@Mn^I{%p1PlU{YVTi=~7FX72>v#+<+e4D}X z(!Cz=dcf-euLryy@Or@OfzNsmG#xmmwbBL?P5L@TlN!5`w;FK7hJ8gGd&!}jSHQw{X3F$_}~{;=HbMN9fOUC z1GI&?#YNW>G~1z5C54{`DE5!>cH%7@MIQ85azi8c4g0Q5G8$oc?Xj*CMxXk}OeQg> z_~!=wH}Q9;%frpVegbCSpZz*Zy4~_~mz3ePkCpp@oNtn!?8pX{6Z&dr+-vA8q;;>Z7Pjz1#x+Dnv7wTonWWO0Ovh@;0= z+(*M}dzIfnzmBf$Q&gnd*o(X}as;sERcicseEB5fBt3?5w@VxMtrZJ33lYh2C9_u)~N^|JEvARR>C z{9_dPTP(`=4#43@-AiZsHv{ak=k%Rr>G$_jCTPN zb<8>bn*;Ra<_6p!M81B3)K>HbpAz7&GsT9}YuBiwuK8|cNC9!vjvMXyf}?A6p0D7$ zkp2jT-QQ`n=L3(4Z#260phAwm$JQQV;piIw)T3*o;{6LOOp1AUdhM`)|78}At$jtj zKQ6*2L`Zrdga;o*qiay5V(&*p_>_P@F2bWC)I@!gA|D?GV{0eG`?Ls8itv;OpB3SA zB79zir$zXJ2vZ_FBSO;GAUsL09i!|Jx_*+LTI2bXs(N(oEY+*by`WrHPAhB}u!zn- zzDeF**((tRHuXCQ~yy64B- zcfF~xHp#k%t+#mkq{ipHJ~ddU_+H`jzwi7A{*{k1Zo2;8ynuoB7=w8;$fgtR2Ol+Y zHx=B~;p&0EWB(LHOX~KS|F?$P#GhGn_}-Tv zn%#BtymE6Eq!Z`ouip@{A?5nW|6&>Dt|`}Vv>!icJ$`$ivU`O7?NLf!4gBiOvy1>E zDZuq}AMv7Jfj9$_0gO!L`UgDj=4(j)-z~@=v>(s?pLT5Paqe%zdihb*;>W-5|IYPu zf5|g_834s2@EiH7R?!=+$GH_0-J?AJtJg>VFH24wn)JVSQQ3V14dO93UsdMf3Irj( z5wd7#b7Z{|36?1S%za}Mp->;GgoNQqCp_uI+T9V1c+v?>GAZ5gq!S;Hu6cqxAv4_4 z)L={WU6xMxn)Y67h})J-TpRWz6Dd#GpizM(uaH!5=QW<+X3xzfJc1iDo|{Y0%_Xd6 z`2{;$aBg*rk_mXe!evfO{J`$2Zc8TgU?il4xd+TmB@=!wnRp*3%OsfyK(FBD`%J9Q z&HyCWM4|8fpKk43ptDfN^S1R0N{q$ruUGKa+}7~hRIlLXi+4@0;7>lsdIiC)>J_}@ zu2=BQJ994UQ#`!_(r574la@-ume`YR>lKvl%=qnCQo9}{=iL?A@v_x)-#eaOfycQ^ zZJ#Yo4K~U)<=j8`%dPFtH^0gV5CYnvSKxbvt0zg3cE`x(kDRmr=5}=s-uHA46pynv zJkB1I)p^;sN#y|zSo58!l;x2LadMFWd^=Pd_A0g7Z+hSABet>iK zA9ix~zuE{V%`T@ZcT+!6aRepITdC@6e5&m$_wricH6YuniBLD~583IeI zb<3<)%~dYd5k7wSosh7>O8e{xt%|8mFyhvwz?jxm8Qz(=l-mK;S1XoA-rBWnu`aN! z<;)7dX7euIJm@(r@f?=aaXD7L#BR}DcXuAHcZnzBa#iWpA&SKkx~_#aob*q-yLATg!I>&0q0=4VN`HsO)X!{7;J(yQ$x;f z_R6l@YNoC^d)O;wOE<`!{~nf+-@irK@24xiYK3wg7o(-CW<8x_8CB;HCC~+0xXngX z(oBsw54LOL304Vr;o!mmzq7xP0TB4Q!gV;trKM(S(%DsN6b*5|z#-hA1&?KPzm~jX ztog#f{!^dQfV{P{7rIv7KD`mv!yUb0FG3OvC~5exkbUXwJoobA^Jiz9xglo? z4(agvl1D%txFbU=NYkzkV)^mcUTvlZ8g>JT=>3D?Xd;pbNA-9l86=&rurx>Oti)bM z#knCnhk!%(N$9psaRl1FgeO`FH~+YT&JrS>p#5709oLhw1Z``PmJwYS*6TnI z+zRM;C;?G8Fbps)xd`Ar!Y) z_6!aA)xO^PYI02Uq8JZ{f;w*PNVcY{TR*npNEFw@(J_&=n6P*3#YcRPHXQjqT;Sl= YCM}xWaLBab$RXnfM~9MQ3y!7ze`^0iV*mgE diff --git a/extra/sqtt/examples/gfx1200/profile_plus_run_1.pkl b/extra/sqtt/examples/gfx1200/profile_plus_run_1.pkl index 9dc3195fcb480d879cb10ad3ade7e8b03115b0e8..bc28d68d1a58d2593ba02852d63bb35a87bc380a 100644 GIT binary patch delta 752 zcmX?njrm(Iqd^19)YondU=TY+qldpFGp{neC^1DZCABOwIdw{pa6nOhT4qivkOGS5 zPSlqWVs`X(nbO0OmzWEbXLgM@p0ask`BuKkTiVQ-GT0}7XcJ+|;G4|ZK7lDieDb_@ ziTVub9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdvd>I%sltF|_k5Ex+dT?NTfNPMOvwx^x zh-;AHl+vUOam~glwNo@QIC^-2vYvjyA;Iy%jy|DN2$;(RHFt7ehd9V7T^+Ix1hg>Y zc8W3BDU(?{6$tr<1-Bt4sQ%%14|4L2cMtW9cXST%408qfhy^5u99E3Luwn#;6(cCD zSTmHpnK?34CO_zutVaqN4tF0T=Xej#5Qq^#4$KGzU~nTF!2vV^65Jxd;09|$3UC8d z2XcbteLaJNv6w54W-uGn;LWGI%mobF=HB4IDBV-rkJB yNcDwADjQq{BTxnNrZ!IbjKB_@ikN_kW*s`l0aVn9QxP*zQOTue9Nch4sd@mY$=%)n literal 60227 zcmeHwYmgk*bzaZx0@%fK0gx0|6zLW$lC$8Vcis<#pm(tVNpP?r_5p@07;5&;^z2My zc4jc$1MHGw0<1(kaJ9NSV7%dDg%TII4ODiud^Qbj~k*-oWW6;%8qsYS*Qy*YRLh>}p-Btu|-g-w%-2>P8KCH{Iy9l2cT3wT52f z_=nmkg$fl-XISX^*tmLxkUuhZyy-MU#Y5Tm3A0kA6?HV*FlGYBAG~K#M0pYX{qu7>lR0 zL_EWYt$3;@EfLltVV>5X5Zzt6ZshZrt?nH``t)@-6Z8W~U#W~16ExIIU-490OC{5r zOmyxpxeM{>>l0nRAISMD5zc7wFsGqj`btEiS|YX?eHFju)0IzGz3XBkmeewtK6Dks zoaQlaxq}O5&b`(w4!H+kDVf&AQzag!7x@g@u0_+lEE~`J zB=WIvEYSz%-nt1+IA^Mhje_Wp&SI&gM=xVh!WT$H`}D>z`-0Q!QH(_*;W)PEDKD6j zNGz$&B%eD9J?n96Q!+M)(w@i&>1H=ip$j-B*v&y;QF(wMCA zo5c5I(Tat+sp-=*rx&KwXVr)bWOaHuHMKCMEtyUabFK=uy3oZ?6KgHuej(uQs+){Z zo9-)M6?NIJ&`~Z|&}2lrp`oK_g(w9{NWQ8!YI(ykb#$*|pXhmKRk>gE;)!$Bies)g z5J@oU#;m)eM7(j+{k%lHQOSA80>B8~uMU|QzkB zcxTOEdGgL`&D0i2%l5`tFBB z_oK*%LDY|mN3)NDOOLTO7`AQJowERc`bxtnhae$qfgD#aCbaOlnxa1$EzHGYKEM7T z@>7{iM$5!K7c?wB_;@Mr#Rv0pW6^d>PQzi#B&|6;i*GCv1+$indIL`+Q(C0Yo}^=C z+ryRXuS83X(sb0u{o9;6k36^O&B8f6>t6CD}nY{t)i&!Szs3*u! z#{x615`+^+1Wrfp_Za>{zyN~OIHHHeZv({0n=aXCwtO>&4L0<^0IAQszS_Z5P zq3kaK_dwvf?GuX!P~?GG8mwtD)@PPZfj5cv;T(sSaq3gjYsf!;Or;i$v&ndnJ8HIxHsp#m$;CA_$HGH$PB!@1Yof-i1Y#b zguS}Fh>d6NnL7BkoT=xk<+4$;&1do^7S7&|Nz+$Lr1{}hci+cjv2a3*2qb#hUw2=* zv|MtCNB7u#plmd(Wjq@Yk0o(Rk<56UL@^fr&{vcF%5N3ql~X zvQ(;=6H+3^{2C?mIxDS=9*UUP@k+$(= z)M`K;5a$ozMSH3Y_4Z(>0Lg_@s#drJ2S@6VG(olHxn_32-4D@F4azZjunA7C*mQ@V zT`t*>1|b^Fj%t6Jegc{i$vZx(`twNTnhES+PlAoC`@VR=fppoZPd$iY@g%Y zE(Fp->)8Qj65tIo8P6?yCV|7cOqi$jZh$&=DTny{NSk3fqjhXj_%KKldHr>Ldk2`lEV}+c9ZLGLX#Y6!?iH=P*7YQ$ zHz~1p_XTE1!uZp}-DC=RtAs1OOUa(AyJ!VnB=VWBR# z!ER?iR}WC3RmAiz+J~8Zr}Hjfq2@Q->2$-OT~2H31Qw8|>Y|2_+*EalEPQUWZr_DQ zjekPgvO(Hh%_$e~S0#T{y8Bfr^Z=faCc<$X7vV_LbA5@YVp^(?)@KL5`|!LFnnOjC zA1+80&9Q+F6-_P|d{4*)KYoQvU(CoL372-HQfQIxi_;3MoR# zz{+89lJ>$@0WZ)ASVCYG@hAI~RYCw)1J-4PV^*2HSrr6???5(ir}$b|SXrQI+58D; zS`9pQp_zz6%A6ERBfWf=yN@YwoP^S~IyJx0$%w|nU}F;zG%>tR<$$Lp5*pIIbu12X ziH!G0+1iF7m- zhKB*6!@CadUalv>^nAM1Bp6Dm2=ooS`+^KwU^*j7p~G&Rlw?N81w+Yz!imuK>BTq( ztqv{RX?ep{8K~NLJQ~q3fV{i;fL+2NEp({3&eJ-i;~0_9YFf7)V~{gM81a4|&@Pq= zjv5M&k6{}|<^#fDh>H>kKq3QqLFZPlos^6~Z$wnPuABE2$_D&92nDd-&Pb-?VJ$7R z-a2FmRAVv|jX`Io4_S;rmjikfQ0C~<<*8&J{W9?TDg#L*`1&sN`XP z(Q&>*7-guTDQ{xSpo^e9(}yY!!f>1SYqzMT{pm9)SUY7>>y&3$y&w#XOm7VrcV8VQ z7FWrfh_uSq;Y?DRaNHYi2D*1>lsCEUA)J?LjeKc&9jQS(9>Y1>anjWyhjcg|jcffr z+u@@}Ro)jy-d0289%YQ9+z9~9T`v$rPo%?aNPC&e)7aYszb*bS~>3-^Tk1$mn6?zKLW6YZ?Dzjbb?GgY<Wp|A({i7Qh#M_pSIu>Sy2oES5r#@$ ze7!LwK}|*A%mat4-n2YIG)Kl>^=ery;Dh#(M=FYLNcI68CFQ+nKhUj%)`>dxM(?bV zPDLTajYrn$E&}UJDucBoEYj9ZM0?qK4HHJF+H~m>Q_*xdN;<5bVAJ6QhG)tf;|}JV zEcQreh~P8(ztfLh%!HS9+9F{ym0XVTC&aN`Q}KX1a9WSS#fo%k#0=1Ay>|zomj$Ou zA-KMg&X}MchWM9}`c5;upE+UK-%4bosOj!FwE#;nD)`bz_+=G$@M^V`Z=TBr&N*<~ zVqF|ml#umh{LLl_gjK*}XY9kCv4>f#fxPa}d8gj0CA%-Hx;sDu2JF8Ag(j-8a#RH+ zSXF{u!yj2Y128<5&)4T`Fnpzk1PF3AkR9knZ-`a7g3@DI<=i^#<1Crg#hN;!4v@Ja zW)#oHW?1$c4B7yP_BOVv)@z7yho)9aw&=L*Am#jxCeK6Rx~f$MtB#VQGNSfKw0)tNNSTzLmZyDh5-kDf=F%tTw{@A`Ww>v z_Y)N@-H8z0jc;8;RVef`1=SUIH;r>>wKLP!XKVGPnm6+?lO^diT`nX2Q*s&K;E_>Q}S zE&fgSG}C0|I-vC>(kvv4+&R{zge8a(`nvF{C6+F(jmo-`6(U}hYvNiA@=si5Wgt(Z zn(zux0#x~~)vC2ncuZqZTehga*&7TRd{u+(Xsh@tgwmbgJ&??8SB3C_JJi9x5^N~s zcain3Jougy+to~q{n|td_G=qU!FE7yH$ZZ*CpOu2u9JO*u=@&{>%?E+WFkp0yIx8S zT`v8Y7HwPAV7p%ZeDnY9`MZ4Z1l+JR0wH;rNyWnHR2W8ukkoj{rMighUb?&TJvu&E z9vwLuWRmc{Z)jgnOXP^$O3!hp_)9_r!dkz5Y;S`uqj(#!kLefMMvQ z8wfK{?hY{PV5h;vv}H){Zk%r%pFcG-bv%rNln(ZO5BYN-lPPw@LVhASngU zNnk?`rU}Mwjam&>a}&kf73TctU=R*!=#-@dZ@d4HX=WiwnjZ0uFShUF>XnAsFd5&* zji(T)6|b?Px=4fX9@Eaj{71T74i%Ua3PSo#wsEFH_z!o@6`b=7NW%H_S0uiT$ejhFPz|tSu)6z~CC-p9w-KbTj#-DjHC?t5)C{ ztc-j{-O-5Y*^)PD-am}jQ{6{(ow&K2QK?o+5aa`aybgQ{RS-K;B^}(3H6Xs z&K*B-iu0#k+~Osz_0`7Jr*-b~%B(*v<&!{i%IV)=Y1%dS$&`z^L--hiz`Fryj(9a$N z+ES?>iNGs+BmxHpX{c1-Ooi=g__0$vDT%|f@cILyJK5)C_tG}U(y7{4+I`U01N)Av9a7jicaVxUUC3u`N8?~Zv_IKI8fH4m+2Efay$VlC341JaoUMH~+u zlV9Rx&H6sAyLY^#SwEy={(`oVJ7a%AJIUOe`KN*YX&`o-{Aw@HTfY2zi)g>RfKIZ7 zBEcOkXz?gK2GF>Z zuBI6cI5!iS$@Bk-nk-6h%#j$?AYGZU`vnlu=Gs=RACu7UB8)3=y&j~+u)@2TgY5;~ z4cHaAC?dJ+cv?7)n`4?U5a_b|AVPpmqI?q5pWs2KvdeWY#QkMe67t2e-4`X^*UJ%n z*J8eFvG$#x{tbq7phs%5d_rcTF}7Av9Fv|E``)$L63ZSCve#HV0SgPwgMFoxWCE5t z;zZjzM=aW5afmH+NqbMRrCr>sw9*V<>1v+5xnEKK{^_&N2Lpjg#IoOj@^b~w9$UXj z@${c|^M`W?CY7`@jK2qzeacR}Q`t%Ur{5&|+YUbzz+^Zi(kWWjDWbXq0!%*{j__A1 zht{^^AK~$e41?lbh8^V?D2YHwr$$jh_+kN#RY^c{^3^4|H{pJ*`pfFB!Mw@G#XyG!NjqEUWNs}>5j z=_ub}XY)gUq`_4v)D$j56aSBS7IDJA34o#P0vOp8z>l@1dbLs0ZF7n2D)mBX1!wuB zQlR`PYcU$yO2kS3Qf260t&6AZkA6>sgPo-YwxfUeGKBimXVvNVTT%K|ob|&t=pP4_ zAK{kW6`f4>^lR1nWl=pK419-OarG}?{qb+KTD^3Y`8!^Z*B@#2QVm{RNvmP#eg0y@ z)o__dDer0A!s)vJq+QPcNF&d-xCD(=2j9=Exu`GjFEyz6+j@Q#Itry6UF?**{1bev z7W9Gl`AaAC@58C?RkKhh3ju1k`eUu#45-e~RbbURX)&0}9}CKH(21t=XLwu6RgNK` z{~)Pj$Ce5!k!Uy~Nlf-qK+q@1(DO&*D3t#{kLSTjWdxI8kmCN^|M<;4>HqrO?|<*e(X<>-5-r#Scg?4Gx00K;LgG8JAa`l*G2mE4+fM! z{XkKEfd7Bs|Nj*9fm}2GQqaR5@g*faiQ%EFhzGF^P~@`ofBxS`RVoq^=>m*$UKSMa zWN;QHBxe0MDZ!~{&z@3Y*$pLOH8fGK;!29TUNf8`DMINsu1hLn=BwA1ZPK68g}M-O zWV+mh=*Am2}KpblsX>wWa!-J%OXoiE{bs@-fUe7WApui<-yv2n8b@ zFT@J@++sey7zx8h(p=0Z@{5U7)G#AaBNi_#jP7cHa;Ekcg5h_DC{{E3G&L2hM|)K9&3ZffStba&J5>W%t~_BL{!>$e}0 zz$LxQcPdL`NS2r5N`|kCj5Iu`E_^ZeljH3G-u~}LA5~>}Ij&lv$S>1nc@eFQ;(=hx zbU6-XD9J;nOZ;W{D=15AmQ0u9UWT$fvanx!ZQ$=uBcM20Uaqe)oaEVBe|z62kV56< zyd>A{R)hpBA@m@S5M`+i>?aWx9VPeAa(WWcSj_qFIRq3Z%hyDCx&OaOAGqV!Q@t{S zw65}Lo>`Xf&ENYzhwK!G1-R=c%hNSkoj>_1!;47l?IOzmgJE83g$D{)KbB9nU(#O= z@v`kAD+@wm;%{_D;#t-!qnGjERsQ!y`H0LY!rt=aNe0TS&EMLoTnBK~9dL{#SlV{# zSa-6r%qU%o!Ev>E8jj97Z;9$k!VOZ=C4~EEOAA~`ZkgIFcAHK2faIiSraif6caRty zAm#30CW7Sl#`;ZAtUG*uW_sSB%Nsf^XZO#~>L)MEzot*WHhuQ=JXFaSq2Y76=?3R# zVUoR)%fW)#9i;nc=^=>bX9p1&&1}#exVBWI*8y-!ROKLTrJFNKz#XoeOVE_1%G^VE zMlI)0&pki=@`ZB?r{}`(h;H2kQY{1CeaLX;-2B44{>s$Z3r(v@7!wNC@8T&NuzrsR z-=tvB`W6L2>!0wqe~JK=TK`OhzmL$P*2o66&fjgV!Od!ouB-LX{Oq)T_SC$7=G3eD z)bzrcS5Dtmg9kRNI<~Irhh^0#r_N>d>6gzvfA-Au!d=xqxLNJ-b+sROVeaIN{=$Vb z-K`#^XwMHF;2%1`KXkz2A9@F?IXh&r@B9#`i^1KHzJ^{n8=cl)JR{pWNblWssum+I z&pL;AokV2qN(thIl@7B?XU@#eiypqyT6cE$F>7j1twhu&<|T+MnkQQ;MbAWbC$HZ# zd#AlSDgEis>$O0A`dj6ngm_pH0!`tEgk2~UQbeZ8&b+YFAM z?)QM-1AY(qJ>d6%-vfRReA;`U>B1|mnbVnQGSDfS+(6xyPQ2hqfxJNQI|POM2W}Uc z2)aYe7!aPYo9@kf@5o`D0|$wWaOA)&7B}?dU@@&*BM2RCtDn{_un)nl0x4}AKfy2i z@$$()`nn<;?BIl72BBBmoz`J<_$(@3>vCEEMc2?#_z9ru+I|7*#x*i%yfm%~4HA-= zo^Tovl!iKD$n_ohR%W^hBDo#Gw07IC{1RZgiI{u8V;aQ1CFN04?tu>U(aSAcv_muB zyidmJz88S|)Um!LWs7>Z??i6<5FQv0T&=USbTk?xFI(#!1jONKI}(*fmN;=@$6zDk z0BwGDVZrkR&2s5fN#UmfiuDt`oqQKZkq3jd;?T(9;lTAtwnrFVd!p-vG4MBk&SVm^ zN^o{CcpHCrdOX}7>?dFjjC`9V-EH~VvNF8(k@66b3n(WAiu|3u1V-k!hTrTJ54>AV zLIY;!l;PmXpSl41;&5>Kr~9V9b^G?`Slrov{geMV%O8q-wv_Ip3`@hrQ=Uv(qwO~k93otH@Dv3y~)xy{r-W;R2Lvo z$Lu$Pw+HCU%?@}!hjUvPAd&hr&~ z7t$Z0u=_iW_I%(m@r_2;9#P2A_t@I!SU9@IKlSL^sCfT83o~LIo>@C0;QuNM$JV|i z-k%iVQz9fi5W<5~(dZgfso48b5k4c}kBjh_2sKgPq{zpqU~G*Kw$Zg|5r0aAr$zX@ z2wxE4iy}ND!k0vt72#PClD-DvNqX&|vP07=-SIvuQL0Raz#0#Ffw2f zoqqy9=j}v}L9ZNv^ES;*Y^bn@&cb+}b-c;8MbSO^dtW8}_QuQidzBuTe3pS6Uh5to zv)}QD#@Zz78aChJ>Eoi$2YqO;P6^!PAFv*ASR!G}#; zO$BdtxO(6pSU=lIN;7xO{J%TYwzW+Jt^WXM4>H^D5ZnL!Mr^-*#c}lIt9{u1Yc;11 z*YV`d$Y<$&UlL^d49irXrT1C-is@UBvUwHOagFjeEJ*LfITHBxR08+>d9(Csy1hwW zaApmG>FT;&v5R{TaPIv&^ov{E`|IOu<@C9Cbi-$dd%yifa4nNRyD^#EcmC0HyKYY? zvkh#WI6r^m7LT3XquiKGu?(}pbmtR(Wl` zJm7|s;UzBso_GXk<+y}*<9hp1`8(~{*5e7~aa0z>zwiISjaxkW_M1TT*;eLT?Z?Rz zGWPZnAi76+@wZIAr4zn%V(tD2Mttc62APy@_|l1wM%R47osb!BX=t!1`aVl10!?c#wrbm!Ok5xK zB@TK3k8?wELW}Bgq-I6FTX$^*&n_cbdSvie9oTPgZSnGpHXuG zpR@NZX?{5^X&9URNptqIWM{{@{($-whLU{$@I$=bBhLTlKIiXq{#BpzpYS>VY^k6w z%uP+7rZar?Sv5j8eESFbKIdP*qsyx9>_9&jOGULPXYYGC|9Bi%TuG}jn=8qO&hk~^4ToIYaDzEk_|M#Z zhOM_9>ona|r$-zCIMmJ%T3mHZqh2>wxmZVd_~92q!U8K7u;Mf=W;?-%JC{OZT32Ow zXYx{R1?a5SOpScCYlUS;U~3nR8oy)nK3+TMyDRbCl{mN>t5jiE=(aA-lkQ3qkz_cX z*3x`d>6&nm6p29!nUG3EbS;LQbJ(>4)&gRRlkTBNG?Ph1BC$j)gS+|lng1S^k>9>W*&m_{zU*?< z!L?|)y5Z!CETifkq6E4=3wPOwN}Aab_mOsOJi)BsCLCNB5OnvK^8f-*SGWvEzr5JY zPP)4)jb&ZjE^r92Uf|LO?yZBTrLE)2a_*{t4P|G_wN@ ztARvx|G{V?4I8y|0@vh(W4N|KE)iXoSWBq5IAj$OaOeRE-Lfc-KwFpblAT(VH^rNwhqX94 zBQ<3T(A*5Z$=eo9V9F4md~mU_YnY=?=#&iU9;V>HDH*cfj2Uu3Ebq;dq2SGwp$Mdv zd>I%sltF|_k5Ex+dT?NTfNPMOvwx^xh-;AHl+q-yNfHhO3}M1;h!MmP^ChzN1Po!u zZHO`05Y5IZwLmX&^oRxrI>-CC1jjo$`gzAY`}?{1cshp=@~I*VZVOBxt~6LGr2%v$ zi+hlhFLH=60z-@u7-Ec|5M#|y_U6b?ncTZnGBQIP?kEm-A0y{@4^OZ=SU?<@2@1e4 zM>c^2XaXe6MSx)r)`k@71`wwxEECZLI|ZLXY*2$XS1dCZ(BkV62q?-=%gjj)4h#u# zElbTSnKF3_)PZan>>0w|3_zzaWO#0VxqAbLOoq2NBQS=2F;p-DRm?kij8i@%umh(e zCZMAH8^<_+iaK#BVg@R5c=n7#389D?$vTc6HrIG#qj-}k8C+Y780Yb6X7F^eF*GnR zGALy5f?X$oq+A#nj3ODL8J{j0X zMvd_S7tj?T|6o@P^t4j~@8pFi!HVy3ZNKi!n8XAP0_<9VL9pG)m$8i(8V1-^0mESX Qr9_}AZb%>iRi)|y03l32^Z)<= literal 97578 zcmeHw3ve9gec#>zB*8aG+LR!r@|u#R0|hSb{a{2H;6oC11Vs>#B8Op4hr2!8E$^j! zkboXG2+MISZ&Jm+xSq+>)^x^?Oq$qE$C;#^Msi+GJI#zZX{Ju6$tY=)xa~AmJ8?&8 zCuV>D@3FUcw+DbANP+T#!~1r>-S7WB_WwQp@1?XqjlEwHs)GBpE)+3 zuaxxh=hH8wi{qI}Rv(v5vt8)rdZngS7h6-`++z-O zc6)revX){^NTo){)I-$rL!(Dq4fH3zLZy1WF`h1E#~ZjvWt#OyrIfm)*UEa48jyU^ z+|8a8v-MPcv7Bj{6KVCBI?J2Ly6OuA<6~;Ju~^lQvERB@C^tgEQ7uy`HwyF3O0(`8 z)oOaWSa?DAjj|>^Fy5@!#*2ly@#%e= z_LPn{vn})PLQXBQYu~6^sH^2lLp@t5>#Vc+d!}CSV$hrAmZ{eDVovKS@1q(xDAp7d z_;oKvnZ1@uwaokQ!J4%)FAO-o!|B?5eM~)k_|lbZoj3m@D>lF0EHqgAlP~1&Gj~yi zzM5Wd78|_A2a>b-53i-3`~x=&!F+JVmZE!QouOLt9jB(6D}Sd_x=?1{X~bg=0Tz8F z%&mvmxbq$ZS*ZW#c||mw>>=gx!SE*wH2MX6dm8<{ysUu5u_b>n`AKu=ky0hwEb5Qq z9f>F!HGsX-Xf@|dZ@tknw`KKAp_DG->ZG{(e9Ih;#AD$^AQ%b90};(1j$v$H{ z#dg%>OOL6!h4Qgnvz%#ED#f~1P1hQQbkQeXlPBRZHC@Z(kFnzlU5$=j@L!~}t-JMZ z!mT$HN@&4&h@I;?5i%M~#I#r>vC5GzU){(p*G0jV$g2%lRW~0BMnj>HmI!b>>q5a$ zD5m+Ny(nl7&YnK|u~vS_-1kgDzaq$P^>Hu+@~zG#H=6J#fW|Q=3{M1OT0FoX*@+rs z3s@uozPH>g&FM9@k=NB`rCRR+9ZqOrQNFX~aep+d1*5^9$s)cp`|?ykAu5G~nCW*F0gh)uK)l zA%94V$DJCR2nC{AILHZC2cS0U^ZwSWR7()6=8+Y+ zLV{g34r~|)LNFTtj|R={sWvmp&^2Os5i`hQiuo>Sc4n9gMopn!DAyb5az^*j>nSi- z#?*SFHmd5yx~}F9y_hS%tO7OE7ds2Td}vI)q!AO1^9lA(c|GR-dM2+E7a&Efu+-%= zvGUH(73Nc@=F*rtr)R)ktEWooDlr};)}H+_dMENJK)t9U^zlvT!- z7#n@KSZ`b$l{NkoqUMw6#q9aX6Q`z5%}%P1U{KN66y2PhoSoF>^+p#PN)->nFoj z&W;l90Z;x5<}+Z1q-1}5hXqSbX0$u#JB-$gBRDvF*-ENe&88cA3d7o{AM3hjRhS=k z;`$KJLSJZr>Bd-l)8>`}aerFoJ+i?#^Ehz3z}EN{4a5ju!>IP_*1_b9<|j*060@ayf7tdo6jN`N0&J;l)Lo;WVrHc7jFV;gxvfwDJ&BoOV;e6bDidc#M-+ z5zU|@NS|#j8B+E zJU>GboKgGEPjjdQAx0rpEp;9dqv1d>tVKf(Fr(pcl;(Ud!7mS4`{;;RfII;E=9L;G zCtV9lG#tVCH4t@RVMQaMU`WIHuHS-U?!@*Cxa@G))#gYj63}8HUY4y>^2i9VQLKWT zT68#SnRjC=tv5?JSkXZPrw1G~=Ji5uvGeE&I59NOR)ajy;z=BfWZ!1*VT)W}uwlqB zoKCdSNHiMMLMxtVap>r5tvr09O`$l>-?rwqA4Q`vEgJ5B0>7`mSemO83z>&&IG<(o zRHjlarmJ=R;jE5Tvuo(1{&?J?NIJHNXp~OtQGfppXm_ztDm3Us-n;jpG)5*Mq&wY! zLw+qNP}J$8?q-Mb^BrH(8UPFq7zha`1dm1tkAnR_=6>E*2tUu*WB|=h zgs~xW5S`7A1%gp68tjjYyCHnWcyO6*_tc36!hR4b4t$SjED!-z80`<3lblH=3w9zo z0U;640!~boXe{UtuLyh?^HRu}Qx3Es7L-6pi#m=sV1pJ+^liB95OJVUsa8O4HdJ=3 zW<-LGm1suRKmzII`9fJAm*;YJblz|7&0tZXGq|8fT4s2q1IQtVh8!B@;(6l%FE--^ z^8>}o6}`sVNGF>jYE9#C9_AM~cz0Fl>mEWM<5?CeWlq09VhTD7+6s@hk^|-*5Jpw# zJ7jTkYGm>)bNeHuLLIaa9)oFVOyDRm_4kA3Hn9kUDrnRej}k3|s#zHy2&OSL(^z1~ zA6_E5#oSRXfjnU=cgvj3>N&NY!Ko^B;V1~NQT529D$jH%`%C!xKOxSA~gX9 z$+rM1k)-#X)qQdc6Sd_E=!aG3aS&+i-D-){0SG*iXuh(3B~`CwQstwZOKWQuJ;E@_ z*O}%3@6a!8F?TbBY{Q=wi4IfdIfR$4zm(#WOcOk8|H<-WP9`wH+ zG(SLNAeB1!2f{AMPG&@KEp(hsEJBha&THl&`m{wI*w}?3Z5py=3VdQ@Fkp6c(bp!R zF-8)l<)K&J)8fT(5tKRfWq_;1ZK)rdtu=KUO8nuVF-hmjdj+}JMO0eN#|eC^IDt>f zI>uCx^dRetb;9>myJa=pJ&+3pmGGTXuQ5X_ZdJW-x^^nVM?luSFBuQhem7u@Au*2N$F}_# z;{VH<0~;)KK!XgZgoIZ@>Yphj9LUe?9Y(KpEEk;f(5ln!g z)vw{U?ehaOf|7%LN6@@nX!h^0Dq2D_P9yx0IN{!Y5tw^mZg3%*yAb~9E`(UBVnK+) z9oc1x1k@lx{TY?R*70_{rkAr2e|E6q;^7EX*WyAKqk|O}k3o4H{I342$z6n;vC3PUOerY~EHy5L5MAs_9_K(wt=A0nO)j6bLXCcExZO2dgCtSN5AAr;l2{4Q-w zNaU@Nf`>T(Mhr|?e6UWCrZISi{HO5r^R46#rXIh?ddLhrTIQCMvoQ0Zf=^z@Us5p# zFINiL*4d=zYy%4Q#(@Dv@fip4o7_VnjQd!IEqjP%k09F|I@hRK0FzsisyP75k(TKx zv;fbzmx@xh@ey1Z58_8w!-{X8%w}sdRp@6>i#-H4=}9h8J4Tbr*`5XsvYH=YP`$|^ zb9b{`FU*(qteVLar_FQRRxMsILR4+oR_$TB_VS9R6fdjE9Kb-gdTSeCwXr_1v6@l` z%xzcn^d;(|(|A3?3LHljqsb)>?^77@Vj;WW?}YzRY8SmIhWn(vAI12vZd1W|Tfs)l z*oT+Q9eis$UCvc7X2P5S3P3$5vm?DwsHe!LhisVkW-H}Js#(`l#R_;n_10(1T_m9( zHe96&laEnzH>Je4WReRQJZ*m8p-)?Gpm*ho!Io$Tx)!=RnkD?U47 zebmYxH6OGv6cRktr)%kEqcX3TH5fH%wQM;ZhBXopesE^$)Rfg{a}Yw1E3H?zE2{E$ zFcN!LF}GE!sks9BLKa|K>T@tbY9%Ld1zn3YU2UsZa=E(RXeHHVSBG-+0a#azc^c-y znWxU3vnn%+$a{5*qI~xa#VFzW<(9d%UMQ6yBTo)Pp58GyJ+Pj6>RhlB)|1PM*DoOX z%t7!t{b&@XVipE4xY%+TS%EdQ(|dBJ6Ed|j{5?+_b{bQRfuvEOT2|fHIK z9zQ=h1vMb9%R@q&L%8F`NUSwRSw~s*ym>xkLaATl6+<9Ja-PvKCc5d%#9SUUcNNnw zEJ6a#1a2*+H%A7gkeSpLF9a?+$VdYwxP4F>sl%w6cZB?4-hKc*J|RWg*dYNhJ)k7^96 zu+7{*b*r@*Zv-t}UB#QSH{NtiuQ=2c!So7pg07ox3db~P@ks8D$1>_aSHU5>)2Khc z)p!mo1+?7<^#vVv>;q6OZY4ir4nSna&U{0#DNnP*-l#bw#gKSn4$>@XU2G+lbbiQq z22gNVY&NP*92PI)C|4+p%O16%%&p+NLi5Zf=)(edlQ!rdhZZ+@BRQ=5LMNS9;tPza zn23(99%xH+HN7_*iiLxTNCKRDyQa5bjfY`|83n)JT0HxCI@p;?f#!u)W(nRIHgc8v z1kh|Z(Ck7W7`q5uI}&h_h-T5IK_VNbz(O);lidi(OBm}H#4Djluk@ySnQA~a6ugOz z#GfVLk0y?@b-mU&g9=VP--MxVTI1FEaJmPAS$AAX$aRdd$sA)48eu64hkH%w!l77< zS(5iv_B6Ia)r@iQEp&}TO}9ho+CtZO1iG}5{@T&TUJ1-rBplV^qKN(Y84kf#4@$fJ z#R10MJ=+#^y5hXc@$g}rlM zrF-LW544~^NK%Mi`ENKJgk*Uojawts3oQ!mVkkB_v@IBo`BBkYgt88Xj6p0S#t#NJ zkxbSrlo(wzM4MS6=gr)hnbEDXrh39-ZcA~!gqHcJP`P4^lH<&XU`$8Gm_H zgROG2rl+9g3b98+&sr>xTh-KOJ?0=ILCvISC)?pf>)}N$4ck_|RH-dqhCMj$SvS}_ zOhTuT*h0&dB!8!#Pgl8aF6e?=)#xi8&e^XPr)q}pl`O~hlWP1XGrBL5vLtCV7yi9vnhmEQ6 zab&p$rOajyN9ZmCb4P>o8)VVms+H&ok1+rV=B8idjUj5KyTp@$PrB|>Z?H37t4GjH zx5S5ou^5>~+V}&uRTzp|E6`j2yoYXp!2^822t`&#Z&|?LqbUy0oqY+UZ%bh95~Nt= zMA1Y7(8q##A>Wo9)J~1xmcZmr776;HeH(USgG54pXt(44~gITkh>QJtC9kHMCEDViW=A0K?@3Ef|Z!W$CI!Y?rd8m2K!|= zTv{EPh4ZjWhRaL`jtewW!SD!0*8og5Ef$M{f#i?b&c;G19$uyV;F0u4ww`WcV!@&w zim|=O7WW4*Xe;nrj2>w4>V;YF_dE169`J)t6&85tgt}ko{si$rkldRIA!x@ZhzA2$ zhcQ$8euAEf5CO4A`>{r^fX9C+do6*XQ}7xhJQUbJZv?)Mv8+dhI|ipt|91(@sy_iQ z$ku|<1r5vJN}fcFzw3n-C-1ioM*y@9-oUXL^tSG z#T_)L%v(dpE*1#{p;V6TfH$S1&w&uRYV!9#3>aIX_h;X-BWMcs*+DxV-zfnv7Vsn2 zce?HEffh<&b4fUyEMO``tFhcfoWAbeUC^M*5rPvA41CW7A!uPt5C>HWF@iRd{eFUL zk?h>Gbw4+4y;&~|u5IhhJ5X?KTc=zf2siGu(B&anflQ=xlzy%j_fI|Rad}9tm|#Q2 z1l?}jX-kF_*-2cDM?_S_`ky3hyLHuJw{{&-Kp*@@T_3rQC@haluEQc0qN@M?u*r1t zU4f4)@Nw1k))fNIS>WTaSqDHrOKQB6i4YDz(LC1Qz})zTB-AAXG7R|yia3$!LGqcu z0<7PZup~<-0CjfQX^0|SEeqm=As;!h5WeF84OSl^2lg-yB_VkK4EAqZ`%q#!x%*(M zWux0Rk!qRi?qhxKKHTlb-EQ3NW&^hy-|Y{ZD8&odiV*ljNa?zE4|1;^ZyNx1>r9*6 zrSoycBvt3+w_$7~M5eN6h*hpeZX{X{9I9egOLGRG-h;Dd-rra?!geay1@6$E}jT-X=-?p!gASjock zd>7mhJfK4u<#2q%;SGYnzzVUThT$+9&Z^-tTR3#@h8==*$?u>!5Fx=VCh7{BCPVr{ zHQnQLDG~%T3DQg_2_n=(ft9fpyteOya|S-Rj;`k-fPf=r*zYi3Fgu}a6zebRB>Oa) z;$SEDq;;zW;?-x zT)>*<!T^bm<@$6Nby(Dt1$Pl*jO3zq)~1lSa!$eh2pSN3JbTxor4hx zldyjU)aJ3+Ai>}s;ky7wvD(Lk?N0~b2o(Q-+(KQ-Y0eB2W>XBSTBWGwfHR$q1^`gG zP)2O8rgATUj{sD=!m4Ndziu)UML*rc*aAF6P=r*p9Q4#8 zup+~Rj79tIQ#PXYgv~Pw^Zs_~-U|tTDRu7!f9c)}p=c=-%&n(V zp1?{dujAj{PVjGumN-ri#D5%X2{7#@8JP87H&vY|;h z7!%IqlFT#P&j=4-l=k>;3o|&rF6f6@F831$-*E8MfQYSd0b)FmH3b2bO;!>!RH(s@A&uRLhv5u3sfj7~~@ZZrk89?k6bEcox5(F+Rqwe-giE zjSKWEy&2tkzKgi8n+4;~(G7E>JP;XwLD1e?-$OU%MacMX)Zz_yA!Pgy0(%EK4sy9h zF4xHA8r|7kBYETl1x!}vPHZCt(xua&rG0N4gz>7_`J|I5V+47|J@`pBjC+w_TQvR= zy=UCV?hF+PTta!^63Ps%=Nx7n;t-Uy@c_GO8V@3Edyk4Fa14h!alH3v*y&z_S>Lwx zA#dITkgHg;p^7!{0rON$1M;7Nh^ui2CPmZ_W+CF%S<@hIaZfw3Ya;{SyS`U8neP?C z0;uRaMNFheFr9&cL=O|vMG!d2bn-FxK=OtBeeWv=jKBi$&^5$#?qP(`Be;wSScYfj zUdtIb_n6u;_rtSm4p9eBG7qlZ9vRJn6X%|Wr}9R&kVA;jM@^5P)naaei+e~^Kw`F6 zC36Rygdd$bIW>9I-^vf!*djYAtTP1{BroD@ADmK#^ZR^0GqDCAwfL0eH+;O_tX2^z zay*}To`t&~_QGpOrczxjlw0OMW)ZXSNCxTS0SnR1G4=VT-qe|^bRK39nOgn|E1D9| zU_`IiGYFKI&LH4oQeX7tSS)I4^o!hk2MX~)qqz6*{c#MghPCqCA_#HV;Oph|9AY;f zgA?1wPMht1EhYK3Q$N=*}8m4O&D!sW=AYI8S2 ztx4Bv>Ba81v^qG#qll+?zxsk+s~`YmMsnlfXtq+)!MDROBUZSYPa~E~rCeMD(_Tf` zn1M2}wM;m?rI?i$GKeCWLm@H4ZSq0FgfCw#%xU~8RjOp0MctQs=tc6U@-jt-X1?WV z6kl9;`62Z5r>NP+#qBsD-fO~19>57Fc_t1IOgoQ_aKC2y3OtNv{$Mf#XVZG)X+&T6 zC`Hd?*KCAGk8&@)=~^aF-WVs(KYnU9HTzU*^8AS>E}Z0Ekq-gu4_(ygor}~axM6(K zYwlpN8D}`yU92S>u5^NS*^8BGA8#Usab&Lw|2Fqb{kr$Ig%^1rt(kivn}J1R2zJ6z z2$pOjr}mQsvCn(udN;JXO$^fNW;_Grn;P~SKY?^pu29BLQVZi=EH~o}(`aezv*cj* zr63R+QxP*_UW5~6E~)(}{SwDh_zK4~5O`k|>m~ew^V#XEYBB(BiqCjfs;U9>n0mO1 z$PfrBD_(|w-}4O?1|jMb0t207t?3N)|H=A)_v2;76GJ zS9WC1iA=r{o#39pv*(##ZY&SgB7&Cq0Gl?3%Y}@7j3;(i+)JB*p}0I~{1tKYM#Pe~ zaW2eV=HubRm#$=aG%MF@`TOJ0ES7nQNU;lOcD<4IlOUqShouPLI7MqWxjCd$vIv9F zeM}aP%Z*{0i33>8TF37Kr}h)4&O8RO<*I)_0j^T_-+;1~lHH{?*}V+igSa{r5Qjys zVNLvlHz;BZ#eAX2EiA?g%jZE-6a~)_*@Oa=AoFv zkmItyp?{!|Y}c=gTZd%V#4E@hlIwxi&=(qtR;!S&Mn$VPYK`=q0vrWc)04kAI+GOsysWxh(c0|;J5RSpuV-kMT8=JuLC zU+0yX`*Dq0&Ye2{*omi}K0ABryua0)GyW4AWi?|rUO0VrW_Bj^%;cG;TgKbeVrqi% zm$*uLjQ`A&|3V4Qjg)wezvg#;g9Mct-xukBMe0y%V1-&|)?2H0wOWHcweFvpM!c?* zGpW-jpG{4km_7Z>sdaUDV7014Jym~DR(*W(Y%+D?sk4urIelVwU9}IcR(rUo_Pvjv zKR%Uu{OQx3y&j}&#}Dn{AKJq|w8!8ddJC%=Ar#qn{s6Fx!Ceu*h8{l?Jdt_=LDD#C z4bpvQovOi*%dyTOUMC@0yHbLnVNbm%01S?}FQdn-~Fk;<*VBZqfH}1n$28^7RJ(*1OutW28)Ppvq14 z>U#Sx%gOXscdEOd?)zI^zYE_T0e1x45pYMq9RYU)+!5HUBhWHo39DyPj4|o4GbUG1 z1Av+bsSA;=Q-_ihR|5!p%^{}3gTS;c^RFfWtgNrZwE@zkSz`3l z20-h5P#z-Xqoc?AYaWpvfp{&4;MQFGNQAWRe8%HYG3osysCcbTLKXk2RX$%KB_M6gnJHp*kM7Gnh6(`&MPj1ZOK{oktI6 z4f+~1aH33Ai7P6gHB|AKDuWM?saMj4#xeR)p~rOc)bXbuS06p525r6iV{UE}&OIgtmok$WbVCt1(55=t zeiDVN3d@R^OFxE%++=2YcGgjFVtkK;2?}QdE5@E3igNsQFoX7atNEef1KT}I6Ku)f zzI>mQR+I<0u82ff$uyVc?1+Mp$}>e@IwZ~hW{wBGX5C1v~ab>#y9>+vT< z6Zt)F-#h2m*1qZI9olVm2_2X|uWa`o|6LPcpV;m_@w?fVcKokvcd@(=E&lz1X?}^P zOV83{s$D#$OO{715qWfJ<-NOod6$yLxQ;CER#ZIs-uC7FNOnz(EFVx_Swg8D=%x=@ z;u36Hub1Z%(|ZUd_py@GcjLtgWGUN5mM0a;1F#QLhBCbi&$cN`xExs?VxMtzc{@u- zmUq${iszjpEPY=2*dVb2ul+uXe4J%@-a5`w8tw=^^5M~Sg=CT|{7oNcZ?W(5T7bPM zzZbi?_R1>5?s+@bTdG~*^v`*(4G@Uw0q2YMtBzM5ftViu!=jq23W;Y=FE5d(bjdUE zi@WH-Y6CaY^#wF48ANdRnAUiZm(GGa{W5=~up(pPPMZutD+o`FhZIeoS2V(2J|?|JN=6(e7g~ zuld<@qJ81rmbtY6e=G1H^s~k=4lT*`(|oTFwN<~vUgIu6+sxEngQ&g7R-*Rm&o>&W z(&awX-j!;jHVR4$bdinbaVfcGcc^8Tk}DQx!Gf4#s`);0U&1e?y^Cwnf>@j25YQmi zg`LmAj5~DR2DEcfH@g z`Z~XuUQ(`&U>oJM-BtA>dweaYOwTIbVdd3f<>Nnn^>vXus$6}WU!vI6=YE>Ki_2-g z5qD98SKsC@Tzd%brsTU<-)>(%_ngd~KCJ9kcD^=hJ|I!oQZ+c(-rgz8imRE%-zU_+x6CYNToyuqN@9v-f(wDw;^tD6ExzD|JXy{z(L#Yo< z?uj z_WH=O%kNp{tfn>bdl(koVxrd5GS1+rchdyzo7-K2Ht7w4KI*S2JZ5Kxo>a@ zS|-zG>?0yjIW6x`7T^+KFU!3Nf;I#vaDFGzWH5wiya^fhc@=`zD+pT1p@^XMK?-^8 zO~wRd97EfFj>GzUEQPExj%0r+wHniQhN--1E9Cg>(tF z8=4vnVULxMkGv{Zwke~}A&XWif)g_^fL=&0_KDMfaa)1+bZ`%pF54i-LOVH8gNqc9P zAd&5<&OfWFmAV8S(;Idra6h6FIG3PvHLl;E8rN(?c&`b%X|l)Q#t$C#CDtT)|L(iE z-5`SRZI|G43BHO;@R8?rSAy3plJEW7LhvEnHNum$<5IMhRbv!!R{GdoQwRwM>uT?v zqL{#f5d5CGje=D$^X~_lY_!YKI+<3NE(;@RvUuhS^{nu>&E4sGs%>xGGGlf_X#v>R ze!`;Qq^@i~g*~7=n){Gie7SLQ42XwT{YgPocL(M2*H`RoBRYr5M1U#>>5Mn?|4(P|PS?WM+z=Rhn%inij$bAX45bZIV|Rx^!-v3j9=>8MIE zZ!kV(z2-ya1^-161p_N3p2Jj%MXPui2E)k#V+@IL1V0q2W&*;aDK|iDLr>Kk=?0Ix zU?edp$!X&Z67wFh@KX>GC-(}sZtHo%V+_!4`KV|C+YXO6bMS34rY>kTorO2TKGdvK zHSFBTAJz17y;;*!Vwcym*6wtxHb{3*KbUm)^gDP@|HWM|b4I=0<$Embhno^D9PHWL z@g(l|OK0#99Q_`w);dhv@R05b*IB3U#9Y#Z9|#=i^^4` zphO=(2ivYs{u+V_dW>+4Td_40``>WObzS*gp2u2-m(Kj&3` z6ULWihhO7WLDEbUVEH#OL{A_QQGRhnwo>iHL1v#!@u*+o=d$k34> zbLn?5U{(cNzr}Ct6?{Xxg6M_0S_MN!y;pxrYj>tvt7Py+Dm988s4KrJzFq@EsW<*5 z?&dS)BS`2!KI@Sq^SOmU&>xVWS&r38YG{O?`v-t6VxB&Wd-@~v-Z!vSnGo}Qf-><( z{(1H(JFNL7@tacl{~la>G0_HxR5~ImOW*(MH-7uF{wMbjetlZvvz-2TOb%zPKM7Ca zBQ6pQypl@g3)NJnS;(exxj?`liD%QnKuC|xaZfoG2*(RKUEdVzhT}^tb8lxzlm0K> zo+lzC(U+9Zy)>wJ{$y$SmpxxrzWCCzveo~JqWJ%r^658T*}CJ&f4Kg$Zw&4n@?PEQ zjW2HWFf?vRqB|udliV|&5DPo^SEVR-i!Ums zr}D0NE{*|;CZ7M`hu14~`E~JJB+6UzLcAxFGobnO+EEI8tDZS?QVk?DSm~?2@nWSs zuh&x5bR$ozYpPyJHS$<{Yt>SnVy31EGHVjp)cA~}eaB%3$HyYLY(eZKI9`iGx0!8VzWZFUaVE{>9SdTI7OSQ*A*Hr+R72eX|Y6MoV0pg1ay*t zoo;NAPM!SN*~zKX+;Nud|9VpzH#+0UeWc&HNW$+%cPdMBNtT!MOQz3>f;7LVF1dfm zc}MQe{?GO66MD=jlKy1*?)SUz&*NRn^HU+)pDdr~ zDj;uVdI4p+hls#@6NDfAE$~FygOuKr{g?QcL7wcpc$FnSu>mygnYfnq%Ip#@oXY=G zQNH_3)m^@2{2R0s5z&?aS~dPHZaRHX$$)4?&NAwF4-aKb!QbN%tv8Y}dl1v&oNNYh z{3n3fK^$|xFCODh@xb_x^uzphN|H`?Z9VD#{(&6W^^{hWxm5Q6tq;zfT1p-}?$q|X z?RH1tP9A~(@Q&OlH+L`OmLt<2wN+F0w_m#2*OTQ|d(VCDj(|G??g+Re;EsSh0`3UB z-$$UuZOpxERG4$n%^hwl%-sQAu-%wPq=4qeJaW~_5ZvnU;Y(LYi$N+=yJ}^wS{Zcf zxvv0@6b~e45jNmHyCObxUY&F@H%-VZi(ejzL_->P9%uJL5DOy|R%~^*a-NRhRwVSY z{G=~Uuya9=pY)~QXMBcLgH!Z5RY-jk&pRY#A{%!%c*?RHS7e=xbH^DVvQg!sxbE+`pPpaE{~dd~=f|V+h-<6X-yJ*ccTWH7KNEq_{^AFP zmsg0LM)S{#lh!wO+7}>u>x!MWcY?)EL-8Apod!9b6>F+HcA5%!{bHvf9*+#0Hhltf zox9eb{&8k0?C9FUQ{@q+tk3ce!EXnPny#$tvoIL+GyPYrZWe&b%Cucv`}vJ=`_?yj z+S{MrMb%O8G!<#Tk)z;g_D{IB;A!nI?0UIa(aVJ$p>J1ksx>*&_Pq8Op$COma}J*N z&fO~pPrFv%82n|8aQKn?x5FcrwbtjJH&=99r~;TfH741R-Y)zU!P7SF)ZqB_*5fSg zJ3oZ#%&Pluzu;+hKl3-iGTM1Zv5efnX|)ur7r-z0kqd!+tL*n&D?^Gco-E|l+4GYp zPEDPfom3xD0~A?3MK>oWXD2n-YWNVly{Ya5;5tlvcF48!3+{X^PbynV+QpLn&&cu zMwU6-zzs5u^ExM8Mwa;zZDjcgc2pZ#<}90$$C9}+Y5eZ9zST8h_Y2DyCF zXxc)-r<-L+i)b0@y92^t>%)MVpU_PvhgR+t|`hGE5m+J|)uIeK`PW`&!Y496w7J9;K`qzMYlGZ&_Q{Mc&=H1|@mh?ux0ta**d{c5n=z2uH9U*Wm-Lq&Y z6w!J^nu6UJLTsD_XGE^bZWJL+mjt_U#gL}z2I-!?>FS=jA@kM|GH*5AGyht3&)!@` z_bltc&O+c+>sQ*IKjgS$D+T4b(kj5p(T|OHdahzhG_6r1wDg?AraxE@k3}*C?{q z>MCfAUU2&HJojOi{Xv}Uhq)7#zG`a4b$r3;?=$kCzi!%RH%7O|(=Vj1tD7d+pA+mm zk1V@97yCT8X?A3FWANABCj0eTbasuyGU`I0i?TFB|es<)N%Z_v-blnjVx?Fao%Z|Lu z)LeFCv2n+m!iuO#;5tV92`%Jg3mc8Z!&;m_S(P6tB64@|Bl|_W|A4yVYhoi*EQd|t+_XiEq0q98BN5su%mut zEE3fsJj%`bozi;m6s7-*h{lurJtGM3g64yq*IuN(qEV@7h^1LwtTY?G(q+UH*NZu} z9WiFK!ODHxh)}thee1Hy=%q?+@p3`G;+tD+=;>N5z35{=%5+~J9pzl@{Zp>jPj_i~ z|12$$kUtvGV)5P>(veUg6w`tU_5`Ai$1(Lf!(=6qlYi9Qn}`Qvp?D-5hy}GkfEO^H z!o!zaK678DnW`e7W4%zyrr5MmQ&H zy!B$Gfv~}uTDp>2`d2b_B!ojEx*dDP!BoQ5JE z;~~b_P#BLwDIFjVBCuY^DCUREEr?cG$+V1n@GPkq_adQ@+mdZ8R`r(Q8^EIFt-pYQ zY8nsXA;x?-5|4!wfnYol3&*u!fNu@X@QrCy&4GI)aASt@2(mGYt6s>F1in8aR0cU_ wI72oEb2S|;WDS?FyP6pF$3j8B77WKEA~_6!K^{Un`|kiI5k`AtPj&7815r!DBLDyZ diff --git a/extra/sqtt/examples/gfx1200/profile_sync_run_1.pkl b/extra/sqtt/examples/gfx1200/profile_sync_run_1.pkl index e1b0a91c43a2aa4729463032b1c5014206289ef5..a16fc7b95c5e580a856a21e2075796c699e7e853 100644 GIT binary patch delta 1024 zcmZ4Rn|0YDCWQu;snQb}z#w*tMh|~UW?p4_QDTZ-N@`hVa_W@H3+Kp9wB?dxcJy_b z(!-LMn43DKhd;TrxFkO}J}0F(zPK_kdCKOqb8J{Ahb%H@%3z;7VUY+^2H)hBizYB- zh)=d#EK#2!-NO_dI3+{Yn=wNUh~>RmG8DX-G8BQdk}m^ehBAmy=@BYQO%D!?4{!}~ zbM_DQ3vmrHoKl*UA+FgtrFM!&21gGsP}b8gI3zee*wH6+3ITJOpyp2ITOtl}iq;ZY z2Lf7{aXZBr?3Bq%mnabO4-0NXOi=y9?;hmj8}AL7=A;G(hJ?75rRJ4PnLGs= zm24U88N%KSK<ecy7ME`vQkdhPO8(FtU9yR4@WnY&iLhQ$8cG1E(S;prVC0o^b#b zb>dXS3{*7n85^e(LJ>2PbsRlxuJOi3@g`F;xVAoKoX4k`!PCXY(7?dRppd}}cAWr{ za$#Vwie!jpcsel%On$iBX?uVG<3WCTpzTZ;{s5(Upii9`xF_G+t-L)>o$&z|)IZo2 q13kUn!;&$L7q2Ryuea~;Wo!ef0(l#|SwL@Z_ecV&;)bhA)dK(*cr+3K literal 97696 zcmeHw3ve9gec#>zB*FK~q68_G*OV+BC~$G_2P4t|ACkx;D1v}wDJ*k5+%0goyqE4l z0(#isSxRDgCsFK+>rN-OrZdiC(!_E)&Lr(LmJ?5$cA6P;(#$xW+M~SFcAB~~Nk?hM zR)7ERaksa(2Y?_*f$|=Q_;$bD@Bcma|2_WirNLj=sdyCnU26HvJ@tHPalV?)YB{}7 z(W|wVx%YgvJeM!%7t*D9{nS;xRBwH*HFCZ63SQrw)vx9=ddnPqX8P0xB&BpwZ{DCVy!h|4pjAe>zwJW&$fI%b95|M zF6v`1q+d)I#xmusJ|>%HxzJ0sa#gD=wkF@&We%mTRnrwTx@88hxEybFUN{x)F2dU);M~<}W=udo^O7&WOEM3fw)p3!^G-~y7F?B_+mh=KOAo-HH zlRYV9YpL2|Dbq5?)9O)mhBuLQ)fWiHN7Zb7v7#Ske`{L4R1XD5v`o2F&(Al?jhb&n ztLo`O{zct4!kYB(Sff@QE97U#DvR}8xfId@!LdduU#Hr}GU=Mm`akJ4cc(JxLSY8k zlRDnaw#@tTb83-Y`$p7!O)Zt{>bY`BXPwR6Kl!2;gWf2$Otq#L<}_P*AJxD?v8JHF zZ+bDx?6p*?Wj=@x)~J?vVZiYnN>}G=qw1kUSFUAiy!jtpw)wS2zRud8d@=W+xq~Y7 zRrOk}g{*H^5hf+~(Yt+)!?txZ7!N>!5Zkw|gYF=Ui^D&zIYW37gx$ z)_ICj% zjLiLbH*!T2S}e$sD|fv6Q9P(c6D#7~7P%L=NM~P@*8yd2d9~e$NEi0UwXnb}8`6aX zSPEk+Bc0f~-IcEAAmAb$PII*P+E%)7D56EoFsvm)u@#PdxypKOxh@JWM_#SNs=DD&5O@~#YoRE|Gdl`K zLs}@(i-P9h%;|H_w{k<~-e>drH9>Z(Pl6$kYjrKT34bCH(t^P-L*s7n0e>K-#RL42 zb;&h0fkgt~yGxDYtX@^?IbCg(D>WPFfIpPz13DN8V_roPJ(EX*{xLmWEi9__YPwXL zD_4sk28p?$f*I4(g(m}=J0V#0c@hiF?{jYT7q6R4==+Q5*)KJAjBAO zf|2=uG-z&3b@)++t`X0R_(5&HnD2^a<%g-@)#PjWQmvjYWpp3Co&tYmRISykBdT7g z>FV6Um*z^ZsK5^OrS8J792`}zXv9b3tb%=$UXQu2mdWYF2uKk(EOj+a+`J33`S}#8 zxiYHG>KSm@YN=wnLc9lwc&U1sad>>sJYCV~P2Zu53mQ7dDqc|YC6#d|Mn@hi)asW; zWR3rV2>K*?F>_(!#HqoWW+t?Gy>8<~se*?z*Tp@e!m!qBM{W143iBgQj2~iK=nHl5 z-57Uo%G{JE22jhqUp5%W9tVaOI2&K3ff&YX7}b8=I*@z?XRGb$Zf2i*AliVtQ5@Sv zG#GK%E<&M%UyDWht}>W`joDf~UvJdes))tKzA%O&p`aEEb0(W@4U5IG7W;$!7F^33 zwgpQr2X5i8B_A~JNr4|;Sfm+FvzqTHSOgwhjy+B*5AnrmLlt~+AoYO9IEfX}3_61J zS=N$Kl^`5FDB5&T&f~jveK3;fcXG-&OWk*8KjZ|&o9<=xe<07IAsnS}7UfgiK0gx? zEg0@QKh2>crD!{3#RKe+G zZ7q(4gIe5yQjzEV2WyMP*>WMDd8~>R39v#wO|qVFF|e_3lD= z6u~YO6kCW>4|W#v#eAKPm9AbJ!?Xn`ny{iU5 z-CXJw^#_7MblQT0mM1{Z_%c(IjYg75gsp_91wahcp@nF(JV^w#M1R_F4<@FNHK&~D z!FV7V)j&6Q&L~#UQP7>MI7IBPmn&sZoOP8Qs~M4C;U=1qRgge>X+B@l$K<)39i8`? zyE9l+=nO9Ck(L==?f`Pop&r5#$M5xm)H;R-aQl8Jwz8 z7mk4N8c`1)Q<1*R#PyO$YLrN-#pS}4B54XRNWKM7i6p@9uI^Ktn6zy!kAAf4JPrbl zy;~`gMgV~)63tiAucd0$OsaH*Gioi(qDL4e`8rb`;2ru`o6MaIA3N}81bHMXOrc5I zwRGW%e8Wc+C1A~|l*{Q_>$?zGTN}(@?+p5151JpMF_3y4`~zVZWG6EsxE8w3CT&8J zBhDM9oT5YkPZ#mG6g;{G8izsy6Ec=&=@0$((=$N?`hrQxCqJ|Dl@=UVz<3AXMl4YagD}~?qD0Uh%wCqOJ z3&*xk89oBC=3U8njCQ*LV-$&T7=J9Ak19z-s#)hP#)z}F-|l_I z_;EI#VjW(77cq{}yY4Qrk=eQ~+vbj`plz64Lg+4niVH{M*d;8qmgNydLt%e^Vyi7S z>73!Zh)+0*swI}_xASRxLomW2ZC08fxq%ik;g zl$7m$Xp)u-S9Njp?3M6gWQgG$3OJ2Skffin{;Uwo4*H>KLCQhEBPd=jHT%z47~4`a zP9*%1IO5)U8JK%wu5&4xyB7Y)u7zy1A||ENY8CWjK?%kCFDQH3C)}~BUdlrL*~Nwn z#KREY`-Mu7l>r(heFt!JL;ZzgI~czV(sz*8-7In{3@wJ31KS;sA|L=_@xE{^tJH^y z=>r20s(=ZvZHj3)^n*9WEjgsJ8WiBAoe9ajHB$002f&Ge8H*1V3er3V&yfE#o_?;C z+{RSoceNifEsMdJ*eQB_wY+9=HS(GKHEB%^qi|hsovN>peR1$0RAR-5eVZ! zmSM>rV%fvUHiypFt8IYEO-a=pfb~er^yFKBXFNbfDckrcu8c?UM^?j%Z=J|ytJ4)I zXi$qi1UKnPE>Sy1lFGTB1`V>BA7)U!$su!Rqg2b!m-MWf$q}o~bKFubUNAybZP-%n zA-eYRiY65=tH~U|K(~5p8(_7uKCrQxR0qs0*Yxxi>Y~$lJb;PJA1@@q>Z5vYM?${OE>D}dA+2;ut}?C zOX)Byl7R4o)03wr+kG|%App77dVQ;+Dt`|{v3C`7OSzJo&7&`51-7X+3p1ovavWDs zwn)>}mRfmkuBO*pNws0?&>VdL))ix(hIw%M>GS8?l^F%(y}n6NzV&s*DB}8+mbtl> zFBSvQR&oyn>Rp4=1MAtR&j-6p2KMlN7)(>Km7{cd~YHq}Ib zml?R8mb~{~>n2+AN_R)jyy_i1BZtJePW@1gCQm-al4S;k;lmKV;fd7Q(=+y-y{CNP z|=Q+Q?4pK8I{Elbq1ZHt=V@;T`%!8)R3%J0SVW zZh>YWyumHoyv{(36+3E&)l_!0@qSGoV#~8no_+fG#97;S*s3gT{yRlk@@|$b#k}XJ zj}_pmyc(o=b<=a;tv2S#nU3|JPx+sg7e6J{^4?bh<7N)kW_b0&Ga-Oyva%)JB|5uMxgDi=C@mr#8P(yl_f zIGatwL@?!h=6t&D^Wi9?O`krO998M+{Dr5Vyf85dJs_^kLsFZAxZ}l0t~E+oM_Bc| zc|K%9v0vjALn1|jo{>=|yXnivYz~?`3h5UYA^m0&w-!^JBcoCXO{$9*1DBixY5zF5 zI_@Xy?dNbRAYoz4JjH}`T>4d=C!q-j%y)qeF0~i?HQs}jrIsO zgZz6YQx0hJkj)>{^MESvwMx0-^N(l@s<6-8H+j3&8E*zHU1h~vvN_&xe6Ii)Q_uww zS|>JoECMy1SgcJqdD@2k=gT-`cN_NSHXE0*Qb6NZbN#}-)X8{F=#YVl-z+v$cj&k{uxNN}EAKDjfo8o9N z6b;6qGt;ZA&}~zk`}}$un7DelPP%*>4Yi@;6b5Q1983se(I707v`CC|IIN0F-FRbe z2J01?XBJr>7QkDyL;nQSxWOBl!@4h&(s?Dmz^EDsCLG;8(3a?`dT%ro1G^~-vva80 zb%e`;^&X2xVWt8`S9|sB=k8#8Dg~k!dYMJIXjso>>XX2;oxrn;fne+sknM25#Uh$U z8xM=3P#1<8D@PUU9vq7WBH_5^5BKUQ*cYlg6hpzCSWg660s?8`KwHzR^|Pqp)C&!m z>!vkcoe$@GU|9Q(>j}B;F*cZc3_>F)zrt3rcb$p_qp&9r#`<#fjm^+Dv-0p`u}~1F zF27ZIRIYm9-bbOt9Pg{BYwYfY7KVQgzbL}6U#RZj=oySa^w`&c(6~mei zqQZ#MdK?jAS|ij0Ef#@len6ajZP21{OW}|A9*PAti@d|Zlre}k#Q2B74J4HH3N=R8 z5Yb|o$eA-YXJ~Z0?5Upem|Id@H=$)76KYqCQ*xYH5R9q|9jiqqBjYcRXfRlARP_|} zTp{)AHX<90RmIgdHWXizgL+Q~LJ(R#R2OT)TVFP5u|S78*6d)5v14inL7B)8D! ziju)o%cUz^I~Rn(?P~NjkMSi;NT%}J_MS$)K>OrI%+7YJwIp?S_BHy9+i+*cSTGJ9 z)_A(Rw6|+5ddg!AK!~~Fmw97|TInwGWaN{sx#SycjaTa!bmJ}Zu^3MD z!C0>?Csyc$KM4OTeV1Y5-`Gh95H%5KocbNMjChcgQI}DbGQQRejSP^Xe$`7_#-m`g z;!%J4?RR^ife7+z(Xc~L?(Hs|e!|Y&!Sg4e1>=C`z&u+9T8PvL zouP$c=Nol!>DU8p2R98y10sUoz&?Y-BV@;5yxV_Tv^&`;5e>qU*nzE)04D-?EVQ#% zIsQ1-0gWA@#Ub)b2rYOgXm9nPTRez~6cZYuR&^>D)u4U>|fMTE?;3tAwM z0B*n)SHC&d3k{apV5~;Ny)!lro;Ym2!FIU`XrT~3Ldanj!2eeaD}$7xu4dfgfe4&t zLCLmvrNbi#Hnl{cKdb8Vwy%I9aT@AC@0^06cMN{SLczXkQ_m8DB8d>J!yQ&SIHG{1 zM0>4r>gl&_t1uP84iOGGeT4+fDzFvxXFmR^4O%RYwFnBt!altlL*Q}(Ou1$Dn>N0Q zi3QpzgmJTKI$NK>_P+0KWA3&&Vdz>i;{Xeb55H)Ob^1f#PN>)4?9ljgTc6{BL=v2vi>W+<(RVpLS^B5Vm>Hg$}q94hJ9?^*i`VmTTPK_dp8+UEyHf!djL?2xvg@ zWtQI`NMM~3SjaJtdUn`29Q=`&$z^<}?BL!RROYIo`(uQo;ShX3IQUGBhvWVT35olk zpEg7B&$_{)DFHx>I?XlsUWL<*M1Rj3#_k?yIF6yq!fARJK7%igX&4IjUkC29L5l$x zW9I#Z@qOF3a)+Yr0WIY=G_*$ z9He$M6X_nMpR>h%lh1iv4w5S-SXVKD-IqJIWJuCMyY>p0=@G>8p;hhH6^Gs0bw&Y& z@SAmhEVSRYLBnYwj0Qlb-GOluj}W7?zf)-A8xGK*)$GKe zM>P>RKaKQvH``$%m`)x)m~z?3mJOs_=6d{Co5v4#!*Mqpcf(o74afJsMNcFp0wOjA z6WqpwCSTDBvFka=d(558Da&!FMXD3iGo10hiwi)pT~H_li)w(7jM8=`oMAbE+{Xjk z6w6ojB8+-f@~G0qf`?d@M94u4WTCVjkND#dh(!ADcjnGQISrR2!gsbEa2yQ$sFQ3Y z0Tvfwgw(#U`haFDf z7-uzwXrnbG?9j~VfABJg^o2^g<8Y~sUJxR|)X_m81R*oYjD_UMX&&x{V+KCBj-Kb@ zfDpmi(ki=>)xwh==_M^Q##LxpU!{OW^h9h>5n425N#Sh9?CT= ze~1UsJuVP+Qm`8*0xFgWkRCv!zwCY|hwMBcP&0u>wF8zwpNCX7mKl&;EF@Nj{AZNv z2$S8lX~9b)00%-2>uC%dEJ=F%j|Gpx2C)Tq1m6W1S~FNl!@^FgYe@pOW8aDOS0yp` z5LQ#@s%p8Q&H-b(8w~&y%*aXU1uAsdX(eVlAfWM2cFGb(utGw7-a!Y6htM{G>Y-St zoPZ6IcL9p%AkiNsfc};vfCUsqG+Bk9+dN_Mg~I&3oqFJ6!e2~1aM53U;9@9R3joe6_)GZ_jxPg-fb3)@1Yy!difkO{*0i#zuty!&To+MKd8m) z?lj2wUj+6pbQR<>iCiX;%OtwHnMCqb1zrw1CHz*tpme9|kF zF^oLpe*8(+jR%lmS2X?^y=Oeg?hF+Pd_Z~N1Ii4o^+BxQl|p zVmwUh#=V3=w>J%DZQId*ymgm9t|HC4D$=~4tkPfMz!)9Q z2ogj5OzFhFF>CH_d*9|RQd{OexN@CCoWYaKWvkrG8A0 z$lL^<_7I|g!tAU{<~Dc*KQeuCa^i@;l^e21?zdAQX9_1sZo}C=IH~N(?e+Q0#44Q9 zR=K0_v09^2L2SseT;>H9;C_!6UP3bE%3{9MGJlRm$-*NUmXC)hL^ns(7aDp)XTH&S zU_oSRxofOwN<4$fyk5&7G+sJ`5Q|BD(KpAUPgA2`;@&z?hz}Y?x`(fiqwp=Pm1Y+~ zhr{AtFQsP@t??+l);@9i>?!7Rc2)5zW$!SYDRO~cS!_-I%|I?}ewtM)JeicMn&_$w zy_gX$Mn+YOdl70)x>`*y+TYS@U<8jKg5pE!i+Z(;kdGP3f`@0>a#07%4#SM-;A$?7 z=q=?^VG*2q6`^A48o2^7M_2Ks`1mll!K!&*Dn-n&OuAGq<>7=L4ala*EnUlmhg*tL zc`<{yfpZA`m?00p2vvz&#O8(jclZ^qYbcGb&7w^{h?Vf=3;9`%U!{uWY@?w2<{o{C z{HMG^@t~PAc^bu+7G8N2J^km@Y~#{aoDd%{;S>+xgi|~dhX6De>bdsX~30$OmZ6 z+zmN#5JuObFw8C?S!0VCJ9O=8KOu)>y^2>iwYZHB(&A=33*4LB<262xbVDvs#!pZS z<6f*bV+_$~W$d%!VDhCP1{+loBVt~}4`nW>{V4r1M^pF)$1D(NUlGeC{D1Sw>8ofm z;%$mgcvh;S0raSPsDelkh$1UqhCkoh(TXfecPwHn)gWWW>IJ9>AcM_w)|5rgFT4oH zz`1f36AU4uE@t32nA}x%WzLFBz7U<@p1-pfm>+Je4wV9el=uLf_JynYjDD0SR@d80 z8-bnJ95nulczMHOMO#0Y-(4i~sbK`0qVY#lHH?%8fx&FrjVUo#OWqo4(hbc9b z7~0xC#wl96$*Un9l1119`zcvCDmV7fOdP;!);fMK__Uuqb@mAeELZ$}984wqZ$Md1 zNqeacb}vKqAdU_Nz+sVUSQG!?>l7h|V!TkK78X;5<@2B@ih}2eWI};RD1b;i)(3kB z(K(bax&PfU*uXd-=AoFvkm9o6&<`jC+k5YcTZd%F_^ZerlIww1*B9!FRxOijMn$XD ztM&A(0v`wL|7Hn*GUzALcUM?Bpe)Z`ZNDa~6X`g?TIRPK&#=(-{KiH3dwZUAXHa|w ziq-Y{X5}&(f*5l3^gNUftgHmK2*zwK*MD(hvl8 z%iJwj^T`vAE4evHo*&R6P<<3*6fq-w#vju@u=V`piRm;{94UZZTN1x(dMb7Nnd#?K zC!RlX_S7`kC9_ZYQq|9p$T$``Z?Eqqx zQI&&4sJA8+kGZv~&)0Zm=004bmh-1BJaOXbXU@%>y5Mg$W{v-bMzx!<6EB=TH$5|* zdUoRMGcDsCYB4pz_;Xw(J;q<~*>9yAC zU8&Y!Pp$i=rx2d&&lNCuUARdumM`9$2aBP*2tO$f}P|oJ*!oJbmtov!_qY ztf}_Fm1+<7)V}-43&$r@Pd;r_e*m#uUwt90`8^z<5Q-P}FKtgE(K z38{4~N`*2GZpxnS)oLwwznGOv@>)vY{blaE`@7)w;1)(e9(Zm6gj@7|9Dx_kfqK1( z-&$8Yd5Vs4l^Fe$0nmCMREJ3Q=*ZFjdPk&2AYPk8ENiZNBqCUMKjTrSnDG8_RNU-R z%z^yfcj)4RM%6Jrp#@S8s-~~uzIJI;od?GzT}To5qxA;3vOY@=MG8j@sIJGI45m!b zxmB4i!MSot=fT2RgMJzMH&Ldj#Pt-=8mf3ym9d6L)obZ|{V4rWp~ZCa)bVGYRF551 zgO*_L$U;g^MDm&*EU5t{x*0)d%cz1(4PHU zJxk+k#oyX|P|B*veR%GTz4 z%7*~avlM4=3_-_O3h?f=}km*stU z@gMh3@k=~idX64b?cyu$fL`4-n(0yJCroWb-1}xQSszETbuik>=++z?pI!2 zLaA-&rVm--5?orZm**1Cdk7`>vXWEx;l*)eDO-k{6AI-4*gce?Ozpt4Ey@xuhnqv} zGmbR3vUIq)o!(G9s>ae6l;;PD6FBt;DDnxG=Q$3)|5|9)FL%&`l?HCu_63KVyBNITCW%~Wz`MWGu=W~&d)`ca*_|@YarPP5I*hA3g|{1h%CJF^e>~%SUFbi5MwvWg#q+_9i$J&s z8Q2?sHr-;ldADV5&chE2oC^Jnu?L5mCk6fGZOKA+_n!6zECKw1b zNHt*ubRhYuT+H0vfHyxHFs+!*m)N7Wt6`z;3~~+;(4c#&h{BMr`@cKJ{e)fu!B3YJEX?IcJ9YdgJ1Z+%AjBE5+J+rGVh+qn0Cf912^-u^nJ*Dv!H+}N*Nf2Y%! z2b6nWRi0T=8iUF+pq4LlYFSdW9w!Qy&9Lr2F44=m8ZOa$mLbM)(Zw>{cwC})xasnVnw&+oDn3ycD&1zH*V8i2;wX5_1n)arU4l3161*GYPRn5Ff>UprpQ zdx_uj-k#p_^Kb8UrI;>3cQaF_3k6ps==`gdV*VsMpb&726w`D046ClMz6~e%K7Fea z&Uai1r{WTPDVN}L3BD@xQw~L8BJM6Q*J`(s;6pGR1XJsA=m$ks_zV#qLsk@U4sd!; z5duy|?|s;%_gs39_DW`QJ&!zXIs=lk} z{iC7i-L+A?-}K%TnS*fK3y=CTYnr@&_g&ns6FsQ70zj7@EW7le&!q?NBs~c8#RwuZ z@xV-70>FSj7)8|H-k>@+XWQ+)Tl5-O0EBOvTPRQlGd_QWiB@f1-pR1LcvYBalf5<9 z*Jp*lX6{VaQXOmWmKn2}X$!!<&I>+U7zz<2gF8=$s{3;uV+a-*X3`2qgtPFCXkC@z zo&I zbrkMK@VeV0cu5s{gp-Iv*Pz#zcp~UW%xKLo!`Q7cQchyHv(&i3B#47ZkzU+j67X-4 zF3x7tYNoz0TFaNN98oFW4#uaXSAEF5=)WXFaA2jxb68@rP#KRwH8nY4j3O}(;}3<& z8HaAjq#K*IuBU4Cbe#taGm;pTbF$QS2JSJMew!`C% zIe1nXRTs3X&O$3;A8M2<8g_2vk7#av{64H-{BZ_xM_9vuI|E)fsJ z34u9?G$?l`^i+e&Zcs#h-`pQyeHC%xP{9R{< zi2Mro+M&o)78Bi5WIL())zl~JmDOJOt9q7=i_Gwe8bm7Jm+h4GPNAF}64|EJq*NFE zkBjlLr+Y-cOyxJ*f-?UPm8(eMW$elGFdO^SZy=OGu9ns)Q~{h<{rvZoeP2e1x%t}9 zzxo?mI-3QtT}#y}uv-5m?iA~nHAEOdggNEQ%pcd#uW47)1%ySsk}qX{UHHx7yT|Wn z^VM>rlB((RMZ}Xz&E*#mxsSYTDZj<)LpAM(*nH+oLtpF3r`LymRV$=R^9{rc`<16F zdg;Vj^~6`(S*fdt?3XW>KI>I}8@8(pskuVBp1M}9UJ=y;g8F5RB8C0ZsZ=PK3jcep zQq5mY(o~8VmFDBaHkV;2HW$AB!=_|i` zRsWNR27hHr;HXRucT7Bd?l4>jTPfAqI{c59w}+ z2POB6r^Ld}-DN4ted3FX>8ZRco{M9EqKW7K@Wc1Ybonddxk!|6%M0`4`&fc=_Z^^FzErFp%Ys-){VT3u7Ma;l!g+FPv@YZR|FB_i$Ojm(wwJObk4 zxt_}EvyJ&wzBE@(VbvtsB2})*aJ#iJgu@*x6s{HzW34j~zm?sn*-$h(myU<$LUY;7 zY&JX_@W;b3eKs4(&PHOvv>phiL*cpE5w^D>1S~cTRjhWR#q)5l=*!jXmD-W9G2G-D zvs$KH9J`Rt>g5vKSP;LlQP;bwSFz$Aj%r~os0Jg!VEATQB1h0UW#7Y#RY;r@IDzSPii=~_+^e<{CS?Zk!RQKH0Vg>1c0E#uQ=v-og|He0JH zG+eZmBZkyskzz<`wVa41C8J8++#;Pi`TV(w$P_x1&3i zrMV=_%lReK%c3C7FRDxKA9CK2ceVff-g{+PUd}I>%FisH%kl!OGK>p?EuYKzC{ufR z*?yUR7-iunu3cWvf0@epk5|~Qy|0S=L^_6)%FFdho*&v7 z60j_1YXIp)QCaE(`^gTAfs*HFxxUHqliT+(6rwy?z9Py?I^YfZz?Pq_dc>`=mB(&L zKUv=XzWx3J-laT06|($f`Gl>2yp`!il(7#Hfq4srAN>}1qU=IS@5%m4{L3Iu_FcTn z5}()r8rDo)%X(#Y5f@J7e^HdTpQ-HSTgHDxOA(H337}QuKjEg^#g&Z4M&vAu#@42R zqp?Ut3v%x<7AYI(W;}^$5gCrn_!9u_qK>&=8ISR&cwqcj`osK%N}5jgZ7u1e2WdvG z{rBik+DB;Zu5tl~Df`UC%+GudppSgKo`UY9T+ri({heOw3@g^3|~C{Q!MQpoMrsuOm?;+etP^93=kQq z@=#z8b=^e{ic?6KQa!0Q*g4RL?kVcSk%u5-uzQ$Nm3g-OnP z!yd8CQ{fS=+Mnh71iurkYO1`Z&%$8PpQ-P$x>*3McBbWG-P|7b3*M$I3Ab)r6czRM z8#xNzX8nY#v+3`AVb=}DvVQEX2z{r5V6Dn2w`cqbhK+sKx9^WF8@vtPR@Wu?pk+#Q zUCRBtE7Nkb+UK4(kFrvI_J0)*-KoKMs5^z9B1qeYof-^ZR-7Mi{}b(xD+hn=7o5%N zr~VdLM%z0^mTq9SYDu^$cO!6bxCq?0%Ye@{Go;Yw$^4u;b7A7dsmW6_6YAq?fC9Ux z=;q|a%!CGG4Ie_s*HmcesuV8%h$`(uyBH8w4wmaW0PNk|!^syNSuv|;=G;1j4Z{wS z`jZiVJQ3Hz4*HW3Kl!r^w)H3b#N`wGAQ6}E@=gmss9#(@mmf6T@{QMhlGWipkAX?nwiny3- z=Qm#7>GB)hFe!I~TU?jl2)kw1RVjBGTdV2S`dN58J@rB(UGSybzE4NN2c2u*aXqd3 zCnvqR2h97xQZ4HBTp3R2x>%>=jL`LpeCNC(L;VcCb^=-`*ol(U=0pYpvAE`6CBIRG zMO_m7#%05zF0GUP+1sxEnHw+trXo_(v07P?-}uh6tJFVxV}b!}D>}U)2@0-{HkEbY zXCe41XLa>uY^_@V%pP@j9re#x)LpCc(niofyYY~+LmB_!4GO=tvKkuuG){j!msOU1 z4^Uwbc{=H+rg`kDYK1j?!Kpu6K|k$h)E>?w2gT_*s!d6HbGXLAD36FpXL~L*X2hx9}{w_4H+AHVlg}#w6G&TG6a5Pj6dlgwp-{w zim=^X{K$S`yIp?d2V6A>$D+3FOVHe$%pjDT;x{dr>vy^=<*wfo8%dN2>MWa6%|**O^Vdwnu-rMmp4NAVCX-({6-)< zoW(aEX6bPAqauAoq+AbRxakx3BO*N{QkUP@q=*Q#Zk!V-chNC*2F^WBL#n#mZzX=? z)GKX$NYD673=&DiZTgX)zEu}~E6@u-|`wm@P&?s28iXc4df|@U> z)$}#o*Dj5!B90q{A3IuaR0_J!k~4ycvE@vz>c%TyOOj-5bvyI}lFzwP?giKQa=IX;I0KTvaG{io16Bm^yR% z-1Dv6?vdP{NyU4&X#HP8M4sd~jUYS>nvZZ^dx7?fdbz40pk`&U+^GACR}olTFU+y+ zh%uuLR_<#?gv!ONTUWb`UMyD^ujch@zS+gPo~~BYi#`UVMEA9k5zf`#H|cu*beEP7 z(9#l(!SiRx$<;$N9`b7;zUV;I@g$~Rcc`o+a&pJa-HCWG7K%rpsU6h9F|Mm^JdKC1 zw0!2?Oe0l6RL5Grm`$;1)k+ST+t@7&irY#qncLYdy_RXTl3sH=dyz-TrPn`1ad>Nm zaviaQGu3oGlVdNc=018r@of?CQB=}O?lBK^YU2;|62e951bSB?3m`a%B4T9f%4{n+ zZf+?xim8Q6rj_)V`w-wZ#lzpqK)|gt-}o;crC}Qaa~sytBhx1*Cyw}AM(e~MY;funM diff --git a/extra/sqtt/examples/gfx950/profile_empty_run_0.pkl b/extra/sqtt/examples/gfx950/profile_empty_run_0.pkl index a5225b04f84a4c3c677842443e24863bde87e298..8a414167fee8885e32f13a33f9008d4a272c2d0a 100644 GIT binary patch delta 1902 zcmaiyZ%7ky7{~8-ckWN>{y58r$cU5_ z435n1boizS1Yh=I)Sx#)Z}Ls#o1z!JsCN=U6a;%y^lX|gX!jdzckDjT=l6Y{=e8FO zTkCXSmR;$5L=`{7noqG#k5AniOD9wQlzel1RMr&7VEWqS@vAai;Qz_xW1NlYj`U~> zJC&S-=crf9Hi-*0>W@pOIBZnd2{}Eb36xTCeH62p8RMZ);yhoB!NC3BdK)aE zw7v`KBNPjQr4-j$A}|#M5nhrCjp!Q8dxk76ci50cJw4;D!ba>6C^(D4e0Fk%V_jGf z5bQLvw^PGDfq+=JXJNfKx>y#X0a{bU0(8GD!~pblrD!S-u>k#g>}PaM6#%tw5Fi6U z@-qTt1ZXWsfJ^|r*|g|qG6VGaH32#V(9gF7s1hLa2Le8-_r z`Jj)XDEq9{qHgBrv)1oVY2EQ#LHi<4(Ao~PwoJf`=5w_ zpvGCKSyi_;Ugnl0^c?A~vmO$4#7MatfZ{KZC>Kz}Bc!Rjfm*%-lx{zsgEz>NLATL- j#xsFx)o(kiYT!Y0&OMFZ={M$uWAZa_Oy&OjFZ=%h=&m{G literal 99408 zcmeHw3zQsJd1lQ+vYw;SNXSn_Ss1%VjB5J*5FTap7z@!FNtQ+|M(C7!x~r$V)zjVX z$7n{ziR@q}a0<*qv*B>Ei{JnbCIM^$F(FxeAiD?7h6O@$4tv7cC2YtpSps6SnY1EYFv>Nf z-e{YfkJYRBV%a#JsmvG?b4I1ves6pDvG$|LzB*^j6|+X$>_2&8;y8jzX4YsIddziG z^-Q){nMoT9cJ2+iVxv~hEa)tkxyqQWH5X9mtg+B&51V~;quD-U_BN;6fq*$OTByz% zqYr1!XUe15YR(uHU2|L*l}5F$)fU>5&kmSt^mFx04V`YA;d8}iLCw}$dX0BJVD~4G zuV^~!g02sbs5_|VJBD|+n|Pl4FVm>eXpT;2vS*D-ZnTLTJ=Vbl}YnW@h-M%0}<&z{RQc=um(+3q)5#U|^2 z`g~!#xq%uC)Qv`~+~h6Zlb$Nvy_|Lm_grfY=DnBgDITw4u<4V&?J} zbbF7v26fR}!uN6?8+ZQ1c>nbL$9Y3EoGg*z_+SLnMH>Ag-aU=}CSKO6E5P$?n|u`- zxPP{qYn6>X$RuWpMm=Den(fxK+1qHg&0BItwm6$9Qe|Po_dz zBpz!&(Z=W6YgF2WJ$5Rx1hsIGr?#dGef%>otS;PASW{SM-EA~Cc1-hgjHhwclT1Tl#F_R`d0a@1~=rKxRbE&sWCEnc3-FM%8z#Cy&;&MzL}>fLS`0 zW?$l=O1>Hx9v(gwJWU@PX82@p;m&1zH<$v;|7LE~6u$P4ps>q-b{B$^xAzo6S3F+} z(eBf(Xwi_Cj3n4JEbQw{O2)NRD8iC1GY4-Gi@ltEd5u;U??QbY=FT-Q z!7Fn1VTsdX$&lad3kMTgDB_)cX8+WoBM-F;Ys}4WEE?ymZCyPG#8GIwCU+zi*Fx3| zc1`YRFs>ycDevT_nm1Kiv(rXhZ59l*RjoDL^+rQ+Eg6dXbvqS|YOxsqo;p4ns`oA< zQ!g*5&3dNN$XDyL*pmo1sla*O&Zk0AEfVz`n`9^w*5War-GLg}m@if`<>GlmCB)+D zI(i+{LaDgj^-W?&N`~=NiLn2OY^WOmj9EjSZspy>kqn1pS~SUD6&q%|&S)sCCHO#j zMq@2|fBStp8I4A@V9?RY#ceYgjbUSq@FaJQj*W2(uyWRDG}Mr~ua(c^T~?ObdID;^mLx{WEa!bicV)w~=kh&RF8tIR_s@lk0oTp+>e~5c<#w zSL<^bq6>~s7iVo1BMYqF&}TC>LOf!g>CGy`m4U@wdgrNm4G+f}o>z+%mEob0 z;n$WM&C|o8#rG40NaHD{j_=(+F*z}{SG^x2iq7hEb71e(UTwx`y6~@RA=iAzQb_1k z)O%Ntc}v}3leBHV5o@L{o-44^g6Sf{e$c>9JcU&XQzI&(VO~j%*m+ZjZMuT?TYopV6$o zTHsNlpB0FHB8iw54)HawbA!M_6V`&Mr2ld_)M!mNn#E?T$yP+HFV>=HuY9pkL`(T? z5y^NsrKMv23$0@by9G-vuh@a=)2}pNp@X(AFVHtm-!XKTFQbruFpLliD=dH7EF_(UYm8HsZ=7A(qg{UQz{utYO$afzXGHJx)Km9 zg86{G@?0Hk8u#*%N+u&()MsIe2gAvrh9~h~nAT%EMxA_X?LfmQh^G&S`HI#7)8fHM zFpPE3zslRNUp88^fKY@g02P2LW{hHfp>r^(l8Ba!J7AzdZAc0QwMfc;$ukGode*&K zpv#2CDCu}<@nAHZ)RKuVYxD-f72Wz8PA2idd=TB0MlinJ24j0;VRpJ&E@od_2M)^` zdbV0FXKD@OwK)UJBOem63B-f3L|BXXK=^nt83Fqz*##MIEEi{sO~Tst&SDQH$aRdf z&(?xk0i2^&6ok9rW_p2dXL{U|!D79eYqev=Ll6V{?Be~Qk53AQw=wJIparpoX!L;Vh}pBY;Q zO=DDMiWOs20BnYVhs;e`tQrKSEdrx$MlXjacKCE;$1rE!m;Ml9jC9M~TCScm>a33h zRLW>I1C$-*7eLUiCIjyA3*-UBo5gB{6C_~0f^dQ?V^2HXXAXeUsX-c&0}yIv3vF}l z{j(+G-tbonFP* zbNM2=zu4eD5Lzr_ZI08MGs$}Y^^dkvKc5c zw#_Mvwy|2%(HZSrrhHc92oM2@it;MOa^lpP7p!h)Z+UiWpkL@Wx6-2v84rIW*wYxq zB2sDIpA_UX<(=jZdY5Gb)6X>}Z0fIVS`&y7vZ%A;`2scxj5j9cu)wY2o)#!R`9NDk ze1^$MERx39RJ~=0Y4G9x!Z*x2EfUW~2s&xlO@y8IimFFc(Bh!S%MF^>OFqqFe5h~Se@^Q7x^-H;<9^UXC1U3E^e|!0I?QytA&8rekBBMT!oNHa+8LNHBBn91K zrmPA%jH|QgYeI{A-x=x!_nmm`{`d=;ltr*0A=Tn?D0hER7%~jsp-qG#Fp4F*)VncX z&On>olA5`D8$;1}%rnENx3q->gZ^9;a}ZJhaNr>Zu*6ZkVlHx_((y2`vtw&=q>$jL zOJ~;@^R-MR2VEG4=#Z*QLL`!MZgqBjNhp+Ax?k>{&fJ{I<>F5 zY$Tq9Pb}C)BD=8;2s=mWR3wG-bQfZ`lvC1a*P2QtK(BO>M6Rd)&eU4D;;efDB%!yh z#X}xy1U*fLAovV*dsSO@?o|0OWO5BPKn4`UuKuEeq?WLh20OpoB;;ILJlw6nu=+5v z5V?Vg=B_i6f^0O&htE}Klz4ZcE^}ZTliH}@i$m6%Gv?J?bD<{W(~IMnj6zgB*Z7RXG(0LnQPVDIeG26V{trl+`n#+S5ZsG?5TSID)(T$R~uL&AhULlNHQG9 zQ2Kpr5Vi-kV64lC3@|;qEX+-?>_VZx^VI1)<6u+(=G$8KPICA2jm&*@OFh(c)qJ1~|VCMbFlOu?3sW`hA?L$g-DHdRkIJWJy=TxU^FAFu#krMqey*k zJ3YXRHU<_G*(cIAR~?wbr$Pna@FIR`#q6J}7IW<*>7FA^n0XABeya}*%yqVT1oB$8 z9Ba&eGDc~e>oW7jhR(maVRKUs?^16y44o~bjrQBk4WuqXf*zg$kHz#j@2}NocB|H$ zI#DySBrde)K_JtB4(|5@A3b<{?<6$qxP}}FQFq|ZvWtO`b%dgJv*vmC0*Hh~jK&*=Xqe=# z!y`m!bP+VXrQh68&YWL>t_hQPx0wbiL2|4|^@UTR(;iv&xB8bQo8Zs+UUL;&^V{YG z6OwZwbA5&+nHZ|TaW<^fQ%pig^kJzbMB6MY8%}lwc3XqX>UqsyGMN&N;4ihI0AEC{ z;$Ev&Yk}af#%i^gk3*Ap^+;B=WL0al8ghmtO!KbU)Cn7B954!W1*fh$5D_*SsDmYJ zk!8=Kt?jXDvC{0cwJoeR`}R&AAVRO-m^ZT67NDiyPAAMhh_abA?i$#IW}us%!ScLD zCRsE!9@D*1~drx^1p5R;gk2!Ey>KiM0x0%@gB^B( zg>TzT<7>ABlrMx+n?6O3>m(z=(O1jczU10z*Jl|=dJy|G_dL3J0$?{=R86=BlGqntC2#5vf z3+2XEtp+X0(L(lNX1ley*SrPrskTsrJ^oWna)u*NRk9usWwR<~{jid>77quzSyX(ICu!b8OZx*p(qvl5eJEQwu@|pBZW+f^8CvGqk{0_){SB;0g%)7R30cXt_U)ZgoUXTakPh+s_#|IX>^O zwJ@9d0O}@CL4aY52Tmr+1-a^95zk4k`o7Mng$FlBrF8PSY!mn9^ z|JHPAj8<#%Q$aM9#jJJ{Ri*C%!4%e%o+KdChxMzy@63>*{KmxL2YP#Y#!&_^qHBx;y`+jt;C@gO2}mZKu$>v;b0othU ztDn6{SWbJQT`?}lzu(NU@myvIN3L^&}eqEE{SsV(|TQipv)GQgvs2*o?Q1#uUV zrXmD=eel>xMfuH7f|M*YGN3)keO}{V{%2+A6Iy1b2IDBA^OTQjWTEr!&ucUFYOAI< zj2Tit(euT55NSj!DxYIzk!vv#B-~7;|FfQWmi^-zG+Sp{*xi5cXw9hXKdkP5b}>qy z0}%)5?R$Ea=iyduUMKwt{am$v)@mLN_I#ANUir|(7qnWvILFKuy}kN`)>wRq)z9*q z1}p~4)^k0raryH{CbYA;=QT2LgG(OP6wEi-qf%R+(I9k)N%F=w@tgKVScVJBu_8rhQ{oR4T#}ws) z(!XY$?*IO?y~>k+^qX&u{O<9eQ~mTu^;4SPk9BxLp+Qx2y-=*dJ!3Jar!u*4a5|U| z8L?nCAM`7E4RM>7D*Gus(97xf*cb7NXOl(o z*7G6C-@8T;xSkcqf4_l0KV0(|;{9;VlN9fRYZMJmnGHn)cmvi`w3$jvD`Xl4#rmJ( zn{`F2S4mi}K<8k-$#N6aL$pz@SMkDgIlOK~n{G4|jl*V*&>zrgake#!NK9G6#0nwG zW@Z{;0rm1L9s&I5W_N|yw>`6VP71g`l#!=H26ERxtIL&S}2o!nj^ z4i7}=l+}qcgio#5x{+-$Fs&?&$Mgda9oaj1Xg@~s+J7`JiPjL{)lH`ex=LLNrBF05W zk%#?U9f%>4%F;8?Pkf((4j1&V*eAsJS!A@b2W2Qul>aneU=%@*9}@+sa{OG)E0c)p zC@<)DYa?>!ci$iPsgv3f?bEgM=Po~kM2d5hDWZJJRY2T|kj7{6-8mY%WNoxbD3{JS}gvTHd$!NLt^2^vDB;5AC12s>b`5YCP&` zeAB_>`zG~+Cl7T#bU#J=zUCg^*Sw0goE|T+x4ac|i`9GC*>%g*{{8yv4~a+Z!)<4a zgM4E7wm8mPoK*UGlQ4Flq53Y{;u^{^dFaFmYjm%$$+gz-c5jjuS zR|p@0{K*lJBOpiMS78J``8mGeC0Fxs)gB&BzL|#?C$01%4&Fj5RA84i~VDeJmB}&NPQ9{QYS7@eWHE=qv~55GiWTK1HgYIY$<*r$^Ko zaBMPVCX^a$w!n=IIAX{`p1i_1lkEtmfNkG$U*zNsog2wB4eY0(&KhTyL0q>Ry`hF} zH{2ueV_s~I(SPJb{lLV&lLys3V`|v(RDa1;g+oS>nE(hw-Q}Gv1-n{lsSfRT=~VSP zTaCnUgdlmUayY87>`5qve6$D$VO-!bXo*TrJUBJwD=R7e4M0X(;k;j^bSs2!`_>?8 z^97PWDaxf$k1o!!!( z_yyNk=uw{d18<&B_wsb!AJHp29Pd-kbW5XNUmAVf-1lXZE|tg4{a@bx{1fk?XW8*x z-WH{9yL73jvK-#$r*tG;pt6+3-PU_}I;)$*HM?(=pVBFw%2VhrKR6)9SEO}@mqwo& z;Pv(>7k}y+_+ylpFi`5 zy=lCDd;WYr`C%_D#hXYnfu;e_LllyRyV2POuO0abO!BQ#;cY_YUj5b&zhj!L+ zTTsM!5Q~weh_RX}uXNy|o(^2ZHd3)}|El4l?JEQqsS+10;%V}1}vvODR$tzADanhwh2W+sciAvuO_6D;x}8ocT>b~jLme-kVuh`NJ65jE+LVGL^sB} zZ48hoi9_)mdP>3rVk#64iDR#p07U&95Cs5*ppOJjldX?*ojf{VKZR6wb;`%~DTgV0mLRX0TVaL)Pzjx~j`Q@_mbkzrP>kGN-5P-$6 zed>pk2lV8;_Zh#@!}W!pV){bfT2xDah^KhdQNKr@=wpxac9x@06kIR#iQG@d@5TFz z|Jd7i>-iS-iQe*aZyqP0IlJNKo;e`u|KV-kG+w{Z$Ldjz2fcYj`5(+4@1k#Gm#_Zp zgeQ+E|IICiCylqy=jUI={nOjm2Qa98QddXvP$XcGj*(w#$H=zyDIAO8L}kV*>F|vV zza^ur917@WW^Q}D9712ZtM1!B#-S$qaBX0YjD!Dr}3gM0Z5c3H8T1}0hDa)| zMSYbuqTz@Z;s^3AX%loy2WDVugEt#9KsMpxGuS)!O27l&f^v=e3VQ{RO{t{(c|%qBo7#Z_6IGSW30dzKZ_+ z&UknKBf=);3*Y=Tk4}Op|AQYN^Q7_idH?)W{W+A*Wy)~VDxI_LJbRALRp;)}wx!F3Tg~+u965=j)6W}H zAAyg{t`(?{5D!MQRM<}+As$X?L5>!dlr605zy$1ELqTV)986&ED*_X6Jm5tIUQA$@ z6}fMuK7zyq-tm{1Ap5X5nmHi0e~AgC!}dlMe(--eGyl&$lYsz)srbN7S9z?{n5}KQ#H=Kw*n{JFU-Atc6IlKc zHV_Y9V1nPcxU`Q$?P+fwQU13Fe{@-Wf-|T6(^Zc?0riSg`JH?M5o?ETK7k)U!ObVI zOY_`r{|N3ngbt$JtIisn{t5>b-`!vHrco)UzQKjR@#b;z0e1bwFP+Qt_x_VNjn{AM8{D?-$DTPP z`uDxe16}k5?DEg7eZ41-DF3!~4|&pf`@DZ2Nc*QtTSsZ@C=Y&kTJV($NJB?y==c&DIws?BE#~VSA(@J3;bkf#40K?Ift7$66c1*w$$Wts zI(KP%_x5KpbmsOl8NBIq3S8%*t#+Q^6&y|`nG9Y=f*|rGlR-L1kcN)Z&`}yX-Y`SQ zR0M~ErEDgHEwV_(l3J3pAC^>L+|+>?*h%tYO5h2WA*r$(Vs?cw4c-N!e4c`Mq1Q!7rDU z$ECPn^9kJ66Ye@BX0U}6lFuN86jDf`?io|1kU|P6I)xM~Pc{(_LFF;#XK@)ul9pJe zV#AgW%&>*_UctM)S$qb_CR|<^?EQQtU(Ga-PRQ`1y?^cz#q)HV_59rH zy?F%W_%|2-qDy%$F248w`KP~}4t@BuJHF)^gJWyBlZFotk(CDhp1KQcTKqUPf@s6z zgkgLKuV*>x3MhZ?3sfDPF#Peo-uA`&8G5rfjY>Jy6+Tt<=5eYkJpSw6^fLqWer=dX zbvo~zIU}wduVRNlSh>QaDL2_Mm-Vfvw!5Dex1}6M7>%BCis)T^0qJD z&(^iy=<#oy?LDP|82Xx=~sdY4!y}+Ui7cGeqWb8-Npo`UNh{;Bg%i|6}vrY z{QYoDTFv{XOH3eD7^MoMRAH?D@~JRJl5pefiwUA391+E>DVM|q+d448Hs_jov-kvF zOknT3LbSlCP%kZNsTy3c4}!AGirj8L3nE^GDpPuJdRm}S5vha9m0$wNC$PtjMv=zR zi76zXKpq687AtB!b50)mAP;@$E!=Hxw~ngdriA_nF5Hto?`wv5PkQQx)v$6z2aDbk z!qFJ~C^(!oI#Ln|Ex1g7jkk7Sf?H|T7IfCl!UQj;Lmwn2keEOo`d~B{Pt}o!J`|)s zM(L07rSZoYj)me{(pR4#7KL)+GW7{=>%auJIWfUo|NPD_yjPod`|)pi^9W4w@R-*$ z-C6(B@A6N7Iq4Jpr{DKJ;|rEP!Bdt#!Qi4k0gVoGui()q`08hPJy$9_IQP7_eer&x zKl7&X_hZWrj7_ih%mF8z|B>z9^ed4a{PTpjyy#!-d*?lE^ZwcMZ|nQt>d7O@Km5Dz z^rZ3fTz2qp|K2}c$_}LLK*|oJ?4bV3B0I2E3}Z=12z=ZvhU3AM7UnjNONtI|?|=li zJD1Fx1qr;O1Dn(oq66xu9ZPro-YYur=Cbl!Crd_eKb+nVBqZ>TzY|hONFYTAQgl!@ z0z0RSiWD74(Lr9r@zhiB%-9HUr;OVsU)57cn44OaVv{^JYO|SQg{3vyu!wqHVUM|Z zno~F)^>u9!PeiroGS6(dqXQ7!K^v=}u~rTsus2-+2>3SR-E6#oz%DCt-$-2o2?)ff z+o1#m*@wYCujY%`2p9t~XUvVL54Vh#p*I(5#+ZoI3g>h(0@SUv5jAHtvSU;AOtx5= zNgE4+{EkN!GozMb)_7FiwMU&gbmXBOBPyh_V<^$oD$@&COyG>jsAQ(gMsDob@uLqM zIy^DbZnguZMg7<#yJ6+7wY*B9c zFTHt0`9F*N*DmF`D0lxm{nOQrkH2f*mreA93OiR z{ix3~_Hx$*T$P}+EY+!6ss%j7n~wTDss`_RhPSgERfF%Y7p@EZpFq67FaOA=?`(e9 zss@Mq{?_x3oqK+Di(fi_|9^X{H;vCfdw%`J9&aAezayV{T^CgWyL|6kyzU1?`R9s9 zx|HYr`>jv@bX_1_7f9CyH_~-MG8)q&zNa%K6G1J(eIqWZYw*er%E+wsqWTBuuJpY?t>QGbx6n{*$mQY zveZ^+st}wUnAmsnpmdrnohGkfr^&WxB9(wegReKFR4Szfmub@Qst(NXs+E8lAe!*7 z8N|N65-@`lO-RfjF@wYm5;KtgYhhcuT)0(YhF=EE5D7*STFmbd=m;GT9a~g!{59w_ z4?otX!)RBRzSpNHrTvJ3^~B(=P%;%w#iLQUV#KixT$oWhh*V@YH)mUV&8X{*;%rW5 z2f4P>Yt41+Hd}2~n(fK)UUNOWr+nE~JB@pm6PnVLA+4HQXEXD9qg-t^^hUOxX=V#7 zqiPOO!nw?x(O7JyonB*Zvzy}yMrEd0G1}=KbD*3+w>%60 diff --git a/extra/sqtt/examples/gfx950/profile_empty_run_1.pkl b/extra/sqtt/examples/gfx950/profile_empty_run_1.pkl index da08cfe5bc706e52a875f1e6a83c08c755a8f902..e2adabb44795927e6dd43d63378438cfd3012718 100644 GIT binary patch delta 1908 zcmai!Z%7ki9LMjuJNHL(+c|Ud?=%JfK+FD6AVki&>N;<_-IOqFxq14ma>6nxK~h?z zaE5YMg9s!d>{Wtrl`cPueH;vSKwVo_OBk-u*FY_W~;h~1jP#)rpcO+h{_eboP; zO#OB3j1wCb_J*8@YXYSd-HKv)L<1*mQG>%v9_(nH(`Mk+q?Ag5Fz=T_Qh31A)34!b zg_Z&ut`$tUPC(ctu(%%X4R9V2DTEudmnPK{anRVI4n>^J1PPs&*!dIM?MOj!4vP15 zOJTnIa@gYy@dG|m0A}Cqcr+J4jJikx*nNp7VmF-11yDaktL{N=_Cl~rEL=E(-JM zJJU|K1`7g$v55Z>2#BTo7PbWk<`0Bu%y(U6DTZhfc-pmVEk{n`eo-NNM38q~E+a((SNty|g%uB#bBX%j$S-dgnY znF0F#jsO*7etna`lz@?aivS%0sCnD1AE^|e!EXe}0??x$guYtyeYL9X+sSnAPeNmD znBVw~2)10_mP<`==vijVj284KR)+b^C7P$hqVH?sQ= Dq8~V0 literal 99294 zcmeHwdz2hkeP+!=(&#lBJ&-M1m_qo`BadqO{ZKs0NTZPvqmkv&h=Vaoxu?5&x?4Tn z?S9Nd#v9qhghzo{Xm~b?er>-|u-Z^uBka(xuSvY%5?6Gz#UV*;+cQ<&9F+ zsMTBM_EWV=u23?j(&brWeBLNGTJLEMKGu2^+1F={`9j8MnZ0LDk53^er{|1TzRTP+ zT}x*Q<=F{i$Q&zPfZd-! zuB_>-3%Wiyr0%1h?;AYWYT$Y9xkRHzy)itK&YUyK+2ICm^h~qfsLbi-j9S?!Q70yf z<|dY6J=o5)I-*YVKC-AlXpl8?`*5>f8!i=QhO0}Be5D-GLgC?NxzM1}!-^R;Ybp?2y-IK3C z1NY2Tvdxll2${r8(WnP3Q=`?KF}v%HmboEoWD0ZX60VL~SMO<=1BqZT5sHOk(Qr7W zr9!FJ6D@qM-A1{UKjcV_Mj~1y8Dgo;nS2lb%=7E>8}fbmP1fC3b8FkwZ{@cQ<_AvZ zw+90G9rzIQJFoQn%kQ2zm%qt;wd1>KYblT(Qge&tky3hYCYx6EgX)?4s#?8JJ{Q0& z9hqQX;_-5>5*QpDd@y*HJ~qto$?p8EtN3m(1y=vfT(2p7^&dh0fdA~y2PgM*Y^mweG1mOobCZ^~S=Xpq5B1Y)J&0RKPq>=VRe;P>Y6r8;J&^TFP&8pw4g270T&S;k=;|U~zRl7R7U? zVt$`{ESijIp*Y`=+O{03bxX|vV9XfmOf%;mk61Jn(ZUhmO5@3(7K$XjYY0`kk-fn& zpKc~Fj4^8%+b2dW5sGWrJ$-9*?2J1BmNQ1Zu7=bj&0G#|vmCxR^N zqc*4-rMjW!_C1m-KdNFnsE@Q4estfEdQKxq&e6w?$<1Bnj(R3<5d6>ySnKm?A`GTx z3bQ(GzC-GakpWn*>vQQUK^`&F^hSjN%fRwBz5l^Q4G+f}UQ`QZl>wrm!MjWK#@Rv9 z;`<0gOyDV|rw)&ePmWI?R`0=xqO&^P96daJSerE(E&!}r&^6b#77~0F_3rI5H`EL^ zOG5_?lz^Q!r!}ju7J8KMXAQ!i zcp{}m!ZE+4JP}G_mFG#G^>9bMIa6;G8qEe<60yE`R>cG@eR0PEzP#WP?W7(v|Fz%?o8)j1cQil?xCghPG^76>z?C6oR@#oSI%umJ7@_R57C_%zn? zvAjkkLeZEO@#|?K8jk=Pcc^$Xwqw-Ex7Kz%oQNh8S}^Q~(-H*!q81YOJpIksFzd}Z zU?_qWzzTpBvqmAek3r?0owm*!?Fr9$TJ8h}{F&@+`%DP64_cV`W(kM1E!qM2yS zZ-OK-+ps}&*kHDn3Uh@9!EK+tB^io_FwlP67LpQLEZM16hPbEvUtcm53u}oWCtljW z4kRVDP$!aO2k#}(9E}RFZKVd*a~qo>8A^aeuy_fss-yJb#r$_eb1V4N7zWPTb#JlB zXfTdh>o+lyQF@0#{(3#{kduy_OXW*CCZl0c85S|*?r$WlMXi0^Q}X~mKZ_Bq`;m26 zFrq*eexFE^cz#K1adKDefCW1nm1+fKV?$*SnE`(X=g*9-f~+yhvxTxTEHE~M!8^=t z87v%xr!5+zWkxRrDfanvWZxiX;1~Y@qKt{AxvNxJFlwxigjGstH4UI0VZSV^?W1a`3sV#Stf26XD(Og20;R_ngWZU`DnLwv~_kJW6|Xb=>BqpM?h<_jMX`k zDNx-+Z3W84f?ls>^zy+GD_N9bNf;fGor@p7I@w%jZe=r2WNew!7I|Z}rlT|3Lb`NL zdmL;NSUm*VHC-qq|Z#V=6K7k5V~eu>(+mYYcN%dFQg>dsO83L8Ufy}k54 zQT!9iEBApbV!BGMO4e8JLduYd4aC$Y9I6hhl_4i%AeO*f@L{4RW03@ir4I72WnqR5 z%3O3~EER@IjxWO|8H40H9P7YQ*j`Q7(sOmHxC03tPolV`f8pA^5^)H=Qypl>olq#L zl;#Z!MfyqElCd~QT3q6u5J&Ns!U_&Rv{lAB_C z9kd}X@3KSzu@-czg@D+9B?hivfuKoplvh$#y9PQCYY_v3i>E~rz8E+ej%vY_zwF!b zsqZ>%v0ETmwOCS2R9$>$x1uadtHI0o2vMciU+OH0T*l&%BBK?wQD2%Lcq|1ma5CQE z1CMLrLv+cnbP8k4#T^x zUH}skk}WQa@^qL4_d!cY=m3pWZwkV-NZg;}0i~=`8am~cSgS9NS5g*Q&Y0{7rjRP? z<_!7MQD#4M@47n`(Afc+X{6^? ztCZ9n-lV7ER0`j}hi0dWAp(Z@yTg|}u*_*4uF+7qmxu%u7a)K3+kR3+SBE-%$XnQc z39U^Cb=uS{&?T&`aVSIh?LMglG#7#$#IX*PeG3T|VS_}qGo{DzTOUjX=Wcy4h{m>= zUE{FL5Oz(Z7^%_DYW6_D2eYXFtY?Hd7Sa%ZA5!1bn&@ZN8~w|P>=S93>yA$2Q=x*d z`6Yf6irG6~DP&tGC%R5HT2rmT;_vo=i@C|RlR#d}sH4y9C99N{xhcI^sO$Wj8#K3N z@iO&h-O$-GT5rA8+(KFtBt66yKxSK8DI_T?)24s{*GAL>6BJS|IoX^!$(nw%P=CEq z1jVXwv>9E!5Zh9AVKVGRa}&L1FteCG=RMWh>_OFUv&hGd5JJP4%g(8WE4ef;FakV;plrtZ6U>hL5q z?6{^JNmBRW&N7gJq;-g*4zlKX_X3E7X^h4jhJ2WWuY*HGY;=$|yrI|JQc9m+g31XK zd$*V-D#3EBNVTO0LuWk#?XUN)N=?iI{rLvnGJCV7e(ri?l zSkBM3%=Lw`b=ib1E>Mgx5RAh%Buaa!pLTMafnfeyHsko}O#-D_yPr|O8sthwxkSbX zgeBuqVE&XJo!UNes@HDfNW+IUX7~_~1Y^l4$hN4Lu%M(ExH8L>yt&Orx0a8A9Y9p% zh=3y}@1i(%>I^V&BU{Kp9rlpf6=VdJxeo9fuo+O)wr4gsVFf*S`sn20gF!5Y4#2*d zEEPz{jxDid0WqlzkZH4-S_L))!~*n%hGVl@g{tImKJzfM;o9D9ZUB6$E)`&y|0I*1 zAxS8$alJhNvLW^1rqMJQ`sNCEh}7~6tf+3K!34>uXP`JH94rKK%)Ej+{Ro3WR0w}? z%s|Fm&~F%l4xd(@S;C$MaA=g%Gf-X}0eC!i{KPneeydX76iAesBY+zKrvvO!g+5|N=fcn-OZYbr3(5fXCS9wgm)yBv&wbv2{kHll^}JE5Ko2q_ zxHrInFf#*ziD8E6i<(bEAX+KIxUqzM=~_c07`rOh5qx|YdRNVxb)E2eCS9(S3*Z)^ z16du~@pCN`aFg^lAIw0#j;!S}1hk>NiCbu-!^RHqzCp%lkZZztvmO8^4{x4~y!e&Q z?e+t?yB?VVSN2g7dD9#rN>6`e@zJ~Rq;Fv8`%EuD|E=a`)){J@JOaRVf-U0e)XhMF ztCDimTPuEq$h>>us$vOU5Aa7FQi-Mu9Bn5%EJ$C&v;d4Ivp+l=NIux94XMYVQ^=JQ zErO)@F{=26k|M=VP#4G)?ivyx#ZOVC*PQw&ewwP4G>6MSiS)BTD+d z*s6w?cSvxU{kAKt+&WnBLn>HUv(D8oh|+IlrLk`lbHZ9mtJYo%9u8mG1655}4=^`F z*tJLpzp9~XIMS(V=_d5@t;}RfJyI1#>5gf_Qwu^7pBZc;f^8CvGql84_yYE-vV7iQYhlLq0n|;Pf&jw^51dSt3v$)JES{5G^?kik z$yNV4t@`j(ksv2e4ud*t)u$8)b|XR(UmT$Yn_N~9S!MC7-E>v)2LLhozT%q+%k*Fg zYaMxLhob!U_=#iPU0tK7f&F@v|82RE=9k!9n#%6?kItK5%rVn{@v zRufWN^p~U#`-EhOM?VpYceVj>Pa#c32on3=Q)d+AO`iZYnXjioevm7@`oI1QWyi<0 z^lTMIQbg)0AJWKD=Ut!CW^0vZRj(Vfq=lmA3X7oAh+I@Y&B`Lzaw4d>*>dl5o_Lo1 zKWb2Loo!-+|LA>Hqday(9eZXuN}mTE2LbN8yOrnRVr@|;Eed_1Qafigj|RIw#JsS4 zaQw4cwN{vC=8N85eO#+AKg9ZH_)Q&_gC*;^KB93E^nV!F&Sjt1$j}Y`d0100<7AIY zZGB3E0IsfQmq2?KGURt8bM6Zt*XEb&R-WNE&bpt5jorMFtC9SUo_X~5twC;Jl#Iq_ za5tMNA3#8V82tkWW^;?FSWvtSF$!JQ8^#Os`O$0M(Sx^YeNW;&W3^DW_h4p;@6jqt z^L4wjNfFf|IoB#8h-WYlR)X%AY`lI?O&F{xL zJg(57D!QI8RN<SFMdrg{M3Gn}wf;K)%ta)(;L3<0jvn(ZCfQo=RtpN|~VpP|a|-G+Zjp&mDj; zp$Lwz3O8>D5?V+Lso_{O8daf8nGB~xP;d+yk%$pVWTLr9CKFF*bE$MZ985$axn#}= z4Js#~aADLAl7K-yapI`e4K*-as=&QTO|PaKd15l_bvRkd19{Y{b9LfP>y}gjIWp(c z@T4${bVDcqAG7cvo2%$R4?5TydZjKz3-w{J{@Z%aZZ4y{%LC|t!S~>w{L-Dei$!?j zU(q3+!)kAS4x7F4;~naGphG;HEQ+_D4^sZ_K1JYqRviERHvar@&0~o7!!>WFcpqG& zXmHSMC>p>Uu%4pLmYZ5WUC%4l{}kV-DO#;U@_Geo2a64so1h+|jZ&?G7oN@Hbt~FT zy{>2+Hfx0bfKCf@%{fG3$`U442vIiDGYIpjmtXM^c*p@&Icy%+;B!(ZAc<7EB+$(- zAlhoX=#C#4Z^ku4iiG{sM7$AVR0CPLv^hYQ@%!Op}3WWo103AAR8D;mPA; z7|E;u(Yz#DLx5K|og(NG(UN=Fpt7_kiSlBd65&HuL0YG%EkTEg^=ljA{_&T;tcdbr zof2WdDlgJSc`I7kgA1x#q>FV;gzoa9|01OCjFwhWUaX@c6zeFmupd2x^LH0+C{C0Y zI9`OK&dj)W-wz{>%8PYg&<7%7RCE-1*w5907$T`GJp=v3_bKRbLH~+>S0Fz*c_uHfJQRMY|&Uaa{P7 zf7UAhF_F;<-Q`JF3F6Y$+tw=&;Xzchr;(m*O(;$&zLPb@)S?BS3$Ace(3W?|JxJ9W z5|ISV5@0#UHB{g-{pMvQ1B)Hcq_Uug;bmR0#U{smSz>f-L9L7)nd_K7AylUV#qVNP zn;TC}j-5`ER~e^zZU5=}^&@9aKcJ62Fm_`6G&B`vAb&UqC+w&1hYHtXCexaLMi+T) zZB2B+nY+=N=tYpGyPk#FDhn&nZ6Li~qUc+bN|(8@X3W-kW9ANAqm@(RQ^&^cJ9Bb+ zd@2aP@WmgZQ_EVuTan@T$bvvH*#L?6G@uRQP506bBf9?1cJ#WuS zEk`^p@3LAxa`@zgK6c;9V<(P}OhL_mZ<~!}Qpg{_5l6QG0OP-r@kCSiUWe@)jqRUfv{3-e;-4OSafY zIVO*vK5dQeWj49d8s5%L(quNU7?mpEd<*WbbMacjM<9Q41mpFC{={+^p);KIT5auU#80ukRx#8M?g?~ z;=2{0h!>%|r|x`$!{EQVcqvm+U#I#-ya+{p5sG*b3MyBGBA+`H`Q7~&<=pY^^5S09 zBktXysK*^I?nQYKy4x4|RtiOVkx$&aLwCBk?-YviE2WF)6z^T!i*lkK5sG+s=uThh zUbG|fxyMh$yW0`>B3*={ya+`;cPR3^$V5%=y;jE_5B z-pdhqDU3kNgr5>4qceVFmy;iP3EgQD_Q5`fA0p2CO}EOt_%M(+c#)*)-3gCpF}pP#7@RE#w%8x9$;%zZ2+ z>Gm{-G5kGf`0+Md7-%gA=r}2A&OT47mR=wm*RwA{oCB^RpWK1ZW_p20dg6!=`AELHjAn}u;TpU&&BB1`% zM?e1lao`6b#1rAg;U|7U0$Iv)?rCoxQT{c_XFHVV^_#g)=}-KED=c&=kGwPo=Z`l+}neDQ47l7JkP|DPu`-k_%7xB-{SQ=`Fe`@!0n3iG)oa>oFU5J`2tJr zQj*{EY2TVZz5nD*qhctp+`%NAUCKYM|CDErIOo^*_V}gq`SXL@y=lCDd;W|Z^yazx z`D52Vcg$P=h0Xl^x!)@rGQ4f^ecUy+>ECtwzQp{0)1PhjS zp6RZ%;i9fKT*Nj~v2Opm;i7$O1Q)3i7cJv%iHmLkE-IeEk|8mX(@vk5OcE18*N}0c zBm*gsmyW*HjH551W}6JcrLpF4^W}g<$*2|)&g)lzMBN+`y_&O%m|*GG6%xI2%|W7e z7L#Dtywq7m4=H1hjam#e@yyQf;_$J@h_&~w&0aQ*U=;~gri4^C3R1oGYS~PqlFg({ zNH&vXGu>u#?^f(M<7tSICncMSv2(9In<*KlLm522+#sz>f&sOBz`+Wbs;7<3#H2L8 zu9)hMHG-*Z@!PK_rjp_}TfBEe#BYqvbk&$hk(fwgqKYmtk;FvT$G7QQR`uID!%8*bPvax|s7ovXHv2@4p-ReSqxvV@LRf60qL+&~RX7OvE`r+gQ1^>PG z8Nb%WRfe8qDns5{R7-`3r+Cv*zelC$o5tH0Dp_61)a$)@#Qgfy`|jzWa$}eO$-YCLJfeK!Z%%sB zoX`JY|9!sct!slB)E=p>Bl##2Gf3~q*ULLHFayq@bso~Yj2S{fQr8iu-mbt$S=WXc zAf~tXjNbq5#RKj&{fR&E<`HZ11F!e)m(Kcs`d0t+hqltLPHM($6Eg^Pn$CN!t!BL8 zgWhL+F)+g&AHMdO;pyGp{hGf&8#8=+pEr*aGyHADFP&qC-y8L&x#!p5>%4iy{CeBJ zSs62YS?!D&`hRq3%n;rpF#|iSfgL*`ReWpuJWdjl_E75n{pS|QlR4K&-aR2tHeA~$ z8#<6dB!y$PePk~o90IK+SBV+cw_yh6xInDQH;T{T#SAv3FZc|dXmBr|!7eNIc(-Yi zh!>#>tkQ0rL_3*C9Z)U@Ge|x|=RIHY8J16~l4e1*vL&>aHq_EANa}06IQkkPKm#qw z&su6X3kroIQ7ybm9oP+Rn1QJd-e}AK*@TPFVDH##0W(P1gw1D=vWYA9&?03Mu*8-+ zo>HGd>N8v~eTGmd0kaujJ_Gq(NeajHE667LIA-YM`V7qRg}@9wFZ^)_dA2PtKJwqZ zc|`gDd*JIG%5&cIH|PD+PxZfa^cnV;-e-J4xk`P8ey-2J9i$NdgKIG$WUf(y8&{{V zu<#>q-?(hR#suqr;mzZeJ)GM14bM9k@Bd3Xy=lCDd;WYi>dhm{o6Wm@^$DEq|5on- zPaaYJeSb9KN#pJF`FCooS8hS+*GOMMk@^Y}6G$D$ZmF+u-Sibg;b>S`%)0mp;Y36W zMTIZ6OS>1@*oFx?)W{WA3^FP>?bXRA*qjG-sW^Ej5FZYwm!lZ>lC|WC=!bJDI-K82`$ClwytRO zu&E6YuwxAck+pUJfxW2+K)}I(7ZP{@fn8SQzMjem5)gRDUjl;6!{TJ-fY|#bAdn8* z>lOIH&*Y74vt;Px{vR5XwR&KGqoUW0bP0!j!pCd2o$oN>y5~xT-5tg8yQ{n@5!I`sSZ|ehtNN?d(~cp(>KE2QSURVy;rGA;Nu(N z_Ir6dtMQG1{@wWkbsl~!rzX7p6YuZu-{noCQYgQC+{*nQ^5$`}0si?9{L;BBf8f7( z(|G;%{Ce-Vy?I3c{(4)agRX#G{?DF^dh&?!Q<-s38gHNXZ_|hT)31@Pfz&^g`iH(W zzO-48HVZG3%|a}cY|}Nc84R&#R9huv*xCjewmO&08wD8#eAozLQ(g;@L8uzgSaCSf z9!khyZ`l$uJlYE6x0;*1hssLGAR&W<4CkeKV(I#+C&rQyE$DYtY$zT~i9=*pWHa=) zVTS&-fEg4IX0Qo;ff?F&X?yqfXEU_t_Ocng>2wTS`>CyVp5SGiPA1t5o!JDE%^+PP zNHa%i<|xe^ubY`;JetsAzM_dZOd9Otvrw%8P*nNkbDLynoyLh zKI%d886=-U@)=&Le1`Y_*EbO*t0;l|fi-iqsO^g}5ejJ`?g_!A>zIfnwQzWK_K*W@ z;DPPZ0zB**{J5{)fm3e%_F-=xQU0%*&vYoy<;HLMihuf39=(RO1s;T6gWW5dB>av- zr;y^N?|YwdwC&7?_Of&rw9b5R)=BZ=9`$?9e0c7Myq(qPH9WOxbv_ghXwPQlS zAM@sM&X0dN?M)xpsyx(b?C2?-C6(Pee4jHTyA9gt*L+XxCcaUb=oM&ywn@2#7!xui+ zp*)v-?EI#GdhjKr&G7p_^*-aNKBt!kVq!eGw}%Fv{g^g{M_b{pf8zD5Mq2@rkvm^t zb#*Bp_=UH9@qYd|`G+0ej|~YbC%t)`+6KqYdea~3r}t~aJk}M*y|et=f7e@H^l$VX zyL`0?oaKLdWVa`eD1Y+XcY4yOFCzUdcirWievPyh!~0+Fea0(+1iNQ=J(mj!-kI~ZFW%2* zpYo>h_iICf-}Um&2tHsB@5wDFsh~ak)ha)j9 z7V|k9G8~SFv?NbiQJvuCHcW7{6BC@jb4v%_t1UbD`2lYp!Mi>4MlWC2S^wi-_fJ37 zL7iZ2VgfHG!fELEsh@bC@r5f;C-~6Mcs-XZJGlQ}yzPtkb1=EH!~3;m2k)Kq=5f;b z-+F^L{c>anKYx?Ayy#!J?_7ue+4Jw+Uo3g@i1Htvd(@NWoPY26q<^}U9Z1=MlpWyM zXij@etAt+}tAubkh2x(5&WjF5!f`FQN~?rDZJ1z>bIrU_m|%2`^9kDZ2_z;M?`|Ih zi3zq1<_9LFilJ07lq!Z&#Snsx>*d@a9EnG?gr5iIa3l#!#Z}4zDc3F|ztz#(SMW`}OM(4Q=G%6x>K#`chrqd;#prgYhi3ubokeEPX zg6oM1qTys4pI}QO7)%9YiDWny;;M%$3Jz{<0|d9O1wa77flHad-j3G-AZS-6u=^$; zgA^RB-VRX;4y1yiR4^;qT-zB6*0lyHcYVBi3$FB%OCj~3Oa?!f4S3} zM+kGD$$hOud2R^y=b!UWSGGR>&Ldwm(G@Z$yzTiX-Ys_9wc!HsaxlT?zv1(Y-A>;G z_OsF06T(m}Uj;nHn_k_uTnc&AU zZyx9S-*e2HemR1J!K$~sm_LWU9q6D8U^6NH{)zpbJfi$`>^D4VG+-kAy>IkSm&ycE znLsKNNM(XrS!H^Fze0wHah$%C^u+}6NJ5KquLUb&f|s{pf|onj%o~LXE^cx23G97Q z@Cm4&b}ZfTdrHUUvU38zTvncIWQgc(!O16(n85p%oR~sl0?8+kd;-ZQAYX~MOg=05 z1Xsr=h$muN-0yUya3UDeV*ISe6*0kWZJ6LT+FJ!3wRSLpz3&Q4z;_z&e&fXic3F}8 zdMXo0Odv+x4kaeYJPdohO0IytfKeh@V}3||xM?&Ey|Gj^Mnt5RU(hp^>XL4y4XIh9 zo*9|0r89-{?1Zrt$nAS%IWuZ0WQ<4E1BcY<<0l{3H>B2##t2F@wDQan_>MT+$S9|0 zN=A0%)YN^)j-MDGB9}D9sb1JhHfTLQwp^J!aZx%fl12yYbNChF6VO@A3BOaC!-+Ut z8LiUj;1z9{;1z2D6F_$GlF=uSn83yYG`9$$NlYLy!J?+aL6Mrn0Yr)f(jUXtVPPv>x)0=QIz5sVxXEZa3GXSMN{!8 z>=vWJkX531FH(`&+@5LbRimcY3v*eWout~D*l2EIx0y<_+-OaXcAJ~oJ>|!nJguGcfQbR&~z8C7!!B`l=pjrwvct%*K!x7{31Fv_!qveBC8 zGW$zeR8lx^=!LR=Zl*Qyl)0hYoYNOG87jGBHn*rZ(zRKmq0bw&dZAKoz2$WILbH>pAKb-CQ)&Kwi diff --git a/extra/sqtt/examples/gfx950/profile_gemm_run_0.pkl b/extra/sqtt/examples/gfx950/profile_gemm_run_0.pkl index a34275a6a29de7249805002365ef13f7cc2f2815..4ebbc8f81a378c12f378c2efd47cbfa0ed9b36b0 100644 GIT binary patch delta 1943 zcmai#Ur19?9LMk3?%bXKHr;Ce-%2qB(Nw4&N}O}* zS0-Swp+LCDKNuO10^v68*7dAXO5kdLw5d5qBbMb$f5PD=7Cy_^h>A{k!a8pzL8-aIMgZ>r@vsW`wg^JNs{IR>|*|`*=xj z$PDj~oC(1J6V8@)f;7mmV0D63Pvw?!`sE-mivL-UOKmx%nSdu_h)yYD zNRk5cN^DL`YezdI4_mQQV4zF}^Ie>iVM>AhSE^EUm(VfZ$gQ9>WplGdP*D zHQ>;4PRIn#KAYh*8P3}ecn%Wxawg1>z=sL5k?7rsavoBGg+f}M$HMY(g8vV^;ykZkaO{!z zFaYY|#4#u~h?Bv_N`z{=?Rw%qrb!=Z9-U0n&P)TE`C+^HO|zN0t7#{hZg#di+v#qz zo=MwHI@4tT-#PcgeP7U)70M#;lX&hq=lj0%oyR@*d!2i013&lI13ra*k2eF2p=x3A z!dy8!6U^u4OSy8T+1NK(UYae;=N`!}&gG7s%`H}&mz#%Hnok4VKa)FKn9emD1CLG} zdj!d1b|Kfy`x=8M%h~C|;@m{;f)2fFrcf!(XD?&~RAXCip;Wy9q$hG0D$PTU{&KF` zJkjW@o&n|I!}+C!+~Fs)Pi5y1PcO~n4l`$li`-&msT?d_XdeIkp2m*MxpKAyo|}!x zxk5FsPM4N5CCPh0XD2Yb7|aL`GMPie>Oo@s;Gz4PRW!Qe9hGVsT)(q9;X`I&X=Zso z_s(Wx=Q|b(m5QjSHtWshGmXAVwb|G{lbbFqWan{pzjk%G*%(TMLWxK?5s4)dBf*i; zk>-b*7@EG^Vl)31Lv3<28H`6FqIUUAzF!V$-k;x|-;p2Ga=Y_;@_P^Ehfd`81@gC| zoB92(ws-jh6TbW#^S8?$=%^!~u~0M^4cqx_44i!M#Cx0h9gTgb3b}LYNpWAQ58!spH!TY-9*+bQ z&eIkjO#~yOar?BL0p-1m%L`|6Wwn~msmn{Hij8g(chiUy+wm}(Ohz1*1JS)Jmo3j< zP^;zaVr6!zyr5Pk^VkEK`OcDyhjBwE9NS1lqQTLq!_7=>?4B(wX6Fk}<<#ly{Ji8` zCz^=HgAqrfqZp95)4Cv{yUIE2rPDd}%<`-@8amC87W*4V( z0Rqq9p);&v+drh{<|{dM_Tc+x7oS$K6x8>dk)J*|tUeK>XVoEkSYS84#n-sCGM&%S zQzAo;oXpuQJux3SQ<%%3nJ0$TGr4Iz#wyqvOJJ1PL8SXz=(p`3}x;gJ2@7d z%T)u1&`ou5sj8k>TFl+2qL0hvMRj&DaCf#mR~c6CzWa%Dbj5V<@-?=Xb7GA)8>h<4 ziy7`$;BI+02f0A+Xy9)2Lc_w7bY>~DT$;&Na~TY8wQ|Ig7fm&Om5@jv}LY2s=GRu&zgOUpg;r%=_}0O0`g3u8JEF>)E;^B_fGXFrsY%b_3~P5-CG;Z)HHHrIy19PqG5>4Va~oa3C z5yyIUk^qw!SG5~cCca4lpLi8$mum!aE5PY1X87)eI31mz&vh$e%{ znDh8G_LlJwoz7*Zm*(fQrHXZ4;v-SoiyZ`-XgnUJZPtbETPhb8&MeIrrth&daL-H* zdzoeWN0PWD6HZTqXgnE;1w%=fq24`TSSVCU64M_9EFb|j91ez^)>I-i5)X!AvdX#? z8oPxsCbVwb%8w?n>VpZb+vbr@ghr#mSk&#oC7C2)w7LYjXQ^x*>_iwEKqTZeiK8LR z@2H#fxa)~rc`-MyLdP#uEkipRiQ_>VbGTEZ$q+Q7kS1H&M?p|iC27kuHaZ%Pj09tj z6P64o5<&3mc)x0T%7okRbn)lNVrfQ!giRE+-3|q1mDikZ(^}v zLgULkamfgj2a2OBL@#jTT1a6Y>5-gjoapz&gII2N0L?M(P|cwT})m;YO^#M zu#R6~7Q$b(u(YTN!3QS#8+#yBmtc~bA(O*2bW9jE0{Q2U|I9#R*L?P=3)n|QeN%{r zB>KW?Rle|8`0)T_>|u4fdR|H}&mVtjfF1_oi3oWUKtfsuUxKm?d}v!^u(SZFSHn0t zlbcob0x7hs=kJ4De@MOeohs4~m}KzfMY1{wxL|4FP*9^3I1eQR0)NoJ+T+_AdzTjr zvjz0C)8J7E{{pbIKr%kjChQG{t1Q%kg5VgvnrKJg~r@0=im;8RIq1qAllXTa=3v`W}`I z7B6XPl_Mw1%Q;>(zdVrtCyhHbwZo#E81<^XBwmj(>0uT6A#}v~3a$IDz3K4VleP`Y zs=GFaO&fwsBq5)CVsGs{gp=5KgCkwH)?4Mw)6X&WroN$TuueZ^;qBI&|iHoW3LCcusCb;XYn9=EE)SpUb!8(#4l{G#p zl{Jez2_-KWin!=;`%2kzcA-LqO?nGUs?lT&ZV6bygg9!UJOayf)JL4^%m*a|zok$~K#Uyyg*l;SaMckmh>M}rlBoTCSWSTt{w@qo<4-D&? zhA3PW#W(gTO2=CP0s2!U1pe^74n3JtXcRvz3vLYBe$3f$5yvJrqPa?KTemdd9L>U~ zG&Xh8xNAp7L$EA_%&SyxyP^u#)`aHCv{AiPx+@o!&gK@NoTwI0DDWd;`2I!{Hq#?j z#gd_5Ebc4?7w>_TVlami{AdI>OiaQX~#RRf{nLuxnGh}y+ZgOFLr^(Oo@{--zA#Zl1A7I2?jsOmj7v*&P`n--@1vdnS1)=U zkAk+tZ4ntwLQYObocHp^9z>?WzO+=q_6Ap)WkizUQFtWml6_<}8baI}2FvAc+am%P zrgIpL#VX?7Ot%Ie3Ai;9M#q8lh7noXEnTKgM#r#Q(n=E!O}MxmMj#rF#jp(Bo~FhXpC3A@V|qcEDJ z9cRSlnXyawdk`iAqov913(RB)N+Pr-2X2!g7`;MK=Y?QUm=?&ej&Mk#E}~4zcRrR3 zEoAO|EVOXvW6{_`G$Jp%6ehz9;kXSlyr5%KC9GzMP|8Fj^e-I86VMP^3LAWo1eZoX z%;N|T4K#Lfa3{sUp!{LfUT#k85s{O7I+fxEY&N#te-bw$A^y}4@S9K?180{CGtCnd zz7ti%S{C2XuPA}y?f5$pClW;!m&VdJIF{}grFR0hv177Y?zA$odqQn&!y?Ke=+ifg z;LsDrH&YW-Q9MMn@5Uc)T@ctgHZxP6Dj`6NI^rYB6TaihfR(=;qRj}<4^1d1Z0xI| zS~Mp5dR*A8G8OXaVvxL71Xo?A?@-$4*2MAS`<)=pfjc0!L0PpJBeV*uBlt zhBu@HHewrhh-!f>!XJ9RF-UtR;vGaV+B-_+x%*UYf1WDkrpdpg@2P=o1%ZV3i+K3Q zWLW&;@b1wQB3BkRqvFRgZxb~Hmd|Gw&dg*HSe^+yI+?8o0@&|@QxBe)7*^?O@{xxh zcx3E2qP1ky6?xqbA}7(vdpS%+_lf2udjS+a7J57=8-_2D9Ce3=g{BOT>2-A#$bMaJ zY!mWQvvEv#QKc8MJV&lB3{Bt>F^t3+79Cu|xpK$%DL}>Wi7S{RE7I93DZhv(i6J7f ziJ092sIomSSXwFtLWhEaYJh!Zclln|mSYx(VJU$9!5A*UHBo3a$zZsn(D^zCH71t| zi&gVMjl93nKX&|n3eg_OozG3<5sN_9=EPW|AMQ$Vlw-$2u9`*oW)?U94)#-`rZGVC z(Zu192|VE@`f{N0`1O??8rzqatEFY^Lyu#>&NlpoMJ*Q)94zCE2xMA5Oc5E4;rIe# zR-8g7%!es{tz#y>o`)%7P=cTq*n~BN7scZv2=QeFD=t(+ zmaR_kOk=MufobP`ZpA)Ghl1pOXJ2gNlaE49tIiZ=5oGn2hA$)rqp=MFC`3CvK=ku2 zjX|u;`=;(cK6YOS3)CR8-a@ge6h$GHF2%qeSB7v9CeTQ3U`NNEfd>|1T$W2E1j`)G zPd_Qb#`pC#wj)5KbfK`=Y&<9Y1E}J#1R0ruZFE?Dayhr06VFT;Qh~x?{+vL~XmyBs z&Q+!n8o-C00<$8(mY9CO*vtqC|8Yo);>Y1HJc1~V;NqDJc#vXC&n;%pAdck-w*2&i z4;>R5_J#s@xOW)NOm<;L^z7UHI0Tg;n-fvLop6&edf@U>5O#y3~PRl zvpBgCAhxpQa`u813PI{myb%Vi=Cg49EG^!J>%o$J4n=f$(tQ;Lu1V0IM0600o%KtBgF_6`1{r_n(1<_UWKDoTQdy z-hck-pFmH4hS)Ct13k0vY}_K4q1I_=1DbDHjIbM1niQ>-@;hRa4C_Ya;dd6o*Z*OHz1WDG2H~e^Ux(ml5)(0n@ReBPB+qoCva`7C6RF98N-k*p!4KaY61V zk3nwBFU??|gwNqI+Fl+9Tc*M@TA|!E$7O8f%p*b|$5tkekp)1To@Vr5fUn0}lN$t>Z41-<-$ELXHNB`U#>mV!y~ zBRnFNx<%$w_n6a?%qlHr%7XVf4exVrmw=pVX!@9GczvkYNPFBakIN(fXLUx5=LY%I zhF5tv@OY@0r;Vi47X2S3h8@yNS~;gYJlg?#0d*=;C=m}#KB_3IzYTdgU&%tQqT>pc zkN>%H>u(0Lb0q}&k_@Z-(;$WLUjEtN4#J0kSf)y*LU!C=o10%clb!!maA|h7k|QvZ ziFf>F5COf=6P|!o_A?ShldQWC*m11{hPo2?y`c2uR&sL`Es&WloQDieY^VOY=sr4{ z;Yvu@bBjB^Xe%f1+kZWXIIFp3JP$wpa4EMq{*XHU`A$*hEM#o>nLg2{d?~n)J)a?0 zZ|2-m`3bFgPyn+er2m%DC-5(VrE=k{IB;NNQ2EVZr86Z%pO={mg8t{VS^88^I?X<7 zq!@f+=1W0}LdQvN!2r%+kSm+|;QO5*ES{Cj%mpw~n5I*z(@(sJ>FYGJ@AEQaZ055F zfIpj?EtA1i4BTh5)lRYH>8C3ZG=KG6M{(pynO)06ncZjkpAARn(zwk4` zU;V${cjk+HKM_yZY4v?U;28cG2Ujs>Fw#mUlP{Dq)60dK%xol?h~=W$nefO=1b+-9 z=rX0{Z`ab4w*P2ZQ$oBQRX(fyEIPvbjp7?@37|fIi%|PNq3DW#Hkpk_LWokE9i7dE4=E495tb|8M8wd=TM)^(7E9`*jQ(mG7vH{Fd<}?Dr`Ki9ENe%#JR}4 z0(LtC=zKxXc18KmKd{!mRV&|TsgTcMJN$$O&-2V^vs$@qhf5XZ16sLWBCmeYrTkyI zln)W%z8wndHIj;={99Z&$v=OAawqxcZ>Zcs{!xN>=}ArrLdJu1s08O0mxK9iC9i0I zsk~ZNg5@RhbSXhHsS0S42vMAyFE3%jXJ##r3vLsL| zB!gD7XOQMWS6<1~EG!x1Tp>l8Yw>EM3`wE(QqoFnrLbs|6mP2%((T#lX)}T-6#{u> zx=g2K&D9tkl86OYAWAZ^9iW_>UKSF*smhe;CJ$!rfA5L0;}4GWj9*(y%ZPM4h-Bht z)eBh<)T$NUuQ*S*vU&*NP#^ zN&5iT^FGRHz=F^Gb4vF+$?c5K`!=V%Z);?1^8HVsLfncP0P7c=rWjF6t@*OS=Kq5G#Y58nTt%-HzJ2TvV)RsHSn)^gO=@@-nnN5@V~WX2ypk$&jG@sqEr@quoQ z$83%7ec+L!$1@K+`k=Yf161rd%|6Fz-X>a}NEO8_--fj%=z6ZM?I*{_Gw*tkyV{Se zxy2#5upC=V$rcdG!k|qD|wfygD>91(%uUp_A z(DJ9X^a(9ZXz6Z^k7X_WNiB_P>2?k7Z|K>8-#FuGX-nJJw`%$=F)E6(RnwFvS8UZ( zoBl1E{QfQ}bpNxr;pvax=bx47*>#x)J}uL)eOIQ9Ka=T|pO@)RX(Pw!B`wb>m%F9E zq`^OC!P_Xll`i9PYOS~CueJSa#kcaqd~<58x8^syeJelAH>cKmYksr&zt;Az72i4@ zJZ_v?>#h0CZl6s8WMZKotDc;4Ij0QIsiuK<@ODcw9g{V~aq32g%eh@nxtvp`!>P4> zO`}#gWjJff^carY<2=)4I4eHqt@TXDnlijK<#rg}+P)RuEzf)~ot@e|a5<-pZ_RT# z<8$f;$MDQQx92whOovm(x285ahaas-+Uy!s(Goe>5}pV1)c5gSe3MI8B%pTRI3 zRFSdKBYilRK_^adtf|?!^x7Xeu^lyGX0#(GIHXebK64^Nq_f1Rc-H;)PFNr|S$rUYTGYRg zP|lvCqiK&1t6JC{Sxt{rmtmO=7)t2iB)xV+xVW@xy+8z7>sk3uQ+lhaIEo{WSbPef zCR)Z}p}078BICHh5jC#A(sLN^L@iX0&>wms>;1=$KKg+A&Le8X(78X6!NK8NfnvP_ ze9CTP-%h}ZwVU8jBgF+L@oG7|DaLR>u4BMY;%yiBOpY+^!?rC9?n9cXe(K#PPdeJ_ zi;rQ+;9$2fYAeN$BX@KMqK5a?q>bz4!|*b|-!k>|cRzCs=8Yk3W8(DX!yo#-d<^=O z`+n9A!}!l9f8c^I>9^Ob`T+eDi_<+C5Q!+fq!)=PevD?Ls1E1I=WFOyx6`Iir0 zY#d#0h@y*)@%7d3O#c@3_m6*1wnep>$5-!GC6cZGRFA3};Sz|Xs<-tCS+7vNN&prL zR8Qqp@9Wg+_(OYme65%=|MKDIG;66(slQ|&A7$rrB5>fjTZua1Ja<>WxRN>G-+os3 z!=AfC73DtV-+fckdG4plEb+-7qliNJ)A`~&@Za{870UUP>i=$UAN4g==b$9g5s8dul%@>pL|O27Uj3CYp_JVG-9)Z!I(0gswo5J5;Np#im=@v z)?1A22C?7g)w-fgtuO`WR#wM6+h1dB*Y>;gZ``wgt6lr|lH%RJ{ayMu?%BW9uKjz- z*QI^*&uwhR(Yr}y)w2gL)!gxkZoeo0<6Zf`+=c&_R@~d~7yOTV@_)Gt|LZBQ_0`{% z|EVtgU-sai#;>NVk9l$MLjP`IG$&YjHK^UooHNv59~iu(=P6fGj%!b)dj%btTfp_8 z*cGUh$_D#}6fPZv-bLkH^EA~%#~btyHcd5y!11B{6N7v7njz!~m+ohxi25t~HRq_D zN?#t_Ew)=D*T+#Z7~qn7MZMq=WeXy=n>ZSlyc{PEsD{$Jd6e!&j>_LmkaA2!iR>46 z1f;|_UK2-pjmV)d9UQG+#0zmp9LQ_Ii=Z#_3$ov+9}H?Gk{|Q;_QVL87(W#)E%dfhAVK>_1UZ|tGp zkTwe_#mpIVM1_8O%J@_^7%|pZA@N~SL7#AsbE{2})7pJp^xITFthXfjV@v~AG1=8_?VbDkr$zNKIvzWI0w9pydop z8giYPU;~4@#A>wC2!IA^XnYo^e*%gzW$ze|*ASo}3*Jjyg=lmZ=<4plMLU)TMRy$0 zzT3>dEE&W)xEHl_fAMJ2SQ33M0rfa>sON#}0K-^v6o@at5q(?{eV#Akg!>@JonW#1 ziZw~ta-L~iWhw0()Ccx>i6*d5xvL$!LFvZsSTrtN*;P1Z4%&f#EyS^8_lr}VaSQGVN650=$FJXw$H6~ zsJ&`@ZjL)lyr@*<&V-yaSx_b$I+j`r||y+ z{uDv@YZB(P;$JhuAKXm)a&yU$* zuHAm3)4%n&oqp{W$-m^@-UsIYtKa=cHr<~3VdVedv+uRRY?S_OZ!FqyC4J>I*(64j z&h<^aK?`qg#j7ZvEnbF3F>wuM?fv>IKSuhBzW{1CvO&ArC(X%eFH?5q$!ayTaMs0? zeXfLW1H*<4Yp=nQdqbM3i@!|M(X&$XI{;tZ$ANWQbQ0f;GQQ4OtBHj|^!EGCw?K>k z9K)_fVEaYXy2&8zlLqN$;lPDK+Go6Bx0*rvYd45NTJ0I6Jq^+|W!%#wx9Vw-dmR{@ z`@9XZQ|O=lF;90cULVpTgY3HEMPbto$`{@Ff4v)&XP zioXw$?V6RL+RmOb1k=z&PZ_FKj?64&mP>f60Nx}~%~mU{gWMb;JEZa?I^WR}FZt7z zp)g)D+WB5t7iFkVDnsv*%1|F|r))jhYRb?b-Ehi~)1!*!XKP32oZjr{oG#G`*F8F= z#-gTC%%MA1=EqZB!Z@i*a$PyEE=B1YorBm5H(ZKcL3E7IOBwTuP9eJep8S)#uXGbX&Td)5BovIr~F|@86s}{Ddy-3MD=zHMVeQrO_%sd8tqg~As>go#wC>2 zQv^{LWoi8Nx2QCoQt(T+7_waxuxltoxuhqR3}i(Ksby(WH2@<^?CI4NbB$T8sgF&< zv!wTz)F=s`7I=zsNqA>Sp_1j+w5zh_)2Ul=Dc@6ObnnsTDKj6rR?kkVr_6}gs`r$c zEuhTc>yJU>XEhDV%xLFi0?r?Jdv^MzGLx6e3^d`btIYhH8$_AWj~X~Xw6?l3bE)Uq zaaATrRq1(lyo7C1x8!~0e0RaK;`0{vyDDQn&rUbb4xTbTbI3eD zw+Q1Z z{-<$8KOI8P`!!`l_XZi^(qmUz&Y1o_DsQJw;ja^2#;QYAlBy7?Q95K>^jlG`j_GCf zF~X{^5L~-P*NSp_=f*<8dZBDITKZU0j`D2mebu3C!gF)Y%FgMYveQ#`G*8V<@41OY zBEh8m3X{ci6B!u^M&x@`yDB>anzDn4ZG^E6(Cy4O^ZomexWuJ$+^7e{HL#u zyWlIQ>0$T2ZTCCYmv&$Lm^}B)r=Il709(baZOgbI(>9 ztc>Uwdi!)`R7Os`+1|&m`;6zF?Nn6l9JG}$ZUyvh=c4cWQQ6K$&P9iA;d9Z}p2DB>8~pU;m%=y zm;RCG$sK#OYyZ}LUE1&T&)&*7dY979McZrTNa~&!XNZ1J{z=*K9>3Kt{I9R@_}JU; z7yOTV&VcTDvEvhG>z(m&n#KOE{Ev6xf7OG3YG0mJTKOXr+I zx*)>izbs3qCKX*t(c$#=>e|}XO9ZmwPf>}#rYimxQi%NWCrW*EEv}SQ=iH7A*J`g& z9jQq8y9Qo>nQkGY$P+r!^OEk+n-oE76e@;R1a7-V*Yu+l{A*WSVovKdy;t=XW9nyV z;Mp0|A-G;oRpC1_)1(~Z@c#S1t=q?*pp5fV`Gy^a@$dfg|IzXJYs9~r|0gyW#(&$+ z&)Vu_`||nzreWv$=4&ME@92eT_4mTGL|E3%FYG6y!I-14AH}B;qA`B6lDB7Ln|5YJ z3j5oPH*0msF34LqlkgkuFuWOj<|S`o|6_gjjfm)<-c90o16Mbuu+L{o^zE6r=B?N) z?0@hMdmpcdu>X)M+qquCe(4^)A3VQ*e_1cH^uHtQzwHG(Ov&Jxa2mr^Phn>r;*_y9 zJB3Xb=KF|4J72^#m5+D&XNOru?^4>C6MLbi|)eixZrkpDxc_MJ4G=7=T}!D#0-tKQ0hUn&ECDU|`g+$^N#zFC%kyE33} zjBNRr+o^q1wOjsmT!yy3@s!IsRiT6&T~{#_{LWJv6|NmpUWqc$v;5m1Hj`a!?|RI* zKISw7JO`_u!p!=bmmotlU@q|%*1HIEtDeH#29%Y}7UsM3Z``wgq^{_5!E<6v*A?&a z)h+Y(`6atNEdM?)3VN849iKSs_vD|{6>t7myYRnWV`19fK8f-v?aYV0mfA%aRhRkq z7S{W_@;}~%|8B|(EdMF)p>bX%_uVc3YM;O5gvatD0(H#g;;PeT9U1e~lom5Hha{b;DRsr$FS7d<~?C zaNa3Zg@3|WqpKU_GniEBEm>=XA^Ot0aB2;GKs%&xQE9zqhGW&oT&nW~&xR+5EX;3l z&%?KR%0^Gw*ec3KjK199sBFaIvEaza#yk((rLyrZscdYg&51XY%~CcHW@Pa^=o{gU zpltNQjFi(9CiZH>jErx)a(dvcdWt$55avBaRapMJ2z%G8%=A1Dp5kuL^WY`QzrH*V zPGLsh>SH^*)7?#$b2o8bz9$OG0EFH23Sp<$=!s;cQ%{j+LW}@NeRzH8dN|>v{G-;by^{s?Xd&!PSDTV z0x41S)0GicU#L`v=I8Gg7v@tvVZJBKZy8}enG7cEA83Q;U?d8k!Ul!;9a5P8v=rv? zR~P0_twvp90`)TkU;c<4hBu>M{BF_(UkZc&A$6Z){p6mT=gh#rI%V%;OEI1qFmL3P z3w6Gorf=|6&KthdnSpbU$#ympMl`hZm7f_nmec#e^LzYDdYR50tsni&z-p0{8d1kyzI zx1Oe;tXRv8`H-f_5WZe6m4`I=0KTBD*AXPawo%K^W?9=FGt!W7ynnytKN{yxhckv5 z6}P|L_Kary)m@iub<~0p|2O`JrVWPi|N8C!)K(|)rF`(*S?BuKD9q@Ep0dzW7PgAA zFcMF=d?#ZhIT{RaP+8b1m4&=i7Vu7`%~uvud#sj!eOqM9Kk=!TQcmWn9+x!=5E6wB zP#TN~B2qOxBeiY^j9*EZQBPU0Kaiy5-#FstnY!#LEZz9ZKtC-1a&0=TP$(m!f6jHJ z3dl1D?i4m%*uUs5Y`)%weOi_BI~VTj(_0mg-??zd#u0SS^?m)7f%P$(KX(r0nFHtb zNmx)8UL7d7Vqjk+EoFZ#&xJ!v5)=u-_B*w~DYIN0?9CG0Z1E zk_?V+P}m=o!v3>T*cacuX3c%GJOg%NU*8Z}*q0lmb(TM)u)RO-in>rB@So`ZA?0QebUpbV@5(?_Lk{Dt`3Vu(8O5SGWFJfyhH znv&gLdIzjEq%fdv1thf=)B071-lTZdC_~iO_=?ldA(RJHP)B4-^^6#l^-RQ#pwOjs4^z|uUFewA|7T>{>BBGvzT?o29D%Gs^5O+xwKUKru7payp z{C#{y>4BqeVlYJJHAR6k0H}0}@Cjb}9q3BFZ73w(#zSfYpzs%p!x&x5A6^gvL0-@K z1%$Dl6a;~r5*G$c%GXy?LVPEP`kE8O)tXMdEdc=AZZRzQOX(0?s;BDBZu$SYuYFJU zw)c+F;7smpFgriLG@Y$3m4lh-rP76^Hn(3@%$cr`;WNHuYNY*lMmZr zjAsMPBS$<9DbfSrR73Un5-lvq);W-F#-zn=L{#62Io5J-?(T0=xl&Hu4v!q-G0B;KRJeGT2>E$ zxt6#a5?^wbSM=8o(eKGWoei+#7) z8pQP#fZ}K8fIcqfM=cBf{xu^M!iBVOpLWeyPPAlR59g@^6SP`XPD+AQQdFTdv_%V4 zW~CxufhrXluB*2?R~SF@F=Q(RC@m={4>iWHrX`^?5f;~^$M}j;-haO}XCHgQG0xA* zhwU(of39%BW&hTn@5hGTZ-Zg{|GD{(ZFRDJ`Fy|SPo3*uquBmlxKB^kuio6UzWzPr z(QqP|j5`VYqp?^pA#1u?_;;JaeRdmf*3A<3ha4>YyxH9d!am3L(^%ObMYP%Wg#8~E zN6LMQldxaYEd0)+_xcy+_1J#r9*%nmyCljRWBZ*bY__nzUs(8^ID?=m_+M4L#Qm3Q z?!q$NQ=2KQi}sH)_fB^=`smp^j_r3IAELjW@}htJh57X{ul7mUm){d`?!8%lUxQy0G**a$wA9yeoff53MmBs2$*B~kIY%j} z7!d0FWEs6Xh|~)D!7BiKdBrLD38W})_Cqj5{e__BcDeql!GF*mBdTV_L8=MPxI`l) z&K9IwOwXELS!;_v*KM4OwrVVlZ*gUy7w*$j2DXkekVpn2j>l1`4((yfh`o+r zd&+>f<=+_l?=*rv%fFYfM(TposA1vZN9%9Bi?F)c%0#~=3|sFj%D>)~iGIvZs*AdC zvs?bvfp!a`H)d0xVJVKw5_-+1{Fzf}&8N5SP_4Pnn1`@G=3MIsCdB)iH~wiQ-IDmE z5|}BG7vVrl;+xe3Qb8o_6~LIWD4#~X#6vjGm{kGbI!fVLsIjL(S8CdVY8~}gQUs;9 zK#B?dWJ$fHnv^9fm+A#WfWA-1Cd{;t>nMe?;WB;_il6^mT$yO~&JOgVh_;GnAsGt= z6OLyGl8Gq4PpGSBVXsssv{#JorLBiIlg&~l?4AX=HPYW3L7C`z7Nnc_)rJxA*#YAN zj@BLLS2bai^X7CN;+nZc$CTxp9VXQCEc84J)-`F&uY0cV*S}}sTLV(CHVd@+JsUE7 zw--WnipEB-B~JwJ_vw_V(!^+nxw^tv#z&e+ zP)3|;(uAl?x++r=$9Q%@AQ*V0JU`#JxU%p3I-4A8#g!_x#0Yj`i6$En`gM();W%jQ`XB`mbE@rLg(YAOF7{>tC>) zInX!v8$ns%GY2|Xv<2%A4WsAbciwLAW6Ebdb701$+^hK>!M-us&PL80d~T4>92mK# zmSx`Q%)y5z^nURCKK`s;#<-LMU*RyM-+$f?Q_{XA?A$tD!C7O=yEDWpge~V9YMqn` zx0#`Od=J1!`0f;@5RMG}C(nbqM`mu-->Vhf23l>`+dnRDK{OQ-6(8Drwimg@)5^*=d^UH8>KzC1g6qSCmQxLE~_Y zN_1hL=)m*g(_3h-QCVHUk+K2xV@0o}ypcBLPMh9pM78Y{mDg*ctu^}Ig2;oi6D3vq zPBFpA`nG6YAfcorzJfq?^%j>I(^KG5;*knvZ($N)Q}$Aw0`LDn_s{HePf*77@#SyW zVHm&uE8{L_7TB}zQ-CrhdHrlLT4-94Rx zWxnOqF2r~t9EXHxZ>3JY1&MW_FYV)NrNOpc z@T90Om5w%OpK4L58OE5eugN+cLMkZ?a~fewdL7Z?8kvSP<-`utt`V$J6{5TuTlY+C zab=r#j9nv~D+J5Si-EJ*a^`GyzJhsL$}E>=vRg-4NG5_Kj$urZ zNNmFgsP~(ch5fXx@Mf}E$^yceEZ9k&&9En!GRv*%g3HqXK(h{|qrL*>$PUT@(NKRcxVMOkLAk%IRn@m?gtlpX;q zjRPyzk~+j^of?N3aljG_*^-;IkA-z3j$l=z0n7fHgq7+6DH~7<(5mz@(3uP===%j6 z(~+JF1)dwJ_DGJ>DXLGmfQ%oNrL}e#kJ5IHYGsKkJPUQu*88N_5`F0#VR&%WGQ#rn zy{9bnlm$hZc!yL%E<$rzE-$J=7lE!c7r6Ve^T7;0$)L_c7C)~R7S(e09P*XNht;|H zr8C+2Or`Kt?nrgH1g*+Y0;PvO0AZ@u3l6D;Swp>=o3G^56HALZsTqmTs81DtnG~aV zuBsl991%m*_OPl;{G=!Jl!fwKMW{#l0~5ad8}qkUp@NG-CnUsawXoKO+r1aMR;&>*(=!RZ-n;nMne`EE#F8ET|{7>I-t`FTr zXC6NIJ$oM~cgQmjpS;ya!;Rl_cR~NfPj3DQW4isU(nsZYs8p=@l=t*~PmJY8&NKl3 zZC}9yi}1egmw3j<3hHa9G<`>*{01G4=kLXz)XRt((*S3(u#Zg{eu5-PyQ(k$*pZFUr$lJ8$|yakFFcUco+UJd+<**`AmZwedGTA)^@-b)l1VqqI zohlWAS6#otoE9*r1XYHn)W62O_+j%uwMN1|mk18VjL<142C`OuB&`m3oxdUFs)lQ% zV1x*XW72N57@wM?M*bA24rWeP6y;LqZAs2Fpo{>@xsJ+te}D1```l8C#`*utkK17w z|2MPmc05yH#IL+{$_B&u`#&;mtCQ_3r|H@Czy7In{cCh)p%;eK3q#s6o`q;692|A@ zEJR~aBQ|(u;SEw*&<;qy!FaR!KKj=#!oGfh>DG_iVR$om`?tU2f-g6Nr_z7pSdWjg zIeHdo`_nxO&YL0C1FS|NM${K4sB3*S>njOE+B;+K^UD%hD(ravePp7Cviu&mqT~j+MJP)!xDa>;@r=gAjZhu87eZzr001VG5 z3Gvjz)iK#ldxdK#mC8%IhF804_2gYJ_gv=n(-ls&6lNv4H;$Q+QmXUnRf1@@jCE8p zDZUWxhCM{}+I%^|)O}aRWVurqA@4U}eXXI~oJmXmZ8YvsS1JXTZW9ujzqe(v5U^(*;3*$Qw(r3~2fb;>WY~4DYrOn6zQd!C7&XTs^O zp`BTlZ? zxiC_YiU*ZInb2BDw~P=>q0~A2EN%a~U!pGHfxjl(sw1`f2BarM3tCzu`gNgHw0+zb z!idJ81gvo0tcpRI7?U+zZ%tqJRK|Lqg=T*5q5RPC)4jJL_1=bbbH5E~Pa+gbh9j|1 zI5rXrIyx3&iTH*;0R2Xjtp7&Z+gNemELk5>NREyL+JA2ZS^t$BS=frQK7WS6`V8ZI z&#};REYLo~>%p8}-zXw#wdYvSk0dzFg3+8>zx$51k@bt0&o9UqrEjsdAfSSj)Id5cF8^<@2? zVSmdQ_D90uU`)QU$YR(ZiH-)t8#L_SZW8uyr|pP0lY`H^$I%OCd>H&u-44U}AO6qY zUcSQ*IeYnTrXvUcZ_(bzmUiUeIqk^7Q0K^jd>9}~$9Ci(T9tHOX%x{n&fD8}iX!^6 z59@F|e|LUfFO!_nUs@mfk%Ls*4r4rWV1LvOiO(ny9qU7k^Q(x$kk?#dOxIiPF)os3 zehae9@5ukM`HPwZFgSWw@6wD{`k#hJAP52v1iPoZ(~?l5-vPkB%I8 zwBNOV>psSJZlhnDKiP&E#u|lWt|jh<6lvIZs-b$EANJ&*lm&NO;%T)D|1Yh$&tE@g zCuN8;a%$;vm-%1U#>WX4z3a-qZrFFGaM^=@8b5jDz1`Rn0g*jIsy)uj{IF&YlEU}w+#rL1F3Gn zu>cNaj1s9kdOfwC_VNDwg>O2{4Iqs3_Y=O;wr4owE6Fd}>Zk=H{zw1r_iQkXzgYdM ztxn=Ar|HPR!M1b#YZS%P6ZU(;{?-xp6X9UO@a9t8M852Lz;5Kl;xW(%lE#o zysCIf`!}+zPOFj(`*LMEug`JM-tmb6_cglixxTNzEKer}y!uxYCkEX4H_CcWvi{_D zyUE@-l4>W2#<`7>kcCNK(6K0ro_1{BLkS2pe79XQB(gQU36}qaoP`1WpW@#Lt3g};?)J!=ITCzkneLuNf<8n?ZXh>=e zTw+YyD=#SR7RYPV+9|&3q`o@1qg>rbX{Wr?J{`E0t_y1&M7oD>&_fQkOs% zI=X@rl%`LB*7g6$!B*52i~;{n<3Ck%x4Y8%TlJK+ZhT#0f0zD^d-iX&)lFCG`31aW z;htaM`uhbqpx){3CcC|#ZevYywWqqQ-)<3{5rX7)$nq)3@inS3)^Yw+%DJ5DQ{v@B zQl56gZ>LVR7&g5^SN;~m`PW4IJ`_6eFJ<$Pcv+CH1Jpa%b%8ah2l;@GS)-u*l@!1n z%Bhqj{L?~*@#6|Q#Tu2;mGl{qMpEmXjuz#BlMtNp^LvZy0=1ql(9;FBj4qIj1VeEl z@>_z05@CGaV1ut3x>G6xe<+oKJ847V&1AEbft%?>z*bZS_(Xu)18>z+7{2j^{hs5X z=QuEnUo3{nblr1(zy2Ku>UM8spx&|zz&t|jmQWT}+#%pN5aO+bsE^SuLP9qEaFtRN z80NK{NKk4_{eARaQFEC&4`syXTrV9BMz;V7-w?@7Euj51l(o3kbc(NJdZi|XdMGsD zgnr;#T={);4g8eezR_hOVd+&y(p}MB5$M1%rcky3VeXSLpYk}ZL6lBw4hJ(;ogP2` zx41IV>M0XFWn!x+6A7FyPB?z#Ad%>NE3NCRgzhpa6L-<3#GA=q{ZG4HA`12Qiv8EG z*zgohiHQ6? z^4h2T;CCJR*e5@7AWs}HmY%BG2?SZ9m($!(YfdK;{*OP9?QG;k!nbbWk0982r`DWK zBz*R(dOvu6S9ZL?Wq$P&3IESsc9@dVuXyk&;wlb(* zQF@ z^cr1}^_`zUPL${+K2`xASS_L>@KaZ7;?mHLF&`2iRIZD*jOEM)DI-u9@E_`e1Taxj z;dzjJN#%h+Nc;y^HM+hc6cH!RsTMv2w9IWer6KJ@&5}Y}P$oYFV}U=EIbjMAWyqKI zROWbp|MQsh{2HUwKJgwq4C9YHv*7Z4;MwyjzjAoV2E+KbeDN7u9rb|gcm9KO{cCih zp{GprPBd(p6Ahy{z8p71De3V`qtQe#xxph1Z#IP~z1euPZk96f@-9*tZd_%;{1F8E zrrlE}yp;)gKpDVC63ji;M`9dvGbz4^<~}}j@0KwO2wWpP0lXlE% zRY3VyO&+UCsyx2h=#5<&{_gh4pz7Sw>0oo2g+Y9Y!~m{8gQg6d_fq^xlJ)|3SF|Lg!>#65uJm2i)xMatc)1Tj4|D446=M5ftc&ka5cq{EJ zyq9d2E`cZ|i(^6Gb#DY+q8Ftkqs(4yl+uuIyyh-V4qc;LK~bb*L0>b*=OnG~sTHr* z5eMXHcqP|@q-dqf`Td@52ej`YOOxASQoQ+;IAx0FbblBA$36L9?aKd4K5xVM{+?rD zz*chRr>jsczTXKA)%**Sz4lG*9SF~D!;W}dqS>GT{GZeWSpeI~Z zl&g@e>nS4!!^kuQ+1q@_=q?0u3|xgwo)-9iP@*5p;!<<*ub`b5jr9F}3`Z&W8>7_R zhMqrQ&?E>>uaOvVEEpi{y2#a*glpR{B_%IiidwtkbK^{w<48nG!LbM{U!3RXe@~a# z7YG#p79q9ANBWAtfP21i>0F_jSM$?PW=iFy*}{A-P|3~D24`|-ht(&SbIZ9*^+G9k zgbRcDbD8O-(uIsxH>}R&D$_?!mb24^#kq;xg~05=_jiCnr7)d)TD|w3>d6OBy!YU+ zTFzCE0HYdQJaeI%s~j3ubBozC^SPNLlaD-{e(<4V!_8_lQ2ZXTSnVr5Pb{W8^s@8w zOViovQaLD?SX!uE#%Gxh;+7G&Rz(dD^<&f(8!tP*+YVG z@5sZ=edNKQG2r~PL6ztPDMdozWN=h|Y7Ksg|A>`#Xr=jdGm!Tezlconuka`Tmd4(D z!=uSqG9Hb`f+OLijCU%28mzP0upPN04CJNZ424+nacc9wUVh!m$TLByZ}@iw^GHq?ActU(@Jw@jve4avkLzNhd;=jkqa*+6fbr|zI z8r#I}+ARJq>L!%pm$aeVHbXl~vp9@=qp$K*^SCmwT>J`h81*515HAslj7F15M42Tc za@d_4D87ugC3PQDFa8%QBhtn1;L3?~EEI}`gHhcz<-ogvbl(O@)8Zno@f|4S^j-`Kvu)`;x;+}hA`Qvc2D|ir+fMF1v*3)uDf}J^-ix~ZJUUSDRSB4_j21ok`8V15whc1RS^?G~N^a+#0*x^5^#n?#yrLYz& z7*(2^^k-wM(xj#4+BaLoF?%b`Tdk-qAt!AfHhak7z4KVQgPne*J0vI_URmh# z=@YT~Z<(%8rmGxWBN{!pR<-n%jc1tQWUp5_CCGY$5x)Nmu!8K&01pKC85U&QK(>A; z!}g)Q-SPTmzaU9jLW_4-=*injO>OkGT9H{@7_Kb99JYxJ)Mv1eS{-h)RbWv>uuuQa zK67^o1jKYSp6A4YsjQFzpvUno=vG$92+*^K5j`n8!~&FPM8K!8$Vp5;3-Bm~4y-a=m-NF&a&4=jQ zf0E?-1z1Qg@`y5TAyCUDh%$>AsMjt;sb~S}S3NDu3e-v)^hFd?1Y+j5#S|r0+t6fs z#D}WM6Bk3WH9)Ix18Vae%}WVTrXlo!TyY;zXUAw>_5&4-A~X3$w*zI4Lza6+?b`Z> zXGnKWDd5K*&`Os9g=awtQFi%%+2x7Lv*?l9`(-MTyou`5$rK84#KR6a{BwI#4*l?C q+VTeVlf6|yTkG#>b}NB8zCzn)6;O5GX?C4Jy;z5>=kH1lH~a+y3qP&^ literal 130773 zcmeIb3z!?%btYKoZh~Zsk0vS6dfH{hu|c|3#QVXB8Z=vkBDYA>BE^X#D;qn^Ob6=x#MVMX|_0@e>Ar^mp^nSzgTNM+uF0zdJ^dVnf#gJbiUObdF*hC6gZFXD=aPK_kA$;q1^nw>7|+cK9ns(xz!Ax zE!GO^ba^>jma+#7aRRf8T2@Gq&F&dj?j8RsXm1JvCy*3v;>ABmXg=THq(;9h?WRN zox^YnoVPD7FPzF()LJ30E-#g>_^XO}7q zYE24{MbII9G#rm;(S+n=6+W6sCA3(=<#s0Fx6T$9bMwUy<<;rj{Jc%{Xfl}6;we|I zsaR5rM0<$7rIN>TI-OTfEzjD=BbtgQwNS+E_K1ap2`w4!!F02@f|E`$W8p+Zi|be4 z+WA;G8PP&+YXmxPtYytzc$7@%t5r3m9$cQC#noI4VX-Lu!>3zvi3>FETf@4K%fAmZ zKrH;98)I+CFt9hcbyqM*F1@_82T4R9G(ZF8x zLZ`yxbY>~LT%O6*@>vXTt$M(AE}Ck-ecy7mvTwe4YG3($t+2F+xf8}SezBH6R|Ax; z>(u+28;XmI^f14Z1z%pAHj6&f75qyyAa?)_quZ-{-G^Tdk8k^u6vfgJcUzJ$vEy1e zE*C@didUe!dS`X{RJB&DE!V^?i1}>43uBQ;QcK2N@4{$2p#>AN%Ffz!&uqlaEjhCu zfLS+nPxH1cl%4tWG^uDJ$yE#U?%n-#ADZl>^w4sxD?OOPTYQCIYKC3;1x#~yTqW`Y z_v!-O%k$>i&Bwy8UCVh``1Psx75>q+pI@;^2v3cu+Y>bwg{BgT^mycLf;LqvX3Gn< z2Sz-eN@}U7q_VEW@dTdUv6zc0K@)ZdJ?o13Y7L9Q*-EjNx6jvj5|4&>%uNqW#PNU) z#+(m-aVy+{1qV#!;$gkL67fh%izeM}F9-|WQh!L<;agZl4wXT@TnAgQiB?%AzE9}~v6W9ox66w;DWH(6t$ zNEoxY2iKiaO43McOVE6lD)zySg`hR3+@@PR6v505_gD$GoX%Gk^Ybb!{bJ2Fw73(X z;m6$WtOPMl>PoQv&T4KI+*HZhwrIs*LpGiqv@Oi=U|0E>+G15mkxS}P{T-nKj&_r$eHwW&3oyh&gU^bZBxcDNm@;P zf(h#iJT3hT7smrVscgMV5vcBXLs!@eT+DgH9TO&EB!BoxR%AP z^a;Yewu@Zplhno=xvrI3Lh5U}n3aBBjG?{+^k2G4zd*R&()7AKQ#XYt083Le4qun6 z{u_zKVI{jSO@@cjcrIC0>u4+rb28}W#6qj7SWlz1xucw`*)`h(QP^&Lw@gJCV1aPbYqV1Ng;(BQRt+d_U}sd8S0CtUao zyD^f0T&JIt_ua9+>$H^ztfq^4XC~j@9#DCx^r38$fWb>qepoSCrl|q$u5p1$5}okskQNk34v4r7$XfSYF&H zIm%pJEr|qHv0$o)r%XC<4(1S7nmBgcyllsUAvgiyVbC3vrfEu52)ZK>+ry`|Q-&=U zmd@lCV4tYAh-fSr4kC!FrE~|Qg>^U-ibu6j(mhyNdN=eG;l`Br5?F=cqQW{a)Mgu} z1cZdf?W$2Xx6Uu+W?VH@8_6yEWm{@k~YLM#gBUP7O7_I0Ycy;_7J440tv?Aiwc@orpwy|*^L@EU3< z-CyS5duO#!oUN&W;P@Uj6o>7+v|NVePkvdO=ok&sD;(Mkh9n*iN3~SgeS#qxKA)>0 zU?nBDbA>w(SpcSmyE_fzh;CrAxzDvRc*Ed{f;-Ip0>%^Y>a#u}24tIXbjN~`cq)vU z;m&qb{#-e?ID$+HidL=ZU}DcwzH8Q(}E0jEFoF?WVLax*jP{1UtgZu%B+PGCBP z^h^T5nFk<}48qah>%wiVKutMAv25qH0wsK;p}5O^5lO-|4b8E~^u2||ohvWT6c=n$ zEQ0kVgd57?B1RulYyNLSTz%m-{5%CXyJY^B9fA8WP|v_eL}ohFQS5FqRn{O-1k|=^5$} zSuz}(v@?z3CXJaFG`=k$Z4%z}9uLYfnrW0=P+6K+XK^L1d}BdGOSNP~cGPm&V!=cN zH%+f6@L1Qmv#&w}VOt?Ujbx#q-d8JQLbAkOyP#tNO5Tn!$Xcd(Li$xXL(W{4B+oY?X#_^Z6bwo zTeni&fUV|+2an@MB*ORn0KX}vIdW#HIMX^Z9#VpbxLh+z$D5c`_ngzEYTySd#DMjDD5HMd+~=`7YsH}%*<4d zm64`J9q|$8Dc@mb#4g_^(PkLrd#02l4)Ik{EgBPjJ**B>h749C*WSKep#K<2WEz90 zD>T)G?wL|$%TGhb(k@UpH%*)?R`pSy8X?R^OP6jE^>kvE_3u%OqS*&=G&NNEaUF@e z+ePjkn1wqg#26>e05T6GE08$zmO|_W{&x$$3yBl18gU{gI7yaY|Ffjp%=dufWqSdBsMh zt@`ld>=I_2o=3c?If85t%-Yc$u4h(0zxFh@&nzw0vdh(cR%j&E*2kM$C|!VpP0*z% zD|;)I6toj*1jx_c3_i$ursR<$%coc_J%8_dS;K2G0-Mpb$3*p977-FX*Bqra6B!R8 zA?^NhWp2N!ug}NI`DqF<8Ea}FS4A@6gCa5hahV_g1Oj~YgveKffCT(_w@GR)_5 z3#Vps$S}_a9y^+=1p-*_v||q+nHpE==;)*Gedy7N!^qf@nO78eyBBAYi~^YB1lli} zm*NEgd_4GsCL2aLkz#dw#)YMfpy@RY7RY`*+uR`ZrB?Hh2&BqDWMz(`T^O3cqhc6I zGE6!|h4Yne_*0;Y;S)#jM^xdAFDlsh~^jIA%tQOwN!#~wU9u|J3jYEoKn zp^Q~Zr4Un>GGPxZW7rcDXr|V%reoE>BMXTx%jGgMX7&}PKPZyNcMLT*B1@!vzPQ+G zJ|iLosN&oNnVx}#bX@)5a(+24o|-bR0zj>BR*+`(I^;g*tJBC2;C)YlS&?Q-LVs8+ zXGDbm*e^v{SE>hsu&q3)^v;AD0X3JU!%L8f$u=IsarjDseM=T z79k9^PD2~e%1y0R}rm^lq^nmQ@xJpV{;6bb2prTsY8+0~g%8#oVLMvlqp_Q>>Dd<5$+e z9lLPKT5cJ6e>yU49y(BFQaTfvcGLnV`iU({0Ek6Nm=fpZit;%0w!+d3)=9)19;fBy z35aC@p3;GG)tr1lj?vh%vyNeTV-V4r+hH>F;~{xDicFK zI&wu4rYM8H%1})*iA^xNJGu02y8UQ7AZd=eDQ#CBOTPo8!lu$r!X3uV(mMFkE=76j z&?A{4pD#TkHmxhlu<|Y1c+vf(^`n=z%Y*lBP?WTiR5s)9n6gdr15C7OS?wtQtImwe zO*$TdxlJ8pEr>DwxEzzGLMVN3Ui^FgJMOS7OJ283uWi?T5hh4bYD_pm0#5; zk@wl3{tXRb1mrVSvsH5C{@UF9(y841=d`8S*=nB9NGaa*yoR)17z(H1nEl5RMOUo{ zFxYgp48{gB_$QhS9H}jiFgFlm>t&q=D4BS7~XHN}dm#_U3 zoXt%y-iLzzV_MyJ-`woER4mA|jK{{OPf^T7=a24>ox_-|`uY&iZ?-=$gZC)>Q z(zg;^8CCfH=KG6E{(q;yO)KfWn*{9g&pz`TU;2RdD}VV@r+$a;Cz1&(t+6f$8q**1 z;40=5jI@%?7K-KU^m1_~I~z_VqWMT}CKQ_q!r6Ac3D!-}x zG&;iTjp7?^3#1``n@ESBRz3l|Zz{t+tY6>zKb5VUm8aUxult8aKDu$p|IyEG8ol?k zTejUhq-^@;$cD`$BPjhF-mWP1qklf4Jms&;a-&i|{^vu=Q=2X*L+_|7ttZRdn_nMF zfAQr_TZbTO{pm;-H-x~vO4tR6GJO^0k z`e(Gb;-{-^3a`hkhMv<8MkSiEUoXG={U0pRVL-3f*(gVuP{->QP`2QYgUX=l&p(x1 z5n6B#n`HxqTCH5&zi%H-3d^T7_`CK!nw!ZlEeh2F_9KjGJV6jP&^V3<|C1OBrzSGjZ9C+b2GE49K0ThNMtrSn-A?#9ziTD zU)fJSWc86p9@NE91N-Keu(7<7E$3T?F?~9RJ(F{& z&u3}(@En}1vrAbh8Cm!cvP)I=9#{9lyK9kLoV*5cy94NcK+kqX`OiPF*S<}M@32+K z=dcrhO2_Bx%w)4VT(-kdMfr#hHz4xpANGKMtp|LJ7!Pey*shUW6y;BG;HLk48*n%M z=Wht^qW>rw-hGl+H0XHH4i#-~aak+mss%;=OYmAn(JD(6>QXfFsS0e;2m#K|SC(+W zXJ&A@741~Ds%X+|&`3#yYFk`bUH}kNmK17*bkJJv6v_hl$|G4?l_is&E3`;!E#7jJ zB`wqmC9A{%g-e^Xct@4cZqH0lTL}bIDCE`Y3hkM-R%3KXBNkGDDe1&cgi3yTS!no{ zD$B_reK7mr`;SZ)^bc@7ucKTBZ1gNYmvq0A-p=&AZga`& zw$8?(-@gkL;#SlV*uLO0&4hYsKi>y9(erbK?G^^4?au{5HvMQ_B3kML{dhe{>kp=~ z1Vc@hKOR4+=PJqr027SqUwF5~WBUGkNwA<93CB#IvJvnpmoz?n?aeG+-6GGDIwH+eTH|^Ac>bOPjK@w=$iD$HIPv>g zS#GXHl2yCYcr4=CX*7{k$auOX;o5Qu-9D@Dkz_8gB*xj0dxJEn_ zjbNLY{zeWFx!xip9#j+K6zt~h-Ke!Wx(6e@)!fcb$itJa4oG@EWn~faG&hJu6H0$5 zoxq%GZa#W=@>q^`qnq=rwjFz4_TXd3-k+U(|KuZwjv@4Y3ZaG5cmd?G_aWEf-1Ky7 z3aJIag0`u9(pu0 z`QFEl96$7E5HGbVWg%2I-D@il9zJsH__6GXiANr5m8MBz5}}mG(bT5WEET@e92Fy_ zf;=nAqJ#pCmQL&CJW7{VLw#C3_Ig_#8q{jo(drKL^^pgUWgmXm`N6WYBEgzgXGL@Zt?~%+S4^JL{U5$?nYCP&_eEUO>9z2|V=&^^bogN{u>t*)2 zUgiy=<*9T@T;*Faw*=q5vuoq=$;s@GKg?Yn#;LW%F*&hZTTIIqkpnB6#6HnE;@7vu zO@wjy;bX`2(S40gZq|plcat=k^H_@D*lisoAN&dZ0rRk4KB1TA^zwqPU_Y;yAJ@yL z^>V+C_mE!xcUpk)dxw7hpY#&StN49SKYvv(KdG1h$aelE{rtUpSeNkVp<+tQzQPyjj)|HC&TI!d7-4^+6QsaL95T5?{ z{qcuo`PJuS`74)Ynf)zU{>AUh@@p%ye2+eITwc=8xny{+@|Se{$8B_LrML5CIxg+? z_VcT4|7z*&^03@o+UxD-*SmeYJS;bt_ImsI^_Kr?+rL_R`*`rUacQr&pI`6x*&jgu z75cH=$tA&s~L~WUVIqN?Q+R*E}0LP_V#siTH%uM>?QMKJZ_KA znJ?qn>G|AV&wT79vjE^ z50^}DFPTrT_U-gMUX0J@ESDYM-X7!G>6wompU->oWj@@GUU&?*x6k9(t9_>HB?q@} zFB#un>eIw4yb*X)jzFu4cR%E(v%(wcGkYWZWJXhH5gR__=P*p$RAgp!YzVtDXuk}R zN038|x191W6Mg@Po&{~KGi{H&zw{Z{5zPyh{T0Ye79UBV7LBhYRB~r&W7-qrs-AO4 zUeg1$Wq4)-CJ1eur1x%!_?BL607RO#aVlSHN-tIw8*$_Yi_hU3Ma$SK6c>9=WEM9h zqQ;H4dXD3@sKwd=`a|zzeelr1#~xB2IG~12o%<7+8XU?ODbqW^d+gTs{RHe+dy{W! z)U$%|rk79;?xX_%ua`8LUU!eYO z`jI~r@?K~jypnz5;$Br?IQviasHzh!q1fB{l&n{%UL^$E2vR-4sovMEHRK=L#^Y@q6Z--&mnqpYm((akh{8Ix3+7J##@(aRj{ZPZ)SSKhDe< zFv8-NK(G9$(4TzDM{7=;G6m8|%q|5}%JNuU8L>jFm>1JT-3#MNo2h$Y?DF+%T~Us$ zFbDTmE>84qf0e0wwm+bMlYRSlabW*mR{HmE*MR;__U+%rf&F{gH=up=uh-a2qIXA? zi+y|Wa=kY_@g43f|KvdVpB*6o%PYOx9~SaY_LcwH0rFo-_nTkC1LZ$9K>la@$WP-} zSFTL-li0F)@V3du}z5S-fDE9*wb@532nCm8COg3j)t{@KQaFu1>Wf`{ZhCiw|Y z28538M4z^HWf(0}AE=)M8!aEkUv6`BoFF{jz#xgniHAP26&?%1B>1RC^rHk9E%TT% z{+%uA8&eU!^^B;+HLOt^R7((vKMEMGjpdvmz!5yTSN1XE)<(}r8Fg+LDTZ`g*h>(j z1z^w!kQ64VrOYHB<4Qc@^6EZ${e8;wEBy4@3W@1AdPz~}^+fa=)2{%a0qYr!J^hT5 z=?OL(G}qWM6-8C#d?V`nxZO{QGtu$^0UH&O%x*o$HAyO)e2haJM(KVGv*e=%YMsxx zA5VauVI=Ee(Vm3RTo?@=7A^F|8Fx_O3K%Vs3|(3rGk^3Jd`CBpMr99qlt!_<7-KP7 zGWTESs`pqC_vSs2SN3@{Fd7C7=iDB_M=$GeQ$Owa_h7C_E;LgNE(XlZ5q%)ZNA-Dt zW6AF@wFc=IOcF@E(JhjGG*0aZyZ~zEnJYtdEHQotccNiT8Z#4zsD$5#7ERE-qdFXt zp=Zq9fSO@6zYTgwg}Jp=@FQBrfDk9l6iH3jhPa7)TJ+w0FVJ0Xl9=#RrWE|*vKPc3+%f)E$p~hQDPOzM zaRZZDVCh__w@H4CG1~2slxuVwaX*aKMf>`i3y2t7l2Fo{`y?s3Euy8dg`@^G^^<0r z=>44rT#OINBJO_7WuhJ(B|HcHy%I(k#K9^h&P=?-*aJ_tC^>N-WNU;W$BA2Jsfj<~ zP#c1e=((gKh^TW=6P4sZbov=!vTb87NIoX{tT;Fm{JHHe*KQwl43^qxiRQ9=mKZR` zl3WQx%0_LBA_s=9zI7zfZ@{P?weO@p(UJVgvh2&$<&gp3D&!nJ*J)E4>6p?H1evsf zVQ{F+yW>N^bgh4aWc!GJb+@Ax$s?`!aYY&{X5Nies*{k8&Nb6EV;J=f;L+DQy1m4v zu|}HC{enVOI(13uPOmEd32w`+%n&CbU1o*2+PMUk#za5UzZb@DQ69|SepApgy+0^(DC@p&iC zApc5wpnt{SKl)`S|N1Rv`JKIB`M2cn{_S4=Czk&Q-?_(uvsV74U(y`3G%VcyUte*r zZ{ht~c(W^BRr#^fv#=+oUccL0-DSKGBsNs~1#r8O9p1Ge>2A*Sb9GlgSgU0h&h&6~ zpDpA2!SFG|8*K9Q-jHtW(l68H@Vwakj=e9mCiB%%3zjYOpKKc-?X7cQNdG zhPGd%v|Ak3KIyRj7y`U-So_R3?0R!pkK7;*Yt?gD_jOp;mC3#?yNi7tc5ecQ^booG z3+W#A&|CgDyTh5T!|=g=?%FYBvhUSVlpEb4-QOKO zrXZm7B41&im*(=-ePk~Aly6^jDzh~2g9;lGWcbhz225HU2t(oWs)QL3>-p{r3QU6D z_s`|Jf!}DG@yO7@9*jB+x)nO2x=fnN4%)=AsF%hLL5*JdU!-#h@edhrf_2i2+40d4 z3?h`7HbWcWYEWcWRfSA8`LmXtI$@p=RkFr^kRbk5Gj>oRlgG@n)24HVykIjXjT(vZ zE!`kue0#|Xy=M^FM^2ve_`Gg|sPq-A|5t4c)i-;_5L{$ao-tIb9++9mE|>Aj0la3S zmaA3S26>aG^w7tY#D-k)-96`fMv?22z_-{V-He|8w$PBp78;_vneXNGW(y79aJG>3 zNyyJ6B)6>VC zD9CnVYbn!aX)-#W#wtUsCE7OyZ92fBpKOz?B-la|283!d;ykk|u}=7`wRFbxWW&9H z&=hPf{CDpMY_YaMVa6g`O;OkulGIKftRHx`kozd|_*|VW^zo|=40*PY+)LouLN~1~ zWQ2djCm?N~vXxd+0v`tIe%g0HU}#txLj@Taf+4+rjiHSwK#?U1% zFx1l+AydZ-4E0mi$z1ED=F#1Sz>KfIu`tk>^8!Ny0z(aqvQPPMJ&d7C3Yi>cC7sTU z^W6u{sy>?tFqDM)-$*y=_UoGwN(3OSMbJv zh9U5>3L5Z~5NQ$-7C}0`-9z$7005Du3}eHXY8&;P34%Ak57~NxV<)XDY)%<8+%}cS z)LSt)m~lFFg1_o8rU+`_)kg;Hlv7U9_3?}y^QakD;07^vPI`eK@qsGO*jWe0PBIeI zlBREn8fH*39@WA!CWQBt_K*CGNMq;o*wlb5xDmR&`DXrKZ~uOeZP&b5`jpdebK)@l z-G4RGH!b!L?%v>9U*5{ws(DkjvC#0AYWHP`wnAICR2xgYl?T!DwrW{p)Z4dJ+j+1t zVo(q_1Irwl`DVn~$CrHOZPiXLs=m$IE*G}~{*Gknj%e(?28|Aql0O6CXpZWi0guQ+kMZP%=B8)L>}N9AO?pR)c2L~d3Fv_H|e z{ei}Ux9z%rp4-($ZYVY-XN!5xnD4)9z3tlk+K%Qwci!JG_&V)gvJt!+xiPI727acV zh6#db%qsQC?GD+UhN)3E*GPke&5y<^3=Clobnw^U+|fCnXoIZ@PG{zGiWEu1gWwrZ z!5pDO{D>PFQ~ov%@RQsdX*!ao20kPv+BVu1h^TK&$fL_`DxLKmpR5mdjU7KIoOq}g zm`*cF;$E^umLyLz7hb=2{kikn5w`jI{>*orI81-)|M-CG^V>|n<-|`paG3s`TbCSl zvVCc6{r~&i>sumA?5ZOp&u#Lu#Ngcavc!-Tc9Uj_874t07}8P^@i|!W*#=Wh1jd4_ z8E7nQ(9^`Ev9Q5>v;O>TPD8-Hm=y1J;_zZ{=YQyLEPU+_=R!pMPi`fZzloz8(^%jw zE#?K7PEX*flJJ-rs_{mA5AAEfuc;0~7`rkDce(K9k9B<1Dt9!~&u-;&Q03j=yoRtb;k-K} z)d^ptPIa_h1ZKkpI&Q4eaYiWDWM)V+fy@Ue_xKN0USLoRDlEE>< zGuSyv)hS0KJS=E~d^U z)fqW3Xc^o7LRN-zsfzf6ilN|FveKw>H)?&gWl^-g2TO6n1xreeYf6kIj0y0H9pn*v+L6oFm(~q&C ztn21L#{^|1Us2L9H6(5(O;mt+we{v@Z|L?L#g@9+rzs7z4h3 z^{i~K^T>c_40wAcc)$3~ZVV)Y(Xf^ZxkLt%!2~`Qkyx8Cuu&QVKQ1E!;-e0{m|QPo z0BJAy6xuK6L4X_vyW}%jo%aS&A%txSxcJF z*k=9|oACfKKW^Oi1ccsW^nsVeA^vzsbJ+Gd%0P_+L@__m3hA zy=Vz8BMZGSjL1SSj4K1icbDj&``Aymm65dg6kG2r^m4tw{KFGe-%C1*9k{p66c-1` z|ME)j_J@W1lM1)t*3z>BcC;#w3`6ma+|7;)mX$<&NY&;(J5^uImLk+IuLyN0? zcl+x!;{G<_zR*on0SG@R<83OLkJ`<7mkKu3kdn<~1!cd@$JWv>c+j2z1KWy0@Q!kM zf-o9sJC-yD{1Y;3pK9%1eW>fJCoO3>8At|RZGm0Kp+aw)a<)n}5?;5YF=d1-nN@=( zDB=p86$4`B_G$5<)K@^k95LG9!dSfizWTrO=$kREX*^&4k4_w>zr*+Ud*tLBXoLM;m&p9c_r~yA&Ln=4a_z8m2XDa+ShmXBY zP``kZPaSlC(q}3fsKL=ef;H$6Kgp>vae0N0x%98{5tXl?-h57H=JTU42CNV|GcfA< z4g;f4RNF~2Mpdh}rTdNJg9fQSOI%)bx9GbqEgvxHAj~erl=OvXHP`;#O+HjP2 zwd)SfppQ=|_;v7qklP-bHgtLUb1Pv0ERYUAEa^QGuMjtR|rBnKPUW? zHDU1Y)R}_Q`C(%SnnqgCx4AwIi^rg2s8MpdznQyHt0<~&n_#pc0mfv4Ff;gzQLE$m ztXm4Q4QET9C9r)@VeGJ%u5nmqeDDAxBnowKdJOS zrPRY1C_`m6u27?m3xl5lk=ZNqoLu%sja1R0%&!;}85|oJq!Op`Q33-(?tTNCa5HUr zPELQQ_ZJ8zUH_R?ah!%y?K5+2fGhnZ*H;jBkUv>ULxLfVm5gsU_|VUa$Hs#S$GuUT zN!#%{Z7ZFl-p1FS4FqFjwI7G)Y%dd%K?Z|@pYLzlu)(LF`8yh4Me$w$s?FrjXt{ZE z%GZ`ET6TJ=e12)U7N~x(R?9A&*?(Xmzpzv}f2No}8#r~omd{lxx$}XueKR>MthHFQ4{os7g&n(HfnA}Qnwt?~VzTPzWTEp#jLqH@>Y{%hC7ee)|H{^ox; zahU!qzy5D~&?_hDp>yc3-RpxrHU`kLN9Kgg6*3kr*~=w^KgU*HMX0BF4u8KDpPxes%1A6D$&?rv!HJZE# zg10e%?=<)F+HVZtTo<2m^yi%S_t6dhx|EFm`rG)^T^k6}X&dLAw1CtB?&3fx_8Dgx z!Hc}~(Fe$fIQ5ib2YP`Ra<(D4zhu|iO;#Vg%Jwh_ZR zf+yF~4&V-|{NMpe1ZATwj?+Xz-!wb1YUV~z>p6@H&=E}onASQvKOv-Y!dZq(FoEPT zL#!pQ&%gTf&TB*b&Cl=CUv}a!{qb);&?E8Rxc;v$ryV#PTxU-_Sm(wD>JTRv^+MA2AM3#~DNu+_40V5|9NyWCm1OW%!P z3|Kb~SQlbi!l4XrlTdtF%`*lZ#*nvhptmtVs=l{z;QFNl(f#X15C%mM3}w*G7*IDf z>b+IjF=&FMv^Uz$m6M)6ilPc*pi{TkOS8cLVn$GMEo~EEwOqnTVb8d#*|HwWcw3qT zkOH(g7V5V&5d1^0O!yBHreE+dK3Uw5E&1wQWu9mWQ<^G*2JwoZ?L|U_WgZWXG+3eN zLs~*!Fr`|V>8KX6rdiou=gPjfOW0HP*N?KFifCciUBa<&aLu0oZI*PQZM2;5VsfJ? z`?T;G%D!_Mq{Y_m`FF$8ao(U^!qh7LjM7u~`+NSqy#u|KeVRv;p66fT{Sn4pW!z<+ z=ihBqj5X-`d;X0qWdI7|3Rhp}PB$oM36FCP4SU+2ccbKWb3yjKdXmsbkKVlgc! z-x6wz48)QVEh={%4BTkE-4YqtPRkB2CfCbYK$?(kZ-KGw-3Z2l^^8dc&R~;D~wBndk3jfPJ zP8_EH7hib32fa)Q{NFZw!L|Omn`leGzPp@#O#9yWEdf9BcG=EawgfykDz^lC7}J36Gv_dFz}g<%VYI^ZejTqUdDBnSCB_zmd1%O zjCZN`ZXJ$!yh~kDIcbUWMd+gLHN^GG>cI8~^lviVk3U%h{rh)C>EFNM0sWio+rNtg z`*+1Rp#5(DoTE62-pOwT^u9tA80dF>$X0N2rZFR)WGl!c0qD+JgW}=<`LC=ne}jg3 z4GZ}vm3|}e>;U<%q?y0l7z_`Tp8^AJlq}=LKJpWe{8oUoZ%l7q{~vtDd4H?o)j_@K z(~7?>YyFnd&$Ow9F$x@{`^n#_L#wVc|BlWJny}!O>An;wlO~)D^lsn_&6|$KlwkLz zKU6Psf0$bacxA^&yQE&vfq++5ozgW{nFFiz?ON|`37GY^2hUf>)w8)`?Ew8z@&3~X4;_5$A@zX+YS^^(`AP@4e_TTKfJdGdu{oO}a zUXw|s|Mk6CXL@%oSGw2V?HXTuUH{4i%jecU*#hMEr;%`%->gfz*#ajA*aBm|OuH9v zg1@4-%dMN=AOx*`2)4vQlozPJf&Vxb+IMG_jvHy|_lFgbW)6nNO*(s{EnDxTfo|!e z)ThZFpdabn@<*lA-7}y+F@Y8*2qsgM%vgkf*(V`_s!k9qMyCxKz{3vb^PsR5`qej9 z7}u>N%f;HF^_+F}b#5zoNkU%S;3l^fjP1jTXh=)C?kGsa6IyDG?-JT+NfO#gOB63A zH=40POP*mYSQkKETJ6Sy8AH% z$-ey~bAm>xw>khT85q6my}&{L@$&)){mqF%#=`EpG5h>&=RB;77jzmy1F3lPHSwp7 zz(c0ZP%0a`60Ry8ovV{K;lc4g0qhm3%@9D`3x;BER;FdJfpMhy5QM=2bzV>!W(K#F zNxxwEQW^yT*p!gwMPt>}e_dt+GNxQ^o6kfuLc$EBrW!U@$S^V0`ed!~d|v0uzLzBA zDf{b2*-r*TT0H3%8AyhcT1?gqj126uWC`t}1?74x`^XZyX?6s7%Dz+Cm+uOa4>I%P zj2~tf`zpI{0%hKd4D>ZOZsaTodSJ z6k{17@K3Z69gt9p5E%8$2lP;zrDjqY)=`3K@?e z)4___0i+>J0P^6&5U`m8X<_VeYvxiK8_((aPM~a?Yq=d5Bf1&jR%<-J*SRrJ_qGRn z+k@B3_TXeZq6OWJfn-WctZ~c0tpYr+h&i3851iaeW2{39ohwE-EIGhS*FY? z;WfH2v=4|GL-46H1oG=V3F7xrE!R;=X+YrTGsDa{N4wjW-7(7DZOd-;jXLSoH*W|g5JginF`*X0v;b@PVwBj*878L9($34wMP!V z@K?_Jd(7Wy%PW_GTI#%pP-b1A#ubJ#uCFBoBh}u@qqDkum0?nmmp*)kHR^in1V9wR z;7mpWIJPB(-YF!1)>0M9Ji{_wRXZ~wVK4>`j_GeBVk|02MaSRtD>?<&Wkh@a392PC z_-rIyUf{;%Bq zWydp|>6Pzo?V;=&`;+ru+U~$%`s0(TqfWLj^@H>!_xd-;Sn&27cxgyCc^cA|L@<~N zMdP7lDiw6eK}rQfcvI0DErs3EQrNUvgdKL9@6_uRcIag(&|3Fq+iD(pUY2>!!-6Omik;e`=@yPtgKutJ8qV(ZK9Ri~coH{nsx!AWIs zNq21lwm4*6r!A@a+bQ@su&#$20RgGy^N2$e*Cp`R8D=g|q+u_()=>M6iOaASTxC)9CUsLLHQ2Bzr^}ztL~zR_=yM>;VOdE5i0E88q6s0EmTQ6P2Wz$L!kPUC7V-;AmGftc`Lh9HovT!G z=L2WUwaOkea%y?j^fsaUCEp7WX1TJc3ijy!Tww3x=d>(v)mglV=$u+yR4ci&IIli2 zuFlOboyyH;tHlrH57d^+7$_42O^{@kY6GxGEzX+i)%<)luO3-i%*)P-P|D{@zf66N z=WFT#DG>=oZI7$Q>yErPCG9@IBrEq%*6~B4IrmBi@FG-o}U*wm{gR@2h&Q?Q2bG zee=Q=YY$tv86)Kl?CN({R?8wRk*U*IyJ1icPNr?HA@f1vQka5G0W${IlX?!fe45pG zH-mxean=zoKi}86vEYR*ys*Vh9=0%yg-|G|MO=-A z5WcYxlr;l$oNl)m3%Aqq!i&l0pK*^dnq#2t%T64o|9=j?t8WbR7eC`(e=}{E=zP`L z$F{y<;u(Fz#8`L3gk0Q_L*&>n;r~O)=e6cIed&*!?YreT9sDx`kFVc5h66pWpYaWa zAGyPcBR5PK4>bED>sY=2t9&lPvu~#{&abpt6}LYm`Fz;N7|sVd@i~U~k}|Gac@fI2 z(ViXS$|`fW*YNqS0sWio+dncV+~sEeR}{u@$D{YIGOk-W*|&ce2lnrZZ$SH^f9|qP zqIdEe3bNh}iy{|pa?>>-I2jxANC0eV8yOpJTn!3(6;kik73JlX-t7+y`6rcr9G)E@ z|CMw<`G*I}Pmzm$@;}>0ei}dd4FzL7>|r{la+rmF;29dwUVEN?_M$LJ3CSxL_w{NCm4Ds>jRf7ZLbOcRfLMrGo zdrO)v8NdM5dD4c5Oj=Skd`wMc$0yt9h+gA-#at`v=r3}pbh?Jl$B?G6O5b?M;IDyi z)JkX@kqHqliPxW{{7&%TKnO z_PNi&W1@fVbz}<24H3OLTz6&vLT_dH&93azETqVSc|O{&RDV-IQEqf&;L1e*{?)}t z7VNz?&dtiAo3ZfHMpEewd>Tu6-e}Vi?{0>=+;+ngG%FAu2wP~XD*9oK3W zjiHKTU-F8X8*#9nQyPyIV#dgu^T8-(`c0!QA^ssZoG}IMw@{ozyrP>*RBxCOojP$NvxVatc6vU4CEaP$F_}zm z?Mq|Dy^f$TwpQHesE**QAFOj@<6B;2!i!9-m&imU9@9e7q_9OMBB{6*S>xxO@0P}f z{*i~fX*uD=vfju^9Qi{h4%45S{g*xH;e1hi%JW}!uUEES_{700O*BQZ zfs5an{sJ$fH%4S)Y@6)AAr*~^u{>i>z&2UuQ!c;k(#IkBkq5bF!O2C{_b|vBXT95= zhL8M9+0I(_G<@q8{s@F2v$++wJq`cqFO7ci_5Ba;_`iExUt>?hgDEG@pgj#YyRqU$ zE_z3%-t1{k!-y-%k3jUgdvE+Sronp}{8;UL%KLxZd~f^HyqbLa`y0Efp|P%X+5n1# zqE}G*>x}Iz)7>r3=DLnhoim`@-39|Fz0$XD#>u3qCM`A?lfn#Ty-xL+HV*NV(zKDl zB+nYFRLVF*!$-%2!yAYh-(Er)ZyWt$Dx*}z2JGIcM1I|v@S}A%J}`DBq$HUYgR6wl zUqpUD=}t@Vvdf1h%=n<6knNJO<&=Qe_qRW4zP<*g*%D2k39*7{&0UG z>cC<8OLrb})KNdU{_z?2`Zs7#L)$YZJY!Z{!IYna$Ewn~s;vUPMhI`C6 z>-91wUfn{8NH?xAVNG)~7Fk}FJ!7K3F)?V*!zJZn-$bA*0}_FT^$ntYJ?!N%Y!J1> zjPt?UpVLbJT@7+igL7WUHg1A6CU4J!K7Wi^L+d7A@tc3oL!(W`#N%CKg4A*P$?=fJ z1fMxeD6?tzkt1CH!IMg-ZPaJ##JMB5XWA&_j)yuO*?Idf5F}{w_rq*?kzmpk@vqP^ zDe(?UA44)LCsu(oK{y1H(Tz@891GZG)EV+gqlF-yx~PVyj0}m+iWy7RNUD?K^Vfo3 zw-s~!OCmmDhBUy>Xpxoab*}7tdm6knrFE006io%SsC)scEpiYGCA6eWU>caF^j4|t z>)&a3D=jg+nA|wZzOmHto(8$}84DmUjm|v{1}$ULo(00U?rAXaRMgmaH5j!Bb9<44 zlk~9v*RL~jFlf(%7dhycvwss93&ND+yU|)s3eWpRh0QT*t-K3guJ?{S5Qg_Qq<*7^ z$i-E^4ar}p`~L?IIlacTuBu{BgL%aysBsQuw=J)0r)@xuE4|yK*n$i{qmItf3VWV) zE;bBfCxn%b`5eWL3gY8lE8n)nwG@l6;&mooalouNuhhX@r*c#IRf{pgm_`X>2(bmL z0o5?INqZ(3+b_(3Mx7uGL*!z`IBNhKh6wOrCJAh5$XI(tIE@K`(+-MVBg~D2W8@|`Mz--6$OG3e3N~(NpDL9 zOyxeVZEs73^S-Bbi&qJ6OGSTM?do5)8uwQ6~PP0bAd3(7EttWoOPg0BOs8zTyL~{Q`43T{{(CAh6XMJ z7{meff)m0MLv)_Oe<_Q!Tsn1vc0(5Oc!FRO@1%>ZVASGeR6c9ZI-oT}>OD6@NG|;R zUgx$z-OEDqvXItG7E(ME)nf89(YC-uJQ~uH-EX%2+hIl7v(kF96)5;izk`#~zsDcE zooVboe1bZfibtXmq%%ds@-3~Uzd|jLn>(hLv*mmRAMsn5$%-!owx%{WN5$#%((+=h zbvQlLyhWT7-t=;73g-e7UmPTaDYbd)Lhf9)I=@t_W~JRPgzEE#ykkk(TT$k?mzb6;fejh77xbu?qG!QqN=!QEK2_h zh7wqO$v%#SP0bDBBa5xlmr*yRlzvYix(ze*9mrN`9OuoU>W5l~m67GrH*iM74r?wE zjs;_>P%x+kQ%Tzd_nzP-g%w!>DB;W;s>pP49)6@{X9YI0n&OxH~G#@=Q>mz2|%!RuqkG|S*JYQie- zoVnvSqjS4M#9{D9wUTa0jA5T1CtA^hEaEn&=jIQQVDAO%EZjPcRwbv3S*4(;qshS$ zi-ZlElCwxea+0XTl98J5L@Gr9hHz;b3Q0Bm{wFVeLzj73gT!6YmkHNNP`X&e>&8S*jA)NzS^O=Vgzll zsINu`T@=4R-!IqChOM8CopeimFq$?P4Tbf?iT}GUMj6IRTMX1i6U#>{SzssAf=cAEsJB&4)3y4%Ck=m}&(z z9LJO!)Y2YIc|dtHnDT;}8p4zh)XG6j`9Wc3SZwrq+X+ zx{9eFsKwiu3empqVrl~@VF6Pep!BDh3WIw50#lu!zP-ez8UdAkgQ+O3_#RU+Q0_0F KT58`BP52G|y)O&^ literal 130351 zcmeHw3zQsJd1m*_NFKc(ejr)4DJx+0$m5!RKNOFnr;TWhBuirlM~HGyPxW-Sdb(R( z)zXY?2O%B_Y+(`H!Gt70$O#Uc5Ns!5A&{Z5(3GRY+%pkZgz7P z@AuzZx2w9UdS>*p#`bL;&8@ocd+S!+@BYv8Lq9zf3Ww--vJ^x>V;XoQx03J zCtHPLr9P9_=bhYZrz=LYR+!gVE^AoR$_1mWX()F}pEo)q)?iCFJBO_SbE*@GSfgX* z#;iVex^S*g8!I-Z^)b;q*M(j;8ZEUs-sPt7CQ|<<0r$sArXW#iX9SxlpTVCTqkvaaJ#!YUw2v|09N8 zD~+5~#+Ad3x=tmnyBFxLZkS^x{%AA;>TUiF)+!d!thF)8BaOC+x+YmekrOd>G(MU* zIilR%jY_JcsnPVw5!Te3#@a?}tX7#CYtEbHMm?d%;$v-enW`Ks77U%u$i#rPK`UYe zlZc+sk#o9ZZK;%$S#}*6Q7VQ~ZC;bM0nEH{Ku`~duJQd?V5gH$J!v)hf3|T zJZ7goWepf+$GTx!FIHv?HC*ktubx7YOf;H_WfF;KE*VoZ+0J`Ac*zI!dZ)bG+UQD5 zs;P96rM9QagZ$%AURl1OJX~IF->oaJFK-wrZ#-Py6v11Dx3zrZmAs@F_0r$X^49zt z%C}zPTZW}QF;Ko^2`>zu{nB5SS8DBE`RXr6gO_zVF|jpVPF~H0TF&)bpXG7)T8l-~ zNi~ygw` zYJ#Wq|HfH22>c-z*LB^vm*b}^mWrvVRLogi-5oR*OC{7)X3-t=XXL7qtID$>#?lEj z6VC;%s&q6C}9O%GS}v-S}~c>qVJa)(v#eD)IfA_mu<$feYT zy%W1@FaISf-cWDPPU$VhEbB_U(KL9q&Vo-yQ)()f3Fvh^noOy=WWbk}>b*@bv}*H; z*(%hHQlm8sbWGY2LoC^Xp2LdZAW1rz>=*@pL_& z#)N0HK?o$COD5H%4U>Jfu5IZ!{1$a(s$KHVM?9Cpqp~4%Z&wpB%yTTkAF!{|RqO%o z^LH~5PiED4oI`d`jq!9;O(%l~@8EUoCLH{Wx?v~@WpBGw0x>|T$AFru+dwaPZ^wr6 z);$Xxdu?LE7?!{oRslQgJTqji(wuX8Rac1|K!@Kh$slq{b(0DdoLDPHrEZvodQp#1 zb`7V@QN=J@BZ^)#bfvWYu~PjB1xrGCthewJ+eejCD$y7Csc7@Wny|ImD3*0P7imN# z(9RTy&~a?4GNYlHQ=`h1UIgXA&}IuwI)RDRrkM?P&W${Dq^VNQ$j%dUDh9_Io>MAy zg`LkvNA9i}=E)J!;z#MMlgB6~kL}raVB)~!9_3yorl7ML-R$2pxksJRP0x8;u}{*a zo-;I^rbWHCg{>P}I$Nq8>tUP@H8HNpPEJZtMT9fZ$WDyHE(PMpbVF-5rwgX8VR}tt z+;h*GvhEBz1{WK3Q=c15(lxwDjUpyv+CD>HyRH_)))1z>(6RD% zKRLODcL=>JVEO5@jn@1be4}yChQabooAXUwogykvX_j6CqSF}VLQ@wD<66z{g|M~K zY}6`5GT7$sHjM;*`Z>F=kayd6^iDt2m-KCHUj_?OZA!N&AgLDlrS4O_%>SLv-hw@vKbHx;(@b}%f`SO zv1t~*$zFR(Z`Jjh0^WSZdb=t#MXwu|5()uc6ffj`>M`fg`4;$%uFUA|((c%u1uq z2{T}hgMb7fmHRsRL2ErIK23+lQ>RwZgf1 ze2-Y>4m;hFxC6vuTJtAjCnF##j4DNQjzM`|>W_wqOv82{5J!-27m;?*Vb)SmiH$3) z)y-K@Y3xFi(|Sp9A~=yrncD?A)QED=eG01q5J+5+X><1=H2|a{HjQot|+`#~SvmD~FWUvFugIsDh}oXNVDpo`^>$UIcOE z6ZiBv=PMY5N~|KRaH2XJx);OSgl+7oHT#M}b)#H>` z?&?>@m6Sy0*0SO>N+Pr5Fp5GAwVO9oP6G9(jZlbF4ocA=d0oY${FM2HDdINV8&a6ZX*rsE4D zoZdPJ-E<)n4t_jUvs;J=Fy$gt$TMckxD4?a$@1*#5G^jVR1C}ANbW|LzZ*rCAImct z%SOS>viX{9KU{AsMeG9&*{aEH3(_-PX5i-AecH#&3+jA+65*oJYo$CRt{3EAsls#4ixwE z1No4S8g_0wkx0xI8_juz|6=h`c7Ya@d+$+>J~H{1BZrl3C!(`Z^@V(GvkjHwlbw91 zV6Cjw#a+12h7vwle83ffb{^akfRuI5O zK=l;Dk!p?IRqF_-yMy`grFOS^nw3RQs|LS9FRN`__VhBHh0;eq85k~_u@{ol%AVts z?xSZm8I6-PP%i3wpe!kkUA#MXmx?nT5I1wBl-b4^eHI)%#iQ?n@`-dBa?3V9-gRK6 zq+A@w(r7SbtIwp|B-p zdMkw}1D3UY%=MmSt&LV)Fk&GI z#buYg9frb997>zMb3N(>Ey2;%@zuN8C|63R5{ZtE0Cthwe~+#Yl}|y=CzbZeFHurD znpLr6g1fw~R4c$1gDu46nTS*l%12qhA|Tb8gRDq27K|6IjgZxVoCJh5f@e4lnH)%c z_#{nGahhOgI@V`~hxNH;p*{_z2^aSUr7}?vbCM|^F@}=j5ao;akub3~LBpEQXb>@m z6^C$DEF09-OdQh56Ojj`;4iaQRUxiE>0yyb-pGm!O=T3!BLt)xbB4od0#M|F=0%Rk<1P@J625NAr zTz3m`j4IFzGmBNP;#2JJ zn3{=qIX?kW>H9Y1(ipAK+4W3B z8t+Cb+ecuGYZn4DbJ?EWH!g?A2PI*jY+E)-mJcAx*l(lQNK1`0vRaLrQo^I`oduM} z4wMM`HXsQfP|>FExozySO~+a^6`pSdO6MTs84Y;nVwrRf%RAj?4XcD1iwNYLqp{a z*5Cmx(Q%nTBKZ^>g3lmyD00siR{hkSS$8VLQ#tf zzV&7N@*!*JOrtX0Ih+q4HepGndSWOPid5f$zxgy(Q7z%p9eZ73-GoUQ~OkeQ$>kXEmqi z9ITfbm|+_*1L~6rpN_S_Z%|lO!M^K zW@~1bVt-SPH}xXPAg2sA-QmHo_S*$HmC>?A}Z>8qxhXhz^P>t~9X)pBfn$3np!~%(xJX!WJMMTf5&#{gs9mgi-n|GS zPef0uykQ7pl6>RHC=*$HMMX#07@x9MunoUs9bgivT$ZynLlT0dFnf$mDfJSI5puV% zb7gN?T&k2!B`ZW{P%cqhWi@}9>jVh?QU^-=71S#3)n=m^iH@kO)~>3k)J%g4f)pCK z61fv83MdThT?KmEm5Kr`J2njt5%DZ2Sjl7%xXxsfX_t=EBE+p9S(v!B0@S!cB5&QX z21G<;01twA|4Tv73*V1ERkBP!Mk^@(2WlyT+Uzz#IIT)n>b$Dw3D7wV9ydOUqu^K(!S#Ivagt>RRFENgl%oeyFfq)i*=PAtqsm`*ERyY z8Niz?1#^Jy=W^^j`VbDq=5(b54bI(GILb&e)(V``agN66#i6TPtHH3?b$tKCo?TIJ zMqKCmHDqgnbeu&?+6RXkrFcyL7je# zA!aIs|FE1))`=jPJPw5_wLUeEQyk6^dc80OMYM68G44Nj=m0w*EJ+nlj1JSO6=tW| z$o_5+mQvr(nzfDFz+AFDRG}YMv~A{$+A?Cy;tchs&}tRty}4ld=$t+RHLo91&grcN z^wNrg%8TOy>?VN@g=r>xQRM=}<{EWa?5ZJOp=GLc+F6oeXFfmdkf&H%G>wjN#X`MN zuYgXE4oquM=mrxDClgYoJ5hvk6PbT1(s>LTc({cUHH=2#6lfAL2;y(BNM=O9Ou&;T zt3~`u;}&uwr5%rvNu4KDdlrfGqdhkF#14$~?d*_NT7k3SHfs&*47E=7j$xYEv3K0B zn~{pG56BKk@2>t1PMOvRq1pxm*kBa2B86IU*Z6*ZFzq=eo>aR}ZB*HzaZ^-!YsZKJ zf<1_}Mb%!)APRt=W8f5Fr_?k9GZBG-Dtjs!(>+j^*m5L`GRSwn-fSSNw^BOp?Q*vOu`T0SY zf^za}3FK5Y>_HQo2C9!DT$Vtk%67@GSStmZdhlJgH{*c!E{sVg- zdO*2vT#36X`8waEVtR!zM??trEX^h(4)>B~Dru%yXhBr&T(7^KD*|sAA78}bg4ooO zxm*mKc$Wl}D}{t7{LKB=@TOM%9PPT7yrET}r($wDTgb$)+|KqgD`b)dC|Q8Oo7}@z zptCLMKZwxsBqqtQr3aS2ok4sKW*xl-AL)#S6bO#nVr-X9iJsr~DmkPT=@^+E0BLxP z9biBrbO?>TQx%ri znA1HRrpRJ>Q$^Kq4A)c(ZP+EVGm|a;@unzBcgzMDwIHmJF^m6*a87LV)h`mc?*#ND z;NuQ-VyM%dC_;^b9Q5@>PT7&1YP`?3`F4y6yMUC-JTyd5HxcoPKtIj{w|vWaPQHDu z$^}V+Y_GS%YwE5*S_P9LQqjFT$?$!K$_s-t1$!|k1u*z}cDfg^h z?%7>DqX_L{&Yaoh29U6n_C&j4UXFd&oM!X6#D3b-8+s7s#8insEhVJ3=zo?v91uQk z$BR%rvm20i25AaHNHRQd^r2Ab-MCp%e7sfzTJ=_I6Gqy*%w~qjkz|9yJX95_3THi%|af zZME5|oMC2*{r&nC)#yHn_0RDe0~Uj8_QQEbeMgE7MvoJ~f?5PnA;XL?IoWj(4+P>ctkjz;65d)35yQ(C0$$L8AEH zgu<&|puFcpD_@|Z7eb*oz4Ha@OQG{ezl|?gn1=%c%?;L~y=D&~8!C&esjd z5JZu-OvT<26u4`_zZo#9BW2TU8oS2Ea8qtif%iT;cC0Y1H|lI_1+fIKxyEX>Gqd+V zX0r;xIt4Bw@5!h!HKxQ<$z)QA!@Wzq0H>tssGdmZiA*tBN)(If!gMKDNJITUktk(L zdTb|b6KFxqPt}?dmR&&-M5kSqBcteij zsREo2%phIY$TP@{R)LX-cA>p}@*n9FulI?6qEGziDSlu$Byc^;hCdhaKL`&zh4>&m z@G*)Hzyl!_HsbV<3e*Fb6jEpEZ56_g<&gcC;>}h_Z8gZwNk}Enk}Nl2Bt+{q$aypY zd-1qK>Xc!GR1TX}f=z&SmD%(2x>02eri|>~RAGI^mXNcMo z>sfr)HX!bQz5H@Rlo#JG5sL4dNEhYpYC~IbL3N9C@x2qFx4h`T2;YdZ^iGQM;`=H> zQ65>?kH6pIE)?2kHqSzIGb;XjoX z-xcxRiu1WR7u((H&cA@S_n|D!sVM)feV!NR^rNC6^3cya{t3kOlo#iEdxP@k_ud~1 zsFT_e?bEgU=Pf^jM2h3zODCr&Pd@70KXEHUnji7(-1%OzH{h(Dk=7{G^is%ct2rB=)T^^vY=jSa?wx1xa-ub}F(0L3*u?Ee;40Pz- ztSdLkikX=Z^sh{U4DQ%QdYP7oD+T5pvck}YwWX`%VY6qLR6I_Y9czQ2mrd*oq?K7i zM8-gutQAc4m&{aEe*$aPT6J_{-|+%DS8;2^uRs2vw)dgqk7)ZI*>~u`acGK6L4@yA z#~L{PAWSXH6^orb^ybM8G)xp&0~maM2tk4F2G7nkSqKscH916Q>4v%{Z97|?2LfmV-SZmR=}wHo)ex(Q=FwEwtv zaR1x1J^LmPK78P^#yhxJ%L!l0H`y)k-E%mv?K^V#{zC`%OxLyI+@^fkWWfn$3o zvVw6`1-Lmk9zZ;KoG!U}A0 zH*awwG{l>PVd@#GZ^0IaDaXXYOgulw6IK{TS* z@h@nbc^yxO9J-u=*Ypf1f6NaV#w|Sjv&VS&Pp5e}Gt0x5pXA|ZKhDF)7~^X3sS)23 z5iVY$H-{)E!roMQCucy;z;c*@e(xv$T5z^=P(Q`7QXCn@NnXT@dPOMmiBQz(z4yj1 zHGQ%Edh7M}OOzM=@7J!k9&zstMg87*aWCo@p|^dJZ?RC67x~1!H}s~9`+lJ)zgW5$ zr}z1ba-tp)ig<76O<(L@v@7y?=TF3Y+Y$F7U4){%2t__`DDr#b#l1)up@_jk;eCL)s=Oz!~+?pjZf$~yCdM8QiLBV0?sjiag z=82D9BO4UbxW_e`Sg9i2UAH z+Gw-Qjz2g#8OS%QTEt5VaXvt(`eoehjo^d2={)fwL!p<(LXQ$Z@iTv8e(C_Qq~Pd^ z@TIZ${(!jil;>^#=+7g{f8+dPeaiFtFMPIN`g?zHlU@H~|H+@{*#Vx;`y+bggu4er zkM>KWYF`?=VD0^i#ag&v?fc5N-tp*@G?p)f9%ucf)H{{`cwAvQ{Ntx|B%PH`LoXij=W(yUm#2c#LyuCM ze>&?=qjspgSYM;>_UF0s_2twbzUZ(2{2Kmz`FO>solwlrrjPwmpZCY9|4H>5zC25< z|IfbsU0+!~KYaem-wjUhz>yP7!NH*aPgY-~Ps!y&aAn9*XH5-Mzlz$HGx#Hu$IOeNWB=!pAuM0U|rXxk|z7ELg}nsy8peG@F3uEdi{Dfp>dc5%yr{a1VxNpiAuGuQv?+8_NVkD-uIX42?6u zvc=HVwfLk+{7fb^g}HDdUQ7H88D>V^mbN<;qWAAw-1oP{&k{dN{A}&G4){3+_r(Z00V`mSAM?(&*RmSVqloDmBC3>$$5KP`2Z#m?})?R~LrggFWPH zCMocGh2Q^XnZR#H1VqAbDFWggE5vGWi?Cc55s<3`pQZQ*q-P|2mhc%oW9vR!w!T;} z%*b8|pCx>Tc0`TbP)J`L*9tz9*F_Sd3>xX3e4 z@u#DHp9u7mALs2XMFcv!MuyZ}aDI zlaJpx8kEl0$0xJ?G(Nx1`uoO%{yd_8Pydwii;JlMeEs(AzC5D*&weTHOXKbH{(bl_ zgVQ@#2Vd?Ul;Xt_UuqIxN_?5Gl$6P1d-fe5imY<45+kR%k~}N%W$TiBz+4)6Hjz#T z`GNy^Hj&G!i6x2`ujs*-%vgljU*GygKX&x)?>E2T&m+n|dBnjaR8Alzk*+-+oc`!K zLV%<~w>0%b(xk z&*R3I|9T`So#VH=)BZGGzk@FyJ>t(J`nUb!;`mbA)*oNq^}E6NQPh9?cY@O;zVy3e zl;XE>mr|Y-zm?**T+!2NUCQ|UEy-YD%tNV^awoQhTI%nh;6 z{a_}OOoGXuWJ+ot`eibmQ)ApQ;iB}*l|A?pD%jVbe(A@T4mK6^OOENh$d1!DF@*Dk zOld6i^XBrJXz->hj7mzHe8NaWZ!MRCFD3ml2>&_wP|`1-fExHZYmMJRnWSGz{`ho5 z(l5s({Zi5|$ymni6Xyn`59?RVJVGi86v?^Z9(eK0A_Ww+?e((6lRP~;>%am{8lQH zQKLb2Z8Pa;Ld_+XW`1k92VXM7p#opt^`2kvBVXsxpT7M8e;!f(#CN~gr##0eUyJ@$ zVEWPZuO0czw>;n<<9`mh4%+h zi06Cvr<^$VDA2#I`K)(;-k->?G0eAU({?1=s^zU~c+T2He(pkU% zO!>Rm+Mdl*`> z8uR1ZhNFm`lX~5NVK5P)kw*@I-CX*RT$UZ}yV1Wf6A$fM()d(P1xK?C^*t2?Qk zTWvL)QoCM^NCyhcc?7wU5OxcS;cAlRDc|Gj0~U+nDI_wY!sWcnugHkXw3Q^#lt9yQ zL9+ZDC}h&fj2h=~(nH8hXOe1aiG<8mJwS66?P6jhyq^iUvABBDc@=(7-%jShTbv{RSfgM>2{$Ofe^N8Hfp$!;Ba#yrA9{;(8xw5 zzFda*(y_LbiKWy)I%Wn2Qf&?MMddJ8_rS~5%M)HY^sME;C%+UqObIU~yetn2FCLfV zDOSp1N;ymq+!QcL!b=G+r5?GoQv^5K*UrpmCY$WTC(p)UW`Bu#K5KgL<(lPyFTp2& z&B$R&d@1pz#Fx!^sYfp5Fr^&knTmcEq*loum*h+EDt7K%hU81f*=aTv6)4if9?xdd zYK~Jj7o}gW?ZKC8-T2b@#K!{VB;9gwXFl!EBiQQ?-uw%F%5yo)R3SKB@yTIQuedB? zFa1))Iw7Sab61a?m+F;c^y-m2rFlND9ZYW>LeIzPRc{S$J4OHdo>@Lswr8bL4%z zou$ZOqJMY2NS%kPqaXc%zkf(SkHj83nC+*XeB$|T_%BWz)h1@yEr)sdd;UD`_4D%g zgVOo>xba{7Y1EFp{Olc{@~tt^zawY=k1x$z{#W1lFTOmY{J;9uH4DmN{_@`jr(Zic zOzT4=QKyuPry+&dC{-YH$ECRRGo#Asw%*n?GJG;FBGvL)t=MSJYjzs=uewnjpKKM1 zmHJFxpO2KbKi17Wt)HnB^(T~j?o%cY9)4u|D0y)lM+sA{PtC&(2AErVy)aeNr^k;T zJ97WQLkC8sp3h7KTv_ZE5Ob7zK3CmbqjVE0-GoXvq1VMtXbx_Q5`p5Exnxq!a0~2< z;>&eC_;MZJfBF7?z3}D60Q#jsFUtX63jR3FmB3W)P~uBR&qw0R_LQVwO8TYLBj2M+ z`lX~_O7Tm$tqf3*TZZaAxokqs1mVkU6c&s!OH}V!--9pLF9&=X^5IK|f++B1FtQ`d zl|l5&-rRorr9U0YfL$w?ZtO_&6J0OVQ+mr$8s(MvQsT={=!)H}OZiJFe<|fJ!&PYq z(~@>Dr5#L|&S<@oc8ao5@W<0ZcGk1eOjeC9(NE-t9(=ihFp}U8E+6{k^1_#re#zx$0dJU^2c96{&-h?97;S{HOS0*HU{>1HoG)4m>YZGCBtz7 zUVinTW__Y*9DWM&T{!g3|Le~q$`_8Ceadrq@*}?yoc>9l_~r6~mqPr~=@rcizoXDi zz6^iWKgMW}8}i<=bay#sFxPyIx3d)S%V$>$@k@8FspT&D^*k;x8&wnWrO_`p_u$LT%Nbu1{mK!)6x6KV&?|n~n@XeWx+?dG zS30yZull&R4v8;0{aMm4CH+#e$0d7Qvd5)$b!lDwPHQbJY?u))BgqZb?#NG;-(=n4 z_UT0CDqj`qJ=sJYMlQJ$kB5GlfM1{367l7YJ^1p*<$y22ANQEQbgHLP9-1zdTbMt-V&m8$R<6ZtdLOyN#U%sPKn11{?j)wf^mXCx6DE?N3qJd@pV2anX6!}V&@0z!~f5zWF($AyP9{I|@ zX*>DE^ZT`*b>gTO0_CH8*Uh;0oIj6SK63PhpmhHHGcWqns2z8C>u>#eMF0Nnjz0-p zKko8>@~v0O-E zWTp2<=C)gUe>^a`XaAnb%d_irAzaOoL-cKK;@Nb;EWo_GaMei62duW|3BqngK4n}f z^-$Lw59^XX`I@6oCR1rO6R5wF%th7g63wUI+=Cx)cJH4rf3TnYoP!_#Ysa5Q?9Wer z|FH%4=g5PP2d2Y(dLVt0@QXvA6lkV5q;zCEf5F;EVHfkbb6u(Vp-bU^f4Jcv=|CqrQ*lu&-vRA!jErzzmrcqzgs`&#JT6vjUT`IqCbxtzkT$Ng3>uY`^7)@r%^lZ z^8Xxq!na05|91ZH*L`W;^85e(8@@cEe6;*MUm7pZ`*+(HgVQB`l=?YRKS%24aA)Sg zV+BK-)n^;6c_}|B|%4O$yMdT+X{jop&P|_bI{qbJN7hQ0ZNZ?m!CbILyoT_1aR!WVQGN)APN~>^| z*vKbGm6=*&s!-F6$~k@9Y&YS7$Q1(zLA6G)H`$38Q7R=@x}w(%T{+yS>)eab7lvRj z^O^FUR+^Dwm|G&rzveV5$-gFIUcP6gO8j`>bh}WCK=-ao(v3Y+&*z`0H-&F9o#gW0&9~{XqnK>a$`C|x-I;YtG3)e@%+FN zbz64i&!O5Y(T=(D8kV2?oNPLsP-C1+?ZuDToSI!Ce%#W7AGf&i=K6@GRk6-(1UeBfS zuOIt+fBQlF>nDETD-LY@?d47)kjfWoFLUx^ulS=M0!jGjz&8mWpTNOd zk{{jU)}k;W zOSk`Scj=8Gi615T@hX@fm+WiFzLxB3$-b8CYke0}1efgVSJXRqHl0!v31(#6Lx0TX zU^sG#?i085;Kyx*lmz?kdeI-jzV?VeI@oVH;KyF^N2hNBhf4aRq(AP8cFM!$)g$lf zNcv-;)hbB(W1(h@DjvKz?zxxr$6~=SBYP$NQPLl!_+!1Uw^TA$&M1vno&K0h#DtKe z2S4Ug88y8`^N+W2{P-5g>%sivZEpPdS0B6BNBqsf2QU7DKaaqlX80q0%5(E5H~d&| zdT8B+C-;8ELRZKL&$r(3=#v7OEf4kUOTmvvn*n1S;N~AMGV_nl*l6wvjViV|NS@+P z2R`+ghg?0!+gXZu*_}kea_cgg_ zJgG*PXx``c9(;K_-+%f3e!cMJOKZLKNe2lD`Xu$!iKRP!PwBXH@wffC>^uy;5*M52 zaPnQ6dqh#*3ElKbi7)*a(Txu!zU)iCl=xEeza{-r^1mhjTk^kmaAC+-lK-8Hrju$S z$ozON3jTMLrz|Qzc}EYvyn}F>px-PXeCc2}fiJz7korxz{P@x-D-h%>DL*OkrNo!z zL7NS}(pn8GK)a6bpV+em{`_D6F_uJC__VsYTor~q*fVFjgb){eqRh%lO!}Gy(Q>ys9Z1%y=5M%7^bvV_Nud1Cd^Iba;bP$ zO$8ihk&?~`MhcFzNAW%idTq5d7z$M<5krDEp4H}xMJl;@rZlISfI_;dozYuHrBUy^^*x1fsO_199-6h7i}l^PNT|si|A@1x zsJ^%%6haL}Oq`fSYvs{LCf{=8Ff_uf73{m#sg{wR4^=A&X!cf2oAXV*Q*FV6k2PSN z>*NR9)ix3_?;A7GXeO3PB%-M#%o5usLp*I%{}}C-hpT^rfL9(Cl~@0i;;828w{R6y zb1a(3si~OQa(a5t)<93q152$rn@A;+v8;;f?ageFng=hX=5#it#*$qx!|oZkXTusQ zwRFrXn`|*3z0;aS|B|t|ioZ+i;)*4A5jCe$S@^N*av}q|SpCQD0~4#k)4>#^<)S$? z?fAp%sfONAJRPSR(LrA|i5L`yv5*!Tr&Eo16rvh3dzit9qKS9{ddEwvM6481317;r Qk|b;=+QzrrQ@Z;90a7nIG5`Po diff --git a/extra/sqtt/examples/gfx950/profile_plus_run_1.pkl b/extra/sqtt/examples/gfx950/profile_plus_run_1.pkl index 4ea36d21248e1839159bbd78461329fcbad814ab..e4580da4b499aae77c82e8ce3b617a5d2ae9d93b 100644 GIT binary patch delta 1874 zcma*mZ%7ky7zgmX-EB8hx4Bugwl(JSX%^SExh-6j;sVeZ;*m6+1zXhQ&D1@k`O6oxmozxQ`RU~&ZEh|k51Bfj zo0x{{Ttq!)mVz^n+(!H2$u=VB{FIi@nG&O0uS^mxZ9*<_7`es!A>!@7W!{93cD-4N zt71$^D(YD1Y|11)-J--z(kaJFL}@ZGe!B26w6ME z?#^o&=sy}soC%MlV#!G2sCn%I@fzQjI(m^&xZP1?-D`o8tnQyfO8DA+kC*#aw&IK? z24yvx64g*RDULZsas+HQ(-SI``AKpW|rM7t^fML)}6V5 zh_5znEHd;SP<<_!Y6P|5!&DQf?ZcRA1{Dor$^mNiG^U)Ogb1cwpfV$vY5}!z4pVMW zy=hE&KoxSBY6Z1@gALUG?d_!5JFO;7c7w|ctU zeMmD({2|*35VjH}aVK#I7!yd0moK&xAS7AX7@RLzU>EoTyMd5px!HwpgE1dBYwz#> zUcIjBs_L21WliMQw&r!cyI#Gn_j|wp@y-l>tgZAZ^f}%RS(}>0%ED~jn9&Mmxn|ZI zZEN$PdNp4xn}?0cthw)`S!uSPY>zzFeiYeP&zL8RIkRmI9zC+}FoKFPXSNG{*4nAM zkt3YFv6m%WsPM8ag_J}o5H=FH)R)2H49ST{aV}e)Mta#?S(MjFRXnZ}8_nb+_y)G*8W zk>l#PdaznCsibxLBHc9`&9Nr_=yV6v+w5DcH7uf5Zefr|s;wsKnqm!wjzzT5=xFTt zhK^MZJ}AHR$^KtI@Us$smigO(JE~;7wsu-uVS*vmwhFutJdPrY0fBC*mK&}XAPk)dP?%S$XP)L_-qp4?ius{Y zyDU$$-I})g8_l+L)r^@d&KYHV+iQP&vTbcjg~O>xCK1h~(-|!q&$Qp)#;?5Jth5We z+^Ly}mWsq!YHPYMz~2sq)rG4HLxr{W)rP`wVdF?))4{^#5PoL(xfZUzkY6cod-*@k z!nW*N3fG_G&kS>WvcGWiGJY|5_{;ybyit?)!l!>B96YZJvB_x^B$O;9Yg`rlXu)=kt1AiL)!3(jqDTCiBj& zbX1Fl6HCr6fk5Q!x~xn0ihOrvQd&BkW>d&JyD}Lq8H+42y9%F_vr5h?&x#lcXS8^f zznDC;Dng58I^~;H*5K6sgAcU}L)PX8i{>eNlcC;=%~PSxs^4fszgRdHO=~e;g{Ssd zJf=lc0qgCbpyG{{*4(sNSDOV>ZB=UxUadfss5TMP67j%#GkDX66MTv9f|FG5jiymA zFR0DBQEB9>^*P{U+5xKAw2Hnvjbt)$EfWd)!9@~4w2>4~@-?y{U#uAA;%QT*ZH=ev zkpzCLkxbAc5lN@gS~#8b4S0dNzP@f^`=~mu59+7l9ttI1pFuTV&8i~cUa3&sH z=^FL`C;Z)vM$$EqMGG~sphx8CtrC~#gtIr(_Q${?W5`mjTi>LFWNG>uFu-i$Q4Z5 z8R^6m=qC-LIu1`4XLU4lVpN?rbKo8t`kYatJ(!qndb7%Qx}l@@*EGr*+Hq`N!_BdV z=hb3GWjnOdk=x6S=J65H;#0KQ$>J`i4)31WH@Rf*jaJ2;u3i3sOLLpyL6b}7&wGgZA+n=zWE zj^S-K#y!`pDeIP?jc~47X`1s*@DPmBc)(g!qz!S~x?Xg=RmlYq0NDLqssTBI0nzdM z!enh*ciG(lA<_)-89b`agT7${78;=kr}4}q8eSu>3$kp?%oyG?h3KJv8A*nNTK{Wj#okEquy-j>~pt$B>ygVAW1_2vOozweQj zQ4WC?>r^?a9!qH9Q8h(>GFq5(VRDnc0=H3E-nCfZ3K@+eDPk|lzIAs?qcz=V7Mram zTO?`4@hra4XezG7Q+zS>tdsF@TuTR_9a?-{tJ+ok2ZowmbX4{x>slSuae0AW`ShB% zN#C}0`(jKoEk#5j6Q1f&$dl{)O1EH%w4#AkhFpuws7f`AZ@2rj{ZjvaZ|p1GNgU2B8nBrb3~)q+eM;* zdzpARtwmzNI}dAU4nm%yUYqm0^kcDjN=v3)Fa6H;G?t1-vBeAC8*Qe&ViB?u0O?b8 zNR_;+Ni3C2XsLv&;?CYNo=AnYWZ1uVW?Q*+0PCof18=rr9u^zscmn&bz#7w7x`KA& ztj4VX(~Z^~cFeTpz&-+7j#;yqU+|1YJe^8ui6p<{?gcPTd&pF*$DVeWt!_Q5&BQRo z@t|rG(L`8FMENcGmhAPk>FCwh1h&Jm!2LHW#T-fW=o>qjv@9P*7l|g`3GpystLAziiq{tsFh6^Bb`xGQI`66;L~r4>2x2IRo1=Q;S}YkL zt?0(WS2wG*D(JJO%4h;cezK&zkJ!q{j8>biWLSmWQ zY;#Nc4$zCKFC2>;4}nt{RddaGhU9sv-x(y9hOI#0Mv!kOF+1omYbmJ2rd8J3+8jt4 zyU^5(nOB_%&M~R;J3&K@sCV3@B783?qOgjoJFN=Ip}4j{ZEggd2dgPGZ<7Vx9X!5@ zNni6t+H4%g^UzLon5yWa_B@nI`KmgJ3etuuX8?w z!Bb)uVTKdw+!&v#w>X`{NAS~wrJDuprrvv~H8(4`(cXUU7Hy2GAUHvCmK!L!^lf!I zE_b6Bi>N))CwHTlgaLoNPwqx-alnP|B`>rFQYh;JeerON;oSf^ACjU(1qSa0U3}dZ z{&pnfE8b@=h{E62W2F=@f)!g7V)*OvAdJ#Zs^q-!L}{FG%0THZ1f{$2#|0ZoyQyko z!j}dMrM;AQnK+?zFJ->u_@MMoYUUzCg3=`G`=!PKr9*5yIpkVm)h`{PymD2)FrlR6 znCmM_&?q@(DPlZ-y{XQs?$8!7u&y>Y+C^evQBQ)C2nUlwj@E^<>GAMQ(Ktx3R3w0P zgLE$pLN3*Vk=c(7yDh}lH;^9O**`c02olYVtX46St`pLQ)s`%CNNz>k?xm1E2 zekTqctN>0g9wv?~9uMYDT^MJW3;I>)2Qdu_M>NUt%`<#mI8^vYO$RVx+s9LN zyM>T|QZAZ8?l9|(^U#fv^3IM4(b5V_xjwlT$+hVG*P_tMlXu3$DPs3ie9iN$EF_*x zB!i_9uJ`}7M{JrLs+L@iAXF_!23)F^oC$o^W&+=M#YsslG8IX_W=b^|VL2*a`%=qs zQH$S;peVEQv>c=HC`9?G08>P)2(;Hoc1#%hI#%?Ds>MpP+ls!h+8WqBxtFXl2F-ah z*R}?rtlrMP!y154huPT=!C>29y22yYkPzh{#Tvw3r_qMxezrfWn5bdL4aZ`!xm>lj zpz=>7I?BGF1@+E5)I$$V-FN>%^@d~NIoSF_y|&hZ&GGSeRxzyA#frGQ-)DRb46<{Tt=YPY!ym5*Q}2(H_!TELQGu}Dv~Rck}J zYRo`n$JJjv9gf7}VJ#f-&Olx(ltI$T9{M%wxf%ph+i`0_@RCVD7TcP;dlVvxD3q;i z-6P-FZX%xvMFV;yw(^d*x5+4UD14^7)Fy0I!kyvqRlKoLgdi5GP+WP*+hG)j(M{B~ z{boD`tC#&mgLmH`BMQZQQw@bjM}WFW>)&nZ!-{9{GtSuBL)}9WgGLB=jvkgi8}elX zz8GvJF3&i`Vo5SR@`^;LL2wXJEgtE?o@|1u2Gk^=tI;*Uox6pW94x_H^>$L?Q5bRc zdTvAJe9frLz+l1=-B4%Zar_*jc4xZpKAxoK)yqU{Gt8^`cm_~0tmb4U38f;h1r9rh3>$@Yh|!o2-WVSdFjE;Spb%*Sd?N8?(um-@~+>hEl=HB+4P&aHSF zbX3H)aqAufSehq#8c|tS@7SU8VK8PIYKXj-jClH+z~k1S6UB5FJ#|Oo(B$jcUwB`eo&XBv_)E`+*_Ti2JDBrevcGs=^aajZ-NfVljO zbSt%yB^_z~Q0>evZ{H2aGT}L$Ko8(iaxTWdA&%jo!bWRgpB{t16X9Qo8HUj+ZVf=Q z3LcC?@Ixa!BSFCn@%JP3$#!;_c|;lRB(k4P+gi1E3Km*a@SU&YlU1z2lhxu(`(U>3 zU=u!6O2-BjB~{&Dp-}fVX=?tjT$l9?2>PeZ7yfWY!kD z)k$@L%mDEU3B1`1%l;UvZGhMIet^%?Lx{14cFz|Z_P}KaDd!Kic^(RVO{=V~I!f9D zSEjG7>(g}2d){Xk{~)5gl`T@NKd4xzeCJE4tJ?Rs`R%ObOwRsFzKRjHp)x=|dGBdk zYYp72PKJEs`D)`#wbImE4O3@8q|yF>wVo8VNh1$SK4sZJAt^pGMQ*ssT7!Dvw1?E% z54NUxPk&HYb!&dS|UZyHmB_DMxB%jk?oh=%Yw7zXSOeOw(!l^NR03K7i`wAS5=(^{**!wnFzb@LLK(?@27fW7)nFP{Y@ z4U6Vr(A(u)k14@AS_2pAUr*ZPUzq2~mB&`YE(#20z-EovErK+#?KQjDx2`F6F`Z9I z&x$L`vtks423!TvZr)F7P!g=vBDR1VE<-mu+g;oL?-l@WauUn|wqL$6ap)*E#m$*w z9>$xytiCYgN32!YrDGqB-HWrCx7I>bvGd5@$=y4{kcqg2f#@D#_z8rP3Hxt-&C`Np(sn|NI7Jhf5vHl6#RGV8aEs9@{C)aEpMDuWb&B&UI0gq>2> z8yJZYJX6_2$;3UO?K`)RU>EARszP({G0pGrGHgRjU!$?O=zGx8H>g^<^K@GTO7pWz zeA!wp(9}B?!yW`YfI3dP$k)TxAQf#}pk*6PjaL&QfYj{@kJiOgIw25- zX1JyFhze9PXDpBmlX>t{4HdA{#wP$~HL$&~<_9QYSzHycC7!L%{~pfSuVKBW#zWK% z0GoQw)yZe!>^;eV2g?WAIYcz5V`4oDd-Y;-oc@r#{N8!zh6_d-^V!nmta<+##Ad@0MDFOuEYy$&|&>0vOXI`m>k zRcIN_+Pih2K{7KHyX=IS32&W>%N^kt&^j=@3#$J>D?h9TXt91JMvY`(6@ zX*-g$#uK);x5G@?1yn9OnjwU`iNq&@ew+tx{gyMFzP(cQf+Qi^eNf>gJ*PnW1C##1 zoHIA&34$cbBh6U;LT0oWTPrAzF=vIIXI**;S?Hx_lGwZQCEdMu#eqnWW@{;7OHevU z`?ITU>9wsr>J?uLju@C~icA=p_44IjU;cT7JE2M|W7`m&!}5a$I=Iy=a$vaVO6xLc5qVW_G!LBrK;r z(XJSm!++kKVdJ^Xe%iyUJb-dys6?NZ6H;6BFHIc|2;XbRi%>kXtB`jVX(~c!GTeLU zsG>aaJcRazhJj;>$Q5(rBY&uD`B}}Ft-%>6Nz|3kXe7>i@_o;1u=apMnugvW&Hc~L zmaEf7`B|-+&o@lUMiTm=pVi>&6UquFposrDp2e2)FHynJg;ua>$qK%#&DN`}n%*#H z$yid)7v~|-r*11>WVeqS?Ic3pKU*341z$X6-};;eZ#lCqK+~VOALRbT0d?Z@ohbbz zB>pf6`e?uMC7gIWuailheyUnOVKmfBYw{Rxh4p4vYQ$`dO{fc@o2) z=NAq53NG6Z=UI&#HGLKpX(wjBq>*b{oK41>gX?B?yJ)ZP=QL>eH}shWXetzQbly_# z#INFEb!s2@Jil<){sO$>o;34yG7zBq-t{?qt~4=HX7kIqn$1=2KtTU6iSD>#Ha`!) z-C|COk?6B0B<5Yvhkg%R1~5DA_raZZCi+=#9$4M^eebg4*HgT*RuR8v@q40B{JS3C z2kCXUM(L-x@BG;htxNy@XJ2@3`;N5u{fJ({kk@$^;CttTQC0ZJDmXH{SkrT@;*6e& zPS2#0(|LH;Fp}Y!Xeax*e%Wdl*k!+edX?X$yr8@fiQ;{u^sRk~@}5yvzeGhJSCqH^ zz)RLwl{1IFgEv?o5BvLT8?CSQt?KVr&eXo6^wC{E_EnUU|Ku3RIq)W)0|Ijpqe7o` zJW#~@NWiIr{l$hgO32Rl_piNPrLSAjdNiB1cQ6YOe??RD$u}ECkh| zLbF+G>>L}zMWHnf+56nsVPnRuR@l-CvIJ*xjg`wM=k9>cW(kUQD$a(y1J=M=M8(+~ z@wggICDTzOl8hz8W-MmLQn`3OmdhoLnS91bLd7%|%ct{ZWJEat^Cz>ulk_;%0|)ln z-B3egk9p$#N4$LbO?io<|L^7mr)frW*}Ku7{r9xn=X-p$5pkDrshYUrya2`jU1TAn_txhP6<)90K-r@qfql8dJ41;hd zb2*mGE*P|IH+D?|tat?#cZV7~YHW zq4`RC5P`eA`xmia5siuWinyk-^sW-+#rsQy58D~(HAHQR`7GXR8xi*pufJXu<;D9; zgyQ`s(nWc@T4fu)P~9S3yzfNlEid{n!ndL<{U$|u@qQJdD32`c!++o6N>OgZSBewm zU$Wn~BHZOqkw|g; zed**B<>_!b_eWfckj6(mJ9oZU>;*V&XQX*bw?y|!A2EOD@m-7^OR)chHt;V$;*P|3 z(XNPo9AARUf59#fP}ueHmM5=I&{l8%$ZF*bZbY>Pn#L?_=-oyXZpw;znXt7wBB?N( zt7vX#5Uv#Xd+GR|v9@-MJZ$j{(~8Fqvu$k@eA(ngAXjD$5{-c_S*w`oFFB|x{SfA? zwdT;|#1VszRdHLy4z^U093qgTUlY_KJZci$G)|$FG3q^Qp%-VwQXywqp!}m^-9I-}^3o_r%ox2lt)V{SGYE za?IEAHFnE;b|1{@6Zapy=fM7nsq<=laH+=QzQ#A+dw9>Je(%x!-8Vf*(SeV-FYqz1 zVl8KPmDp3h26KzmyXfq?YHDIazi+>|)d5^~x44NMQqNL-i?%pK zIVSfXIbx6QIX1b*9^T$f(quOAHUbCP-XmyV_q%I^IHEW4Cuo^@6Ay>HbvXi8^ax}I zICg&iEqt@lf1HQ^;iq``WAEkRqo3vByY2Y1XZiI~gCpK2B3vrJH(ityp*LM#%Mp+x zaQR1|*Y8ujB}C}$sn}MEEu+}Ui+EA52t_^-iu%0Q-uUIFi++kw)Z-0BecpbFa-#nt zpSQfYUMgMGFY?+v}_ zOI?e0MLzHNiFj{2;##DOP?Q&;$mb13es8?E7U?1s@gfxQ-cXERul7ZLZ$0AL8;bGq z#>;Ct0$0Kav@JN3Hgh^7B>UWiiXm5) zofAZwoTrUg__D?HarOPyUua&x`#$o%cud$_aN3C=d5aqiSCnd)(Udvc+0> z+?sg#55E8557E7RU;kd#UrN12{q-X%%i+I&N=MQeDoa_sZGDudE0oTmyVEzyPw5md z(!J$34U6#=b$5rajGY}Gpo-AvKf0dx+1=4N6Gyp3OX#2S4kixoQ??)S=W)-!n{q+v%EQ#=pU(Nys2wUV=GVj%{yY~x zznuEN_91`$XV&rO>wWIjH$J|+|K?+#>hb$=>i=(>f83X6x%E%J`Z-@&>VbGZe|63C zf$41=;RIK3aOnTTrLWPe;YCRhAyqgD@EdGa-k{Q zg%f!#@iTOo8M`fgcdEqq?_B!SZ;78JewO&zy6rOH=P1rZb{|~q;O7`lLW=NXR$&0K zIDYQu__@G2Xr{gJX2s7(R~&v8tZaAqX6K*}DKG!ZOFrhG*i+79zcTjqUm*^<^lm@@ zOL-)ReVI9IhQB{>q5Sp3lD}56QZb}egH5^3XLBcF(s#aeHjGml^hxP#SUMYqWivnG zY#B}l&yBy1B@dMdKcGG%@w3Fw5E@%|*~;}fqtOiQk@#8SXP8Hn=^P4qSjQ#8&+$0CbO+<- z6r2ur{C6+TUk`Hpd>@cAbg02%J35rFJ@JEqI+Skw@)JMe&m-`6|7%u{@?1x>|Bep^ zrq?z+{vod8EHs=Q94Yjhsh>_PUGZzLesfSRJ5Ns?X0ML3w+?}CgW9Klxbf{fC;a#L z+kM?S&ffaNiqLruZUOcCbfBC2c{|I|fex=bYwJLJTjtlO{|s)UO9%R?Dd#@K^G$!n ziKF`#y+!$s{>GuN`}4TxPwrbm>3n|t*`NB;s2z9tSO3kQNA&OYoBlj-e!0tk^Vz@f z-(x7_U94h z3;((}zAWtUV*{%H;SGcU$%bxa;!C0bEAW&zbmPkxclqz}QsB#P-^1IvRQU3>2b}v5 z&-dwHbmHi_iE(h_%P+p{&*R2-zxKO9=^Vek^N;;$)Q-FSyZ*(WNA%CUVQGAs{!DLt z`8Vxg{3z=G?k@(XOMK}+k5TI1MqO5UQvX)!-*QXO=0eTX$QQPjpKID)QW^!L#*Zq! zgr_%3#}LvnL=&fq6)Wrv6OPscID;`nlP7rulI_T}d<{rJ+s zE`q=0SVkZkC-n1|PFa!5>!U%$i%#;czArhJS=)GOTeW(yrsJ zk&sSksTlJ}wxoLLRbB9sc@7iw%w-ghd+19C+^qz7DaGTQUX?Q;DdS;VDbv%b_lO$sC6YeEGx^ zr+cWMIrP)teZZebl>fOuJkz5*$0uLe@mOH`q2Vh>{qn@!{(Jmw<>ISf9^K2^S&n#I zS;N&Y9YU7qSqH-hsbBuWLFYci^S$;*oj6bVj{0Tmd4C?aczfn0e|mVMa6#-rO|zg^jH69aQd4}JpQrA{*6XOQ6$jB z$ql3GNkan7i?1#y{ZvUmRnkw@8|bHMeTT8(SC>ZQ zl?o0yymU^O(h!oTxT!dkk~`w_G~A+&Vx<=QTIxprm3ONY!_2}1I7(ZQ7PQo_r^ zfH=bAoQ{c=I+#)iQ^L!>lGMSpqz?$fXjdRKh%2G*5*_G{(V7B}}P=d7@*-9?rf3 z^QHCV2tLGNBo|f0Cb#t!;%}gA+#xA5nS{nIdAe`rHFMxN=N21-t$8P(c)sI5=ft_k0_8h8m}kD>&*RpCy!~52>3n|N{T+WAwc{@T>3{X-5&e65 z;-$d(tFHZ5#^^|{-!UDx6k|c&Vk>*FwgV)K-eJ^dp~$jhhXW2TyoMD6FFOYjCBn&w77Nn8 zOoSmFk1P{kZs@|78{F&X<-(Vn0_aPD;8p^@6yk9jD+hJb6~C7FQsPTRx!@x(rFdM5 z$EA2&ipL9AOYyk0AvfP>@pvK_kv!k5E@kpzEpx$vdp!c3;x!hMtLQ^l=xEO%W}DzGn&=9rst|s zJT83{?bf7tT#Cm>)nf@QJgTPXkMvR0;ZRo~e;LUnwRn(=^+Yrp(^AVk!e?U_zT60S zblLHxpTAsL_)_whTs|!2<5E5@ePGUprlovb%E#m62tMnUrRta$d9x06OQWIBnRC_p zg2a~+Uj|FqS0KKOCgWN-5#XID5yjy?k!9k`OX*6o*FHMt&m+oT zebU52=aga3*OFh^exu%HQ7fVP76me%=3h#+ODt5$R8j|72jg)W4KRL`om) z7w4pGsedW;FQx0tH_*Jt(Z7tvV8z3Y#69*su|!(4{bDZZo@jFyzTE6yKQ9-){O}5w zj|=`%;!8ia%L_lE0Mv zrIe3L`FN%Tph_U94!^dqf_9_2Yc34b#<9q!dXw31JU?8cW~cC*`l{NKLozsEyE z-Tp5(DQ9oQ@Y^4zzq$3}N2r7g z9R13V?)!0H9#Q_T_0Rj#c>BD6e|ao8{UYgCssn}1p-@R*r6`k${?c!(hJnC`PZgU5 zwUB#6uhpyhV%ZFFFU;o2QT36Q*)sL!Ld_f(ky_ytv(DD-G#oEzHge-rbt6}-%x2An zP=5P+I+eT*&4{aY+VU9445>2f#y?~b^q*rFem@b<${f{%uPpqS3`ezipnuCGkmYjmkZ$)aIac{`^0Ctv`*=KiB;Fcm6!0 ze=kgZ!q>L9{7(*k(w9e+pZtyId};3a{}0#y$G~)nAEkYcw9k?DInw@H%D0#6VSNSi zkI5AL3kC9zsc>9Na?h4a+UH!?g&(hTub-ETfAr%=2crr3HpglLu{oiw8+k^YzeD%q zq|%lD{2gz)%Ct1y9xSPplj~gYqvRiZ^M{gul>FnJIN{>)F0U-a`0Y40uaSEeH4g)i zd9_$k>&7W^<8ypeoh?_Vjk4Y-o;Jsutr{p=R}2_ba+}$e>_m*H#k?zBHOmcCJy@-n zTulF^K{#Z87DsU;&89l8GTv75k92Ue)Q^<%Yso+Q4`2MAmS3mhaV;9?KPZ)oXz691 zuk)5J{P-3^N`jxcT=)_4Yma`UgWgsGew6ZShkumv>(=yn9{(-n*Yf-vd47&OKS!RQ zWA2pa=e+Uf=cFSsEyhuz$3IXymeJCjnOu^8+}ed7x4QA;y!LdU{LU?(ygTX7Bk<>| z^FQ6AJeOalemyun+(ZA77(js^9YpEyg4A~}|Ja?%OI_&sIor9r<FRpZ??I-|%|QRet@?f9G#MNPhjnfzNcFm3V%C8F%8Sm*P%Q zzN7#6+J1i?_xu=tC@7sj|ND;n)2JPH`G5HVe;(1lt5^TOPM^E`=g7YY{_|_TJfi$B zKDqI^eNZzM_a#D+{x{zWPM7j)DZiHTYbn2$@@px-{vMWJr_*795TFS3mwHc#n z>W!w+Y>Y4-X3OM_eWhFJyp;#*t%}~r70j6ysK;Hk<49rCq?BJv`E?MogSMf>k2sicRGrssOpG%MTh*Gz z3BM5yFuGMYb%VRWH)lcucRzKB4vUfY*V6v_7VFBwkI8T(u0^;P$Zq~I8IFgwDEAz> zB!0Z13qRhl67VDB*B<>x2LrAI{3zwuQht3A&aaX9QQ}95AEo}I)PI!vkGC;ny}H?G zmC1CE90eV2RVrqk`739M6vy{Tv23w$owbn{X;r3>(6F~q+pu?bNn2bPv9bKmX#~V3*yl;T{f4tF+AOFYi9rfvs@pMPO z?3@3~pGWAA{j>V+9_6`wTsa+_u57q+{6GHHM*{A#zuW(#U_eC}_UJ>3co7EaN51+A z-p+FLBmZa}_Ydiya(b$wcILnG3@*FQ-~QlFJNF@;?-Sp4;ymR$`jLP2U4I_8e&f}z z2c`4*@yi3pd(0na{(XJKpGWj>wE6Q(mcQlZFZ%L`^433n-Iqr9DbkJvZs}o{Nx25Jo$KEDMJH$gWLH5G${;lz2dMLTbtqw z#OiKCJ|LTX#jW39)Mss-3&~Gjar|Ti^6f;>`8CN1qk1cOyD)(zm9#4#Fs(%(ZPojUp@-|DS`5Di7zF8DfvsuUrPQ`@|Tx`zl=tr zS|Z4PI~k3owHRkMmz00s+=VZ1UYYpP!Ei$U?bUgtep4<=#}}9Wq*GSlKd=3^h!^3- z^dBklrNo!#=^seqONlQfzLfY<;!BAyFAKg*rL_q6LhjM8jKM#7e3||w-`a&Q-%2P= z$p0=Ee+l`xN59g+h${hKcI#IsF@M zYTE|F+Xg$J7%8DSoMFt;=wuD=wZ3q&Xr5|YDYs6FU2(|C5Te71 zje6Z!;HM+ohZm0sD@186D7A5}Fm0SnDjbDPV1TtCQW6<0(vg-N!tZh9u{KHlSC{h8 z>Ml(oh7`x|h@>;|Ofr^Ez-?}Zzn@C)LL%~7n{zF_X4Y}6&D@O6ep~JA8fz`P%vD>J zW_xm1zqO8CQ@&iQoy9fFiC;Kn$g0*gbH=>hC|8>ec-^TR&0K+HRIM$PaLPDoHae}e zvqRQ(PIElLtjrcGW;@$w4VPz73H-8ZkAbxomj64C?48`bGu#$qG29u8sFq3bmpB`W($_XB3Tnt< z;KV3es}DUib>IC5+l3)(6??C>O9iB7l~NG_jozvmI+nd%s^bJ7tG{u&ogHYET1dpW zZ^D9+iewVeOgx>@V$PFtw^8~Q+AR!~eh&e!yiZhK`ZmQ;&80uUx4@b+>4+9}JdSkL zOmm>C=Kf{Y91SND;Y1uGl<3$JEMD`#xzwDF;E)gd{5a2u+oNF(=IbU#l?}ESkFH^j zhLe$43a1sv9CL@oySQrET|~|4H2k>4gKyq`V1L(nU}83SJDAC2VoBU-a5eVRajFp= w^i+dMluV_xRF47kRU;mSzZz75WBWR%^92o;(;G2Y0;)+wwNyrsXr}i60TXH%1ONa4 diff --git a/test/amd/test_sqtt_examples.py b/test/amd/test_sqtt_examples.py index fd1820e11c038..f8326c2f93415 100644 --- a/test/amd/test_sqtt_examples.py +++ b/test/amd/test_sqtt_examples.py @@ -88,7 +88,6 @@ def worker(): if t.is_alive(): raise RuntimeError("rocprof decoder timeout") return occupancy_records, wave_insts -@unittest.skip("TODO: fix to not require unpickling UOps.") class SQTTExamplesTestBase(unittest.TestCase): target: str examples: dict From 9607787ce124fddadd16b36ab26d3717084e325b Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 25 Aug 2026 08:32:45 -0400 Subject: [PATCH 197/306] delete more explicitly set dtype to UOp [PR] (#17726) * delete more explicitly set dtype to UOp [PR] * not that --- examples/mlperf/optim.py | 2 +- extra/gemm/moe_routing.py | 2 +- extra/hcq2/ops_amd2.py | 6 +- .../quantize_fp8_delayed/__init__.py | 2 +- .../llama_kernels/quantize_mxfp4/__init__.py | 2 +- test/backend/test_isel.py | 2 +- test/null/test_uop_vmin_vmax.py | 2 +- test/null/test_uops.py | 14 ++-- test/null/test_validate_oob.py | 70 +++++++++---------- tinygrad/runtime/support/hcq2.py | 2 +- 10 files changed, 52 insertions(+), 52 deletions(-) diff --git a/examples/mlperf/optim.py b/examples/mlperf/optim.py index f93ccef1193b0..5e013193018bd 100644 --- a/examples/mlperf/optim.py +++ b/examples/mlperf/optim.py @@ -15,7 +15,7 @@ def stochastic_round_bf16(x:Tensor) -> Tensor: bits = x.bitcast(dtypes.uint32) if isinstance(x.device, tuple): shape = x.uop.shard_shape if x.uop.axis is not None else x.shape - noise = Tensor(UOp(Ops.MSTACK, dtypes.default_float, tuple(Tensor.rand(*shape, device=d).uop for d in x.device))) + noise = Tensor(UOp(Ops.MSTACK, src=tuple(Tensor.rand(*shape, device=d).uop for d in x.device))) else: noise = x.rand_like() noise = (noise * 0xFFFF).cast(dtypes.uint32) diff --git a/extra/gemm/moe_routing.py b/extra/gemm/moe_routing.py index 30ba001c3f98d..eb4982c6e630f 100644 --- a/extra/gemm/moe_routing.py +++ b/extra/gemm/moe_routing.py @@ -53,7 +53,7 @@ def _bwd_kernel(gtab:UOp, gout:UOp, idx:UOp) -> UOp: g, m, j, jo, ji = _kv_ranges(Gk, M, Dk, _blk_for(Dk)) row = idx.index(g, m).cast(dtypes.weakint) val = gout.index(g, m, j).load().cast(dtypes.float32) - atomic = UOp(Ops.CUSTOM, dtypes.void, (gtab.index(g, row, j), val), arg=atomic_str) + atomic = UOp(Ops.CUSTOM, src=(gtab.index(g, row, j), val), arg=atomic_str) return atomic.end(g, m, jo, ji).sink(arg=KernelInfo(name=f"ggather_bwd_{M}_{Dk}", opts_to_apply=())) grad_table = Tensor.custom_kernel(gt, go, Tensor(idx_u, device=dev), fxn=_bwd_kernel)[0] return (None, grad_table.cast(table_u.dtype).uop, None) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index c4f7b6bc3b3d0..bfd238210ab65 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -87,7 +87,7 @@ def release_mem(ctx, address=0x0, value=0, data_sel=0, int_sel=2, ctxid=0, cache def memory_barrier(ctx): pf = '' if ctx.nbio.version[0] == 2 else '0' if ctx.nbio.version[:2] != (7, 11) else '1' - return UOp(Ops.LINEAR, dtypes.void, ( + return UOp(Ops.LINEAR, src=( wait_reg_mem(ctx, reg=getattr(ctx.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_REQ').addr[0], reg_done=getattr(ctx.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE').addr[0], value=0xffffffff), acquire_mem(ctx))) @@ -135,7 +135,7 @@ def pm4_program(ctx, call, prg): wreg(ctx, ctx.gc.regCOMPUTE_START_X, 0, 0, 0, *(info.local_size or (1, 1, 1)), 0, 0), pkt3(ctx, PM4Ops.DISPATCH_DIRECT, *info.global_size, dispatch_init), pkt3(ctx, PM4Ops.EVENT_WRITE, ctx.pm4.EVENT_TYPE(ctx.soc.CS_PARTIAL_FLUSH) | ctx.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))] - return UOp(Ops.LINEAR, dtypes.void, tuple(ins)) + return UOp(Ops.LINEAR, src=tuple(ins)) pm_pm4_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), pm4_program), @@ -207,7 +207,7 @@ def sdma_timestamp(ctx, ins, dst): pm_sdma_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), sdma_copy), - (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP, dtypes.void, ())), + (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP)), (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val")), name="ins"), sdma_wait), (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),), name="ins"), sdma_timestamp), (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val")), name="ins"), sdma_store), diff --git a/extra/llama_kernels/quantize_fp8_delayed/__init__.py b/extra/llama_kernels/quantize_fp8_delayed/__init__.py index 6e1c25a1fcf75..db9bb5fd689af 100644 --- a/extra/llama_kernels/quantize_fp8_delayed/__init__.py +++ b/extra/llama_kernels/quantize_fp8_delayed/__init__.py @@ -50,7 +50,7 @@ def _custom_quantize_fp8_with_amax(fp8_out:UOp, amax_out:UOp, x:UOp, amax_state: else: raise NotImplementedError(f"no atomic max for device {device}") amax_idx = amax_out.reshape((1,)).index(UOp.const(0)) max_val = lds[0].load() - atomic = UOp(Ops.CUSTOM, dtypes.void, (amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=atomic_arg) + atomic = UOp(Ops.CUSTOM, src=(amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=atomic_arg) return atomic.end(tid, wg).sink(arg=KernelInfo(f"quantize_fp8_with_amax_{n_elems}", opts_to_apply=())) @functools.cache diff --git a/extra/llama_kernels/quantize_mxfp4/__init__.py b/extra/llama_kernels/quantize_mxfp4/__init__.py index 6cdd983c6f2eb..e68fbb146bfbd 100644 --- a/extra/llama_kernels/quantize_mxfp4/__init__.py +++ b/extra/llama_kernels/quantize_mxfp4/__init__.py @@ -12,7 +12,7 @@ def _custom_quantize_mxfp4(row_fp4:UOp, row_scale:UOp, col_fp4:UOp, col_scale:UO mem = M*N*2 + M*N + M*N//16 # read bf16, write row+col fp4 + e8m0 outputs = (row_fp4, row_scale, col_fp4, col_scale) sink = UOp.sink(*(o.base for o in outputs), x.base, - *(UOp(Ops.CUSTOM, dtypes.void, (o.base.index(0),), arg="") for o in outputs), + *(UOp(Ops.CUSTOM, src=(o.base.index(0),), arg="") for o in outputs), UOp.special(256, "lidx0"), UOp.special(M//128, "gidx0"), UOp.special(N//64, "gidx1"), arg=KernelInfo(name, estimates=Estimates(ops=12*M*N, mem=mem))) src = (pathlib.Path(__file__).parent/"quantize_mxfp4.cpp").read_text() diff --git a/test/backend/test_isel.py b/test/backend/test_isel.py index a3b986c619254..f219195a52ca8 100644 --- a/test/backend/test_isel.py +++ b/test/backend/test_isel.py @@ -7,7 +7,7 @@ from tinygrad.renderer.isa import IselContext # INDEX on a register value with a constant index extracts a single element (the old GEP) -def lane(y:UOp, i:int) -> UOp: return y.index(UOp.cconst(i, dtypes.int), dtype=y.dtype) +def lane(y:UOp, i:int) -> UOp: return y.index(UOp.cconst(i, dtypes.int)) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only x86") class TestIselX86(unittest.TestCase): diff --git a/test/null/test_uop_vmin_vmax.py b/test/null/test_uop_vmin_vmax.py index 2dc3f27c35ad7..6398c28bb2f51 100644 --- a/test/null/test_uop_vmin_vmax.py +++ b/test/null/test_uop_vmin_vmax.py @@ -82,7 +82,7 @@ def test_vmin_vmax_variable_inside_special(self): def test_vmin_vmax_multiplication_0_inf(self): # vmin and vmax for multiplication with a variable x = UOp.const(0.0) - y = UOp.load(UOp.param(0, dtypes.float, (1,)), UOp.const(0), dtype=dtypes.float) + y = UOp.load(UOp.param(0, dtypes.float, (1,)), UOp.const(0)) uop = x * y # TODO: these should be 0, but definitely should not be nan self.assertEqual(uop.vmin, -math.inf) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index aaded43fdf08c..f4230bf9e833f 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -56,7 +56,7 @@ def test_invalid_dtype_and_consumers(self): if u.is_invalid)), (dtypes.float32, dtypes.float32, dtypes.bool)) invalid, value = UOp.invalid(), UOp.const(1, dtypes.float32) for u in (UOp.param(0, dtypes.bool, ()).where(value, invalid), value+invalid, UOp.stack(value, invalid)): self.assertIs(u.src[-1], invalid) - for u in (UOp(Ops.STACK, dtypes.float32, src=(value, invalid)), UOp(Ops.ADD, dtypes.float32, src=(value, invalid)), + for u in (UOp(Ops.STACK, src=(value, invalid)), UOp(Ops.ADD, src=(value, invalid)), UOp.const(True).where(value, invalid), UOp(Ops.CMPLT, src=(invalid, value)), UOp(Ops.CMPLT, src=(value, invalid)), UOp.param(0, dtypes.float32, (4,)).index(invalid)): type_verify(u, spec_shared) gate, value = UOp.param(0, dtypes.bool, ()), UOp.param(1, dtypes.float, ()) @@ -64,7 +64,7 @@ def test_invalid_dtype_and_consumers(self): type_verify(out.sink(), spec_program) def test_remove_invalid_stack_lanes(self): - stack = UOp(Ops.STACK, dtypes.half, (UOp.const(1, dtypes.half), UOp.invalid())) + stack = UOp(Ops.STACK, src=(UOp.const(1, dtypes.half), UOp.invalid())) out = graph_rewrite(stack, pm_remove_invalid) self.assertEqual(out.src, (UOp.const(1, dtypes.half), UOp.const(0, dtypes.half))) type_verify(out.sink(), spec_program) @@ -284,7 +284,7 @@ def test_division_power_of_two(self): g = UOp.param(0, dt, (3,)) c = UOp.const(2) l = g.index(c) - a = UOp(Ops.CDIV, dt, (l, c)) + a = UOp(Ops.CDIV, src=(l, c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) Device[Device.DEFAULT].renderer.render(uops) ops = [x.op for x in uops] @@ -296,7 +296,7 @@ def test_floormod_power_of_two(self): for dt in (dtypes.int32, dtypes.uint32): g = UOp.param(0, dt, (9,)) c = UOp.const(8) - a = UOp(Ops.FLOORMOD, dt, (g.index(c), c)) + a = UOp(Ops.FLOORMOD, src=(g.index(c), c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) ops = [x.op for x in uops] self.assertIn(Ops.AND, ops, f"For dtype={dt} FLOORMOD by pow2 did not simplify to AND") @@ -308,7 +308,7 @@ def test_floordiv_power_of_two(self): for dt in (dtypes.int32, dtypes.uint32, dtypes.int64, dtypes.uint64): g = UOp.param(0, dt, (3,)) c = UOp.const(2) - a = UOp(Ops.FLOORDIV, dt, (g.index(c), c)) + a = UOp(Ops.FLOORDIV, src=(g.index(c), c)) uops = to_uops_list([a], ren=Device[Device.DEFAULT].renderer) ops = [x.op for x in uops] self.assertIn(Ops.SHR, ops, f"For dtype={dt} FLOORDIV by power of two did not simplify to shift") @@ -469,10 +469,10 @@ def test_render_ssimplified_marg_outside_toposort(self): self.assertEqual(UOp.range(1, 0, src=(shrink,), dtype=dtypes.int).render(simplify=False), "r0") def test_render_vectorize_empty(self): - u = UOp(Ops.STACK, dtype=dtypes.void, src=()) + u = UOp(Ops.STACK, src=()) self.assertEqual(u.render(simplify=False), "{}") def test_render_vectorize_empty_simplified(self): - u = UOp(Ops.STACK, dtype=dtypes.void, src=()) + u = UOp(Ops.STACK, src=()) self.assertEqual(u.render(), "{}") def test_render_vectorize_same(self): u = UOp(Ops.STACK, src=(UOp.const(0),)*3) diff --git a/test/null/test_validate_oob.py b/test/null/test_validate_oob.py index abc7a9bc06419..5a256f8db6e89 100644 --- a/test/null/test_validate_oob.py +++ b/test/null/test_validate_oob.py @@ -14,37 +14,37 @@ class TestValidateOOB(unittest.TestCase): def test_const_index(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.const(0)).load(dtype=dtypes.int)]) # valid - to_uops_list([buf.index(UOp.const(15)).load(dtype=dtypes.int)]) # valid (last element) + to_uops_list([buf.index(UOp.const(0)).load()]) # valid + to_uops_list([buf.index(UOp.const(15)).load()]) # valid (last element) with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.const(16)).load(dtype=dtypes.int)]) # off by one + to_uops_list([buf.index(UOp.const(16)).load()]) # off by one with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.const(42)).load(dtype=dtypes.int)]) # way out + to_uops_list([buf.index(UOp.const(42)).load()]) # way out def test_variable_index(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(Variable("i", 0, 15)).load(dtype=dtypes.int)]) # valid + to_uops_list([buf.index(Variable("i", 0, 15)).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("i", 0, 20)).load(dtype=dtypes.int)]) # oob + to_uops_list([buf.index(Variable("i", 0, 20)).load()]) # oob with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("i", -5, 10)).load(dtype=dtypes.int)]) # negative + to_uops_list([buf.index(Variable("i", -5, 10)).load()]) # negative def test_range_with_mask(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) r = UOp.range(42, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r.valid(r < 16)).load(dtype=dtypes.int)]) # valid + to_uops_list([buf.index(r.valid(r < 16)).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r.valid(r < 17)).load(dtype=dtypes.int)]) # oob + to_uops_list([buf.index(r.valid(r < 17)).load()]) # oob def test_variable_with_mask(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) v = Variable("v", -5, 80) - to_uops_list([buf.index(v.valid((v >= 0) & (v < 16))).load(dtype=dtypes.int)]) # valid + to_uops_list([buf.index(v.valid((v >= 0) & (v < 16))).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(v.valid(v < 20)).load(dtype=dtypes.int)]) # negative not masked + to_uops_list([buf.index(v.valid(v < 20)).load()]) # negative not masked def test_gated_store(self): with Context(CHECK_OOB=1, SPEC=2): @@ -58,62 +58,62 @@ def test_gated_store(self): def test_floordiv(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2).load(dtype=dtypes.int)]) # 0..16 oob + to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2).load()]) # 0..16 oob def test_mod(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) r = UOp.range(100, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r % 16).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r % 16).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r % 20).load(dtype=dtypes.int)]) # 0..19 oob + to_uops_list([buf.index(r % 20).load()]) # 0..19 oob def test_shr(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2).load(dtype=dtypes.int)]) # 0..31 oob + to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2).load()]) # 0..31 oob def test_shl(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (64,)) r = UOp.range(8, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r << 2).load(dtype=dtypes.int)]) # 0..28 valid + to_uops_list([buf.index(r << 2).load()]) # 0..28 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r << 4).load(dtype=dtypes.int)]) # 0..112 oob + to_uops_list([buf.index(r << 4).load()]) # 0..112 oob def test_and(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) r = UOp.range(100, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r & 15).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r & 15).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r & 31).load(dtype=dtypes.int)]) # 0..31 oob + to_uops_list([buf.index(r & 31).load()]) # 0..31 oob # align masks round down to a multiple of 2^k - to_uops_list([buf.index((r & -4).valid(r < 16)).load(dtype=dtypes.int)]) # 0..12 valid + to_uops_list([buf.index((r & -4).valid(r < 16)).load()]) # 0..12 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r & -2).load(dtype=dtypes.int)]) # 0..100 oob + to_uops_list([buf.index(r & -2).load()]) # 0..100 oob # other masks can't be modeled as mod with self.assertRaisesRegex(RuntimeError, "z3 int AND only supports"): - to_uops_list([buf.index(r & 21).load(dtype=dtypes.int)]) + to_uops_list([buf.index(r & 21).load()]) def test_max(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) - to_uops_list([buf.index(Variable("v", -10, 15).maximum(0)).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(Variable("v", -10, 15).maximum(0)).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0)).load(dtype=dtypes.int)]) # 0..20 oob + to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0)).load()]) # 0..20 oob def test_xor_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (16,)) r = UOp.range(32, 0, AxisType.GLOBAL) - to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16)))).load(dtype=dtypes.int)]) # 0..15 valid + to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16)))).load()]) # 0..15 valid with self.assertRaises(RuntimeError): - to_uops_list([buf.index(r.valid((r < 10) ^ (r >= 20))).load(dtype=dtypes.int)]) # 0..9,20..31 oob + to_uops_list([buf.index(r.valid((r < 10) ^ (r >= 20))).load()]) # 0..9,20..31 oob # cast patterns def test_float_cast_in_index(self): @@ -121,13 +121,13 @@ def test_float_cast_in_index(self): buf = UOp.param(0, dtypes.int, (16,)) r = UOp.range(20, 0) i = (r.cast(dtypes.float) * 0.68).trunc().cast(dtypes.int) - to_uops_list([buf.index(i.valid((i >= 0) & (i < 16))).load(dtype=dtypes.int)]) + to_uops_list([buf.index(i.valid((i >= 0) & (i < 16))).load()]) def test_bool_cast_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): buf = UOp.param(0, dtypes.int, (1,)) r = UOp.range(20, 0) - to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not())).load(dtype=dtypes.int)]) # only r=0 valid + to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not())).load()]) # only r=0 valid # load result as index/mask def test_load_as_index(self): @@ -135,18 +135,18 @@ def test_load_as_index(self): buf0 = UOp.param(0, dtypes.int, (16,)) buf1 = UOp.param(1, dtypes.int, (64,)) r = UOp.range(42, 0, AxisType.GLOBAL) - ld0 = buf0.index(r.valid(r < 8)).load(dtype=dtypes.int).cast(dtypes.weakint) - to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32))).load(dtype=dtypes.int)]) # valid + ld0 = buf0.index(r.valid(r < 8)).load().cast(dtypes.weakint) + to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32))).load()]) # valid with self.assertRaises(RuntimeError): - to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 64))).load(dtype=dtypes.int)]) # oob + to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 64))).load()]) # oob def test_load_from_shrink_as_index(self): with Context(CHECK_OOB=1, SPEC=2): buf0 = UOp.param(0, dtypes.int, (16,)) buf1 = UOp.param(1, dtypes.int, (64,)) shrink = UOp(Ops.SHRINK, src=(buf0, UOp.const(0, dtypes.int), UOp.const(4))) - ld0 = shrink.load(dtype=dtypes.int).index(0) - to_uops_list([buf1.index(ld0.valid((ld0 >= 0) & (ld0 < 64))).load(dtype=dtypes.int)]) + ld0 = shrink.load().index(0) + to_uops_list([buf1.index(ld0.valid((ld0 >= 0) & (ld0 < 64))).load()]) def test_load_bool_as_mask(self): with Context(CHECK_OOB=1, SPEC=2): diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index fd3ee107e7e25..47f7ba5941b73 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -502,7 +502,7 @@ def bufferize_buf(ctx:bool, buf:UOp) -> UOp|None: # 7. resolve patches def push_stack(op, s): return UOp(Ops.STACK, - src=tuple(op.replace(dtype=op.dtype, src=tuple(x if y is s else y for y in op.src)) for x in s.src)) + src=tuple(op.replace(src=tuple(x if y is s else y for y in op.src)) for x in s.src)) def fold_binary(buf:UOp, blob:UOp) -> UOp: for b in (m.bufs if isinstance(m:=buf.buffer, MultiBuffer) else (m,)): From 9f01775cf4418669e4764b43fbf077f311fe1e6d Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 25 Aug 2026 08:43:59 -0400 Subject: [PATCH 198/306] dtype_from_uop for THREEFRY and FDIV (#17727) --- test/backend/test_ops.py | 10 +++++----- test/null/test_dtype_spec.py | 2 +- test/unit/test_dtype_weak.py | 6 +++--- tinygrad/uop/ops.py | 4 ++++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index d3bb8e20ee373..35dec0269b44a 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -865,9 +865,9 @@ def test_lshift(self): lambda: (ten << Tensor([0,2,4], dtype=dtypes.uint32)).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.__lshift__(2), lambda: ten.__lshift__(2).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.bitwise_left_shift(2), lambda: ten.lshift(2).cast(dtypes.int32), forward_only=True) - self.helper_test_exception([], lambda: torch.tensor([1.0]) << 2, lambda: Tensor([1.0]) << 2, expected=RuntimeError) - self.helper_test_exception([], lambda: tor << torch.tensor([1.0]), lambda: ten << Tensor([1.0]), expected=RuntimeError) - self.helper_test_exception([], lambda: tor << 1.0, lambda: ten << 1.0, expected=RuntimeError) + self.helper_test_exception([], lambda: torch.tensor([1.0]) << 2, lambda: (Tensor([1.0]) << 2).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor << torch.tensor([1.0]), lambda: (ten << Tensor([1.0])).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor << 1.0, lambda: (ten << 1.0).realize(), expected=RuntimeError) def test_rshift(self): data = [[0,1,2],[1<<8,1<<16,1<<31-1]] @@ -881,8 +881,8 @@ def test_rshift(self): lambda: (ten >> Tensor([0,2,4], dtype=dtypes.uint32)).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.__rshift__(2), lambda: ten.__rshift__(2).cast(dtypes.int32), forward_only=True) helper_test_op([], lambda: tor.bitwise_right_shift(2), lambda: ten.rshift(2).cast(dtypes.int32), forward_only=True) - self.helper_test_exception([], lambda: torch.tensor([4.0]) >> 1, lambda: Tensor([4.0]) >> 1, expected=RuntimeError) - self.helper_test_exception([], lambda: tor >> torch.tensor([1.0]), lambda: ten >> Tensor([1.0]), expected=RuntimeError) + self.helper_test_exception([], lambda: torch.tensor([4.0]) >> 1, lambda: (Tensor([4.0]) >> 1).realize(), expected=RuntimeError) + self.helper_test_exception([], lambda: tor >> torch.tensor([1.0]), lambda: (ten >> Tensor([1.0])).realize(), expected=RuntimeError) def test_lshift_signed(self): data = [[-1, -3, 1, 7], [0, -2147483648, 2147483647, -1]] diff --git a/test/null/test_dtype_spec.py b/test/null/test_dtype_spec.py index 8460d9c86db88..363ae5b75d70f 100644 --- a/test/null/test_dtype_spec.py +++ b/test/null/test_dtype_spec.py @@ -419,7 +419,7 @@ def test_where_two_scalars(self): self.check_where_alternate_input_other(3, True, dtypes.weakint) def test_where_non_bool_cond_raises(self): - with self.assertRaises(RuntimeError): Tensor([1, 0, 2]).where(1, 0) + with self.assertRaises(RuntimeError): Tensor([1, 0, 2]).where(1, 0).dtype self.check_where_alternate_input_other(False, True, dtypes.bool) @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes)) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 84e9de85d4459..5ed5f379c4e46 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -183,11 +183,11 @@ def test_weak_int_binop(self): self.assertEqual(dtype_from_uop(Ops.SHL, (UOp.const(1, dtypes.int8), UOp.const(1, dtypes.uint32)), None), dtypes.int8) self.assertEqual(UOp.const(1).alu(Ops.SHL, UOp.const(1, dtypes.uint)).dtype, dtypes.weakint) self.assertEqual((v & 3).dtype, dtypes.weakint) - with self.assertRaises(RuntimeError): Tensor.const(1.0) << Tensor.const(1.0) - with self.assertRaises(RuntimeError): UOp.const(1, dtypes.int32).alu(Ops.SHL, UOp.const(1, dtypes.float64)) + with self.assertRaises(RuntimeError): (Tensor.const(1.0) << Tensor.const(1.0)).dtype + with self.assertRaises(RuntimeError): UOp.const(1, dtypes.int32).alu(Ops.SHL, UOp.const(1, dtypes.float64)).dtype for op in (Ops.SHL, Ops.SHR): with self.assertRaises(RuntimeError): - UOp.const(1, dtypes.float32).alu(op, UOp.const(1, dtypes.int32)) + UOp.const(1, dtypes.float32).alu(op, UOp.const(1, dtypes.int32)).dtype # float bitwise builds, the spec rejects it with Context(SPEC=1): f32, wf = UOp.const(1.0, dtypes.float32), UOp.const(1.0) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 12bce3a8af9f7..2ff57a458e6a6 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -158,6 +158,10 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: return in_tuple.src[arg].dtype case Ops.GETADDR: return dtypes.uint64 + case Ops.THREEFRY: + return dtypes.uint64 + case Ops.FDIV: + return least_upper_float(promo_dtype(src)) case Ops.SHL | Ops.SHR: if not all(dtypes.is_int(x.dtype) or x.base.is_invalid for x in src): raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") From 023bfdb380dc023c2670000c5951defe8127082a Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:23:22 +0300 Subject: [PATCH 199/306] fix viz for hcq2 (#17729) --- tinygrad/engine/realize.py | 31 +++++++++++++++++-------------- tinygrad/runtime/support/hcq2.py | 5 +++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 3f1ae94f6e33a..85bb40d5ec52f 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -26,11 +26,12 @@ def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return (0,), tuple(range(1, len(get_call_arg_uops(call)))) return (), () -def get_call_kernels(call:UOp) -> list[tuple[str, UOp]]: - if (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": return [(d, k) for devs, k, _ in call.arg.aux.kernels for d in devs] - if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return [(to_tuple(ast.device)[0], call)] +def get_call_kernels(call:UOp) -> list[tuple[str, UOp, tuple[str, Estimates, bytes]|None]]: + if (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": + return [(d, call, (name, estimates, profile_key)) for devices,name,estimates,_,profile_key in call.arg.aux.kernels for d in devices] + if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "graph": return [(to_tuple(ast.device)[0], call, None)] if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "validate": return [] - return [(d, call) for d in to_tuple(call.src[1].device)] + return [(d, call, None) for d in to_tuple(call.src[1].device)] def get_call_name(call:UOp, bufs:Sequence[Buffer|UOp], var_vals:dict[str, int]|None=None) -> str: def _uop_sz_to_str(uop:UOp) -> str: return size_to_str(sym_infer(prod(uop.shape) * uop.dtype.itemsize, var_vals or {})) @@ -66,24 +67,25 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No if DEBUG < 2 and not PROFILE: return kernels = get_call_kernels(call) # everything below is the per kernel display: exec events for the profiler and DEBUG=2 lines - args = resolve_params(call, ctx.input_uops) if kernels and kernels[0][1] is call else [] + args = resolve_params(call, ctx.input_uops) if kernels and kernels[0][2] is None else [] lanes = list(unwrap_multi(call, [args[g] for g in call.src[0].arg.globals] if call.src[0].op is Ops.PROGRAM else args)) if args else [] - for i, (device, kcall) in enumerate(kernels): + for i, (device, kcall, stats) in enumerate(kernels): et, bufs = ets[i] if i < len(ets) else None, lanes[i][0] if i < len(lanes) else [] + display_name = get_call_name(kcall, bufs, ctx.var_vals) if stats is None else stats[0] if PROFILE: # backdate the event to the start of the call, the viz matches a device range with the exec event before it outputs, inputs = get_call_outs_ins(kcall) cpu_events.append(ProfilePointEvent(device, "exec", len(cpu_events), {"var_vals": ctx.var_vals, - "bufs": [b.trace_num for b in bufs], "name": get_call_name(kcall, bufs, ctx.var_vals), "outputs": outputs, "inputs": inputs}, ts=st)) + "bufs": [b.trace_num for b in bufs], "name": display_name, "outputs": outputs, "inputs": inputs}, ts=st)) if DEBUG < 2 or not ctx.update_stats: continue if et is None: Device[device].synchronize() et, st = float(perf_counter_us() - st)*1e-6, perf_counter_us() GlobalCounters.time_sum_s += et - estimates = estimate_uop(kcall) - display_name = get_call_name(kcall, bufs, ctx.var_vals) + estimates = estimate_uop(kcall) if stats is None else stats[1] op_est, mem_est, lds_est = (sym_infer(x, ctx.var_vals) for x in (estimates.ops, estimates.mem, estimates.lds)) - header_color = 'magenta' if ctx.jit else ('green' if kcall.src[0].key not in first_run_cache else None) + key = kcall.src[0].key if stats is None else stats[2] + header_color = 'magenta' if ctx.jit else ('green' if key not in first_run_cache else None) ptm = colored(time_to_str(et, w=9), "yellow" if et > 0.01 else None) if et is not None else "" flops, membw, ldsbw = op_est/(et or 1e-20), mem_est/(et or 1e-20), lds_est/(et or 1e-20) flops_str = f"{flops*1e-9:7.0f} GFLOPS" if flops < 1e14 else colored(f"{flops*1e-12:7.0f} TFLOPS", 'green') @@ -92,7 +94,7 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No print(f"{colored(f'*** {device[:7]:7s} {GlobalCounters.kernel_count:4d}', header_color)}"+ f" {ansipad(display_name, 46)} arg {len(bufs):2d} mem {GlobalCounters.mem_used/1e9:6.2f} GB"+ ("" if et is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) - first_run_cache.add(kcall.src[0].key) + first_run_cache.add(key) local_size_cache: dict[bytes, tuple[int, ...]] = {} def optimize_local_size(call:UOp, prg:UOp) -> UOp|None: @@ -221,13 +223,14 @@ def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer()._buf.va_addr + base}), call, ast) - def _prof_tm(device:str, stat_call:UOp, prof:tuple[int, ...]) -> float|None: - (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, stat_call.arg.name, prof[0], prof[1], stat_call.key) + def _prof_tm(device:str, name:str, prof:tuple[int, ...], profile_key:bytes) -> float|None: + (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, name, prof[0], prof[1], profile_key) if not ctx.wait: return None d.synchronize(timeout=ctx.timeout) st, en = (d.signal(x)._buf.cpu_view().view(fmt='Q')[0] for x in prof) return float(en-st)/d.timestamp_divider/1e6 - return [_prof_tm(device, k, prof) for devices, k, prof in info.kernels if prof for device in devices] if PROFILE or ctx.wait else [] + return [_prof_tm(device, name, prof, profile_key) for devices,name,_,prof,profile_key in info.kernels + if prof for device in devices] if PROFILE or ctx.wait else [] # flatten LINEAR-in-LINEAR: any nested LINEAR child gets inlined into its parent's src pm_flatten_linear = PatternMatcher([ diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 47f7ba5941b73..f72dc3c656029 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -32,7 +32,8 @@ class HCQInfo: input_idxs:tuple[tuple[tuple[str, ...], tuple[int, ...]], ...] = () # per inputs table: (devices, indexes into input_uops) inputs:int|None = None # index of the inputs table in call.src - kernels:tuple[tuple[tuple[str, ...], UOp, tuple[int, ...]], ...] = () # per kernel: (devices, a call carrying its name and estimates, timestamps) + # per kernel: (devices, name, estimates, timestamps, profile key) + kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...], bytes], ...] = () def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c @@ -271,7 +272,7 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] - kerns.append((devices, make_call(name, call.src[0], info), tuple(ts_ids))) + kerns.append((devices, name, info.estimates, tuple(ts_ids), make_call(name, call.src[0], info).key)) ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_buf(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] From 7dc8b666e7ac56016ce215e50cccb6d5cb722f00 Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 25 Aug 2026 11:46:31 -0400 Subject: [PATCH 200/306] wgsl cast before load for packed [pr] (#17731) instead of explicit dtype on load --- test/backend/test_renderer_failures.py | 2 +- tinygrad/renderer/wgsl.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/backend/test_renderer_failures.py b/test/backend/test_renderer_failures.py index 403a938cdc5de..9ec18c1a9b3d6 100644 --- a/test/backend/test_renderer_failures.py +++ b/test/backend/test_renderer_failures.py @@ -80,7 +80,7 @@ class TestWGSLFailures(unittest.TestCase): def test_folded_packed_store(self): b = UOp.param(0, dtypes.char, (4,)) idx = b.index(UOp.const(0).cast(dtypes.int)) - store = UOp.store(idx, UOp.load(idx, dtype=dtypes.uint32) & UOp.const(0xffffff00).cast(dtypes.uint32)) + store = UOp.store(idx, idx.cast(dtypes.uint32).load() & UOp.const(0xffffff00).cast(dtypes.uint32)) src = Device[Device.DEFAULT].renderer.render(UOp.sink(store, arg=KernelInfo()).toposort()) self.assertIn("atomicAnd(&data0_4[0],4294967040u);", src) self.assertNotIn("atomicAdd", src) diff --git a/tinygrad/renderer/wgsl.py b/tinygrad/renderer/wgsl.py index b1b26d4666edd..6f01288916170 100644 --- a/tinygrad/renderer/wgsl.py +++ b/tinygrad/renderer/wgsl.py @@ -18,19 +18,19 @@ def packed_store(s:UOp): # bool does its mask math at int32: renderer rewrites run after weak dtypes are lowered, and bool & 0xFF would create a weakint const if var.dtype == dtypes.bool: var = var.cast(dtypes.int32) new_v, wmask = (var & mask).cast(dtypes.uint32) << shift_am, ((mask << shift_am) ^ 0xFFFFFFFF).cast(dtypes.uint32) - buf = idx.load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ()), dtype=dtypes.uint32) + buf = idx.cast(dtypes.uint32).load(*((UOp.const(0, dtypes.uint32), *gate) if gate else ())) return idx.store((buf & wmask) | new_v, *gate) # load for char: sign_extend(buf[idx/4] >> ((idx%4)*8)) def packed_load(root:UOp): bidx, *alt = root.src idx, shift_am, mask = packed_field(bidx, dtype:=root.dtype) - load = idx.load(*((alt[0].cast(dtypes.uint32), *alt[1:]) if alt else ()), dtype=dtypes.uint32, arg=root.arg) + load = idx.cast(dtypes.uint32).load(*((alt[0].cast(dtypes.uint32), *alt[1:]) if alt else ()), arg=root.arg) val = (load >> shift_am) & mask return sign_extend(val, 8*dtype.itemsize).cast(dtype) if dtype in [dtypes.char, dtypes.short] else val.cast(dtype) def is_packed(x:UOp): - dt = x.src[1].dtype if x.op is Ops.STORE else x.dtype + dt = x.src[1].dtype if x.op is Ops.STORE else x.buf_uop.dtype return dt.itemsize < 4 and dt != dtypes.half and x.buf_uop.addrspace != AddrSpace.REG def _packed_size(u:UOp): return ceildiv(u.max_numel(), 4//u.dtype.itemsize) if is_packed(u) else u.max_numel() def is_nan(a): @@ -38,12 +38,12 @@ def is_nan(a): return (a.bitcast(getattr(dtypes, f"uint{bs}")) & ((1 << (bs - 1)) - 1)) > (((1 << exp) - 1) << mant) # the read-modify-write packed_store emits: a load of the very index being stored to, masked (a gated store loads with 3 srcs) -packed_rmw = UPat(Ops.LOAD, src=(UPat.var("b"),), allow_any_len=True) & UPat.var("wmask") +packed_rmw = UPat(Ops.LOAD, src=(UPat(Ops.CAST, dtype=dtypes.uint32, src=(UPat.var("b"),)),), allow_any_len=True) & UPat.var("wmask") wgsl_matcher = PatternMatcher([ (UPat((Ops.CMPLT, Ops.XOR), src=(UPat(name="a", dtype=dtypes.bool), UPat.var("b")), name="c"), lambda a,b,c: a.cast(dtypes.int).alu(c.op, b.cast(dtypes.int)).cast(dtypes.bool)), - (UPat(Ops.LOAD, name="l"), lambda l: packed_load(l) if is_packed(l) else None), + (UPat(Ops.LOAD, src=(UPat(Ops.INDEX),), allow_any_len=True, name="l"), lambda l: packed_load(l) if is_packed(l) else None), (UPat(Ops.STORE, name="s"), lambda s: packed_store(s) if is_packed(s) else None), (UPat.var("a") << UPat.var("b"),lambda a,b:(a.bitcast(dtypes.uint32)<> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), @@ -65,6 +65,7 @@ class WGSLRenderer(CStyleLanguage): dtypes.char: "i32", dtypes.int32: "i32", dtypes.uint32: "u32", dtypes.bool: "bool", dtypes.half: "f16" } string_rewrite = PatternMatcher([ + (UPat(Ops.CAST, dtype=dtypes.uint32, src=(UPat(Ops.INDEX, name="x"),)), lambda ctx,x: ctx[x] if is_packed(x) else None), (UPat(Ops.NEG, dtypes.uints, src=(UPat.var('x'))), lambda ctx,x: f"(0-{ctx[x]})"), (UPat.cvar("c").cast(dtypes.bool), lambda c: "true" if c.val else "false"), (UPat.cvar("c").cast((dtypes.uchar, dtypes.ushort, dtypes.uint32)), From d851aca9ae1faf4210cc0da4508bead7da57d7ee Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:03:19 +0300 Subject: [PATCH 201/306] hcq2: fix copy/call in usb (#17732) --- tinygrad/runtime/support/usb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index b2330470aa6dc..6ba42fef9c551 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -297,7 +297,7 @@ def usb_stage_copy(dst:UOp, src:UOp) -> UOp|None: sram.copy_to_device(d.device).call(d, sram)] else: pad = UOp.new_buffer("CPU", round_up(nb, 512), dtypes.uint8)[0:nb] - submit = make_submit(UOp(Ops.CALL, src=(UOp(Ops.COPY, dtypes.void, ()), sram, s)), devs=devs, queue="COPY:0") + submit = make_submit(s.copy_to_device(sram.device).call(sram, s), devs=devs, queue="COPY:0") pull = usb_bulk(devs, (submit,), 0x81, pad.getaddr((HCQ_RUNTIME_DEV.value,)), round_up(nb, 512), 10000) ops += [UOp.custom_function("hcq", pull.sink()).call(pad, sram, s, name="hcq_copyout", aux=HCQInfo(devs)), pad.copy_to_device("CPU").call(d, pad)] From b831ca62d9f4792d74ccad26639e92edfe9ce5ef Mon Sep 17 00:00:00 2001 From: George Hotz Date: Tue, 25 Aug 2026 13:20:15 -0700 Subject: [PATCH 202/306] Reapply "disk cache: thread-local db conn (#17694)" This reverts commit df528499ced865a714baff5a4ab2ad52ff2652ed. --- tinygrad/helpers.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 7390709b05d12..4c8bc4028ae26 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -1,7 +1,7 @@ from __future__ import annotations import time START_TIME = time.perf_counter() -import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc +import os, functools, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc, threading from collections import defaultdict import shutil, math, types, copyreg, inspect, importlib, decimal, itertools, difflib from dataclasses import dataclass, field, replace @@ -398,18 +398,17 @@ def my_gc_callback(phase, info): CACHEDB: str = getenv("CACHEDB", os.path.abspath(os.path.join(cache_dir, "cache.db"))) VERSION = 22 -_db_connection = None +_db_connection = threading.local() def db_connection(): - global _db_connection - if _db_connection is None: + if (conn:=getattr(_db_connection, "conn", None)) is None: os.makedirs(CACHEDB.rsplit(os.sep, 1)[0], exist_ok=True) - _db_connection = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") + conn = _db_connection.conn = sqlite3.connect(CACHEDB, timeout=60, isolation_level="IMMEDIATE") # another connection has set it already or is in the process of setting it # that connection will lock the database - with contextlib.suppress(sqlite3.OperationalError): _db_connection.execute("PRAGMA journal_mode=WAL").fetchone() - _db_connection.execute("PRAGMA synchronous=NORMAL") - if DEBUG >= 8: _db_connection.set_trace_callback(print) - return _db_connection + with contextlib.suppress(sqlite3.OperationalError): conn.execute("PRAGMA journal_mode=WAL").fetchone() + conn.execute("PRAGMA synchronous=NORMAL") + if DEBUG >= 8: conn.set_trace_callback(print) + return conn def diskcache_clear(): cur = db_connection().cursor() From 7fef98c86e995f32cde00c46d35378e62742ed36 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 25 Aug 2026 13:21:08 -0700 Subject: [PATCH 203/306] benchmarks: remove usage of sudo from linux runners (#17735) --- .github/workflows/benchmark.yml | 47 ++++++++++++++------------------- extra/hcq/hcq_smi.py | 13 ++++++--- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 691618c62414f..80d4674050e99 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -102,12 +102,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -149,12 +148,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -198,12 +196,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: Symlink models and datasets run: | mkdir -p extra/datasets @@ -241,12 +238,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -287,12 +283,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: Symlink models and datasets run: | mkdir -p weights @@ -340,12 +335,11 @@ jobs: - name: Setup (AMD) if: ${{ matrix.dev == 'AMD' }} run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py amd rmmod - ./extra/hcq/hcq_smi.py amd kill_pids + ./extra/hcq/hcq_smi.py amd rmmod --expect + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless - name: Setup (NV) if: ${{ matrix.dev == 'NV' }} - run: sudo lsof -tQ /dev/nvidia* | { xargs -r sudo kill -9 || true; } + run: lsof -tQ /dev/nvidia* | { xargs -r kill -9 || true; } - name: setup staging db if: github.ref == 'refs/heads/update_benchmark_staging' run: | @@ -588,9 +582,8 @@ jobs: uses: actions/checkout@v6 - name: Setup run: | - ./extra/amdpci/setup_python_cap.sh - ./extra/hcq/hcq_smi.py ${{ matrix.dev }} rmmod - ./extra/hcq/hcq_smi.py ${{ matrix.dev }} kill_pids + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} rmmod --expect + ./extra/hcq/hcq_smi.py ${{ matrix.dev }} kill_pids --sudoless mkdir -p extra/datasets ln -s /raid/datasets/imagenet extra/datasets/imagenet - name: setup staging db diff --git a/extra/hcq/hcq_smi.py b/extra/hcq/hcq_smi.py index 2482038b71a8f..814c8f4309c6a 100755 --- a/extra/hcq/hcq_smi.py +++ b/extra/hcq/hcq_smi.py @@ -16,9 +16,12 @@ def _do_reset_device(pci_bus): os.system(f"sudo sh -c 'echo 1 > /sys/bus/pci/dev def _is_module_loaded(name: str) -> bool: return os.path.isdir(f"/sys/module/{name}") def cmd_remove_module(args): - modules = ["nvidia_drm", "nvidia_modeset", "nvidia_uvm", "nvidia", "ast"] if args.backend == "nv" else ["amdgpu"] + modules = ["nvidia_drm", "nvidia_modeset", "nvidia_uvm", "nvidia"] if args.backend == "nv" else ["amdgpu"] to_unload = [m for m in modules if _is_module_loaded(m)] if not to_unload: print("Kernel modules are not loaded") + elif getattr(args, "expect", False): + print(f"Kernel modules are loaded: {to_unload}") + sys.exit(1) else: print("Removing kernel modules:", ", ".join(to_unload)) try: subprocess.run(["sudo", "modprobe", "-r", *to_unload], check=True) @@ -60,17 +63,19 @@ def cmd_show_pids(args): def cmd_kill_pids(args): devs = scan_devs_based_on_lock(prefix:={"amd":"am", "nv":"nv"}[args.backend], args) + use_sudo = not getattr(args, "sudoless", False) for dev in devs: for i in range(128): if i > 0: time.sleep(0.2) try: - try: pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] + try: pid = subprocess.check_output((['sudo'] if use_sudo else []) + + ['lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] except subprocess.CalledProcessError: break print(f"Killing process {pid} (which uses {dev})") - subprocess.run(['sudo', 'kill', '-9', pid], check=True) + subprocess.run((['sudo'] if use_sudo else []) + ['kill', '-9', pid], check=True) except subprocess.CalledProcessError as e: print(f"Failed to kill process for device {dev}: {e}", file=sys.stderr) @@ -79,6 +84,7 @@ def add_common_commands(parent_subparsers): p_insmod.set_defaults(func=cmd_insert_module) p_rmmod = parent_subparsers.add_parser("rmmod", help="Remove a kernel module") + p_rmmod.add_argument("--expect", action="store_true", help="Just assert that module is already unloaded") p_rmmod.set_defaults(func=cmd_remove_module) p_reset = parent_subparsers.add_parser("reset", help="Reset a device") @@ -91,6 +97,7 @@ def add_common_commands(parent_subparsers): p_reset = parent_subparsers.add_parser("kill_pids", help="Kill pids of processes using the device") p_reset.add_argument("--pci_bus", default="", help="PCI bus ID of the device") + p_reset.add_argument("--sudoless", action="store_true", help="Do not use sudo when detecting or killing pids") p_reset.set_defaults(func=cmd_kill_pids) if __name__ == "__main__": From 2824504f90a249eada7ea1050b3ebd7a4f0d44c9 Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Tue, 25 Aug 2026 14:11:53 -0700 Subject: [PATCH 204/306] usb amd: yield between signal polls (#17712) * AMD_USB_POLL_US * com * its micro --- tinygrad/runtime/ops_amd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index e5f968133d0f8..3a30cb6791e78 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -25,6 +25,7 @@ SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE = \ ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("SQTT_LIMIT_SE", 0), ContextVar("SQTT_SIMD_SEL", 0), ContextVar("SQTT_TOKEN_EXCLUDE", 0) PMC = ContextVar("PMC", abs(VIZ.value)>=2) +AMD_USB_POLL_US = getenv("AMD_USB_POLL_US", 500) # microseconds to sleep between USB signal polls. 0 disables EVENT_INDEX_PARTIAL_FLUSH = 4 # based on a comment in nvd.h WAIT_REG_MEM_FUNCTION_EQ = 3 # == WAIT_REG_MEM_FUNCTION_NEQ = 4 # != @@ -45,8 +46,9 @@ class AMDSignal(HCQSignal): def __init__(self, *args, **kwargs): super().__init__(*args, **{**kwargs, 'timestamp_divider': 100}) def _sleep(self, time_spent_since_last_sleep_ms:int): - # Reasonable to sleep for long workloads (which take more than 200ms) and only timeline signals. - if time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200) + # USB signals live in VRAM across the link, so yield between polls. Native AMD only blocks after 200 ms. + if self.owner is not None and self.owner.is_usb() and AMD_USB_POLL_US: time.sleep(AMD_USB_POLL_US / 1e6) + elif time_spent_since_last_sleep_ms > 200 and self.owner is not None: self.owner.iface.sleep(200) class AMDComputeQueue(HWQueue): def __init__(self, dev:AMDDevice): From 55032514ce2295dd706e095ddc0756f28af5ce9c Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 25 Aug 2026 14:53:15 -0700 Subject: [PATCH 205/306] pin onnxruntime==1.24.1 (#17738) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18939ec5a4afe..a039dcbffbd02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ testing = [ "pillow", "onnx==1.19.0", "onnx2torch", - "onnxruntime", + "onnxruntime==1.24.1", "opencv-python", "transformers", "sentencepiece", From 07268b724fe63e45ba33be193ee679dbf02b163f Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Tue, 25 Aug 2026 18:43:30 -0400 Subject: [PATCH 206/306] fix: external_test_gpu_crash on python 3.14 (#17739) --- test/external/external_test_gpu_crash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/external/external_test_gpu_crash.py b/test/external/external_test_gpu_crash.py index ab31ae35647f3..ebe0f1eb2d6d5 100644 --- a/test/external/external_test_gpu_crash.py +++ b/test/external/external_test_gpu_crash.py @@ -23,7 +23,7 @@ def setUpClass(cls): cls.is_cdna = cls.dev.target[0] < 10 ins = importlib.import_module('tinygrad.runtime.autogen.amd.' + ('cdna' if cls.is_cdna else 'rdna3') + '.ins') for rdna3_name, cdna3_name in RDNA3_CDNA3_MAP.items(): - setattr(cls, rdna3_name, getattr(ins, cdna3_name if cls.is_cdna else rdna3_name)) + setattr(cls, rdna3_name, staticmethod(getattr(ins, cdna3_name if cls.is_cdna else rdna3_name))) def setUp(self): # Verify device works before each test From 6ece327cf3dd6cc55b3d2bf81b557ce3de17b0a9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 25 Aug 2026 20:15:06 -0400 Subject: [PATCH 207/306] CUSTOM arg is (str, dtype) [PR] (#17737) --- extra/gemm/moe_routing.py | 2 +- .../quantize_fp8_delayed/__init__.py | 2 +- .../llama_kernels/quantize_mxfp4/__init__.py | 2 +- test/null/test_viz.py | 12 ++--- tinygrad/llm/kernels/amd.py | 16 +++---- tinygrad/nn/__init__.py | 2 +- tinygrad/renderer/cstyle.py | 2 +- tinygrad/runtime/ops_qcom.py | 5 +- tinygrad/uop/ops.py | 3 +- tinygrad/uop/spec.py | 5 +- tinygrad/uop/upat.py | 46 ++++++++++--------- 11 files changed, 52 insertions(+), 45 deletions(-) diff --git a/extra/gemm/moe_routing.py b/extra/gemm/moe_routing.py index eb4982c6e630f..321f241aa00bc 100644 --- a/extra/gemm/moe_routing.py +++ b/extra/gemm/moe_routing.py @@ -53,7 +53,7 @@ def _bwd_kernel(gtab:UOp, gout:UOp, idx:UOp) -> UOp: g, m, j, jo, ji = _kv_ranges(Gk, M, Dk, _blk_for(Dk)) row = idx.index(g, m).cast(dtypes.weakint) val = gout.index(g, m, j).load().cast(dtypes.float32) - atomic = UOp(Ops.CUSTOM, src=(gtab.index(g, row, j), val), arg=atomic_str) + atomic = UOp(Ops.CUSTOM, src=(gtab.index(g, row, j), val), arg=(atomic_str, dtypes.void)) return atomic.end(g, m, jo, ji).sink(arg=KernelInfo(name=f"ggather_bwd_{M}_{Dk}", opts_to_apply=())) grad_table = Tensor.custom_kernel(gt, go, Tensor(idx_u, device=dev), fxn=_bwd_kernel)[0] return (None, grad_table.cast(table_u.dtype).uop, None) diff --git a/extra/llama_kernels/quantize_fp8_delayed/__init__.py b/extra/llama_kernels/quantize_fp8_delayed/__init__.py index db9bb5fd689af..f7d42b44e19d3 100644 --- a/extra/llama_kernels/quantize_fp8_delayed/__init__.py +++ b/extra/llama_kernels/quantize_fp8_delayed/__init__.py @@ -50,7 +50,7 @@ def _custom_quantize_fp8_with_amax(fp8_out:UOp, amax_out:UOp, x:UOp, amax_state: else: raise NotImplementedError(f"no atomic max for device {device}") amax_idx = amax_out.reshape((1,)).index(UOp.const(0)) max_val = lds[0].load() - atomic = UOp(Ops.CUSTOM, src=(amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=atomic_arg) + atomic = UOp(Ops.CUSTOM, src=(amax_idx, max_val.bitcast(dtypes.int32), max_val, amax_idx.load()), arg=(atomic_arg, dtypes.void)) return atomic.end(tid, wg).sink(arg=KernelInfo(f"quantize_fp8_with_amax_{n_elems}", opts_to_apply=())) @functools.cache diff --git a/extra/llama_kernels/quantize_mxfp4/__init__.py b/extra/llama_kernels/quantize_mxfp4/__init__.py index e68fbb146bfbd..48682b8a1079b 100644 --- a/extra/llama_kernels/quantize_mxfp4/__init__.py +++ b/extra/llama_kernels/quantize_mxfp4/__init__.py @@ -12,7 +12,7 @@ def _custom_quantize_mxfp4(row_fp4:UOp, row_scale:UOp, col_fp4:UOp, col_scale:UO mem = M*N*2 + M*N + M*N//16 # read bf16, write row+col fp4 + e8m0 outputs = (row_fp4, row_scale, col_fp4, col_scale) sink = UOp.sink(*(o.base for o in outputs), x.base, - *(UOp(Ops.CUSTOM, src=(o.base.index(0),), arg="") for o in outputs), + *(UOp(Ops.CUSTOM, src=(o.base.index(0),), arg=("", dtypes.void)) for o in outputs), UOp.special(256, "lidx0"), UOp.special(M//128, "gidx0"), UOp.special(N//64, "gidx1"), arg=KernelInfo(name, estimates=Estimates(ops=12*M*N, mem=mem))) src = (pathlib.Path(__file__).parent/"quantize_mxfp4.cpp").read_text() diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 9812cefda2b89..06ce496137d57 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -185,18 +185,18 @@ def test_colored_label(self): @dataclass(frozen=True) class TestStruct: colored_field: str - a = UOp(Ops.CUSTOM, arg=TestStruct(colored("xyz", "magenta")+colored("12345", "blue"))) + a = UOp(Ops.PYLITERAL, arg=TestStruct(colored("xyz", "magenta")+colored("12345", "blue"))) a2 = uop_to_json(VizData(), a)[id(a)] - self.assertEqual(ansistrip(a2["label"]), f"CUSTOM\n{TestStruct.__qualname__}(colored_field='xyz12345')") + self.assertEqual(ansistrip(a2["label"]), f"PYLITERAL\n{TestStruct.__qualname__}(colored_field='xyz12345')") def test_colored_label_multiline(self): with save_viz() as viz: arg = colored("x", "green")+"\n"+colored("y", "red")+colored("z", "yellow")+colored("ww\nw", "magenta") src = [Tensor.empty(1).uop for _ in range(10)] - a = UOp(Ops.CUSTOM, src=tuple(src), arg=arg) + a = UOp(Ops.PYLITERAL, src=tuple(src), arg=arg) exec_rewrite(a, [PatternMatcher([])]) a2 = next(viz.get_details(0, 0))["graph"][id(a)] - self.assertEqual(ansistrip(a2["label"]), "CUSTOM\nx\nyzww\nw") + self.assertEqual(ansistrip(a2["label"]), "PYLITERAL\nx\nyzww\nw") def test_inf_loop(self): a = UOp.const(3) @@ -347,7 +347,7 @@ def test_gc_uop_in_arg(self): init = bufs_allocated() a = UOp.new_buffer("NULL", 10, dtypes.char) a.buffer.allocate() - exec_rewrite(UOp(Ops.CUSTOM, src=(a,), arg=a), [PatternMatcher([])]) + exec_rewrite(UOp(Ops.PYLITERAL, src=(a,), arg=a), [PatternMatcher([])]) del a self.assertEqual(bufs_allocated()-init, 0) lst = viz.list_items() @@ -474,7 +474,7 @@ def test_view_source(self): def custom_fn(X:UOp): X = X.flatten() i = UOp.range(X.numel(), 0) - custom_op = UOp(Ops.CUSTOMI, src=(X[i],), arg="{} + undeclared_name") + custom_op = UOp(Ops.CUSTOMI, src=(X[i],), arg=("{} + undeclared_name", X.dtype)) return X[i].store(custom_op).end(i).sink(arg=KernelInfo(name=f"custom_fn_{X.numel()}")) x = Tensor.custom_kernel(Tensor.empty(1, device="CPU"), fxn=custom_fn)[0] with save_viz() as viz: diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py index b206c3415fa5d..ecb4d5a79ee63 100644 --- a/tinygrad/llm/kernels/amd.py +++ b/tinygrad/llm/kernels/amd.py @@ -35,8 +35,8 @@ def amd_custom_kernels_supported(device:str|tuple[str, ...]|None) -> bool: def warp_reduce(val:UOp, maximum:bool=False, full_wave:bool=False) -> UOp: for offset in ((16, 8, 4, 2, 1) if full_wave else (8, 4, 2, 1)): if val.op is Ops.INDEX and val.addrspace == AddrSpace.REG: val = val.load() - other = UOp(Ops.CUSTOM, dtypes.float, (val,), arg= - f"__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {{0}}), {0x1f | offset<<10}))") + other = UOp(Ops.CUSTOM, src=(val,), arg= + (f"__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {{0}}), {0x1f | offset<<10}))", dtypes.float)) val = val.maximum(other) if maximum else val + other return val @@ -77,14 +77,14 @@ def __call__(self, x:Tensor) -> Tensor: return super().__call__(x) def _amd_dp4a(a:UOp, b:UOp, c:UOp) -> UOp: - return UOp(Ops.CUSTOMI, dtypes.int32, (a.int(), b.int(), c), arg="__builtin_amdgcn_sudot4(true, {}, true, {}, {}, false)") + return UOp(Ops.CUSTOMI, src=(a.int(), b.int(), c), arg=("__builtin_amdgcn_sudot4(true, {}, true, {}, {}, false)", dtypes.int32)) def _amd_byte_perm(a:UOp, b:UOp, selectors:UOp) -> UOp: - return UOp(Ops.CUSTOMI, dtypes.uint32, tuple(x.cast(dtypes.uint32) for x in (a, b, selectors)), arg="__builtin_amdgcn_perm({}, {}, {})") + return UOp(Ops.CUSTOMI, src=tuple(x.cast(dtypes.uint32) for x in (a, b, selectors)), arg=("__builtin_amdgcn_perm({}, {}, {})", dtypes.uint32)) def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: assert ptr.op is Ops.INDEX - if lanes is None: return UOp(Ops.CUSTOMI, ptr.dtype, (ptr,), arg="__builtin_nontemporal_load({0})") + if lanes is None: return UOp(Ops.CUSTOMI, src=(ptr,), arg=("__builtin_nontemporal_load({0})", ptr.dtype)) buf, coords = ptr.src[0], ptr.src[1:] idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0)) return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load(dtype=ptr.dtype) @@ -191,7 +191,7 @@ def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): output_waves = 2 if out_features % (32*output_tiles) == 0 else 1 token_block, output_block = UOp.range(out.shape[0]//token_tile, 0), UOp.range(out_features//(16*output_tiles*output_waves), 1) lane, wave = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) - hw_lane = UOp(Ops.CUSTOM, dtypes.int32, (lane.int(),), arg="__builtin_amdgcn_mbcnt_lo(-1, 0)").cast(dtypes.weakint) + hw_lane = UOp(Ops.CUSTOM, src=(lane.int(),), arg=("__builtin_amdgcn_mbcnt_lo(-1, 0)", dtypes.int32)).cast(dtypes.weakint) col, half = hw_lane % 16, hw_lane // 16 outputs = tuple((output_block*output_waves+wave)*(16*output_tiles) + tile*16 + col for tile in range(output_tiles)) inputs = tuple(token_block*token_tile + tile*16 + col for tile in range(token_tile//16)) @@ -201,8 +201,8 @@ def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): def _wmma_stores(out, outputs, tokens, accs, update, half): def values(acc:UOp) -> tuple[UOp, ...]: vals = tuple(acc.after(update)[i].load() for i in range(8)) - swapped = tuple(UOp(Ops.CUSTOM, dtypes.float32, (value,), - arg="__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {0}), 50688))") for value in vals) + swapped = tuple(UOp(Ops.CUSTOM, src=(value,), + arg=("__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {0}), 50688))", dtypes.float32)) for value in vals) low = half.eq(0) return tuple(low.where(vals[i], swapped[i+4]) if j == 0 else low.where(swapped[i], vals[i+4]) for i in range(4) for j in range(2)) return [out[token, output].store(value) for output,output_accs in zip(outputs, accs) diff --git a/tinygrad/nn/__init__.py b/tinygrad/nn/__init__.py index e98186111e0ef..9b663935a457f 100644 --- a/tinygrad/nn/__init__.py +++ b/tinygrad/nn/__init__.py @@ -359,7 +359,7 @@ def _embedding_bwd_kernel(grad_weight:UOp, grad_emb:UOp, idx:UOp) -> UOp: if device in ("CPU", "NULL"): atomic_arg = "__atomic_fetch_add({0}, {1}, __ATOMIC_RELAXED);" elif device == "AMD": atomic_arg = "__hip_atomic_fetch_add({0}, {1}, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);" else: raise NotImplementedError(f"no atomics for device {device}") - atomic = UOp(Ops.CUSTOM, src=(grad_weight.index(local_token_id, j_idx), grad_val), arg = atomic_arg) + atomic = UOp(Ops.CUSTOM, src=(grad_weight.index(local_token_id, j_idx), grad_val), arg=(atomic_arg, dtypes.void)) return atomic.end(i, j_outer, j_inner).sink(arg=KernelInfo(name="embedding_bwd", opts_to_apply=())) grad_weight_uop = grad_weight_uop.custom_kernel(grad_emb, idx, fxn=_embedding_bwd_kernel)[0] diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index ac9f2eda85792..f139ce9dc2232 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -71,7 +71,7 @@ f"({', '.join(f'({ctx.render_type(y)})({ctx[y]})' for y in x.src[1:])}))" + (";" if x.dtype is dtypes.void else "")), # custom passes through with format - (UPat((Ops.CUSTOM, Ops.CUSTOMI), name="x"), lambda ctx,x: x.arg.format(*[ctx[y] for y in x.src])), + (UPat((Ops.CUSTOM, Ops.CUSTOMI), name="x"), lambda ctx,x: x.arg[0].format(*[ctx[y] for y in x.src])), ]) def create_non_native_float_pats(dts:tuple[DType, ...], casting:bool=True): diff --git a/tinygrad/runtime/ops_qcom.py b/tinygrad/runtime/ops_qcom.py index f69a14c5547c0..7b12d8cc82ddb 100644 --- a/tinygrad/runtime/ops_qcom.py +++ b/tinygrad/runtime/ops_qcom.py @@ -22,8 +22,9 @@ def dcache_flush(): from tinygrad.codegen import to_program buf, n = UOp.param(0, dtypes.uint8, shape=(1,)), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) i = UOp.range(n, 0, dtype=dtypes.int) - flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg='__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");') - sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg='__asm__ volatile("dsb sy" ::: "memory");'), arg=KernelInfo(name="dcache_flush"), tag=1) + flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg=('__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");', dtypes.void)) + sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg=('__asm__ volatile("dsb sy" ::: "memory");', dtypes.void)), + arg=KernelInfo(name="dcache_flush"), tag=1) prg = to_program(sink, Device["CPU"].renderer) return Device["CPU"].runtime(prg.to_elf()) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 2ff57a458e6a6..359c55c15efe5 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -123,7 +123,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: # a CALL of an opaque body is void, a CALL of an address can return a value return dtypes.void if src[0].dtype is dtypes.void else None case Ops.CUSTOM | Ops.CUSTOMI: - return None + assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"CUSTOM/CUSTOMI arg must be (str, DType), got {arg}" + return arg[1] case Ops.INS: return None case Ops.NOOP: diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 09b9c17c9060a..3fa820c480dc2 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -100,8 +100,9 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS})),), allow_any_len=True, name="x"), lambda x: matches_dtype(x.src[0], x.dtype)), - # CUSTOM (inline and non inline) - (UPat((Ops.CUSTOMI, Ops.CUSTOM)), lambda: True), + # CUSTOM (inline and non inline): the arg is the source string and the dtype it produces, void for a bare statement + (UPat((Ops.CUSTOMI, Ops.CUSTOM), name="x"), + lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[0], str) and isinstance(x.arg[1], DType)), # CALL of an external function (UPat(Ops.CALL, src=(UPat(),), allow_any_len=True, name="x"), diff --git a/tinygrad/uop/upat.py b/tinygrad/uop/upat.py index 9bbcab452f01f..55ab01eaebcf5 100644 --- a/tinygrad/uop/upat.py +++ b/tinygrad/uop/upat.py @@ -2,6 +2,7 @@ import itertools, inspect, functools, types from tinygrad.helpers import partition, dedup, Context from tinygrad.uop.ops import UPat, UOp, Ops, PatternMatcher, graph_rewrite, deconstruct_function +from tinygrad.dtype import dtypes class UPatCompileError(Exception): pass @@ -18,40 +19,42 @@ def _get_clause(self:UPat, base:UOp, depth=0) -> UOp: # build the and_clause for acceptance and_clause:list[UOp] = [] if self.op is not None: - if len(self.op) > 1: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(int(x) for x in self.op))), arg="{0}.op in {1}")) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg="{0}.op == "+str(self.op[0].value))) + if len(self.op) > 1: + and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(int(x) for x in self.op))), arg=("{0}.op in {1}", dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("{0}.op == "+str(self.op[0].value), dtypes.void))) if self.arg is not None: - if isinstance(self.arg, int): and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg="{0}.arg == "+str(int(self.arg)))) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.arg)), arg="{0}.arg == {1}")) + if isinstance(self.arg, int): and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("{0}.arg == "+str(int(self.arg)), dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.arg)), arg=("{0}.arg == {1}", dtypes.void))) if self.strict_length or self.required_len > 0: - and_clause.append(UOp(Ops.CUSTOM, src=(base,), arg=("len({0}.src)"+(" == " if self.strict_length else " >= ")+str(self.required_len)))) - if self.name is not None: and_clause.append(UOp(Ops.STORE, src=(UOp(Ops.CUSTOMI, arg=self.name), base))) + and_clause.append(UOp(Ops.CUSTOM, src=(base,), + arg=("len({0}.src)"+(" == " if self.strict_length else " >= ")+str(self.required_len), dtypes.void))) + if self.name is not None: and_clause.append(UOp(Ops.STORE, src=(UOp(Ops.CUSTOMI, arg=(self.name, dtypes.void)), base))) if self.match_dtype is not None: if len(self.match_dtype) > 1: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_dtype))), - arg="{0}.dtype in {1}")) + arg=("{0}.dtype in {1}", dtypes.void))) else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_dtype[0])), - arg="{0}.dtype == {1}")) + arg=("{0}.dtype == {1}", dtypes.void))) if self.match_tag is not None: if len(self.match_tag) > 1: - and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_tag))), arg="{0}.tag in {1}")) - else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_tag[0])), arg="{0}.tag == {1}")) + and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=tuple(self.match_tag))), arg=("{0}.tag in {1}", dtypes.void))) + else: and_clause.append(UOp(Ops.CUSTOM, src=(base, UOp(Ops.PYLITERAL, arg=self.match_tag[0])), arg=("{0}.tag == {1}", dtypes.void))) if self.src is not None: # single match if len(self.src) == 1 and isinstance(self.src[0], tuple): and_clause += [_get_clause(s, base.index(i), depth) for i,s in enumerate(self.src[0])] # repeat match elif len(self.src) == 1 and isinstance(self.src[0], itertools.repeat): - it = UOp(Ops.CUSTOMI, arg=f"ituop{depth}") + it = UOp(Ops.CUSTOMI, arg=(f"ituop{depth}", dtypes.void)) match = _get_clause(next(self.src[0]), it, depth+1) - and_clause.append(UOp(Ops.CUSTOM, src=(match, it, base), arg="all([{0} for {1} in {2}.src])")) + and_clause.append(UOp(Ops.CUSTOM, src=(match, it, base), arg=("all([{0} for {1} in {2}.src])", dtypes.void))) # multi match (fork) elif len(self.src) > 1 and all(isinstance(x, tuple) for x in self.src): fork_cond = [UOp(Ops.AND, src=tuple([_get_clause(s, base.index(i), depth) for i,s in enumerate(ss)])) for ss in self.src] and_clause.append(UOp(Ops.OR, src=tuple(fork_cond))) else: raise RuntimeError("broken") - return UOp(Ops.AND, src=tuple(and_clause)) if and_clause else UOp(Ops.CUSTOMI, arg="True") + return UOp(Ops.AND, src=tuple(and_clause)) if and_clause else UOp(Ops.CUSTOMI, arg=("True", dtypes.void)) # *** pattern matcher *** @@ -91,7 +94,7 @@ def do_process_and(a:UOp) -> UOp|None: for store in stores: if store.src[0] in dict_stores: # duplicate store is an identity compare - new_src.append(UOp(Ops.CUSTOM, src=(dict_stores[store.src[0]], store.src[1]), arg="{0} is {1}")) + new_src.append(UOp(Ops.CUSTOM, src=(dict_stores[store.src[0]], store.src[1]), arg=("{0} is {1}", dtypes.void))) found = True else: dict_stores[store.src[0]] = store.src[1] @@ -108,17 +111,18 @@ def do_process_and(a:UOp) -> UOp|None: # renderer def wrap(ctx, x) -> UOp: ctx[ret:=f"a{len(ctx)}"] = x.arg - return UOp(Ops.CUSTOMI, arg=ret) + return UOp(Ops.CUSTOMI, arg=(ret, dtypes.void)) pm_renderer = PatternMatcher([ (UPat(Ops.PYLITERAL, name="x"), wrap), # AND of CUSTOMI fragments inside a CUSTOM becomes a single CUSTOMI (joined with " and ") (UPat(Ops.CUSTOM, src=(UPat(Ops.AND, src=UPat(Ops.CUSTOMI), name="x"), UPat(), UPat()), name="r"), - lambda r,x: r.replace(src=(UOp(Ops.CUSTOMI, arg="(" + ' and '.join(y.arg for y in x.src) + ")"),)+r.src[1:])), + lambda r,x: r.replace(src=(UOp(Ops.CUSTOMI, arg=("(" + ' and '.join(y.arg[0] for y in x.src) + ")", dtypes.void)),)+r.src[1:])), - (UPat(Ops.CUSTOM, src=UPat(Ops.CUSTOMI), name="x"), lambda x: UOp(Ops.CUSTOMI, arg=x.arg.format(*[y.arg for y in x.src]))), - (UPat(Ops.INDEX, src=(UPat(Ops.CUSTOMI, name="x"), UPat(Ops.CONST, name="c")), name="g"), lambda x,c,g: x.replace(arg=x.arg+f".src[{c.val}]")) + (UPat(Ops.CUSTOM, src=UPat(Ops.CUSTOMI), name="x"), lambda x: UOp(Ops.CUSTOMI, arg=(x.arg[0].format(*[y.arg[0] for y in x.src]), dtypes.void))), + (UPat(Ops.INDEX, src=(UPat(Ops.CUSTOMI, name="x"), UPat(Ops.CONST, name="c")), name="g"), + lambda x,c,g: x.replace(arg=(x.arg[0]+f".src[{c.val}]", dtypes.void))) ], compiled=False) def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: @@ -131,8 +135,8 @@ def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: for ss in s.src: or_pieces.extend(_final_render(ss, has_ctx, depth+1)) elif s.op is Ops.STORE: assert s.src[0].op is Ops.CUSTOMI and s.src[1].op is Ops.CUSTOMI - store_pieces.append(f"{s.src[0].arg}={s.src[1].arg}") - elif s.op is Ops.CUSTOMI: and_pieces.append(s.arg) + store_pieces.append(f"{s.src[0].arg[0]}={s.src[1].arg[0]}") + elif s.op is Ops.CUSTOMI: and_pieces.append(s.arg[0]) else: raise UPatCompileError(f"can't compile this {s}") # if we have an or, render it if len(or_pieces): @@ -145,7 +149,7 @@ def _final_render(x:UOp, has_ctx:bool, depth=1) -> list[str]: return [f"{' '*depth}if {and_clause}: return _ret"] def _get_code(self:UPat, has_ctx:bool): - ret = _get_clause(self, UOp(Ops.CUSTOMI, arg="uop")) + ret = _get_clause(self, UOp(Ops.CUSTOMI, arg=("uop", dtypes.void))) try: # TODO: this should be tracked in a "system" rewrite, not untracked or tracked with kernel with Context(TRACK_MATCH_STATS=0): From dc04c7820e25987c31f2068875a515c881b79bb3 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:15:28 -0700 Subject: [PATCH 208/306] lil fixes from new_rangeify (#17741) * lil fixes from new_rangeify * gpt sol review --- test/null/test_uop_graph.py | 5 +++++ test/null/test_uop_symbolic.py | 5 +++++ tinygrad/codegen/__init__.py | 7 ++++--- tinygrad/codegen/simplify.py | 8 ++++---- tinygrad/uop/symbolic.py | 2 ++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 0c8616bce66c9..9cc3f88688a11 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -202,6 +202,11 @@ def test_devectorize_derives_lane_dtype(self): invalid_lane_mul = next(u for u in out.src[0].toposort() if u.op is Ops.MUL) self.assertIs(invalid_lane_mul.dtype, dtypes.bool) + def test_devectorize_zero_sized_scalar_expand(self): + from tinygrad.codegen import devectorizer2 + expanded = UOp.const(1.0).reshape(1, 1).expand(0, 3) + self.assertEqual(graph_rewrite(expanded, devectorizer2).shape, (0, 3)) + def test_gep_vec_const_fold(self): for vec_size in [2, 4, 8]: consts = [UOp.const(float(i), dtypes.float) for i in range(vec_size)] diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index b127a89dbc9e1..910e63f2415bf 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -1455,6 +1455,11 @@ def test_invalid_gate_simplifies_vectorize(self): self.assertEqual(idx, (r0 < 3).where(expected_vec, UOp.invalid())) class TestRangeSplitting(unittest.TestCase): + def test_end_preserves_constant_backedge(self): + loop, backedge = UOp.loop(0), UOp.const(False) + end = graph_rewrite(UOp(Ops.NOOP).end(loop, backedge), sym) + self.assertEqual(end.src, (UOp(Ops.NOOP), loop, backedge)) + def test_range_split_on_mod(self): # test that mark_range_mod splits RANGE(8) into RANGE(4)*2 + RANGE(2) when used with %2 from tinygrad.codegen.simplify import pm_split_ranges, pm_flatten_range diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 5ffbcf2e846ca..f1c304b965e01 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -26,7 +26,7 @@ from tinygrad.codegen.late.linearizer import CFGContext, pm_split_ends, pm_add_control_flow, linearize from tinygrad.codegen.late.regalloc import LinearScanRegallocContext, pm_regalloc_rewrite from tinygrad.codegen.late.coalesce import memory_coalescing, pm_simplify_add_image -from tinygrad.helpers import all_same, flatten, argsort, partition +from tinygrad.helpers import all_same, all_int, flatten, argsort, partition from tinygrad.uop.ops import _broadcast_shape, identity_element from tinygrad.schedule.rangeify import BufferizeOpts @@ -162,9 +162,10 @@ def do_stack_wmma(u:UOp): (UPat(Ops.RESHAPE, dtype=dtypes.void, name="x"), lambda x: x.src[0]), # reshape of a single element shaped value to scalar is an index (UPat(Ops.RESHAPE, name="x"), lambda x: x.src[0].index(0) if x.marg == () and x.src[0].shape == (1,) else None), - # EXPAND on scalar -> STACK + # EXPAND on scalar -> nested STACKs with the same shape (UPat(Ops.EXPAND, src=(UPat.var("x"), UPat()), name="out"), - lambda x,out: UOp.stack(*([x]*out.max_numel())) if x.shape == () and out.shape == (out.max_numel(),) else None), + lambda x,out: functools.reduce(lambda x,s: UOp.stack(*([x]*s)), reversed(out.shape), x) + if x.shape == () and all_int(out.shape) and 0 not in out.shape else None), ]) def fix_group_for_reduce(x:UOp): diff --git a/tinygrad/codegen/simplify.py b/tinygrad/codegen/simplify.py index b25de5089fc0b..582e721662ed3 100644 --- a/tinygrad/codegen/simplify.py +++ b/tinygrad/codegen/simplify.py @@ -35,10 +35,10 @@ def simplify_merge_adjacent(u:UOp) -> UOp|None: nidx = graph_rewrite(u, _substitute+symbolic+pm_flatten_range, ctx={r0:new_range//s1, r1:new_range%s1}, name=f"check_merge_{r0.arg[0]}_{r1.arg[0]}") - # check if it simplifies - if count_divmod(nidx) <= count_divmod(u): - u = nidx - return u + # check if it simplifies. return after one merge so the next rewrite uses the new ranges, + # rather than continuing with stale pairs from the original ended_ranges. + if count_divmod(nidx) <= count_divmod(u): return nidx + return None def mark_gated(ctx, idx): if len(idx.src) > 1 and idx.src[1].op is Ops.WHERE: diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 81868f4ff20a6..55e69655f0139 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -312,6 +312,8 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: else y.src for y in x.src[1:]]))))), # after/end with 1 src is just src[0] (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), + # ranges can be subbed for CONSTs, remove them from ENDs while preserving a constant bool backedge + (UPat(Ops.END, name="x"), lambda x: x.replace(src=(x.src[0],)+tuple(r for r in x.src[1:] if r.op is not Ops.CONST or r.dtype is dtypes.bool))), # the rules above key on bare CONSTs, so a redundantly committed const has to be uncast in the same fixpoint ])+div_and_mod_symbolic+pm_uncast_const From b49c03fb1c494fa34c9e86fece8bef3006e1165e Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Tue, 25 Aug 2026 18:06:04 -0700 Subject: [PATCH 209/306] benchmarks: don't use sudo on mac (#17740) --- .github/workflows/benchmark.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 80d4674050e99..798d2261b3ab4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -431,23 +431,20 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: Kill stale pids run: | - PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids - PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids - # since sudo is required for usbgpu on macos, do not write bytecode, as some of the files are owned by root + ./extra/hcq/hcq_smi.py amd kill_pids --sudoless + ./extra/hcq/hcq_smi.py nv kill_pids --sudoless - name: UsbGPU boot time - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus + run: GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU tiny tests - run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py + run: GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py - name: UsbGPU copy speeds - run: sudo -E PYTHONDONTWRITEBYTECODE=1 SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds - #- name: UsbGPU openpilot test - # run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx + run: SIZE=64000000 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds - name: UsbGPU (USB4/TB) install script - run: PYTHONPATH=. sh extra/setup_tinygpu_osx.sh + run: sh extra/setup_tinygpu_osx.sh - name: UsbGPU (USB4/TB) boot time - run: PYTHONPATH=. DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus + run: DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU (USB4/TB) tiny tests - run: PYTHONPATH=. DEV=PCI+NV:NAK python3.11 test/test_tiny.py + run: DEV=PCI+NV:NAK python3.11 test/test_tiny.py testcommalatest: name: comma Benchmark (0.11.2) From 625c05df1efadb587ee44f86d5d6fa4a0578e84c Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:21:55 -0700 Subject: [PATCH 210/306] fix am_smi to respect dev (#17742) --- extra/amdpci/am_smi.py | 6 ++++-- test/backend/test_asm_gemm.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/extra/amdpci/am_smi.py b/extra/amdpci/am_smi.py index ed43f107589d6..888e77ccbc239 100755 --- a/extra/amdpci/am_smi.py +++ b/extra/amdpci/am_smi.py @@ -84,7 +84,8 @@ def read_pci_state(self): with open(f"/sys/bus/pci/devices/{self.pcibus}/power_state", "r") as f: return f.read().strip().rstrip() class SMICtx: - def __init__(self): + def __init__(self, dev_filter=None): + self.dev_filter = dev_filter self.devs = [] self.opened_pcidevs = [] self.opened_pci_resources = {} @@ -135,6 +136,7 @@ def rescan_devs(self): pattern = os.path.join('/tmp', 'am_*.lock') for d in [f[8:-5] for f in glob.glob(pattern)]: if d.startswith("usb"): continue + if self.dev_filter is not None and d != self.dev_filter: continue if d not in self.opened_pcidevs: self._open_am_device(d) @@ -406,7 +408,7 @@ def draw(self, once): try: if not args.list: os.system('clear') - smi_ctx = SMICtx() + smi_ctx = SMICtx(args.dev) while True: smi_ctx.rescan_devs() smi_ctx.draw(args.list) diff --git a/test/backend/test_asm_gemm.py b/test/backend/test_asm_gemm.py index 6e34b996c0279..594de42b2ae4e 100644 --- a/test/backend/test_asm_gemm.py +++ b/test/backend/test_asm_gemm.py @@ -188,7 +188,7 @@ def test_empty(self): M, N, K = getenv("M", 16384), getenv("N", 4096), getenv("K", 14336) a = Tensor.empty(M, K, dtype=dtypes.bfloat16) b = Tensor.empty(N, K, dtype=dtypes.bfloat16) - asm_gemm(a, b.T, mxfp4=True).realize() + for _ in range(getenv("CNT", 1)): asm_gemm(a, b.T, mxfp4=True).realize() # test the Asm GEMM with Llama shapes, only run on the real machine for speed From 9860e5d2852aeb021e949da37caefd2d553b583a Mon Sep 17 00:00:00 2001 From: b1tg <33436708+b1tg@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:30:35 +0800 Subject: [PATCH 211/306] llm tokenizer: fix tekken, add gpt4o (#17733) --- test/null/test_llm_tokenizer.py | 20 ++++++++++++++++++++ tinygrad/llm/cli.py | 19 +++++++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/test/null/test_llm_tokenizer.py b/test/null/test_llm_tokenizer.py index 2675dae44de0f..a526c81a3607b 100644 --- a/test/null/test_llm_tokenizer.py +++ b/test/null/test_llm_tokenizer.py @@ -95,6 +95,26 @@ def test_tekken_from_gguf_kv(self): self.assertEqual(template.end_turn(), "[/INST]") self.assertEqual(template.role("assistant"), "") + def test_tekken_gpt4o_split(self): + split = {p: SimpleTokenizer({}, {}, p)._split_to_word.findall for p in ("tekken", "gpt-4o")} + shared = { + "HelloWorld": ["Hello", "World"], + " ÜNICODE": [" ÜNICODE"], # Ü: non-ascii upper joins the run + "é café": ["é", " café"], # first é is e + U+0301 combining acute (NFD) + "เพื่อน วิ": ["เพื่อน", " วิ"], # thai vowel marks stay in the word + "a/b\r\n x": ["a", "/b", "\r\n", " x"], # punct tail eats / + } + for s, want in shared.items(): + self.assertEqual(split["tekken"](s), want, f"tekken {s!r}") + self.assertEqual(split["gpt-4o"](s), want, f"gpt-4o {s!r}") + differ = [ + ("12345", list("12345"), ["123", "45"]), # digits: tekken single, o200k groups {1,3} + ("it's I'M don'T", ["it", "'s", " I", "'M", " don", "'T"], ["it's", " I'M", " don'T"]), # contraction: o200k inline suffix + ] + for s, tk, go in differ: + self.assertEqual(split["tekken"](s), tk, f"tekken {s!r}") + self.assertEqual(split["gpt-4o"](s), go, f"gpt-4o {s!r}") + def test_stream_decoder(self): """stream_decoder buffers incomplete UTF-8: token 25677 has 3/4 of emoji, token 138 completes it.""" bs = [*range(33, 127), *range(161, 173), *range(174, 256)] diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index 4a17a9f6d02b8..e0e2336e2c943 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -12,22 +12,29 @@ class SimpleTokenizer: def __init__(self, normal_tokens:dict[str, int], special_tokens:dict[str, int], preset:str="llama3", bos_id:int|None=None, eos_id:int=0, eot_id:int|None=None): preset = {"qwen35":"qwen2","qwen35moe":"qwen2"}.get(preset, preset) - if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4"): + if preset not in ("llama3","llama-v3","llama-bpe","qwen2","olmo","kimi-k2","tekken","glm4","gpt-4o"): raise ValueError(f"Invalid tokenizer preset '{preset}'") # https://github.com/openai/gpt-2/blob/9b63575ef42771a015060c964af2c3da4cf7c8ab/src/encoder.py#L9 bs = [*range(33, 127), *range(161, 173), *range(174, 256)] # bytes that map to themselves self._byte_decoder = {chr(b): b for b in bs} | {chr(256+i): b for i,b in enumerate(b for b in range(256) if b not in bs)} # https://github.com/ggml-org/llama.cpp/blob/94933c8c2eeaa9a7983e3f6c08af76bd86724094/src/llama-vocab.cpp#L286 - # 0x323b0 is one past the max codepoint in unicode categories L/N/Z (0x323af is max L) + # each limit is one past the category's max codepoint (Z→U+3000, N→U+1FBF9, L→U+323AF, M→U+E01EF) # compact adjacent codepoints into ranges: listing them all makes re spend seconds on large prompts - def ucat_range(pre:str) -> str: - cps = enumerate(cp for cp in range(0x323b0) if unicodedata.category(chr(cp)).startswith(pre)) + def ucat_range(pre:str|tuple[str, ...]) -> str: + limits = {"Z": 0x3001, "N": 0x1fbfa, "L": 0x323b0, "M": 0xe01f0} + limit = max(limits[p if p in limits else p[0]] for p in (pre if isinstance(pre, tuple) else (pre,))) + cps = enumerate(cp for cp in range(limit) if unicodedata.category(chr(cp)).startswith(pre)) runs = [list(g) for _, g in itertools.groupby(cps, lambda e: e[1]-e[0])] return "".join(re.escape(chr(g[0][1])) + (f"-{re.escape(chr(g[-1][1]))}" if len(g) > 1 else "") for g in runs) r_ws, r_p_N, r_p_L = r"\t\n\x0b\x0c\r\x85" + ucat_range("Z"), ucat_range("N"), ucat_range("L") - self._split_to_word = re.compile("(?i:'s|'t|'re|'ve|'m|'ll|'d)|" + \ - f"[^\\r\\n{r_p_N}{r_p_L}]?[{r_p_L}]+|[{r_p_N}]{{1,3}}| ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*|[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+") + contr, r_l, r_n = "(?i:'s|'t|'re|'ve|'m|'ll|'d)", f"[^\\r\\n{r_p_N}{r_p_L}]?", f"[{r_p_N}]" if preset == "tekken" else f"[{r_p_N}]{{1,3}}" + r_p, r_w, r_t = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n]*", f"{contr}|{r_l}[{r_p_L}]+", f"[{r_ws}]*[\\r\\n]+|[{r_ws}]+(?![^{r_ws}])|[{r_ws}]+" + if preset in ("tekken", "gpt-4o"): + r_up, r_lo = ucat_range(("Lu","Lt","Lm","Lo","M")), ucat_range(("Ll","Lm","Lo","M")) + sfx = f"{contr}?" if preset == "gpt-4o" else "" + r_p, r_w = f" ?[^{r_ws}{r_p_N}{r_p_L}]+[\\r\\n/]*", f"{r_l}[{r_up}]*[{r_lo}]+{sfx}|{r_l}[{r_up}]+[{r_lo}]*{sfx}" + self._split_to_word = re.compile(f"{r_w}|{r_n}|{r_p}|{r_t}") self._split_to_sentence = re.compile("|".join(re.escape(tok) for tok in special_tokens.keys()) if special_tokens else r"(?!)") self._normal_tokens = {bytes(self._byte_decoder[c] for c in tok): tid for tok, tid in normal_tokens.items()} From ecf79e260d91e763ba7f39e1944088317e9257f4 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:46:10 +0800 Subject: [PATCH 212/306] better all2all schedule test (#17746) * better all2all schedule test * deconstruct those numbers * reorder --- test/unit/test_allreduce.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/unit/test_allreduce.py b/test/unit/test_allreduce.py index 59663089cc04c..26ff252507db3 100644 --- a/test/unit/test_allreduce.py +++ b/test/unit/test_allreduce.py @@ -3,6 +3,7 @@ from tinygrad.helpers import Context from tinygrad.uop.ops import Ops from test.helpers import KernelCountException +from tinygrad.engine.realize import run_linear class TestRingAllReduce(unittest.TestCase): def test_schedule_ring(self): @@ -21,13 +22,26 @@ def test_schedule_ring(self): def test_schedule_all2all(self): with Context(ALL2ALL=2): N = 4 + M = N*100 ds = tuple(f"CPU:{i}" for i in range(N)) - t = Tensor.empty(N, N*100).shard(ds, axis=0).realize() - linear = t.sum(0).mul(2.0).contiguous().linear_with_vars()[0] + x = Tensor.arange(N*M, dtype=dtypes.float).reshape(N, M) + t = (x*x).clone().shard(ds, axis=0).realize() + out = t.sum(0).mul(2.).contiguous() + linear, var_vals = out.linear_with_vars() copies = [si for si in linear.src if si.src[0].op is Ops.COPY] sinks = [si for si in linear.src if si.src[0].op is Ops.SINK] - if len(copies) != 24: raise KernelCountException(24, len(copies)) - if len(sinks) != 26: raise KernelCountException(26, len(sinks)) + # N*(N-1) copies for input and output + copy_count = N*(N-1)*2 + if len(copies) != copy_count: raise KernelCountException(copy_count, len(copies)) + # N*N shrinks becoming contigs, N ALU, N extra contig, reassembly (cat), and mul + sink_count = (N*N)+(N)+(N)+(1)+(1) + if len(sinks) != sink_count: raise KernelCountException(sink_count, len(sinks)) + # correctness + run_linear(linear, var_vals) + expected = [2*sum((d*M+i)**2 for d in range(N)) for i in range(M)] + dev_nums = Tensor.arange(1, N+1, dtype=dtypes.float).reshape(N, 1).expand(N, M).shard(ds, axis=0) + shards = out.reshape(1, M).expand(N, M)+dev_nums + self.assertListEqual(shards.tolist(), [[x+d+1 for x in expected] for d in range(N)]) @Context(RING=0, ALL2ALL=0) def test_schedule_naive(self): From 4c20f1d357937d65f606e0aebf4802208aeccfb3 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Wed, 26 Aug 2026 10:08:04 -0400 Subject: [PATCH 213/306] fix asinh precision loss on negative inputs (#17749) --- test/backend/test_ops.py | 4 ++-- tinygrad/mixin/elementwise.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 35dec0269b44a..05c940966aa25 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1807,9 +1807,9 @@ def test_hardtanh(self): helper_test_op([()], lambda x: torch.nn.functional.hardtanh(x, -val, val), lambda x: x.hardtanh(-val, val), grad_atol=1e-6) def test_asinh(self): helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6) - # TODO: this one has larger tol? - helper_test_op([(45,65)], lambda x: x.asinh(), atol=1e-2, rtol=2e-2, grad_rtol=2e-2, low=-300, high=-297) + helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=300, high=303) + helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-1e10, high=-1e9) def test_acosh(self): helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-6) helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-3, grad_rtol=1e-2, low=-300, high=-297) diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 3818ef0b5e2b8..dec78a65e0e20 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -870,7 +870,7 @@ def asinh(self) -> Self: print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).asinh().numpy()) ``` """ - return (self + (self.square() + 1).sqrt()).log() + return self.sign() * (self.abs() + (self.square() + 1).sqrt()).log() def acosh(self) -> Self: """ From 4bdc86513124455e149a431677b93b13238ac1a9 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 26 Aug 2026 10:50:42 -0400 Subject: [PATCH 214/306] delete unused rewrite rules [PR] (#17748) --- test/backend/test_multitensor.py | 5 +++++ test/backend/test_setitem.py | 8 ++++++++ test/null/test_uop_symbolic.py | 12 +++++++++++- tinygrad/codegen/late/coalesce.py | 1 - tinygrad/schedule/rangeify.py | 8 -------- tinygrad/uop/render.py | 2 -- tinygrad/uop/symbolic.py | 3 --- 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index dc0deb0d04237..2501f824bbfb2 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -58,6 +58,11 @@ def test_shard(self): assert X.uop.ended_ranges == X.uop.src[1:] (X + X).realize() + def test_shard_invalids_contiguous(self): + # every store is Invalid, so none of them should become a (empty) kernel + t = Tensor.invalids(8).shard(devices_2, axis=0).contiguous() + self.assertEqual(len([c for c in t.schedule_linear().src if c.src[0].op is Ops.SINK]), 1) + @unittest.expectedFailure # TODO: fix def test_shard_empty(self): GlobalCounters.reset() diff --git a/test/backend/test_setitem.py b/test/backend/test_setitem.py index 2017756da7cc7..03ee238681f7a 100644 --- a/test/backend/test_setitem.py +++ b/test/backend/test_setitem.py @@ -301,6 +301,14 @@ def test_setitem_multiple_disjoint_on_invalid(self): self.assertListEqual(z[2:5].tolist(), [2, 2, 2]) self.assertListEqual(z[6:7].tolist(), [3]) +class TestAssignBitcast(unittest.TestCase): + def test_assign_through_bitcast(self): + # the dest is unrealized, so callify cannot fold the BITCAST into a buffer view and the STORE keeps a + # BITCAST dest; the bitcast has to move to the value side or the store never reaches the buffer + a = Tensor.full((4,), 1.0, dtype=dtypes.float32).contiguous() + a.bitcast(dtypes.uint32).assign(Tensor([0x40800000, 0x40400000, 0x40000000, 0x3f800000], dtype=dtypes.uint32)).realize() + np.testing.assert_allclose(a.numpy(), [4.0, 3.0, 2.0, 1.0]) + class TestWithGrad(unittest.TestCase): def test_basic_setitem_works(self): z = Tensor.rand(8, 8) diff --git a/test/null/test_uop_symbolic.py b/test/null/test_uop_symbolic.py index 910e63f2415bf..cc5fe2d601b19 100644 --- a/test/null/test_uop_symbolic.py +++ b/test/null/test_uop_symbolic.py @@ -5,7 +5,7 @@ from tinygrad.dtype import dtypes, ConstType, DType, Invalid from tinygrad.uop.ops import UOp, Ops, graph_rewrite, sym_infer from tinygrad.uop.spec import spec_shared, type_verify -from tinygrad.uop.symbolic import sym, commutative, pm_simplify_valid, pm_move_where_on_load +from tinygrad.uop.symbolic import sym, commutative, pm_simplify_valid, pm_move_where_on_load, symbolic_simple from tinygrad.uop.validate import uops_to_z3 def check_uop_against_string(self, v:UOp, s:str): @@ -448,10 +448,20 @@ def test_and_fold(self): def test_and_remove(self): self.helper_test_variable(uand([uconst(1), Variable("a", 0, 1)]), 0, 1, "a") + def test_zero_div_zero_bottom_up(self): + # codegen runs symbolic_simple bottom_up, so the 0/0 is rewritten before its consts fold. + # without the guard the unsound x/x -> 1 below it claims this one. + z = UOp.const(0.0) + self.assertTrue(math.isnan(graph_rewrite(z/z, symbolic_simple, bottom_up=True).arg)) + def test_masked_shr_fold(self): x = UOp.variable('x', 0, 255, dtype=dtypes.uint32, param=True) self.helper_test_variable((x & -4) >> 2, 0, 63, "(x>>2)") + def test_masked_idiv_fold(self): + x = UOp.variable('x', 0, 255, dtype=dtypes.uint32, param=True) + self.helper_test_variable((x & -4) // 4, 0, 63, "(x//4)") + def test_bool_or_not_tautology(self): a = Variable("a", 0, 10) c = a<10 diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index f3bfcbeec87ec..305de98153b73 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -95,7 +95,6 @@ def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: pm_simplify_add_image = PatternMatcher([ (UPat(Ops.SHRINK, src=(UPat(Ops.PARAM, name="buf"), UPat(name="x"), UPat(arg=4))), transform_to_image), # image load/store is always float - (UPat(Ops.INDEX, dtype=dtypes.float, name="x").load(dtype=dtypes.half), lambda x: x.load().cast(dtypes.half)), (UPat(Ops.INDEX, dtype=dtypes.float, name="x").store(UPat(name="d", dtype=dtypes.half)), lambda x,d: x.store(d.cast(dtypes.float))), (UPat.var("x", dtype=dtypes.float).cast(dtypes.half).cast(dtypes.float), lambda x: x), ]) diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 0b8b152d11005..769f5b69123bf 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -132,8 +132,6 @@ def after_all_invalid(after:UOp): (UPat(Ops.STAGE, name="b"), cleanup_dead_axes), # remove noop buffers. if we look at the next index we can remove even more of these (UPat(Ops.INDEX, name="idx").f(Ops.STAGE, allow_any_len=True, name="b2"), remove_noop_bufferize), - (UPat(Ops.INDEX, src=(UPat(Ops.STAGE),), allow_any_len=True, name="idx").f(Ops.NOOP).f(Ops.STAGE, allow_any_len=True, name="b2"), - remove_noop_bufferize), # no buffers for a const, in either spelling (UPat.cvar('c').or_casted().f(Ops.STAGE, allow_any_len=True, name="b"), lambda c,b: b.const_like(c.val)), # indexing a const is the const @@ -141,8 +139,6 @@ def after_all_invalid(after:UOp): # indexing an after with all fully invalid stores is invalid (UPat(Ops.INDEX, src=(UPat(Ops.AFTER, name="after"),), allow_any_len=True, name="idx"), lambda idx,after: idx.const_like(Invalid) if after_all_invalid(after) else None), - # hack if a noop turned to a const - (UPat(Ops.NOOP, src=(UPat.cvar().or_casted("c"),)), lambda c: c), # a deviceless MSTACK src is the same value on every device, so indexing the stack is just indexing that value (UPat(Ops.MSTACK, src=(UPat.var("s"),), allow_any_len=True).f(Ops.INDEX, allow_any_len=True, name="idx"), lambda s,idx: idx.replace(src=(s,)+idx.src[1:]) if s.device is None else None), @@ -364,10 +360,6 @@ def get_contiguous(ctx:LocalAddBufferContext, x:UOp): rangeify_codegen = PatternMatcher([ (UPat(Ops.CONTIGUOUS, name="x"), get_contiguous), - - # no NOOP in the kernel graph - # TODO: this can be moved into codegen? - (UPat(Ops.NOOP, name="x"), lambda x: x.src[0] if len(x.src) else None), ]) pm_add_param_range_tags = PatternMatcher([ diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 7f070a06a7251..4bdb0ec5ad97c 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -57,8 +57,6 @@ def strip_binary_parens(x:UOp, left:str, right:str, code_for_op) -> str: ]) renderer_infer = PatternMatcher([ - (UPat(Ops.CMOD, name="x"), lambda ctx,x: f"cmod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), - (UPat(Ops.CDIV, name="x"), lambda ctx,x: f"cdiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.FLOORMOD, name="x"), lambda ctx,x: f"floormod({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.FLOORDIV, name="x"), lambda ctx,x: f"floordiv({ctx[x.src[0]]}, {ctx[x.src[1]]})"), (UPat(Ops.CAST, name="x"), diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 55e69655f0139..a3ab02b26519f 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -448,9 +448,6 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: ]) sym = symbolic+pm_simplify_valid+PatternMatcher([ - # reorder ALU/VECTORIZE - (UPat(GroupOp.ALU, src=(UPat(Ops.STACK, src=UPat(name='x')), UPat(Ops.STACK, src=UPat(name='y'))), name='alu'), - lambda x,y,alu: UOp(Ops.STACK, src=(UOp(alu.op, src=(x,y)),))), # ** where ** # push cast to branches (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), From 0abcf09b7498be654abebd960d9898976aa8bfea Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 26 Aug 2026 13:25:52 -0400 Subject: [PATCH 215/306] never bufferize_to_store weak input [PR] (#17751) github github --- tinygrad/schedule/rangeify.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 769f5b69123bf..5fc268a03c6cb 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools -from tinygrad.dtype import dtypes, AddrSpace, Invalid, strong_dtype +from tinygrad.dtype import dtypes, AddrSpace, Invalid from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg, shape_to_shape_arg from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic @@ -217,7 +217,7 @@ def visitor(u:UOp) -> frozenset[UOp]: def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): size = prod(x.shape) - dtype = strong_dtype(x.dtype) # a BUFFER is never weak: store at the concrete dtype, the .cast(x.dtype) on the result keeps readers unchanged + if x.dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {x.dtype}") rngs = sorted(idx.ranges, key=lambda x: x.arg) assert size > 0 and isinstance(size, int), f"no zero sized or symbolic sized buffers {size}" @@ -238,15 +238,15 @@ def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): # NOTE: the local BUFFER needs to be disambiguated here if x.arg.addrspace == AddrSpace.GLOBAL: - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(next(ctx), dtype, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) - do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) - return buf.after(do_store).cast(x.dtype) + buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(next(ctx), x.dtype, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) + do_store = buf.index(idx).store(x.src[0]).end(*rngs) + return buf.after(do_store) if allow_locals: # handle locals - buf = UOp.placeholder((size,), dtype, next(ctx), AddrSpace.LOCAL) - do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) - return buf.after(do_store).cast(x.dtype) + buf = UOp.placeholder((size,), x.dtype, next(ctx), AddrSpace.LOCAL) + do_store = buf.index(idx).store(x.src[0]).end(*rngs) + return buf.after(do_store) # collapse any BUFFERIZE to single input BUFFERIZE def flatten_bufferize(x:UOp): @@ -271,11 +271,6 @@ def remove_noop_afters(x:UOp) -> UOp|None: pm_add_buffers = pm_mops+pm_flatten_bufferize+PatternMatcher([ (UPat(Ops.STAGE, src=(UPat(), UPat(name="idx")), name="x"), lambda ctx,x,idx: bufferize_to_store(ctx, x, idx, allow_locals=False)), - # INDEX of a buffer through the weak cast added above: index the buffer directly and cast the loaded value instead. - # this must run in the same rewrite that adds the cast, or the expander expands the whole casted buffer into one big VECTORIZE - (UPat(Ops.INDEX, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("buf"),)),), allow_any_len=True, name="u"), - lambda u,buf: u.replace(dtype=None, src=(buf,)+u.src[1:]).cast(u.dtype)), - # move RESHAPEs through MSELECT/MSTACK (UPat((Ops.MSELECT, Ops.MSTACK), src=UPat(Ops.RESHAPE), name="m"), lambda m: m.replace(src=tuple([x.src[0].base for x in m.src])).reshape(m.shape)), From ea6c82f3bebf55437a1b6d1a772a09638a683572 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:33:31 -0700 Subject: [PATCH 216/306] small changes from new rangeify (#17752) --- test/backend/test_schedule.py | 2 +- tinygrad/schedule/multi.py | 4 +++- tinygrad/schedule/prepare.py | 11 ++++++++++- tinygrad/uop/symbolic.py | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/backend/test_schedule.py b/test/backend/test_schedule.py index 4f4bffa10aebc..85bedd2dc0fae 100644 --- a/test/backend/test_schedule.py +++ b/test/backend/test_schedule.py @@ -147,7 +147,7 @@ def test_copy_multi_scalar(self): devs = ("CPU:0", "CPU:1") x = Tensor.ones(2, device="CPU").shard(devs, axis=0).realize() out = (x.sum()*2).reshape(1).to("CPU") - run_linear(*check_schedule(out, 5)) + run_linear(*check_schedule(out, 3)) np.testing.assert_equal(out.numpy(), [4.]) class TestLimitBufs(unittest.TestCase): diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index 45d29d260b12f..bc97293776ff2 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -35,9 +35,11 @@ def lower_broadcast_copy(c:UOp, x:UOp): (UPat(Ops.MSELECT, src=(UPat(Ops.MSTACK, name="mstack"),), name="ms"), lambda mstack, ms: mstack.src[ms.arg]), # move shrink before MSTACK (UPat(Ops.SHRINK, src=(UPat(Ops.MSTACK, name="ms"),), allow_any_len=True, name="shrink"), mstack_early_shrink), - # move MSELECT before movement ops + # move MSELECT before movement/ALU ops (UPat(Ops.MSELECT, src=(UPat(GroupOp.Movement, src=(UPat.var("s"),), allow_any_len=True, name="v"),), name="ms"), lambda s,v,ms: v.replace(src=(s.mselect(ms.arg),)+v.src[1:])), + (UPat(Ops.MSELECT, src=(UPat(GroupOp.ALU, name="a"),), name="ms"), lambda a,ms: + a.replace(src=tuple(s.mselect(ms.arg) if isinstance(s.device, tuple) else s for s in a.src))), ]) _early_allreduce = PatternMatcher([ diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 7fac55277d8da..aea39e4d0a771 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -8,6 +8,10 @@ from tinygrad.schedule.allreduce import create_allreduce_function from tinygrad.schedule.multi import multi_pm +def walk_mop(u:UOp): + if u.op in GroupOp.Movement or u.op in {Ops.INDEX, Ops.UNSHARD}: return walk_mop(u.src[0]) + return u + def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): if (x:=src).op is Ops.CAST and x.dtype == dtypes.half and FLOAT16: x, after = x.src[0], after.cast(dtypes.float) while True: @@ -174,11 +178,16 @@ def expand_bitcast(bc:UOp) -> UOp|None: lambda reduce,x: reduce.const_like(identity_element(reduce.arg[0], reduce.dtype)) if 0 in x.shape and 0 not in reduce.shape else None), # handle size 0 (UPat(GroupOp.All-{Ops.SINK}, name="x"), lambda x: x.const_like(0).rtag(x.tag) if x._shape is not None and 0 in x.shape else None), + + # remove movement ops from SINK/AFTER. TODO: should be generic + (UPat(Ops.SINK, name="s"), lambda s: s.replace(src=tuple(walk_mop(u) for u in s.src if u.op is not Ops.NOOP))), + (UPat(Ops.AFTER, name="s"), lambda s: s.replace(src=(s.src[0],)+tuple(walk_mop(u) for u in s.src[1:] if u.op is not Ops.NOOP))), ]) def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): input_src = copy.src[0] - if not input_src.has_buffer_identity(after_ok=True): input_src = input_src.contiguous() + # if it's a COPY, we need to give the input buffer identity + if not input_src.has_buffer_identity(after_ok=True) and copy.op is Ops.COPY: input_src = input_src.contiguous() input_src = input_src.flatten() if existing_buf is not None: # if the existing buffer is not a full buffer, we can't use it diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index a3ab02b26519f..3ed54afa794b3 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -189,6 +189,7 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: # a conditional with the same results either way is a noop, also fold const conditionals (UPat.var().where(UPat.var("val"), UPat.var("val")), lambda val: val), (UPat.cvar("gate").where(UPat.var("c0"), UPat.var("c1")).named("w"), fold_const_where), + (UPat.var("gate").where(UPat.var("x"), 0) != 0, lambda gate,x: gate & (x != 0)), # a.where(b.where(c, d), d) -> (a & b).where(c, d) (UPat.var("a").where(UPat.var("b").where(UPat.var("c"), UPat.var("d")), UPat.var("d")), lambda a,b,c,d: (a&b).where(c,d)), # a.where(c, b.where(c, d)) -> (a | b).where(c, d) From 6a3b297548ab809d5390e3b2cfb20a326752ab9f Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 26 Aug 2026 14:07:29 -0400 Subject: [PATCH 217/306] fix PTX NIR SPEC=2 for bool [pr] (#17753) * fix PTX NIR SPEC=2 for bool [pr] storing bool with uint8 needs to pass SPEC * the fix --- test/backend/test_ops.py | 3 +++ tinygrad/renderer/__init__.py | 7 ++++++- tinygrad/renderer/nir.py | 11 ++++++----- tinygrad/renderer/ptx.py | 10 +++++----- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 05c940966aa25..303a7cc37d72b 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -6,6 +6,7 @@ from tinygrad import Tensor, Device, dtypes from tinygrad.tensor import _to_np_dtype from tinygrad.renderer.nir import NIRRenderer +from tinygrad.renderer.isa.x86 import X86Renderer TINY_BACKEND = getenv("TINY_BACKEND") if TINY_BACKEND: @@ -808,6 +809,8 @@ def test_xor(self): helper_test_op([], lambda: tor^0x1337, lambda: ten^0x1337, forward_only=True) helper_test_op([], lambda: 0x1337^tor, lambda: 0x1337^ten, forward_only=True) + # TODO: x86 PARAM dtype fails SPEC=2 + @Context(SPEC=1 if isinstance(Device[Device.DEFAULT].renderer, X86Renderer) else 2) def test_and(self): data = [[1,-8,1],[32,1,6]] tor = torch.tensor(data, dtype=torch.int) diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 728e6148d3ddc..86cd044a06cf2 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -1,12 +1,17 @@ from __future__ import annotations from typing import Callable, cast -from dataclasses import dataclass +from dataclasses import dataclass, replace from tinygrad.helpers import prod, Target, EMULATED_DTYPES from tinygrad.uop.ops import Ops, UOp, sint, ssimplify, smin, GroupOp, PatternMatcher from tinygrad.dtype import AddrSpace, DType, dtypes from tinygrad.codegen.opt.tc import TensorCore from tinygrad.device import Compiler +# an access takes its dtype from the buffer it indexes, so accessing at another dtype restates the storage on the buffer that owns it +def with_storage(x:UOp, dt:DType) -> UOp: + if x.op in {Ops.PARAM, Ops.BUFFER}: return x.replace(dtype=None, arg=replace(x.arg, dtype=dt)) + return x.replace(dtype=None, src=(with_storage(x.src[0], dt),)+x.src[1:]) + @dataclass(frozen=True) class Estimates: # number of FLOPS used in the Kernel diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index cbfa3e46db45d..bcf0fdf2e0829 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -1,7 +1,7 @@ from typing import Callable, Any from tinygrad.dtype import AddrSpace, DType, dtypes, truncate from tinygrad.helpers import DEBUG, OSX, unwrap, fromimport, Target, is_image_shape, round_up -from tinygrad.renderer import Renderer +from tinygrad.renderer import Renderer, with_storage from tinygrad.renderer.cstyle import CUDARenderer from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str from tinygrad.runtime.autogen import mesa, libc @@ -123,11 +123,12 @@ class NIRRenderer(Renderer): extra_matcher = PatternMatcher([ # from ptx (UPat.var('x', dtype=dtypes.bool) uint8 + # a bool is one bit in NIR but a byte in memory, so every access to a bool buffer goes through a uint8 view of it (UPat(Ops.LOAD, dtypes.bool, name="x"), - lambda x: x.replace(dtype=dtypes.uint8, src=x.src[0:1]+((x.src[1].cast(dtypes.uint8),) if len(x.src)>=2 else ())+x.src[2:]).cast(dtypes.bool)), - (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), - lambda x: x.replace(src=(x.src[0], x.src[1].cast(dtypes.uint8))+x.src[2:])), + lambda x: x.replace(dtype=None, src=(with_storage(x.src[0], dtypes.uint8),)+((x.src[1].cast(dtypes.uint8),) if len(x.src)>=2 else ()) + +x.src[2:]).cast(dtypes.bool)), + (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:])), # NIR requires shift amount to be 32 bit: https://docs.mesa3d.org/nir/alu.html#nir-alu-op-ishl (UPat((Ops.SHL, Ops.SHR), name="x"), lambda x: x.replace(src=(x.src[0], x.src[1].cast(dtypes.uint))) if x.src[1].dtype.bitsize != 32 else None), # OpConvertFToU is undefined if Result Type is not wide enough, cast through int32 diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index 120fa39021453..dcf763a89dedd 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -4,7 +4,7 @@ from tinygrad.codegen.opt import tc from tinygrad.uop.ops import Ops, UOp, PatternMatcher, UPat, GroupOp from tinygrad.dtype import dtypes, DType, AddrSpace -from tinygrad.renderer import Renderer +from tinygrad.renderer import Renderer, with_storage from tinygrad.renderer.cstyle import CUDARenderer from tinygrad.helpers import flatten, prod, unwrap, Target @@ -45,12 +45,12 @@ def render_val(x, dtype): # upcast to float32 all the ops that don't support half (UPat(doesnt_support_half, dtype=dtypes.half, name="x"), lambda x: (UOp(x.op, src=tuple(vv.cast(dtypes.float32) for vv in x.src), arg=x.arg).cast(dtypes.half))), - # load/store bool -> uint8 (only for memory, not registers) + # a bool is a predicate register in PTX but a byte in memory, so a bool buffer is accessed through a uint8 view of it (UPat(Ops.LOAD, dtypes.bool, src=(UPat(name="idx"),), name="x", allow_any_len=True), - lambda x,idx: UOp(x.op, dtypes.uint8, x.src[0:1] + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + x.src[2:]).cast(dtypes.bool) \ - if idx.addrspace != AddrSpace.REG else None), + lambda x,idx: x.replace(dtype=None, src=(with_storage(idx, dtypes.uint8),) + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + + x.src[2:]).cast(dtypes.bool) if idx.addrspace != AddrSpace.REG else None), (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), - lambda x,idx: UOp(x.op, src=(x.src[0], x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), # ptx shr and shl instructions require y to be uint (UPat.var("x") << UPat.var("y"), lambda x,y: UOp(Ops.SHL, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), (UPat.var("x") >> UPat.var("y"), lambda x,y: UOp(Ops.SHR, src=(x,y.cast(dtypes.uint))) if y.dtype != dtypes.uint else None), From cc72b9f7be253f42d858ea1a51d8b8e685f98535 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 26 Aug 2026 11:55:54 -0700 Subject: [PATCH 218/306] cleanup BENCHMARK_LOG (#17754) --- extra/bench_log.py | 18 +++++++++--------- test/testextra/test_bench_log.py | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/extra/bench_log.py b/extra/bench_log.py index 62695c69329e9..26bc55d8b6e11 100644 --- a/extra/bench_log.py +++ b/extra/bench_log.py @@ -35,7 +35,7 @@ def __enter__(self): return self def __exit__(self, *_): self.time = time.monotonic() - self.start - _events[self.event]["wall"].append(self.time) + _events[self.event]["wall"].append((self.time, BENCHMARK_LOG.value)) return False class KernelTimeEvent: @@ -47,19 +47,19 @@ def __enter__(self): self.start = GlobalCounters.time_sum_s return self def __exit__(self, *_): - _events[self.event]["kernel"].append(GlobalCounters.time_sum_s - self.start) + _events[self.event]["kernel"].append((GlobalCounters.time_sum_s - self.start, BENCHMARK_LOG.value)) return False def log_event_instant(event:InstantBenchEvent, value:float): - _events[event].append(value) + _events[event].append((value, BENCHMARK_LOG.value)) if BENCHMARK_LOG: INFLUXDB_HOST = getenv("INFLUXDB_HOST", "") INFLUXDB_ORG = getenv("INFLUXDB_ORG", "tiny") INFLUXDB_TOKEN = getenv("INFLUXDB_TOKEN", "") - def _create_point(run_id, i, attempt, ref, commit, name, value, run): - point = Point(BENCHMARK_LOG.value).tag("id", run_id).tag("index", i) + def _create_point(run_id, i, attempt, ref, commit, name, value, log_name, run): + point = Point(log_name.replace(':', '_').replace('.', '_')).tag("id", run_id).tag("index", i) point = point.tag("device", Device.DEFAULT) point = point.tag("attempt", attempt).tag("ref", ref).tag("commit", commit) point = point.field(name, value).field("x", run) @@ -91,12 +91,12 @@ def write_events(): run_id = str(uuid.uuid4()) if isinstance(event, BenchEvent): for event_type, values in _events[event].items(): - for i, value in enumerate(values): - point = _create_point(run_id, i, attempt, ref, commit, f"{event.value}_{event_type}", value, run) + for i, (value, log_name) in enumerate(values): + point = _create_point(run_id, i, attempt, ref, commit, f"{event.value}_{event_type}", value, log_name, run) points.append(point) else: - for i, value in enumerate(_events[event]): - point = _create_point(run_id, i, attempt, ref, commit, event.value, value, run) + for i, (value, log_name) in enumerate(_events[event]): + point = _create_point(run_id, i, attempt, ref, commit, event.value, value, log_name, run) points.append(point) write_options = WriteOptions(write_type=WriteType.synchronous, retry_interval=5000, max_retries=5, max_retry_delay=30000, exponential_base=2) diff --git a/test/testextra/test_bench_log.py b/test/testextra/test_bench_log.py index 975bef2fe1833..4067985a02034 100644 --- a/test/testextra/test_bench_log.py +++ b/test/testextra/test_bench_log.py @@ -21,7 +21,7 @@ def test_log_single_wall_time(self): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 1) - self.assertGreater(_events[event]["wall"][0], 0) + self.assertGreater(_events[event]["wall"][0][0], 0) def test_log_double_wall_time(self): for event in BenchEvent: @@ -35,8 +35,8 @@ def test_log_double_wall_time(self): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 2) - self.assertGreater(_events[event]["wall"][0], 0) - self.assertGreater(_events[event]["wall"][1], 0) + self.assertGreater(_events[event]["wall"][0][0], 0) + self.assertGreater(_events[event]["wall"][1][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci timing is not accurate") def test_log_single_kernel_time(self): @@ -52,8 +52,8 @@ def test_log_single_kernel_time(self): # check event list for event in BenchEvent: self.assertEqual(len(_events[event]["kernel"]), 1) - self.assertLess(_events[event]["kernel"][0], wall_times[0]) - self.assertGreater(_events[event]["kernel"][0], 0) + self.assertLess(_events[event]["kernel"][0][0], wall_times[0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci cuda timing is not accurate") def test_interleaved_wall_kernel_time(self): @@ -74,8 +74,8 @@ def test_interleaved_wall_kernel_time(self): for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 1) self.assertEqual(len(_events[event]["kernel"]), 1) - self.assertLess(_events[event]["kernel"][0], wall_times[0]) - self.assertGreater(_events[event]["kernel"][0], 0) + self.assertLess(_events[event]["kernel"][0][0], wall_times[0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) @skipIf(_SKIP_KERNEL_TIMING, "ci cuda timing is not accurate") def test_stacked_wall_kernel_time(self): @@ -93,10 +93,10 @@ def test_stacked_wall_kernel_time(self): for event in BenchEvent: self.assertEqual(len(_events[event]["wall"]), 2) self.assertEqual(len(_events[event]["kernel"]), 2) - self.assertLess(_events[event]["kernel"][0], _events[event]["wall"][0]) - self.assertGreater(_events[event]["kernel"][0], 0) - self.assertLess(_events[event]["kernel"][1], _events[event]["wall"][1]) - self.assertGreater(_events[event]["kernel"][1], 0) + self.assertLess(_events[event]["kernel"][0][0], _events[event]["wall"][0][0]) + self.assertGreater(_events[event]["kernel"][0][0], 0) + self.assertLess(_events[event]["kernel"][1][0], _events[event]["wall"][1][0]) + self.assertGreater(_events[event]["kernel"][1][0], 0) def test_log_instant_event(self): for event in InstantBenchEvent: @@ -105,7 +105,7 @@ def test_log_instant_event(self): # check event list for event in InstantBenchEvent: self.assertEqual(len(_events[event]), 1) - self.assertEqual(_events[event][0], 1000) + self.assertEqual(_events[event][0][0], 1000) if __name__ == '__main__': unittest.main() From 6042b8727250dbb7d2cc59ceea8d36b47e776a19 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:07:28 -0700 Subject: [PATCH 219/306] delete PCONTIG [PR] (#17756) * delete PCONTIG * cleanups --- test/backend/test_rangeify.py | 82 +++++------------------------------ tinygrad/helpers.py | 1 - tinygrad/schedule/indexing.py | 13 ++---- tinygrad/schedule/rangeify.py | 21 ++------- 4 files changed, 18 insertions(+), 99 deletions(-) diff --git a/test/backend/test_rangeify.py b/test/backend/test_rangeify.py index 011af5bf9e4d5..2eeba7a749958 100644 --- a/test/backend/test_rangeify.py +++ b/test/backend/test_rangeify.py @@ -1,6 +1,6 @@ import unittest -from tinygrad import Tensor, nn, Device, dtypes, Variable -from tinygrad.helpers import Context, GlobalCounters, getenv, PCONTIG, DEBUG +from tinygrad import Tensor, Device, dtypes, Variable +from tinygrad.helpers import Context, GlobalCounters, getenv, DEBUG from tinygrad.uop.ops import graph_rewrite, PatternMatcher, UPat, Ops, UOp from tinygrad.codegen.opt import OptOps, Opt from tinygrad.renderer.ptx import PTXRenderer @@ -14,7 +14,7 @@ def setUp(self): self.ref = (self.a @ self.b @ self.c).realize() def _test(self, opts): - with Context(PCONTIG=2, DEBUG=max(2, DEBUG.value)): + with Context(DEBUG=max(2, DEBUG.value)): out = (self.a @ self.b @ self.c).contiguous(arg=opts).realize() with Context(DEBUG=0): @@ -88,16 +88,15 @@ def test_matmul_relu_cat(self): res = Tensor.cat(a, c, dim=0) self.assertEqual(res.numpy()[-1, :16].tolist(), [512] * 16) - def test_pcontig_multi_gather(self): + def test_multi_gather(self): # regression test: local bufferize must have device set for const_like to work - with Context(PCONTIG=2): - # NOTE: with uint type, this will become a long and fail on WEBGPU - forest = Tensor(list(range(8)), dtype='int') - idx = Tensor([0, 0], dtype='int') - node_val = forest.gather(0, idx) - idx2 = idx * 2 + 1 - node_val2 = forest.gather(0, idx2) - result = (node_val + node_val2).numpy() + # NOTE: with uint type, this will become a long and fail on WEBGPU + forest = Tensor(list(range(8)), dtype='int') + idx = Tensor([0, 0], dtype='int') + node_val = forest.gather(0, idx) + idx2 = idx * 2 + 1 + node_val2 = forest.gather(0, idx2) + result = (node_val + node_val2).numpy() self.assertEqual(result.tolist(), [1, 1]) if getenv("BIG") > 2: @@ -118,65 +117,6 @@ def fa(): GlobalCounters.reset() return q.scaled_dot_product_attention(k, v) -def fa_bw(): - Tensor.manual_seed(1337) - with Context(DEBUG=0): - q,k,v = [Tensor.rand(BS, HEADS, SEQLEN, EMB).contiguous().realize() for _ in range(3)] - attn_output = nn.Linear(HEADS*EMB, HEADS*EMB, bias=False) - attn_output.weight.realize() - target = Tensor.rand(BS, SEQLEN, HEADS*EMB).contiguous().realize() - - GlobalCounters.reset() - attn = q.scaled_dot_product_attention(k, v).contiguous().contiguous_backward() - attn = attn.transpose(1, 2).reshape(BS, SEQLEN, -1) - out = attn_output(attn) - loss = (out - target).square().mean() - loss.backward() - #ret = [out, Tensor.stack(q.grad, k.grad, v.grad, dim=-1)] - #ret = [out, Tensor.stack(q.grad, k.grad, dim=-1), v.grad] - ret = [out, q.grad, k.grad, v.grad] - Tensor.realize(*ret) - return ret - -@unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, (NIRRenderer, PTXRenderer)), "broken in LVP and PTX") -class TestPcontig(unittest.TestCase): - def test_flash_attention_bw(self): - with Context(PCONTIG=max(2, PCONTIG.value), DEBUG=2): - grads = fa_bw() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - - with Context(PCONTIG=0, DEBUG=2): - cmp_grads = fa_bw() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - - with Context(DEBUG=0): - mses = [((x-y)**2).sum().item() for x,y in zip(grads, cmp_grads)] - mse = sum(mses) - print(f"mse: {mse}") - self.assertLessEqual(mse, 1e-6) - - def test_flash_attention(self, opts=None): - with Context(PCONTIG=2, DEBUG=max(2, DEBUG.value)): - ret = fa().realize() if opts is None else fa().contiguous(arg=opts).realize() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - with Context(DEBUG=2): - cmp = fa().realize() - print(f"{GlobalCounters.global_ops/1e9:.2f} GFLOPS") - with Context(DEBUG=0): - mse = ((cmp-ret)**2).sum().item() - print(f"mse: {mse}") - self.assertLessEqual(mse, 1e-6) - - def test_flash_attention_opt(self): - opts = () - # columns in top matrix - opts += (Opt(OptOps.UPCAST, 0, 4),) - # columns in bottom matrix - opts += (Opt(OptOps.UPCAST, 3, 4),) - # rows in all the matrix - opts += (Opt(OptOps.UPCAST, 4, 4),) - self.test_flash_attention(opts) - # contiguous + reduce can support ranges? @unittest.skip("pm_rangeify no longer exists. test this in a different way") diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 4c8bc4028ae26..471745973eaf2 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -271,7 +271,6 @@ def _get_cpu_count() -> int: SPEC = ContextVar("SPEC", 1) # TODO: disable by default due to speed CHECK_OOB = ContextVar("CHECK_OOB", 0) -PCONTIG = ContextVar("PCONTIG", 0) # partial contiguous in rangeify DEBUG_RANGEIFY = ContextVar("DEBUG_RANGEIFY", 0) # set to 1, this uses tuplize in the linearizer sort order TUPLE_ORDER = ContextVar("TUPLE_ORDER", 1) diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index 549e3fdae3724..cd1bf9d435f2a 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -5,7 +5,7 @@ from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, graph_rewrite, sint, AxisType, rewrite_group, broadcast_axes from tinygrad.uop.ops import gate_kernel_sink from tinygrad.uop.symbolic import symbolic, pm_simplify_valid, pm_drop_and_clauses -from tinygrad.helpers import argsort, all_same, cpu_profile, PCONTIG, colored, Context, SPEC +from tinygrad.helpers import argsort, all_same, cpu_profile, colored, Context, SPEC @dataclass class IndexingContext: @@ -248,13 +248,12 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: local_rngs, valids = zip(*[(r.get_idx(), r.get_valid()) for r in valid_rngs]) rngs_valids.append((local_rngs, valids)) - # TODO: in RANGEIFY > 1 all_all_same isn't required all_all_same = all(all_same(local_rngs) for local_rngs,_ in rngs_valids) _out_rngs = [] _realize_axis = [] for i,(local_rngs,valids) in enumerate(rngs_valids): # we compare the ranges without their valids - if all_all_same or (PCONTIG and all_same(local_rngs)): + if all_all_same: # the new valid is the OR of all the children valids minimum_valid = UOp.const(False).usum(valids) _out_rngs.append(graph_rewrite(local_rngs[0].valid(minimum_valid), symbolic, name="minimum_valid")) @@ -268,15 +267,11 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: # if this element is a reduce and there's ended ranges, we might have to end some other ranges if len(ending_ranges[x]) and x.op in GroupOp.Elementwise.union({Ops.REDUCE}): - _realize_axis = rctx.realize_map.get(x) or [] - for i,r in enumerate(out_rngs): - if i in _realize_axis: continue - if not (PCONTIG > 1) or any(any(rr.arg > e.arg for e in ending_ranges[x]) for rr in r.ranges): - _realize_axis.append(i) + _realize_axis = list(range(len(out_rngs))) ending_ranges[x] = [] if len(_realize_axis): rctx.realize_map[x] = _realize_axis - out_rngs = tuple([(rctx.new_range(x.shape[i]) if i in _realize_axis else r) for i,r in enumerate(out_rngs)]) + out_rngs = tuple(rctx.new_range(x.shape[i]) for i in range(len(out_rngs))) ending_ranges[x] += broadcast_ending_ranges # TODO: some ops don't have shape, enable this after the `.st` property is removed diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 5fc268a03c6cb..874b55624be57 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -6,7 +6,7 @@ from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import prod, dedup, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC -from tinygrad.helpers import PCONTIG, partition, get_single_element +from tinygrad.helpers import get_single_element from tinygrad.codegen.simplify import pm_flatten_range, pm_reduce_simplify from tinygrad.codegen.opt import Opt from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, apply_movement_op @@ -83,7 +83,7 @@ def red_gate(x:UOp): accessed_buffers = dedup(accessed_buffers) # if this is generated from multiple buffers, don't remove this buffer - if len(accessed_buffers) > 3 and not (PCONTIG > 2): return None + if len(accessed_buffers) > 3: return None # if any reduces access a buffer, don't remove this buffer buffer_in_reduce = False @@ -94,22 +94,7 @@ def buf_gate(x:UOp): UOp.sink(*[x.src[0] for x in reduces]).toposort(gate=buf_gate) del buf_gate if buffer_in_reduce: - if PCONTIG > 2: - out_in_ratio = (prod(buf.shape)+1) / (sum([x.numel() for x in accessed_buffers])+1) - if out_in_ratio < 10: return None - # here we have to check the indexes, we might do a partial contig here - local_indexes = [x for x in indexes if x.src[0].op is Ops.STAGE and x.src[0].arg.addrspace == AddrSpace.LOCAL] - exclude_ranges = UOp.group(*[UOp.group(*x.src[1:]) for x in local_indexes]).ranges - subs = [(k,v) for k,v in zip(buf.src[1:], idx.src[1:]) if k.op is not Ops.CONST] - # if it's bufferized or a reduce, it's pcontig - is_pcontig, is_subs = partition(subs, lambda x: x[0] in exclude_ranges or any([r.arg[-1] == AxisType.REDUCE for r in x[1].ranges])) - if not len(is_subs): - return None - if len(is_pcontig): - ret = src.substitute(dict(is_subs), extra_pm=pm_gate_substitute) - return ret.bufferize(*[x[0] for x in is_pcontig], arg=BufferizeOpts(None, AddrSpace.LOCAL)).index(*[x[1] for x in is_pcontig]) - else: - return None + return None # if it makes it here, the bufferize is removed # this is the ranges replaced From 6074c002e1edf04b56cfd972d4706114f1e1d1fb Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:36:36 +0300 Subject: [PATCH 220/306] hcq2: fix jit (#17747) * hcq2: fix reduce * Dx * inputs table * emoty commit --- tinygrad/engine/jit.py | 10 ++---- tinygrad/engine/realize.py | 14 ++++---- tinygrad/runtime/support/hcq2.py | 57 +++++++++++++++++--------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index 221889f7d677b..6918807bc2fa0 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -7,7 +7,7 @@ from tinygrad.uop.ops import UOp, PatternMatcher, Variable, sym_infer, Ops, buffers, rewrite_group, graph_rewrite from tinygrad.renderer import Estimates from tinygrad.engine.realize import capturing, compile_linear, link_linear, run_linear, graph_cache, estimate_uop, get_runtime -from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_outs_ins +from tinygrad.engine.realize import unwrap_multi, resolve_params, get_call_arg_uops, get_call_written_bufs from tinygrad.schedule.memory import memory_plan_rewrite, _collect_bufs from tinygrad.nn.state import get_parameters from tinygrad.uop.movement import mop_cleanup @@ -173,13 +173,7 @@ def __reduce__(self): return self.__class__, (self.ret, self._linear, self.expec @functools.cached_property def _written_uops(self) -> set[UOp]: - out: set[UOp] = set() - for call in self.linear.toposort(): - if call.op is not Ops.CALL: continue - arg_uops = get_call_arg_uops(call) - outs, ins = get_call_outs_ins(call) - out |= {b for k in set(outs) - set(ins) if (b:=u if (cv:=(u:=arg_uops[k]).contiguous_view()) is None else cv[0]).op is Ops.BUFFER} - return out + return {b for call in self.linear.toposort() if call.op is Ops.CALL for b in get_call_written_bufs(call)} def __call__(self, input_uops:list[UOp], var_vals:dict[str, int]) -> ReturnType: concrete = tuple(_copy_input(u) if u in self._written_uops else u for u in input_uops) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 85bb40d5ec52f..31b3a14b87349 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -2,7 +2,7 @@ from typing import cast, Iterator, Any, Sequence import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple, tqdm +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple, tqdm, dedup from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry @@ -26,6 +26,10 @@ def get_call_outs_ins(call:UOp) -> tuple[tuple[int, ...], tuple[int, ...]]: if ast.op is Ops.CUSTOM_FUNCTION and ast.arg == "encdec": return (0,), tuple(range(1, len(get_call_arg_uops(call)))) return (), () +def get_call_written_bufs(call:UOp) -> list[UOp]: + arg_uops, (outs, ins) = get_call_arg_uops(call), get_call_outs_ins(call) + return dedup([b for k in outs if k not in ins and (b:=u if (cv:=(u:=arg_uops[k]).contiguous_view()) is None else cv[0]).op is Ops.BUFFER]) + def get_call_kernels(call:UOp) -> list[tuple[str, UOp, tuple[str, Estimates, bytes]|None]]: if (ast:=call.src[0]).op is Ops.CUSTOM_FUNCTION and ast.arg == "hcq": return [(d, call, (name, estimates, profile_key)) for devices,name,estimates,_,profile_key in call.arg.aux.kernels for d in devices] @@ -213,14 +217,12 @@ def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: dev = cast(Any, Device[(info:= call.arg.aux).device[0]]) - addrs = [(b.bufs[j] if isinstance(b:=_resolve(ctx.input_uops[k], ctx.input_uops).buffer, MultiBuffer) else b).get_buf(dev_name).va_addr - for devs, idxs in info.input_idxs for j, dev_name in enumerate(devs) for k in idxs] + addrs = [cast(Buffer, _resolve(u, ctx.input_uops).buffer).get_buf(d).va_addr for d, u in info.input_addrs] dev.rt_buffer()._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) if info.inputs is not None: - tables = [UOp.from_buffer(dev.rt_buffer().view(len(idxs), dtypes.uint64, base + j*len(idxs)*8), HCQ_RUNTIME_DEV.value) - for devs, idxs in info.input_idxs for j in range(len(devs))] - call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*tables)}) + table = UOp.from_buffer(dev.rt_buffer().view(len(info.input_addrs), dtypes.uint64, base), HCQ_RUNTIME_DEV.value) + call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*[table]*len(info.device))}) exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer()._buf.va_addr + base}), call, ast) def _prof_tm(device:str, name:str, prof:tuple[int, ...], profile_key:bytes) -> float|None: diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f72dc3c656029..d8bd869e2355b 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -30,9 +30,8 @@ class HCQInfo: device:tuple[str, ...] estimates:Estimates = Estimates() - input_idxs:tuple[tuple[tuple[str, ...], tuple[int, ...]], ...] = () # per inputs table: (devices, indexes into input_uops) - inputs:int|None = None # index of the inputs table in call.src - # per kernel: (devices, name, estimates, timestamps, profile key) + inputs:int|None = None + input_addrs:tuple[tuple[str, UOp], ...] = () # (device, lane arg uop) kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...], bytes], ...] = () def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c @@ -44,6 +43,8 @@ def unwrap_mstack(u:UOp) -> tuple[UOp, ...]: def unwrap_view(v:UOp) -> tuple[UOp, int]: return unwrap_view(v.src[0]) if v.op is Ops.BITCAST else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) +def _lane(u:UOp, lane:int) -> UOp: return u.src[lane] if u.op is Ops.MSTACK else u.mselect(lane) if len(to_tuple(u.device)) > 1 else u + # patches def is_value_known_at_link(val:UOp) -> bool: @@ -154,11 +155,12 @@ def copy_with_kernel(call:UOp, dst:UOp, src:UOp) -> UOp|None: class HCQDepsTracker(DepsTracker): @staticmethod def _key(buf:Any) -> tuple[Any, int, int]: + if isinstance(buf, UOp) and buf.op is Ops.MSELECT: buf = buf.src[0] return (buf.arg.slot, 0, buf.max_numel() * buf.dtype.itemsize) if isinstance(buf, UOp) else DepsTracker._key(buf) def _get_call_bufs_by_lane(call:UOp, devices:tuple[str, ...]) -> list[list[Any]]: - refs = get_call_arg_uops(call) - return [[b if b.op is Ops.PARAM else mb.bufs[lane] if isinstance(mb:=b.buffer, MultiBuffer) else mb for b in refs] for lane in range(len(devices))] + return [[b if (b:=_lane(a, lane)).op is Ops.PARAM or (b.op is Ops.MSELECT and b.src[0].op is Ops.PARAM) else b.buffer + for a in get_call_arg_uops(call)] for lane in range(len(devices))] def _get_deps(ctx:DepsTracker, bufs_by_lane:list[list[Any]], write, key:tuple[tuple[str, ...], str, int]) -> list[tuple[tuple, int, int]]: dep_lanes:list[tuple[tuple, int, int]] = [] @@ -223,8 +225,8 @@ def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? if len(calls) == 1: return calls[0] devs, queue = get_submit(calls[0]).src[0].arg body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() - return make_call(f"submit {queue} ({len(calls)})", body, - replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) + return make_call(f"submit {queue} ({len(calls)})", body, replace(calls[0].arg.aux, + estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) def _merge_queues(submits:list[UOp]) -> list[UOp]: new_src:list[UOp] = [] @@ -325,18 +327,20 @@ def trim_link_patches(ctx:tuple[list[UOp], list[UOp]], a:UOp) -> UOp|None: return a.src[0].after(*kept, *[d for p in afters for d in p.src[1:]]) if links else None pm_trim_link_patches = PatternMatcher([(UPat(Ops.AFTER, src=(UPat((Ops.PARAM, Ops.MSTACK)),), allow_any_len=True, name="a"), trim_link_patches)]) -def make_addr_table(call:UOp, gaddrs:list[UOp], name:str) -> tuple[UOp, dict[UOp, UOp], tuple[UOp, ...], dict[UOp, int]]: +def _dnum(stride:int) -> UOp: return UOp.variable("_device_num", 0, stride - 1, dtypes.int, param=True) if stride > 1 else UOp.const(0, dtypes.int) + +def make_addr_table(call:UOp, gaddrs:list[UOp], name:str, stride:int=1) -> tuple[UOp, dict[UOp, UOp], tuple[UOp, ...], dict[UOp, int]]: bare = {g: g.replace(src=(g.src[0].without_after,)) for g in gaddrs} - order = sorted(dedup(bare.values()), key=lambda g: ((b:=unwrap_mstack(g.buf_uop)[0]).arg.slot, repr(b.tag))) - slots = {g:i for i,g in enumerate(order)} - table = UOp.placeholder((len(order),), dtypes.uint64, next(UOp.unique_num), device=call.arg.aux.device).rtag(name) + # slot-major layout: slot i of lane j lives at i*stride+j, every lane reads through the same table base + slots = {g:i*stride for i,g in enumerate(sorted(dedup(bare.values()), key=lambda g: g.key))} + table = UOp.placeholder((len(slots)*stride,), dtypes.uint64, next(UOp.unique_num), device=call.arg.aux.device).rtag(name) - reads = {g: table.after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()).index(UOp.const(slots[bare[g]], dtypes.int)).load() for g in gaddrs} - fills = (table.after(*make_patches(table, [(i*table.dtype.itemsize, addr) for addr, i in slots.items()])),) if slots else () + reads = {g: table.after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()).index(_dnum(stride) + slots[bare[g]]).load() for g in gaddrs} + fills = (table.after(*make_patches(table, [(i*table.dtype.itemsize, addr) for addr, i in slots.items()])),) if slots and stride == 1 else () return table, reads, fills, {g:slots[bare[g]] for g in gaddrs} -def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp]) -> dict[UOp, UOp]: +def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp], stride:int) -> dict[UOp, UOp]: (dst,), words = dedup(p.buf_uop for p in patches), [(unwrap_view(p.src[0].src[0])[1] + off.val*(val.dtype.itemsize//p.buf_uop.dtype.itemsize), slots[val]) for p in patches for off,val in zip(p.src[0].src[1].src, p.src[1].src)] @@ -344,13 +348,13 @@ def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patc pairs = UOp.placeholder((2*len(words),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words)))) r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst)) - off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()))) + off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()-(stride-1)))) # SHRINK(offset, length): a const length keeps the end bound from becoming an expression the program spec rejects patch = UOp(Ops.SHRINK, src=(dst, off, off.const_like(table.dtype.itemsize//dst.dtype.itemsize))).bitcast(table.dtype).index(0) \ - .store(table.index(slot).load()).end(r) + .store(table.index(slot + _dnum(stride)).load()).end(r) return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: patch} -def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) +def is_input_addr(g:UOp) -> bool: return any(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) def split_patches(call:UOp) -> UOp|None: rt_patches:list[UOp] = [] @@ -358,20 +362,22 @@ def split_patches(call:UOp) -> UOp|None: body = graph_rewrite(call.src[0], pm_trim_link_patches, ctx=(rt_patches, lt_patches), name=f"trim link-time patches ({call.arg.name})") # split patches. addresses read in the body go through the tables too + lanes = len(to_tuple(call.arg.aux.device)) inputs, internals = partition(dedup([g for p in rt_patches for g in get_getaddrs(p)] + get_getaddrs(body)), is_input_addr) runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) - tables = [make_addr_table(call, gs, n) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] + tables = [make_addr_table(call, gs, n, lanes if n == "inputs" else 1) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec ipatches = [p for p in rt_patches if p.tag == "inputs" and all(v in tables[0][3] for v in p.src[1].src)] # only getaddrs go to the table - gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches) if ipatches else {} + gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches, lanes) if ipatches else {} body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}).substitute(reads) lt_srcs = collections.defaultdict(list) for p in lt_patches: lt_srcs[p.buf_uop].append(p) - return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), - arg=replace(call.arg, aux=replace(call.arg.aux, input_idxs=((to_tuple(inputs[0].arg), - tuple(sorted(dedup(b.arg.slot for g in inputs for b in unwrap_mstack(g.buf_uop))))),) if inputs else call.arg.aux.input_idxs))) + + bufs = [u for _, u in sorted(dedup([(i, g.src[0].without_after) for g, i in tables[0][3].items()]))] + aux = replace(call.arg.aux, input_addrs=tuple((d, _lane(u, j)) for u in bufs for j,d in enumerate(call.arg.aux.device))) if inputs else call.arg.aux + return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), arg=replace(call.arg, aux=aux)) pm_split_patches = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), split_patches)]) # ***************** @@ -440,13 +446,12 @@ def _lane_arg(a:UOp, lane:int, table:UOp) -> UOp: return table if a.tag == "inpu def merge_batch(batch:list[UOp]) -> UOp: tables = UOp.variable("hcq_inputs_ptr", 0, 2**64-1, dtypes.uint64, param=True) - lanes = [(c, j, sum(len(idxs) * 8 for _, idxs in c.arg.aux.input_idxs)) for c in batch for j in range(len(c.arg.aux.device))] # (call, lane, bytes) - offs = itertools.accumulate((table_bytes for _, _, table_bytes in lanes), initial=0) # every lane owns the next table of the region + offs = itertools.accumulate((8 * len(c.arg.aux.input_addrs) for c in batch), initial=0) # every call owns the next table of the region cmds = [c.src[0].src[0].call(*[_lane_arg(a.without_after, j, tables + off) for a in c.src[1:]], UOp.variable("_device_num", 0, 1 << 30).bind(j)) - for (c, j, _), off in zip(lanes, offs)] + for c, off in zip(batch, offs) for j in range(len(c.arg.aux.device))] info = HCQInfo((HCQ_RUNTIME_DEV.value,), sum((c.arg.aux.estimates for c in batch), start=Estimates()).simplify(), - input_idxs=tuple(x for c in batch for x in c.arg.aux.input_idxs), kernels=tuple(k for c in batch for k in c.arg.aux.kernels)) + input_addrs=tuple(x for c in batch for x in c.arg.aux.input_addrs), kernels=tuple(k for c in batch for k in c.arg.aux.kernels)) body = UOp.custom_function("hcq", make_submit(*cmds, devs=HCQ_RUNTIME_DEV.value, queue="SUBMIT:0").sink()) return body.call(*[s for c in batch for s in c.src[1:] if s.without_after.tag != "inputs"], name=f"hcq_submitter ({len(batch)})", aux=info) From c015351ac5c00c10c58dbdaf530ef5b2883ab948 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 26 Aug 2026 13:53:30 -0700 Subject: [PATCH 221/306] fix _get_cpu_count for docker --cpus=N in python 3.13+ (#17760) --- tinygrad/helpers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 471745973eaf2..6b9276a2fae61 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -250,16 +250,16 @@ def target(self, dev:str, **kwargs) -> Target: DEFAULT_FLOAT, DEFAULT_INT = ContextVar("DEFAULT_FLOAT", "float32"), ContextVar("DEFAULT_INT", "int32") CAPTURE_PROCESS_REPLAY = ContextVar("CAPTURE_PROCESS_REPLAY", 0) def _get_cpu_count() -> int: - # os.process_cpu_count (3.13+) respects cgroup limits - if hasattr(os, "process_cpu_count"): return max(1, os.process_cpu_count() or 1) - # cgroup v2 (containers with --cpus=N) + # os.process_cpu_count is available in 3.13+, then try affinity, then fallback to cpu_count + count = (os.process_cpu_count() if hasattr(os, "process_cpu_count") else + len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count()) or 1 + # limit with cgroup v2 (containers with --cpus=N) try: with open("/sys/fs/cgroup/cpu.max") as f: quota, period = f.read().strip().split() - if quota != "max": return max(1, int(quota) // int(period)) + if quota != "max": count = min(count, max(1, int(quota) // int(period))) except (FileNotFoundError, ValueError, ZeroDivisionError): pass - # fall back to affinity (respects taskset but not cgroup quota) - return max(1, len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)) + return count NUM_CPU_THREADS = ContextVar("NUM_CPU_THREADS", _get_cpu_count()) NULL_ALLOW_COPYOUT = ContextVar("NULL_ALLOW_COPYOUT", 0) # VIZ implies PROFILE, but you can run PROFILE without VIZ From a0a901c8e4af76975ea288348a0d5a0c4870459b Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:23:25 -0700 Subject: [PATCH 222/306] faster qwen 3.8 (#17720) * faster qwen * test fix * dead code * fix gguf issue * pretty nt loads * lil * use warp --- test/unit/test_attention.py | 2 +- test/unit/test_llm_amd.py | 8 +- tinygrad/codegen/late/coalesce.py | 9 +- tinygrad/llm/kernels/amd.py | 363 +++++++++++++++++++++--------- tinygrad/renderer/cstyle.py | 11 +- tinygrad/uop/ops.py | 3 +- 6 files changed, 281 insertions(+), 115 deletions(-) diff --git a/test/unit/test_attention.py b/test/unit/test_attention.py index 2efa3afe5059e..6ed281b7eec29 100644 --- a/test/unit/test_attention.py +++ b/test/unit/test_attention.py @@ -16,7 +16,7 @@ def apply_rope(x:Tensor, start_pos:int): class TestLinear(unittest.TestCase): def test_recovers_packed_ggml_weight(self): - for ggml_type,packed_size,words in ((13, 176, 44), (14, 210, 210), (23, 136, 34)): + for ggml_type,packed_size,words in ((13, 176, 44), (14, 210, 53), (23, 136, 34)): packed = Tensor.empty(packed_size+4, dtype=dtypes.uint8, device="CPU")[4:] decoded = ggml_data_to_tensor(packed, 256, ggml_type).reshape(1, 256) linear = Linear(256, 1, bias=False) diff --git a/test/unit/test_llm_amd.py b/test/unit/test_llm_amd.py index eaf08ab233178..2aa4cce2f8e99 100644 --- a/test/unit/test_llm_amd.py +++ b/test/unit/test_llm_amd.py @@ -20,11 +20,13 @@ def test_word_quant_weights_use_typed_buffer_view(self): def test_values_and_scales(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") x = np.linspace(-3.1, 2.7, 64, dtype=np.float32).reshape(2, 32) - quant, scale = q8_quantize(Tensor(x), 2, 32) + quant, scale, gsum = q8_quantize(Tensor(x), 2, 32) scale_np = np.maximum(np.max(np.abs(x), axis=-1, keepdims=True) / 127, 1e-8) expected = np.clip(np.rint(x / scale_np), -127, 127).astype(np.int8) np.testing.assert_array_equal(quant.bitcast(dtypes.int8).reshape(2, 32).numpy(), expected) np.testing.assert_allclose(scale.numpy(), scale_np, rtol=1e-6) + # xsum holds the two per-16 sums per 32-wide group + np.testing.assert_array_equal(gsum.numpy().reshape(2, 2), expected.reshape(2, 2, 16).sum(-1).astype(np.float32)) def test_q6_linear_compiles(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") @@ -36,7 +38,9 @@ def test_q6_linear_compiles(self): linear = Linear(256, 1, bias=False) nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) self.assertTrue(np.isfinite(linear(Tensor.randn(1, 256)).realize().item())) - self.assertEqual(linear.weight.uop.buf_uop.buffer.offset, 4) + # the Q6 weight is repacked: 210-byte blocks padded to 212 (one block = 53 words) + self.assertEqual(linear.weight.uop.buf_uop.buffer.nbytes, 53*4) + self.assertEqual(linear.weight.dtype, dtypes.uint32) def test_q4_k_linear(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index 305de98153b73..4b1da0d785979 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -103,7 +103,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: if getenv("DMC"): return sink # collect - memory: defaultdict[tuple[Ops, UOp, UOp|str, UOp], dict[int, list[UOp]]] = defaultdict(dict) + memory: defaultdict[tuple[Ops, UOp, UOp|str, UOp, object], dict[int, list[UOp]]] = defaultdict(dict) for u in sink.toposort(): # TODO: this should handle images too, it's just memory coalescing if u.op in {Ops.LOAD, Ops.STORE}: @@ -118,11 +118,12 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: elif idx.op is Ops.CONST and idx.val is Invalid: root_src, arg = "INVALID", 0 elif idx.op is Ops.CONST: root_src, arg = "CONST", idx.val else: root_src, arg = idx, 0 - memory[(u.op, buf, root_src, valid)].setdefault(arg, []).append(u) + # loads/stores only coalesce with others carrying the same arg (e.g. the nontemporal flag) + memory[(u.op, buf, root_src, valid, u.arg)].setdefault(arg, []).append(u) # build replacements replacements = {} - for (op,buf,base,valid),offsets in memory.items(): + for (op,buf,base,valid,ld_arg),offsets in memory.items(): # allowed lengths (copied in) lengths = [] must_divide = True @@ -157,7 +158,7 @@ def memory_coalescing(sink:UOp, ctx:Renderer) -> UOp: store = idx.store(UOp.stack(*datas) if len(datas) > 1 else datas[0]) for i,g in enumerate(grp): replacements[offsets[g][0]] = store else: - ld = idx.load() + ld = idx.load(arg=ld_arg) for i,g in enumerate(grp): for oo in offsets[g]: replacements[oo] = ld.index(i) if len(grp) > 1 else ld diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py index ecb4d5a79ee63..ef8a715e5e37f 100644 --- a/tinygrad/llm/kernels/amd.py +++ b/tinygrad/llm/kernels/amd.py @@ -7,12 +7,13 @@ from tinygrad.helpers import prod from tinygrad.uop.ops import AxisType, KernelInfo, Ops, resolve -BLOCK_M, BLOCK_N, DECODE_HEAD_TILE, WARP_SIZE = 32, 32, 8, 32 +BLOCK_M, BLOCK_N, WARP_SIZE = 32, 32, 32 WMMA_M, WMMA_N, WMMA_K = 16, 16, 16 WAVES_M, WAVES_N, LANES_PER_WAVE_M, LANES_PER_WAVE_N = 2, 2, 2, 16 WMMA_ACC, THREADS_PER_BLOCK = WMMA_M // LANES_PER_WAVE_M, WARP_SIZE * WAVES_M * WAVES_N LDS_PAD, WMMA_ARG, LOG2E = 4, ((WMMA_M, WMMA_N, WMMA_K), 'AMD', 32), math.log2(math.e) Q4_K, Q5_K, Q6_K, IQ4_XS, GGML_BLOCK_SIZE, Q8_GROUP_SIZE, Q4_WORDS, Q5_WORDS, Q6_BYTES, IQ4_WORDS = 12, 13, 14, 23, 256, 32, 36, 44, 210, 34 +Q6_PADDED, Q6_WORDS = 212, 53 # the 210-byte Q6 blocks are padded to 212 bytes so they are word-addressable QUANT_SIZES = {Q4_K: Q4_WORDS*4, Q5_K: Q5_WORDS*4, Q6_K: Q6_BYTES, IQ4_XS: IQ4_WORDS*4} # bytes per 256-weight block def kernel_var(x:UOp) -> UOp: @@ -54,21 +55,43 @@ def __init__(self, in_features:int, out_features:int, bias=True): self.in_features, self.out_features = in_features, out_features def set_quantized(self, decoded:Tensor): packed_sizes = {decoded.numel() // 256 * type_size:typ for typ,type_size in QUANT_SIZES.items()} - raw = next((u for u in decoded.uop.toposort() if u.op is Ops.SHRINK and u.dtype == dtypes.uint8 and prod(u.shape) in packed_sizes), None) + graph = decoded.uop.toposort() + raw = next((u for u in graph if u.op is Ops.SHRINK and u.dtype == dtypes.uint8 and prod(u.shape) in packed_sizes), None) if raw is None: return + ggml_type = packed_sizes[prod(raw.shape)] + # the packed byte rate alone can't distinguish same-rate formats (Q4_0 vs Q4_K, Q5_0 vs Q5_K, MXFP4 vs IQ4_XS). + # the supported formats are 256-wide superblocks: their decode views the packed bytes at the superblock width + # (ggml_data_to_tensor reshapes to (-1, QUANT_SIZES[type])), while same-rate 32-wide formats reshape to 17-22 + if not any(u.op is Ops.RESHAPE and u.shape[-1:] == (QUANT_SIZES[ggml_type],) for u in graph): return raw_offset = raw.contiguous_view_offset() assert raw_offset is not None and raw_offset % 4 == 0 and raw.buf_uop.dtype == dtypes.uint8 - self.ggml_type = packed_sizes[prod(raw.shape)] + self.ggml_type = ggml_type # store a typed buffer view: a lazy BITCAST is decomposed into byte-combining ALU before custom-kernel # scheduling and would copy the entire packed weight on every JIT graph - packed_dtype = dtypes.uint8 if self.ggml_type == Q6_K else dtypes.uint32 - self.weight = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer) - .view(raw.max_numel() * raw.dtype.itemsize // packed_dtype.itemsize, packed_dtype, raw_offset))) + if self.ggml_type == Q6_K: + # Q6 blocks are 210 bytes, so consecutive blocks are only 2-byte aligned. pad each block to 212 bytes + # (a one-time copy at load) so the kernel can do all its reads as aligned u32 words + nbytes, nblocks = raw.max_numel(), raw.max_numel() // Q6_BYTES + byte_view = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer).view(nbytes, dtypes.uint8, raw_offset))) + padded = byte_view.reshape((nblocks, Q6_BYTES)).pad_to((nblocks, Q6_PADDED)).contiguous().realize() + self.weight = Tensor(UOp.from_buffer(cast(Buffer, padded.uop.buf_uop.buffer).view(nblocks * Q6_WORDS, dtypes.uint32, 0))) + else: + self.weight = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer) + .view(raw.max_numel() * raw.dtype.itemsize // dtypes.uint32.itemsize, dtypes.uint32, raw_offset))) def __call__(self, x:Tensor) -> Tensor: supported = self.use_custom_quant and amd_custom_kernels_supported(self.weight.device) if self.ggml_type is None and supported: self.set_quantized(self.weight) - if self.ggml_type is None: self.use_custom_quant = supported = False # not a supported quant format + if self.ggml_type is None: + # tiny dense fp16 matmul (e.g. the ssm beta/alpha head rows): single fp16 gemv kernel instead of a + # generic matmul schedule, and realize the densely packed weight once if it is still a lazy ggml view + if self.weight.dtype in (dtypes.half, dtypes.float, dtypes.bfloat16) and self.out_features <= 2048 \ + and self.in_features % (WARP_SIZE*4) == 0: + numel, max_shape = x.numel(), x.max_shape + if isinstance(numel, int) or prod(max_shape) // self.in_features <= 32: + out = f16_gemv(self, x if isinstance(numel, int) else x.pad_to(max_shape)) + return out if isinstance(numel, int) else out.shrink(tuple((0, s) for s in (*x.shape[:-1], self.out_features))) + self.use_custom_quant = supported = False # not a supported quant format if self.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) and supported: if isinstance(x.numel(), int): return q8_linear(self, x) # symbolic token count: pad to the max chunk size so the kernels see static shapes, garbage rows are sliced off @@ -77,14 +100,20 @@ def __call__(self, x:Tensor) -> Tensor: return super().__call__(x) def _amd_dp4a(a:UOp, b:UOp, c:UOp) -> UOp: - return UOp(Ops.CUSTOMI, src=(a.int(), b.int(), c), arg=("__builtin_amdgcn_sudot4(true, {}, true, {}, {}, false)", dtypes.int32)) + # int8 4-wide dot, widened to scalar multiply-adds (2% decode slower than the sudot4 builtin, but portable) + for i in range(4): + av = ((a >> (8*i)) & 255).cast(dtypes.uint8).bitcast(dtypes.int8).int() + bv = ((b >> (8*i)) & 255).cast(dtypes.uint8).bitcast(dtypes.int8).int() + c = c + av*bv + return c def _amd_byte_perm(a:UOp, b:UOp, selectors:UOp) -> UOp: return UOp(Ops.CUSTOMI, src=tuple(x.cast(dtypes.uint32) for x in (a, b, selectors)), arg=("__builtin_amdgcn_perm({}, {}, {})", dtypes.uint32)) def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: assert ptr.op is Ops.INDEX - if lanes is None: return UOp(Ops.CUSTOMI, src=(ptr,), arg=("__builtin_nontemporal_load({0})", ptr.dtype)) + # nontemporal scalar load: streamed weights must not evict the activations/KV cache from L2 + if lanes is None: return ptr.load(arg="nontemporal") buf, coords = ptr.src[0], ptr.src[1:] idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0)) return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load(dtype=ptr.dtype) @@ -93,16 +122,19 @@ def _load_byte(raw:UOp, base:UOp, offset:UOp) -> UOp: return (raw[base + offset/ def _half(value:UOp) -> UOp: return value.cast(dtypes.uint16).bitcast(dtypes.float16).float() def _iq4_bytes(packed:UOp, shift:int) -> UOp: + # the non-linear iq4nl table as a byte lookup: 3 byte_perms beat any arithmetic/select-tree form (~60% decode) selectors = (packed >> shift) & 0x0f0f0f0f low = _amd_byte_perm(UOp.const(0xf6eaddcf, dtypes.uint32), UOp.const(0xbfad9881, dtypes.uint32), selectors) high = _amd_byte_perm(UOp.const(0x71594535, dtypes.uint32), UOp.const(0x26190d01, dtypes.uint32), selectors & 0x07070707) return _amd_byte_perm(high, low, 0x03020100 | ((selectors & 0x08080808) >> 1)) def _q5_scales(raw:UOp, base:UOp, subgroup:UOp) -> tuple[UOp, UOp, UOp, UOp]: - scale = (subgroup < 4).where(_load_byte(raw, base, 4 + subgroup) & 63, - (_load_byte(raw, base, 8 + subgroup) & 15) | ((_load_byte(raw, base, subgroup) >> 6) << 4)) - minimum = (subgroup < 4).where(_load_byte(raw, base, 8 + subgroup) & 63, - (_load_byte(raw, base, 8 + subgroup) >> 4) | ((_load_byte(raw, base, 4 + subgroup) >> 6) << 4)) + # scales/mins (6-bit each) live in block bytes 4-15: three words total, same for the whole super-block's lanes + w1, w2, w3 = _amd_load(raw[base+1]), _amd_load(raw[base+2]), _amd_load(raw[base+3]) + sb = (subgroup & 3) * 8 # byte within word + byte1, byte2, byte3 = (w1 >> sb) & 255, (w2 >> sb) & 255, (w3 >> sb) & 255 + scale = (subgroup < 4).where(byte1 & 63, (byte3 & 15) | ((byte1 >> 6) << 4)) + minimum = (subgroup < 4).where(byte2 & 63, (byte3 >> 4) | ((byte2 >> 6) << 4)) d, dmin = (raw[base] & 0xffff).cast(dtypes.uint16), (raw[base] >> 16).cast(dtypes.uint16) return _half(d), _half(dmin), scale.float(), minimum.float() @@ -118,52 +150,66 @@ def iq4_half_lut(device:str) -> Tensor: dtype=dtypes.float16, device=device).bitcast(dtypes.uint32).contiguous() @functools.cache -def _q8_quantize_kernel(q:UOp, scale:UOp, x:UOp, tokens:int, in_features:int) -> UOp: +def _q8_quantize_kernel(q:UOp, scale:UOp, xsum:UOp, x:UOp, tokens:int, in_features:int) -> UOp: groups = in_features//Q8_GROUP_SIZE - token_group, lane = UOp.range(tokens*groups, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) + token_group, lane = UOp.range(tokens*groups, 0, axis_type=AxisType.GLOBAL), UOp.range(32, 1, axis_type=AxisType.LOCAL) token, group = token_group//groups, token_group%groups x = x.reshape(tokens, groups, 32) group_scale = (warp_reduce(x[token, group, lane].float().abs(), maximum=True, full_wave=True) / 127).maximum(1e-8) word_lane = lane.minimum(7) xs = tuple(x[token, group, word_lane*4+i].float() for i in range(4)) - word = sum(((v/group_scale).round().clip(-127, 127).cast(dtypes.int8).cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) - for i,v in enumerate(xs)), UOp.const(0, dtypes.uint32)) - stores = (q[token, group, lane.valid(lane < 8)].store(word), scale[token, group.valid(lane.eq(0))].store(group_scale)) + qs = tuple((v/group_scale).round().clip(-127, 127).cast(dtypes.int8) for v in xs) + word = sum((v.cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) for i, v in enumerate(qs)), UOp.const(0, dtypes.uint32)) + # per-16 sums of the quantized values (lanes 0-3 / 4-7): Q4_K/Q5_K need the 32-sum, Q6_K the 16-sums + part = (lane < 8).where(sum((v.cast(dtypes.int32) for v in qs), UOp.const(0, dtypes.int32)), UOp.const(0, dtypes.int32)) + gsum = [warp_reduce(((lane & 4).eq(h*4)).where(part, UOp.const(0, dtypes.int32)), full_wave=True) for h in range(2)] + store_half = (lane & 4) >> 2 + stores = (q[token, group, lane.valid(lane < 8)].store(word), + UOp.group(scale[token, group.valid(lane.eq(0))].store(group_scale), + xsum[token, group, store_half.valid(lane.eq(0) | lane.eq(4))].store( + store_half.eq(0).where(gsum[0].float(), gsum[1].float())))) return UOp.group(*stores).end(token_group, lane).sink(arg=KernelInfo(name="q8_quantize", opts_to_apply=())) -def q8_quantize(x:Tensor, tokens:int, in_features:int) -> tuple[Tensor, Tensor]: +def q8_quantize(x:Tensor, tokens:int, in_features:int) -> tuple[Tensor, Tensor, Tensor]: groups = in_features//Q8_GROUP_SIZE q = Tensor.empty(tokens, groups, 8, dtype=dtypes.uint32, device=x.device) scale = Tensor.empty(tokens, groups, dtype=dtypes.float32, device=x.device) - q, scale = Tensor.custom_kernel(q, scale, x, fxn=functools.partial(_q8_quantize_kernel, tokens=tokens, in_features=in_features))[:2] - return q, scale + xsum = Tensor.empty(tokens, groups, 2, dtype=dtypes.float32, device=x.device) + q, scale, xsum = Tensor.custom_kernel(q, scale, xsum, x, fxn=functools.partial(_q8_quantize_kernel, tokens=tokens, in_features=in_features))[:3] + return q, scale, xsum def _decode_linear(out:UOp, out_features:int, group_count:int, group_dot, name:str) -> UOp: - chunks = (group_count+31)//32 - token_output_chunk, lane = UOp.range(out.shape[0]*out_features*chunks, 0), UOp.range(32, 1, axis_type=AxisType.LOCAL) - token, output, chunk = token_output_chunk // (out_features*chunks), (token_output_chunk//chunks) % out_features, token_output_chunk % chunks - group = lane+chunk*32 - value = group_dot(token, output, group) if group_count % 32 == 0 else \ - (group < group_count).where(group_dot(token, output, group.minimum(group_count-1)), UOp.const(0, dtypes.float32)) + chunks = out.shape[2] + # two-dim global grid instead of one flat grid: no div/mods needed to decompose the gid + token_output = UOp.range(out.shape[0]*out_features, 0, axis_type=AxisType.GLOBAL) + chunk, lane = UOp.range(chunks, 1, axis_type=AxisType.GLOBAL), UOp.range(32, 2, axis_type=AxisType.LOCAL) + token, output = token_output // out_features, token_output % out_features + group = (lane+chunk*32).minimum(group_count-1) + value = group_dot(token, output, group) if chunks*32 == group_count else \ + (lane+chunk*32 < group_count).where(group_dot(token, output, group), UOp.const(0, dtypes.float32)) total = warp_reduce(value, full_wave=True) - return out[token, output, chunk.valid(lane.eq(0))].store(total.cast(out.dtype)).end(token_output_chunk, lane).sink( + return out[token, output, chunk.valid(lane.eq(0))].store(total.cast(out.dtype)).end(token_output, chunk, lane).sink( arg=KernelInfo(name=name, opts_to_apply=())) @functools.cache -def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: +def _quant_decode_kernel(out:UOp, raw:UOp, xq:UOp, xd:UOp, xs:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: group_count = in_features // Q8_GROUP_SIZE def group_dot(token:UOp, output:UOp, group:UOp) -> UOp: block, subgroup = group // 8, group % 8 xwords = _amd_load(xq[token, group, 0], 8) if ggml_type in (Q4_K, Q5_K): base = (output * in_features//GGML_BLOCK_SIZE + block) * (Q4_WORDS if ggml_type == Q4_K else Q5_WORDS) - qs_base, dot, qsum = base + (4 if ggml_type == Q4_K else 12) + (subgroup//2)*8, UOp.const(0, dtypes.int32), UOp.const(0, dtypes.int32) + qs_base, dot = base + (4 if ggml_type == Q4_K else 12) + (subgroup//2)*8, UOp.const(0, dtypes.int32) + # vectorize the 8 packed-weight words and (for Q5_K) the 32-byte high-bit bitmap + qs_pair = (_amd_load(raw[qs_base], 4), _amd_load(raw[qs_base+4], 4)) + if ggml_type == Q5_K: qh_pair = (_amd_load(raw[base+4], 4), _amd_load(raw[base+8], 4)) for word_idx in range(8): - word = (raw[qs_base+word_idx] >> ((subgroup&1)*4).cast(dtypes.uint32)) & 0x0f0f0f0f - if ggml_type == Q5_K: word |= ((raw[base+4+word_idx] >> subgroup.cast(dtypes.uint32)) & 0x01010101) << 4 - dot, qsum = _amd_dp4a(word, xwords[word_idx], dot), _amd_dp4a(UOp.const(0x01010101, dtypes.uint32), xwords[word_idx], qsum) + word = (qs_pair[word_idx//4][word_idx%4] >> ((subgroup&1)*4).cast(dtypes.uint32)) & 0x0f0f0f0f + if ggml_type == Q5_K: word |= ((qh_pair[word_idx//4][word_idx%4] >> subgroup.cast(dtypes.uint32)) & 0x01010101) << 4 + dot = _amd_dp4a(word, xwords[word_idx], dot) d, dmin, scale, minimum = _q5_scales(raw, base, subgroup) - return (dot.float()*d*scale - qsum.float()*dmin*minimum) * xd[token, group] + gsum = xs[token, group, 0].load() + xs[token, group, 1].load() + return (dot.float()*d*scale - gsum*dmin*minimum) * xd[token, group] if ggml_type == IQ4_XS: base = (output * in_features//GGML_BLOCK_SIZE + block) * IQ4_WORDS dot = UOp.const(0, dtypes.int32) @@ -172,45 +218,57 @@ def group_dot(token:UOp, output:UOp, group:UOp) -> UOp: dot = _amd_dp4a(_iq4_bytes(packed, 4*(word_idx//4)), xwords[word_idx], dot) d, scale = _iq4_scales(raw, base, subgroup) return dot.float() * xd[token, group] * d * scale - base = (output*in_features//GGML_BLOCK_SIZE+block)*Q6_BYTES + # the packed rows were padded to 212 bytes (53 words) per 256-block in set_quantized: everything is word-aligned + base = (output*in_features//GGML_BLOCK_SIZE+block)*Q6_WORDS + # the subgroup's 8 ql words and 8 qh words are contiguous: two 16-byte vector loads each + lows = tuple(_amd_load(raw[base + (subgroup//4)*16 + (subgroup%2)*8 + half*4], 4) for half in range(2)) + highs = tuple(_amd_load(raw[base + 32 + (subgroup//4)*8 + half*4], 4) for half in range(2)) dots = [UOp.const(0, dtypes.int32)] * 2 for word_idx in range(8): - pos, within = subgroup*32 + word_idx*4, (subgroup*32 + word_idx*4)%128 - low = _amd_load(raw[base + (pos//128)*64 + within%64], 4) >> ((within//64)*4).cast(dtypes.uint8) - high = _amd_load(raw[base + 128 + (pos//128)*32 + within%32], 4) >> ((within//32)*2).cast(dtypes.uint8) - quant = ((low & 15) | ((high & 3) << 4)).bitcast(dtypes.int8) - 32 - word = sum((quant[i].cast(dtypes.uint8).cast(dtypes.uint32) << (i*8) for i in range(4)), UOp.const(0, dtypes.uint32)) + within = (subgroup*32 + word_idx*4)%128 + low = lows[word_idx//4][word_idx%4] >> ((within//64)*4).cast(dtypes.uint32) + high = highs[word_idx//4][word_idx%4] >> ((within//32)*2).cast(dtypes.uint32) + # 4 values per word: (low nibble) | (2 high bits << 4). values stay positive, so the int8-bitcast/-32 of the + # naive dequant is skipped and the -32 offset is applied later via the per-16 sums of the quantized inputs + word = (low & 0x0f0f0f0f) | ((high & 0x03030303) << 4) dots[word_idx//4] = _amd_dp4a(word, xwords[word_idx], dots[word_idx//4]) - scales = [raw[base + 192 + subgroup*2+i].cast(dtypes.uint8).bitcast(dtypes.int8).float() for i in range(2)] - dbits = raw[base+208].cast(dtypes.uint16) | (raw[base+209].cast(dtypes.uint16) << 8) - return (dots[0].float()*scales[0] + dots[1].float()*scales[1]) * xd[token, group] * _half(dbits) + scales = [((raw[base + 48 + (subgroup*2+i)//4] >> (((subgroup*2+i)%4)*8).cast(dtypes.uint32)) & 255) + .cast(dtypes.uint8).bitcast(dtypes.int8).float() for i in range(2)] + gsum = [xs[token, group, i].load() * 32 for i in range(2)] + return ((dots[0].float() - gsum[0])*scales[0] + (dots[1].float() - gsum[1])*scales[1]) * xd[token, group] * _half(raw[base+52] & 0xffff) names = {Q4_K: "linear_q4_k", Q5_K: "linear_q5_k", IQ4_XS: "linear_iq4_xs", Q6_K: "linear_q6"} return _decode_linear(out, out_features, group_count, group_dot, names[ggml_type]) def _wmma_layout(out:UOp, out_features:int, token_tile:int, output_tiles:int): output_waves = 2 if out_features % (32*output_tiles) == 0 else 1 token_block, output_block = UOp.range(out.shape[0]//token_tile, 0), UOp.range(out_features//(16*output_tiles*output_waves), 1) - lane, wave = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) - hw_lane = UOp(Ops.CUSTOM, src=(lane.int(),), arg=("__builtin_amdgcn_mbcnt_lo(-1, 0)", dtypes.int32)).cast(dtypes.weakint) - col, half = hw_lane % 16, hw_lane // 16 + # lane is a hardware WARP range (like the flash kernel): the fragment math stays visible without being + # range-split into nested loops, which would scramble the WMMA fragment layout + lane, wave = UOp.range(WARP_SIZE, -1, axis_type=AxisType.WARP), UOp.range(output_waves, 3, axis_type=AxisType.LOCAL) + col, half = lane % 16, lane // 16 outputs = tuple((output_block*output_waves+wave)*(16*output_tiles) + tile*16 + col for tile in range(output_tiles)) inputs = tuple(token_block*token_tile + tile*16 + col for tile in range(token_tile//16)) tokens = tuple(tuple(token_block*token_tile + tile*16 + half*8 + i for i in range(8)) for tile in range(token_tile//16)) return output_waves, token_block, output_block, lane, wave, half, outputs, inputs, tokens -def _wmma_stores(out, outputs, tokens, accs, update, half): - def values(acc:UOp) -> tuple[UOp, ...]: - vals = tuple(acc.after(update)[i].load() for i in range(8)) - swapped = tuple(UOp(Ops.CUSTOM, src=(value,), - arg=("__builtin_bit_cast(float, __builtin_amdgcn_ds_swizzle(__builtin_bit_cast(int, {0}), 50688))", dtypes.float32)) for value in vals) +def _wmma_stores(out, outputs, tokens, accs, update, half, lane, wave, output_waves): + # the accumulator fragment halves are exchanged between lane pairs (l, l^16) through LDS (a ds_swizzle without CUSTOM) + flat_accs = [acc for output_accs in accs for acc in output_accs] + lds = UOp.placeholder((output_waves, 32, len(flat_accs)*8), dtypes.float32, slot=33, addrspace=AddrSpace.LOCAL) + stores = [lds[wave, lane, a*8+i].store(acc.after(update)[i].load()) for a,acc in enumerate(flat_accs) for i in range(8)] + lds = lds.after(UOp.barrier(UOp.group(*stores))) + def values(ai:int) -> tuple[UOp, ...]: + own = tuple(lds[wave, lane, ai*8+i].load() for i in range(8)) + peer = tuple(lds[wave, lane ^ 16, ai*8+i].load() for i in range(8)) low = half.eq(0) - return tuple(low.where(vals[i], swapped[i+4]) if j == 0 else low.where(swapped[i], vals[i+4]) for i in range(4) for j in range(2)) - return [out[token, output].store(value) for output,output_accs in zip(outputs, accs) - for tile_tokens,acc in zip(tokens, output_accs) for token,value in zip(tile_tokens, values(acc))] + return tuple(low.where(own[i], peer[i+4]) if j == 0 else low.where(peer[i], own[i+4]) for i in range(4) for j in range(2)) + tt = len(tokens) + return [out[token, output].store(value) for ot,(output,output_accs) in enumerate(zip(outputs, accs)) + for tile,(tile_tokens,_acc) in enumerate(zip(tokens, output_accs)) for token,value in zip(tile_tokens, values(ot*tt+tile))] def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, dequant, name): x = x.reshape(out.shape[0], in_features) - _, token_block, output_block, lane, wave, physical_half, outputs, input_tokens, tokens = layout + output_waves, token_block, output_block, lane, wave, physical_half, outputs, input_tokens, tokens = layout token_tile, output_tiles = len(tokens)*16, len(outputs) output_words = in_features // GGML_BLOCK_SIZE * type_words accs = tuple(tuple(UOp.placeholder((8,), dtypes.float32, slot=ot*(token_tile//16)+tile, addrspace=AddrSpace.REG) @@ -229,8 +287,8 @@ def _quant_linear_wmma(out, x, out_features, in_features, type_words, layout, de wmma_accs[output_tile][tile] = UOp.wmma(afrag, bfrag, previous, *WMMA_ARG) update = UOp.group(*(acc.store(value) for output_accs,output_values in zip(accs, wmma_accs) for acc,value in zip(output_accs, output_values))).end(group) - return UOp.group(*_wmma_stores(out, outputs, tokens, accs, update, physical_half)).end(token_block, output_block, lane, wave).sink( - arg=KernelInfo(name=name, opts_to_apply=())) + stores = _wmma_stores(out, outputs, tokens, accs, update, physical_half, lane, wave, output_waves) + return UOp.group(*stores).end(token_block, output_block, lane, wave).sink(arg=KernelInfo(name=name, opts_to_apply=())) @functools.cache def _q5_linear_f16_wmma_kernel(out:UOp, raw:UOp, x:UOp, out_features:int, in_features:int, ggml_type:int) -> UOp: @@ -264,9 +322,10 @@ def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: pairs = tuple(lut[((raw[base + 2 + subgroup*4 + word] >> (byte*8)) & 255).cast(dtypes.weakint)] for word in range(4) for byte in range(4)) return tuple((_half((pair >> (half*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in pairs) - def nibble(packed:UOp, index:int): return (packed >> (8*index+4*half)) & 15 - lut_pairs = (lut[(nibble(packed, i) | nibble(packed, i+1)<<4).cast(dtypes.weakint)] - for packed in (raw[base+2+subgroup*4+i] for i in range(4)) for i in (0, 2)) + # a subgroup-half gathers the lo (half=0) or hi (half=1) nibbles of byte pairs of each packed word + lut_pairs = (lut[(((raw[base+2+subgroup*4+i] >> (8*j+4*half)) & 15) | + (((raw[base+2+subgroup*4+i] >> (8*j+8+4*half)) & 15) << 4)).cast(dtypes.weakint)] + for i in range(4) for j in (0, 2)) return tuple((_half((pair >> (i*16)) & 0xffff)*scale).cast(dtypes.float16) for pair in lut_pairs for i in range(2)) return _quant_linear_wmma(out, x, out_features, in_features, IQ4_WORDS, layout, dequant, "linear_iq4_xs_f16_wmma") @@ -287,67 +346,163 @@ def run(fxn:Callable[..., UOp], out:UOp, *srcs:UOp) -> Tensor: fxn = _iq4_linear_f16_wmma_kernel if layer.ggml_type == IQ4_XS else functools.partial(_q5_linear_f16_wmma_kernel, ggml_type=layer.ggml_type) extra = (iq4_half_lut(str(x.device)).uop,) if layer.ggml_type == IQ4_XS else () return run(fxn, out, raw, x.cast(dtypes.float16).contiguous().uop, *extra) - xq, xd = q8_quantize(x, tokens, in_features) + xq_, xd, xs = q8_quantize(x, tokens, in_features) decode = functools.partial(_quant_decode_kernel, ggml_type=layer.ggml_type) out = Tensor.empty(tokens, out_features, (in_features+1023)//1024, dtype=dtypes.float32, device=x.device).uop - return run(decode, out, raw, xq.uop, xd.uop) + return run(decode, out, raw, xq_.uop, xd.uop, xs.uop) + +# ******** tiny dense fp16 gemv ******** + +@functools.cache +def _amd_f16_gemv_kernel(out:UOp, w:UOp, x:UOp, *rest:UOp, in_features:int, out_features:int, tokens:int) -> UOp: + bias: UOp|None = rest[0] if rest else None + # one block per (token, output row), 32 lanes accumulate 4-wide chunks of the row + lanes, val_chunk = WARP_SIZE, 4 + token, out_row = UOp.range(tokens, 0, AxisType.GLOBAL), UOp.range(out_features, 1, AxisType.GLOBAL) + lane = UOp.range(lanes, 2, axis_type=AxisType.LOCAL) + per = in_features // (lanes * val_chunk) + assert per * lanes * val_chunk == in_features + w = w.reshape((out_features, per, lanes*val_chunk)) + x = x.reshape((tokens, per, lanes*val_chunk)) + acc = UOp.const(0, dtypes.float32) + for i in range(per): + for j in range(val_chunk): + acc = acc + w[out_row, i, lane*val_chunk + j].load().float() * x[token, i, lane*val_chunk + j].load().float() + total = warp_reduce(acc, full_wave=True) + if bias is not None: total = total + bias[token, out_row].load().float() + return out[token, out_row.valid(lane.eq(0))].store(total).end(token, out_row, lane).sink(arg=KernelInfo(name="linear_f16_gemv", opts_to_apply=())) + +def _view_back(t:Tensor) -> Tensor: + """strip top-of-chain CAST(s) from a lazy weight: reading the raw file bytes in the kernel instead of + materializing the cast into a fresh buffer every step""" + uop = t.uop + while uop.op is Ops.CAST: uop = uop.src[0] + return Tensor(uop).reshape(t.shape) + +def f16_gemv(layer:Linear, x:Tensor) -> Tensor: + tokens = prod(x.shape[:-1]) + assert isinstance(tokens, int) + weight = _view_back(layer.weight) + x = x.contiguous() if x.dtype == dtypes.half else x.cast(dtypes.half).contiguous() + out = Tensor.empty(tokens, layer.out_features, dtype=dtypes.float32, device=x.device) + fxn = functools.partial(_amd_f16_gemv_kernel, in_features=layer.in_features, out_features=layer.out_features, tokens=tokens) + srcs = (out, weight.reshape(-1), x.reshape(tokens, layer.in_features)) + (() if layer.bias is None else (_view_back(layer.bias),)) + return Tensor.custom_kernel(*srcs, fxn=fxn)[0].reshape(*x.shape[:-1], layer.out_features) # ******** flash attention on the KV cache ******** +def _vec_load(ptr:UOp, lanes:int) -> tuple[UOp, ...]: + if lanes == 1: return (ptr.load().float(),) + vec = _amd_load(ptr, lanes) + return tuple(vec[i].float() for i in range(lanes)) + @functools.cache -def _amd_flash_attention_decode_partial(out, stats, q, cache_kv, valid_kv_len, max_kv_len, block_n): +def _amd_flash_attention_decode_partial(out, stats, q, cache_kv, valid_kv_len, max_kv_len, block_n, waves=4): valid_kv_len = _unbind(valid_kv_len) _, B, H_KV, N, D = cast(tuple[int, int, int, int, int], cache_kv.shape) _, H, M, _ = cast(tuple[int, int, int, int], q.shape) assert M == 1 and H % H_KV == 0 and D % WARP_SIZE == 0 and max_kv_len <= N and max_kv_len % block_n == 0 - G, CHUNK, DV, heads_per_wave = H // H_KV, block_n, D // WARP_SIZE, 2 - head_tile = min(DECODE_HEAD_TILE, G) # share each KV stream across two GQA heads per wave - assert G % head_tile == 0 and head_tile % heads_per_wave == 0 - decode_waves, decode_group = head_tile // heads_per_wave, 4 - block_bhkv = UOp.range(B*H_KV*(G//head_tile), 0, AxisType.GLOBAL) - valid_chunks = (valid_kv_len+CHUNK-1)//CHUNK - group_count = min(valid_chunks, out.shape[2]) if isinstance(valid_chunks, int) else valid_chunks.minimum(out.shape[2]) - block_n, lane = UOp.range(group_count, 1, AxisType.GLOBAL), UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL) - wave = UOp.range(decode_waves, 3, axis_type=AxisType.LOCAL) - head_group, bhkv = block_bhkv % (G//head_tile), block_bhkv // (G//head_tile) - b, kv_head = bhkv // H_KV, bhkv % H_KV - dims = tuple(lane + i*WARP_SIZE for i in range(DV)) - acc, row_max, row_sum = _reg((heads_per_wave, DV), 0, 0), _reg((heads_per_wave,), 1, -math.inf), _reg((heads_per_wave,), 2, 0) - groups_per_chunk, offset = CHUNK // decode_group, UOp.range(((valid_chunks+group_count-1)//group_count)*(CHUNK//decode_group), 100, AxisType.REDUCE) - chunk = block_n + (offset // groups_per_chunk) * group_count - keys = tuple(chunk*CHUNK + (offset % groups_per_chunk)*decode_group + i for i in range(decode_group)) - valid = tuple(key < valid_kv_len for key in keys) - kvals, vvals = (tuple(tuple(is_valid.where(cache_kv[kv, b, kv_head, key, d].float(), UOp.const(0, dtypes.float)) for d in dims) - for key,is_valid in zip(keys, valid)) for kv in range(2)) - q_heads = tuple(kv_head*G + head_group*head_tile + wave*heads_per_wave + head for head in range(heads_per_wave)) - updates:list[UOp] = [] - for head,q_head in enumerate(q_heads): - scores = tuple(warp_reduce(sum((q[b, q_head, 0, d].float()*k for d,k in zip(dims, key_kvals)), - UOp.const(0, dtypes.float)), full_wave=True) / math.sqrt(D) for key_kvals in kvals) - prev_acc, prev_max, prev_sum = acc.after(offset)[head], row_max.after(offset)[head], row_sum.after(offset)[head] - new_max = functools.reduce(lambda a,vs:a.maximum(vs[0].where(vs[1], UOp.const(-math.inf, dtypes.float))), zip(valid, scores), prev_max) - alpha = ((prev_max-new_max)*LOG2E).exp2() - betas = tuple(is_valid.where(((score-new_max)*LOG2E).exp2(), UOp.const(0, dtypes.float)) for is_valid,score in zip(valid, scores)) - updates += [acc[head].store(prev_acc*alpha + sum((UOp.stack(*value)*beta for value,beta in zip(vvals, betas)), acc[head].const_like(0))), - row_sum[head].store(prev_sum*alpha + sum(betas, UOp.const(0, dtypes.float))), row_max[head].store(new_max)] - update = UOp.group(*updates).end(offset) - acc, row_max, row_sum = acc.after(update), row_max.after(update), row_sum.after(update) - stores = [out[b, q_head, block_n, d].store(acc[head, i]) for head,q_head in enumerate(q_heads) for i,d in enumerate(dims)] + \ - [stats[b, q_head.valid(lane.eq(0)), block_n, i].store(x[head]) for head,q_head in enumerate(q_heads) for i,x in enumerate((row_max, row_sum))] - return UOp.group(*stores).end(lane, wave, block_n, block_bhkv).sink(arg=KernelInfo(name="flash_decode_partial", opts_to_apply=())) + G, CHUNK, DPL, WAVES = H // H_KV, block_n, D // WARP_SIZE, waves + assert CHUNK % WAVES == 0 + SEC = CHUNK // WAVES # keys each wave scans independently + live_chunks = (valid_kv_len+CHUNK-1)//CHUNK + live_chunks = min(live_chunks, out.shape[2]) if isinstance(live_chunks, int) else live_chunks.minimum(out.shape[2]) + block_bhkv, block_chunk = UOp.range(B*H_KV, 0, AxisType.GLOBAL), UOp.range(live_chunks, 1, AxisType.GLOBAL) + lane, wave = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL), UOp.range(WAVES, 3, axis_type=AxisType.LOCAL) + b, kv_head = block_bhkv // H_KV, block_bhkv % H_KV + # per-lane query fragments for every GQA head, kept packed in registers; unpacked at use + qf = tuple(_vec_load(q[b, kv_head*G+h, 0, lane*DPL], DPL) for h in range(G)) + zerof = UOp.const(0, dtypes.float) + valids: list[UOp] = [] + scores: list[list[UOp]] = [[zerof]*G for _ in range(SEC)] + vfrags: list[tuple[UOp, ...]] = [()]*SEC + for j in range(SEC): + key = block_chunk*CHUNK + wave*SEC + j + valid = key < valid_kv_len + valids.append(valid) + kfrag = _vec_load(cache_kv[0, b, kv_head, key, lane*DPL], DPL) + # V is prefetched in the score pass so both streams are in flight together + vfrags[j] = _vec_load(cache_kv[1, b, kv_head, key, lane*DPL], DPL) + for h in range(G): + s = warp_reduce(sum((qf[h][i]*kfrag[i] for i in range(DPL)), UOp.const(0, dtypes.float)), full_wave=True) * (1/math.sqrt(D)) + scores[j][h] = valid.where(s, UOp.const(-math.inf, dtypes.float)) + ninf = UOp.const(-math.inf, dtypes.float) + row_max = [functools.reduce(UOp.maximum, (scores[j][h] for j in range(SEC)), ninf) for h in range(G)] + accs:list[list[UOp]] = [[UOp.const(0, dtypes.float)] * DPL for _ in range(G)] + row_sums:list[UOp] = [UOp.const(0, dtypes.float) for _ in range(G)] + for j in range(SEC): + for h in range(G): + beta = valids[j].where(((scores[j][h]-row_max[h])*LOG2E).exp2(), UOp.const(0, dtypes.float)) + accs[h] = [a + beta*v for a, v in zip(accs[h], vfrags[j])] + row_sums[h] = row_sums[h] + beta + # exchange across the block's waves through LDS (fp16 halves LDS so more blocks fit per CU) + acc_lds = UOp.placeholder((WAVES, G, D), dtypes.half, slot=0, addrspace=AddrSpace.LOCAL) + ml_lds = UOp.placeholder((WAVES, G, 2), dtypes.float, slot=1, addrspace=AddrSpace.LOCAL) + lds_acc = acc_lds.reshape(WAVES, G, WARP_SIZE, DPL) + stores = [lds_acc[wave, h, lane].store(UOp.stack(*accs[h]).cast(dtypes.half)) for h in range(G)] + # NOTE: duplicate stores of the same value from every lane are harmless here + stores += [ml_lds[wave, h, i].store(x) for h in range(G) for i, x in enumerate((row_max[h], row_sums[h]))] + barrier = UOp.barrier(UOp.group(*stores)) + acc_lds, ml_lds = acc_lds.after(barrier), ml_lds.after(barrier) + tid = wave*WARP_SIZE + lane + final_stores:list[UOp] = [] + for i in range(-(-G*D//(WAVES*WARP_SIZE))): + flat = tid + i*WAVES*WARP_SIZE + h, d = flat // D, flat % D + M = functools.reduce(UOp.maximum, (ml_lds[w, h, 0].load() for w in range(WAVES)), ninf) + val = sum((((ml_lds[w, h, 0].load()-M)*LOG2E).exp2() * acc_lds[w, h, d].load().float() for w in range(WAVES)), UOp.const(0, dtypes.float)) + oidx = out[b, kv_head*G + h, block_chunk, d] + if G*D % (WAVES*WARP_SIZE): oidx = out[b, (kv_head*G + h).valid(flat < G*D), block_chunk, d] + final_stores.append(oidx.store(val)) + hstat = tid + M = functools.reduce(UOp.maximum, (ml_lds[w, hstat, 0].load() for w in range(WAVES)), ninf) + L = sum((((ml_lds[w, hstat, 0].load()-M)*LOG2E).exp2() * ml_lds[w, hstat, 1].load() for w in range(WAVES)), UOp.const(0, dtypes.float)) + q_head = (kv_head*G + hstat).valid(hstat < G) if WAVES*WARP_SIZE > G else kv_head*G + hstat + final_stores += [stats[b, q_head, block_chunk, 0].store(M), stats[b, q_head, block_chunk, 1].store(L)] + return UOp.group(*final_stores).end(lane, wave, block_chunk, block_bhkv).sink(arg=KernelInfo(name="flash_decode_partial", opts_to_apply=())) + +@functools.cache +def _amd_flash_decode_combine(o:UOp, partial:UOp, stats:UOp, live:int|UOp) -> UOp: + # one wave per (batch, head, 64-dim tile): every lane redundantly weights its chunks; no cross-lane traffic + live = _unbind(live) + B, H, C, D = cast(tuple[int, int, int, int], partial.shape) + DT = 64 if D % 64 == 0 else WARP_SIZE # dims per block + assert D % DT == 0 + block_bh, block_dt = UOp.range(B*H, 0, AxisType.GLOBAL), UOp.range(D//DT, 1, AxisType.GLOBAL) + lane = UOp.range(WARP_SIZE, 2, axis_type=AxisType.LOCAL) + b, h = block_bh // H, block_bh % H + NPD = DT // WARP_SIZE # output dims per lane + dims = tuple(block_dt*DT + lane*NPD + i for i in range(NPD)) + chunk = UOp.range(live, 100, AxisType.REDUCE) + def iloop(ph, val): return ph.store(ph.const_like(val)) + chunk_max = UOp.placeholder((1,), dtypes.float, slot=0, addrspace=AddrSpace.REG) + chunk_max_i = chunk_max.after(iloop(chunk_max, -math.inf)) + update0 = chunk_max_i.store(chunk_max_i.after(chunk).maximum(stats[b, h, chunk, 0].load())).end(chunk) + chunk_max = chunk_max_i.after(update0) + chunk2 = UOp.range(live, 101, AxisType.REDUCE) + acc = UOp.placeholder((NPD,), dtypes.float, slot=1, addrspace=AddrSpace.REG) + weight_sum = UOp.placeholder((1,), dtypes.float, slot=2, addrspace=AddrSpace.REG) + acc_i, weight_sum_i = acc.after(iloop(acc, 0)), weight_sum.after(iloop(weight_sum, 0)) + w = ((stats[b, h, chunk2, 0].load()-chunk_max)*LOG2E).exp2() + update1 = UOp.group(*[acc_i[i].store(acc_i.after(chunk2)[i].load() + w*partial[b, h, chunk2, d].load()) for i, d in enumerate(dims)], + weight_sum_i[0].store(weight_sum_i.after(chunk2)[0].load() + w*stats[b, h, chunk2, 1].load())).end(chunk2) + acc, weight_sum = acc_i.after(update1), weight_sum_i.after(update1) + inv = 1 / weight_sum[0].load() + return UOp.group(*[o[b, h, 0, d].store(acc[i].load() * inv) for i, d in enumerate(dims)]) \ + .end(lane, block_dt, block_bh).sink(arg=KernelInfo(name="flash_decode_combine", opts_to_apply=())) def amd_flash_attention_decode(q:Tensor, cache_kv:Tensor, valid_kv_len:int|UOp, max_kv_len:int) -> Tensor: B, H, D = cache_kv.shape[1], q.shape[1], cache_kv.shape[4] - chunks = min(64, max_kv_len // 128) + chunks = min(256, max_kv_len // 64) partial = Tensor.empty(B, H, chunks, D, dtype="float32", device=q.device) stats = Tensor.empty(B, H, chunks, 2, dtype="float32", device=q.device) - fxn = functools.partial(_amd_flash_attention_decode_partial, valid_kv_len=valid_kv_len, max_kv_len=max_kv_len, block_n=128) + fxn = functools.partial(_amd_flash_attention_decode_partial, valid_kv_len=valid_kv_len, max_kv_len=max_kv_len, block_n=64, waves=16) partial, stats = Tensor.custom_kernel(partial, stats, q, cache_kv, fxn=fxn)[:2] - live = (valid_kv_len+127)//128 + live = (valid_kv_len+63)//64 live = min(live, chunks) if isinstance(live, int) else live.minimum(chunks) - partial, stats = partial[:, :, :live], stats[:, :, :live] - weights = ((stats[..., 0]-stats[..., 0].max(2, keepdim=True))*LOG2E).exp2() - return ((partial*weights.unsqueeze(-1)).sum(2) / (stats[..., 1]*weights).sum(2, keepdim=True)).unsqueeze(2) + out = Tensor.empty(B, H, 1, D, dtype="float32", device=q.device) + fxn = functools.partial(_amd_flash_decode_combine, live=live) + return Tensor.custom_kernel(out, partial, stats, fxn=fxn)[0] @functools.cache def _amd_flash_attention(o:UOp, q:UOp, cache:UOp, valid_kv_len:int|UOp, q_start:int|UOp|None=None) -> UOp: diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index f139ce9dc2232..b2b23fc02daca 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -188,10 +188,12 @@ def _render_dtype(self, dtype:DType, sz:int=1, addrspace=AddrSpace.ALU, mutable= return prefix + self.type_map.get(dtype, dtype.name) + suffix def render_type(self, u:UOp): return self._render_dtype(u.dtype, u.max_numel(), u.addrspace, shape=u._shape) - def render_access(self, u:UOp): + def render_ptr(self, u:UOp): + # the address of an access, vector-cast if the access reads/writes more lanes than the pointer's scalar type if u.max_numel() > 1 or u.dtype != u.src[0].dtype: - return f"*(({self._render_dtype(u.dtype, u.max_numel(), u.addrspace, override_ptr=True, shape=u._shape)})({self[u]}))" - else: return f"*{self[u]}" + return f"(({self._render_dtype(u.dtype, u.max_numel(), u.addrspace, override_ptr=True, shape=u._shape)})({self[u]}))" + else: return f"{self[u]}" + def render_access(self, u:UOp): return f"*{self.render_ptr(u)}" def render_cast(self, u:UOp, val:str) -> str: return f"({self.render_type(u)})({val})" # LEGACY @@ -509,6 +511,9 @@ def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 = (UPat(Ops.CAST, dtypes.float, (UPat.var("y", dtypes.fp8s),), name="x",), lambda ctx,x,y: f"__builtin_amdgcn_cvt_f32_{('fp8', 'bf8')[fp8_index(y.dtype)]}((unsigned int){ctx[x.src[0]]}, 0)"), ]) + base_rewrite + # a LOAD flagged nontemporal renders as the cache-bypassing builtin (only used on global loads) + self.string_rewrite = PatternMatcher([(UPat(Ops.LOAD, arg="nontemporal", src=(UPat.var("bidx"),)), + lambda ctx,bidx: f"__builtin_nontemporal_load({ctx.render_ptr(bidx)})")]) + self.string_rewrite # https://clang.llvm.org/docs/AttributeReference.html#amdgpu-flat-work-group-size # NOTE: this makes hlb_cifar10 twice as fast, there may be more gains in tweaking these parameters diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 359c55c15efe5..28b7662de51fd 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -325,7 +325,8 @@ def topovisit(self, visitor:Callable[[UOp], T], cache:dict[UOp, T]) -> T: @functools.cached_property def tuplize(self:UOp) -> tuple: - return (self.op.value, self.arg, self.dtype,)+tuple([x.tuplize for x in self.src]) + # arg goes through repr: args of different types (None, str, tuple) must stay mutually comparable for the sort + return (self.op.value, repr(self.arg), self.dtype,)+tuple([x.tuplize for x in self.src]) # *** uop shape stuff *** From 22ffff32582c43457d821ee9747245235c5caff0 Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Wed, 26 Aug 2026 18:32:21 -0400 Subject: [PATCH 223/306] gptoss: single jit (#17525) --- examples/mlperf/model_train.py | 55 +++++++++++++--------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index 367962bd21c6e..a69850ae93004 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -1674,8 +1674,7 @@ def train_gptoss(): config = {} BASEDIR = config["BASEDIR"] = Path(getenv("BASEDIR", "/raid/datasets/c4-8b/")) BS = config["BS"] = getenv("BS", 16) - grad_acc = config["GRADIENT_ACC_STEPS"] = getenv("GRADIENT_ACC_STEPS", 1) - GBS = config["GLOBAL_BATCH_SIZE"] = BS * grad_acc + GBS = config["GLOBAL_BATCH_SIZE"] = BS SEED = config["SEED"] = getenv("SEED", 5760) DATA_SEED = config["DATA_SEED"] = getenv("DATA_SEED", SEED) SEQLEN = config["SEQLEN"] = getenv("SEQLEN", 8192) @@ -1737,8 +1736,8 @@ def train_gptoss(): params_wd = [p for p in params if p.ndim >= 3] params_no_wd = [p for p in params if p.ndim < 3] optim = GradAccClipAdamWGroup( - GradAccClipAdamW(params_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay, grad_acc=grad_acc, device=optim_device), - GradAccClipAdamW(params_no_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=0.0, grad_acc=grad_acc, device=optim_device), + GradAccClipAdamW(params_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay, grad_acc=1, device=optim_device), + GradAccClipAdamW(params_no_wd, lr=0.0, b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=0.0, grad_acc=1, device=optim_device), ) for p in optim.params: @@ -1770,7 +1769,7 @@ def _scale_key(n): @TinyJit @Context(TRAINING=1) - def minibatch(tokens:Tensor): + def step(tokens:Tensor): if is_dp: tokens = tokens.to(None).shard(device, 0) if not is_sharding: tokens = tokens.to(None) @@ -1784,22 +1783,20 @@ def minibatch(tokens:Tensor): for g, new_g in zip(grads, loss.gradient(*optim.params)): apply_grad(g, new_g.uop) - loss_cpu = loss.flatten().float().to("CPU") - return loss_cpu.realize(*grads) + Tensor.realize(loss, *grads) - @TinyJit - def optim_step(): - grad_norm = clip_grads(grads, grad_acc, 1.0) + grad_norm = clip_grads(grads, 1, 1.0) optim.fstep(grads, grad_norm) scheduler.step() for g in grads: g.assign(0) + loss_cpu = loss.flatten().float().to("CPU") lr_cpu = optim.lr.float().to("CPU") grad_norm_cpu = grad_norm.float().to("CPU") - Tensor.realize(lr_cpu, grad_norm_cpu, *grads, *fp8_inv_scales) + Tensor.realize(loss_cpu, lr_cpu, grad_norm_cpu, *grads, *fp8_inv_scales) - return lr_cpu, grad_norm_cpu + return loss_cpu, lr_cpu, grad_norm_cpu @TinyJit @Context(TRAINING=0) @@ -1848,30 +1845,20 @@ def get_eval_iter(): profile_marker(f"train @ {i}") st = time.perf_counter() - stopped = False - losses, data_time, dev_time = [], 0, 0 - for _ in range(grad_acc if i >= 2 else 1): - ist = time.perf_counter() - try: tokens = next(train_iter) - except StopIteration: - stopped = True - break - mst = time.perf_counter() - data_time += mst - ist - losses.append(minibatch(tokens).item()) - dev_time += time.perf_counter() - mst - if stopped: break + ist = time.perf_counter() - gt = time.perf_counter() - ret = optim_step() - lr, grad_norm = ret[0].item(), ret[1].item() + try: tokens = next(train_iter) + except StopIteration: break + mst = time.perf_counter() + data_time = mst - ist + + ret = step(tokens) + dev_time = time.perf_counter() - mst + + loss, lr, grad_norm = ret[0].item(), ret[1].item(), ret[2].item() et = time.perf_counter() - loss = sum(losses) / len(losses) - optim_time = et - gt - dev_time += optim_time step_time = et - st - gbs_time = gt - st if BENCHMARK: step_times.append(step_time) i += 1 @@ -1881,7 +1868,7 @@ def get_eval_iter(): gflops = GlobalCounters.global_ops / 1e9 / dev_time mfu = ((6 * num_params * SEQLEN * GBS) / (dev_time * device_count * 4.6e15)) * 100 tqdm.write( - f"{i:5} {step_time:.3f} s step, {gbs_time:.3f} s gbs, {optim_time:.3f} s optim, {data_time:.3f} s data, {loss:.4f} loss, " \ + f"{i:5} {step_time:.3f} s step, {dev_time:.3f} s dev, {data_time:.3f} s data, {loss:.4f} loss, " \ f"{lr:.12f} LR, {grad_norm:.6f} grad_norm, {mem_gb:.2f} GB used, {gflops:9.2f} GFLOPS, {mfu:5.2f}% MFU") if DEBUG >= 1: tqdm.write(" mem per device: " + ', '.join(f"{dev}: {mem/1e9:.2f} GB" for dev, mem in sorted(GlobalCounters.mem_used_per_device.items()))) @@ -1891,8 +1878,6 @@ def get_eval_iter(): "train/lr": lr, "train/grad_norm": grad_norm, "train/step_time": step_time, - "train/gbs_time": gbs_time, - "train/optim_time": optim_time, "train/dev_time": dev_time, "train/data_time": data_time, "train/mem": mem_gb, From 0820c7a4248cb2d062f418b458e58701a7ee7e50 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 26 Aug 2026 15:53:55 -0700 Subject: [PATCH 224/306] benchmarks: openpilot matrix (#17714) --- .github/workflows/benchmark.yml | 90 ++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 798d2261b3ab4..ca506e4d95ff0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -446,43 +446,55 @@ jobs: - name: UsbGPU (USB4/TB) tiny tests run: DEV=PCI+NV:NAK python3.11 test/test_tiny.py - testcommalatest: - name: comma Benchmark (0.11.2) - runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 - defaults: - run: - shell: bash -e -o pipefail {0} - if: github.repository_owner == 'tinygrad' - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: setup staging db - if: github.ref == 'refs/heads/update_benchmark_staging' - run: | - echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV - rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - - name: reset process replay - run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 supercombo (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=ir3_openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=41 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_2_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 - - name: Run process replay tests - uses: ./.github/actions/process-replay - - testcommaold: - name: comma Benchmark (0.11.0) + testcomma: + strategy: + matrix: + dev: ['QCOM', 'QCOM:IR3'] + version: ['0.11.0', '0.11.2'] + model: ['vision', 'policy', 'supercombo', 'dmonitoring'] + # exclude non-existent models + exclude: [{ version: '0.11.0', model: supercombo }, { version: '0.11.2', model: vision }, { version: '0.11.2', model: policy }] + include: + - version: '0.11.0' + model: vision + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx + timing: 17 + - version: '0.11.0' + model: policy + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx + timing: 3.2 + - version: '0.11.0' + model: dmonitoring + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + timing: 11 + - version: '0.11.2' + model: supercombo + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b + timing: 26 + - dev: QCOM:IR3 + version: '0.11.2' + model: supercombo + timing: 41 + - version: '0.11.2' + model: dmonitoring + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 + timing: 11 + # IR3 dmonitoring is slightly slower + - dev: QCOM:IR3 + model: dmonitoring + timing: 12 + fail-fast: false + name: openpilot ${{ matrix.version }} compile3 ${{ matrix.model }} (DEV=${{ matrix.dev }}) runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} if: github.repository_owner == 'tinygrad' + env: + DEV: ${{ matrix.dev }} + ASSERT_MIN_STEP_TIME: ${{ matrix.timing }} + BENCHMARK_LOG: ${{ matrix.dev == 'QCOM:IR3' && 'ir3_' || '' }}openpilot_${{ matrix.version }}_${{ matrix.model }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -493,16 +505,10 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: reset process replay run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_vision (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_0_vision_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=18 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_policy - run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx - - name: openpilot compile3 0.11.0 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + - name: compile + run: FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }} + - name: run pickle + run: BENCHMARK_LOG="${BENCHMARK_LOG}_run_pickle" RUN_PICKLE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py - name: Run process replay tests uses: ./.github/actions/process-replay From 118e812b316f5020b3d29a2b035fe1bd5cdc67b8 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:54:12 +0800 Subject: [PATCH 225/306] simplify more same device copies (#17766) * simplify more same device copies * stack of contigs or copies * rename --- test/unit/test_allreduce.py | 4 ++-- tinygrad/schedule/multi.py | 3 ++- tinygrad/schedule/prepare.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/unit/test_allreduce.py b/test/unit/test_allreduce.py index 26ff252507db3..602f5aa57748e 100644 --- a/test/unit/test_allreduce.py +++ b/test/unit/test_allreduce.py @@ -33,8 +33,8 @@ def test_schedule_all2all(self): # N*(N-1) copies for input and output copy_count = N*(N-1)*2 if len(copies) != copy_count: raise KernelCountException(copy_count, len(copies)) - # N*N shrinks becoming contigs, N ALU, N extra contig, reassembly (cat), and mul - sink_count = (N*N)+(N)+(N)+(1)+(1) + # N*(N-1) shrinks from other devices becoming contigs, N ALU, N extra contig, reassembly (cat), and mul + sink_count = (N*(N-1))+(N)+(N)+(1)+(1) if len(sinks) != sink_count: raise KernelCountException(sink_count, len(sinks)) # correctness run_linear(linear, var_vals) diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index bc97293776ff2..c60fd8aa7d9e8 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -15,7 +15,8 @@ def mstack_early_shrink(ms:UOp, shrink:UOp): ret:list[UOp] = [] for i, x in enumerate(ms.src): if x.op is Ops.COPY: - ret.append(_apply_shrink(shrink.marg, x.src[0], i).copy_to_device(x.device)) + src = _apply_shrink(shrink.marg, x.src[0], i) + ret.append(src.contiguous() if src.device == x.device else src.copy_to_device(x.device)) else: ret.append(_apply_shrink(shrink.marg, x, i).contiguous()) return ms.replace(src=tuple(ret)) diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index aea39e4d0a771..1df40dc59a448 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -139,13 +139,13 @@ def expand_bitcast(bc:UOp) -> UOp|None: # ** copy rules ** + # copy to same device is a no-op + (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), + # COPY transfers a contiguous range, so materialize a source that's resized (shrink/pad/expand) or reordered (permute/flip) (UPat(Ops.COPY, src=(UPat(GroupOp.Movement, name="r"),), name="c"), lambda c,r: c.replace(src=(r.contiguous(),)) if resolve(r.numel() != r.base.numel(), False) or r.contiguous_view_offset() is None else None), - # copy to same device is a no-op - (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), - # copy on reshape is reshape on copy (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), From d16a2510ad01f5b7942cc190447ddf4ab9e7ae9a Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:01:28 +0800 Subject: [PATCH 226/306] remove some dead scheduler tests (#17768) * remove tests that use .uop * that test does nothing --- test/backend/test_schedule.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/backend/test_schedule.py b/test/backend/test_schedule.py index 85bedd2dc0fae..d802dcf4edacf 100644 --- a/test/backend/test_schedule.py +++ b/test/backend/test_schedule.py @@ -365,16 +365,6 @@ def test_alu_before_copy(self): b = a.to("CPU") self.assertListEqual(b.tolist(), [2.]) - def test_copy_to_same_device(self): - a = Tensor.empty(4).uop - b = a.copy_to_device(a.device) - check_schedule(b, 1, filter_sink=False) # TODO: 0? - - def test_copy_to_same_device_alt(self): - a = Tensor.empty(4, 4).uop - b = a.copy_to_device(a.device) - check_schedule(b, 1, filter_sink=False) # TODO: 0? - def test_copy_to_same_device_sched(self): a = Tensor.ones(4).contiguous().realize().uop.buf_uop t = Tensor(a.copy_to_device(a.device)) @@ -384,14 +374,6 @@ def test_copy_to_same_device_sched(self): assert t.uop.is_realized, f"didn't realize Tensor {t}" self.assertListEqual(t.tolist(), [1.,1.,1.,1.]) - @unittest.skip("same-device copies are no-ops") - def test_self_assign_same_device_copy(self): - a = Tensor.ones(4, 4).contiguous().realize() - # use copy_to_device to bypass Tensor.to() shortcircuit and force a real same-device COPY in the graph - a.assign(Tensor(a.uop.copy_to_device(a.device), a.device)) - run_linear(*check_schedule(a, 2, filter_sink=False)) - self.assertListEqual(a.tolist(), [[1.]*4]*4) - def test_clone(self): a = Tensor.empty(4) check_schedule(a.clone(), 1, filter_sink=False) From 0d6681584955f1002644a5444aebb224aaacc211 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:45:33 +0800 Subject: [PATCH 227/306] set BEAM_PADTO=0 in llama (#17769) --- .../llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh | 2 +- .../llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh | 2 +- .../llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh index ede63df0a5a84..0da8692faf19c 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh @@ -44,7 +44,7 @@ export SEED=${SEED:-5760} export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 export FAKEDATA=${FAKEDATA:-1} BENCHMARK=${BENCHMARK:-10} if [ -z "$FULL_LAYERS" ]; then diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index 4ee7eacff844c..4579bc6453f32 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -44,6 +44,6 @@ export SEED=${SEED:-$RANDOM} export DATA_SEED=${DATA_SEED:-5760} export JITBEAM=${JITBEAM:-3} -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 python3 examples/mlperf/model_train.py diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh index 6dc5ce1a14816..8815f4484c53e 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/run_and_time.sh @@ -44,7 +44,7 @@ export SEED=$RANDOM export DATA_SEED=$SEED export JITBEAM=3 -export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=1 +export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0 export LOGMLPERF=1 From 2d4e7afcbfbe749f751f8ffe668314b122809ad7 Mon Sep 17 00:00:00 2001 From: tonyfloatersu Date: Thu, 27 Aug 2026 05:24:37 -0400 Subject: [PATCH 228/306] mockgpu/amd: use generated KFD ioctl metadata (#17761) * init commit * init commit * minimize to single file scope --- test/mockgpu/amd/amddriver.py | 47 +++++++++++++++-------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/test/mockgpu/amd/amddriver.py b/test/mockgpu/amd/amddriver.py index 6d619bff6b10c..0ec6d98141c0b 100644 --- a/test/mockgpu/amd/amddriver.py +++ b/test/mockgpu/amd/amddriver.py @@ -1,20 +1,15 @@ -import pathlib, re, ctypes, mmap, collections, functools, copy, os +import ctypes, mmap, collections, functools, copy, os from tinygrad.runtime.autogen import kfd, amdgpu_drm, libc import tinygrad.runtime.autogen.am.am as am from tinygrad.helpers import from_mv from test.mockgpu.driver import VirtDriver, VirtFileDesc, TextFileDesc, DirFileDesc, VirtFile from test.mockgpu.amd.amdgpu import AMDGPU, gpu_props, GFX_TARGET_VERSION, MOCKGPU_ARCH -def ioctls_from_header(): - # hdrpy = (pathlib.Path(__file__).parent.parent.parent.parent / "tinygrad" / "runtime" / "autogen" / "kfd.py").read_text() - # pattern = r'# (AMDKFD_IOC_[A-Z0-9_]+)\s=\s_(IOW?R?).*\(( 0x[0-9a-fA-F]+) ,\s+struct\s([A-Za-z0-9_]+)\s+\)' - # matches = re.findall(pattern, hdrpy, re.MULTILINE) - hdr = (pathlib.Path(__file__).parent.parent.parent.parent / "extra" / "hip_gpu_driver" / "kfd_ioctl.h").read_text().replace("\\\n", "") - pattern = r'#define\s+(AMDKFD_IOC_[A-Z0-9_]+)\s+AMDKFD_(IOW?R?)\((0x[0-9a-fA-F]+),\s+struct\s([A-Za-z0-9_]+)\)' - matches = re.findall(pattern, hdr, re.MULTILINE) - return type("KFD_IOCTLS", (object, ), {name: int(nr, 0x10) for name, _, nr, _ in matches}), \ - {int(nr, 0x10): getattr(kfd, "struct_"+sname, None) for name, idir, nr, sname in matches} -kfd_ioctls, kfd_headers = ioctls_from_header() +def _ioctl_nr(ioctl: functools.partial) -> int: return ioctl.args[2] + +kfd_ioctl_info = { + _ioctl_nr(ioctl): (name, ioctl.args[3]) for name, ioctl in vars(kfd).items() + if name.startswith("AMDKFD_IOC_") and isinstance(ioctl, functools.partial)} class KFDFileDesc(VirtFileDesc): def __init__(self, fd, driver): @@ -116,42 +111,44 @@ def open(self, name, flags, mode, virtfile): return virtfile.fdcls(self._alloc_f def kfd_ioctl(self, req, argp): nr = req & 0xFF - struct = kfd_headers[nr].from_address(argp) + if nr not in kfd_ioctl_info: raise RuntimeError(f"unknown kfd ioctl, {nr} unknown") + name, struct_type = kfd_ioctl_info[nr] + struct = struct_type.from_address(argp) - if nr == kfd_ioctls.AMDKFD_IOC_ACQUIRE_VM: pass - elif nr == kfd_ioctls.AMDKFD_IOC_RUNTIME_ENABLE: pass - elif nr == kfd_ioctls.AMDKFD_IOC_GET_VERSION: + if nr == _ioctl_nr(kfd.AMDKFD_IOC_ACQUIRE_VM): pass + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_RUNTIME_ENABLE): pass + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_GET_VERSION): struct.major_version = 1 struct.minor_version = 14 - elif nr == kfd_ioctls.AMDKFD_IOC_ALLOC_MEMORY_OF_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_ALLOC_MEMORY_OF_GPU): if struct.gpu_id not in self.gpus: return -1 struct.handle = self._alloc_handle() self.object_by_handle[struct.handle] = copy.deepcopy(struct) # save memory struct to know what mem it is # Track signal memory (uncached + coherent) - progress queues when written to if struct.flags & kfd.KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED: self.track_address(struct.va_addr, struct.va_addr + struct.size, lambda mv,off: None, lambda mv, off: self._emulate_execute()) - elif nr == kfd_ioctls.AMDKFD_IOC_FREE_MEMORY_OF_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU): self.object_by_handle.pop(struct.handle) - elif nr == kfd_ioctls.AMDKFD_IOC_MAP_MEMORY_TO_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU): dev_ids = (ctypes.c_int32 * struct.n_devices).from_address(struct.device_ids_array_ptr) for i in range(struct.n_devices): gpu = self.gpus[dev_ids[i]] mem_obj = self.object_by_handle[struct.handle] gpu.map_range(mem_obj.va_addr, mem_obj.size) struct.n_success = i + 1 - elif nr == kfd_ioctls.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU): dev_ids = (ctypes.c_int32 * struct.n_devices).from_address(struct.device_ids_array_ptr) for i in range(struct.n_devices): gpu = self.gpus[dev_ids[i]] mem_obj = self.object_by_handle[struct.handle] gpu.unmap_range(mem_obj.va_addr, mem_obj.size) struct.n_success = i + 1 - elif nr == kfd_ioctls.AMDKFD_IOC_CREATE_EVENT: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_CREATE_EVENT): struct.event_slot_index = self._alloc_next_event_slot() struct.event_id = struct.event_slot_index if struct.event_type == kfd.KFD_IOC_EVENT_MEMORY: self.mmu_event_ids.append(struct.event_id) - elif nr == kfd_ioctls.AMDKFD_IOC_CREATE_QUEUE: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_CREATE_QUEUE): gpu = self.gpus[struct.gpu_id] if struct.queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA: gpu.add_sdma_queue(struct.ring_base_address, struct.ring_size, struct.read_pointer_address, struct.write_pointer_address) @@ -162,7 +159,7 @@ def kfd_ioctl(self, req, argp): # Track writes to doorbell, calling callback struct.doorbell_offset = self._alloc_doorbell(struct.gpu_id) self.track_address(struct.doorbell_offset, struct.doorbell_offset + 8, lambda mv,off: None, lambda mv, off: self._emulate_execute()) - elif nr == kfd_ioctls.AMDKFD_IOC_WAIT_EVENTS: + elif nr == _ioctl_nr(kfd.AMDKFD_IOC_WAIT_EVENTS): evs = (kfd.struct_kfd_event_data * struct.num_events).from_address(struct.events_ptr) for ev in evs: if ev.event_id in self.mmu_event_ids and "MOCKGPU_EMU_FAULTADDR" in os.environ: @@ -170,11 +167,7 @@ def kfd_ioctl(self, req, argp): ev.memory_exception_data.va = int(os.environ["MOCKGPU_EMU_FAULTADDR"], 16) ev.memory_exception_data.failure.NotPresent = 1 else: - name = "unknown" - for k,v in kfd_ioctls.__dict__.items(): - if nr == v: name = k - assert False, f"unknown kfd ioctl, {nr} {name}" - exit(1) + raise RuntimeError(f"unsupported kfd ioctl, {nr} {name}") return 0 def _emulate_execute(self): From f19d89e29ac9da39044c0271c0b8c656052e1231 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:24:59 +0300 Subject: [PATCH 229/306] hcq2: a bit cleaner deps (#17771) --- tinygrad/runtime/support/hcq2.py | 180 +++++++++++++++---------------- 1 file changed, 84 insertions(+), 96 deletions(-) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index d8bd869e2355b..18059762f3f29 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import cast, TypeVar, Generic, Any, Sequence, Iterable import struct, functools, time, collections, itertools, decimal, statistics -from dataclasses import replace, dataclass +from dataclasses import replace, dataclass, field from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE from tinygrad.helpers import to_tuple, round_up, partition, panic, ContextVar, perf_counter_us, Context from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker @@ -158,136 +158,127 @@ def _key(buf:Any) -> tuple[Any, int, int]: if isinstance(buf, UOp) and buf.op is Ops.MSELECT: buf = buf.src[0] return (buf.arg.slot, 0, buf.max_numel() * buf.dtype.itemsize) if isinstance(buf, UOp) else DepsTracker._key(buf) +@dataclass(frozen=True) +class Dep: dev:str; queue:str; tag:int; lane:int # producer submit (dev, queue, tag) awaited by consumer lane # noqa: E702 + +@dataclass +class BatchCtx: + batch:list[tuple[UOp, tuple[str, ...]]]; batch_info:list[tuple[tuple[str, ...], str]]; profile:bool # noqa: E702 + tracker:HCQDepsTracker = field(default_factory=HCQDepsTracker); signal_tags:set[int] = field(default_factory=set) # noqa: E702 + slots:dict[str, int] = field(default_factory=lambda: collections.defaultdict(lambda: next(UOp.unique_num))) + def _get_call_bufs_by_lane(call:UOp, devices:tuple[str, ...]) -> list[list[Any]]: return [[b if (b:=_lane(a, lane)).op is Ops.PARAM or (b.op is Ops.MSELECT and b.src[0].op is Ops.PARAM) else b.buffer for a in get_call_arg_uops(call)] for lane in range(len(devices))] -def _get_deps(ctx:DepsTracker, bufs_by_lane:list[list[Any]], write, key:tuple[tuple[str, ...], str, int]) -> list[tuple[tuple, int, int]]: - dep_lanes:list[tuple[tuple, int, int]] = [] +def _wait_ins(ctx:BatchCtx, bufs_by_lane:list[list[Any]], write, devices:tuple[str, ...], queue:str, tag:int) -> list[UOp]: + deps:list[Dep] = [] for lane, bufs in enumerate(bufs_by_lane): written = write if write is not None else list(range(len(bufs))) - dep_lanes += [(dep, dlane, lane) for dep, dlane in ctx.access_resources(bufs, written, (key, lane))] - return dep_lanes + deps += [Dep(d, q, t, lane) for d, q, t in ctx.tracker.access_resources(bufs, written, (devices[lane], queue, tag)) if t < tag] -def _build_wait_ins(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int]], devices:tuple[str, ...], queue:str) -> tuple[list[UOp], set[int]]: - # opt1: same-queue ops are fifo-ordered + # same-queue submits are fifo-ordered, no wait needed if devices[0].split(":")[0] in {"AMD", "QCOM", "CPU"} or queue.startswith("COPY"): - dep_lanes = [(dep, dlane, lane) for dep, dlane, lane in dep_lanes if (dep[0][dlane], dep[1]) != (devices[lane], queue)] - - # opt2: keep latest dep per (dep device, queue, cur lane) - latest = {((dep[0][dlane], dep[1]), lane): (dep, dlane) for dep, dlane, lane in sorted(dep_lanes, key=lambda x: x[0][2])} - deps:dict[tuple, dict[int, list[int]]] = collections.defaultdict(lambda: collections.defaultdict(list)) - for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane].append(dlane) + deps = [d for d in deps if (d.dev, d.queue) != (devices[d.lane], queue)] + latest = {(d.dev, d.queue, d.lane): d for d in sorted(deps, key=lambda d: d.tag)} + # keep only the latest signal + rows:dict[tuple[str, int], dict[int, list[str]]] = collections.defaultdict(lambda: collections.defaultdict(list)) + for d in latest.values(): rows[(d.queue, d.tag)][d.lane].append(d.dev) waits = [] - for (ddevs, dqueue, dtag), by_lane in deps.items(): - for ls in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): - s = UOp.mstack(*[make_buf(d, tag="sentinel_signal") if dl is None else make_buf(ddevs[dl], slots[dqueue]) for dl, d in zip(ls, devices)]) - waits.append(UOp(Ops.INS, arg="wait", src=(s, UOp.const(dtag + 1, dtypes.uint64)))) - return waits, {dtag for _, _, dtag in deps} - -def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[tuple[tuple[str, ...], str]], - tracker:HCQDepsTracker, slots:dict[str, int]) -> tuple[list[UOp], list[UOp], set[int]]: + for (dqueue, dtag), by_lane in rows.items(): + for ds in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): + sig = UOp.mstack(*[make_buf(d, tag="sentinel_signal") if dd is None else make_buf(dd, ctx.slots[dqueue]) for dd, d in zip(ds, devices)]) + waits.append(UOp(Ops.INS, arg="wait", src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) + ctx.signal_tags |= {t for _, t in rows} + return waits + +def _merge_submits(calls:list[UOp]) -> UOp: # TODO: simplify? + if len(calls) == 1: return calls[0] + devs, queue = get_submit(calls[0]).src[0].arg + body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() + return make_call(f"submit {queue} ({len(calls)})", body, replace(calls[0].arg.aux, + estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) + +def _merge_queues(submits:list[UOp]) -> list[UOp]: + merged:list[UOp] = [] + opened:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> hcq calls in submit order + limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) + for call in submits: + devs, queue = key = get_submit(call).src[0].arg + if (group:=opened.pop(key, None)) is None: + # first submit on this queue: close open groups on the same queue with shared devices, so submit order is kept + for k in [k for k in opened if k[1] == queue and set(k[0]) & set(devs)]: merged.append(_merge_submits(opened.pop(k))) + group = [] + elif limits[key] and len(group) >= limits[key]: merged, group, limits[key] = merged + [_merge_submits(group)], [], limits[key] * 2 + opened[key] = group + [call] + return merged + [_merge_submits(g) for g in opened.values()] + +def _make_finalizers(ctx:BatchCtx) -> tuple[list[UOp], list[UOp], list[UOp]]: # collect all buffers which belong to devices dev_bufs:dict[str, dict[int, Any]] = collections.defaultdict(dict) - for call, devices in batch: + for call, devices in ctx.batch: for b in itertools.chain.from_iterable(_get_call_bufs_by_lane(call, devices)): for bd in to_tuple(b.device): dev_bufs[bd][id(b)] = b - n, fences, resets, fins, signal_tags = len(batch_info), [], [], [], set() + n, fences, resets, fins = len(ctx.batch_info), [], [], [] for _, devgroup in itertools.groupby(sorted(dev_bufs), key=lambda d: d.split(":")[0]): - devs = tuple(devgroup) - - # to finalize the batch, sync all accesses from other devices to buffers that belong to this device - fin_deps = [dl for dl in _get_deps(tracker, [list(dev_bufs[d].values()) for d in devs], None, key=(devs, "COMPUTE:0", n)) if dl[0][2] < n] - waits, cur_signal_tags = _build_wait_ins(slots, fin_deps, devs, "COMPUTE:0") - signal_tags |= cur_signal_tags - - # wait the syncs and signal the device epoch, then bump the timeline on the host + sched_epoch = make_buf(devs:=tuple(devgroup), next(UOp.unique_num), tag="epoch") tl_signal, tl_value = make_buf(devs, tag="timeline_signal"), make_buf(devs, tag="timeline_value") - fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") - epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() - # fence once per device group on this schedule's previous epoch - qs = dedup([qn for bdevs, qn in batch_info if set(bdevs) & set(devs)]) - sched_epoch = make_buf(devs, next(UOp.unique_num), tag="epoch") + # fence: spin until the device timeline reaches this schedule's previous epoch + done = tl_signal.after(loop:=UOp.loop(0)).index(0).load() + fences.append(make_call("hcq_fence", UOp.sink(done.end(loop, done < sched_epoch.index(0).load())), HCQInfo(devs))) - wait_device_epoch = (done:=tl_signal.after(loop:=UOp.loop(0)).index(0).load()).end(loop, done < sched_epoch.index(0).load()) - fences.append(make_call("hcq_fence", UOp.sink(wait_device_epoch), HCQInfo(devs))) - - # queues of other groups wait on these signals, reset them after every group reached its epoch - rst = functools.reduce(lambda a,q: a+(make_buf(devs, slots[q]).after(*a[-1:]).index(0).store(0),), qs, cast(tuple[UOp, ...], ())) + # reset: queues of other groups wait on this group's signals, zero them only after every group reached its epoch + qs = dedup([qn for bdevs, qn in ctx.batch_info if set(bdevs) & set(devs)]) + rst:tuple[UOp, ...] = () + for q in qs: rst += (make_buf(devs, ctx.slots[q]).after(*rst[-1:]).index(0).store(0),) if rst: resets.append(make_call("hcq_reset", UOp.sink(*rst), HCQInfo(devs))) - fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) - return fences + resets, fins, signal_tags -def _merged_hcq_call(calls:list[UOp]) -> UOp: # TODO: simplify? - if len(calls) == 1: return calls[0] - devs, queue = get_submit(calls[0]).src[0].arg - body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() - return make_call(f"submit {queue} ({len(calls)})", body, replace(calls[0].arg.aux, - estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) - -def _merge_queues(submits:list[UOp]) -> list[UOp]: - new_src:list[UOp] = [] - opened_qs:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> list of hcq calls, kept in submit order - limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) - - for call in submits: - devs, queue = get_submit(call).src[0].arg - if (old:=opened_qs.pop(key:=(devs, queue), None)) is not None: - if limits[key] and len(old) >= limits[key]: new_src, old, limits[key] = new_src + [_merged_hcq_call(old)], [], limits[key] * 2 - new_rec = old + [call] - else: - # no such queue opened: close every open submit on this queue that shares a device, so submit order is kept - closing = [k for k in opened_qs if k[1] == queue and set(k[0]) & set(devs)] - new_src += [_merged_hcq_call(opened_qs.pop(k)) for k in closing] - new_rec = [call] - opened_qs[(devs, queue)] = new_rec - return new_src + [_merged_hcq_call(c) for c in opened_qs.values()] - -def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: - batch_info = [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch] - - # schedule deps - signal_tags:set[int] = set() - slots:dict[str, int] = collections.defaultdict(lambda: next(UOp.unique_num)) - deps_tracker = HCQDepsTracker() - call_waits:list[list[UOp]] = [] - for tag, ((call, _), (devices, queue)) in enumerate(zip(batch, batch_info)): - deps = _get_deps(deps_tracker, _get_call_bufs_by_lane(call, devices), get_call_outs_ins(call)[0], key=(devices, queue, tag)) - cmds, cur_signal_tags = _build_wait_ins(slots, deps, devices, queue) - call_waits.append(cmds) - signal_tags |= cur_signal_tags - - # build fences and finalizers - fences, fins, finalizer_signal_tags = _build_finalizers(batch, batch_info, deps_tracker, slots) - signal_tags |= finalizer_signal_tags + # finalizer: bump the host timeline and remember this schedule's epoch for the next fence + waits = _wait_ins(ctx, [list(dev_bufs[d].values()) for d in devs], None, devs, "COMPUTE:0", n) + fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") + epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() + fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) + return fences, resets, fins +def _emit_submits(ctx:BatchCtx, call_waits:list[list[UOp]]) -> tuple[list[UOp], list[tuple]]: + # one submit per call: timeline sync on first queue use, timestamps, the call, and a signal if someone waits on it src, kerns = [], [] - for tag, ((call, _), (devices, queue), q) in enumerate(zip(batch, batch_info, call_waits)): + for tag, ((call, _), (devices, queue), q) in enumerate(zip(ctx.batch, ctx.batch_info, call_waits)): # first queue use, sync prior device work with the device timeline - if batch_info.index((devices, queue)) == tag: + if (devices, queue) not in ctx.batch_info[:tag]: epoch = make_buf(devices, tag="timeline_value").index(0) - 1 q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_buf(devices, tag="timeline_signal"), epoch))] + q # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) - ts_ids = [next(UOp.unique_num) for _ in range(2)] if profile else [] + ts_ids = [next(UOp.unique_num) for _ in range(2)] if ctx.profile else [] kerns.append((devices, name, info.estimates, tuple(ts_ids), make_call(name, call.src[0], info).key)) ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_buf(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] # signal the queue if someone waits for us - if tag in signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_buf(devices, slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] + if tag in ctx.signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_buf(devices, ctx.slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] src.append(make_call(f"submit {name}", make_submit(*q, devs=devices, queue=queue).sink(), info)) + return src, kerns + +def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> list[UOp]: + ctx = BatchCtx(batch, [(devices, "COMPUTE:0" if call.src[0].op is Ops.PROGRAM else "COPY:0") for call, devices in batch], profile) + + call_waits = [_wait_ins(ctx, _get_call_bufs_by_lane(call, devices), get_call_outs_ins(call)[0], devices, queue, tag) + for tag, ((call, _), (devices, queue)) in enumerate(zip(ctx.batch, ctx.batch_info))] + fences, resets, fins = _make_finalizers(ctx) + submits, kerns = _emit_submits(ctx, call_waits) # append batch timestamps to finalizers fins = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, kernels=tuple(x for x in kerns if set(x[0]) & set(a.device))))) for f in fins] - return fences + _merge_queues(src) + fins + return fences + resets + _merge_queues(submits) + fins -def sched_hcq_batches(l:UOp, profile:bool) -> UOp: +@rewrite_group(new_ctx=False) +def sched_batches(l:UOp, profile:bool) -> UOp: srcs:list[UOp] = [] batch:list[tuple[UOp, tuple[str, ...]]] = [] for call in l.src: @@ -295,9 +286,6 @@ def sched_hcq_batches(l:UOp, profile:bool) -> UOp: else: srcs, batch = srcs + _finalize_batch(batch, profile) + [call], [] return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) -pm_schedule_and_merge = PatternMatcher([(UPat(Ops.LINEAR, name="l"), - lambda ctx, l: sched_hcq_batches(l, ctx[1]).substitute(ctx[0], walk=True, enter_calls=True))]) - # ***************** # 4.2. hcq lowering: ops to ir @@ -486,8 +474,8 @@ def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: linear = linear.substitute(back_map:={s.param_like(i): s for i,s in enumerate(input_uops)} if input_uops is not None else {}, walk=True) linear = graph_rewrite(linear, pm_insert_copy_staging+pm_flatten_linear, name="insert copy staging") - # schedule - linear = graph_rewrite(linear, pm_schedule_and_merge, ctx=({s:p for p,s in back_map.items()}, profile), walk=True, name="schedule and merge hcq") + # schedule on real buffers + linear = sched_batches(linear, profile).substitute({s:p for p,s in back_map.items()}, walk=True, enter_calls=True) # lower to hcq programs, then pack the programs of every batch into one C submitter (needs a C runtime device for the program addresses) linear = hcq_lower(linear, pm_encode_cmdbufs+pm_pack_placeholders) From 5c3d0444650599afb4e39ec3bcc91860c8a4b6f1 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Thu, 27 Aug 2026 09:51:17 -0400 Subject: [PATCH 230/306] fix asinh gradient at zero (#17758) --- test/backend/test_ops.py | 1 + tinygrad/mixin/elementwise.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 303a7cc37d72b..d0263b9f5b982 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1813,6 +1813,7 @@ def test_asinh(self): helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=300, high=303) helper_test_op([(45,65)], lambda x: x.asinh(), grad_atol=1e-6, low=-1e10, high=-1e9) + helper_test_op(None, lambda x: x.asinh(), grad_atol=1e-6, vals=[[-1.0, 0.0, 1.0]]) def test_acosh(self): helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-6) helper_test_op([(45,65)], lambda x: x.acosh(), grad_atol=1e-3, grad_rtol=1e-2, low=-300, high=-297) diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index dec78a65e0e20..724b022d6ed71 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -870,7 +870,7 @@ def asinh(self) -> Self: print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).asinh().numpy()) ``` """ - return self.sign() * (self.abs() + (self.square() + 1).sqrt()).log() + return (sg:=(self<0).where(-1.0, 1.0)) * (self*sg + (self.square() + 1).sqrt()).log() def acosh(self) -> Self: """ From 7eb763a3c290c96972dbabfa24acfb87d05d80e2 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:29:17 +0300 Subject: [PATCH 231/306] bnxt to extra (#17772) * bnxt to extra * x * x * les --- .github/workflows/autogen.yml | 2 +- extra/bnxt_driver/bnxtdev.py | 238 ++ extra/bnxt_driver/connect.py | 118 + extra/bnxt_driver/loopback.py | 46 + test/unit/test_bnxt.py | 115 + test/unit/test_bnxt_transport.py | 40 + tinygrad/runtime/autogen/__init__.py | 17 + tinygrad/runtime/autogen/bnxt.py | 5407 ++++++++++++++++++++++++ tinygrad/runtime/support/mlx/mlxdev.py | 6 +- tinygrad/runtime/support/system.py | 2 + 10 files changed, 5986 insertions(+), 5 deletions(-) create mode 100644 extra/bnxt_driver/bnxtdev.py create mode 100644 extra/bnxt_driver/connect.py create mode 100644 extra/bnxt_driver/loopback.py create mode 100644 test/unit/test_bnxt.py create mode 100644 test/unit/test_bnxt_transport.py create mode 100644 tinygrad/runtime/autogen/bnxt.py diff --git a/.github/workflows/autogen.yml b/.github/workflows/autogen.yml index 6b3fb964b79b5..a6d0e7eb2306d 100644 --- a/.github/workflows/autogen.yml +++ b/.github/workflows/autogen.yml @@ -54,7 +54,7 @@ jobs: python3 -c "from tinygrad.runtime.autogen import mesa" python3 -c "from tinygrad.runtime.autogen import avcodec" python3 -c "from tinygrad.runtime.autogen import llvm_qcom" - python3 -c "from tinygrad.runtime.autogen import mlx5" + python3 -c "from tinygrad.runtime.autogen import mlx5, bnxt" python3 -c "from tinygrad.runtime.autogen import ggml_common" REGEN=1 python3 -c "from tinygrad.runtime.autogen import libclang" - name: Check for differences diff --git a/extra/bnxt_driver/bnxtdev.py b/extra/bnxt_driver/bnxtdev.py new file mode 100644 index 0000000000000..bf898625d07de --- /dev/null +++ b/extra/bnxt_driver/bnxtdev.py @@ -0,0 +1,238 @@ +import ctypes, struct +from tinygrad.helpers import ceildiv, getenv, wait_cond, DEBUG +from tinygrad.runtime.autogen import bnxt, pci +from tinygrad.runtime.support.system import PCIDevice, System, ipv4_to_gid + +BNXT_DEBUG = getenv("BNXT_DEBUG", 0) +BNXT_ACCESS, BNXT_INIT_MASK, BNXT_RTR_MASK, BNXT_RTS_MASK = 3, 0xd, 0x41515ad, 0xae005 +BNXT_CHIMP_COMM, BNXT_CHIMP_COMM_TRIGGER = 0x0, 0x100 +BNXT_BACKING_STORE = ((0, 2), (1, 0), (2, 2), (3, 0), (4, 2), (5, 0), (6, 0), (14, 2), (15, 0)) + +def db_value(xid, typ, index, epoch): + return (xid & bnxt.DBC_DBC_XID_MASK | bnxt.DBC_DBC_PATH_ROCE | typ | bnxt.BNXT_QPLIB_DBR_VALID) << 32 | \ + index & bnxt.DBC_DBC_INDEX_MASK | epoch << bnxt.BNXT_QPLIB_DBR_EPOCH_SHIFT + +def _pbl(dev, paddrs, queue=False): + if len(paddrs) == 1: return 0, paddrs[0] + values = [p | bnxt.PTU_PTE_VALID for p in paddrs] + if queue: + values[-1] |= bnxt.PTU_PTE_LAST + if len(values) > 1: values[-2] |= bnxt.PTU_PTE_NEXT_TO_LAST + table, table_paddrs = dev.pci_dev.alloc_sysmem(ceildiv(len(values), 512) * 0x1000) + table[:len(values) * 8] = struct.pack(f"<{len(values)}Q", *values) + if len(table_paddrs) == 1: return 1, table_paddrs[0] + top, top_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + top[:len(table_paddrs) * 8] = struct.pack(f"<{len(table_paddrs)}Q", *(p | bnxt.PTU_PTE_VALID for p in table_paddrs)) + return 2, top_paddrs[0] + +def _queue(dev, stride:int=16, aux=False): + mem, paddrs = dev.pci_dev.alloc_sysmem(0x1000 + aux * 0x400) + level, base = _pbl(dev, paddrs, queue=True) + return {"mem":mem, "paddrs":paddrs, "stride":stride, "prod":0, "cons":0, "level":level, "base":base} + +def _qread(q, i): + off = (i & 15) * q["stride"] + return q["mem"][off:off + q["stride"]] + +def _qwrite(q, i, data, aux=False): + off = 0x1000 + i % 128 * 8 if aux else (i & 15) * q["stride"] + q["mem"][off:off + len(data)] = data + +class BNXTDev: + def __init__(self, pci_dev:PCIDevice, ip:str=getenv("BNXT_IP", "10.0.0.1")): + self.pci_dev, self.devfmt = pci_dev, pci_dev.pcibus + self.bar0, self.db = pci_dev.map_bar(0, fmt='I'), pci_dev.map_bar(2, fmt='Q') + pci_dev.write_config(pci.PCI_COMMAND, pci_dev.read_config(pci.PCI_COMMAND, 2) | pci.PCI_COMMAND_MASTER, 2) + self.resp, self.resp_pa = pci_dev.alloc_sysmem(0x1000) + self.seq = 0 + + ver = self.hwrm("ver_get") + if DEBUG >= 2: print(f"bnxt {self.devfmt}: firmware {ver.hwrm_fw_maj_8b}.{ver.hwrm_fw_min_8b}.{ver.hwrm_fw_bld_8b}") + self.hwrm("func_reset", timeout_ms=40000) + caps = self.hwrm("func_qcaps", fid=0xffff) + self.mac, self.port_id = int.from_bytes(bytes(caps.mac_address), 'big'), caps.port_id + self.hwrm("func_drv_rgtr") + self.db_off = self.hwrm("func_qcfg", fid=0xffff).legacy_l2_db_size_kb * 1024 + + self.setup_backing_store() + self._open_rcfw() + self._open_l2() + self.local_gid = ipv4_to_gid(ip) + gids, mac = (ctypes.c_uint32 * 4)(*(int.from_bytes(self.local_gid[i:i + 4], 'big') for i in (12, 8, 4, 0))), self.mac.to_bytes(6, 'big') + smac = (ctypes.c_uint16 * 3)(*(int.from_bytes(mac[i:i + 2], 'big') for i in (0, 2, 4))) + self.gid_id = self.rcfw("add_gid", gid=gids, src_mac=smac).xid + + if DEBUG >= 2: print(f"bnxt {self.devfmt}: booted mac={self.mac.to_bytes(6, 'big').hex(':')} gid={self.local_gid.hex()}") + + def hwrm(self, name, timeout_ms=10000, **fields): + inp, out = getattr(bnxt, f"struct_hwrm_{name}_input"), getattr(bnxt, f"struct_hwrm_{name}_output") + opcode = getattr(bnxt, f"HWRM_{name.upper()}") + self.seq = (self.seq + 1) & 0xffff + data = bytes(inp(req_type=opcode, cmpl_ring=bnxt.BNXT_HWRM_NO_CMPL_RING, seq_id=self.seq, target_id=bnxt.BNXT_HWRM_TARGET, + resp_addr=self.resp_pa[0], **fields)) + self.resp[:] = bytes(len(self.resp)) + System.memory_barrier() + for i, w in enumerate(memoryview(bytearray(data.ljust(bnxt.HWRM_MAX_REQ_LEN, b'\0'))).cast('I')): + self.bar0[BNXT_CHIMP_COMM // 4 + i] = w + self.bar0[BNXT_CHIMP_COMM_TRIGGER // 4] = 1 + def hdr(): return bnxt.struct_hwrm_resp_hdr.from_buffer_copy(bytes(self.resp[:8])) + wait_cond(lambda: (n := hdr().resp_len) and hdr().seq_id == self.seq and self.resp[n - 1], timeout_ms=timeout_ms, msg=f"HWRM {name}") + ret = out.from_buffer_copy(bytes(self.resp[:ctypes.sizeof(out)])) + assert ret.error_code == 0, f"HWRM {name}: {ret.error_code}" + return ret + + def setup_backing_store(self): + counts: dict[int, int] = {} + for typ, extra in BNXT_BACKING_STORE: + caps = self.hwrm("func_backing_store_qcaps_v2", type=typ) + size, splits = caps.entry_size, tuple(getattr(caps, f"split_entry_{j}") for j in range(caps.subtype_valid_cnt)) + counts[typ] = n = counts[0] if typ == 15 else max(caps.min_num_entries, sum(splits) + extra) + # a zero bitmap means the type has a single instance 0 + for instance in [i for i in range(8) if caps.instance_bit_map >> i & 1] or [0]: + mem, paddrs = self.pci_dev.alloc_sysmem(ceildiv(n * size, 0x1000) * 0x1000) + if caps.ctx_init_value: + for off in range(caps.ctx_init_offset, len(mem), size): mem[off] = caps.ctx_init_value + lvl, base = _pbl(self, paddrs) + self.hwrm("func_backing_store_cfg_v2", type=typ, instance=instance, entry_size=size, num_entries=n, page_dir=base, + page_size_pbl_level=lvl, subtype_valid_cnt=len(splits), + flags=bnxt.FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE if typ == 15 else 0, + **{f"split_entry_{j}": v for j, v in enumerate(splits)}) + + def _open_rcfw(self): + self.rcfw_first = True + + self.creq = _queue(self) + self.creq_id = self.hwrm("ring_alloc", ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_NQ, page_tbl_addr=self.creq["base"], + page_size=12, page_tbl_depth=self.creq["level"], length=16, int_mode=bnxt.RING_ALLOC_REQ_INT_MODE_MSIX).ring_id + + self.cmdq = _queue(self) + self.doorbell(self.creq_id, bnxt.DBC_DBC_TYPE_NQ_ARM, 0, 0) + init = bnxt.struct_cmdq_init(cmdq_pbl=self.cmdq["base"], creq_ring_id=self.creq_id, + cmdq_size_cmdq_lvl=16 << bnxt.CMDQ_INIT_CMDQ_SIZE_SFT) + + System.memory_barrier() + for i, w in enumerate(memoryview(bytearray(bytes(init))).cast('I')): self.bar0[bnxt.RCFW_COMM_BASE_OFFSET // 4 + i] = w + + _, p = self.pci_dev.alloc_sysmem(0x1000) + self.rcfw("initialize_fw", stat_ctx_id=self.hwrm("stat_ctx_alloc", stats_dma_addr=p[0], stats_dma_length=176).stat_ctx_id, + flags=bnxt.CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED) + + # RoCE notification ring: never armed or serviced, but CQ and L2 ring allocation require one + nq = _queue(self) + self.nq_id = self.hwrm("ring_alloc", ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_NQ, page_tbl_addr=nq["base"], + page_size=12, page_tbl_depth=nq["level"], length=16, logical_id=1, int_mode=bnxt.RING_ALLOC_REQ_INT_MODE_MSIX).ring_id + + def rcfw(self, name, timeout_ms=20000, **fields): + req_t, resp_t = getattr(bnxt, f"struct_cmdq_{name}"), getattr(bnxt, f"struct_creq_{name}_resp") + op = getattr(bnxt, f"CMDQ_BASE_OPCODE_{name.upper()}") + data = bytes(req_t(opcode=op, cmd_size=(slots := ceildiv(ctypes.sizeof(req_t), 16)), **fields)).ljust(slots * 16, b'\0') + for i in range(slots): _qwrite(self.cmdq, self.cmdq["prod"] + i, data[i * 16:(i + 1) * 16]) + + self.cmdq["prod"] += slots + prod = self.cmdq["prod"] & 0xffff + if self.rcfw_first: prod, self.rcfw_first = prod | 1 << bnxt.FIRMWARE_FIRST_FLAG, False + + System.memory_barrier() + + self.bar0[(bnxt.RCFW_COMM_BASE_OFFSET + bnxt.RCFW_PF_VF_COMM_PROD_OFFSET) // 4] = prod + self.bar0[(bnxt.RCFW_COMM_BASE_OFFSET + bnxt.RCFW_COMM_TRIG_OFFSET) // 4] = bnxt.RCFW_CMDQ_TRIG_VAL + + def poll(): + h = bnxt.struct_creq_base.from_buffer_copy(bytes(_qread(self.creq, self.creq["cons"]))) + return bool(h.v & bnxt.CREQ_BASE_V) != bool((self.creq["cons"] // 16) & 1) + wait_cond(poll, timeout_ms=timeout_ms, msg=f"RCFW {name}") + + ret = resp_t.from_buffer_copy(bytes(_qread(self.creq, self.creq["cons"]))) + self.creq["cons"] += 1 + + # NQ_ARM also publishes the CREQ consumer index, which is what frees ring space for the next command + self.doorbell(self.creq_id, bnxt.DBC_DBC_TYPE_NQ_ARM, self.creq["cons"] & 15, (self.creq["cons"] // 16) & 1) + assert ret.status == 0, f"RCFW {name}: {ret.status}" + + if BNXT_DEBUG >= 1: print(f"bnxt {self.devfmt}: rcfw {name} xid={getattr(ret, 'xid', 0):#x}") + return ret + + def doorbell(self, xid, typ, index, epoch): + System.memory_barrier() + self.db[self.db_off // 8] = db_value(xid, typ, index, epoch) + + # L2 receive path, required for RoCE ingress even though no ethernet receive buffers are posted + def _open_l2(self): + cq = _queue(self) + ci = self.hwrm("ring_alloc", enables=bnxt.RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID, ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_L2_CMPL, + page_tbl_addr=cq["base"], page_size=12, page_tbl_depth=cq["level"], length=16, nq_ring_id=self.nq_id).ring_id + rx = _queue(self) + ri = self.hwrm("ring_alloc", enables=bnxt.RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID | + bnxt.RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID, ring_type=bnxt.RING_ALLOC_REQ_RING_TYPE_RX, page_tbl_addr=rx["base"], + page_size=12, page_tbl_depth=rx["level"], length=16, rx_buf_size=640, nq_ring_id=self.nq_id).ring_id + vi = self.hwrm("vnic_alloc").vnic_id + self.hwrm("vnic_cfg", enables=bnxt.VNIC_CFG_REQ_ENABLES_MRU | bnxt.VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | + bnxt.VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID, vnic_id=vi, mru=9018, + default_rx_ring_id=ri, default_cmpl_ring_id=ci) + self.hwrm("cfa_l2_filter_alloc", flags=bnxt.CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX, + enables=bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK | + bnxt.CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID, l2_addr=tuple(self.mac.to_bytes(6, 'big')), l2_addr_mask=(0xff,) * 6, dst_id=vi) + + def register_mem(self, paddrs:list[int], size:int, log_page_size:int=12) -> int: + level, base = _pbl(self, paddrs[:ceildiv(size, 1 << log_page_size)]) + return self.rcfw("register_mr", flags=bnxt.CMDQ_REGISTER_MR_FLAGS_ALLOC_MR, + log2_pg_size_lvl=level << bnxt.CMDQ_REGISTER_MR_LVL_SFT | log_page_size << bnxt.CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT, + access=bnxt.CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE | bnxt.CMDQ_REGISTER_MR_ACCESS_REMOTE_WRITE, + log2_pbl_pg_size=12, pbl=base, va=paddrs[0], mr_size=size).xid + +class BNXTQP: + def __init__(self, dev:BNXTDev): + self.dev, self.sq_psn, self.msn = dev, 0, 0 + + self.cqq = _queue(dev, ctypes.sizeof(bnxt.struct_cq_base)) + self.cq_id = dev.rcfw("create_cq", cq_size=16, pbl=self.cqq["base"], + pg_size_lvl=self.cqq["level"], cq_fco_cnq_id=dev.nq_id).xid + + self.sq = _queue(dev, aux=True) + self.qpn = dev.rcfw("create_qp", type=bnxt.CMDQ_CREATE_QP_TYPE_RC, + sq_size=16, sq_fwo_sq_sge=1, scq_cid=self.cq_id, rcq_cid=self.cq_id, + sq_pbl=self.sq["base"], sq_pg_size_sq_lvl=self.sq["level"]).xid + self.qp_op(1, BNXT_INIT_MASK, access=BNXT_ACCESS, pkey=0xffff) + + def qp_op(self, state, mask, network_type=0, **fields): + self.dev.rcfw("modify_qp", qp_cid=self.qpn, modify_mask=mask, + network_type_en_sqd_async_notify_new_state=state | network_type, **fields) + + def connect(self, qpn:int, gid:bytes, mac:int): + network_type = bnxt.CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 + dgid = (ctypes.c_uint32 * 4)(*(int.from_bytes(gid[i:i + 4], 'little') for i in (0, 4, 8, 12))) + dmac = (ctypes.c_uint16 * 3)(*(int.from_bytes(mac.to_bytes(6, 'big')[i:i + 2], 'little') for i in (0, 2, 4))) + + self.qp_op(2, BNXT_RTR_MASK, network_type=network_type, qp_type=bnxt.CMDQ_MODIFY_QP_QP_TYPE_RC, access=BNXT_ACCESS, + pkey=0xffff, dgid=dgid, sgid_index=self.dev.gid_id, hop_limit=64, dest_mac=dmac, + path_mtu_pingpong_push_enable=bnxt.CMDQ_MODIFY_QP_PATH_MTU_MTU_1024, max_dest_rd_atomic=4, + dest_qp_id=qpn) + self.qp_op(3, BNXT_RTS_MASK, network_type=network_type, qp_type=bnxt.CMDQ_MODIFY_QP_QP_TYPE_RC, access=BNXT_ACCESS, + max_rd_atomic=1) + + if BNXT_DEBUG >= 1: print(f"bnxt: QP {self.qpn:#x} connected (remote={qpn:#x})") + + def _poll(self, timeout): + def poll(): + base = bnxt.struct_cq_base.from_buffer_copy(bytes(_qread(self.cqq, self.cqq["cons"]))) + return bool(base.cqe_type_toggle & bnxt.CQ_BASE_TOGGLE) == (not bool((self.cqq["cons"] // 16) & 1)) + wait_cond(poll, timeout_ms=timeout, msg="BNXT CQ") + raw = bytes(_qread(self.cqq, self.cqq["cons"])) + self.cqq["cons"] += 1 + self.dev.doorbell(self.cq_id, bnxt.DBC_DBC_TYPE_CQ, self.cqq["cons"] & 15, (self.cqq["cons"] // 16) & 1) + return raw + + def rdma_write(self, rva, rkey, lva, lkey, size, timeout_ms=20000): + start = self.sq["prod"] & 15 + hdr = bytes(bnxt.struct_sq_rdma_hdr(wqe_type=bnxt.SQ_RDMA_HDR_WQE_TYPE_WRITE_WQE, + flags=bnxt.SQ_SEND_FLAGS_SIGNAL_COMP, wqe_size=3, length=size, remote_va=rva, remote_key=rkey)) + for i, data in enumerate((hdr[:16], hdr[16:32], bytes(bnxt.struct_sq_sge(va_or_pa=lva, l_key=lkey, size=size)))): + _qwrite(self.sq, start + i, data) + nxt = (self.sq_psn + max(1, ceildiv(size, 1024))) & 0xffffff + value = start << bnxt.SQ_MSN_SEARCH_START_IDX_SFT | nxt << bnxt.SQ_MSN_SEARCH_NEXT_PSN_SFT | self.sq_psn + _qwrite(self.sq, self.msn, struct.pack(" dict[str, Any]: + for line in iter(stream.readline, ""): + print(f" [remote] {line}", end="") + try: value = json.loads(line) + except json.JSONDecodeError: continue + if isinstance(value, dict): return value + raise RuntimeError(f"remote exited before publishing {what}") + +def wait_line(stream:IO[str], text:str) -> str: + for line in iter(stream.readline, ""): + print(f" [remote] {line}", end="") + if text in line: return line + raise RuntimeError(f"remote exited before reporting {text!r}") + +def send_line(stream:IO[str], value:str|dict[str, Any]): + stream.write((json.dumps(value) if isinstance(value, dict) else value) + "\n") + stream.flush() + +def qp_info(dev:BNXTDev, qp:BNXTQP) -> dict[str, Any]: + return {"qpn":qp.qpn, "mac":dev.mac.to_bytes(6, "big").hex(), "gid":dev.local_gid.hex()} + +def server(): + dev = BNXTDev(PCIDevice("bnxt", os.getenv("BNXT_PCI", "0000:41:00.0")), ip=os.getenv("BNXT_IP", REMOTE_IP)) + qp = BNXTQP(dev) + print(json.dumps(qp_info(dev, qp)), flush=True) + + peer = json.loads(sys.stdin.readline()) + qp.connect(peer["qpn"], bytes.fromhex(peer["gid"]), int(peer["mac"], 16)) + print("connected", flush=True) + + target, target_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + target[:0x1000] = bytes(0x1000) + rkey = dev.register_mem(target_paddrs, 0x1000) + print(json.dumps({"target_addr":target_paddrs[0], "rkey":rkey}), flush=True) + + assert sys.stdin.readline().strip() == "done" + received = bytes(target).rstrip(b"\0") + print(f"AS TEXT: {received.decode(errors='replace')!r}", flush=True) + print(json.dumps({"data":received.hex()}), flush=True) + +def sync_remote(): + if os.getenv("SYNC", "1") == "0": return + print("syncing BNXT driver to remote") + subprocess.run(["rsync", "-azR", *SYNC_FILES, f"{REMOTE}:~/tinygrad/"], cwd=TINYGRAD, check=True) + +def start_remote() -> subprocess.Popen[str]: + print("booting remote") + command = (f"cd ~/tinygrad && sudo env PYTHONPATH=. PYTHONUNBUFFERED=1 BNXT_DEBUG={os.getenv('BNXT_DEBUG', '0')} " + f"BNXT_PCI={REMOTE_PCI} BNXT_IP={REMOTE_IP} python3 extra/bnxt_driver/connect.py --server") + return subprocess.Popen(SSH + [command], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stderr, text=True) + +def client(): + assert 0 < len(MESSAGE) <= 0x1000 + sync_remote() + remote = start_remote() + assert remote.stdin is not None and remote.stdout is not None + remote_info = read_json(remote.stdout, "QP information") + print("booting local") + dev = BNXTDev(PCIDevice("bnxt", LOCAL_PCI), ip=LOCAL_IP) + qp = BNXTQP(dev) + + send_line(remote.stdin, qp_info(dev, qp)) + wait_line(remote.stdout, "connected") + qp.connect(remote_info["qpn"], bytes.fromhex(remote_info["gid"]), int(remote_info["mac"], 16)) + print("both QPs in RTS") + + remote_target = read_json(remote.stdout, "MR information") + source, source_paddrs = dev.pci_dev.alloc_sysmem(0x1000) + source[:len(MESSAGE)] = MESSAGE + lkey = dev.register_mem(source_paddrs, 0x1000) + print(f"RDMA WRITE {len(MESSAGE)}B to remote phys 0x{remote_target['target_addr']:x}") + qp.rdma_write(remote_target["target_addr"], remote_target["rkey"], source_paddrs[0], lkey, len(MESSAGE)) + + send_line(remote.stdin, "done") + wait_line(remote.stdout, "AS TEXT") + result = read_json(remote.stdout, "RDMA result") + assert bytes.fromhex(result["data"]) == MESSAGE + print("RDMA WRITE data verified") + + remote.stdin.close() + assert remote.wait() == 0 + print("RDMA WRITE test complete") + +if __name__ == "__main__": + server() if "--server" in sys.argv else client() diff --git a/extra/bnxt_driver/loopback.py b/extra/bnxt_driver/loopback.py new file mode 100644 index 0000000000000..42b25a44a3837 --- /dev/null +++ b/extra/bnxt_driver/loopback.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Local BNXT RoCEv2 RDMA WRITE loopback using the firmware's PHY loopback mode. + +The kernel bnxt_en/bnxt_re modules must be unloaded first. + + sudo PYTHONPATH=. BNXT_PCI=0000:41:00.0 BNXT_IP=10.0.200.5 python3 extra/bnxt_driver/loopback.py +""" +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "../..")) + +from extra.bnxt_driver.bnxtdev import BNXTDev, BNXTQP +from tinygrad.runtime.autogen import bnxt +from tinygrad.runtime.support.system import PCIDevice + +BUF_SIZE = 0x1000 +BNXT_PCI = os.getenv("BNXT_PCI", "0000:41:00.0") +BNXT_IP = os.getenv("BNXT_IP", "10.0.200.5") + +if __name__ == "__main__": + print(f"[init] BNXT at {BNXT_PCI}") + dev = BNXTDev(PCIDevice("bnxt", BNXT_PCI), ip=BNXT_IP) + tx_qp, rx_qp = BNXTQP(dev), BNXTQP(dev) + print(f"[init] loopback-connect TX QP 0x{tx_qp.qpn:x} <-> RX QP 0x{rx_qp.qpn:x}") + tx_qp.connect(rx_qp.qpn, dev.local_gid, dev.mac) + rx_qp.connect(tx_qp.qpn, dev.local_gid, dev.mac) + + src, src_paddrs = dev.pci_dev.alloc_sysmem(BUF_SIZE) + dst, dst_paddrs = dev.pci_dev.alloc_sysmem(BUF_SIZE) + message = b"Hello from BNXT RoCE PHY loopback!" + src[:BUF_SIZE], dst[:BUF_SIZE] = bytes(BUF_SIZE), bytes(BUF_SIZE) + src[:len(message)] = message + lkey = dev.register_mem(src_paddrs, BUF_SIZE) + rkey = dev.register_mem(dst_paddrs, BUF_SIZE) + + print("[loopback] enabling local PHY loopback") + dev.hwrm("port_phy_cfg", port_id=dev.port_id, enables=bnxt.PORT_PHY_CFG_REQ_ENABLES_LPBK, lpbk=bnxt.PORT_PHY_CFG_REQ_LPBK_LOCAL) + time.sleep(1) + tx_qp.rdma_write(dst_paddrs[0], rkey, src_paddrs[0], lkey, len(message)) + got = bytes(dst[:len(message)]) + print(f"[result] {got!r}") + assert got == message + print("BNXT RoCE PHY loopback RDMA WRITE passed") + dev.hwrm("port_phy_cfg", port_id=dev.port_id, enables=bnxt.PORT_PHY_CFG_REQ_ENABLES_LPBK, lpbk=bnxt.PORT_PHY_CFG_REQ_LPBK_NONE) diff --git a/test/unit/test_bnxt.py b/test/unit/test_bnxt.py new file mode 100644 index 0000000000000..0716da3daec52 --- /dev/null +++ b/test/unit/test_bnxt.py @@ -0,0 +1,115 @@ +import struct, unittest +from types import SimpleNamespace +from unittest.mock import patch + +from tinygrad.runtime.autogen import bnxt +from extra.bnxt_driver.bnxtdev import BNXT_BACKING_STORE, BNXTDev, BNXTQP, _queue, _qwrite, ipv4_to_gid + +class FakePCI: + def __init__(self): self.next_addr, self.allocations = 0x100000, [] + def alloc_sysmem(self, size, contiguous=False): + pages = [self.next_addr+i*0x1000 for i in range((size+0xfff)//0x1000)] + self.next_addr += len(pages)*0x1000 + self.allocations.append(mem := bytearray(size)) + return mem, pages + +class FakeDev: + def __init__(self): self.pci_dev, self.calls = FakePCI(), [] + def hwrm(self, name, **fields): + self.calls.append((name, fields)) + typ = fields.get("type", 0) + return SimpleNamespace(ctx_init_value=0x5a, ctx_init_offset=4, entry_size=16 if typ == 0 else 4, + subtype_valid_cnt=typ == 0, split_entry_0=2, instance_bit_map=5 if typ == 0 else 1, min_num_entries=0) + +class FakeRCFW: + def __init__(self): self.calls, self.doorbells = [], [] + def exec(self, name, **fields): + self.calls.append((name, fields)) + return SimpleNamespace(xid={"create_cq":77, "create_qp":88, "register_mr":0x5678}.get(name, 0)) + def doorbell(self, *args, **kwargs): self.doorbells.append((args, kwargs)) + +class FakeQPDev: + def __init__(self): self.pci_dev, self.fw, self.gid_id, self.nq_id = FakePCI(), FakeRCFW(), 9, 41 + def rcfw(self, *args, **kwargs): return self.fw.exec(*args, **kwargs) + def doorbell(self, *args, **kwargs): self.fw.doorbell(*args, **kwargs) + +class TestMemory(unittest.TestCase): + def test_cmdq_and_sq_aux(self): + dev = FakeDev() + cmdq, sq = _queue(dev), _queue(dev, aux=True) + self.assertEqual((cmdq["level"], cmdq["base"]), (0, 0x100000)) + _qwrite(sq, 3, b"ABCDEFGH", aux=True) + self.assertEqual(bytes(sq["mem"][0x1018:0x1020]), b"ABCDEFGH") + + def test_f320_backing_layout_and_final_marker(self): + self.assertEqual(len(BNXT_BACKING_STORE), 9) + dev = FakeDev() + small = ((0, 6), (15, 0)) + with patch("extra.bnxt_driver.bnxtdev.BNXT_BACKING_STORE", small): BNXTDev.setup_backing_store(dev) + cfg = [fields for name, fields in dev.calls if name == "func_backing_store_cfg_v2"] + self.assertEqual([(x["type"], x["instance"]) for x in cfg], [(0, 0), (0, 2), (15, 0)]) + self.assertTrue(all(not x["flags"] for x in cfg[:-1])) + self.assertEqual(cfg[-1]["flags"], bnxt.FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE) + self.assertEqual((dev.pci_dev.allocations[0][4], dev.pci_dev.allocations[0][20]), (0x5a, 0x5a)) + +class TestRCFW(unittest.TestCase): + def setUp(self): + patch("extra.bnxt_driver.bnxtdev.System.memory_barrier").start() + self.addCleanup(patch.stopall) + + def test_doorbell_encodes_xid_type_and_index(self): + dev = BNXTDev.__new__(BNXTDev) + dev.db, dev.db_off = [0]*1024, 0x1000 + dev.doorbell(0x123456, bnxt.DBC_DBC_TYPE_CQ_ARMALL, 0x456, epoch=1) + key = dev.db[0x1000//8] + self.assertEqual(key >> 32, + 0x123456 & bnxt.DBC_DBC_XID_MASK | bnxt.DBC_DBC_PATH_ROCE | bnxt.DBC_DBC_TYPE_CQ_ARMALL | bnxt.BNXT_QPLIB_DBR_VALID) + self.assertEqual(key & 0xffffffff, 0x456 | 1<> 20) ^ ((lqpn * rqpn) >> 40)) & 0xFFFFF return ((v & 0x3FFF) ^ ((v & 0xFC000) >> 14)) | 0xC000 diff --git a/tinygrad/runtime/support/system.py b/tinygrad/runtime/support/system.py index 1f11403d30ef0..b6bd184aad8a7 100644 --- a/tinygrad/runtime/support/system.py +++ b/tinygrad/runtime/support/system.py @@ -10,6 +10,8 @@ MAP_FIXED, MAP_FIXED_NOREPLACE = 0x10, 0x100000 MAP_LOCKED, MAP_POPULATE, MAP_NORESERVE = 0 if OSX else 0x2000, getattr(mmap, "MAP_POPULATE", 0 if OSX else 0x008000), 0x400 +def ipv4_to_gid(ip:str) -> bytes: return bytes(10) + b'\xff\xff' + socket.inet_aton(ip) + class _System: def write_sysfs(self, path:str, value:str, msg:str, expected:str|None=None): if FileIOInterface(path, os.O_RDONLY).read().splitlines()[0] != (expected or value): From 980c42d611c3f304217b8be24eca880a97c1dd71 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:42:37 +0300 Subject: [PATCH 232/306] hcq2 tiny speedups (#17773) --- tinygrad/runtime/support/hcq2.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 18059762f3f29..88c914bf66657 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -245,10 +245,11 @@ def _make_finalizers(ctx:BatchCtx) -> tuple[list[UOp], list[UOp], list[UOp]]: def _emit_submits(ctx:BatchCtx, call_waits:list[list[UOp]]) -> tuple[list[UOp], list[tuple]]: # one submit per call: timeline sync on first queue use, timestamps, the call, and a signal if someone waits on it - src, kerns = [], [] + src, kerns, seen_queues = [], [], set() for tag, ((call, _), (devices, queue), q) in enumerate(zip(ctx.batch, ctx.batch_info, call_waits)): # first queue use, sync prior device work with the device timeline - if (devices, queue) not in ctx.batch_info[:tag]: + if (devices, queue) not in seen_queues: + seen_queues.add((devices, queue)) epoch = make_buf(devices, tag="timeline_value").index(0) - 1 q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_buf(devices, tag="timeline_signal"), epoch))] + q @@ -351,14 +352,14 @@ def split_patches(call:UOp) -> UOp|None: # split patches. addresses read in the body go through the tables too lanes = len(to_tuple(call.arg.aux.device)) - inputs, internals = partition(dedup([g for p in rt_patches for g in get_getaddrs(p)] + get_getaddrs(body)), is_input_addr) + inputs, internals = partition(dedup(get_getaddrs(UOp.sink(body, *rt_patches))), is_input_addr) runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) tables = [make_addr_table(call, gs, n, lanes if n == "inputs" else 1) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec ipatches = [p for p in rt_patches if p.tag == "inputs" and all(v in tables[0][3] for v in p.src[1].src)] # only getaddrs go to the table gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches, lanes) if ipatches else {} - body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches}).substitute(reads) + body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches} | reads, walk=True) lt_srcs = collections.defaultdict(list) for p in lt_patches: lt_srcs[p.buf_uop].append(p) From 54d69f439fa371506e5daa6600067ec22a8e8f55 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 27 Aug 2026 10:47:21 -0400 Subject: [PATCH 233/306] fix test_assign for no subbuffer backends (#17757) * test_assign to test/backend * fix --- .pre-commit-config.yaml | 2 +- test/{unit => backend}/test_assign.py | 79 ++++++++++++++++++++++++++- test/unit/test_multitensor.py | 76 -------------------------- tinygrad/schedule/prepare.py | 7 +-- 4 files changed, 80 insertions(+), 84 deletions(-) rename test/{unit => backend}/test_assign.py (92%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5fdd5aaed665..380ddca53d38d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: pass_filenames: false - id: tests name: comprehensive test suite - entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/backend/test_ops.py test/backend/test_schedule.py test/unit/test_assign.py test/backend/test_tensor.py test/backend/test_jit.py test/unit/test_schedule_cache.py test/null/test_pattern_matcher.py test/null/test_uop_symbolic.py test/unit/test_helpers.py + entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/backend/test_ops.py test/backend/test_schedule.py test/backend/test_assign.py test/backend/test_tensor.py test/backend/test_jit.py test/unit/test_schedule_cache.py test/null/test_pattern_matcher.py test/null/test_uop_symbolic.py test/unit/test_helpers.py language: system always_run: true pass_filenames: false diff --git a/test/unit/test_assign.py b/test/backend/test_assign.py similarity index 92% rename from test/unit/test_assign.py rename to test/backend/test_assign.py index 61b9b5266dc43..d10d8888c8536 100644 --- a/test/unit/test_assign.py +++ b/test/backend/test_assign.py @@ -1,10 +1,10 @@ #!/usr/bin/env python import unittest import numpy as np -from tinygrad import dtypes, Tensor, TinyJit, GlobalCounters, Variable +from tinygrad import Device, dtypes, Tensor, TinyJit, GlobalCounters, Variable from tinygrad.uop.ops import Ops, UOp from tinygrad.helpers import temp, DEV, Context -from test.helpers import assert_kernel_count +from test.helpers import assert_kernel_count, needs_second_gpu N = 200 # has to be bigger than the cache to fail @@ -1079,5 +1079,80 @@ def test_running_stats_are_realized(self): with Context(TRAINING=1): bn(x).realize() self.assertTrue(bn.running_mean.uop.base.is_realized) +class TestMultiAssign(unittest.TestCase): + device = tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) + + @needs_second_gpu + def setUp(self): pass + + def test_multi_assign_realized(self): + out = Tensor.zeros(4).shard(self.device, 0).contiguous().realize() + ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_unrealized(self): + out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_both_unrealized(self): + out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4).contiguous().realize().shard(self.device, 0) + out.assign(ones).realize() + self.assertListEqual(out.tolist(), [1,1,1,1]) + + def test_multi_assign_scalar(self): + out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(0).realize() + self.assertListEqual(out.tolist(), [0,0,0,0]) + + def test_multi_assign_const_like(self): + out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() + out.assign(out.const_like(7)).realize() + self.assertListEqual(out.tolist(), [7,7,7,7]) + + def test_multi_assign_piece(self): + out = Tensor.zeros(4,4).shard(self.device, 0).contiguous().realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_piece_noncontig(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + @unittest.expectedFailure + def test_multi_assign_piece_unrealized(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0) + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + out[:, 2:3].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_var_offset(self): + out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() + ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() + vi = Variable("i", 0, 3).bind(2) + out[:, vi:vi+1].assign(ones).realize() + self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) + + def test_multi_assign_var_offset_jit_none(self): self.test_multi_assign_var_offset_jit(None) + def test_multi_assign_var_offset_jit(self, shard_axis=0): + out = Tensor.zeros(4,6).contiguous().realize().shard(self.device, shard_axis).realize() + ones = Tensor.ones(4,1).shard(self.device, shard_axis).contiguous().realize() + + @TinyJit + def f(out:Tensor, vi): + out[:, vi:vi+1].assign(ones).realize() + ones.assign(ones+1).realize() + + vi = Variable("i", 0, 5) + for i in range(1,5): + GlobalCounters.reset() + f(out, vi.bind(i)) + self.assertListEqual(out.tolist(), [[0,1,2,3,4,0]]*4) if __name__ == "__main__": unittest.main() diff --git a/test/unit/test_multitensor.py b/test/unit/test_multitensor.py index d93019bc6015e..8ceebe6e1ba71 100644 --- a/test/unit/test_multitensor.py +++ b/test/unit/test_multitensor.py @@ -852,82 +852,6 @@ def test_from_npy(self): np.testing.assert_equal(ll.numpy(), np.arange(100)+1) @unittest.skipIf(not_support_multi_device(), "need multi") -class TestMultiAssign(unittest.TestCase): - device = tuple(f"{Device.DEFAULT}:{i}" for i in range(2)) - - @needs_second_gpu - def setUp(self): pass - - def test_multi_assign_realized(self): - out = Tensor.zeros(4).shard(self.device, 0).contiguous().realize() - ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_unrealized(self): - out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_both_unrealized(self): - out = Tensor.zeros(4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4).contiguous().realize().shard(self.device, 0) - out.assign(ones).realize() - self.assertListEqual(out.tolist(), [1,1,1,1]) - - def test_multi_assign_scalar(self): - out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(0).realize() - self.assertListEqual(out.tolist(), [0,0,0,0]) - - def test_multi_assign_const_like(self): - out = Tensor.ones(4).shard(self.device, 0).contiguous().realize() - out.assign(out.const_like(7)).realize() - self.assertListEqual(out.tolist(), [7,7,7,7]) - - def test_multi_assign_piece(self): - out = Tensor.zeros(4,4).shard(self.device, 0).contiguous().realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_piece_noncontig(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - @unittest.expectedFailure - def test_multi_assign_piece_unrealized(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0) - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - out[:, 2:3].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_var_offset(self): - out = Tensor.zeros(4,4).contiguous().realize().shard(self.device, 0).realize() - ones = Tensor.ones(4,1).shard(self.device, 0).contiguous().realize() - vi = Variable("i", 0, 3).bind(2) - out[:, vi:vi+1].assign(ones).realize() - self.assertListEqual(out.tolist(), [[0,0,1,0], [0,0,1,0], [0,0,1,0], [0,0,1,0]]) - - def test_multi_assign_var_offset_jit_none(self): self.test_multi_assign_var_offset_jit(None) - def test_multi_assign_var_offset_jit(self, shard_axis=0): - out = Tensor.zeros(4,6).contiguous().realize().shard(self.device, shard_axis).realize() - ones = Tensor.ones(4,1).shard(self.device, shard_axis).contiguous().realize() - - @TinyJit - def f(out:Tensor, vi): - out[:, vi:vi+1].assign(ones).realize() - ones.assign(ones+1).realize() - - vi = Variable("i", 0, 5) - for i in range(1,5): - GlobalCounters.reset() - f(out, vi.bind(i)) - self.assertListEqual(out.tolist(), [[0,1,2,3,4,0]]*4) - @unittest.skipIf(not_support_multi_device(), "need multi") class TestMultiSetitem(unittest.TestCase): device = tuple(f"{Device.DEFAULT}:{i}" for i in range(4)) diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 1df40dc59a448..4bc9c10267ed0 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -9,7 +9,8 @@ from tinygrad.schedule.multi import multi_pm def walk_mop(u:UOp): - if u.op in GroupOp.Movement or u.op in {Ops.INDEX, Ops.UNSHARD}: return walk_mop(u.src[0]) + if u.op in GroupOp.Movement or u.op in {Ops.INDEX, Ops.UNSHARD, Ops.BITCAST}: return walk_mop(u.src[0]) + if u.op is Ops.AFTER and (b:=walk_mop(u.src[0])) is not u.src[0]: return b.after(*u.src[1:]) return u def found_after(ctx:dict[UOp, UOp], after:UOp, src:UOp): @@ -142,10 +143,6 @@ def expand_bitcast(bc:UOp) -> UOp|None: # copy to same device is a no-op (UPat(Ops.COPY, src=(UPat.var("x"),), name="copy"), lambda x,copy: x if x.device == copy.device else None), - # COPY transfers a contiguous range, so materialize a source that's resized (shrink/pad/expand) or reordered (permute/flip) - (UPat(Ops.COPY, src=(UPat(GroupOp.Movement, name="r"),), name="c"), - lambda c,r: c.replace(src=(r.contiguous(),)) if resolve(r.numel() != r.base.numel(), False) or r.contiguous_view_offset() is None else None), - # copy on reshape is reshape on copy (UPat(Ops.COPY, src=(UPat(Ops.RESHAPE, name="shp"),), name="cpy"), lambda shp,cpy: shp.src[0].copy_to_device(cpy.device).reshape(shp.shape)), From 14cbecef8f43edc2ee5453fc0942655c8f1ccbbb Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 27 Aug 2026 13:32:16 -0700 Subject: [PATCH 234/306] autogen: use deb for comgr 2 (#17782) --- .github/workflows/autogen.yml | 43 ++-------------------------- tinygrad/runtime/autogen/__init__.py | 26 ++++++++--------- 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/.github/workflows/autogen.yml b/.github/workflows/autogen.yml index a6d0e7eb2306d..df891d2687fc0 100644 --- a/.github/workflows/autogen.yml +++ b/.github/workflows/autogen.yml @@ -40,10 +40,10 @@ jobs: run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev libdrm-dev liburing-dev - name: Regenerate autogen files run: | - find tinygrad/runtime/autogen -type f -name "*.py" -not -path "*/amd/*" -not -name "__init__.py" -not -name "comgr.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete + find tinygrad/runtime/autogen -type f -name "*.py" -not -path "*/amd/*" -not -name "__init__.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete python3 -c "from tinygrad.runtime.autogen import opencl" python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv_610, nv" - python3 -c "from tinygrad.runtime.autogen import comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm" + python3 -c "from tinygrad.runtime.autogen import comgr, comgr_3, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm" python3 -c "from tinygrad.runtime.autogen.am import *" python3 -c "from tinygrad.runtime.autogen.nv_regs import *" python3 -c "from tinygrad.runtime.autogen import libc, kfd, io_uring, pci, vfio" @@ -102,42 +102,3 @@ jobs: with: name: autogen-macos-patch path: autogen-macos.patch - - autogen-comgr-2: - name: In-tree Autogen (comgr 2) - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: Setup Environment - uses: ./.github/actions/setup-tinygrad - with: - key: 'autogen-comgr' - - name: Install autogen support packages - run: | - wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - sudo tee /etc/apt/sources.list.d/rocm.list < autogen-comgr2.patch - echo "Autogen mismatch detected. Patch available at: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts" - exit 1 - fi - - name: Upload patch artifact - if: failure() - uses: actions/upload-artifact@v7 - with: - name: autogen-comgr2-patch - path: autogen-comgr2.patch diff --git a/tinygrad/runtime/autogen/__init__.py b/tinygrad/runtime/autogen/__init__.py index 9b54a01a5547e..299f979a0789f 100644 --- a/tinygrad/runtime/autogen/__init__.py +++ b/tinygrad/runtime/autogen/__init__.py @@ -17,6 +17,7 @@ cudart_src = "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-12.0.146-archive.tar.xz" nvrtc_src = "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/linux-x86_64/cuda_nvrtc-linux-x86_64-12.0.140-archive.tar.xz" opencl_src = "https://github.com/KhronosGroup/OpenCL-Headers/archive/2e30669d48718fd460f085b4b35b160dad51ce9d.tar.gz" +comgr_2_src = "https://repo.radeon.com/rocm/apt/6.2/pool/main/c/comgr/comgr_2.8.0.60200-66~24.04_amd64.deb" macossdk = "/var/db/xcode_select_link/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" llvm_lib = ( @@ -53,6 +54,8 @@ def load(name, files, **kwargs): if srcs: td.cleanup() return importlib.import_module(f"{path}.{name.replace('/', '.')}") +def _extract_deb(path:str): subprocess.run("ar x *.deb && tar xf data.tar.*", cwd=path, shell=True, check=True) + def __getattr__(nm): match nm: case "libc": @@ -102,14 +105,11 @@ def __getattr__(nm): # this defines all syscall numbers. should probably unify linux autogen? case "io_uring": return load("io_uring", ["{}/liburing.h", "{}/usr/include/linux/io_uring.h", "{}/usr/include/asm-generic/unistd.h"], - args=["-I{}/usr/include"], srcs=[linux_headers_deb, liburing_src], rules=[('__NR', 'NR')], - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) + args=["-I{}/usr/include"], srcs=[linux_headers_deb, liburing_src], rules=[('__NR', 'NR')], preprocess=_extract_deb) case "llvm": return load("llvm", lambda: [system("llvm-config-20 --includedir")+"/llvm-c/**/*.h"], dll=llvm_lib, args=lambda: system("llvm-config-20 --cflags").split(), recsym=True, prolog=["from tinygrad.helpers import WIN, OSX"]) - case "pci": return load("pci", ["{}/usr/include/linux/pci_regs.h"], srcs=linux_headers_deb, - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) - case "vfio": return load("vfio", ["{}/usr/include/linux/vfio.h"], args=["-I{}/usr/include"], srcs=linux_headers_deb, - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_deb.split('/')[-1]} && tar xf data.tar.xz", cwd=path, shell=True, check=True)) + case "pci": return load("pci", ["{}/usr/include/linux/pci_regs.h"], srcs=linux_headers_deb, preprocess=_extract_deb) + case "vfio": return load("vfio", ["{}/usr/include/linux/vfio.h"], args=["-I{}/usr/include"], srcs=linux_headers_deb, preprocess=_extract_deb) # could add rule: WGPU_COMMA -> ',' case "webgpu": return load("webgpu", [root/"extra/webgpu/webgpu.h"], dll=webgpu_lib, prolog=["from tinygrad.helpers import WIN, OSX", "import sysconfig, os"]) @@ -119,9 +119,10 @@ def __getattr__(nm): dll="os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamdhip64.so'", args=["-D__HIP_PLATFORM_AMD__", "-I/opt/rocm/include", "-x", "c++"], prolog=["import os"]) case "comgr" | "comgr_3": - return load("comgr_3" if nm == "comgr_3" else "comgr", ["/opt/rocm/include/amd_comgr/amd_comgr.h"], - dll= "[os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamd_comgr.so', 'amd_comgr']", - args=["-D__HIP_PLATFORM_AMD__", "-I/opt/rocm/include", "-x", "c++"], prolog=["import os"]) + prefix = "{}/opt/rocm-6.2.0" if nm == "comgr" else "/opt/rocm" + return load(nm, [f"{prefix}/include/amd_comgr/amd_comgr.h"], dll="[os.getenv('ROCM_PATH', '/opt/rocm')+'/lib/libamd_comgr.so', 'amd_comgr']", + args=["-D__HIP_PLATFORM_AMD__", f"-I{prefix}/include", "-x", "c++"], prolog=["import os"], srcs=comgr_2_src if nm == "comgr" else None, + **({'preprocess':_extract_deb} if nm == "comgr" else {})) case "hsa": return load("hsa", [*[f"{{}}/projects/rocr-runtime/runtime/hsa-runtime/core/inc/{s}.h" for s in ["registers"]], *[f"{{}}/projects/rocr-runtime/runtime/hsa-runtime/inc/{s}.h" for s in [ "hsa", "hsa_ext_amd", "amd_hsa_signal", "amd_hsa_queue", "amd_hsa_kernel_code", @@ -184,8 +185,7 @@ def __getattr__(nm): return load("mlx5", [root/"extra/mlx_driver/mlx5.h", f"{kh}/mlx5_ifc.h"], srcs=linux_headers_kern_deb, args=["-Du8=unsigned char", "-Du16=unsigned short", "-Du32=unsigned int", "-Du64=unsigned long long", "-D__be16=unsigned short", "-D__be32=unsigned int", "-D__be64=unsigned long long", f"-I{kh}"], - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_kern_deb.split('/')[-1]} && tar xf data.tar.xz", - cwd=path, shell=True, check=True)) + preprocess=_extract_deb) case "bnxt": kh = "{}/usr/src/linux-headers-6.18.9+deb14-common/include" return load("bnxt", [f"{kh}/linux/bnxt/hsi.h", *[f"{{}}/{s.split('/')[-1]}" for s in bnxt_src]], @@ -197,7 +197,5 @@ def __getattr__(nm): r"((cmdq|creq)_(base|init|add_gid|create_(cq|qp)|initialize_fw|modify_qp|query_version|register_mr)(_resp)?" r"|cq_(base|req)|sq_(rdma_hdr|sge))$", r"(BNXT|CMDQ|CREQ|CQ|SQ|DBC|PTU|RCFW|HWRM|VNIC|RING_ALLOC|STAT_CTX|CFA_L2_FILTER|PORT_PHY_CFG|FIRMWARE_FIRST" - r"|FUNC_(QCAPS|QCFG|RESET|DRV_RGTR|BACKING_STORE))_"], - preprocess=lambda path: subprocess.run(f"ar x {linux_headers_kern_deb.split('/')[-1]} && tar xf data.tar.xz", - cwd=path, shell=True, check=True)) + r"|FUNC_(QCAPS|QCFG|RESET|DRV_RGTR|BACKING_STORE))_"], preprocess=_extract_deb) case _: raise AttributeError(f"no such autogen: {nm}") From f267a49639cd5f1b81588965e0c07456b8e1dd67 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:07:33 -0700 Subject: [PATCH 235/306] UOp.param accepts single sint as shape (int,) (#17784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UOp.param accepts single sint as shape (int,) Change UOp.param signature from shape:tuple[sint, ...]|None to shape:tuple[sint, ...]|sint|None. A single sint means 1D shape (sint,). Update all callers from (n,) to n syntax. No param_from_shape — multi-dim shapes stay as tuples. * use single int syntax in hcq2 copy_with_kernel --- test/amd/test_emu2_pcode.py | 10 ++-- test/backend/test_isel.py | 2 +- test/backend/test_linearizer_dumb.py | 6 +-- test/backend/test_renderer_failures.py | 18 +++---- test/backend/test_uops.py | 16 +++--- test/backend/test_wait_loop.py | 2 +- test/external/external_benchmark_op_conv.py | 6 +-- test/external/external_test_gpu_crash.py | 2 +- test/helpers.py | 2 +- test/mockgpu/amd/emu.py | 12 ++--- test/null/test_gpudims.py | 2 +- test/null/test_linearizer_failures.py | 6 +-- test/null/test_simplify_valid_idx.py | 10 ++-- test/null/test_transcendental_helpers.py | 2 +- test/null/test_uop_graph.py | 54 ++++++++++----------- test/null/test_uop_symbolic.py | 8 +-- test/null/test_uop_vmin_vmax.py | 4 +- test/null/test_uops.py | 32 ++++++------ test/null/test_uops_stats.py | 4 +- test/null/test_validate_oob.py | 46 +++++++++--------- test/unit/test_dtype_weak.py | 4 +- tinygrad/runtime/ops_cpu.py | 14 +++--- tinygrad/runtime/ops_qcom.py | 2 +- tinygrad/runtime/support/hcq2.py | 2 +- tinygrad/uop/ops.py | 3 +- 25 files changed, 135 insertions(+), 134 deletions(-) diff --git a/test/amd/test_emu2_pcode.py b/test/amd/test_emu2_pcode.py index dc20ff2fd121d..ff6366372af70 100644 --- a/test/amd/test_emu2_pcode.py +++ b/test/amd/test_emu2_pcode.py @@ -152,7 +152,7 @@ class TestDSPcodePatterns(unittest.TestCase): def test_global_atomic_add_f32_parsing(self): """Test GLOBAL_ATOMIC_ADD_F32 keeps memory values in float dtype.""" - vmem = UOp.param(2, dtypes.uint32, (1024,)) + vmem = UOp.param(2, dtypes.uint32, 1024) srcs = { 'ADDR': UOp.const(0, dtypes.uint64), 'DATA': UOp.const(0x3f800000, dtypes.uint32), @@ -183,7 +183,7 @@ def test_ds_store_b32_pcode(self): def test_mem_read_parsing(self): """Test MEM[addr].type read expression parsing.""" # Create a mock LDS buffer - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) addr = UOp.const(0, dtypes.uint32) vrs = {'_lds': lds, 'ADDR': addr, 'OFFSET': UOp.const(0, dtypes.uint32)} @@ -218,7 +218,7 @@ def test_ds_load_2addr_b32_parsing(self): pcode = PCODE.get(DSOp.DS_LOAD_2ADDR_B32) self.assertIsNotNone(pcode) assert pcode is not None - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) srcs = { 'ADDR': UOp.const(0, dtypes.uint32), 'OFFSET0': UOp.const(0, dtypes.uint32), @@ -299,7 +299,7 @@ def test_permlane64_altlane_concat(self): self.assertIs(parsed.simplify(), UOp.const(expected, dtypes.uint32)) def test_permlane64_wave64_pcode_indices(self): - vgpr = UOp.param(0, dtypes.uint32, (256,)) + vgpr = UOp.param(0, dtypes.uint32, 256) srcs = { 'SRC0': UOp.const(0, dtypes.uint32), 'VDST': UOp.const(1, dtypes.uint32), @@ -330,7 +330,7 @@ class TestAllPcode(unittest.TestCase): def _make_srcs(self): """Create dummy source variables for pcode parsing.""" u32, u64 = lambda v=0: UOp.const(v, dtypes.uint32), lambda v=0: UOp.const(v, dtypes.uint64) - lds = UOp.param(3, dtypes.uint32, (16384,)) + lds = UOp.param(3, dtypes.uint32, 16384) return {'laneId': u32(), 'laneID': u32(), 'S0': u32(), 'S1': u32(), 'S2': u32(), 'S3': u32(), 'SRC0': u32(), 'D0': u32(), 'D1': u32(), 'DST': u32(), 'VDST': u32(), 'SDST': u32(), 'VCC': u64(), 'VCCZ': u32(), 'EXEC': u64(), 'EXEC_LO': u32(), 'EXECZ': u32(), 'SCC': u32(), diff --git a/test/backend/test_isel.py b/test/backend/test_isel.py index f219195a52ca8..54033ba393da1 100644 --- a/test/backend/test_isel.py +++ b/test/backend/test_isel.py @@ -46,7 +46,7 @@ def test_vinsertps(self): # complex address is [base + index*scale + displacement] def test_complex_address(self): a = UOp.variable("a", 0, 0, dtypes.int32) - load = UOp.param(0, dtypes.int32, (16,)).index(a + UOp.cconst(1, dtypes.int32)).load() + load = UOp.param(0, dtypes.int32, 16).index(a + UOp.cconst(1, dtypes.int32)).load() n = self.isel_rewrite(load) # displacement is the constant in "a" scaled to the buffer element size, dtype is int8 when the value fits otherwise int32 self.assertTrue(n.src[2].dtype is dtypes.int8 and n.src[2].src[0].op is Ops.CONST and n.src[2].src[0].val == 4) diff --git a/test/backend/test_linearizer_dumb.py b/test/backend/test_linearizer_dumb.py index 317f0abd3bfd4..364bcc2578ffb 100644 --- a/test/backend/test_linearizer_dumb.py +++ b/test/backend/test_linearizer_dumb.py @@ -11,16 +11,16 @@ class TestLinearizerFailure(unittest.TestCase): @unittest.skipUnless(Device.DEFAULT == "METAL", "only tested on METAL") def test_failure_beam_mnist(self): - c0 = UOp.param(0, dtypes.uchar, (4014080,)) + c0 = UOp.param(0, dtypes.uchar, 4014080) c1 = UOp.range(UOp.const(512), 0, AxisType.GLOBAL) c2 = UOp.range(UOp.const(784), 1, AxisType.GLOBAL) c3 = UOp.range(UOp.const(10), 3, AxisType.GLOBAL) - c4 = UOp.param(1, dtypes.int, (512,)) + c4 = UOp.param(1, dtypes.int, 512) c5 = c4.index(c1.valid(UOp.const(True))) c6 = UOp.range(UOp.const(6000), 1004, AxisType.REDUCE) c7 = UOp.range(UOp.const(3750), 2006, AxisType.REDUCE) c8 = UOp.range(UOp.const(16), 2007, AxisType.GROUP_REDUCE) - c9 = UOp.param(2, dtypes.uchar, (47040000,)) + c9 = UOp.param(2, dtypes.uchar, 47040000) c10 = c9.index((((c3*UOp.const(4704000))+c2)+(c6*UOp.const(784))).valid(UOp.const(True))) c11 = c5.alu(Ops.CMPNE, ((((c3*UOp.const(6000))+c6)+((c7*UOp.const(16))+c8)).alu(Ops.CMPLT, UOp.const(59999)).where(UOp.const(0).cast(dtypes.int), UOp.const(1).cast(dtypes.int)).reduce(c7, c8, arg=Ops.ADD)+UOp.const(-1).cast(dtypes.int))).where(UOp.const(0).cast(dtypes.uchar), c10).reduce(c6, arg=Ops.ADD) c12 = c0.index((((c1*UOp.const(7840))+(c2*UOp.const(10)))+c3).valid(UOp.const(True))).store(c11).end(c1, c2, c3) diff --git a/test/backend/test_renderer_failures.py b/test/backend/test_renderer_failures.py index 9ec18c1a9b3d6..8f5460d1b967b 100644 --- a/test/backend/test_renderer_failures.py +++ b/test/backend/test_renderer_failures.py @@ -23,8 +23,8 @@ def _test_uop_result(inputs:list[Tensor], sink:UOp, local_size=None): def _setup_and_test_alu(alu_op:Ops, input_val:ConstType, *alu_src_uops:UOp): dtype = alu_src_uops[0].dtype - a = UOp.param(0, dtype, (1,)) - b = UOp.param(1, dtype, (1,)) + a = UOp.param(0, dtype, 1) + b = UOp.param(1, dtype, 1) idx = UOp.const(0) ld = b.index(idx).load() alu = ld.alu(alu_op, *alu_src_uops) @@ -34,7 +34,7 @@ def _setup_and_test_alu(alu_op:Ops, input_val:ConstType, *alu_src_uops:UOp): class TestRendererFailures(unittest.TestCase): @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer") def test_gated_store_with_alu(self): - a = UOp.param(0, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) gate_alu = (lidx0:=UOp.special(4, 'lidx0')).ne(0) gated_alu_store = UOp(Ops.STORE, src=(a.index(lidx0.valid(gate_alu)), UOp.const(1).cast(dtypes.int))) sink = UOp(Ops.SINK, src=(gated_alu_store,), arg=KernelInfo()) @@ -43,7 +43,7 @@ def test_gated_store_with_alu(self): @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer") def test_gated_store_with_alu_2d(self): - a = UOp.param(0, dtypes.int, (8,)) + a = UOp.param(0, dtypes.int, 8) gate_alu_0 = (lidx0:=UOp.special(4, 'lidx0')).ne(0) gate_alu_1 = (lidx1:=UOp.special(2, 'lidx1')).ne(0) gated_alu_store = UOp(Ops.STORE, src=(a.index((lidx0+lidx1*4).valid(gate_alu_0&gate_alu_1)), UOp.const(1).cast(dtypes.int))) @@ -78,7 +78,7 @@ def test_repeat_sub(self): self._test_src_strip_paren(Ops.SUB, should_strip_pare @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, WGSLRenderer), "tests for wgsl renderer") class TestWGSLFailures(unittest.TestCase): def test_folded_packed_store(self): - b = UOp.param(0, dtypes.char, (4,)) + b = UOp.param(0, dtypes.char, 4) idx = b.index(UOp.const(0).cast(dtypes.int)) store = UOp.store(idx, idx.cast(dtypes.uint32).load() & UOp.const(0xffffff00).cast(dtypes.uint32)) src = Device[Device.DEFAULT].renderer.render(UOp.sink(store, arg=KernelInfo()).toposort()) @@ -93,9 +93,9 @@ def test_multiply_infinity(self): # WGSL has a specific select(alt, val, gate) ternary operator instead of gate?val:alt def test_gated_load(self): - a = UOp.param(0, dtypes.int, (4,)) - b = UOp.param(1, dtypes.int, (4,)) - c = UOp.param(2, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) + b = UOp.param(1, dtypes.int, 4) + c = UOp.param(2, dtypes.int, 4) lidx0 = UOp.special(4, "lidx0") gate = lidx0.ne(0) alt = c.index(lidx0).load() @@ -110,7 +110,7 @@ def test_gated_load(self): class TestPTXFailures(unittest.TestCase): @unittest.skip("INDEX can only have a gate ALU parent, not an IF") def test_gated_store_with_if(self): - a = UOp.param(0, dtypes.int, (4,)) + a = UOp.param(0, dtypes.int, 4) gate_alu = (lidx0:=UOp.special(4, 'lidx0')).ne(0) val = UOp.const(1).cast(dtypes.int) if_uop = UOp(Ops.IF, src=(gate_alu,)) diff --git a/test/backend/test_uops.py b/test/backend/test_uops.py index e1c0c324432f0..d966f8eb47bd4 100644 --- a/test/backend/test_uops.py +++ b/test/backend/test_uops.py @@ -20,7 +20,7 @@ def run_uops(uops_list:list[UOp], bufs:list[Buffer]): def uop(uops:list[UOp], op:Ops, dtype:Optional[DType], src:tuple[UOp, ...], arg:Any=None) -> UOp: if op is Ops.CONST: uops.append(UOp.const(arg).cast(dtype)) - elif op is Ops.PARAM: uops.append(UOp.param(arg, dtype, shape=(1,))) + elif op is Ops.PARAM: uops.append(UOp.param(arg, dtype, 1)) else: uops.append(UOp(op, dtype, tuple(src), arg)) return uops[-1] @@ -62,7 +62,7 @@ def _test_uops_result(output_dtype, uops, res): class TestBitcastBufferView(unittest.TestCase): @Context(SPEC=2) def test_render(self): - buf = UOp.param(0, dtypes.uint32, (4,)) + buf = UOp.param(0, dtypes.uint32, 4) uops = to_uops_list([buf.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0).store(1)], ren=Device[Device.DEFAULT].renderer) idx = next(u for u in uops if u.op is Ops.INDEX and u.src[0].op is Ops.BITCAST) self.assertEqual(idx.src[0].src[0].op, Ops.SHRINK) @@ -71,7 +71,7 @@ def test_render(self): @Context(SPEC=2) def test_load(self): val = 0x1122334455667788 - src, out = UOp.param(0, dtypes.uint32, (4,)), UOp.param(1, dtypes.uint64, (1,)) + src, out = UOp.param(0, dtypes.uint32, 4), UOp.param(1, dtypes.uint64, 1) ibuf = Buffer(Device.DEFAULT, 4, dtypes.uint32, initial_value=np.array([0, 0x55667788, 0x11223344, 0], dtype=np.uint32).tobytes()) obuf = Buffer(Device.DEFAULT, 1, dtypes.uint64).allocate() run_uops([out.index(0).store(src.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0))], [ibuf, obuf]) @@ -80,7 +80,7 @@ def test_load(self): @Context(SPEC=2) def test_store(self): val = 0x1122334455667788 - dst = UOp.param(0, dtypes.uint32, (6,)) + dst = UOp.param(0, dtypes.uint32, 6) buf = Buffer(Device.DEFAULT, 6, dtypes.uint32, initial_value=bytes(24)) view = dst.shrink(((1, 5),)).bitcast(dtypes.uint64) # two stores through one view: it must inline, not get a declared vector-pointer run_uops([view.index(0).store(val ^ 0xff), view.index(1).store(val)], [buf]) @@ -249,8 +249,8 @@ def test_local_indirect(self): @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "This only tests assembly backends") class TestAssembly(unittest.TestCase): def test_bitshift_left(self): - g1 = UOp.param(0, dtypes.int32, shape=(3,)) - out = UOp.param(1, dtypes.int32, shape=(2,)) + g1 = UOp.param(0, dtypes.int32, 3) + out = UOp.param(1, dtypes.int32, 2) c1 = UOp.const(2) c2 = UOp.const(3) l1 = g1.index(c1) @@ -278,7 +278,7 @@ def test_mulacc_unrolled(self): self.assertGreaterEqual(len([x.op for x in uops if x.op is Ops.MULACC]), 4) def test_mulacc_shl(self): - g1 = UOp.param(0, dtypes.int32, shape=(2,)) + g1 = UOp.param(0, dtypes.int32, 2) c1 = UOp.const(0) c2 = UOp.const(1) expr = g1.index(c1) * UOp.const(4096) + g1.index(c2) @@ -287,7 +287,7 @@ def test_mulacc_shl(self): self.assertIn(Ops.MULACC, [x.op for x in uops]) def test_use_cmpeq(self): - g = UOp.param(0, dtypes.uint32, shape=(8,)) + g = UOp.param(0, dtypes.uint32, 8) c = UOp.const(7) comp = g.index(c).ne(c).ne(True) uops = to_uops_list([comp], ren=Device[Device.DEFAULT].renderer) diff --git a/test/backend/test_wait_loop.py b/test/backend/test_wait_loop.py index 83e0543e61a28..6073c9422bdfe 100644 --- a/test/backend/test_wait_loop.py +++ b/test/backend/test_wait_loop.py @@ -43,7 +43,7 @@ def nested_loop_kernel(C:UOp) -> UOp: return C[0].store(i[0].load()).sink(arg=KernelInfo(name="nested_loop", opts_to_apply=())) def wait_ext_kernel() -> UOp: - sig = UOp.param(0, dtypes.int, (1,), volatile=True) + sig = UOp.param(0, dtypes.int, 1, volatile=True) l = UOp.loop(0) v = sig.after(l)[0].load() e = v.end(l, v < 1) diff --git a/test/external/external_benchmark_op_conv.py b/test/external/external_benchmark_op_conv.py index 4076facd5d8c8..9280ef4a050c3 100644 --- a/test/external/external_benchmark_op_conv.py +++ b/test/external/external_benchmark_op_conv.py @@ -28,7 +28,7 @@ def vision_conv_143(): c48 = (c24&c32).where(c34.index(c45), UOp.const(0.0, dtypes.float)) c49 = UOp.param(2, dtypes.half, shape=(64, 49, 4)) c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196)) - c63 = UOp.param(3, dtypes.float, (128,)) + c63 = UOp.param(3, dtypes.float, 128) c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5) c67 = c0.index((c2*128+c5+c8*4096)).store(c65).end(c8, c2, c5) @@ -54,7 +54,7 @@ def vision_conv_153(): c48 = (c24&c32).where(c34.index(c45), UOp.const(0.0, dtypes.float)) c49 = UOp.param(2, dtypes.half, shape=(128, 49, 4)) c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196)) - c63 = UOp.param(3, dtypes.float, (256,)) + c63 = UOp.param(3, dtypes.float, 256) c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5) c67 = c0.index((c2*256+c5+c8*4096)).store(c65).end(c8, c2, c5) @@ -73,7 +73,7 @@ def dm_conv_172(): c18 = UOp.range(8, 2, AxisType.REDUCE) c23 = UOp.param(2, dtypes.half, shape=(240, 128, 4)) c35 = c5.index((c7*4+c10+c13*128+c18*1536))*c23.index((c10*4+c2%4+c7*16+c2//4*512)) - c37 = UOp.param(3, dtypes.float, (960,)) + c37 = UOp.param(3, dtypes.float, 960) c39 = c35.reduce(c7, c10, arg=Ops.ADD)+c37.index(c2) c50 = (1.0+((c39+0.044708251953125*(c39*(c39*c39)))*-2.3021129851685216).exp2()).reciprocal()*c39 c53 = c50.reduce(c18, c13, arg=Ops.ADD)*0.010416666666666666 diff --git a/test/external/external_test_gpu_crash.py b/test/external/external_test_gpu_crash.py index ebe0f1eb2d6d5..238cdc8c2e121 100644 --- a/test/external/external_test_gpu_crash.py +++ b/test/external/external_test_gpu_crash.py @@ -36,7 +36,7 @@ def setUp(self): def _run_insts(self, insts: list[Inst]): buf = UOp.new_buffer("AMD", 64, dtypes.uint8) - sink = UOp.sink(UOp.param(0, dtypes.uint8, (64,), device="AMD"), UOp.special(1, "lidx0"), arg=KernelInfo("test")) + sink = UOp.sink(UOp.param(0, dtypes.uint8, 64, device="AMD"), UOp.special(1, "lidx0"), arg=KernelInfo("test")) prg = UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=i) for i in insts)))) run_linear(UOp(Ops.LINEAR, src=(prg.call(buf),)), wait=True) diff --git a/test/helpers.py b/test/helpers.py index e6d9916d4ea95..8bdd7d9bbf30d 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -122,7 +122,7 @@ def eval_uop(uop:UOp, inputs:list[tuple[DType, list[Any]]]|None=None, vals:tuple for buf_dt, data in inputs or []: bufs.append(buf:=allocator.alloc(len(data) * buf_dt.itemsize)) allocator._copyin(buf, memoryview(struct.pack(str(len(data)) + (buf_dt.fmt or ""), *data))) - g = UOp.param(0, uop.dtype, (1,)) + g = UOp.param(0, uop.dtype, 1) prg = to_program(UOp.store(g.index(UOp.const(0)), uop).sink(arg=KernelInfo()), PythonRenderer(Target("PYTHON"))) prog = dev.runtime(prg.to_elf()) prog(out_buf:=allocator.alloc(uop.dtype.itemsize), *bufs, vals=vals) diff --git a/test/mockgpu/amd/emu.py b/test/mockgpu/amd/emu.py index 6c8b3380f4a8f..9d0ac9ef60779 100644 --- a/test/mockgpu/amd/emu.py +++ b/test/mockgpu/amd/emu.py @@ -321,10 +321,10 @@ def to_wide(x: UOp) -> UOp: return (x.bitcast(narrow_dt) if x.dtype.itemsize == class _Ctx: """Context for instruction compilation - holds buffers and helpers.""" __slots__ = ('inst_size', 'dyn_fields', '_axis_id', 'wave_size', 'vgpr', 'accvgpr') - sgpr = UOp.param(0, dtypes.uint32, (SGPR_COUNT,)) - vmem = UOp.param(2, dtypes.uint32, (1 << 46,)) - lds = UOp.param(3, dtypes.uint32, (16384,)) - scratch = UOp.param(4, dtypes.uint8, (1 << 30,)) + sgpr = UOp.param(0, dtypes.uint32, SGPR_COUNT) + vmem = UOp.param(2, dtypes.uint32, 1 << 46) + lds = UOp.param(3, dtypes.uint32, 16384) + scratch = UOp.param(4, dtypes.uint8, 1 << 30) # Cache PARAM UOps by wave_size so all _Ctx instances with same wave_size share identical UOp references _vgpr_cache: dict[int, UOp] = {} _accvgpr_cache: dict[int, UOp] = {} @@ -332,10 +332,10 @@ class _Ctx: def __init__(self, inst_size: int, wave_size: int = 32): self.inst_size, self._axis_id, self.wave_size = inst_size, 0, wave_size self.dyn_fields: list[tuple[int, int]] = [] # (lo, hi) of fields read dynamically - if wave_size not in _Ctx._vgpr_cache: _Ctx._vgpr_cache[wave_size] = UOp.param(1, dtypes.uint32, (256 * wave_size,)) + if wave_size not in _Ctx._vgpr_cache: _Ctx._vgpr_cache[wave_size] = UOp.param(1, dtypes.uint32, 256 * wave_size) self.vgpr = _Ctx._vgpr_cache[wave_size] if wave_size == 64: - if wave_size not in _Ctx._accvgpr_cache: _Ctx._accvgpr_cache[wave_size] = UOp.param(5, dtypes.uint32, (256 * wave_size,)) + if wave_size not in _Ctx._accvgpr_cache: _Ctx._accvgpr_cache[wave_size] = UOp.param(5, dtypes.uint32, 256 * wave_size) self.accvgpr = _Ctx._accvgpr_cache[wave_size] else: self.accvgpr = self.vgpr diff --git a/test/null/test_gpudims.py b/test/null/test_gpudims.py index 8d73d907bc1a0..cde27aadcdba4 100644 --- a/test/null/test_gpudims.py +++ b/test/null/test_gpudims.py @@ -107,7 +107,7 @@ def test_grouped_dims_high_rank(self): def test_global_prod_max(self): g, l = UOp.range(256, 0, AxisType.GLOBAL), UOp.range(256, 1, AxisType.LOCAL) - sink = UOp.param(0, dtypes.float, (512,)).index(g + l).store(UOp.const(1.0)).end(g, l).sink(arg=KernelInfo()) + sink = UOp.param(0, dtypes.float, 512).index(g + l).store(UOp.const(1.0)).end(g, l).sink(arg=KernelInfo()) class R(Renderer): global_max, local_max, global_prod_max = (256, 256, 256), (128, 128, 128), (128, 128, 128) specials = [u for u in add_gpudims(R(Target()), sink).toposort() if u.op is Ops.SPECIAL] self.assertGreater(len([s for s in specials if "lidx" in s.arg]), 1) diff --git a/test/null/test_linearizer_failures.py b/test/null/test_linearizer_failures.py index 620cd8d0f3106..5c89d4c3d8ce9 100644 --- a/test/null/test_linearizer_failures.py +++ b/test/null/test_linearizer_failures.py @@ -7,14 +7,14 @@ class TestLinearizerFailures(unittest.TestCase): def test_fail_1(self): - c0 = UOp.param(0, dtypes.float, (64,)) + c0 = UOp.param(0, dtypes.float, 64) c1 = UOp.range(UOp.const(2), 1, AxisType.WEAK) c2 = UOp.range(UOp.const(32), 2, AxisType.WEAK) c3 = ((c1*UOp.const(32))+c2) - c4 = UOp.param(1, dtypes.float, (163840,)) + c4 = UOp.param(1, dtypes.float, 163840) c5 = UOp.range(UOp.const(2560), 0, AxisType.REDUCE) c6 = c4.index(((((((c5//UOp.const(8))%UOp.const(8))*UOp.const(8))+(c5%UOp.const(8)))+(((c2*UOp.const(40))+(c5//UOp.const(64)))*UOp.const(64)))+(c1*UOp.const(81920)))) - c7 = UOp.param(2, dtypes.float, (64,)) + c7 = UOp.param(2, dtypes.float, 64) c8 = c7.index(c3) c9 = ((((c6+(c8*UOp.const(-1.0)))*(c6+(c8*UOp.const(-1.0)))).reduce(c5, arg=Ops.ADD)*UOp.const(0.000390625))+UOp.const(1e-05)).sqrt().reciprocal() c10 = c0.index(c3).store(c9).end(c1, c2) diff --git a/test/null/test_simplify_valid_idx.py b/test/null/test_simplify_valid_idx.py index 2166e69e8a07a..05f831fb83a74 100644 --- a/test/null/test_simplify_valid_idx.py +++ b/test/null/test_simplify_valid_idx.py @@ -15,7 +15,7 @@ def simplify_valid_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move def simplify_image_idx(sink: UOp) -> UOp: return graph_rewrite(sink, sym+pm_move_where_on_load+indexing_simplify, name="simplify_image_idx") def get_gated_load_uop(valid:UOp, idx:UOp): - return UOp.param(0, dtypes.float, (1024,)).index(idx.valid(valid)).load() + return UOp.param(0, dtypes.float, 1024).index(idx.valid(valid)).load() def get_load_image_uop(image_shape:tuple[int, ...], valid:UOp, idx:tuple[UOp, UOp]): return UOp.param(0, dtypes.float, image_shape).index(idx[1].valid(valid), idx[0].valid(valid)).load() @@ -505,7 +505,7 @@ def test_drop_true_gate_on_index(self): from tinygrad.codegen.late.coalesce import indexing_simplify from tinygrad.uop.ops import graph_rewrite from tinygrad.uop.symbolic import sym - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) idx = UOp.const(0) true_gate = UOp.const(True) index_with_gate = buf.index(idx.valid(true_gate)) @@ -551,7 +551,7 @@ def test_range_no_shrink_when_unguarded_elsewhere(self): # one load guards r < 4, but another load uses r without a gate -> no shrink r = Range(0, 204) load1 = get_gated_load_uop(r < UOp.const(4), r) - load2 = UOp.param(1, dtypes.float, (204,)).index(r).load() + load2 = UOp.param(1, dtypes.float, 204).index(r).load() ranges = self.get_ranges(UOp.sink(load1, load2)) self.assert_range_end(ranges, 204) @@ -575,7 +575,7 @@ def test_range_shrink_store_where_invalid(self): from tinygrad.dtype import Invalid r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) - ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r < 4).where(x, Invalid)).sink()) + ranges = self.get_ranges(UOp.param(0, dtypes.float, 204).index(r).store((r < 4).where(x, Invalid)).sink()) self.assert_range_end(ranges, 4) def test_range_shrink_store_where_invalid_flipped(self): @@ -583,7 +583,7 @@ def test_range_shrink_store_where_invalid_flipped(self): from tinygrad.dtype import Invalid r = Range(0, 204) x = (r < 4).where(UOp.const(1.0), Invalid) - ranges = self.get_ranges(UOp.param(0, dtypes.float, (204,)).index(r).store((r >= 4).where(Invalid, x)).sink()) + ranges = self.get_ranges(UOp.param(0, dtypes.float, 204).index(r).store((r >= 4).where(Invalid, x)).sink()) self.assert_range_end(ranges, 4) if __name__ == '__main__': diff --git a/test/null/test_transcendental_helpers.py b/test/null/test_transcendental_helpers.py index 1653b176c74b2..7602184241d49 100644 --- a/test/null/test_transcendental_helpers.py +++ b/test/null/test_transcendental_helpers.py @@ -9,7 +9,7 @@ class TestTranscendentalFunctions(unittest.TestCase): def test_payne_hanek_reduction(self): # TODO: Test constant input when constant folding is fixed (or maybe test both variants) # Load input value from a buffer to prevent constant folding - input_buf = UOp.param(1, dtypes.double, (1,)) + input_buf = UOp.param(1, dtypes.double, 1) loaded_value = input_buf.index(UOp.const(0)).load() def eval_payne_hanek_reduction(v:float) -> tuple[float, int]: return tuple(eval_uop(u, [(dtypes.float64, [v])]) for u in payne_hanek_reduction(loaded_value)) diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 9cc3f88688a11..602144f6e3ace 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -214,8 +214,8 @@ def test_gep_vec_const_fold(self): for i, const in enumerate(consts): self.assertIs(vec.index(i), const) def test_cast_alu_fold(self): - d0 = UOp.param(0, dtypes.bool, (1,)) - d1 = UOp.param(1, dtypes.int, (1,)) + d0 = UOp.param(0, dtypes.bool, 1) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0) ld = d1.index(idx) alu = (ld<1).cast(dtypes.bool) @@ -224,8 +224,8 @@ def test_cast_alu_fold(self): self.assertEqual(len([x for x in uops if x.op is Ops.CAST and x.src[0].op is not Ops.CONST]), 0) def test_double_cast_fold(self): - d0 = UOp.param(0, dtypes.float, (1,)) - d1 = UOp.param(1, dtypes.int, (1,)) + d0 = UOp.param(0, dtypes.float, 1) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0, dtypes.int) ld = d1.index(idx) alu = ld.cast(dtypes.float).cast(dtypes.float) @@ -243,7 +243,7 @@ def test_depth_2_const_fold(self): def test_bitcast_to_same_dtype_fold(self): for dt in dtypes.ints + dtypes.floats + (dtypes.bool,): - d0 = UOp.param(0, dt, (1,)) + d0 = UOp.param(0, dt, 1) v = d0.index(UOp.const(0)) uops = to_uops_list([v.bitcast(dt)]) self.assertEqual(len([x for x in uops if x.op is Ops.BITCAST and x.dtype is dt]), 0, f"dtype = {dt}") @@ -255,10 +255,10 @@ def test_sub_with_cast_folds(self): def test_where_on_gated_load_fold(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = (ridx0<50).where(ld, 5) - out = UOp.param(1, dtypes.long, (100,)) + out = UOp.param(1, dtypes.long, 100) uops = to_uops_list([out.index(ridx0).store(w)]) expected = full_rewrite(UOp.const(5, dtypes.long).sink()).src[0] for u in uops: @@ -267,7 +267,7 @@ def test_where_on_gated_load_fold(self): def test_where_on_gated_load_folds_swapped_branches(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) ld = d0.index(ridx0.valid((ridx0<50).logical_not())) w = (ridx0<50).where(5, ld) uops = to_uops_list([w]) @@ -278,11 +278,11 @@ def test_where_on_gated_load_folds_swapped_branches(self): def test_where_on_gated_load_with_cast(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.int, (100,)) + d0 = UOp.param(0, dtypes.int, 100) gate_idx = ridx0.valid((ridx0<50)) ld = d0.index(gate_idx).cast(dtypes.float) w = (ridx0<50).where(ld, 5.0) - out = UOp.param(1, dtypes.float, (100,)) + out = UOp.param(1, dtypes.float, 100) uops = to_uops_list([out.index(ridx0).store(w)]) expected = full_rewrite(UOp.const(5, dtypes.int).sink()).src[0] for u in uops: @@ -291,27 +291,27 @@ def test_where_on_gated_load_with_cast(self): def test_where_on_casted_gated_load_extra_cond(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.float, (100,)) + d0 = UOp.param(0, dtypes.float, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = ((ridx0<50) & (ridx0>30)).where(ld, UOp.const(0.0)).cast(dtypes.half) - out = UOp.param(1, dtypes.half, (100,)) + out = UOp.param(1, dtypes.half, 100) uops = to_uops_list([out.index(ridx0).store(w)]) for u in uops: assert u.op is not Ops.WHERE def test_where_on_casted_gated_load_extra_cond_swapped(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.float, (100,)) + d0 = UOp.param(0, dtypes.float, 100) ld = d0.index(ridx0.valid(ridx0<50)) w = ((ridx0<50) & (ridx0>30)).where(UOp.const(0.0), ld).cast(dtypes.half) - out = UOp.param(1, dtypes.half, (100,)) + out = UOp.param(1, dtypes.half, 100) uops = to_uops_list([out.index(ridx0).store(w)]) for u in uops: assert u.op is not Ops.WHERE def test_where_in_store_becomes_gate(self): ridx0 = UOp.range(100, 0) - d0 = UOp.param(0, dtypes.long, (100,)) + d0 = UOp.param(0, dtypes.long, 100) idx = d0.index(ridx0) ld = idx.load() val = (ridx0<50).where(5, ld) @@ -325,14 +325,14 @@ def test_where_in_store_becomes_gate(self): def test_load_idx_becomes_int(self): # mnist indexing with split reduceop # Make sure we are not doign math on the loaded index, which would promote it to long - c0 = UOp.param(0, dtypes.uchar, (128000,)) + c0 = UOp.param(0, dtypes.uchar, 128000) c1 = UOp.range(UOp.const(512), 1, AxisType.WEAK) c2 = UOp.range(UOp.const(250), 2, AxisType.WEAK) - c3 = UOp.param(1, dtypes.int, (512,)) + c3 = UOp.param(1, dtypes.int, 512) c4 = c3.index(c1) c5 = UOp.range(UOp.const(240), 0, AxisType.REDUCE) c6 = ((c2*UOp.const(240))+c5) - c7 = UOp.param(2, dtypes.uchar, (60000,)) + c7 = UOp.param(2, dtypes.uchar, 60000) c8 = c7.index(c6) c9 = ((c4<0).where((c4+60000), c4)!=c6.cast(dtypes.int)).where(0, c8.cast(dtypes.uint).cast(dtypes.uchar)).reduce(c5, arg=Ops.ADD) c10 = c0.index(((c1*UOp.const(250))+c2)).store(c9).end(c1, c2) @@ -342,14 +342,14 @@ def test_load_idx_becomes_int(self): def test_load_idx_no_math_on_loaded(self): # test the (x+y) NOOP rule. This rule matches patterns that EMERGE during simplification.""" def test_store_load_folding(self): # store(idx, load(idx)) -> NOOP, including emergent patterns like store(idx, load(idx) + 0) - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) index = buf.index(UOp.const(0)) # Direct: store(idx, load(idx)) -> NOOP self.assertEqual(graph_rewrite(index.store(index.load()), sym).op, Ops.NOOP) @@ -1417,7 +1417,7 @@ def test_store_load_folding(self): class TestMoveWhereOnLoad(unittest.TestCase): def test_bool_index_preserves_dtype(self): - buf = UOp.param(0, dtypes.bool, (8,)) + buf = UOp.param(0, dtypes.bool, 8) a = Variable("a", 0, 7) r = UOp.range(8, 0) # cond has a range that the rewrite can move into the valid: gate (a<4) goes into load valid @@ -1475,7 +1475,7 @@ def test_range_split_on_mod(self): from tinygrad.codegen.simplify import pm_split_ranges, pm_flatten_range r0 = UOp.range(uconst(8), 0) # create a simple expression using the range with mod: store range%2 to a buffer - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) val = (r0 % uconst(2)).cast(dtypes.int) store = UOp(Ops.STORE, src=(buf.index(uconst(0)), val)) sink = UOp(Ops.SINK, src=(UOp(Ops.END, src=(store, r0)),)) diff --git a/test/null/test_uop_vmin_vmax.py b/test/null/test_uop_vmin_vmax.py index 6398c28bb2f51..fcc7834089f61 100644 --- a/test/null/test_uop_vmin_vmax.py +++ b/test/null/test_uop_vmin_vmax.py @@ -82,7 +82,7 @@ def test_vmin_vmax_variable_inside_special(self): def test_vmin_vmax_multiplication_0_inf(self): # vmin and vmax for multiplication with a variable x = UOp.const(0.0) - y = UOp.load(UOp.param(0, dtypes.float, (1,)), UOp.const(0)) + y = UOp.load(UOp.param(0, dtypes.float, 1), UOp.const(0)) uop = x * y # TODO: these should be 0, but definitely should not be nan self.assertEqual(uop.vmin, -math.inf) @@ -332,7 +332,7 @@ def test_vmin_vmax_vconst_with_bools(self): def test_vmin_vmax_vector_with_gep(self): # vmin and vmax for a vector constant of bool values - d1 = UOp.param(1, dtypes.int, (1,)) + d1 = UOp.param(1, dtypes.int, 1) idx = UOp.const(0) val = UOp(Ops.LOAD, src=(d1.index(idx),)) uop = (val // 32) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index f4230bf9e833f..636dfb0957cbd 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -58,7 +58,7 @@ def test_invalid_dtype_and_consumers(self): for u in (UOp.param(0, dtypes.bool, ()).where(value, invalid), value+invalid, UOp.stack(value, invalid)): self.assertIs(u.src[-1], invalid) for u in (UOp(Ops.STACK, src=(value, invalid)), UOp(Ops.ADD, src=(value, invalid)), UOp.const(True).where(value, invalid), UOp(Ops.CMPLT, src=(invalid, value)), UOp(Ops.CMPLT, src=(value, invalid)), - UOp.param(0, dtypes.float32, (4,)).index(invalid)): type_verify(u, spec_shared) + UOp.param(0, dtypes.float32, 4).index(invalid)): type_verify(u, spec_shared) gate, value = UOp.param(0, dtypes.bool, ()), UOp.param(1, dtypes.float, ()) self.assertIs((out:=graph_rewrite(gate.where(value, UOp.invalid()), pm_remove_invalid)).src[2], UOp.const(0, dtypes.float)) type_verify(out.sink(), spec_program) @@ -73,7 +73,7 @@ class TestLowerIndexDtype(unittest.TestCase): def test_gated_shrink_lowers_to_selected_width(self): # coalesce builds gated SHRINKs for masked vectorized loads; lowering must resolve them at the # width the offset bounds select (this one needs long) - buf = UOp.param(0, dtypes.float, (2**31+64,)) + buf = UOp.param(0, dtypes.float, 2**31+64) i = UOp.variable("i", 0, 2**28) shrink = UOp(Ops.SHRINK, src=(buf, (i*24).valid(i < 2**28), UOp.const(4))) lowered = graph_rewrite(shrink.sink(), pm_lower_weak) @@ -221,7 +221,7 @@ def test_overflow(self): class TestGatedStoreRewrite(unittest.TestCase): def test_tiny_gate_store(self): - gmem = UOp.param(0, dtypes.float, (8,)) + gmem = UOp.param(0, dtypes.float, 8) gidx0 = UOp.special(4, 'gidx0') gate = gidx0= 0) & (v < 16))).load()]) # valid with self.assertRaises(RuntimeError): @@ -48,7 +48,7 @@ def test_variable_with_mask(self): def test_gated_store(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) v = Variable("v", 0, 20) to_uops_list([buf.index(v.valid(v < 16)).store(0)]) # valid with self.assertRaises(RuntimeError): @@ -57,14 +57,14 @@ def test_gated_store(self): # ALU ops in index def test_floordiv(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2).load()]) # 0..16 oob def test_mod(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(100, 0, AxisType.GLOBAL) to_uops_list([buf.index(r % 16).load()]) # 0..15 valid with self.assertRaises(RuntimeError): @@ -72,14 +72,14 @@ def test_mod(self): def test_shr(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2).load()]) # 0..15 valid with self.assertRaises(RuntimeError): to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2).load()]) # 0..31 oob def test_shl(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (64,)) + buf = UOp.param(0, dtypes.int, 64) r = UOp.range(8, 0, AxisType.GLOBAL) to_uops_list([buf.index(r << 2).load()]) # 0..28 valid with self.assertRaises(RuntimeError): @@ -87,7 +87,7 @@ def test_shl(self): def test_and(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(100, 0, AxisType.GLOBAL) to_uops_list([buf.index(r & 15).load()]) # 0..15 valid with self.assertRaises(RuntimeError): @@ -102,14 +102,14 @@ def test_and(self): def test_max(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) to_uops_list([buf.index(Variable("v", -10, 15).maximum(0)).load()]) # 0..15 valid with self.assertRaises(RuntimeError): to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0)).load()]) # 0..20 oob def test_xor_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(32, 0, AxisType.GLOBAL) to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16)))).load()]) # 0..15 valid with self.assertRaises(RuntimeError): @@ -118,22 +118,22 @@ def test_xor_in_mask(self): # cast patterns def test_float_cast_in_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (16,)) + buf = UOp.param(0, dtypes.int, 16) r = UOp.range(20, 0) i = (r.cast(dtypes.float) * 0.68).trunc().cast(dtypes.int) to_uops_list([buf.index(i.valid((i >= 0) & (i < 16))).load()]) def test_bool_cast_in_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf = UOp.param(0, dtypes.int, (1,)) + buf = UOp.param(0, dtypes.int, 1) r = UOp.range(20, 0) to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not())).load()]) # only r=0 valid # load result as index/mask def test_load_as_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf0 = UOp.param(0, dtypes.int, (16,)) - buf1 = UOp.param(1, dtypes.int, (64,)) + buf0 = UOp.param(0, dtypes.int, 16) + buf1 = UOp.param(1, dtypes.int, 64) r = UOp.range(42, 0, AxisType.GLOBAL) ld0 = buf0.index(r.valid(r < 8)).load().cast(dtypes.weakint) to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32))).load()]) # valid @@ -142,16 +142,16 @@ def test_load_as_index(self): def test_load_from_shrink_as_index(self): with Context(CHECK_OOB=1, SPEC=2): - buf0 = UOp.param(0, dtypes.int, (16,)) - buf1 = UOp.param(1, dtypes.int, (64,)) + buf0 = UOp.param(0, dtypes.int, 16) + buf1 = UOp.param(1, dtypes.int, 64) shrink = UOp(Ops.SHRINK, src=(buf0, UOp.const(0, dtypes.int), UOp.const(4))) ld0 = shrink.load().index(0) to_uops_list([buf1.index(ld0.valid((ld0 >= 0) & (ld0 < 64))).load()]) def test_load_bool_as_mask(self): with Context(CHECK_OOB=1, SPEC=2): - buf_bool = UOp.param(0, dtypes.bool, (16,)) - buf_int = UOp.param(1, dtypes.int, (8,)) + buf_bool = UOp.param(0, dtypes.bool, 16) + buf_int = UOp.param(1, dtypes.int, 8) gidx = UOp(Ops.SPECIAL, src=(UOp.const(16),), arg="gidx0") ld_bool = buf_bool.index(gidx).load() with self.assertRaises(RuntimeError): @@ -162,7 +162,7 @@ def test_load_bool_as_mask(self): def test_in_bounds_access_gated_local(self): with Context(CHECK_OOB=1): # Define buffers - gbuf = UOp.param(0, dtypes.uint, (400,)) + gbuf = UOp.param(0, dtypes.uint, 400) sbuf = UOp.placeholder((8,), dtypes.uint, slot=0, addrspace=AddrSpace.LOCAL) # Define indices, valids and barrier @@ -186,8 +186,8 @@ def test_in_bounds_access_gated_local(self): @unittest.skip("Bool load is not supported yet") def test_load_mask(self): with Context(CHECK_OOB=1): - glbl0 = UOp.param(0, dtypes.int, (16,)) - mask = UOp.param(0, dtypes.bool, (16,)) + glbl0 = UOp.param(0, dtypes.int, 16) + mask = UOp.param(0, dtypes.bool, 16) ridx = UOp.range(20, 0) ld0 = UOp(Ops.LOAD, src=(glbl0.index(UOp.const(ridx<16&mask, ridx)))) to_uops_list([ld0]) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 5ed5f379c4e46..88025ef476805 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -116,7 +116,7 @@ def test_uop_scalar_const_lifts_kind(self): def test_store_weak_value_uses_destination_dtype(self): with Context(DEFAULT_FLOAT=dtypes.float16): - dst = UOp.param(0, dtypes.bfloat16, (1,)).index(UOp.const(0).cast(dtypes.int32)) + dst = UOp.param(0, dtypes.bfloat16, 1).index(UOp.const(0).cast(dtypes.int32)) gate = UOp.const(True) out = graph_rewrite(dst.store(UOp.const(5.0), gate), pm_commit_weak) # a bare weak CONST commits directly: the pass runs without symbolic, so a CAST here would survive it @@ -132,7 +132,7 @@ def test_weak_srcs_commit_only_at_a_concrete_lub(self): def test_derivable_const_rounds_at_the_derived_width(self): # re-rounds a derivable const in place (still bare) so value-keyed folds (x*1 -> x, x*-1 -> NEG) still fire - x = UOp.param(0, dtypes.float32, (1,)).index(UOp.const(0).cast(dtypes.int32)).load() + x = UOp.param(0, dtypes.float32, 1).index(UOp.const(0).cast(dtypes.int32)).load() mul = graph_rewrite(x * UOp.const(-0.9999999893980771), symbolic_simple+pm_commit_weak) self.assertIs(mul.src[1], UOp.const(-1.0)) self.assertIs(graph_rewrite(x * UOp.const(1.0000000106), symbolic_simple+pm_commit_weak), x) diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 1ce6b428ea299..012781ca7b6a3 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -26,24 +26,24 @@ def signal_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) - return UOp.param(0, dtypes.uint32, (1,))[0].store(val.cast(dtypes.uint32)) + return UOp.param(0, dtypes.uint32, 1)[0].store(val.cast(dtypes.uint32)) def wait_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) - return (v:=UOp.param(0, dtypes.uint32, (1,), volatile=True).after(l:=UOp.loop(0))[0].load()).end(l, v < val.cast(dtypes.uint32)) + return (v:=UOp.param(0, dtypes.uint32, 1, volatile=True).after(l:=UOp.loop(0))[0].load()).end(l, v < val.cast(dtypes.uint32)) def timestamp_prog(): if WIN: val = UOp.const(0, dtypes.uint64) else: - fn, ts = UOp.param(1, dtypes.uint64, (1,)), UOp.placeholder((2,), dtypes.uint64, slot=0, addrspace=AddrSpace.REG) + fn, ts = UOp.param(1, dtypes.uint64, 1), UOp.placeholder((2,), dtypes.uint64, slot=0, addrspace=AddrSpace.REG) call = fn[0].load().call(UOp.const(6 if OSX else 1, dtypes.int), ts[0], ret_dtype=dtypes.void) # clock_gettime(CLOCK_MONOTONIC, &ts) val = ts.after(call)[0].load() * 1_000_000_000 + ts.after(call)[1].load() - return UOp.param(0, dtypes.uint64, (1,))[0].store(val) + return UOp.param(0, dtypes.uint64, 1)[0].store(val) def worker_prog(): - ring = UOp.param(0, dtypes.uint64, (RING_SLOTS * CMD_SIZE,), volatile=True) - wait, done = UOp.param(1, dtypes.uint64, (1,), volatile=True), UOp.param(2, dtypes.uint64, (1,), volatile=True) - sem, cur = UOp.param(3, dtypes.uint64, (1,)), UOp.range(2**64-1, 0, dtype=dtypes.uint64) # sem is unused on windows, it has to come last + ring = UOp.param(0, dtypes.uint64, RING_SLOTS * CMD_SIZE, volatile=True) + wait, done = UOp.param(1, dtypes.uint64, 1, volatile=True), UOp.param(2, dtypes.uint64, 1, volatile=True) + sem, cur = UOp.param(3, dtypes.uint64, 1), UOp.range(2**64-1, 0, dtype=dtypes.uint64) # sem is unused on windows, it has to come last # spin on windows, sem_wait to sleep on posix if WIN: ready = (v:=wait.after(lw:=UOp.loop(1), cur)[0].load()).end(lw, v <= cur) diff --git a/tinygrad/runtime/ops_qcom.py b/tinygrad/runtime/ops_qcom.py index 7b12d8cc82ddb..63fc9badef8b1 100644 --- a/tinygrad/runtime/ops_qcom.py +++ b/tinygrad/runtime/ops_qcom.py @@ -20,7 +20,7 @@ def dcache_flush(): from tinygrad.uop.ops import UOp, Ops, KernelInfo from tinygrad.codegen import to_program - buf, n = UOp.param(0, dtypes.uint8, shape=(1,)), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) + buf, n = UOp.param(0, dtypes.uint8, 1), UOp.param(1, dtypes.int, shape=(), name="n", addrspace=AddrSpace.ALU) i = UOp.range(n, 0, dtype=dtypes.int) flush = UOp(Ops.CUSTOM, src=(buf.index(i * 64),), arg=('__asm__ volatile("dc cvac, %0" :: "r"({0}) : "memory");', dtypes.void)) sink = UOp.sink(flush.end(i), UOp(Ops.CUSTOM, arg=('__asm__ volatile("dsb sy" ::: "memory");', dtypes.void)), diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 88c914bf66657..3bc3806b29dee 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -139,7 +139,7 @@ def _get_enqueue_devs(call:UOp) -> Any|None: def copy_with_kernel(call:UOp, dst:UOp, src:UOp) -> UOp|None: if (devs:=_get_enqueue_devs(call)) is None or Device[(dev:=to_tuple(devs)[0])].has_copy_queue: return None - d, s = (UOp.param(i, dst.dtype, (n:=dst.max_numel(),), device=devs) for i in range(2)) + d, s = (UOp.param(i, dst.dtype, n:=dst.max_numel(), device=devs) for i in range(2)) ast = d.index(r:=UOp.range(n, 0)).store(s.index(r).load()).end(r).sink(arg=KernelInfo(name="copy"), tag=1) return call.replace(src=(to_program(ast, Device[dev].renderer), dst, src)) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 28b7662de51fd..aafd8147c726c 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1166,9 +1166,10 @@ def set(self:UOp, val:UOp|ConstType, end:UOp|tuple[UOp, ...]|list[UOp]=()) -> UO # TODO: this should replace placeholder @staticmethod - def param(slot:int, dtype:DType, shape:tuple[sint, ...]|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, + def param(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, axis:int|None=None, volatile:bool=False): if dtype in dtypes.weaks: raise RuntimeError(f"cannot create param for weak dtype {dtype}") + if isinstance(shape, (int, UOp)): shape = (shape,) if shape is not None and axis is not None and isinstance(device, tuple): shape = tuple(s*len(device) if i == axis else s for i,s in enumerate(shape)) src: tuple[UOp, ...] = (UOp(Ops.NOOP) if shape is None else shape_to_shape_arg(shape),) From 38fa0643cf512a8f95d245db0065670c79f90a87 Mon Sep 17 00:00:00 2001 From: George Hotz Date: Thu, 27 Aug 2026 14:27:22 -0700 Subject: [PATCH 236/306] hotfix: update max test time from 300 -> 90 --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 8c84ede27fe19..61d5ea1acd30e 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,7 @@ @pytest.hookimpl(wrapper=True) def pytest_runtest_call(item): - t = threading.Timer(int(os.getenv("TEST_TIMEOUT", 300)), os.kill, args=(os.getpid(), signal.SIGABRT)) + t = threading.Timer(int(os.getenv("TEST_TIMEOUT", 90)), os.kill, args=(os.getpid(), signal.SIGABRT)) t.start() try: yield finally: From ee3161e9240111720de7fde40fe17a249dfc6e3f Mon Sep 17 00:00:00 2001 From: George Hotz Date: Thu, 27 Aug 2026 14:44:06 -0700 Subject: [PATCH 237/306] hotfix: decrease dims in test_attention to get below the 90s limit --- test/unit/test_attention.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/unit/test_attention.py b/test/unit/test_attention.py index 6ed281b7eec29..7a39473781f34 100644 --- a/test/unit/test_attention.py +++ b/test/unit/test_attention.py @@ -73,10 +73,10 @@ def _tensor_linspace(self, start:float, stop:float, shape:tuple[int, ...]) -> Te return Tensor.linspace(start, stop, int(np.prod(shape)), dtype=dtypes.float32).reshape(*shape) def _make_config(self, **kwargs): - return TransformerConfig(**({"num_blocks":1, "dim":32, "hidden_dim":64, "n_heads":1, "n_kv_heads":1, - "norm_eps":1e-5, "vocab_size":32, "head_dim":32, "rope_theta":10000.0, - "rope_dim":32, "v_head_dim":32, "max_context":4, "ssm_layers":(True,), - "ssm":SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32)} | kwargs)) + return TransformerConfig(**({"num_blocks":1, "dim":8, "hidden_dim":16, "n_heads":1, "n_kv_heads":1, + "norm_eps":1e-5, "vocab_size":32, "head_dim":8, "rope_theta":10000.0, + "rope_dim":8, "v_head_dim":8, "max_context":4, "ssm_layers":(True,), + "ssm":SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4)} | kwargs)) def _make_block(self, config:TransformerConfig) -> GatedDeltaNetBlock: block = GatedDeltaNetBlock(config, config.ssm) @@ -229,7 +229,7 @@ def test_kda_channel_decay(self): np.testing.assert_allclose(block.recurrent_state.numpy(), initial_state.numpy() * alpha[..., None], rtol=1e-5, atol=1e-5) def test_kda_prefill_matches_decode(self): - config = self._make_config(ssm=SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32, kda=True)) + config = self._make_config(ssm=SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4, kda=True)) block = GatedDeltaNetBlock(config, config.ssm) for p in nn.state.get_parameters(block): p.replace(self._tensor_linspace(-0.05, 0.05, p.shape) if len(p.shape) > 1 else self._tensor_linspace(0.05, 0.1, p.shape)) @@ -245,7 +245,7 @@ def test_kda_prefill_matches_decode(self): def test_varied_chunk_sizes_match_decode(self): for kda in (False, True): - ssm = SSMConfig(conv_kernel=2, state_size=32, group_count=1, time_step_rank=1, inner_size=32, kda=kda) + ssm = SSMConfig(conv_kernel=2, state_size=4, group_count=1, time_step_rank=1, inner_size=4, kda=kda) config = self._make_config(ssm=ssm) if kda: block = GatedDeltaNetBlock(config, config.ssm) @@ -267,7 +267,8 @@ def test_varied_chunk_sizes_match_decode(self): np.testing.assert_allclose(chunked_recurrent, decode_recurrent, rtol=1e-3, atol=1e-3, err_msg=f"{kda=} {chunking=}") def test_start_zero_resets_realized_state(self): - config, x = self._make_config(max_context=3), self._tensor_linspace(-1, 1, (1, 3, 32)) + config = self._make_config(max_context=3) + x = self._tensor_linspace(-1, 1, (1, 3, config.dim)) block = self._make_block(config) self._run_attention(block, x, 0) restarted = self._run_attention(block, x[:, :2], 0) From f06832bf6f2bcfdfe52964c8cbbb33cbd88ccae3 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 27 Aug 2026 16:01:55 -0700 Subject: [PATCH 238/306] test llm with --no_chat_template (#17785) --- .github/workflows/test.yml | 8 ++++---- tinygrad/llm/cli.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38b87c21cd13b..3b2e2bd58691f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -390,10 +390,10 @@ jobs: run: | parallel --link --tagstring '[{1}]' '{2}' \ ::: llama 'llama q4' qwen3.5 qwen \ - ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ - $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --model qwen3:0.6b | tee /dev/stderr | grep -i hen' + ::: $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model llama3.2:1b-q4 | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a male chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3.5:0.8b | tee /dev/stderr | grep -i rooster' \ + $'echo "What\'s a female chicken called? Answer with only one word." | python3 -m tinygrad.llm --no_chat_template --model qwen3:0.6b | tee /dev/stderr | grep -i hen' # NOTE: qwen is dumb and only knows about female chickens # ****** Models Tests ****** diff --git a/tinygrad/llm/cli.py b/tinygrad/llm/cli.py index e0e2336e2c943..417ffa7518b50 100644 --- a/tinygrad/llm/cli.py +++ b/tinygrad/llm/cli.py @@ -145,6 +145,7 @@ def main(): parser.add_argument("--serve", nargs='?', type=int, const=8000, metavar="PORT", help="Run OpenAI compatible API (optional port, default 8000)") parser.add_argument("--warmup", action="store_true", help="warmup the JIT") parser.add_argument("--benchmark", nargs='?', type=int, const=20, metavar="COUNT", help="Benchmark tok/s (optional count, default 20)") + parser.add_argument("--no_chat_template", action="store_true", help="Don't use the model's chat template, always use the fallback template") args = parser.parse_args() # load the model @@ -160,7 +161,7 @@ def main(): # use the model's chat template if jinja2 is available (enables model-specific formatting) template: jinja2.Template|FallbackTemplate = FallbackTemplate(tok) - if (ct := kv.get('tokenizer.chat_template')) is not None: + if not args.no_chat_template and (ct := kv.get('tokenizer.chat_template')) is not None: try: import jinja2 env = jinja2.Environment() From fede358811ec40bcf10e11e3fc93d9ef55ebd943 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 27 Aug 2026 19:59:27 -0400 Subject: [PATCH 239/306] fix fancy indexing with uint8 index (#17787) * fix fancy indexing with uint8 index * fix --- test/backend/test_ops.py | 4 ++++ tinygrad/renderer/isa/x86.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index d0263b9f5b982..30f564d6d18e2 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -2963,6 +2963,10 @@ def test_fancy_indexing_inf(self): data = [math.inf, -math.inf, math.nan] helper_test_op((), lambda: torch.tensor(data)[torch.tensor([0, 1, 2])], lambda: Tensor(data)[Tensor([0, 1, 2])]) + def test_fancy_indexing_index_dtypes(self): + helper_test_op((), lambda: torch.tensor([10., 20., 30., 40.])[torch.tensor([1, 2, 3, 0])], + lambda: Tensor([10., 20., 30., 40.])[Tensor([1, 2, 3, 0], dtype=dtypes.uint8)]) + @slow_test def test_slice_fancy_indexing_no_dim_collapse(self): a,b,c,d,e,i,j,k,o,p = self._get_index_randoms() diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index f90c8d5686919..79a8a17945c16 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -282,7 +282,7 @@ def shift(x:UOp, op:X86Ops) -> UOp: # it is materialized as an immediate so the address stays correct if the base register is ever spilled and refilled def fold_address(x:UOp) -> tuple[UOp, UOp, UOp, UOp]: def _disp(v:int) -> UOp: return imm(dtypes.int32 if abs(v) > dtypes.int8.max else dtypes.int8, v) - def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v + def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v.cast(dtypes.uint32) if v.dtype.itemsize < 4 else v if x.op not in {Ops.INDEX, Ops.SHRINK}: return (x, UOp(Ops.NOOP), _disp(0), imm(dtypes.uint8, x.dtype.itemsize)) base, idx = x.src[0], x.src[1] # buffers are indexed by element, everything else (the stack pointer) by byte From e6ad2907a8abd9f0fe7eb7e013b6ca6e6bc51c6f Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 27 Aug 2026 17:11:27 -0700 Subject: [PATCH 240/306] onnxruntime respects NUM_CPU_THREADS (#17788) --- extra/onnx_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extra/onnx_helpers.py b/extra/onnx_helpers.py index 7d3af8fa704c9..c11a8d41f8353 100644 --- a/extra/onnx_helpers.py +++ b/extra/onnx_helpers.py @@ -1,10 +1,12 @@ from tinygrad import Tensor +from tinygrad.helpers import NUM_CPU_THREADS from tinygrad.tensor import _to_np_dtype from tinygrad.nn.onnx import OnnxRunner, OnnxValue import numpy as np import onnxruntime as ort ort_options = ort.SessionOptions() ort_options.log_severity_level = 3 +ort_options.intra_op_num_threads = NUM_CPU_THREADS.value def get_example_inputs(graph_inputs:dict[str, OnnxValue], config={}): """ From 38e0fe103b3c6e647e247e068d9228a6ba108713 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 27 Aug 2026 21:03:43 -0400 Subject: [PATCH 241/306] fix where with float64 branches (#17789) * fix where with float64 branches * fix --- test/backend/test_dtype.py | 5 +++++ tinygrad/renderer/isa/x86.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/backend/test_dtype.py b/test/backend/test_dtype.py index 94d37090e54e7..fac4a39b08675 100644 --- a/test/backend/test_dtype.py +++ b/test/backend/test_dtype.py @@ -258,6 +258,11 @@ def test_float64_increased_precision(self): a = [2, 3, 4] np.testing.assert_allclose(func(Tensor(a, dtype=self.DTYPE)).numpy(), func(torch.tensor(a, dtype=torch.float64)), rtol=1e-12, atol=1e-12) + def test_float32_compare_selecting_float64(self): + a = Tensor([1.0, 2.0, 5.0, 9.0], dtype=dtypes.float32) + p, q = Tensor([10., 20., 30., 40.], dtype=self.DTYPE), Tensor([50., 60., 70., 80.], dtype=self.DTYPE) + _test_op(lambda: (a < 3.0).where(p, q), self.DTYPE, [10., 20., 70., 80.]) + def test_float64_to_float32_cast_inf(self): _test_op(lambda: Tensor([3.4e40, 3.4e38, 1, 0], dtype=dtypes.float64).cast(dtypes.float32), dtypes.float32, [float('inf'), 3.4e38, 1, 0]) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 79a8a17945c16..fa105f841825a 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -4,7 +4,7 @@ from typing import cast from tinygrad.dtype import dtypes, DType, truncate, AddrSpace from tinygrad.uop import FastEnum, auto, Ops, GroupOp -from tinygrad.uop.ops import UOp, UPat, PatternMatcher +from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype from tinygrad.renderer.isa import ISARenderer, IselContext, Register, PreRegAllocContext, greg from tinygrad.helpers import getenv, NUM_CPU_THREADS, unwrap, Target @@ -150,9 +150,9 @@ class X86GroupOp: # no cmpne for packed ints, y != x => !(y==x) (UPat(Ops.CMPNE, src=(UPat.var("y", dtypes.ints), UPat.var("x")), name="cmp"), lambda y,x,cmp: UOp(Ops.CMPEQ, src=(y,x))^True if y.max_numel() > 1 else None), - # float WHERE needs a mask unless its comparison already has a float operand + # a float WHERE blends at the width of its value, so it needs a comparison at that width to make the mask (UPat.var("m", dtypes.bool).where(UPat.var("a", dtypes.floats+(dtypes.weakfloat,)), UPat.var("b")).named("w"), - lambda m,a,b,w: m.cast(w.dtype).ne(0).where(a, b) if w.dtype in dtypes.floats and not dtypes.is_float(m.src[0].dtype) else None), + lambda m,a,b,w: m.cast(w.dtype).ne(0).where(a, b) if w.dtype in dtypes.floats and promo_dtype(m.src) is not w.dtype else None), # rewrite -x -> 0 - x (UPat(Ops.NEG, name="x"), lambda x: UOp(Ops.SUB, src=(x.const_like(0),) + x.src)), # TODO: add support for mod, requires support for accessing the 2nd+ reg of a multi output instruction From 6b361e4de7e3e892c3e7351d1a6bd45c85ca3894 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:13:24 -0700 Subject: [PATCH 242/306] move shape into arg for param/buffer (#17783) * move shape into arg for param/buffer * no param_from_shape * drop gratuitous syntax changes * image is a in-graph view, folded into the param arg at render; drop dead multi param sharding * view_as helper, simpler resolve_function, spec update * spec: param/buffer are flat storage, no shape input * image dims live in the param arg from transform_to_image; tighten kernel graph spec * kernel graph spec: only RESHAPE/SHRINK over storage values, not all movement * kernel graph: call args are storage, not views (pm_no_view_args); assert in spec * strip views at the kernel graph level (pm_no_views), move into rangeify * touchups --- extra/llama_kernels/__init__.py | 6 ++-- spec/tinyspec.pdf | Bin 98812 -> 98831 bytes spec/tinyspec.tex | 12 +++---- test/null/test_uop_graph.py | 3 +- test/null/test_uops.py | 3 +- test/null/test_viz.py | 2 +- test/unit/test_call.py | 5 +++ tinygrad/codegen/__init__.py | 10 +++++- tinygrad/codegen/decomp/dtype.py | 4 +-- tinygrad/codegen/gpudims.py | 2 +- tinygrad/codegen/late/coalesce.py | 6 ++-- tinygrad/engine/jit.py | 2 +- tinygrad/mixin/gradient.py | 6 +++- tinygrad/schedule/__init__.py | 2 +- tinygrad/schedule/multi.py | 5 --- tinygrad/schedule/prepare.py | 19 ++++++++--- tinygrad/schedule/rangeify.py | 14 +++++--- tinygrad/tensor.py | 8 ++--- tinygrad/uop/ops.py | 53 ++++++++++++++++++++---------- tinygrad/uop/render.py | 2 +- tinygrad/uop/spec.py | 20 +++++------ 21 files changed, 113 insertions(+), 71 deletions(-) diff --git a/extra/llama_kernels/__init__.py b/extra/llama_kernels/__init__.py index d16187e176352..9308bc169515a 100644 --- a/extra/llama_kernels/__init__.py +++ b/extra/llama_kernels/__init__.py @@ -1,8 +1,6 @@ from __future__ import annotations import functools, pathlib -from dataclasses import replace -from tinygrad import Tensor, dtypes -from tinygrad.uop.ops import shape_to_shape_arg +from tinygrad import Tensor from tinygrad.runtime.support.compiler_amd import HIPCCCompiler FP8_MAX = 448.0 @@ -12,7 +10,7 @@ @functools.cache def _local_abs_max_fxn(x_p, device): x = Tensor(x_p, device=device) - inner = Tensor(x.uop.replace(src=(shape_to_shape_arg(x.uop.shard_shape),), arg=replace(x.uop.arg, axis=None))) if x.uop.axis is not None else x + inner = Tensor(x.uop.src[0]) if x.uop.axis is not None else x # the per-shard view of the flat param return (inner.abs().max(),) def local_abs_max(x:Tensor) -> Tensor: diff --git a/spec/tinyspec.pdf b/spec/tinyspec.pdf index 4e380f5d9db2f68a7b1fa922f8b1c353bd40f2f9..e9dd6e4207c8a08c2c6dd4faad5240554cf38ece 100644 GIT binary patch delta 25824 zcmV)2K+M1Vg9eX;29P&@Ff%?r3UhRFWnpa!c-rk;NpBp-6~5PN>*6%H?!%qis;P_u0oIE%Y zSfKw7Z$4n`9e%#aKkqJn{^`}>4}TE{;g}RgYjJq>csLp+tqJaba8!zUZk#^6I{*ES za~n9<5sPEZ@>o^ewHB zzQGu+8p;gi(jG=&#;LCA@146#^J!(;g=SbzGw1|Hx99qEYWjQKpKfJDJl55n3BE$tR9&)73k zx|}NB2DdUk?`OvSjuN~JLfFGmC>FGHWjme^E7K=!L1RQ{MO%sfNb}B2e z6yxUg?d^;SoCVOMF~P|qK1+tn1@o+w-sMdy5r@U0j@Cd;>f$5sRwiysezX2Y;}}Pz z%E}P!U{uj8+A>;@*@=S%Lya*jW(`L^>L8fYI4RVBI$0UFJiL`f@T{5(Uj8Q9U{`)C zW+rOHIbSEOFzCwW4Mem$ocooKj8u2b!UoAT)#JhGQ1Pg|*%H#OM$dEmqL>W2W3G{G(2$4E3;MjW|7o}ZNEN!i%~x>1`^@PS@XV-ThY6te%R zUW>L9;xzRnk_YBx;7u;*1ke>Fvg=B&;I1=&O-r|;;ZA*f{yg2tFz>k5&0)z#l(>c5 z+d>7^Y1r--<4shgVIoM9$k`4`07fY8o_dLjq8Robs~1&$&^9;_RG zg=TOsN#l9b?<%5770?sPZ{w3{u$E<{TKwS&J**mGWp9+?=BKs6t$^2!fR`X6=q@y` zz*=NWc%!-z*|^*_Hl{B&S0 zI4;BCKVXQEzyuC0R?RxYkBEpyzN z<+bZ@e{HvQZRHsv91+U3)xG!K4YKE%>GqN;1|&0fgCe%#spj*&aLo$W03up{`RWMb zG~b22E(?x%Q*9StYv9|_J@ zCisboKCC4v=O+lO#h=8?7tsf4{5ATJrY-ehT%hCMT053n5uI}NOILXLEp4djo^v)~ zymz(OB=EZ|3q#7Md@~Y-M#W5H;@m-F;=EmriO=k4%z{eW8siBVsSS;P+17`t)JbcZ zL8LMfyN;6SUe6af0Bn;p8zOsPdtOSwj`Yo4S#;?Mlw^@A8kt;3UHG)TQFVQpRy0%2 z_+3};QFIOj=H^f07bi=80J)f3T4f_s0n&3&YH1^mWJI|_nsLP)W$(S}VnkhOFa$BD zULMQsW^`x}ZgKaS)Clf>nRBwwh$VH2ZUfAfWgDF&;i!;9(}Kgy&#YU)y_sD$J%;dC|->K}g)oMTiP z`0kt4(8pYTT10ezy*yaOoUl%mt49}euXr+BIq@+#u^f|7xTn|Q9vy5bd@X`M!wjN7 zwc5Fl0&~2Vek;1(3}8FW5E-fRL_snU>lZ(!` z*a~C8Fr|2>LyIWZ3udh76I-7niUl%*SAn7|Ze543iDEUp4?v*l!?HfN(?X3YUKlAu zaA9Vsi49s6L;dRZ=+#D@gb;U{NN6Q^CFSw6AX|76-+} zyeb8+2Xnn&;L;Fy=L^>38z4_I;godiqCick)pSQGe-EXAY-KhBNJ<~UUCIeq%mJ#b zn)qePU#xw(yDkG)vKZB0OTKW6u(~q4#hb_3F1VV1_Cbo3!1YMEz=?#LZuX)q?QWMz5KM2*$xP?EO=?-%4W^cL(UWV$@=#w}38XU=#ncUj<> zao!i8@cSv((_h8F^{=L@1!O>isb=+`4?hy}xS8hokTkYvbFde)#@fQnP@~bxiF8$e zl;1|If0!-@x=UZH0`rANIQ45fAQ){lMrnVQLXB0IDI^6xf&D8rWKeI03t@YaHxOgc z6MCORdFrlHXv!u}S(Pe?KWsS<3TWiqd4y5+CDIM1_Z=$_nj@~UY?Ayb61}zqQ85;K zOg`4tev=+Ll~{FK+ZXlc{bcOuvVbdp0<tzc7@LLk%Nm^_ zWgfFTtIU9wec6|ggBF7|8k@F;KyNI`4O)rbXX&fm*z&5Y>Ktfth3EFvBtf-*j8#or zEE9*HUD;7lw>R$#yY#>=L3&6r^}sJaT^)LDVYG$_V}EtWJXqky#l=$>|B60b`mh^# z3&IKf=+!QQ2GHU`^DE55nkl7IYT4?$g=wcD8o@UgYSL`um=1lr;A1-t-}Hbz+#4mq zg7m@_QFuSi$Y7t)wXN4Y^r?$~m3wWfT2$Wp&?!4Ujc7Q!KBaz=zz zyj}hdt6f33!o8ADwq1%y_IZR2ljuM}10zA`AOMv+to*Ey9IxLkt9d7AKz`*Do-zknP(cpt;KQWAk`OR2nCt_VDzd`I3x-e zF&gfG)y5>ctPCpHFO1Z3=rHX&8hd9@+mk|X0@j+btmbZn zJNSkflDJb^4c^rtoA4{!QpBe}5In<$lZoN^R_EMKtm?jMB zvZU>1cg7<&u`_EF>4`s9CpRpZi zIe57t$S73TJp!PGQ;ZUOzVNQch2|gQsHYN&wA`g@{#k}MK8H7t5h322cTEsc#vO?oE(@*MW=@TP-v;5Fhlkg$6HkoA9~1W(+!q zEhJ-k(K>uh*9___;1ZX5UjXLsb&Yh`=xrniW3<~#z$3~+=3Gx2_jZ){x*uMoP|YBvt2VKsSu7jRyG5+s(R>u24Bu~PMo8lgM4 z+}&2}4ai6xHv6SXbf)RB1!Sf^a+D_(MoIP``I?I@r*609lr~APAWk=-fQxsr)MX##(aLG*_59K9)Zy%- z7XL8Hx4-)ZALKwMw3A#9JAXGmJ_>Vma%Ev{3V7P>TuEE z8;A|$6gzYC_#la)kvNmUcAWU<=j(l8H%Vzm$Q_|DMI zaJM=C-|p!h#=ghvtMvN5zHiJD{tBn@*C?_3=BvrzPv7spc(eQDOMkQDd~-^1ZFg^u zyE|^u5_V(42(^22`1YPLdSHw{F($2l?eBz97*z}}$(W}7cW?f<`@@^Ei!e7=n8j%= zjP`R2vnj`w>Zj)X5z|S5>4d>l2P}XWJ>s*&XTlQPkLA1kHc0qI;~ko%v6Ck-R-OtQ z#}%pfWc^y?uh4ybMt>ahNWM25_A%Z|Vt*Gu9Q}oSn_n=~BrJZ2pZ$BK$d9|tIK#;8 z{?kzRjyBGX(z|y%DdlFB60#2defQ_x*NKLF2c~P%vSp9KC$VYVs}qAFE5f3hHYN?` zW9eF;6!V)5*R9&PmfDr;{t*7U04F%^^^b5*5Kn@1#=~Cl`+rK{aNhQe(pj#lKO@AI z$lBqMWwbq|S)5X^4dSCH?NsQq65_Y>$(OF3>x39eC59URh2w)uoHcrHy+BZ&h>Lb6 z!8gW*i4)yC+{SxI?)2u2urlb6+DL1zaRg9A3LEZax08*@4J^QP&5i4|HT2i;m#od7 zwf3D5!mA{tSbym(3_(LNWRqNY2DAz&vmzdnGEtf*NKjPGeIU0u=F@fj=43YR1Z$Wi z_SQC%nY3}r7PU!>(4PCS%=f~#@UiGlQfP1HymAYgNtqDXgp{H@ zwv@0$8AFwBl`7rAq}i;)O(WCAWv2fyXS!n?MCh5WV1LY^oSr!kjV(RK?TJXJaFW1k zR_$-3Mwnqn$UEW0MphHY-lB^!b7%5@t$l5+&3@VwT_sOdwNy4tHY~clKw{H4xf5R; z9`)aZ$=`x&6xlU~J0FwgP?Z-tmW6dP3&}spLy|ZjUU#bALzrL{68w&uJ~JH*arlr-HayBa0yZdoB!clH|CY3*ulNPNm`I)7v|)^=fkX zU8S9zOWM_w|K7iu_~yUOD8eE6irH+F-`-BenKt55o2=uVz=|jt*?HI~ElDaMq)Oda z8={lk1i29Tt`N~s6?jQ?tjb`JZJN6RxE4pOv429(4?m>atQL%}b_`Lj%eS=O;$B&+ zeO2F96j|D4L5Q#vI6KO`_X1_b*OUJ@w|7=Rq^8mXlWlzTcTRc?gY>k#2~nvhrGyko zsr^~F|2oSO+5jcnC2y;Gf0rdG+P~IZ=7k!|!$x}q?o6c$Ou`tZAq#_&64gKSe?M#JBbE{=m2)5SXnW$`4>>@blYF`9T;!UzkY4XJVN zCS#I<@xCsW5JH!7~k?>Z>QA&16s z2j`4TKBdWKo6VxS>Qtz68#&-I)Y0cqJX(wibqeHo!8X!Nr>hsU#0^p+z)kYR!^o#J zRU$M!V_KL>p6M25(1T(JQZ34VT&v${3pUw066Cdqt`11}Zwl=Mtv;5nXG?eJBuz`+$r=Er+ipq-kz0;J&5SpDiyAHrl0;UMJi4p)b-vHl2u zitr$*wUo^wY>u2saC3$6AEV@Wg)SQZHZw6i!VQ`=0#*!SYr=z&9a|ILub-b;8|wBH z6{qP(>pCI1mkFs}gzAK_8Tlp8nb~Kv(Avk}P|-rry6rtr==gM9lVEjTO&FMqV8pVS z8|a{eo&l{7tBxQLmeo{m1g9!H4b2#TG_@TCjT6T*<1_Y@Q;R0yN^DZBBBDhk+~ zx|qDXWNj`w0a$uKM{Lx%JX&0UQNHlwJr&@%6yA@6mt6oU7-%J(Y79@XG{*+Np&W0& z%pGVA+v2<@8&+#;L=rkUa6x zK>L-Y+LUlGZ;QGNbW1-pXx~qga|2nWGGVxX4(37Gyv+0%gz@y@3L!kT{sa-s}+#TL>0Y=aKqTSmv`V6@$T$k`^1>!71jw9LV0B-k1gA`YFi0LxD?AwH1Xtyy|4 zR^Nbhz1W;MUPAht;_Kjgc((qRfEJ;B?XYoVOBr=zK?!+hQ!TO$9-c+eTeUSM5u}l? zwU3nKkBRlJvq{%cbQaQN;}(r^XRZSH{7so1zIuiyFGfCG40Q{ub5R-=& z9el4%O7V{0|5-&x-rue<`7Tn52g}7uS#JN6nQ_D_<23dic~gQXfUuu8d&~@usN%ef6h=rdNX1e|ir5B+et(Q7 zJ6ES#+i8ysmz_~|K399wjLHpua~6iT+>j>j`E=ztW||tejFn7P%|gk}a}1O9vp}H< zE|4MC&x^E;ZhFxg7Z)p8F+0zP+5=E)nGGA(DUwe#87DoYNOic$99&;<#8!M-dMAOL zDkKkWNR;D|jG5LPT&_W006z{PJb!g%jLQ}ntvVc~uqqD1^xEa?4I##-+$oqr`iRMt z3}Ch?Ah3do7I3>N;zM!II@#0&D=5CkGCE}RY#F$Tx7`fsIA3NyU1wc&IKf?;szQsw zoP`dC$hJUQHHT>hHzyTUc$tqdJh-i#2fAg#y~n}JpocG;qN_{EHW4~47n7hF6@SAH z8X%2S@JNpbp@6Ph16C9?34u?+z|T{V2lzcg4{G4* z3J?L91>bFhe0&hNM;c6vMdfhnGJm`@+RdSX4Z%suT;A6yjn2-`EaE{8kWoJ`9(+k_ z$PlBe4Un@oUE-gEpcvy%cRSF384-fUrQ7ARUvC$Y3v^gI!{=RPDZ{MIL|1L-UqAiw zkEaC@eGe@vd&%?M{5qwd+7QgT^U>%D`c~_+Y($fWuBFazX3?m=Ge<{xo_}=;VtqTU z6U)xg7!u7q>8TFY!ffB-n0He9e1_gD*CMdlLALIxqJaYxSnK(E^k7*8&EcdQlKS4R zy0X%>ph5!8%v$+^3i$?MgtqydO|1S3YWaD}dcK$|CO%_R{JdfkaxgT*i7q;dJrmPp z8NEl6+A{vUXS?gO^^|XPS$9d;68Kq~mxu#!Qt#|YmELj`r>}8+3 zDOzvp-NUad`wLcaU3T`6)a7xmE&`vkU|(y8pF^~MIsbzzksH2l-hch`7PjN&Fh2&$ z>6)Z|5#@ST^*ei9!7Hf3WJC1u+2fSy=nGJ&@n^sFGt zR8KXEqt8~jZiT01`3oJZo)x*Gbo1?;2k>B~*wP2jEpv14-|Rx3Abhs!@^@2^sh#v% zaldYVGnVxtr>DDSLw^@SzY1V$q-41cvp9Gsisj&=V4#wK?hSs5A{nu-fDBvsAg-jO z=L$rukIy5Nm)Ugwc&-YEeM&?Q=DD+m4CA=mD_TVzH}<8o77sjVJ=i8{7s_>(p0kg@ z{q$n}x9N^N-YAWHwH=*v(ZBuZX!I6sDRcdX84>@hEy>H3cmZ(SbF$8z0YIv0RNf`AvsQgpB$-5Wy32%*rBKcsfN__^ObO# z(bo}M`;x!T4+6xN3iqbo%?p8aru#Jr>eh2*@Z~L8=sDlFF)pLHAAIM%Z1(2jlBW!@ zz_c3#oCF1W4}U-W%q3xxQ=g8AS?Ut^eX)uo@r!f*Dr8xscIY337#Tr5epQFps9mW# zD0C}Xv}>3DpTS>hI$nRM2zt?d+hUUPb9?5uLJ5!Cm&U^m{Izu77Mi9$R>4M#R?Aa3KQ~s&bTZleusgRV3M^_H|6{Xssb-x5D-9Um;@o}6 z0t1DvHg8MOy}Vg#TrsAP7t297%)}-voa?Y1Ans?@)BziE%(Gke*Ove<`n2r!>v z{{8R%4}B&m50lj@Is!H_lMyQ$e`{Z2VE{Xb4dj+YH_M0Na7Ixa$+l!TPJVs9S6zEo za~X}IF)SM#k>1dj3w_GiLm?1Z&bh1M_f9*J%Ja;NlN?>|}>7|V?`pm-gpMB-)Ar^$B+YJ$#)5<{? zDWA>jr>hcIgmUbx8|N!p$9cJ^C+5sxX67SDw(nm3_wH)2K&~gPh4p!X+9ZZ5ue7;j z;SkRvD#BKx5 zwz9o_v3+rMHCpacti$T?0!Q&058jCP_M0@atZx#NHCFL>&(BvoskfhB?YOb{Fyy|P zFlA=`x5M>f6g??rm|Bom%WLN<$^>TZKM8mc@bQg73K^NMyl@_@Lj3e_wP@5pOV{(G3e!^D_F#6@XE%N@@Y9vB+yDGSHdvrt zu2z%bT6CQEf30u`Z&h&>GpvmGZcHv6$R*8(G&LhKVOkIL+5`0HUakq6cOFJv0K6nH zva>cwQtNRDcm;Fy{{>x-OV@Y*xFHd=Q3O3%5aanr=iz>aiZj}Z40O+u>SmbWgC?AT zD^BvZ1Jz=VtY`wU|HS8CpRSIBB!WuTJVj{HS?C5^e>{Z4y*+Oj{$LSpL$q~lbc7Y0 zRGTKh&02hzS(_&RZ7}G!rh>$Gn(GN5`*XmSkVQWNl)qo`1RpKn8Yf5;pR_SbbTLIc zs-4KWrhEQyMtHDCk_Fnj{bIaIhV-eo{DJ|$91i3t&6w|5OV)TZaoXfU#wZCeXPG|9 z*yG69f9qFo#`e!m+%jk5L!XZ$!bu71(!j58ei`}-foMczrpKIyv^m3}5|JVVtRpwV zXJ*Y5IWtKny&j55l_&osChspMH^02t4u>{!@=hq3>`_S8n(cm^ti{bH=P^rixfxNy z!Ju++RPs>HaZXf{x3W5HTyc{v;TLgrW8*}{e<^J;1zuc@p^p-T4YcLU+yk2>R^I1A zNeW$LspLE%81!qG&ZicF%%Lpd-)k3>(W51TGRkUKCl*JOk&s(%%Wd@xx78Zkv?Y8s z7pi(p#@z-B>!SgVE_6}N{I#=t9QUZELXfmGq}Q~GaS=f#b1~a3ul4%tNeDQkNAR4r$|DY5!yr>#>O;2)!u83Fz{WEW&X2pps!) z$O2x^t`|#`oSpGqr;dnAO;i}wOqdtCb0~@l* zV&yJ#DMs9HhLt`J6K5RH76HUviS+9s`C3G6sQp2rrD_o>@w15cluKnnWTFhDe@jmV zZA`^GSM==Z7twhtg>)F>xMEQ4FycNQ^<*VEx279U1=VrBe@HqqU_swCt$j*5e?PV) z&wYr?o=xl;sUJ@ps(TTu%XL4Z81y637?yAfT_vRV_WPT|?ajNNMna$ey*={;rJWPsAV||z zjC)Js{psF$B)S?xn|Plo{ekiRe-QYk<^ko?;CesbJnuXW6I-~3aE>6f^xh>eGo(J! zKwPSVlg5rRXsODW=A0(Llu{a}rqKh;jb8Ns`PK0nq}wN_ZT7h5tZaWgorJDELgkOO zGT|;*+U?PJ151OjNq%f;=&gobG>gV%a<7^PH$+oHw2(y;B66#a(RqvoWTKKr;v6!gfZ^LfUl)&T~VA zW(kz*U=-MVz{x$b?nx9+V>e+ZiGH)PVsp1Lu)^_x#Ot-X1gCfm zl)E5MKVykhq3~W^xWu)3e+Nfd!n2Vn?9L`gIZ3D?X$?6?Vf7$VoVz6(XgiMPC1<{X zzYyE9)|DtTI}W(v&T-5V7bjt>rxfCoCzc1-z@sa*xP_&ZUU-eM)$Q46;k#@YV|Nq;->YE_8)TC);z8iKRPnufQ%HDE)X-GoYM!a+UeMvat@=OzTdSeO! z6?ZA?b@9H2>>&MI)j|d2$?IL8e`HB|9GI|`2HJ|fYdzg!^2}!d`)0hqxv9F}U#EHz zIypM=pq8nbu?>zgCjBXO6e=qneUeKdAD=Z*RH99eWXneTe{-;V+S}&n!m7s*eHJG2 z68I+=DOVXOk#x3rlPb7k12vsT-{+lnP17YSl-{@f;EhAZvA3W=X3-Vh5UfXb8i_yO z*q@au65evJ8NS3v!{h-Xz5*b-EQRi35 zZq8NW`IV0&WYV}EV1$iy-nGk4__(3Yd9G}VBkh1tf2XRw0O1A(VHRVmsAi0>cFG`F zv5whQ9HT@@5{7MA^{lCG;fg*bJ9deC@^iz)Q+ftmPc@Zc2>|ZN>V$2#|4elPwjTA5 zYY7Mqj7pJR9_MHtVud9*&m?Vq!0^~%8{f4p;u|BHCa~a{BKa;SwQ5KmmcW`*fAO27 zh-+ddf6$a9kX~8rD!J<9&alWJb}6Mi7CR(yapud)m7O*^v$ErrosM`5*o-4#DU#X# zeq7R$h%_|Umv|o3u%&kPltiGk%_r1V)p;%X?c&i$#Wg5$YVpD`A>VWHG2OE4PHzOJ zY8GkRI#dc+=bZ2Ax!7dD=b)qbiE|P@bW(84_w>6A#Q`qRO*j=u3$VilFn*uXKJ%TTZGl>{s*&dkPRd3@ioWpM z`U%-+N}JtnrJBxqTLe>=$Eyk|>{iY$*18q|18XT0z_1*%BmNT}KXbyJk>n|f=gcgFH{?Ms7nxRD)=50Hseq5$ndL(`6HuHx8DBfw!^YPwyN zvj5u8rC|Fekj-@@%$H{p!b*b06CIi*d%5}B*NaPUUq(Xv6+#1Oqn7DZV(#56f1~Dp z?!+LUDS29L`azt*kQNrgP~vzo;yBfaq=j`(_4Rbr{T;0QqRwtO+RSqINptYx%sg>1 z2gmZniw=zxrKcP~7a|Z*klP1TBeRIV)H4_O&|f^s3BWX(AY0a(S&YdQQ37ek2^Vpq zA94RdI6(`g;I06elp`TpQHNWqf9uZQT9KMSqGK@g6G`pnoK!N27xDtch8oge9Pc1L zG#5_t0JH-j;K}S{`rkZ={{3j~9Qp@Zlpl78Q})~_{Y8f+pSa+T<=0N0MeQ(~tB<|x zd)A$1M)~2GrLwgrsq+~7vbbQ*P0vt+DirCEO$;p;PBPS?poQX~u6m=1f05+Z(KmSs zGy9RmS4N5pF zC|^Z&>8d}0QP!WQ;FJ0ir$9K zv3zY6>&DRK0QuLfd}}K@tBB?e!WCTG_&4yXzj%^eIA4bRzteS^e~Hf^kSfQQgTLj7 zAB1hAEZ;a{T$Jr98ais6m;{~LK)4j;y}x*phLVHUDEZfD=supjyNz0V_b9Yo%ygay zx<3~{;AMs;sScZ#4n6vfp*yb;;TUn$Jh7bCJWm8tpBwVU_jkTf%sOgbfh$6e4G4VLH( zm$LqmmBS*1eO4y+WD?I8nd`b0mfgJ6u`Z;~2M50YVcfQd^D#abvRB&nsI@Rek8|7a zFR$LMwE%h?XpxtU_4Xy&VP!e;1$`Qm(KaStcCS4*BRZh<|w3`MqS(D;3D{7 z-6NxiFs%mNP)fJLwhQi0MK}jdpaUvXN7QHDmg;D#)u9fQ3a47h*AS*PF*8tG2iS*R z15&tmpeYq)X~mY2q(G9OhM4(yyIg++ZFTfBj|2uQ(PIgjlHdkcW{{f&&4{e)6?f`0 zO$P0I9&EP#QU`FkM8eF4Stn*BsRMrOZbu2!qzsUkU(KC?u9P~He(|Ss%2dp04NU!g z;#n!72BBlxSLw+l!*k%OcgHTyPQj3tO`g(l=tbK+?eLV>+_czpH$?LzZ9adlhHA^c zD{SYIoq5}AWI1963Bt82!oW@INAdwgy@0lM+B-5fRmwGv(Z4KMwrnPENJ+EYNiz*; zLek$;foO>BIYZyl#0~8%ZwIP%Lll^wU0i<;+_jo(ZbNl~Gp!)%v}tPPUP#|<4_Sz54>#I{kS&*> z1QDmXjj%w9Rv9yFGd3Nlqlqr5-7veUTrPzn=^-~7UFk19&-=>-fLrZ>th84iXW2(q zxKv;?#~rW0MIQ=e2aRnnX*ZE_0BzF-Vp>uH!YGcYHX%ciOG(+#uET!|muLv3W4}r} z5gd4YDTK*kx>ogyuQ;Jvk2F4+lzOxAlIi&f>&=I}f6_;Uel;&?%0ii*j~%;`ZpOV+ z7~g8@yv7N)1@vhfhWXK*Ku(b5V}q)VO5W`E*XosXbv;`wnlf1lfM)MV?mu@-KyL(N zZFgnnk4gFCx@Xqc%E*6*-q{VhCi7Ds=|8R4zhZssBLeWwhy zxPgZoTN2!btVt^|5)yU(b4{)kD2sYv*3(_Dsv~Bzzw*=Txz~e#oyN za=hkeQWQJ=w6@F8#`k?S(#Oqt<-`zsGHyfk_bQe5CZ@qx&#Bu$*h#WqSC+mq>2Ftj zywJNnd)=I+94~);jbyVuFDeaSP3dsQWj{LfvDa{I$uEcKV0(=~wOgFm9uW9WFmGQz zHt=5QHVNctU0811TWU`G zwl!zobXNnunQJ(kz1Q9z5oKY<(R4SnmNnCEZF-@nO@)8ax>{Rs>V;kF9K#c<@HWGw z-CWW3INw%jz@*Cci0$)ANAZPAFr1|9Sak4+?E}bS$cRMpnQ#Lm%)>*fSCd}LFyP1Ukb3hsQx=N|KC7!0mS!%W%qWl z?A{cXU9aQfAlP(DI`|H;_Lb(|fox;(EE)+X@3&W5t7?C7a@yl~B)q0XOJM@;g`{Ee zU#IQz(+`u2|M^tTgI5P@2TV$Zb$zhG2&85@erA8LCY+`6AsJ3X-_C8vnB&P>=*L_{ z{;h{XT0RWCBc@a#F5A(hxGays;&J_2#Kg^^b5B<2(%BOoD3$Y4sOwi^%b51;wKJ^n zu%bz=C;=v9`PAkxFi`+Br&OjZ8nJI9IIjX#TDWT%_X z%!R~a1;)cADmWsFkL>h^h!BOn#rT{qGQWQwgKd`Ey1hw1l=&(Dd^b2NO?A8W2qw>OA zA66p&ED~vT?)?FmSBQrL(Q6jXmHL8PY>8ZL>dsY>X~Xv*ZU-qcb-bVjzB<8K=^F>x zp{7Nxzzwfwr2T2K{r;N;YpR@1q~Cw!beu1ns7dZzR7|m;)s7S@XzRu!{_X)#Bp@1B z!{}CbWYabSuE{b+?jlWz7wOXrJpj`;KWHDd?62;n0n9Vft`wNtsuogF7r=izy}OE0OEs}1{OcvyvDzHd!`ik!DdJ=^@O@*IB|g$??_ zMIVtUMP=0O+Oo+!Li^0`AtJ>w?PG9OC_%-DP}3w}G}dIE=(o4MbM$nNd|@h!e!?GA z27R2reH3snpiTWw9WspYP|=}qumU%jU**MKu+!a!a4`^$mg{v&=uC<6(`yu@r3}_t zTd?2brx{l#sqMcv3aH=DHgtb`2HpM6wU_V^GlBY?ubx?@AYGfX7U)#~pLA5l0O;V!~2KHw^ zhTC#S=?O-LpQ;F#VWb&s{_@&oiRCWPR4^<{g0MFEEPGTrPTi1Qqw*bAN~ikOq1l3-AForH#j~%3UhRFWnpa!c-qw) zyK*GAG51$&NCrCw8jY95V&E!URnDwpW;q~lwF*=UD>zj*$4`t&RP_A~u`pZpMQNh{rT_GpXgBo22U^7(^=ZI z&MiV&rB5$^mw!D80Y3?$q!8vR1kxwG^7rR|zx;zhbP$wWlmi8{LtJXq^I17TR|K%(G9@pDY&+W3D3KV25ph(tvcd4wJ5aSM(RDhK29j?i6=^# z$`gpFM}plT-zFmh&Nm1j7<8#bGa_J5cVW@!P?5;W#cM5X!b#Z(mC4Pk0t68y4i!^W zGLfZ!dFolgRE*E&yZ9DaYa?svSIsO{g)s%pgaHf|uK9@x6WX=(XGPp7vpcTHoANh{ zjhsrGiRUw@Tc zo}GI5?(r;*dH5kM06dw+TjkL43h5SusEMI}3wq*ISSY5zaY5q=c>ClIa1s&^3VZ1C zUg&?}4;w%D+l)SBLamgb($T-0?qL_{wZuW$2Bh@@#z^yp6xjgo7h2%WM-ar2*$^%h z;w5+$@RA$t-Zkr+hmQ|NM<)m7CT8Uha=t?Weu}P z5NAk|l|i34gM^9(Hk8KYtAa9=d(A6;rS6KN} zHVJJu$LTbLJdjvP>*NY~P{~!v#EWaA2g%JsMMGq|Mila9HO5#IB|5}=H7wFs_+L6ANa&>)HLpGK;-Nflw_|)d@{Lz7*2>N zL^$pgHw6n@=Zaj|)HOg>q7;$SNwce~vwE0o%S&_Or_NI?$ZqE%G#fV99@`TI%-9o#DU|mfqBxchZ?##QEKIWm|T)WRL^6D94inU~yG4k9Z&q=^B?0vN zO1z=N$91FM?lXK#(s6xD$3^v!rq7&j3TwgHx8fHtc)MR%ZXz>h9a6Sr6@|Ygw>cs+ z$}UDb%80(E&>WHCBazcq3^YFr<=2$Tnnupn92m3@7YQJAN3vTXk zjeLV^&__`_69)FRW3-%eJVqRf#dN^}yiwQiy}9S@9&e|V{yQnSSvJ-3aG zwI&a-{@YQFoD&O3ssS~^mw8ML$6b0fB6DT8q;1k__x`z_Caa=n`1M zC(HNk`XI8lt_pq&YUd|kX%&4@R@ST%sk?Oh2pLBAjVf?|4xVNW`iudTs%~u{aefi+ zRNOWiHq-7njX_B1RONs&_fnrakIpDj)hEONGBppmZ6=?(p3urHo>5T$9JP_FR@Y&H z`0C{<8?9eZf$9z2oI%z8abIWu`D#mf&P82z;eV`CTDxyhTV^|0tll1Q2x0_T1Hx>f zJcn|eI@F|pY8!NF5@_hg$VQqnBiN}l(PHb+`bjFfCTVQas;*6&&0$v*xV8=IZYn3P z*)=DHL!&7l4dr4Tk8GXMOB#ugqBPs9daS&G+#v1%f(O#_ zgrw3YYTs0ZSgaATn*>XQG_8$lD;>3C9z^mj$KdOKR(M@s?OO0om;2@gZrRbuWN2BM z0;*ftd?P%n8ILOVc-q*HUAb;fZiByDBBF(-jD#k?vQEF9CkyxR?PuvtnfCeWmV#P`4oS@m<=GTeD?aO zSEE~YhX+p67+1RB^Vq4ccewBr9d^DRFMF(~YtZfBwCkO~d{mCIFq*(c)fI?FUxdZO z)GT5rE*}j=7qW$YLvSt(uxxDe#I|kQwr!hVY}>YN+x%iDC$>*cl9T(ts<*gr^%k?6 z)l^MY*L3%Eddte$i+)3qC4p_<`ptvYbN~yk9q?M^_)`oZ=2IxSnq=NXV;3Un2Vfe8 zw8?hqAoI?W>X6|d!hWRg0Mta2AFf2z#r)*kWFvu>#3Sa0{^xtPvWR#YSuWT zWlzP0AaZ3IKE4~M`V3ShKh$041kiYEEaju>Nvt#!qb7@Szn6$`iYv!gHJ|2+bvz%X z;+yU>vjv7vqoSwciWN=sgRIqPVg9bjaGTJpPj}}!H-Mp(&>5|f3s?>#Ymc+@55)2r zf8}{P<{OSvOo2rbmdAxBY-V&~hk!q>36T3t&WTNAuOk>%Fm`hV&-d7(!RR=wl}&~l#yM)GWV|~mov5J zC(af)H#yZX_2HfL2w2w`YgH#`Mtv));AVD_H239+kcFBE&_SKmb8s{0j;lr$BD+2%R&=1bRh&{Ft^X&{c zsh%Lu98GJ`24BCk@zxRYv8dGRERHyAGbPjrY z6m4C{svEXUynFB#AlEjk0j*0Kk6s~5{vo!bp-+zScZ*oZ1{i4As_q=PDZ^5%Y89Ct zCjD|D-Uj#eNV{Tt@jcq|?&r1&sD8O{`4IYf0c2$LrKVb`^MP`y(CEQNWGRJ3hP#^Pj+|bFWuXZ(}A_ZTaY<`P+$Q5TjS|JjEBxyA8c}YQ7Hi3 z0x}}K{+Y)be+5p2t_RtyTO!L(7@lIQPt%%Yf|hsloiTZq;ojzzD(3>bS3STOgoEu4 zGkk}!C}oV=WThTtwFb|d=Nxj@t9JyP;wNC1`gl9~e%;$A>ri&VMc5F%k-=&#`;_4k zizkNQBuHM+b%3-X$?U25mBk0&ol{O`fmYuTx*ZFKCDB=u_uQ{3qRPms=mn&q0qlkEe#ow~&$B(M_)yfNK4TeuIy7;Kqh|UiL z%*^XpS~hUq0-tl;0^AspOzh+U&fNHs^Lkzmk56C5`RATqT-TB1B-)>%f;kpe*)rPy zMw;hZnwj-y`8rx*LRvj(d(i;$9S!dl+8ghi7Bf6lBN?N`Wi+x}o#XJkyIA$H@p4%A z@QF!XbBNjJ#{>Q!Egy`rySo{^b(pZ@pQ0hA`uw5fsG=%KVc+Pt#{_tQ7A0yK&+J}W zQPOC=W@i&q)8i0gm5~r_mp_xwXT9Wisp4Js*px8ipW(%fc&29Lj7AXAVT@*3*m3pO zrV=iy(aJZ+IC!)elTF$L2UKUSg2cP_8EACTYh#2Xi7+jQozWebJCs`}5Hj+_lxS~b z>LbyI=syx);l@!KW0PS3=;nk!Y1&Hm#GGikQH!uj$M7Suo3i#~eXv9$nVTXvT4a^>SsFk9aHx-_G$MiTxmEWSyh-Q=?fmN z#iz$|Q%9%_TA@GbknWItCmj;rsT$j;@uh0ZENA7}YM7mQ@9+l!VV1bdYrPGXNDcT^=u5RjZL|Q+K zU|!~FW=J`hK~w)?4-6wgYv^7VDhuoa{>=z0AN5p7I5sT&3w(4rlwX36_rToK;%v5=%82q~cBOFMFkt6D0LMA+ z?O}KN_SmSfsuIZ)KOvnzwcP+jr`9eLy5I2kwhIqFj6Jb#)=MFj({tU0=#FziL;F;8u-XCPgG4$NmSlU(_^aEqoM3_gEA_$}k{v{+##h(=Z;16p5m zysFL#fwbJH0*>p0h^L|T(~1)k2*u)TD!eeLtie-i_xXeAkmyih*pbpB#!Xr>tgN!~ z(^sEp1;oF{Tgb6{3C>6q>^oI^M!}WR1x}#GeDh|z3js>>mG?C6)S4{%Y)Nd_rBM|QM+ zR}XdLD^SM}6~gckm8g%{R_xAJ8`kb=Z%2ry`PkMJ`DN7Odm5mM7z)XCBn20FQtM7_ z&q4V^+6#kx1g0Mmd>lxS^D-(z#y%q4;^-L7tMzKVW=<<^+Xf*3+G% z(#be(a8Y@`VDG_VN1JeaHjM(K&8f3=c+IZLJT>nkUCobdkOmK9moDywqAKMVi~>(9s)kVz}=G9p2qVa z3Kvgn1v1j9WBi6uuY1&&FkR1##v_6Fig^QEwYJN?;dy|xs6lo`%17Lc%A;DheT0`| zGJJOyBgX}fPh*7eHfrVvZX@`A%ZEpv> zBK)YV$T1=|H~Btq=dol|<%RTxtlT@1)EN5AbCL2e3J$O~WYZYmhwB=YF|J`?>(HSq zIEle;b6ZIX{Z42Q>%4e+Eb;$3%d};#&cJyZ*Oo*WG zbXEPbyX4%!ekSTX(gBnuJKZXpdSL?+EtkDDr%r^HA+zY8HX1&BAY83i{}kDM)zdmxZj1`t#ZwV%@S>vh28)v2xU6L zSSFUbOwbG4RSz|u3o>6M9>AX94^wEaBHM$;BtctWQhkK64?9Y@|v<}6Lia@2%#!__^1JP#`3iGDxd~YXL}rk;8&e!{A}t$t#5jeYekUXP4{N}m}>A= zC)Hq_7dVRssL}X*s9IVn_0#Q83|f{_bDojlO?aXSvby4x#vnQMS3&<>1>T;$>`;K* zYp)dlg7^ciui&{Ce+(kVtP%Sm0_#zSP(8%DoBt>)7~m!_D8*VHXJ z!&swHh>_utY!OykOxuk|BN8D|j8Smkt;#5*L?OnBD2RLHtrcHnqv66(Z1&z9SFIP(h^=iP6t78_>zd zo&V)^?%8suAw~0BVFV?sGt720E@wgY^bnXKpw(FFam893$MBVM1Blsh>-L3ljlEgcBP zVdI@KIXOadUjDnhc-iS>bK2D}`HB<2yjg1h!NxhcY-eU&234u0-Xs)=+#-@_Q%Ch9 zl=ekcB?lNB>;M+t;`o7^3DEqzJtzNH%Rw8sNJlyO^asl`@Eid4XAqhl>3E`$nAa#& zc`6Qz3p}#FLTYQ698O}zdOHvGCaAK<6v>a`(u$}lL9lN$?hKE9m2YffBjhU! z)7;PZ(8D1>iyRr{!D%q=SK?(>kCGYhO0B&w`X}uZu};$Vk>7W`Y%h{o$F-m#q6lcT zs%H7DO`7of`y22b%;WWXd=rVU&Zt8Z1#IN1%TqIDO?kc~n`hy3YG?}O2TC{kZh0zV zECe8AmvHbw315HZs9>PS)z&QimTH$?J`dIwgn#GlyvNHCBw4OZVfh~#sa0ZA$3f;` z7vF$p{lX(e8DJ4edYza;$oN_3;lLfQRBlmmjrjoyFCl0nxvx~$mb#6B(V3?YeQ=X` z$FQjJ_cWXj$}XRGaN@RnD_L$~?2{vhJ)sZol2ceF=cU zw7|A|%EFM4qCfIlhW|sw^4ZJdz`$}LIzal(0=ok>u)%yHH^pGmJDf_#@}G2Xs`XoL z1~3{W`n=CZ25*YOPw-hgV`Fc!$gk1MF~l?(q&KB@7;)a}kBrrkxxx2=PFlNtW}(U1 z1^uP|EE>BwQC&4Zi*W#oOd1QVObDQma{Mo`txCEqF>Fhkk59|Hb2D3q1mY8`-Cael zZgC7tDrtJBDjIePmQJEkh`)~xd+qR4H?u*i9+X8AfO&B2YC)IVa)Bf^Cw5zzGT`OU zV}WjJcXxl^&K0{Mjp}%;r!{Mx=F7}6f^;|#9H-lLac!Zt&@#P=5WZ5(=L0Z&)x27x zFa!MdGwR=J2-01<<%qL6M}905*y*5sAAPZ2DPyTscqN$8y{1UAb(<-|M|h1l0ssOM z+?wm?F@Gner(;!`QvE5tYQ-H*)?8D$Rxy%Zm%geGa!D$_YLJ*wb}KU9 z0z--?QzBR$@8f?U@;X-71*Je^9j$0nS$IT#eE%qRCHy>dZfhqYZHN0I`8ESh2O&^I zg-;>lQ#4?!y;~hd>vkz>H|2(T4 z*V}R3;YJHMXgu7ZN}Zfw zqt~lK9z@OK3o)3^$k0{H(^=m}5Y;WD7Gvg;=-7c=?C zjc`o1qX(fDX&Xm1i^k-g=$e91vPFn5#Bq>U&&EkzGq!eP zz#o)F&$>pUAPyfO+|Dp?=O7pJhKE>$&x^QeF9AED8iDf`pj-EGz-9je(pu z#-@r6TDZyp8DoASpp^rZ?O}&{jjEX`CIHg_xFNr~fW=S9l(RqTL7J#sWEyQAxwjNS za4>uAw}`=(MxtTO)qLlikQu>!1*MKVu zpw(iHkR3CRTeH>#1CGf`B7GDu{vEobwa4zRaa1$GO|ly`u1>0l-I+>IAA#VJb2qY@ z{E@i}o0*@YT+*s*2n&}`rl_{aN?@V0d8<#PEZ=Zt-1RX|XPz&Wu{2|C5)UDQ{Z~5O z24J}@= zYnVvbdBCzG=v4RWtlKAWB0!<2e2B>Dz&E4DP1iToj9Idx8h2*j%niI=RugN0NX52r z%e-|3B?S;Z7Cs9R3MG_FtxgXctMK#w$vk{ez}J^AZgb|#`$7L9nv4jInq7t{eus`7 z_Vx{)6tIIufc;4u_&u`h@nvlokh!rY6_#T2aaXt*q1}$^3l5LLb8MlzXOK|}9JXBU zKV@^hIJy~~FLMc_8dh9p3RqVCAzI>Jy!$NPzXZsURqkiPXe$^!B^<5CaH`8^xp7*H z@vUvcw`C=_$}jX|tp&h}W6W9fogY%;J~7N@U{sW`VlWm#=>PRL8WqqiF#BSa*J(@n zlqJogaq#eg%tSxU!^Szdl7dSN^asKQ=&`Bp&k(16Ukfmf?sg7X}eovj1JJzJ! z;se_8ENDCrPnBE^W_Rz*;(UzftU1SUngv5Mi00smi1n3h4l--3+OFu9*^0uzqC}qu zTAFrzjGhVaUfSR3fay4X#`Iu`5CWQl$cNZcif+)ks+D%j1TSa8t$^)fpZb#+ z-xs?Xx7Ygb{2LwR{G4JCjDvvV{VgQOX!j&Tv>}KFJO^=u_S=0{?#o#COcq55y-MpUm0VU!NaM29qBR7{gfQ5 z;20lc{Y7ajy?O=}qc<}OOn&H60s;Kt*L`on1x1lfM^qau=CJ2E#M3#dDjtXgo3=-j z88YCOT8Y)(oMa)V(;cI^q4se=3hp^D%G!@w7@K*P=D z6ENdZ-6HD+{Rf-UD!!9EUP4pjC;4}Xr0ay;Axf<6UDRus#8y!|r-FGeSQfz9lv#kc zr^Y_9`|hyM_RDq(Ej;O+R}}^_TC|K?odV9pFXRr*qp7i!-r-$YTNsF13?4GE+yWK| z4JVSV4rL0Bn`{CF*F|CjhTDT`3`_F}N2a4FEm1m#CXTLp$$g&>`@AB_S9Sv8hFMt& zx0>2C+gN-89a|z-D*CgsxQPrOxpj4Ot1G#-s$t zEt@tLvz@WSRud`Jzf!7uf!fBM`%kBjqTFjeA5`_G;uUgM3D=`GCLonMni8DE+!};N z$Lwhpf;0_f%{rO|7ZPDzq!`-IL7)%}J%Yuu-Lhc}C#^?$(QHptR|()NQ_LaMmP7mk zFE_A(EwSf;*3K%O?FhjgWhCqIy6E&h3^8{ZYTCuoacZe%!DGZ!Vme_kU^)t2+gSc% zM}XdGyEF!EsS(TFH>+?9hspAQ=oYE6FlO>*&0c#>GBs98 zN-S)tbo_nt;_}K?Utd*xzByIwmI&kddFujh*^xV}sO#kroFJ+R`AD4n<^%Y<-nMD) zgQxwp%hEmbi4V}daf6}SKbe45vQv9LesL^E-}?D_JZjzbC**Z6qT-GEYk$}@>DJhz zXq1GP8BFvBM&KuLt(;u}( zD8?Qp3lQt*KFUp(T$?u&Os%X3YPD=4wp87j^ZdPaejnw7abwMw@0`W#oJjAw{}O8; zPO0h&83mkCHYjIfYN3===m$r3kX^c`y>;^NKS|bX;#D2*+{(iH(x~~z)A;O<*RgXQ zq*DimQupwwEbQkA06s}gZui~!cfk~9RwKeFuLLO}|8#C)gtpTh!HNcMV(`|tV<=2x z<&wq?={V8p?IK2$Jg8 zjxIk2;d=AsnM(8I-}T1sMso)cBN=YZvnkoj1-A&vq??Z?9F;jDlh#DyPvtYXjX z#8>DWbUrIF%_jJ_@c7LvgBIA_7CKHdw+`ct8@kVlf!0AHEpoZyzWVLDs%nLXYK04R z2cPl|FRR{9ri0SP)Kbd+>B#jR#fHGM*=&dL6)O2}X_}1f(OQG+Z8eWugZN)SE(Iow|H9eH@E3eo>_)Do{e{-dJ01mZ9J!0Jb2t-s+gG2uXDpmy z-ixyx?{1(!w0CgO&-<^-sUq0d2SD?iyO|JQHjrqb3okD)`)NFPC2=)xKWQ9x6#VH_ z5u|A>LP=*f58Rw?Eh)`VMe*et8*p~cShocvAraBL3*uHa)UTdWg z^yU5V^g)9)*ng0N5ANYNIppqoF}2g!eThlXMneGpmn@QFa_F(g>;Z?4zy9{wT%Zy0 zvIC@tA95+01H=ci&^7u^NRV$|KwPVGa7W?cg`kgiAS9uaPkn}qM;!*%v#&x3V+h$W zsp^RGy2~Je(%HMwvUHYI1i(nJP#>(uarlDk%-ERZ7A^MoW+U}{((mHRhU|XMq4-1y z%u$Bh0*@5NTSL2k#At5mpl6fU(sCXa(TR=`%bSO5xQ;VPFVxbmsGDQSpW&};-TZ86 z>-Q?xu!h57?p_Tx@-0WQBaarUbN!-*(d50_WxUd6C&Zw;bZ7YDGXO)duyME&?T>rN zes6z2mlYGup)JUw1D$)jLNVRfCR*p8eTA4@Y>(ZQCU!)=@Q=gB~Wni}j$I1MKW}&va-wTjv`z9Otrb(TD$mzk@^8&m@5cMbFd%fZ~dBahdS& zm~*jNn6R*Muv)UQ^O&=nvU0GSa+tB1ad5K>^8f!jGFt(lk^m0&|IJHttms)UxPVMN zo@e_Q#r4mfHEl)k*t>08D(>VN&Rw%tW@o?e>AF&h>#L@0=5$n~OE)CBD3P3kb-R@A zcB7T=x|E;2`}ywYDjgL3bpHckW>EoAH|wBJmgyEU(6>sfb!=%M8n-sGP=erCk2z^M~|}DA^jZzSC`M zB-{Ss^}TAiSCh?ah@Kr+VX_#bVAyC~2H;^X=T2XM1T5g2GNWhi6!yJtjY3d%?nW{6KZedPHbe?<9#KAH~o z)4m{kQ9N?~#3L5}bk#;78|~UfC&u5|9^0+=3%9<#-$|GsXZqn~2h4S0 zr4))}x*1rb5=wEQ&Rap)0?vMhu`r~eFT?-KeZVVOA_PhP2 z_i4QteBLU16&#cOehc~t6Q)I;sytmm9tzP{=qmnS3*7%nEBVh1=VWU8Uv`O!tGl_58N(-Yo++%_2qEXi!j=M+rv@H#W8Kw;r03N z4~!Ylm=gG$ABD#6)-Wbd7-x*}H;jo3K5rh6itz^;c|1xc=sA&}i}RJkPm#Z+r|4T+ zA%24~TGi(n(#AbB$Ba{1)jeAK5a-j%v`$lCyioI$_lch{ z|4dYt8Q8~Q#6HD;y=hgf_K(+p1uNx&^Tlh;jU!fakGo-ByTehq#d&VbgqLQEmuKu5 zIGw2qr@^VTOZ(}5zoWq2dCtw@$mQadwRt<1h!ycCZ9!wiXh~b~mZUj3+EpUtDG^`~ zTrE-=?oA+oJG)3NlsZ5R7Vfl`nFK7O6a9r>{s0CUVH7=o%s@Y1oa8(|!;`cq2TpL} z1QAEKC0r7nc&hSsWNG@1P9!f$*vd#|yPdL~W~zsNlt`AByhi4o%!lB-Jq6chF;FE^ z(OXXr;gDQ9Q69ksAC+{01%>vqOncgT_MmX(lq90GVN$k4sc3g~YH<##9(TR_=I)0d z@6$Zlod`F7?3qr~-Tm=?wwPVG*IcDJKfix7UrH=OlRn(O)YJQSTU6m>%sgYMU!LCG zLox2Yxxb%X0$V)zQ7^%YEI#9fi{bK&5YDDe3KP4BNP4c@#lQxMHP-{-L{uP^Ra-*ZRrz_YUu2U(XAG9v zvWLz;OgXI=E;RFv0M5zOo;kBYCJ9IJY=zH(rc*<3w$0f-SyPx_9AP$`r|J$y?@Xpg zOBFnSY$JYQp&+K`#wHKTkAAkjZ&Ox@@)CUM+sIQGxybXU_G%z6#H=H{$zrV1W+jKC zv_!KR%r{R4{SNl{516>g89V>&#GJy1<)1Alk1AE;Gr>pO`}^;aHp<5f1Yln7iXDL)nSRsl(2=^ z+d_KTXxQ!+{YhltVPr^@$k`5(14c+@pL>XcqAnSu#k+}8v9+5bmP>Bh-N0f#RrUV1 z)Z(pimDvG8W^nqp!bz)U8*7qmDayt?IR0Mld?SC&;z|C3t|&zwRw}pB6*!6nd9ZGO z5Sq@tM2%-vzl(@UnL&@wzx7Yb&RXV?V)45tRJUq?m90? z)#I_taC>aKb!_PtL^#~%W6MY9niC|?GxNqhTL5@7W^|pwfgw>c!?~ZQxKb^Dn)`w) z=XlbqMd0d1cL^#7Q3*u=98|krLXzM&ifake_@>ds^f$#SvX$Onw}Z4mf9t~%T1LX# zN8}};Ro6O3B3mZ#sj|tsVYpL zlD8A%|L9~Bd8@*ME;KylBy;A2*x*Bzx8dwFso~5m)`~XI7>$lG@p47pMny?DN@ONs z31CEK9wkQ;>AFnJO21s(#HDIDe=`>zeTfvuycH~lRY_TUs-n(+D{EHkM%mR|4Do!O zyCuzOO3>^f>X4GY)$7=AZCJL%GTRSVLnN&qXZcJ@a_|Co&QHHQO@-7WQ>~mJ9rtU1j6qRKa@bUKke*N|LaI%El;% z#5I%#>_)->sdPDiNlElun1wmZYhIzO{1S31ytY7U7nusDeL~r@x`-^7U7%q4%k-H^j=VOjvASQLD*E4{qjBvS_+eN~sIQ zQ?y5cv|0z!L|9i+9Kvc1XaXj!c5dVA6iLV5vc6X%SgGlMVosT#$PPC+-|Wj&n{6(x zS@jvuIjWGV2Pd21 zTM`7)qZa2BQc|k~fz}Hh!fz!Aa7$9MMu&#{fa8lcyiFN?jQoIt^YGHavvjAf!?)xI zR4hoAsx<|FuWi<>Mt@Vx&~v7VnB;cE^!onjU4XCUr`)x|cp3+CO3FvKM9ilJDg9TY@SgbEi*Tkji1^8s9!|hyMj`_hr;Acyg zy^Z>RN$0dQQShgtK*!XK+(WE%+Rb6 z=6<))hJdLUYG;0VKcyCV9&X_3UQ8^X5<8QBlHptkZV)F*$_w@4MO^xz?*MMp?qj2W^b$yXlA118^bK5nw zxQ3&n#92Sk-X~-i<3#TBcxVvmMinoIsUg$mQwC9Gyq}R%*R`>ml*!9(p90+xh%B9d z61!L676hLz1BKsDZ$18%cennFu} zS~-!XKK0v(^$*kKz;~21{0OJqHk|r39T1E*TF#WaNum1M)D)5epTPd*>O8nN!-bS% zmkmDYi{7L>ls9gAu%_1pVMW@656oYGIS+Jpa4~ltzL$N0bc1ev$I64Ih^tYz(%^_> zuW3M3h{Z0^$J!>BUZGQtRkyW$sO;}2WP6tdT;ZWj{+dMnmd?tc8sa5dXID8s9#5L{ z5+Ry5|JUe`tkFs*C)5Q=y*49~KoM2{s$!HCu@|GtO9vf@ax5IUTUeoh8YNez4d$Ai^UeymNR|lTl z;JF>OpHMwx)#l2_#NkI*_9I>ixcfF=m+G#IlO9q`-Q5?Pt`5C6Fj`%NvA?=q9In9j zi}U9${ssN8@rT*KTM$m*M<;iG5j21n^NQVI9@dwVd(g%v>Mcw=4bcd`SzkLP=44uQ zMB^SSHEh!Zws5bP1OwJ{n?>Q)C9naGY$BN~Pryim^) z*zZs}6w|l?cYBw;h6y6hdF-?j{w0VRLlW4&~4I@^_pG9qdKx%Ef%BF=t zB3&RuB&LvHr{wx^EX7oPsSrSAsF&G>jm3e4As$wJF2+G3KZ&=QuGJBqFI~t}oAfAe6qH;`t)aE6$!pzkp~NZM7cJU1 z`9f1f`>KMhFEuvZHATE?8+hAv^&t{sn)*m-xnQ3p$r0?w*BiFV(d%~lg@$lzaU9tT z`PmpkPG;Y0wJH?$iNZcc!x_*Sr=P?4r!A)=j0ta-zJ%8siz8Wo>7_TH8LOK%lA$-W zbq1wH$+sq8tr=rAcO%@vH)s+T-V7a5wP#@?!GfBuLLW;;v0u;QK)h~dsQv0gP19gdw4 z=*q${x_&5|9sUe|?Lgz;^@bpf(fgEI4Nfsg?D@)DG*_B`w57gEDAKq~)BH1rH!g)Y zK6^DK=EiP^g@$egSt76yd3hKY;kw~lT4=PT9yp0i)A}77j~A`9fd?EUJ?33c&o0xtB`ad zvu?szsrp8Z(4K4VZY%Z%WTXz8{nDgBfa$k@%+yEr@+5=YJx8AUa;8@E`lKCpalGw= zJLS&T$N4`)A<(;RF@K?G)q480_iCsh`;UChwU*NXLyRq_v`Kt_VY&$gT&#(uuKOsw z+>TC8HrkysoP5;qPtLsh-G2c;Lvh%X9uGTzI6giKb98cLVQmU{+U;CRj~mAkzUNo) z6zn3{ad-7QUJ;Oh*g#IPqnqW!?lKe-M+t1liGO~+es|AwlUz!R=nxBBl07s1sCRu; z^zO$U-!c5-J3~9e-QxUzyQdErdym)G>Gi(8FU%7D3a9bcD6#wY>%rhp?{{zB?!Nec z%Ir8_oKjrd-P_~tj+?ZEU6?RJ?cN@~yJw6Z7~@ZjN$X$xJ7E+?6~jw1rfL8E+duC9 z@V4wC%*_>Maas$b{oKMV%5kOosX2edbW&hCVKCJJ3*beM`0VhRumtyG`7XZ=5`NKm zhh}MP^907qQ(@z{BK4lEUyJ+|x{uF)h+`hf_lCnh#(PQZ@8XA}zmRY93uc;x#Sihb zf3Fnzakm&}7`fel8tUHB!nsj;_hBccT&z+;*1^B;{=EAp(U9-JbWK{e>@oNx7L9v# zVo+p7SX9%(q``bFT?>?Aev{$4RSVZryK>zh!e1BQ1joJp5$*}%Ns!KX*b9DtUkMz} z+n!N6%Qf|9gt!t}yEe6 zalW>O{u+MG+Wc8--w7eSNibTMY^O#ZL6FRiuNPkW-P{WU|7}DO4#`){W}E!(b|}uY5SQ9y9q$BIM9Ij` z!$xUIQUM`V>b}|#o#ZCSg~)e>h=!`bOR8g427_$V*cHIFIAV=|6?%U7A>CrNV05u# zh;m)NrTrH7%3AHK`ZlA;)HVx3gr&gQwaj}jP*!|7`G0$RX9YxRC_OOQ#z%jrq{lEw zPs^JSm3mT2NRgD-91hA&F>@KJNfVxQ~X5yhOWPwzM~5m?5M84mKhkpJ0*3ny#- z_i2(n5?(+RU!AlJUy|-|As1y7x1AL2i5FfBRl_X72(Oa?6c>Lrk(%|?XJjQ$rjAW+ zR9unYbx@8&4vpguP8peeN|Vht8%1^1sZi%Oa=>M%qtBsuv=|fW6v**{ZKRpDs~5Ax z4N@Y&O>*O5AXSH7 zcMx6j+{wwbAj<+N6GODoi+Y#W@31R8#HSKtQf@Bga;uz zwkEt^KR+`!)a@xMPScOpbwYA46H+}1)d^uU@=KmGvsa_g+Q;8e(L&I=?LANE_;6j5 zV0B(i7?_J-#Ilha=%9n10j&?Kjvx@0)l_c{PE~dq8Zm!pXgdlTCyr&tXY47b7EQvH z*rZrRM2kqc@hl#7F?n~%+MIO)u=IeA*jnTAXmJ5X`NEI)RDk1Bc)uPz?*d4{Kr87~ zV|aq4IX3tW<#_u#cc3+Fi}RjrwOU&vqL|Ya(|efJiL%+KJ3L2FtPf+Qe8NNy{#|M% z+u=(LG#P(`_ zBo3yqH+#f*vTNCmWYD2{u}L&P+MomYmeH{>7;S$Sa<)n1I_RhrEpzZ02{y-sh(qTr z!1667#8>2YbCzC;)i)qrFE$&;OGtlHd>vd5&({AO&?2<29X5_^Dx+>)P(t3>REunb zhi4J=R&7p61Zm`J?Q2T%$HaQq*`&)TItppBaf?Q_sR3whdiu4Hs-_B}i({2Uoe6`8 znWP450De+z6_ZC79e;} zb_)0WWmW_-&@%Utxc_8wj$}{p&jt1;SQ(?O>R^A!sL#Rlac_AS2SJiq5Sao>u$t;| z7UU*NafWHraK>|^2dyGlE++{+R76P92<$fNdZO~dz@}JqQGd2Vz8+v2AFt^=V3UK^ zCI~$wlVlrhs#YPJ`)vr6%O59H0|5-&x-r8O<`7TnPo@$-&q`qt6!|!n_<2?eV=gHF z=b;j4h?0WzNG0eY`5RtA!3sO<@01w96mfI{Q>3dSc~gQXfUuu8d&~@usN%ef6h=rd zNX1k~ir5B+et(Q7J5{He+i8ysmz_~|J{Nn_jLNP2<|qtrxgky5^LFJpW||tejFn7P z%|gk}a}1O9vp}H*!h^_dv^iBdfRY)G%kSNC^88fXpxSWH$0Dc@ocz^2380Rf8T6H)|VO1Q2>9xz( z8$ygvxl=HM^bwOO8Ng^$Kwt$EE#P)h#E0UZb+V}mR#1G7Wpv2s$ue*gZ@UrFak|XB zU1wf(*x;^BRiVXTPC^H($hJUQHHT>hHzyTUc$tqdJh-i#2fAg#z1M^1K@VRxMOT-S zZ6dTS7n5Ka6@RN8G(bq#jVncCFDlxU!i$KbQ*NVck?akK=k6PksW{3A$ojZx2R@JNpbp@6Ph16C9?34u?+ zz|Sqn1NU1(35gUE-gEpcvy1ce|qhG$I6zOSj9bUvDRo3v^gI!^d4^ zDZ{MIL>F!7UqAiwkEaO{eGe@vd&%?M{5qtcS`f^-)6wV#eXI3ZwnmdyT}z$c%%V|! zXO52YJb&{P#QJtxC#IdFF(ev!(yb2F!ffB-n0He9d{(_zu0>$8gKXVXMFUq*V6Ert zwFk>0XbzihNa}mL>dH#jf(i*VGi&7wD&!l45!&W+HnI8(sO9UF^}HD?CO%_R{Ip^c zaxgT*i7q;dJrl!Z8NEl6+A{vQXS?gO^^|1yO~Ry4t(SxH_q zYbZ+}C$mbXuGlNDJ=})P8lxfgYTaJmvlT$Pual;Gem4}8_%U<-aoo8~M(Gi4F6oz6 z^BuhdOXfy8d)eo1isqa8@bK%*{(@Cpmz_N%b$J}Ci@@hB*q7Sj=Mb%5&i~*_h3&XGjE{kGx+bZwqFm3aerJy}cm-9MY=|CSJr0?Uz5s<9fA(8XW0#A@7hg2A zW4BRS(aqkP`kR(P70ztFMjS&=JBH{XqU01sx0Eq(CZGB@}B%}(SA z!e^^4e?J77+DR`J_sjM-Vp%V8dbn%0>VHD$R{?B|lq}a_CI|0Cu^fCY7^ozmdxLLL zBqR0}kYN)a#FdovT!D!7@o|K5olWPDr>bz+r$ppno;z#EFpkr`qE*y!W3QdHc;J=R zgKeUAp$+tDc({oBuuM(@y;GS+XH5%Is;lDuw9K!0=4 zFoQaRFYTg`B@uO(v@XI>?T?vWAPoOg@ZDd=(!=ZYK90!){Aa-=Gi4X+4c zhoY`XwMtE2&xF%#e4U;9lE01*0>qXI_lDlh3xRZ|`!xva)?;Pxl(~*4a8YjC}%PpLr8!KfxnQVU8om&$HwJVMPv01uQ zv$nuW1ID*FcOSCAK%uM6+fsBdFV-4YjOpY0@o#?fUo=^lsFRr;I{`Jb)gAu`A5@I! z8v%-XT~gNa4wqWJhHBpqPk{h6T{IC1gOju3v+^bt1Am{dk(XGDXt^gQ^7BuyH>W_4 z8g89pBYPR0a+lOS3+ME1oMFw=8K~~GJ)82>Hlr3)JKMPvjOM2{gfgnZN3}FT_jQFb zJN#v*qLvdeVuN|xD8W4;+gf!}k8RlZWt+Za9O}b6#~9e-@=MA6*9S*>(;&@h;pyz# z!GweE&JTJ{zRlD6_3wZ8f8%2(_mh<>Is!B{lhG<0f4laoL>RyhVgtE3(M|GUIGj-& zN3ty$j`Q#Hz1pj~n#*Vujbqv1h-`M(^49My#m$c!wwdui+Zetv%uSB|+uXj!wO4q3 z6<@FNzdvp!t5T;bBr%NxVkj5mX;NgL;67<~Kg?d^;Co2$VpoSWEe zg}G!EkhQ{hZYsX}58zIGik_e+i65fyZ9z|n!bc(H ze~4zSpl)AoUtV2}mb;WuzdF3Y9=yhbH?q2YkwzBvO?)!iNEYw;`D!QB_D@$krVT!f zMrCrslrAm#o8fvfikhUs%6Mk&{~x7we4`hsnT$%hLf=DQER|it$OTT-2MFR?HyE!IoA!{ znIeeTY^*j5UD3D23?=vFW#|5*GiArHrXy;j+$8jDE2nx$Tp=FOPM`rnu=8%(e>GZo zM8T4wxrwuI0ifWF@cs|;Ok8W+v}3qi2K2D~F5Ro@0nNmi(rQ!R`4Y<*QYgGln}4%5 zxq}T6)|uZUMaUjV7%&bvBz$GHd1#?ra=;yIdtF z%@prA-};1!J0o45I{?0jyvF2Gf1X@Ie@Ig^A`_-{z?D68j_&1}Ky&9|_m3MAK@&yLkprdSbN6g^W=WAf${w$=Kt_ z*z0fKjP0MDn3-+N86lrGe~btx$&F0|zrOip=qos)5uTYIa~i_r42MiaiV(1l+z6lP zHB)5uBxd2%P)w=}_9roUe=)iF<>hubv=)=IT#96meCAbe_v6efZq_;RS~A7Whyo5q zisO=pLW{M$lDw7GVdIkNYzaS)s~a09DjO-CDe&@Y41FXgY@iHVf94+MAhF^;moQT3 zB1(to=ne@eMH>}q8OLC3ulUARCCTl*2 z1CA;#E9^>E{dUdgf1rAA#qO6rGLp`#=LDl`r2*;DLe+gIRo)`J?8x`gun+#TN zGnZn-{bpF{V=!^rvTPAR+?7bb9+Is^)S7x31e%=|p%Ocbcu%=h=0zsTK>Ang>!N{- zcdqE!(=Vd4R0`=Z#&N}<+F`_fEb7Tha&ApGo(igCZ2ypSq{o7;YbX1Zbbf3}4jqEa zZc6MLsUJrhf2z9#tIKt#ARlxJ(inzt3SA|n_xAgn!|lzxpGHC>A;i*Dcm@Rsqj|Iz zB>mp0+?Oao&J~o2-vcb|q+`e3b?60Axfg^67&jw@NlYp?#@?PVL20zaHwe;{;p5(t zcz?P#j6_#`XcO<#rP(ju?*qToJRn^fTf7rq`gmVO;g>yD}nIZL&24X@M zoHTZnUQ1=hH0N~mrIgY*HI43Erq!aE&#jKvAl*JWZL-HbXJz~2=_EZ-mHN?E#_a`5 zyFD6jU}+FG!Hz8rUC*$KdeKr$?p3h{H1Ckjm^d?}+&#WnZ!WGn+>i|(cAS`r!|mIX z<;d2Uf34H8iDgpx>;lSl_J-EYY~1c)eDaU=)vmau)>Z zXDpE_6uFc3>?N+%I~d9mZXTJ!?$m$~lYknMf7Xz5ruBeV-B@K7(x2H>|b7J7Mf3to2=?T_7@GeD2s%a?wW^ zduDZFa@f_7n07`}z7_gXPNE4O^O!Chke1W5sw?%ZM{;w&?m|HoMKM3%onF;9R?=$) zMSA8~(-MDZ#{h>RbtVNK-wivECq=Dle`RmEzBD8uY$M)T_P!(>aXHXLp5B;5K*emz zdR@G)Av;JvSG7=Y7zuCYHW zRV2J+PKo1r#ddU3%~@4mJ-UWO6W!1uB=C~Yk5fXm3N=mls6|nr-Dz&+uClth3g@Z&{o7@k|94(QNMdNjTmF#Ay z5{Fklj*v-Xdw}6L(s|b|JK^GnI_J5vDTcHILY=Di0)*=sgjuw%qM9+jf7(fnV8z(k ztl}8SOOnuU%gAR@UhCZ5tW;Cia53`+oTPevtdyZvVhb?kc7KdvPp zG|)0dc6pqmd59I3;5?Hs#x($A^{|cW+7|JR=1mir_e_y|my=pHr1ndgn^S-Bo1}?*nH8&r9NnCdfjZ03tb`mm~cI=|=jL01KHHFq9hOQ}A z0z|3_As>+lPY&NZp3WrI=rO%-&JWXmB=0gNIf^%>>J}@Aan7vO{*%<#t=l^+MqiZ2 z*C6GtY<0&Fi97nme}v)y7wE>H3Zw;?;R0rMpVB_zPTsaaEm_sba|I_wqaj6K*lqoU zXf&nG?zU1*XT1%AsmtS42^F?0XBTT-^MGM)DdWMg7_%hn_kESV$_Hn1A59lFkcFC6 zm4qTAQ%A4buK&>brqKl1vfczSCRaoWq<|AH;zU2< z{)2FW7D~Zg0Wc{?LbRd|w^Y{+-x{8pK%!&N;fbVne<3H8OyY&S0I{Kl^cTlFh!4$$ zlRNwzJDL7B52AlRnmvd9o)+bY9paQdH%foep~)vMxMSG06=zY~&t~gmFZ-T# zrbG(qa*Jx41;1=V|6uv6fPl{ARQOM)^6$ObaL^9TqNdr zci16PQzOy<)ocY&8pu^}X?;|_7Lx8KdG>YpSX0&VHq}?L=u`s!x~R*GiznId6xHjI zt}JvDcQKxhqQLJi5`%t3ccxB{wKxZ7?zk57e}#=rQF|>|MW7N$)^$ITEv-QbM|tI| zs4i^v$CJFda*Ez+Mt=mNwcdXE*G1~K9O;Akt|X&%V9ob^0Fio>7R^+=Hj=!zp&^#9 ztzz96x*Qy$-CRAwRev~+r>=hd7%4q z0R&!pXp-u%Y3b0T-x#{{8WD~WM?KD%l`YepmvQYK0Zw!Tru7$3G7Y~R#jqV$wMh`8 z63+AO&GsVZ^dlY~g7xfUBEO_0VqeA0K2@6Kw70LXCcb1Gc_gn^^Go_Lh*52Q(jxM! zzj%^gDBXd$Lf36Aw?)y&&s>Jh7bDJRDpT{LYd7!5A)y1496BX`#$C_~4VLH(7oz@= zmBS*1eO4y+WD?I6nd`b0mfgJ6vCf6d2M50YVcfQd^D!7d&c@~%q|ppyrd5fUo8FtxG5`VaKDjOf~?>Bg)*8xzY)B_Kv)x6c1uQRrOOjmD?X zba;fj7(<7KN=f@Vb4^@=<7nI?nw zJr6e9eyIbvTq0rS!mJZBlGFh|cDJJhYElNs%dh6nKvzngNx%5hIb|y5v<9aBKJl!S zP=nAh?W^=;lHoaU)w^RCXQyCD%O+20IP{`zo_2W3Yi?TXxf`PSkv4xHS3|XB-xaoV z$9Ll?xdN9G$HBl zsX#Qu_MD+_Y2t=G{IKvvo-kF)F}D_klt zn&XaF;Gz!&vV+F9m$aKmIe@ll12HWr0bvwJRGW~Y$fcxgXxD$?g-bMq(y?Eqod^y* zz7)dbFkP#9#aEnAtw$Q4OiI1kc**p9g!Se_-aqLhLcf}qG-aVo&&Q5kNjKx(DU5G5 zbzb9y+XDKu4a5BCP9P`9^07hHMkR0d`)l>exw@V$7EPI~1VFQQB=?^?CZIQhv9`N1 z^T(w8aoscPYh{1rL+|VcU2-k;ihxd%fE=YMK|R)HyhevWpmc3t7xp?CI{`M8*DK4eBSuf8tzU5kGs<&5ywYd7(tLM~hAnYXBuPaMmne?|SK3?eE zp1p3)QjUL@zDBayo)?t{u%>jlZt>djdf4r;-e~zEHvWQ~sdyRGhb>3&uIGY4=v@R?+?kzQ^ecPHd zZ@Q}i-^?|f&E9KokBG7`<7m2@S<9Mfw>G`d)24sIXkD!>IQ7D=b&lbQRd}0W(r&J3 zdz^2pG+_vtT+tvu>|4X-zi7y4%T~z-an*VR0xd7t(!LoZhSaxp; z%dXdPaS&`eB^`W+So=zI??AS(covO>llR-JtyQ(ZI63WcJQ7}0qNOka_d?RJ_^;D; z`RRwr#s7RN=fSIkwF4%l!n!`#U<6V#9Y23FSQE}t`H&2!p>OB5W6benE%ak9BLCJy zAuS&U-Vsx(5SQ&}Qe2kDVDY$qEn?#4(77iobm{Dg4wTAyDb)2Vv1Lqq_SzX%cv#V- zRzy(~vV3ZD7?>yknp3J*v?x4GMdv3z-k+yjRh9S#jjM3?;T{h{{T>%K@rI+M0_=b8 zM2syn@Xf4Hp==T|x|y7IXapuLB-iqmnOy7czAYTvZ--x%c-saZiEem=tFh+rBX>XJ{LHm+NG^1)9`4_%sS%4Bc2HwZ`*i|0l%3;5?Z%(QFS653XXZj; zu>#}a5)~W~#YcAfLqv$e-eP=C7ny%wkHNM}i$Hb?299LZl8>Dv)yU=}-6n{H#(g`r z5mK`Nm$b#|QOBs>TU)w2!}q+qaISpX+-gHuZE9?J{9~kPWQMuWxQsQ_r?`>F9 zR^W!$Gt&Mv*?#}cf;Cl6C(?g!ayrhJP1GcJE-I#2&}v5t6|{BZ5r6jpC=w8jt6_Aj zJF;n;0oP<1BX^Oe#EbOlg&u(En;*1~TJ~4>(g5Ze=~iiY1&k#LHx*rFuuAiA;m9b> zdn6br?%Jtb<<5?isq^qW-KCdQ%GHMbF+8loFyFVPKSj=4q@HblR(XGpjKT)};G&O6 zl%g`~c5T^Y9-)2a_YjfdnD#L^E0myOM5t*JFdAzzPxRZ{-Z^@@N4_wXML*#WDuX`G z-#!Ys7tp4Drw$oLc&O-5I9P!j%&+odFWBjBL%0|SN6Ym(C3L35_~|tY(ozQNtS#8@ z@zac}lhpR#8wJ$wXB&UIJ%jFk=h{nnh#A2Hv4)%J*KiD+i`*%fZcT2>9W0HHA8`N0)+oXReDXr-J4t(DeS>gD8 zl$oM_p$IF!Q6xJgmph~6>S5qHR!^$QgE}b0T{SX~x3BGJ=%8Afzb2{C2m|}GAH!|A zqx1wL!%tO&%P`UmHh+2Tvcz&1Xet<%B|%u5e3m_`9H(x`u2Ff#h9($R9Prup#8|H5 zy95?1d@PAwv?388v5zSAI;gNgJ_>Vma%Ev{3V7PpTRW2^ zw++tyD<<3xrcHw2%i7w&Rko^}S+$wvVE0C&K&3#A?0;{7Z?f6bJ$JXO(_L*%&vcUn z@%R7&=+kc}c@p$ro-CZK^o!?zr%%7qZ$H!D_v!EF^7+@7)1Uti;zW-cFnD^op3c&y zac&XHe=2=?x%}%%2>3||C513oA&@@dmA^m#`{f@rg>Pq2a#0Qx&uG8`8z_U zGS98?`5YzX_%$qR@I9x%cq;IN`7- znJ8tdOdygT8Fqtxn}P^5-(Y+Y(4{iXh(JBvg+-%7MJg*7ueG#^CS~KS%x)ePW)M;1 ze^4<;B{Nx?r(P7y#rSN#i*Hf1Hj0*h)htp~1XHG&2!N-BYkp!Pgmx_bSrHe?;*KNo zqWsNbqo>kl=J`BRt7l>LY*@saR?tQ*R$FDOnS81o=(mw4fOY~@3y7*HKR!Ip^hqtw zVqWCoKLnvxN>FL*-%Z!Bi}YIJU~B`?dI4jk`9h9tOzsyN;LS%c z#1L!BBe-h5E zkR?E}O_nd85>zO)hXCP;Y{3C8A^C`MC)jOyc7_K5fD|NH(lWu4crH!?-!hoR-vTm( z5zHwjKF*GbvlZke}!Zy_nM7pjAD#;w9Z1IYg2v=3YNc-CuruW4m;9Z z=gPaXNog}2r`-_oKx!qelPlsuC08YrEUt|mBsU9{43YU7Nyv-U7-P+p=n(6c(#UeZ zZg*=NCR>e?E1?K77nfy{Cr~yIP~8B6dj}q?!5`RskQ2L6)3~>Rkh5P6fBI+mAJ5|YbIH#QY20%%dU#Tq=ujuqK)v=)#Br*a6KKB<3}E4O?;YT z@qB+@2kI);2j!lc6j(+0OBh&oT($CuaWGAAY4xhbI)y2~xD(n?`|vGy1S)= z6gVlBcQJtHuSOCTcFC- zW8*E32Y2npcq1vwF(;>5c)HGvxg45ZT`ZpFwjI~j8Mrsx-t^bzfBd1wyTyoa)^Nck z1@!wyykWz~ZKGfAGki<3aed3iMfH%TKRMr2)-ndqaaO5PmGMe|EAueh%3=eSG+t;|TlyU}ugNVedbC%tn7JXOO#m$NQW`HvEP8 z>F_ya4)P7H!5&4eOc>a=juAQMbc{3<#B>1x-syz$nJoL}t!`pBoeYeLe|V{uQcsI9 zdTt9FYfBzt{kNkQIcFA-Rs(8;FY}lhj=S^-B6DN6WNp%FfA{`pJx!>hDItB#iSCDl zpQq95ka6Gle1Oy2+RX1}euq8sSHP7m-At}P>yJ!^{q1E{pQrH8k-08hr4dxKQ>P!k zpZxfrRLQreRIf0uFXdA$nu_UOKm&29%31{54F$VU=}uOt02%A5IW?ARs|aQ9ai)O= zLyBRgs*DVMe*{nwELS>fp>||O*qYS6Mk__dUuALJnhd2~ROwujXAhus&3B-!e>6H8 z>bc>G>|r_OAAbqYiRz)7di|`1B()3#at+C`sD#|&ec$y!AwOCL>VD351ZdU|DVpT^ zb(inClI8nyeNb6jM+LtHwepj%w2Hnc8*4U+R9(8ge}z0p_l+`e4xVNO`b+_otZpqJ z@%bX&skChrY^K$5YJ-r{sVV_w?&UsJ9-UF5noo!UWGWtV*-SolHK7$Oo>5T!9JP_F zQrBUH`0D8@8__SAK=p=hE})8kT-VuuzG5l&xv0zT{2%L-*6thB7HkK^>hXX>5F-c; zh_Z#we>rrdLek4!fklwJlJ0 zT{&^gE}RqzjZi)U_D)gLh0;bB9q|rVj3) zq_Ss>q|_#A-(-YXtr4;71WSxGEk?DOj#@Dfe=7NwWAJq;ypFH97QEBtzJ7sQI2wfv ztw>Wqb?Y?WD35B|qY55R3;VGu*Y(M5$ahOcwD5Eyq0X;t({FdH$T)?*KC95laml=4 zwSZMC+5sOnYnP)i#rxI~r2ajd7@N8svisB3RrGc?7H=A{W0S_aj6|gX{!+!2wg9!{ zf77)d4o;mr!?b9dN2Xy_b@3^2fp6Q`F;73-uM8BVa)W3#cJq3_G7M8tS7}g&Y@D^> zvp+*H$E+JtlRZ*|)Uli)#h5KC6+1ZAjkAWUZDFn2GqZSSk&i_7fwF1AB7Qix;>WZn zkX)t=En;`{1pYE@7SjtSGhnAzj9}hZf9}qBIaW5y0q{;WLtMP`6d@#^;_p|pVG1js zy?*M==oap9!)b+lMNk|JuqG1R-Q8V=!QEYhySoJU32wpN-Ccr9U~qQ{3;q z)gIoS_RzP!byZh?{fWcAB0(>O(BmUI!Bq)(c5UBU?ZFeR+d0p>4=1Z*a)IfGpqqvl zQpo~=`7P%oZZl41qWD6D6J34VhDau$Y9o@k5xDuT?{56#H}PY4`t!&9&U}|zsl6UI z$T*EnqO>n_H?(e=f0Il2fo8Le{7;-e?nR*iHB%=Kb?zR>R;NNL%r4CQzPhSwNk7~6 zbtjO1vX)OShF;BY9i0t(iuHg1JFRC2J&NhUOKG4Y#oE-xGgVaNWCNLSt!zk{2DQ>w zCY4#E5zWu=jLVFH09E2FbjTv1@T>4o5H3=uPTLGtFY$?LTs^h(+I-bGRZhSPB`Q`L zjYCsl>xB1qV2ac<3Fp3U8fi$@S3($-8>qBe6YHd0qgf)?be%;=;FTmN(OTt_-!449qpn$R)l!d0@jl7p%DHUaOEjoR@k*P)XJ=}T(F%OLysFZ`L zA)j84k+DL2kQ5d0UgRo&U!(06BalQEc*;&bX&K#6W?@kMqw%CoP@}7m#`euXSJA+| zopRrhP9Mi|7aHkrUR1hn%;_oV+GAxyNdet?bbJc}U_>GsK)*D_mB!6^HI<^b?H_ht zCqXcJ1hz`@-TxaVgs$c-H_XoqI`#j<)HVkGK-E&EGhe2qEr@Ckef`My1ch_zAXTcY zK|`)ERTShCLxk~#Ct_fnyaWb#G-$&|h8!RNbCv(A8|Hzb%wT)aF+=}yj>6#Z>6~F| z^P|{TA9J?-I(PXwZ1(1WIsMn;8xh&r*jjzc)u6`{yKzvEIY)$WMWqLdO@u1zZ%%Oq zeM3dMj?2qNV{*H{uhiw4b+ZK9u@1vqhe zEl?$$atcuAa+43`$Gn8@6I6U5^6$HG>NUx0?YHuO;D0X!y@~}2^m?Rneiee|YjygX z3j1Y+GiM zhKXZFUVp@jRHW_IR3SyrGeX-hh3AMj+K_1#H%vL8zdYq{Gz&ZwyWYyt>s*Svdw;n*W=hF(5u*UidY zroGxsW1tq~xs~ao(ue8f5FA7sYlOLVxDN4MzrY=fkSSwf& z7pz>&NJ%)V&_oewU;awU7K^LWp}-kS?=tDeOm{Skc}0yrXO(p8$Nlqa^-e9Nep*zADE=(wJxRU`1|{fO`Q{m6iHXf*@$$BjQBxBX)4V*g)A<`XZ5&v{h2 zx!%1IZ-%37wu0WrQ1f(4GpEtiW_v4QXsf?q4>m~oXM<0r{@OsxRGYl*PzJb|)*jjC zX#`~pLf|*TP!{`FRyM}?IC940QLjI6BXCXT>T1e(4H>Sa&z$O8!#33(V+g18^hb=> zV>||ElLfP~d)AnU4xgb;uai?iz)vEY(_W&N@h;Wx-Z=Fsw$!KtDdkH^P^jD5f1*x{ zH4e#en6+=yD5JWKzUHCQte9Q?V^Jv5-&l1jpJKZ5Fc5um%ZJB?nJxIR#KYHRe-HnT zb<*PhB7vP6JrNGemG4IHI}E}CQ4w0u-6HLR6nwD?vCpF)hv6+~ZUuH#0wta?4WsMB zIdOFD3T zVoq!F-xiD*%65TvjDVdECN%p?~9JKxRmiMdOjol?!_abPUoTHMSS~ zdqfX{SqfKY;yZ!DVC;s$k=8Twvj`}V*E|y6;qfnJh`p;*Kh!l7;d-Zqf+jWIJD2mi z7L)K*eZ1?&0&%l_u8)bZIzVe++0(jP)w1HDxEQ)Y2}auD>l+Sn!klX^kpZWMIyue`UlwHthkURZ<6%w zBZJTdJYtulogdxAH&b^tm^aXK!}D-l=l8mU|0(w7OD!L7@pMxH%%K`i)!tJbNSV8# z#WeY`O^d2#kIubYfAE+69zr~YQMe*#_!E* ziB_zh>Hb-Q-}91 z^^R6iwTtMwxgt=8>#ZKd%HlXcWG8~KJU6DxdJG=x(RuZ=6eGQ)PH*W^z`f3nXls-! z@TPtXG&fOKmEHb`rzDc&sFveRT(E)le`~+5PKMZ|xfee5j8MSM+fCk?at&TY{&^T@ zM82f-YN_azEt+4R64S+OEK*NNGrVDs@10sX|lBC1FExmMRDo6Z_ z%EDH6M2?%PH9&BY5F>HyzG^&*2sWe`RZI( zs|1j>l|6)GC3BZ*58S?vejPByOuPrPxJJeCepJ8YE*3dE|>qh(7xve*=J}bCdqNbzQRvasT zwp6O>jF$tpzL|gVggeGo>C$3is@%`?&QzEug<1Qw7%YO@*rB=Abx_%d#S{E$2K@BXaK&Y2JF6~%IK*(M#$PVALYrCkNU7CjOta3^~O`i%+lA&9r zUCZ@ck?S{x{6#A0$Heg^1_wm$5NF3RKB&^TGqOxU3#6jN!bGsT3?6S_iQlqr5>Ut~ zfnN5hlmyuTqf$1;+Gw}jtV{hRd?|LSUHREY9Rp>VbLT&WgHoYLl}3Ml$$bwQO8pEV zui(1AHL^l`!}cTt@@mSv#!1P`!$t6@bPB0sJH^kYd9pT31>*}AX$KV-E4hlU@<7eZ zXPcv!O)jlts|MlyR8JH}n|tV=0Wh>}u_6!%uO1)rwpr0sD&e zDd#I#fj>LC^x`ysE$ksh>wYF_h6Cl*L8DgD ze{JLT!Rt5GZp6sN?FMW3J5*~hk8XA+f)oZzhB80dw>#2Kc zlnpfv^C5(@v{Ff|v^-@@(%{&Y(8qJJU8y7qv#reZWMVW8T-P7p*O5a6HW=9G35OUP zvBVC->BTO42E{#Wc4l8M9}Ra=K$sGo2h|qHPuU@dLA>N8JEl86IT(;J12wy<`@i6E z{ESynw_^ctr7kCtAt%f!GRZ7D&Z%J`M05SuaLP=L=%cyTz#e1m<^epD=k0>c-Kmc) zyD{t2&gdWIJJbBl!ERmh;v^0vuvvohL|wwFvo(WDl3LDp$$>t7?mC_((DtQZ@_yAl zM(H`cAI5^<6akO-&w}Tp??2d>!QqLrE8n{tx?ai? zYVvJ5rI-cWMhJPJw&70tc<6JlAx8Gyqy8`|L_BkTP_YHslQkzBr#BHj0sX$A zG;zV0cq1w6({|>z8)RZztg=$t;Cdb6jXxpkUSHB1k%*r}lmFDH?}}EWGnqcIkt9EI z4nPHthQ-G>qYa{7cQZ)4jHBB0acPDfg4$+PWET|D9B>N{Aac8QLe7lOOk}-P`5NZ6 zS=N|-yoU|FlKSJ@x~-HP>50bUJmW03WBrQdD>eFjz~4AjAY(-l5}}QMPuDMQ2EMd! zW}cS5_VP@*Iy-l0cy0*}yk4k)arju_<>s0#SXSX0nAo;=1}k8ie0HGDe;3FhoIn?x zyF?J9Rnz1#?fRdosH&Q0Fm|lZ09Uig-=K=e$ywxQQnFaS(nQa^N~sev8iM+t(-jK= zHt*!WSzC13K?L2iEJd_F-(X`ZMW5)d4J8VOb@HKoaa^D;7(0b z*kfSN1dF6}V^ZJdtPG;50Lsql;4FpWMO`XYL@lKHF%HS%I_iuSvjfe%>e$zw6u--=?)XQ$V0B=j9z$K}*0n{)lK6mQv2LO?q{)mWHh}ObmS^4D;ap z{KMnZ+EHhkBFmx#d{-9{lHy-@khGVqGyhhFu!YDOMzzDYFg#9luH~&}dA-;xH@3Ki zpx>U&YT9r(sO480N_AH=lQld#eC}zn?-x(aphY;3Aoq89WA zQ5gWGl+mnwri_@Zkg3oJpaVD?Rmj-f)_NVq&$UhQ^D(Cq5oFkES{W%C5vOh2L|TK4 zb`u|n*0!t_i#>$_!crXR_7AP7rNwI+wkx5n4j7n{!V#55b$`^DY}(SU?2szbRF83bBJyvD_59jJ^&zWy zJWqtND5*?CAxiLYlw?tnli~sBc4;62bw#AFy_C)&m@#!kZD1qfNhTwJkiyG09P)Mv%khH2ew`v5}h9uw46Z9wv3&I;ukhU zAVr3CAN(E{HBUIrlxBBA2E>Fyg!MrAeiH>IG+|Lb0$g)KM_FtyStS&uii1*%Qp?DS zp;DpT+ALSWU(0?lB%3k)K_){YWFK!IMrpA7GPI?~v-SlzQ&N32i1}w4t`dJ)22sU^ zhN-HfB8Rt0Sbx|-Rhn9WR6ohdvWjLS9K4M@SDx{ZS<%7SX)&xIRRK%LDlYl(MVp&D zX^}7oD%5@> zES2HqeBXoAxZV zM%&{NQbR|=vx_(FsLizi-|or^{0;=7oi%*O{Xne2o=`XMUDtJ zPcjfsZE)t=N+KfL(q9N@kI(`rn}x=zjvJk6?d7wPpomZ?^OP~iSR~h!$RGEBs1{mS zGVpx)`K7k?qF;;$@b5n#1kb%@K<#NJwPko)2^tp8YA#yFyz0+c>=MztWxYd#uTSMA zj47Ee{APC*+N&&<8UQ>5Oby+AtZW&A5So;|Jx=H5r%hZ6wYhCRek=!h>#|E=VHR zQ3zLQI;Jl8^-TZ6Oa=;W*I)6i|Eox;wrNZOG#zf~MdV|8#N1KPjDP3SS~mkLGKD|k zQYpR8uq3S7LPvAhesPS}?j(tu!egJ*?wdRTnA8(xL&q+JrzR?`De*^jWyDpcdq^ba zIQOP5&rig=?{Ksr=z`=ghk2a}9((jN3cBpVrPXAkwt^X1avp{Wja^qtM9A9sytwmx zxT3i9qF|@5>+4@yBd8Ph_5l=)cJ1nSzQ(ig$dV*cCnyC)jmmR_6Ex;mEIX}j4V|c3 z#0pxP^lZ*qI(s3vB2iKZXdszt21rXy6fRzNqbDt1{DB^5j%jk$dS{{lZpEHZhDnsz z58|<|cFIY!rNc}J-n$4Pr!kL`7mNxk-7XHFrEgr-f(`PwXUXW|LBs3$wS^hJnD zmSGf0aOqbgkXiIj1a=`TPL@GMIn^_|`8UhS_b_rItf9&ASBUCvU-A{~psVAp8LboF;h!1gXR)2-$m24q zu82-yqcCHaTBy(meRtXUrNQRML);PP=wDkx`X07GCk51GUDt-&0CnKk7|E&tGN$lI zX)12gZtMuR?=n6yg8X^v843Q3bd+b&oX`^cc81Ucun}g;4zSIz@#9>ua9c@jP-qau zrtlPBIF$?(&E$;rP&)}`vb8>dx$*ClD9^7+wVlG+dIg?Nl~!(ucFtTC*n@xtJZXe+ zUUV$bYCW>a5d2(OR*_dNA%Dc`;Jha`{*NoE+9pCyN;Y^BMJ*4qd$Hyg5wXS6$BYH3 zOs}i7M9}8|cL`T`TS5U2=7ACB7v?LYb-aJQ@wOXQ885zDuZ(>QLQ+N3Gu*B{Lak*^ ztI>t+fB|ANIn7eL%*Lc>^k@>0G}ft7ERFl6>wB4R zempP2XfaPCcKumFpf;hrU}o~;MjNmNZ;p|$ATf0iIa|NjsIso^{$5IpeWVsC@n=z1 z;he1QxF;9|aZ&+8^p|eyyOb{p_H2N|osP&I8yxo-1xJ`%Oyb_@5`gC=#Uan=_$L%BJ4CC(9Pmzac=IP1eeuupCIvg_-{ zHHSL-Pn2B}StNTl&UE8WY{ubJgvl#Fpp*6YZ7F_K={M-HsmWAaN(0stIN7k@EC9%& zsF2dSPr-xmPc2cf<3#@S49;xTS*hyy$4rA;V(wN=O69K(Zn_q5w3U9%v zO!A>2sVoN>A>X*(Gr5NWss^x+C8BHS)M_30slzw0N32G*T;%kN>1}uzT z=%0I;nj@>#dqV2v&hN8M$s|tAkd2mr_rTBnZ`3mxq;Yx}P7$4%mb*18&GO0aBgvM7 znHs|6OMm2_i^-G#;YfzkBJFs_7Mr7kLR!{<7B8`skjnafb`bGRzHNJ;p-9dQ!J8>O zAdl#X1Kyf!QfG)_;N#y&ZEE3@+O~W9AmTq*n6F18RP!n5@TgS_#V%9n6#9bSc}Acy zJbhhvPj6ZZ^~N*cLt&ccO9S2fw5N~4CO8L52nzT2?x|O8Xw`y&{RPLLOg2k_f=qWK zJ4^5PCyvQ;aUi8%zyj5~y`8nNZy)x&{GYXY;oH_Ce<=#Cp1iBku(ngjoa-U z9k;jZ{Q~l6=&uUS6=T4O2pollL*7@5C$6`mFjU{8QL^XgAovJ|+$hLDd4Q*3|+9>_CRq4B}e+9QT+TibAq$^3b$O z(E5yZ#WkjOGaiOsfOvVJmUu+C+1S$Y3k>MSPr7?3udcCaYT?W7)rym+KCL6G_VMbk zY7nuTKN|lS%YhMglIYvz!OJaqh5zr9;%Vn%OEDOi)0{Ntaf)BW7dpE;lk)@a&0f>f z2AIVCT1Kol`YxgRD#?anIvy7;kmfgot8~_kOgY=PN>|CM-@o{JRv5`w9jPIUwKNce z`!koBS`c&Tt-=S}Acyv8`;A}5(D4g#Ac#3cjp3wkb2Gr>Zwht;{LNN7w(e$HCv^hb zw>VQsS*M@I7@LaXQd{}r(Q4-eu}ckPLZSh=gP_q=B;b|4Gos?>S;x_l9?m}I)@3-I z5<@zE;a%DMsVpCXrd&>a79RNh&cBz?IDX*xxOC&C6EQ}NE{$uD<9@_o?c5Br=E|yriq|eWe_;dXl=kMHxnY=T0 z+^X9Z{vIj$z{ll-wR;J!1VlEvHCUMS+Do`j0|A^a)Mx`*Nn0>hLqytQ96H9cak+BW%0#Ua*+RXB&(2R9x3?7KtQ`280bNC_ zYQ4*ss=ch<*|FSP>|lay%}DQ(cB5Dmm;IS;w6x@pNL`fo|LW4nUV>BQQ1Rcql9#}P z19=}ncMD$E$fuz<(5vO`uQt;ZubSM-t9@qwS`?;^*JMwok>7f64_t2={i7v2=H_%x4pf7aG+0VPF)C7v^vs%1LIRNYySbOE zzOE;jU(deeNzyTSbSMYR`b=w+D=h{MsU(N+T{12yAf_(u)d^?!N~feA>(VWnr-`;L zj@&~kzi@F<^)O+>%L3{|tXp!9vVfA5y`gobQ>HPuQrX9A z!$eD#9r523j(zDVI1VlWE!A`xclZSztRT4GJMcp1VH>)G4IgGt@;A#2VwGw9`6yFe zQznzqpzXb;hw`w0&3{&QMDtaQYSE@1`36ZE`{t-z(4!7l5jZ?zLd49wL{6;9za8u9 zUD5STGhFdS9S#mt7>WxE<7ABagoRlSGJJ-r&AsI??`_ce2;8=;_0gZrU47wNJ~p9v z^m&rpgCzAt`kG#;V(%f&PGP9yVvsJPv`TKmza})6vPO$s+#`U-diq diff --git a/spec/tinyspec.tex b/spec/tinyspec.tex index 4444d84563b42..66358aaa83ac0 100644 --- a/spec/tinyspec.tex +++ b/spec/tinyspec.tex @@ -50,10 +50,10 @@ \subsection*{Source Ops \normalfont\small--- leaf nodes} \toprule \textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ \midrule -\op{Param} & $(\mathbf{s})$ & slot, dtype, device?, addrspace? & - Placeholder with shape $\mathbf{s}$. Substituted in \op{Function}. \\[4pt] -\op{Buffer} & $(\mathbf{s})$ & slot, dtype, device, addrspace & - Concrete buffer slot with shape $\mathbf{s}$. If device is a tuple, it creates the fully sized buffer across multiple devices. \\ +\op{Param} & () & slot, dtype, size?, device?, addrspace? & + Placeholder with flat storage of $\mathrm{size}$ elements. Substituted in \op{Call}. \\[4pt] +\op{Buffer} & () & slot, dtype, size, device, addrspace & + Concrete buffer slot with flat storage of $\mathrm{size}$ elements. \\ \op{Const} & () & value, dtype & A scalar constant with shape $(\ )$. \\ & & & Form vector consts with \op{Stack} \\ @@ -256,9 +256,9 @@ \subsection*{Derived Properties} \toprule \textbf{Op} & \textbf{dtype} & \textbf{shape} & \textbf{device} & \textbf{min\_max} \\ \midrule -\op{Buffer} & from arg & from $\mathrm{src}[0]$ & from arg & dtype range \\ +\op{Buffer} & from arg & from arg ($\mathrm{size}$) & from arg & dtype range \\ \op{Const} & from arg & $()$ & \textsc{null} & $[v, v]$ \\ -\op{Param} & from arg & from $\mathrm{src}[0]$ & from arg & from src or dtype range \\[3pt] +\op{Param} & from arg & from arg ($\mathrm{size}$) & from arg & from src or dtype range \\[3pt] Movement ops & $\mathrm{src}[0].\mathrm{dtype}$ & (see op) & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Unshard} & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0]$, each $a_k \times n_k$ & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Reduce} & $\mathrm{src}[0].\mathrm{dtype}$ & remove first $n$ axes & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\[3pt] diff --git a/test/null/test_uop_graph.py b/test/null/test_uop_graph.py index 602144f6e3ace..577e39afe3bc5 100644 --- a/test/null/test_uop_graph.py +++ b/test/null/test_uop_graph.py @@ -441,7 +441,8 @@ def test_pm_mops_partial_reshape_index_removes_reshape(self): self.assertEqual(result.op, Ops.INDEX) self.assertIs(result.src[0], src) self.assertEqual(result.shape, (4,)) - self.assertNotIn(Ops.RESHAPE, [u.op for u in result.toposort()]) + # the only RESHAPE is src itself: the view of the flat param, the extra reshape was folded into the INDEX + self.assertEqual([u for u in result.toposort() if u.op is Ops.RESHAPE], [src]) def test_pm_mops_partial_reshape_index_suffix_mismatch_does_nothing(self): from tinygrad.schedule.prepare import pm_mops diff --git a/test/null/test_uops.py b/test/null/test_uops.py index 636dfb0957cbd..2eff8f78d9097 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -84,11 +84,10 @@ def test_gated_shrink_lowers_to_selected_width(self): def test_reg_buffer_size_lowers(self): reg = UOp.placeholder((4,), dtypes.float, 0, addrspace=AddrSpace.REG) - self.assertEqual(reg.src[0].dtype, dtypes.weakint) + self.assertEqual(reg.arg.size, 4) lowered = graph_rewrite(reg.sink(), pm_lower_weak) self.assertTrue(all(u.op is Ops.CONST for u in lowered.backward_slice_with_self if u.dtype in dtypes.weaks), "lowering must resolve every weak width, except a typed literal's value half") - self.assertEqual(next(u for u in lowered.backward_slice_with_self if u.op is Ops.BUFFER).src[0].dtype, dtypes.int) class TestSafeCast(unittest.TestCase): def test_cast_folds(self): diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 06ce496137d57..2e97f9a30d12b 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -244,7 +244,7 @@ def test_const_node_visibility(self): self.assertEqual(len(lst), 1) graphs = [x["graph"] for x in viz.get_details(0, 0)] # const is always in the graph, client side hides exclude=True nodes by default - self.assertEqual(list(graphs[0]), [id(a.src[0]), id(a), id(z), id(alu), id(y), id(sink)]) + self.assertEqual(list(graphs[0]), [id(a), id(z), id(alu), id(y), id(sink)]) self.assertTrue(graphs[0][id(z)]["exclude"]) self.assertTrue(graphs[0][id(y)]["exclude"]) self.assertFalse(graphs[0][id(alu)]["exclude"]) diff --git a/test/unit/test_call.py b/test/unit/test_call.py index 238b52fa1cc4a..ec8041f77d876 100644 --- a/test/unit/test_call.py +++ b/test/unit/test_call.py @@ -52,6 +52,11 @@ def test_call_plus_backward_auto(self): np.testing.assert_allclose(a.grad.numpy(), gt_a_grad, rtol=1e-5) np.testing.assert_allclose(b.grad.numpy(), gt_b_grad, rtol=1e-5) + def test_call_scalar_param_shape_mismatch(self): + scalar_fxn = UOp.param(0, dtypes.float, ()) * 2 + with self.assertRaisesRegex(TypeError, "shape mismatch: expected scalar"): + Tensor.call(Tensor.ones(2), fxn=scalar_fxn).realize() + def test_call_gemm(self): M, K, N = 4, 8, 4 a = Tensor.randn(M, K) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index f1c304b965e01..72f3e2ead3683 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -391,7 +391,15 @@ def full_rewrite_to_sink(ast:UOp, ren:Renderer, optimize:bool=True) -> UOp: sink = graph_rewrite(sink, pm_number_params, ctx=[num_params], name="number params with -1", walk=True) if VIZ: graph_rewrite(sink, PatternMatcher([]), name="View Output AST") - if SPEC: type_verify(sink, spec_program) + if SPEC: + import os + if os.environ.get("DBGTV"): + try: type_verify(sink, spec_program) + except RuntimeError: + from tinygrad.uop.render import print_uops + print_uops(list(sink.toposort())) + raise + else: type_verify(sink, spec_program) # return the rewritten sink return sink diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index 61ea1195cea64..cde69c6f75127 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -140,8 +140,8 @@ def f2f_store(st, idx, val, fr:DType, to:DType): pm_long_decomp: PatternMatcher = PatternMatcher([ # the decomp's own bottom-up rewrite can mint bare consts mid-flight: word splitting commits them at the long sibling's dtype (UPat(GroupOp.All, name='x'), lambda x: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in l2i_dt), None))), - (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), src=(UPat.var("sz"),), name="x"), lambda x,sz: - UOp(x.op, src=(sz*2,), arg=replace(x.arg, dtype=l2i_dt[x.dtype]), tag=x.tag)), + (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), name="x"), lambda x: + UOp(x.op, arg=replace(x.arg, dtype=l2i_dt[x.dtype], size=None if x.arg.size is None else x.arg.size*2), tag=x.tag)), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: diff --git a/tinygrad/codegen/gpudims.py b/tinygrad/codegen/gpudims.py index f479007c820bb..10f45d89c89c9 100644 --- a/tinygrad/codegen/gpudims.py +++ b/tinygrad/codegen/gpudims.py @@ -73,7 +73,7 @@ def add_gpudims(ctx:Renderer, s:UOp): subs = {} for r in s_topo: # look for local INDEXes that are not used in the GLOBAL store, then add them as an INVALID - if r.op is Ops.STORE and (idx := r.src[0]).src[0].addrspace == AddrSpace.GLOBAL: + if r.op is Ops.STORE and len((idx := r.src[0]).src) and idx.src[0].addrspace == AddrSpace.GLOBAL: missing_locals = [all_ranges[rng] for rng in local_dims if all_ranges[rng] not in idx.ranges] if len(missing_locals): assert len(idx.src) == 2, "index has 2 sources" diff --git a/tinygrad/codegen/late/coalesce.py b/tinygrad/codegen/late/coalesce.py index 4b1da0d785979..d62997865283f 100644 --- a/tinygrad/codegen/late/coalesce.py +++ b/tinygrad/codegen/late/coalesce.py @@ -1,7 +1,8 @@ import itertools, functools from collections import defaultdict +from dataclasses import replace from tinygrad.dtype import dtypes, AddrSpace, Invalid, DType -from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat, GroupOp, shape_to_shape_arg, graph_rewrite +from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat, GroupOp, graph_rewrite from tinygrad.uop.symbolic import uop_given_valid, parse_valid, invalid_gate, sym from tinygrad.helpers import getenv, IMAGE, OSX, ceildiv, is_image_shape from tinygrad.renderer import Renderer @@ -85,7 +86,8 @@ def transform_to_image(ctx, buf:UOp, x:UOp) -> UOp|None: if len(cands) == 0: return None # and tiebreak with indexing complexity (ie. number of nodes) h, w, cidx = cands[0] if len(cands) == 1 else min(cands, key=lambda cand: len(cand[2].index(1).simplify().backward_slice)) - buf = buf.replace(src=(shape_to_shape_arg((h, w, 4)),)) + # the image dims are stored in the param's arg, the size stays the flat buffer len + buf = buf.replace(arg=replace(buf.arg, image=(h, w))) shapes[buf.arg.slot] = (h, w) if valid.op is not Ops.CONST or valid.val is not True: return buf.index(cidx.src[1].valid(valid), cidx.src[0].valid(valid)) diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index 6918807bc2fa0..bfb7f29d3572f 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -67,7 +67,7 @@ def jit_lower(linear:UOp, held_bufs:set[UOp], input_uops:list[UOp]) -> UOp: if VIZ: graph_rewrite(linear, PatternMatcher([]), name="View captured linear") # parametrize input buffers: map each input buffer UOp to a PARAM with the correct slot index - linear = linear.substitute({u: UOp.param(i, u.dtype, u.shape, u.device) for i,u in enumerate(input_uops)}, walk=True) + linear = linear.substitute({u: UOp.param(i, u.dtype, u.max_numel(), u.device) for i,u in enumerate(input_uops)}, walk=True) linear = memory_plan_rewrite(linear, held_bufs) linear = compile_linear(linear, beam=getenv("JITBEAM", BEAM.value)) if JIT < 2: linear = graph_split_rewrite(linear, max_batch_size=JIT_BATCH_SIZE.value) diff --git a/tinygrad/mixin/gradient.py b/tinygrad/mixin/gradient.py index 4b10a4393ca2c..9899fbe0865a5 100644 --- a/tinygrad/mixin/gradient.py +++ b/tinygrad/mixin/gradient.py @@ -31,6 +31,10 @@ def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else return (None,) + k.arg.grad_fxn(on_dev(ctx, 0), k) assert fxn.op is Ops.TUPLE, f"expected TUPLE body for gradient, got {fxn.op}" params = {x.arg.slot:x for x in fxn.toposort(enter_calls=False) if x.op == Ops.PARAM} + # grads are collected at the flat param storage: reshape to each arg's view (max view shrunk to symbolic) + def shaped_grad(grad:UOp, i:int) -> UOp: + a = args[i] + return grad.view_as(a.shard_shape, a.axis) if a.axis is not None and isinstance(a.device, tuple) else grad.view_as(a._shape) grad_args = ctx.src root_grad = UOp(Ops.TUPLE, src=tuple(UOp(Ops.NOOP) if g.op is Ops.NOOP else g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args))) @@ -39,7 +43,7 @@ def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(fxn.src)} if k.arg.precompile else {} fwd_outs = tuple(k.gettuple(i) for i in range(len(fxn.src))) if k.arg.precompile else () # collect needed gradient bodies, compact unused params, create a single backward CALL - grad_bodies = [(i, grads[p]) for i in needed if (p:=params.get(i)) is not None and p in grads] + grad_bodies = [(i, shaped_grad(grads[p], i)) for i in needed if (p:=params.get(i)) is not None and p in grads] bwd_body = UOp.maketuple(*(gb for _, gb in grad_bodies)).substitute(fwd_subs, walk=True) bwd_body = renumber_invalid_outputs(bwd_body) bwd_body, compact_args = _compact_params(bwd_body, (*args, *grad_args, *fwd_outs)) diff --git a/tinygrad/schedule/__init__.py b/tinygrad/schedule/__init__.py index 9608e032fa4f9..b0543d4cb69c1 100644 --- a/tinygrad/schedule/__init__.py +++ b/tinygrad/schedule/__init__.py @@ -94,7 +94,7 @@ def create_new_buffer(ctx:tuple[dict[UOp, UOp], tuple[UOp, ...]], b:UOp): # only resolve buffer PARAMs (slot>=0); ALU/shape vars use slot=-1 and must not be swapped for call args (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx[1][x.arg.slot] if x.arg.slot >= 0 else None), # create new BUFFERs - (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: + (UPat(Ops.BUFFER, src=(), name="b"), lambda ctx,b: create_new_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), ]) diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index c60fd8aa7d9e8..bbee3258be469 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -280,13 +280,8 @@ def rewrite_into_function(call:UOp): for i, s in enumerate(new_body.src)]) return call.replace(src=(new_body,)+new_args) -def param_to_multi(p:UOp): - if p.axis is None: return None - return UOp.param(p.arg.slot, p.dtype, p.shard_shape, p.device, p.arg.vmin_vmax, p.arg.multiple_of, p.arg.name, p.arg.addrspace).unshard(p.axis) - # NOTE: this is the same pattern as unrolled ranges multi_pm = PatternMatcher([ - (UPat(Ops.PARAM, name="p"), param_to_multi), (UPat(GroupOp.ALU, name="root", custom_early_reject=set([Ops.UNSHARD])), alu_multi), (UPat(Ops.REDUCE, src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), reduce_multi), (UPat(Ops.RESHAPE, src=(UPat(Ops.UNSHARD, name="multi"), UPat()), name="root"), reshape_multi), diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 4bc9c10267ed0..d83329de8ebd4 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -1,7 +1,7 @@ import itertools from tinygrad.dtype import dtypes, to_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp -from tinygrad.uop.ops import graph_rewrite, rewrite_group, shape_to_shape_arg, ParamArg, identity_element +from tinygrad.uop.ops import graph_rewrite, rewrite_group, ParamArg, identity_element from tinygrad.uop.movement import mop_cleanup from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort from tinygrad.schedule.indexing import apply_movement_op @@ -100,10 +100,19 @@ def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: if [x.arg.slot for x in params] != list(range(len(params))): raise RuntimeError(f"params not in order: {[x.arg.slot for x in params]}") if len(params) != len(args): raise TypeError(f"expected {len(params)} args, got {len(args)}") + # params have a flat storage size in the arg, the logical shape is a view (RESHAPE/SHRINK/UNSHARD) on top of it. + # substitute args by their flat max-shaped storage view so the movement views on the params stay valid + def flat_storage(a:UOp) -> tuple[int, UOp]: # returns (size, view of a as flat max-shaped storage) + shp = a.max_shard_shape if a.axis is not None and isinstance(a.device, tuple) else a.max_shape + return (n:=prod(shp)), a if a.shape == (n,) else a.pad_to(shp).reshape((n,)) dict_map = {x:args[x.arg.slot] for x in params} for i, (p, a) in enumerate(dict_map.items()): - if p.axis != a.axis: raise TypeError(f"arg {i} axis mismatch: expected {p.axis}, got {a.axis}") - if p.max_shape != a.max_shape: raise TypeError(f"arg {i} shape mismatch: expected {p.shape}, got {a.shape}") + if p.arg.size is not None: + n, flat = flat_storage(a) + if p.arg.size != n: raise TypeError(f"arg {i} shape mismatch: expected size {p.arg.size}, got {a.shape}") + dict_map[p] = flat + elif a.shape != (): + raise TypeError(f"arg {i} shape mismatch: expected scalar, got {a.shape}") if p.dtype != a.dtype: raise TypeError(f"arg {i} dtype mismatch: expected {p.dtype}, got {a.dtype}") return c.src[0].substitute(dict_map, walk=True) @@ -192,9 +201,9 @@ def convert_copy_to_store(ctx, copy:UOp, existing_buf:UOp|None=None): # if there's already a buffer, we just use it return existing_buf.flatten().store(input_src) # create the output buffer - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg(input_src.max_shape),), arg=ParamArg(next(ctx), copy.dtype, device=copy.device)) + buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), copy.dtype, size=prod(input_src.max_shape), device=copy.device)) # reshape back to input - return buf.after(buf.store(input_src)).reshape(copy.shape) + return buf.reshape(input_src.max_shape).after(buf.store(input_src)).reshape(copy.shape) pm_copy_to_store = PatternMatcher([ (UPat(name="existing_buf").store(UPat(Ops.COPY, name="copy")), convert_copy_to_store), diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 874b55624be57..c2e8b925afc3c 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -2,7 +2,7 @@ from typing import cast import itertools from tinygrad.dtype import dtypes, AddrSpace, Invalid -from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg, shape_to_shape_arg +from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic from tinygrad.helpers import prod, dedup, DEBUG_RANGEIFY, VIZ, MAX_KERNEL_BUFFERS, SPEC @@ -162,6 +162,12 @@ def no_indexing_calls(u:UOp): (UPat(Ops.CALL, name="u"), no_indexing_calls), ]) +# the kernel graph is what gets executed: no shape views left in it, the storage of a value is just the storage +pm_no_views = PatternMatcher([ + (UPat((Ops.RESHAPE, Ops.SHRINK), name="v", src=(UPat((Ops.AFTER, Ops.PARAM, Ops.UNSHARD, Ops.MSTACK, Ops.BUFFER)),), allow_any_len=True), lambda v: + v.src[0]), +]) + DEVICE_MAX_BUFS = {"METAL": 31, "WEBGPU": 8, "CPU": 31} # TODO: get from device? @dataclass class LimitBufsContext: @@ -223,7 +229,7 @@ def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): # NOTE: the local BUFFER needs to be disambiguated here if x.arg.addrspace == AddrSpace.GLOBAL: - buf = UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(next(ctx), x.dtype, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) + buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), x.dtype, size=size, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) do_store = buf.index(idx).store(x.src[0]).end(*rngs) return buf.after(do_store) @@ -282,8 +288,7 @@ class LocalAddBufferContext: def debuf(ctx:LocalAddBufferContext, buf:UOp): # Variables (ALU buffers with a value range) are scalar symbolic values, not real buffers: they become ALU params with no slot if buf.is_variable: return buf.replace(op=Ops.PARAM) - param = UOp(Ops.PARAM, src=(UOp.const(prod(buf.max_shape)),), - arg=ParamArg(ctx.dg, buf.dtype, addrspace=buf.addrspace, device=buf.device)) + param = UOp(Ops.PARAM, arg=ParamArg(ctx.dg, buf.dtype, prod(buf.max_shape), addrspace=buf.addrspace, device=buf.device)) ret = param.reshape(buf.max_shape) # if the buffer has symbolic shape, shrink the max-sized view to the actual shape if buf.max_shape != buf.shape: ret = ret.shrink(tuple((0, s) for s in buf.shape)) @@ -383,6 +388,7 @@ def get_kernel_graph(tsink:UOp) -> UOp: tsink = graph_rewrite(tsink, pm_add_buffers+pm_add_param_range_tags, ctx=itertools.count(paramarg_start), bottom_up=True, name="stage to store") tsink = graph_rewrite(tsink, split_kernels, bottom_up=True, name="split kernels") tsink = graph_rewrite(tsink, pm_no_indexing_calls, name="remove indexing from call args") + tsink = graph_rewrite(tsink, pm_no_views, name="remove views from the kernel graph") if VIZ: graph_rewrite(tsink, PatternMatcher([]), name="View Kernel Graph") if SPEC: diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 28d0dfb3065ce..8a77549ecedbf 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -198,9 +198,7 @@ def finalize_after(ctx:AllocCtx, x:UOp): def replace_input_buffer(ctx:AllocCtx, b:UOp): ctx.replacements.append(b) - if b.is_bound_var or b.is_variable: return b.param_like(len(ctx.replacements)-1) - return UOp.param(len(ctx.replacements)-1, b.dtype, b.shape, b.device, - addrspace=b.addrspace if b.addrspace is not None else AddrSpace.GLOBAL) + return b.param_like(len(ctx.replacements)-1) def replace_input_view(ctx:AllocCtx, b:UOp): return replace_input_buffer(ctx, b) if b in ctx.views else None @@ -211,7 +209,7 @@ def replace_input_view(ctx:AllocCtx, b:UOp): return replace_input_buffer(ctx, b) pm_replace_buf = PatternMatcher([ # replace BUFFER with PARAM for cache key normalization - (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: + (UPat(Ops.BUFFER, src=(), name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), # replace SHRINK with PARAM (UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), name="b", allow_any_len=True), replace_input_view), @@ -381,7 +379,7 @@ def dtype(self) -> DType: return self.uop.dtype # ***** data handlers **** def as_param(self, slot:int): - return Tensor(UOp.param(slot, self.dtype, self.uop.shard_shape, self.device, axis=self.uop.axis)) + return Tensor(self.uop.param_like(slot)) def call(self, *lst:Tensor, fxn:Tensor|UOp, grad_fxn:Callable|None=None) -> Tensor: fret = fxn._uop.call(*[t.uop for t in (self,)+lst], grad_fxn=grad_fxn) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index aafd8147c726c..638a05a899c4b 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -23,6 +23,8 @@ def __repr__(self): return str(self) class ParamArg: slot: int dtype: DType + # number of elements in the buffer. always a concrete int (never symbolic), None for scalars (shape ()) + size: int|None = None vmin_vmax: tuple[PyConst, PyConst]|None = None multiple_of: int|None = None name: str|None = None @@ -30,10 +32,13 @@ class ParamArg: axis: int|None = None device: str|tuple[str, ...]|None = None volatile: bool = False + # (h, w) if this is an image2d buffer, then size == h*w*4 + image: tuple[int, int]|None = None def __repr__(self): fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("axis", None), ("device", None), - ("volatile", False)) - args = [repr(self.slot), repr(self.dtype)] + [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] + ("volatile", False), ("image", None)) + args = [repr(self.slot), repr(self.dtype)] + ([repr(self.size)] if self.size is not None else []) + \ + [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] return f"ParamArg({', '.join(args)})" axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.THREAD: "t", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", AxisType.LOOP: "L", AxisType.UPCAST: "u", AxisType.GROUP_REDUCE: "G", AxisType.REDUCE: "R", AxisType.UNROLL: "r"} @@ -381,9 +386,10 @@ def _shape(self) -> tuple[sint, ...]|None: case Ops.GETADDR: return () case Ops.RANGE | Ops.SPECIAL: return () case Ops.BINARY: return (len(self.arg),) - case Ops.BUFFER: - if len(self.src): return self.src[0].as_shape - return () + case Ops.BUFFER | Ops.PARAM: + # PARAM/BUFFER don't have a shape input, they have a size in the arg: int gives shape (size,), None gives () + if (img:=self.arg.image) is not None: return (img[0], img[1], 4) + return () if self.arg.size is None else (self.arg.size,) case Ops.CUSTOM | Ops.CUSTOMI: if self.dtype is dtypes.void: return None input_shapes = [x._shape for x in self.src if x._shape is not None] @@ -394,10 +400,6 @@ def _shape(self) -> tuple[sint, ...]|None: # STAGE adds the existing shape to the front, opposite of INDEX return tuple([int(r.vmax+1) for r in self.src[1:]])+self.src[0].shape - # param has shape as the only arg - case Ops.PARAM: - return self.src[0].as_shape - # wmma output shape = accumulator shape (src[2]) case Ops.WMMA: wmma_b = _broadcast_shape(self.src[0].shape[:-1], self.src[1].shape[:-1], self.src[2].shape[:-1]) @@ -821,8 +823,9 @@ def getaddr(self, device=None) -> UOp: @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): if dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {dtype}") + assert isinstance(size, int), f"new_buffer size must be a concrete int, got {size}" slot = next(UOp.unique_num) if num is None else num - return UOp(Ops.BUFFER, src=(shape_to_shape_arg((size,)),), arg=ParamArg(slot, dtype, device=device)) + return UOp(Ops.BUFFER, arg=ParamArg(slot, dtype, size=size, device=device)) @staticmethod def from_buffer(opaque:Buffer, device:str|tuple[str, ...]|None=None): if (uop:=UOp.new_buffer(device or opaque.device, opaque.size, opaque.dtype, num=-id(opaque))) not in buffers: buffers[uop] = opaque.ref(1) @@ -978,7 +981,7 @@ def variable(name:str, min_val:PyConst, max_val:PyConst, dtype:DType=dtypes.weak # a Variable is a 0-d BUFFER in the ALU addrspace; binding it is storing a CONST into it # param=True creates the kernel-side form directly: an ALU PARAM (what the BUFFER becomes inside kernels) arg = ParamArg(-1, dtype, name=name, vmin_vmax=(min_val, max_val), multiple_of=multiple_of, addrspace=AddrSpace.ALU) - return UOp(Ops.PARAM if param else Ops.BUFFER, src=(shape_to_shape_arg(()),), arg=arg) + return UOp(Ops.PARAM if param else Ops.BUFFER, arg=arg) @property def is_variable(self) -> bool: # a Variable is a 0-d BUFFER in the ALU addrspace that carries a value range (it becomes a PARAM inside kernels) @@ -1148,11 +1151,11 @@ def placeholder(shape:tuple[int, ...], dtype:DType, slot:int|None=None, addrspac dtype = strong_dtype(dtype) # storage is never weak: a placeholder commits the width of what's put in it if slot is None: slot = next(UOp.unique_num) if addrspace is AddrSpace.GLOBAL: - ret = UOp(Ops.PARAM, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace, device=device,volatile=volatile)) + ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, size=prod(shape), addrspace=addrspace, device=device, volatile=volatile)) else: assert addrspace in (AddrSpace.LOCAL, AddrSpace.REG) assert device is None, "LOCAL and REG placeholders cannot have a device" - ret = UOp(Ops.BUFFER, src=(shape_to_shape_arg((prod(shape),)),), arg=ParamArg(slot, dtype, addrspace=addrspace)) + ret = UOp(Ops.BUFFER, arg=ParamArg(slot, dtype, size=prod(shape), addrspace=addrspace)) if tag is not None: ret = ret.rtag(tag) if len(shape) > 1: ret = ret.reshape(shape) return ret @@ -1168,19 +1171,35 @@ def set(self:UOp, val:UOp|ConstType, end:UOp|tuple[UOp, ...]|list[UOp]=()) -> UO @staticmethod def param(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, axis:int|None=None, volatile:bool=False): + """create a PARAM: a single sint or 1-d shape gives a flat param of that size, a None shape gives a scalar param. + the arg only stores the concrete max size (never symbolic): a multi-dim shape is a RESHAPE on top of the flat param, + a symbolic shape is a max-size param shrunk to the real shape""" if dtype in dtypes.weaks: raise RuntimeError(f"cannot create param for weak dtype {dtype}") if isinstance(shape, (int, UOp)): shape = (shape,) if shape is not None and axis is not None and isinstance(device, tuple): shape = tuple(s*len(device) if i == axis else s for i,s in enumerate(shape)) - src: tuple[UOp, ...] = (UOp(Ops.NOOP) if shape is None else shape_to_shape_arg(shape),) - return UOp(Ops.PARAM, src=src, arg=ParamArg(slot, dtype, vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + if shape is None or len(shape) == 0: + return UOp(Ops.PARAM, arg=ParamArg(slot, dtype, None, vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + max_shape = to_max_shape(shape) + ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, prod(max_shape), vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + return ret.view_as(shape) def param_like(self, slot:int): # Variables become ALU params in the call body; the stored value (if bound) stays in the call args if self.is_bound_var or self.is_variable: b = self.src[0] if self.op is Ops.AFTER else self - return UOp(Ops.PARAM, src=b.src, arg=replace(b.arg, slot=slot, name=f"p{slot}")) + return UOp(Ops.PARAM, arg=replace(b.arg, slot=slot, name=f"p{slot}")) addrspace = self.addrspace if self.addrspace is not None else AddrSpace.GLOBAL - return UOp.param(slot, self.dtype, self.shard_shape if self.axis is not None else self._shape, self.device, addrspace=addrspace, axis=self.axis) + # multi-device values become a per-shard sized param wrapped in UNSHARD: the sharding lives in the graph, not the arg + if self.axis is not None and isinstance(self.device, tuple): + return UOp(Ops.PARAM, arg=ParamArg(slot, self.dtype, prod(to_max_shape(self.shard_shape)), + addrspace=addrspace, device=self.device)).view_as(self.shard_shape, self.axis) + return UOp.param(slot, self.dtype, self._shape, self.device, addrspace=addrspace) + def view_as(self:UOp, shape:tuple[sint, ...], axis:int|None=None) -> UOp: + """view flat storage as the given (possibly symbolic) shape, optionally sharded on axis, the UNSHARD gives back the multiplied shape""" + max_shape = to_max_shape(shape) + ret = self.reshape(max_shape) if len(shape) > 1 else self + if tuple(max_shape) != tuple(shape): ret = ret.shrink_to(shape) + return ret if axis is None else ret.unshard(axis) @staticmethod def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, src=src, arg=name) diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 4bdb0ec5ad97c..914e398a15c1c 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -84,7 +84,7 @@ def render_marg(ctx,x:UOp): (UPat(Ops.CONST, src=(), name="x"), lambda x: f"UOp.const({x.val})"), (UPat((Ops.CAST, Ops.BITCAST), name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({x.dtype})" if x.dtype != x.src[0].dtype else None), (UPat(Ops.SPECIAL, src=(UPat(Ops.CONST),), name="x"), lambda x: f"UOp.special({x.src[0].val}, {repr(x.arg)})"), - (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: + (UPat(Ops.BUFFER, src=(), name="x"), lambda x: f"UOp.new_buffer({repr(x.arg.device)}, {x.max_numel()}, {x.dtype}, {x.arg.slot})" if isinstance(x.arg, ParamArg) and x.addrspace is AddrSpace.GLOBAL else None), (UPat(Ops.COPY, src=(UPat(name="x"),), name="copy"), lambda ctx,x,copy: f"{ctx[x]}.copy_to_device({repr(copy.arg)})"), diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 3fa820c480dc2..6d0751c5cbefc 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -87,10 +87,9 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base # a loop-ended END requires a trailing bool condition for the backedge (loop again while true) (UPat(Ops.END, src=(UPat(), UPat(Ops.RANGE, dtypes.void), UPat(dtype=dtypes.bool))), lambda: True), - # PARAM - (UPat(Ops.PARAM, name="x"), lambda x: isinstance(x.arg, ParamArg)), - (UPat(Ops.BUFFER, src=(UPat(),), name="x"), lambda x: - isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.REG, AddrSpace.LOCAL)), + # PARAM/BUFFER have a size in the arg, no shape input + (UPat(Ops.PARAM, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), + (UPat(Ops.BUFFER, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.REG, AddrSpace.LOCAL)), # GROUP of stores (or groups, or NOOPs) (UPat(Ops.GROUP, dtypes.void, src=UPat((Ops.GROUP, Ops.STORE, Ops.NOOP, Ops.INS, Ops.END))), lambda: True), @@ -141,12 +140,12 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < lambda u: dtypes.is_float(u.dtype) or u.src[0].base.is_invalid), # BUFFER - (UPat(Ops.BUFFER, src=(UPat(),), name="buf"), lambda buf: - (isinstance(buf.dtype, DType) and matches_dtype(buf.src[0], dtypes.weakint) and is_device(buf.arg.device)) + (UPat(Ops.BUFFER, src=(), name="buf"), lambda buf: + (isinstance(buf.dtype, DType) and isinstance(buf.arg.size, int) and is_device(buf.arg.device)) if isinstance(buf.arg, ParamArg) and buf.addrspace is AddrSpace.GLOBAL else None), # a Variable is a 0-d ALU BUFFER with a value range and no device - (UPat(Ops.BUFFER, src=(UPat(),), name="buf"), lambda buf: buf.arg.device is None if buf.is_variable else None), + (UPat(Ops.BUFFER, src=(), name="buf"), lambda buf: buf.arg.device is None if buf.is_variable else None), # custom function (UPat(Ops.CUSTOM_FUNCTION, name="x"), lambda x: isinstance(x.arg, str)), @@ -260,11 +259,10 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.STACK, name="s"), lambda s: all(x.op in (Ops.CONST, Ops.PARAM) or x.is_variable or x.is_bound_var for x in s.src) or None), # linear for more kernels (TODO: we should enter non sink calls) #(UPat(Ops.LINEAR), lambda: True), - # param is outside buffer, buffer is local buffer - (UPat(Ops.PARAM, name="x"), lambda x: isinstance(x.arg, ParamArg)), + # param is outside buffer, buffer is local buffer. params have a size in the arg, no shape input + (UPat(Ops.PARAM, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), (UPat(Ops.BUFFER, name="x"), lambda x: isinstance(x.arg, ParamArg) and x.addrspace in (AddrSpace.GLOBAL, AddrSpace.ALU)), - # RESHAPE/BITCAST are NOOPs in the kernel graph (do we need them?) - (UPat((Ops.RESHAPE, Ops.BITCAST)), lambda: True), + (UPat(Ops.BITCAST), lambda: True), # mstack/mselect (UPat(Ops.MSTACK, name="x"), lambda x: all(isinstance(s.device, str) for s in x.src) or (all_same(x.src) and x.src[0].device is None)), (UPat(Ops.MSELECT, name="x"), lambda x: isinstance(x.src[0].device, tuple) and x.arg < len(x.src[0].device)), From bc485d271f13e3e1d6b42bb605f94f0c4b499eb7 Mon Sep 17 00:00:00 2001 From: chenyu Date: Thu, 27 Aug 2026 21:14:34 -0400 Subject: [PATCH 243/306] fix CMPLT between float16 and const 0 (#17791) * fix CMPLT between float16 and const 0 * fix --- test/backend/test_dtype.py | 5 +++++ tinygrad/renderer/isa/x86.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/backend/test_dtype.py b/test/backend/test_dtype.py index fac4a39b08675..7ce56ad3cdfe6 100644 --- a/test/backend/test_dtype.py +++ b/test/backend/test_dtype.py @@ -428,6 +428,11 @@ def test_max_w_alu(self): t = Tensor([[1, 2], [3, 4]], dtype=d) (t*t).max().item() + def test_where_float16_compare_to_const(self): + # t > 0 is CMPLT(0, t): the float16 operand is on the right + t = Tensor([-1.0, 1.0], dtype=dtypes.float16) + np.testing.assert_equal((t > 0).where(Tensor.ones(2, dtype=dtypes.float16), Tensor.zeros(2, dtype=dtypes.float16)).numpy(), [0.0, 1.0]) + @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, f"no bfloat16 on {Device.DEFAULT}") class TestOpsBFloat16(unittest.TestCase): def test_cast(self): diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index fa105f841825a..15870a9f6f8b0 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -145,8 +145,8 @@ class X86GroupOp: # float16 alus are done in float32 (UPat(GroupOp.ALU, dtypes.float16, name="x"), lambda x: UOp(x.op, src=tuple(s.cast(dtypes.float) if s.dtype != dtypes.bool else s for s in x.src)).cast(x.dtype)), - (UPat(GroupOp.Comparison, src=(UPat.var("a", dtypes.float16), UPat.var("b")), name="x"), - lambda x,a,b: UOp(x.op, src=(a.cast(dtypes.float32), b.cast(dtypes.float32))).cast(x.dtype)), + (UPat(GroupOp.Comparison, src=[UPat(dtype=dtypes.float16), UPat()], name="x"), + lambda x: UOp(x.op, src=tuple(s.cast(dtypes.float32) for s in x.src)).cast(x.dtype)), # no cmpne for packed ints, y != x => !(y==x) (UPat(Ops.CMPNE, src=(UPat.var("y", dtypes.ints), UPat.var("x")), name="cmp"), lambda y,x,cmp: UOp(Ops.CMPEQ, src=(y,x))^True if y.max_numel() > 1 else None), From 4e6bdac412c5b2be4b233b82a11fe1e9d6a81e4c Mon Sep 17 00:00:00 2001 From: George Hotz Date: Thu, 27 Aug 2026 18:22:56 -0700 Subject: [PATCH 244/306] hotfix: disable HCQ2 --- .github/workflows/benchmark.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ca506e4d95ff0..c1a75ecf5dd11 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -94,7 +94,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -140,7 +140,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -188,7 +188,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -230,7 +230,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -275,7 +275,7 @@ jobs: shell: bash -e -o pipefail {0} env: DEV: ${{ matrix.dev }} - HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }} + HCQ2: '0' if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code From 39d7fd0981c70f8ef812a3b523ce2c8b40c5f80e Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:41:09 -0700 Subject: [PATCH 245/306] axis arg is dead (#17793) --- tinygrad/uop/ops.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 638a05a899c4b..397cdeb5bf9a3 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -29,13 +29,12 @@ class ParamArg: multiple_of: int|None = None name: str|None = None addrspace: AddrSpace|None = AddrSpace.GLOBAL - axis: int|None = None device: str|tuple[str, ...]|None = None volatile: bool = False # (h, w) if this is an image2d buffer, then size == h*w*4 image: tuple[int, int]|None = None def __repr__(self): - fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("axis", None), ("device", None), + fields = (("vmin_vmax", None), ("multiple_of", None), ("name", None), ("addrspace", AddrSpace.GLOBAL), ("device", None), ("volatile", False), ("image", None)) args = [repr(self.slot), repr(self.dtype)] + ([repr(self.size)] if self.size is not None else []) + \ [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] @@ -710,7 +709,7 @@ def axis(self) -> int|None: if self.op is Ops.GETTUPLE: in_tuple = self.src[0].src[0] if self.src[0].op is Ops.FUNCTION else self.src[0] return in_tuple.src[self.arg].axis if in_tuple.op is Ops.TUPLE else None - if self.op is Ops.PARAM: return self.arg.axis + if self.op is Ops.PARAM: return None # NOTE: they all have to share an axis, we always choose [-1]. src axes are right-aligned into the output shape if self.op in GroupOp.ALU.union({Ops.STACK}): return axes[-1] if (axes := dedup([x.axis+len(self.shape)-len(x.shape) for x in self.src if x.axis is not None])) else None @@ -1170,18 +1169,16 @@ def set(self:UOp, val:UOp|ConstType, end:UOp|tuple[UOp, ...]|list[UOp]=()) -> UO # TODO: this should replace placeholder @staticmethod def param(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, vmin_vmax:tuple[PyConst, PyConst]|None=None, - multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, axis:int|None=None, volatile:bool=False): + multiple_of:int|None=None, name=None, addrspace=AddrSpace.GLOBAL, volatile:bool=False): """create a PARAM: a single sint or 1-d shape gives a flat param of that size, a None shape gives a scalar param. the arg only stores the concrete max size (never symbolic): a multi-dim shape is a RESHAPE on top of the flat param, a symbolic shape is a max-size param shrunk to the real shape""" if dtype in dtypes.weaks: raise RuntimeError(f"cannot create param for weak dtype {dtype}") if isinstance(shape, (int, UOp)): shape = (shape,) - if shape is not None and axis is not None and isinstance(device, tuple): - shape = tuple(s*len(device) if i == axis else s for i,s in enumerate(shape)) if shape is None or len(shape) == 0: - return UOp(Ops.PARAM, arg=ParamArg(slot, dtype, None, vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + return UOp(Ops.PARAM, arg=ParamArg(slot, dtype, None, vmin_vmax, multiple_of, name, addrspace, device, volatile)) max_shape = to_max_shape(shape) - ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, prod(max_shape), vmin_vmax, multiple_of, name, addrspace, axis, device, volatile)) + ret = UOp(Ops.PARAM, arg=ParamArg(slot, dtype, prod(max_shape), vmin_vmax, multiple_of, name, addrspace, device, volatile)) return ret.view_as(shape) def param_like(self, slot:int): # Variables become ALU params in the call body; the stored value (if bound) stays in the call args From 4456e62343babef414ca7f17420740dd39654ce4 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Thu, 27 Aug 2026 19:06:08 -0700 Subject: [PATCH 246/306] ci: skip apt steps if packages are already installed (#17794) --- .github/actions/setup-tinygrad/action.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index d55221f688724..372f227e30db7 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -194,21 +194,29 @@ runs: echo "pkgs=$pkgs" >> "$GITHUB_OUTPUT" echo "hash=$(echo -n "$pkgs" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + installed=true + for pkg in $pkgs; do + info=$(dpkg-query -W -f='${db:Status-Abbrev} ${Version}' "$pkg" 2> /dev/null || true) + echo "${pkg}: ${info:-not in dpkg database}" + [[ "$info" == ii* ]] || installed=false + done + echo "installed=$installed" >> "$GITHUB_OUTPUT" + - name: Cache apt (PR) - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name == 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name == 'pull_request' && steps.apt-pkgs.outputs.installed == 'false' uses: actions/cache/restore@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Cache apt - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name != 'pull_request' + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && github.event_name != 'pull_request' && steps.apt-pkgs.outputs.installed == 'false' uses: actions/cache@v5 with: path: /var/cache/apt/archives/ key: ${{ runner.os }}-${{ runner.arch }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }} - name: Run apt Update + Install - if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') + if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true' || inputs.qemu == 'true' || inputs.ninja == 'true') && steps.apt-pkgs.outputs.installed == 'false' shell: bash run: | sudo apt -qq update || true From 53cad325c794f50b87bea8730b9495177957de0c Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:09:11 +0800 Subject: [PATCH 247/306] copy movement op tests construct copies (#17792) * copy movement op tests construct copies * hop 3 gpus * assert the amount of bytes moved * work * arange 4, everyone copies 16B first --- test/backend/test_schedule.py | 37 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/test/backend/test_schedule.py b/test/backend/test_schedule.py index d802dcf4edacf..0f7738d6b3620 100644 --- a/test/backend/test_schedule.py +++ b/test/backend/test_schedule.py @@ -379,27 +379,30 @@ def test_clone(self): check_schedule(a.clone(), 1, filter_sink=False) def test_shrink_copy(self): - a = Tensor.arange(4) - view = a.shrink(((0, 2),)) - b = view.clone() - run_linear(*check_schedule(b, 1, filter_sink=False)) - self.assertEqual(b.uop.base.buffer.size, 2) - self.assertEqual(b.uop.numel(), 2) - self.assertListEqual(b.tolist(), [0, 1]) + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").shrink(((1, 3),)).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # extra E kernel, copy exactly 4 bytes + self.assertEqual(GlobalCounters.global_mem, 4*4 + 2*4*2 + 2*4) + self.assertListEqual(b.tolist(), [1, 2]) def test_expanded_copy(self): - a = Tensor.arange(2) - view = a.reshape(2, 1).expand(2, 2) - b = view.clone() - run_linear(*check_schedule(b, 1, filter_sink=False)) - self.assertEqual(b.uop.base.buffer.size, 4) - self.assertEqual(b.uop.numel(), 4) - self.assertListEqual(b.tolist(), [[0, 0], [1, 1]]) + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").reshape(4, 1).expand(4, 2).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # TODO: expands before copy + self.assertEqual(GlobalCounters.global_mem, 4*4 + (4*4 + 8*4) + 8*4) + self.assertListEqual(b.tolist(), [[0, 0], [1, 1], [2, 2], [3, 3]]) def test_permuted_copy(self): - a = Tensor.arange(4) - b = a.reshape(2, 2).permute(1, 0) - b.realize() + a = Tensor.arange(4).clone("CPU:1").realize() + b = a.to("CPU:2").reshape(2, 2).permute(1, 0).to("CPU:3") + GlobalCounters.reset() + run_linear(*check_schedule(b, 3, filter_sink=False)) + # permutes before copy + self.assertEqual(GlobalCounters.global_mem, 4*4 + (4*4 + 4*4) + 4*4) self.assertListEqual(b.tolist(), [[0, 2], [1, 3]]) def test_permute_on_disk(self): From 26c4b6319edd61f9a1e6cf7a07c057426418e596 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Thu, 27 Aug 2026 22:17:05 -0400 Subject: [PATCH 248/306] fix interpolate with align_corners and an output size of 1 (#17790) --- test/backend/test_ops.py | 2 +- tinygrad/mixin/op.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 30f564d6d18e2..1aba62cb0b62e 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -2810,7 +2810,7 @@ def test_interpolate_linear(self): lambda x: Tensor.interpolate(x, size=out_sz, mode="linear")) def test_interpolate_linear_corners_aligned(self): - for in_sz, out_sz in [((52,),(29,)), ((29,),(52,))]: + for in_sz, out_sz in [((52,),(29,)), ((29,),(52,)), ((29,),(1,))]: helper_test_op([(2,3)+in_sz], lambda x: torch.nn.functional.interpolate(x, size=out_sz, mode="linear", align_corners=True), lambda x: Tensor.interpolate(x, size=out_sz, mode="linear", align_corners=True)) diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 9f65649042f75..6991683052256 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -1067,7 +1067,7 @@ def interpolate(self, size:tuple[int, ...], mode:str="linear", align_corners:boo reshape[i] = expand[i] = size[i] if mode == "linear": arr = type(self).arange(size[i]) - num, den = (arr*(in_sz-1), size[i]-1) if align_corners else ((arr*2+1)*in_sz - size[i], size[i]*2) + num, den = (arr*(in_sz-1), max(size[i]-1, 1)) if align_corners else ((arr*2+1)*in_sz - size[i], size[i]*2) num = num.clip(0, (in_sz-1)*den) low, high, perc = [y.reshape(reshape).expand(expand) for y in (num//den, (num+den-1)//den, (num % den).cast(dtypes.float32)/den)] x = x.gather(i, low).lerp(x.gather(i, high), perc) From 92d92c70a956e0a300ed12a3a41ac8529694f02f Mon Sep 17 00:00:00 2001 From: wozeparrot Date: Fri, 28 Aug 2026 01:06:33 -0400 Subject: [PATCH 249/306] gptoss: faster grad handling (#17795) --- examples/mlperf/model_train.py | 8 +++----- examples/mlperf/models/gpt_oss.py | 6 +++--- examples/mlperf/optim.py | 5 +++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/mlperf/model_train.py b/examples/mlperf/model_train.py index a69850ae93004..d72399d83e817 100644 --- a/examples/mlperf/model_train.py +++ b/examples/mlperf/model_train.py @@ -1667,7 +1667,7 @@ def get_eval_iter(): def train_gptoss(): from examples.mlperf.models.gpt_oss import GPTOSS, GPT_OSS_20B, apply_grad, FP8_DTYPE from examples.mlperf.lr_schedulers import CosineAnnealingLRWithWarmup - from examples.mlperf.optim import GradAccClipAdamW, GradAccClipAdamWGroup, clip_grads + from examples.mlperf.optim import GradAccClipAdamW, GradAccClipAdamWGroup, fclip_grads BENCHMARK = getenv("BENCHMARK") @@ -1785,12 +1785,10 @@ def step(tokens:Tensor): Tensor.realize(loss, *grads) - grad_norm = clip_grads(grads, 1, 1.0) - optim.fstep(grads, grad_norm) + clipped_grads, grad_norm = fclip_grads(grads, 1.0) + optim.fstep(clipped_grads, grad_norm) scheduler.step() - for g in grads: g.assign(0) - loss_cpu = loss.flatten().float().to("CPU") lr_cpu = optim.lr.float().to("CPU") grad_norm_cpu = grad_norm.float().to("CPU") diff --git a/examples/mlperf/models/gpt_oss.py b/examples/mlperf/models/gpt_oss.py index 00cc5cad2f32a..227d5ea6e50b1 100644 --- a/examples/mlperf/models/gpt_oss.py +++ b/examples/mlperf/models/gpt_oss.py @@ -282,14 +282,14 @@ def apply_grad(grad_buf:Tensor, new_grad:UOp): pads = _get_pads(new_grad) if len(pads) <= 1: new_grad = new_grad.cast(grad_buf.dtype) - grad_buf.uop = grad_buf.uop.after(grad_buf.uop.store(grad_buf.uop + new_grad)) + grad_buf.uop = grad_buf.uop.after(grad_buf.uop.store(new_grad)) return cur = grad_buf.uop for pad in sorted(pads, key=lambda p: p.marg[0][0] if p.op == Ops.PAD else 0, reverse=True): if pad.op == Ops.PAD: - grad_shrink = tuple([(p[0], s+p[0]) for s,p in zip(pad.src[0].shape, pad.marg)]) + grad_shrink = tuple((p[0], s+p[0]) for s,p in zip(pad.src[0].shape, pad.marg)) buf_slice = cur.shrink(grad_shrink) - cur = cur.after(buf_slice.store(buf_slice + pad.src[0].cast(cur.dtype))) + cur = cur.after(buf_slice.store(pad.src[0].cast(cur.dtype))) else: cur = cur.after(cur.store(cur + pad.cast(cur.dtype))) grad_buf.uop = cur diff --git a/examples/mlperf/optim.py b/examples/mlperf/optim.py index 5e013193018bd..d7e3d861b9eaf 100644 --- a/examples/mlperf/optim.py +++ b/examples/mlperf/optim.py @@ -27,6 +27,11 @@ def clip_grads(grads:list[Tensor], grad_acc, clip_norm) -> Tensor: for g in grads: g.assign((g * (clip_norm / (total_norm + 1e-6)).clamp(max_=1.0)).cast(g.dtype)) return total_norm +def fclip_grads(grads:list[Tensor], clip_norm) -> Tensor: + total_norm = Tensor.stack(*[g.float().square().sum() for g in grads]).sum().sqrt().contiguous() + scale = (clip_norm / (total_norm + 1e-6)).clamp(max_=1.0) + return [(g * scale).cast(g.dtype) for g in grads], total_norm + class GradAccClipAdamW(Optimizer): def __init__(self, params:list[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-6, weight_decay=0.0, grad_acc=1, clip_norm=1.0, device=None, fused=FUSE_OPTIM): super().__init__(params, lr, device, fused) From 6242b09066db994cf30177f543e8ccd2ee66af16 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:59:35 +0800 Subject: [PATCH 250/306] cleaner mxfp4 gemm prelude (#17796) * cleaner mxfp4 prelude * rename sgprs * min diff --- extra/gemm/gemm_mxfp4.py | 183 +++++++++++++++------------------------ 1 file changed, 68 insertions(+), 115 deletions(-) diff --git a/extra/gemm/gemm_mxfp4.py b/extra/gemm/gemm_mxfp4.py index 678e1ffd5d9bf..4edbf1779941f 100644 --- a/extra/gemm/gemm_mxfp4.py +++ b/extra/gemm/gemm_mxfp4.py @@ -20,34 +20,39 @@ def v_mfma_fp4(dst, a, b, opsel, opsel_hi, scale_a, scale_b): def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k = Kernel() scale_k = K // 32 + k.emit(s_and_b32(s[1], s[1], LIT, 65535)) if (tile_m, tile_n) == (128, 512): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) k.emit(s_mov_b32(s[47], s[2])) k.emit(s_mov_b32(s[48], s[3])) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[36], N)) - k.emit(s_mov_b32(s[37], K)) - k.emit(s_mov_b32(s[38], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[39], scale_k)) - k.emit(s_mov_b32(s[40], scale_k)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(v_readfirstlane_b32_e32(v[46], v[3])) - k.emit(s_waitcnt(49279)) + k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[8], 0)) + k.emit(s_mov_b32(s[9], 0)) + k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) + k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[36], N)) + k.emit(s_mov_b32(s[37], K)) + k.emit(s_mov_b32(s[38], K)) + k.emit(s_mov_b32(s[43], M)) + k.emit(s_mov_b32(s[44], N)) + k.emit(s_mov_b32(s[45], K)) + k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) + k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) + k.emit(s_mov_b32(s[39], scale_k)) + k.emit(s_mov_b32(s[40], scale_k)) + k.emit(v_lshrrev_b32_e32(v[1], 10)) + k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) + k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) + k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) + k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) + k.emit(v_lshrrev_b32_e32(v[3], 6)) + k.emit(v_and_b32_e32(v[0], 63)) + if (tile_m, tile_n) == (256, 256): + k.emit(s_mov_b32(s[49], s[2])) + k.emit(s_mov_b32(s[47], s[3])) + k.emit(v_readfirstlane_b32_e32(v[46], v[3])) + k.emit(s_waitcnt(49279)) + + if (tile_m, tile_n) == (128, 512): for i in range(2): k.emit(s_mov_b32(s[6 + i * 8], -16)) k.emit(s_mov_b32(s[10 + i * 12], -16)) @@ -1213,31 +1218,6 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_waitcnt()) k.emit(s_endpgm()) elif (tile_m, tile_n) == (192, 256): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[36], N)) - k.emit(s_mov_b32(s[37], K)) - k.emit(s_mov_b32(s[38], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[39], scale_k)) - k.emit(s_mov_b32(s[40], scale_k)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(v_readfirstlane_b32_e32(v[46], v[3])) - k.emit(s_waitcnt(49279)) k.emit(s_mul_i32(s[63], LIT, 8, 192)) k.emit(v_cvt_f32_u32_e32(v[4], s[63])) k.emit(s_sub_i32(s[62], 0, s[63])) @@ -2234,49 +2214,22 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_waitcnt()) k.emit(s_endpgm()) elif (tile_m, tile_n) == (256, 256): - k.emit(s_and_b32(s[1], s[1], LIT, 65535)) - k.emit(s_load_dwordx2(s[4:5], s[0:1], s[0], 0, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[8], 0)) - k.emit(s_mov_b32(s[9], 0)) - k.emit(s_load_dwordx2(s[12:13], s[0:1], s[0], 8, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[16:17], s[0:1], s[0], 16, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[40], N)) - k.emit(s_mov_b32(s[41], K)) - k.emit(s_mov_b32(s[42], K)) - k.emit(s_mov_b32(s[43], M)) - k.emit(s_mov_b32(s[44], N)) - k.emit(s_mov_b32(s[45], K)) - k.emit(s_load_dwordx2(s[20:21], s[0:1], s[0], 24, 0, 0, 0, 1)) - k.emit(s_load_dwordx2(s[24:25], s[0:1], s[0], 32, 0, 0, 0, 1)) - k.emit(s_mov_b32(s[36], scale_k)) - k.emit(s_mov_b32(s[37], scale_k)) - k.emit(v_lshrrev_b32_e32(v[1], 10)) - k.emit(v_lshrrev_b32_e32(v[2], 10, v[1])) - k.emit(v_and_b32_e32(v[2], LIT, v[2], 1023)) - k.emit(v_and_b32_e32(v[1], LIT, v[1], 1023)) - k.emit(v_and_b32_e32(v[0], LIT, v[0], 1023)) - k.emit(v_lshrrev_b32_e32(v[3], 6)) - k.emit(v_and_b32_e32(v[0], 63)) - k.emit(s_mov_b32(s[46], s[2])) - k.emit(s_mov_b32(s[47], s[3])) - k.emit(v_readfirstlane_b32_e32(v[49], v[3])) - k.emit(s_waitcnt(49279)) k.emit(s_add_u32(s[55], s[44], LIT, 255)) k.emit(s_lshr_b32(s[54], s[55], 8)) k.emit(s_mul_i32(s[48], s[54], s[47])) - k.emit(s_add_i32(s[48], s[48], s[46])) + k.emit(s_add_i32(s[48], s[48], s[49])) k.emit(s_add_u32(s[55], s[43], LIT, 255)) k.emit(s_lshr_b32(s[52], s[55], 8)) k.emit(s_lshl_b32(s[52], s[52], 5)) - k.emit(s_mov_b32(s[46], 0)) + k.emit(s_mov_b32(s[49], 0)) k.label('L2_00E8') k.emit(s_cmp_lt_i32(s[48], s[52])) k.emit(s_cbranch_scc1(3), target='L2_00FC') k.emit(s_sub_i32(s[48], s[48], s[52])) - k.emit(s_add_i32(s[46], s[46], 32)) + k.emit(s_add_i32(s[49], s[49], 32)) k.emit(s_branch(65531), target='L2_00E8') k.label('L2_00FC') - k.emit(s_sub_i32(s[54], s[54], s[46])) + k.emit(s_sub_i32(s[54], s[54], s[49])) k.emit(s_cmp_lt_i32(s[54], 32)) k.emit(s_cbranch_scc1(3), target='L2_0114') k.emit(s_lshr_b32(s[47], s[48], 5)) @@ -2311,7 +2264,7 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_mul_i32(s[52], s[54], s[47])) k.emit(s_sub_i32(s[52], s[48], s[52])) k.label('L2_0194') - k.emit(s_add_i32(s[46], s[52], s[46])) + k.emit(s_add_i32(s[49], s[52], s[49])) k.emit(s_mov_b32(s[6], -16)) k.emit(s_mov_b32(s[10], -16)) k.emit(s_mov_b32(s[18], -16)) @@ -2328,18 +2281,18 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_or_b32(s[9], s[9], LIT, 262144)) k.emit(s_or_b32(s[17], s[17], LIT, 262144)) k.emit(s_or_b32(s[13], s[13], LIT, 262144)) - k.emit(s_lshr_b32(s[41], s[41], 1)) - k.emit(s_mul_i32(s[52], s[41], s[43])) + k.emit(s_lshr_b32(s[37], s[37], 1)) + k.emit(s_mul_i32(s[52], s[37], s[43])) k.emit(s_mov_b32(s[14], s[52])) - k.emit(s_lshr_b32(s[42], s[42], 1)) - k.emit(s_mul_i32(s[52], s[42], s[44])) + k.emit(s_lshr_b32(s[38], s[38], 1)) + k.emit(s_mul_i32(s[52], s[38], s[44])) k.emit(s_mov_b32(s[18], s[52])) k.emit(s_add_u32(s[52], s[43], 31)) k.emit(s_lshr_b32(s[52], s[52], 5)) k.emit(s_lshl_b32(s[52], s[52], 5)) - k.emit(s_mul_i32(s[53], s[52], s[36])) + k.emit(s_mul_i32(s[53], s[52], s[39])) k.emit(s_mov_b32(s[22], s[53])) - k.emit(s_mul_i32(s[53], s[44], s[37])) + k.emit(s_mul_i32(s[53], s[44], s[40])) k.emit(s_mov_b32(s[26], s[53])) k.emit(s_mov_b32(s[23], LIT, 131072)) k.emit(s_mov_b32(s[27], LIT, 131072)) @@ -2356,23 +2309,23 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[5], v[5], v[6])) k.emit(v_and_b32_e32(v[4], 1, v[4])) k.emit(v_add_u32_e32(v[5], v[5], v[4])) - k.emit(v_mul_lo_u32(v[212], s[41], v[5])) + k.emit(v_mul_lo_u32(v[212], s[37], v[5])) k.emit(v_and_b32_e32(v[4], 7)) k.emit(v_lshlrev_b32_e32(v[4], 4, v[4])) k.emit(v_add_u32_e32(v[212], v[212], v[4])) - k.emit(s_lshr_b32(s[52], s[49], 1)) + k.emit(s_lshr_b32(s[52], s[46], 1)) k.emit(s_mul_i32(s[52], s[52], 8)) - k.emit(s_and_b32(s[53], s[49], 1)) + k.emit(s_and_b32(s[53], s[46], 1)) k.emit(s_mul_i32(s[53], s[53], 2)) k.emit(s_add_u32(s[52], s[52], s[53])) k.emit(s_mul_i32(s[53], s[47], LIT, 256)) k.emit(s_add_u32(s[52], s[52], s[53])) - k.emit(s_mul_i32(s[52], s[41], s[52])) + k.emit(s_mul_i32(s[52], s[37], s[52])) k.emit(v_add_u32_e32(v[212], s[52], v[212])) - k.emit(s_mul_i32(s[52], s[41], 32)) + k.emit(s_mul_i32(s[52], s[37], 32)) for i in range(7): k.emit(v_add_u32_e32(v[213 + i * 1], s[52], v[212 + i * 1])) - k.emit(s_mul_i32(s[59], LIT, s[49], 1056)) + k.emit(s_mul_i32(s[59], LIT, s[46], 1056)) k.emit(s_add_u32(s[59], LIT, s[59], 4096)) k.emit(v_and_b32_e32(v[4], 15)) k.emit(v_lshrrev_b32_e32(v[5], 3, v[4])) @@ -2396,35 +2349,35 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_add_u32_e32(v[221], LIT, v[220], 33792)) k.emit(v_lshlrev_b32_e32(v[222], 2)) k.emit(s_mul_i32(s[52], s[47], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 32)) + k.emit(s_mul_i32(s[53], s[46], 32)) k.emit(s_add_i32(s[52], s[53], s[52])) - k.emit(s_mul_i32(s[53], s[52], s[36])) + k.emit(s_mul_i32(s[53], s[52], s[39])) k.emit(v_add_u32_e32(v[222], s[53], v[222])) - k.emit(s_mul_i32(s[53], LIT, s[36], 128)) + k.emit(s_mul_i32(s[53], LIT, s[39], 128)) k.emit(v_add_u32_e32(v[223], s[53], v[222])) - k.emit(s_mul_i32(s[60], s[49], LIT, 256)) + k.emit(s_mul_i32(s[60], s[46], LIT, 256)) k.emit(s_add_i32(s[60], s[60], 0)) k.emit(v_lshlrev_b32_e32(v[224], 2)) k.emit(v_add_u32_e32(v[224], 0, v[224])) k.emit(v_lshlrev_b32_e32(v[225], 4)) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_u32(s[52], s[52], s[53])) - k.emit(s_mul_i32(s[52], s[52], s[42])) + k.emit(s_mul_i32(s[52], s[52], s[38])) k.emit(v_add_u32_e32(v[225], s[52], v[225])) - k.emit(s_mul_i32(s[52], 16, s[42])) + k.emit(s_mul_i32(s[52], 16, s[38])) k.emit(v_add_u32_e32(v[226], s[52], v[225])) k.emit(v_add_u32_e32(v[227], s[52], v[226])) k.emit(v_add_u32_e32(v[228], s[52], v[227])) for i in range(4): k.emit(v_add_u32_e32(v[229 + i * 1], LIT, v[225 + i * 1], 1024)) k.emit(v_lshlrev_b32_e32(v[233], 2)) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_i32(s[52], s[53], s[52])) - k.emit(s_mul_i32(s[53], s[52], s[37])) + k.emit(s_mul_i32(s[53], s[52], s[40])) k.emit(v_add_u32_e32(v[233], s[53], v[233])) - k.emit(s_mul_i32(s[52], 32, s[37])) + k.emit(s_mul_i32(s[52], 32, s[40])) k.emit(v_add_u32_e32(v[234], s[52], v[233])) k.emit(s_mov_b32(s[61], LIT, 128)) k.emit(s_mov_b32(s[62], LIT, 2048)) @@ -2510,18 +2463,18 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(ds_read_b32(v[201], v[224], v[0], v[0], 0, 0, 1)) k.emit(ds_read_b32(v[202], v[224], v[0], v[0], 0, 0, 2)) k.emit(ds_read_b32(v[203], v[224], v[0], v[0], 0, 0, 3)) - k.emit(s_lshl_b32(s[40], s[40], 1)) + k.emit(s_lshl_b32(s[36], s[36], 1)) k.emit(s_mul_i32(s[52], s[47], LIT, 256)) - k.emit(s_mul_hi_u32(s[53], s[52], s[40])) + k.emit(s_mul_hi_u32(s[53], s[52], s[36])) k.emit(s_add_u32(s[5], s[5], s[53])) - k.emit(s_mul_i32(s[53], s[52], s[40])) + k.emit(s_mul_i32(s[53], s[52], s[36])) k.emit(s_add_u32(s[4], s[4], s[53])) k.emit(s_addc_u32(s[5], 0, s[5])) k.emit(s_sub_i32(s[52], s[43], s[52])) - k.emit(s_mul_i32(s[52], s[52], s[40])) + k.emit(s_mul_i32(s[52], s[52], s[36])) k.emit(s_mov_b32(s[6], s[52])) k.emit(v_and_b32_e64(v[235], v[0], 15)) - k.emit(v_mul_lo_u32(v[235], v[235], s[40])) + k.emit(v_mul_lo_u32(v[235], v[235], s[36])) k.emit(v_lshrrev_b32_e32(v[4], 5)) k.emit(v_mul_i32_i24_e32(v[4], 16, v[4])) k.emit(v_add_u32_e32(v[235], v[4], v[235])) @@ -2529,12 +2482,12 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(v_and_b32_e32(v[4], 1, v[4])) k.emit(v_mul_i32_i24_e32(v[4], 32, v[4])) k.emit(v_add_u32_e32(v[235], v[4], v[235])) - k.emit(s_mul_i32(s[52], s[46], LIT, 256)) - k.emit(s_mul_i32(s[53], s[49], 64)) + k.emit(s_mul_i32(s[52], s[49], LIT, 256)) + k.emit(s_mul_i32(s[53], s[46], 64)) k.emit(s_add_i32(s[52], s[52], s[53])) k.emit(s_lshl_b32(s[52], s[52], 1)) k.emit(v_add_u32_e32(v[235], s[52], v[235])) - k.emit(s_mul_i32(s[53], s[40], 16)) + k.emit(s_mul_i32(s[53], s[36], 16)) for i in range(15): k.emit(v_add_u32_e64(v[236 + i * 1], v[235 + i * 1], s[53])) k.emit(s_mov_b32(s[50], 0)) @@ -2543,7 +2496,7 @@ def build_kernel(M: int, N: int, K: int, tile_m: int, tile_n: int): k.emit(s_cmp_lt_u32(LIT, s[51], 512 + i * -256)) k.emit(s_cselect_b32(s[61 + i * 1], s[61 + i * 1], 0)) k.emit(s_cselect_b32(s[63 + i * 1], s[63 + i * 1], 0)) - k.emit(s_cmp_lt_i32(s[49], 2)) + k.emit(s_cmp_lt_i32(s[46], 2)) k.emit(s_cbranch_scc0(1367), target='L2_25B8') k.label('L2_105C') k.emit(s_waitcnt(122)) From 76dce1eb8d20698cfa8a43c1598efdc44f8aff72 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:29:00 +0300 Subject: [PATCH 251/306] tiny hcq2 changes (#17797) * tiny hcq2 changes * x * x --- test/backend/test_jit.py | 3 ++- test/device/test_hcq2.py | 4 ++-- tinygrad/engine/realize.py | 10 +++++----- tinygrad/helpers.py | 2 +- tinygrad/runtime/ops_amd.py | 4 ++-- tinygrad/runtime/support/hcq.py | 14 +++----------- tinygrad/runtime/support/hcq2.py | 12 ++++++------ tinygrad/runtime/support/memory.py | 13 +++++++++++-- 8 files changed, 32 insertions(+), 30 deletions(-) diff --git a/test/backend/test_jit.py b/test/backend/test_jit.py index 23db717e2638f..a9c34ad678644 100644 --- a/test/backend/test_jit.py +++ b/test/backend/test_jit.py @@ -6,7 +6,7 @@ from test.unit.test_jit import _simple_test from tinygrad import Tensor, Variable, TinyJit, Device, dtypes from tinygrad.engine.jit import graph_class -from tinygrad.helpers import JIT, DEV, GlobalCounters +from tinygrad.helpers import JIT, DEV, GlobalCounters, HCQ2 from tinygrad.uop.ops import Ops from tinygrad.renderer.isa.x86 import X86Renderer @@ -235,6 +235,7 @@ def w2(x) -> Tensor: return (weights*2).contiguous().to(Device.DEFAULT) + x assert_jit_cache_len(w2_prune, 1) class TestJitFree(unittest.TestCase): + @unittest.skipIf(HCQ2, "hcq2 keeps refs to intermediate buffers") def test_free_intermediates(self): ext_tensor = Tensor([1,24,23,45,1]) @TinyJit diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index e312e980d3f74..43c90f93c5dc0 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -3,10 +3,10 @@ from tinygrad import Device, Tensor from tinygrad.device import Buffer from tinygrad.dtype import dtypes -from tinygrad.helpers import getenv +from tinygrad.helpers import HCQ2 from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in -@unittest.skipUnless(getenv("HCQ2") and all_devices_in(Device.DEFAULT, HCQ_DEVS), "hcq2 device required") +@unittest.skipUnless(HCQ2 and all_devices_in(Device.DEFAULT, HCQ_DEVS), "hcq2 device required") class TestHCQ2(unittest.TestCase): def test_copy_without_copy_queue(self): with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 31b3a14b87349..baac566d768ef 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -2,8 +2,8 @@ from typing import cast, Iterator, Any, Sequence import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, getenv, to_tuple, tqdm, dedup -from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, to_tuple, tqdm, dedup +from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, HCQ2, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.dtype import dtypes @@ -305,17 +305,17 @@ def lower_and_compile(linear:UOp) -> UOp: (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="validate", name="ast"),), name="call", allow_any_len=True), exec_validate), ]) -if getenv("HCQ2"): from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize +from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:list[UOp]|None=None, profile:bool|None=None) -> UOp: if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) linear = lower_and_compile(linear) linear = graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) - if getenv("HCQ2"): linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) + if HCQ2: linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) return linear -def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if getenv("HCQ2") else linear +def link_linear(linear:UOp, cache=True) -> UOp: return hcq_link(linear, cache=cache) if HCQ2 else linear def run_linear(linear:UOp, var_vals:dict[str, int]|None=None, input_uops:Sequence[UOp]=(), update_stats=True, jit=False, wait=False): inputs = list(input_uops) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 6b9276a2fae61..b3e7fdded8669 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -240,7 +240,7 @@ def target(self, dev:str, **kwargs) -> Target: SPLIT_REDUCEOP, NO_MEMORY_PLANNER, LRU = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("LRU", 1) RING, ALL2ALL, ALLREDUCE_CAST = ContextVar("RING", 1), ContextVar("ALL2ALL", 0), ContextVar("ALLREDUCE_CAST", 1) CACHELEVEL, IGNORE_BEAM_CACHE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0) -VALIDATE_WITH_CPU = ContextVar("VALIDATE_WITH_CPU", 0) +VALIDATE_WITH_CPU, HCQ2 = ContextVar("VALIDATE_WITH_CPU", 0), ContextVar("HCQ2", 0) # TODO: this is broken for some indexing DISABLE_FAST_IDIV = ContextVar("DISABLE_FAST_IDIV", 1) FUSE_OPTIM = ContextVar("FUSE_OPTIM", 0) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 3a30cb6791e78..b3094e11acdc6 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -8,7 +8,7 @@ from tinygrad.uop.ops import sint from tinygrad.device import Compiled, BufferSpec, TinyELF from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, colored, prod, ContextVar, TracingKey -from tinygrad.helpers import VIZ, ceildiv, unwrap, pluralize +from tinygrad.helpers import VIZ, HCQ2, ceildiv, unwrap, pluralize from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer from tinygrad.runtime.autogen import kfd, hsa, sqtt, amdgpu_kd, amdgpu_drm @@ -1153,4 +1153,4 @@ def device_props(self): return self.iface.props def hw_copy_queues(self): return [(f"SDMA:{i}", functools.partial(unwrap(self.hw_copy_queue_t), queue_idx=i)) for i in self.sdma_queues] -if getenv("HCQ2"): from extra.hcq2.ops_amd2 import * # noqa: F401, F403 # pylint: disable=unused-import +if HCQ2: from extra.hcq2.ops_amd2 import * # noqa: F401, F403 # pylint: disable=unused-import diff --git a/tinygrad/runtime/support/hcq.py b/tinygrad/runtime/support/hcq.py index 1481fe7e0cdbd..14db571e3a1b8 100644 --- a/tinygrad/runtime/support/hcq.py +++ b/tinygrad/runtime/support/hcq.py @@ -1,24 +1,16 @@ from __future__ import annotations from typing import cast, Callable, Type, TypeVar, Generic, Any -import contextlib, decimal, statistics, time, ctypes, array, os, struct, collections, itertools +import contextlib, decimal, statistics, time, ctypes, array, os, collections, itertools try: import fcntl # windows misses that except ImportError: fcntl = None #type:ignore[assignment] -from tinygrad.helpers import DEV, PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, unwrap +from tinygrad.helpers import DEV, PROFILE, getenv, from_mv, cpu_profile, ProfileRangeEvent, unwrap from tinygrad.helpers import suppress_finalizing, pluralize, TracingKey from tinygrad.device import Device, BufferSpec, Compiled, LRUAllocator, ProfileDeviceEvent, ProfileProgramEvent, Program, TinyELF from tinygrad.uop.ops import sym_infer, sint, UOp from tinygrad.runtime.autogen import libc -from tinygrad.runtime.support.memory import BumpAllocator +from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface from tinygrad.renderer import Renderer -class MMIOInterface: - def __init__(self, addr:int, nbytes:int, fmt='B'): self.mv, self.addr, self.nbytes, self.fmt = to_mv(addr, nbytes).cast(fmt), addr, nbytes, fmt - def __len__(self): return self.nbytes // struct.calcsize(self.fmt) - def __getitem__(self, k): return (self.mv[k] if self.fmt == 'B' else self.mv[k].tolist()) if isinstance(k, slice) else self.mv[k] - def __setitem__(self, k, v): self.mv[k] = v - def view(self, offset:int=0, size:int|None=None, fmt=None) -> MMIOInterface: - return MMIOInterface(self.addr+offset, (self.nbytes - offset) if size is None else size, fmt=fmt or self.fmt) - class FileIOInterface: """ Hardware Abstraction Layer for HCQ devices. The class provides a unified interface for interacting with hardware devices. diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 3bc3806b29dee..f91a4b9cec698 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import cast, TypeVar, Generic, Any, Sequence, Iterable +from typing import cast, TypeVar, Generic, Any, Sequence, Iterable, TYPE_CHECKING import struct, functools, time, collections, itertools, decimal, statistics from dataclasses import replace, dataclass, field from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE @@ -9,11 +9,11 @@ from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp from tinygrad.uop.symbolic import symbolic from tinygrad.dtype import dtypes, truncate, DType -from tinygrad.runtime.support.hcq import MMIOInterface, HCQBuffer -from tinygrad.runtime.support.memory import BumpAllocator +from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface from tinygrad.renderer import Renderer, Estimates -from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop -from tinygrad.engine.realize import pm_flatten_linear, lower_and_compile +from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop, pm_flatten_linear,lower_and_compile + +if TYPE_CHECKING: from tinygrad.runtime.support.hcq import HCQBuffer # TODO: remove that # ***************** # 0. helpers @@ -462,7 +462,7 @@ def hcq_lower(linear:UOp, pm_encode:PatternMatcher) -> UOp: linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") # and compile it - return lower_and_compile(graph_rewrite(linear, pm_replace_params, walk=True, name="replace params")) + with Context(EMULATED_DTYPES=""): return lower_and_compile(graph_rewrite(linear, pm_replace_params, walk=True, name="replace params")) @rewrite_group(lambda linear,input_uops,profile,ret: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: diff --git a/tinygrad/runtime/support/memory.py b/tinygrad/runtime/support/memory.py index 30d0fbc5eb84f..317c2e212173e 100644 --- a/tinygrad/runtime/support/memory.py +++ b/tinygrad/runtime/support/memory.py @@ -1,6 +1,15 @@ -import collections, functools, dataclasses, enum +from __future__ import annotations +import collections, functools, dataclasses, enum, struct from typing import Any, ClassVar -from tinygrad.helpers import round_up, getenv +from tinygrad.helpers import round_up, getenv, to_mv + +class MMIOInterface: + def __init__(self, addr:int, nbytes:int, fmt='B'): self.mv, self.addr, self.nbytes, self.fmt = to_mv(addr, nbytes).cast(fmt), addr, nbytes, fmt + def __len__(self): return self.nbytes // struct.calcsize(self.fmt) + def __getitem__(self, k): return (self.mv[k] if self.fmt == 'B' else self.mv[k].tolist()) if isinstance(k, slice) else self.mv[k] + def __setitem__(self, k, v): self.mv[k] = v + def view(self, offset:int=0, size:int|None=None, fmt=None) -> MMIOInterface: + return MMIOInterface(self.addr+offset, (self.nbytes - offset) if size is None else size, fmt=fmt or self.fmt) class BumpAllocator: def __init__(self, size:int, base:int=0, wrap:bool=True): self.size, self.ptr, self.base, self.wrap = size, 0, base, wrap From 55c8e18930cf9f32099e829e5fe858cb1d88ab6a Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:51:55 +0300 Subject: [PATCH 252/306] hcq2 spec2 (#17798) --- test/backend/test_multitensor.py | 7 +++++++ tinygrad/runtime/support/hcq2.py | 4 ++-- tinygrad/uop/spec.py | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/backend/test_multitensor.py b/test/backend/test_multitensor.py index 2501f824bbfb2..360ee267960e2 100644 --- a/test/backend/test_multitensor.py +++ b/test/backend/test_multitensor.py @@ -187,6 +187,13 @@ def test_allreduce_ring_jit(self): a,b = jit_allreduce(Tensor.rand(256, 256)) np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5) + def test_allreduce_all2all_jit(self): + with Context(ALL2ALL=2): + jit_allreduce = TinyJit(_test_allreduce) + for _ in range(5): + a,b = jit_allreduce(Tensor.rand(256, 256)) + np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5) + def test_multitensor_jit_input(self): @TinyJit def f(x): return (x+1).contiguous().sum() diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f91a4b9cec698..fd52192dc478f 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -168,8 +168,8 @@ class BatchCtx: slots:dict[str, int] = field(default_factory=lambda: collections.defaultdict(lambda: next(UOp.unique_num))) def _get_call_bufs_by_lane(call:UOp, devices:tuple[str, ...]) -> list[list[Any]]: - return [[b if (b:=_lane(a, lane)).op is Ops.PARAM or (b.op is Ops.MSELECT and b.src[0].op is Ops.PARAM) else b.buffer - for a in get_call_arg_uops(call)] for lane in range(len(devices))] + def dep_buf(b:UOp) -> Any: return base if (base:=(b.src[0] if b.op is Ops.MSELECT else b).base).op is Ops.PARAM else b.buffer + return [[dep_buf(_lane(a, lane)) for a in get_call_arg_uops(call)] for lane in range(len(devices))] def _wait_ins(ctx:BatchCtx, bufs_by_lane:list[list[Any]], write, devices:tuple[str, ...], queue:str, tag:int) -> list[UOp]: deps:list[Dep] = [] diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 6d0751c5cbefc..ad92bcd92400b 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -228,7 +228,8 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < ])+spec_shared spec_hcq = PatternMatcher([ - (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),), name="x"), lambda x: is_device(x.arg)), + (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.SHRINK, Ops.BITCAST, Ops.MSTACK)).or_after(),), name="x"), + lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), ])+spec_shared From 6083de5cfff4f23cead0b025e64e3e09cc9745fb Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 09:17:18 -0400 Subject: [PATCH 253/306] fix nan compares (#17801) * fix nan compares * fix --- test/backend/test_ops.py | 7 +++++++ tinygrad/renderer/isa/x86.py | 31 ++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 1aba62cb0b62e..3f62a4f8b9944 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -359,6 +359,13 @@ def test_where_permute(self): lambda x: torch.where(x > 0.5, 4, 2).type(torch.int32).permute((1, 0)), lambda x: (x > 0.5).where(4, 2).clone().permute((1, 0)), forward_only=True) + @unittest.skipIf(Device.DEFAULT == "WEBGPU", "software vulkan evaluates a NaN != x as false") + def test_where_nan_cond(self): + # a NaN compares false against everything except !=. + for fxn in (lambda x: x<1, lambda x: x>1, lambda x: x!=1, lambda x: x==1): + helper_test_op(None, lambda x,a,b: torch.where(fxn(x), a, b), lambda x,a,b: fxn(x).where(a, b), forward_only=True, + vals=[[math.nan, 1.0, 2.0, -1.0], [10, 20, 30, 40], [-1, -2, -3, -4]]) + def _test_cmp(self, fxn, reverse=True): # test different dtypes helper_test_op(None, fxn, fxn, forward_only=True, vals=[[0.,1,2], [2.,1,0]]) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 15870a9f6f8b0..a542071a1f786 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -33,7 +33,6 @@ class X86Ops(FastEnum): # bitcasts VMOVD = auto(); VMOVQ = auto(); VMOVDm = auto(); VMOVQm = auto() # comparisons - VUCOMISS = auto(); VUCOMISD = auto() VCMPSS = auto(); VCMPSD = auto(); VCMPPS = auto(); VCMPPD = auto() VPCMPGTB = auto(); VPCMPGTW = auto(); VPCMPGTD = auto(); VPCMPGTQ = auto() VPCMPEQB = auto(); VPCMPEQW = auto(); VPCMPEQD = auto(); VPCMPEQQ = auto() @@ -95,7 +94,7 @@ class X86GroupOp: X86Ops.VPMULLW, X86Ops.VPMULLD, X86Ops.VROUNDSS, X86Ops.VROUNDSD, X86Ops.VSQRTSS, X86Ops.VSQRTSD, X86Ops.VINSERTPS, X86Ops.VPINSRB, X86Ops.VPINSRW, X86Ops.VPINSRD, X86Ops.VPINSRQ, X86Ops.VPAND, X86Ops.VPOR, X86Ops.VPXOR, X86Ops.VPSLLVD, X86Ops.VPSLLVQ, X86Ops.VPSRLVD, X86Ops.VPSRLVQ, X86Ops.VPSRAVD, X86Ops.CMOVNE, X86Ops.CMOVE, X86Ops.CMOVL, X86Ops.CMOVB, - X86Ops.VCVTSI2SS, X86Ops.VCVTSI2SD, X86Ops.VCVTSS2SD, X86Ops.VCVTSD2SS, X86Ops.VUCOMISS, X86Ops.VUCOMISD, X86Ops.IDIV, X86Ops.DIV} + X86Ops.VCVTSI2SS, X86Ops.VCVTSI2SD, X86Ops.VCVTSS2SD, X86Ops.VCVTSD2SS, X86Ops.IDIV, X86Ops.DIV} # X86Ops that can write to memory WriteMem = {X86Ops.MOVm, X86Ops.MOVi, X86Ops.VMOVSSm, X86Ops.VMOVSDm, X86Ops.VMOVUPSm, X86Ops.VMOVDm, X86Ops.VMOVQm, @@ -110,7 +109,7 @@ class X86GroupOp: # X86Ops that write flags or can modify flags to undefined values WriteFlags = {X86Ops.CMP, X86Ops.CMPi, X86Ops.ADD, X86Ops.ADDi, X86Ops.SUB, X86Ops.SUBi, X86Ops.IMUL, X86Ops.IMULi, X86Ops.IDIV, X86Ops.DIV, X86Ops.SHL, X86Ops.SHLi, X86Ops.SHR, X86Ops.SHRi, X86Ops.SAR, X86Ops.SARi, X86Ops.AND, X86Ops.ANDi, X86Ops.XOR, X86Ops.XORi, - X86Ops.OR, X86Ops.ORi, X86Ops.VUCOMISS, X86Ops.VUCOMISD} + X86Ops.OR, X86Ops.ORi} # X86Ops whose first src is the rm field Rm1st = ReadMem1st | (ReadMem2nd & TwoAddress) | {X86Ops.VPSRLDQ} @@ -177,6 +176,12 @@ def gated_store(addr:UOp, gate:UOp, val:UOp): sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.cconst(0, dtypes.int32), dtype=dtypes.uint64)) return UOp(Ops.AFTER, addr.dtype, (sel,)).store(val) +# a gate the flags can be picked with, or the bool compared to zero that replaces one they can't: only an integer +# comparison sets the flags, see cmp. NOTE: the 0 is int so the bool zero-extends and compares as int (a byte compare renders +# different kernels) +def flag_gate(m:UOp) -> UOp|None: + return None if m.op in GroupOp.Comparison and m.src[0].dtype not in dtypes.floats else m.ne(UOp.cconst(0, dtypes.int)) + # legalize the new style graph for isel. NOTE: this runs after the spec is verified, some of these rewrites violate it pre_isel_matcher = PatternMatcher([ # noop casts: zero extending scalar 32bit int, same-width signed/unsigned, narrowing scalar int @@ -191,11 +196,9 @@ def gated_store(addr:UOp, gate:UOp, val:UOp): # gated load/store become a conditional move on the address, the load/store are unconditional (UPat((Ops.INDEX, Ops.SHRINK), name="addr").load(UPat.var("alt"), UPat.var("gate"), name="x"), gated_load), (UPat((Ops.INDEX, Ops.SHRINK), name="addr").store(UPat.var("val"), UPat.var("gate")), gated_store), - # TODO: remove this once we allow all flag producing ops in cmove - # if gate in scalar int cmove is not a comparison need to add one to set the flag - # NOTE: the 0 is int so the bool gate zero-extends and compares as int (a byte compare renders different kernels) - (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), - lambda m,a,b: m.ne(UOp.cconst(0, dtypes.int)).where(a,b) if m.op not in GroupOp.Comparison else None), + # a conditional backedge picks with the flags, and so does the cmove, which is legalized in isel + (UPat(Ops.END, src=(UPat(), UPat(), UPat.var("m", dtypes.bool)), name="x"), + lambda m,x: x.replace(src=x.src[:2]+(g,)) if (g:=flag_gate(m)) is not None else None), ]) # ***** X86 registers ***** @@ -230,9 +233,10 @@ def to_imm(c:UOp) -> UOp|None: if c.dtype in dtypes.int64s: return imm(dtypes.int32, v.val) if not v.overflows(dtypes.int32) else None if c.dtype in dtypes.ints+(dtypes.bool,): return imm(c.dtype, v.val) return None +# the flag path, which only an integer comparison can take: an x86 float compare sets carry, zero and parity together when an +# operand is NaN, so a NaN reads as "below" and as "equal", and it clears sign and overflow, so nothing reads as "less" def cmp(x:UOp) -> UOp: - if x.src[0].dtype is dtypes.float32: return x.ins(X86Ops.VUCOMISS, dtype=dtypes.void) - if x.src[0].dtype is dtypes.float64: return x.ins(X86Ops.VUCOMISD, dtype=dtypes.void) + if x.src[0].dtype in dtypes.floats: raise RuntimeError(f"no flag compare for {x.src[0].dtype}, a float gate must be a mask") return x.ins(X86Ops.CMP, dtype=dtypes.void) if (i:=to_imm(x.src[1])) is None else x.ins(X86Ops.CMPi, dtype=dtypes.void, src=(x.src[0], i)) def vcmp(x:UOp) -> UOp: v = imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) @@ -382,6 +386,9 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(GroupOp.Comparison, dtypes.bool, (UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: UOp(Ops.AND, src=(x.replace(dtype=y.dtype).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).f(Ops.NOOP, dtype=dtypes.bool)), # conditional moves that use flags + # TODO: remove this once we allow all flag producing ops in cmove + # the blends took every float gate a mask can serve, so a gate that is still not an integer comparison becomes one here + (UPat.var("m", dtypes.bool).where(UPat.var("a"), UPat.var("b")), lambda m,a,b: g.where(a, b) if (g:=flag_gate(m)) is not None else None), (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.sints), UPat()), name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVL, src=(b, a, cmp(m)))), (UPat(Ops.CMPLT, name="m").where(UPat.var("a"), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.CMOVB, src=(b, a, cmp(m)))), @@ -695,7 +702,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No if x.arg in X86GroupOp.Rm2nd: if len(x.src) > 4: address, rest = x.src[1:5], x.src[:1] + x.src[5:] else: address, rest = (x.src[1], None, None, None), x.src[:1] + x.src[2:] - # cmp/vucomiss reg, rm don't define a new register + # cmp reg, rm doesn't define a new register return _encode(x, *address, *rest) if x.dtype is not dtypes.void else _encode(rest[0], *address) return None @@ -770,8 +777,6 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No X86Ops.VPSUBB: lambda x: encode(x, 0xF8, pp=1, sel=1), X86Ops.VPSUBW: lambda x: encode(x, 0xF9, pp=1, sel=1), X86Ops.VPSUBD: lambda x: encode(x, 0xFA, pp=1, sel=1), X86Ops.VPSUBQ: lambda x: encode(x, 0xFB, pp=1, sel=1), X86Ops.VPSRAVD: lambda x: encode(x, 0x46, pp=1, sel=2), - # float cmp - X86Ops.VUCOMISS: lambda x: encode(x, 0x2E, pp=0, sel=1), X86Ops.VUCOMISD: lambda x: encode(x, 0x2E, pp=1, sel=1), # scalar / packed float binary X86Ops.VADDSS: lambda x: encode(x, 0x58, pp=2, sel=1), X86Ops.VADDPS: lambda x: encode(x, 0x58, pp=0, sel=1), X86Ops.VADDSD: lambda x: encode(x, 0x58, pp=3, sel=1), X86Ops.VADDPD: lambda x: encode(x, 0x58, pp=1, sel=1), From cb857a5a407b415eb75599e8596b78d5c2fdca7e Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 10:18:10 -0400 Subject: [PATCH 254/306] remove X86 ymm thingy (#17803) --- test/backend/test_encodings.py | 7 ------- tinygrad/dtype.py | 2 -- tinygrad/renderer/isa/x86.py | 7 +++---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/test/backend/test_encodings.py b/test/backend/test_encodings.py index 38e1373beb60c..bd4dd8c53578d 100644 --- a/test/backend/test_encodings.py +++ b/test/backend/test_encodings.py @@ -100,13 +100,6 @@ def test_long_vex_encoding(self): # vaddss xmm0, xmm0, xmm8 self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C4 C1 7A 58 C0")) - # test ymm encoding - def test_ymm_encoding(self): - xmm0, xmm1 = def_reg(dtypes._uint256, XMM[0]), def_reg(dtypes._uint256, XMM[1]) - add = ins(X86Ops.VADDPS, dtypes._uint256, (xmm0, xmm1), XMM[0]) - # vaddps ymm0, ymm0, ymm1 - self.assertEqual(bytes.fromhex(self.encode(add)), bytes.fromhex("C5 FC 58 C1")) - # test encoding where register is in the immediate field def test_reg_in_imm_field(self): xmm0, xmm1, xmm2 = def_reg(dtypes.float32, XMM[0]), def_reg(dtypes.float32, XMM[1]), def_reg(dtypes.float32, XMM[2]) diff --git a/tinygrad/dtype.py b/tinygrad/dtype.py index 31891c5e85adf..8d586d2c045c0 100644 --- a/tinygrad/dtype.py +++ b/tinygrad/dtype.py @@ -120,8 +120,6 @@ def finfo(dtype:DType) -> tuple[int, int]: uint32: Final[DType] = DType.new(6, 32, "unsigned int", 'I') int64: Final[DType] = DType.new(7, 64, "long", 'q') uint64: Final[DType] = DType.new(8, 64, "unsigned long", 'Q') - _uint128: Final[DType] = DType.new(8, 128, "uint128", None) - _uint256: Final[DType] = DType.new(8, 256, "uint256", None) weakfloat: Final[DType] = DType.new(9, 800, "weakfloat", None) fp8e4m3: Final[DType] = DType.new(10, 8, "float8_e4m3", None) fp8e5m2: Final[DType] = DType.new(11, 8, "float8_e5m2", None) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index a542071a1f786..0b67a6da60b33 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -220,7 +220,7 @@ def flag_gate(m:UOp) -> UOp|None: reg_strs = {"rax": {4:"eax", 2:"ax", 1:"al"}, "rcx": {4:"ecx", 2:"cx", 1:"cl"}, "rdx": {4:"edx", 2:"dx", 1:"dl"}, "rbx": {4:"ebx", 2:"bx", 1:"bl"}, "rsp": {4:"esp", 2:"sp", 1:"spl"}, "rbp": {4:"ebp", 2:"bp", 1:"bpl"}, "rsi": {4:"esi", 2:"si", 1:"sil"}, "rdi": {4:"edi", 2:"di", 1:"dil"}, - **{f"r{i}": {4:f"r{i}d", 2:f"r{i}w", 1:f"r{i}b"} for i in range(8, 16)}, **{f"xmm{i}": {64:f"zmm{i}", 32:f"ymm{i}"} for i in range(16)}} + **{f"r{i}": {4:f"r{i}d", 2:f"r{i}w", 1:f"r{i}b"} for i in range(8, 16)}} # ***** X86 instruction selection ***** def base(x:UOp, i:int) -> UOp: return s.src[0] if (s:=x.src[i]).op is Ops.INDEX else s @@ -636,9 +636,8 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No r, _x, b = reg >> 3, idx >> 3, rm >> 3 if sel: # VEX bytes vvvv = cast(Register, greg(vvvv_uop)).index if vvvv_uop is not None else 0 - l = (max(reg_sz, rm_sz) > 16) & 0b1 - if sel == 1 and _x == b == we == 0: inst += bytes([0xC5, (~r & 0b1) << 7 | (~vvvv & 0b1111) << 3 | l << 2 | pp]) - else: inst += bytes([0xC4, (~r & 0b1) << 7 | (~_x & 0b1) << 6 | (~b & 0b1) << 5 | sel, we << 7 | (~vvvv & 0b1111) << 3 | l << 2 | pp]) + if sel == 1 and _x == b == we == 0: inst += bytes([0xC5, (~r & 0b1) << 7 | (~vvvv & 0b1111) << 3 | pp]) + else: inst += bytes([0xC4, (~r & 0b1) << 7 | (~_x & 0b1) << 6 | (~b & 0b1) << 5 | sel, we << 7 | (~vvvv & 0b1111) << 3 | pp]) else: # optional PREFIX and REX bytes # PREFIX byte signaling 16 bit variant of instruction if sz == 2: inst += bytes([0x66]) From 7fdc58b1cc43c10a839e089855a948e7fc2f61c6 Mon Sep 17 00:00:00 2001 From: Joel Lisenby Date: Fri, 28 Aug 2026 09:38:34 -0700 Subject: [PATCH 255/306] add support for UD quants (#17806) --- test/unit/test_gguf.py | 99 ++++++++++++++++++++++++++++++++++++++++-- tinygrad/llm/gguf.py | 71 ++++++++++++++++++++++++++++-- 2 files changed, 163 insertions(+), 7 deletions(-) diff --git a/test/unit/test_gguf.py b/test/unit/test_gguf.py index 8b74d88aa516d..7fcdda8c1f99e 100644 --- a/test/unit/test_gguf.py +++ b/test/unit/test_gguf.py @@ -5,17 +5,33 @@ from tinygrad.runtime.autogen import ggml_common as _ggml import numpy as np from gguf import GGUFReader, GGUFValueType, GGMLQuantizationType, GGML_QUANT_SIZES, dequantize, quantize -from gguf.quants import IQ2_S, IQ3_S, IQ3_XXS +from gguf.quants import IQ1_S, IQ2_S, IQ2_XS, IQ2_XXS, IQ3_S, IQ3_XXS ggml_test_block_count = 4 supported_dtypes = Device[Device.DEFAULT].renderer.supported_dtypes() class TestGGUFTables(unittest.TestCase): + def test_iq2_xxs_grid_matches_gguf_py(self): + IQ2_XXS.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2xxs_grid, (256, 8)).numpy() + np.testing.assert_equal(grid, IQ2_XXS.grid.reshape(256, 8)) + + def test_iq2_xs_grid_matches_gguf_py(self): + IQ2_XS.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2xs_grid, (512, 8)).numpy() + np.testing.assert_equal(grid, IQ2_XS.grid.reshape(512, 8)) + def test_iq2_s_grid_matches_gguf_py(self): IQ2_S.init_grid() grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq2s_grid, (1024, 8)).numpy() np.testing.assert_equal(grid, IQ2_S.grid.reshape(1024, 8)) + def test_iq1_s_grid_matches_gguf_py(self): + IQ1_S.init_grid() + grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq1s_grid, (2048, 8)).numpy() + grid = np.where(grid > 127, grid - 256, grid) + np.testing.assert_equal(grid, IQ1_S.grid.reshape(2048, 8)) + def test_iq3_xxs_grid_matches_gguf_py(self): IQ3_XXS.init_grid() grid = _ggml_iq_grid(Device.DEFAULT, _ggml.iq3xxs_grid, (256, 4)).numpy() @@ -39,6 +55,56 @@ def test_dequantization_q8_0_hardcoded(self): expected = np.arange(1, 33, dtype=np.float32) * 2.0 np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 32, GGMLQuantizationType.Q8_0.value).numpy().flatten(), expected) + def test_dequantization_q2_k_hardcoded(self): + # Q2_K: scales[16] + qs[64] + d(fp16) + dmin(fp16). 16 sub-blocks of 16, x = d*(scale&0xF)*q - dmin*(scale>>4) + scales, qs = bytes([0x11]*16), bytes([0x55]*64) # scale=1, min=1; qs=0x55 -> 2-bit quants of 1 + d, dmin = np.float16(1.0).tobytes(), np.float16(0.0).tobytes() + block = np.frombuffer(scales + qs + d + dmin, dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 10).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_q3_k_hardcoded(self): + # Q3_K: hmask[32] + qs[64] + scales[12] + d(fp16). 16 sub-blocks of 16, x = d * (scale-32) * (q - (hbit?0:4)) + # 6-bit scales 32..47 so (scale-32) = 0..15; qs=0x55 -> 2-bit quants of 1; d=1.0 + scales = bytes([0x80, 0x91, 0xA2, 0xB3, 0xC4, 0xD5, 0xE6, 0xF7, 0xAA, 0xAA, 0xAA, 0xAA]) + d = np.float16(1.0).tobytes() + qs, ones = bytes([0x55]*64), np.ones(16, dtype=np.float32) + # hmask all-ones: high bit set, q=1; hmask zeros: subtract 4, q=-3 + for hmask, q in ((bytes([0xFF]*32), 1.0), (bytes([0x00]*32), -3.0)): + block = np.frombuffer(hmask + qs + scales + d, dtype=np.uint8).copy() + expected = np.concatenate([q * s * ones for s in range(16)]) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 11).numpy().flatten(), expected) + + def test_dequantization_iq2_xxs_hardcoded(self): + # IQ2_XXS: d + 8 groups of (4 grid bytes + uint32 signs/scale). grid[0]=all 0x08, scale=0, signs=0 + # db = 1.0 * (0.5 + 0) * 0.25 = 0.125; 0.125 * 8 = 1.0 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(64), dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 16).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_iq2_xs_hardcoded(self): + # IQ2_XS: d + 32 uint16 qs + 8 scale bytes. qs=0 -> grid[0]=all 0x08, signs=0; scales=0 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(64) + bytes(8), dtype=np.uint8).copy() + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 17).numpy().flatten(), np.ones(256, dtype=np.float32)) + + def test_dequantization_iq1_s_hardcoded(self): + # IQ1_S: d + qs[32] + qh[16]. qs=qh=0 -> grid[0]=all -1, scale=1, delta=+0.125 -> -0.875 + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(48), dtype=np.uint8).copy() + expected = np.full(256, -0.875, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 19).numpy().flatten(), expected) + + def test_dequantization_iq1_m_hardcoded(self): + # IQ1_M: qs[32] + qh[16] + scales[8]. f16 1.0=0x3C00 packed in high nibbles; qs=qh=0 -> -0.875 + scales = bytes([0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x30]) + block = np.frombuffer(bytes(48) + scales, dtype=np.uint8).copy() + expected = np.full(256, -0.875, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 256, 29).numpy().flatten(), expected) + + def test_dequantization_iq4_nl_hardcoded(self): + # IQ4_NL: 2-byte fp16 scale + 16 packed bytes. low nibbles first, then high + lut = list(_ggml.kvalues_iq4nl) + block = np.frombuffer(np.float16(1.0).tobytes() + bytes(range(16)), dtype=np.uint8).copy() + expected = np.array(lut + [lut[0]]*16, dtype=np.float32) + np.testing.assert_equal(ggml_data_to_tensor(Tensor(block), 32, 20).numpy().flatten(), expected) + def test_dequantization_mxfp4_hardcoded(self): # MXFP4: 1 byte shared exponent E + 16 packed bytes (32 x 4-bit values) # nibble: bit3=sign, bit2:1=exp, bit0=mant; E=128 gives scale=1.0 @@ -52,13 +118,20 @@ def test_dequantization_q4_1(self): self._test_dequantization(GGMLQuantizationTy def test_dequantization_q5_0(self): self._test_dequantization(GGMLQuantizationType.Q5_0) def test_dequantization_q5_1(self): self._test_dequantization(GGMLQuantizationType.Q5_1) def test_dequantization_q8_0(self): self._test_dequantization(GGMLQuantizationType.Q8_0) + def test_dequantization_q2_k(self): self._test_dequantization(GGMLQuantizationType.Q2_K) + def test_dequantization_q3_k(self): self._test_dequantization(GGMLQuantizationType.Q3_K) def test_dequantization_q4_k(self): self._test_dequantization(GGMLQuantizationType.Q4_K) def test_dequantization_q5_k(self): self._test_dequantization(GGMLQuantizationType.Q5_K) def test_dequantization_q6_k(self): self._test_dequantization(GGMLQuantizationType.Q6_K) + def test_dequantization_iq2_xxs(self): self._test_dequantization(GGMLQuantizationType.IQ2_XXS) + def test_dequantization_iq2_xs(self): self._test_dequantization(GGMLQuantizationType.IQ2_XS) def test_dequantization_iq3_xxs(self): self._test_dequantization(GGMLQuantizationType.IQ3_XXS) + def test_dequantization_iq1_s(self): self._test_dequantization(GGMLQuantizationType.IQ1_S) + def test_dequantization_iq4_nl(self): self._test_dequantization(GGMLQuantizationType.IQ4_NL) def test_dequantization_iq3_s(self): self._test_dequantization(GGMLQuantizationType.IQ3_S) def test_dequantization_iq2_s(self): self._test_dequantization(GGMLQuantizationType.IQ2_S) def test_dequantization_iq4_xs(self): self._test_dequantization(GGMLQuantizationType.IQ4_XS) + def test_dequantization_iq1_m(self): self._test_dequantization(GGMLQuantizationType.IQ1_M) def test_dequantization_mxfp4(self): self._test_dequantization(GGMLQuantizationType.MXFP4) @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, "Backend must support bfloat16") def test_dequantization_bf16(self): self._test_dequantization(GGMLQuantizationType.BF16) @@ -203,12 +276,23 @@ def _test_gguf_gemv(self, qtype: GGMLQuantizationType): q_data = rng.integers(0, 256, size=n_blocks * type_size, dtype=np.uint8).reshape(n_blocks, type_size) scales = np.float16(rng.standard_normal(n_blocks * 4)).view(np.uint8).reshape(n_blocks, -1) if qtype in (GGMLQuantizationType.Q5_0, GGMLQuantizationType.Q8_0, - GGMLQuantizationType.IQ3_XXS, - GGMLQuantizationType.IQ2_S, + GGMLQuantizationType.IQ2_XXS, GGMLQuantizationType.IQ2_XS, + GGMLQuantizationType.IQ3_XXS, GGMLQuantizationType.IQ4_NL, + GGMLQuantizationType.IQ1_S, GGMLQuantizationType.IQ2_S, GGMLQuantizationType.IQ3_S, GGMLQuantizationType.IQ4_XS): q_data[:, :2] = scales[:, :2] # d at offset 0 elif qtype in (GGMLQuantizationType.Q5_1, GGMLQuantizationType.Q4_K, GGMLQuantizationType.Q5_K): q_data[:, :4] = scales[:, :4] # d, m/dmin at offset 0 - elif qtype == GGMLQuantizationType.Q6_K: q_data[:, -2:] = scales[:, :2] # d at end + elif qtype == GGMLQuantizationType.Q2_K: q_data[:, -4:] = scales[:, :4] # d, dmin at end + elif qtype in (GGMLQuantizationType.Q6_K, GGMLQuantizationType.Q3_K): q_data[:, -2:] = scales[:, :2] # d at end + elif qtype == GGMLQuantizationType.IQ1_M: + s = np.float16(rng.standard_normal(n_blocks)).view(np.uint16) + sc = q_data[:, -8:].copy().view(np.uint16).reshape(n_blocks, 4) + sc &= np.uint16(0x0FFF) + sc[:, 0] |= (s & np.uint16(0x000F)) << 12 + sc[:, 1] |= (s & np.uint16(0x00F0)) << 8 + sc[:, 2] |= (s & np.uint16(0x0F00)) << 4 + sc[:, 3] |= (s & np.uint16(0xF000)) + q_data[:, -8:] = sc.reshape(n_blocks, -1).view(np.uint8) elif qtype == GGMLQuantizationType.MXFP4: q_data[:, 0] = rng.integers(120, 136, size=n_blocks, dtype=np.uint8) # constrain byte0 q_data = q_data.flatten() ref = dequantize(q_data, qtype).reshape(rows, cols) @@ -235,13 +319,20 @@ def _test_gguf_gemv(self, qtype: GGMLQuantizationType): def test_gguf_gemv_q8_0(self): self._test_gguf_gemv(GGMLQuantizationType.Q8_0) def test_gguf_gemv_q5_0(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_0) def test_gguf_gemv_q5_1(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_1) + def test_gguf_gemv_q2_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q2_K) + def test_gguf_gemv_q3_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q3_K) def test_gguf_gemv_q4_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q4_K) def test_gguf_gemv_q5_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q5_K) def test_gguf_gemv_q6_k(self): self._test_gguf_gemv(GGMLQuantizationType.Q6_K) + def test_gguf_gemv_iq2_xxs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_XXS) + def test_gguf_gemv_iq2_xs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_XS) def test_gguf_gemv_iq3_xxs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ3_XXS) + def test_gguf_gemv_iq1_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ1_S) + def test_gguf_gemv_iq4_nl(self): self._test_gguf_gemv(GGMLQuantizationType.IQ4_NL) def test_gguf_gemv_iq3_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ3_S) def test_gguf_gemv_iq2_s(self): self._test_gguf_gemv(GGMLQuantizationType.IQ2_S) def test_gguf_gemv_iq4_xs(self): self._test_gguf_gemv(GGMLQuantizationType.IQ4_XS) + def test_gguf_gemv_iq1_m(self): self._test_gguf_gemv(GGMLQuantizationType.IQ1_M) def test_gguf_gemv_mxfp4(self): self._test_gguf_gemv(GGMLQuantizationType.MXFP4) @unittest.skipUnless(dtypes.bfloat16 in supported_dtypes, "Backend must support bfloat16") def test_gguf_gemv_bf16(self): self._test_gguf_gemv(GGMLQuantizationType.BF16) diff --git a/tinygrad/llm/gguf.py b/tinygrad/llm/gguf.py index 49772dabf1dd2..60f937d3ab3da 100644 --- a/tinygrad/llm/gguf.py +++ b/tinygrad/llm/gguf.py @@ -18,7 +18,9 @@ def _ggml_iq_grid(device: str, grid: tuple[int, ...], grid_shape: tuple[int, int # quant types {ggml_type: (number of elements, number of bytes)} _GGML_QUANT = {2:(32,18), 3:(32,20), 6:(32,22), 7:(32,24), 8:(32,34), - 12:(256,144), 13:(256,176), 14:(256,210), 18:(256,98), 21:(256,110), 22:(256,82), 23:(256,136), 39:(32,17), 41:(128,18)} + 10:(256,84), 11:(256,110), 12:(256,144), 13:(256,176), 14:(256,210), + 16:(256,66), 17:(256,74), 18:(256,98), 19:(256,50), 20:(32,18), 21:(256,110), 22:(256,82), 23:(256,136), + 29:(256,56), 39:(32,17), 41:(128,18)} def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: """ @@ -27,8 +29,9 @@ def ggml_data_to_tensor(t: Tensor, n: int, ggml_type: int) -> Tensor: Supported native types: float32 (id: 0), float16 (id: 1), int8 (id: 24), int16 (id: 25), int32 (id: 26), int64 (id: 27), float64 (id: 28), bfloat16 (id: 30) Supported quantized types: Q4_0 (id: 2), Q4_1 (id: 3), Q5_0 (id: 6), - Q5_1 (id: 7), Q8_0 (id: 8), Q4_K (id: 12), Q5_K (id: 13), - Q6_K (id: 14), IQ3_XXS (id: 18), IQ3_S (id: 21), IQ2_S (id: 22), IQ4_XS (id: 23), MXFP4 (id: 39), Q1_0 (id: 41) + Q5_1 (id: 7), Q8_0 (id: 8), Q2_K (id: 10), Q3_K (id: 11), Q4_K (id: 12), Q5_K (id: 13), + Q6_K (id: 14), IQ2_XXS (id: 16), IQ2_XS (id: 17), IQ3_XXS (id: 18), IQ1_S (id: 19), + IQ4_NL (id: 20), IQ3_S (id: 21), IQ2_S (id: 22), IQ4_XS (id: 23), IQ1_M (id: 29), MXFP4 (id: 39), Q1_0 (id: 41) """ # https://github.com/ggerganov/ggml/blob/323951f1bdcdfbd5b5ff3a9a7c3770e63b1a560e/include/ggml.h#L356 @@ -54,6 +57,19 @@ def q_to_uint8(t: Tensor, b: int) -> Tensor: q = q_to_uint8(blocks[:,qh_off+4:], 4).bitcast(dtypes.int8) + qh * 16 return q * d + (blocks[:,2:4].bitcast(dtypes.float16).cast(dtypes.float32) if ggml_type == 7 else -16 * d) if ggml_type == 8: return blocks[:,:2].bitcast(dtypes.float16).cast(dtypes.float32) * blocks[:,2:].bitcast(dtypes.int8) + # Q2_K: 256 elements per 84-byte block (scales:16, qs:64, d:2, dmin:2) + if ggml_type == 10: + d, dmin = (blocks[:,i:i+2].bitcast(dtypes.float16).cast(dtypes.float32).unsqueeze(-1) for i in [80, 82]) + sc = blocks[:, :16] + q = q_to_uint8(blocks[:, 16:80].reshape((-1, 2, 32)), 2).reshape((-1, 16, 16)) + return (d * sc.bitwise_and(0xF).unsqueeze(-1) * q - dmin * sc.rshift(4).unsqueeze(-1)).flatten(-2) + # Q3_K: 256 elements per 110-byte block (hmask:32, qs:64, scales:12, d:2) + if ggml_type == 11: + d = blocks[:,-2:].bitcast(dtypes.float16).cast(dtypes.float32).unsqueeze(-1) + sc = q_to_uint8(blocks[:,96:104], 4).bitwise_or(q_to_uint8(blocks[:,104:108], 2).lshift(4)).bitcast(dtypes.int8) - 32 + q = q_to_uint8(blocks[:,32:96].reshape((-1, 2, 32)), 2).reshape((-1, 16, 16)) + qh = q_to_uint8(blocks[:,:32], 1).reshape((-1, 16, 16)) + return (d * sc.unsqueeze(-1) * (q.bitcast(dtypes.int8) - qh.bitwise_xor(1).lshift(2).bitcast(dtypes.int8))).flatten(-2) # Q4_K: 256 elements per 144-byte block (d:2, dmin:2, scales:12, qs:128) # Q5_K: 256 elements per 176-byte block (d:2, dmin:2, scales:12, qh:32, qs:128) if ggml_type in (12, 13): @@ -79,6 +95,41 @@ def q_to_uint8(t: Tensor, b: int) -> Tensor: signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8)) grid = _ggml_iq_grid(t.device, _ggml.iq3xxs_grid, (256, 4))[blocks[:, 2:66]].reshape((-1, 8, 4, 8)) return (db * grid * signs).flatten(-3) + # IQ2_XXS: 256 elements per 66-byte block (d:2, qs:64). 8 groups of 32: 4 grid bytes + packed signs/scale. + if ggml_type == 16: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + qs_u32 = blocks[:, 2:].bitcast(dtypes.uint32).reshape((-1, 8, 2)) + db = d * (qs_u32[:, :, 1].rshift(28).cast(dtypes.float32) + 0.5).reshape((-1, 8, 1, 1)) * 0.25 + sign_idx = qs_u32[:, :, 1].unsqueeze(-1).rshift(Tensor.const((0, 7, 14, 21), dtypes.uint32)) + sign_idx = sign_idx.bitwise_and(0x7F).reshape((-1, 32)).cast(dtypes.int32) + even_signs = Tensor([i | (0x80 if i.bit_count() % 2 else 0) for i in range(128)], dtype=dtypes.uint8, device=t.device) + signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 8, 4, 8)) + grid = _ggml_iq_grid(t.device, _ggml.iq2xxs_grid, (256, 8))[blocks[:, 2:].reshape((-1, 8, 8))[:, :, :4]].reshape((-1, 8, 4, 8)) + return (db * grid * signs).flatten(-3) + # IQ2_XS: 256 elements per 74-byte block (d:2, qs:64 as uint16, scales:8) + if ggml_type == 17: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + db = d * (q_to_uint8(blocks[:, 66:74].reshape((-1, 8, 1)), 4).reshape((-1, 16)).cast(dtypes.float32) + 0.5).reshape((-1, 16, 1, 1)) * 0.25 + qs = blocks[:, 2:66].bitcast(dtypes.uint16) + sign_idx = qs.rshift(9).cast(dtypes.int32) + even_signs = Tensor([i | (0x80 if i.bit_count() % 2 else 0) for i in range(128)], dtype=dtypes.uint8, device=t.device) + signs = (q_to_uint8(even_signs[sign_idx].reshape((-1, 32, 1)), 1) == 0).where(1.0, -1.0).reshape((-1, 16, 2, 8)) + grid = _ggml_iq_grid(t.device, _ggml.iq2xs_grid, (512, 8))[qs.bitwise_and(511)].reshape((-1, 16, 2, 8)) + return (db * grid * signs).flatten(-3) + # IQ1_S: 256 elements per 50-byte block (d:2, qs:32, qh:16). grid bytes are int8 {-1,0,1}. + if ggml_type == 19: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) + qh = blocks[:, 34:50].bitcast(dtypes.uint16) + dl = d * (qh.rshift(12).bitwise_and(7).cast(dtypes.float32) * 2 + 1).reshape((-1, 8, 1, 1)) + delta = (qh.bitwise_and(0x8000) == 0).where(0.125, -0.125).reshape((-1, 8, 1, 1)) + qh_hi = qh.unsqueeze(-1).rshift(Tensor.const((0, 3, 6, 9), dtypes.uint16)).bitwise_and(7).lshift(8) + q = blocks[:, 2:34].cast(dtypes.uint16) + qh_hi.reshape((-1, 32)) + grid = _ggml_iq_grid(t.device, _ggml.iq1s_grid, (2048, 8))[q].reshape((-1, 8, 4, 8)) + grid = (grid > 127).where(grid - 256, grid) + return (dl * (grid + delta)).flatten(-3) + if ggml_type == 20: + d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32) + return d * Tensor(list(_ggml.kvalues_iq4nl), dtype=dtypes.float32, device=t.device)[q_to_uint8(blocks[:, 2:], 4)] if ggml_type == 21: d = blocks[:, :2].bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1)) scales = (1 + 2 * q_to_uint8(blocks[:, 106:110].reshape((-1, 4, 1)), 4).reshape((-1, 8))).cast(dtypes.float32).reshape((-1, 8, 1, 1)) @@ -102,6 +153,20 @@ def q_to_uint8(t: Tensor, b: int) -> Tensor: scales = (scales_l.bitwise_or(scales_h.lshift(4)).bitcast(dtypes.int8) - 32).cast(dtypes.float32).reshape((-1, 8, 1)) q = (qs:=blocks[:, 8:].reshape((-1, 8, 16))).bitwise_and(0xF).cat(qs.rshift(4), dim=2) return (d * scales * iq4_xs_lut[q]).flatten(-2) + # IQ1_M: 256 elements per 56-byte block (qs:32, qh:16, scales:8). f16 scale packed in high nibbles. + if ggml_type == 29: + sc16 = blocks[:, 48:56].bitcast(dtypes.uint16) + d = sc16.bitwise_and(0xF000).rshift(Tensor.const((12, 8, 4, 0), dtypes.uint16)) + d = d[:, 0:1].bitwise_or(d[:, 1:2]).bitwise_or(d[:, 2:3]).bitwise_or(d[:, 3:4]) + d = d.bitcast(dtypes.float16).cast(dtypes.float32).reshape((-1, 1, 1, 1, 1)) + scales = sc16.unsqueeze(-1).rshift(Tensor.const((0, 3, 6, 9), dtypes.uint16)).bitwise_and(7) + dl = d * (scales.cast(dtypes.float32) * 2 + 1).reshape((-1, 8, 2, 1, 1)) + qh_n = Tensor.stack(blocks[:, 32:48].bitwise_and(0x0F), blocks[:, 32:48].rshift(4), dim=-1).reshape((-1, 32)) + q = blocks[:, :32].cast(dtypes.uint16) + qh_n.bitwise_and(7).cast(dtypes.uint16).lshift(8) + delta = (qh_n.bitwise_and(0x08) == 0).where(0.125, -0.125).reshape((-1, 8, 2, 2, 1)) + grid = _ggml_iq_grid(t.device, _ggml.iq1s_grid, (2048, 8))[q].reshape((-1, 8, 2, 2, 8)) + grid = (grid > 127).where(grid - 256, grid) + return (dl * (grid + delta)).flatten(-4) if ggml_type == 39: e = blocks[:, 0].cast(dtypes.uint32) small_bits = Tensor([0x00200000, 0x00400000], dtype=dtypes.uint32, device=t.device)[e.clip(0, 1).cast(dtypes.int32)] # e = 0 or e = 1 case From 2eacd4fa68c42c1f092ffcacac0cd0ac89930919 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Fri, 28 Aug 2026 12:44:43 -0400 Subject: [PATCH 256/306] fix Conv2d same padding with per axis dilation (#17805) --- test/backend/test_nn.py | 2 +- tinygrad/nn/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/backend/test_nn.py b/test/backend/test_nn.py index 75195d88e0a91..c12ec91266a6c 100644 --- a/test/backend/test_nn.py +++ b/test/backend/test_nn.py @@ -135,7 +135,7 @@ def test_conv2d_same_padding_odd_input(self): def test_conv2d_same_padding_large_kernel(self): self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=16, DIMS=[28, 33], C2=32, K=9, S=1, P='same') def test_conv2d_same_padding_with_dilation(self): - self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=3, DIMS=[28, 28], C2=32, K=3, S=1, P='same', D=3) + self._test_conv(Conv2d, torch.nn.Conv2d, BS=16, C1=3, DIMS=[28, 31], C2=32, K=(3,5), S=1, P='same', D=(2,3)) def test_conv2d_same_padding_invalid_stride(self): self.assertRaises(ValueError, Conv2d, in_channels=16, out_channels=32, kernel_size=2, stride=2, padding='same') diff --git a/tinygrad/nn/__init__.py b/tinygrad/nn/__init__.py index 9b663935a457f..a74a4c295fb3d 100644 --- a/tinygrad/nn/__init__.py +++ b/tinygrad/nn/__init__.py @@ -99,7 +99,7 @@ def __init__(self, in_channels:int, out_channels:int, kernel_size:int|tuple[int, if isinstance(padding, str): if padding.lower() != 'same': raise ValueError(f"Invalid padding string {padding!r}, only 'same' is supported") if stride != 1: raise ValueError("padding='same' is not supported for strided convolutions") - pad = [(d*(k-1)//2, d*(k-1) - d*(k-1)//2) for d,k in zip(make_tuple(dilation, len(self.kernel_size)), self.kernel_size[::-1])] + pad = [(d*(k-1)//2, d*(k-1) - d*(k-1)//2) for d,k in zip(make_tuple(dilation, len(self.kernel_size))[::-1], self.kernel_size[::-1])] padding = tuple(flatten(pad)) self.stride, self.dilation, self.groups, self.padding = stride, dilation, groups, padding scale = 1 / math.sqrt(in_channels * prod(self.kernel_size)) From 287679a88af09f41d9686e7753a0d6f7fd42a1ba Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 12:50:50 -0400 Subject: [PATCH 257/306] delete X86 not used codes [PR] (#17807) --- tinygrad/renderer/isa/x86.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 0b67a6da60b33..f359ae253f7c7 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -117,8 +117,6 @@ class X86GroupOp: # X86Ops whose second src is the rm field Rm2nd = ReadMem2nd - All = set(X86Ops) - # ***** X86 legalization ***** extra_matcher = PatternMatcher([ @@ -354,8 +352,6 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: isel_matcher = PatternMatcher([ # **** Op -> Op **** - # cast of void is a noop - (UPat.var("y").cast(name="x"), lambda y,x: y if y.dtype == dtypes.void else None), # range is lowered to acc, cmp, jmp after regalloc (UPat(Ops.RANGE, src=(UPat.cvar("c").cast(),), allow_any_len=True, name="x"), lambda c,x: x.replace(src=(imm(x.dtype, c.val),) + x.src[1:])), (UPat(Ops.RANGE, name="x"), lambda ctx,x: x.replace(tag=(ctx.vreg(WGPR),)) if not isinstance(x.tag, tuple) else None), From ae63f2355770c14c9a391293b228a2b668bb0bb8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 13:13:47 -0400 Subject: [PATCH 258/306] fix onnx MaxUnpool (#17809) pads can be tuple --- test/external/external_test_onnx_ops.py | 9 +++++++++ tinygrad/nn/onnx.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/external/external_test_onnx_ops.py b/test/external/external_test_onnx_ops.py index e3ac1fe6995d7..55256f153ae5f 100644 --- a/test/external/external_test_onnx_ops.py +++ b/test/external/external_test_onnx_ops.py @@ -235,6 +235,15 @@ def test_maxunpool_export_with_output_shape(self): outputs = ["y"] self.helper_test_single_op("MaxUnpool", inputs, attributes, outputs) + def test_maxunpool_pads(self): + # per-axis pads shrink the output: spatial dim is (i-1)*stride + kernel - pad_begin - pad_end -> (2, 4), and indices index into that output + # NOTE: indices must be in bounds of that output; ORT aborts the process on out-of-bounds indices + xT = np.array([[[[5, 6], [7, 8]]]], dtype=np.float32) + xI = np.array([[[[0, 3], [4, 7]]]], dtype=np.int64) + inputs = {"x": xT, "indices": xI} + attributes = {"kernel_shape": [2, 2], "strides": [2, 2], "pads": [1, 0, 1, 0]} + self.helper_test_single_op("MaxUnpool", inputs, attributes, ["y"]) + def test_averagepool_3d_dilations_large_count_include_pad_is_1_ceil_mode_is_True(self): # https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-13 inputs = {"x": np.random.randn(1, 1, 32, 32, 32).astype(np.float32)} diff --git a/tinygrad/nn/onnx.py b/tinygrad/nn/onnx.py index a8a19f077454f..4083e3be7550f 100644 --- a/tinygrad/nn/onnx.py +++ b/tinygrad/nn/onnx.py @@ -750,7 +750,7 @@ def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OP def MaxUnpool(xT: Tensor, xI: Tensor, outshape: list[int]|None=None, kernel_shape:list[int]|None=None, pads:list[int]|int=0, strides:list[int]|int=1): if kernel_shape is None: kernel_shape = [] - pads_: int | tuple[int, ...] = tuple(pads) if isinstance(pads, list) else pads + pads_: int | tuple[int, ...] = pads if isinstance(pads, int) else _onnx_pads_to_tiny_pads(pads) return Tensor.max_unpool2d(xT, xI, tuple(kernel_shape), strides, 1, pads_, outshape if outshape is None else tuple(outshape)) def GlobalAveragePool(X:Tensor): return X.mean(axis=tuple(range(2, X.ndim)), keepdim=True) From 112e50ec3a1a2f740e86ff2f5e70dc38abbcc8f0 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 13:34:43 -0400 Subject: [PATCH 259/306] fix onnx.py type annotations (#17811) --- tinygrad/nn/onnx.py | 76 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tinygrad/nn/onnx.py b/tinygrad/nn/onnx.py index 4083e3be7550f..7723e74b3e1bf 100644 --- a/tinygrad/nn/onnx.py +++ b/tinygrad/nn/onnx.py @@ -556,8 +556,8 @@ def run_branch(branch:OnnxRunner): return tuple(t if cond else e for t,e in zip(then_out.values(), else_out.values())) def Identity(x:Tensor): return x - def Constant(sparse_value:Tensor|None=None, value:Tensor|None=None, value_float:float|None=None, value_floats:list[float]|None=None, - value_int:int|None=None, value_ints:list[int]|None=None, value_string:str|None=None, value_strings:list[str]|None=None): + def Constant(sparse_value:Tensor|None=None, value:Tensor|None=None, value_float:float|None=None, value_floats:tuple[float, ...]|None=None, + value_int:int|None=None, value_ints:tuple[int, ...]|None=None, value_string:str|None=None, value_strings:tuple[str, ...]|None=None): if value is not None: return value if value_float is not None: return Tensor(value_float, dtype=dtypes.float32) if value_floats is not None: return Tensor(list(value_floats), dtype=dtypes.float32) @@ -594,7 +594,7 @@ def Shape(data:Tensor, end:int|None=None, start:int=0): return Tensor(data.shape # ***** Unary Ops (math) ***** def Not(x:Tensor): return x.logical_not() - def Clip(x: Tensor, min:Tensor|None=None, max:Tensor|None=None): return x if min is None and max is None else x.clip(min, max) # noqa: A002 # pylint: disable=redefined-builtin + def Clip(x: Tensor, min:Tensor|float|None=None, max:Tensor|float|None=None): return x if min is None and max is None else x.clip(min, max) # noqa: A002 # pylint: disable=redefined-builtin def IsInf(x:Tensor, detect_negative:int=1, detect_positive:int=1): return x.isinf(bool(detect_positive), bool(detect_negative)) # ***** Unary Ops (activation) ***** @@ -643,26 +643,26 @@ def Max(*data_0:Tensor): return functools.reduce(Tensor.maximum, data_0) def Min(*data_0:Tensor): return functools.reduce(Tensor.minimum, data_0) def Sum(*data_0:Tensor): return functools.reduce(Tensor.add, data_0) def Mean(*data_0:Tensor): return Sum(*data_0) / len(data_0) - def ReduceMax(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMax(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.max(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceMin(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMin(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.min(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceSum(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceSum(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.sum(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceMean(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceMean(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.mean(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceSumSquare(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceSumSquare(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.square(), axes, keepdims, noop_with_empty_axes) - def ReduceProd(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceProd(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return data.prod(_axes(axes, noop_with_empty_axes), keepdim=keepdims) - def ReduceL1(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceL1(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.abs(), axes, keepdims, noop_with_empty_axes) - def ReduceL2(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceL2(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): dtype = dtypes.float if data.dtype in (dtypes.float16, dtypes.bfloat16) else data.dtype return ReduceSum(data.cast(dtype).square(), axes, keepdims, noop_with_empty_axes).sqrt().cast(data.dtype) - def ReduceLogSum(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceLogSum(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data, axes, keepdims, noop_with_empty_axes).log() - def ReduceLogSumExp(data:Tensor, axes:list[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): + def ReduceLogSumExp(data:Tensor, axes:Sequence[int]|None=None, keepdims:int=1, noop_with_empty_axes:int=0): return ReduceSum(data.exp(), axes, keepdims, noop_with_empty_axes).log() def ArgMax(x:Tensor, axis:int=0, keepdims:int=1, select_last_index:int=0): if select_last_index: return ((int(x.shape[axis])-1) - x.flip(axis).argmax(axis, keepdim=keepdims)).cast(dtypes.int64) @@ -671,32 +671,32 @@ def ArgMin(x, axis:int=0, keepdims:int=1, select_last_index:int=0): return ArgMax(-x, axis=axis, keepdims=keepdims, select_last_index=select_last_index) # ***** Movement Ops ***** - def Reshape(data:Tensor, shape:list[int], allowzero:int=0): + def Reshape(data:Tensor, shape:Sequence[int], allowzero:int=0): return data.reshape([x if x != 0 else (0 if allowzero else data.shape[i]) for i,x in enumerate(shape)]) def Flatten(x:Tensor, axis:int=1): return x.reshape(prod(x.shape[0:axis]), -1) def Expand(x:Tensor, shape:list[int]): return x.expand(_broadcast_shape(x.shape, tuple(shape))) def Shrink(x:Tensor, bias:float=0.0, lambd:float=0.5): return (x < -lambd)*(x+bias) + (x > lambd)*(x-bias) - def Transpose(x:Tensor, perm:list[int]|None=None): return x.permute(order=perm or list(range(x.ndim)[::-1])) + def Transpose(x:Tensor, perm:tuple[int, ...]|None=None): return x.permute(order=perm or list(range(x.ndim)[::-1])) - def Squeeze(data:Tensor, axes:list[int]|None=None): + def Squeeze(data:Tensor, axes:Sequence[int]|None=None): return data.squeeze() if axes is None else functools.reduce(lambda d, dim: d.squeeze(dim), sorted(axes, reverse=True), data) - def Unsqueeze(data:Tensor, axes:list[int]): return functools.reduce(lambda d, dim: d.unsqueeze(dim), sorted(axes), data) + def Unsqueeze(data:Tensor, axes:Sequence[int]): return functools.reduce(lambda d, dim: d.unsqueeze(dim), sorted(axes), data) def Tile(x:Tensor, repeats:list[int]): return x.repeat(repeats) def Concat(*xs:Tensor, axis:int): return Tensor.cat(*xs, dim=axis) - def Slice(data:Tensor, starts:list[int], ends:list[int], axes:list[int]|None=None, steps:list[int]|None=None): + def Slice(data:Tensor, starts:Sequence[int], ends:Sequence[int], axes:Sequence[int]|None=None, steps:list[int]|None=None): axes = axes or list(range(data.ndim)) steps = steps or [1] * data.ndim slices = [slice(None)] * data.ndim for i, axis in enumerate(axes): slices[axis] = slice(starts[i], ends[i], steps[i]) return data[tuple(slices)] - def Split(data:Tensor, split:list[int]|None=None, num_outputs:int=0, axis:int=0): + def Split(data:Tensor, split:Sequence[int]|None=None, num_outputs:int=0, axis:int=0): sz = int(data.shape[axis]) if split is None: split = [sz // num_outputs + (1 if i < sz % num_outputs else 0) for i in range(num_outputs)] return data.split(split, axis) - def Pad(x:Tensor, pads:list[int], constant_value:ConstType|None=None, axes:list[int]|None=None, + def Pad(x:Tensor, pads:Sequence[int], constant_value:ConstType|None=None, axes:list[int]|None=None, mode:Literal["constant", "reflect", "edge", "wrap"]="constant", value=0): value = _resolve_const(value if constant_value is None else constant_value) axes = axes or list(range(x.ndim)) @@ -704,7 +704,7 @@ def Pad(x:Tensor, pads:list[int], constant_value:ConstType|None=None, axes:list[ for i,axis in enumerate(axes): real_pads[axis%x.ndim], real_pads[axis%x.ndim+x.ndim] = pads[i], pads[i+len(axes)] return x.pad(padding=_onnx_pads_to_tiny_pads(real_pads), mode={"edge":"replicate", "wrap":"circular"}.get(mode, mode), value=value) - def CenterCropPad(t:Tensor, shape:list[int], axes:list[int]|None=None): + def CenterCropPad(t:Tensor, shape:list[int], axes:tuple[int, ...]|None=None): shrink_arg:list[None|tuple[sint,sint]] = [None] * t.ndim pad_arg:list[None|tuple[sint,sint]] = [None] * t.ndim for s, x in zip(shape, axes or range(t.ndim)): @@ -714,26 +714,26 @@ def CenterCropPad(t:Tensor, shape:list[int], axes:list[int]|None=None): return t.shrink(tuple(shrink_arg)).pad(tuple(pad_arg)) # ***** Processing Ops ***** - def AveragePool(X: Tensor, kernel_shape:list[int], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, count_include_pad:int=0, - dilations:list[int]|int=1, pads:list[int]|int=0, strides:list[int]|int=1): + def AveragePool(X: Tensor, kernel_shape:tuple[int, ...], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, count_include_pad:int=0, + dilations:tuple[int, ...]|int=1, pads:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): pool_pads = _resolve_pool_pads(X, pads, kernel_shape, dilations, strides, auto_pad) return X.avg_pool2d(tuple(kernel_shape), strides, dilations, pool_pads, ceil_mode=ceil_mode, count_include_pad=count_include_pad) - def MaxPool(X: Tensor, kernel_shape:list[int], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, dilations:list[int]|int=1, pads:list[int]|int=0, - storage_order:int=0, strides:list[int]|int=1): + def MaxPool(X: Tensor, kernel_shape:tuple[int, ...], auto_pad:AUTO_PAD_OPTIONS="NOTSET", ceil_mode:int=0, dilations:tuple[int, ...]|int=1, + pads:tuple[int, ...]|int=0, storage_order:int=0, strides:tuple[int, ...]|int=1): pool_pads = _resolve_pool_pads(X, pads, kernel_shape, dilations, strides, auto_pad) out = X.max_pool2d(tuple(kernel_shape), strides, dilations, pool_pads, ceil_mode=ceil_mode, return_indices=True) ret, idx = cast(tuple[Tensor, Tensor], out) return ret, idx.transpose(-2, -1).cast(dtypes.int64) if storage_order else idx.cast(dtypes.int64) - def Conv(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:list[int]|int=1, group:int=1, - kernel_shape:list[int]|None=None, pads:list[int]|int=0, strides:list[int]|int=1): + def Conv(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:tuple[int, ...]|int=1, group:int=1, + kernel_shape:tuple[int, ...]|None=None, pads:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): return X.conv2d(W, B, stride=strides, groups=group, dilation=dilations, padding=_resolve_pool_pads(X, pads, kernel_shape or W.shape[2:], dilations, strides, auto_pad)) - def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:list[int]|int=1, group:int=1, - kernel_shape:list[int]|None=None, pads:list[int]|None=None, output_shape:list[int]|None=None, output_padding:list[int]|int=0, - strides:list[int]|int=1): + def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OPTIONS="NOTSET", dilations:tuple[int, ...]|int=1, group:int=1, + kernel_shape:tuple[int, ...]|None=None, pads:Sequence[int]|None=None, output_shape:Sequence[int]|None=None, + output_padding:tuple[int, ...]|int=0, strides:tuple[int, ...]|int=1): input_shape_, kernel_shape_ = X.shape[2:], (kernel_shape or W.shape[2:]) strides_, dilations_, output_padding_ = (make_tuple(x, len(input_shape_)) for x in (strides, dilations, output_padding)) if output_shape is not None: # we pad according to output_shape @@ -747,8 +747,8 @@ def ConvTranspose(X: Tensor, W: Tensor, B:Tensor|None=None, auto_pad:AUTO_PAD_OP pads = _onnx_pads_to_tiny_pads(pads) return X.conv_transpose2d(W, B, group, strides_, dilations_, pads, output_padding_) - def MaxUnpool(xT: Tensor, xI: Tensor, outshape: list[int]|None=None, kernel_shape:list[int]|None=None, pads:list[int]|int=0, - strides:list[int]|int=1): + def MaxUnpool(xT: Tensor, xI: Tensor, outshape: list[int]|None=None, kernel_shape:Sequence[int]|None=None, pads:tuple[int, ...]|int=0, + strides:tuple[int, ...]|int=1): if kernel_shape is None: kernel_shape = [] pads_: int | tuple[int, ...] = pads if isinstance(pads, int) else _onnx_pads_to_tiny_pads(pads) return Tensor.max_unpool2d(xT, xI, tuple(kernel_shape), strides, 1, pads_, outshape if outshape is None else tuple(outshape)) @@ -761,7 +761,7 @@ def Gemm(A:Tensor, B:Tensor, C:Tensor|None=None, alpha:float=1.0, beta:float=1.0 if C is not None: ret = ret + beta * (C if broadcast == 0 else C.reshape([-1 if i < len(C.shape) else 1 for i in range(ret.ndim)][::-1])) return ret - def Einsum(*Inputs:list[Tensor], equation:str): return Tensor.einsum(equation, *Inputs) + def Einsum(*Inputs:Tensor, equation:str): return Tensor.einsum(equation, *Inputs) def CumSum(X:Tensor, axis:int|list[int], exclusive:int=0, reverse:int=0): axis = X._resolve_dim(_resolve_const(axis)) @@ -774,8 +774,8 @@ def Trilu(x:Tensor, k:int|list[int]=0, upper:int=1): k_ = _resolve_const(k) return x.triu(k_) if upper else x.tril(k_) - def Resize(X:Tensor, roi:list[float]|None=None, scales:list[float]|None=None, sizes:list[int]|None=None, antialias:int=0, - axes:list[int]|None=None, coordinate_transformation_mode:str='half_pixel', cubic_coeff_a:float=-0.75, exclude_outside:int=0, + def Resize(X:Tensor, roi:list[float]|None=None, scales:Sequence[float]|None=None, sizes:list[int]|None=None, antialias:int=0, + axes:Sequence[int]|None=None, coordinate_transformation_mode:str='half_pixel', cubic_coeff_a:float=-0.75, exclude_outside:int=0, extrapolation_value:float=0.0, keep_aspect_ratio_policy:str='stretch', mode:str='nearest', nearest_mode:str='round_prefer_floor'): def _apply_transformation(input_sz, output_sz, scale_dim, mode): index = Tensor.arange(output_sz) @@ -876,7 +876,7 @@ def W1_2(x: Tensor): return polyN(x, [A, -5 * A, 8 * A, -4 * A]) gathered_values = [X.gather(i, idx) for idx in expanded_indices] X = sum(v * c for v, c in zip(gathered_values, expanded_coeffs)) return X.permute(*argsort(perm)) if perm else X - def Upsample(X, scales, mode): return Resize(X=X, scales=scales, mode=mode) # deprecated + def Upsample(X:Tensor, scales:Sequence[float], mode:str): return Resize(X=X, scales=scales, mode=mode) # deprecated def TopK(X:Tensor, K:int|list[int], axis:int=-1, largest:int=1, sorted:int=1): # noqa: A002 # pylint: disable=redefined-builtin val, idx = X.topk(_resolve_const(K), axis, bool(largest), bool(sorted)) @@ -937,7 +937,7 @@ def embedding(x:Tensor, vocab_size, weight:Tensor) -> Tensor: if segment_embedding is not None: embedding_sum = embedding_sum + embedding(segment_ids, segment_embedding.shape[0], segment_embedding) out = embedding_sum.layernorm(eps=epsilon) * gamma + beta return out, None, embedding_sum - def MeanVarianceNormalization(x:Tensor, axis:list[int]|None=None): + def MeanVarianceNormalization(x:Tensor, axis:Sequence[int]|None=None): if axis is None: axis = [0,2,3] return (x - x.mean(axis, keepdim=True)) / (x.std(axis, keepdim=True, correction=0) + 1e-9) @@ -1001,7 +1001,7 @@ def generate_grid(steps): def attention_contrib(x:Tensor, weights:Tensor, bias:Tensor|None=None, mask_index:Tensor|None=None, past:Tensor|None=None, attention_bias:Tensor|None=None, past_sequence_length:Tensor|None=None, do_rotary:int=0, mask_filter_value:float=-10000.0, - num_heads:int|None=None, past_present_share_buffer:int|None=None, qkv_hidden_sizes:list[int]|None=None, + num_heads:int|None=None, past_present_share_buffer:int|None=None, qkv_hidden_sizes:Sequence[int]|None=None, rotary_embedding_dim:int|None=None, scale:float|None=None, unidirectional:int=0): assert not do_rotary and not attention_bias, "TODO" if qkv_hidden_sizes is None: qkv_hidden_sizes = [int(weights.shape[1] // 3)] * 3 From f196af2327ceed2eeaef0b4d4248d479180a0087 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 13:47:47 -0400 Subject: [PATCH 260/306] fix onnx MeanVarianceNormalization arg (#17812) axes, not axis --- test/external/external_test_onnx_ops.py | 6 ++++++ tinygrad/nn/onnx.py | 9 ++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/external/external_test_onnx_ops.py b/test/external/external_test_onnx_ops.py index 55256f153ae5f..02071da6958c9 100644 --- a/test/external/external_test_onnx_ops.py +++ b/test/external/external_test_onnx_ops.py @@ -54,6 +54,12 @@ def test_squeeze(self): outputs = ["squeezed"] self.helper_test_single_op("Squeeze", inputs, attributes, outputs) + def test_mean_variance_normalization_axes(self): + inputs = {"x": np.random.randn(2, 3, 4, 5).astype(np.float32)} + attributes = {"axes": [2, 3]} + outputs = ["out"] + self.helper_test_single_op("MeanVarianceNormalization", inputs, attributes, outputs) + def test_conv(self): # test VALID auto_pad inputs = { diff --git a/tinygrad/nn/onnx.py b/tinygrad/nn/onnx.py index 7723e74b3e1bf..bc8ca04f8228e 100644 --- a/tinygrad/nn/onnx.py +++ b/tinygrad/nn/onnx.py @@ -937,9 +937,8 @@ def embedding(x:Tensor, vocab_size, weight:Tensor) -> Tensor: if segment_embedding is not None: embedding_sum = embedding_sum + embedding(segment_ids, segment_embedding.shape[0], segment_embedding) out = embedding_sum.layernorm(eps=epsilon) * gamma + beta return out, None, embedding_sum - def MeanVarianceNormalization(x:Tensor, axis:Sequence[int]|None=None): - if axis is None: axis = [0,2,3] - return (x - x.mean(axis, keepdim=True)) / (x.std(axis, keepdim=True, correction=0) + 1e-9) + def MeanVarianceNormalization(x:Tensor, axes:Sequence[int]=(0,2,3)): + return (x - x.mean(axes, keepdim=True)) / (x.std(axes, keepdim=True, correction=0) + 1e-9) def LpNormalization(x:Tensor, axis:int=-1, p:int=2): return x / (x.abs().sum(axis, keepdim=True) if p == 1 else x.square().sum(axis, keepdim=True).sqrt()) @@ -1246,8 +1245,8 @@ def QLinearGlobalAveragePool(X:Tensor, x_scale:Tensor, x_zero_point:Tensor, y_sc ret = _qlinearop_float(GlobalAveragePool, [X], [x_zero_point], [x_scale], y_scale, y_zero_point) return ret.permute(0, *range(2, ret.ndim), 1) if channels_last else ret # NCHW -> NHWC - def ConvInteger(x: Tensor, w: Tensor, x_zero_point:Tensor = Tensor(0), w_zero_point:Tensor = Tensor(0), B: Tensor | None = None, **opts) -> Tensor: - return _op_integer(Conv, [x,w], [x_zero_point,w_zero_point], **{"B":B, **opts}) + def ConvInteger(x: Tensor, w: Tensor, x_zero_point:Tensor = Tensor(0), w_zero_point:Tensor = Tensor(0), **opts) -> Tensor: + return _op_integer(Conv, [x,w], [x_zero_point,w_zero_point], **opts) def MatMulInteger(A: Tensor, B: Tensor, a_zero_point: Tensor = Tensor(0), b_zero_point: Tensor = Tensor(0)) -> Tensor: return _op_integer(Tensor.matmul, [A,B], [a_zero_point,b_zero_point]) From 959958135d167b7a181daa89b8e6cf5c3b9c9824 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Fri, 28 Aug 2026 20:48:59 +0300 Subject: [PATCH 261/306] am: vf (#17802) * am: vf * x * Dx * Dx * Dx * x * x --- tinygrad/runtime/autogen/am/__init__.py | 7 +- tinygrad/runtime/autogen/am/am.py | 149 +++++++++++++++++++++++- tinygrad/runtime/autogen/am/regs.py | 23 ++++ tinygrad/runtime/ops_amd.py | 6 +- tinygrad/runtime/support/am/amdev.py | 96 +++++++++++---- tinygrad/runtime/support/am/ip.py | 43 +++++-- 6 files changed, 286 insertions(+), 38 deletions(-) diff --git a/tinygrad/runtime/autogen/am/__init__.py b/tinygrad/runtime/autogen/am/__init__.py index b413cc6ecc269..6d6507fb6695c 100644 --- a/tinygrad/runtime/autogen/am/__init__.py +++ b/tinygrad/runtime/autogen/am/__init__.py @@ -22,11 +22,12 @@ reg_patterns = { "gc": ["GCVM", "GCMC_VM", "CP_(HQD|MQD|MEC|ME_CNTL|PERFMON|RB_WPTR_POLL_CNTL|INT_CNTL|STAT|PFP_PRGRM|ME_PRGRM|COHER_START)", "COMPUTE_", "(SQ|GL2C|TCC)_PERFCOUNTER", "SQ_THREAD_TRACE", "SPI_(CONFIG_CNTL|COMPUTE_QUEUE_RESET)", "GRBM", "SH_MEM", "RLC", "TCP", "GB_ADDR_CONFIG", - "SDMA[01]_(WATCHDOG_CNTL|UTCL1_(CNTL|PAGE)|MCU_CNTL|F32_CNTL|CNTL|QUEUE0_|RLC_CGCG_CTRL)", "SCRATCH_REG[67]"], + "SDMA[01]_(WATCHDOG_CNTL|UTCL1_(CNTL|PAGE)|MCU_CNTL|F32_CNTL|CNTL|QUEUE0_|RLC_CGCG_CTRL)", "SCRATCH_REG[0-367]"], "mmhub": ["MMVM", "MMMC_VM", "MM_ATC_L2_MISC_CG"], "nbio": (nbio:=["BIF_BX_PF[01]_GPU_HDP_FLUSH", "BIF_BX_PF0_RSMU", "BIF_BX0_(REMAP_HDP_MEM_FLUSH_CNTL|BIF_DOORBELL_INT_CNTL|PCIE_INDEX2|PCIE_DATA2)", "BIFC_(DOORBELL_ACCESS_EN_PF|GFX_INT_MONITOR_MASK)", "XCC_DOORBELL_FENCE", "DOORBELL0_CTRL_ENTRY", "GDC_S2A0_S2A_DOORBELL_ENTRY", - "S2A_DOORBELL_ENTRY", "RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN", "RCC_DEV0_EPF2_STRAP2"]), + "S2A_DOORBELL_ENTRY", "RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN", + "BIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL", "RCC_DEV0_EPF2_STRAP2"]), "nbif": nbio, "mp": ["MP([01]|ASP)_SMN_C2PMSG"], "hdp": ["HDP_MEM_POWER_CTRL"], "oss": ["IH_"], "sdma": ["SDMA_GFX", "SDMA_CNTL"] } @@ -38,7 +39,7 @@ def __getattr__(nm): case "am": return load("am/am", [root/f"extra/amdpci/headers/{s}.h" for s in ["v11_structs", "v12_structs", "amdgpu_vm", "discovery", "amdgpu_ucode", "psp_gfx_if", "amdgpu_psp", "amdgpu_irq", "amdgpu_doorbell"]] + [f"{AMD}/amdkfd/soc15_int.h"] + \ [f"{AMDINC}/ivsrcid/{s}.h" for s in [f"gfx/irqsrcs_gfx_{x}_0" for x in ('9','11_0','12_0')] + [f"sdma0/irqsrcs_sdma0_{x}_0" for x in (4,5)]] + \ - [f"{AMDINC}/{s}.h" for s in ["v9_structs", "soc15_ih_clientid"]], args=inc, srcs=am_src, rules=kern_rules) + [f"{AMDINC}/{s}.h" for s in ["v9_structs", "soc15_ih_clientid"]] + [f"{AMD}/amdgpu/{s}.h" for s in ["mxgpu_nv", "amdgpu_virt"]], args=inc, srcs=am_src, rules=kern_rules) case "pm4_soc15": return load("am/pm4_soc15", [f"{AMD}/amdkfd/kfd_pm4_headers_ai.h", f"{AMD}/amdgpu/soc15d.h"], srcs=am_src) case "pm4_nv": return load("am/pm4_nv", [f"{AMD}/amdkfd/kfd_pm4_headers_ai.h", f"{AMD}/amdgpu/nvd.h"], srcs=am_src) case "sdma_4_0_0": return load("am/sdma_4_0_0", [root/"extra/hip_gpu_driver/sdma_registers.h", f"{AMD}/amdgpu/vega10_sdma_pkt_open.h"], diff --git a/tinygrad/runtime/autogen/am/am.py b/tinygrad/runtime/autogen/am/am.py index 15bc876f49a49..bbe5ee5a1b3a8 100644 --- a/tinygrad/runtime/autogen/am/am.py +++ b/tinygrad/runtime/autogen/am/am.py @@ -4110,6 +4110,110 @@ class struct_v9_gfx_meta_data(c.Struct): struct_v9_gfx_meta_data.register_fields([('ce_payload', struct_v9_ce_ib_state, 0), ('reserved1', c.Array[uint32_t, Literal[54]], 40), ('de_payload', struct_v9_de_ib_state, 256), ('DeIbBaseAddrLo', uint32_t, 364), ('DeIbBaseAddrHi', uint32_t, 368), ('reserved2', c.Array[uint32_t, Literal[931]], 372)]) enum_soc15_ih_clientid: dict[int, str] = {(SOC15_IH_CLIENTID_IH:=0): 'SOC15_IH_CLIENTID_IH', (SOC15_IH_CLIENTID_ACP:=1): 'SOC15_IH_CLIENTID_ACP', (SOC15_IH_CLIENTID_ATHUB:=2): 'SOC15_IH_CLIENTID_ATHUB', (SOC15_IH_CLIENTID_BIF:=3): 'SOC15_IH_CLIENTID_BIF', (SOC15_IH_CLIENTID_DCE:=4): 'SOC15_IH_CLIENTID_DCE', (SOC15_IH_CLIENTID_ISP:=5): 'SOC15_IH_CLIENTID_ISP', (SOC15_IH_CLIENTID_PCIE0:=6): 'SOC15_IH_CLIENTID_PCIE0', (SOC15_IH_CLIENTID_RLC:=7): 'SOC15_IH_CLIENTID_RLC', (SOC15_IH_CLIENTID_SDMA0:=8): 'SOC15_IH_CLIENTID_SDMA0', (SOC15_IH_CLIENTID_SDMA1:=9): 'SOC15_IH_CLIENTID_SDMA1', (SOC15_IH_CLIENTID_SE0SH:=10): 'SOC15_IH_CLIENTID_SE0SH', (SOC15_IH_CLIENTID_SE1SH:=11): 'SOC15_IH_CLIENTID_SE1SH', (SOC15_IH_CLIENTID_SE2SH:=12): 'SOC15_IH_CLIENTID_SE2SH', (SOC15_IH_CLIENTID_SE3SH:=13): 'SOC15_IH_CLIENTID_SE3SH', (SOC15_IH_CLIENTID_UVD1:=14): 'SOC15_IH_CLIENTID_UVD1', (SOC15_IH_CLIENTID_THM:=15): 'SOC15_IH_CLIENTID_THM', (SOC15_IH_CLIENTID_UVD:=16): 'SOC15_IH_CLIENTID_UVD', (SOC15_IH_CLIENTID_VCE0:=17): 'SOC15_IH_CLIENTID_VCE0', (SOC15_IH_CLIENTID_VMC:=18): 'SOC15_IH_CLIENTID_VMC', (SOC15_IH_CLIENTID_XDMA:=19): 'SOC15_IH_CLIENTID_XDMA', (SOC15_IH_CLIENTID_GRBM_CP:=20): 'SOC15_IH_CLIENTID_GRBM_CP', (SOC15_IH_CLIENTID_ATS:=21): 'SOC15_IH_CLIENTID_ATS', (SOC15_IH_CLIENTID_ROM_SMUIO:=22): 'SOC15_IH_CLIENTID_ROM_SMUIO', (SOC15_IH_CLIENTID_DF:=23): 'SOC15_IH_CLIENTID_DF', (SOC15_IH_CLIENTID_VCE1:=24): 'SOC15_IH_CLIENTID_VCE1', (SOC15_IH_CLIENTID_PWR:=25): 'SOC15_IH_CLIENTID_PWR', (SOC15_IH_CLIENTID_RESERVED:=26): 'SOC15_IH_CLIENTID_RESERVED', (SOC15_IH_CLIENTID_UTCL2:=27): 'SOC15_IH_CLIENTID_UTCL2', (SOC15_IH_CLIENTID_EA:=28): 'SOC15_IH_CLIENTID_EA', (SOC15_IH_CLIENTID_UTCL2LOG:=29): 'SOC15_IH_CLIENTID_UTCL2LOG', (SOC15_IH_CLIENTID_MP0:=30): 'SOC15_IH_CLIENTID_MP0', (SOC15_IH_CLIENTID_MP1:=31): 'SOC15_IH_CLIENTID_MP1', (SOC15_IH_CLIENTID_MAX:=32): 'SOC15_IH_CLIENTID_MAX', (SOC15_IH_CLIENTID_VCN:=16): 'SOC15_IH_CLIENTID_VCN', (SOC15_IH_CLIENTID_VCN1:=14): 'SOC15_IH_CLIENTID_VCN1', (SOC15_IH_CLIENTID_SDMA2:=1): 'SOC15_IH_CLIENTID_SDMA2', (SOC15_IH_CLIENTID_SDMA3:=4): 'SOC15_IH_CLIENTID_SDMA3', (SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid:=5): 'SOC15_IH_CLIENTID_SDMA3_Sienna_Cichlid', (SOC15_IH_CLIENTID_SDMA4:=5): 'SOC15_IH_CLIENTID_SDMA4', (SOC15_IH_CLIENTID_SDMA5:=17): 'SOC15_IH_CLIENTID_SDMA5', (SOC15_IH_CLIENTID_SDMA6:=19): 'SOC15_IH_CLIENTID_SDMA6', (SOC15_IH_CLIENTID_SDMA7:=24): 'SOC15_IH_CLIENTID_SDMA7', (SOC15_IH_CLIENTID_VMC1:=6): 'SOC15_IH_CLIENTID_VMC1'} enum_soc21_ih_clientid: dict[int, str] = {(SOC21_IH_CLIENTID_IH:=0): 'SOC21_IH_CLIENTID_IH', (SOC21_IH_CLIENTID_ATHUB:=2): 'SOC21_IH_CLIENTID_ATHUB', (SOC21_IH_CLIENTID_BIF:=3): 'SOC21_IH_CLIENTID_BIF', (SOC21_IH_CLIENTID_DCN:=4): 'SOC21_IH_CLIENTID_DCN', (SOC21_IH_CLIENTID_ISP:=5): 'SOC21_IH_CLIENTID_ISP', (SOC21_IH_CLIENTID_MP3:=6): 'SOC21_IH_CLIENTID_MP3', (SOC21_IH_CLIENTID_RLC:=7): 'SOC21_IH_CLIENTID_RLC', (SOC21_IH_CLIENTID_GFX:=10): 'SOC21_IH_CLIENTID_GFX', (SOC21_IH_CLIENTID_IMU:=11): 'SOC21_IH_CLIENTID_IMU', (SOC21_IH_CLIENTID_VCN1:=14): 'SOC21_IH_CLIENTID_VCN1', (SOC21_IH_CLIENTID_THM:=15): 'SOC21_IH_CLIENTID_THM', (SOC21_IH_CLIENTID_VCN:=16): 'SOC21_IH_CLIENTID_VCN', (SOC21_IH_CLIENTID_VPE1:=17): 'SOC21_IH_CLIENTID_VPE1', (SOC21_IH_CLIENTID_VMC:=18): 'SOC21_IH_CLIENTID_VMC', (SOC21_IH_CLIENTID_GRBM_CP:=20): 'SOC21_IH_CLIENTID_GRBM_CP', (SOC21_IH_CLIENTID_ROM_SMUIO:=22): 'SOC21_IH_CLIENTID_ROM_SMUIO', (SOC21_IH_CLIENTID_DF:=23): 'SOC21_IH_CLIENTID_DF', (SOC21_IH_CLIENTID_VPE:=24): 'SOC21_IH_CLIENTID_VPE', (SOC21_IH_CLIENTID_PWR:=25): 'SOC21_IH_CLIENTID_PWR', (SOC21_IH_CLIENTID_LSDMA:=26): 'SOC21_IH_CLIENTID_LSDMA', (SOC21_IH_CLIENTID_MP0:=30): 'SOC21_IH_CLIENTID_MP0', (SOC21_IH_CLIENTID_MP1:=31): 'SOC21_IH_CLIENTID_MP1', (SOC21_IH_CLIENTID_MAX:=32): 'SOC21_IH_CLIENTID_MAX'} +enum_idh_request: dict[int, str] = {(IDH_REQ_GPU_INIT_ACCESS:=1): 'IDH_REQ_GPU_INIT_ACCESS', (IDH_REL_GPU_INIT_ACCESS:=2): 'IDH_REL_GPU_INIT_ACCESS', (IDH_REQ_GPU_FINI_ACCESS:=3): 'IDH_REQ_GPU_FINI_ACCESS', (IDH_REL_GPU_FINI_ACCESS:=4): 'IDH_REL_GPU_FINI_ACCESS', (IDH_REQ_GPU_RESET_ACCESS:=5): 'IDH_REQ_GPU_RESET_ACCESS', (IDH_REQ_GPU_INIT_DATA:=6): 'IDH_REQ_GPU_INIT_DATA', (IDH_LOG_VF_ERROR:=200): 'IDH_LOG_VF_ERROR', (IDH_READY_TO_RESET:=201): 'IDH_READY_TO_RESET', (IDH_RAS_POISON:=202): 'IDH_RAS_POISON', (IDH_REQ_RAS_ERROR_COUNT:=203): 'IDH_REQ_RAS_ERROR_COUNT', (IDH_REQ_RAS_CPER_DUMP:=204): 'IDH_REQ_RAS_CPER_DUMP', (IDH_REQ_RAS_BAD_PAGES:=205): 'IDH_REQ_RAS_BAD_PAGES', (IDH_REQ_RAS_CHK_CRITI:=206): 'IDH_REQ_RAS_CHK_CRITI'} +enum_idh_event: dict[int, str] = {(IDH_CLR_MSG_BUF:=0): 'IDH_CLR_MSG_BUF', (IDH_READY_TO_ACCESS_GPU:=1): 'IDH_READY_TO_ACCESS_GPU', (IDH_FLR_NOTIFICATION:=2): 'IDH_FLR_NOTIFICATION', (IDH_FLR_NOTIFICATION_CMPL:=3): 'IDH_FLR_NOTIFICATION_CMPL', (IDH_SUCCESS:=4): 'IDH_SUCCESS', (IDH_FAIL:=5): 'IDH_FAIL', (IDH_QUERY_ALIVE:=6): 'IDH_QUERY_ALIVE', (IDH_REQ_GPU_INIT_DATA_READY:=7): 'IDH_REQ_GPU_INIT_DATA_READY', (IDH_RAS_POISON_READY:=8): 'IDH_RAS_POISON_READY', (IDH_PF_SOFT_FLR_NOTIFICATION:=9): 'IDH_PF_SOFT_FLR_NOTIFICATION', (IDH_RAS_ERROR_DETECTED:=10): 'IDH_RAS_ERROR_DETECTED', (IDH_RAS_ERROR_COUNT_READY:=11): 'IDH_RAS_ERROR_COUNT_READY', (IDH_RAS_CPER_DUMP_READY:=14): 'IDH_RAS_CPER_DUMP_READY', (IDH_RAS_BAD_PAGES_READY:=15): 'IDH_RAS_BAD_PAGES_READY', (IDH_RAS_BAD_PAGES_NOTIFICATION:=16): 'IDH_RAS_BAD_PAGES_NOTIFICATION', (IDH_UNRECOV_ERR_NOTIFICATION:=17): 'IDH_UNRECOV_ERR_NOTIFICATION', (IDH_REQ_RAS_CHK_CRITI_READY:=18): 'IDH_REQ_RAS_CHK_CRITI_READY', (IDH_TEXT_MESSAGE:=255): 'IDH_TEXT_MESSAGE'} +class struct_amdgpu_virt_ops(c.Struct): pass +enum_amdgpu_sriov_vf_mode: dict[int, str] = {(SRIOV_VF_MODE_BARE_METAL:=0): 'SRIOV_VF_MODE_BARE_METAL', (SRIOV_VF_MODE_ONE_VF:=1): 'SRIOV_VF_MODE_ONE_VF', (SRIOV_VF_MODE_MULTI_VF:=2): 'SRIOV_VF_MODE_MULTI_VF'} +@c.record +class struct_amdgpu_mm_table(c.Struct): + SIZE = 24 + bo: c.POINTER[struct_amdgpu_bo] + cpu_addr: c.POINTER[ctypes.c_uint32] + gpu_addr: int +class struct_amdgpu_bo(c.Struct): pass +struct_amdgpu_mm_table.register_fields([('bo', c.POINTER[struct_amdgpu_bo], 0), ('cpu_addr', c.POINTER[uint32_t], 8), ('gpu_addr', uint64_t, 16)]) +class struct_amdgpu_vf_error_buffer(c.Struct): pass +@c.record +class struct_amdgpu_virt_fw_reserve(c.Struct): + SIZE = 32 + p_pf2vf: c.POINTER[struct_amd_sriov_msg_pf2vf_info_header] + p_vf2pf: c.POINTER[struct_amd_sriov_msg_vf2pf_info_header] + ras_telemetry: ctypes.c_void_p + checksum_key: int +@c.record +class struct_amd_sriov_msg_pf2vf_info_header(c.Struct): + SIZE = 16 + size: int + version: int + reserved: c.Array[ctypes.c_uint32, Literal[2]] +struct_amd_sriov_msg_pf2vf_info_header.register_fields([('size', uint32_t, 0), ('version', uint32_t, 4), ('reserved', c.Array[uint32_t, Literal[2]], 8)]) +@c.record +class struct_amd_sriov_msg_vf2pf_info_header(c.Struct): + SIZE = 16 + size: int + version: int + reserved: c.Array[ctypes.c_uint32, Literal[2]] +struct_amd_sriov_msg_vf2pf_info_header.register_fields([('size', uint32_t, 0), ('version', uint32_t, 4), ('reserved', c.Array[uint32_t, Literal[2]], 8)]) +struct_amdgpu_virt_fw_reserve.register_fields([('p_pf2vf', c.POINTER[struct_amd_sriov_msg_pf2vf_info_header], 0), ('p_vf2pf', c.POINTER[struct_amd_sriov_msg_vf2pf_info_header], 8), ('ras_telemetry', ctypes.c_void_p, 16), ('checksum_key', ctypes.c_uint32, 24)]) +enum_AMDGIM_FEATURE_FLAG: dict[int, str] = {(AMDGIM_FEATURE_ERROR_LOG_COLLECT:=1): 'AMDGIM_FEATURE_ERROR_LOG_COLLECT', (AMDGIM_FEATURE_GIM_LOAD_UCODES:=2): 'AMDGIM_FEATURE_GIM_LOAD_UCODES', (AMDGIM_FEATURE_GIM_FLR_VRAMLOST:=4): 'AMDGIM_FEATURE_GIM_FLR_VRAMLOST', (AMDGIM_FEATURE_GIM_MM_BW_MGR:=8): 'AMDGIM_FEATURE_GIM_MM_BW_MGR', (AMDGIM_FEATURE_PP_ONE_VF:=16): 'AMDGIM_FEATURE_PP_ONE_VF', (AMDGIM_FEATURE_INDIRECT_REG_ACCESS:=32): 'AMDGIM_FEATURE_INDIRECT_REG_ACCESS', (AMDGIM_FEATURE_AV1_SUPPORT:=64): 'AMDGIM_FEATURE_AV1_SUPPORT', (AMDGIM_FEATURE_VCN_RB_DECOUPLE:=128): 'AMDGIM_FEATURE_VCN_RB_DECOUPLE', (AMDGIM_FEATURE_MES_INFO_ENABLE:=256): 'AMDGIM_FEATURE_MES_INFO_ENABLE', (AMDGIM_FEATURE_RAS_CAPS:=512): 'AMDGIM_FEATURE_RAS_CAPS', (AMDGIM_FEATURE_RAS_TELEMETRY:=1024): 'AMDGIM_FEATURE_RAS_TELEMETRY', (AMDGIM_FEATURE_RAS_CPER:=2048): 'AMDGIM_FEATURE_RAS_CPER', (AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK:=4096): 'AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK'} +enum_AMDGIM_REG_ACCESS_FLAG: dict[int, str] = {(AMDGIM_FEATURE_IH_REG_PSP_EN:=1): 'AMDGIM_FEATURE_IH_REG_PSP_EN', (AMDGIM_FEATURE_MMHUB_REG_RLC_EN:=2): 'AMDGIM_FEATURE_MMHUB_REG_RLC_EN', (AMDGIM_FEATURE_GC_REG_RLC_EN:=4): 'AMDGIM_FEATURE_GC_REG_RLC_EN', (AMDGIM_FEATURE_L1_TLB_CNTL_PSP_EN:=8): 'AMDGIM_FEATURE_L1_TLB_CNTL_PSP_EN', (AMDGIM_FEATURE_REG_ACCESS_SQ_CONFIG:=16): 'AMDGIM_FEATURE_REG_ACCESS_SQ_CONFIG'} +@c.record +class struct_amdgim_pf2vf_info_v1(c.Struct): + SIZE = 44 + header: struct_amd_sriov_msg_pf2vf_info_header + uvd_enc_max_pixels_count: int + uvd_enc_max_bandwidth: int + vce_enc_max_pixels_count: int + vce_enc_max_bandwidth: int + mecfw_kboffset: int + feature_flags: int + checksum: int +struct_amdgim_pf2vf_info_v1.register_fields([('header', struct_amd_sriov_msg_pf2vf_info_header, 0), ('uvd_enc_max_pixels_count', ctypes.c_uint32, 16), ('uvd_enc_max_bandwidth', ctypes.c_uint32, 20), ('vce_enc_max_pixels_count', ctypes.c_uint32, 24), ('vce_enc_max_bandwidth', ctypes.c_uint32, 28), ('mecfw_kboffset', ctypes.c_uint32, 32), ('feature_flags', ctypes.c_uint32, 36), ('checksum', ctypes.c_uint32, 40)]) +@c.record +class struct_amdgim_vf2pf_info_v1(c.Struct): + SIZE = 128 + header: struct_amd_sriov_msg_vf2pf_info_header + driver_version: c.Array[ctypes.c_char, Literal[64]] + driver_cert: int + os_info: int + fb_usage: int + gfx_usage: int + gfx_health: int + compute_usage: int + compute_health: int + vce_enc_usage: int + vce_enc_health: int + uvd_enc_usage: int + uvd_enc_health: int + checksum: int +struct_amdgim_vf2pf_info_v1.register_fields([('header', struct_amd_sriov_msg_vf2pf_info_header, 0), ('driver_version', c.Array[ctypes.c_char, Literal[64]], 16), ('driver_cert', ctypes.c_uint32, 80), ('os_info', ctypes.c_uint32, 84), ('fb_usage', ctypes.c_uint32, 88), ('gfx_usage', ctypes.c_uint32, 92), ('gfx_health', ctypes.c_uint32, 96), ('compute_usage', ctypes.c_uint32, 100), ('compute_health', ctypes.c_uint32, 104), ('vce_enc_usage', ctypes.c_uint32, 108), ('vce_enc_health', ctypes.c_uint32, 112), ('uvd_enc_usage', ctypes.c_uint32, 116), ('uvd_enc_health', ctypes.c_uint32, 120), ('checksum', ctypes.c_uint32, 124)]) +@c.record +class struct_amdgim_vf2pf_info_v2(c.Struct): + SIZE = 1024 + header: struct_amd_sriov_msg_vf2pf_info_header + checksum: int + driver_version: c.Array[ctypes.c_ubyte, Literal[64]] + driver_cert: int + os_info: int + fb_usage: int + gfx_usage: int + gfx_health: int + compute_usage: int + compute_health: int + vce_enc_usage: int + vce_enc_health: int + uvd_enc_usage: int + uvd_enc_health: int + reserved: c.Array[ctypes.c_uint32, Literal[224]] +struct_amdgim_vf2pf_info_v2.register_fields([('header', struct_amd_sriov_msg_vf2pf_info_header, 0), ('checksum', uint32_t, 16), ('driver_version', c.Array[uint8_t, Literal[64]], 20), ('driver_cert', uint32_t, 84), ('os_info', uint32_t, 88), ('fb_usage', uint32_t, 92), ('gfx_usage', uint32_t, 96), ('gfx_health', uint32_t, 100), ('compute_usage', uint32_t, 104), ('compute_health', uint32_t, 108), ('vce_enc_usage', uint32_t, 112), ('vce_enc_health', uint32_t, 116), ('uvd_enc_usage', uint32_t, 120), ('uvd_enc_health', uint32_t, 124), ('reserved', c.Array[uint32_t, Literal[224]], 128)]) +@c.record +class struct_amdgpu_virt_ras_err_handler_data(c.Struct): + SIZE = 24 + bps: c.POINTER[struct_eeprom_table_record] + bps_bo: c.POINTER[c.POINTER[struct_amdgpu_bo]] + count: int + last_reserved: int +class struct_eeprom_table_record(c.Struct): pass +struct_amdgpu_virt_ras_err_handler_data.register_fields([('bps', c.POINTER[struct_eeprom_table_record], 0), ('bps_bo', c.POINTER[c.POINTER[struct_amdgpu_bo]], 8), ('count', ctypes.c_int32, 16), ('last_reserved', ctypes.c_int32, 20)]) +class struct_amdgpu_virt_ras(c.Struct): pass +@c.record +class struct_amdgpu_virt_region(c.Struct): + SIZE = 8 + offset: int + size_kb: int +struct_amdgpu_virt_region.register_fields([('offset', uint32_t, 0), ('size_kb', uint32_t, 4)]) +class struct_amdgpu_virt(c.Struct): pass +class struct_amdgpu_video_codec_info(c.Struct): pass AMDGPU_VM_MAX_UPDATE_SIZE = 0x3FFFF AMDGPU_PTE_VALID = (1 << 0) AMDGPU_PTE_SYSTEM = (1 << 1) @@ -4465,4 +4569,47 @@ class struct_v9_gfx_meta_data(c.Struct): SDMA0_5_0__SRCID__SDMA_DOORBELL_INVALID = 244 SDMA0_5_0__SRCID__SDMA_FROZEN = 245 SDMA0_5_0__SRCID__SDMA_POLL_TIMEOUT = 246 -SDMA0_5_0__SRCID__SDMA_SRBMWRITE = 247 \ No newline at end of file +SDMA0_5_0__SRCID__SDMA_SRBMWRITE = 247 +NV_MAILBOX_POLL_ACK_TIMEDOUT = 500 +NV_MAILBOX_POLL_MSG_TIMEDOUT = 15000 +NV_MAILBOX_POLL_FLR_TIMEDOUT = 10000 +NV_MAILBOX_POLL_MSG_REP_MAX = 11 +mmMAILBOX_CONTROL = 0xE5E +NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE = (mmMAILBOX_CONTROL * 4) +NV_MAIBOX_CONTROL_RCV_OFFSET_BYTE = (NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE + 1) +mmMAILBOX_MSGBUF_TRN_DW0 = 0xE56 +mmMAILBOX_MSGBUF_TRN_DW1 = 0xE57 +mmMAILBOX_MSGBUF_TRN_DW2 = 0xE58 +mmMAILBOX_MSGBUF_TRN_DW3 = 0xE59 +mmMAILBOX_MSGBUF_RCV_DW0 = 0xE5A +mmMAILBOX_MSGBUF_RCV_DW1 = 0xE5B +mmMAILBOX_MSGBUF_RCV_DW2 = 0xE5C +mmMAILBOX_MSGBUF_RCV_DW3 = 0xE5D +mmMAILBOX_INT_CNTL = 0xE5F +AMDGPU_SRIOV_CAPS_SRIOV_VBIOS = (1 << 0) +AMDGPU_SRIOV_CAPS_ENABLE_IOV = (1 << 1) +AMDGPU_SRIOV_CAPS_IS_VF = (1 << 2) +AMDGPU_PASSTHROUGH_MODE = (1 << 3) +AMDGPU_SRIOV_CAPS_RUNTIME = (1 << 4) +AMDGPU_VF_MMIO_ACCESS_PROTECT = (1 << 5) +AMDGPU_RLCG_GC_WRITE_LEGACY = (0x8 << 28) +AMDGPU_RLCG_GC_WRITE = (0x0 << 28) +AMDGPU_RLCG_GC_READ = (0x1 << 28) +AMDGPU_RLCG_MMHUB_WRITE = (0x2 << 28) +AMDGPU_RLCG_VFGATE_DISABLED = 0x4000000 +AMDGPU_RLCG_WRONG_OPERATION_TYPE = 0x2000000 +AMDGPU_RLCG_REG_NOT_IN_RANGE = 0x1000000 +AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK = 0xFFFFF +AMDGPU_RLCG_SCRATCH1_ERROR_MASK = 0xF000000 +mmRCC_IOV_FUNC_IDENTIFIER = 0xDE5 +mmBIF_IOV_FUNC_IDENTIFIER = 0x1503 +AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT = 2 +AMDGPU_SRIOV_CRIT_DATA_SIGNATURE = "INDA" +AMDGPU_SRIOV_CRIT_DATA_SIG_LEN = 4 +AMDGPU_VF_ERROR_ENTRY_SIZE = 16 +AMDGIM_DATAEXCHANGE_OFFSET = (64 * 1024) +AMDGIM_GET_STRUCTURE_RESERVED_SIZE = lambda total,u8,u16,u32,u64: (total - (((u8)+3) / 4 + ((u16)+1) / 2 + (u32) + (u64)*2)) # type: ignore +AMDGPU_VIRT_CAPS_LIST = lambda X: X(AMDGPU_VIRT_CAP_POWER_LIMIT) # type: ignore +amdgpu_sriov_fullaccess = lambda adev: (amdgpu_sriov_vf((adev)) and not amdgpu_sriov_runtime((adev))) # type: ignore +amdgpu_sriov_rlcg_error_report_enabled = lambda adev: (amdgpu_sriov_reg_indirect_mmhub(adev) or amdgpu_sriov_reg_indirect_gc(adev)) # type: ignore +amdgpu_sriov_multi_vf_mode = lambda adev: (amdgpu_sriov_vf(adev) and not amdgpu_sriov_is_pp_one_vf(adev)) # type: ignore \ No newline at end of file diff --git a/tinygrad/runtime/autogen/am/regs.py b/tinygrad/runtime/autogen/am/regs.py index 6cb6f94e02f4b..e5b9f556ea9c5 100644 --- a/tinygrad/runtime/autogen/am/regs.py +++ b/tinygrad/runtime/autogen/am/regs.py @@ -510,6 +510,10 @@ 'regTCP_PERFCOUNTER_FILTER': (4793, 0, {'buffer': (0, 0), 'flat': (1, 1), 'dim': (2, 4), 'data_format': (5, 10), 'num_format': (11, 14), 'sw_mode': (15, 19), 'num_samples': (20, 21), 'opcode_type': (22, 24), 'glc': (25, 25), 'slc': (26, 26), 'compression_enable': (27, 27), 'addr_mode': (28, 30)}), 'regTCP_PERFCOUNTER_FILTER_EN': (4794, 0, {'buffer': (0, 0), 'flat': (1, 1), 'dim': (2, 2), 'data_format': (3, 3), 'num_format': (4, 4), 'sw_mode': (5, 5), 'num_samples': (6, 6), 'opcode_type': (7, 7), 'glc': (8, 8), 'slc': (9, 9), 'compression_enable': (10, 10), 'addr_mode': (11, 11)}), 'regCP_PERFMON_CNTX_CNTL': (216, 1, {'perfmon_enable': (31, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regCP_COHER_START_DELAY': (8315, 1, {'start_delay_count': (0, 5)}), @@ -1791,6 +1795,10 @@ 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tc_td_ram_clken_disable': (20, 20), 'tc_td_data_clken_disable': (21, 21), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), 'regTCP_DEBUG_INDEX': (6565, 1, {'index': (0, 4)}), 'regTCP_DEBUG_DATA': (6566, 1, {'data': (0, 17)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -3366,6 +3374,10 @@ 'regTCP_CNTL': (6562, 1, {'force_hit': (0, 0), 'force_miss': (1, 1), 'flat_buf_cache_swizzle': (5, 5), 'td_data_en_override': (6, 6), 'enable_128b_dcc_comp_read_for_indep64': (7, 7), 'disable_write_combining': (9, 9), 'force_eow_total_cnt': (15, 20), 'force_eow_set_cnt': (22, 26), 'disable_z_map': (28, 28), 'force_order_between_read_write_to_same_address': (29, 29), 'astc_ve_msb_tolerant': (31, 31)}), 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tc_td_ram_clken_disable': (20, 20), 'tc_td_data_clken_disable': (21, 21), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), 'regTCP_CREDIT': (6564, 1, {'lfifo_ram_depth': (0, 9), 'gl1_req_credit': (10, 15), 'req_fifo_credit': (16, 22), 'td_ram_credit': (23, 27), 'td_data_credit': (29, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -4791,6 +4803,10 @@ 'regTCP_STATUS': (6561, 1, {'tcp_busy': (0, 0), 'input_busy': (1, 1), 'adrs_busy': (2, 2), 'tagrams_busy': (3, 3), 'cntrl_busy': (4, 4), 'lfifo_busy': (5, 5), 'read_busy': (6, 6), 'format_busy': (7, 7), 'vm_busy': (8, 8), 'memif_busy': (9, 9), 'gcr_busy': (10, 10), 'ofifo_busy': (11, 11), 'ofifo_queue_busy': (12, 13), 'xnack_prt': (15, 15)}), 'regTCP_CNTL': (6562, 1, {'force_hit': (0, 0), 'force_miss': (1, 1), 'flat_buf_cache_swizzle': (5, 5), 'td_data_en_override': (6, 6), 'enable_128b_dcc_comp_read_for_indep64': (7, 7), 'disable_decompression_power_opt': (8, 8), 'disable_write_combining': (9, 9), 'force_eow_total_cnt': (15, 20), 'force_eow_set_cnt': (22, 26), 'disable_z_map': (28, 28), 'force_order_between_read_write_to_same_address': (29, 29), 'astc_ve_msb_tolerant': (31, 31)}), 'regTCP_CNTL2': (6563, 1, {'ls_disable_clocks': (0, 7), 'tcp_fmt_mgcg_disable': (8, 8), 'tcpf_latency_bypass_disable': (9, 9), 'tcp_write_data_mgcg_disable': (10, 10), 'tcp_inner_block_mgcg_disable': (11, 11), 'tcp_adrs_img_calc_mgcg_disable': (12, 12), 'v64_combine_enable': (13, 13), 'tagram_addr_swizzle_disable': (14, 14), 'return_order_override': (15, 15), 'power_opt_disable': (16, 16), 'gcr_rsp_fgcg_disable': (17, 17), 'perf_en_override': (18, 19), 'tcp_gl1_req_clken_disable': (22, 22), 'tcp_gl1r_src_clken_disable': (23, 23), 'tcp_force_2x_to_load': (24, 24), 'spare_bit': (26, 26), 'tagram_xy_bias_override': (27, 28), 'tcp_req_mgcg_disable': (29, 29), 'tcp_miss_mgcg_disable': (30, 30), 'disable_mipmap_param_calc_self_gating': (31, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -6048,6 +6064,10 @@ 'regCP_ME_CNTL': (2051, 1, {'ce_invalidate_icache': (4, 4), 'pfp_invalidate_icache': (6, 6), 'me_invalidate_icache': (8, 8), 'pfp_pipe0_disable': (12, 12), 'pfp_pipe1_disable': (13, 13), 'me_pipe0_disable': (14, 14), 'me_pipe1_disable': (15, 15), 'ce_pipe0_reset': (16, 16), 'ce_pipe1_reset': (17, 17), 'pfp_pipe0_reset': (18, 18), 'pfp_pipe1_reset': (19, 19), 'me_pipe0_reset': (20, 20), 'me_pipe1_reset': (21, 21), 'ce_halt': (24, 24), 'ce_step': (25, 25), 'pfp_halt': (26, 26), 'pfp_step': (27, 27), 'me_halt': (28, 28), 'me_step': (29, 29)}), 'regGRBM_GFX_CNTL': (2304, 1, {'pipeid': (0, 1), 'meid': (2, 3), 'vmid': (4, 7), 'queueid': (8, 10), 'ctxid': (11, 13)}), 'regGRBM_NOWHERE': (2305, 1, {'data': (0, 31)}), + 'regSCRATCH_REG0': (8256, 1, {'scratch_reg0': (0, 31)}), + 'regSCRATCH_REG1': (8257, 1, {'scratch_reg1': (0, 31)}), + 'regSCRATCH_REG2': (8258, 1, {'scratch_reg2': (0, 31)}), + 'regSCRATCH_REG3': (8259, 1, {'scratch_reg3': (0, 31)}), 'regSCRATCH_REG6': (8262, 1, {'scratch_reg6': (0, 31)}), 'regSCRATCH_REG7': (8263, 1, {'scratch_reg7': (0, 31)}), 'regRLC_GPM_PERF_COUNT_0': (8512, 1, {'feature_sel': (0, 3), 'se_index': (4, 7), 'sa_index': (8, 11), 'wgp_index': (12, 15), 'event_sel': (16, 17), 'unused': (18, 19), 'enable': (20, 20), 'reserved': (21, 31)}), @@ -8977,6 +8997,7 @@ 'regBIF_BX_PF0_GPU_HDP_FLUSH_REQ': (262, 2, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regBIF_BX_PF0_GPU_HDP_FLUSH_DONE': (263, 2, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN': (192, 2, {'bif_doorbell_aper_en': (0, 0)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), 'regRCC_DEV0_EPF2_STRAP2': (53506, 5, {'strap_no_soft_reset_dev0_f2': (7, 7), 'strap_resize_bar_en_dev0_f2': (8, 8), 'strap_max_pasid_width_dev0_f2': (9, 13), 'strap_msi_pervector_mask_cap_dev0_f2': (14, 14), 'strap_aer_en_dev0_f2': (16, 16), 'strap_acs_en_dev0_f2': (17, 17), 'strap_cpl_abort_err_en_dev0_f2': (20, 20), 'strap_dpa_en_dev0_f2': (21, 21), 'strap_vc_en_dev0_f2': (23, 23), 'strap_msi_multi_cap_dev0_f2': (24, 26), 'strap_pasid_en_dev0_f2': (28, 28), 'strap_pasid_exe_permission_supported_dev0_f2': (29, 29), 'strap_pasid_global_invalidate_supported_dev0_f2': (30, 30), 'strap_pasid_priv_mode_supported_dev0_f2': (31, 31)}), 'regRCC_DEV0_EPF2_STRAP20': (53524, 5, {}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_REQ': (36390, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), @@ -9084,6 +9105,7 @@ 'regS2A_DOORBELL_ENTRY_14_CTRL': (31374, 5, {'s2a_doorbell_port14_enable': (0, 0), 's2a_doorbell_port14_awid': (1, 5), 's2a_doorbell_port14_fence_enable': (6, 6), 's2a_doorbell_port14_range_offset': (7, 16), 's2a_doorbell_port14_range_size': (17, 24), 's2a_doorbell_port14_64bit_support_dis': (25, 25), 's2a_doorbell_port14_need_deduct_range_offset': (26, 26), 's2a_doorbell_port14_awaddr_31_28_value': (28, 31)}), 'regS2A_DOORBELL_ENTRY_15_CTRL': (31375, 5, {'s2a_doorbell_port15_enable': (0, 0), 's2a_doorbell_port15_awid': (1, 5), 's2a_doorbell_port15_fence_enable': (6, 6), 's2a_doorbell_port15_range_offset': (7, 16), 's2a_doorbell_port15_range_size': (17, 24), 's2a_doorbell_port15_64bit_support_dis': (25, 25), 's2a_doorbell_port15_need_deduct_range_offset': (26, 26), 's2a_doorbell_port15_awaddr_31_28_value': (28, 31)}), 'regXCC_DOORBELL_FENCE': (29708, 5, {'xcc_0_doorbell_fence': (0, 0), 'xcc_1_doorbell_fence': (1, 1), 'xcc_2_doorbell_fence': (2, 2), 'xcc_3_doorbell_fence': (3, 3), 'xcc_4_doorbell_fence': (4, 4), 'xcc_5_doorbell_fence': (5, 5), 'xcc_6_doorbell_fence': (6, 6), 'xcc_7_doorbell_fence': (7, 7), 'shub_slv_mode': (16, 16), 'rmote_cp_sent': (17, 17), 'cp_0_sent': (18, 18), 'cp_1_sent': (19, 19), 'cp_2_sent': (20, 20), 'cp_3_sent': (21, 21), 'cp_4_sent': (22, 22), 'cp_5_sent': (23, 23), 'cp_6_sent': (24, 24), 'cp_7_sent': (25, 25), 'remote_client_sent': (26, 26), 'remote_client_clr_pending': (27, 27)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), } nbio_7_11_0 = { 'regBIF_BX0_PCIE_INDEX2': (32782, 0, {'pcie_index2': (0, 31)}), @@ -9131,6 +9153,7 @@ 'regRCC_DEV0_EPF2_STRAP20': (53524, 5, {}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_REQ': (36390, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), 'regBIF_BX_PF1_GPU_HDP_FLUSH_DONE': (36391, 5, {'cp0': (0, 0), 'cp1': (1, 1), 'cp2': (2, 2), 'cp3': (3, 3), 'cp4': (4, 4), 'cp5': (5, 5), 'cp6': (6, 6), 'cp7': (7, 7), 'cp8': (8, 8), 'cp9': (9, 9), 'sdma0': (10, 10), 'sdma1': (11, 11), 'rsvd_eng0': (12, 12), 'rsvd_eng1': (13, 13), 'rsvd_eng2': (14, 14), 'rsvd_eng3': (15, 15), 'rsvd_eng4': (16, 16), 'rsvd_eng5': (17, 17), 'rsvd_eng6': (18, 18), 'rsvd_eng7': (19, 19), 'rsvd_eng8': (20, 20), 'rsvd_eng9': (21, 21), 'rsvd_eng10': (22, 22), 'rsvd_eng11': (23, 23), 'rsvd_eng12': (24, 24), 'rsvd_eng13': (25, 25), 'rsvd_eng14': (26, 26), 'rsvd_eng15': (27, 27), 'rsvd_eng16': (28, 28), 'rsvd_eng17': (29, 29), 'rsvd_eng18': (30, 30), 'rsvd_eng19': (31, 31)}), + 'regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL': (247, 2, {'hdp_mem_flush_addr': (0, 0)}), } mp_11_0_0 = { 'mmMP0_SMN_C2PMSG_32': (96, 0, {'content': (0, 31)}), diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index b3094e11acdc6..8d8c48f8de01a 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -897,7 +897,7 @@ def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return ((self.drm_dev_info.cu class PCIIface(PCIIfaceBase): def __init__(self, dev, dev_id): - super().__init__(dev, dev_id, vendor=0x1002, devices=((0xffff, (0x74a1,0x744c,0x7480,0x7550,0x7551,0x7590,0x75a0,0x75a8)),), vram_bar=0, + super().__init__(dev, dev_id, vendor=0x1002, devices=((0xffff, (0x74a1,0x74b5,0x744c,0x7480,0x7550,0x7551,0x7590,0x75a0,0x75a8)),), vram_bar=0, va_start=AMMemoryManager.va_allocator.base, va_size=AMMemoryManager.va_allocator.size, dev_impl_t=AMDev) self._compute_props() @@ -1079,6 +1079,10 @@ def __init__(self, device:str=""): self.sqtt_wptrs = self.allocator.alloc(round_up(self.se_cnt * self.xccs * 4, 0x1000), BufferSpec(cpu_access=True, nolru=True)) self.sqtt_next_cmd_id = itertools.count(0) + if self.is_am(): + self.iface.dev_impl.gmc.vf_owner = self + if self.iface.dev_impl.vf_access: self.iface.dev_impl.release_vf_access() + def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0, idx=0): ring = self.iface.alloc(ring_size, uncached=True, cpu_access=True) gart = self.iface.alloc(0x100, uncached=True, cpu_access=True) diff --git a/tinygrad/runtime/support/am/amdev.py b/tinygrad/runtime/support/am/amdev.py index e28190aaf9238..9e7d02452de84 100644 --- a/tinygrad/runtime/support/am/amdev.py +++ b/tinygrad/runtime/support/am/amdev.py @@ -1,6 +1,6 @@ from __future__ import annotations -import ctypes, collections, dataclasses, functools, hashlib, array -from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw, to_mv +import ctypes, collections, dataclasses, functools, hashlib, array, contextlib +from tinygrad.helpers import mv_address, getenv, DEBUG, lo32, hi32, fetch_fw, to_mv, wait_cond from tinygrad.runtime.autogen import pci from tinygrad.runtime.autogen.am import am, fw from tinygrad.runtime.support.amd import AMDReg, import_module, import_asic_regs @@ -14,10 +14,11 @@ class AMRegister(AMDReg): adev:AMDev - def read(self, inst=0): return self.adev.rreg(self.addr[inst]) + def read(self, inst=0, direct=False): return self.adev.rreg(self.addr[inst], inst=inst, direct=direct) def read_bitfields(self, inst=0) -> dict[str, int]: return self.decode(self.read(inst=inst)) - def write(self, _am_val:int=0, inst=0, **kwargs): self.adev.wreg(self.addr[inst], _am_val | self.encode(**kwargs)) + def write(self, _am_val:int=0, inst=0, direct=False, **kwargs): + self.adev.wreg(self.addr[inst], _am_val|self.encode(**kwargs), inst=inst, direct=direct) def update(self, inst=0, **kwargs): self.write(self.read(inst=inst) & ~self.fields_mask(*kwargs.keys()), inst=inst, **kwargs) @@ -159,6 +160,12 @@ def __init__(self, pci_dev:PCIDevice, reset_mode=False): self._disable_aspm() self.vram, self.doorbell64, self.mmio = self.pci_dev.map_bar(0), self.pci_dev.map_bar(2, fmt='Q'), self.pci_dev.map_bar(5, fmt='I') + # VF related + self.is_vf = bool(self.mmio[am.mmRCC_IOV_FUNC_IDENTIFIER] & 1) + self.vf_mailbox = self.mmio.view(am.NV_MAIBOX_CONTROL_TRN_OFFSET_BYTE, 2, fmt='B') + self.vf_access = self._vf_mailbox_request(am.IDH_REQ_GPU_INIT_ACCESS) if self.is_vf else 0 + self.vf_rlc_gated:list[tuple[int, int]] = [] + self._run_discovery() self._build_regs() @@ -185,14 +192,14 @@ def __init__(self, pci_dev:PCIDevice, reset_mode=False): # Init hw for IP blocks where it is needed if not self.partial_boot: - if self.psp.is_sos_alive() and self.smu.is_smu_alive(): + if not self.is_vf and self.psp.is_sos_alive() and self.smu.is_smu_alive(): # skip in vf mode, these are pf funcs. self.pci_dev.write_config_flush(pci.PCI_COMMAND, self.pci_dev.read_config(pci.PCI_COMMAND, 2) & ~pci.PCI_COMMAND_MASTER, 2) if self.is_hive(): if reset_mode: return # in reset mode, do not raise raise RuntimeError("Malformed state. Use extra/amdpci/hive_reset.py to reset the hive") self.smu.mode1_reset() self.pci_dev.write_config_flush(pci.PCI_COMMAND, self.pci_dev.read_config(pci.PCI_COMMAND, 2) | pci.PCI_COMMAND_MASTER, 2) - self.init_hw(self.soc, self.gmc, self.ih, self.psp, self.smu) + self.init_hw(self.soc, self.gmc, self.ih, *(() if self.is_vf else (self.psp, self.smu))) # Booting done self.is_booting = False @@ -200,13 +207,15 @@ def __init__(self, pci_dev:PCIDevice, reset_mode=False): # Re-initialize main blocks self.init_hw(self.gfx, self.sdma) - if (max_power:=getenv("AM_POWER_LIMIT", 0.0)) > 0: - self.smu.set_power_limit(max_power) - self.smu.set_clocks(level=None) - else: self.smu.set_clocks(level=-1) # last level, max perf. - for ip in [self.soc, self.gfx]: ip.set_clockgating_state() - self.reg("regSCRATCH_REG7").write(AMDev.Version) - self.reg("regSCRATCH_REG6").write(1) # set initialized state. + if not self.is_vf: # skip in vf mode, these are pf funcs. + if (max_power:=getenv("AM_POWER_LIMIT", 0.0)) > 0: + self.smu.set_power_limit(max_power) + self.smu.set_clocks(level=None) + else: self.smu.set_clocks(level=-1) # last level, max perf. + for ip in [self.soc, self.gfx]: ip.set_clockgating_state() + self.reg("regSCRATCH_REG7").write(AMDev.Version) + self.reg("regSCRATCH_REG6").write(1) # set initialized state. + if DEBUG >= 2: print(f"am {self.devfmt}: boot done") def init_sw(self, smi_dev=False): @@ -237,10 +246,33 @@ def init_hw(self, *blocks:AM_IP): def fini(self): if DEBUG >= 2: print(f"am {self.devfmt}: Finalizing") + # a VF may only touch the engines inside an access window, take one so the host does not have to FLR the VF later + if self.is_vf and not self.vf_access: + with contextlib.suppress(TimeoutError): self.vf_access = self._vf_mailbox_request(am.IDH_REQ_GPU_FINI_ACCESS) for ip in [self.sdma, self.gfx]: ip.fini_hw() - self.smu.set_clocks(level=0) + if not self.is_vf: self.smu.set_clocks(level=0) self.ih.interrupt_handler() - self.reg("regSCRATCH_REG6").write(self.is_err_state) # set finalized state. + if not self.is_vf: self.reg("regSCRATCH_REG6").write(self.is_err_state) # set finalized state. + if self.vf_access: self.release_vf_access() + + def release_vf_access(self): + rel, self.vf_access = self.vf_access, 0 # give back the same lease that was taken + with contextlib.suppress(TimeoutError): self._vf_mailbox_request(rel, wait_ready=False) + + def _vf_mailbox_request(self, req:int, wait_ready=True) -> int: + self.vf_mailbox[0] = 0 # drop TRN_MSG_VALID + + wait_cond(lambda: self.vf_mailbox[0] & 2, value=0, timeout_ms=1000, msg="VF mailbox acknowledgement did not clear") + for i, val in enumerate((req, 0, 0, 0)): self.mmio[am.mmMAILBOX_MSGBUF_TRN_DW0 + i] = val + + self.vf_mailbox[0] = 1 # set TRN_MSG_VALID + wait_cond(lambda: self.vf_mailbox[0] & 2, value=2, timeout_ms=am.NV_MAILBOX_POLL_ACK_TIMEDOUT, msg=f"VF mailbox request {req:#x} was not acked") + self.vf_mailbox[0] = 0 + if wait_ready: + wait_cond(lambda: self.mmio[am.mmMAILBOX_MSGBUF_RCV_DW0], value=am.IDH_READY_TO_ACCESS_GPU, timeout_ms=am.NV_MAILBOX_POLL_MSG_TIMEDOUT, + msg="VF mailbox: the pf never granted access") + self.vf_mailbox[1] = 2 # ack + return req + 1 def recover(self, force=False) -> bool: if not force and not self.is_err_state: return False @@ -260,20 +292,36 @@ def xgmi2paddr(self, xgmi_paddr:int) -> int: return xgmi_paddr - self.gmc.paddr_ def reg(self, reg:str) -> AMRegister: return self.__dict__[reg] - def rreg(self, reg:int) -> int: + def rreg(self, reg:int, inst=0, direct=False) -> int: + if not direct and any(lo <= reg <= hi for lo, hi in self.vf_rlc_gated): return self.rlcg_rw(reg, 0, inst, read=True) val = self.indirect_rreg(reg) if reg >= len(self.mmio) else self.mmio[reg] if AM_DEBUG >= 4 and getattr(self, '_prev_rreg', None) != (reg, val): print(f"am {self.devfmt}: Reading register {reg:#x} with value {val:#x}") self._prev_rreg = (reg, val) return val - def wreg(self, reg:int, val:int): + def wreg(self, reg:int, val:int, inst=0, direct=False): if AM_DEBUG >= 4: print(f"am {self.devfmt}: Writing register {reg:#x} with value {val:#x}") - if reg >= len(self.mmio): self.indirect_wreg(reg, val) + if not direct and any(lo <= reg <= hi for lo, hi in self.vf_rlc_gated): self.rlcg_rw(reg, val, inst) + elif reg >= len(self.mmio): self.indirect_wreg(reg, val) else: self.mmio[reg] = val - def wreg_pair(self, reg_base:str, lo_suffix:str, hi_suffix:str, val:int, inst:int=0): - self.reg(f"{reg_base}{lo_suffix}").write(lo32(val), inst=inst) - self.reg(f"{reg_base}{hi_suffix}").write(hi32(val), inst=inst) + def rlcg_rw(self, addr:int, val:int, inst:int, read=False) -> int: + # the rlc gateway takes the grbm selection through its own scratch registers + if addr in {self.reg("regGRBM_GFX_CNTL").addr[inst], self.reg("regGRBM_GFX_INDEX").addr[inst]}: + self.reg("regSCRATCH_REG2" if addr == self.reg("regGRBM_GFX_CNTL").addr[inst] else "regSCRATCH_REG3").write(val, inst=inst, direct=True) + return val + + self.wreg_pair("regSCRATCH_REG", "0", "1", (addr | (0x1 << 28 if read else 0)) << 32 | val, inst=inst, direct=True) + self.reg("regRLC_SPARE_INT").write(1, inst=inst, direct=True) + wait_cond(lambda: self.reg("regSCRATCH_REG1").read(inst=inst, direct=True) & 0xFFFFF, value=0, msg=f"RLC gateway timeout on {addr:#x}") + + if AM_DEBUG >= 1 and (err:=self.reg("regSCRATCH_REG1").read(inst=inst, direct=True) & 0xF000000): + print(f"am {self.devfmt}: RLC gateway refused {addr:#x}: {err:#x}") + return self.reg("regSCRATCH_REG0").read(inst=inst, direct=True) + + def wreg_pair(self, reg_base:str, lo_suffix:str, hi_suffix:str, val:int, inst:int=0, direct=False): + self.reg(f"{reg_base}{lo_suffix}").write(lo32(val), inst=inst, direct=direct) + self.reg(f"{reg_base}{hi_suffix}").write(hi32(val), inst=inst, direct=direct) def indirect_rreg(self, reg:int) -> int: self.reg("regBIF_BX_PF0_RSMU_INDEX").write(reg * 4) @@ -353,7 +401,11 @@ def _build_regs(self): if self.ip_ver[am.SDMA0_HWIP] in {(4,4,2), (4,4,4)}: mods += [("sdma", am.SDMA0_HWIP)] for prefix, hwip in mods: - self.__dict__.update(import_asic_regs(prefix, self.ip_ver[hwip], cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[hwip]))) + regs = import_asic_regs(prefix, self.ip_ver[hwip], cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[hwip])) + self.__dict__.update(regs) + if prefix == "gc" and self.is_vf: + ext = {seg: max(r.offset for r in regs.values() if r.segment == seg) for seg in {r.segment for r in regs.values()}} + self.vf_rlc_gated = sorted((bases[seg], bases[seg] + off) for bases in self.regs_offset[hwip].values() for seg, off in ext.items()) self.__dict__.update(import_asic_regs('mp', (11, 0, 0), cls=functools.partial(AMRegister, adev=self, bases=self.regs_offset[am.MP1_HWIP]))) # Live AIDs like the kernel: 4 SDMAs per AID; the AID lives iff its group's alive-mask is 0xf/0x3/0xc. diff --git a/tinygrad/runtime/support/am/ip.py b/tinygrad/runtime/support/am/ip.py index 4f9a585449171..6155542c9e8cb 100644 --- a/tinygrad/runtime/support/am/ip.py +++ b/tinygrad/runtime/support/am/ip.py @@ -1,5 +1,5 @@ import ctypes, time, contextlib, functools -from typing import Iterable, Literal +from typing import Any, Iterable, Literal from tinygrad.helpers import to_mv, data64, lo32, hi32, DEBUG, wait_cond, pad_bytes, getbits from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.amd import import_soc @@ -50,6 +50,8 @@ def doorbell_enable(self, port, awid=0, awaddr_31_28_value=0, offset=0, size=0, else: reg.write(val) class AM_GMC(AM_IP): + vf_owner:Any = None + def init_sw(self): self.vmhubs = len(self.adev.regs_offset[am.MMHUB_HWIP]) @@ -86,22 +88,36 @@ def init_sw(self): def init_hw(self): self.init_hub("MM", insts=self.mm_insts) - def flush_hdp(self): self.adev.wreg(self.adev.reg("regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL").read() // 4, 0x0) + def flush_hdp(self): + if self.adev.is_vf: self.adev.reg("regBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL").write(0x0) + else: self.adev.wreg(self.adev.reg("regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL").read() // 4, 0x0) + def flush_tlb(self, ip:Literal["MM", "GC"], vmid, flush_type=0): self.flush_hdp() # Can't issue TLB invalidation if the hub isn't initialized. if not self.hub_initted[ip]: return + req = self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").encode(flush_type=flush_type, per_vmid_invalidate_req=(1 << vmid), + invalidate_l2_ptes=1, invalidate_l2_pde0=1, invalidate_l2_pde1=1, invalidate_l2_pde2=1, invalidate_l1_ptes=1, + clear_protection_fault_status_addr=0) + + if ip == "GC" and self.adev.is_vf and (dev:=self.vf_owner) is not None: # the cp runs invalidations once its queues are up + from tinygrad.runtime.ops_amd import WAIT_REG_MEM_FUNCTION_EQ + dev.hw_compute_queue_t().wait_reg_mem(req, mask=1 << vmid, reg_done=self.adev.regGCVM_INVALIDATE_ENG17_ACK.addr[0], + reg=self.adev.regGCVM_INVALIDATE_ENG17_REQ.addr[0], op=WAIT_REG_MEM_FUNCTION_EQ).signal(dev.timeline_signal, dev.next_timeline()).submit(dev) + dev.timeline_signal.wait(dev.timeline_value - 1) + return + + use_sema = ip == "MM" and not self.adev.is_vf # vf can't use sema for inst in (self.adev.gmc.mm_insts if ip == "MM" else range(self.adev.gfx.xccs)): - if ip == "MM": wait_cond(lambda: self.adev.regMMVM_INVALIDATE_ENG17_SEM.read(inst=inst) & 0x1, value=1, msg="mm flush_tlb timeout") + if use_sema: wait_cond(lambda: self.adev.regMMVM_INVALIDATE_ENG17_SEM.read(inst=inst) & 0x1, value=1, msg="mm flush_tlb timeout") - self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").write(flush_type=flush_type, per_vmid_invalidate_req=(1 << vmid), invalidate_l2_ptes=1, - invalidate_l2_pde0=1, invalidate_l2_pde1=1, invalidate_l2_pde2=1, invalidate_l1_ptes=1, clear_protection_fault_status_addr=0, inst=inst) + self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_REQ").write(req, inst=inst) wait_cond(lambda: self.adev.reg(f"reg{ip}VM_INVALIDATE_ENG17_ACK").read(inst=inst) & (1 << vmid), value=(1 << vmid), msg="flush_tlb timeout") - if ip == "MM": self.adev.regMMVM_INVALIDATE_ENG17_SEM.write(0x0, inst=inst) + if use_sema: self.adev.regMMVM_INVALIDATE_ENG17_SEM.write(0x0, inst=inst) if self.adev.ip_ver[am.GC_HWIP] >= (11,0,0) and ip == "MM": self.adev.regMMVM_L2_BANK_SELECT_RESERVED_CID2.update(reserved_cache_private_invalidation=1, inst=inst) @@ -256,8 +272,9 @@ def init_sw(self): def init_hw(self): # Wait for RLC autoload to complete - wait_cond(lambda: self.adev.regCP_STAT.read() == 0 or self.adev.regRLC_RLCS_BOOTLOAD_STATUS.read_bitfields()['bootload_complete'] == 0, - value=True, msg="RLC autoload timeout") + if not self.adev.is_vf: # VF boots with the RLC already up + wait_cond(lambda: self.adev.regCP_STAT.read() == 0 or self.adev.regRLC_RLCS_BOOTLOAD_STATUS.read_bitfields()['bootload_complete'] == 0, + value=True, msg="RLC autoload timeout") self.adev.gmc.init_hub("GC", insts=range(self.xccs)) if self.adev.partial_boot: return self.reset_mec() @@ -301,8 +318,11 @@ def init_hw(self): self._enable_mec() - # Set 1 partition - if self.xccs > 1: self.adev.psp._spatial_partition_cmd(1) + if self.adev.is_vf: # the host PF shuts a VF down when it leaves its access window with no cp scheduler, point the RLC at the kiq slot + for xcc in range(self.xccs): self.adev.reg("regRLC_CP_SCHEDULERS").update(scheduler0=(2 << 5) | (1 << 3) | 0x80, inst=xcc) + + # set 1 partition on bare metal. a VF uses the spatial partition its host PF assigned. + if self.xccs > 1 and not self.adev.is_vf: self.adev.psp._spatial_partition_cmd(1) def fini_hw(self): self._dequeue_hqds() @@ -344,7 +364,7 @@ def setup_ring(self, ring_addr:int, ring_size:int, rptr_addr:int, wptr_addr:int, mqd_st_mv = to_mv(ctypes.addressof(mqd_struct), ctypes.sizeof(mqd_struct)).cast('I') for i, reg in enumerate(range(self.adev.regCP_MQD_BASE_ADDR.addr[xcc], self.adev.regCP_HQD_PQ_WPTR_HI.addr[xcc] + 1)): - self.adev.wreg(reg, mqd_st_mv[0x80 + i]) + self.adev.wreg(reg, mqd_st_mv[0x80 + i], inst=xcc) self.adev.regCP_HQD_ACTIVE.write(0x1, inst=xcc) self.adev.gmc.flush_hdp() @@ -489,6 +509,7 @@ def interrupt_handler(self): self.drain() + if self.adev.is_vf: return # fatal RAS events are handled by the host PF bif_intr = self.adev.regBIF_BX0_BIF_DOORBELL_INT_CNTL.read_bitfields() athub_err, cntlr_err = bif_intr['ras_athub_err_event_interrupt_status'], bif_intr['ras_cntlr_interrupt_status'] if athub_err or cntlr_err: From 49627aecb57c1290b211e3e65f83c280e596905f Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Fri, 28 Aug 2026 10:53:19 -0700 Subject: [PATCH 262/306] ci: use fetch to download libraries (#17814) --- .github/actions/setup-tinygrad/action.yml | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index 372f227e30db7..e30bc8a470eca 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -248,10 +248,10 @@ runs: if: inputs.amd == 'true' && runner.os == 'macOS' shell: bash run: | - sudo mkdir -p /usr/local/lib - curl -s -H "Authorization: token $GH_TOKEN" curl -s https://api.github.com/repos/tinygrad/amdcomgr_dylib/releases/latest | \ - jq -r '.assets[] | select(.name == "libamd_comgr.dylib").browser_download_url' | \ - sudo xargs curl -fL -o /usr/local/lib/libamd_comgr.dylib + sudo "$VIRTUAL_ENV/bin/python" -c " + from tinygrad.helpers import fetch + fetch('https://github.com/tinygrad/amdcomgr_dylib/releases/download/v7.2.0/libamd_comgr.dylib', name='/usr/local/lib/libamd_comgr.dylib', + sha256='7712fbe4fcb9fcdea49aeac989876448df975ce0a8ce7c9b15b55c15e7a05935').chmod(0o644)" # **** CUDA **** - name: Install CUDA @@ -269,8 +269,11 @@ runs: if: inputs.ocelot == 'true' shell: bash run: | - sudo mkdir -p /usr/local/lib - sudo curl --output-dir /usr/local/lib -fLO https://github.com/tinygrad/gpuocelot/releases/download/v0.1.0/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }} + sudo "$VIRTUAL_ENV/bin/python" -c " + from tinygrad.helpers import fetch + fetch('https://github.com/tinygrad/gpuocelot/releases/download/v0.1.0/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', + name='/usr/local/lib/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', + sha256='${{ runner.os == 'Linux' && 'a24705276a9a187111371465987b3258f8836ef512a34266e3075bc4714e125a' || '5106c998c795a36dec79eb7b2aae324a93d1338236d36eeaae232649ec457663' }}').chmod(0o644)" # **** WebGPU **** @@ -278,8 +281,11 @@ runs: if: inputs.webgpu == 'true' shell: bash run: | - sudo mkdir -p /usr/local/lib - sudo curl --output-dir /usr/local/lib -fLO https://github.com/wpmed92/pydawn/releases/download/v0.1.6/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }} + sudo "$VIRTUAL_ENV/bin/python" -c " + from tinygrad.helpers import fetch + fetch('https://github.com/wpmed92/pydawn/releases/download/v0.1.6/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', + name='/usr/local/lib/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', + sha256='${{ runner.os == 'Linux' && 'cf36091d266a32c9d5080f14662de44cece241987939713282ea0ff558db81c6' || '7e87c7acefda8b6af1a1c5debfedcf62958311284b8fd8d9bcf93e312e6636e3' }}').chmod(0o644)" # **** LLVM **** @@ -293,7 +299,10 @@ runs: if: inputs.opencl == 'true' shell: bash run: | - sudo curl -fL https://github.com/sirhcm/tinymesa/releases/download/rusticl-v1/libRusticlOpenCL.so.1.0.0 -o /usr/lib/libRusticlOpenCL.so + sudo "$VIRTUAL_ENV/bin/python" -c " + from tinygrad.helpers import fetch + fetch('https://github.com/sirhcm/tinymesa/releases/download/rusticl-v1/libRusticlOpenCL.so.1.0.0', name='/usr/lib/libRusticlOpenCL.so', + sha256='d4f48566d8fd33f6cdd8ef6de35a71966e8a8517e6f68ff3c52dbb43765a2513').chmod(0o644)" sudo mkdir -p /etc/OpenCL/vendors echo "/usr/lib/libRusticlOpenCL.so" | sudo tee /etc/OpenCL/vendors/rusticl.icd echo "RUSTICL_ENABLE=llvmpipe" >> "$GITHUB_ENV" From b3dad4fa0cbdaf1c7b3f2d3823ed7e7a83fc8e27 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 14:30:34 -0400 Subject: [PATCH 263/306] update X86 cast related stuff (#17816) --- test/backend/test_dtype.py | 5 ++++- tinygrad/codegen/late/regalloc.py | 2 +- tinygrad/renderer/isa/__init__.py | 2 +- tinygrad/renderer/isa/x86.py | 14 +++++--------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/test/backend/test_dtype.py b/test/backend/test_dtype.py index 7ce56ad3cdfe6..27de82721c0d3 100644 --- a/test/backend/test_dtype.py +++ b/test/backend/test_dtype.py @@ -328,7 +328,10 @@ def test_uint16_to_int8_overflow(self): class TestInt32DType(TestDType): DTYPE = dtypes.int32 class TestUint32DType(TestDType): DTYPE = dtypes.uint32 -class TestInt64DType(TestDType): DTYPE = dtypes.int64 +class TestInt64DType(TestDType): + DTYPE = dtypes.int64 + def test_int64_to_uint32_to_int64(self): + _test_op(lambda: Tensor([0x12345678ABCDEF01], dtype=dtypes.int64).cast(dtypes.uint32).cast(dtypes.int64), dtypes.int64, [2882400001]) @unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, PTXRenderer), "PTX does indexing math with longs") class TestEmulatedInt64DType(TestInt64DType): diff --git a/tinygrad/codegen/late/regalloc.py b/tinygrad/codegen/late/regalloc.py index b43c934c844aa..611f2780c4438 100644 --- a/tinygrad/codegen/late/regalloc.py +++ b/tinygrad/codegen/late/regalloc.py @@ -4,7 +4,7 @@ from tinygrad.renderer.isa import ISARenderer, Register, greg from tinygrad.dtype import dtypes -PSEUDO_OPS = {Ops.CONST, Ops.CAST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} +PSEUDO_OPS = {Ops.CONST, Ops.CAST, Ops.BITCAST, Ops.NOOP, Ops.AFTER, Ops.BARRIER, Ops.GROUP, Ops.STACK} class LinearScanRegallocContext: # returns the uop that defines the virtual register diff --git a/tinygrad/renderer/isa/__init__.py b/tinygrad/renderer/isa/__init__.py index fb74f1569def1..a9d1b38eea1d3 100644 --- a/tinygrad/renderer/isa/__init__.py +++ b/tinygrad/renderer/isa/__init__.py @@ -27,7 +27,7 @@ def vreg(self, cons:tuple[Register, ...]|Register): return Register(f"v{next(self.reg_n)}", 0, _cons=cons if isinstance(cons, tuple) else (cons,)) def greg(u:UOp): - if u.op in {Ops.NOOP, Ops.AFTER} and u.src: return greg(u.src[0]) + if u.op in {Ops.NOOP, Ops.AFTER, Ops.BITCAST} and u.src: return greg(u.src[0]) if isinstance(u.tag, tuple): return u.tag[0] return u.tag diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index f359ae253f7c7..256e9aacc3eb8 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -182,15 +182,10 @@ def flag_gate(m:UOp) -> UOp|None: # legalize the new style graph for isel. NOTE: this runs after the spec is verified, some of these rewrites violate it pre_isel_matcher = PatternMatcher([ - # noop casts: zero extending scalar 32bit int, same-width signed/unsigned, narrowing scalar int - (UPat.var("y", dtypes.uint32).cast(dtypes.int64s, name="x"), lambda y,x: x.replace(op=Ops.NOOP, arg=None) if y.max_numel() == 1 else None), + # widening a scalar uint32 is free, the 32bit write that produced it already zeroed the upper half + (UPat.var("y", dtypes.uint32).cast(dtypes.int64s, name="x"), lambda y,x: x.replace(op=Ops.BITCAST) if y.max_numel() == 1 else None), (UPat.var("y", dtypes.ints+(dtypes.bool,)).cast(dtypes.ints, name="x"), - lambda y,x: x.replace(op=Ops.NOOP, arg=None) if x.dtype.itemsize == y.dtype.itemsize and y.max_numel() == 1 else None), - (UPat.var("y", dtypes.ints).cast(dtypes.ints, name="x"), - lambda y,x: x.replace(op=Ops.NOOP, arg=None) if x.dtype.itemsize < y.dtype.itemsize and y.max_numel() == 1 else None), - # bitcasts between scalar floats and ints are real, rest are noops - (UPat.var("y").bitcast().named("x"), lambda y,x: None if y.dtype in dtypes.floats and x.dtype in dtypes.ints or \ - y.dtype in dtypes.ints and x.dtype in dtypes.floats else x.replace(op=Ops.NOOP, arg=None)), + lambda y,x: x.replace(op=Ops.BITCAST) if x.dtype.itemsize == y.dtype.itemsize else None), # gated load/store become a conditional move on the address, the load/store are unconditional (UPat((Ops.INDEX, Ops.SHRINK), name="addr").load(UPat.var("alt"), UPat.var("gate"), name="x"), gated_load), (UPat((Ops.INDEX, Ops.SHRINK), name="addr").store(UPat.var("val"), UPat.var("gate")), gated_store), @@ -503,11 +498,12 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat.var("y", dtypes.float64).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSD2SS, src=(y, y))), (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float32, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SS, src=(def_reg(x.dtype), y))), (UPat.var("y", (dtypes.int32, dtypes.int64)).cast(dtypes.float64, name="x"), lambda y,x: x.ins(X86Ops.VCVTSI2SD, src=(def_reg(x.dtype), y))), - (UPat(dtype=dtypes.uints+(dtypes.bool,)).cast(dtypes.ints, name="x"), lambda x: + (UPat(dtype=(dtypes.uint8, dtypes.uint16, dtypes.bool)).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOVZX) if x.max_numel() == 1 and x.src[0].dtype.itemsize < x.dtype.itemsize else None), (UPat(dtype=dtypes.int32).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.MOVSXD) if x.max_numel() == 1 else None), (UPat(dtype=dtypes.sints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOVSX) if x.max_numel() == 1 and x.src[0].dtype.itemsize < x.dtype.itemsize else None), + (UPat(dtype=dtypes.ints).cast(dtypes.ints, name="x"), lambda x: x.ins(X86Ops.MOV) if x.max_numel() == 1 else None), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int16s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBW)), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int32s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBD)), (UPat(dtype=(dtypes.uint8, dtypes.bool)).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VPMOVZXBQ)), From bd2711b7c1e4f0a5d9e7ca38f842242f2aa7fc32 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 15:05:17 -0400 Subject: [PATCH 264/306] put dtype to Ops.INS arg (#17817) --- docs/abstractions4.py | 2 +- extra/gemm/amd_asm_matmul.py | 2 +- extra/gemm/cdna_asm_gemm.py | 2 +- extra/gemm/rdna4_asm_matmul.py | 2 +- extra/hcq2/ops_amd2.py | 20 ++--- extra/mmapeak/mmapeak.py | 4 +- test/amd/test_asm_kernel.py | 12 +-- test/backend/test_encodings.py | 4 +- test/backend/test_isel.py | 8 +- test/external/external_test_gpu_crash.py | 2 +- .../external_test_schedule_scaling.py | 4 +- test/null/test_viz.py | 2 +- tinygrad/codegen/__init__.py | 2 +- tinygrad/renderer/amd/elf.py | 2 +- tinygrad/renderer/isa/x86.py | 80 +++++++++---------- tinygrad/runtime/ops_cpu.py | 10 +-- tinygrad/runtime/support/hcq2.py | 12 +-- tinygrad/uop/ops.py | 12 +-- tinygrad/uop/spec.py | 4 +- 19 files changed, 95 insertions(+), 91 deletions(-) diff --git a/docs/abstractions4.py b/docs/abstractions4.py index d28a3e3a3474c..9e00f732f9f60 100644 --- a/docs/abstractions4.py +++ b/docs/abstractions4.py @@ -122,7 +122,7 @@ def finalize(self, sink:UOp) -> UOp: offset_dwords = (self.labels[inst._target] - inst._pos - inst.size()) // 4 if not -32768 <= offset_dwords <= 32767: raise ValueError(f"branch to '{inst._target}' offset {offset_dwords} exceeds simm16 range") inst.simm16 = offset_dwords - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in self.instructions])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in self.instructions])))) CU_COUNT = 32 LANES = 64 diff --git a/extra/gemm/amd_asm_matmul.py b/extra/gemm/amd_asm_matmul.py index a6c79df7436cb..2feabab25f2de 100644 --- a/extra/gemm/amd_asm_matmul.py +++ b/extra/gemm/amd_asm_matmul.py @@ -462,7 +462,7 @@ def asm_kernel(A:UOp, B:UOp, C:UOp) -> UOp: lds = UOp.placeholder((lds_size,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(A.base, B.base, C.base, lds, *gidxs, *lidxs, arg=KernelInfo(name=colored("kernel", "cyan"), estimates=Estimates(ops=N*N*N*2, mem=N*N*4*3))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) c = Tensor.custom_kernel(a, b, c, fxn=asm_kernel)[2] linear = c.schedule_linear() diff --git a/extra/gemm/cdna_asm_gemm.py b/extra/gemm/cdna_asm_gemm.py index e94cf6c2520cf..8c1f18aee6d8d 100644 --- a/extra/gemm/cdna_asm_gemm.py +++ b/extra/gemm/cdna_asm_gemm.py @@ -125,7 +125,7 @@ def custom_mxfp4_gemm(C:UOp, A:UOp, B:UOp, scale_a:UOp, scale_b:UOp, *extra:UOp, arg=KernelInfo(f"mxfp4_gemm_{M}_{N}_{K}", estimates=Estimates(ops=2*M*N*K, mem=(M*half_k+N*half_k)*A.dtype.itemsize+M*N*C.dtype.itemsize))) insts = build_kernel(M, N, K, tile_m, tile_n) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=x) for x in insts)))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts)))) def _mxfp4_gemm_quantized(a_q:Tensor, b_q:Tensor, scale_a:Tensor, scale_b:Tensor) -> Tensor: M, half_k = a_q.shape diff --git a/extra/gemm/rdna4_asm_matmul.py b/extra/gemm/rdna4_asm_matmul.py index 4f655bcefed47..5b1817fd7d4e8 100644 --- a/extra/gemm/rdna4_asm_matmul.py +++ b/extra/gemm/rdna4_asm_matmul.py @@ -223,7 +223,7 @@ def asm_kernel(A, B, C): lds = UOp.placeholder((lds_size,), dtypes.uint8, 0, AddrSpace.LOCAL) sink = UOp.sink(A.base, B.base, C.base, lds, *gidxs, *lidxs, arg=KernelInfo(name=colored("kernel","cyan"), estimates=Estimates(ops=N*N*N*2, mem=N*N*2*3))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) c = Tensor.custom_kernel(a, b, c, fxn=asm_kernel)[2] linear = c.schedule_linear() diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index bfd238210ab65..538d8d4edbda3 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -37,7 +37,7 @@ class PM4Ops(FastEnum): RELEASE_MEM = auto(); DISPATCH_DIRECT = auto(); EVENT_WRITE = auto() # noqa: E702 def pkt3(ctx, op:PM4Ops, *vals): - return UOp(Ops.INS, arg=op, src=tuple(UOp.const(x, dtypes.uint32) + return UOp(Ops.INS, arg=(op, dtypes.void), src=tuple(UOp.const(x, dtypes.uint32) for x in (ctx.pm4.PACKET3(getattr(ctx.pm4, f"PACKET3_{op.name}"), len(vals) - 1), *vals))) def wreg(ctx, reg:AMDReg, *args:sint, **kwargs:int): @@ -140,10 +140,10 @@ def pm4_program(ctx, call, prg): pm_pm4_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), pm4_program), - (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), - (UPat(Ops.INS, arg="barrier"), pm4_barrier), - (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),)), pm4_timestamp), - (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), pm4_barrier), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), pm4_timestamp), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), ]) def queue_ptrs(devs, qname:str, q:AMDQueueDesc) -> tuple[UOp, ...]: @@ -207,10 +207,10 @@ def sdma_timestamp(ctx, ins, dst): pm_sdma_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), sdma_copy), - (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP)), - (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val")), name="ins"), sdma_wait), - (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),), name="ins"), sdma_timestamp), - (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val")), name="ins"), sdma_store), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.NOOP)), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val")), name="ins"), sdma_wait), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),), name="ins"), sdma_timestamp), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val")), name="ins"), sdma_store), ]) def sdma_submit(cmdbuf, devs): @@ -254,7 +254,7 @@ def amd_usb_submit(ctx, lin): if nb:=usb_arm_bytes(ctx.pre, Device[ctx.devs[0]].iface.usb_sram): poke = (ctx.sdma.SDMA_OP_WRITE, *data64_le(Device[ctx.devs[0]].iface.cq_buf.va_addr + 12), 0, 0) - lin = lin.replace(src=lin.src + (UOp(Ops.INS, arg="poke", src=tuple(UOp.const(x, dtypes.uint32) for x in poke)),)) + lin = lin.replace(src=lin.src + (UOp(Ops.INS, arg=("poke", dtypes.void), src=tuple(UOp.const(x, dtypes.uint32) for x in poke)),)) ib_host, ib_gpu, pkt_dw = usb_ib(ctx.devs, lin, 32 if comp else 0x100, nb) pkt = (ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER,2),*data64_le(ib_gpu.getaddr(ctx.devs)),pkt_dw|ctx.pm4.INDIRECT_BUFFER_VALID) if comp else () diff --git a/extra/mmapeak/mmapeak.py b/extra/mmapeak/mmapeak.py index 485e09efb601a..4488346ea195f 100644 --- a/extra/mmapeak/mmapeak.py +++ b/extra/mmapeak/mmapeak.py @@ -3,7 +3,7 @@ # TODO: there is a timing bug without this os.environ["AMD_AQL"] = "1" -from tinygrad import Tensor, Device, GlobalCounters, Context +from tinygrad import Tensor, Device, GlobalCounters, Context, dtypes from tinygrad.helpers import getenv, DEV from tinygrad.uop.ops import UOp, Ops, KernelInfo from tinygrad.renderer import Estimates @@ -37,7 +37,7 @@ def fxn(A:UOp) -> UOp: gidx = UOp.special(NUM_WORKGROUPS, "gidx0") FLOPs = FLOPS_PER_MATMUL * NUM_WAVES * NUM_WORKGROUPS * INTERNAL_LOOP * INSTRUCTIONS_PER_LOOP sink = UOp.sink(A.base, threads, gidx, arg=KernelInfo(inst.op.name.lower(), estimates=Estimates(ops=FLOPs, mem=0))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) dummy = Tensor.zeros(1).contiguous().realize() out = Tensor.custom_kernel(dummy, fxn=fxn)[0] linear = out.schedule_linear() diff --git a/test/amd/test_asm_kernel.py b/test/amd/test_asm_kernel.py index c7b6b036c4dce..9bbb81e330775 100644 --- a/test/amd/test_asm_kernel.py +++ b/test/amd/test_asm_kernel.py @@ -30,7 +30,7 @@ def custom_add_one(A:UOp) -> UOp: s_endpgm(), ] sink = UOp.sink(A.base, threads, arg=KernelInfo(f"custom_add_one_{A.numel()}", estimates=Estimates(ops=A.numel(), mem=A.numel()*4*2))) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_add_var(A:UOp, B:UOp) -> UOp: A,B = A.flatten(), B.flatten() @@ -49,7 +49,7 @@ def custom_add_var(A:UOp, B:UOp) -> UOp: s_endpgm(), ] sink = UOp.sink(A.base, B.base, var, threads, arg=KernelInfo(f"custom_add_var_{A.numel()}")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_wave_sync(A:UOp, arch:str) -> UOp: # 4 waves across 1024 WG — enough to saturate a SIMD with many concurrent WGs @@ -63,7 +63,7 @@ def custom_wave_sync(A:UOp, arch:str) -> UOp: insts += [s_nop(0)]*4 insts.append(s_endpgm()) sink = UOp.sink(A.base, threads, wg, arg=KernelInfo("custom_wave_sync")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_lds_sync(A:UOp, arch:str) -> UOp: A = A.flatten() @@ -97,7 +97,7 @@ def custom_lds_sync(A:UOp, arch:str) -> UOp: isa.s_endpgm(), ] sink = UOp.sink(A.base, lds, threads, wg, arg=KernelInfo("custom_lds_sync")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_handwritten(A:UOp) -> UOp: A = A.flatten() @@ -143,7 +143,7 @@ def custom_handwritten(A:UOp) -> UOp: k.emit(r4.s_endpgm()) insts = k.finalize() sink = UOp.sink(A.base, threads, wg, lds, arg=KernelInfo("custom_handwritten")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) def custom_data_deps(A:UOp) -> UOp: A = A.flatten() @@ -159,7 +159,7 @@ def custom_data_deps(A:UOp) -> UOp: k.emit(s_endpgm()) insts = k.finalize() sink = UOp.sink(A.base, threads, arg=KernelInfo("custom_data_deps")) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) @unittest.skipUnless(Device.DEFAULT == "AMD", "requires AMD device") class TestAsmKernel(unittest.TestCase): diff --git a/test/backend/test_encodings.py b/test/backend/test_encodings.py index bd4dd8c53578d..01423a0eedfe2 100644 --- a/test/backend/test_encodings.py +++ b/test/backend/test_encodings.py @@ -4,7 +4,7 @@ from tinygrad.dtype import dtypes from tinygrad.renderer.isa.x86 import X86Ops, X86Renderer, RBP, RDI, RSP, RSI, RAX, RDX, XMM, GPR, imm, def_reg -def ins(op, dt, src, tag=None): return UOp(Ops.INS, arg=op, dtype=dt, src=src, tag=tag) +def ins(op, dt, src, tag=None): return UOp(Ops.INS, arg=(op, dt), src=src, tag=tag) @unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, X86Renderer), "only on x86") class TestEncodingsX86(unittest.TestCase): @@ -136,7 +136,7 @@ def test_disp_imm_same_value(self): # cmoves have the cmp as the last src even though it is not explicitly used, the cmp doesn't define a reg and is ignored in the encoding def test_cmove_ignore_cmp(self): - cmove = ins(X86Ops.CMOVE, dtypes.int32, (def_reg(dtypes.int32, RAX), UOp(Ops.INS, arg=X86Ops.CMP)), RDX) + cmove = ins(X86Ops.CMOVE, dtypes.int32, (def_reg(dtypes.int32, RAX), UOp(Ops.INS, arg=(X86Ops.CMP, dtypes.void))), RDX) # cmove edx, eax self.assertEqual(bytes.fromhex(self.encode(cmove)), bytes.fromhex("0F 44 D0")) diff --git a/test/backend/test_isel.py b/test/backend/test_isel.py index 54033ba393da1..edb037d909b00 100644 --- a/test/backend/test_isel.py +++ b/test/backend/test_isel.py @@ -20,7 +20,7 @@ def _check_op(self, dt_op, expr): with self.subTest(dtype=dt): v = [UOp.variable(str(i), 0, 0, dt) for i in range(nargs)] n = self.isel_rewrite(expr(*v)) - self.assertIs(n.arg, op) + self.assertIs(n.arg[0], op) def test_cmove(self): a = UOp.variable("a", 0, 0, dtypes.int32) @@ -29,9 +29,9 @@ def test_cmove(self): d = (a != b).where(a, b) f = c + d n = self.isel_rewrite(f) - self.assertTrue(n.src[0].arg is X86Ops.CMOVL and n.src[1].arg is X86Ops.CMOVNE) + self.assertTrue(n.src[0].arg[0] is X86Ops.CMOVL and n.src[1].arg[0] is X86Ops.CMOVNE) # both comparisons become the same instruction - self.assertTrue(n.src[0].src[2] == n.src[1].src[2] and n.src[0].src[2].arg is X86Ops.CMP) + self.assertTrue(n.src[0].src[2] == n.src[1].src[2] and n.src[0].src[2].arg[0] is X86Ops.CMP) def test_vinsertps(self): a = UOp.variable("a", 0, 0, dtypes.float32) @@ -41,7 +41,7 @@ def test_vinsertps(self): valid = [UOp.stack(lane(a, 0), lane(b, 1), lane(a, 2), lane(b, 3)), UOp.stack(lane(a, 3), lane(b, 2), lane(c, 1), d)] - for shuf in valid: self.assertIs(self.isel_rewrite(shuf).arg, X86Ops.VINSERTPS) + for shuf in valid: self.assertIs(self.isel_rewrite(shuf).arg[0], X86Ops.VINSERTPS) # complex address is [base + index*scale + displacement] def test_complex_address(self): diff --git a/test/external/external_test_gpu_crash.py b/test/external/external_test_gpu_crash.py index 238cdc8c2e121..16e3c8fe6711a 100644 --- a/test/external/external_test_gpu_crash.py +++ b/test/external/external_test_gpu_crash.py @@ -37,7 +37,7 @@ def setUp(self): def _run_insts(self, insts: list[Inst]): buf = UOp.new_buffer("AMD", 64, dtypes.uint8) sink = UOp.sink(UOp.param(0, dtypes.uint8, 64, device="AMD"), UOp.special(1, "lidx0"), arg=KernelInfo("test")) - prg = UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=i) for i in insts)))) + prg = UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(i, dtypes.void)) for i in insts)))) run_linear(UOp(Ops.LINEAR, src=(prg.call(buf),)), wait=True) def _assert_gpu_fault(self, func): diff --git a/test/external/external_test_schedule_scaling.py b/test/external/external_test_schedule_scaling.py index e97697ae32045..2e62a809cb813 100644 --- a/test/external/external_test_schedule_scaling.py +++ b/test/external/external_test_schedule_scaling.py @@ -1,5 +1,5 @@ import unittest, time, itertools -from tinygrad import Tensor, Context +from tinygrad import Tensor, Context, dtypes class TestScheduleScaling(unittest.TestCase): """Test that .schedule() scales linearly with graph size (no O(n^2) behavior).""" @@ -138,7 +138,7 @@ def test_custom_kernel_assign_scaling(self): def custom_kernel_assign(n): def custom_asm(out): return UOp(Ops.PROGRAM, src=(UOp.sink(out, arg=KernelInfo(f"fxn_{next(count)}")), - UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=s_nop(i)) for i in range(n*8))))) + UOp(Ops.LINEAR, src=tuple(UOp(Ops.INS, arg=(s_nop(i), dtypes.void)) for i in range(n*8))))) call = Tensor.custom_kernel(Tensor.empty(1), fxn=custom_asm)[0] return Tensor.cat(*[Tensor.empty(1).assign(call+i) for i in range(n)]) self._assert_linear(custom_kernel_assign, n_small=50, n_large=500) diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 2e97f9a30d12b..49f23c65680a3 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -841,7 +841,7 @@ def fxn(out:UOp) -> UOp: lidx = UOp.special(1, "lidx0") gidx = UOp.special(1, "gidx0") sink = UOp.sink(out.base, lidx, gidx, arg=KernelInfo(name=name)) - return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=x) for x in insts])))) + return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.LINEAR, src=tuple([UOp(Ops.INS, arg=(x, dtypes.void)) for x in insts])))) with save_viz() as viz: with Context(DEV="NULL::gfx1100"): out = Tensor.custom_kernel(Tensor.empty(1), fxn=fxn)[0] diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 72f3e2ead3683..062c02d04b5c7 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -443,7 +443,7 @@ def do_estimates(prg:UOp, sink:UOp, lin:UOp) -> UOp|None: return prg.replace(src=(sink.replace(arg=replace(sink.arg, estimates=Estimates.from_uops(lin.src, ignore_indexing=True))),)+prg.src[1:]) def do_assemble(ctx:Renderer, prg:UOp, lin:UOp) -> UOp: - src = "\n".join(str(u.arg) for u in lin.src) + src = "\n".join(str(u.arg[0]) for u in lin.src) if DEBUG >= 4: print(src) binary = ctx.asm(prg, lin) return prg.replace(src=prg.src[:2]+(UOp(Ops.SOURCE, arg=src), UOp(Ops.BINARY, arg=binary))) diff --git a/tinygrad/renderer/amd/elf.py b/tinygrad/renderer/amd/elf.py index 718700cf84640..8c918c063d3f7 100644 --- a/tinygrad/renderer/amd/elf.py +++ b/tinygrad/renderer/amd/elf.py @@ -13,7 +13,7 @@ _arch_map = {"gfx9": "cdna", "gfx10": "rdna3", "gfx11": "rdna3", "gfx12": "rdna4"} def assemble_linear(prg:UOp, lin:UOp, arch:str) -> bytes: - insts = [u.arg for u in lin.src] + insts = [u.arg[0] for u in lin.src] # ** scan for max vgpr/sgpr/accvgpr max_vgpr, max_sgpr, max_accvgpr = 0, 0, 0 diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 256e9aacc3eb8..1946957f85116 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -219,7 +219,7 @@ def flag_gate(m:UOp) -> UOp|None: def base(x:UOp, i:int) -> UOp: return s.src[0] if (s:=x.src[i]).op is Ops.INDEX else s def lane(x:UOp, i:int) -> int: return s.src[1].src[0].val if (s:=x.src[i]).op is Ops.INDEX else 0 def to_int(dt:DType): return {dtypes.float16: dtypes.int16, dtypes.float32: dtypes.int32, dtypes.float64: dtypes.int64}[dt] -def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, dt, arg=X86Ops.DEFINE, tag=None if reg is None else (reg,)) +def def_reg(dt:DType, reg:Register|None=None) -> UOp: return UOp(Ops.INS, arg=(X86Ops.DEFINE, dt), tag=None if reg is None else (reg,)) def imm(dt:DType, v:int) -> UOp: return UOp.cconst(truncate[dt](v), dt).rtag() def to_imm(c:UOp) -> UOp|None: if not (c.op is Ops.CAST and (v:=c.src[0]).op is Ops.CONST): return None @@ -259,8 +259,8 @@ def idiv(ctx:IselContext, x:UOp) -> UOp: elif x.dtype in dtypes.uints: ext = [x.ins(X86Ops.MOVi, src=(imm(min(dtypes.uint32, x.dtype), 0),), tag=(RDX,))] else: ext = [x.ins(X86Ops.SARi, src=(x.src[0], imm(dtypes.uint8, x.dtype.itemsize * 8 - 1)), tag=(RDX,))] # for 8bit need to zero/sign extend al to ah - if x.dtype is dtypes.uint8: dividend = UOp(Ops.INS, arg=X86Ops.MOVZX, dtype=dtypes.int16, src=(x.src[0],), tag=(RAX,)) - elif x.dtype is dtypes.int8: dividend = UOp(Ops.INS, arg=X86Ops.MOVSX, dtype=dtypes.int16, src=(x.src[0],), tag=(RAX,)) + if x.dtype is dtypes.uint8: dividend = UOp(Ops.INS, arg=(X86Ops.MOVZX, dtypes.int16), src=(x.src[0],), tag=(RAX,)) + elif x.dtype is dtypes.int8: dividend = UOp(Ops.INS, arg=(X86Ops.MOVSX, dtypes.int16), src=(x.src[0],), tag=(RAX,)) else: dividend = x.ins(X86Ops.MOV, src=(x.src[0],), tag=(RAX,)) # divisor can't be in rax or rdx divisor = x.ins(X86Ops.MOV, src=(x.src[1],), tag=tuple(r for r in WGPR if r not in (RAX, RDX))) @@ -298,7 +298,7 @@ def abi(ctx:IselContext, x:UOp) -> UOp|None: # the shape srcs of a PARAM are not values, tag them so they aren't materialized into registers def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(dtype=dt, src=tuple(s.rtag() for s in x.src), tag=(r,)),) def _stack_arg(disp:int): - return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=X86Ops.FRAME_INDEX, dtype=dtypes.int32, tag=disp), imm(dtypes.uint8, 8)) + return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=(X86Ops.FRAME_INDEX, dtypes.int32), tag=disp), imm(dtypes.uint8, 8)) if sys.platform == "win32": src = _reg_arg((RCX, RDX, GPR[8], GPR[9])[i]) if i < 4 else _stack_arg((i-3)*8+32) else: src = _reg_arg((RDI, RSI, RDX, RCX, GPR[8], GPR[9])[i]) if i < 6 else _stack_arg((i-5)*8) # this move "cleanses" the abi register constraint @@ -309,7 +309,7 @@ def _stack_arg(disp:int): XMM_OPS = {op for op in X86Ops if op.name.startswith('V')} - GPR_DEST_OPS def _is_vec_xmm(y: UOp) -> bool: - return (y.op is Ops.INS and y.arg in XMM_OPS) or (y.op not in (Ops.BUFFER, Ops.PARAM, Ops.AFTER, Ops.INS) and y.max_numel() > 1) + return (y.op is Ops.INS and y.arg[0] in XMM_OPS) or (y.op not in (Ops.BUFFER, Ops.PARAM, Ops.AFTER, Ops.INS) and y.max_numel() > 1) def _xmm_sz(x: UOp) -> X86Ops: bits = x.max_numel() * x.dtype.itemsize @@ -325,10 +325,10 @@ def _xmm_sz_m(x: UOp) -> X86Ops: def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # register placeholders with real registers - if x.arg is X86Ops.DEFINE and x.tag is not None: return None - if x.arg is X86Ops.LOOP_CMP: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.DEFINE and x.tag is not None: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.LOOP_CMP: return None # this is an immediate - if x.arg is X86Ops.FRAME_INDEX: return None + if x.op is Ops.INS and x.arg[0] is X86Ops.FRAME_INDEX: return None # no register definition if x.dtype is dtypes.void: return None # already allocated vregs @@ -337,10 +337,10 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: defs = [] if isinstance(x.tag, tuple): defs = [ctx.vreg(x.tag)] elif x.op is Ops.BUFFER: defs = [ctx.vreg(WGPR)] - elif x.dtype in dtypes.floats or (x.op is Ops.INS and x.arg in XMM_OPS) or x.max_numel() > 1: defs = [ctx.vreg(XMM)] + elif x.dtype in dtypes.floats or (x.op is Ops.INS and x.arg[0] in XMM_OPS) or x.max_numel() > 1: defs = [ctx.vreg(XMM)] elif x.dtype in dtypes.ints+(dtypes.bool,): defs = [ctx.vreg(WGPR)] # TODO: add this once the scheduler can track register pressure - # if x.arg in X86GroupOp.WriteFlags: defs.append(ctx.vreg(RFLAGS)) + # if x.arg[0] in X86GroupOp.WriteFlags: defs.append(ctx.vreg(RFLAGS)) # the size src of a BUFFER is not a value, tag it so it isn't materialized into a register if x.op is Ops.BUFFER: return x.replace(src=tuple(s.rtag() for s in x.src), tag=tuple(defs)) return x.replace(tag=tuple(defs)) @@ -358,7 +358,7 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # so regalloc builds the prologue/epilogue naturally (UPat(Ops.SINK, name="x"), lambda x: x.replace(src=(x.ins(X86Ops.RET, src=x.src + tuple(def_reg(dtypes.uint64 if r in GPR else dtypes.float64, r) for r in CALLEE_SAVED)),)) \ - if not x.src or x.src[0].arg is not X86Ops.RET else None), + if not x.src or x.src[0].op is not Ops.INS or x.src[0].arg[0] is not X86Ops.RET else None), # function abi constraints (UPat((Ops.PARAM, Ops.SPECIAL), name="x"), abi), # constants that can't be immediates, move them to registers @@ -554,7 +554,7 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: # so we rematerialize. This is different from rematerialization you might want to do in regalloc because it is not optional, # regalloc shouldn't rematerialize if a src of the instruction is dead, but here you need to as there's no fallback load from stack def flag_rematerialize(ctx:PreRegAllocContext, x:UOp): - flag_def = x if x.arg in X86GroupOp.WriteFlags or x.op in (Ops.RANGE, Ops.END) else x.src[-1] if x.arg in X86GroupOp.ReadFlags else None + flag_def = x if x.op in (Ops.RANGE, Ops.END) or x.arg[0] in X86GroupOp.WriteFlags else x.src[-1] if x.arg[0] in X86GroupOp.ReadFlags else None if flag_def is None: return None if ctx.lock is not None and ctx.lock is not flag_def: ctx.clobbered.add(ctx.lock) ctx.lock = flag_def @@ -570,19 +570,19 @@ def flag_rematerialize(ctx:PreRegAllocContext, x:UOp): # TODO: control flow should be overhauled so that this isn't necessary def lower_range(ctx, x:UOp) -> tuple[UOp, list[UOp]]: loop_label = "_".join(str(i) for i in x.arg[:-1]) - label = UOp(Ops.INS, arg=X86Ops.LABEL, tag=f".LOOP_{loop_label}") + label = UOp(Ops.INS, arg=(X86Ops.LABEL, dtypes.void), tag=f".LOOP_{loop_label}") # loop, cmp on backedge all we need is a jmp tag if x.dtype is dtypes.void: return (label, [label]) else: acc = x.ins(X86Ops.MOVi, src=(imm(x.dtype, 0),) + x.src[1:]) - cmp = UOp(Ops.INS, arg=X86Ops.CMPi if x.src[0].op is Ops.CAST else X86Ops.CMP, src=(acc, x.src[0])) - jump_out = UOp(Ops.INS, arg=X86Ops.JGE, src=(cmp,), tag=f".LOOP_OUT_{loop_label}") + cmp = UOp(Ops.INS, arg=(X86Ops.CMPi if x.src[0].op is Ops.CAST else X86Ops.CMP, dtypes.void), src=(acc, x.src[0])) + jump_out = UOp(Ops.INS, arg=(X86Ops.JGE, dtypes.void), src=(cmp,), tag=f".LOOP_OUT_{loop_label}") ctx.loop_label[acc] = loop_label return (acc, [acc, label, cmp, jump_out]) def lower_end(ctx, x:UOp) -> tuple[UOp, list[UOp]]: - end_label = UOp(Ops.INS, arg=X86Ops.LABEL, tag=f".LOOP_OUT_{ctx.loop_label[x.src[1]]}") - jmp = UOp(Ops.INS, arg=X86Ops.JMP, tag=f".LOOP_{ctx.loop_label[x.src[1]]}") + end_label = UOp(Ops.INS, arg=(X86Ops.LABEL, dtypes.void), tag=f".LOOP_OUT_{ctx.loop_label[x.src[1]]}") + jmp = UOp(Ops.INS, arg=(X86Ops.JMP, dtypes.void), tag=f".LOOP_{ctx.loop_label[x.src[1]]}") inc = x.src[1].ins(X86Ops.ADDi, src=(imm(x.src[1].dtype, 1),)) return (inc, [inc, jmp, end_label]) @@ -594,16 +594,16 @@ def lower_loop(ctx, x:UOp) -> tuple[UOp, list[UOp]]: # final rewrite to match the isa spec post_regalloc_matcher = PatternMatcher([ # rewrite FRAME_INDEX to IMM now that the stack size is known - (UPat(Ops.INS, arg=X86Ops.FRAME_INDEX, name="x"), lambda ctx,x: (nx:=UOp.cconst(ctx.stack_size + x.tag, x.dtype), [nx])), + (UPat(Ops.INS, name="x"), lambda ctx,x: (nx:=UOp.cconst(ctx.stack_size + x.tag, x.dtype), [nx]) if x.arg[0] is X86Ops.FRAME_INDEX else None), # expand the cmp here so we can preserve rng src edge to get label from ctx - (UPat(Ops.INS, arg=X86Ops.LOOP_CMP, name="x"), lower_loop), + (UPat(Ops.INS, name="x"), lambda ctx,x: lower_loop(ctx, x) if x.arg[0] is X86Ops.LOOP_CMP else None), # rewrite RANGE to ACC = 0 -> LABEL -> JUMP if ACC >= loop bound (UPat(Ops.RANGE, name="x"), lower_range), # rewrite END to ACC + 1 -> JUMP -> LABEL, also add the out of loop JUMP to the src so this becomes the jump target (UPat(Ops.END, name="x"), lower_end), # rewrite two address instructions to two address form, if reused src wasn't coalesced insert a move (UPat(Ops.INS, name="x"), lambda ctx,x: (nx:=x.replace(src=x.src[1:]), - [ctx.ren.copy(x.src[0], greg(x)), nx] if greg(x) != greg(x.src[0]) else [nx]) if x.arg in X86GroupOp.TwoAddress else None), + [ctx.ren.copy(x.src[0], greg(x)), nx] if greg(x) != greg(x.src[0]) else [nx]) if x.arg[0] in X86GroupOp.TwoAddress else None), ]) # ***** X86 instruction encoding ***** @@ -636,7 +636,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # bit signaling 64 bit variant of instruction w = sz == 8 # legacy 8bit opcode is 1 less than 16-64bit variants - demote = (rm_sz == 1 or reg_sz == 1) and x.arg not in X86GroupOp.ReadFlags | {X86Ops.LEA} + demote = (rm_sz == 1 or reg_sz == 1) and x.arg[0] not in X86GroupOp.ReadFlags | {X86Ops.LEA} # REX byte is required when 64 bit or an extended reg is used (index 8 - 15) or lower 8 bits of (rsp, rbp, rsi, rdi) are accessed if w | r | _x | b | (reg_sz == 1 & reg >> 2) | (rm_sz == 1 & rm >> 2) | (demote and disp_uop is None and rm >= 4): inst += bytes([0b0100 << 4 | w << 3 | r << 2 | _x << 1 | b]) @@ -678,19 +678,19 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # get the encoding structure of the uop # when a uop writes to memory it takes the form of a store, dtype is void, no definition address:tuple[UOp|None, ...] - if x.arg in X86GroupOp.WriteMem: + if x.arg[0] in X86GroupOp.WriteMem: if len(x.src) > 4: address, rest = x.src[:4], x.src[4:] else: address, rest = (x, None, None, None), x.src imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(rest[0], *address, *(None, *rest[1:])) if reg is None else _encode(None, *address, *(None, *imm_uop)) - if x.arg in X86GroupOp.Rm1st: + if x.arg[0] in X86GroupOp.Rm1st: if len(x.src) > 3: address, rest = x.src[:4], x.src[4:] else: address, rest = (x.src[0], None, None, None), x.src[1:] imm_uop = rest[:1] if rest and rest[0].op is Ops.CAST else (None,) return _encode(x, *address, *(None, *imm_uop)) if reg is None else _encode(None, *address, *(x if sel else None, *imm_uop)) - if x.arg in X86GroupOp.Rm2nd: + if x.arg[0] in X86GroupOp.Rm2nd: if len(x.src) > 4: address, rest = x.src[1:5], x.src[:1] + x.src[5:] else: address, rest = (x.src[1], None, None, None), x.src[:1] + x.src[2:] # cmp reg, rm doesn't define a new register @@ -819,7 +819,7 @@ def __init__(self, target:Target): super().__init__(target) from tinygrad.runtime.support.compiler_cpu import X86Compiler self.compiler = X86Compiler() - def is_two_address(self, x:UOp) -> bool: return x.arg in X86GroupOp.TwoAddress + def is_two_address(self, x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] in X86GroupOp.TwoAddress def stack_pointer(self) -> UOp: return def_reg(dtypes.uint64, RSP) # the value of a BUFFER is its address, it moves through registers and the stack as a 64bit int def copy(self, x:UOp, reg:Register): @@ -832,15 +832,15 @@ def spill(self, disp:UOp, x:UOp) -> UOp: if x.op is Ops.BUFFER: x = x.replace(dtype=dtypes.uint64) is_xmm = isinstance(x.tag, tuple) and x.tag[0].cons[0].size == 16 op = X86Ops.VMOVUPSm if is_xmm else X86Ops.MOVm - return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=op, tag=x.tag) + return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=(op, dtypes.void), tag=x.tag) def fill(self, disp:UOp, x:UOp, reg:Register) -> UOp: is_xmm = reg.cons[0].size == 16 dt = dtypes.uint64 if x.op is Ops.BUFFER else x.dtype - return UOp(Ops.INS, dt, fold_address(self.stack_pointer().index(disp)), X86Ops.VMOVUPS if is_xmm else X86Ops.MOV, (reg,)) + return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)), arg=(X86Ops.VMOVUPS if is_xmm else X86Ops.MOV, dt), tag=(reg,)) def asm_str(self, uops:list[UOp], function_name:str) -> str: - def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg))[-1] in ('i', 'm') else o[7:]).lower():7s}" + def _format_op(x:UOp) -> str: return f" {(o[7:-1] if (o:=str(x.arg[0]))[-1] in ('i', 'm') else o[7:]).lower():7s}" def _format_operands(x:UOp) -> str: def _format(src:tuple[UOp, ...]) -> list[str]: return [str(s.src[0].val) if s.op is Ops.CAST else reg_strs[o].get(s.dtype.itemsize, o) if \ @@ -848,17 +848,17 @@ def _format(src:tuple[UOp, ...]) -> list[str]: def _mem_adress(base:UOp, idx:UOp, disp:UOp, sz:UOp) -> list[str]: return [f"[{greg(base)}" + (f" + {greg(idx)}*{sz.src[0].val}" if greg(idx) else "") + (f" + {d}" if (d:=disp.src[0].val) else "") + "]"] - if len(x.src) > 4 and x.arg in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) - elif len(x.src) > 3 and x.arg in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) - elif len(x.src) > 4 and x.arg in X86GroupOp.Rm2nd: ret = _format((x, x.src[0])) + _mem_adress(*x.src[1:5]) + _format(x.src[5:]) + if len(x.src) > 4 and x.arg[0] in X86GroupOp.WriteMem: ret = _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 3 and x.arg[0] in X86GroupOp.Rm1st: ret = _format((x,)) + _mem_adress(*x.src[:4]) + _format(x.src[4:]) + elif len(x.src) > 4 and x.arg[0] in X86GroupOp.Rm2nd: ret = _format((x, x.src[0])) + _mem_adress(*x.src[1:5]) + _format(x.src[5:]) else: ret = _format((x,) + x.src) return ", ".join(ret) asm = [f".{function_name}:"] for u in uops: - if u.op is not Ops.INS or u.arg is X86Ops.DEFINE: continue - if u.arg is X86Ops.LABEL: asm.append(f"{str(u.tag)}:") - elif u.arg is X86Ops.RET: asm.append(_format_op(u)) + if u.op is not Ops.INS or u.arg[0] is X86Ops.DEFINE: continue + if u.arg[0] is X86Ops.LABEL: asm.append(f"{str(u.tag)}:") + elif u.arg[0] is X86Ops.RET: asm.append(_format_op(u)) else: asm.append(_format_op(u) + " " + _format_operands(u)) return "\n".join(asm) @@ -867,15 +867,15 @@ def render(self, uops:list[UOp]) -> str: jumps: dict[UOp, int] = {} binary = bytearray() for u in uops: - if u.op is not Ops.INS or u.arg is X86Ops.DEFINE: continue - if u.arg is X86Ops.LOOP_CMP: continue - if u.arg is X86Ops.LABEL: + if u.op is not Ops.INS or u.arg[0] is X86Ops.DEFINE: continue + if u.arg[0] is X86Ops.LOOP_CMP: continue + if u.arg[0] is X86Ops.LABEL: targets[u.tag] = len(binary) continue - if u.arg not in encodings or (l:=encodings[u.arg](u)) is None: - raise RuntimeError(f"failed to encode {u.arg} with {u.dtype} srcs {[x.dtype for x in u.src]}") + if u.arg[0] not in encodings or (l:=encodings[u.arg[0]](u)) is None: + raise RuntimeError(f"failed to encode {u.arg[0]} with {u.dtype} srcs {[x.dtype for x in u.src]}") binary.extend(l) - if u.arg in (X86Ops.JL, X86Ops.JB, X86Ops.JE, X86Ops.JNE, X86Ops.JGE, X86Ops.JMP): jumps[u] = len(binary) + if u.arg[0] in (X86Ops.JL, X86Ops.JB, X86Ops.JE, X86Ops.JNE, X86Ops.JGE, X86Ops.JMP): jumps[u] = len(binary) # fixup jump targets now that encoding size is known for u in uops: if (t:=jumps.get(u)) is not None: binary[t-4:t] = (targets[u.tag] - t).to_bytes(4, 'little', signed=True) diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 012781ca7b6a3..6ac0d650d2fa7 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -62,7 +62,7 @@ def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: progs = [get_runtime(d, prog) if isinstance(prog, UOp) else cast(CPUDevice, Device[d]).prgs[prog] for d in devs] addrs = tuple(UOp.const(p.addr, dtypes.uint64) for p in progs) words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs)),) + args - return UOp(Ops.INS, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg="cmd") + return UOp(Ops.INS, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg=("cmd", dtypes.void)) def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] @@ -74,12 +74,12 @@ def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: pm_cpu_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), - (UPat(Ops.INS, arg="barrier"), lambda: UOp(Ops.NOOP)), - (UPat(Ops.INS, arg="wait", src=(UPat(name="dst"), UPat(name="val"))), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.NOOP)), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), lambda ctx, dst, val: cpu_cmd(ctx, wait_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), - (UPat(Ops.INS, arg="store", src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), lambda ctx, dst, val: cpu_cmd(ctx, signal_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), - (UPat(Ops.INS, arg="timestamp", src=(UPat(name="dst"),)), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), lambda ctx, dst: cpu_cmd(ctx, timestamp_prog, dst.getaddr(ctx), *(() if WIN else (make_buf(ctx, tag="func:clock_gettime").getaddr(ctx),)))), ]) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index fd52192dc478f..f447b90fb3b08 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -189,7 +189,7 @@ def _wait_ins(ctx:BatchCtx, bufs_by_lane:list[list[Any]], write, devices:tuple[s for (dqueue, dtag), by_lane in rows.items(): for ds in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): sig = UOp.mstack(*[make_buf(d, tag="sentinel_signal") if dd is None else make_buf(dd, ctx.slots[dqueue]) for dd, d in zip(ds, devices)]) - waits.append(UOp(Ops.INS, arg="wait", src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) + waits.append(UOp(Ops.INS, arg=("wait", dtypes.void), src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) ctx.signal_tags |= {t for _, t in rows} return waits @@ -238,7 +238,7 @@ def _make_finalizers(ctx:BatchCtx) -> tuple[list[UOp], list[UOp], list[UOp]]: # finalizer: bump the host timeline and remember this schedule's epoch for the next fence waits = _wait_ins(ctx, [list(dev_bufs[d].values()) for d in devs], None, devs, "COMPUTE:0", n) - fin_submit = make_submit(*waits, UOp(Ops.INS, arg="store", src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") + fin_submit = make_submit(*waits, UOp(Ops.INS, arg=("store", dtypes.void), src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() fins.append(make_call("hcq_finalizer", UOp.sink(epoch_slot.store(epoch + 1), sched_epoch.after(fin_submit).index(0).store(epoch)), HCQInfo(devs))) return fences, resets, fins @@ -251,18 +251,20 @@ def _emit_submits(ctx:BatchCtx, call_waits:list[list[UOp]]) -> tuple[list[UOp], if (devices, queue) not in seen_queues: seen_queues.add((devices, queue)) epoch = make_buf(devices, tag="timeline_value").index(0) - 1 - q = [UOp(Ops.INS, arg="barrier", src=()), UOp(Ops.INS, arg="wait", src=(make_buf(devices, tag="timeline_signal"), epoch))] + q + q = [UOp(Ops.INS, arg=("barrier", dtypes.void), src=()), + UOp(Ops.INS, arg=("wait", dtypes.void), src=(make_buf(devices, tag="timeline_signal"), epoch))] + q # and make hcq call name, info = get_call_name(call, get_call_arg_uops(call)), HCQInfo(devices, estimate_uop(call)) ts_ids = [next(UOp.unique_num) for _ in range(2)] if ctx.profile else [] kerns.append((devices, name, info.estimates, tuple(ts_ids), make_call(name, call.src[0], info).key)) - ts_ins = [UOp(Ops.INS, arg="timestamp", src=(make_buf(devices, s),)) for s in ts_ids] + ts_ins = [UOp(Ops.INS, arg=("timestamp", dtypes.void), src=(make_buf(devices, s),)) for s in ts_ids] q += ts_ins[:1] + [call.replace(arg=replace(call.arg, aux=info))] + ts_ins[1:] # signal the queue if someone waits for us - if tag in ctx.signal_tags: q += [UOp(Ops.INS, arg="store", src=(make_buf(devices, ctx.slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] + if tag in ctx.signal_tags: + q += [UOp(Ops.INS, arg=("store", dtypes.void), src=(make_buf(devices, ctx.slots[queue]), UOp.const(tag + 1, dtypes.uint64)))] src.append(make_call(f"submit {name}", make_submit(*q, devs=devices, queue=queue).sink(), info)) return src, kerns diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 397cdeb5bf9a3..07ffb8983c925 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -124,13 +124,14 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: # always void return dtypes.void case Ops.CALL: - # a CALL of an opaque body is void, a CALL of an address can return a value - return dtypes.void if src[0].dtype is dtypes.void else None + # a CALL of an opaque body (CallInfo arg) is void, a CALL of an address states its return dtype in the arg + return arg if isinstance(arg, DType) else dtypes.void case Ops.CUSTOM | Ops.CUSTOMI: assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"CUSTOM/CUSTOMI arg must be (str, DType), got {arg}" return arg[1] case Ops.INS: - return None + # arg is (instruction, dtype), a queue command or an asm line is void + return arg[1] case Ops.NOOP: # NOOP can be void or carry any dtype (e.g. x.f(Ops.NOOP) or substitute base with NOOP) return None @@ -607,7 +608,8 @@ def after(self, *src:UOp, **kwargs): return UOp(Ops.AFTER, src=(self,)+src, **kw @property def without_after(self) -> UOp: return self.src[0] if self.op is Ops.AFTER else self def barrier(self, *src:UOp): return UOp(Ops.BARRIER, src=(self,)+src) - def ins(self, arg, **kwargs): return UOp(Ops.INS, kwargs.pop("dtype", self.dtype), kwargs.pop("src", self.src), arg, kwargs.pop("tag", self.tag)) + def ins(self, arg, **kwargs): + return UOp(Ops.INS, src=kwargs.pop("src", self.src), arg=(arg, kwargs.pop("dtype", self.dtype)), tag=kwargs.pop("tag", self.tag)) def contract(self, *rngs:UOp): assert all(x.arg[-1] == AxisType.UPCAST for x in rngs), "all contract ranges must be upcast" return UOp.stack(*[self.substitute(dict(zip(rngs, [r.const_like(i) for r,i in zip(rngs, idx)]))) @@ -1205,7 +1207,7 @@ def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.CUSTOM_FUNCTION} def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None, name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: - if ret_dtype is not None: return UOp(Ops.CALL, ret_dtype, src=(self,)+srcs) + if ret_dtype is not None: return UOp(Ops.CALL, src=(self,)+srcs, arg=ret_dtype) # calls are launched per device, so an open DEVICE range is allowed to cross the call boundary assert all(r.arg[-1] is AxisType.DEVICE for r in self.ranges), \ f"ranges {self.ranges} are leaking out of the call in {self.pyrender()}" diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index ad92bcd92400b..87e2b60f19464 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -105,7 +105,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base # CALL of an external function (UPat(Ops.CALL, src=(UPat(),), allow_any_len=True, name="x"), - lambda x: matches_dtype(x.src[0], dtypes.uint64) if x.src[0].dtype is not dtypes.void else None), + lambda x: matches_dtype(x.src[0], dtypes.uint64) and isinstance(x.arg, DType) if x.src[0].dtype is not dtypes.void else None), # pattern compiler IR ops (not in tensor/program graphs, but spec-compliant) (UPat(Ops.PYLITERAL), lambda: True), @@ -114,7 +114,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base (UPat(Ops.BARRIER, dtypes.void), lambda: True), # assembly instruction - (UPat(Ops.INS), lambda: True), + (UPat(Ops.INS, name="x"), lambda x: isinstance(x.arg, tuple) and len(x.arg) == 2 and isinstance(x.arg[1], DType)), # LOAD(idx) / STORE(idx, val) with gates on the LOAD/STORE (UPat((Ops.INDEX, Ops.SHRINK), name="uidx").or_casted().load(), validate_index), From 793c1664f31168ffe631be2431bbb59bce6943a8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 15:18:41 -0400 Subject: [PATCH 265/306] don't set UOp dtype in x86 (#17818) --- tinygrad/codegen/__init__.py | 2 +- tinygrad/renderer/isa/x86.py | 88 +++++++++++++++--------------------- 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 062c02d04b5c7..5be1e1b2b1c59 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -418,7 +418,7 @@ def line_rewrite(lst:list[UOp], pm:PatternMatcher, ctx=None) -> list[UOp]: newlst = [] replaced: dict[UOp, UOp] = {} for u in lst: - nu = u.replace(src=tuple([replaced.get(x, x) for x in u.src])) + nu = u.replace(dtype=None, src=tuple([replaced.get(x, x) for x in u.src])) ret: tuple[UOp, list[UOp]] = pm.rewrite(nu, ctx) or (nu, [nu]) replaced[u] = ret[0] newlst.extend(ret[1]) diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 1946957f85116..fa7c1ab1f96d4 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -2,6 +2,7 @@ # allow semicolons to put multiple ops on one line import sys, struct, functools from typing import cast +from dataclasses import replace from tinygrad.dtype import dtypes, DType, truncate, AddrSpace from tinygrad.uop import FastEnum, auto, Ops, GroupOp from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype @@ -34,12 +35,10 @@ class X86Ops(FastEnum): VMOVD = auto(); VMOVQ = auto(); VMOVDm = auto(); VMOVQm = auto() # comparisons VCMPSS = auto(); VCMPSD = auto(); VCMPPS = auto(); VCMPPD = auto() - VPCMPGTB = auto(); VPCMPGTW = auto(); VPCMPGTD = auto(); VPCMPGTQ = auto() - VPCMPEQB = auto(); VPCMPEQW = auto(); VPCMPEQD = auto(); VPCMPEQQ = auto() SETNE = auto(); SETE = auto(); SETL = auto(); SETB = auto() # where CMOVNE = auto(); CMOVE = auto(); CMOVL = auto(); CMOVB = auto() - VPBLENDVB = auto(); VBLENDVPS = auto(); VBLENDVPD = auto() + VBLENDVPS = auto(); VBLENDVPD = auto() # jumps JNE = auto(); JE = auto(); JL = auto(); JB = auto(); JGE = auto(); JMP = auto() # vectorize / gep @@ -89,8 +88,7 @@ class X86GroupOp: X86Ops.VADDSS, X86Ops.VADDSD, X86Ops.VADDPS, X86Ops.VADDPD, X86Ops.VSUBSS, X86Ops.VSUBSD, X86Ops.VSUBPS, X86Ops.VSUBPD, X86Ops.VMULSS, X86Ops.VMULSD, X86Ops.VMULPS, X86Ops.VMULPD, X86Ops.VDIVSS, X86Ops.VDIVSD, X86Ops.VDIVPS, X86Ops.VDIVPD, X86Ops.VPADDB, X86Ops.VPADDW, X86Ops.VPADDD, X86Ops.VPADDQ, X86Ops.VPSUBB, X86Ops.VPSUBW, X86Ops.VPSUBD, X86Ops.VPSUBQ, - X86Ops.VPCMPEQB, X86Ops.VPCMPEQW, X86Ops.VPCMPEQD, X86Ops.VPCMPEQQ, X86Ops.VPBLENDVB, X86Ops.VBLENDVPS, X86Ops.VBLENDVPD, - X86Ops.VPCMPGTB, X86Ops.VPCMPGTW, X86Ops.VPCMPGTD, X86Ops.VPCMPGTQ, X86Ops.VCMPSS, X86Ops.VCMPSD, X86Ops.VCMPPS, X86Ops.VCMPPD, + X86Ops.VBLENDVPS, X86Ops.VBLENDVPD, X86Ops.VCMPSS, X86Ops.VCMPSD, X86Ops.VCMPPS, X86Ops.VCMPPD, X86Ops.VPMULLW, X86Ops.VPMULLD, X86Ops.VROUNDSS, X86Ops.VROUNDSD, X86Ops.VSQRTSS, X86Ops.VSQRTSD, X86Ops.VINSERTPS, X86Ops.VPINSRB, X86Ops.VPINSRW, X86Ops.VPINSRD, X86Ops.VPINSRQ, X86Ops.VPAND, X86Ops.VPOR, X86Ops.VPXOR, X86Ops.VPSLLVD, X86Ops.VPSLLVQ, X86Ops.VPSRLVD, X86Ops.VPSRLVQ, X86Ops.VPSRAVD, X86Ops.CMOVNE, X86Ops.CMOVE, X86Ops.CMOVL, X86Ops.CMOVB, @@ -163,16 +161,15 @@ def scratch_buffer(elem_dt:DType, count:int, slot:int) -> UOp: def gated_load(ctx, addr:UOp, alt:UOp, gate:UOp, x:UOp): local = scratch_buffer(addr.src[0].dtype, x.max_numel(), next(ctx)) - local_idx = local.index(UOp.cconst(0, dtypes.int32), dtype=dtypes.uint64) - # the selected address is a 64bit value, the AFTER orders the load after the scratch store and carries the element dtype for the encoder - sel = gate.where(addr.replace(dtype=dtypes.uint64), local_idx) - ptr = UOp(Ops.AFTER, addr.dtype, (sel, (local_idx if x.max_numel() == 1 else local).store(alt))) - return ptr.load(dtype=x.dtype) + local_idx = local.index(UOp.cconst(0, dtypes.int32)) + # the AFTER orders the load after the scratch store + sel = gate.where(addr, local_idx) + return UOp(Ops.AFTER, src=(sel, (local_idx if x.max_numel() == 1 else local).store(alt))).load() def gated_store(addr:UOp, gate:UOp, val:UOp): local = scratch_buffer(addr.src[0].dtype, val.max_numel(), -1) - sel = gate.where(addr.replace(dtype=dtypes.uint64), local.index(UOp.cconst(0, dtypes.int32), dtype=dtypes.uint64)) - return UOp(Ops.AFTER, addr.dtype, (sel,)).store(val) + sel = gate.where(addr, local.index(UOp.cconst(0, dtypes.int32))) + return UOp(Ops.AFTER, src=(sel,)).store(val) # a gate the flags can be picked with, or the bool compared to zero that replaces one they can't: only an integer # comparison sets the flags, see cmp. NOTE: the 0 is int so the bool zero-extends and compares as int (a byte compare renders @@ -231,10 +228,11 @@ def to_imm(c:UOp) -> UOp|None: def cmp(x:UOp) -> UOp: if x.src[0].dtype in dtypes.floats: raise RuntimeError(f"no flag compare for {x.src[0].dtype}, a float gate must be a mask") return x.ins(X86Ops.CMP, dtype=dtypes.void) if (i:=to_imm(x.src[1])) is None else x.ins(X86Ops.CMPi, dtype=dtypes.void, src=(x.src[0], i)) -def vcmp(x:UOp) -> UOp: - v = imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) - if x.dtype is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, src=x.src + (v,)) - return x.ins(X86Ops.VCMPSD if x.max_numel() == 1 else X86Ops.VCMPPD, src=x.src + (v,)) +# comparisons that produce masks, the mask has the width of the operands +def mask(x:UOp) -> UOp: + dt, v = x.src[0].dtype, imm(dtypes.uint8, {Ops.CMPLT: 1, Ops.CMPNE: 4, Ops.CMPEQ: 0}[x.op]) + if dt is dtypes.float32: return x.ins(X86Ops.VCMPSS if x.max_numel() == 1 else X86Ops.VCMPPS, dtype=dt, src=x.src + (v,)) + return x.ins(X86Ops.VCMPSD if x.max_numel() == 1 else X86Ops.VCMPPD, dtype=dt, src=x.src + (v,)) # vinsertps xmm2, xmm0, xmm1, imm # inserts any 32 bit element in xmm1 into any position in xmm0 according to immm, result is written to xmm2 @@ -247,9 +245,9 @@ def _insert(ret:UOp, i:int) -> UOp: # vpinsq xmm2, xmm0, rax, imm # inserts element in rax into any position in xmm0, result is written to xmm2 according to imm -def vpins(x:UOp) -> UOp: +def vpins(x:UOp, srcs:tuple[UOp, ...]) -> UOp: op = {1: X86Ops.VPINSRB, 2: X86Ops.VPINSRW, 4: X86Ops.VPINSRD, 8: X86Ops.VPINSRQ}[x.dtype.itemsize] - return functools.reduce(lambda ret,i: x.ins(op, src=(ret, x.src[i], imm(dtypes.uint8, i))), range(len(x.src)), def_reg(x.dtype)) + return functools.reduce(lambda ret,i: x.ins(op, src=(ret, srcs[i], imm(dtypes.uint8, i))), range(len(srcs)), def_reg(x.dtype)) # we don't call ctx.vreg on the srcs to avoid duplicates, a rewrite will assign the tuple of valid registers to a vreg def idiv(ctx:IselContext, x:UOp) -> UOp: @@ -290,13 +288,17 @@ def _cast(v:UOp) -> UOp: return v.cast(dtypes.int64) if v.vmin < 0 else v.cast(d if idx.op is Ops.CAST and idx.src[0].op is Ops.CONST: return (base, UOp(Ops.NOOP), _disp(idx.src[0].val * scale), sz) return (base, _cast(idx), _disp(0), sz) +# addresses are 64bit values +def lea(x:UOp) -> UOp: return x.ins(X86Ops.LEA, dtype=dtypes.uint64, src=fold_address(x)) + def abi(ctx:IselContext, x:UOp) -> UOp|None: if isinstance(x.tag, tuple): return None i = ctx.func_args.index(x) # buffer params hold addresses, their value moves as a 64bit int dt = dtypes.uint64 if x.op is Ops.PARAM and x.arg.addrspace is AddrSpace.GLOBAL else x.dtype + arg = replace(x.arg, dtype=dt) if x.op is Ops.PARAM else x.arg # the shape srcs of a PARAM are not values, tag them so they aren't materialized into registers - def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(dtype=dt, src=tuple(s.rtag() for s in x.src), tag=(r,)),) + def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(dtype=None, arg=arg, src=tuple(s.rtag() for s in x.src), tag=(r,)),) def _stack_arg(disp:int): return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=(X86Ops.FRAME_INDEX, dtypes.int32), tag=disp), imm(dtypes.uint8, 8)) if sys.platform == "win32": src = _reg_arg((RCX, RDX, GPR[8], GPR[9])[i]) if i < 4 else _stack_arg((i-3)*8+32) @@ -361,21 +363,22 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: if not x.src or x.src[0].op is not Ops.INS or x.src[0].arg[0] is not X86Ops.RET else None), # function abi constraints (UPat((Ops.PARAM, Ops.SPECIAL), name="x"), abi), + # conditional moves between addresses, lea both srcs + (UPat.var("m").where(UPat((Ops.INDEX, Ops.SHRINK), name="a"), UPat((Ops.INDEX, Ops.SHRINK), name="b")), lambda m,a,b: + m.where(lea(a), lea(b)) if not _is_vec_xmm(a.src[0]) else None), # constants that can't be immediates, move them to registers (UPat.cvar("c").cast(dtypes.int64s, name="x"), lambda c,x: x.ins(X86Ops.MOVABS, src=(imm(x.dtype, c.val),)) if not x.tag else None), (UPat.cvar("c").cast(dtypes.ints+(dtypes.bool,), name="x"), lambda c,x: x.ins(X86Ops.MOVi, src=(imm(x.dtype, c.val),)) if not x.tag else None), (UPat.cvar("c").cast(dtypes.floats, name="x"), lambda c,x: UOp.cconst(struct.unpack((dt:=to_int(x.dtype)).fmt, struct.pack(x.dtype.fmt, c.val))[0], dt).bitcast(x.dtype) if not x.tag else None), - # conditional moves that use masks NOTE: these currently assume a mask producing cmp exists - (UPat.var("m").where(UPat.var("a", dtypes.int8s+dtypes.int16s+dtypes.int32s+(dtypes.int64,)), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VPBLENDVB, src=(b, a, m.replace(dtype=m.src[0].dtype))) if a.max_numel() > 1 else None), - (UPat.var("m").where(UPat.var("a", dtypes.float32), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VBLENDVPS, src=(b, a, m.replace(dtype=m.src[0].dtype)))), - (UPat.var("m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: - a.ins(X86Ops.VBLENDVPD, src=(b, a, m.replace(dtype=m.src[0].dtype)))), + # conditional moves that use masks, the mask has the width of the values + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float32), UPat()), name="m").where(UPat.var("a", dtypes.float32), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPS, src=(b, a, mask(m)))), + (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float64), UPat()), name="m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: + a.ins(X86Ops.VBLENDVPD, src=(b, a, mask(m)))), # in this case we have a mask producing comparison whose user expects a bool, so we convert to bool (UPat(GroupOp.Comparison, dtypes.bool, (UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: - UOp(Ops.AND, src=(x.replace(dtype=y.dtype).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).f(Ops.NOOP, dtype=dtypes.bool)), + UOp(Ops.AND, src=(mask(x).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).bitcast(dtypes.bool)), # conditional moves that use flags # TODO: remove this once we allow all flag producing ops in cmove # the blends took every float gate a mask can serve, so a gate that is still not an integer comparison becomes one here @@ -395,16 +398,6 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(Ops.CMPLT, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), (UPat(Ops.CMPEQ, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), (UPat(Ops.CMPNE, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), - # comparisons that produce masks (these aren't bool dtype) - (UPat(GroupOp.Comparison, src=(UPat(dtype=(dtypes.float32, dtypes.float64)), UPat()), name="x"), vcmp), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int8s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQB)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int16s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQW)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int32s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQD)), - (UPat(Ops.CMPEQ, src=(UPat(dtype=dtypes.int64s), UPat()), name="x"), lambda x: x.ins(X86Ops.VPCMPEQQ)), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int8s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTB, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int16s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTW, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int32s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTD, src=(b, a))), - (UPat(Ops.CMPLT, src=(UPat.var("a", dtypes.int64s), UPat.var("b")), name="x"), lambda a,b,x: x.ins(X86Ops.VPCMPGTQ, src=(b, a))), # float unary (UPat.var("y", dtypes.float32).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSS, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPS)), (UPat.var("y", dtypes.float64).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSD, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPD)), @@ -413,10 +406,9 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat.var("y", dtypes.float64).trunc().named("x"), lambda y,x: x.ins(X86Ops.VROUNDSD, src=(y, y, imm(dtypes.uint8, 3))) if x.max_numel() == 1 else x.ins(X86Ops.VROUNDPD, src=(y, imm(dtypes.uint8, 3)))), # for float16 we route the srcs through gprs, this is suboptimal for values in xmms, in that case we want vpunpcklwd - (UPat(Ops.STACK, dtypes.float16, name="x"), lambda x: - vpins(x.replace(src=tuple(s.bitcast(dtypes.int16) for s in x.src)))), + (UPat(Ops.STACK, dtypes.float16, name="x"), lambda x: vpins(x, tuple(s.bitcast(dtypes.int16) for s in x.src))), (UPat(Ops.STACK, dtypes.float32, name="x"), vinsertps), - (UPat(Ops.STACK, dtypes.ints+(dtypes.bool,), name="x"), vpins), + (UPat(Ops.STACK, dtypes.ints+(dtypes.bool,), name="x"), lambda x: vpins(x, x.src)), # INDEX on a vector register value extracts a single element (UPat.var("y", dtypes.int8s+(dtypes.bool,)).index(UPat.cvar("c").cast(), name="x"), lambda y,c,x: x.ins(X86Ops.VPEXTRB, src=(y, imm(dtypes.uint8, c.val))) if _is_vec_xmm(y) else None), @@ -518,14 +510,13 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(dtype=dtypes.int32).cast(dtypes.int64s, name="x"), lambda x: x.ins(X86Ops.VPMOVSXDQ)), # bitcasts between scalar floats and ints (UPat.var("y", dtypes.float16).bitcast(dtypes.int16s).named("x"), lambda y,x: x.ins(X86Ops.VPEXTRW, src=(y, imm(dtypes.uint8, 0)))), - (UPat(dtype=dtypes.int16s).bitcast(dtypes.float16).named("x"), vpins), + (UPat(dtype=dtypes.int16s).bitcast(dtypes.float16).named("x"), lambda x: vpins(x, x.src)), (UPat(dtype=dtypes.int32s).bitcast(dtypes.float32).named("x"), lambda x: x.ins(X86Ops.VMOVD)), (UPat(dtype=dtypes.int64s).bitcast(dtypes.float64).named("x"), lambda x: x.ins(X86Ops.VMOVQ)), (UPat(dtype=dtypes.float32).bitcast(dtypes.int32s).named("x"), lambda x: x.ins(X86Ops.VMOVDm)), (UPat(dtype=dtypes.float64).bitcast(dtypes.int64s).named("x"), lambda x: x.ins(X86Ops.VMOVQm)), # index on a buffer (or the stack pointer) computes an address, addresses are 64bit values - (UPat((Ops.INDEX, Ops.SHRINK), name="x"), - lambda x: x.ins(X86Ops.LEA, dtype=dtypes.uint64, src=fold_address(x)) if not _is_vec_xmm(x.src[0]) else None), + (UPat((Ops.INDEX, Ops.SHRINK), name="x"), lambda x: lea(x) if not _is_vec_xmm(x.src[0]) else None), # TODO: fuse stores, very few cases -- store cmp becomes setcc, store gep int becomes vpextr, store bitcast to int becomes vmovd/q # copy, load, store # NOTE: copy here violates the spec, it only happens post register allocation when a reg to reg move needs to be inserted @@ -758,10 +749,6 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No # packed int binary X86Ops.VPSLLVD: lambda x: encode(x, 0x47, pp=1, sel=2), X86Ops.VPSLLVQ: lambda x: encode(x, 0x47, pp=1, sel=2, we=1), X86Ops.VPSRLVD: lambda x: encode(x, 0x45, pp=1, sel=2), X86Ops.VPSRLVQ: lambda x: encode(x, 0x45, pp=1, sel=2, we=1), - X86Ops.VPCMPGTB: lambda x: encode(x, 0x64, pp=1, sel=1), X86Ops.VPCMPGTW: lambda x: encode(x, 0x65, pp=1, sel=1), - X86Ops.VPCMPGTD: lambda x: encode(x, 0x66, pp=1, sel=1), X86Ops.VPCMPGTQ: lambda x: encode(x, 0x37, pp=1, sel=2), - X86Ops.VPCMPEQB: lambda x: encode(x, 0x74, pp=1, sel=1), X86Ops.VPCMPEQW: lambda x: encode(x, 0x75, pp=1, sel=1), - X86Ops.VPCMPEQD: lambda x: encode(x, 0x76, pp=1, sel=1), X86Ops.VPCMPEQQ: lambda x: encode(x, 0x29, pp=1, sel=2), X86Ops.VPMULLW: lambda x: encode(x, 0xD5, pp=1, sel=1), X86Ops.VPMULLD: lambda x: encode(x, 0x40, pp=1, sel=2), X86Ops.VPADDB: lambda x: encode(x, 0xFC, pp=1, sel=1), X86Ops.VPADDW: lambda x: encode(x, 0xFD, pp=1, sel=1), X86Ops.VPADDD: lambda x: encode(x, 0xFE, pp=1, sel=1), X86Ops.VPADDQ: lambda x: encode(x, 0xD4, pp=1, sel=1), @@ -783,7 +770,6 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No X86Ops.CMOVB: lambda x: encode(x, 0x0F42), X86Ops.CMOVL: lambda x: encode(x, 0x0F4C), X86Ops.CMOVE: lambda x: encode(x, 0x0F44), X86Ops.CMOVNE: lambda x: encode(x, 0x0F45), X86Ops.VBLENDVPS: lambda x: encode(x, 0x4A, pp=1, sel=3), X86Ops.VBLENDVPD: lambda x: encode(x, 0x4B, pp=1, sel=3), - X86Ops.VPBLENDVB: lambda x: encode(x, 0x4C, pp=1, sel=3), # shuffles X86Ops.VPSRLDQ: lambda x: encode(x, 0x73, reg=3, pp=1, sel=1), X86Ops.VPINSRB: lambda x: encode(x, 0x20, pp=1, sel=3), X86Ops.VPINSRW: lambda x: encode(x, 0xC4, pp=1, sel=1), @@ -823,13 +809,13 @@ def is_two_address(self, x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] in def stack_pointer(self) -> UOp: return def_reg(dtypes.uint64, RSP) # the value of a BUFFER is its address, it moves through registers and the stack as a 64bit int def copy(self, x:UOp, reg:Register): - dt = dtypes.uint64 if x.op is Ops.BUFFER else x.dtype - ret = isel_matcher.rewrite(UOp(Ops.COPY, dt, (x,), tag=reg)) + if x.op is Ops.BUFFER: x = x.replace(dtype=None, arg=replace(x.arg, dtype=dtypes.uint64)) + ret = isel_matcher.rewrite(UOp(Ops.COPY, src=(x,), tag=reg)) assert ret is not None, f"failed to copy {x}" return ret def spill(self, disp:UOp, x:UOp) -> UOp: - if x.op is Ops.BUFFER: x = x.replace(dtype=dtypes.uint64) + if x.op is Ops.BUFFER: x = x.replace(dtype=None, arg=replace(x.arg, dtype=dtypes.uint64)) is_xmm = isinstance(x.tag, tuple) and x.tag[0].cons[0].size == 16 op = X86Ops.VMOVUPSm if is_xmm else X86Ops.MOVm return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=(op, dtypes.void), tag=x.tag) From cb0db2d0420de4145ab0e30baa241c6213ae9593 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 15:28:09 -0400 Subject: [PATCH 266/306] dtype_from_uop is never None (#17819) --- tinygrad/codegen/__init__.py | 4 ++-- tinygrad/renderer/__init__.py | 4 ++-- tinygrad/renderer/isa/x86.py | 6 +++--- tinygrad/renderer/nir.py | 2 +- tinygrad/renderer/ptx.py | 2 +- tinygrad/uop/ops.py | 33 ++++++++++++--------------------- tinygrad/uop/symbolic.py | 2 +- tinygrad/uop/weak.py | 16 ++++++++-------- 8 files changed, 30 insertions(+), 39 deletions(-) diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 5be1e1b2b1c59..4f01a8eca58d9 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -126,7 +126,7 @@ def do_devectorize(b:UOp): if not all(x.shape == b.shape or x.base.is_invalid for x in b.src): return None src = [] for idx_c in itertools.product(*[[UOp.const(i) for i in range(x)] for x in b.shape]): - src.append(b.replace(dtype=None, src=tuple(x.base if x.base.is_invalid else x.index(*idx_c) for x in b.src))) + src.append(b.replace(src=tuple(x.base if x.base.is_invalid else x.index(*idx_c) for x in b.src))) return UOp.stack(*src).reshape(b.shape) if b.op is not Ops.STORE else UOp.group(*src) def do_stack_wmma(u:UOp): @@ -418,7 +418,7 @@ def line_rewrite(lst:list[UOp], pm:PatternMatcher, ctx=None) -> list[UOp]: newlst = [] replaced: dict[UOp, UOp] = {} for u in lst: - nu = u.replace(dtype=None, src=tuple([replaced.get(x, x) for x in u.src])) + nu = u.replace(src=tuple([replaced.get(x, x) for x in u.src])) ret: tuple[UOp, list[UOp]] = pm.rewrite(nu, ctx) or (nu, [nu]) replaced[u] = ret[0] newlst.extend(ret[1]) diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 86cd044a06cf2..40682e7ccf64f 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -9,8 +9,8 @@ # an access takes its dtype from the buffer it indexes, so accessing at another dtype restates the storage on the buffer that owns it def with_storage(x:UOp, dt:DType) -> UOp: - if x.op in {Ops.PARAM, Ops.BUFFER}: return x.replace(dtype=None, arg=replace(x.arg, dtype=dt)) - return x.replace(dtype=None, src=(with_storage(x.src[0], dt),)+x.src[1:]) + if x.op in {Ops.PARAM, Ops.BUFFER}: return x.replace(arg=replace(x.arg, dtype=dt)) + return x.replace(src=(with_storage(x.src[0], dt),)+x.src[1:]) @dataclass(frozen=True) class Estimates: diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index fa7c1ab1f96d4..3bef96b2faea1 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -298,7 +298,7 @@ def abi(ctx:IselContext, x:UOp) -> UOp|None: dt = dtypes.uint64 if x.op is Ops.PARAM and x.arg.addrspace is AddrSpace.GLOBAL else x.dtype arg = replace(x.arg, dtype=dt) if x.op is Ops.PARAM else x.arg # the shape srcs of a PARAM are not values, tag them so they aren't materialized into registers - def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(dtype=None, arg=arg, src=tuple(s.rtag() for s in x.src), tag=(r,)),) + def _reg_arg(r:Register) -> tuple[UOp, ...]: return (x.replace(arg=arg, src=tuple(s.rtag() for s in x.src), tag=(r,)),) def _stack_arg(disp:int): return (def_reg(dtypes.uint64, RSP), UOp(Ops.NOOP), UOp(Ops.INS, arg=(X86Ops.FRAME_INDEX, dtypes.int32), tag=disp), imm(dtypes.uint8, 8)) if sys.platform == "win32": src = _reg_arg((RCX, RDX, GPR[8], GPR[9])[i]) if i < 4 else _stack_arg((i-3)*8+32) @@ -809,13 +809,13 @@ def is_two_address(self, x:UOp) -> bool: return x.op is Ops.INS and x.arg[0] in def stack_pointer(self) -> UOp: return def_reg(dtypes.uint64, RSP) # the value of a BUFFER is its address, it moves through registers and the stack as a 64bit int def copy(self, x:UOp, reg:Register): - if x.op is Ops.BUFFER: x = x.replace(dtype=None, arg=replace(x.arg, dtype=dtypes.uint64)) + if x.op is Ops.BUFFER: x = x.replace(arg=replace(x.arg, dtype=dtypes.uint64)) ret = isel_matcher.rewrite(UOp(Ops.COPY, src=(x,), tag=reg)) assert ret is not None, f"failed to copy {x}" return ret def spill(self, disp:UOp, x:UOp) -> UOp: - if x.op is Ops.BUFFER: x = x.replace(dtype=None, arg=replace(x.arg, dtype=dtypes.uint64)) + if x.op is Ops.BUFFER: x = x.replace(arg=replace(x.arg, dtype=dtypes.uint64)) is_xmm = isinstance(x.tag, tuple) and x.tag[0].cons[0].size == 16 op = X86Ops.VMOVUPSm if is_xmm else X86Ops.MOVm return UOp(Ops.INS, src=fold_address(self.stack_pointer().index(disp)) + (x,), arg=(op, dtypes.void), tag=x.tag) diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index bcf0fdf2e0829..71c00de8a1635 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -125,7 +125,7 @@ class NIRRenderer(Renderer): (UPat.var('x', dtype=dtypes.bool)=2 else ()) + lambda x: x.replace(src=(with_storage(x.src[0], dtypes.uint8),)+((x.src[1].cast(dtypes.uint8),) if len(x.src)>=2 else ()) +x.src[2:]).cast(dtypes.bool)), (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:])), diff --git a/tinygrad/renderer/ptx.py b/tinygrad/renderer/ptx.py index dcf763a89dedd..0b895fbe27927 100644 --- a/tinygrad/renderer/ptx.py +++ b/tinygrad/renderer/ptx.py @@ -47,7 +47,7 @@ def render_val(x, dtype): lambda x: (UOp(x.op, src=tuple(vv.cast(dtypes.float32) for vv in x.src), arg=x.arg).cast(dtypes.half))), # a bool is a predicate register in PTX but a byte in memory, so a bool buffer is accessed through a uint8 view of it (UPat(Ops.LOAD, dtypes.bool, src=(UPat(name="idx"),), name="x", allow_any_len=True), - lambda x,idx: x.replace(dtype=None, src=(with_storage(idx, dtypes.uint8),) + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8),) + ((x.src[1].cast(dtypes.uint8),) if len(x.src) >= 2 else ()) + x.src[2:]).cast(dtypes.bool) if idx.addrspace != AddrSpace.REG else None), (UPat(Ops.STORE, src=(UPat(name="idx"), UPat(dtype=dtypes.bool)), name="x", allow_any_len=True), lambda x,idx: x.replace(src=(with_storage(idx, dtypes.uint8), x.src[1].cast(dtypes.uint8))+x.src[2:]) if idx.addrspace != AddrSpace.REG else None), diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 07ffb8983c925..ab4be971b8330 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -115,11 +115,11 @@ def promo_dtype(src:tuple[UOp,...]) -> DType: dts = [x.dtype for x in src] return dts[0] if all_same(dts) else least_upper_dtype(*dts) -def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: - # here are the dtype production rules, eventually this will go in UOp as a recursive property +def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: + # here are the dtype production rules, total over all Ops match op: case Ops.STORE | Ops.LINEAR | Ops.SINK | Ops.PROGRAM | Ops.SOURCE | \ - Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | \ + Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | Ops.NOOP | \ Ops.TUPLE | Ops.FUNCTION | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: # always void return dtypes.void @@ -131,10 +131,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType|None: return arg[1] case Ops.INS: # arg is (instruction, dtype), a queue command or an asm line is void + assert isinstance(arg, tuple) and len(arg) == 2 and isinstance(arg[1], DType), f"INS arg must be (instruction, DType), got {arg}" return arg[1] - case Ops.NOOP: - # NOOP can be void or carry any dtype (e.g. x.f(Ops.NOOP) or substitute base with NOOP) - return None case Ops.INDEX: # an image access is always float, no matter the storage dtype # TODO: should there be a CAST so src[0].dtype just work? @@ -197,9 +195,9 @@ class UOpMetaClass(type): ucache:dict[tuple, weakref.ReferenceType[UOp]] = {} def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, metadata:tuple[Metadata,...]|None=None, _buffer:Buffer|None=None): - if dtype is None: dtype = dtype_from_uop(op, src, arg) or dtypes.void + if dtype is None: dtype = dtype_from_uop(op, src, arg) # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py - if SPEC == 2 and (expected_dtype:=dtype_from_uop(op, src, arg)) is not None and expected_dtype != dtype: + elif SPEC == 2 and (expected_dtype:=dtype_from_uop(op, src, arg)) != dtype: raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(*key)) @@ -257,11 +255,10 @@ def __reduce__(self): if self.op is Ops.BUFFER and self.realized is not None: args.append(self.realized) return UOp, tuple(args) def replace(self, **kwargs) -> UOp: - new_args = (kwargs.pop("op", self.op), kwargs.pop("dtype", self.dtype), kwargs.pop("src", self.src), - kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) + new_args = (kwargs.pop("op", self.op), kwargs.pop("src", self.src), kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) assert len(kwargs) == 0, f"unused kwargs in replace {list(kwargs)}" - if (self.op, self.dtype, self.src, self.arg, self.tag) == new_args: return self - return UOp(*new_args) + if (self.op, self.src, self.arg, self.tag) == new_args: return self + return UOp(new_args[0], src=new_args[1], arg=new_args[2], tag=new_args[3]) def rtag(self, tag=True): return self.replace(tag=tag) @property def val(self): @@ -282,8 +279,6 @@ def argstr(self): return repr(self.arg) def tagstr(self): return f", tag={self.tag}" if self.tag is not None else "" - def f(self, op, **kwargs): return UOp(op, dtype=kwargs.pop("dtype", self.dtype), src=(self,), **kwargs) - @functools.cached_property def backward_slice(self:UOp) -> dict[UOp, None]: res: dict[UOp, None] = self.toposort() @@ -807,7 +802,7 @@ def _mop(self, op:Ops, arg) -> UOp: case Ops.PERMUTE | Ops.FLIP: src_args = [] case Ops.STACK: srcs = (self,)+tuple(arg) - dtype = cast(DType, dtype_from_uop(Ops.STACK, srcs, None)) + dtype = dtype_from_uop(Ops.STACK, srcs, None) return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else UOp.const(u.val, dtype) if u.op is Ops.CONST else u.cast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] @@ -1694,7 +1689,7 @@ def walk_rewrite(self, root:UOp) -> UOp: else: # rebuild node with rewritten srcs new_src = tuple(self.replace.get(x, x) for x in n.src) - new_n = UOp(n.op, _rebuild_dtype(n, new_src), new_src, n.arg, n.tag) if new_src != n.src else n + new_n = UOp(n.op, src=new_src, arg=n.arg, tag=n.tag) if new_src != n.src else n # top-down: try pm on rebuilt node, use result as-is (no re-traversal) if self.pm is not None and (rewritten:=self.pm_rewrite(new_n)) is not None: new_n = rewritten self.replace[n] = new_n @@ -1753,7 +1748,7 @@ def unified_rewrite(self, root:UOp) -> UOp: continue else: # if srcs changed from rewrites, construct a new UOp with the new srcs - new_src_n = UOp(new_n.op, _rebuild_dtype(new_n, new_src), new_src, new_n.arg, new_n.tag) + new_src_n = UOp(new_n.op, src=new_src, arg=new_n.arg, tag=new_n.tag) # trigger a rewrite of new_src_n, then after that rewrite is done, link it back to n stack.append((n, 2, new_src_n)) stack.append((new_src_n, 0, new_src_n)) @@ -1773,10 +1768,6 @@ def graph_rewrite(sink:UOp, pm:PatternMatcher, ctx=None, bottom_up=False, name=N rewrite_ctx = RewriteContext(pm if not bottom_up else None, pm if bottom_up else bpm, ctx, enter_calls) return rewrite_ctx.walk_rewrite(sink) if walk else rewrite_ctx.unified_rewrite(sink) -def _rebuild_dtype(n:UOp, new_src:tuple[UOp,...]) -> DType: - # TODO: delete this once the dtype field is removed, every rebuild will re-derive - if all(a.dtype is b.dtype for a,b in zip(n.src, new_src)): return n.dtype - return dtype_from_uop(n.op, new_src, n.arg) or n.dtype def sint_to_uop(x:sint, dtype=dtypes.weakint) -> UOp: return UOp.const(x, dtype) def to_max_shape(shape:tuple[sint, ...]) -> tuple[int, ...]: return tuple(int(x.vmax) if isinstance(x, UOp) else x for x in shape) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 3ed54afa794b3..bf7b40db8865d 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -150,7 +150,7 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=bare_const, name="a"), fold_const_alu), (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=casted_const, name="a"), fold_const_alu), (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=[casted_const, bare_const], name="a"), lambda a: - a.replace(dtype=None, src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in a.src)) + a.replace(src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in a.src)) if (dt:=promo_dtype(a.src)) not in dtypes.weaks else None), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index ccf51c644d7df..8f946f33f00aa 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -1,6 +1,6 @@ from dataclasses import replace from tinygrad.dtype import dtypes, DType, AddrSpace, Invalid, least_upper_dtype, strong_dtype, weak_dtype -from tinygrad.helpers import unwrap + from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, dtype_from_uop, promo_dtype def default_dtype(u:UOp): @@ -18,14 +18,14 @@ def commit_weak_consts(u:UOp, dt:DType|None) -> UOp|None: # the concrete dtypes u commits its srcs at: the operands' meet and u's own derived dtype, None if either is weak def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: if u.op not in GroupOp.Broadcastable or (meet:=promo_dtype(src)) in dtypes.weaks \ - or (result:=unwrap(dtype_from_uop(u.op, src, u.arg))) in dtypes.weaks: return None + or (result:=dtype_from_uop(u.op, src, u.arg)) in dtypes.weaks: return None return meet, result def commit_srcs_at(u:UOp, dt:DType) -> UOp|None: # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too dts = derived_dtypes(u, u.src) - ret = u.replace(dtype=None, src=tuple(UOp.const(dt.const(s.val)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is not None else - commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) + ret = u.replace(src=tuple(UOp.const(dt.const(s.val)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is not None else + commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) return None if ret is u else ret def commit_weak_srcs(u:UOp) -> UOp|None: @@ -60,10 +60,10 @@ def lower_weak_node(u:UOp) -> UOp|None: # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait if u.op in _lower_weak_ops and src != u.src and not any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary - else unwrap(dtype_from_uop(u.op, src, u.arg))) - return u.replace(dtype=None, src=src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) + else dtype_from_uop(u.op, src, u.arg)) + return u.replace(src=src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) for s in src[start:])).cast(u.dtype) - return None if src == u.src else u.replace(dtype=None, src=src) + return None if src == u.src else u.replace(src=src) pm_lower_weak = PatternMatcher([ # a gated long index into a small buffer narrows; its out-of-gate value is discarded @@ -74,7 +74,7 @@ def lower_weak_node(u:UOp) -> UOp|None: (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), lambda u,x: x.cast(default_dtype(u.src[0])).cast(default_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), - lambda u: u.replace(dtype=None, arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), + lambda u: u.replace(arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), (UPat(GroupOp.All, name="u"), lower_weak_node), ]) From 086f45cb8163d764d0a569c610ddaffaaa6a02f4 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 16:17:29 -0400 Subject: [PATCH 267/306] dtype is not a UOp field anymore (#17821) --- extra/gemm/cdna_asm_gemm.py | 2 +- test/backend/test_uops.py | 2 +- test/external/external_benchmark_pyrender.py | 4 +-- test/null/test_pattern_matcher.py | 2 +- test/null/test_uop_repr.py | 18 ++++++------ test/null/test_uops.py | 8 +---- test/unit/test_dtype_weak.py | 2 +- tinygrad/llm/kernels/amd.py | 2 +- tinygrad/schedule/multi.py | 2 +- tinygrad/uop/ops.py | 31 ++++++++++---------- tinygrad/uop/render.py | 8 ++--- tinygrad/uop/spec.py | 2 +- tinygrad/viz/serve.py | 6 ++-- 13 files changed, 39 insertions(+), 50 deletions(-) diff --git a/extra/gemm/cdna_asm_gemm.py b/extra/gemm/cdna_asm_gemm.py index 8c1f18aee6d8d..801f407795c14 100644 --- a/extra/gemm/cdna_asm_gemm.py +++ b/extra/gemm/cdna_asm_gemm.py @@ -215,7 +215,7 @@ def custom_uop_gemm(C:UOp, A:UOp, B:UOp) -> UOp: k = UOp.range(K, 0, AxisType.REDUCE) mul = (A.flatten().index((m*UOp.const(K)+k))* B.flatten().index((k*UOp.const(N)+n))).cast(dtypes.float32) - red = mul.reduce(k, arg=Ops.ADD, dtype=dtypes.float32).cast(C.dtype) + red = mul.reduce(k, arg=Ops.ADD).cast(C.dtype) store = C.flatten().index((m*UOp.const(N)+n)).store(red).end(m, n) return store.sink(arg=KernelInfo(name=f'uop_gemm_{M}_{N}_{K}')) diff --git a/test/backend/test_uops.py b/test/backend/test_uops.py index d966f8eb47bd4..67fd41ddc2db2 100644 --- a/test/backend/test_uops.py +++ b/test/backend/test_uops.py @@ -21,7 +21,7 @@ def run_uops(uops_list:list[UOp], bufs:list[Buffer]): def uop(uops:list[UOp], op:Ops, dtype:Optional[DType], src:tuple[UOp, ...], arg:Any=None) -> UOp: if op is Ops.CONST: uops.append(UOp.const(arg).cast(dtype)) elif op is Ops.PARAM: uops.append(UOp.param(arg, dtype, 1)) - else: uops.append(UOp(op, dtype, tuple(src), arg)) + else: uops.append(UOp(op, tuple(src), arg)) return uops[-1] def _test_single_value(vals, op, dts): diff --git a/test/external/external_benchmark_pyrender.py b/test/external/external_benchmark_pyrender.py index 2183e2a99b494..48238e5296ea2 100644 --- a/test/external/external_benchmark_pyrender.py +++ b/test/external/external_benchmark_pyrender.py @@ -7,8 +7,8 @@ @functools.cache def create_uop(a:int) -> UOp: - op, dtype, src, arg, *rest = trace.uop_fields[a] - return UOp(op, dtype, tuple(create_uop(s) for s in src), arg, *rest) + op, src, arg, *rest = trace.uop_fields[a] + return UOp(op, tuple(create_uop(s) for s in src), arg, *rest) if __name__ == "__main__": # load rewrite trace diff --git a/test/null/test_pattern_matcher.py b/test/null/test_pattern_matcher.py index ca9b6bf0efa65..20dc78f9f4ee9 100644 --- a/test/null/test_pattern_matcher.py +++ b/test/null/test_pattern_matcher.py @@ -40,7 +40,7 @@ def fxn(x): assert len(x.src) == 0 return UOp(Ops.CONST, src=(UOp(Ops.CONST),)) matcher = PatternMatcher([(UPat(Ops.CONST, src=(), name="x"), fxn)]) - c1 = UOp(Ops.CONST, dtypes.float, arg=1.0) + c1 = UOp(Ops.CONST, arg=1.0) # second rewrite shouldn't match anything c1 = matcher.rewrite(c1) c1 = matcher.rewrite(c1) diff --git a/test/null/test_uop_repr.py b/test/null/test_uop_repr.py index 1c2b1bd252f1c..f03262a223df3 100644 --- a/test/null/test_uop_repr.py +++ b/test/null/test_uop_repr.py @@ -4,21 +4,21 @@ class TestUOpRepr(unittest.TestCase): def test_simple_const(self): a = UOp.const(42) - self.assertEqual(repr(a), "UOp(Ops.CONST, dtypes.weakint, arg=42, src=())") + self.assertEqual(repr(a), "UOp(Ops.CONST, arg=42, src=())") def test_different_consts(self): a, b = UOp.const(42), UOp.const(3) expected = ( - "UOp(Ops.ADD, dtypes.weakint, arg=None, src=(\n" + - " UOp(Ops.CONST, dtypes.weakint, arg=42, src=()),\n" + - " UOp(Ops.CONST, dtypes.weakint, arg=3, src=()),))" + "UOp(Ops.ADD, arg=None, src=(\n" + + " UOp(Ops.CONST, arg=42, src=()),\n" + + " UOp(Ops.CONST, arg=3, src=()),))" ) self.assertEqual(repr(a+b), expected) def test_walrus_operator_indentation(self): # The reference should have the same indentation as the definition a = UOp.const(42) expected = ( - "UOp(Ops.ADD, dtypes.weakint, arg=None, src=(\n" + - " x0:=UOp(Ops.CONST, dtypes.weakint, arg=42, src=()),\n" + + "UOp(Ops.ADD, arg=None, src=(\n" + + " x0:=UOp(Ops.CONST, arg=42, src=()),\n" + " x0,))" ) self.assertEqual(repr(a+a), expected) @@ -26,9 +26,9 @@ def test_nested_walrus_indentation(self): # Ensure indentation is consistent at multiple levels b = (a:=UOp.const(1)) + a expected = ( - "UOp(Ops.MUL, dtypes.weakint, arg=None, src=(\n" + - " x0:=UOp(Ops.ADD, dtypes.weakint, arg=None, src=(\n" + - " x1:=UOp(Ops.CONST, dtypes.weakint, arg=1, src=()),\n" + + "UOp(Ops.MUL, arg=None, src=(\n" + + " x0:=UOp(Ops.ADD, arg=None, src=(\n" + + " x1:=UOp(Ops.CONST, arg=1, src=()),\n" + " x1,)),\n" + " x0,))" ) diff --git a/test/null/test_uops.py b/test/null/test_uops.py index 2eff8f78d9097..2138b3070deb2 100644 --- a/test/null/test_uops.py +++ b/test/null/test_uops.py @@ -38,15 +38,9 @@ def test_const_default_dtype_is_derived(self): self.assertEqual(UOp(Ops.CONST, arg=ConstFloat(3.0)).dtype, dtypes.weakfloat) self.assertEqual(UOp(Ops.CONST, arg=True).dtype, dtypes.bool) self.assertEqual(UOp(Ops.CONST, arg=Invalid).dtype, dtypes.bool) - # an explicit (strong) const dtype is legal until the field is removed + # UOp.const at a strong dtype builds the CAST that carries it self.assertEqual(UOp.const(3, dtypes.int32).dtype, dtypes.int32) - def test_invalid_stated_dtype(self): - # UOp.const normalizes a stated dtype away (const_like/full pass their position's); the core constructor does not, - # and the spec is what rejects a non-bool Invalid - self.assertIs(UOp.const(Invalid, dtypes.float32), UOp.invalid()) - with self.assertRaises(RuntimeError): type_verify(UOp(Ops.CONST, dtypes.float32, arg=Invalid), spec_shared) - def test_invalid_dtype_and_consumers(self): invalid = UOp.invalid() self.assertIs(invalid.dtype, dtypes.bool) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 88025ef476805..77bedaf2b9b96 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -191,7 +191,7 @@ def test_weak_int_binop(self): # float bitwise builds, the spec rejects it with Context(SPEC=1): f32, wf = UOp.const(1.0, dtypes.float32), UOp.const(1.0) - for bad in (f32.alu(Ops.AND, f32), UOp(Ops.AND, dtypes.float32, (f32, f32)), UOp(Ops.AND, dtypes.int32, (wf, wf))): + for bad in (f32.alu(Ops.AND, f32), UOp(Ops.AND, (f32, f32)), UOp(Ops.AND, (wf, wf))): with self.assertRaises(RuntimeError): type_verify([bad], spec_shared) def test_integer_values(self): diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py index ef8a715e5e37f..23a869a64d89e 100644 --- a/tinygrad/llm/kernels/amd.py +++ b/tinygrad/llm/kernels/amd.py @@ -116,7 +116,7 @@ def _amd_load(ptr:UOp, lanes:int|None=None) -> UOp: if lanes is None: return ptr.load(arg="nontemporal") buf, coords = ptr.src[0], ptr.src[1:] idx = sum((coord*math.prod(buf.shape[i+1:]) for i,coord in enumerate(coords)), UOp.const(0)) - return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load(dtype=ptr.dtype) + return UOp(Ops.SHRINK, src=(buf.flatten(), idx, UOp.const(lanes))).load() def _load_byte(raw:UOp, base:UOp, offset:UOp) -> UOp: return (raw[base + offset//4] >> ((offset&3)*8).cast(dtypes.uint32)) & 255 def _half(value:UOp) -> UOp: return value.cast(dtypes.uint16).bitcast(dtypes.float16).float() diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index bbee3258be469..06a366ff19ca1 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -266,7 +266,7 @@ def store_dest_multi(root:UOp, multi:UOp): def passthrough_multi(root:UOp, multi:UOp): new_src = (multi.src[0],)+tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src[1:]) - return UOp(root.op, root.dtype, src=new_src, arg=root.arg).unshard(multi.arg, multi.src[1:]) + return UOp(root.op, src=new_src, arg=root.arg).unshard(multi.arg, multi.src[1:]) def rewrite_into_function(call:UOp): if call.arg.precompile: return None diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index ab4be971b8330..dec5f695dc6ce 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -193,14 +193,13 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: class UOpMetaClass(type): ucache:dict[tuple, weakref.ReferenceType[UOp]] = {} - def __call__(cls, op:Ops, dtype:DType|None=None, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, + def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, metadata:tuple[Metadata,...]|None=None, _buffer:Buffer|None=None): - if dtype is None: dtype = dtype_from_uop(op, src, arg) - # TODO: delete this once the dtype field is removed, for now it just re-implements spec.py - elif SPEC == 2 and (expected_dtype:=dtype_from_uop(op, src, arg)) != dtype: - raise RuntimeError(f"bad dtype {dtype}, expected {expected_dtype} on {op}") - if (wret:=UOpMetaClass.ucache.get(key:=(op, dtype, src, arg, tag), None)) is not None and (ret:=wret()) is not None: return ret - UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(*key)) + # NOTE: the key must separate nodes of different dtype: a CONST's dtype is the type of its arg, and True == 1 as dict keys + if (wret:=UOpMetaClass.ucache.get(key:=(op, src, arg, tag, type(arg)), None)) is not None and (ret:=wret()) is not None: return ret + UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(op, src, arg, tag)) + # derive at construction: bottom up, so no recursion, and a bad node fails where it is built + created.__dict__["dtype"] = dtype_from_uop(op, src, arg) if metadata is not None: all_metadata[created] = metadata # NOTE: this value is set by pickle when pickling a realized tensor if _buffer is not None: @@ -241,24 +240,25 @@ def __get__(self, x:UOp|None, owner=None): @dataclass(eq=False, slots=True) class UOp(RandMixin, metaclass=UOpMetaClass): op:Ops - dtype:DType = dtypes.void src:tuple[UOp, ...] = tuple() arg:Any = None tag:Any = None + @functools.cached_property + def dtype(self) -> DType: return dtype_from_uop(self.op, self.src, self.arg) def __del__(self): # NOTE: getattr because this object may be partially constructed (e.g. if __init__ raised, like the BEAM timeout SIGALRM) if Ops is not None and getattr(self, 'op', None) is Ops.BUFFER and (buffer:=buffers.get(self)) is not None: buffer.ref(-1) - try: del UOpMetaClass.ucache[(self.op, self.dtype, self.src, self.arg, self.tag)] + try: del UOpMetaClass.ucache[(self.op, self.src, self.arg, self.tag, type(self.arg))] except (AttributeError, KeyError): pass def __reduce__(self): - args = [self.op, self.dtype, self.src, self.arg, self.tag, self.metadata] + args = [self.op, self.src, self.arg, self.tag, self.metadata] if self.op is Ops.BUFFER and self.realized is not None: args.append(self.realized) return UOp, tuple(args) def replace(self, **kwargs) -> UOp: new_args = (kwargs.pop("op", self.op), kwargs.pop("src", self.src), kwargs.pop("arg", self.arg), kwargs.pop("tag", self.tag)) assert len(kwargs) == 0, f"unused kwargs in replace {list(kwargs)}" if (self.op, self.src, self.arg, self.tag) == new_args: return self - return UOp(new_args[0], src=new_args[1], arg=new_args[2], tag=new_args[3]) + return UOp(*new_args) def rtag(self, tag=True): return self.replace(tag=tag) @property def val(self): @@ -544,7 +544,7 @@ def __index__(self): return self.__int__() @recursive_property def trace_num(self): num = next(ucount) - uop_fields[num] = (self.op, self.dtype, tuple(s.trace_num for s in self.src), self.arg, self.tag)+((self.metadata,) if TRACEMETA>=2 else ()) + uop_fields[num] = (self.op, tuple(s.trace_num for s in self.src), self.arg, self.tag)+((self.metadata,) if TRACEMETA>=2 else ()) return num # *** uop syntactic sugar *** @@ -603,8 +603,7 @@ def after(self, *src:UOp, **kwargs): return UOp(Ops.AFTER, src=(self,)+src, **kw @property def without_after(self) -> UOp: return self.src[0] if self.op is Ops.AFTER else self def barrier(self, *src:UOp): return UOp(Ops.BARRIER, src=(self,)+src) - def ins(self, arg, **kwargs): - return UOp(Ops.INS, src=kwargs.pop("src", self.src), arg=(arg, kwargs.pop("dtype", self.dtype)), tag=kwargs.pop("tag", self.tag)) + def ins(self, arg, **kwargs): return UOp(Ops.INS, kwargs.pop("src", self.src), (arg, kwargs.pop("dtype", self.dtype)), kwargs.pop("tag", self.tag)) def contract(self, *rngs:UOp): assert all(x.arg[-1] == AxisType.UPCAST for x in rngs), "all contract ranges must be upcast" return UOp.stack(*[self.substitute(dict(zip(rngs, [r.const_like(i) for r,i in zip(rngs, idx)]))) @@ -1689,7 +1688,7 @@ def walk_rewrite(self, root:UOp) -> UOp: else: # rebuild node with rewritten srcs new_src = tuple(self.replace.get(x, x) for x in n.src) - new_n = UOp(n.op, src=new_src, arg=n.arg, tag=n.tag) if new_src != n.src else n + new_n = UOp(n.op, new_src, n.arg, n.tag) if new_src != n.src else n # top-down: try pm on rebuilt node, use result as-is (no re-traversal) if self.pm is not None and (rewritten:=self.pm_rewrite(new_n)) is not None: new_n = rewritten self.replace[n] = new_n @@ -1748,7 +1747,7 @@ def unified_rewrite(self, root:UOp) -> UOp: continue else: # if srcs changed from rewrites, construct a new UOp with the new srcs - new_src_n = UOp(new_n.op, src=new_src, arg=new_n.arg, tag=new_n.tag) + new_src_n = UOp(new_n.op, new_src, new_n.arg, new_n.tag) # trigger a rewrite of new_src_n, then after that rewrite is done, link it back to n stack.append((n, 2, new_src_n)) stack.append((new_src_n, 0, new_src_n)) diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index 914e398a15c1c..c30625e1629fb 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -11,7 +11,7 @@ def dfs(x:UOp, cache:dict): if cache is None: dfs(x, cache:={}) if (cx:=cache.setdefault(x, [0,0,False]))[2]: return f"{' '*d}x{cx[0]}" cx[2], srcs = True, (''.join(f'\n{pretty_print(s, cache, d+2)},' for s in x.src)) - return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, {x.dtype}, arg={x.argstr()}{x.tagstr()}, src=({srcs}))" + return f"{' '*d}{f'x{cx[0]}:=' * (cx[1]>1)}{type(x).__name__}({x.op}, arg={x.argstr()}{x.tagstr()}, src=({srcs}))" # ***** uop helpers ***** @@ -94,10 +94,6 @@ def render_marg(ctx,x:UOp): (UPat(Ops.RANGE, src=(UPat(Ops.CONST, name="c"),), allow_any_len=True, name="x"), lambda ctx,x,c: "UOp.range("+', '.join([str(c.val)] + [repr(y) for y in x.arg])+ (f', src={srcs(ctx, x.src[1:])}' if len(x.src) > 1 else '')+")"), - # TODO: index shouldn't mismatch dtype - (UPat(Ops.INDEX, src=(UPat(), UPat()), allow_any_len=True, name="x"), lambda ctx,x: - f"{ctx[x.src[0]]}.index({ctx[x.src[1]]}, "+''.join([f"{ctx[xx]}, " for xx in x.src[2:]])+ - f"dtype={x.dtype})" if x.src[0].dtype != x.dtype else None), # TODO: movement ops simplify stuff, this can break SPEC=2 #(UPat(GroupOp.Movement, name="x"), lambda ctx,x: f"{ctx[x.src[0]]}.{x.op.name.lower()}({render_marg(ctx,x)})"), # NOTE: CMPNE doesn't work cause there's no __rne__ @@ -117,7 +113,7 @@ def render_marg(ctx,x:UOp): # NOTE: you can remove pm_pyrender_extra and it'll still be correct pm_pyrender = pm_pyrender_extra+PatternMatcher([ - (UPat(GroupOp.All, name="u"), lambda ctx,u: f"UOp({u.op}, {u.dtype}, {srcs(ctx,u.src)}"+(f", {repr(u.arg)})" if u.arg is not None else ")")), + (UPat(GroupOp.All, name="u"), lambda ctx,u: f"UOp({u.op}, {srcs(ctx,u.src)}"+(f", {repr(u.arg)})" if u.arg is not None else ")")), ]) def _render_with_splits(lst:list[UOp], pm:PatternMatcher, to_render:set[UOp], split_depth:int=100) -> dict[str, str]: diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 87e2b60f19464..e77fdd5d4ab6d 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -54,7 +54,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base (UPat(Ops.NOOP), lambda: True), # CONST is everywhere; Invalid is a bool const - (UPat(Ops.CONST, src=(), name="x"), lambda x: x.dtype is dtypes.bool if x.is_invalid else type(x.val) is type(x.dtype.const(x.val))), + (UPat(Ops.CONST, src=(), name="x"), lambda x: x.is_invalid or type(x.val) is type(x.dtype.const(x.val))), # STACK is everywhere too (UPat(Ops.STACK, dtype=dtypes.void, src=()), lambda: True), diff --git a/tinygrad/viz/serve.py b/tinygrad/viz/serve.py index cb7f54b63cf8a..32c973932c6c4 100755 --- a/tinygrad/viz/serve.py +++ b/tinygrad/viz/serve.py @@ -171,9 +171,9 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: def _reconstruct(data:VizData, a:int, depth:int|None=None): if depth is None and a in data.all_uops: return data.all_uops[a] - op, dtype, src, arg, *rest = data.trace.uop_fields[a] - if depth is not None and depth <= 0: return UOp(op, dtype, (), arg, *rest) - ret = UOp(op, dtype, tuple(_reconstruct(data, s, None if depth is None else depth-1) for s in src), arg, *rest) + op, src, arg, *rest = data.trace.uop_fields[a] + if depth is not None and depth <= 0: return UOp(op, (), arg, *rest) + ret = UOp(op, tuple(_reconstruct(data, s, None if depth is None else depth-1) for s in src), arg, *rest) if depth is None: data.all_uops[a] = ret return ret From 0640cfce35a1a267f1aa69fe14718d1cae0153a4 Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 17:00:24 -0400 Subject: [PATCH 268/306] remove spec dtype check that's trivially true [PR] (#17823) --- tinygrad/renderer/cstyle.py | 2 +- tinygrad/runtime/ops_python.py | 1 - tinygrad/uop/spec.py | 28 +++++++++++++--------------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index b2b23fc02daca..680bd4608dfdf 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -247,7 +247,7 @@ def _render(self, uops:list[UOp]) -> tuple[str, list[str], list[tuple[str,tuple[ (u.op in {Ops.STACK, *(GroupOp.ALU-{Ops.WHERE}), Ops.CAST, Ops.BITCAST} and child_count[u] == 1 and not getenv("EXPAND_SSA"))): r[u] = l else: - if u.op not in {Ops.RANGE, Ops.STORE, Ops.BUFFER} and u.dtype != dtypes.void: + if u.op not in {Ops.RANGE, Ops.BUFFER} and u.dtype != dtypes.void: l = f"{self.render_type(u)} {r[u]} = {l}" + (";" if u.op is not Ops.SPECIAL else "") kernel.append("\n".join(" "*depth + line for line in l.split("\n"))) if prefix: c[prefix] += 1 # if it was used, increment diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index 9a7ea339af9dc..a5d532ead263c 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -80,7 +80,6 @@ def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tup # in the python emulator, the warp is always in sync i += 1 continue - assert u.dtype is not None, f"{u.op} is missing a dtype" if u.op is Ops.STORE: assert len(src_values) == 2, f"STORE must be lowered to 2 srcs, got {len(src_values)}" store_gate = exec_masks[-1] diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index e77fdd5d4ab6d..65305dacad6d6 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -68,8 +68,8 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base (UPat(GroupOp.Comparison, dtype=dtypes.bool, src=(UPat.var("x"), UPat.var("y"))), lambda x,y: matches_dtype(x, y.dtype) or matches_dtype(y, x.dtype) or x.dtype in dtypes.weaks or y.dtype in dtypes.weaks), (UPat((Ops.AND, Ops.OR, Ops.XOR, Ops.SHL, Ops.SHR), name="x"), lambda x: False if any(dtypes.is_float(s.dtype) for s in x.src) else None), - (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: (matches_dtype(x, a.dtype) or x.dtype is dtypes.weakint) - and (matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid)), + (UPat((Ops.SHL, Ops.SHR), src=(UPat.var("x"), UPat.var("c")), name="a"), lambda a,x,c: + matches_dtype(c, a.dtype) or c.dtype in (dtypes.uint, dtypes.weakint) or x.base.is_invalid), (UPat((Ops.CDIV, Ops.CMOD, Ops.FLOORDIV, Ops.FLOORMOD), name="x"), lambda x: None if dtypes.is_int(x.dtype) or any(s.base.is_invalid for s in x.src) else False), (UPat(GroupOp.ALU, name="x"), lambda x: all(matches_dtype(y, x.dtype) or y.dtype in dtypes.weaks for y in x.src)), @@ -78,8 +78,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base (UPat((Ops.BITCAST, Ops.CAST), src=(UPat(),), name="x"), lambda x: isinstance(x.arg, DType)), # RANGE can be in the big graph now. a void RANGE is a bound-less loop header, the arg is an axis id like RANGE - (UPat(Ops.RANGE, src=(UPat.var("x"),), allow_any_len=True, name="rng"), lambda rng,x: - matches_dtype(x, rng.dtype) and isinstance(rng.arg, tuple) and len(rng.arg) >= 2 and \ + (UPat(Ops.RANGE, src=(UPat(),), allow_any_len=True, name="rng"), lambda rng: isinstance(rng.arg, tuple) and len(rng.arg) >= 2 and \ all(isinstance(ra, int) for ra in rng.arg[0:-1]) and isinstance(rng.arg[-1], AxisType)), (UPat(Ops.INDEX, name="x"), lambda x: len(x.src)>0 and all(dtypes.is_int(y.dtype) or y.base.is_invalid for y in x.src[1:]) or None), # END closes RANGEs @@ -97,7 +96,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base # AFTER on Movement Op, PARAM, BUFFER, CONTIGUOUS, or another AFTER (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.BUFFER, Ops.CONTIGUOUS, Ops.INDEX, Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS})),), - allow_any_len=True, name="x"), lambda x: matches_dtype(x.src[0], x.dtype)), + allow_any_len=True), lambda: True), # CUSTOM (inline and non inline): the arg is the source string and the dtype it produces, void for a bare statement (UPat((Ops.CUSTOMI, Ops.CUSTOM), name="x"), @@ -132,7 +131,7 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base def is_device(d): return isinstance(d, str) or (isinstance(d, tuple) and all(isinstance(s, str) for s in d)) -def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < len(t.src) and matches_dtype(t.src[g.arg], g.dtype) +def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < len(t.src) # these ops can exist in tensor but not programs. example: movement spec_tensor = PatternMatcher([ @@ -160,7 +159,7 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), valid_gettuple), # SPECIAL is index before index lowering. custom_kernel currently has this - (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.weakint),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), + (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.weakint),), name="s"), lambda s: isinstance(s.arg, str)), # movement ops (UPat((Ops.RESHAPE, Ops.EXPAND), src=(UPat(), UPat())), lambda: True), @@ -173,20 +172,19 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < and isinstance(x.arg[1], int) and all(y.dtype in (dtypes.weakint, dtypes.int) for y in x.src[1:])), # COPY - (UPat(Ops.COPY, name="copy", src=(UPat.var("x"),)), lambda copy,x: matches_dtype(x, copy.dtype) and is_device(copy.arg)), - (UPat(Ops.ALLREDUCE, name="red", src=(UPat.var("x"),)), lambda red,x: matches_dtype(x, red.dtype) and isinstance(red.arg, tuple) and - len(red.arg) == 2 and red.arg[0] in GroupOp.Reduce and is_device(red.arg[1])), + (UPat(Ops.COPY, name="copy", src=(UPat(),)), lambda copy: is_device(copy.arg)), + (UPat(Ops.ALLREDUCE, name="red", src=(UPat(),)), + lambda red: isinstance(red.arg, tuple) and len(red.arg) == 2 and red.arg[0] in GroupOp.Reduce and is_device(red.arg[1])), # UNSHARD/MSELECT/MSTACK # an UNSHARD carries the value and one sharding range per sharded axis (usually a DEVICE RANGE, but can be a derived expression) - (UPat(Ops.UNSHARD, name="multi"), lambda multi: len(multi.src) == 1+len(multi.arg) and matches_dtype(multi.src[0], multi.dtype) + (UPat(Ops.UNSHARD, name="multi"), lambda multi: len(multi.src) == 1+len(multi.arg) and all(isinstance(a, int) for a in multi.arg) and all(r.dtype in dtypes.weaks for r in multi.src[1:])), (UPat(Ops.MSELECT, name="x"), lambda x: isinstance(x.src[0].device, tuple) and x.arg < len(x.src[0].device)), (UPat(Ops.MSTACK, name="x"), lambda x: all(isinstance(s.device, str) for s in x.src) or (all_same(x.src) and x.src[0].device is None)), # CONTIGUOUS ensures the source UOp realizes - (UPat((Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD), name="root", src=(UPat.var("x"),), arg=None), - lambda root,x: matches_dtype(x, root.dtype)), + (UPat((Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD), src=(UPat(),), arg=None), lambda: True), # TODO: this should not be here. STAGE is transformed to BUFFER later (UPat(Ops.STAGE, src=(UPat(),), allow_any_len=True), lambda: True), @@ -224,7 +222,7 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.ENDIF, dtype=dtypes.void, src=(UPat(Ops.IF),)), lambda: True), # SPECIAL is int32 after index lowering - (UPat(Ops.SPECIAL, src=(UPat.var("x", dtypes.int32),), name="s"), lambda s,x: matches_dtype(x, s.dtype) and isinstance(s.arg, str)), + (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.int32),), name="s"), lambda s: isinstance(s.arg, str)), ])+spec_shared spec_hcq = PatternMatcher([ @@ -271,7 +269,7 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < (UPat(Ops.CALL, src=(UPat((Ops.SINK, Ops.LINEAR, Ops.PROGRAM, Ops.CUSTOM_FUNCTION)),), allow_any_len=True), lambda: True), # after on PARAM or AFTER (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.AFTER, Ops.BUFFER, Ops.MSTACK, Ops.MSELECT, Ops.BITCAST, Ops.RESHAPE})),), - allow_any_len=True, name="x"), lambda x: matches_dtype(x.src[0], x.dtype)), + allow_any_len=True), lambda: True), ]) # **** pyrender (move this) **** From 812d5cef626213c0b4ac4b7dde1111a5ab1713e7 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:54:58 -0700 Subject: [PATCH 269/306] delete Ops.FUNCTION (#17824) * delete Ops.FUNCTION: value-producing calls are just CALLs with a TUPLE body a CALL with a TUPLE body (always void dtype) is value-producing and gradient-able, outputs are extracted with GETTUPLE like before; all other CALLs are opaque. TUPLE/GETTUPLE are untouched * match the TUPLE body in the UPat, not in the rewrite * remove FUNCTION from tinyspec (regen pdf) and viz * fix and spec --- spec/tinyspec.pdf | Bin 98831 -> 98739 bytes spec/tinyspec.tex | 13 ++++++------ test/null/test_viz.py | 2 +- test/unit/test_call.py | 10 ++++----- test/unit/test_function.py | 9 ++++++++ tinygrad/mixin/gradient.py | 14 ++++++------- tinygrad/schedule/indexing.py | 4 ++-- tinygrad/schedule/multi.py | 12 ++++++----- tinygrad/schedule/prepare.py | 4 ++-- tinygrad/tensor.py | 11 +++++----- tinygrad/uop/__init__.py | 4 ++-- tinygrad/uop/ops.py | 38 +++++++++++++++++----------------- tinygrad/uop/render.py | 4 ++-- tinygrad/uop/spec.py | 6 +++--- tinygrad/uop/symbolic.py | 2 +- tinygrad/viz/js/worker.js | 2 +- tinygrad/viz/serve.py | 8 +++---- 17 files changed, 76 insertions(+), 67 deletions(-) diff --git a/spec/tinyspec.pdf b/spec/tinyspec.pdf index e9dd6e4207c8a08c2c6dd4faad5240554cf38ece..6e52ace8e9abee7717ec7b8c997e7f6eae04c9ef 100644 GIT binary patch delta 19059 zcmV)QK(xP)ga)&N29Py>G%`6kK0XR_baG{3Z3=kW?OaK3B)1j5=U4PmLeLfNizDk0 zATf}eow>z1sHaC0!*Ohn4F|}t&-b{K#j0+NBzq3Ap=pWD;_~>G?=7suj|YBW_@5uF zI9To%PyZd>euuXA`1vCJyubMQr#FW`{DmL5T?E%k@xzU z=fB@EW;|m`;OBfVH2$`RF?m8eV~oFLOg!S}?fp?Pen&0$N67>|C-QT#zjF8#`CEF5 zzNHc3H)x|#eeNM`oI`W;IF(V|y|s6-KaEVYP!GeX2dzM>`dqtCMZcH%X;+$;)jZ^R z;!o&*rd5_6n8%=h#XiKjX;h5%k2ik>DdmCl#cR!tBT{mYvteGl!%?`!d2Y;vmS&5V zXUrKWomUlhgI#Ht=F|OrM}f2RoSVau%f%~e^K>jNR>V)5g4&4ClBVMAlIF|Nt}a5J z(gN&(t3@iqy$K88&Ms05rNBKNUszR^fMIl6f1#H@fIvon7)1{=te-DVaxT*|JaLP1 zpadsM(BkN{giEXwcU8WQBu#(Oj^txvw$hTxZl`3Yp6ae|r6tS9JVvIS%$wl6-38ZX zF%TsZ(Q9`O=8#-EQ69koAC+{01ZC}In0B{y??K?oE{R2H!X#{IrJ|kHsl_>ndYtv< z>znU?xJ~_kWVa)nuxC20?&goTv%&1bz2++Q`RT*k`A}jJn&jd7p&mZG-&z%R#>^9j z`uXAgEd=A{>)YGeA+W`R9`z8c$m}y7xEK!42;pp+q!6(i9O7s+#H0+~pWRBuZSilC z-=G{ri;zhef*uSanpj%|4KiD?H#ktO_3E&?B_EZ4=ZjTXDa1Nz89OfCh|G9K&KWO# z6Lhf6e`PfjB;uU56G!M|Wpf8SXdTAfib;B^+r@CF#hU72;k2l*R91BfNmr%kxqgvV z2JJCuX3Oq6`!J=nUbxWAHw-vmruNL4jb##Z6w6lV9M*Jd7Myi+)=$ABI#!|ZO_}3Q6j$tTly~26iP1A{GmM>@Cy;^2y3z! ztF&41;m9pfZ3gwtlR`hm8G`y|D>cf@UEE_Jn#e7dOdv~7iu|N#bOG(C_Q?1^4bWdWFm7O)&b&m4XH~uni%OY6kI#SgPs&bP=9Z%IyC+mPYJiokQVNrw)(W=(UfTm+ zf{efckOiK3mnThlZV)a_Xo-|77{>L%;Sh|p0l!)>n_QiO3OdYH=p6PHTQeT@Jszde z|NQ3g<6*JhGT{&Z!GYN1s2v5vuL7fgAHF*%rWfgqH0m9HCvo02=|4Dvix@2R^D+V zhHb+=fo4|Ywo7xnZM(H?X%>Ye%zUloy>rbDlI5An_DB^Cl4-L+5mWF~wfT;J+p2|& z0}-@*F$AHTYusL&7)QS?x{Ix)bE*%^Lm1y)rPJt%{sf1Kw2a$oACZp2gdN_J&Uu zl^{24aa?+}2Cfs`Qh-AcXp?k*hmdtgdG`^jHU?i-Fq?PPv8{)7G`zJY4>i!C+|;0| zg=?=`PjEIe!B2E_e$7cKK0#P5{KUh26?qWaT_X>%+b9qH0L}TWrDK$eV3dnpy28qD zNka+uoRbP;t*ON%p4}y0=t4gEo1Q4t%4ZT2`gRf%`t3?gcxFdp1}bfTON=96BsU~x zTOOudCyZG%Je3LCHI$6|dOphmVCxjw;K>4;^T+`^+&4F6!K6n}l2|HBWHcdV;M2IH zV)_!TsHd2ncNx7!(a8~*t3QE1tjO^J#A0q~l#NIQNY6m2QAZp}i86^W^e$2a@(V;?rxMA-zuHnor)`~VF7R4o)4IHk>)2PS^M~N7k;tXnjrrk2` zO~++o{Pg+kCbAzG4tmZ!`Z2CJMp$4ptV#-^Usc4}%9_=0 zvr^KxdhMmGb?FvcW^3junAiG#7I`CgoQoXqIzRmUuwkhYp^^=M?ac*Lj953EF9#Q@ zgNLH&UXyRBx>~gWDDb>acw0up|TN=6T5~Ww%u@T z5%iTa=eg@=7UnFk9&j1`VQ+LnlmJ-jU0GSxz0Q=(M-Vm*_Np`E?T?+)yL*)&n_<& zfop;;Sv9!3K4OsLQnE2+d3{m6(?~h?rcxBZUKCp3Q0X*(4?^n%3=;<>o91g=S?O^6 zN*1YYTv@Fb+JxV7WpQ9qvPOmmPZlFq3K zA{_fp-9Eo1e;gF&^UM@1AJp}Jl1nX0SeNl0+W>lE;Z_Q}P7IWwS_ygN6nK{oNLpso zi4+uEg_W8B#^j`mB8i_S|AqXQy=$UyIg(NAwP1sPS%lUV$t>PH&St^Zl=FhG7_LRj z8BRp(B<`vokrk+{DEnRG-JQk=!Zx!G-WT}6Vp(fgtRI{F{$rE1el69}({wnxpK>P- z=;Ez(%0-q39D%kYnbXol#-FkRZBsK+s7=>7LjbCg@^x**cB=O?iB`~gY9I5FmVyGA zZETx=UsWvIk;H@)n5^E@ToX3uD~1@DSdf(k_JS3`@WC$Jd^y)KtAZ#5V{6lZ-+cCbxBeH}U(*687)u;lY|3<^8S#)kv3c=KW=P zkyRchN%6Vvnrd8w(5`mY4{Pt1V$6u^=O6(qiTeNj91BKsDPd)z1yHo#cIvNr@s9z~+^`8$v5c0U0#`zF9HYjy} zFc%}o%0SIfvC+ziG?|p&hONJ!4hOmmS26?hfqFRgYZ@RZZGd>?t`ev|^D>2`z$egu zIgbqL&2S-fFX9HgcY1tlb0}8bR0~ay3CxPH2_J~RG9IWw;9_n(e9!wxNg~kPbX-}` zlW0#7>@^LD3clE5w6V6-oV3t?Df_D1(mv#%_XD!M$pWtMkS2eLqkc z4ApydgY%~j{uzDP_+d8i7MK(G(aBu|O$A3@u`ATW`cQJU)L8Djg=wcQ8o@W~OVDg% znHIIXILEpgwrK%dxYu)n0qMDa&8+avG$Mm}d{eq!;?SqYS8k=LNKwA|p<;A=>d`P% zehT#@hW!?)Q@#;n!mk36yxsHdM!SM=`Sfx;*)$;{(Z&%rRH6k1HI!Ic{>*CwQe#3~ z)({@*2N5DMg#0Q}Lzj9i^e3ryDi~2NDK*SoNtW2Z{J3?%iB})(tXA{~8@n z1Bjt!TITFi&K#u-O;fOtr7vf91|Dr{7>o3+jQZR#^YdEdnI!GcgH3T4>OVAH>10%J)m1{& z=2!{APyuMD>W8A);ondWG&Wu?2-092tjvv>&%m+g3->)P6#r;Tb(IjLah9g|XLN5| za&LULYD&zN-3|$Vm8R?WNRWJom@KLi;iGiT*Obs`luTMM#-xM>cbbq|k$kI!#&|jh zH7%jRg>LO%s%l0f^MGmw=7x=y=4+~Ew3UmM^{oUO3^2dj)ze|4zL5-!R(3A{4=)Rm zb2Ta4+Y#aqb8DhVBu5X1%@xSXn7qc^{7G|1<-xfN6`p{9UfwrXYd020tj)@OzmrPmsBh#5?YYM8wtR0uMyjycPfZFqn0^b$Ol@S(Pcrb`Q~0S*XKFRgPxc~0 zFYDlK9o#8bzTVF-yZ2^W%x|!)N>3m5E-?zy|A^OIEIGBiC8sn=yoNZPgd8r`_)?c` zltm&{I%x-gy;GXAjbaqXS_s;~G~vX8jT##-+bDz5a^Vq{nBV&ODBC3IjyOjmVH~}& zP33y_xz_Uc<(=iqOmn-by!&GHL|2-cQgRgDa-FT~1h4c>*YM)aPT9P$BSbP(j#p%| zl*^wk2MaEoDJswbAbltnmlXBF-CNT(Ut=^4#c~;c#Zp&B6E+ET)lzL4O^2KPMj1ah zn&|sdM$_PG2;u z##22rVDZaVz(Q!_KbhDNy`um{*Ikr63yMxrzs&yBrcAkMS8fFM*8xq5K8nUi)2Dh{ zD!28OYwZgTnj3M`ITRb-q*CsJCv)3G(`W2|CO{i&S}=9RZD1;xG%ZZq)xevKjyL(> z{V9C?1{L{GZ-l%e(?2_aj_V|NM_FTza(NIXRS_gNO^N0j)O2sD*ero)%ed<^?L>Ro z+jL`arK3SETs7nkkk}&gm`H+@IeR_IHK37pVgy9FHDTQ01%O}v?tg0qhhqw5Ze(+l zBM&rxFf}kfJ_>Vma%Ev{3V7PhTU&2rw+(*pub8J|jKU`pMTt_^8DJM^0`#eA{g%yx z$5Ss_bXz2C(Ea%(b=Mgk&)C^TlZ9_OXDEuKK9Zl5-H$uHQ~Xcw2zI2y66C+#<3}F* zz+d0yuMh3#Le}tis~~^>^S6`1pFZr~-S57C`i6E|FTq;v{Ot8AMXFS`@?#?Y{5!v8K;8g(<4OKfiq%sHNk)1qpBIgtDt{&2)0b1Cb<#e6 zj;GJ%Rs6VHD3F4%`)_BFTUfLv>vkV`DYUS3USJ*k$L=q?@AA6!jveK|VQ*66-wa#8 zI+u-_%y+m2F#0a(BYN?wMGpKg_%I}M!@`6b(c)w^9Fx2NxCu3kUWD*>jaa_Mh z3|>1J7s)XO;QeZqb+lh5Z7kEMg?(;+VThdr=<0*0!dKsf{_}7%9SR7FM@J6d+p1EpbXZ%9m)6 z^`j(K0uzsAw5*aIlEfIY@rfwOUIfZsWHa&;h>F&CPI*x<*yogI$)6^d;v#N;iBpV& zEa`l9e#@s2B(tCi?4a9>5>$Npm_=nn1FM6GSxwomqPnXp#jwq`6q7W2){tBWl(&oW zDzxy2hcu_iAUBI42#dC?!LVb*5;f$;DLASPkuhge$Y;u~k(hF)!E%$StzVpz4OGL| zUqBP|3j}Lm+-P+)Z8Wu@7szIRFxbhivI))CgBQ{KmU!ID?rczz0;>4tm6(o8kl0&K z(7}2Mh!K`C?+Br_ zj5CLMhI&Nr#bJuw%^$bj=rsu-m__9Jbpmj~&JuuG0Lf1iKtZ9;g86}e(cL<|T3H>v z2J^M%NOb6NJHRHp2KC}O`Oy`iFK)IS_@^gu5%gmuYOkb@Abs$V(WnMj1;QS>V}L#N z&^^vyKC=zb{VEs>GCGcPe~RyY{W-4^xch?^@lN3|6t_%(tK|S73@wxkhWFSLP1eF0DDZa~Xu(JF(ef z{%l}fEY!m0EcXKU$GNnQ*+-WI(At%Gac-QnkzFp^jM345>2^VGIeqO!E(E(oukoy)}+vl@tBinDXmK+x?&hEm;pl$ z9!b<-)vEe%(Zh7t8k~Yx8JKEnHcnn#N)#ugt;2Y1AZ!|cRnQ*i2t7422s?(s%vlA)^|3Y9 zo(0Zz4GS|2-cgt29bD#WO5NJkp{EQu|WWX2w#a(G8X&Kb^&! zH9B*DgQMJ%F$*_hYx1s9nAp4+6`OmchHXsfPibYjb9y083n^SRgOYr%&ZdQsTjkAJ z9o~BbDd~JZa~h-K9bOwo${czIxC|p!D&d@rv;jVQUoN#M@SQV!@mm4lKZO~kKx00yTBZV$)Z8&c8 zyUhIrF5zJ~zPOCQ#w^-f+rq)nxVz-2A=dbi509Ym^NE(J9l%T}Zd{ErJ>_;)#m6Te zKA^_d@>D*2o6Z=B(J9AOeB9OBVMcaTVkP9FN)1oX=Gi`QIHO&=1BIT3d6qj^RkoFX z)oMLVAGoEdXu+$b!VqTL24T4|liusW%dCe%IqbCF_l=NmEqJz~NmWQ{#j6EMk3#rj zCd9ji`xaqBRvL6nNbzyx647*c(FmtpUL+blLlq7uoM${z9d5k6OKj z&~3FeasR;>BsokcE{GA5)6X;TWrbvaRum#@#;M&)pVtjvL-80PgtYge2e7(9a}Y=i z#{N8Kc>+o40UcxA7$W9o+3&t%K0XWI!=w&VHK6=z)pd9iw7W$!Im9`V3f<#!8(oqg zZPbH~Sw{D=dhq_{)|(B|M%ZZ*g;wR%sz$d9v(Ao&#=jInsm7mbyR!dgMaVXPHdsjX zPd6Kt3-4hJq|eJ}-#X07%N(y#=s!RG{LjY?BQoiw`B*P!p8cn({p`ZUYwwj{_tJvBCPMFqB_e7DYjU?l|U)K?Z=|JS9=GP8B5b$r&9UEf^R3`Yox|g>8A~;sO&3r)yb|g($thB=jMQaqu%E7dH*nV zD--E*RGQ!U-84+nuf_An8KmF>Bq4jg3bVP2su73jrbh4CCEFlfvg}Khf^DTfK7883 zUv`ytRkMe@U5@kZV)*wG*w^ClON=&7CpYFyx#+e3_^%trj{RxA3_ekJfbLb68(lSO zcHE*@_!Oo9(Zkorsna!oE|!n^OWt~%r(EFLDp>UMv=J{aVQk?&-EE~0!*>k%bo9j= zZkXY5^Y}|EBd#bXO8f8UKHv;S6-yt)TeQEEAATdAIQV?q%Rfv>re5f^;eMTdGnb7a z=cZb#)kTDH7N)JaB`fDJ8w(#ou`YZa7<`hz7dZ4ei$vzo2pP41k%P2FgqH?HIv}4{ zD6hkGeSGc|j`c*Y9O$J|V~6pysViR*4P1`BR%*?_t6k4-GwmYEbx5z(uOR)5Vg2{1 zM#48D;TO~BxfK1|U!9HK^Tm|;{$?04|1XvBbt-Wm%r|EE96^qD`5+6q>aJ~FqM<5T z@ny>JKgHbrCLKM0yv;x7HF>0e=|V})tMGhbv{{*SV+aQpb!DnmYx;I8IBA{hKyBxe zzs(oI)B{xSO|zQ>0fw#shInp0-wblRh1H&Oy@^6$aX-7}V{DFaaVb;AT3~t`ggwa? z7&CmiZIVf|%t=mjI#*20HXqx6&W*?;>7!RCA5}Z!$ zylIOu=r0}Nakx7<+_pTY^UK{z%quGde(oMKtjFE8E>Q>b8iTz~w#-~12O=zo*(TNAUZ9V88ZGd4ax3UhRFWnpa!c-rk;NpBp- z8NKIMOm0I2YRBi%qG8z}#p&s; z<=dB6z4>v&HZ%Tb8^bq-xyjLgoBeBCdx@W~?_7Lm_u_dH_z|W^K7H1Sqfn{&kx@`n`brS zN8Wt<{4X1;CgyMtrViin`+3&xMlk$sw7>7#e-PY1p!vkO<>dj*@f{vw`*<@iH#5#B zO@lPU3p`=41c`t7u8fR*NzTf>au@EA2h4JR7e3Q0CM^BW&xuyjS*CeSD_okHZ73Bl z)pRmLI$g7Qa?Hv^DURtirWYpu(C67t|Jhfr9%4aAx?K~Y8Lb?25#m``KV22L!lj+h zx-nc)D$dJzJu#d0WaRAP%-eUb{(E;dSRhlAQrx(_K&2ByrIVK}9O79-g_}xL{0A_9 zG(JU7(38YMQCPj8KiBwVtTo^Oo19YkK62nTuH+Cz4HG^J^+%*^WqbQ#`{L?qwA`gw z`_4Ae!jXEYWw-sHPZ$khTNAErp(O$cDP=Q!X|4S z3_wp_C2p*(DC3#6`y}9Az{fXwDV)iFbmfIJZx!OFhpR=S26A#zVAD~+yc)zo=Ns;x z1!E+4@!v!E4ijSz9v7t~Eq6%OmYv&oZ};EK-;QMxLTQwd@y z-WSW2oTQhX`;X3)YlbzoQ#2e>ZW4O7mFqqvt`LuCC(wW(*m*bY8Z8u0u(X@oI13jD zN1PGf%VM61YmJ+(8L4TAQMTWudsRK4nHW=AZR$H;U>QRSg+J5g->gmUjB{c=gs?KkcVluX3om4Z{*b0-L?%q@5WV&RdUP*06qdR)4``^PPbpot>r$bx9cJ~|I~Ayf>}R%D=kmQ*)~3C?T68F9sbN#1rrH4m8; zO~7}b`26eB)p3$Up_0*0F|_C`bQ4=Vgu}f(Zy5ey5p7en_1NeLD;TLZO}@`sdfJU(_>9Tm|WqIiAXU7tTQ)?&-9upvfL*m zLn*1svCrn4dZBCc+1o2WOXbf&Goyt)5|Dt+5$h>PaePYX-*MdIRgC z{f!=U(aij{vwNI>_Q<9|khIfR*OZQF5k*Y;VzwJr>-#08M)paoPemqcK8GV3Ra{n~ zD_QmZn$H15z{-#2biTa0R`WKl@lR{vwO~exh3DzSP;xOL;Nt7G$|LR9Cs*x`$O12l zu@sj|a&$fD=};eGvjNrcT@ifQnhVf&52@Mz1jo`Se5>32OOTT62_VG@wbFB2Gc z7V(~P|H_L@l!5eb0XopeWV~}l&z^o~o2C9l=lPE-1~v3zG{&NytW?a_bmOVST8!-; zQX}*cn(L5%WS=U*k1fdo9JmBhV%Mn0INDGR5UeiOAVEF=3DV65x-aP}#b0l~zdh`4 z-~BWa8VMnmrphnS6)>7dYeCZQoyvWQ&E;H$GVy!F?$^?>kKzmyvcySaN9nax=9uOjWL_%tjO)b!<}$4opggxaUXum)6=Rb< z?l~*l9WU|#N(ae%w3Tst!P53evj8j&!Y0_Ur2+74IM)j_F}YX8nxOd_HHL{Z#!3U-DP?GOfHCJwi6Pxc5~W42Dmrq0m@)Q>bCKW{umdM7Dt=64Z?XJSqf1I?Tk zbJH0)18G-XaGp^LYED79@BV>fOl0T5VOvA$aw zSOT-3c)ivvV3d!6au=k}XDpH02F}R~m$=q{gkY0E(RO4Cy8{D4z$J2z6gHC++^9jO z7$A&IXgjXO2{t=kw%|%imRyO{!Ep%bI>!+tE>1G99vz5Jo>(41pi_qIB{4`~x}K1y z9zN^8%Uv4(J)u)uSGQWXPOmT4c#6685by&~#Y|7NClM z%A7=?_AJv7IRBJj&HcIy z1yvNq{D60Q6_|_uO`A$lq-TybE%Aqcc1#$Ga7`-ReK&*!Pm1tkWpBCbGbABwBmS!D z@ed^7h|7T{^7O_eQa5H()$8JY4cS5ZxvGT{kteHnef~Kj;c$wd0kQI{GA*LOwnKBB^nj9LaWn9_`P< z?kQ)Q^8~9NgZEXK$V{$tP{2z*KUyj4i?)qC z)YKkRw0x3BN0r6R zBU1PgGcefA6@(;jptSc&J>1Wi8w@->I|vKn963|>KITwLvQ-D)@@K2m=6Xgema{4j z5@l_2H)wOTL^hM$c}!WDH&cFZ#F$p`5Z`FtAhWz@ zisZYT)UqM9U&7oR#f#s6Bt={kGjuu~M|owjtK_PaJHsNq*rk&4SnN>5#X&41R(9Ix z%*u{eb~@rM!e*S)N}0^|_v4n9K&GLXy2SIKvn-S~rz8RmXEvd)D$m2ow~Oa)VhR*F zwRmo4F5YwTvD~tRj5h*PK{#5lg-i)+sQIoQR!s&xBpt<1oKx_BscSAR2>?H|PFCjq zeHdsqkyF}+zLS9rISt$b4iQr=W+f zEih@xsz#oxa8fiHQuKxG>nB8`DQ$JPm1+b3Zjek}p07%%uw6M&SnHY(80MBTJ{T5b zmSp|DugX{XG>bfp(+09ov#N?vWMt~>RonG7+d*S*~P&Gc(O%6n2cjLK!8J7YPy}5w*T5YOt5|J$!0qe=F5SEu##Z$L?>CvUT*&O_2SaomyytJ zh0qAJ(aW@dGBNiq$f&*#NDT6sf~D1_x6}-VG_Vkw3df5P$E8N@Uv(h+emeU87FK@I zXD7Ib$mo;iG{l*8;$jYt)rl7fjg+OQ8bFsK5K&OuN2o?-5x)t^=K0WHJjn@!X*5B$ zthYgo$rVunDZ~jEaiSk__dz&83#H_)1Td*aLbM`(gj?$ChHni|Adu)7ba*1EU8qSV zlXxL7Ky0WX{l=LK;zM)cBo9D41OzOZolO7R2hqPD&7MPlPmA)lg*au;je;*aH1)&< zcMQ9+;w)4U*-`*EhOE0a&|R(tm$fbo9eq* zbXEYrUDV~p#gps@fDQe6lq(C}#9hp%qbTryyNkr2AJLtu(_<~p$(cK@#e897Q`TM! zRuQNaBv|Y?}o(H-=H-Ny89-5>&ZCX0?=r@M$yp9OR5l21Fn3XNloR@L!9RZUn zFs;9Ml4*FI6J9KbuM13q7?t8Y4{x@A7cr+F@$isX&pu}4my|^8s=T=>vz+$!_0`0e z97i6htJVCHJ`G~jTc5Ou{OT{Bey5(|2<-@Bmdg*Oy28>HI_>J}oVe6XuLb+U(XxtbDtpKnDG92iC>FUiV zheZkdtV-<3BAzQU*P#`bXkJ3BbK&yAf$x79!}b9FcDa$g0^1{Kq0b%z?Dv;f@75rI zPUw7u?X0ohz62arlp{Z`gAiL|q!`rP8sTp1>+lb1c2uNAZw04ojLWXV^^gDccmD(5 zWvEsPWo~41baG{3Z3<;>WN%`#$vPYke>pHQK0XR_baG{3Z3=kW)muB0B)1LD{VOKi z4W>-^$tMgrLP0w_b1o8L)0_fB4 zCwUU|U!E+Stn`Zy|4yHNqu+j^zwh(kFZJ`UFQ-5MUHTI}YQW&><$5|xo5#6De<-W; z>E-gTCn4ZxA(RxtT!ldTgxB!>`KOnE&=lPaN-oNQ0$L#rHR}1SoS-WOLTE>yZ{f52 zDnI3EdP)>7LK`jXJeHP~VI+~D)of|N#R>(>9_J7p-@@o*7$sm zlSa~LBPbHdbweF-f!sJ5Jq1_3nfSbGWY7|s->PGtQ;S0DZQ|a;>*IvOo@AnwX)=LC zdSciO_H7Cx(tLyQQ9zf*G$SJQbQczl4i%}aT)fuOW}1{uvj%pDQE>*5e>4tN;HV5t zmgcD!Mc`t3w%ouEd5!L7AoRSBg&%Tn?pJ#cHc;Gm}q^1N}D21i&XiwSZ`f^5etfOrO-^ES5zcULMa* zJ$(0gmc~5%kOu&s%;K$bf9S#p=@x^iNuUdQl2jZhrjz4>#?$2d$(_kbNjxa*XP5WR z{+IA!(?|F=;}1col@e6m`ghYc>>|CEI2hYNv|hj%X}*vn87!WD-6 z_&9?WyO3pJgZqk>BEug?=}zCt4rh9fe$k}o%^T?J3WO$foY6{Be_g`46|zJ~w%PLK zQ-%tq_7EUGQ7kyZC1f8_??k(;&(8250+50XOIl`F63^91;9CK+^jk!RI3jQ=5T9np z%vp_@n{~|un|a-98GY~G>N-&Nm0dsgX#7SX*k&{iAX(ffA$Hc>5RQ0blr=#_f+Rzl ztPJ`r86;9P1fVpoe_U0Pq26N~(U`=T?r2?vLf5AL8WpU3BTvBQX%0Kl+~z8~vRP>h z9OvB-@jz-Nt&=O_L1kAZvn;O64zintMusSSO(c}XT8yz~N^*#8OL=6y-?qE84O6T} z$(2w9nTsnjDHAAL2B>ZT!My{I&EOAgKFTS$QN!HZK*-rIe<%sBMtTan!aOme5OKRx z+7v7Sof~m+QO71(sZt_NXUlGi5=f0fFC-fYPdAHCqvG{+REZzMC~MNwAr>$9_jRDH zVtrKZsY!uVgujG=b;nIBj~GXw2`*f(IMyl50mh~3TMFW&UX%wy{cW2t!;V@Y*xACs zgMfXb?iZ8oe{F#6;VsEDQW^eCLu$U~4mhxer8n*7oowb7bN+B!+1A}H9i+fXsS+o= zgAQ7s^$x@$=#cAf>D^~Ll4#B12)*TMJVxEx!3rs=CN~exXW7t4wzo)?Ysbc09FOkW zjqyfSlw(d#ad^7UjJY0~U0W=j=C&Qz))}}r+}`xpf9Cw5#=FIcZ`N?ZWd-#6M!aFe z$8Dou?lXK#wsC#S#zpgxraw8~RMtwf=jsRABmi{Vxak6=zL8_S;O>fErG%MaFGB)cPzUV5=ku|Fbg(N7(z%9<$Nk${FM?-|;?Ykqv)gem;ConS(r| zHQJ+Sl?em;)-fXIoQ{!(ikL1Uz&o8#K9gnNywy$Yrjvm&=?^c>Qrc-TMbEsjv9{zP z)_*%{k#lANX*Hlm_%e^F;kZjrATl?0OV%c>e|GPG*3*P4niA5-oalZ?_<0(=4jK1- z&j&cYtaCJ ze@6ff!AhmG6>3Ljq$UO3YqU~S{8bjmt;ty0MU~GbW%dY4*K!AV{e$UftmnojiiZs; z|MW|IPBahA>h-f4lGHL17-~o^h)T#U-uGP(6!N20pzh~jXL+8tG9eJ5lt)_4}TUJ@OVs8A-8PhAaY1%uSiK($rvwON(64lBS{ zPfyu|c2VV<_j7ZBG`tISoc-q;c80zdb=jT&W1Z02U4z4;Fh{fCgl2(o^ne{7+1 z4V`IP6O(FX&nW~@lZ}xAR)Pplghhwkf%TnJbgcwGMs;P`0)Jf+;K~cqT^CLovkM

` zY?46%=>;NoonM)ehGP=XY_tk_e^kl05_7Lh-)(%uvFMF1_w@(d!p$UP;3CZd)UDGy zQy#_Ip(_4P3;C%E*Y(A1$ahOcwD5ESp-!)2({FdH$T)?*KC9TdaVfkBY5}X(vjaYB z)*eS;iTBKOOTV7YjLqE*#og)VDtbE`i#1H_$kKL~k*FN7Uz)h`7NC`Qf4Ww~!HI`X zFfDkq$TW;l(kTUe`h%p%@dq$4r-K*?IJh#$_a_%ZDPB$p{;OV}Mf zfWJn(-ck>o%HYM-Mu(ioGpoFw+PVvaf<=LewiGZ%aJ}s0{T$ zkh+4dn8PL2MlCshf5%13PT0*JcRFa5S+oYGB8O=1Rf8ZyPzs1a_ht|6ytkx8ls8bt zuP8Z!D7o$fn$Q3!6}?IjvsBF)Z&gn6-Ez!AFgd6iMqS!_Bq1;>w5I20fmRw{0kA7JL0t1EdFA$wN}DgDAhZ}60hzw!e=v5MKcN_y;sSFps_=up zA^ToB zCz7(yC5-D2SyklHvtv>bXxfHa2~nm-Z_TR)=8{<gT!M zeJ4mMO*&ZFe^r8kJ_jo)5gCsV;98#Kr^!4v+ewrM@RMYJ+$jf`G-?t(&m&%3l+nU` z`w=g_eP{I09L;x#rYO-E^CxwwJgW-CQ#YN2u%bh23>cIqyo*K2K^m~q_Wi`hG^!v; zXmb2=eE&u5zyI>cf zj?(*+&S`9cf{{%gXf}dUrwxD8MYsBJ^e1l(Fm$)4(0;~wGT(2XkLq85t08#G17tT0 zJh#{-!6v1HJqMXdGJV1eFD+y^5^Vx;mQah{UOXGGiOJU2#F8F||MO4(16(rCuL@;u zWOHEevj_Glu$gRq(%!K5S!)HPpmpPGPa+J8iqTM+v ze_e)wg|#RZN{_K}EGDP%asyHEi*v{<(agEZq~cV;Y7N&|4?26tXyX$+ZrK*Wgz z0G7yuiUbfC2mm`$KGh7S6ttKyi>8_fq=KX;{5jD<2=E37KnnV;sipxe%`7Znf98a; zAeMkg&`kgxypi1bj20?2jToqCbDk*@&7%k1s~{@2=FCRPP$ zCsrltB~~>lAXY8m9jl(;4OI~Ue~wj7z{V#Mn|gg+D}k)WD3 z5wQubwuLXXur*Xqb3Z!jq(PUkBrNP@`KE0lu7jvAaVW_k3R|R{G z9{4pYCJQ>jseK$gkQe7n2l?KZi6)q9Q(*XS+Qggzb(aZ3P( zJej76m28=q*C$q@@pdh4e|Nj)-Q^B-^0XQ!Zo^xwP26(3yXEcO@zw!R@&s?V)0S%t zU89mq%t{gZ?4%fu27A+QJ^I!JFHw}k?azwAVmf=YD9#tt$>{tsk$!l6Ud$eS8-5*K z4L@Ig%l>$j|IGh*{5bX=O(zF~Me*q1Um_kl#kv@ki&?XL()yh|n|b-~A_!Ay#9A*}wt?f-K9bb=i+fNB%k5 zmfv9waPuUbX#+V1jjCMoZ5$ii>`GZ{n6ol(3f9?BH^^8`1C3*&wPykWr zyss?mHt*}ahnBw9Jz-J!aECEH>S9GKH1HIC8*gwtba5(Nc>^w~zIP~mAAwq!gPIU# zg&qi%ddlfOLZu#J&VDHuXb6F9Zu^R+7C|1cfF!_oBtbbR8fi{6(wubV!gn;a!}z(5 zCFarGmg8%#fBRB~LY0LL1=br19|{w{!)#R8*afqNB97wYcRpUxY@y`wx||IZ#A+No zFj#`VGCc=FZ?IbBdzSVUmtjgHX-Y$Gr47?cd|`agA%fSiS;ivdN0U`OME?kRr6Z|m zB$xM`3SoNR$({w(X&6PZJvw>abCQC_JT@y8lwpx{f1s}ssY4_kL_&1l!(1p8NesmR z5=c!;u|2Y0bOqAboD8T5dV&K+j1$&{k2NZO<$DIA2Q!VEG%*c%6;Nz>URW1wbKi(K zG)_(_WiU)Mkfw|Vx>Ev#OdSc2b^?Gh7G&Ma+X#Trs~ni-4Wl8B#wW7E(Lj9<1(Mk$ zl7PE#e~u*VSm^BI8j=$&;uI1{zLL5WHQMrMg^^sq@aR}JL=I3100ZE}a7TcGBe&Xi-%^f;!UCd>2on%Aj*b=Z zJ5UJ9q&^Tk(1;8(r$h#%MuVY-081!|S6In%Ln>M&jv(5QO2>WgOxTw;+94em$W`2) zDr`@jSyIVh5YXFVOeKP4qAQ3FkZl|y>{}JAgbkKS<%0IDq9hM`2Srl~EGXTFUSTs} ze?Zy{Hk78kcCi$+%BB9qN^hX8!QP==0s*y))(nUui4}AwXn(4pJ4M`cv^gvp2njO_ zC~*QC#e)vX252(F*b0J{XrNAm^q7vOzyo3A)3W8P!4#0TOweOFK3GpJq)yr%I7aWu}Rl^ z_+a&QH2zx0!mS20@e8xMi#WWhnpM?p&g80lT%5(%=caX6b+FR1+Tv9}xBa=hp9~ATR+2{(< zZg}!Qap}Pa*y@~p2#-NR&Zj1CbqCv>pVL^#&NFJI`VJmeuv19&9vLM>w^z@L(Z!e8 zXZP#HhtY8H1?RgBL+w%cy@SIb+rjC8tUEpbz}ekrM~I3?=ev`OagpWha6Gt}X9nYb z9~XZijGj#A#V`5tpU!Llo|J2QIk+s^@uMDxqj5oxBg$+F5G@w;dCxKRZj< zcb>DSlk@2is$fZ51Z+5f5YvzeiCKxW={qDGro#ef%S}==lV$JY(fgB+D`CVo$)eWD zVs1$mlafW1WHGu%7L$-g*2n_y4`d0CXy|_t5j?U7BuguPDOq%#EVct#+Q#on7P~9uSChWXjIBaY6)B1y$+yOvS{T~vPkk)JiQP=?{hrx0o00L zN)=nDitj*`w(&c&MQ*Ugc4y1(@$-NEkH?8LTQV$S-LVzoUWSFMR>LAi$`wb3h2GwY zX&R)2AKI`qLzu8d)O}mg#kXx7yfalcJzMu98F<<|o}vNYd1lKe)}z^cvHxW-L(Y)B z7?gGBwG#gL^=~g;o&2HR(-0wBB~O`Z-_j zY%7*NVd_UpvD6;k=UUgEs%!6R#Gt;>bjZ%9mzM)JF6MJK9}nhV*abZ&E?`{P9@}RJ z>@~wP;FUf`?BDFqQWrH%BzmY})6LYrKY0J_?Co+Z`z@Z!Vm&Yv_X=YS;nBC9%SP3l zeMz>KUb+kK4kA(ffah`>e5s+W$AzLxT-Y*vXH&c3?%O-K`}@0xryp1DB*33<)sGYF z!J*9W!xwjh(=SuLcvJdu8tETIr1?YgYkmQ8&WU5d zbtV7d$Ag#8PjUF|{nq3dsM|XRrjtWQ-hRzZujrsn=@{TmuXV@3<#os4GfsFq+vmx7 zUR%;_0dzbC(1|qw^9nwHtyn8%4 zAB-o{1^YanPG=!u5Nh$hPjEnHgCX@vG+M*Cn~!*S{QB2_)0at3DT2BUA8|cFEbo;d z)}}o{Tzi^;4H^Rr`opiy$Jv26c=--57V3&McL_4upiCXcdahV@$rZN@-_>DM8=-~2 zLum2({n^=zT#rho=jFzwSOeKK{Lw<;7xqj~CQ>#!KBU<7IKmRFvU%>AuL6GISEP1*7zM8_(8l zpp5i4JY3*6JX~n!N(+K1SL#VG)g|fGJbGuYxDBqj?p%5K>SXurt5TM2W=GX}woBjZ zRS2s(B`%g+dDz573wS9_>S-?BCC$}5dgs@x@(r2^EYHIK0!h6L=$Elw0TY*Ve*qdo zWHmKrGdMXpGh;U}GcYn?G-Wn7WHdQ3Gh|_9Wi>NmK0IVKHD)t7IXN?9H!w3WGGR1j zHaBE6IWaS2VP$1CGh&y|e*s5-H$FZJb98cLVQmU{+BMS2OO;^&#o=?bY}Onq8_ZkT z;82!k!@ZShla`rf-pt#*64jzjiwassM2JEoMGzrDU`YrGEm{<~38HOL{Q)h)RhwFB z?|V3I9)6r}h7SOyr}KbHl*$r0N7l+}xkc8=ZL(f&mkn~KY?4X2OQz(1UfCj>_Ne z?3Y<0nf6t)WFgVk?q#+tBD#NhDUii-zFZ_16TJ_@SD>#xni+{^tD?C-(ewS$3n|~H zjJWtIF4vm8EFrFIxjhwsz4I|TFd7|v>2sD6Lz8j2ch$=>;=Z2s@RpZy;?diVPG2k|rGP7r^`+`EZ?CGIt3!Cm)P{?L=|wPe|4_kqw~ z-0R5dnfE^&dca*pCdb`9WUA1;o^1Kx?j>8x-5bcBZ|)44Zg6iTvzOet&=c-W;eQ@- q?~(gtyF4b3%M?UuP0{;Q^_+2@d=Yatp3NSV|3MC~)PeuyiR`MVK delta 19169 zcmZ77LvUaXuqfczwlOg$wvCBx+nP8hwr$(CZQGa_6LXTh|5e?^t-I?*FM82cU0-)~ zQxH^KFjPD{z{WC!Q5d95q$Krn{cd#W*=AjXuki5+%ADr8K;$=fN; z?Z{U6gDcW*L&*Q%yHDNoMePl4WMPe{ts>c~>e~J4Uxu0?1!$-JR|A47anB|0DI%88tG}DEzr&L{Tb_m&$Wgo zck1FVJ6PFpqi`OAY%XrSMl`H$uGFC>vx2W>SP!r!ffK!7v ziEz9~Bw;vOQG4+>r%U$1%4{>lTvML)3__yq^xeh!-Fqj`-b{>tMs$PfbNJ1I$1Eqn zmtJBU5Z%mDAF=-Sc+!YhCJZ(lz%k-aP=>e-g+4--JkLNL+=OMg#$uRTXI1XZrrm{V zV-fgkG5Q&c)(tPRKBPpA)$C%%n~O}hL}e#zB4#PB6Y(YWaD=mmR3QHbaznzVvKeW3 z1RCuA3%BA4)VM#GQzC`T5I1Sg;2aZ)BtAtQSQ;WY#h#`kon-wX8}*zYOd>OX5U#Um zu^`=6X+V{DvUetN!3?iMNMq`*(zx`?mos)TVZ}tNXkI==7G_~j1ZRp~HA3}qGfR{! zSy6vgvVXrwT_7<(w})|QuXTr?R=0;Dg4!-Ug&|M%xyx&&JFK!y&q06h-|dlIZdoWk z5HJ*w<Z0Ho^Lv`}*u^cUG*hoR_s*v2pA=zJbN_$zqrZ0&j9LeIzNtA0t z)dl)Z7VY?rh#*MO7-l+iL`i`QlJq4mfM%IT3%8B0J4O7qIul<1i*nOY!;j9K(t4QN zfU1V+Z1MzAteGeKBPBjs)qQ#pYJz>`Xy9D&uDp8#ZLwBj&gG*p9ih{tc{+9v@ykfY zYABeE&lQYIuoQNCE5;*nah$U)8 z?`v1YSaPvQ!k+s1X*=hjqal5d=6k*+rh?NU_7IlK9*_NF6?HdQt1L!u;y_tDaU=Rf z$27=KhF9!iM#DULtIKsvicL#`phGA(FZoz{V*{*HDNE@#Kwr@Wr4$CE@w-Alsulf^ z?TI`Q`owHsj`0j)9+RT+Ql@qvsAZLtZ7V#G_x1kb+Zkv2+rBzuG$E1P9(S$HxX4(0Sf5UhGfxB+3XTupm^5#j?P&zN_~LZIYOxTMvSU?0yvS3^cF#-Rg2 z!|=eB|EG)FgZvI7U{Z(T{Qw~R`u)lwX%Up1B94+H?QaHQSLuIjyKHF@Bi-vY zR=p5`cjrAoQN&v%tixl79E=jkaqa!X=8j7m^`;5|^OLZ)CJ0-dv6g|sWjo`z<q6BH>Gw(u#kxL{;?;COv9EXetF~{-|9kJj#7Q= zxF(KMIPE5O-g)*r+`JVi#*CY=j9}mQ-z^$dIH5Zbjj`fXm@l z)qNDqP4g{MwJxO&z&EfmXGjBaI#mbJ7=NFz=B#HzRMyo}(5)O#hR~H5c2=G8H6lWw znu6w(3Z`l#Z+IGe0ov*x`+IyT!;_J2{?VQ`Z>lzGj`NeOhQ}L&UHN&g3x`p+d&@fc zZ5KVf=wz7>D_V%e$S;$_ZDEtcZoQa@xhu=*K$$JPBUqdY05N7`HLPM;+9pG|m<6>f zHt}444x1aqHi;Gghqs}(zD$4Yp5()X#y=?hOU&}Yjd}fn{S3Xo678~p2lA# zS0sp&6NN);Dp@~}j&vPuGcpy>MHfYDE3#Nqu?~?;rx8N4zdaYmVtQ6g5X@!zL{r!1 zu9hhFPA?hD0L)E>OXFNPgNM}G)kVoI3_EC3 zhciCl52M?2EDjj4@LKnJr(p&}9DM97s+C1OJ^qm?fEZD57AisNfSp)1PU=bIBqoy^ zACe#$7+WN1wqsScQt7aVlYRB&A;65FNDgGGn?j+v{0AW5;kZ0&%E(3aYVAy9A#p-`}8vm*}7 zr8~`!u5%Z`kaq?Jx8hz*m<9C`Td?#a+cE~?F+I(OH7rvVtOmCgqiO%|sIBmXbj55; z$^*rcJY7n6)+{cFyOI$hTyX3lUwL(!qYPLwa1~kyYm&nGYf77`kX-qSt0WD8T;gwQ#sMxRLp-2239C5`T3Ma>YBp(5 z?c34C2(HnHvtl6I>ljtv$=*N6=p*vU!yj_)Oqu8IGS4KJY>rd=-(9Bl{o<&iumx z1C|s!h;tI})^_$Wi1+-L=3!qBs6jGX^T%uc1w%Aw=Bp$}ObL0{ah<8H8`WHU$4Eab zKtRWISz-lS87akSEk_|1vvQ`G*arC*`w!bA))l2`C*RE|gW{0ItOtX;)_#W5E_VLy z2hG}&!ASp8U1=4hsZd~r`BNUUAliL<#>s0uTdSo@J$BRR=HRqq(ZXZ|;Hs9qCs7Tg zGeURD8dQNjgh+5b+3=9y*s)1i0VyIeYH2j&kOBdpbnK>z4x#Wy-U*y2CiCcCd7_>d zwJMgAlnPoYwBdrb+{kd2f3`h?m7XaU5ldeP>5wzYY8z}5|IwiJw1O)!V|o7)SY7c; zBH~HWbko_wemea$y*g_I{OQ1&X>=3I_ZGFZhCCE4Z*`Af#5%1eWQQ$xzYK~5nY9u2G3T_U#h~Dl!ld}8P|Lwm7h{#drhpOtpp}cNYrq$LLR5^a!AdHns}?G z{ZoB6(&B@#g4pr0T0?~#=qC`hBX%v$t9I7!E2}ODES*D6pO?u2;g)DMIURcRusppD zC8ajbzwlj;{fxq1Qp`{NKLzv%z3hydLt#t+w=q7={n3#T73}Xw0p0g{d=L0R*nej$ zjDb+|hr$nZ(?gjwB`Ve%7hNG%Wu$lmoY>ZjwnynmzFqq!yoh^`{`}kc6a#^egP2Ml zf-~cWdGjqbYkZGDMO2EnU8T0_?)y9quRx3?SEoRUUt+&sTw}%hIP*arT1L0t7gCj; zaQk3SqC%@)q|~!#Fk;e~kT5uzdYoX?rvGw#avFn}|C?~AlsxHxe8c)9Imrz0t)6t2Q+YBT?pyj^zHi7Z1kHt%~cURo1|_ubR3 z!&&1~ig02dqy+9Ps#LHt|Gu(ZB2cG!sQP&c^7&!jKw9Wfu5rE6Rohe=sbQmjQ~M^p z;k30PY25`t0tw`j&BS|l>$D0#LozJz25Ux~j!9OqO<*81J`Eo$6}iaau$l30{8xn@ zFb4l?G{WYJ5qFY2(aV^TglYuZ^`Ftp&Sy;5ym;Ogid#t{4q&U94b><1!Mv}bxXCxy zg}jwd^tR-hic2eJCN$K~U7iHty~MIL8hLooH{8H-fqjos&a-lwdO9V-MPu=`x%(&? z$Db+=Z$JG=zNErW&jXczAz#Soy#D1H)@Q+%Llx+OMIUp@Ysh9zM|ll|mfRj4q~p}8 zYy4pOAu7>m-o3ZAV<9KRN9AQ`kmf}`1)P(qFpqC&?%7gVV{%uiBrywsa?wX$$FjvYFF|0z*LBPJ>v^*B z>Cxyfq=fV{@mppTLz#WsaWaQ=9f>?=fq#^H!?GFi@xr9l`P09Jo8^P6BCKzw$u{V| z58^*URLx09sN4VtE9d{Eu*n$ee-ysOkb-&K*6SH#0mTnT{DZ;k^6*L$QINou)c-sC z_4M@8*N9WvN@ASg5LCQz&%0+?-vG1c!)eSN;rn%Ve%7&#`+M)^oN%;v?6T_dy|>q4 z8tn_N>~4*^;p3!gVqanJZ?8ww2k=L>>Bz@L#>vjzA-0Rxei7}r$skONx6A8uF0+?m zv>@|;UcUYttc0V6@N*))^wZtptNmjrt1*g+QzL!EiXr;h1-^-9Eajr?H}SBv)E`BH$c^3wOxfC^n-4&Fq4B4@x*guWQ2|9E&~~K1j$YKNtq>#|P8r`kh&OBIFVV_Yu17p9qP|u= zYeGA0dzeV8&AAN5k)a61`wlq@lFerKa_A9K%ngtFO*z0h8WMrN!?!)^Ep_vF5?#-7 z7Pl#7Q|knwCZB6*~0#~cAHi`lh^W`gECk}4isyi(<2CrGMgsp zdO$V9Q8)b{N@P*Wq>@rpo$)1U<2qS#`*&i>aRXtKPX4^gk(kl!G>O$B9koQh`HzXW zZik+8H1#})lxzBBu#Xg00XP|hadPt6J zX~eAbo5`Zngh1>p1E@8eF38U1Mvh_NAH9x{DtAomw^qM)D3(Y!k{IVb7?w~+AKNXA zVs^2(2WounYG(X4(e)@4)LK+IesR!ub_*FVadn^@5D-1>1yAe++F)$3hw>69ohYVY z^Q4BjvA|CVPKYLQ5gWqDJlL$x;^o+EE9e&BWHl`eY0C?@0dg*q?_%<4opV+U)tn9v z_GN+XT4YfqU*3!ej){NTv*8a7iR4QTPL{TKYoDy0U-e3IvSl|{lD>EQGG0&mW~c{A z-y%0`llFGyM>04Lm9wgka{42waW_2IsTrhcNs485H1G?PwjnxD-fB@66fr%d-Kr-I zKIy^=EC9i7+k!Q-7jEf zBWp?DoNeyvLdB(v;7N8qkM8nb2tkj{-LOg(R3*U}(iNXt?cY38XqzAuTNDmeUO*ZJ z&1T=K?nwyV)R2TX#Lce~xIc#oiQn5OL8R7etEy?1z|UeFC^)FF{P)|g=+RqV^B`Ll zeTkvHc*fWu?4=Q2XtO)=xhtjwBIZmFtw9+00>lcZ8%mSn%-rOJ7qjC@02#1dX`8v@1iBGo(?zweFG_ zE{y^`z>%0CuBbWJ#9xsw3r@!D>bJcx>1QbqP&FSeE8Jenog)$ZNospunoabd-RYH~ z8-b`qg(Id=a_D4g;O8?}D&FF30rE^a3U+KQ@tIB+@QsO7Dr1zKw6)`Zd8o%o!$zH; zPosYcluVds%@lTCsFu6nbMLb(FPtj!X-C7l0dRjj1~?Xfl~rLk4I?W5=$E@69vUxW zQ~SZje~ds*#yp_1bOzq!S^cQ%Wlusd&7#5HZ$aQECkX9xEfzX%r`rIE0qf_N(9|i1 zLpD5tcCUyjSv+4Je_NFeo-dFxy5mmt@tb88ha-<93!mNsICqHv`sIba;Mk@nf&OpJ@CY7q0B ze=yfD?31X48MvdkQcn29g8%&CxBc!DpZxgCWS=`%SWAYBR?mz1FU#| zQS9$Xk+%79e@%hiz=Ow4+PN9dXegj}S(@Zw^*wzCpxH zb})4WW1GRCctRcY-MYLyE&PjT49v1~$x9y6jx^W`{osq>dfYdPxr_~qruogWJ;eo) z`Oa&8g|k;zy(B|7eL0-;D8u2#4E(eFpj^-zQ_l++bf2Z%CSUc?z3wiZIj7yx~Y z<%sn;StW|{K*mkfiN$JD;|$zf$k*5kKaBJH~EY zP?{bH4UQ#6jx~ZpJ=+*mUjczb-!!2m;?`j|6~4NSjTnYYr?D4u6^{}JzFSxtOyrYM zb+SVG_&}d*jVZ5BOaP@xJm1}d5b%*wvYG72#ISj1e1kIP638!7Zk`^n zAYaEr|ACa#1Tv+-Oezq+Riov_puXqt+OwX`r7!ZHb$Ge&<1^YPJ!R80<3 z410_=f(IQ6l~sBTHy?>^ns|hLv6q}{NN^yJVk7w z3J!{}-O0R!euAQKfn16vSSMqIis31k^6IGuXTnJRJyw8b8J-yU_>1C<^bKEMcmR&~ z>Th!T0EUw*s7i4aE8wXTNabIj$LT$d5Lg zAuUJgw}6KCQuHLEp9@xK+jczV&712nmw6^$TXf1VwHc9#Sr>$)rzW8SSjKqL)w!tV z7<;b;JZx+_EoR=iuuX79n+!z!RhmQvHe4$HVkx)w2?AXMCtz6JN85`kAe)vP*-k3? zC?Psyl|Z*vSRX_XiImUHEP4tHr-p~1q@EUO@Nx6_5|MPYY?~%s_&t0g8NslXhNM0) z0?VPUb*-c!8#Ep+)K$HvIGkKcRwa5@Fn{&+4cI&*Att zYo(>{han1DYwh~h$D!YB`p=O2!X}pFVO(k(-`q6>10d*DDh|59zg8Pu9fJ)vKw+SN zP9+}tm0twedjA3r1vDLU&o0Qv2c=yS?^l%gvEv0xH{0e+K>*@_^rY_Y8codX>>T}} zFk-xt;7=kSX`6ABqAJArCJw#Byr6uXBL#Qfc>otRaI|FWxXHi)I|1HRlV|8u&x~fM zdPYpc4mdLq{Jf7VgGBX3W}x++c-a2OQ!*^bRv@&SBcbM+3{NWS2OSkr;enYtRrlK$a97FT+&Fj%n z3u5M=7(elSx3;r!u_a#wY-U>fEfDSv$|$(?4akwrzGRBqBbm6#^3`q8b=mZk>@#ll>8kD^n%?v?V`XV&{Jh#SP_!x0XQH}* z-dA^;Hp6Q`-}Cx_!*&G+u|mVd+jCAGf%?m}7xXy;YH7s3Ugq~w?arwJw>oOO`7U_dCv?(_2Kx*Rhpo%=+FEY$e^uIy$i~NDx zQ0I@J^GZ@%Tzm>ARvkA-mYlZX`sGmz7v?J?hIG==6pC=vzD=hsPRRLO(u9S~2Uzm2 zQ(9wBx~1ke#r1za8h^H($MWq`$N01m=C~~MeV$o9V%gESe1_tpepfL}q}jn*BH@`U zO5T>}VKkB#-XyMy<5Ye!FVO`b0C2q@OPXGKq}lz>Fx}CfNSXoJIuwZi`nflgVq`MUK=xGQ9H5E zX^-vm$zfs{LPLKuK14PIq!+T_ziA4iM3ubQaIFip7vF|Rvb!jkFIrQrqTgr4!8o|W z{C;X-Yvp3s`}y{2=y_|n;Kmd()8i8)%-m@2zW0>0 zzc~$YB78r@VB$}G{yFcPxwg^uj#yi}dt&+iI5KbCWjWyd-o5%W+-&4LbZv^G@%sVr zDk{$DR^3gB`riEK`>-~XHpiLgJMsqQf@8FN^5*N6>o7f>oNWK_?{5BT`^ z6aSR5sv0HT>q~x{)%;(}9}fq)b_Z+$DO4gRKAu5`}E*rp6~G`DOPat79wiq(xfmpP2S99SrwW_u=7`3ae1>tFrPmmBMPp<1e%kSEW8P4ZCMy5ADE%NWtvBh#(NKRkcmIqE#F^G%03+rpcHhNj$ z%R9YqXgFPTSlmi{icq=5?DTnOYs+E1%_JjOQQJ-(Q$3%WV4 zX)0UN@63lmdP2R3a@Z6ORILt_7+b^nlUB{z zKjcAI+kk8771!#r{DPy@hcw;^?V7KYE-BH1d_HKC;ptAZjjJNxMAGi=uvRbx!JRtP z&nQnyBLw?RC*>vFBHKRMmkTc;UZ>ia-DUUQewI*-%)Qc?KCP^6oxjH(_o?Y{I0HCT znHzc8{paCzfv7xeuNM`7j=!N#k#v<@&QxjAbSD3_JBMbzUafl4fwpV*ON}FpgPId2 zYjyOi{Vztsaf+ERHg+-=BW>izP4LGv{aV;ecOnTrkZy9O#wHFa3r1lPI53-k2-Aj<3jI{AB`6@0}ww%QV`;d}Vh zAg?wyh&)hcl~3GqLF)`khc9@jhu;C`%MjI0wAszpH5fBnHH)%uM%yP{D~s~mj_Ite zG;r8G^AeWi!=*npUR)3?xzFGZf|y|+k4g$7ubElKm06s(GH1M_)cea(!lPa?5puKh z)3{qE^s%DtUmz&ZM}YK)dM5p^mh7qrtJCHmkx^<`CW{P#iS?uL^-B$h(dRk2Z5E3o z(Y!daL21~_#r`@6Kiqe?0825`(y{E&X6~T^w1giJ)nuH57UnY;+^o`+Jw%bxDOBI& zlz)Fxa`ttvBZYA|-^){HTq#MdW?uVcSC7n4o-)fici<`kg9G`_POA5cvrh7=iEWKM zwz^J93h|R}S+u&Ig>bs{R(r|jF6@U0haUN~bX3MYw zrwiy0_H3||7K$hWSthMDsIVlH65@62r|g~(+E(zknkjtPu`9iPjqX6fUlxJJGP)|w z{A+3QCU_|aN=4$ES&P@2Bb-o$DP5Si80wxr6LrbHi&x6>Co<J4y~hT(`MvTpbC{qpO8LIosfa3;+~cN?{MQ+OAvPz94JLy zGc8)JsGNpLfMj)~YxQl4cG(a?3e!V9410DTPe14gyyPbuX^(^H&(&hfCTHi|Ikza0 zQ}Jn_-9(ZuH_ZY#S$B_o<-G>D2OcaO2bJ4eMFM^f;P{Ob#>b2;+HS5LcX| zc4I~0n(0u0=*g6VyR8&o0dmEu7m(s%F%;foukn@P5o{%&^9`TMmAk^LAXdPy^hT|k zauY1Xc_drJNp*zNqbPoP{>U1-p6SYy5Gf)Vo>cdMf`qXVYsMn~9H`h%qZVp_n;iZN zn)-Y4;poQe26CYTI~WRwlWZnwUDZCihyMY9D9J(JBavECkM@>5{5;?FB=5u%$@0jf z`3$)GLU}DY0X<&Y_Y&OW`^$%juG1ds7)#RpxJ&VqwOC+LNVhUDX*8C_vSKp*B%887 zSsAaabOC%iM*s2qZRHNHc#Epc?z#Q6+7{r;39jWAIgV>#IbhV>yk77Cmkp|y_Ok`7 z&T2yS*rGyL)z!R+^P=p|c6ZF6Gf=$F7Q zB=HooIdds{R3NfO^laanm`lTjSfTd^e@o1YY`Q_ux`+mw!zN!EVyycdvsJ4T1t2AN zbL5%yPw+S1NMk6Na2ioX_%$^eyR-o;^@Df*!?+$z3Fhf+i8F1-JTo?ZYiz z(2`n%O$jttvy@WY(&eO9h}@D!PbA9OT@?71ys;+~7QOy`C|ixIDb%dINbLu+PSa@E zR7O>07>7wzsn8p;H?-4O6ij*wa=6?WwV}hV2)_U1 zGiA9eQ1fBTyKIHD<(DdJpPB%oi3Y!&kU*YDV?PqEo=S)6;x)KPglVT%im&-tP$ba{ zTR6U6{NE%Bnc6FpyUxB=QX=aT-7f!prMV98f1$g1nZSW!y3<^`-SviEk$mc|h(P|C zxYWx7Or^aD((6;8HY0m%_;-x&r!7D}%)9$KDmwQJ%Dhl{xOwjKQ!9Xs2EW)`YOkO_Rt}g^Tj+8f!ByrxIATn%`$_cTcp8D?%zV zmn+K#e1}25QTovMr{e1cE9rA%q);WM`NwnBc4$4pOX4-}4~%+s4N=ZFK{z&n^ zri)5XsP}-$^XZhdGTDj&Hoc)0FUG@sc|^olH%pd2J0#Q8H5CztwBjYbDyE3hq@oHyd!OY zN~~0V55>GXpCez6 zbmfAH*IqV2T>449VD`%y!$Ee1K^^5NTdQh_%B)tW&0RCPmpZOacv5fUB9;v8<*^H4 z1e9@Jjtf%?zJ3Uj$uDnF;322bEfI9yzA^F2_%)@=XvomE`LzFPn1r z(R;UR*&x^U1i~2P_z2ZPkZWPlrC6KV!*w~bej|PBPQB#D@6EXq!w<^%88`CjGz+vr znLuC!oiLNayW?$ZElSqGKtkjLQv}?yDD_;`_-2Q`=)~`4m?V$JQ`_!cIC~%-3u{pE z(1ZDqYbCrm#7+L`$#waYu=-!&tUcG}wDXI!%kAMbpA$3m*!^%6QVd2(8G#V?4-}>N zEofD;rtrRHR_xb#1HK8^ezr^!oz*O@=m`K*3CwB~7Tc+i*y&rCOBlXnzZN7tnT%9h zDNLvGa&}KUAycT(gy{iY`uZeWl_diE9&9)iZ_$9@FNpvfw){OvGdRiq#Fgp$9$)y^ zr{xX7JG6E(@$Ybz=WXgw@i5A~_Wju7HThieYo@HKxSlKT)f{t)w}bSO#uloI2OQv| zvE6W1ZcZ_X7UMi#4uJt%o@KcpKjbjK)~hHY;m_Ivrzb4yN5ZhyG(yCFe@M_|;YD$GMsnWOYZ~xne*%xhMQL z=igb~ysAgIxgN$^H0f<^BR6dZXu(p18Y?~`bgd9&xM-;j#NEy-1q1%fq^LZuB7CBT zuX=sp>SI@IITs_JbR|Zw2DGfdfQnrxX|t(BY)VD$hR!zl*kN9d&L4xktpU|OHgr5y z;U|cLn7Ui9oVS&I9*KHf2BH7@mfSKU=1Agc$I^oOj>CzA?TYC4xQum^uW9jkth3W0 zayEt9F^+rw%MVQqTBUwlvpf^X?e2=v_I{ym)|+3iox__jLoH77ffaiSf`Q0bHD1|9c^@g5Tv#-p2f*VJij23xyYx zHe$R_tD`nsZEVg>;?M%83ioIH%iH}35y9Nz`9><|CbFf8H%Ovrua5ikcOsSW_wmHw zgiN2um;3u8-RCpwr*GbuA7dWi%l-NGcQNlw2^>n{=h~9iX7enAh*pUL@G}kIhXoac zEht7gp@mqa>b`$D1^WI43Av&q>n4LkHjC<7FF$B;3v0jy&&LYvy|RAO3Q{ofQ^H{v z#H2#Tme^6$kz+Jx+99@U9`vu&Kn77o|8!D;^i;`rkSeznKo?v7)nUW;>1>yGY5`V^ z6d7Vse+RSw+U5B>vcTXBfc?27&<$!Ma%JL!7G z(y}45IoFt>R|{c=oDK#>z^Z+VfJNVId9)DUrf%Y`9nZSwjEc#Y$%;5-Emof$yb*7( z9kVRLs;X&JlM_&bdmI}F$%pi}233MQ9t#`G;HwzXn$F_u=F83TdcC=$kMVpZWdPyJ z7|}VNzk@r+3M!8P3gEpCRW+$g`<;Z3U_CzmAUUF;A;Wo}-RqwL2gvw>uUpXqrXn@T z`HKsESvN-5=Pkp+8&JhhV53qGAs8DVH`vGOQZlt+3Qx45~|{ zPU>0~bGnm=X8#a;7D~@nan7%$pl0qnNzh5VPR!QO%Q8&>99F$PGNmFF{O~Bb<9Gzq z6y(H8HxNAxbF){%OhRypT8k_h6nDDW5WQw5SO9z~X|^p}W9*zQm@i&a`tlE) zr{aX##Bh{N5%k22h`6X}rv%2mIs2$4It9U*vY1K5Sx7nAKi8Glc3NEpjpNq@4K4J%6HUIoy|jSNs3+m}GFnLWgii{5 zO?lmt`*3F_c}J_`$`&^`6@Mq1d#oU?3hd*6>aSM$^aC+=lq*-Ge9*04rRS!}XP)`& z1BQEm&2R!J)xLP1&N=8uOWj@0a$n5Q8GhSwGt>FqW*QGhRpOj|i^O8-o5+h2l~2C5 zlh`%)n_M_+1l8wZC0$~NXJAqpL%NozJEV~`Ze{~ay|y%_jhbk?1^+09oO&!f+j?fn zt#q2g>bF}{`ivcKZ4E)Y#`+Qq+u$u4mq+(F%GfCBXKL#Df6Xw0o>?}pV!cG1aK zc1-*!zH~0z0nTP%qwNrz$)n3a)BXpm1)*}QDP~9?MH}}I)=5SQ-pTOZ(s_b}L$ZV= zbp$p+_~W(Y$xMmaRRWx5WNcUxSII^l(u8GWW*H!X^BoWE>IxCIHuTm6(Ppgb$a5T! zf4)259z(tV9B__Z9o)_Ho>m9wxX1N+k9?gI5CJ1Up6s&e7@r`Uc}XQ+bg1VRljdv} zJ_&-z5!V5*Zk5pc4DyeY7W+a@LgeTOz@s8lxoHH4zbz)Fg%Y{pYfC968y#M%5|n)~ z&GQ%Hn4TBg4;QnxV!K7_m#>ao$||6+g2Z<^;wCIOk80sE-f8c}$A4*yXZmlcUl{+r zMm?QQV}@?PDzO&A$JSJ(pu#~OnVVj_ELpilnY^Hn;M<*GMUsO;g@KXo-ej3R8~aj^ z|LAZDqm7Ym8^)DL4X>6QX_^#GymS>~m z5}7Py>lJJZ#+UQ&(zKnFR{KD5MKr{S$Hs(lwmA#3 zcoFf1Y+F@c)!hNxBsU;|(5v*dG4Ay055Y=XrARmV$&D7l#fy~Id#b^j)u;_w5VFB+ zH8Iw7ge@jSV)qPV`@p5Thn~T;_N{!{#|fnU#=^L%rB)_brOt*A8DAxC9PP8m=6al7 zhrPUgP@m3^!jQ7)V9LHm`Mw>7A7n7TzUoGaM~PNL#-LS7&ENvI9E_4ikuNo5QV-r7 zs2p;22P^$%`xL-q+7l$eb zV0*20W8ME8`;h=VzP~N7U*Qe{Zr-Sk~ z8JnK?mfS?AngA9}NGW*3>FvBGbf=7my>5Zuvi4n!I8&et=Pabt^ z^UVpZyH~GnEhc=K@obhal@32d0a4F9iKTecP8yp)3Bdd30HjrhQ!ANAx@4;~UqAL8 zbt_OALB78jULE1YUor`Ax%p88dML%9gZy(CxY;WamdKFpYp@9 zGWovtD!@VGp*Ej`sw2KwTY#D%%Jotx%ps;2Q{Hf#CEEIMkVIg-%f#vzGJ%SogeRIm z!3VNZse$>YG}UEPrzXXf^HdLpQe11OQr34Nn5-q*#>WrKbL5Hp{*ZSdT0RjLN$3|I zJW&Iq0~-YVVP%js?^>EBZMWg?0;n07^>Nuq8=wpl3oepIkqLSTL2w)&O}FKOkr}cv z=;zXis5inwit?|ilsn8EbxPVNmp`@!m-5mwvsk7+Rphe9wtOTR{HI38YQ~;CV{X2y zYQs&cgbk?A`K4S;P7)F4cVma*v+mCKd(|(M$NUws+ z6##O`9|WbQ6hG>qt7BMI+TuY>Xt^5nsU8g0Pk3oB0* zu|bKW>=BC9oyFT+bw_NGD=myarDKFCl|S@ zPG#A$u{n6E?FGRjv^&02YD&qWcIVo4+4ct0M;iDVPp%PskW_0j8K z!=s1G!nfk_%;`<=`w>V@?@odujxv@rP<|vaCh1$qm~XcKikd#GayyBc zYo5B_F8^iTQSV79o+=w58??MNeM_(LnjkRccEeH<-^s`78LCaYRBpkoE$V6GYOc5k z>l%1Jn}Lu<)b{Ypv4&+TfhR{R1D3|r+?SHgFMB+AB0)qy>SGpAx?qj-S|wG>a3s(x zpqi-Qwz^C!ApRXA(=H0jF^mD&qT(hC-wG zcnl~kY`U=v1wVbF2ieD3(EU;KH4VcXL?W|~w2eV1SwqEUquI$TrlKV;8JjxDDwxg) zMqHLvV>73!DO7=gK=4~dk(2hpaESd|2$vHKylKd}?16q}q0@XW+A~5-6cCsZkRW8T zFqZC6{eWWk>M6W(vVagw%?0_%2`pw*x`gdc z2hvFCEX`p3z_l?Sf}P2Iw^0vy<({jwmd8&f%H}|_uhX)Ym41cX|H007jH9USP@?X zyEPuGItaln>uO*z_APxAJUKH?Ij>P&8yq68NC8xsrN=VUSv}XpQI@PZGj4ktrZCME zNt>In)QbfY!+sS_G=o{`G_m8Z=~>;jrYqE0oQ@F4+E`v(eW(rS7eV-$kC+NaaqsMf z{c6vC5o_PAuFa~#*RNw#b8&N@y5OkjrRA3lut~V@(!cHigtNr`=B;x$1Fz8TH${bL1Z zy{YXRlW0%FX-ke_+y;T5RN`s4d=fxc!D=t9(xUl-ZhO3Ag2@%bx2;C9W%Pi$A&Ryh6knwj|(V(hTfn;fEG=VA4MJ zPCZd$6it&*P7B?Z!S59J-{b5(t-s4RYCoRgq|JrTW&R7x3BL&XbO<3(O4JQdQ# zk2mmW_?Hum$Huxv*1u#OD&y1 zMe0lrEhpTzD}cQ6X}>h$fuhKyAgc5gu-kI)<7*w17xadKP1vGI_XB!dl8aHA>toF1 zblSr-R}^nD7s|_)Q$?7g0^ph0#-3!({>mUYAXK5eU>F3#W@va>y!5ti-D(3e>xLN^*lZ4%&}E7|)S17B0vCO$F#$?&yy6K%QR_(?Y!%@m z-3x{4N2pD_>Hn6;D2iQ{GydhzO74LtWpJG;!~Bv-L;o+43U2k1fsin>fD$V(c4vYP z$p&aL!npM_q_T;KMF81Qu%yC=ENMB2Sj9Fe9r4)9|oPAtCA+ zk*tQ;j*Ux_6y?%q)*%#|nDLur3q>?q~HIz}oU&)n}MhAyvH^xD}|8NMUgnadA!t8An%+#l|aZ6KOZ((Wwkw052uz zhIf7+*t_Fh;w(C1-$v+V$Vxjg?DYE6B7}lR2XsvjINKY{i}3X5`R>uj_dmTjc{{xv zOme&Pwz#++4`x&W{FmN$0V#jz>FR1+Tv9~Mv6DmX9~ATR+2{(=>{uvcojrG9pnutwMV$kNt+SF+d*vZ(H4Ir{Bz_rp5~b26Pw zx62i`#+A75{6nrFlBl?X!{#QgP?;515Td4Bfrqvu>2#F#w>doWMyl8wsN&o0N(*`^ zQ+$mndEUjWG&kRsD1L(|x;s(M-kd#u`yyfO-gG?N0!_-(hJ=46>OLj~__x^86<6W5&*;%8C=}wj3pT2wd zYJVjP-=a||kJJ*jxO*)?tz^;4r(}`jt$2DjAiN^-unSNtekoOKohrTqRodF`%oe%9 z7TcXIyT{M>KOTQ4(rn4Fh;_$Sh~;%^#|+|J7+^yunYEujo6r7vI(2AD>h?uwqV!nEBnU2v%eo#Buk;Z zt=g=psZD>bsZHoqZF!}(cW1l%FW;lDlcOfR$%oe4POg6D%b9J((kD#)NGX=u!>e8E z+EaDyU5yyjH<}LF`SkK~z{bUV&gSF6{0qCFbK(NVh3&C@cEDaUoB^-&F=GE_f0nwa zX(G`>4V!MJ_Wi;8XJ>DhQ`v8ET^8$sp}1FI2#sejmQL8=QWb^2M9d zkJFsqki`kZ1M=lY3{%q7LqypnM7*hwuFMe|-oL$r_kVD3^z_4rWzf1!(#Z8tQL20G zDENQ29R(;kB@K>@cIZ`Wyl+iA3g#hq6xz^B=pyS;qU;hSwlv?FDI1=@wO!mf>uP?1 zU-JvF=bShOxvu0t{CM#4`6(8^z2BM~19f}Hz;trx$lI^E=@lKcDIEj)BZoD|!120c z@EI#St?lz$qKxCUAJKW~0j@K`tH6t$|ZD;Pe`BF0YVti9`DA*OTYR zd+6$EBk#;@q2oUSZb>+08zKmJRamXUU(C?oy4X5qX3Yr=Z!0bqcsV!0tTvEagJyqg zpt&{Bd@cG)tj)`#v;EUI!BuVGo877x5A8LPZaG&GaIMc(U&rrL>5Wg=w*Y5>n|D|EF@p)35@61Vu?O^TZrb==ahG!0v-(cBt0 zS;I}W8#%~T;r8P7t7lK&1hddISN(t1%p7lG9GttIFlA}4(&A;F6gu5#=0fO5y%n=! z$S$wPi_z8iJ3AlE&c{Uv{VAY~A$}Q8#tXWAd40Z$GDChan@#^J!+y$snl8SuNpTTg zSB^m`-aQ_j55|+}f_)xOr?aqO5OT5JCpaLp!H~)%8m-~n%||>ue*Np|%Vd8~DT2BU zA8~zySl(-cSey0@;@W8fCTI*Sq%#f2*?~BC`3^4@>WVdY2{PKCOdZDhUa{=5SKQKk zSBFt;gckk|p~dU>XJ;>tEAMq{tWoQG#Y+57A`4$e77mfjW@OPsn*2^tMhlP$XH+XNELK5K0G%!VL3TuH#22nF*7wX zW-~N7WHe$jH8f&1V>4qlI5d}Ve*s5-G(J8Gb98cLVQmU{+BMQ!XwGo}#_{Kwd0&jZ z%;vD!hB<6=+GbAMoLouUIqs!E7RfHzNz^~~(MfW;JRy(E ztn86}GDoz0@>PYhn8>{GGFhgG&a{_=oF%8p8FD7k{X2XGdWNFe!f5VVw67!D|2lf= zuJ2PyoUVz>#ZfP1#3e1)v(X!W715i?=m34r9AYpNm)oDc%q52PtdVbC<`JXCak=x# z%Y0%i5tn<-UdoC4{c(Bl&r1dI@Jd`BJ&Qg`MW2>L$8Y(ZdgA$rdjaucz`ZK;4|gT; zHvf$cq5rt6h!0=fYl)AI?rP#|*1bOTOZP(JZ;^Xb==1JHWa5{58=1^hxEGV9@7+7d zvRZcyS(ktR_R!zmOUTyzzjrM38TV2$UGDA}qnK3_a{#PUhaZd&%A^cP)AL rr@KvVk$dDmc|abNhviZ7>IL9GET&pcmy3Y`9ttrwG72RnMNdWwps`#& diff --git a/spec/tinyspec.tex b/spec/tinyspec.tex index 66358aaa83ac0..911ee82c03b97 100644 --- a/spec/tinyspec.tex +++ b/spec/tinyspec.tex @@ -102,9 +102,8 @@ \subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstractio \toprule \textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ \midrule -\op{Function} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in \op{Tuple} body with $a_k$. Gradient-able. \\ -\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Opaque invocation of a compiled kernel or custom function. \\ -\op{Tuple} & $(v_0, v_1, \ldots)$ & --- & Pack values; required as \op{Function} body to return a value. \\ +\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in body with $a_k$. \\ +\op{Tuple} & $(v_0, v_1, \ldots)$ & --- & Pack values; required as a value-producing \op{Call} body. \\ \op{GetTuple} & $(T,)$ & idx & Extract element at idx from a \op{Tuple}. \\ \bottomrule \end{tabular} @@ -272,7 +271,7 @@ \subsection*{Derived Properties} Other binary & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\ \op{CmpLt}, \op{CmpNe} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\ \op{Where} & $\mathrm{src}[1].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min(b,c),\, \max(B,C)]$ \\[3pt] -\op{Function}, \op{Call} & $\mathrm{src}[0].\mathrm{dtype}$ & substitute \op{Param} shapes & $\mathrm{src}[1].\mathrm{device}$ & dtype range \\ +\op{Call} & $\mathrm{src}[0].\mathrm{dtype}$ & substitute \op{Param} shapes & $\mathrm{src}[1].\mathrm{device}$ & dtype range \\ \op{Range} & index & $()$ & \textsc{null} & $[0,\, n{-}1]$ \\ \op{Index} & $\mathrm{src}[0].\mathrm{dtype}$ & remaining dims & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Store} & void & $()$ & $\mathrm{src}[0].\mathrm{device}$ & --- \\ @@ -421,7 +420,7 @@ \subsection*{{\color{multipurple}Multi-Device Collectives} \normalfont\small--- %% ============================================================ \subsection*{{\color{callblue}The \texttt{@function} Decorator} \normalfont\small--- graph capture via tracing} -The \texttt{@function} decorator transforms a Python function on Tensors into a single \op{Function} node. +The \texttt{@function} decorator transforms a Python function on Tensors into a single \op{Call} node. \begin{lstlisting} @function @@ -436,11 +435,11 @@ \subsection*{{\color{callblue}The \texttt{@function} Decorator} \normalfont\smal \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from the result. \item \textbf{Parameterizes}: replaces each input UOp with a \op{Param}$(k)$ placeholder. \item \textbf{Wraps the body} in a \op{Tuple} (even for single returns) and creates\\ - \op{Function}(\op{Tuple}(body), $x$, $y$). + \op{Call}(\op{Tuple}(body), $x$, $y$). \item \textbf{Returns} the result via \op{GetTuple}$(0)$, or one \op{GetTuple} per element for tuple returns. \end{enumerate} -The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers. At schedule time, the \op{Function} is resolved by substituting each \op{Param}$(k)$ back with its corresponding argument $a_k$, or lowered into an opaque \op{Call} if it is to be compiled as a reusable kernel. +The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers. At schedule time, the value-producing \op{Call} is resolved by substituting each \op{Param}$(k)$ back with its corresponding argument $a_k$, or its body is lowered into a \op{Sink} (opaque) if it is to be compiled as a reusable kernel. %% ============================================================ \subsection*{Lowering Pipeline \normalfont\small--- from Tensor graph to machine code} diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 49f23c65680a3..998d9b9efc836 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -228,7 +228,7 @@ def test_enter_calls_rewrite(self): with save_viz() as viz: inner = UOp.const(3) call = UOp(Ops.CALL, src=(UOp(Ops.SINK, src=(inner,)),)) - func = UOp(Ops.FUNCTION, src=(UOp(Ops.TUPLE, src=(call,)),)) + func = UOp(Ops.CALL, src=(UOp(Ops.TUPLE, src=(call,)),)) graph_rewrite(func, TrackedPatternMatcher(pm.patterns), enter_calls=True) details = list(viz.get_details(0, 0)) self.assertTrue(details[-1]["change"], "viz replay should detect change inside CALL") diff --git a/test/unit/test_call.py b/test/unit/test_call.py index ec8041f77d876..d681802746fce 100644 --- a/test/unit/test_call.py +++ b/test/unit/test_call.py @@ -223,8 +223,8 @@ def f(x:Tensor, scale:UOp) -> Tensor: return x * scale a = Tensor.ones(3) x = f(a, UOp.variable("scale_a", 1, 100).bind(2)) y = f(a, UOp.variable("scale_b", 1, 100).bind(3)) - fx = next(u for u in x.uop.toposort() if u.op is Ops.FUNCTION) - fy = next(u for u in y.uop.toposort() if u.op is Ops.FUNCTION) + fx = next(u for u in x.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) + fy = next(u for u in y.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) self.assertEqual(fx.src[0].key, fy.src[0].key) np.testing.assert_equal(x.numpy(), [2, 2, 2]) np.testing.assert_equal(y.numpy(), [3, 3, 3]) @@ -251,9 +251,9 @@ def f(x:Tensor) -> Tensor: return x + Tensor.full(x.shape, -1.0) a = Tensor.empty(4, 8) b = Tensor.empty(4, 8) r0, r1 = f(a), f(b) - # find the FUNCTION nodes - c0 = next(u for u in r0.uop.toposort() if u.op is Ops.FUNCTION) - c1 = next(u for u in r1.uop.toposort() if u.op is Ops.FUNCTION) + # find the value-producing call nodes + c0 = next(u for u in r0.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) + c1 = next(u for u in r1.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) # the function bodies (src[0]) should have identical keys self.assertEqual(c0.src[0].key, c1.src[0].key) diff --git a/test/unit/test_function.py b/test/unit/test_function.py index 865f55ca5a7d7..ec1cd33a15601 100644 --- a/test/unit/test_function.py +++ b/test/unit/test_function.py @@ -425,6 +425,15 @@ def f(u1:Tensor, u2:Tensor): return (u1+1, u2+2) np.testing.assert_allclose(x.grad.numpy(), [1., 1., 1.]) np.testing.assert_allclose(y.grad.numpy(), [1., 1., 1.]) + def test_grad_fxn_more_outputs_than_inputs(self): + def grad_fxn(grad:UOp, call:UOp): return (grad,) + + x = Tensor([2.]).contiguous() + @function(grad_fxn=grad_fxn) + def f(x:Tensor): return (x+1, x+2) + _, y = f(x) + self.assertEqual(y.sum().gradient(x)[0].item(), 1.0) + def test_grad_unused_tuple_output_recursive(self): # only one output is used @function(precompile=True, precompile_backward=True) diff --git a/tinygrad/mixin/gradient.py b/tinygrad/mixin/gradient.py index 9899fbe0865a5..0d154e7451591 100644 --- a/tinygrad/mixin/gradient.py +++ b/tinygrad/mixin/gradient.py @@ -24,7 +24,7 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: fxn, args = k.src[0], k.src[1:] if k.arg.grad_fxn is not None: # put const on a device, also TODO why do we still have NOOP... - def on_dev(g, i): return g.clone(device=args[i].device if k.op is Ops.CALL else k.device) if g.device is None else g + def on_dev(g, i): return g.clone(device=k.device if fxn.op is Ops.TUPLE else args[i].device) if g.device is None else g if ctx.op is Ops.TUPLE: real = [on_dev(g, i) for i,g in enumerate(ctx.src) if g.op is not Ops.NOOP] return (None,) + (k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) @@ -102,18 +102,18 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp grads: dict[UOp, UOp] = {root: root_grad} for t0 in reversed(walk): if t0 not in grads or grads[t0].op is Ops.NOOP: continue - # GETTUPLE: accumulate gradient into a TUPLE UOp on the FUNCTION, process when we hit the FUNCTION + # GETTUPLE: accumulate gradient into a TUPLE UOp on the value-producing call, process when we hit the call if t0.op is Ops.GETTUPLE: - k = t0.src[0] # the FUNCTION - assert k.op is Ops.FUNCTION and k.src[0].op is Ops.TUPLE + k = t0.src[0] # the call + assert k.op is Ops.CALL and k.src[0].op is Ops.TUPLE n_outputs = len(k.src[0].src) prev = grads[k].src if k in grads else tuple(UOp(Ops.NOOP) for _ in range(n_outputs)) grads[k] = UOp.maketuple(*(prev[i] + grads[t0] if i == t0.arg and prev[i].op is not Ops.NOOP else grads[t0] if i == t0.arg else prev[i] for i in range(n_outputs))) continue - # FUNCTION/CALL: pass needed param set so backward only computes required gradients - # (FUNCTION uses implicit TUPLE gradient or grad_fxn; CALL requires an explicit grad_fxn) - if t0.op in {Ops.FUNCTION, Ops.CALL}: + # CALL: pass needed param set so backward only computes required gradients + # (value-producing calls use implicit TUPLE gradient or grad_fxn; opaque calls require an explicit grad_fxn) + if t0.op is Ops.CALL: needed = {i for i, arg in enumerate(t0.src[1:]) if arg in targets or in_target_path.get(arg, False)} lgrads:tuple[UOp|None, ...]|None = call_gradient(grads[t0], t0, needed) else: diff --git a/tinygrad/schedule/indexing.py b/tinygrad/schedule/indexing.py index cd1bf9d435f2a..26af302e40c1f 100644 --- a/tinygrad/schedule/indexing.py +++ b/tinygrad/schedule/indexing.py @@ -23,7 +23,7 @@ def new_range(self, s:sint, axistype:AxisType=AxisType.WEAK) -> UOp: ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.AFTER, Ops.BUFFER, Ops.CONST, Ops.MSELECT, Ops.MSTACK, Ops.PARAM, - Ops.LOAD, Ops.CALL, Ops.FUNCTION} + Ops.LOAD, Ops.CALL} def realize(ctx:IndexingContext, tr:UOp) -> None: ctx.realize_map[tr] = None @@ -204,7 +204,7 @@ def run_rangeify(tsink:UOp, debug:bool=False) -> UOp: ending_ranges: dict[UOp, list[UOp]] = {} for x in reversed(tsink_toposort): # no ranges on kernels, they are internal - if x.op in {Ops.CALL, Ops.FUNCTION, Ops.LINEAR}: continue + if x.op in {Ops.CALL, Ops.LINEAR}: continue # AFTER doesn't have range if x.op is Ops.AFTER: continue diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index 06a366ff19ca1..b8b377c2f9dd0 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -272,7 +272,7 @@ def rewrite_into_function(call:UOp): if call.arg.precompile: return None new_body = graph_rewrite(call.src[0], multi_pm, name="subcall") new_args = tuple(a.src[0] if a.op is Ops.UNSHARD else a for a in call.src[1:]) - # after multi resolution, TUPLE elements may be UNSHARD — strip UNSHARD from body, create per-shard FUNCTION, wrap each GETTUPLE in its own UNSHARD + # after multi resolution, TUPLE elements may be UNSHARD — strip UNSHARD from body, create per-shard call, wrap each GETTUPLE in its own UNSHARD assert new_body.op is Ops.TUPLE if any(s.op is Ops.UNSHARD for s in new_body.src): shard_call = call.replace(src=(UOp.maketuple(*[s.src[0] if s.op is Ops.UNSHARD else s for s in new_body.src]),)+new_args) @@ -300,11 +300,13 @@ def rewrite_into_function(call:UOp): # resolve TUPLE+GETTUPLE (needed in multi) (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), # rewrite into FUNCTION calls explicitly for UNSHARD (value-producing) - (UPat(Ops.FUNCTION, name="call"), rewrite_into_function), - (UPat((Ops.CALL, Ops.FUNCTION, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), - # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — FUNCTION is handled by rewrite_into_function + # rewrite into value-producing calls explicitly for UNSHARD + (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="call"), rewrite_into_function), + (UPat((Ops.CALL, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), + # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — value-producing CALLs are handled by rewrite_into_function (UPat(Ops.CALL, dtype=dtypes.void, name="root", custom_early_reject=set([Ops.UNSHARD])), lambda root: - UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg)), + UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg) + if root.src[0].op is not Ops.TUPLE else None), (UPat((Ops.CAST, Ops.BITCAST, Ops.CONTIGUOUS, Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), passthrough_multi), # STORE of a sharded value into an unsharded dest (e.g. a fragment into a full output tile) diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index d83329de8ebd4..8d9f4bc3989b5 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -129,8 +129,8 @@ def expand_bitcast(bc:UOp) -> UOp|None: return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) earliest_rewrites = mop_cleanup+PatternMatcher([ - # resolve FUNCTION calls (inline the body) - (UPat(Ops.FUNCTION, name="c"), resolve_function), + # resolve value-producing calls (inline the body) + (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="c"), resolve_function), # resolve TUPLE+GETTUPLE (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 8a77549ecedbf..24c39a1ae35aa 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -107,8 +107,7 @@ def _precompiled_output_redirect(s:UOp, t:UOp) -> UOp|None: return None def transform_precompiled_call(c:UOp) -> UOp|None: - if not c.arg.precompile: return None - assert c.src[0].op is Ops.TUPLE, f"expected TUPLE body for precompiled FUNCTION, got {c.src[0].op}" + if c.arg is None or not c.arg.precompile: return None input_buffers = tuple(x.contiguous() if x.op is not Ops.AFTER else x for x in c.src[1:]) # add the outputs to the call @@ -131,20 +130,20 @@ def transform_precompiled_call(c:UOp) -> UOp|None: items.append(t.after(t.store(s.after(*after_deps)))) fxn = UOp.sink(*(x.substitute(subs) for x in items)) - # body switches from TUPLE to SINK, so the node becomes an opaque CALL (not FUNCTION) + # body switches from TUPLE to SINK, so the node becomes an opaque CALL new_call = UOp(Ops.CALL, src=(fxn, *input_buffers, *outs), arg=c.arg) rets = tuple(o.after(new_call) for o in outs) # if the CALL has symbolic shapes, shrink the max-sized output to the actual symbolic shape - # NOTE: must use resolved shapes from the FUNCTION (which substitutes PARAMs with external args), not raw body shapes + # NOTE: must use the resolved shapes of the CALL (which substitutes PARAMs with external args), not raw body shapes rets = tuple(r.shrink_to(rs.shape) for r,rs in zip(rets, resolved)) return UOp.maketuple(*rets) # NOTE: adding rules to here is bad. these all need to run before the schedule cache pm_early_transform_tensor_graph = PatternMatcher([ - # transform precompiled FUNCTIONs into CALLs (body becomes SINK with stores) - (UPat(Ops.FUNCTION, name="c"), transform_precompiled_call), + # transform precompiled value-producing calls into opaque CALLs (body becomes SINK with stores) + (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="c"), transform_precompiled_call), # resolve TUPLE+GETTUPLE (for precompiled calls) (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), diff --git a/tinygrad/uop/__init__.py b/tinygrad/uop/__init__.py index 39b73f3a8025d..abe700810da2f 100644 --- a/tinygrad/uop/__init__.py +++ b/tinygrad/uop/__init__.py @@ -23,8 +23,8 @@ class Ops(FastEnum): # uops that aren't rendered NOOP = auto(); REWRITE_ERROR = auto() - # FUNCTION has a TUPLE body and is gradient-able; CALL is an opaque kernel invocation - PARAM = auto(); FUNCTION = auto(); CALL = auto() + # CALL is a kernel invocation; a CALL with a TUPLE body is value-producing (and gradient-able), the rest are opaque + PARAM = auto(); CALL = auto() # renderer # LINEAR is a list of UOps, SOURCE has a str arg that's human readable, BINARY has bytes arg that's compiled diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index dec5f695dc6ce..5746412eb6a26 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -49,7 +49,7 @@ def __repr__(self): axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.THREAD: 0, AxisType.GLOBAL: 0, AxisType.WARP: 1, AxisType.LOCAL: 2, AxisType.UPCAST: 3, AxisType.GROUP_REDUCE: 2, AxisType.REDUCE: 4, AxisType.UNROLL: 5} -range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.FUNCTION: 1, Ops.LINEAR: 0} +range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.LINEAR: 0} # https://en.wikipedia.org/wiki/Identity_element def identity_element(op:Ops, dt:DType) -> PyConst: return dt.const({Ops.ADD:0, Ops.MUL:1, Ops.MAX:dt.min}[op]) @@ -120,7 +120,7 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: match op: case Ops.STORE | Ops.LINEAR | Ops.SINK | Ops.PROGRAM | Ops.SOURCE | \ Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | Ops.NOOP | \ - Ops.TUPLE | Ops.FUNCTION | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: + Ops.TUPLE | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: # always void return dtypes.void case Ops.CALL: @@ -157,8 +157,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: # WMMA output dtype is the accumulator dtype (src[2]) return src[2].dtype case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a FUNCTION) - in_tuple = src[0].src[0] if src[0].op is Ops.FUNCTION else src[0] + # GETTUPLE extracts from a TUPLE (possibly through a CALL body) + in_tuple = src[0].src[0] if src[0].op is Ops.CALL else src[0] return in_tuple.src[arg].dtype case Ops.GETADDR: return dtypes.uint64 @@ -306,7 +306,7 @@ def toposort(self, gate:Callable|None=None, enter_calls=True) -> dict[UOp, None] if not visited: if gate is None or gate(node): stack.append((node, True)) # push node back on stack to process after its srcs - for s in reversed(node.src if enter_calls or node.op not in {Ops.CALL, Ops.FUNCTION} else node.src[1:]): + for s in reversed(node.src if enter_calls or node.op is not Ops.CALL else node.src[1:]): stack.append((s, False)) # push srcs on the stack else: cache[node] = None # second time i'm seeing this node, add it to returned toposort return cache @@ -335,7 +335,7 @@ def _shape(self) -> tuple[sint, ...]|None: match self.op: # late ops don't have shape case Ops.IF | Ops.BARRIER | Ops.SINK | Ops.REWRITE_ERROR | Ops.ENDIF | Ops.GROUP | \ - Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE | Ops.TUPLE | Ops.FUNCTION: + Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE | Ops.TUPLE: return None # a void CALL has no shape, the return value of a CALL has the shape of its dtype @@ -356,13 +356,13 @@ def _shape(self) -> tuple[sint, ...]|None: return self.src[0]._shape if len(self.src) >= 1 else None case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a FUNCTION) - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.FUNCTION else self.src[0] + # GETTUPLE extracts from a TUPLE (possibly through a CALL body) + in_tuple = self.src[0].src[0] if self.src[0].op is Ops.CALL else self.src[0] assert in_tuple.op is Ops.TUPLE inner_shape = in_tuple.src[self.arg]._shape if inner_shape is None: return None - # if through a FUNCTION, substitute internal PARAMs in the shape with corresponding args - if self.src[0].op is Ops.FUNCTION: + # if through a CALL, substitute internal PARAMs in the shape with corresponding args + if self.src[0].op is Ops.CALL: return tuple(graph_rewrite(s, _pm_resolve_params, self.src[0].src[1:], walk=True) if isinstance(s, UOp) else s for s in inner_shape) return inner_shape @@ -554,8 +554,8 @@ def sink(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument def maketuple(*srcs:UOp): # pylint: disable=no-self-argument return UOp(Ops.TUPLE, src=srcs) def gettuple(self, idx:int) -> UOp: - in_tuple = self.src[0] if self.op is Ops.FUNCTION else self - assert in_tuple.op is Ops.TUPLE, f"gettuple requires FUNCTION or TUPLE source, got {self.op}" + in_tuple = self.src[0] if self.op is Ops.CALL else self + assert in_tuple.op is Ops.TUPLE, f"gettuple requires CALL or TUPLE source, got {self.op}" return UOp(Ops.GETTUPLE, src=(self,), arg=idx) def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] @@ -703,7 +703,7 @@ def axis(self) -> int|None: return self.arg[0] # GETTUPLE: axis comes from the specific TUPLE element, not src[0] if self.op is Ops.GETTUPLE: - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.FUNCTION else self.src[0] + in_tuple = self.src[0].src[0] if self.src[0].op is Ops.CALL else self.src[0] return in_tuple.src[self.arg].axis if in_tuple.op is Ops.TUPLE else None if self.op is Ops.PARAM: return None # NOTE: they all have to share an axis, we always choose [-1]. src axes are right-aligned into the output shape @@ -1197,7 +1197,7 @@ def view_as(self:UOp, shape:tuple[sint, ...], axis:int|None=None) -> UOp: @staticmethod def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, src=src, arg=name) - # opaque bodies stay as Ops.CALL; value-producing bodies become Ops.FUNCTION (wrapped in TUPLE) + # opaque bodies stay as plain CALLs; value-producing bodies get a TUPLE body on the CALL _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.CUSTOM_FUNCTION} def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None, name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: @@ -1207,9 +1207,9 @@ def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None f"ranges {self.ranges} are leaking out of the call in {self.pyrender()}" if self.op in UOp._OPAQUE_CALL_BODIES: return UOp(Ops.CALL, src=(self,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) - # value-producing bodies are always wrapped in TUPLE so FUNCTION dtype is always void + # value-producing bodies are always wrapped in TUPLE so the CALL dtype is always void body = self if self.op is Ops.TUPLE else UOp.maketuple(self) - return UOp(Ops.FUNCTION, src=(body,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) + return UOp(Ops.CALL, src=(body,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) def custom_kernel(*srcs:UOp, fxn:Callable, grad_fxn:Callable|None=None) -> list[UOp]: placeholders = [UOp.placeholder_like(s, slot=i) for i,s in enumerate(srcs)] kernel = fxn(*placeholders).call(*srcs, grad_fxn=grad_fxn) @@ -1681,7 +1681,7 @@ def walk_rewrite(self, root:UOp) -> UOp: continue # no rewrite, process children then come back to rebuild stack.append((n, True)) - if not self.enter_calls and (n.op is Ops.FUNCTION or (n.op is Ops.CALL and n.src[0].op in UOp._OPAQUE_CALL_BODIES)): + if not self.enter_calls and n.op is Ops.CALL and n.src[0].op in UOp._OPAQUE_CALL_BODIES|{Ops.TUPLE}: self.replace[n.src[0]] = n.src[0] for x in reversed(n.src): if x not in self.replace: stack.append((x, False)) @@ -1719,11 +1719,11 @@ def unified_rewrite(self, root:UOp) -> UOp: if n in waitlist: stack.extend(waitlist.pop(n)) continue stack.append((n, 1, new_n)) - # NOTE: CALL/FUNCTION are handled as a special case. + # NOTE: CALLs are handled as a special case. # The function that is called is not included in the graph_rewrite. # If you want to graph_rewrite a call, you can # A CALL of an address is not a body, its srcs are regular dataflow - if not self.enter_calls and (new_n.op is Ops.FUNCTION or (new_n.op is Ops.CALL and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES)): + if not self.enter_calls and new_n.op is Ops.CALL and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES|{Ops.TUPLE}: self.replace[new_n.src[0]] = new_n.src[0] for x in reversed(new_n.src): if x in on_stack: continue diff --git a/tinygrad/uop/render.py b/tinygrad/uop/render.py index c30625e1629fb..5d2bbf14ac87f 100644 --- a/tinygrad/uop/render.py +++ b/tinygrad/uop/render.py @@ -140,7 +140,7 @@ def pyrender(ast:UOp) -> str: cmap = consumer_map_from_toposort(lst) not_rendered = {Ops.CONST} always_rendered = {Ops.PARAM, Ops.LOAD, Ops.SPECIAL, Ops.RANGE, Ops.CONTIGUOUS, Ops.STACK, - Ops.BUFFER, Ops.COPY, Ops.CALL, Ops.FUNCTION, Ops.WHERE, Ops.END} + Ops.BUFFER, Ops.COPY, Ops.CALL, Ops.WHERE, Ops.END} to_render: set[UOp] = {ast} for u in lst: @@ -148,7 +148,7 @@ def pyrender(ast:UOp) -> str: for s in u.src: to_render.add(s) if u.op is Ops.STORE: to_render.add(u.src[1]) if u.op is Ops.REDUCE: to_render.add(u.src[0]) - if u.op is Ops.FUNCTION or (u.op is Ops.CALL and u.src[0].dtype is dtypes.void): raise NotImplementedError("call can't be pyrendered") + if u.op is Ops.CALL and u.src[0].dtype is dtypes.void: raise NotImplementedError("call can't be pyrendered") if u.op in not_rendered: continue # checking the consumers is not enough, you have to make sure it's not used twice by the one consumer if len(cmap[u]) == 1 and len([x for x in list(cmap[u].keys())[0].src if x is u]) == 1 and u.op not in always_rendered: continue diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index 65305dacad6d6..c3de59420be7b 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -152,10 +152,10 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # CALL (UPat(Ops.CALL, dtypes.void, src=(UPat((Ops.SINK, Ops.LINEAR, Ops.PROGRAM, Ops.COPY, Ops.CUSTOM_FUNCTION)),), allow_any_len=True), lambda: True), - # FUNCTION + TUPLE must have void dtype, GETTUPLE can only appear on FUNCTION or TUPLE - (UPat(Ops.FUNCTION, dtypes.void, src=(UPat(Ops.TUPLE),), allow_any_len=True), lambda: True), + # value-producing CALLs and TUPLEs must have void dtype, GETTUPLE can only appear on CALL or TUPLE + (UPat(Ops.CALL, dtypes.void, src=(UPat(Ops.TUPLE),), allow_any_len=True), lambda: True), (UPat(Ops.TUPLE, dtypes.void), lambda: True), - (UPat(Ops.GETTUPLE, src=(UPat(Ops.FUNCTION, src=(UPat(Ops.TUPLE, name="t"),), allow_any_len=True),), name="g"), valid_gettuple), + (UPat(Ops.GETTUPLE, src=(UPat(Ops.CALL, src=(UPat(Ops.TUPLE, name="t"),), allow_any_len=True),), name="g"), valid_gettuple), (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), valid_gettuple), # SPECIAL is index before index lowering. custom_kernel currently has this diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index bf7b40db8865d..50d94c69b3136 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -309,7 +309,7 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: ((UPat.var("x", dtypes.weakint) + UPat.cvar("c")).cast(dtypes.sints, name="cast"), lambda x,c,cast:x.cast(cast.dtype)+cast.const_like(c.val)), # only RANGE/IF/STORE/KERNEL have side effects (UPat(Ops.AFTER, name="x"), lambda x: x.replace(src=(x.src[0],)+ - tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.FUNCTION, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} + tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} else y.src for y in x.src[1:]]))))), # after/end with 1 src is just src[0] (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), diff --git a/tinygrad/viz/js/worker.js b/tinygrad/viz/js/worker.js index 00de46c5ff55b..55e2e2fc58f9c 100644 --- a/tinygrad/viz/js/worker.js +++ b/tinygrad/viz/js/worker.js @@ -55,7 +55,7 @@ const layoutUOp = (g, { graph, change }, opts) => { height += lineHeight; } const op = label.split("\n", 1)[0]; - const callNode = op === "CALL" || op === "FUNCTION", programNode = op === "PROGRAM"; + const callNode = op === "CALL", programNode = op === "PROGRAM"; const collapsePorts = callNode ? [0] : programNode ? [0, 1] : null; if (callNode) callCount++; g.setNode(k, {...rectDims(width, height), label, labelX:0, ref, id:k, color, tag, callNode, collapsePorts, exclude, addrspace, diff --git a/tinygrad/viz/serve.py b/tinygrad/viz/serve.py index 32c973932c6c4..beeff6fc7dcda 100755 --- a/tinygrad/viz/serve.py +++ b/tinygrad/viz/serve.py @@ -51,7 +51,7 @@ def stream_json(self, source:Generator): Ops.WMMA: "#efefc0", Ops.UNSHARD: "#f6ccff", Ops.INS: "#eec4ff", **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", - Ops.CALL: "#00B7C8", Ops.FUNCTION: "#C07788", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", + Ops.CALL: "#00B7C8", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", Ops.LINEAR: "#7DF4FF", Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D", Ops.STAGE: "#AC640D", Ops.REWRITE_ERROR: "#1a1b26", Ops.AFTER: "#8A7866", Ops.END: "#524C46"} @@ -145,7 +145,7 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: label += f"\n({multirange_str(rngs, color=True)})" if u._shape is not None: label += f"\n{shape_to_str(u.shape)}" - if u.op in {Ops.CALL, Ops.FUNCTION}: + if u.op is Ops.CALL: label += f"\n{u.src[0].key.hex()[:8]}\n{u.src[0].op}" if u.op in {Ops.INDEX, Ops.STAGE}: label += f"\n{u.render()}" if sum(len(s.toposort()) for s in u.src[1:]) < 30 else "\nINDEX TOO LARGE" @@ -156,10 +156,10 @@ def uop_to_json(data:VizData, x:UOp) -> dict[int, dict]: label += "\n"+' '.join([f"{range_str(s, color=True)}({s.vmax+1})" for s in trngs]) except Exception: label += "\n" - ref = data.ref_map.get(canonicalize_ast(u.src[0])) if u.op in {Ops.CALL, Ops.FUNCTION} else None + ref = data.ref_map.get(canonicalize_ast(u.src[0])) if u.op is Ops.CALL else None if ref is not None: label += f"\ncodegen@{fmt_colored(data.ctxs[ref]['name'])}" # NOTE: kernel already has metadata in arg - if TRACEMETA >= 2 and u.metadata is not None and u.op not in {Ops.CALL, Ops.FUNCTION}: label += "\n"+str(u.metadata) + if TRACEMETA >= 2 and u.metadata is not None and u.op is not Ops.CALL: label += "\n"+str(u.metadata) # limit SOURCE labels line count if u.op is Ops.SOURCE and len(lines:=label.split("\n")) > 40: label = "\n".join(lines[:30]) + "\n..." From a9830e2b4d625a292c53b3c814bdb5add5978c3f Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 19:43:47 -0400 Subject: [PATCH 270/306] make UOp.dtype recursive_property [PR] (#17827) --- tinygrad/uop/ops.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 5746412eb6a26..09dd9c9271ec7 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -198,8 +198,6 @@ def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None # NOTE: the key must separate nodes of different dtype: a CONST's dtype is the type of its arg, and True == 1 as dict keys if (wret:=UOpMetaClass.ucache.get(key:=(op, src, arg, tag, type(arg)), None)) is not None and (ret:=wret()) is not None: return ret UOpMetaClass.ucache[key] = weakref.ref(created:=super().__call__(op, src, arg, tag)) - # derive at construction: bottom up, so no recursion, and a bad node fails where it is built - created.__dict__["dtype"] = dtype_from_uop(op, src, arg) if metadata is not None: all_metadata[created] = metadata # NOTE: this value is set by pickle when pickling a realized tensor if _buffer is not None: @@ -243,7 +241,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): src:tuple[UOp, ...] = tuple() arg:Any = None tag:Any = None - @functools.cached_property + @recursive_property def dtype(self) -> DType: return dtype_from_uop(self.op, self.src, self.arg) def __del__(self): # NOTE: getattr because this object may be partially constructed (e.g. if __init__ raised, like the BEAM timeout SIGALRM) From 12f34d8eb1c3ecfedd73041e388520a293bc649a Mon Sep 17 00:00:00 2001 From: chenyu Date: Fri, 28 Aug 2026 21:57:02 -0400 Subject: [PATCH 271/306] some tri cleanups [PR] (#17829) --- tinygrad/mixin/creation.py | 3 +-- tinygrad/mixin/op.py | 6 +++--- tinygrad/nn/onnx.py | 5 ++--- tinygrad/uop/ops.py | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tinygrad/mixin/creation.py b/tinygrad/mixin/creation.py index c290bceb23e52..eb6d82481742b 100644 --- a/tinygrad/mixin/creation.py +++ b/tinygrad/mixin/creation.py @@ -78,8 +78,7 @@ def full(cls, shape:'tuple[sint, ...]', fill_value:'ConstType|UOp', dtype:DTypeL from tinygrad.uop.ops import UOp new_shape = argfix(shape) dt = to_dtype(dtype) if dtype is not None else fill_value.dtype if isinstance(fill_value, UOp) else dtypes.from_py(fill_value) - val = cls.const(fill_value, dt) - val = val.reshape((1,)*len(new_shape)).expand(new_shape) + val = cls.const(fill_value, dt).expand(new_shape) if not buffer: return val ret = val.empty_like(dt if dtype is not None else None, device) return cls._wrap_uop(ret._uop.after(ret._uop.store(val._uop))) diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 6991683052256..8ed61f849d862 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -812,7 +812,7 @@ def cummax(self, axis:int=0) -> tuple[Self, Self]: if self.ndim == 0: return self._split_cumalu(axis, Ops.MAX), type(self).zeros(self.shape, dtype=dtypes.int32, buffer=False) values, n = self._split_cumalu(axis, Ops.MAX), int(self.shape[axis]) x, values_t = self.transpose(axis, -1), values.transpose(axis, -1) - match = x.unsqueeze(-1).eq(values_t.unsqueeze(-2)) * type(self).ones(n, n, dtype=dtypes.bool, buffer=False).triu() + match = x.unsqueeze(-1).eq(values_t.unsqueeze(-2)) * self._tri(n, n) idx = (-(match * type(self).arange(n, 0, -1).reshape(n, 1)).max(-2) + n).cast(dtypes.int32) return values, idx.transpose(-1, axis) @@ -859,7 +859,7 @@ def logcumsumexp(self, axis=0) -> Self: last_dim_size = x.shape[-1] x_unsqueezed = x.unsqueeze(-2) x_cummax = x.cummax(-1)[0].detach() - mask = type(self).ones(last_dim_size, last_dim_size, buffer=False, dtype=dtypes.bool).tril() + mask = self._tri(last_dim_size, last_dim_size, 1).logical_not() ret = mask.where(x_unsqueezed - x_cummax.unsqueeze(-1), self.dtype.min).exp().sum(-1).log() + x_cummax return ret.transpose(-1, axis) @@ -956,7 +956,7 @@ def sort(self, dim:int=-1, descending:bool=False) -> tuple[Self, Self]: x = blue_box.cat(flipped_green_box.flip(flip_dims), dim=crossover_dim) x = x.flatten(dim, dim+n_stages-1).shrink_to(self.shape) # compute indices for sorted values - mask = type(self).ones(orig_len, orig_len, dtype=dtypes.bool, buffer=False).tril() + mask = self._tri(orig_len, orig_len, 1).logical_not() mask = mask.reshape((None, None) + (1,)*(self.ndim-dim-1)) def compute_counts(t:Self): return (mask & t.unsqueeze(dim).eq(t.unsqueeze(dim+1))).sum(dim+1) count_orig, count_sorted = compute_counts(self), compute_counts(x) diff --git a/tinygrad/nn/onnx.py b/tinygrad/nn/onnx.py index bc8ca04f8228e..8050142b8f497 100644 --- a/tinygrad/nn/onnx.py +++ b/tinygrad/nn/onnx.py @@ -1037,7 +1037,7 @@ def attention_contrib(x:Tensor, weights:Tensor, bias:Tensor|None=None, mask_inde attn_scores = mask.where(attn_scores, mask_filter_value) if unidirectional: - causal_mask = Tensor.ones((seq_len, seq_len), dtype=dtypes.bool, buffer=False).tril() + causal_mask = Tensor._tri(seq_len, seq_len, 1).logical_not() attn_scores = causal_mask.where(attn_scores, mask_filter_value) output = attn_scores.softmax(-1) @ v @@ -1069,8 +1069,7 @@ def attention_onnx(Q:Tensor, K:Tensor, V:Tensor, attn_mask:Tensor|None=None, pas qk_matmul_return_val = scores if is_causal: - causal_mask = Tensor.ones(Q.shape[-2], K.shape[-2], dtype=dtypes.bool, buffer=False).tril(0) - scores = scores.masked_fill(causal_mask.logical_not(), -float("inf")) + scores = scores.masked_fill(Tensor._tri(Q.shape[-2], K.shape[-2], 1), -float("inf")) if attn_mask is not None: mask_to_add = attn_mask.where(0, -float("inf")) if attn_mask.dtype == dtypes.bool else attn_mask diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 09dd9c9271ec7..90bacf99a2b99 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -615,7 +615,7 @@ def const(b:ConstLike, dtype:DType|None=None): if isinstance(b, tuple): return UOp.stack(*[UOp.const(c, dtype) for c in b]) # .cast folds away at exactly the dtypes a CONST derives (bool/weakint/weakfloat): bare there, the pair everywhere else return UOp(Ops.CONST, arg=dtype.const(b), src=()).cast(dtype) - # weak CONST with width on the CAST. TODO: this is the final const + # a forced CAST for bool: .cast(bool) folds, so UOp.const cannot state the width @staticmethod def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @staticmethod From 02d67a8333d0753cce84865722735d8b950da1a8 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:01:47 +0800 Subject: [PATCH 272/306] parallel beam and dataset cache for digitalocean_mi350x (#17830) --- .../tinybox_8xMI350X/buid_dataset_cache.py | 13 ++++++++ extra/runbook_digitalocean_mi350x.md | 31 ++++++++++--------- 2 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py new file mode 100644 index 0000000000000..53510dd5815fd --- /dev/null +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from examples.mlperf.dataloader import get_llama3_dataset +from tinygrad.helpers import getenv + +BASEDIR = Path(getenv("BASEDIR", "/raid/datasets/c4-8b/")) +SAMPLES = getenv("SAMPLES", 1_200_000 * 32) +EVAL_SAMPLES = getenv("EVAL_SAMPLES", 1024) +SEQLEN = getenv("SEQLEN", 8192) +DATA_SEED = getenv("DATA_SEED", 5760) + +get_llama3_dataset(SAMPLES, SEQLEN, BASEDIR, seed=DATA_SEED, val=False, small=True) +get_llama3_dataset(EVAL_SAMPLES, SEQLEN, BASEDIR, seed=0, val=True, small=True) diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md index 69d59215f0618..9ec209fde642c 100644 --- a/extra/runbook_digitalocean_mi350x.md +++ b/extra/runbook_digitalocean_mi350x.md @@ -76,8 +76,23 @@ rclone config create mlc-training s3 provider=Cloudflare \ endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com mkdir -p /raid/datasets/c4-8b -rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /raid/datasets/c4-8b/ -P +(rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /raid/datasets/c4-8b/ -P && \ + PYTHONPATH=. python3 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/buid_dataset_cache.py) \ + > /root/dataset_cache.log 2>&1 & ``` +Leave this running and proceed to the beam step while the dataset downloads and its cache builds. + +### 3.1 Smoke test (beam search, 2 layers, fake data) +Always run beam first to validate the pipeline: +```bash +tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' +``` + +The beam test runs 10 training steps with 2 layers. Expected results: +- ~0.29s per step after warmup +- ~700K GFLOPS, ~7% MFU (low because only 2 layers) +- ~380 GB VRAM used +- Loss stable at ~12.55 with random init Files downloaded (~85GB total, ~6 minutes): - `c4-train.en_6_text_document.bin` (79 GB) @@ -110,19 +125,7 @@ tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/liba ``` Attach with `tmux attach -t train`. -### 5.1 Smoke test (beam search, 2 layers, real data) -Always run beam first to validate the pipeline: -```bash -tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' -``` - -The beam test runs 10 training steps with 2 layers. Expected results: -- ~0.29s per step after warmup -- ~700K GFLOPS, ~7% MFU (low because only 2 layers) -- ~380 GB VRAM used -- Loss stable at ~12.55 with random init - -### 5.2 Full training run +### 5.1 Full training run ```bash tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' ``` From 9aa3c48200db9d589e0984ca57379387d8a54c93 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:12:41 +0800 Subject: [PATCH 273/306] use am in digitalocean_mi350x writeup (#17832) * mi350x am writeup * min diff * edit * edit --- extra/runbook_digitalocean_mi350x.md | 43 +++++++--------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md index 9ec209fde642c..32df97ba44bfb 100644 --- a/extra/runbook_digitalocean_mi350x.md +++ b/extra/runbook_digitalocean_mi350x.md @@ -49,13 +49,13 @@ ldconfig curl -sL https://raw.githubusercontent.com/geohot/configuration/master/.tmux.conf -o ~/.tmux.conf ``` -### 1.6 Reload amdgpu driver -tinygrad's HCQ backend needs `/dev/kfd` which is created by the amdgpu kernel driver. -If the driver was unloaded, reload it: +### 1.6 Verify GPU PCI access +The AM userspace driver accesses the GPUs directly over PCI. Do not load `amdgpu`. `/dev/kfd` is not required. ```bash -modprobe amdgpu -ls /dev/kfd # should exist +rmmod amdgpu +lspci -nnk -d 1002: ``` +The MI350X devices should not show a `Kernel driver in use: amdgpu`. ## Phase 2: Clone tinygrad ```bash @@ -85,7 +85,7 @@ Leave this running and proceed to the beam step while the dataset downloads and ### 3.1 Smoke test (beam search, 2 layers, fake data) Always run beam first to validate the pipeline: ```bash -tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' +tmux new-session -d -s beam 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh 2>&1 | tee /root/beam.log' ``` The beam test runs 10 training steps with 2 layers. Expected results: @@ -121,13 +121,13 @@ wandb login Run training in tmux so it survives SSH disconnects: ```bash -tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' ``` Attach with `tmux attach -t train`. ### 5.1 Full training run ```bash -tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' +tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/libamd_comgr.so COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so CC=/opt/rocm/core-7.14/lib/llvm/bin/clang DEV=PCI+AMD:HIP ROCM_PATH=/opt/rocm WANDB=1 bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh 2>&1 | tee /root/train.log' ``` ## Environment Variable Reference @@ -137,7 +137,7 @@ tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/liba | `COMGR_PATH` | `/opt/rocm/lib/libamd_comgr.so` | tinygrad's DLL loader needs explicit path to find comgr 3.3 | | `COMGR_3_PATH` | `/opt/rocm/lib/libamd_comgr.so` | comgr 3.x uses a separate `comgr_3` module with its own path var | | `CC` | `/opt/rocm/core-7.14/lib/llvm/bin/clang` | System clang doesn't know gfx950; must use ROCm's bundled clang | -| `DEV` | `AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | +| `DEV` | `PCI+AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | | `ROCM_PATH` | `/opt/rocm` | Script defaults to `/opt/rocm-7.1.1` which doesn't exist | | `WANDB` | `1` | Enable wandb logging (off by default) | @@ -153,7 +153,7 @@ tmux new-session -d -s train 'cd /root/tinygrad && COMGR_PATH=/opt/rocm/lib/liba | ASM GEMM | `extra/gemm/cdna_asm_gemm.py` — gfx950 MFMA assembly, MXFP4 | | Flash attention | `extra/thunder/amd/fa.py` | | Fused kernels | `extra/llama_kernels/` — rmsnorm, silu, quantize, fused_ce | -| GPU driver | `tinygrad/runtime/ops_amd.py` — HCQ, direct KFD ioctl | +| GPU driver | `tinygrad/runtime/ops_amd.py` — HCQ, using the AM userspace PCI interface | | Renderer | `tinygrad/renderer/cstyle.py` — HIPRenderer for gfx950 | | comgr compiler | `tinygrad/runtime/support/compiler_amd.py` — HIPCompiler using comgr 3.3 | @@ -193,28 +193,5 @@ $ lspci -nn | grep AMD ``` CPU flags include `hypervisor`. `dmesg` shows `Hypervisor detected: KVM`. -### Working path: amdgpu driver (KFDIface) -The amdgpu driver loads on boot and binds to all 8 GPUs, creating `/dev/kfd` and 64 renderD nodes (`/dev/dri/renderD128` through `/dev/dri/renderD191`). tinygrad's `KFDIface` enumerates GPUs through `/sys/devices/virtual/kfd/kfd/topology/nodes` and uses `/dev/kfd` for ioctl. No PCI device ID patching is needed — the KFD path does not use `PCIIface` or `AMDev._run_discovery()`. - -This is the working configuration. No code changes to tinygrad are required. - -### PCIIface path (does not work on this VM) -For reference, the `PCIIface` path was also explored but does not work in this KVM guest: - -- `PCIIface` in `ops_amd.py` does not list device ID `0x75b0`. Adding it allows PCI detection but `AMDev._run_discovery()` fails because the VRAM BAR reads all `0xFF`. -- This was observed with the GPU unbound from any driver, after PCI reset, and with VFIO bound. -- VFIO binding (`vfio-pci` with `enable_unsafe_noiommu_mode=1`) succeeded but VRAM BAR still reads all `0xFF`. -- No IOMMU in guest — `dmesg` has no `AMD-Vi` entries, PCI devices have no `iommu_group` symlink. - -### amdgpu driver behavior -On first boot, amdgpu loaded and bound to all 8 GPUs. On one boot it failed to initialize: -``` -[ 799.780369] amdgpu 0000:83:00.0: Failed to alloc msi vectors -[ 799.781476] amdgpu 0000:83:00.0: sw_init of IP block failed -22 -[ 799.782724] amdgpu 0000:83:00.0: amdgpu_device_ip_init failed -[ 799.793885] amdgpu 0000:83:00.0: Fatal error during GPU init -``` -On a subsequent boot, amdgpu initialized successfully (SMU initialized, VRAM ready). After unbinding all 8 GPUs from amdgpu, `rmmod amdgpu` wedged the module (stuck in "Unloading" state in `/proc/modules`), requiring a full VM reboot. - ### No fan control No `fan*` or `pwm*` hwmon entries exist. Only `temp*`, `power*`, `freq*` are exposed. GPU temps read 56-63°C, power ~265W per GPU. From 661440e6723f5cb9c5cde523346a4a6cb28c4dd8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 11:01:33 -0400 Subject: [PATCH 274/306] fix bufferize_to_store for weak input (#17833) --- test/backend/test_ops.py | 3 +++ tinygrad/schedule/rangeify.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 3f62a4f8b9944..4abc87e6ea3a2 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1768,6 +1768,9 @@ def test_normalize(self): helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=3, dim=0), lambda x: x.normalize(p=3, dim=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=0), lambda x: x.normalize(p=0), atol=1e-7, grad_atol=1e-7) helper_test_op([(45,65)], lambda x: torch.nn.functional.normalize(x, p=-1), lambda x: x.normalize(p=-1), atol=1e-7, grad_atol=1e-7) + def test_normalize_int(self): + helper_test_op(None, lambda x: torch.nn.functional.normalize(x.float(), p=2), lambda x: x.normalize(p=2), forward_only=True, + vals=[[[3, 4], [6, 8]]]) def test_logsumexp(self): helper_test_op([(45,65)], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index c2e8b925afc3c..79d8d3784cc9a 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools -from tinygrad.dtype import dtypes, AddrSpace, Invalid +from tinygrad.dtype import dtypes, AddrSpace, Invalid, strong_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic @@ -208,7 +208,7 @@ def visitor(u:UOp) -> frozenset[UOp]: def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): size = prod(x.shape) - if x.dtype in dtypes.weaks: raise RuntimeError(f"cannot create storage for weak dtype {x.dtype}") + dtype = strong_dtype(x.dtype) # a BUFFER is never weak: store at the concrete dtype, the .cast(x.dtype) on the result keeps readers unchanged rngs = sorted(idx.ranges, key=lambda x: x.arg) assert size > 0 and isinstance(size, int), f"no zero sized or symbolic sized buffers {size}" @@ -229,15 +229,15 @@ def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): # NOTE: the local BUFFER needs to be disambiguated here if x.arg.addrspace == AddrSpace.GLOBAL: - buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), x.dtype, size=size, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) - do_store = buf.index(idx).store(x.src[0]).end(*rngs) - return buf.after(do_store) + buf = UOp(Ops.BUFFER, arg=ParamArg(next(ctx), dtype, size=size, device=x.arg.device, addrspace=AddrSpace.GLOBAL)) + do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) + return buf.after(do_store).cast(x.dtype) if allow_locals: # handle locals - buf = UOp.placeholder((size,), x.dtype, next(ctx), AddrSpace.LOCAL) - do_store = buf.index(idx).store(x.src[0]).end(*rngs) - return buf.after(do_store) + buf = UOp.placeholder((size,), dtype, next(ctx), AddrSpace.LOCAL) + do_store = buf.index(idx).store(x.src[0].cast(dtype)).end(*rngs) + return buf.after(do_store).cast(x.dtype) # collapse any BUFFERIZE to single input BUFFERIZE def flatten_bufferize(x:UOp): @@ -262,6 +262,11 @@ def remove_noop_afters(x:UOp) -> UOp|None: pm_add_buffers = pm_mops+pm_flatten_bufferize+PatternMatcher([ (UPat(Ops.STAGE, src=(UPat(), UPat(name="idx")), name="x"), lambda ctx,x,idx: bufferize_to_store(ctx, x, idx, allow_locals=False)), + # INDEX of a buffer through the weak cast added above: index the buffer directly and cast the loaded value instead. + # this must run in the same rewrite that adds the cast, or the expander expands the whole casted buffer into one big VECTORIZE + (UPat(Ops.INDEX, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("buf"),)),), allow_any_len=True, name="u"), + lambda u,buf: u.replace(src=(buf,)+u.src[1:]).cast(u.dtype)), + # move RESHAPEs through MSELECT/MSTACK (UPat((Ops.MSELECT, Ops.MSTACK), src=UPat(Ops.RESHAPE), name="m"), lambda m: m.replace(src=tuple([x.src[0].base for x in m.src])).reshape(m.shape)), From 8c7906d200f04674aa6ed09126ebc85126f3293b Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 11:36:44 -0400 Subject: [PATCH 275/306] weak.py cleanups [PR] (#17834) --- tinygrad/uop/weak.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 8f946f33f00aa..857a891631cf7 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -23,10 +23,9 @@ def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: def commit_srcs_at(u:UOp, dt:DType) -> UOp|None: # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too - dts = derived_dtypes(u, u.src) - ret = u.replace(src=tuple(UOp.const(dt.const(s.val)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is not None else - commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in u.src)) - return None if ret is u else ret + bare = derived_dtypes(u, u.src) is not None + src = tuple(s if s.dtype not in dtypes.weaks else UOp.const(dt.const(s.val)) if bare and s.op is Ops.CONST else commit_weak(s, dt) for s in u.src) + return None if (ret := u.replace(src=src)) is u else ret def commit_weak_srcs(u:UOp) -> UOp|None: if not any(s.dtype in dtypes.weaks for s in u.src) or (dt:=least_upper_dtype(*(s.dtype for s in u.src))) in dtypes.weaks: return None @@ -53,17 +52,15 @@ def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: def lower_weak_node(u:UOp) -> UOp|None: if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const, not a consumer src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) - dts = derived_dtypes(u, src) - src = tuple(commit_weak(s, default_dtype(s)) if s.op is Ops.CONST and s.dtype in dtypes.weaks and dts is None else s - for s in src) + if derived_dtypes(u, src) is None: + src = tuple(commit_weak(s, default_dtype(s)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) + if src == u.src: return None start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification + if u.op not in _lower_weak_ops or any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): return u.replace(src=src) # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait - if u.op in _lower_weak_ops and src != u.src and not any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): - dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary - else dtype_from_uop(u.op, src, u.arg)) - return u.replace(src=src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) - for s in src[start:])).cast(u.dtype) - return None if src == u.src else u.replace(src=src) + dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary else dtype_from_uop(u.op, src, u.arg)) + src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) for s in src[start:]) + return u.replace(src=src).cast(u.dtype) pm_lower_weak = PatternMatcher([ # a gated long index into a small buffer narrows; its out-of-gate value is discarded From 0a1bfc9fadb7815e52c6ceec94215176736030f5 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Sat, 29 Aug 2026 11:48:10 -0400 Subject: [PATCH 276/306] fix logsumexp on fully masked slices (#17822) Co-authored-by: chenyu --- test/backend/test_ops.py | 1 + tinygrad/mixin/op.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 4abc87e6ea3a2..1109dbec4b6e0 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1783,6 +1783,7 @@ def test_logsumexp(self): helper_test_op([(45)], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), atol=1e-7, grad_atol=1e-7) helper_test_op([()], lambda x: torch.logsumexp(x, dim=-1), lambda x: x.logsumexp(-1), atol=1e-7, grad_atol=1e-7) + helper_test_op(None, lambda x: torch.logsumexp(x, dim=0), lambda x: x.logsumexp(0), vals=[[-math.inf, -math.inf]], forward_only=True) @slow_test def test_logcumsumexp(self): diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 8ed61f849d862..74722fa814e8f 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -654,7 +654,7 @@ def logsumexp(self, axis=None, keepdim=False) -> Self: print(t.logsumexp(axis=1).numpy()) ``` """ - m = self.max(axis=axis, keepdim=True).detach() + m = (mx:=self.max(axis=axis, keepdim=True).detach()).isfinite().where(mx, 0) return (self - m).exp().sum(axis=axis, keepdim=keepdim).log() + (m if keepdim else m.squeeze(axis)) def _softmax(self, axis, dtype:DTypeLike|None=None) -> tuple[Self, Self, Self]: From 2f5572ab204f88cd12c8eb3012fc293f47a33296 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 12:01:12 -0400 Subject: [PATCH 277/306] UPat(RandMixin) [PR] (#17835) --- tinygrad/uop/ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 90bacf99a2b99..2df84c8a7ed6b 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -231,7 +231,6 @@ def __get__(self, x:UOp|None, owner=None): return x.__dict__[self.nm] # we import this late so we can use resolve/smax in mixins -from tinygrad.mixin.op import OpMixin from tinygrad.mixin.rand import RandMixin # NOTE: this should be frozen, but frozen is slower @@ -1334,7 +1333,7 @@ def get_location() -> tuple[str, int]: frm = frm.f_back return frm.f_code.co_filename, frm.f_lineno -class UPat(OpMixin): +class UPat(RandMixin): __slots__ = ("op", "match_dtype", "match_tag", "arg", "name", "src", "is_any") def __init__(self, op:Ops|tuple[Ops, ...]|set[Ops]|None=None, dtype:DType|tuple[DType, ...]|set[DType]|None=None, src:tuple[UPat, ...]|list[UPat]|UPat|None=None, arg:Any=None, From d7913e206bce728598b1d8e7578e104608a50ec2 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 13:53:49 -0400 Subject: [PATCH 278/306] fix weak const that overflows int32 [pr] (#17837) --- test/backend/test_tensor_variable.py | 13 +++++++++++-- tinygrad/tensor.py | 8 ++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/backend/test_tensor_variable.py b/test/backend/test_tensor_variable.py index 8544d04e2bbaa..efab3b0aab940 100644 --- a/test/backend/test_tensor_variable.py +++ b/test/backend/test_tensor_variable.py @@ -1,7 +1,7 @@ import unittest import numpy as np from tinygrad import Device, Tensor, Variable, TinyJit, dtypes -from tinygrad.helpers import CHECK_OOB +from tinygrad.helpers import CHECK_OOB, Context class TestTensorVariable(unittest.TestCase): def test_add_tvar(self): @@ -35,7 +35,16 @@ def test_variable_defers_like_a_literal(self): vv = Variable("a", 1, 10).bind(2) self.assertEqual(Tensor(vv).dtype, dtypes.weakint) self.assertEqual((Tensor(vv) + Tensor([1], dtype=dtypes.int8)).dtype, dtypes.int8) # takes the concrete side, no widening - self.assertEqual(Tensor(vv).item(), 2) # a read commits at default_int + self.assertEqual(Tensor(vv).item(), 2) # a read commits by bounds, like a kernel + + def test_weak_read_widens_by_bounds(self): + self.assertEqual(Tensor(2**40).item(), 2**40) + self.assertEqual(Tensor(Variable("b", 0, 2**40).bind(2**35+3)).item(), 2**35+3) + + @unittest.expectedFailure + def test_long_variable_emulated(self): + # the long decomp splits the variable into two int PARAMs that share its name, so both bind the full value and truncate + with Context(EMULATED_DTYPES="long"): self.assertEqual(Tensor(Variable("c", 0, 2**40).bind(2**35+3)).item(), 2**35+3) def test_variable_tensor_dtype_arg(self): vv = Variable("a", 1, 10).bind(2) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 24c39a1ae35aa..eddea628e5646 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -4,11 +4,11 @@ from dataclasses import dataclass, field from typing import Any, Callable, cast, get_args, ParamSpec, TypeGuard, TypeVar, Generic, TYPE_CHECKING if TYPE_CHECKING: import numpy -from tinygrad.dtype import DType, DTypeLike, dtypes, ConstType, least_upper_dtype, to_dtype, strong_dtype, \ - _from_np_dtype, _to_np_dtype, PyConst, AddrSpace +from tinygrad.dtype import DType, DTypeLike, dtypes, ConstType, least_upper_dtype, to_dtype, _from_np_dtype, _to_np_dtype, PyConst, AddrSpace from tinygrad.helpers import all_int, getenv, fetch, Metadata, TRACEMETA, TracingKey from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ, pluralize from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, UPat, PatternMatcher, GroupOp, ParamArg, graph_rewrite, rewrite_group +from tinygrad.uop.weak import default_dtype from tinygrad.mixin.rand import RandMixin from tinygrad.schedule import create_linear_with_vars from tinygrad.device import Buffer, canonicalize_device @@ -480,7 +480,7 @@ def data(self) -> memoryview: print(np.frombuffer(t.data(), dtype=np.int32)) ``` """ - if self.dtype in dtypes.weaks: return self.cast(strong_dtype(self.dtype)).data() + if self.dtype in dtypes.weaks: return self.cast(default_dtype(self.uop)).data() if 0 in self.shape: return memoryview(bytearray(0)).cast(self.dtype.fmt) # type: ignore[arg-type,return-value] assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" buf = self._buffer() @@ -521,7 +521,7 @@ def numpy(self) -> 'numpy.ndarray': print(repr(t.numpy())) ``` """ - if self.dtype in dtypes.weaks: return self.cast(strong_dtype(self.dtype)).numpy() + if self.dtype in dtypes.weaks: return self.cast(default_dtype(self.uop)).numpy() assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" import numpy as np if self.dtype in { dtypes.bfloat16, *dtypes.fp8s }: return self.float().numpy() From 0bd725376b00f686f3da6a20859649e7795ad938 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 17:38:45 -0400 Subject: [PATCH 279/306] fix weak const overflow with movements [pr] (#17840) * fix weak const overflow with movements [pr] * fix --- test/unit/test_dtype_weak.py | 15 +++++++++++++++ tinygrad/mixin/elementwise.py | 3 ++- tinygrad/uop/ops.py | 4 ++-- tinygrad/uop/weak.py | 4 +++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 77bedaf2b9b96..c6ddb2e8fcbb7 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -226,6 +226,21 @@ def test_computed_float_index_lowers(self): self.assertNotIn(out.uop.buffer.dtype, dtypes.weaks) +class TestWeakBounds(unittest.TestCase): + def test_bounds_survive_movement(self): + moved = Tensor(5).reshape(1).expand(2).pad((1, 1)).detach().contiguous_backward() + self.assertEqual((moved.uop.vmin, moved.uop.vmax), (0, 5)) + self.assertEqual(moved.numpy().dtype, Tensor(5).numpy().dtype) # a moved weak int reads at the same dtype as the bare one + + def test_wide_src_keeps_its_width(self): + # the node's result fits int32, its variable does not: the shift runs at long, only the result narrows + v = UOp.variable("v", 0, 2**40).bind(2**35+7) + for t in (Tensor(v) // 2**31, (Tensor(v) - 1) // 2**31, Tensor(v).reshape(1) // 2**31): self.assertEqual(t.item(), 16) + + def test_padded_weak_const_keeps_its_zeros(self): + self.assertEqual(Tensor(1).expand(1).cat(Tensor(2).expand(2), Tensor(3).expand(3)).tolist(), [1, 2, 2, 3, 3, 3]) + self.assertEqual((Tensor(5).reshape(1).pad((1, 1)) == 5).tolist(), [False, True, False]) + class TestWeakStorageBoundary(unittest.TestCase): # weak has no storage: a weak assignment source casts when it defers to the destination, everything else raises def test_weak_source(self): diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 724b022d6ed71..a640356be2a64 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -25,7 +25,8 @@ def _broadcasted(self, y: 'Self|ConstType|UOp', reverse: bool = False) -> tuple[ # keep weak CONST weak, might lift weakint -> weakfloat def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const - if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST: return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) + if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST and t._uop.vmin == t._uop.vmax: + return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) return t.cast(out_dtype) return promote(x), promote(y) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 2df84c8a7ed6b..00e86f3cf90ec 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1095,10 +1095,10 @@ def _min_max(self) -> tuple[PyConst, PyConst]: # NOTE: returned UOp is assumed to be CONST if self.op in (Ops.PARAM, Ops.BUFFER) and isinstance(self.arg, ParamArg) and self.arg.vmin_vmax is not None: return self.arg.vmin_vmax if self.op in (Ops.RANGE, Ops.SPECIAL) and self.dtype is not dtypes.void: return 0, (self.src[0]-1).vmax - if self.op is Ops.AFTER: return self.src[0]._min_max if self.op is Ops.STACK: return min(x.vmin for x in self.src), max(x.vmax for x in self.src) if self.op is Ops.CONST and self.val is not Invalid: return self.val, self.val - if self.op is Ops.INDEX: return self.src[0]._min_max + if self.op is Ops.PAD: return min(self.src[0].vmin, 0), max(self.src[0].vmax, 0) # PAD adds zeros + if self.op in GroupOp.Movement|{Ops.INDEX, Ops.AFTER, Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD}: return self.src[0]._min_max if self.op is Ops.CAST: # rounding is monotone (truncation toward zero into an int, to-nearest onto the value grid into a float) smin, smax = self.src[0]._min_max diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 857a891631cf7..374b1277ea759 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -35,7 +35,9 @@ def commit_weak_srcs(u:UOp) -> UOp|None: def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: # only within the kind: an int cast of a weakfloat node is a value conversion, not a statement about the node's width if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None - return None if (ret:=commit_srcs_at(u, least_upper_dtype(c.dtype, default_dtype(u)))) is None else ret.cast(c.dtype) + # every weak src commits at the one width: the node's own bounds and each src's, none of them narrowed + dt = least_upper_dtype(c.dtype, default_dtype(u), *(default_dtype(s) for s in u.src if s.dtype in dtypes.weaks)) + return None if (ret:=commit_srcs_at(u, dt)) is None else ret.cast(c.dtype) # rides every round that can mint a weak const, and must reach fixpoint before pm_lower_weak below defaults one pm_commit_weak = PatternMatcher([ From 0b58ff679cecf5c15a41dbde291cfaf410002881 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 19:18:11 -0400 Subject: [PATCH 280/306] cannot decomp long Variable (#17842) raise instead of silent error. also fix negative WEBGPU variable --- test/backend/test_tensor_variable.py | 8 ++++---- tinygrad/codegen/decomp/dtype.py | 10 +++++++--- tinygrad/runtime/ops_webgpu.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/test/backend/test_tensor_variable.py b/test/backend/test_tensor_variable.py index efab3b0aab940..834d8c998e8f0 100644 --- a/test/backend/test_tensor_variable.py +++ b/test/backend/test_tensor_variable.py @@ -41,10 +41,8 @@ def test_weak_read_widens_by_bounds(self): self.assertEqual(Tensor(2**40).item(), 2**40) self.assertEqual(Tensor(Variable("b", 0, 2**40).bind(2**35+3)).item(), 2**35+3) - @unittest.expectedFailure - def test_long_variable_emulated(self): - # the long decomp splits the variable into two int PARAMs that share its name, so both bind the full value and truncate - with Context(EMULATED_DTYPES="long"): self.assertEqual(Tensor(Variable("c", 0, 2**40).bind(2**35+3)).item(), 2**35+3) + def test_long_variable_emulated_raises(self): + with Context(EMULATED_DTYPES="long"), self.assertRaises(RuntimeError): Tensor(Variable("c", 0, 2**40).bind(2**35+3)).item() def test_variable_tensor_dtype_arg(self): vv = Variable("a", 1, 10).bind(2) @@ -59,6 +57,8 @@ def test_unbound_variable_tensor(self): # bound variables in an expression are fine self.assertEqual(Tensor(Variable("u", 1, 10).bind(2) + 1).item(), 3) + def test_negative_variable_on_device(self): self.assertEqual(Tensor(Variable("n", -10, 10).bind(-3)).clone().item(), -3) + def test_shrink_beyond_buffer_variable(self): # TODO: shrink by a variable whose vmax exceeds the dim should fail at build, today only CHECK_OOB=1 rejects it t = Tensor.ones(3).contiguous()[:Variable("a", 1, 10).bind(5)] diff --git a/tinygrad/codegen/decomp/dtype.py b/tinygrad/codegen/decomp/dtype.py index cde69c6f75127..f02e781d06bd4 100644 --- a/tinygrad/codegen/decomp/dtype.py +++ b/tinygrad/codegen/decomp/dtype.py @@ -1,5 +1,5 @@ from dataclasses import replace -from tinygrad.dtype import dtypes, DType, truncate +from tinygrad.dtype import dtypes, DType, AddrSpace, truncate from tinygrad.helpers import flatten, DEBUG, EMULATED_DTYPES from tinygrad.uop import GroupOp from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, graph_rewrite @@ -80,6 +80,11 @@ def l2i(op: Ops, dt: DType, *uops:UOp): case Ops.MAX: return l2i(Ops.WHERE, dt, l2i(Ops.CMPLT, dt, *uops), b0, b1, a0, a1) case _: raise NotImplementedError(f"long decomposition of {op} unsupported") +def l2i_define(x:UOp) -> UOp: + # cannot decomp a Variable + if x.addrspace == AddrSpace.ALU: raise RuntimeError(f"long decomposition of variable {x.arg.name} unsupported") + return UOp(x.op, arg=replace(x.arg, dtype=l2i_dt[x.dtype], size=None if x.arg.size is None else x.arg.size*2), tag=x.tag) + def split_l2i(ctx:dict, op: Ops, dt: DType, *uops:UOp): # l2i does arithmetic on its inputs; rules enter here to split them to 32-bit words first, l2i recurses on itself. # both word halves of a node ask for the same split, so ctx memos it for the pass @@ -140,8 +145,7 @@ def f2f_store(st, idx, val, fr:DType, to:DType): pm_long_decomp: PatternMatcher = PatternMatcher([ # the decomp's own bottom-up rewrite can mint bare consts mid-flight: word splitting commits them at the long sibling's dtype (UPat(GroupOp.All, name='x'), lambda x: commit_weak_consts(x, next((s.dtype for s in x.src if s.dtype in l2i_dt), None))), - (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), name="x"), lambda x: - UOp(x.op, arg=replace(x.arg, dtype=l2i_dt[x.dtype], size=None if x.arg.size is None else x.arg.size*2), tag=x.tag)), + (UPat(GroupOp.Defines, tuple(l2i_dt.keys()), name="x"), l2i_define), (UPat(Ops.INDEX, tuple(l2i_dt.keys()), name='x'), lambda x: reindex(x, x.tag[0]).replace(tag=None) if x.tag is not None else None), (UPat(Ops.STORE, src=(UPat.var('idx', tuple(l2i_dt.keys())), UPat.var('val')), name='st'), lambda st,idx,val: diff --git a/tinygrad/runtime/ops_webgpu.py b/tinygrad/runtime/ops_webgpu.py index dc3466e4acead..de2103f281a5d 100644 --- a/tinygrad/runtime/ops_webgpu.py +++ b/tinygrad/runtime/ops_webgpu.py @@ -201,7 +201,7 @@ def pop_error(self) -> str: return DevicePopErrorScope(self.device_res)[1] def create_uniform(self, val:int|float) -> webgpu.WGPUBuffer: buf = webgpu.wgpuDeviceCreateBuffer(self.device_res, webgpu.WGPUBufferDescriptor(size=4, usage=webgpu.WGPUBufferUsage_Uniform | webgpu.WGPUBufferUsage_CopyDst)) - self.write_buffer(buf, val.to_bytes(4, "little") if isinstance(val, int) else struct.pack(' webgpu.WGPUBuffer: size = webgpu.wgpuBufferGetSize(buf) From 1a5a364f3ee99ee3d684d5f2e28ced2828f4b293 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sat, 29 Aug 2026 21:25:41 -0400 Subject: [PATCH 281/306] weak dtype overflow touchups [pr] (#17843) * weak dtype overflow touchups [pr] * one more --- test/unit/test_dtype_weak.py | 21 ++++++++++++++++++++- tinygrad/dtype.py | 10 +++++++++- tinygrad/mixin/creation.py | 2 +- tinygrad/mixin/dtype.py | 5 ++++- tinygrad/mixin/op.py | 23 ++++++++++------------- tinygrad/mixin/reduce.py | 6 +++--- tinygrad/schedule/rangeify.py | 4 ++-- tinygrad/tensor.py | 5 ++--- tinygrad/uop/ops.py | 4 ++-- tinygrad/uop/weak.py | 15 ++++++--------- 10 files changed, 59 insertions(+), 36 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index c6ddb2e8fcbb7..6495ed5b7b6f0 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -229,7 +229,7 @@ def test_computed_float_index_lowers(self): class TestWeakBounds(unittest.TestCase): def test_bounds_survive_movement(self): moved = Tensor(5).reshape(1).expand(2).pad((1, 1)).detach().contiguous_backward() - self.assertEqual((moved.uop.vmin, moved.uop.vmax), (0, 5)) + self.assertEqual((moved.uop.vmin, moved.uop.vmax, moved.uop.bufferize().vmax), (0, 5, 5)) self.assertEqual(moved.numpy().dtype, Tensor(5).numpy().dtype) # a moved weak int reads at the same dtype as the bare one def test_wide_src_keeps_its_width(self): @@ -255,6 +255,25 @@ def test_weak_source(self): ddst = Tensor.empty(2, dtype=dtypes.int32, device=f"DISK:{td}/t") with self.assertRaises(RuntimeError): ddst.assign(w05.expand(2)) + def test_weak_commits_by_bounds(self): + big = Tensor(2**40) + edges = (big.clone(), big.sum(), big.reshape(1).max(), big.reshape(1).mean(), Tensor.stack(big, Tensor(1)).sum() - 1, + Tensor([2**40]), big.full_like(2**40)) + for t in edges: self.assertEqual(t.item(), 2**40) + self.assertEqual(Tensor(UOp.variable("b", 0, 2**40).bind(2**35+3)).clone().item(), 2**35+3) + self.assertEqual(Tensor([10, 20, 30])[[2**32+1]].tolist(), [0]) # a wide list index is out of range, not wrapped + with Context(DEFAULT_INT=dtypes.int64): self.assertEqual(Tensor(2).clone().dtype, dtypes.int64) + + def test_literal_beyond_any_int_raises(self): + for make in (lambda: Tensor(2**64).item(), lambda: Tensor([2**64]), lambda: Tensor.full((2,), -2**63-1)): + with self.assertRaises(OverflowError): make() + + def test_weak_sentinels_commit_first(self): + # max_pool2d, scatter_reduce and cummax pad with the dtype's min/max, which a weak dtype does not have + self.assertEqual(Tensor(-5).expand(1, 1, 2, 2).max_pool2d(2, padding=1).dtype, Tensor(-5).clone().dtype) + self.assertEqual(Tensor(-5).expand(2).scatter_reduce(0, Tensor([0]), Tensor(-5).expand(1), "amax", include_self=False).tolist(), [-5, -5]) + self.assertEqual(Tensor(2**40).expand(3).cummax(0)[0].tolist(), [2**40]*3) + def test_weak_has_no_storage(self): import numpy as np with self.assertRaises(RuntimeError): Tensor(np.ones(2, dtype=np.float32), dtype=dtypes.weakfloat) diff --git a/tinygrad/dtype.py b/tinygrad/dtype.py index 8d586d2c045c0..edbce227d753a 100644 --- a/tinygrad/dtype.py +++ b/tinygrad/dtype.py @@ -101,7 +101,11 @@ def from_py(x) -> DType: if isinstance(x, float): return dtypes.weakfloat if isinstance(x, int): return dtypes.weakint # put this in the last is faster because there are more items than lists/tuples to check - if isinstance(x, (list, tuple)): return strong_dtype(max(dtypes.from_py(xi) for xi in x)) if x else dtypes.default_float + if isinstance(x, (list, tuple)): + dt = max(dtypes.from_py(xi) for xi in x) if x else dtypes.weakfloat + if dt is not dtypes.weakint: return strong_dtype(dt) + ints = [xi for xi in x if isinstance(xi, int)] # a vconst also holds Invalid + return commit_int(min(ints), max(ints)) raise RuntimeError(f"Could not infer dtype of {x} with type {type(x)}") @staticmethod def finfo(dtype:DType) -> tuple[int, int]: @@ -162,6 +166,10 @@ def to_dtype(dtype:DTypeLike) -> DType: return dtype if isinstance(dtype, DType) assert dtypes.is_int(dtypes.default_int), f"{DEFAULT_INT.value} is not an int dtype" def strong_dtype(dtype:DType) -> DType: return {dtypes.weakint: dtypes.default_int, dtypes.weakfloat: dtypes.default_float}.get(dtype, dtype) +def commit_int(lo:int|float, hi:int|float, default_int:DType|None=None) -> DType: + if lo == hi and not dtypes.long.min <= lo <= dtypes.ulong.max: raise OverflowError(f"{lo} does not fit any int") + ladder = (dtypes.default_int if default_int is None else default_int, dtypes.int, dtypes.long, dtypes.ulong) + return next((dt for dt in ladder if dt.min <= lo and hi <= dt.max), dtypes.long) def weak_dtype(dtype:DType) -> DType: return dtypes.weakfloat if dtypes.is_float(dtype) else dtypes.weakint if dtypes.is_int(dtype) else dtype diff --git a/tinygrad/mixin/creation.py b/tinygrad/mixin/creation.py index eb6d82481742b..0015cc6014fe6 100644 --- a/tinygrad/mixin/creation.py +++ b/tinygrad/mixin/creation.py @@ -80,7 +80,7 @@ def full(cls, shape:'tuple[sint, ...]', fill_value:'ConstType|UOp', dtype:DTypeL dt = to_dtype(dtype) if dtype is not None else fill_value.dtype if isinstance(fill_value, UOp) else dtypes.from_py(fill_value) val = cls.const(fill_value, dt).expand(new_shape) if not buffer: return val - ret = val.empty_like(dt if dtype is not None else None, device) + ret = val.empty_like(None if dt in dtypes.weaks else dt, device) return cls._wrap_uop(ret._uop.after(ret._uop.store(val._uop))) def full_like(self, fill_value:ConstType, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None, buffer=True) -> Self: diff --git a/tinygrad/mixin/dtype.py b/tinygrad/mixin/dtype.py index b46bac675b32b..688a612aae479 100644 --- a/tinygrad/mixin/dtype.py +++ b/tinygrad/mixin/dtype.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING, Self -from tinygrad.dtype import DType, DTypeLike, dtypes, to_dtype +from tinygrad.dtype import DType, DTypeLike, dtypes, to_dtype, strong_dtype, commit_int from tinygrad.uop import Ops if TYPE_CHECKING: @@ -13,6 +13,9 @@ def _uop(self) -> 'UOp': raise NotImplementedError @classmethod def _wrap_uop(cls, u:'UOp') -> Self: raise NotImplementedError + def commit_dtype(self, default_int:DType|None=None) -> DType: + return commit_int(self._uop.vmin, self._uop.vmax, default_int) if self.dtype is dtypes.weakint else strong_dtype(self.dtype) + def cast(self, dtype:DTypeLike) -> Self: """ Casts `self` to the given `dtype`. diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 74722fa814e8f..5b7f190d0bf70 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -6,7 +6,7 @@ from tinygrad.mixin.reduce import ReduceMixin from tinygrad.uop import Ops from tinygrad.uop.ops import _broadcast_shape, resolve, smax, smin, identity_element -from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, PyConst, dtypes, least_upper_dtype, sum_acc_dtype, to_dtype +from tinygrad.dtype import ConstType, DType, DTypeLike, Invalid, PyConst, dtypes, least_upper_dtype, sum_acc_dtype, to_dtype, commit_int from tinygrad.helpers import all_int, argfix, argsort, ceildiv, flatten, flat_to_grouped, fully_flatten, get_shape, make_tuple, merge_dicts, prod from tinygrad.helpers import resolve_pool_pads, round_up, IMAGE, FLOAT16, WINO @@ -82,8 +82,7 @@ def is_adv(i): return isinstance(i,(list,tuple)) or (isinstance(i,type(self)) an parsed = {"size":size, "boundary":(0, size), "stride":1, "collapse_dim":False} if isinstance(index,(list,tuple)): flat = fully_flatten(index) - inferred = dtypes.bool if (flat and all(isinstance(s,bool) for s in flat)) else \ - (dtypes.default_int if flat and all_int(flat) else dtypes.default_float) + inferred = dtypes.from_py(flat) if not dtypes.is_int(inferred): raise IndexError(f"{index=} contains non-int element") index = self._wrap_uop(UOp._frompy([i+size if i<0 else i for i in flat], inferred, self.device)).reshape(get_shape(index)) elif is_adv(index): @@ -186,9 +185,7 @@ def arange(cls, start, stop=None, step=1, dtype:DTypeLike|None=None) -> Self: if stop is None: stop, start = start, 0 lo, hi = (start, stop-step) if step > 0 else (stop-step, start) if dtype is None: - dtype = dtypes.default_float if any(isinstance(x, float) for x in (start, stop, step)) else dtypes.default_int - # an int range too large for default_int picks int64 - if dtype is dtypes.default_int and (lo < dtype.min or dtype.max < hi): dtype = dtypes.int64 + dtype = dtypes.default_float if any(isinstance(x, float) for x in (start, stop, step)) else commit_int(lo, hi) if lo < (dt:=to_dtype(dtype)).min or dt.max < hi: raise OverflowError(f"arange [{start}, {stop}) is not representable in dtype {dtype}") # NOTE: this matches numpy, torch raises RuntimeError if stop-start and step have different signs if (output_len:=ceildiv(stop-start, step)) <= 0: return cls.full((0,), 0, dtype=dtype, buffer=False) @@ -519,7 +516,7 @@ def mean(self, axis:int|Sequence[int]|None=None, keepdim=False) -> Self: ``` """ output_dtype = self.dtype if dtypes.is_float(self.dtype) else dtypes.float32 - numerator = self.cast(sum_acc_dtype(self.dtype)).sum(axis=axis, keepdim=keepdim) + numerator = self.cast(sum_acc_dtype(self.commit_dtype())).sum(axis=axis, keepdim=keepdim) denominator = prod([si for si, so in zip(self.shape, self.sum(axis=axis, keepdim=True).shape) if resolve(si != so)]) return numerator.div(denominator).cast(output_dtype) @@ -548,7 +545,7 @@ def var(self, axis:int|Sequence[int]|None=None, keepdim=False, correction=1) -> output_dtype = self.dtype if dtypes.is_float(self.dtype) else dtypes.float32 squares = (self - self.mean(axis=axis, keepdim=True)).square() n = prod([si for si, so in zip(self.shape, squares.sum(axis=axis, keepdim=True).shape) if resolve(si != so)]) - numerator = squares.cast(sum_acc_dtype(self.dtype)).sum(axis=axis, keepdim=keepdim) + numerator = squares.cast(sum_acc_dtype(self.commit_dtype())).sum(axis=axis, keepdim=keepdim) return numerator.div(smax(n - correction, 0)).cast(output_dtype) def var_mean(self, axis:int|Sequence[int]|None=None, keepdim=False, correction=1) -> tuple[Self, Self]: @@ -755,7 +752,7 @@ def cat(self, *args:Self, dim:int=0) -> Self: def _cumalu(self, axis:int, op:Ops) -> Self: assert self.shape[axis] != 0 and op in (Ops.ADD, Ops.MAX, Ops.MUL) pads = (None,)*(self.ndim-1) + ((self.shape[axis]-1, 0),) - pooled = self.transpose(axis,-1)._pad_constant(pads, identity_element(op, self.dtype))._pool((self.shape[axis],)) + pooled = self.transpose(axis,-1)._pad_constant(pads, identity_element(op, self.commit_dtype()))._pool((self.shape[axis],)) return getattr(pooled, {Ops.ADD: "sum", Ops.MAX: "max", Ops.MUL: "prod"}[op])(-1).transpose(axis, -1) def _split_cumalu(self, axis:int, op:Ops) -> Self: @@ -764,7 +761,7 @@ def _split_cumalu(self, axis:int, op:Ops) -> Self: # TODO: someday the optimizer will find this on its own # for now this is a two stage cumsum SPLIT = 256 - value = identity_element(op, self.dtype) + value = identity_element(op, self.commit_dtype()) if not isinstance(s:=self.shape[axis], int) or s <= SPLIT*2: return self._cumalu(axis, op) chunks = self.transpose(axis,-1)._pad_constant((None,)*(self.ndim-1)+((round_up(s,SPLIT)-s,0),), value).unflatten(-1,(-1,SPLIT))._cumalu(-1, op) base = chunks[..., -1]._cumalu(-1, op)._pad_constant((None,)*(chunks.ndim-2) + ((1, -1),), value) @@ -1129,8 +1126,8 @@ def scatter_reduce(self, dim:int, index:Self, src:Self, reduce:Literal["sum", "p def _inv_mask(a:Self|PyConst, b:Self|PyConst) -> Self: return mask.any(-1).logical_not().where(a, b) if reduce == "sum": return mask.where(src, 0).sum(-1).add(self if include_self else _inv_mask(self, 0)) if reduce == "prod": return mask.where(src, 1).prod(-1).mul(self if include_self else _inv_mask(self, 1)) - if reduce == "amax": return mask.where(src, m := src.dtype.min).max(-1).maximum(self if include_self else _inv_mask(self, m)) - if reduce == "amin": return mask.where(src, m := src.dtype.max).min(-1).minimum(self if include_self else _inv_mask(self, m)) + if reduce == "amax": return mask.where(src, m := src.commit_dtype().min).max(-1).maximum(self if include_self else _inv_mask(self, m)) + if reduce == "amin": return mask.where(src, m := src.commit_dtype().max).min(-1).minimum(self if include_self else _inv_mask(self, m)) if reduce == "mean": count = mask.where(1, 0).sum(-1).add(1 if include_self else _inv_mask(1, 0)) return mask.where(src, 0).sum(-1).add(self if include_self else _inv_mask(self, 0)).div(count) @@ -1372,7 +1369,7 @@ def max_pool2d(self, kernel_size:tuple[int, ...]=(2,2), stride=None, dilation=1, s_ = stride if stride is not None else k_ pads = resolve_pool_pads(padding, len(k_)) if ceil_mode: pads = self._apply_ceil_mode(pads, k_, s_, dilation) - pooled = self._pad_constant(((0,0),)*(self.ndim-len(k_)) + flat_to_grouped(pads), self.dtype.min)._pool(k_, s_, dilation) + pooled = self._pad_constant(((0,0),)*(self.ndim-len(k_)) + flat_to_grouped(pads), self.commit_dtype().min)._pool(k_, s_, dilation) if not return_indices: return pooled.max(axis) spatial_sz = int(prod(spatial_shape := self.shape[-len(k_):])) idx = type(self).arange(spatial_sz, 0, -1).reshape(spatial_shape) diff --git a/tinygrad/mixin/reduce.py b/tinygrad/mixin/reduce.py index 78695a40e218d..19931d7e60394 100644 --- a/tinygrad/mixin/reduce.py +++ b/tinygrad/mixin/reduce.py @@ -1,6 +1,6 @@ from typing import Self, Sequence from tinygrad.uop import Ops -from tinygrad.dtype import DTypeLike, dtypes, strong_dtype, sum_acc_dtype, to_dtype +from tinygrad.dtype import DTypeLike, dtypes, sum_acc_dtype, to_dtype from tinygrad.helpers import make_tuple from tinygrad.mixin.dtype import DTypeMixin from tinygrad.mixin.movement import MovementMixin @@ -11,7 +11,7 @@ def _rop(self, op: Ops, axis: tuple[int, ...]) -> Self: raise NotImplementedError def _reduce(self, op:Ops, axis:int|Sequence[int]|None=None, keepdim=False) -> Self: - self = self.cast(strong_dtype(self.dtype)) + self = self.cast(self.commit_dtype()) axis = tuple(self._resolve_dim(x) for x in (range(self.ndim) if axis is None else make_tuple(axis, 1))) if self.ndim == 0: axis = () ret = self._rop(op, axis) @@ -41,7 +41,7 @@ def sum(self, axis:int|Sequence[int]|None=None, keepdim=False, dtype:DTypeLike|N print(t.sum(axis=1).numpy()) ``` """ - ret = self.cast(sum_acc_dtype(self.dtype) if dtype is None else to_dtype(dtype))._reduce(Ops.ADD, axis, keepdim) + ret = self.cast(sum_acc_dtype(self.commit_dtype()) if dtype is None else to_dtype(dtype))._reduce(Ops.ADD, axis, keepdim) return ret.cast(self.dtype) if dtype is None and self.dtype in (dtypes.float16, dtypes.bfloat16, *dtypes.fp8s) else ret def prod(self, axis:int|Sequence[int]|None=None, keepdim=False, dtype:DTypeLike|None=None) -> Self: diff --git a/tinygrad/schedule/rangeify.py b/tinygrad/schedule/rangeify.py index 79d8d3784cc9a..d2bbdf2eb6cb5 100644 --- a/tinygrad/schedule/rangeify.py +++ b/tinygrad/schedule/rangeify.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field, replace from typing import cast import itertools -from tinygrad.dtype import dtypes, AddrSpace, Invalid, strong_dtype +from tinygrad.dtype import dtypes, AddrSpace, Invalid from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, KernelInfo, ParamArg from tinygrad.uop.ops import graph_rewrite, sint, AxisType, BottomUpGate, rewrite_group from tinygrad.uop.symbolic import symbolic @@ -208,7 +208,7 @@ def visitor(u:UOp) -> frozenset[UOp]: def bufferize_to_store(ctx:itertools.count, x:UOp, idx:UOp, allow_locals=True): size = prod(x.shape) - dtype = strong_dtype(x.dtype) # a BUFFER is never weak: store at the concrete dtype, the .cast(x.dtype) on the result keeps readers unchanged + dtype = x.commit_dtype() # a BUFFER is never weak: store at the committed dtype, the .cast(x.dtype) on the result keeps readers unchanged rngs = sorted(idx.ranges, key=lambda x: x.arg) assert size > 0 and isinstance(size, int), f"no zero sized or symbolic sized buffers {size}" diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index eddea628e5646..4d0afe3331ab6 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -8,7 +8,6 @@ from tinygrad.helpers import all_int, getenv, fetch, Metadata, TRACEMETA, TracingKey from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ, pluralize from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, UPat, PatternMatcher, GroupOp, ParamArg, graph_rewrite, rewrite_group -from tinygrad.uop.weak import default_dtype from tinygrad.mixin.rand import RandMixin from tinygrad.schedule import create_linear_with_vars from tinygrad.device import Buffer, canonicalize_device @@ -480,7 +479,7 @@ def data(self) -> memoryview: print(np.frombuffer(t.data(), dtype=np.int32)) ``` """ - if self.dtype in dtypes.weaks: return self.cast(default_dtype(self.uop)).data() + if self.dtype in dtypes.weaks: return self.cast(self.commit_dtype()).data() if 0 in self.shape: return memoryview(bytearray(0)).cast(self.dtype.fmt) # type: ignore[arg-type,return-value] assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" buf = self._buffer() @@ -521,7 +520,7 @@ def numpy(self) -> 'numpy.ndarray': print(repr(t.numpy())) ``` """ - if self.dtype in dtypes.weaks: return self.cast(default_dtype(self.uop)).numpy() + if self.dtype in dtypes.weaks: return self.cast(self.commit_dtype()).numpy() assert all_int(self.shape), f"no data if shape is symbolic, {self.shape=}" import numpy as np if self.dtype in { dtypes.bfloat16, *dtypes.fp8s }: return self.float().numpy() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 00e86f3cf90ec..8f6e4b17c36b6 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -826,7 +826,7 @@ def from_buffer(opaque:Buffer, device:str|tuple[str, ...]|None=None): def empty_like(self, dtype:DTypeLike|None=None, device:str|tuple[str, ...]|None=None) -> UOp: device = canonicalize_device(self.device if device is None else device) axis = self.axis if isinstance(device, tuple) else None - ret = UOp.empty(self.shard_shape if axis is not None else self.shape, dtype=strong_dtype(self.dtype) if dtype is None else dtype, device=device) + ret = UOp.empty(self.shard_shape if axis is not None else self.shape, dtype=self.commit_dtype() if dtype is None else dtype, device=device) return ret.unshard(axis) if axis is not None else ret @staticmethod def _frompy(x:list|tuple|bytes, dtype:DType, device:str|tuple[str, ...]|None=None) -> UOp: @@ -1098,7 +1098,7 @@ def _min_max(self) -> tuple[PyConst, PyConst]: if self.op is Ops.STACK: return min(x.vmin for x in self.src), max(x.vmax for x in self.src) if self.op is Ops.CONST and self.val is not Invalid: return self.val, self.val if self.op is Ops.PAD: return min(self.src[0].vmin, 0), max(self.src[0].vmax, 0) # PAD adds zeros - if self.op in GroupOp.Movement|{Ops.INDEX, Ops.AFTER, Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD}: return self.src[0]._min_max + if self.op in GroupOp.Movement|{Ops.INDEX, Ops.STAGE, Ops.AFTER, Ops.DETACH, Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD}: return self.src[0]._min_max if self.op is Ops.CAST: # rounding is monotone (truncation toward zero into an int, to-nearest onto the value grid into a float) smin, smax = self.src[0]._min_max diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index 374b1277ea759..d8cede571daae 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -3,10 +3,6 @@ from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, dtype_from_uop, promo_dtype -def default_dtype(u:UOp): - if u.dtype is dtypes.weakfloat: return dtypes.default_float - return dtypes.long if u.overflows(dtypes.int32) else dtypes.int - def commit_weak(s:UOp, dt:DType) -> UOp: # a CONST re-mints, never takes a cast: at bool/weakint/weakfloat a CAST would be a second spelling of one const return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) @@ -36,7 +32,7 @@ def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: # only within the kind: an int cast of a weakfloat node is a value conversion, not a statement about the node's width if c.dtype in dtypes.weaks or weak_dtype(c.dtype) is not u.dtype: return None # every weak src commits at the one width: the node's own bounds and each src's, none of them narrowed - dt = least_upper_dtype(c.dtype, default_dtype(u), *(default_dtype(s) for s in u.src if s.dtype in dtypes.weaks)) + dt = least_upper_dtype(c.dtype, u.commit_dtype(dtypes.int), *(s.commit_dtype(dtypes.int) for s in u.src if s.dtype in dtypes.weaks)) return None if (ret:=commit_srcs_at(u, dt)) is None else ret.cast(c.dtype) # rides every round that can mint a weak const, and must reach fixpoint before pm_lower_weak below defaults one @@ -55,12 +51,13 @@ def lower_weak_node(u:UOp) -> UOp|None: if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const, not a consumer src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) if derived_dtypes(u, src) is None: - src = tuple(commit_weak(s, default_dtype(s)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) + src = tuple(commit_weak(s, s.commit_dtype(dtypes.int)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) if src == u.src: return None start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification if u.op not in _lower_weak_ops or any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): return u.replace(src=src) # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait - dt = strong_dtype(least_upper_dtype(default_dtype(u), *(s.dtype for s in src)) if u.op in GroupOp.Binary else dtype_from_uop(u.op, src, u.arg)) + dt = strong_dtype(least_upper_dtype(u.commit_dtype(dtypes.int), *(s.dtype for s in src)) if u.op in GroupOp.Binary else + dtype_from_uop(u.op, src, u.arg)) src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) for s in src[start:]) return u.replace(src=src).cast(u.dtype) @@ -71,9 +68,9 @@ def lower_weak_node(u:UOp) -> UOp|None: lambda u,buf,gate,idx: u.replace(src=(buf, idx.cast(dtypes.int).valid(gate))+u.src[2:]) if buf.max_numel()-1 <= dtypes.int32.max else None), # two stacked weak casts are two kind conversions: each resolves at its own kind's default (UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat(Ops.CAST, dtype=dtypes.weaks, src=(UPat.var("x"),)),), name="u"), - lambda u,x: x.cast(default_dtype(u.src[0])).cast(default_dtype(u)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), + lambda u,x: x.cast(u.src[0].commit_dtype(dtypes.int)).cast(u.commit_dtype(dtypes.int)).cast(u.dtype) if x.dtype not in dtypes.weaks else None), (UPat((Ops.PARAM, Ops.BUFFER), dtype=dtypes.weakint, name="u"), - lambda u: u.replace(arg=replace(u.arg, dtype=default_dtype(u))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), + lambda u: u.replace(arg=replace(u.arg, dtype=u.commit_dtype(dtypes.int))).cast(dtypes.weakint) if u.addrspace == AddrSpace.ALU else None), (UPat(GroupOp.All, name="u"), lower_weak_node), ]) From 61f7fd3735b25cd06da9d781e6b5df2bfa0eeb2b Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 30 Aug 2026 08:01:10 -0400 Subject: [PATCH 282/306] remove dead symbolic rules [pr] (#17844) --- tinygrad/uop/symbolic.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 50d94c69b3136..d615be38a45fc 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -139,8 +139,7 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat.var("x", dtype=dtypes.ints+(dtypes.bool, dtypes.weakint)) != UPat.var("x"), lambda x: x.const_like(False, dtypes.bool)), # x != x -> False (only ints) # ** constant folding ** - # a CAST to a concrete dtype over a CONST is a value conversion: evaluate it once, at the CAST's dtype - # TODO: delete this once CONST has no dtype + # canonicalize casted CONST (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.val)), # collapse committed const conversions when the target has a native constant format. fmt-less targets are emulated and would re-expand this pair. (UPat(Ops.CAST, dtypes.all, name="root", src=(UPat(Ops.CAST, dtypes.all, src=(UPat(Ops.CONST, name="c"),)),)), @@ -158,7 +157,6 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat.var('x', dtype=dtypes.bool).maximum(UPat.var('y', dtype=dtypes.bool)), lambda x,y: x|y), # *** div rules *** (UPat.cvar('x', arg=0) / 0, lambda x: x.const_like(float('nan'))), # 0/0 -> nan - ((UPat.var("x") * 0) / 0, lambda x: x.const_like(float('nan'))), # (x*0)/0 -> nan # can be wrong if x or x2 is 0 (UPat.var("x") / UPat.var("x"), lambda x: x.const_like(1)), # x/x -> 1 ((UPat.var("x") * UPat.var("x2")) / UPat.var("x2"), lambda x,x2: x), # (x*x2)/x2 -> x @@ -390,7 +388,6 @@ def simplify_valid(valid:UOp) -> UOp|None: def reduce_mul_chain(r:UOp) -> UOp|None: if r.arg[0] not in {Ops.ADD, Ops.MAX}: return None - if r.dtype != r.src[0].dtype: return None inside, outside = [], [] for m in r.src[0].split_uop(Ops.MUL): m_parents = m.backward_slice @@ -471,8 +468,6 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")), lambda x,d: 1-d), # x*/(1+x) -> 1-1/(1+x) (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")*UPat.var("y")), lambda x,y,d: y*(1-d)), (UPat.var("x") * ((1+UPat.var("x")).reciprocal().named("d")+UPat.var("y")), lambda x,y,d: (1-d)+x*y), - # move const multiply after REDUCE (NOTE: the mul chain can do this, but only if it's a same dtype reduce) - ((UPat.var("x")*UPat.cvar("c")).reduce(arg=Ops.ADD, name="r", allow_any_len=True), lambda x,c,r: r.replace(src=(x,)+r.src[1:])*c.val), # reduce mul chain, move muls after the reduce (UPat(Ops.MUL).reduce(name="r", allow_any_len=True), reduce_mul_chain), # ** combine terms (opinionated) ** From 7cc973a4e033e5c497eaa96de1b5b286103519e6 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 30 Aug 2026 09:28:21 -0400 Subject: [PATCH 283/306] clean up unneeded dtype check in rules [PR] (#17845) --- tinygrad/codegen/decomp/op.py | 2 +- tinygrad/codegen/simplify.py | 2 +- tinygrad/renderer/isa/x86.py | 10 +++++----- tinygrad/schedule/prepare.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tinygrad/codegen/decomp/op.py b/tinygrad/codegen/decomp/op.py index 4e1d5f81c4505..16043212255eb 100644 --- a/tinygrad/codegen/decomp/op.py +++ b/tinygrad/codegen/decomp/op.py @@ -84,7 +84,7 @@ def get_simplifying_rewrite_patterns(ops:tuple[Ops, ...]) -> PatternMatcher: if Ops.AND in ops: pat.append((UPat.var("x", dtypes.ints)%UPat.cvar("c"), lambda x,c: x & (c.val-1) if c.val in powers_of_two else None)) pat.append((UPat.var("a")%UPat.var("b"), floormod_to_mod)) # no real hardware supports THREEFRY, but NullRenderer does - if Ops.THREEFRY not in ops: pat.append((UPat(Ops.THREEFRY, dtype=dtypes.uint64, src=(UPat.var("x"), UPat.var("key"))), threefry2x32)) + if Ops.THREEFRY not in ops: pat.append((UPat(Ops.THREEFRY, src=(UPat.var("x"), UPat.var("key"))), threefry2x32)) # MAX can be rewritten as CMPLT + WHERE (max function is annoying on many cstyle backends) if Ops.MAX not in ops and Ops.CMPLT in ops: pat.append((UPat(Ops.MAX, name="m"), lambda m: (m.src[0] < m.src[1]).where(m.src[1], m.src[0]))) return PatternMatcher(pat) diff --git a/tinygrad/codegen/simplify.py b/tinygrad/codegen/simplify.py index 582e721662ed3..21c928a769a27 100644 --- a/tinygrad/codegen/simplify.py +++ b/tinygrad/codegen/simplify.py @@ -84,7 +84,7 @@ def reduce_unparented(red:UOp) -> UOp|None: assert all(x.op is Ops.RANGE for x in red.src[1:]), "some reduce srcs aren't ranges" reduce_parented, reduce_unparented = partition(red.src[1:], lambda x: x in red.src[0].ranges) if len(reduce_unparented) == 0: return None - ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) or red.dtype != red.src[0].dtype else red.src[0] + ret = red.replace(src=(red.src[0],)+tuple(reduce_parented)) if len(reduce_parented) else red.src[0] if red.arg[0] is Ops.ADD: for r in reduce_unparented: ret = ret * r.src[0] if red.arg[0] is Ops.MUL: diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 3bef96b2faea1..042486d3d7c4e 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -377,7 +377,7 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(GroupOp.Comparison, src=(UPat(dtype=dtypes.float64), UPat()), name="m").where(UPat.var("a", dtypes.float64), UPat.var("b")), lambda m,a,b: a.ins(X86Ops.VBLENDVPD, src=(b, a, mask(m)))), # in this case we have a mask producing comparison whose user expects a bool, so we convert to bool - (UPat(GroupOp.Comparison, dtypes.bool, (UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: + (UPat(GroupOp.Comparison, src=(UPat.var("y", (dtypes.float32, dtypes.float64)), UPat()), name="x"), lambda y,x: UOp(Ops.AND, src=(mask(x).bitcast(dt:=to_int(y.dtype)), UOp.cconst(1, dt))).bitcast(dtypes.bool)), # conditional moves that use flags # TODO: remove this once we allow all flag producing ops in cmove @@ -394,10 +394,10 @@ def alloc_vregs(ctx:IselContext, x:UOp) -> UOp|None: (UPat(Ops.IF, src=(UPat(Ops.CMPEQ, name="y"),), name="x"), lambda y,x: x.ins(X86Ops.JE, src=(cmp(y),))), (UPat(Ops.IF, src=(UPat(Ops.CMPNE, name="y"),), name="x"), lambda y,x: x.ins(X86Ops.JNE, src=(cmp(y),))), # comparisons whose user doesn't use the flag, move flag result to register - (UPat(Ops.CMPLT, dtypes.bool, (UPat(dtype=dtypes.uints), UPat()), name="x"), lambda x: x.ins(X86Ops.SETB, src=(cmp(x),))), - (UPat(Ops.CMPLT, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), - (UPat(Ops.CMPEQ, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), - (UPat(Ops.CMPNE, dtypes.bool, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), + (UPat(Ops.CMPLT, src=(UPat(dtype=dtypes.uints), UPat()), name="x"), lambda x: x.ins(X86Ops.SETB, src=(cmp(x),))), + (UPat(Ops.CMPLT, name="x"), lambda x: x.ins(X86Ops.SETL, src=(cmp(x),))), + (UPat(Ops.CMPEQ, name="x"), lambda x: x.ins(X86Ops.SETE, src=(cmp(x),))), + (UPat(Ops.CMPNE, name="x"), lambda x: x.ins(X86Ops.SETNE, src=(cmp(x),))), # float unary (UPat.var("y", dtypes.float32).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSS, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPS)), (UPat.var("y", dtypes.float64).sqrt().named("x"), lambda y,x: x.ins(X86Ops.VSQRTSD, src=(y, y)) if x.max_numel() == 1 else x.ins(X86Ops.VSQRTPD)), diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 8d9f4bc3989b5..037afb6df7424 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -38,7 +38,7 @@ def _mop_index(r:UOp, idx:UOp): if r.op is Ops.RESHAPE: src_prefix = len(r.src[0].shape) - len(r.shape[len(idxs):]) if src_prefix >= 0 and r.src[0].shape[src_prefix:] == r.shape[len(idxs):]: - if src_prefix == 0: return r.src[0] if r.src[0].dtype == idx.dtype else None + if src_prefix == 0: return r.src[0] ret = r.src[0].index(*apply_movement_op(r.op, r.src[0].shape[:src_prefix], r.shape[:len(idxs)], idxs), arg=idx.arg) return ret if ret.shape == idx.shape else None From 9c49e35aab2fbb23d91cd98e6a703329765f3773 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 30 Aug 2026 12:38:18 -0400 Subject: [PATCH 284/306] fix promote weak in _broadcasted [pr] (#17849) --- test/unit/test_dtype_weak.py | 1 + tinygrad/mixin/elementwise.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 6495ed5b7b6f0..c247392956432 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -240,6 +240,7 @@ def test_wide_src_keeps_its_width(self): def test_padded_weak_const_keeps_its_zeros(self): self.assertEqual(Tensor(1).expand(1).cat(Tensor(2).expand(2), Tensor(3).expand(3)).tolist(), [1, 2, 2, 3, 3, 3]) self.assertEqual((Tensor(5).reshape(1).pad((1, 1)) == 5).tolist(), [False, True, False]) + self.assertEqual((Tensor(5).reshape(1,1).expand(1,2).pad(((0,2),(0,0))) + Tensor([[1],[2],[3]])).tolist(), [[6,6],[2,2],[3,3]]) class TestWeakStorageBoundary(unittest.TestCase): # weak has no storage: a weak assignment source casts when it defers to the destination, everything else raises diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index a640356be2a64..527ca0a5370cf 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -25,8 +25,8 @@ def _broadcasted(self, y: 'Self|ConstType|UOp', reverse: bool = False) -> tuple[ # keep weak CONST weak, might lift weakint -> weakfloat def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const - if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST and t._uop.vmin == t._uop.vmax: - return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) + if t.dtype in dtypes.weaks and (b:=t._uop.base).op is Ops.CONST: + return t if t.dtype == (dt:=weak_dtype(out_dtype)) else t._wrap_uop(t._uop.substitute({b: b.const_like(b.val, dt)})) return t.cast(out_dtype) return promote(x), promote(y) From 902a1efe062866282c739677dd3e9c62cebdc55f Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 30 Aug 2026 13:09:36 -0400 Subject: [PATCH 285/306] Revert "fix promote weak in _broadcasted [pr] (#17849)" (#17851) This reverts commit 9c49e35aab2fbb23d91cd98e6a703329765f3773. --- test/unit/test_dtype_weak.py | 1 - tinygrad/mixin/elementwise.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index c247392956432..6495ed5b7b6f0 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -240,7 +240,6 @@ def test_wide_src_keeps_its_width(self): def test_padded_weak_const_keeps_its_zeros(self): self.assertEqual(Tensor(1).expand(1).cat(Tensor(2).expand(2), Tensor(3).expand(3)).tolist(), [1, 2, 2, 3, 3, 3]) self.assertEqual((Tensor(5).reshape(1).pad((1, 1)) == 5).tolist(), [False, True, False]) - self.assertEqual((Tensor(5).reshape(1,1).expand(1,2).pad(((0,2),(0,0))) + Tensor([[1],[2],[3]])).tolist(), [[6,6],[2,2],[3,3]]) class TestWeakStorageBoundary(unittest.TestCase): # weak has no storage: a weak assignment source casts when it defers to the destination, everything else raises diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 527ca0a5370cf..a640356be2a64 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -25,8 +25,8 @@ def _broadcasted(self, y: 'Self|ConstType|UOp', reverse: bool = False) -> tuple[ # keep weak CONST weak, might lift weakint -> weakfloat def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const - if t.dtype in dtypes.weaks and (b:=t._uop.base).op is Ops.CONST: - return t if t.dtype == (dt:=weak_dtype(out_dtype)) else t._wrap_uop(t._uop.substitute({b: b.const_like(b.val, dt)})) + if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST and t._uop.vmin == t._uop.vmax: + return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) return t.cast(out_dtype) return promote(x), promote(y) From f5b00de3199f2d7fc0ce2d568dd9103aa200cc85 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Sun, 30 Aug 2026 13:17:26 -0400 Subject: [PATCH 286/306] fix weak const promote dropping the movement ops (#17846) Co-authored-by: chenyu --- test/unit/test_dtype_weak.py | 1 + tinygrad/mixin/elementwise.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index 6495ed5b7b6f0..c247392956432 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -240,6 +240,7 @@ def test_wide_src_keeps_its_width(self): def test_padded_weak_const_keeps_its_zeros(self): self.assertEqual(Tensor(1).expand(1).cat(Tensor(2).expand(2), Tensor(3).expand(3)).tolist(), [1, 2, 2, 3, 3, 3]) self.assertEqual((Tensor(5).reshape(1).pad((1, 1)) == 5).tolist(), [False, True, False]) + self.assertEqual((Tensor(5).reshape(1,1).expand(1,2).pad(((0,2),(0,0))) + Tensor([[1],[2],[3]])).tolist(), [[6,6],[2,2],[3,3]]) class TestWeakStorageBoundary(unittest.TestCase): # weak has no storage: a weak assignment source casts when it defers to the destination, everything else raises diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index a640356be2a64..326cc3d64daf7 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -1,7 +1,7 @@ import math, functools, operator from typing import TYPE_CHECKING, Literal, Self from tinygrad.uop import Ops -from tinygrad.dtype import dtypes, ConstType, PyConst, least_upper_dtype, least_upper_float, weak_dtype +from tinygrad.dtype import dtypes, ConstType, DType, PyConst, least_upper_dtype, least_upper_float, weak_dtype from tinygrad.helpers import argfix, polyN from tinygrad.mixin.creation import CreationMixin @@ -9,6 +9,9 @@ from tinygrad.uop.ops import UOp, sint +def remint(u:'UOp', dt:DType) -> 'UOp': + return u.const_like(u.val, dt) if u.op is Ops.CONST else u.replace(src=(remint(u.src[0], dt),)+u.src[1:]) + class ElementwiseMixin(CreationMixin): # required to implement def alu(self, op: Ops, *src: Self) -> Self: @@ -26,7 +29,7 @@ def _broadcasted(self, y: 'Self|ConstType|UOp', reverse: bool = False) -> tuple[ def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST and t._uop.vmin == t._uop.vmax: - return t._wrap_uop(t._uop.const_like(t._uop.base.val, weak_dtype(out_dtype))) + return t._wrap_uop(remint(t._uop, weak_dtype(out_dtype))) return t.cast(out_dtype) return promote(x), promote(y) From 1c183e6de91ce83affc36f393641baee58f0e78e Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Sun, 30 Aug 2026 15:12:52 -0400 Subject: [PATCH 287/306] fix elu/celu/selu gradient being nan on large inputs (#17852) * fix elu/celu/selu gradient being nan on large inputs * shrink shape for the elu/celu/selu overflow checks * route selu and celu through elu --- test/backend/test_ops.py | 4 ++++ tinygrad/mixin/elementwise.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 1109dbec4b6e0..a47c90ae1d9ef 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -978,9 +978,12 @@ def test_leaky_relu(self): def test_celu(self): for val in range(1, 5): helper_test_op([(45,65)], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val)) + helper_test_op([(3,3)], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val), low=300, high=400) helper_test_op([()], lambda x: torch.nn.functional.celu(x,val), lambda x: x.celu(val)) def test_selu(self): helper_test_op([(45,65)], torch.nn.functional.selu, Tensor.selu) + helper_test_op([(3,3)], torch.nn.functional.selu, Tensor.selu, low=300, high=400) + helper_test_op(None, torch.nn.functional.selu, Tensor.selu, vals=[[-1.,0.,1.]]) helper_test_op([()], torch.nn.functional.selu, Tensor.selu) def test_silu(self): helper_test_op([(45,65)], torch.nn.functional.silu, Tensor.silu) @@ -1117,6 +1120,7 @@ def test_quick_gelu_extreme(self): def test_elu(self): helper_test_op([(45,65)], torch.nn.functional.elu, Tensor.elu) helper_test_op([(45,65)], lambda x: torch.nn.functional.elu(x, alpha=0.1), lambda x: Tensor.elu(x, alpha=0.1)) + helper_test_op([(3,3)], torch.nn.functional.elu, Tensor.elu, low=300, high=400) helper_test_op([()], torch.nn.functional.elu, Tensor.elu) def test_relu6(self): helper_test_op([(45,65)], torch.nn.functional.relu6, Tensor.relu6) diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 326cc3d64daf7..914b207568b2c 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -971,7 +971,7 @@ def elu(self, alpha=1.0) -> Self: print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).elu().numpy()) ``` """ - return self.relu() - alpha*(1-self.exp()).relu() + return (self > 0).where(self, alpha*((self - self.relu()).exp() - 1)) def celu(self, alpha=1.0) -> Self: """ @@ -983,7 +983,7 @@ def celu(self, alpha=1.0) -> Self: print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).celu().numpy()) ``` """ - return self.maximum(0) + (alpha * ((self / alpha).exp() - 1)).minimum(0) + return alpha * (self / alpha).elu() def selu(self, alpha=1.67326, gamma=1.0507) -> Self: """ @@ -995,7 +995,7 @@ def selu(self, alpha=1.67326, gamma=1.0507) -> Self: print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).selu().numpy()) ``` """ - return gamma * (self >= 0).where(self, alpha * (self.exp() - 1)) + return gamma * self.elu(alpha) def softplus(self, beta=1.0) -> Self: """ From 85841ce0d127191faebf88e8010aca1bb7f21913 Mon Sep 17 00:00:00 2001 From: chenyu Date: Sun, 30 Aug 2026 15:33:59 -0400 Subject: [PATCH 288/306] more const cast cleanups [PR] (#17854) --- test/unit/test_dtype_weak.py | 7 +++++++ test/unit/test_invalid_tensor.py | 6 ++++++ tinygrad/mixin/elementwise.py | 10 +++++----- tinygrad/renderer/nir.py | 4 ++-- tinygrad/uop/ops.py | 4 +++- tinygrad/uop/symbolic.py | 11 +++++------ tinygrad/uop/weak.py | 18 +++++++----------- 7 files changed, 35 insertions(+), 25 deletions(-) diff --git a/test/unit/test_dtype_weak.py b/test/unit/test_dtype_weak.py index c247392956432..b8b022829ec45 100644 --- a/test/unit/test_dtype_weak.py +++ b/test/unit/test_dtype_weak.py @@ -54,6 +54,13 @@ def test_minimum_reflects_weak_operand(self): self.assertEqual((r.dtype, r.tolist()), (dt, [1])) self.assertNotIn(Ops.CAST, [u.op for u in r._uop.toposort()]) + def test_promote_keeps_shape_args(self): + # the shape arg is the same CONST as the value, only the value lifts + self.assertEqual((Tensor(5).expand(5) + 1.5).tolist(), [6.5]*5) + self.assertEqual((Tensor(2).reshape(1,1).expand(2,2).pad(((0,2),(0,0))) + 0.5).tolist(), [[2.5,2.5],[2.5,2.5],[0.5,0.5],[0.5,0.5]]) + x, _ = Tensor(5).reshape(1).pad((1,1))._broadcasted(0.5) + self.assertEqual((x._uop.op, x._uop.base.dtype, x._uop.src[1].dtype), (Ops.PAD, dtypes.weakfloat, dtypes.weakint)) + def test_broadcasted_keeps_const_weak(self): # a python scalar stays a bare weak CONST through _broadcasted, lifted only to the KIND of the lub x, y = Tensor([1], dtype=dtypes.int8)._broadcasted(3) diff --git a/test/unit/test_invalid_tensor.py b/test/unit/test_invalid_tensor.py index 1c8630af36b25..1277ce6b8c1f9 100644 --- a/test/unit/test_invalid_tensor.py +++ b/test/unit/test_invalid_tensor.py @@ -23,6 +23,12 @@ def test_where_x_invalid(self): out = mask.where(Tensor([1.0, 2.0, 3.0, 4.0]), Invalid) self._invalid_test_helper(out, [1.0, 2.0, None, None]) + def test_where_padded_invalid_cast(self): + # a padded Invalid is not Invalid: its zeros take the cast + a, b = Tensor.full((1,), Invalid).pad((1,1)), Tensor.full((1,), Invalid).pad((2,0)) + out = Tensor([True, False, True]).where(a, b).cast(dtypes.float) + Tensor([1., 2., 3.]) + self.assertEqual((out.dtype, out.tolist()), (dtypes.float, [1.0, 2.0, 3.0])) + def test_where_invalid_x(self): mask = Tensor.arange(4) < 2 out = mask.where(Invalid, Tensor([1.0, 2.0, 3.0, 4.0])) diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 914b207568b2c..57bb9fdee352a 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -10,7 +10,7 @@ def remint(u:'UOp', dt:DType) -> 'UOp': - return u.const_like(u.val, dt) if u.op is Ops.CONST else u.replace(src=(remint(u.src[0], dt),)+u.src[1:]) + return u.ccast(dt) if u.op is Ops.CONST else u.replace(src=(remint(u.src[0], dt),)+u.src[1:]) class ElementwiseMixin(CreationMixin): # required to implement @@ -28,8 +28,8 @@ def _broadcasted(self, y: 'Self|ConstType|UOp', reverse: bool = False) -> tuple[ # keep weak CONST weak, might lift weakint -> weakfloat def promote(t): if t._uop.base.is_invalid: return t # invalid bool is weak const - if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST and t._uop.vmin == t._uop.vmax: - return t._wrap_uop(remint(t._uop, weak_dtype(out_dtype))) + if t.dtype in dtypes.weaks and t._uop.base.op is Ops.CONST: + return t if t.dtype == (dt:=weak_dtype(out_dtype)) else t._wrap_uop(remint(t._uop, dt)) return t.cast(out_dtype) return promote(x), promote(y) @@ -119,7 +119,7 @@ def sub(self, x: Self | ConstType, reverse: bool = False) -> Self: ``` """ a, b = self._broadcasted(x, reverse) - # alu, not +: _broadcasted already promoted these, and a second promote would cast -b (only a bare weak CONST is kept weak) + # alu, not +: _broadcasted already promoted these, and a second promote would cast -b (only a weak CONST is kept weak) return a.alu(Ops.ADD, -b) def mul(self, x: Self | ConstType, reverse: bool = False) -> Self: @@ -251,7 +251,7 @@ def div(self, x: 'Self|ConstType|UOp', reverse: bool = False, rounding_mode: Lit if rounding_mode == "trunc": return a.alu(Ops.CDIV, b) if rounding_mode == "floor": return a.alu(Ops.FLOORDIV, b) if dtypes.is_int(a.dtype) or a.dtype == dtypes.bool: a = a.cast(dtypes.default_float) - # alu, not *: _broadcasted already promoted these, and a second promote would cast 1/b (only a bare weak CONST is kept weak) + # alu, not *: _broadcasted already promoted these, and a second promote would cast 1/b (only a weak CONST is kept weak) d = a.alu(Ops.MUL, b.reciprocal()) if rounding_mode is None: return d if rounding_mode == "trunc": return d.trunc() diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index 71c00de8a1635..3dcc4ebf8a62d 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -136,8 +136,8 @@ class NIRRenderer(Renderer): (UPat(Ops.CAST, (dtypes.uchar, dtypes.ushort), src=(UPat.var("x", dtypes.floats),), name="c"), lambda x,c: x.cast(dtypes.int32).cast(c.dtype)), # load/store use pointer arithmetic, and the cast does nothing. NOTE: this doesn't apply to image indexing cause it's 1-D # nor to REG/ALU register picks, which keep their own index dtype - (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), lambda x,buf,off: x.replace( - src=(buf,UOp.const(off.val, dtypes.long) if off.op is Ops.CONST else off.cast(dtypes.long))+x.src[2:]) + (UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True, name="x"), + lambda x,buf,off: x.replace(src=(buf,off.ccast(dtypes.long))+x.src[2:]) if buf.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) and not is_image_shape(buf._shape) else None), # images need index to be int for nir (coordinates only: the INDEX keeps its access dtype) (UPat.var("buf").index(UPat.var("idx_y"), UPat.var("idx_x"), name="x"), diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 8f6e4b17c36b6..e141df8aeef4d 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -614,6 +614,8 @@ def const(b:ConstLike, dtype:DType|None=None): if isinstance(b, tuple): return UOp.stack(*[UOp.const(c, dtype) for c in b]) # .cast folds away at exactly the dtypes a CONST derives (bool/weakint/weakfloat): bare there, the pair everywhere else return UOp(Ops.CONST, arg=dtype.const(b), src=()).cast(dtype) + # cast, except for CONST, in which case rebuild a new CONST at the dtype + def ccast(self, dtype:DType): return UOp.const(self.val, dtype) if self.op is Ops.CONST else self.cast(dtype) # a forced CAST for bool: .cast(bool) folds, so UOp.const cannot state the width @staticmethod def cconst(b:ConstLike, dtype:DType): return UOp(Ops.CAST, src=(UOp.const(b),), arg=dtype) @@ -799,7 +801,7 @@ def _mop(self, op:Ops, arg) -> UOp: case Ops.STACK: srcs = (self,)+tuple(arg) dtype = dtype_from_uop(Ops.STACK, srcs, None) - return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else UOp.const(u.val, dtype) if u.op is Ops.CONST else u.cast(dtype) for u in srcs)) + return UOp(Ops.STACK, src=tuple(u if u.base.is_invalid else u.ccast(dtype) for u in srcs)) case _: raise RuntimeError(f"{op} is not a MovementOp") usrcs = [shape_to_shape_arg(arg) for arg in src_args] if len(usrcs) == 0: return UOp(op, src=(self,), arg=arg) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index d615be38a45fc..b44b15abfe4a9 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -6,7 +6,7 @@ from tinygrad.helpers import partition, all_same, prod, flatten, unwrap, IMAGE, dedup from tinygrad.uop.divandmod import div_and_mod_symbolic from tinygrad.uop.movement import mop_cleanup -from tinygrad.uop.weak import pm_uncast_const, commit_weak +from tinygrad.uop.weak import pm_uncast_const # TODO: symbolic shouldn't be importing from codegen from tinygrad.codegen.decomp.transcendental import xpow @@ -101,7 +101,7 @@ def const_arg(u:UOp): def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: # folding a strong dtype WHERE to a weak const branch keeps the strong dtype ret = c0 if gate.val else c1 - return commit_weak(ret, w.dtype) if ret.op is Ops.CONST and ret.dtype in dtypes.weaks and w.dtype not in dtypes.weaks else ret + return ret.ccast(w.dtype) if ret.op is Ops.CONST and ret.dtype in dtypes.weaks and w.dtype not in dtypes.weaks else ret symbolic_simple = pm_data_invalid + PatternMatcher([ # ** self folding ** @@ -149,7 +149,7 @@ def fold_const_where(gate:UOp, c0:UOp, c1:UOp, w:UOp) -> UOp: (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=bare_const, name="a"), fold_const_alu), (UPat(GroupOp.ALU-{Ops.THREEFRY}, src=casted_const, name="a"), fold_const_alu), (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=[casted_const, bare_const], name="a"), lambda a: - a.replace(src=tuple(commit_weak(s, dt) if s.dtype in dtypes.weaks else s for s in a.src)) + a.replace(src=tuple(s.ccast(dt) if s.dtype in dtypes.weaks else s for s in a.src)) if (dt:=promo_dtype(a.src)) not in dtypes.weaks else None), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), @@ -296,9 +296,8 @@ def fold_where_closure(cond:UOp, t:UOp, f:UOp) -> UOp|None: # cast/long folding # if the intermediate cast doesnt narrow we can do it in one cast (UPat.var('x').cast(name="a").cast(name="b"), lambda x,a,b: x.cast(b.dtype) if can_lossless_cast(x.dtype, a.dtype) else None), - # commit_weak, not .cast: a weak b.dtype is not a const spelling, and a CAST(weakfloat, CONST) reaches no commit round (UPat.var('x', dtypes.ints+(dtypes.weakint,)).cast(dtypes.ints+(dtypes.weakint,), name="a").cast(name="b"), - lambda x,a,b: commit_weak(x, b.dtype) if a.dtype.min<=x.vmin and x.vmax<=a.dtype.max else None), + lambda x,a,b: x.ccast(b.dtype) if not x.overflows(a.dtype) else None), # try to do math in int instead of long, keep weak const weak (UPat(GroupOp.Binary, src=(UPat.var("x", (dtypes.long, dtypes.weakint)), UPat.var("y", (dtypes.long, dtypes.weakint))), name="u"), lambda u,x,y: (UOp.const(x.val) if x.op is Ops.CONST else x.cast(dtypes.int)).alu(u.op, @@ -449,7 +448,7 @@ def gated_given_valid(cond:UOp, x:UOp, i:UOp) -> UOp|None: # ** where ** # push cast to branches (UPat.var("s").where(UPat.var("a"), UPat.var("b")).cast().named("cast"), - lambda s,a,b,cast: s.where(commit_weak(a, cast.dtype), commit_weak(b, cast.dtype))), + lambda s,a,b,cast: s.where(a.ccast(cast.dtype), b.ccast(cast.dtype))), # ** pow ** ((UPat(Ops.POW, name="p"), lambda p: xpow(*p.src))), # ** load/store folding ** diff --git a/tinygrad/uop/weak.py b/tinygrad/uop/weak.py index d8cede571daae..c4c9bd485a5eb 100644 --- a/tinygrad/uop/weak.py +++ b/tinygrad/uop/weak.py @@ -3,13 +3,9 @@ from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher, GroupOp, dtype_from_uop, promo_dtype -def commit_weak(s:UOp, dt:DType) -> UOp: - # a CONST re-mints, never takes a cast: at bool/weakint/weakfloat a CAST would be a second spelling of one const - return UOp.const(s.val, dt) if s.op is Ops.CONST else s.cast(dt) - # the decomps and float emulation commit bare consts at a dtype another src already states def commit_weak_consts(u:UOp, dt:DType|None) -> UOp|None: - return None if dt is None else u.replace(src=tuple(commit_weak(s, dt) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in u.src)) + return None if dt is None else u.replace(src=tuple(s.ccast(dt) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in u.src)) # the concrete dtypes u commits its srcs at: the operands' meet and u's own derived dtype, None if either is weak def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: @@ -20,7 +16,7 @@ def derived_dtypes(u:UOp, src:tuple[UOp, ...]) -> tuple[DType, DType]|None: def commit_srcs_at(u:UOp, dt:DType) -> UOp|None: # the root re-derives: a shift's dtype is its lhs's, so committing the lhs commits the node too bare = derived_dtypes(u, u.src) is not None - src = tuple(s if s.dtype not in dtypes.weaks else UOp.const(dt.const(s.val)) if bare and s.op is Ops.CONST else commit_weak(s, dt) for s in u.src) + src = tuple(s if s.dtype not in dtypes.weaks else UOp.const(dt.const(s.val)) if bare and s.op is Ops.CONST else s.ccast(dt) for s in u.src) return None if (ret := u.replace(src=src)) is u else ret def commit_weak_srcs(u:UOp) -> UOp|None: @@ -39,7 +35,7 @@ def cast_weak_srcs(c:UOp, u:UOp) -> UOp|None: pm_commit_weak = PatternMatcher([ (UPat(GroupOp.Broadcastable, name="u"), commit_weak_srcs), (UPat(Ops.STORE, src=(UPat(), UPat(dtype=dtypes.weaks)), allow_any_len=True, name="u"), - lambda u: u.replace(src=(u.src[0], commit_weak(u.src[1], u.src[0].dtype), *u.src[2:]))), + lambda u: u.replace(src=(u.src[0], u.src[1].ccast(u.src[0].dtype), *u.src[2:]))), # no CONST arm: a concrete CAST over a weak CONST is already committed, minted that way by UOp.const (UPat(Ops.CAST, name="c", src=(UPat(GroupOp.ALU, dtype=dtypes.weaks, name="u"),)), cast_weak_srcs), ]) @@ -51,14 +47,14 @@ def lower_weak_node(u:UOp) -> UOp|None: if u.op is Ops.CAST and u.src[0].op is Ops.CONST: return None # a committed const, not a consumer src = tuple(s.src[0] if s.op is Ops.CAST and s.dtype in dtypes.weaks else s for s in u.src) if derived_dtypes(u, src) is None: - src = tuple(commit_weak(s, s.commit_dtype(dtypes.int)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) + src = tuple(s.ccast(s.commit_dtype(dtypes.int)) if s.op is Ops.CONST and s.dtype in dtypes.weaks else s for s in src) if src == u.src: return None start = 1 if u.op is Ops.WHERE else 0 # WHERE's cond is bool, never part of the width unification if u.op not in _lower_weak_ops or any(s.dtype in dtypes.weaks and s.op is not Ops.CONST for s in src[start:]): return u.replace(src=src) # resolve whole once every weak expression lowered: a Binary widens from its own bounds too, derivable consts wait dt = strong_dtype(least_upper_dtype(u.commit_dtype(dtypes.int), *(s.dtype for s in src)) if u.op in GroupOp.Binary else dtype_from_uop(u.op, src, u.arg)) - src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else commit_weak(s, dt) for s in src[start:]) + src = src[:start]+tuple(s if s.base.is_invalid or s.dtype in dtypes.weaks else s.ccast(dt) for s in src[start:]) return u.replace(src=src).cast(u.dtype) pm_lower_weak = PatternMatcher([ @@ -89,8 +85,8 @@ def cast_const(u:UOp, s:UOp) -> UOp: if s.op is not Ops.CONST or s.is_invalid: return s # Invalid never commits # bool is the one strong bare dtype: cconst, since .cast(bool) would fold at construction if s.dtype is dtypes.bool: return UOp.cconst(s.val, s.dtype) - # commit at the dtype its consumer derives; where nothing does, commit_weak is the identity and spec_program rejects it - return commit_weak(s, dts[0]) if (dts:=derived_dtypes(u, u.src)) is not None else s + # commit at the dtype its consumer derives + return s.ccast(dts[0]) if (dts:=derived_dtypes(u, u.src)) is not None else s # commit every remaining bare const, keyed on the consumer: "bare" is a property of the edge def cast_consts(u:UOp) -> UOp|None: From fdb4109f5f0c82d61f11311ae331a7e9a150e797 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Sun, 30 Aug 2026 15:34:33 -0400 Subject: [PATCH 289/306] fix logcumsumexp on fully masked prefixes (#17853) --- test/backend/test_ops.py | 1 + tinygrad/mixin/op.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index a47c90ae1d9ef..238fdf7f66806 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1803,6 +1803,7 @@ def test_logcumsumexp(self): def test_logcumsumexp_numerical(self): helper_test_op(None, lambda x: torch.logcumsumexp(x, dim=0), lambda x: x.logcumsumexp(), atol=1e-7, grad_atol=1e-7, vals=[[0.0, 100.0]]) + helper_test_op(None, lambda x: torch.logcumsumexp(x, dim=0), lambda x: x.logcumsumexp(), vals=[[-math.inf, 0.0, 1.0]], forward_only=True) def test_sinh(self): helper_test_op([(45,65)], lambda x: x.sinh(), grad_atol=1e-6) diff --git a/tinygrad/mixin/op.py b/tinygrad/mixin/op.py index 5b7f190d0bf70..6aa3d097f3a40 100644 --- a/tinygrad/mixin/op.py +++ b/tinygrad/mixin/op.py @@ -855,7 +855,7 @@ def logcumsumexp(self, axis=0) -> Self: x = self.transpose(axis, -1) last_dim_size = x.shape[-1] x_unsqueezed = x.unsqueeze(-2) - x_cummax = x.cummax(-1)[0].detach() + x_cummax = (mx:=x.cummax(-1)[0].detach()).isfinite().where(mx, 0) mask = self._tri(last_dim_size, last_dim_size, 1).logical_not() ret = mask.where(x_unsqueezed - x_cummax.unsqueeze(-1), self.dtype.min).exp().sum(-1).log() + x_cummax return ret.transpose(-1, axis) From 593aa1e17b1cb0b1b2011482bfa0cf325d6bd9f0 Mon Sep 17 00:00:00 2001 From: Teddy Tennant Date: Sun, 30 Aug 2026 16:47:01 -0400 Subject: [PATCH 290/306] fix strided setitem being a no-op on a realized tensor (#17855) --- test/backend/test_setitem.py | 13 ++++++++----- tinygrad/tensor.py | 4 +++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/backend/test_setitem.py b/test/backend/test_setitem.py index 03ee238681f7a..4ee3290b52972 100644 --- a/test/backend/test_setitem.py +++ b/test/backend/test_setitem.py @@ -18,13 +18,16 @@ def test_simple_setitem(self): ((4,4,4,4), (slice(1,3), slice(None), slice(None), slice(0,3)), 4), ((6,6), (slice(1,5,2), slice(0,5,3)), 1.0), ((6,6), (slice(5,1,-2), slice(5,0,-3)), 1.0), + ((6,6), (slice(None), slice(0,6,2)), 1.0), ) for shp, slc, val in cases: - t = Tensor.zeros(shp).contiguous() - t[slc] = val - n = np.zeros(shp) - n[slc] = val.numpy() if isinstance(val, Tensor) else val - np.testing.assert_allclose(t.numpy(), n) + for realize in (False, True): + t = Tensor.zeros(shp).contiguous() + if realize: t.realize() + t[slc] = val + n = np.zeros(shp) + n[slc] = val.numpy() if isinstance(val, Tensor) else val + np.testing.assert_allclose(t.numpy(), n) def test_padded_setitem(self): t = Tensor.arange(10) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 4d0afe3331ab6..4daf6c21a59ba 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -257,6 +257,8 @@ def visitor(node: UOp) -> bool: return True if node in applied_map else any(in_s if s is ns: continue t.uop = ns +def _tensor_holds(u:UOp) -> bool: return any((t:=tref()) is not None and t.uop is u for tref in list(all_tensors)) + # **** Tensor helper functions **** def is_numpy_ndarray(x) -> "TypeGuard[numpy.ndarray]": return str(type(x)) == "" @@ -450,7 +452,7 @@ def assign(self, x:Tensor|PyConst|list|tuple) -> Tensor: # STORE+AFTER: STORE is the write effect (void), AFTER wraps the view for correct shape/ranging assign = self.uop.after(self.uop.store(x.uop)) ib = self.uop - while not ib.has_buffer_identity() and ib.op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH}: ib = ib.src[0] + while ib.op in GroupOp.Movement|{Ops.BITCAST, Ops.DETACH} and not (ib.has_buffer_identity() and _tensor_holds(ib)): ib = ib.src[0] if ib is not self.uop and ib.has_buffer_identity(after_ok=True): # view assign: replace at the buffer-identity level (e.g. RESHAPE(BUFFER)) so @function's substitution catches it _apply_map_to_tensors({ib: ib.after(assign)}, name="Embed View Assign") From a6c810eb03d19255da8f85f6fcb8fc1b48c306cd Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:25:48 +0800 Subject: [PATCH 291/306] mi350p: fix pmc (#17858) --- tinygrad/runtime/ops_amd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index 8d8c48f8de01a..f73b3cf0a91c8 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -1065,7 +1065,8 @@ def __init__(self, device:str=""): for k in (PMC_COUNTERS:=getenv("PMC_COUNTERS", pmc_default).split(",")): if k not in self.pmc_counters: raise RuntimeError(f"PMC counter {k} is not supported. Available: {','.join(self.pmc_counters.keys())}") - cast(AMDComputeQueue, unwrap(self.hw_compute_queue_t)()).pmc_start([(k, *self.pmc_counters[k]) for k in PMC_COUNTERS]).submit(self) + with (q:=cast(AMDComputeQueue, unwrap(self.hw_compute_queue_t)())).pred_exec((1 << self.xccs) - 1): + q.pmc_start([(k, *self.pmc_counters[k]) for k in PMC_COUNTERS]).submit(self) self.pmc_buffer = self.allocator.alloc(self.pmc_sched[-1].off + self.pmc_sched[-1].size, BufferSpec(nolru=True, uncached=True)) self.allocator._copyin(self.pmc_buffer, memoryview(bytearray(self.pmc_buffer.size))) # zero pmc buffers, some counters have only lo part. From b48cb9fce794c62f409a8c3d704d5c8896e3139f Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:45:31 +0800 Subject: [PATCH 292/306] viz: add cdna waveend (#17859) --- tinygrad/renderer/amd/sqtt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad/renderer/amd/sqtt.py b/tinygrad/renderer/amd/sqtt.py index 08f565b85e7f9..bac6888313dce 100644 --- a/tinygrad/renderer/amd/sqtt.py +++ b/tinygrad/renderer/amd/sqtt.py @@ -664,7 +664,7 @@ def simd_select(p) -> bool: return getattr(p, "cu", 0) == 0 and getattr(p, "simd if isinstance(p, (WAVESTART, WAVESTART_RDNA4, CDNA_WAVESTART)): assert p.wave not in wave_pc, "only one inflight wave per unit" wave_pc[p.wave] = next(iter(pc_map)) - elif isinstance(p, (WAVEEND, WAVEEND_RDNA4)): + elif isinstance(p, (WAVEEND, WAVEEND_RDNA4, CDNA_WAVEEND)): pc = wave_pc.pop(p.wave) yield (p, InstructionInfo(pc, p.wave, s_endpgm())) elif isinstance(p, IMMEDIATE_MASK): From 3123f947b0af7b0ead22e439465c2990ef010ece Mon Sep 17 00:00:00 2001 From: project224 <115223468+project224@users.noreply.github.com> Date: Mon, 31 Aug 2026 14:57:11 +0200 Subject: [PATCH 293/306] fix erf gradient being zero at x=0 (#17861) erf used self.sign() * f(self.abs()). Both sign() and abs() have zero gradient at exactly 0, so d/dx erf(0) came out as 0.0 instead of 2/sqrt(pi) ~= 1.1283792. Values on either side (including +/-1e-8) were already correct, so the error was a single point discontinuity. Derive the sign factor once with a where() and use it to form abs(x) as s*x, so the gradient at 0 takes the right-hand limit instead of being annihilated. Forward output is unchanged (erf(0) is still exactly 0.0 in float32) and the line count is unchanged. Claude-Session: https://claude.ai/code/session_01EXJpYja2w3bcjXzCaZwyWx Co-authored-by: projectai225 Co-authored-by: Claude Opus 5 --- test/backend/test_ops.py | 1 + tinygrad/mixin/elementwise.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 238fdf7f66806..9ede61e9ec945 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -1096,6 +1096,7 @@ def test_erf(self): helper_test_op([(45,65)], torch.erf, Tensor.erf) helper_test_op([(45,65)], torch.erf, Tensor.erf, low=300, high=400) helper_test_op([(45,65)], torch.erf, Tensor.erf, low=-400, high=-300) + helper_test_op(None, torch.erf, Tensor.erf, vals=[[-1., 0., 1.]]) helper_test_op([()], torch.erf, Tensor.erf) def test_gelu(self): diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 57bb9fdee352a..41f07b8fbf123 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -1066,8 +1066,8 @@ def erf(self) -> Self: ``` """ # https://personal.math.ubc.ca/~cbm/aands/page_299.htm 7.1.26 - t = 1.0 / (1.0 + 0.3275911 * self.abs()) - return self.sign() * (1.0 - t * polyN(t, [1.061405429, -1.453152027, 1.421413741, -0.284496736, 0.254829592]) * (-self.square()).exp()) + t = 1.0 / (1.0 + 0.3275911 * (s:=(self >= 0).where(1.0, -1.0)) * self) + return s * (1.0 - t * polyN(t, [1.061405429, -1.453152027, 1.421413741, -0.284496736, 0.254829592]) * (-self.square()).exp()) def softsign(self) -> Self: """ From e3c4b3d9b2e0e088cc29f93e1d26949641e7f330 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 31 Aug 2026 09:51:03 -0400 Subject: [PATCH 294/306] fix logaddexp and asin at 0 (#17862) * fix logaddexp and asin at 0 * pi/2 --- test/backend/test_ops.py | 5 +++++ tinygrad/mixin/elementwise.py | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/backend/test_ops.py b/test/backend/test_ops.py index 9ede61e9ec945..5483cb35d21e8 100644 --- a/test/backend/test_ops.py +++ b/test/backend/test_ops.py @@ -955,15 +955,18 @@ def test_asin(self): helper_test_op([(45,65)], lambda x: x.asin(), low=-1, high=1) helper_test_op([(45,65)], lambda x: x.asin(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.asin(), low=300, high=303) + helper_test_op(None, lambda x: x.asin(), vals=[[-0.5, 0., 0.5]]) def test_acos(self): # high grad atol helper_test_op([(45,65)], lambda x: x.acos(), low=-1, high=1) helper_test_op([(45,65)], lambda x: x.acos(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.acos(), low=300, high=303) + helper_test_op(None, lambda x: x.acos(), vals=[[-0.5, 0., 0.5]]) def test_atan(self): helper_test_op([(45,65)], lambda x: x.atan()) helper_test_op([(45,65)], lambda x: x.atan(), low=-300, high=-297) helper_test_op([(45,65)], lambda x: x.atan(), low=300, high=303) + helper_test_op(None, lambda x: x.atan(), vals=[[-0.5, 0., 0.5]]) def test_relu(self): helper_test_op([(64,64)], lambda x: x.relu()) @@ -1050,6 +1053,7 @@ def test_logaddexp(self): helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-1.], [-1.0, 2, 3]]) helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-100.0, -200, -300], [-1.0, 2, 3]]) helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[1.0, 2000, 30000], [-1.0, 2, 3]]) + helper_test_op(None, torch.logaddexp, Tensor.logaddexp, vals=[[-math.inf, math.inf, 1.0, -math.inf], [-math.inf, math.inf, -math.inf, 1.0]]) def test_softsign(self): helper_test_op([(45,65)], torch.nn.functional.softsign, Tensor.softsign) @@ -1091,6 +1095,7 @@ def test_softplus(self): helper_test_op([(45,65)], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6, low=300, high=400) helper_test_op([(45,65)], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6, low=-400, high=-300) helper_test_op([()], torch.nn.functional.softplus, Tensor.softplus, grad_atol=1e-6) + helper_test_op(None, torch.nn.functional.softplus, Tensor.softplus, vals=[[-math.inf, math.inf, 0.0]], forward_only=True) def test_erf(self): helper_test_op([(45,65)], torch.erf, Tensor.erf) diff --git a/tinygrad/mixin/elementwise.py b/tinygrad/mixin/elementwise.py index 41f07b8fbf123..bed5e5dabcae8 100644 --- a/tinygrad/mixin/elementwise.py +++ b/tinygrad/mixin/elementwise.py @@ -420,7 +420,7 @@ def logaddexp(self, other: Self | ConstType) -> Self: Calculates (self.exp()+other.exp()).log(), elementwise. """ a, b = self._broadcasted(other) - m = a.maximum(b) + m = (mx:=a.maximum(b)).isfinite().where(mx, 0) return ((a-m).exp() + (b-m).exp()).log() + m def where(self, x: 'Self | ConstType | sint', y: 'Self | ConstType | sint') -> Self: @@ -936,10 +936,10 @@ def asin(self) -> Self: print(Tensor([-0.9, -0.6, -0.3, 0., 0.3, 0.6, 0.9]).asin().numpy()) ``` """ - # https://personal.math.ubc.ca/~cbm/aands/page_81.htm 4.4.46 - coefficients = [-0.0012624911, 0.0066700901, -0.0170881256, 0.0308918810, -0.0501743046, 0.0889789874, -0.2145988016, 1.5707963050] - x = math.pi / 2 - (1.0 - self.abs()).sqrt() * polyN(self.abs(), coefficients) - return self.sign() * x + # https://personal.math.ubc.ca/~cbm/aands/page_81.htm 4.4.46, with a0 = pi/2 so asin(0) is exactly 0 + coefficients = [-0.0012624911, 0.0066700901, -0.0170881256, 0.0308918810, -0.0501743046, 0.0889789874, -0.2145988016, math.pi / 2] + a = (s:=(self >= 0).where(1.0, -1.0)) * self + return s * (math.pi / 2 - (1.0 - a).sqrt() * polyN(a, coefficients)) def acos(self) -> Self: """ From af079e8cc84c398b605c3765aa7bd3830bb36a1b Mon Sep 17 00:00:00 2001 From: b1tg <33436708+b1tg@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:55:37 +0800 Subject: [PATCH 295/306] llm: fix q6k fastpath (#17847) --- test/unit/test_llm_amd.py | 8 +++++--- tinygrad/llm/kernels/amd.py | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/unit/test_llm_amd.py b/test/unit/test_llm_amd.py index 2aa4cce2f8e99..505e9a6b80aa9 100644 --- a/test/unit/test_llm_amd.py +++ b/test/unit/test_llm_amd.py @@ -1,6 +1,6 @@ import unittest import numpy as np -from tinygrad import Tensor, UOp, dtypes, nn +from tinygrad import Tensor, UOp, dtypes, nn, function from tinygrad.llm.kernels.amd import Linear, amd_custom_kernels_supported, q8_quantize, flash_attention from tinygrad.llm.gguf import ggml_data_to_tensor @@ -28,7 +28,7 @@ def test_values_and_scales(self): # xsum holds the two per-16 sums per 32-wide group np.testing.assert_array_equal(gsum.numpy().reshape(2, 2), expected.reshape(2, 2, 16).sum(-1).astype(np.float32)) - def test_q6_linear_compiles(self): + def test_q6_linear_compiles_in_function(self): if not amd_custom_kernels_supported(Tensor.empty(1).device): self.skipTest("RDNA3 required") rng = np.random.default_rng(42) packed = rng.integers(0, 256, 210, dtype=np.uint8) @@ -37,7 +37,9 @@ def test_q6_linear_compiles(self): decoded = ggml_data_to_tensor(raw, 256, 14).reshape(1, 256) linear = Linear(256, 1, bias=False) nn.state.load_state_dict(linear, {"weight":decoded}, verbose=False, realize=False) - self.assertTrue(np.isfinite(linear(Tensor.randn(1, 256)).realize().item())) + @function(allow_implicit=True) + def run(x:Tensor): return linear(x) + self.assertTrue(np.isfinite(run(Tensor.randn(1, 256)).realize().item())) # the Q6 weight is repacked: 210-byte blocks padded to 212 (one block = 53 words) self.assertEqual(linear.weight.uop.buf_uop.buffer.nbytes, 53*4) self.assertEqual(linear.weight.dtype, dtypes.uint32) diff --git a/tinygrad/llm/kernels/amd.py b/tinygrad/llm/kernels/amd.py index 23a869a64d89e..3cc387d91a49e 100644 --- a/tinygrad/llm/kernels/amd.py +++ b/tinygrad/llm/kernels/amd.py @@ -70,11 +70,11 @@ def set_quantized(self, decoded:Tensor): # scheduling and would copy the entire packed weight on every JIT graph if self.ggml_type == Q6_K: # Q6 blocks are 210 bytes, so consecutive blocks are only 2-byte aligned. pad each block to 212 bytes - # (a one-time copy at load) so the kernel can do all its reads as aligned u32 words + # the kernel can do all its reads as aligned u32 words nbytes, nblocks = raw.max_numel(), raw.max_numel() // Q6_BYTES byte_view = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer).view(nbytes, dtypes.uint8, raw_offset))) - padded = byte_view.reshape((nblocks, Q6_BYTES)).pad_to((nblocks, Q6_PADDED)).contiguous().realize() - self.weight = Tensor(UOp.from_buffer(cast(Buffer, padded.uop.buf_uop.buffer).view(nblocks * Q6_WORDS, dtypes.uint32, 0))) + padded = byte_view.reshape((nblocks, Q6_BYTES)).pad_to((nblocks, Q6_PADDED)).bitcast(dtypes.uint32) + self.weight = padded.contiguous().reshape(nblocks * Q6_WORDS) else: self.weight = Tensor(UOp.from_buffer(cast(Buffer, raw.buf_uop.buffer) .view(raw.max_numel() * raw.dtype.itemsize // dtypes.uint32.itemsize, dtypes.uint32, raw_offset))) @@ -332,7 +332,7 @@ def dequant(base:UOp, subgroup:UOp, half:int) -> tuple[UOp, ...]: def q8_linear(layer:Linear, x:Tensor) -> Tensor: assert layer.ggml_type in (Q4_K, Q5_K, Q6_K, IQ4_XS) tokens = int(x.numel()) // layer.in_features - raw, out_features, in_features = layer.weight.uop.buf_uop, layer.out_features, layer.in_features + raw, out_features, in_features = layer.weight.uop, layer.out_features, layer.in_features def run(fxn:Callable[..., UOp], out:UOp, *srcs:UOp) -> Tensor: all_srcs = (out,)+srcs params = tuple(UOp.placeholder_like(src, slot=i) for i,src in enumerate(all_srcs)) From d0259936421fca5af8a8f1100e50010bb0a6983d Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:34:21 +0300 Subject: [PATCH 296/306] hcq2 rewrite patches (#17838) * hcq2 rewrite patches * hcq2 rewrite patches * fix fix * fix fix fix * h --- extra/hcq2/ops_amd2.py | 258 +++++++--------- test/device/test_hcq2.py | 23 +- test/helpers.py | 7 +- tinygrad/device.py | 5 +- tinygrad/engine/realize.py | 30 +- tinygrad/runtime/ops_cpu.py | 73 ++--- tinygrad/runtime/ops_python.py | 3 +- tinygrad/runtime/support/hcq2.py | 491 +++++++++++++++---------------- tinygrad/runtime/support/usb.py | 35 +-- tinygrad/uop/ops.py | 2 +- tinygrad/uop/spec.py | 3 +- 11 files changed, 440 insertions(+), 490 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 538d8d4edbda3..8de14a207333b 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -1,33 +1,32 @@ from __future__ import annotations -from typing import cast, Any, Callable -import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit +from typing import cast +import os, ctypes, struct, functools, importlib, mmap, errno, contextlib, sys, itertools, atexit assert sys.platform != 'win32' from dataclasses import dataclass -from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, encode_kernargs_clike, make_cmdbuf -from tinygrad.runtime.support.hcq2 import make_binary_patch +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_buf, hcq_size_var from tinygrad.uop.ops import sint, UOp -from tinygrad.device import Compiled, BufferSpec, Buffer, Device +from tinygrad.device import BufferSpec, Buffer, Device from tinygrad.dtype import dtypes -from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, ProfileEvent, lo32, hi32, colored, prod, ContextVar, TracingKey -from tinygrad.helpers import VIZ, ceildiv, unwrap, pluralize, to_tuple +from tinygrad.helpers import getenv, round_up, data64_le, DEBUG, PROFILE, lo32, hi32 +from tinygrad.helpers import ceildiv, unwrap, pluralize, to_tuple from tinygrad.renderer.cstyle import HIPRenderer, HIPCCRenderer from tinygrad.renderer.llvmir import AMDLLVMRenderer -from tinygrad.runtime.autogen import kfd, hsa, sqtt, amdgpu_kd, amdgpu_drm +from tinygrad.runtime.autogen import kfd, hsa, amdgpu_kd, amdgpu_drm from tinygrad.runtime.autogen.am import am from tinygrad.runtime.support.elf import elf_loader from tinygrad.runtime.support.hcq import FileIOInterface, HCQBuffer, MMIOInterface, hcq_filter_visible_devices from tinygrad.runtime.support.am.amdev import AMDev, AMMemoryManager from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_soc, import_pmc from tinygrad.runtime.support.system import PCIIfaceBase, PCIAllocationMeta, USBPCIDevice, MAP_FIXED, MAP_NORESERVE -from tinygrad.runtime.support.usb import USB3, usb_ib, usb_push, usb_arm_bytes, pm_usb_stage, pm_usb_hostio, pm_usb_bufferize +from tinygrad.runtime.support.usb import USB3, pm_usb_bufferize from tinygrad.runtime.support.memory import AddrSpace, BumpAllocator -from tinygrad.runtime.ops_amd import SQTT, SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE, SQTT_SIMD_SEL, SQTT_TOKEN_EXCLUDE, PMC -from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_EQ, WAIT_REG_MEM_FUNCTION_NEQ, WAIT_REG_MEM_FUNCTION_GEQ +from tinygrad.runtime.ops_amd import SQTT, PMC +from tinygrad.runtime.ops_amd import EVENT_INDEX_PARTIAL_FLUSH, WAIT_REG_MEM_FUNCTION_GEQ if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import -from tinygrad.engine.realize import get_runtime, pm_flatten_linear +from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops from tinygrad.uop import FastEnum, auto -from tinygrad.uop.ops import Ops, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import Ops, UPat, PatternMatcher # ***************** # PM4 @@ -36,9 +35,11 @@ class PM4Ops(FastEnum): SET_SH_REG = auto(); SET_UCONFIG_REG = auto(); WAIT_REG_MEM = auto(); ACQUIRE_MEM = auto() # noqa: E702 RELEASE_MEM = auto(); DISPATCH_DIRECT = auto(); EVENT_WRITE = auto() # noqa: E702 +def _dw(vals) -> int: return sum(2 if isinstance(x, UOp) and x.dtype.itemsize == 8 else 1 for x in vals) + def pkt3(ctx, op:PM4Ops, *vals): - return UOp(Ops.INS, arg=(op, dtypes.void), src=tuple(UOp.const(x, dtypes.uint32) - for x in (ctx.pm4.PACKET3(getattr(ctx.pm4, f"PACKET3_{op.name}"), len(vals) - 1), *vals))) + return UOp(Ops.LINEAR, src=tuple(x if isinstance(x, UOp) else UOp.const(x, dtypes.uint32) + for x in (ctx.pm4.PACKET3(getattr(ctx.pm4, f"PACKET3_{op.name}"), _dw(vals) - 1), *vals))) def wreg(ctx, reg:AMDReg, *args:sint, **kwargs:int): if bool(args) == bool(kwargs): raise RuntimeError('One (and only one) of *args or **kwargs must be specified') @@ -52,7 +53,7 @@ def wreg(ctx, reg:AMDReg, *args:sint, **kwargs:int): def wait_reg_mem(ctx, value, mask=0xffffffff, mem=None, reg=None, reg_done=0, op=WAIT_REG_MEM_FUNCTION_GEQ): wrm_info_dw = ctx.pm4.WAIT_REG_MEM_MEM_SPACE(int(mem is not None)) | ctx.pm4.WAIT_REG_MEM_OPERATION(int(mem is None and reg_done > 0)) \ | ctx.pm4.WAIT_REG_MEM_FUNCTION(op) | ctx.pm4.WAIT_REG_MEM_ENGINE(0) - return pkt3(ctx, PM4Ops.WAIT_REG_MEM, wrm_info_dw, *(data64_le(mem) if mem is not None else (reg, reg_done)), value, mask, 4) + return pkt3(ctx, PM4Ops.WAIT_REG_MEM, wrm_info_dw, *((mem,) if mem is not None else (reg, reg_done)), value, mask, 4) def acquire_mem(ctx, addr=0x0, sz=(1 << 64)-1, gli=1, glm=1, glk=1, glv=1, gl1=1, gl2=1): if ctx.target[0] != 9: @@ -83,7 +84,9 @@ def release_mem(ctx, address=0x0, value=0, data_sel=0, int_sel=2, ctxid=0, cache event_dw = ctx.pm4.EVENT_TYPE(ctx.pm4.CACHE_FLUSH_AND_INV_TS_EVENT) | ctx.pm4.EVENT_INDEX(ctx.pm4.event_index__mec_release_mem__end_of_pipe) memsel_dw = ctx.pm4.DATA_SEL(data_sel) | ctx.pm4.INT_SEL(int_sel) ctxid = 0 - return pkt3(ctx, PM4Ops.RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, *data64_le(address), *data64_le(value), ctxid) + addr_w = address if isinstance(address, UOp) else UOp.const(address, dtypes.uint64) + val_w = value.cast(dtypes.uint64) if isinstance(value, UOp) else UOp.const(value, dtypes.uint64) + return pkt3(ctx, PM4Ops.RELEASE_MEM, event_dw | cache_flags_dw, memsel_dw, addr_w, val_w, ctxid) def memory_barrier(ctx): pf = '' if ctx.nbio.version[0] == 2 else '0' if ctx.nbio.version[:2] != (7, 11) else '1' @@ -92,7 +95,7 @@ def memory_barrier(ctx): reg_done=getattr(ctx.nbio, f'regBIF_BX_PF{pf}_GPU_HDP_FLUSH_DONE').addr[0], value=0xffffffff), acquire_mem(ctx))) -def pm4_wait(ctx, dst, val): return wait_reg_mem(ctx, val, mem=dst.getaddr(ctx.devs)) +def pm4_wait(ctx, dst, val): return wait_reg_mem(ctx, val.cast(dtypes.uint32), mem=dst.getaddr(ctx.devs)) def pm4_barrier(ctx): return memory_barrier(ctx) @@ -106,28 +109,32 @@ def pm4_timestamp(ctx, dst): ctx.pm4.int_sel__mec_release_mem__none) def pm4_program(ctx, call, prg): - data, info = prg.arg - lib_gpu = prg.src[0] - args = encode_kernargs_clike(call, prg, ctx.devs) - prog_addr = lib_gpu.getaddr(ctx.devs) + data.entry_point_offset + data, lib = amd_build_program(ctx.dev, prg) + info = prg.arg + + # kernargs: a nested blob linear inside a getaddr, input addresses and variable values are filled per call through the input table + ka_words = [get_call_arg_uops(call)[gi].getaddr(ctx.devs) for gi in info.globals] + list(get_call_var_uops(call, prg)) + pad = data.kernargs_alloc_size - sum(w.dtype.itemsize for w in ka_words) + assert pad >= 0 and pad % 4 == 0, f"bad kernargs padding {pad}" + ka = UOp(Ops.LINEAR, src=tuple(ka_words) + (UOp.const(0, dtypes.uint32),) * (pad // 4)).rtag("kernargs") + + prog_addr = lib.getaddr(ctx.devs) + data.entry_point_offset scratch_addr = UOp.placeholder((data.private_segment_size,), dtypes.uint8, 0, device=ctx.devs).rtag("scratch").getaddr(ctx.devs) - args_addr = args.getaddr(ctx.devs) + args_addr = ka.getaddr(ctx.devs) - user_regs = [] - if data.enable_private_segment_sgpr: - scratch_hilo = data64_le(scratch_addr) - user_regs = [scratch_hilo[0], scratch_hilo[1] | 1 << 31, 0xffffffff, 0x20c14000] - if data.enable_dispatch_ptr: user_regs += [*data64_le(args_addr + data.kernargs_segment_size)] - user_regs += [*data64_le(args_addr)] + user_regs:list = [] + if data.enable_private_segment_sgpr: user_regs = [scratch_addr | (1 << 63), 0xffffffff, 0x20c14000] + if data.enable_dispatch_ptr: user_regs += [args_addr + data.kernargs_segment_size] + user_regs += [args_addr] dispatch_init = ctx.gc.regCOMPUTE_DISPATCH_INITIATOR.encode( **({'cs_w32_en': int(data.wave32)} if ctx.target[0] != 9 else {}), force_start_at_000=1, compute_shader_en=1) ins = [acquire_mem(ctx, gli=0, gl2=0), - wreg(ctx, ctx.gc.regCOMPUTE_PGM_LO, *data64_le(prog_addr >> 8)), + wreg(ctx, ctx.gc.regCOMPUTE_PGM_LO, prog_addr >> 8), wreg(ctx, ctx.gc.regCOMPUTE_PGM_RSRC1, data.rsrc1, data.rsrc2), wreg(ctx, ctx.gc.regCOMPUTE_PGM_RSRC3, data.rsrc3), wreg(ctx, ctx.gc.regCOMPUTE_TMPRING_SIZE, ctx.tmpring_size(data.private_segment_size))] - ins += [wreg(ctx, ctx.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO, *data64_le((scratch_addr + data.private_segment_size // ctx.xccs * xcc_id) >> 8)) + ins += [wreg(ctx, ctx.gc.regCOMPUTE_DISPATCH_SCRATCH_BASE_LO, (scratch_addr + data.private_segment_size // ctx.xccs * xcc_id) >> 8) for xcc_id in range(ctx.xccs)] ins += [wreg(ctx, ctx.gc.regCOMPUTE_RESTART_X, 0, 0, 0), wreg(ctx, ctx.gc.regCOMPUTE_USER_DATA_0, *user_regs), @@ -137,142 +144,95 @@ def pm4_program(ctx, call, prg): pkt3(ctx, PM4Ops.EVENT_WRITE, ctx.pm4.EVENT_TYPE(ctx.soc.CS_PARTIAL_FLUSH) | ctx.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))] return UOp(Ops.LINEAR, src=tuple(ins)) -pm_pm4_opsel = PatternMatcher([ +def pm4_ib(ctx, submit:UOp, lin:UOp) -> UOp|None: + # the ring only carries a packet pointing at the ib: the host fence at the start of the batch guarantees the ib is free to reuse + if lin.tag is not None or any(w.op in {Ops.CALL, Ops.INS, Ops.LINEAR, Ops.NOOP} for w in lin.src): return None # wait for the flat word linear + assert (size_dw:=sum(w.dtype.itemsize for w in lin.src) // 4) < (1 << 20), f"indirect buffer of {size_dw} dwords doesn't fit one packet" + pkt = (UOp.const(ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER, 2), dtypes.uint32), lin.rtag("indirect").getaddr(ctx.devs), + UOp.const(size_dw | ctx.pm4.INDIRECT_BUFFER_VALID, dtypes.uint32)) + return submit.replace(src=(UOp(Ops.LINEAR, src=pkt, arg=lin.arg).rtag(("cmdbuf", ctx.queue)),)) + +pm_pm4_encode = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), pm4_program), + (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), pm4_ib), - (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), (UPat(Ops.INS, arg=("barrier", dtypes.void)), pm4_barrier), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), pm4_wait), (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), pm4_timestamp), (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), pm4_store), ]) -def queue_ptrs(devs, qname:str, q:AMDQueueDesc) -> tuple[UOp, ...]: - return tuple(UOp.placeholder((b.size,), b.dtype, 0, device=devs).rtag(f"{qname}_{n}") - for n, b in (("ring", q.ring), ("write_ptr", q.write_ptr), ("doorbell", q.doorbell), ("put_value", q.put_value))) - -def pm4_submit(ctx, lin): - # ensure compute queues are allocated - for d in (devs:=ctx.devs): q = Device[d].compute_queue - ring, wptr, doorbell, put_ptr = queue_ptrs(devs, "COMPUTE:0", q) - - # the host fence at the start of the batch guarantees the ib is free to reuse - size_dw = sum(len(ins.src) for ins in lin.src) - assert size_dw < (1 << 20), f"indirect buffer of {size_dw} dwords doesn't fit one packet" - - ib = UOp.placeholder((size_dw,), dtypes.uint32, next(UOp.unique_num), device=devs, volatile=True).rtag("cmdbuf") - cmdbuf = make_cmdbuf(lin, devs, buf=ib) - - # the ring itself only carries a packet pointing at the ib, wrapping the ring - put = put_ptr.index(zero:=UOp.const(0, dtypes.int)) - pkt = (ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER, 2), *data64_le(cmdbuf.getaddr(devs)), size_dw | ctx.pm4.INDIRECT_BUFFER_VALID) - write_pkt = UOp.barrier(*[ring.index(((put + off) % q.ring.size).cast(dtypes.int)).store(UOp.const(x, dtypes.uint32)) for off,x in enumerate(pkt)]) - - # advance the put/write pointers past the packet - bump_put_ptr = put_ptr.index(zero).store(put + len(pkt)) - bump_wptr = wptr.index(zero).store(put + len(pkt)) - flush = UOp.barrier(write_pkt, bump_put_ptr, bump_wptr) - return doorbell.after(flush).index(zero).store(put + len(pkt)) - -pm_pm4_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), pm4_submit)]) - # ***************** # SDMA -class SDMAOps(FastEnum): COPY = auto(); POLL_REGMEM = auto(); FENCE = auto(); TRAP = auto(); TIMESTAMP = auto() # noqa: E702 - def sdma_copy(ctx, call): sz = call.src[2].max_numel() * call.src[2].dtype.itemsize hdr = ctx.sdma.SDMA_OP_COPY | ctx.sdma.SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_COPY_LINEAR) - return call.ins(SDMAOps.COPY, src=tuple(x for off in range(0, sz, ctx.max_copy_size) for x in ( + return UOp(Ops.LINEAR, src=tuple(x for off in range(0, sz, ctx.max_copy_size) for x in ( *(UOp.const(v, dtypes.uint32) for v in (hdr, ctx.sdma.SDMA_PKT_COPY_LINEAR_COUNT_COUNT(min(sz-off, ctx.max_copy_size)-1), 0)), *(a + UOp.const(off, dtypes.uint64) if off else a for a in (call.src[2].getaddr(ctx.devs), call.src[1].getaddr(ctx.devs)))))) -def sdma_wait(ctx, ins, dst, val): +def sdma_wait(ctx, dst, val): op = ctx.sdma.SDMA_OP_POLL_REGMEM | ctx.sdma.SDMA_PKT_POLL_REGMEM_HEADER_FUNC(WAIT_REG_MEM_FUNCTION_GEQ) \ | ctx.sdma.SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1) - return ins.ins(SDMAOps.POLL_REGMEM, src=tuple(UOp.const(x, dtypes.uint32) for x in ( - op, *data64_le(dst.getaddr(ctx.devs)), val, 0xffffffff, - ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff)))) + return UOp(Ops.LINEAR, src=( + UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs), val.cast(dtypes.uint32), UOp.const(0xffffffff, dtypes.uint32), + UOp.const(ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(0x04) | ctx.sdma.SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff), dtypes.uint32))) -def sdma_store(ctx, ins, dst, val): +def sdma_store(ctx, dst, val): # a fence packet then a trap op = ctx.sdma.SDMA_OP_FENCE | (ctx.sdma.SDMA_PKT_FENCE_HEADER_MTYPE(3) if ctx.target[0] != 9 else 0) - return UOp(Ops.LINEAR, src=( - ins.ins(SDMAOps.FENCE, src=tuple(UOp.const(x, dtypes.uint32) for x in (op, *data64_le(dst.getaddr(ctx.devs)), val))), - ins.ins(SDMAOps.TRAP, src=tuple(UOp.const(x, dtypes.uint32) for x in (ctx.sdma.SDMA_OP_TRAP, 0))))) + return UOp(Ops.LINEAR, src=(UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs), val.cast(dtypes.uint32), + UOp.const(ctx.sdma.SDMA_OP_TRAP, dtypes.uint32), UOp.const(0, dtypes.uint32))) -def sdma_timestamp(ctx, ins, dst): +def sdma_timestamp(ctx, dst): op = ctx.sdma.SDMA_OP_TIMESTAMP | ctx.sdma.SDMA_PKT_TIMESTAMP_GET_HEADER_SUB_OP(ctx.sdma.SDMA_SUBOP_TIMESTAMP_GET_GLOBAL) - return ins.ins(SDMAOps.TIMESTAMP, src=tuple(UOp.const(x, dtypes.uint32) for x in (op, *data64_le(dst.getaddr(ctx.devs))))) + return UOp(Ops.LINEAR, src=(UOp.const(op, dtypes.uint32), dst.getaddr(ctx.devs))) -pm_sdma_opsel = PatternMatcher([ +pm_sdma_encode = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY),), name="call", allow_any_len=True), sdma_copy), - (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.NOOP)), - (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val")), name="ins"), sdma_wait), - (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),), name="ins"), sdma_timestamp), - (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val")), name="ins"), sdma_store), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.LINEAR)), + (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), sdma_wait), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), sdma_timestamp), + (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), sdma_store), ]) -def sdma_submit(cmdbuf, devs): - # the cmdbuf to submit + the patch writes that fill it - size_dw, zero = cmdbuf.nbytes() // dtypes.uint32.itemsize, UOp.const(0, dtypes.int) - - # the sdma queue's ring and its host-side ring/write/put pointers - for d in devs: q = Device[d].sdma_queue(0) - ring, wptr, doorbell, put_ptr = queue_ptrs(devs, "COPY:0", q) - - # sdma needs the cmdbuf contiguous: if it won't fit before the ring end, restart at 0 and zero the tail - put_b = put_ptr.index(zero) - tail_off_dw = ((put_b % (q.ring.size * 4)) // 4).cast(dtypes.int) - fits = (size_dw <= q.ring.size - tail_off_dw).cast(dtypes.int) - start_dw = fits * tail_off_dw - zero_amt_dw = (1 - fits) * (q.ring.size - tail_off_dw) - - # zero the wrapped tail, then copy the cmdbuf into the ring - zi = UOp.range(zero_amt_dw, 0, dtype=dtypes.int, src=(cmdbuf,)) - zero_tail = ring.index(tail_off_dw + zi).store(UOp.const(0, dtypes.uint32)).end(zi) - i = UOp.range(UOp.const(size_dw, dtypes.int), 0, dtype=dtypes.int, src=(cmdbuf,)) - copy_to_ring = ring.index(start_dw + i).store(cmdbuf.index(i).load()).end(i) - - # advance the put/write pointers past the zeroed tail and the cmdbuf - next_put_b = put_b + ((zero_amt_dw + size_dw) * 4).cast(put_b.dtype) - bump_put_ptr = put_ptr.index(zero).store(next_put_b) - bump_wptr = wptr.index(zero).store(next_put_b) - - # ring the doorbell once the writes have landed - flush = UOp.barrier(zero_tail, copy_to_ring, bump_put_ptr, bump_wptr) - return doorbell.after(flush).index(zero).store(next_put_b) - -pm_sdma_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), - lambda ctx, lin: sdma_submit(make_cmdbuf(lin, ctx.devs), ctx.devs))]) - # ***************** -# USB submit - -def amd_usb_submit(ctx, lin): - for d in ctx.devs: q = Device[d].compute_queue if (comp:=ctx.qname.startswith("COMPUTE")) else Device[d].sdma_queue(0) - - if nb:=usb_arm_bytes(ctx.pre, Device[ctx.devs[0]].iface.usb_sram): - poke = (ctx.sdma.SDMA_OP_WRITE, *data64_le(Device[ctx.devs[0]].iface.cq_buf.va_addr + 12), 0, 0) - lin = lin.replace(src=lin.src + (UOp(Ops.INS, arg=("poke", dtypes.void), src=tuple(UOp.const(x, dtypes.uint32) for x in poke)),)) - - ib_host, ib_gpu, pkt_dw = usb_ib(ctx.devs, lin, 32 if comp else 0x100, nb) - pkt = (ctx.pm4.PACKET3(ctx.pm4.PACKET3_INDIRECT_BUFFER,2),*data64_le(ib_gpu.getaddr(ctx.devs)),pkt_dw|ctx.pm4.INDIRECT_BUFFER_VALID) if comp else () - return usb_push(ctx.devs, *queue_ptrs(ctx.devs, ctx.qname, q), ib_host, ib_gpu, pkt, 4 if comp else 1) - -pm_usb_submit = PatternMatcher([(UPat(Ops.LINEAR, name="lin"), amd_usb_submit)]) - -@dataclass(frozen=True) -class AMDEncodeCtx: # encode-time constants for one queue: devs (every cmdbuf address resolves into these) + gfx version + packet/ip modules - devs: tuple[str, ...]; target: tuple[int, ...]; pm4: Any; sdma: Any; soc: Any # noqa: E702 - gc: AMDIP; nbio: AMDIP; xccs: int; max_copy_size: int; tmpring_size: Callable; qname: str; pre: UOp # pre: the queue before opsel - -def encode_queue(q:UOp) -> UOp|None: - d = Device[(devs:=to_tuple(q.arg[0]))[0]] - ctx = AMDEncodeCtx(devs, d.target, d.pm4, d.sdma, d.soc, d.gc, d.nbio, d.xccs, d.max_copy_size, d.tmpring_size, q.arg[1], q) - opsel = pm_pm4_opsel if (comp:=q.arg[1].startswith("COMPUTE")) else pm_sdma_opsel - submit = d.pm_submit if d.pm_submit is not None else (pm_pm4_submit if comp else pm_sdma_submit) - return submit.rewrite(graph_rewrite(q, opsel + pm_flatten_linear, walk=True, ctx=ctx, name=f"{q.arg[1]} opsel"), ctx) +# queue submit + +def _queue_bufs(ctx, q:AMDQueueDesc) -> tuple[UOp, UOp, UOp, UOp]: + ring = UOp.placeholder((q.ring.size,), q.ring.dtype, 0, device=ctx.devs, volatile=True).rtag(f"{ctx.queue}_ring") + return (ring, *(make_buf(ctx.devs, tag=f"{ctx.queue}_{n}") for n in ("write_ptr", "doorbell", "put_value"))) + +def pm4_submit(ctx, cmdbuf:UOp) -> UOp: + for d in ctx.devs: q = Device[d].compute_queue + ring, wptr, doorbell, put = _queue_bufs(ctx, q) + p, size_dw = put.after(cmdbuf).index(0).load(), hcq_size_var(cmdbuf) // 4 + i = UOp.range(size_dw, 10, dtype=dtypes.int, src=(cmdbuf, ring)) + copy = ring.index(((p + i.cast(p.dtype)) % q.ring.size).cast(dtypes.int)).store(cmdbuf.bitcast(dtypes.uint32).index(i).load()).end(i) + next_put = p + size_dw.cast(p.dtype) + flush = UOp.barrier(copy, put.index(0).store(next_put), wptr.index(0).store(next_put)) + return doorbell.after(flush).index(0).store(next_put) + +def sdma_submit(ctx, cmdbuf:UOp) -> UOp: + # sdma needs the cmdbuf contiguous in the ring: if it won't fit before the ring end, restart at 0 and zero the tail + for d in ctx.devs: q = unwrap(Device[d].sdma_queue(int(ctx.queue.split(":")[1]))) + (ring, wptr, doorbell, put), rs = _queue_bufs(ctx, q), q.ring.size + size_dw = hcq_size_var(cmdbuf) // 4 + put_b = put.after(cmdbuf).index(0).load() + tail = ((put_b % (rs * 4)) // 4).cast(dtypes.int) + fits = (size_dw <= rs - tail).cast(dtypes.int) + start_dw, zero_amt = fits * tail, (1 - fits) * (rs - tail) + zi = UOp.range(zero_amt, 10, dtype=dtypes.int, src=(ring,)) + zero_tail = ring.index(tail + zi).store(UOp.const(0, dtypes.uint32)).end(zi) + i = UOp.range(size_dw, 11, dtype=dtypes.int, src=(cmdbuf, ring)) + copy = ring.index(start_dw + i).store(cmdbuf.bitcast(dtypes.uint32).index(i).load()).end(i) + next_put = put_b + ((zero_amt + size_dw) * 4).cast(put_b.dtype) + flush = UOp.barrier(zero_tail, copy, put.index(0).store(next_put), wptr.index(0).store(next_put)) + return doorbell.after(flush).index(0).store(next_put) + +pm_pm4_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), pm4_submit)]) +pm_sdma_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), sdma_submit)]) @dataclass(frozen=True) class AMDProgramData: @@ -280,9 +240,8 @@ class AMDProgramData: private_segment_size:int; kernargs_segment_size:int; kernargs_alloc_size:int enable_dispatch_ptr:int; enable_private_segment_sgpr:int -_amd_program_cache:dict[tuple[bytes, tuple[str, ...]], UOp] = {} -def amd_build_program(prg:UOp) -> UOp: - dev = Device[to_tuple(prg.device)[0]] # TODO: rm this +_amd_program_cache:dict[tuple[bytes, tuple[str, ...]], tuple[AMDProgramData, UOp]] = {} +def amd_build_program(dev, prg:UOp) -> tuple[AMDProgramData, UOp]: # key on the full device tuple: the same lib can be built for different device sets, each needs its own program buffer if (cached:=_amd_program_cache.get(key:=(lib:=prg.src[3].arg, to_tuple(prg.device)))) is None: image, sections, relocs = elf_loader(lib) @@ -303,7 +262,7 @@ def amd_build_program(prg:UOp) -> UOp: enable_private_segment_sgpr=desc.kernel_code_properties & hsa.AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER) image = bytes(image).ljust(round_up(len(image), 4), b"\x00") # the program is uploaded as whole dwords buf = UOp.placeholder((len(image),), dtypes.uint8, next(UOp.unique_num), device=prg.device).rtag("program") - cached = _amd_program_cache[key] = prg.replace(src=(buf.after(make_binary_patch(buf, image)),), arg=(data, prg.arg)) + cached = _amd_program_cache[key] = (data, buf.after(buf.store(UOp(Ops.BINARY, src=(), arg=image).bitcast(buf.dtype)))) return cached class AMDAllocator(HCQAllocator['AMDDevice']): @@ -587,17 +546,10 @@ def usb_sram(self) -> Buffer: def _mock(iface, name=None): return type(name or f"MOCK{iface.__name__}", (iface,), {}) class AMDDevice(HCQ2Compiled): - pm_lower = PatternMatcher([ - # prep program - (UPat(Ops.PROGRAM, src=(UPat(), UPat(), UPat(), UPat(Ops.BINARY)), name="prg"), amd_build_program), - - # encoding of cmdbuf - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue), - ]) - pm_submit: PatternMatcher|None = None - timestamp_divider = 100.0 # AMD GPU clock: ticks/us max_scratch_psize = 0 + pm_encode = {"COMPUTE": pm_pm4_encode, "COPY": pm_sdma_encode} + pm_lower = {"COMPUTE": pm_pm4_submit, "COPY": pm_sdma_submit} ifaces = [KFDIface, PCIIface, USBIface, _mock(KFDIface, "MOCKIface"), _mock(KFDIface), _mock(PCIIface), _mock(USBIface)] @@ -648,7 +600,7 @@ def __init__(self, device:str=""): if self.is_usb: self.pm_bufferize = pm_usb_bufferize + self.pm_bufferize - self.pm_stage_copy, self.pm_host_lower, self.pm_submit = pm_usb_stage, pm_usb_hostio, pm_usb_submit + raise NotImplementedError("usb amd is not migrated to sealed submits yet") # a usb pm_lower can override the whole submit graph self.pmc_enabled:bool = PROFILE > 0 and PMC > 0 if self.pmc_enabled: diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index 43c90f93c5dc0..364d02f0ec739 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -4,7 +4,7 @@ from tinygrad.device import Buffer from tinygrad.dtype import dtypes from tinygrad.helpers import HCQ2 -from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in +from tinygrad.runtime.support.hcq2 import HCQ_DEVS, all_devices_in, hcq_compile_cache, link_linear_cache @unittest.skipUnless(HCQ2 and all_devices_in(Device.DEFAULT, HCQ_DEVS), "hcq2 device required") class TestHCQ2(unittest.TestCase): @@ -29,5 +29,26 @@ def test_overlapping_device_tuples(self): out = Tensor.ones(8).shard(d2, axis=0).contiguous().realize() np.testing.assert_equal(out.numpy(), np.ones(8)) + def relowers(self, t:Tensor) -> tuple[int, int]: + # a miss in either cache relowers the whole submit, a hit costs ~0.1ms + before = (len(hcq_compile_cache), len(link_linear_cache)) + t.realize() + return (len(hcq_compile_cache) - before[0], len(link_linear_cache) - before[1]) + + def test_relower_only_on_new_kernel(self): + a, b = (Tensor.empty(64, 64).contiguous().realize() for _ in range(2)) + self.relowers(a.sin()) + self.assertEqual(self.relowers(a.sin()), (0, 0)) # nothing changed + self.assertEqual(self.relowers(b.sin()), (0, 0)) # new buffers, patched in at link time + self.assertEqual(self.relowers(a.cos()), (1, 1)) # new kernel, though only the code address moved + self.assertEqual(self.relowers(a.cos()), (0, 0)) + self.assertEqual(self.relowers(Tensor.empty(32, 32).contiguous().realize().sin()), (1, 1)) # new shape + + def test_dtype_sweep_relowers_every_dtype(self): + # test_dtype sweeps dtypes at one shape, so nearly every kernel is new: this is where hcq2 ci time goes + src = Tensor.empty(64, 64).contiguous().realize() + dts = (dtypes.int8, dtypes.uint8, dtypes.int16, dtypes.uint16, dtypes.int32) + self.assertEqual([self.relowers(src.cast(dt).contiguous())[0] for dt in dts], [1] * len(dts)) + if __name__ == "__main__": unittest.main() diff --git a/test/helpers.py b/test/helpers.py index 8bdd7d9bbf30d..3dadc0a7a762c 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -86,9 +86,10 @@ def assert_jit_cache_len(fxn, expected_len): if linear is None or not linear.src: if expected_len != 0: raise KernelCountException(expected_len, 0) return - if expected_len and all(call_is_hcq(call) for call in linear.src): # HCQ2: one batch submitter, or fence + reset + merged calls + finalizer - from tinygrad.runtime.support.hcq2 import HCQ_RUNTIME_DEV - expected_len = 1 if HCQ_RUNTIME_DEV.value == "CPU" else 4 + if expected_len and any(call_is_hcq(call) for call in linear.src): # HCQ2: kernels batch into submits, the finalizers carry the batch's kernels + count = sum(len(call.arg.aux.kernels) if call_is_hcq(call) else 1 for call in linear.src) + if count != expected_len: raise KernelCountException(expected_len, count) + return if call_is_graph(linear.src[0]): if len(linear.src) != 1: raise KernelCountException(1, len(linear.src)) inner = linear.src[0].src[0].src[0] # LINEAR UOp inside CUSTOM_FUNCTION diff --git a/tinygrad/device.py b/tinygrad/device.py index c21f7e275b395..22bf270856972 100644 --- a/tinygrad/device.py +++ b/tinygrad/device.py @@ -54,7 +54,7 @@ def _select_device(self) -> str: return device except StopIteration as exc: raise RuntimeError("no usable devices") from exc Device: _Device = _Device() -atexit.register(lambda: [Device[dn].finalize() for dn in Device._opened_devices]) +atexit.register(lambda: [Device[dn].finalize() for dn in tuple(Device._opened_devices)]) def canonicalize_device(device:str|tuple|list|None) -> str|tuple[str, ...]: if not isinstance(device, (tuple, list)): return Device.canonicalize(device) @@ -346,7 +346,8 @@ class Compiled: has_copy_queue:bool = True - pm_lower:Any = None + pm_encode:Any = None # per queue kind: queue ops -> flat command words + pm_lower:Any = None # per queue kind: custom_function(submit, cmdbuf) -> the queue push pm_bufferize:Any = None def __init__(self, device:str, allocator:Allocator, renderers:list[type[Renderer]], runtime:type[Program[Self]]|None, graph=None, arch=None): diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index baac566d768ef..e825bcb7f24e8 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -2,11 +2,10 @@ from typing import cast, Iterator, Any, Sequence import random, itertools, math, weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, to_tuple, tqdm, dedup +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, to_tuple, tqdm, dedup, to_mv from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, HCQ2, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry -from tinygrad.dtype import dtypes from tinygrad.renderer import Estimates, Renderer from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context from tinygrad.codegen.opt.postrange import args_from_ast @@ -169,7 +168,9 @@ def unwrap_multi(call:UOp, resolved:list[UOp]) -> Iterator[tuple[list[Buffer], d # the DEVICE axis is bound per device at launch: it's a RANGE in the AST and the _device_num variable after codegen has_dnum = any((x.op is Ops.RANGE and x.arg[-1] is AxisType.DEVICE) or (x.op is Ops.PARAM and x.arg.name == '_device_num') for x in call.src[0].toposort()) - for j, per_dev in enumerate(zip(*[cast(MultiBuffer, b).bufs for b in bufs])): yield list(per_dev), {"_device_num": j} if has_dnum else {} + lanes = max(len(b.bufs) for b in bufs if isinstance(b, MultiBuffer)) # a single buffer is shared by every lane + per_lane = [b.bufs if isinstance(b, MultiBuffer) else (b,)*lanes for b in bufs] + for j, per_dev in enumerate(zip(*per_lane)): yield list(per_dev), {"_device_num": j} if has_dnum else {} def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: for bufs, device_vars in unwrap_multi(call, resolve_params(call, ctx.input_uops)): @@ -183,10 +184,10 @@ def exec_copy(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: else: dest.allocator._copyin(dest._buf, src.as_memoryview(allow_zero_copy=True)) return [] -def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: +def exec_kernel(ctx:ExecContext, call:UOp, ast:UOp, devices=None) -> list[float|None]: ets:list[float|None] = [] resolved = resolve_params(call, ctx.input_uops) - for device, (bufs, device_vars) in zip(to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): + for device, (bufs, device_vars) in zip(devices or to_tuple(call.src[1].device), unwrap_multi(call, [resolved[i] for i in ast.arg.globals])): var_vals = {**ctx.var_vals, **device_vars} prg_bufs = [b.ensure_allocated() for b in bufs] rt = get_runtime(device, ast, cache=ctx.cache) @@ -216,14 +217,17 @@ def exec_graph(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: return [get_graph_runtime(ast, ctx.input_uops)(ctx.input_uops, ctx.var_vals, wait=ctx.wait)] def exec_hcq(ctx:ExecContext, call:UOp, ast:UOp) -> list[float|None]: - dev = cast(Any, Device[(info:= call.arg.aux).device[0]]) - addrs = [cast(Buffer, _resolve(u, ctx.input_uops).buffer).get_buf(d).va_addr for d, u in info.input_addrs] - dev.rt_buffer()._buf.cpu_view().view(offset=(base:=dev.rt_allocator.alloc(len(addrs) * 8)), fmt='Q')[:len(addrs)] = array.array('Q', addrs) + info = call.arg.aux + assert len(ast.arg.globals) == len(info.args), f"{call.arg.name}: an arg is dead in the rendered body, the args after it would mis-bind" - if info.inputs is not None: - table = UOp.from_buffer(dev.rt_buffer().view(len(info.input_addrs), dtypes.uint64, base), HCQ_RUNTIME_DEV.value) - call = call.substitute({call.src[1+info.inputs]: UOp.mstack(*[table]*len(info.device))}) - exec_kernel(replace(ctx, var_vals={**ctx.var_vals, "hcq_inputs_ptr": dev.rt_buffer()._buf.va_addr + base}), call, ast) + # fill the inputs table with the address of every input the sealed cmdbufs reference + if info.table is not None: + addrs = [cast(Buffer, _resolve(_lane(u, lane), ctx.input_uops).buffer).get_buf(dev).va_addr for u, lane, dev in info.inputs] + tab = cast(Buffer, call.src[info.table].without_after.buffer) + to_mv(tab._buf.va_addr, len(addrs) * 8).cast('Q')[:] = array.array('Q', addrs) + + # every lane's body runs on the runtime device, info.device is only the lane count + exec_kernel(replace(ctx, var_vals={**ctx.var_vals, **dict(info.vals)}), call, ast, devices=(HCQ_RUNTIME_DEV.value,)*len(info.device)) def _prof_tm(device:str, name:str, prof:tuple[int, ...], profile_key:bytes) -> float|None: (d:=cast(Any, Device[device])).prof_ents[prof[0]] = ProfileGraphEntry(device, name, prof[0], prof[1], profile_key) @@ -305,7 +309,7 @@ def lower_and_compile(linear:UOp) -> UOp: (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="validate", name="ast"),), name="call", allow_any_len=True), exec_validate), ]) -from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize +from tinygrad.runtime.support.hcq2 import hcq_compile, hcq_link, _lane, HCQ_RUNTIME_DEV # noqa: E402 # down here, hcq2 imports realize def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:list[UOp]|None=None, profile:bool|None=None) -> UOp: if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 6ac0d650d2fa7..048c23ce117bc 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -2,10 +2,10 @@ import platform, sys, os, ctypes, functools, mmap, threading, array, struct, time from dataclasses import dataclass, replace from typing import cast, Callable -from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le, to_tuple +from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device from tinygrad.runtime.support.hcq import HCQBuffer, MMIOInterface -from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_cmdbuf, make_buf +from tinygrad.runtime.support.hcq2 import HCQ2Compiled, HCQAllocator, make_buf, hcq_size_var from tinygrad.runtime.support.c import DLL from tinygrad.renderer.cstyle import ClangRenderer from tinygrad.renderer.llvmir import CPULLVMRenderer @@ -14,10 +14,10 @@ from tinygrad.runtime.support.elf import jit_loader from tinygrad.runtime.autogen import libc from tinygrad.codegen import do_to_program -from tinygrad.engine.realize import pm_flatten_linear, get_call_arg_uops, get_call_var_uops, get_runtime +from tinygrad.engine.realize import get_call_arg_uops, get_call_var_uops, get_runtime from tinygrad import UOp, dtypes from tinygrad.dtype import AddrSpace -from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher, graph_rewrite +from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher MAX_ARGS, CMD_SIZE, RING_SLOTS, FUNCS = 63, 64, (16 << 10), (() if WIN else ('clock_gettime', 'sem_wait', 'sem_post')) @@ -62,50 +62,51 @@ def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: progs = [get_runtime(d, prog) if isinstance(prog, UOp) else cast(CPUDevice, Device[d]).prgs[prog] for d in devs] addrs = tuple(UOp.const(p.addr, dtypes.uint64) for p in progs) words = ((addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs)),) + args - return UOp(Ops.INS, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words)), arg=("cmd", dtypes.void)) + return UOp(Ops.LINEAR, src=words + (UOp.const(0, dtypes.uint64),) * (CMD_SIZE - len(words))) -def cpu_exec(ctx:tuple[str, ...], call:UOp, prg:UOp) -> UOp: - args = [get_call_arg_uops(call)[i].getaddr(ctx) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] - if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(ctx, prg, *args) +def cpu_exec(ctx, call:UOp, prg:UOp) -> UOp: + devs = ctx.devs + args = [get_call_arg_uops(call)[i].getaddr(devs) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] + if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(devs, prg, *args) - la = [cpu_cmd(ctx,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t, dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] + la = [cpu_cmd(devs,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t,dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] return UOp(Ops.LINEAR, src=tuple(la)) pm_cpu_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), - (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.NOOP)), + (UPat(Ops.INS, arg=("barrier", dtypes.void)), lambda: UOp(Ops.LINEAR)), (UPat(Ops.INS, arg=("wait", dtypes.void), src=(UPat(name="dst"), UPat(name="val"))), - lambda ctx, dst, val: cpu_cmd(ctx, wait_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), + lambda ctx, dst, val: cpu_cmd(ctx.devs, wait_prog, dst.getaddr(ctx.devs), val.cast(dtypes.uint64))), (UPat(Ops.INS, arg=("store", dtypes.void), src=(UPat((Ops.BUFFER, Ops.PARAM), name="dst"), UPat(name="val"))), - lambda ctx, dst, val: cpu_cmd(ctx, signal_prog, dst.getaddr(ctx), val.cast(dtypes.uint64))), - (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), - lambda ctx, dst: cpu_cmd(ctx, timestamp_prog, dst.getaddr(ctx), *(() if WIN else (make_buf(ctx, tag="func:clock_gettime").getaddr(ctx),)))), + lambda ctx, dst, val: cpu_cmd(ctx.devs, signal_prog, dst.getaddr(ctx.devs), val.cast(dtypes.uint64))), + (UPat(Ops.INS, arg=("timestamp", dtypes.void), src=(UPat(name="dst"),)), lambda ctx, dst: cpu_cmd(ctx.devs, timestamp_prog, dst.getaddr(ctx.devs), + *(() if WIN else (make_buf(ctx.devs, tag="func:clock_gettime").getaddr(ctx.devs),)))), ]) -def encode_queue(q:UOp) -> UOp: - devs, queue = to_tuple(q.arg[0]), q.arg[1] - lin = graph_rewrite(q, pm_cpu_opsel+pm_flatten_linear, ctx=devs, walk=True, name=f"{queue} opsel") - - cnt = sum(len(ins.src) for ins in lin.src) // CMD_SIZE - assert cnt < RING_SLOTS, f"submit of {cnt} entries doesn't fit the ring" - cmdbuf = make_cmdbuf(lin, devs, buf=UOp.placeholder((cnt*CMD_SIZE,), dtypes.uint64, next(UOp.unique_num), device=devs).rtag("cmdbuf")) - ring = UOp.placeholder((ring_words:=RING_SLOTS*CMD_SIZE,), dtypes.uint64, 0, device=devs, volatile=True).rtag(f"{queue}_ring") - put, done, sem, sysbuf = (make_buf(devs, tag=f"{queue}_{name}") for name in ("put", "done", "sem", "sys")) +def cpu_submit(ctx, cmdbuf:UOp) -> UOp: + # copy the cmd entries into the worker ring and post the semaphore once per entry + assert ctx.nbytes % (CMD_SIZE * 8) == 0 and ctx.nbytes // (CMD_SIZE * 8) < RING_SLOTS, f"submit of {ctx.nbytes} bytes doesn't fit the ring" + devs, cnt, cb = ctx.devs, hcq_size_var(cmdbuf) // (CMD_SIZE * 8), cmdbuf.bitcast(dtypes.uint64) + ring, put, done, sem = (make_buf(devs, tag=f"{ctx.queue}_{n}") for n in ("ring", "put", "done", "sem")) # submits are serialized on the submitter, so they can bump put without atomics - ran = done.after(l:=UOp.loop(next(UOp.unique_num))).index(0).load() - room = ran.end(l, put.index(0).load() - ran > RING_SLOTS - cnt) # wait until cnt entries fit in the ring - base = ((put.after(room).index(0).load() % RING_SLOTS) * CMD_SIZE).cast(dtypes.int) - e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(cmdbuf, ring)) - copy = UOp.group(*[ring.index((base + e*CMD_SIZE + w) % ring_words).store(cmdbuf.index(e*CMD_SIZE + w).load()) for w in range(CMD_SIZE)]) - - bumped = put.after(copy.end(e)).index(0).store(put.index(0).load() + cnt) - if WIN: return sysbuf.after(bumped).index(0).store(put.after(bumped).index(0).load()) - - e = UOp.range(cnt, next(UOp.unique_num), dtype=dtypes.int, src=(bumped,)) + ran = done.after(l:=UOp.loop(10)).index(0).load() + room = ran.end(l, put.index(0).load() - ran > (RING_SLOTS - cnt).cast(ran.dtype)) # wait until cnt entries fit in the ring + base = ((put.after(room, cmdbuf).index(0).load() % RING_SLOTS) * CMD_SIZE).cast(dtypes.int) + e = UOp.range(cnt, 11, dtype=dtypes.int, src=(cmdbuf, ring)) + # the slot is a multiple of CMD_SIZE, so a word can never wrap on its own: take the modulo once per entry, not per word + slot = (base + e*CMD_SIZE) % (RING_SLOTS * CMD_SIZE) + w = UOp.range(CMD_SIZE, 12, dtype=dtypes.int, src=(cmdbuf, ring)) + copy = ring.index(slot + w).store(cb.index(e*CMD_SIZE + w).load()).end(w) + + bumped = put.after(copy.end(e)).index(0).store(put.index(0).load() + cnt.cast(dtypes.uint64)) + if WIN: return make_buf(devs, tag=f"{ctx.queue}_sys").after(bumped).index(0).store(put.after(bumped).index(0).load()) + e = UOp.range(cnt, 13, dtype=dtypes.int, src=(bumped,)) return make_buf(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) +pm_cpu_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), cpu_submit)]) + # ***************** # NOTE: MAP_JIT is added to mmap module in python 3.13 @@ -194,7 +195,7 @@ def _unmap(self, mb): pass # CPU _do_map returns a view wrapper, nothing to rel class CPUDevice(HCQ2Compiled): wait_timeout_ms, has_copy_queue = 30000, False - pm_lower = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="q"),)), encode_queue)]) + pm_encode, pm_lower = {"COMPUTE": pm_cpu_opsel, "SUBMIT": pm_cpu_opsel}, {"COMPUTE": pm_cpu_submit, "SUBMIT": pm_cpu_submit} def __init__(self, device:str=""): self.workers:list[CPUWorker] = [] @@ -202,8 +203,8 @@ def __init__(self, device:str=""): arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") self.pm_bufferize = PatternMatcher( - [(UPat(Ops.PARAM, tag=f"{q}_{n}"), lambda ctx, q=q,n=n: getattr(ctx[0].worker(q), n)) - for q in ("COMPUTE:0", "SUBMIT:0") for n in ("ring", "put", "sem", "sys", "done")] + + [(UPat(Ops.PARAM, tag=f"{q}_{n}"), lambda ctx, q=q, n=n: getattr(ctx[0].worker(q), n)) + for q in ("COMPUTE:0", "SUBMIT:0") for n in ("ring","put","sem","sys","done")] + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): diff --git a/tinygrad/runtime/ops_python.py b/tinygrad/runtime/ops_python.py index a5d532ead263c..ebe40f4e8e3eb 100644 --- a/tinygrad/runtime/ops_python.py +++ b/tinygrad/runtime/ops_python.py @@ -15,7 +15,8 @@ def _load(m, i, dtype: DType): if i is None: return 0.0 if i < 0 or i >= len(m): raise IndexError(f"load out of bounds, size is {len(m)} and access is {i}") - return from_storage_scalar(m[i], dtype) + if (w:=m.nbytes // len(m)) >= dtype.itemsize: return from_storage_scalar(m[i], dtype) + return sum(m[i+k] << (8*w*k) for k in range(dtype.itemsize // w)) # a bitcast can read wider than the buffer, _store splits it the same way def load(inp, j, dtype: DType): if len(inp) >= 3: return [_load(m, x+j if x is not None else None, dtype) if gate else default for (m,x),default,gate in zip(*inp[:3])] diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index f447b90fb3b08..af3ad32a63a54 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -1,17 +1,18 @@ from __future__ import annotations -from typing import cast, TypeVar, Generic, Any, Sequence, Iterable, TYPE_CHECKING +from typing import cast, TypeVar, Generic, Any, Sequence, TYPE_CHECKING import struct, functools, time, collections, itertools, decimal, statistics from dataclasses import replace, dataclass, field -from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE -from tinygrad.helpers import to_tuple, round_up, partition, panic, ContextVar, perf_counter_us, Context +from tinygrad.helpers import suppress_finalizing, dedup, pluralize, JIT_BATCH_SIZE, unwrap, PROFILE, all_same, all_int +from tinygrad.helpers import to_tuple, ContextVar, perf_counter_us, Context, panic, partition, round_up, flatten, next_power2 from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator, MultiBuffer, DepsTracker from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp from tinygrad.uop.symbolic import symbolic -from tinygrad.dtype import dtypes, truncate, DType +from tinygrad.dtype import dtypes, DType, AddrSpace from tinygrad.runtime.support.memory import BumpAllocator, MMIOInterface from tinygrad.renderer import Renderer, Estimates -from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop, pm_flatten_linear,lower_and_compile +from tinygrad.engine.realize import to_program, get_call_arg_uops, get_call_name, get_call_outs_ins, estimate_uop, pm_flatten_linear +from tinygrad.engine.realize import lower_and_compile if TYPE_CHECKING: from tinygrad.runtime.support.hcq import HCQBuffer # TODO: remove that @@ -19,76 +20,41 @@ # 0. helpers HCQDeviceType = TypeVar('HCQDeviceType', bound='HCQ2Compiled') - HCQ_RUNTIME_DEV = ContextVar("HCQ_RUNTIME_DEV", "CPU") - HCQ_DEVS = frozenset(("AMD", "CPU")) -HCQ_CACHE_TAGS = frozenset(("program", "systems")) @dataclass(frozen=True) class HCQInfo: device:tuple[str, ...] estimates:Estimates = Estimates() - inputs:int|None = None - input_addrs:tuple[tuple[str, UOp], ...] = () # (device, lane arg uop) kernels:tuple[tuple[tuple[str, ...], str, Estimates, tuple[int, ...], bytes], ...] = () + args:tuple[tuple[UOp, UOp], ...] = () # placeholder -> the canonical body param it becomes, in call src order + table:Any = None # the inputs table placeholder (its src position after lower) + inputs:tuple[tuple[UOp, int, str], ...] = () # per table slot: (src, lane, device) the exec resolves to an address + vals:tuple[tuple[str, int], ...] = () # bound values of the body variables (the queue byte size, table slots) + def all_devices_in(d:Any, c:frozenset[str]) -> bool: return {x.split(":")[0] for x in to_tuple(d)} <= c def unwrap_mstack(u:UOp) -> tuple[UOp, ...]: if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s)) return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,) -def unwrap_view(v:UOp) -> tuple[UOp, int]: - return unwrap_view(v.src[0]) if v.op is Ops.BITCAST else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) +def unwrap_view(v:UOp) -> tuple[UOp, int]: # look through views to (base, element offset) + return unwrap_view(v.src[0]) if v.op in (Ops.BITCAST, Ops.AFTER) else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0) def _lane(u:UOp, lane:int) -> UOp: return u.src[lane] if u.op is Ops.MSTACK else u.mselect(lane) if len(to_tuple(u.device)) > 1 else u -# patches - -def is_value_known_at_link(val:UOp) -> bool: - runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)] - addressed_bufs = [b for g in val.toposort() if g.op is Ops.GETADDR for b in unwrap_mstack(g.buf_uop)] - - # addr of input params is not known at link time - return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs) - -def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]: - # group patches into stacks: (tag, type, offset). offset is used for shrink later - groups:dict[tuple[str|None, DType, sint], list[tuple[sint, UOp]]] = collections.defaultdict(list) - for off, val in patches: - tag = "link" if is_value_known_at_link(val) else "inputs" if val.op is Ops.GETADDR else None - groups[(tag, (v:=(val.bitcast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val)).dtype, off % v.dtype.itemsize)].append((off, v)) - - ret, bit = [], buf.dtype.itemsize - for (tag, dt, r), ps in groups.items(): - view = buf.shrink(((r // bit, (max(off for off,_ in ps) + dt.itemsize) // bit),)).bitcast(dt) - offs = UOp(Ops.STACK, src=tuple(UOp.const((off - r) // dt.itemsize, dtypes.int) for off,_ in ps)) - ret.append(view.index(offs).store(UOp(Ops.STACK, src=tuple(val for _,val in ps))).rtag(tag)) - return tuple(ret) - -def make_binary_patch(buf:UOp, blob:bytes) -> UOp: return buf.store(UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype)).rtag("link") - -def make_cmdbuf(lin, devs, buf:UOp|None=None, dep:tuple[UOp, ...]=()): - blob, patches = bytearray(), [] - for s in (s for ins in lin.src for s in ins.src): - if not (is_const:=(s.op is Ops.CAST and s.src[0].op is Ops.CONST)): patches.append((len(blob), s)) - blob.extend(struct.pack(f'<{s.dtype.fmt}', s.val if is_const else 0x0)) - cmdbuf = buf if buf is not None else UOp.placeholder((len(blob) // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("cmdbuf") - return cmdbuf.after(*dep, make_binary_patch(cmdbuf, bytes(blob)), *make_patches(cmdbuf, patches)) - def make_submit(*cmds, devs:str|tuple[str, ...], queue:str) -> UOp: return UOp.custom_function("submit_cmdbuf", UOp(Ops.LINEAR, src=tuple(cmds), arg=(to_tuple(devs), queue))) -def get_submit(ast:UOp) -> UOp: return next(u for u in ast.toposort() if u.op is Ops.CUSTOM_FUNCTION and u.arg == "submit_cmdbuf") +def get_submit(ast:UOp) -> UOp|None: + return next((u for u in ast.toposort() if u.op is Ops.CUSTOM_FUNCTION and u.arg == "submit_cmdbuf"), None) def make_call(name:str, body:UOp, info:HCQInfo) -> UOp: return UOp.custom_function("hcq", body).call(name=name, aux=info) -def encode_kernargs_clike(call:UOp, prg:UOp, devs:str|tuple[str, ...]) -> UOp: - data, info = prg.arg - buf = UOp.placeholder((data.kernargs_alloc_size // 4,), dtypes.uint32, next(UOp.unique_num), device=devs).rtag("kernargs") - words = [get_call_arg_uops(call)[gi].getaddr(devs) for gi in info.globals] + list(info.vars) - return buf.after(*make_patches(buf, list(zip(itertools.accumulate((w.dtype.itemsize for w in words), initial=0), words)))) +def hcq_size_var(cmdbuf:UOp) -> UOp: # the sealed byte count, bounded by the cmdbuf it walks so the submit copy loops stay in bounds + return UOp.variable("hcq_size", 0, cmdbuf.max_numel() * cmdbuf.dtype.itemsize, dtypes.uint32, param=True) def make_buf(devs, slot:int=0, tag:str="signal") -> UOp: return UOp.placeholder((1,), dtypes.uint64, slot, device=devs, volatile=True, tag=tag) @@ -193,10 +159,10 @@ def _wait_ins(ctx:BatchCtx, bufs_by_lane:list[list[Any]], write, devices:tuple[s ctx.signal_tags |= {t for _, t in rows} return waits -def _merge_submits(calls:list[UOp]) -> UOp: # TODO: simplify? +def _merge_submits(calls:list[UOp]) -> UOp: if len(calls) == 1: return calls[0] - devs, queue = get_submit(calls[0]).src[0].arg - body = make_submit(*[cmd for c in calls for cmd in get_submit(c).src[0].src], devs=devs, queue=queue).sink() + devs, queue = unwrap(get_submit(calls[0])).src[0].arg + body = make_submit(*[cmd for c in calls for cmd in unwrap(get_submit(c)).src[0].src], devs=devs, queue=queue).sink() return make_call(f"submit {queue} ({len(calls)})", body, replace(calls[0].arg.aux, estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) @@ -205,7 +171,7 @@ def _merge_queues(submits:list[UOp]) -> list[UOp]: opened:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> hcq calls in submit order limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) for call in submits: - devs, queue = key = get_submit(call).src[0].arg + devs, queue = key = unwrap(get_submit(call)).src[0].arg if (group:=opened.pop(key, None)) is None: # first submit on this queue: close open groups on the same queue with shared devices, so submit order is kept for k in [k for k in opened if k[1] == queue and set(k[0]) & set(devs)]: merged.append(_merge_submits(opened.pop(k))) @@ -236,7 +202,8 @@ def _make_finalizers(ctx:BatchCtx) -> tuple[list[UOp], list[UOp], list[UOp]]: for q in qs: rst += (make_buf(devs, ctx.slots[q]).after(*rst[-1:]).index(0).store(0),) if rst: resets.append(make_call("hcq_reset", UOp.sink(*rst), HCQInfo(devs))) - # finalizer: bump the host timeline and remember this schedule's epoch for the next fence + # finalizer: the submit stores the host timeline into the device timeline signal, then the host bumps the + # timeline and remembers this schedule's epoch for the next fence waits = _wait_ins(ctx, [list(dev_bufs[d].values()) for d in devs], None, devs, "COMPUTE:0", n) fin_submit = make_submit(*waits, UOp(Ops.INS, arg=("store", dtypes.void), src=(tl_signal, tl_value.index(0))), devs=devs, queue="COMPUTE:0") epoch = (epoch_slot:=tl_value.after(fin_submit).index(0)).load() @@ -276,7 +243,7 @@ def _finalize_batch(batch:list[tuple[UOp, tuple[str, ...]]], profile:bool) -> li fences, resets, fins = _make_finalizers(ctx) submits, kerns = _emit_submits(ctx, call_waits) - # append batch timestamps to finalizers + # append batch kernels to the finalizers, their exec collects the profiles after everything is in flight fins = [f.replace(arg=replace(f.arg, aux=replace(a:=f.arg.aux, kernels=tuple(x for x in kerns if set(x[0]) & set(a.device))))) for f in fins] return fences + resets + _merge_queues(submits) + fins @@ -290,181 +257,201 @@ def sched_batches(l:UOp, profile:bool) -> UOp: return l.replace(src=tuple(srcs + _finalize_batch(batch, profile))) # ***************** -# 4.2. hcq lowering: ops to ir - -def encode_host_call(call:UOp) -> UOp|None: - if (pm:=getattr(Device[call.arg.aux.device[0]], "pm_host_lower", None)) is None: return None - body = graph_rewrite(call.src[0], pm, name="lower host access", enter_calls=True) - return None if body is call.src[0] else call.replace(src=(body, *call.src[1:])) - -def encode_cmdbuf(submit:UOp, lin:UOp) -> UOp|None: - if (pm:=Device.get_class(lin.arg[0][0]).pm_lower) is None: return None - return graph_rewrite(submit, pm, name=f"encode {lin.arg[0]}", enter_calls=True) -pm_encode_cmdbufs = PatternMatcher([ - (UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(Ops.LINEAR, name="lin"),), name="submit"), encode_cmdbuf), - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), encode_host_call)]) +# 3. encode: the backend rewrites the ops of every submit into flat command words. a word is a const, a uop the +# link or the exec resolves to a value, or a getaddr of a nested LINEAR (an indirect blob like kernargs) + +class EncodeCtx: # devs/queue (and for the submit lowering, the sealed byte count) plus everything on the device + def __init__(self, dev, devs:tuple[str, ...], queue:str, nbytes:int=0): self.dev, self.devs, self.queue, self.nbytes = dev, devs, queue, nbytes + def __getattr__(self, name): return getattr(self.dev, name) + +def encode_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None or (lin:=submit.src[0]).op is not Ops.LINEAR: return None + if not any(w.op in {Ops.INS, Ops.CALL} for w in lin.src): return None # already flat words + devs, queue = lin.arg + ctx = EncodeCtx(dev:=Device[devs[0]], devs, queue) + body = graph_rewrite(call.src[0], dev.pm_encode[queue.split(":")[0]] + pm_flatten_linear, ctx=ctx, name=f"encode {queue}") + return call.replace(src=(body, *call.src[1:])) if body is not call.src[0] else None +pm_encode = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), encode_call)]) # ***************** +# 4. seal: pack the words into one blob per linear tag. consts bake into the blobs, link values fold in at link time, +# and the rest the body stores each call from the exec-filled inputs table and the link-filled offset table. +# the submit keeps only its buffer: custom_function(submit) -> placeholder + +def is_link_value(w:UOp) -> bool: # resolvable when the linear links: no variables or memory reads, no input-param or program addresses + if w.op is Ops.GETADDR: return all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(w.buf_uop)) + if w.op in {Ops.LOAD, Ops.INDEX} or w.is_variable or (w.op is Ops.PARAM and w.arg.addrspace is AddrSpace.ALU): return False + return all(is_link_value(s) for s in w.src) + +def blobify(ph:UOp, data:bytes, stores:Sequence[tuple[int, UOp]]=()) -> UOp: + binary = ph.store(UOp(Ops.BINARY, src=(), arg=data).bitcast(ph.dtype)) + return ph.after(binary, *[ph.shrink(((o, o + w.dtype.itemsize),)).bitcast(w.dtype).index(0).store(w) for o, w in stores]) + +def seal_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None or (lin:=submit.src[0]).op is not Ops.LINEAR: return None + devs, queue = lin.arg + + # pack every linear into its own blob + patches:list[tuple[UOp, int, UOp]] = [] # (linear, local byte offset, word) + blobs:dict[UOp, bytearray] = {l: bytearray() for l in [lin] + [u for u in lin.toposort() if u.op is Ops.LINEAR and u is not lin]} + + for l, blob in blobs.items(): + for w in l.src: + if is_uop:=isinstance(c:=w.ssimplify(), UOp): patches.append((l, len(blob), w)) # the original w: simplify can rebuild nested linears + blob += (b"\xbe" * w.dtype.itemsize) if is_uop else cast(int, c).to_bytes(w.dtype.itemsize, 'little') + + # merge blobs based on tags + tags, offs, datas = {l: l.tag or ("cmdbuf", queue) for l in blobs}, {}, collections.defaultdict[Any, bytearray](bytearray) + for l, b in blobs.items(): + offs[l] = len(datas[tags[l]]) + datas[tags[l]] += b.ljust(round_up(len(b), 128), b"\xbf") + + bufs = {t: UOp.placeholder((next_power2(len(d)),), dtypes.uint8, next(UOp.unique_num), device=devs).rtag(t) for t, d in datas.items()} + views = {l: bufs[tags[l]][offs[l]:offs[l] + len(blobs[l])] for l in blobs} + + # place the words in the merged blobs, then split: link words fold at link time, runtime words the body stores every call + placed = UOp.sink(*[w for _, _, w in patches]).substitute(views).src + links, runtime = partition([(bufs[tags[l]], offs[l] + o, w) for (l, o, _), w in zip(patches, placed)], lambda p: is_link_value(p[2])) + + rt_sink = UOp.sink(*[w for _, _, w in runtime]) + rt_vars = {u: u.src[0] for u in rt_sink.toposort() if u.is_bound_var} + + # all getaddrs are one input table. the body walks it on the host, so it's a CPU buffer: an emulated runtime device has no memory of its own + gaddrs = dedup([g for g in rt_sink.toposort() if g.op is Ops.GETADDR]) + table_srcs = dedup([g.src[0].without_after for g in gaddrs]) + slots = {src: i * len(devs) for i, src in enumerate(table_srcs)} + table = UOp.placeholder((tsz:=next_power2(len(slots)*len(devs)),), dtypes.uint64, next(UOp.unique_num), device="CPU").rtag("inputs") + dvar = UOp.variable("_device_num", 0, len(devs) - 1, dtypes.int, param=True) if len(devs) > 1 else UOp.const(0, dtypes.int) + reads = {g: table.index(slots[g.src[0].without_after] + dvar).load() for g in gaddrs} + + # group rt-patches by target and uop + groups:dict[tuple[UOp, UOp], list[tuple[int, int]]] = collections.defaultdict(list) + for (buf, off, w), v in zip(runtime, rt_sink.substitute(reads | rt_vars).src): + if w.op is Ops.GETADDR: groups[(buf, table)].append((off, slots[w.src[0].without_after])) + else: groups[(buf, v)].append((off, 0)) + + stores, vals, base = [], [], UOp.const(0, dtypes.int) + offtbl = UOp.placeholder((osz:=next_power2(2 * len(runtime)),), dtypes.uint32, next(UOp.unique_num), device="CPU").rtag("offtbl") + for j, ((buf, v), grp) in enumerate(groups.items()): + n = UOp.variable(f"hcq_off_len{j}", 0, osz // 2, dtypes.uint32, param=True) + vals.append((n.arg.name, len(grp))) + + # the lens sum to the entry count, so both masks are no-ops. they just keep the table indices provably in bounds + r = UOp.range(n, 20 + j, dtype=dtypes.int, src=(buf,)) + ent = 2 * ((base + r) & (osz // 2 - 1)) + off = offtbl.index(ent).load().cast(dtypes.int) + val = table.index((offtbl.index(ent + 1).load().cast(dtypes.int) + dvar) & (tsz - 1)).load() if v is table else v # reindex table + stores.append(buf.shrink(((off, off + val.dtype.itemsize),)).bitcast(val.dtype).index(0).store(val).end(r)) + base = base + n.cast(dtypes.int) + + patched = bufs[tags[lin]].after(*stores) + body = call.src[0].substitute({submit: submit.replace(src=(patched,))}) + + # link-time patches are just stores + link_stores:dict[UOp, list[tuple[int, UOp]]] = collections.defaultdict(list) + for b, off, w in links: link_stores[b].append((off, w)) + + # blobs, blobs + data_blobs = [blobify(offtbl, struct.pack(f"<{2*len(runtime)}I", *flatten(flatten(groups.values()))))] if runtime else [] + link_blobs = [blobify(bufs[t], bytes(d), link_stores[bufs[t]]) for t, d in datas.items()] + prog_blobs = dedup([u for u in UOp.sink(*[w for _, _, w in links]).toposort() if u.op is Ops.AFTER]) + info = replace(call.arg.aux, table=table if table_srcs else call.arg.aux.table, vals=(("hcq_size", len(blobs[lin])), *vals), + inputs=call.arg.aux.inputs + tuple((src, lane, dev) for src in table_srcs for lane, dev in enumerate(devs))) + return call.replace(src=(body, *dedup([*call.src[1:], *link_blobs, *data_blobs, *prog_blobs])), arg=replace(call.arg, aux=info)) + +pm_seal = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), seal_call)]) -def get_getaddrs(p:UOp) -> list[UOp]: return [u for u in p.toposort(gate=lambda u: u.op is not Ops.AFTER) if u.op is Ops.GETADDR] - -def trim_link_patches(ctx:tuple[list[UOp], list[UOp]], a:UOp) -> UOp|None: - links, kept = partition(a.src[1:], lambda p: p.tag == "link") - ctx[0].extend(kept) - - # keep all patches from the link-time patches' subtrees in the C code - afters = [u for u in UOp.sink(*links).toposort() if u.op is Ops.AFTER] - ctx[1].extend(UOp.sink(*links).substitute({p: p.src[0] for p in afters}).src) - return a.src[0].after(*kept, *[d for p in afters for d in p.src[1:]]) if links else None -pm_trim_link_patches = PatternMatcher([(UPat(Ops.AFTER, src=(UPat((Ops.PARAM, Ops.MSTACK)),), allow_any_len=True, name="a"), trim_link_patches)]) - -def _dnum(stride:int) -> UOp: return UOp.variable("_device_num", 0, stride - 1, dtypes.int, param=True) if stride > 1 else UOp.const(0, dtypes.int) - -def make_addr_table(call:UOp, gaddrs:list[UOp], name:str, stride:int=1) -> tuple[UOp, dict[UOp, UOp], tuple[UOp, ...], dict[UOp, int]]: - bare = {g: g.replace(src=(g.src[0].without_after,)) for g in gaddrs} - - # slot-major layout: slot i of lane j lives at i*stride+j, every lane reads through the same table base - slots = {g:i*stride for i,g in enumerate(sorted(dedup(bare.values()), key=lambda g: g.key))} - table = UOp.placeholder((len(slots)*stride,), dtypes.uint64, next(UOp.unique_num), device=call.arg.aux.device).rtag(name) - - reads = {g: table.after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()).index(_dnum(stride) + slots[bare[g]]).load() for g in gaddrs} - fills = (table.after(*make_patches(table, [(i*table.dtype.itemsize, addr) for addr, i in slots.items()])),) if slots and stride == 1 else () - return table, reads, fills, {g:slots[bare[g]] for g in gaddrs} - -def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp], stride:int) -> dict[UOp, UOp]: - (dst,), words = dedup(p.buf_uop for p in patches), [(unwrap_view(p.src[0].src[0])[1] + off.val*(val.dtype.itemsize//p.buf_uop.dtype.itemsize), - slots[val]) for p in patches for off,val in zip(p.src[0].src[1].src, p.src[1].src)] - - # build a runtime loop that writes every input address - pairs = UOp.placeholder((2*len(words),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems") - lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words)))) - r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst)) - off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel()-(stride-1)))) - # SHRINK(offset, length): a const length keeps the end bound from becoming an expression the program spec rejects - patch = UOp(Ops.SHRINK, src=(dst, off, off.const_like(table.dtype.itemsize//dst.dtype.itemsize))).bitcast(table.dtype).index(0) \ - .store(table.index(slot + _dnum(stride)).load()).end(r) - return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: patch} - -def is_input_addr(g:UOp) -> bool: return any(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop)) - -def split_patches(call:UOp) -> UOp|None: - rt_patches:list[UOp] = [] - lt_patches:list[UOp] = [] - body = graph_rewrite(call.src[0], pm_trim_link_patches, ctx=(rt_patches, lt_patches), name=f"trim link-time patches ({call.arg.name})") - - # split patches. addresses read in the body go through the tables too - lanes = len(to_tuple(call.arg.aux.device)) - inputs, internals = partition(dedup(get_getaddrs(UOp.sink(body, *rt_patches))), is_input_addr) - runtimes, systems = partition(internals, lambda g: any(x.tag in {"program", "kernargs", "cmdbuf"} for x in unwrap_mstack(g.buf_uop))) - tables = [make_addr_table(call, gs, n, lanes if n == "inputs" else 1) for gs,n in ((inputs, "inputs"), (runtimes, "runtime"), (systems, "systems"))] - reads, fills = {k:v for _,r,_,_ in tables for k,v in r.items()}, [f for t in tables[1:] for f in t[2]] # inputs table is filled by exec +# ***************** +# 5. lower submits: the backend's pm_lower turns custom_function(submit, cmdbuf) into the queue push. per-call values +# it needs (like the cmdbuf address) are written as plain uops: the body lowering routes them through the input table - ipatches = [p for p in rt_patches if p.tag == "inputs" and all(v in tables[0][3] for v in p.src[1].src)] # only getaddrs go to the table - gathers = make_gather_loop(ipatches, tables[0][0], tables[0][3], lt_patches, lanes) if ipatches else {} - body = body.substitute({p:p.substitute(gathers | reads) for p in rt_patches} | reads, walk=True) +def lower_submit_call(call:UOp) -> UOp|None: + if (submit:=get_submit(call.src[0])) is None: return None - lt_srcs = collections.defaultdict(list) - for p in lt_patches: lt_srcs[p.buf_uop].append(p) + devs, queue = to_tuple((cmdbuf:=submit.src[0]).device), cmdbuf.without_after.tag[1] + ctx = EncodeCtx(Device[devs[0]], devs, queue, dict(call.arg.aux.vals)["hcq_size"]) + return call.replace(src=(call.src[0].substitute({submit: unwrap(ctx.dev.pm_lower[queue.split(":")[0]].rewrite(submit, ctx=ctx))}), *call.src[1:])) - bufs = [u for _, u in sorted(dedup([(i, g.src[0].without_after) for g, i in tables[0][3].items()]))] - aux = replace(call.arg.aux, input_addrs=tuple((d, _lane(u, j)) for u in bufs for j,d in enumerate(call.arg.aux.device))) if inputs else call.arg.aux - return call.replace(src=(body, *call.src[1:], *[b.after(*ps) for b,ps in lt_srcs.items()], *fills), arg=replace(call.arg, aux=aux)) -pm_split_patches = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), split_patches)]) +pm_lower_submit = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), lower_submit_call)]) # ***************** +# 7. lower the hcq bodies (submits, fences, finalizers) to plain programs: per-call addresses go through the input +# table, placeholders become canonically-sized params so one body program is shared across batches -def _rank_ranges(uops:Iterable[UOp]) -> dict[UOp, UOp]: - return {r: r.replace(arg=(i,)+r.arg[1:]) for i,r in enumerate(sorted([u for u in uops if u.op is Ops.RANGE], key=lambda r: r.arg))} +def lower_hcq_call(call:UOp) -> UOp|None: + if call.arg.aux.args or get_submit(call.src[0]) is not None: return None # lowered already, or the submit isn't lowered yet -def replace_params(call:UOp) -> UOp|None: - body, variables, param_ops = call.src[0], call.src[0].variables(), {Ops.PARAM, Ops.MSTACK} - tops = body.toposort(gate=lambda u: u.op not in param_ops) - args = dedup([s for u in tops for s in u.src if s.op in param_ops and s not in variables]) + variables = (body:=call.src[0]).variables() + tops = body.toposort(gate=lambda u: u.op not in {Ops.PARAM, Ops.MSTACK}) + placeholders = dedup([s for u in tops for s in u.src if s.op in {Ops.PARAM, Ops.MSTACK} and s not in variables]) - patched, refhold = partition(call.src[1:], lambda x: x.src[0] in args) - by_root = {p.src[0]: p for p in patched} - c_args = [by_root.get(a, a) for a in args] + # args -> params + args = {b: UOp.param(i, b.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=any(x.arg.volatile for x in unwrap_mstack(b))) + for i, b in enumerate(placeholders)} - # keep buffers whose addresses become link-time constants alive and mapped - held = args + [r.without_after for r in refhold] - addrs = dedup([g.src[0].without_after for g in call.toposort() if g.op is Ops.GETADDR]) - refhold += [a for a in addrs if a not in held and all(b.op is not Ops.PARAM or b.tag is not None for b in unwrap_mstack(a))] + # vars slots aft args + base = max([len(args)] + [v.arg.slot + 1 for v in variables]) # above every existing slot, or a renumber can cycle + vrs = {v: v.replace(arg=replace(v.arg, slot=base + i)) for i, v in enumerate(sorted(variables, key=lambda v: (v.arg.name, v.arg.slot)))} - sub = {(b:=u.without_after): UOp.param(i, u.dtype, shape=b.shape, device=HCQ_RUNTIME_DEV.value, volatile=b.op is Ops.PARAM and b.arg.volatile) - for i,u in enumerate(c_args)} | {v: v.replace(arg=replace(v.arg, slot=-1)) for v in variables if v.op is Ops.PARAM} | _rank_ranges(tops) - info = replace(call.arg.aux, inputs=next((i for i,u in enumerate(c_args + refhold) if u.without_after.tag == "inputs"), None)) - prg_sink = body.src[0].substitute(sub).replace(arg=KernelInfo("hcq_submit"), tag=1) - return call.replace(src=(body.replace(src=(prg_sink,)), *c_args, *refhold), arg=replace(call.arg, aux=info)) -pm_replace_params = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), replace_params)]) + # reenum ranges + rngs = {r: r.replace(arg=(i,)+r.arg[1:]) for i,r in enumerate(sorted([u for u in tops if u.op is Ops.RANGE], key=lambda r: r.arg))} -# ***************** + sink = body.src[0].substitute(cast(dict[UOp, UOp], args) | vrs | rngs).replace(arg=KernelInfo("hcq_submit"), tag=1) -def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp: - base = bv.src[0].after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ()) - addr = UOp(Ops.GETADDR, src=(base,), arg=g.arg) - return addr if bv.op is Ops.BITCAST else addr + UOp.const(bv.src[1].val * bv.dtype.itemsize, dtypes.uint64) + # args the link writes into keep their seal after + patched = {s.without_after: s for s in call.src[1:] if s.op is Ops.AFTER} + arg_src = [patched.get(b, b) for b in placeholders] + src = (body.replace(src=(sink,)), *arg_src, *[s for s in call.src[1:] if s not in arg_src]) -pm_early_simplify = PatternMatcher([ - (UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv").or_after(),), name="g"), resolve_getaddr_view), - (UPat(Ops.SHRINK, src=(UPat(Ops.SHRINK, name="bv"), UPat(), UPat()), name="x"), - lambda bv,x: bv.src[0].shrink(((start:=bv.src[1]+x.src[1], start+x.src[2]),))), -]) + table = None if (t:=call.arg.aux.table) is None else next(i for i, x in enumerate(src) if x.without_after is t) + return call.replace(src=src, arg=replace(call.arg, aux=replace(call.arg.aux, table=table, args=tuple(args.items())))) -# ***************** -# 5.3. pack placeholders buffers - -def pack_hcq_placeholders(call:UOp) -> UOp|None: - bufs = [b for b in call.src[0].toposort() if b.op is Ops.PARAM and b.tag in {"scratch", "kernargs"}] - offs:dict[UOp, int] = {} - sizes:dict[Any, int] = {} - for b in bufs: - if b.tag == "scratch": sizes[b.tag] = max(sizes.get(b.tag, 0), b.max_numel()) - else: - offs[b] = round_up(sizes.get(b.tag, 0), 128 // b.dtype.itemsize) - sizes[b.tag] = offs[b] + b.max_numel() - counts = collections.Counter(b.tag for b in bufs) - bases = {b.tag:UOp.placeholder((sizes[b.tag],), b.dtype, next(UOp.unique_num), device=b.device).rtag(b.tag) for b in bufs if counts[b.tag] > 1} - subs = {b:bases[b.tag][(off:=offs.get(b, 0)):off+b.max_numel()] for b in bufs if b.tag in bases} - return call.replace(src=(call.src[0].substitute(subs, walk=True), *call.src[1:])) if subs else None -pm_pack_placeholders = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq"),), name="call", allow_any_len=True), pack_hcq_placeholders)]) +pm_lower_hcq = PatternMatcher([ + (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), lower_hcq_call)]) # ***************** -# 9. merge submitters - -def _lane_arg(a:UOp, lane:int, table:UOp) -> UOp: return table if a.tag == "inputs" else a.mselect(lane) if len(to_tuple(a.device)) > 1 else a - -def merge_batch(batch:list[UOp]) -> UOp: - tables = UOp.variable("hcq_inputs_ptr", 0, 2**64-1, dtypes.uint64, param=True) - offs = itertools.accumulate((8 * len(c.arg.aux.input_addrs) for c in batch), initial=0) # every call owns the next table of the region - cmds = [c.src[0].src[0].call(*[_lane_arg(a.without_after, j, tables + off) for a in c.src[1:]], UOp.variable("_device_num", 0, 1 << 30).bind(j)) - for c, off in zip(batch, offs) for j in range(len(c.arg.aux.device))] - - info = HCQInfo((HCQ_RUNTIME_DEV.value,), sum((c.arg.aux.estimates for c in batch), start=Estimates()).simplify(), - input_addrs=tuple(x for c in batch for x in c.arg.aux.input_addrs), kernels=tuple(k for c in batch for k in c.arg.aux.kernels)) +# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 ring: a submit whose cmds call the +# compiled piece programs, so the worker runs the batch in fifo order and the python exec is one ring push + +def _lane_arg(a:UOp, lane:int) -> UOp: return a.mselect(lane) if len(to_tuple(a.device)) > 1 else a + +def _batch_hcq_calls(calls:list[UOp]) -> UOp: + # flatten tables + views for each cmd + table = UOp.placeholder((next_power2(sum(len(c.arg.aux.inputs) for c in calls)),), dtypes.uint64, next(UOp.unique_num), + device=HCQ_RUNTIME_DEV.value).rtag("inputs") + offs = itertools.accumulate((len(c.arg.aux.inputs) for c in calls), initial=0) + views = {c: table[off:off + len(c.arg.aux.inputs)] for c, off in zip(calls, offs)} + + # build submitter + def cmd(c:UOp, j:int) -> UOp: + args = [views[c] if i == c.arg.aux.table else _lane_arg(c.src[i].without_after, j) for i in range(1, 1 + len(c.arg.aux.args))] + vals = [UOp.variable(n, 0, 0xffffffff).bind(v) for n, v in c.arg.aux.vals] + return c.src[0].src[0].call(*args, *vals, UOp.variable("_device_num", 0, 1 << 30).bind(j)) + cmds = [cmd(c, j) for c in calls for j in range(len(to_tuple(c.arg.aux.device)))] body = UOp.custom_function("hcq", make_submit(*cmds, devs=HCQ_RUNTIME_DEV.value, queue="SUBMIT:0").sink()) - return body.call(*[s for c in batch for s in c.src[1:] if s.without_after.tag != "inputs"], name=f"hcq_submitter ({len(batch)})", aux=info) - -def merge_submitters(linear:UOp) -> UOp: - batches = [(k, list(g)) for k, g in itertools.groupby(linear.src, key=lambda c: isinstance(c.arg.aux, HCQInfo))] - return linear.replace(src=tuple(c for is_hcq, b in batches for c in ([merge_batch(b)] if is_hcq else b))) -# ***************** -# hcq schedule + # update info + estimates = sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify() + kernels = sum((c.arg.aux.kernels for c in calls), start=()) + inputs = sum((c.arg.aux.inputs for c in calls), start=()) + cargo = dedup([table] + [s for c in calls for s in c.src[1:] if s.without_after.tag != "inputs"]) + info = HCQInfo((HCQ_RUNTIME_DEV.value,), estimates, kernels=kernels, table=table, inputs=inputs) + return body.call(*cargo, name=f"hcq_submitter ({len(calls)})", aux=info) -hcq_compile_cache:dict[tuple[bytes, bool], UOp] = {} +def batch_hcq_linear(linear:UOp) -> UOp: + def _key(c:UOp): return c.op is Ops.CALL and c.src[0].op is Ops.CUSTOM_FUNCTION and c.src[0].arg == "hcq" + return linear.replace(src=tuple(flatten([_batch_hcq_calls(list(g))] if k else g for k, g in itertools.groupby(linear.src, key=_key)))) -def hcq_lower(linear:UOp, pm_encode:PatternMatcher) -> UOp: - # lowering to hcq ir - linear = graph_rewrite(linear, pm_encode, walk=True, name="encode and pack", enter_calls=True) +# ***************** +# 8. hcq compile - # patches and runtime uops - linear = graph_rewrite(linear, pm_early_simplify+symbolic, bottom_up=False, name="simplify patches", enter_calls=True) - linear = graph_rewrite(linear, pm_split_patches, walk=True, name="split patches") +def hcq_lower(linear:UOp, name:str="lower") -> UOp: + linear = graph_rewrite(linear, pm_encode + pm_seal + pm_lower_submit + pm_lower_hcq, bottom_up=True, name=name) + with Context(EMULATED_DTYPES=""): return lower_and_compile(linear) - # and compile it - with Context(EMULATED_DTYPES=""): return lower_and_compile(graph_rewrite(linear, pm_replace_params, walk=True, name="replace params")) +hcq_compile_cache:dict[tuple[bytes, bool], UOp] = {} @rewrite_group(lambda linear,input_uops,profile,ret: f"HCQ Compile {pluralize('Kernel', len(ret.src))}") def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: @@ -480,80 +467,87 @@ def hcq_compile(linear:UOp, input_uops:list[UOp]|None, profile:bool) -> UOp: # schedule on real buffers linear = sched_batches(linear, profile).substitute({s:p for p,s in back_map.items()}, walk=True, enter_calls=True) - # lower to hcq programs, then pack the programs of every batch into one C submitter (needs a C runtime device for the program addresses) - linear = hcq_lower(linear, pm_encode_cmdbufs+pm_pack_placeholders) - final_linear = hcq_compile_cache[cache_key] = hcq_lower(merge_submitters(linear), pm_encode_cmdbufs) if HCQ_RUNTIME_DEV.value == "CPU" else linear + # lower + linear = hcq_lower(linear) + final_linear = hcq_compile_cache[cache_key] = \ + hcq_lower(batch_hcq_linear(linear), name="lower c submitter") if HCQ_RUNTIME_DEV.value == "CPU" else linear return final_linear # ***************** -# 6. bufferize placeholders: replace placeholders with real buffers. +# 9. bufferize placeholders: replace placeholders with real buffers -def bufferize_buf(ctx:bool, buf:UOp) -> UOp|None: +def bufferize_buf(ctx:tuple[bool, list[UOp]], buf:UOp) -> UOp|None: if buf.tag is None: return None - return UOp.mstack(*(UOp.from_buffer((dv:=Device[dev]).pm_bufferize.rewrite(buf, ctx=(dv, ctx)), HCQ_RUNTIME_DEV.value) + return UOp.mstack(*(UOp.from_buffer((dv:=Device[dev]).pm_bufferize.rewrite(buf, ctx=(dv, ctx[0])), HCQ_RUNTIME_DEV.value) for dev in to_tuple(buf.device))) pm_bufferize = PatternMatcher([(UPat(Ops.PARAM, name="buf"), bufferize_buf)]) # ***************** -# 7. resolve patches +# 10. link: bufferize the placeholders, then the patch stores fold into plain memory writes + +def push_stack(op:UOp) -> UOp|None: + if not (ns:=[s for s in op.src if s.op is Ops.STACK]) or not all_same([len(s.src) for s in ns]): return None + return UOp(Ops.STACK, src=tuple(op.replace(src=tuple(s.src[i] if s.op is Ops.STACK else s for s in op.src)) for i in range(len(ns[0].src)))) -def push_stack(op, s): return UOp(Ops.STACK, - src=tuple(op.replace(src=tuple(x if y is s else y for y in op.src)) for x in s.src)) +def _bufs(buf:UOp) -> list[Buffer]: # one Buffer per lane + if buf.op is Ops.MSTACK: return [cast(Buffer, x.buffer) for x in buf.src] + return list(m.bufs) if isinstance(m:=buf.buffer, MultiBuffer) else [m] def fold_binary(buf:UOp, blob:UOp) -> UOp: - for b in (m.bufs if isinstance(m:=buf.buffer, MultiBuffer) else (m,)): - b.ensure_allocated()._buf.cpu_view().view(fmt='B')[:len(blob.arg)] = blob.arg + for b in _bufs(buf): + if getattr(b, '_hcq_written', None) is not blob.arg: # programs are shared across linears, write them once + cast(Any, b.ensure_allocated())._hcq_written = blob.arg + b._buf.cpu_view().view(fmt='B')[:len(blob.arg)] = blob.arg return UOp(Ops.NOOP) -def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp: +def fold_word_store(view:UOp, idx:UOp, val:UOp) -> UOp|None: + vals = [v.ssimplify() for v in (val.src if val.op is Ops.STACK else (val,))] + if not all_int(vals): return None buf, start = unwrap_view(view) - for off,val in zip(off.src, val.src): - for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)): - data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype](v.val)) - bo = start*buf.dtype.itemsize + off.val*val.dtype.itemsize - b.ensure_allocated()._buf.cpu_view().view(fmt='B')[bo:bo+len(data)] = data + width, bo = view.dtype.itemsize, start * buf.dtype.itemsize + idx.val * view.dtype.itemsize + for b, c in zip(_bufs(buf), itertools.cycle(vals)): # a single value broadcasts over the lanes + b.ensure_allocated()._buf.cpu_view().view(fmt='B')[bo:bo+width] = (c & (1 << 8 * width) - 1).to_bytes(width, 'little') return UOp(Ops.NOOP) -def resolve_getaddr(buf:UOp, g:UOp) -> UOp: - assert buf.op in (Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), f"{buf.op}" - - devs, b = g.arg, buf.buffer - bufs = tuple(cast(Buffer, x.buffer) for x in buf.src) if buf.op is Ops.MSTACK else tuple(b.bufs if isinstance(b, MultiBuffer) else (b,)*len(devs)) +def resolve_getaddr(ctx:tuple[bool, list[UOp]], buf:UOp, g:UOp) -> UOp: + ctx[1].append(buf) # the address bakes into the blob, the linked linear refholds the buffer (amd scratch outlives its realloc) + devs, bufs = to_tuple(g.arg), _bufs(buf) + if len(bufs) == 1: bufs = bufs * len(devs) # one buffer shared by every lane assert len(bufs) == len(devs), f"can't resolve {len(bufs)} buffers on {len(devs)} devices" addrs = tuple(UOp.const(x.get_buf(d).va_addr, dtypes.uint64) for x, d in zip(bufs, devs)) return addrs[0] if len(addrs) == 1 else UOp(Ops.STACK, src=addrs) +def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp: + addr = UOp(Ops.GETADDR, src=(bv.src[0],), arg=g.arg) + return addr if bv.op is Ops.BITCAST else addr + UOp.const(bv.src[1].val * bv.dtype.itemsize, dtypes.uint64) + pm_resolve_patches = PatternMatcher([ # multi - (UPat(GroupOp.ALU, src=[UPat(Ops.STACK, name="s"), UPat.any(UPat(Ops.CONST), UPat(Ops.CAST, src=(UPat(Ops.CONST),)))], name="op"), push_stack), - (UPat(Ops.CAST, src=(UPat(Ops.STACK, name="s"),), name="op"), push_stack), + (UPat(GroupOp.ALU | {Ops.CAST}, name="op"), push_stack), # getaddr - (UPat(Ops.GETADDR, src=(UPat(name="buf"),), name="g"), resolve_getaddr), + (UPat(Ops.GETADDR, src=(UPat(Ops.AFTER, name="a"),), name="g"), lambda a, g: g.replace(src=(a.src[0],))), + (UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv"),), name="g"), resolve_getaddr_view), + (UPat(Ops.GETADDR, src=(UPat((Ops.BUFFER, Ops.MSTACK, Ops.MSELECT), name="buf"),), name="g"), resolve_getaddr), # folders (UPat(name="buf").store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast())), fold_binary), (UPat((Ops.BITCAST, Ops.SHRINK, Ops.BUFFER, Ops.MSTACK), name="view") - .index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store), + .index(UPat(Ops.CONST, name="idx")).store(UPat(name="val")), fold_word_store), ]) pm_assert_no_afters = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: panic(RuntimeError, f"AFTER left at hcq_link: {a.src[0].op}"))]) -def link_buf_key(a:UOp): return a.key, to_tuple(a.device) -link_buf_cache:dict[tuple[bytes, tuple[str, ...]], UOp] = {} link_linear_cache:dict[bytes, UOp] = {} @rewrite_group(lambda _,cache,ret: f"HCQ Link {pluralize('Kernel', len(ret.src))}") def hcq_link(linear:UOp, cache=True) -> UOp: if (linked:=link_linear_cache.get(linear_key:=linear.key)) is not None: return linked - - bufs = {(j,i):a for j,c in enumerate(linear.src) for i,a in enumerate(c.src[1:], 1) - if a.op is Ops.AFTER and unwrap_mstack(a.src[0])[0].tag in HCQ_CACHE_TAGS} - linear = linear.substitute({x:link_buf_cache[k] for a in bufs.values() if (k:=link_buf_key(a)) in link_buf_cache for x in (a, a.src[0])}, walk=True) - linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_assert_no_afters, bpm=pm_bufferize, ctx=cache, bottom_up=False, + refs:list[UOp] = [] + linear = graph_rewrite(linear, pm_resolve_patches+symbolic+pm_assert_no_afters, bpm=pm_bufferize, ctx=(cache, refs), bottom_up=False, name="resolve patches") - for (j,i),a in bufs.items(): link_buf_cache.setdefault(link_buf_key(a), linear.src[j].src[i]) + if refs: linear = linear.replace(src=(linear.src[0].replace(src=linear.src[0].src + tuple(dedup(refs))), *linear.src[1:])) if cache: link_linear_cache[linear_key] = linear return linear @@ -563,7 +557,7 @@ def hcq_link(linear:UOp, cache=True) -> UOp: class HCQ2Compiled(Compiled): timestamp_divider: float = 1000.0 wait_timeout_ms: float = 30000.0 - rt_nbytes: int = 64 << 20 # scratch that single-run placeholders are carved out of + rt_nbytes: int = 64 << 20 # the pool every per-linear buffer is carved out of def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Renderer]], runtime, can_recover:bool=False, arch=None): self.can_recover = can_recover @@ -580,6 +574,7 @@ def __init__(self, device:str, allocator:HCQAllocator, compilers:list[type[Rende super().__init__(device, allocator, compilers, runtime, None, arch=arch) self.rt_allocator = BumpAllocator(self.rt_nbytes) + self.prog_bufs:dict[UOp, Buffer] = {} self.prof_ents:dict[int, ProfileGraphEntry] = {} def collect_prof(self): @@ -606,11 +601,15 @@ def _at_profile_finalize(self): def rt_buffer(self, uncached:bool=True) -> Buffer: return Buffer(self.device, self.rt_allocator.size, dtypes.uint8, options=BufferSpec(uncached=uncached, cpu_access=True), preallocate=True) + def rt_view(self, nbytes:int, dtype:DType=dtypes.uint8, uncached:bool=True) -> Buffer: + return self.rt_buffer(uncached).view(nbytes // dtype.itemsize, dtype, self.rt_allocator.alloc(max(nbytes, 1), alignment=128)).ensure_allocated() + def new_buffer(self, b:UOp, cache:bool) -> Buffer: - if cache or b.tag in HCQ_CACHE_TAGS: - return Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(uncached=b.tag not in ("program","kernargs"), cpu_access=True,nolru=True)) - return self.rt_buffer(uncached=b.tag!="kernargs").view(b.max_numel(), b.dtype, - self.rt_allocator.alloc(b.max_numel() * b.dtype.itemsize, alignment=128)) + if b.tag == "program": # program buffers are shared across linears, keyed on the placeholder + if (buf:=self.prog_bufs.get(b)) is None: + buf = self.prog_bufs[b] = Buffer(self.device, b.max_numel(), b.dtype, options=BufferSpec(cpu_access=True, nolru=True)).ensure_allocated() + return buf + return self.rt_view(b.max_numel() * b.dtype.itemsize, b.dtype) @functools.cache def signal(self, name:str|int, init_value:int=0, device:str|None=None) -> Buffer: diff --git a/tinygrad/runtime/support/usb.py b/tinygrad/runtime/support/usb.py index 6ba42fef9c551..7fdb1309967d5 100644 --- a/tinygrad/runtime/support/usb.py +++ b/tinygrad/runtime/support/usb.py @@ -1,11 +1,11 @@ import ctypes, struct, time, functools, itertools from typing import Any, cast from tinygrad.runtime.autogen import libusb -from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv, unwrap, dedup, to_tuple +from tinygrad.helpers import DEBUG, DEV, to_mv, from_mv, round_up, ceildiv, unwrap, to_tuple from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, UPat, Ops, PatternMatcher from tinygrad.device import Buffer, BufferSpec, Device -from tinygrad.runtime.support.hcq2 import HCQInfo, make_buf, make_cmdbuf, make_submit, HCQ_RUNTIME_DEV +from tinygrad.runtime.support.hcq2 import HCQInfo, make_buf, make_submit, HCQ_RUNTIME_DEV from tinygrad.runtime.support.hcq import MMIOInterface from tinygrad.runtime.support import c @@ -263,9 +263,6 @@ def usb_stream(devs, dep:tuple[UOp, ...], addr:UOp, data:UOp, nbytes:int, write: 0x40, 0xF0, (0x60 if write else 0x20) | (0x0F << 8), 1 if write else 2, hdr.index(0), 12, 5000) return usb_bulk(devs, (arm,), 0x02 if write else 0x81, data, nbytes) -def usb_writes(devs, ws:list[tuple[UOp, UOp, int]]) -> tuple[UOp, ...]: - return functools.reduce(lambda dep, w: (usb_stream(devs, dep, w[0], w[1], w[2], True),), ws, ()) - def usb_load(b:UOp, idx:UOp, dt) -> UOp: got = UOp.placeholder((1,), dt, device=(devs:=to_tuple(b.device)), tag="usb_scratch") addr = b.getaddr((HCQ_RUNTIME_DEV.value,)) + (idx*dt.itemsize).cast(dtypes.uint64) @@ -304,34 +301,6 @@ def usb_stage_copy(dst:UOp, src:UOp) -> UOp|None: return UOp(Ops.LINEAR, src=tuple(ops)) pm_usb_stage = PatternMatcher([(UPat(Ops.CALL, src=(UPat(Ops.COPY), UPat(name="dst"), UPat(name="src"))), usb_stage_copy)]) -def usb_arm_bytes(lin:UOp, sram:Buffer) -> int: - dsts = [c.src[1] for c in lin.src if c.op is Ops.CALL and c.src[0].op is Ops.COPY] # the rest of a linear is INS, some with no srcs - return next((d.nbytes() for d in dsts if d.base.op is Ops.BUFFER and d.base.buffer is sram), 0) - -def usb_ib(devs, lin:UOp, align:int, arm:int=0) -> tuple[UOp, UOp, int]: - pkt_dw = sum(s.dtype.itemsize for ins in lin.src for s in ins.src) // 4 # by bytes: sdma packs 64-bit addresses as single srcs - kargs = dedup([b for b in lin.toposort() if b.op is Ops.PARAM and b.tag == "kernargs"]) - offs, up_dw = {}, round_up(pkt_dw, align) - for k in kargs: offs[k], up_dw = up_dw, round_up(up_dw + k.max_numel(), 32) - ib_gpu = UOp.placeholder((up_dw,), dtypes.uint32, device=devs, tag="cmdbuf") - ib_host = UOp.placeholder((up_dw,), dtypes.uint32, device=devs, tag="usb_scratch") - gsubs = {g: g.replace(src=(d if a.op is not Ops.AFTER else d.after(*a.src[1:]),)) for g in lin.toposort() if g.op is Ops.GETADDR - for a in [g.src[0]] if (k:=a.src[0] if a.op is Ops.AFTER else a) in offs for d in [ib_gpu[offs[k]:offs[k] + k.max_numel()]]} - lin = lin.substitute(gsubs, walk=True).substitute({k: ib_host[offs[k]:offs[k] + k.max_numel()] for k in kargs}, walk=True) - return make_cmdbuf(lin, devs, buf=ib_host, dep=(usb_scsi(devs, True, arm),) if arm else ()), ib_gpu, pkt_dw - -def usb_push(devs, ring:UOp, wptr:UOp, doorbell:UOp, put_ptr:UOp, ib_host:UOp, ib_gpu:UOp, pkt:tuple, unit:int) -> UOp: - stage = UOp.placeholder(((n:=round_up(len(pkt), 4)) + 2,), dtypes.uint32, device=devs, tag="usb_scratch") - put, step = put_ptr.index(zero:=UOp.const(0, dtypes.int)), (n * 4 if pkt else ib_host.nbytes()) // unit - st = stage.after(*[stage.index(i).store(UOp.const(v, dtypes.uint32)) for i, v in enumerate(pkt)], - *[stage.index(n + i).store((((put + step) >> (32 * i)) & 0xffffffff).cast(dtypes.uint32)) for i in (0, 1)]) - - writes = [(ib_gpu.getaddr((HCQ_RUNTIME_DEV.value,)), ib_host.index(zero), ib_gpu.nbytes())] if pkt else [] - writes += [(ring.getaddr((HCQ_RUNTIME_DEV.value,)) + ((put % (ring.nbytes() // unit)) * unit).cast(dtypes.uint64), - (st if pkt else ib_host).index(zero), step * unit)] - writes += [(p.getaddr((HCQ_RUNTIME_DEV.value,)), st.index(n), 8) for p in (wptr, doorbell)] - return put_ptr.after(*usb_writes(devs, writes)).index(zero).store(put + step) - USB_HOST_TAGS = {"signal", "timeline_signal"} pm_usb_hostio = PatternMatcher([ (UPat(Ops.LOAD, src=(UPat(Ops.INDEX, src=(UPat(Ops.PARAM, tag=USB_HOST_TAGS).or_after(name="b"), UPat(name="idx"))),), diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index e141df8aeef4d..c0907ad6df9bf 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -812,7 +812,7 @@ def _mop(self, op:Ops, arg) -> UOp: unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.SHRINK, Ops.BITCAST, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self + if self.without_after.op not in {Ops.BUFFER, Ops.SHRINK, Ops.BITCAST, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM, Ops.LINEAR}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index c3de59420be7b..ae8b3bc785ca1 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -226,7 +226,8 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < ])+spec_shared spec_hcq = PatternMatcher([ - (UPat(Ops.GETADDR, dtypes.uint64, src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.SHRINK, Ops.BITCAST, Ops.MSTACK)).or_after(),), name="x"), + (UPat(Ops.GETADDR, dtypes.uint64, name="x", + src=(UPat((Ops.BUFFER, Ops.PARAM, Ops.SHRINK, Ops.BITCAST, Ops.MSTACK, Ops.MSELECT, Ops.LINEAR)).or_after(),)), lambda x: is_device(x.arg)), (UPat(Ops.PROGRAM, dtypes.void, src=(UPat((Ops.BUFFER, Ops.PARAM)).or_after(),)), lambda: True), ])+spec_shared From 60cc1d782762c3301d929a32ade13f600e4dfc32 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 31 Aug 2026 08:51:49 -0700 Subject: [PATCH 297/306] add Ops.RETURNED, remove TUPLE/GETTUPLE (#17804) * delete Ops.FUNCTION/GETTUPLE/TUPLE: call outputs are AFTER on RETURNED placeholders value-producing calls: the body is a plain parametric program that stores outputs into output PARAMs (slots after the input PARAMs). the RETURNED placeholders are inputs to the call, bound to the output PARAMs positionally wherever the call is resolved, and callers AFTER on them like normal buffers. gradient flows through the generic AFTER rule; everything is just Ops.CALL. * RETURNED identity is its placement in the call srcs, not a nonce slot=-1 merging collapses duplicate-signature outputs into one uop (t+1,t+2 grads and multi-grad backward calls dedupe wrongly), and skipping the uop cache breaks schedule_cache (stale linear hits since structural keys assume interning). instead the RETURNED's placement (output index among call srcs) is its identity: identical call constructions merge deterministically, positions never collide. * resolve RETURNED afters in the tensor graph like values (master parity with gettuple) - remove the CONTIGUOUS wrap of tagged call-output afters, it forced call outputs (e.g. local shard amax) into their own buffer/kernel instead of inlining - inline RETURNED afters at transform time via returned_after_finalize, dissolving to values for consumers; calls with bound-variable or unresolved UNSHARD args keep the schedule-time resolution path - allow movement ops (flat-storage views) in kernel graph value positions in the spec - port embedding backward + extra/llama_kernels (local_abs_max, rmsnorm) to the new API * use SINK, not GROUP, for gradient value containers spec.py only blesses GROUP of stores/groups/loops; the gradient value bundles (the forward values, root_grad seeds, and the after->call gradient edge) are plain value containers, and SINK-of-values is already in the spec. also fix extra/llama_kernels/rmsnorm: returned_outputs is a property * CALL is positional: RETURNS work in any src position, convention lives in call_outputs - all resolution paths (gradient, precompile transform, binding) locate RETURNEDs by identity, not by "last srcs"; only call_outputs builds the args-first layout - grad_fxn padding aligns grads with the call's actual src positions - add test_two_return/precompiled * source-compat shim for maketuple/gettuple so foreign code built before the redesign keeps working UOp.maketuple returns a _LegacyTupleValues holder; .call builds call_outputs; CALL.gettuple(i) is returned_outputs[i]. the produced graphs are identical to the new-api versions, so nn/extra/mlperf code is reverted to upstream text * simplify function.py call construction + drop the resolved-call cache - function.py: single and tuple returns both build the call through call_outputs - tensor.py: resolve_function is deterministic and interned, the global cache was unneeded * bind zero-offset views of flat storage to the storage instead of padding them call args need offset 0 and enough length, not views: flat_storage collapses the zero-offset contiguous view chain to the sized storage base, so resolved call args are storage-bare like master (no PAD/SHRINK chains in the kernel graph) * spec.py: drop stray rebase-collision edits, keep only the RETURNED changes * test_multitensor: revert to master, the gettuple shim covers it * materialize all tagged RETURNED afters into real buffers call outputs need real storage regardless of whether they are finals of the current realize: deferred/stateful outputs (the fp8 grad-amax mailbox) are consumed by later realize steps as call args, where a resolved value would have no ranges * call input buffers: wrap RETURNED-based afters, not real-buffer afters precompiled call input binding kept any AFTER unwrapped; an AFTER on a RETURNED placeholder has no storage behind it, so its value leaked into the kernel graph with no consumer able to register ranges (llama3 8B fp8 mailbox pipeline crash). materialize afters whose base has no buffer identity instead. this was the fix matching master for the REDUCE-has-no-ranges crash and restores the llama-kernels amax kernel count * call slots are src positions, always; never rearrange one upstream cause behind the three P1 findings: the raw CALL machinery binds positionally (resolve_function params, gradient padding) but a second args-first convention crept in where RETURNEDs get moved to trailing slots. position is identity now: - transform_precompiled_call keeps RETURNEDs' original src positions: outs take their places, other args become input buffers; no slot renumbering - implicit gradients are emitted aligned to original src positions (None at RETURNED positions) - flat_storage drops the hand-rolled contiguity analysis: reshape itself is the flat-prefix check (it raises ValueError); strided views materialize first * nits on call slot positions; regression tests for interspersed RETURNED - flat_storage back to pad_to().reshape() (reshape keeps movement views, it is not a contiguity check) - input_buffer checks has_buffer_identity(after_ok=True) - TestArgOrder: interspersed RETURNED (plain + precompiled transform), its gradient, padded and strided function inputs * device fixes * TestArgOrder: padded regression uses zero-start padded/shrunk view * TestArgOrder: clone to force buffer identity in padded/strided regression tests * slim: revert prepare formatting, drop reverted-bug tests, restore viz guards, clean comments, mirror returned on param * gut transform_precompiled_call, delete returned_after_finalize the transform keeps master's shape; the prepare-stage resolve_AFTER rule already inlines plain call outputs, and materialization is owned by the input-buffer rule (afters on real buffers bind, afters on RETURNEDs contiguous) * update spec for returned * transform_precompiled_call: inline the input-buffer rule, drop sorted() (body stores are already slot-ordered) * drop dead RETURNED-era rules: prepare's after-shell strip (leftover from returned_after_finalize, which is gone), redundant pattern-covered SINK check, defensive slot-sorts (bodies are slot-ordered by construction) * drop final_tags: final outputs of value calls materialize at sink construction The set of finals is already known precisely (the big_sink's srcs), so track nothing: wrap each final AFTER-on-RETURNED in CONTIGUOUS right after numbering. Precompiled calls are excluded - transform_precompiled_call in the flatten pass gives their outputs real buffers, and wrapping before that transform leaves a stale tag that breaks the output copy. * drop unused default_dtype import --- spec/tinyspec.pdf | Bin 98739 -> 99412 bytes spec/tinyspec.tex | 53 +++++++++------- test/null/test_viz.py | 3 +- test/unit/test_call.py | 60 ++++++++++++++++-- test/unit/test_function.py | 16 ++++- tinygrad/function.py | 11 ++-- tinygrad/mixin/gradient.py | 64 ++++++++++--------- tinygrad/nn/__init__.py | 2 +- tinygrad/schedule/multi.py | 24 +++---- tinygrad/schedule/prepare.py | 11 ++-- tinygrad/tensor.py | 47 +++++++++----- tinygrad/uop/__init__.py | 6 +- tinygrad/uop/ops.py | 117 ++++++++++++++++++++++------------- tinygrad/uop/spec.py | 13 ++-- tinygrad/viz/serve.py | 2 +- 15 files changed, 264 insertions(+), 165 deletions(-) diff --git a/spec/tinyspec.pdf b/spec/tinyspec.pdf index 6e52ace8e9abee7717ec7b8c997e7f6eae04c9ef..5e8ebe58b3b28885d72f51e27f594eda73dd59ed 100644 GIT binary patch delta 42432 zcmY(KLv){y)3=+(Mq}HyZ8o-T8+UBmW@9^PY_qYG#_2ca^IN=&_rIInZ05|F^SQ2R z4uM*Zgi7QBSXl)H;auHZ%#H2gyf&`%ragTSERD_|`1E|w-} z?#00B-2u_`U+6_F%V#>vUF6*6TZzegeBBli#gF^k@wt(=v>~OW{<4ZQU0jlY5L@p4@Ql+;eeR>IeIUB5i87vzLck&xbB@wxWYsQOoUzl|G9>b4dHHQg zC8m8;-544sYv6A;993<$LXAvw8TAhk%Cm1TDDIOZgz5vntr%TGt7;3F)Igy|I~wQ^ zvF^G>W5`#3W2B`uhY}F{S{0(QV-;vCC7*d~fmSfV|xxyhSq$-Rd-f z)B+vB&LIw#!7@a-Kn`^Xa^nXlaqelto2OK({v3$tgv%T1J6MU8@=uVmz8C5jeD~9; zX_tQ<0-T6BDQu*g?Baluj0hs6=$^kmj4IP_E% zl5BWLDA)d$^UB!!wzI>cy6Jh1>>Y&P27Jpdp2%15or(#zurMZ1Hm`KN$KtR(7^Y3K zRu1c?jy={qY}XbUq@6nEy3?O#IryZ5EKQ6{ezHT9E3AeH64wI|rN`V7(~4bYt4GFc1)1=q>$wi%nAD;d<*Q~?*mpEWBQVg%@L{JW&j= z=GJJDH)=^oX>sl#RVpc_B-ZlXlc2m@ZZ;!Ii=PtTd2Vi1IJ)P`1?hR-o9a1{r=v91 z%3c>4s>w5@a#TB)oZB#lhj#CDmrego-|L5tNlOO$RxXb02#DKSSjAv9U9oF#EJMi} zQS{8RZ{BaGhM?rXvryUMW*+tBAxM`7z$G9wbQm}oJI~EeKn}5W{TY|g3cm!Yl7%k9 zm#$bzEPT>lGpo?F|5O~&EF$itrg~Ir?Y?p&!Ti$}ZKm(bVa4fKd}$Os%J99fySS_2 z5yS`}vBPUdt6%(m=F-KUYx>nnSElOdsYT#(G1L4FJ}k#s6Nai${mVHOYjBtWZ>K%4nr?c<%>l{&y?C5=G37^D|9>g#|Ez5xkoub*0lD5 zH0qbO@d#R+g48U)Xa5(oNJMnVGWss?`562)AV4_ZnZ84)@O-r$ALf>`5K3ieXLRy- zbd>Wai){Ef{GE6X7+ovkxCJy^Ni+M#k4SMi$PM|)PVejJDDfpu^ULhNxSC=QG@-oP zWvMtRbj<^{KGmBWo9ScH@c4g#EEQOlu1C2UOiN;ii%UyOq7KbD z^ERumTX;}lN=Mef#DP`C-pX~bW^fAF({m8gRTOblSCD3zDm4H6vQRLvpnB2p5Xyi= z^RdCPB1y_8j$=%)`}AGR@TacQx=~9PBBD{o`6GOqIi^h={Ll7&H^}*i?|iA{=*HhQ zy_vM##H!p2Rfi$MVIZohLX66zyE_Ytmm`%{7U9TKUZQgC!iQDe zP`n2B+w&NTL)KSf#>+(jRyO(Ty88ChI;1}+#kSA+e45Lh9oZkm(8RGu$MY!A*JH5r zIWYED$F@INakqZU;~cJr8P#HK3|mlOLsL@NqKy`zTs{{gI_iq$gA3ZWYz%C6JFgx8 z4kaUBUYFjLc5J))u)u+0yDNoa# z4Qj7~QR>Cocn(GFguX^>I^8)?fH8-qDe$h#bi*dO}rCTynF`KsDZvuk=vEr$Lqg36lK zY)`}tPu%WrZE5b*`yxcYk%-OgSFiXyZsEI-8a>^JKnhJE9I20iQhbRs*4ABSe%}b! z%O%B@8++aBQ1&P$Ub@m&qwyOcJpcgt#~pS*fbkkCs(EQ-1?vX zL@~mUVj)%#up#?mdL7;zPb_NE_rxmkSG`21UT6a=p$!}(DPr9`4kKlcv-OS@)#j;Y zCBREF7F|V5p7lfn)AIOqajCQB(e=2Q5%;^OB|qH6cc)ZejM-_{!gR?0cW2rb;34Vw}T_ z-qm!Jw%y>Q7qIc#th?cO&WP77jKx3hT`_!($}Q@~2p+bpq$Kc};#u*z;>MovTD3M( z`w7~o5+&qXvOp!bg<^usjM+AvggO7rW(PR*#vFl?RUyTQlJDts+y0;+AMOWRC}Iig zVvTFskaORu@=O@Va$ zH4}j0V&yKY77n71+=hF-!niXL9n*qx^;i4ZX_Hl%D$03s=}vZ%jFq~f+MD_^HJ(Ym zWJV5jvvR9|>lL4nzamIR5Ze9jwkz(nNwMOh5AJd{8s}%R{VwHEq1S2L>so%7c>T@Q zWeu7mGQ8&dsD_$;tG6hnrnrOD4bD7YNHwtICWsa>=Wuxz%f+=jW1E+I`4b}FU&V?5 z=R&;|G)3aWMs85xy=TP1bT?&;gSG zmxzRiU-klX4;RgH3i_ij``$mqc-SxW_z5a?c1Bpn0)vwj1u@j9bg6ZxC``0vClg-^ zS2c=C*Yl=`TPGo-$YFG}OZIEQgo<^mP^Y6}p(QRm1jQNd^%ftRSrHYYjHV3#keXu< z7=aS(@ca_F>jRBuyevycORwd{6tI#fB9+7SE#{#%_?4g61DG)B$7p0-rftp6e6dnf#iF_=^(7N>hzmu- zG@DrGoBiu(J{Oysi{>AP915Oowzxr)?B+QI!|^%gf-1aEf3Qe+g1??UfFVtu*Jj#R zHiG@tz4a!nei*!YX!@qccHWW)n$z=+~cxJ+g(g@%O;?e z4o>V;F8y~6P47C5`<1OYH|Sk@9azV3EPI=|L8t-IxL~V|S?b44Z^B4Hb}`30KO4dh zte@Ro50dU1uxW37{c3o|bP|1h0_BF%{3*pB-jOsJ+4%blMhVZ&sQy&qz4&}2qe7dH z=bm5SI`5D-DM+Y10Neit7Bjk=4&?2q{&Skss>CEebLe+Wy#w`$ITI_m7jo8+aTQvm zQFv}CB_BbqUR2#OGqLm(wx(sM-4d7=F2BwbH$Hj}`NF?mX6I*}+PS}WulHe1iSBN? z?{7>$`mjsx(&$q9K3}>A_9_y(pFF$k`v4BUS*!~5H4o3Hj_!R*24gJB*q`0qhsf=* zRZ^o6ERxT43C{tyLN|_~%AYhtLQzyLUw79D-9lR!)`Pfu#+((cfe2b+Sz*?-m4?9} z@bZ-JOV*fJDO%!p0@|@SZB(8)m}#L(?_edYecD)`-tE|y*qz%z_RQA&V5hRr6VUP_ z9rJ3OGvsUi!~Dq~E$ybzR6(b0gZY^pGtSd}RchT?ORXy`6>UfHE~~r)k~j2p!~0l{ zcCL5NIA)!>eey;Y)MfgmMfeo_<{UAHzOsF#%sa*L;2Y|RqTRh=hGXw(P1%iZ7nEj{ zZCQIg@_eqlHruEitn!zGapP&&1c*zf?DKH<6v&*UA4V5E$vWfCm0_C3&9{6_wIB_L zo`s^KQ+$?QeoD$}hj7=D(#6-1K~!roX2S@ID=ZvRV@5K_CNF^_bE-1^`@Urf ziC{@lSYE`)i2Gs1sTZPal_|HwaDYgE<%HVz%{tFM?Sa1mt`}w^@Rr~#bpeJ~T4+Zy z#w4|GG`9tpKP$z7U2HOu2M{fd@_mvGp)p8}B{$UYNXkt16Cb7`oZ4I# zr^zNZ47z!4VbOf0fh!m`{MnbFF;y(At;>D!2u}8^) zOPmGv;U#wH$6fepN$81TgU9|Lhw=yQpP6h4=2@@)8CD-f4_A*MSHO0ub1vVjagdrk zbPTy?&5W~$)&w1PY1DW4!5)mYdD5VD0iZ{wf}SH-T9sVz5E*Bx!uYc3^tb>L=A*wb5f zN*v+fFR>X!*sjRCY-+}KQTuin)6jMLeSk_T?uo|#lATLS=Z}Ofk+3Bh@98&3ME*0f zS|m&Pj8$;1J)_fVNemW$0WaL8PKbXt2g`*L_f>ZJSft&0xqxT?44i$%0wyP9u6ZzdOn$Q9S3NE<*?YA(7mZIyW=>LZ(1*^8F=mt{~)C|~@9B|Rkp@F$%}Jj&h$ zY`gef&8u5&fp|$mW;l#hT4bvVN$+p1SWV%CT%4$}1{IA9!ro1h5dp5|&-k~eP-jdi zaIRceF0z@ zzkJ1Lqp+2*>i%9cfOTnKLKmtA?@G<<*O)bR=NMiK{j?2)wb4HA@JWu#mAEU|e0q;C zucMbmnwRjS=!P4=>SMg~h)iZ9y1=wYSD&!6v(BUN1LEQEcUl%b!sRHh^f;lbwK0kl zg&m=Ia|cyFQp)6NN!_Khyd(>5P|81_*fZ!lQIiTI8i7K0-8j z*I42D^Bhdcu?DXOVx6Y^v1z3wgSbyaeC5R9T4B(~*_HK*4*A~wy$23k0g1-IhGKKy zl{uTA6wsbI4vbWZK||`D4viFbBL(=lMr}t7;vz*gZjZCH99s|yLe@ejN76Nv?0sud zzdz7+ygG4+63>aqD3j!g2V*6~88?r)yHomP>XJia#@UCqNMJ0PuSwIMF9tg5u%o@a1G0Ep`P{7h`iWrw|Zf|{$ zg`*p9YNU5=<(?{>B7Oa`t|vM%LlIC$X|%|1f+M&%+4G&MK{*CJfH2wLvDJw{x&9NLipAP?4Dq zrZvR2yegY*wlvF7rXfbe6w^R_cZU36TXl6HK`JG)kKLa2C#pf?j;KMSU>a;AW|>pVa%D( zf|_>F<7#mq|~fc!0?ezG|RF2RhhUSpbNGl7HVzvUtZ*? zJc{6%NylGh$Uy$d!Iu}+GF}fKHF4x%%n0$7fr`d$nHGFcO@1bLjAvjyAj^O0n2L>x z;p`@=R$4&@s0LF-HxzJLPRl1$Cilc(Nvbe;jj&5bZ-$3{Vtb~)mcu)5xDH9MPEMR& z+GTrps_)>d@7HpaMzK0-#+y>=0?@uGLCi2QozdFrnres5iA7ONiABqEF-KFMTj?nIv4AuM^q7V1RXs_Puq4l^Cyj$!@d8iH$;tL3!dRq9)w)-2&)N5o} z9JLD1Y?c9M&G@oj=ZY+fWX$n!&gJ#L_+XZ#GK8FAI0Va7$tT(U$<1Y<0cJE=)ObE| zxz0TGyP_fS=Bb?KA<>w*iP(Xno9O*XPKR@Qu3}!e+5;8kk*=p-x2Fmg)0o;Sjk^Obs}>rRc24oRB~xtlEARtj7_I*h_-Z!2iSc z=eri{{1HSSWQagJega$HePMWg-8)LA_WCyG^jLL+02)DUTJmZ)z&NBj@4$KWqp^5! zrh<|8RFI7ZdO!lsUz^9v)8BL?wE1qikaQ&dqMN2)zU3GW{w*2Ss#5 z)B@@Q^ib*~0wZVn+!o!|IzZnty-y$X`SCZSfRIua;&jBhC4&jWpYU91{}uYV_mroe z<3$76>QA=34cjm;U~jQjS;dAW<(=;hZL(joXpaYUIX=U4$(C-S{Gw`>5!6`nsp;L= z(t2V0Qav_w4X|rgr2yBIHxig%yMRd7pU2-EEVlm2j?L24=pw94{Py?Phdm8_L^fEO z{f?#TuYM7&J~@TCH)o>*w$Ot)Xv!kUAJl}O&}%J>9U>1FKt}&;9!IX6kfKGJe6O$u z;R7yzT-?pE^Yb?p0qx5GD~;Q;&s_FiPdhLTl_xCYu`wB_OsdWL^o8NGz(dm1iR#`RMKXdY)f8@@sYJED7 zv$Uq>0w#Z+H&~n!12^&f-`XN zj)?2bKM4Hr0Lv8_H*C{glPB`@J&iGN z0ItaQ0NJnW#Z?K+Urrc~s^MQWN7wa-6GPqi{(33Uk88Qq6OHh4h}HC)pUJFum;a7NjmBGK>_x=m-U^zAH+DIN8U-Zwb{ zdbyLSMVQzIP(c($L4rf4Q)wf0LXJhOi0`R1Arz3uVGaU~$Yyz@zq~CAQ0kQQz7RReWen9YG%h-k z+%e2-XV8D6-m(U1MPE#HuvzMXEe*Y{^?%I(U@m|Ek*5d}@PV;4`w&=x14x%~?bzC} zzjSgD!@yh{zpXi>e=kkoT~H$;y_uurIcUZzRKJjPmb(K#>Iu=i67b1+z zFN`V7k$&`AO|Sbp5BJklM`mfDD!&eRImJt_J1eE5N#ctC{SU<^m%P3>s#RC|){Jc? zMAxT*S?+HbAsPuB9N?h<4H&N(w=3I^k>@l%AF3u!Gq5sD;TS*RRPGA_Ut`#Bu(Sfi z28S8+cR|$KKWlg|wfJ#C4RJSXMX)fNaqb?0R}FDjrUqr;$I-0~1OgAt&KF}OxKcZg zMG5~9%k>|;=g~9Zzx9p@uR7yaqmaMb@x#g0OHbaoA%B-egd)+)AHybk>?}NV!l+PGHfl7d-g22W0^(HgM8A6& zPihO)G7?N3l5%zTep71-7E%+Ww3oG+vdVY-DPp!vwIpJEkPoY2*YD748=~OF+k~oi z(q0Vu<}LFVOqpbv22dGPpUjTtgs^m6167>J?mFrRW1S}pC1f0<93$x#HeA)JwlD0u z;FS*;)bPRZ=ANRxKk;eZpCYEXhSFB<^{}(FsPV4|6AHD3v)ewJ+f%4j+~&^w&13_o zZWIcPvLEtcweZnkzj@6KdCv8WL~vqdj2&B?Ao!l<&`51RPlms>Wx7Sg-tZ4O}1cs$%<>@sv^hOOxVfGkQ{ z@@ian(qg{0-t9pvCm^gGrM^AC5T00O8_WYAiL7k9cjMP*%Y@Cq*2oikOq|>o2);P&Up$`j5?lthCt}%RS+=5wbcNh3 zR;cP|LEkM)d-5I(9>gh-Egn8h(gPjC z65ACmHve;2RXela4CYSJB^H=?Icgm}wa(9JY4J}Ln0}$Y#9nB_c)Xgrnu>{8aKUmR z#Eih=j$O-iWPofpJWVq9&&aH4jIQtv==x^dR$!nP%3u|cg=S5Ffut1@UE38BUx_01BEO})@%DDlh=<)YSFd)2>B|`dNF^0@yq7g$y+s*EFz)T+w z9kOh zjzEL{HJt-=b9MslJ5zT@d95rdOQY@0LiT^B+W4_6{eQ3txi6t0I5fGpBDyszQ21J= zgxC~!dUWb2gxfAe1vyekcvVN^MP`Y*gqXU>uvm+W5^X>|L%mTGl#^)7p#^6kk%wOyeGrqBNd|@N{9v=G-Wq z#kaBEV@m+x`NfG?iuj-3oNR7=Vi2aq6D);qr&$#Fp>$VmvfBICSy$U=t^Q5s@1JzerzvZN|n(m7I4 z2dOIvG60k+nagA_7=RrlvH@ZiJ}74DIJumO9zHe}7|c z4tuAsb+BuxB(Iv0kI%q5Jd>t5TFb3D`krs3>~s8~M>zF2j{c#W3Z3SINcvoLQpKcA z4)tHF9QR}7pb6`wjpkuu$%OPML;cSepYTnaxS6x)vpW5m{+3N6JO8GUiDpLG^?nzJ z1U7(dWf`-{0qR{LCzVH;*nIY+>$tV809^8)Wu73Fk#_Tq3)&=%wnV$xuH#w{vX6M~ ztJU=>C`N9k%{mUyiTO>@4qA;tJ_jc`>vW#270XfR{a`+l;C5?5;jfaDz4v+h{o=4S zvk11PefL-rUy)UEeAnh|VaSO-@qlY0&4b^%v{%d5lkDNlucje7-e&{LH zP*gJ@PQzuZCkxs^no`u0pXL^X$D!^?Hdqz=*Xn{88SWFJwWxQO68V%W$01^tjBvEu zl%$+?tw&Sy-u0`S|L|WPpKs*{x4Kfk^m~JiIue;F$@#6m+!ovCi%1!78$aWwSR)|m zB`KlU?%-R)xK^8BGXE-VYGe>u(#G=3>uG!D^YP>(feck8(aOB~8}4`q2Wb|nO4y$L zwcLG`)^Xv|sZ(smBuE7LuO3eca;2%r905H0?+$-TMH5#NdiRoHSlJ6|K@o?-T5M0@ z(l=1AGkF&-|685cze~{J2g&ZZx;=0V+VjD3jOUvhDh+5bmdGCD_}e>mZ7KZ$D^TC0 zCV8I?A2v&SItQkwn#l;Y6Oiqlb9b=HbtJIYoXU5KsT?`G8cr@ALDQL{6+ifKUlHa98ejPvv^7>6s zo6=ftpBJw{x=RTpu;+8z^DFpNz}%pCR>{nimokMFx*ky3Ssu}s>U&mB>it@yFQMVZ zhQzGLPiO}#!%_LS;TKP)d`*$ldgS$!>XLi`LtVq}S_4XhJ37(3H#&C6llkX$Z0T(% z1)Ah_FRyEqZ6u!*RN=1Yl8DCKpIFM_@lU1j!qVH1IoDgcZ4W*@ zT|bI-!CCs4Zw!@!QDJaZweR^T=~(2ksT;1D0(N@MRo6mVamdnDhSX27Hm9~)7xDxm zWX{s+g`q`|rZW}&-Dl>aZMz3kT_rA85({dBQGejOn{rkOLzci4LMSlHI9GHjXXr+< zwn^jE7 z!uRALFE`~T-7DqENb&iPwRkX#7$|~USoyC=xzreX78NQMa|@y58mugc_k+q^&~Y*k z?d0Bz2vX8T^>V*9Dv`A*n0IVPfx zdJ;ZqA#>c>Xd{CDFnKn?8}1MOm1{e8-@FhPrp50c>#?N?Yo8*J~VEk!66Dv zEIEycima3$Ebizw8oWMK8KQ!M&xMuDnFJH*h$-&t0$27+c0m6*i#YNexdG`XtmLG+ zq^wyU!%nGe-MX;C;aRtLg+7;KP|Uww`x2MKMCV~f>O}393r>g&GQL`IM6&3WbI4CS zS#q)A1y2fn)6BXa9~ZVm4xO?#W`xSTJ#~EYsSP#R=jzy6s;0c$${@|AoJMXO zL)U_LdE7NVJ3W*oG^u;JJ$E)H2k+}0F>U`$gQ#cx9f>>{7r|iBzPF;+b-uMu&N+s zp5~k}o=CBp1_Qj#4B!qZyvodl`*k4i>TF zMfl%zF@V0gX5KaqCeAP;__S#wcxg+W!m`K8UeV?E?yDtCj}7nk>o40HvhYgy z&ovF`H@OE6j>u3Qk1M$lw~izN1?0piwPJ`3S4Pz+>NUNx#vG&|e?6N%{nUWK)_RM9 zM-extMyh>3o@5jaH2VBAcB(&9Glu0C_Y{3OCH z0$+Ai2f>U(Mxvw4XIytNs#H5pKmts#WIl^)bVtgVr+85^3ENeYd<=tuABef`QzPS_ z6ID`}H?u8$orv7uNV!(b3(Xh%eF$eaol08sZ*X}X1oNdE@_D1R4%yWSdNp2Xd*&mB3uIAt6jjgARK)O2kgyZu1A`vuY6&(*Mhx;f%>h7tRQDtDu1P~bRy8=jTZ%M;4 z(+I*xb(cb8oD2+@-kx|FR&HTz${4n=#pqmXhxNd~0h zT2)K7i2bXRR@hp!4lQFfqQjQtZ{Mz^W_vP+daaSF7)DI?{@oIlzJVV9Xu!HXwzSH)(ldLj>0{e! zEzej+=*)iHsrI(w=4h*(!j9ms`GdLN)t?8-9y0N7KkMp@DG5V_e*yRfMDGkz@5FX^ zMSh2|&dP1oR?@vd15I!-_&{>c6IQ|^ioh{7)tO-4Q*be9lE z;O{8}1(iHb1_Gadcw(LgQlW>DC4`pKlj+ui;~lW^fpi2}tA_lO?5V*Gc-Fr-VNKNH zL8A4|d@(#9k#7;hKEO2#S$_?=i{|kR#|VxiI!fReOI{*1PKe zCd;rghnZb{va9yEjT-kQ3uFQsiM$jyx9P*ZJjrmD&c+VG*lJnL(sR$*kDQFqG`?Tl zTt>haW4CG*R|-8P==;g2i4sTDE&@I-AxmK!SRVcP!XtdC3?R&5C0%4=q|fv$_c4|h zAfx&{eRS9=k#H1TtkdcHE^A{akF7tvfi9kG2?7nWJ5fJM8+Up|xoYU8g8bG*F6oi7 z5sBL++jPAM>NMGuuo-0Oe#8SWmtkc$L|^}bbjaq5KR$5g4rc#P`TwtSG_ir>U?FBE z{;x=k*g90=h(Fmgkp0KT)3!LJ9B-y~-(<{r zo5rv-)Y1`O`0*CXvvjWW#&@!tfYbA_k-Ao7w`MS?a+RLQiNmQ(rcip*X`r2 z+y3dp0MjJy4`ij&egW!Z^ne-_!Mmq1GVXaVIUzcX+jPht!~_6%of5I__d*;f_B!^q zIlAL<{e{SmWU63;*fW~59wLx7`uJ{D>4}@@6RvvkF{f18sG#hdhjeG$Ei1_Mtq)hlM{JbJ8wv_RYoO?{k|DEmD1BvbJ16hIAJSdpOH)o4`k%^rqZh%k<2#b^6|WYEptZvR&kk}mI;*YdiJ zLr6(ffZ(xhXKOq>`_k42z?;IyfZB5l#vF3YslPGey`WEO4E2r(wsI}46oG;a7r*SN3I<)_=kx_r&gekSUBbK5#&1+Y{?_hCEX@X+_^4q53~zWK zcCamGeCLlHAF>gfz}3NNGII$RaG!^~a-8I+>)V!LJijsloY`Lz-Ddm5Isq%xU%A13 z`YlM-9MY}A4_#>Yqrx%}6!d#Zhpjut1JeN}AZ&5kCQGi?0mL}WJe|_*j{vv@oN?Qj zDjknIFqKP^mP3ee75m6lJN>_%&w@N;A7QGVv*kPL5%aI?AxM2awTFAE5nG6AI&dGP z@?n#V%H96Jd6Ygypkz|T^v@ptej~&$JjAJBP|cWJIgxl{^EB{X#K0g>fUCyY5Fkr z1{IpHYl}H6@C#7}l2vn?q_7!l^%1?um~4dwE64Zi?wccdvll)M)(yL$@AVrBtu%5o zEP2#gw?96Ng6!_(W>bH_8+m67^U=k;Ywrd`!HVxZK!AnwH^d58o*gca;$8?bAF>wP z$}PnK+P<8W>1jSesWqFtU#=2Ga@t%Q!UcbCVLWL@kUUK*+s5gt`>%Yk*l3F#zD zKvFZts2gR|f3Jd%5y8J3HCwTTqz=_mW!J>d9tyHmMGC=pYg-A^Q{$wLX&pvrWm}Ou zeb-+Zd`-QGz1DI?U)(9Ivv+pF@N2kMrV-8sG(rNX%XE!5A(2l>b0a9{QPZl(vSaL~ z+}YR8e?k`zR%!g>2=!K=M_673yDnM_#J?u6l9kZbi83dr6GmG`X(n>ah$*U#4@gQP zHll!}C!vA;PoswF{7-rwFE_Y56i;2;KM)vCYT|e*~xrS8cXi}J7N2ZJ4j~| z_~!kxvW4?3TgCR0;m2Dc&yc%1xV%f1F9FNJNV(7_t3-*DU!vB_=i%&u1f|0XsW{Y# zWe;j0Q#Ej1GMU>(nMr7Q@nTWaebhJZPQ4`Ev>>ZSWz#~b(np1I&=G1vCO(&FY2_Md zZtPU4bB_Hy{vBn`<_k_S;}T(kYbAOH+;j@FmpFtTH^nebHiqInjLoJfsKti89mwbtlb2SJHe(f_udRyVTk$M1CXZDB>?T5+ zk_GsW1yEDJ-nZu!!f{z&0;hB_v@khHN#Bh-IKBHKVTbwkhh&c*G*Zm*oG8{ZurMXu z#XF%0QjApQQ{6}frQ%}Ld9Ug_QHj5Q$dXJWEw%pA6di%x_Bp!yQ@WKSwr}!SAlS;b};8p?Xgd!JiliV)0IN0d?*_t1j-dsZIyD&UN2U5xi$1SE7 z656t9Q#9(2`_@0XuJEqP*DRg8bZU2+CiMvX^T5=$PRmn$c*)k8NXTCYXhU7M&)&!% zXEZ-pQ z1kgYCrHOB8mvb6;&Wf0R%41zVfFg9Tqq)^=qO33EzU~yg(Ei8S;8;5q(0i7riTn^y zo9COg$yVDV_AF>L+i3!Lel!@0BzrN;m2!K$TH9w7#Sy*QlqTeWCbfwl&9=hNZq#XS z6y!td^Htk}8dA*q>*)Q>7kBHP&5AsIAG>(uytQ9-C@g#p)O3DlFCNBE1-NCZ*8d0F z4G|Ye9&v+g+u7>{h$r)m{`C3jrS|@t!12zmI$QJi;D-SmIF1$bU>DnkhwQ*SPxUGTr{mF3R^sVm}Ae|hPFaGOb!C+gyy>TzA zlml1d-rBae4Bx%V6HcX(WFIcm-C2C)+*i9OES+|Nx^|kDZ~&iS#4Youxv+O8*v;?c z-N1aY%jY65_@n_aJmAjp4}O4sTyYmtUMz++?7U^Nt$9I!pf_sa-^8W*CWt8ai%g3? zm6W)^hG<9eIzT^S<6cubAKx+{DQku7gC`G1`p{Fyv#!0 zhi>Gl{P8vaE(AUUiL#5F{;@gG1gL5~_QuPF7%kEK-Qxp>H#D<-p_^)lnytuldv*=% zYkv)`OjsuzCf#PE)nei~Glhna0KLu;MEwPN?@y;Lggv}kd?qTiH5Vl@9wIj78J1AC zgE(o(X58TZeC-6L@l>{rM6N6^?SybR56rjmp(5W%^GG-eNhN3B3iy_uCkC$?Wgp0pn%TyUG6yGvqUCrM>b>XZjil||7igyci1FBt|-?`57JuVI$0uI`uPb_&UQ;4Z0HwhmM4H7nz^@s8g58^g5E)_*n*=9{H=%7$iohtV>Tms9T+hSf(d(f4O8!j@cyet zvW{&JDXA@Pdu zbSeC1U<*mZ?A&XC{`oxAtjrPLEFwES7D}imU*v7c?(KsMepQV}tklqjcOrwrCb=IS zg9bBpCrMQ5@ZD>8SKwPTg#4im_V#(a-G;?)4(-uj`lU2?#ov49!I}L*Ay3LvpXY!9 z?r;8*cUr{yz((@@^5S{~4RfM8e4K?Is_>wLy1YsC>r3veK7=`tet=Awp8a~k!4NG1 z$lvl!^|D9e{LhJH+9^4;-8?emYot64ZHdELy21(ShGhUU?0<_Xfke9q0fS6bAtjJQ zD+RKAcjs|-VM*X$SSS4}D&T~3?+koCDi8Mtaz606k24R7Nkj||L@yc{r(yXH%zEMt zmW^}k)UDC%vr!Bv)hi3j;nXy@8TF4THTNm1Yyytx^9=>|&yNUQZ+RI8gJRQ>V3{0S z^8MkaiF!who%gmbx`bRzShKM;Kk4Y`bb{lw4_(uN+!FCi!_AG zxqECS04GqB1FeyB5E>f+YLBKJ44$b4C}z)9(M|-C*#jp`_4*lFSKk#NhnmPEZByo& zq|y$z5dUz{*00|}3`K(rZsuc8x%*}OHV#XmXjF2DlD;&R?Txw^1D>}5Fk`Zm^%O2? zZh(!Ai|zl%yDRup@%Zg``SqD4p+>xBjnC0ozA%Z{;|$>rjmWrhV=&)|l0%Z-{i72O zs{gQUSFgw>HLPb{TQq9jXx{uar1D0)H_WHct+|gu{!%OmD6{RKpL=>a&G`4GM5G8j z$J|r-EC&uOMQr}1w>LP&1d3`FE0EXvgmiCm3Xh8M zsS61fRZ8;bCShMog<-DFo+#c=s{g4%EtwWOp>9}d?`dslgr|iDYc$t_VbS4tbhX0l z!5-RfsDU}c-6O(c5X?fra;RS8Drq`gf%datszOZmaqM`%cie9C1AG{ThJRt1K!U~z zfg;cNM&f6Qb$6Z+qshj$eu7eks2sd7(V;FpKO^*nxxSZF?5z?~3f|D-wn&t>pGTf9 z7tj0TR6boFzQKkG+=3pli zL{a}4;P(_jsbu4f>7C+9v{R7h^R)fe%zmp%FPQ0l2n@7#D4$LYfezFCb%+ zPS|Lgz_yrLe&@6kN0Ky?;Qu%pdtzHM!CIuRIqCa9uHG>?6E0d7p4hf++qP{@Y@5%- z<`dhtZCexDnb=Og_f*}wbFY_07*G|I;mDk_A7pG$qJ#W@$|xeK^)wfa^r%9Tai9`xjE~>R)5gVnu^Ui zvKBj`X&K<{CAEhcTIm+5^en=)Q{?*0{7(tZ5OqwD%!DQe*Y9puzI z{7t?dn=~?;gg1!d7}H2uFO6(D4Vq6MP9h>=-;yTc>Zfi+iw!_9kvfTA9GO&u^?s%D`%qL~6_mjSBnSNr~>& zglk$InS^g5plYady?W?CFjT)n-C+_%?CzY4!40OIidF6alHm(QST%UV?RTW+R8;Eh z2u5nz2goP)HdNl@;cI>`8DV=QrRG ztcl7itKtrTud0?&3fIoQ5lRj0jDAvSFQbLfO#Gt2(^k|kafx>U&)(aZ)UdrkKlyd2 zd&B_YtjaVArt@W=!#oseL%eoAC>;FzLvr0R1#sn_&oe}E$8LfXAm6flmBBX@NI~CI z#bqRId5v;fburm`=`cYIp(yK|{huT||4#2oj42{h+ePKcVeR4Glsse4S6hSMuC-0uztIaHm7noVnZ;c06_!^#oKO5f$*ah`@h#kk63ez{}I^}8ihWUN{qavPL29snc3 z)g?0xc8uZ|o`o@z{V8%E0#?Ci0mr?^R>X(D%Z_bQ0T5k{6j|#SR$r+j^+Evcr&F3w z*Wy~avqk8C>lI5#yGo4Te4h7LG%s{97kY2Y(P0uQY~oy;jBPq_mR&8svHopO*w6`= zj?%7IT!h_C@b!(I-0yC20M%~32qwICHnPi3k>4R z3Zwk6_@@7im_(KR_-RvY(3NTG1>u9bxwSSb7tZD;T*x}58>NblPU4zrTL=I;bk?ha zFgvhREis2JTye32sXETQm=m=rIc2dyR3-vVlVihu+aRZD0i-HsT6v{(>7nE&RV`4V z%g3c#rDj8F$dC&)c(9xm()+{8rL3=uLsF=5rfi0vaS$|7q9RlW6cf_ngU0#;ndYRye& zuWrLxbnVy)*tO2gu!$8;CRn-ex-S`TB~Cc)is#0-l0mxzqeF0y9%vy10!rXW;X>LF zDr?YunF6)ym9ur#9HXks%MTf|Q` zV%a$by3*c&UFPGWmJl=9&zcMa%&EwL26|fiURRUS!x)6;uRGNN3CnPhErLP1A*YJF zMl?g<@cfEA7zh_Q$F{c_{i2$yDGPyPD!RI@Q))Oe;IKe?9YMk*;?FTN#d~~){CqR} zaeHS@R<_{i4;lvEpQ67A7;bPBO;#Vg3oRI7zcWyc8(@qotrS$BbE}sFoFut75n@K` zDd)1oCzN43W!lFk<`GzO2sP2{IWo0=|0>}Nvt`G7R3ZW#TLSxrPAd>t=pRQq5Wb35 zd9sV{FPsW@ynY=EoPA1eG%CuU?7 zQAZiJTb=O5QWA@jYAw>w=v|SR6z5t8`LS$`jdKJ9m79it+~A{f%x`8~UnE}<6ptFp2J zFfRbWaG|+CE4299_^M>hXSo9}WA1aui@We?3Xe!f>i_BP?juOB1ej9^aL~C>Nb4WL zdYbH7skX)9)Fz1iNnJ@xlQ3l)KSnj81}V+C_!z)2ET&RFR#0-0ph!b5%c+`&EddG7 zi8P)AR2J*m7}r30fTLm>-~s-ff;I;g{h#WLsvGg{Ww72614&!I z>6>-?qRL56M{?qbwnyCevR9Lp;Wa3>klQ=gk{ZFQk>=w7> zA_J`nk`wDPPg~8ueC+Q5s zWL5W3=)|4^6PRjL$EH!sC)GGlRaH?VHrAv&1T&nYBCYgo9vJ4G*xw!tH>k|P!-fre z#Z|01M`ZLBoFy27T3=XgkrSdJu$yrIFjyiuRkKpi+#>nQx-Sa=f%(fO_Wtdj7&V4j zv0=nuW}9@u8dy-Rq0QB~NBxnM8g_t_`u$w@c#5A_C@AGyB7EuQVGE+xxHNs0uvUU8 zZh6&DHsX}oisjA#37o84A|QW{yv%JiQIx6uaoZ(JE7L(G=WeXQjF&sxSe)mnN6+3s zv%>8cH~Nxq>pa@Rl@Z1|M*33+OsqRM&Bxo=J(sXwv|bqjjNWRuE5h71(TRri#TGS5 z2BhA1UoM}lW0RwtL9N=E99aj<{oa5NZ)zVrO5H}-oLI{Nd}b$vI_ttxp+$yz`=y-o zpYP9T%-p1;raSd>w-qrN=h0)p$M(;!LwD8irbV?D1Q0f0gfSsN0X{OQbaaEWFl`4J zQjQc(=Wef!=`zdfj56WV!(tacUWaSFRL|gL@Qc$vQH7~eH!aNJY8#z=)~%xD5UQ{` zI(MQsn(_vpYyEWFawowlt?T8;^5wljs9B-b-?7uC?)cGd>s2*bo#g5DOd5Q8lihdA z$ELN@X2fJ4dY=#gd93G1Xwpc($<`C&Yg9_Eb`hPPoiu%I_EOuWD%OdWi#Db?E9EGm zw^%a;$J)`G&~zJy5j|p|-5}QaiIq3we5sGo*MYP2^*7<#yMRbfmR?2~nHcj=YCaD1 zDftp^19GznJo^JmwD5egJS@3Lb%k&>Ya4pwBwmC@dOj)+Wj>FI{FXcH0jxZYl|9XU zLI+4eu34rIF0jXhQ1+EMYa*l%b$~6h76UCiKEJS?_fuUTDS-AGH-J`M%SpMQQ+K0a z{~UYF694^l8q3o(jHH=sDDH(jvZeI&_^>)Qso<<+27q1>0a@ zXO6X$&V_QeLEI<`dhzn#dO(>l8bHILQyV$z$yaM95}CWN>-C5Y z1~iJ|=7zDG=Uu(SJDR}kkD$I-EeF<}6Fu!uS*MO~N!&DbA@Q7lJGNByM;K(4t;>c! zwYJ0(Fj|90z+1TVt*i{EHD~rI-s9oki;W?^kbPDFBbS_7CZ8f7g1wZZ83I~`d}F?e zY{lmj<#(*=i2>AXeEDwfZGgz2Acy_){WCta0*s#@~FLTaw?p{Wap)pf==i##_#uMsWR$M?pl zF1wrP?P?KKB6Hfvw=>?E?uHsw>XdMV)7HukVp z$G50fV{k+e{lmG5x|S!x_Io9z);b%+l$9TA!&aOUW`g~9ja;z(nA(?G2Mxbr{JskL z;k4SdOmGsmEebUVMLs$pQQlR_20`}^>^$s;D)J+{+CI&M2%fm?JZ55kvaJg7IZddj z4RVD@wWVX$X;EQ~wM+!16*9J{oH_GVCiSw?=^pe=+D zMzx zJt5k~fNk(MJeuNkK3Q-aJTGfGMeJwFaTkxlj`i8MMQtx1SKOaGc_IBJ{N~i}u(q3b z>ofoW4i0*k7FGL{TZp>-WeM3mfFT)RQku%)(1{Pb2KTA1jioan zn2_`_p)Rh0BQD00DXC~YO_ImAuU08fdhh^YWA8#|wsTze8EPrjA@BcAN&xqx1cBVy z*ig>VE_V;mD@|J2ETCZRe=dPWalD|MY`~vPT)=C(xE2hFyD}wvgF;cyB_`?+X)HPH z{ydZ+=XG72BXTnJ z9~Uo)_}|s3i-umN&DM1x{aN=UkTGNP>^fYM?fcSywM`=xiNQhqRKFI=@VNC{L$?#F z2mv>KSKChPqH*S!C-PG6*-Tb;^*HH!Zbss|i=h0YAD{TD zLZUS;4)QtGu^3YqR9QxpGqxidlvLfC{<{w{nac@W59*}%7 zD`$QE#?su}zF) zoidH`R(FL|O;5YAQh7jw%7}VIbTWr*j)=E-cHLj*gE=#s z60ZyaTOHRH8{Ii-_XE%?loCZTWO5_La#eP52B-|VOG@(LGI0_W$!cj5=^!QXaXS~r zl3=Y-qckf8x#sl)7!~}f({ab%p(`XW?#5&qw+;akClVvMepwu2+oIX^HO*Z}NOhvH zJ$wK=hw8kU8avzc$&<(50L_bLZkv8k&Btek2_!Fu0D`hZ_->bvW4}spMBC*tm0h;p zdVSyPPH0N7vP15_yHMA?XBq1c3oycJWgK1N#H)QJJ9=t{9{K3Xx}=9jjQggj503HM z`}@9f6@!Gf>T!^@O7?5dmOC0x3 z9ww;DO8JSHAWQmXbbINQI@Vi1YK6`?aJj!93=BmFjV0*8n!{EWse<0U?A*+fP3QpM z-@;b2Ogmjv=TTh)e5jY$K@RNdc&wU4WmU;)op{R#{d82LP8&?Gbx*mBo_{ut__#I3 z@5!sn18^vy8I^orZU_I?Vg5pZn${q-a$0HQ;Z7E^cjkYv_2S|FJp96o=Kp;9fuCK! zY;JPo@9Nkw2BqRw`OjWjVuJkdMQR(c@h zaF-^kz>CR=nT=VM$5fM0CS?+!R#VrUnxVF@@Z#}cO>`am5|+=mwA_)`QCoXN|4lH` zZbV8LMCKz4N-U6ap1$bw4DiC=Vul2J((tIll%HBbjC+}xmLr9&ghy9P@hP!>pO?8N znM$j-U6%1fSw*g6V)y$w>5%&~YFY|L%B(weNt^s%jL4CP2PDh(4=UZ2zDUi%=_8zG zE1YHrc$HF(PyKtc7I8n4(HGRLw?hrkF_8y28Ho%SkR4|J4{CZx@zh4J8#FW`!#a3Y zX=sjI2d%Pato%_B-5`8_rnFuU4maz~nEu51E-pt*(Ig|!n~lJgvPwxsgks|foc~?M zF4IPwg1t(=Q+S$6J`%af`|QV}nTdb{{Zz;&3W0vU{P~5GfX_bq3Frl$)01R)38UK0zlRc_+)I<%A@1YctL} zZ<81&pY%_%Qk5U~wS}7(T8|)khL~sE6fZ-~b%lBc#={YTrNQ|MJbMwXhv)ZHh8P7A zY4HsoW$&_OViG`4(Z*^S)ifB><_fJEdW-{elcaz|(Wn4H^Cbxhp0 zA+RG*Z_=ERw39}35MEb`Q{O^-DeJkmE#ph9qevn{uN9r5%s|_mI}K)=C>SrW_P5Sz zLdGj|Fr^@OFqQ9&^w++aC~r8q3A!>EX>A z-%Oh_6AcJoTbGoxIdntV7PTUiYSUDy$mG;+MHS`Dt-if^hh62=dhL$i&X+0rRoKc)bX>6H^tz^;KCA{SJb| zRNG;r3QzL!I5pLa|5{Tl_*$C&*5-kchmhUvEg!&HJ`%>>w?zPhcfV;P4J0b_5OOd` zRDNtP7%@{Ct{WK)aY`*Bmmb{8$8i-pCxMb?IwxD99YldsnNX-rjMm~ou>+0q#`WV2YJ~DAsbYjiMd`rFFJ~-6z`-%qtB}5=CmqIT*M`;-e4e9gBg!mjGPSopa{|kqGS0&-{lxuY4CY*q9L2S-EmZ;KZ z=W`caOqcmem}S}`$YSB>R5?uiaD5x=xCG4GqITW)Jz#zI-Oyhqcnt0bgGqtk38Ln@ ztFuOqq{!0p^`z++o5W>}Ku-S_<1py2N6WLOY2tiNy2tm;wEwn+>J9QGN^UST$!luM zmIo(C(pL`QOFJobz-*jS{xdk1_GE|rz9(Re#5^cIp}4DXa+azi)tmnvM5U1a3<99% ziMQ~fEFiDu523NfxZl6)U6!_Cha1$u?mKu7&I^h(K3BsA1DNau1mX*<&*z|)7<}yy#Bv%dZZQfiWFRAV2|WHJiDNX zKF8qyz-hcGO0jS>&$1W78v~L|*F(FY12P{577@psNbWZtn-D~(3u%b~?VM+tJ|Fc! z^n=1djacZB7VFSYKTNE*uV)wAi+>pK&v=Cfpa9VA(-ZnMn*Fu*9WYby{MSx6MH)sY zZk*lyrB2^?$Rc+zdgf&J*yQw-+fc z$<+%Y3{UbBjN=djS0h5}ldA{~bXGFy1ruTSf5OD3D2om6378-ChlM3+v=Npb8%<$U}bo_=I8sySf z=p;|=q%2!cUmwYyfyU*q^`C(O6GO8nlR-%tlW*ht>#B)(d zJ3>@vXRrK!`G!64BTE3-s*QwfYWCaM__vwa5uCdVEd1S46F5*D1e{I9*XVz)swpqG zK?}gl`oEr&zc!3Iy4FfMQjGe*FL(9@s|Vi9*p8#3U(d zb1lBh{c`?OtJS>lcm&I!u+5qRqf%;#)TS#hV`^@mIPyMyOrFk7#cU6JiH2Zm2y-PK z6&qQx?Gj-T0fS*7O(HZdDo#cUG8Qt=?)bdjVVw8nt>)p^5cvO`UP65^hjwGDEg*s$ zduJ&!@0X9o;`VN`)tqZmd~;HGg>#gaXe~ugG#lydA!kP#x?i;R+WPIBYyyl726%a; zbMgu=_X}YZ%+^wJ~xs z@k>6wwk_w*&Kn^yfSBcp-d>!HJHhv6uU!o@`A)}spvZY|>qocc3=+&kE}y03Z)vN1 zB}v6Fj}$=#JiIiNob>GL2TS?9AE`tnC4@3bRnzi@WTa!c9@*G3!=j!1Qk8G@Ul`&u zr9l;K<=)GnHY6Cbp(=Fc)U=sV1)3@_>ZB?1H3h2W>a?+wfc(aCe2Ty{KYkh|ym^o0 zT;c5MS7+tdd@@<35}9>JEJdh!g+~MncC@+WdbRbYXR^zg=QjLL0!D;cyw4FhJc4jO zMt3`RzoY-+l0R9aHPPlm`ru_q=L_D$ZHxa1OqeZ+5I#)1Gg$0Q`;R0vQyrV9!P|HJ} zJECD8C`Q4g9J9uM0yT9h@))nR$p${b_O|yMvG9p8u@$NE){1s|Q0&n~r^_AMAd}WM z*@OlRqw8&^LNKm)eDyYQyO569v1LQ={q0Y5zaO5@0M;g?d`o}mU~j1r(u6R>jtoCl z|NV17Z9ku&zQ@|fZmPn(LUAUqx4Hf=b2Qd^pKfy(0(NvYuO6EALZ|V#h+v+WrXMYA zKnA;_8B;9ceW!Tt7ES|HnNNtvQ{mxjnCXtZ0Fa7U{=J#LIMDzv|GioRMT+d(j@7Tn ziLa$gN37RVHJ;tp@as4OyqacEk%aIwdG*<^S$sKX{qfB+a|ttF@LT=$5$I^Ua~k+= zjC+zfj32$;p5@4r)*1m`kyB3F)^9~mi+)0%4^W!U>LH65D_O(y-M7L=IKq=P7M$OM zuP#rbi{>*wRZJV&i~4OP0Pl|x=Hhw8?xeixD4gZ0Fd9oSxYdHfv(Jx-;`N*>xd-|0 z88Wy)@o3Ox6{iW7eU8-yZ8MDtyRq5xCc4!qQt?L8fMrh}z-+M)gDi!5%9#}R8i zMwJf7b8eI+?2)upfwv_FK{gYimsaHRN&RZNOwRE&clBB|ru7LGMDeG0Uii&tVPT%!Ivs1ki))+REjlp`Fu6%05Oo#><>Z~j4WrXH$6n$il~D|?3C&c~3o2AjJ$ z9sXSI1p=gz%)HTla_rrQauIweTnJfscJR=1Vw(}4nXBsu12+SgFKe+A$b z1hA@+%l1y1ldfh}oqXSC=uGkwrU2dB^x6-6q3jaVfe}%n-aqgJh3qJVpmfG%f|!v0 zDu27(=~y+PFCO4rUD#Ewco?8IaJ#@${2-(}`rCckZc29bHbFdIG`FH}B5qs)C-tC{ z&r<19CRP0JX}m*xJPm!1d~!&PL;G8bS573k-XGQk>|}pW*QU3^tK=zn^8)0WdhK*9 zlj;|c6F9Wc0&IBCh;QH5<)C1Na{ZR4yA42JRwF(0y4(m-1()Y zogFdU6QaRCu1Sl$c7I-)U>|P1@Q1sA>0Xg6)%~!|F-H4=^v(wluzkabA)*d#BxXuM zUQj+TyqdYkjNcGQmVn0dvDJiT(Im$t8@#W9K%^th#nMhf&x9E@GZNWB+x2O`N6wn02ef9Lv_DJW$3Fx!q5!6XpDkoq)%j#}ATVpGWNFgVU{;Jr!J~UGCG_S>FiQaV90fiK!QZ&N=ryvt)HqRXuA+`9={Ng1BtfH3ZTBw$Zj?`d z216X<^Msa@1)g}&zQ-th{saReh9vvH0Hk{d68p9go0Y5IFO`7fyr7C?HrIyNM~>17&#=GydlzkTy;@m zIMOvE5GWMit{HK)J8jZsCwwo+(R^z*yW@B)Ip#tz2UG#+e{SC3@Ss}qKz8n6I!mRA z7ZL*m0Mpy{$n_Ey)u{#am`-h!kKzY+!|rDpdq};$aDeDsS5}u7_QQiGan#yhgm`Fu z)NYY*MMkXqiv;g7NK|bH@o&YZj$XsFe|~-4kw2Gi&>kdrYneW_P|b+cf1SXG0F?jU z?_pLQ+xb?A0Lok_IdiP`J65P}6f_yOuWt}(fG#<4Q-ytrr4@v}H6HznvN4F4%VdVY5Pxth2gfS0GuK^5(mo@AR91ceQ>At-NOaEY)h``)|yk^|LZ z1mt9q{F-smx;yG)e)gov95A$j{Hp{yIU%<&+-!ccySuU`vw@MuB%&Ta{Y=fc6(0SP za&N9(0C{|b7r+O>UzuX4gW+>VrZCt0AmqQE77>1fxj2>3khmM%dX2eX1EQs2#cDr` zm0FkYiv_7Afoz?};$e0>Fg_}O_u42X0mzr`lc2X&OZ{qnnz1G0K(V##VMU1FgfI*p zL`2)joG2?gfJcF9;wk?vrD1I?Q8K?N$Mo^2$RcxjaT<6{hD}fBAonlfjWca1&3MV2 zQ7ZSBuDgT zi#DlsZ(ENvIHY+kyPZPiL5a`U+d9x&^!gC{pR3ScT>GCM4)K zL8%_XzP|Zzf!DTTPlsi8IwttI0?<_tl>Y0U7zZ#=b(2#TCof8n4pYf3nQi3Blaa?{ zETw=vtNe$zH*0j!7uN&Rrig0klqw__7X}P5*tvgV()KEON0?RzT8Tyqy$ivJHW?6t z76E_P4{xU#{Em1BGvLNcPOdeIV7$cqE##d%K!{vvs*pP+8PB;khKA@L3V31NaAacw zI6qEnh&-(YV&8p;8|S8XC`AlA>0T(%m8C`yu@5tdHt=)t`b_+3<^*c?6b+bdkz8}7 zD`aE_P^C8WkFZwvv&Ar9Hut|xphZ$5Y@LS|dezcRUSxd7n%>C?8$(owNbDj5kLkz7 zCi*Fxyz?aXno|WztcV<&0L?_gyEG$BCpFDa*JXyOCU<&$4?@ZQ0)Ec@>S7NvAlKjL zlaQLeWDjGhbZ0(w%%K!>Zfh6mgC6QcA2YZEl}ARnjYld;g%SZuG)82tawf6XG^Ol2 zbD}3}9lOdA2@PgV(lOl-uJfXK$=LkEe{fvhK;hWl3C^20c-6qN24FD#UtxEEg2Z?>c3Bv?e-$J|mP7ImzX6@;A>+8t&iXd7pRcKM`&+6M(YnR}ur-?CM|ClxX$4!B5y zibrE>>*(rUGt;zXFepDexEC%{sDp@k)|;<4J*%#4AoQR z0I}jpy$C*fj!Dk<>>OrSTtcLKqF+Qng6r`JLgdk4a`f~c{3z$P zxfGx>R=`p-H1B-YMEc9`I{N7wqC^xCDKkolnIkUjnN-n+N;^;gu9M&Ix#Iq^igxYj zuGbskr*oGW#4>Tk7j*QkpW`XzYOQ3Qv{2N?*xHJISyBU?$e*M_k%3^hcm_IZy1FIS zTYp?a11OqctmcSY-s-zs4oQ9xf;Zo3I*EyJ;Mgr<3%emcV7gT6(YU%14)KgPd zwDEJkiMXcvDu5(4XlZefAl#_R%!=spTQ0$O3mqB=zbNS-DgWoJ?E(^n5D}D}D-8|< zgb2`Va{(SvSc9dGe(GJVDT0EmJjx6`j!Z0^loP@u4=Xu{j)BpQS@YNCuV4$J6_kx0 z-HWb&PPNZwt(fQa#}1;1k>1D7OHaa0$jwpqWy?3He?7Vnhw!I!TzR zs(#8d)fFE>&vBcFtRlK76lyge76QbXKKy}L)%pz%_l+YDCw1C zu#r)UkDqc8KTa+Z*{qc0nWRXmd+eD$DVf3w#r!l(Hn7KH3S}0`Y!!VfO%*)qzc|u# zg&B%5RpiP`mRW6hTB39nYD!jAvMOqpIcY;S6eUzumKpy?9yq7~mzAMV1<0sJrOyi! z<|dcrAXCu-K^Ghr4nYdQI)|HvV-;P>Tzs%|NW5iSg6@nlSuv&=PPI>~7zr7}At4d@ zIjpYp??rao!`cn;v#&X4uBJ%d{|hfSB;d5yj_#T^xQHvU3_l$&o$4n;K`G zX7FM2)e_T7M-D!5>ga~YLEJqhRyQ$}OS`I#noJv$$ahnm_%ta-YO{4m;NOA8GxI7r z(i<<^_MwxRT%DeGzzDoCc)f>mzJb-I{qV8iB2*pus=s-d+P?oUh$HYgdxoJ?WYJz( z;>65ae5?9onK5>pmqEmlRdQxly+3-*vVD{E)55G@ zR2nc=Y?RRRi3pd3Hp%ThMH}ye*&YY=okliB8R)lFz75s30#|%qSbSKL->S)k%fUL< zIx!j{MI_Rct;ITZJhex)@D=$cRfIXkLH67}I~e;Tf|^;-t{?rP7mV%Y((hbaq5Go1|5?PK7-_ zJ>%x*MC$rP;A>F3t88+D-U&@GQi zW`A?-M=Bsg$==$WeZKDKY3q(vA-sRWw!&hp-tEmT7dB@5<#1BP!);1usqF{~hL4~% zz#|O;Z^?=4m{ItzKQD;L2N=x+%rSVO8)=kkVo0J!Z;!7Z;Ex1(_1nTlL6;Dg#; z1e$ks-Z64nZRP@4&Hm?%0L&%ClMvIGU|rq&fz$IQ#`~Nq`6Y{~ZJ(Tk<75eJ8Ev-? z7_954P)6!R5H85DJr=;owlSpS7wGIQ>`&NO77htbSNoB`CZ~5p6TUc%gFI3Ec2X{y z-eRRUz%`r43}1&bNBtsvg^+HV*!pdY${egWl{C7?N`bWmTuGA>a_07kwC(&D- zfGxJIU$DeX*JAo&%S-u1>$D0r-NlJ4pe40U`hYm2b=yB~{QETGliC?=7Kp0QBAh<* zHEi$Jb>Yzh@!$FfiFauU8~0%U^?~!qtUD)6fZCUNYj^ZpvC9i-ZtY&^1tadAvV2*5_`I}ZOrfL92yq-@2OPjCW9Esmnje(~R&Vmu$BZhGbk z((N@q*){qLxc7HB5xL?@D@j}{yAS-9Ce@2OC~4P_wrexWf=W(8w_R2tP00^lYNxxYLcz5QisRTpCiWvEo_D2(wedA?a>i$b{JOqay)digIK;97LmsJRf z2SbuzCNOa4aNvgbA(Fg&DoXUIQD>48_VJ>;6}v)y)StnxB+YOdDM1x(3l(aLJ{yZ1>|a2 z1yTHWWTj+%YQ$+}`&cPgq}`1(H@xJGj2Ogq3%}-5R(7X~m#6<1#l~bxMF#=N6(-;& zecpVwMl01ia2=D|-dSh~jLIj-K@Kc5yk6YGxS3DP9ma0h#w_M2AgRKc$B+ zWlv{alr4>CO7bO?|G9X{^`r&^B%@ICrTOPU$dMp1eQo=iS#=CdMi)wQQ-?*@t9qai z>Cd#opvBDe6k7PYMuG5G!^Gr`x`kljrjto75!5cwE{MMuIB`zTA?mr2xsI(diQuLX zr6Du0v~H6eHhdv2Hs3i2>ncH$6%KA;iZM*k)#ty0GjX#}MMXQVn|dw+epC5gwnf3U z-f>b{=!uBr3kbR_)_77k(k)i(iaaFv4bP$xUp6JBTtlRs3ts3#x3%%Dugbd9uXY{wOm8~iQ{eKBr8jWT;u z3?V!n7naFr*|Es{ueWAox@c9b0}J~{#1 zlihG5um?*RAxZG6LZp`7)|4G|lXv!ij@~?i+5?Oit6E{+b&&oxN)eA$kA#?g>A<~_ z_OI06@j$DqM++-@!%o84+ZzQbrAO?Ecf;dD zM;G;Q0Z)8pqmFDwCCIRqY(PzDV{Jo6lowE6^yy0g+L_&+s{PhyZ4YEX* zLx)$pQ`ZIqTo}W?4q`gU$HHqJvIv^8guXcz;IlJ7SB(h`ot<;%`5nwpOOk9lr0@0l z3=e84QwHA+AHa{)p+(|HgeM(}W5+-Za^G`dLk^11u9A~M+VyAl3-j-R+L6X7xxBw4 z^4;dwE9_NN+v|QW9d&PcyP_oZ`Yyv+?$z?7f=4Jh}?#sLYh-;T(nh}r0>KW0@z%= zQNgGEIGzzUq~gf7uebb_>(ZtvdoSMqrU}4Zp0pr;XM{e;WJ6W5HxoNF@hDBJMDD9# z+(Y%6G#;r->*6_Aul!g&_JU9nWHu;W8nuuBG;TKAkDPRNmIj|3{YHa%_i9{UgTKkz zuhuZ5f)F6MUh&<_YeY38J>k}>IsANRtx{b>_^0Tkc&K3znrZ;0Hr@4aP=0oA!{uoM z;SFi+iZOS5{~Al7X4vZ})qvOGX+7VvI;EfaWNTpC_xXOzgZS9*4HVs(G-lKNF?fp; z@EqM)m+9Rtl}@V$?$C5&1u0fiJfXbv`{?mlQ+@6m%uv#czu*z2fPDZS25~cM?SaKv zdXS6DY^0o2u4IbgNVA;G&BnviA%!Myu4=|{wli}z)zv~V+I6*eD(8pfztTY?oz%ZW zTmx9X--V{vg1aMh_X} zc~!Gy5k~J(ug}8W>>i{xa~qj2tH$~gyn@*nc;VoQ4OjKe-v&&WyYhIzd}oJq67Ok^ zNn=tCVEE$N3C5nW%T~+KrBaUpc-dTkZ>$|t@-Dgk35q}Jr)o(R?Sh%#D z`>SknGPWzY>aHH?mrgC&;`2y@QBSj~VzfXVYb2jNo#4i0(=yhU)6hBtQY%l$o3j<( zBfj=sUI;!s&~zZ=C5H|_=9S5=6_7bm(Q@Q}J}6{p6-yaVZ>wq@Lul*_;1)G+Ef6LZ zlYpdgq8UXU%BsU-dHKN0i85)|?6xr0B9$lOk57R=v@?>k$_qjNv~PFv{(MzleaM^1 zaKa6kkzWSSwC!K{FRJeY=o1g+yKLuD%}f9vLD$DmAHvfCzr}}nRphNmtkcjNy_Gjw zYQIqb{VoaTWh7DHbCfZ*z{rfS%}+;@d<0QGaieFT0-g#iBMAL`uF-ZrvylrR!<{py zX(K53T;J2DH0cUJnKyd%gj@!MU)BhCn;Bh2&|xv<_)VpuHIX_sIJdk0(u(`KjZx{1ht3p>&YzQ=I zF#d!7QSA>YI_Doio%Y_z&6WGE)n8h!v-|aS?H-c2Y0WXS8H@U+o6%4fQ#xZ%s1q#u z!`Q-4dvXHFNDu7B_KahrY4Y3Phrd)r7vANjypORbimBB)iKJHaq{cAv=x_pYe^n3M zY3%&6caW7dZI_ZuizhWQyFoF+r4FIYHrCF%{l}Hb9eHB_&q)+gwr3$%anW;W<(`@k z^@FW3v$QPbRckh4GVU-a7z1VJvSkI5LW5UrP@U)nQ&H7dnw|w$l*t2#=c&7lk04dC zkt-whh<0=5)iMqDpv4U12e<5cw48oWZ3Q~ zcuqz_T|(Fanfi!*7Foexq4=HousAXID@e*sU%BIN&%RriFVgVoajkD^pJhDk`ml!RZ8nxpb^Txaw3Ia~DGD)ZQ8tLt3qkaZF2{4?ljMnU zWu9*Z@MIsI(RcAoS&FK=TnUeeG|+TQy(V5Yr2>P zHbl7<1=f&ih z5oG8=Yt!=ATh`mhu$A}!E9@+y>I{}O3Is`T2@Z$g?iS$S?h@SH-TlYi-Q9z`yIXJx z?yiC0@N)0$j^4CaPkL2P`m3(JWB=fbc6S#z(JDH`d)v8)X6NKp%*@;vcnTv5-WzK8 zs044{%^QQM-2%^^6oqe*$(9C`A@Dm{omywgBWqSe2s|5$G4g>($?%Om)~Jd%TPggaYtU!Xb@@efj5@ckQ9KoE0UkOv5cgM}(! zvQU6Psj{I7xye#tO-!_=dudIzZt=I}h5g68t!+=sfd2J{+duKIUV_gwpV#9@(^5~p zXi_i?IeK`Am%ArTVG_`gWm@?KIXwKc0xn}Usu+SD3I4y7@n6X3$Z8PP#Qm(*-m#6iLP->I=n`3d_yD zVx>ohbY_gPOW@*kvQT}O!l;l;yExuwGb?A|7R1CWm>D(W;R3qYHy+v*FYfWKB!_*=GVgj8DG*NO=dcw9HQQ7#IJ>x!vOpC}k1#1*jH;Yn zst47eiUS8JmDVipiU87pPYZp5w`pa0`pMdbpHmq1T9&jfOQPmRd4*bF#LEK@SWQO{ zPPg^jgbKr5a{#U%7#O={%E6QCa?fREVs|b|zu5RV2W*m49}r`crBgksbPmhHmq%M@ zRc`b2Wd+QGe%}Bgu6Y;5%A&9$3j0F|wNd%QN_%Bv3tug1*rPK9xs+JTH5Ld?ZN}3z z1-gXWl%&fJ7T`~L#{+MRHcM51Sd}#`a9i>=#s8Li`T;;D1;7`Rlu6J;C@IY@AWMl_ zl&RD7Wew(ZW>HZZVbPV5oiZxcOE9fjs~WgYc-MJ^KdD4=($fsuh4~DcSzwRRjeQx@ zOX*I5qX+QnN2CmsE$@@rtq2W#rcBGGEHizCbwVs{c&)WAL|pw{bzGU~tXHrHew>T{ z26`(EDuBL4tP@}n!Dv^cs`eDOk_Fqban7gBYhJ3gAPXmHayMV7>rui~$^aVwA0UR+ z1Ke0Dm{lP7#O3WNCnr`x=g42NqdV6$CE@iAIrv((C&B=k{*8o3B^1yhlaRG6`9zPr$O}l5 z;3s7K{5{SM-^LppeBaU9Kkic&_4e`<=H}5!hq7jqMPW3zQQh|Mt-tQGjX2UO!se@N zA7o=C0yi4fBf(C^LFI4smptS#pKgiu`a0CC=fdyH5Llz1i`RTC^LW4bilMJw`Y_;7 z8nDPjHz7OBk3wY7M#}Z`Y>P_on=nDP4n>} zS97A0?YP~TgAro~X|Ek?hYAJ$g%9#;Pfe(#E`PQ5uJG;e-fnRRp?VTWfC*+YiY90k+5rm3EEia{l6Ee*3 zGhgK~VaO?S0q>?4K7NE4$xj1natFhKM2My7UR!LQAvZ{m!Y3tp4@dTC{_5N2!@Kww z==^)rJ29X^&)Ha5D)uylF;rrhj#d8X)$>$Tv~mBAeSzss1>10oiXXhi&IA#Q0HN4D zy>@RMMmn4Z>f7Tdrp`6|`II(Bts?h3v}N)($EW@#p48OI9qXgB51dkL|A;71i>{|Y zbi#M=TdV_9cC{t30I?0Irf8s^b>Io;q70{+`aan=zUlSa+dvKL^hhS~JmTf^lXxRh zg$#L$44LUoZ}0j}@FMvxf5QY0@K|&1sugGCw)CSl-gQ@};w=(8Nyqz|{7ZJsRTZvY z&It%DXFAxq+EC+P-PInW#JZ50tLI)&`1OggT#P>cOB?QLoA$e0Qf@`PHNu)wkN=R8 zwZF*&a9n5W+_H5=y8C5@w)kQNMtWIMaIA%ta85N zfvPpO_?^Cs3gO}o1hpH0Q6zEFG4e)_bKAWPXR zN)5(sB*`W6=wA1i?DBnfpBx407X%_bx6HVXEJ91O2@>x1~# z{~5z;1trE7meZcVV4?`#B&dQ%0se?GY3?Hg1Dt7#f>8O_tj_$m^W?P*8tAY4=~Ze> zOLL1ul-g%0u5p*yMS3w;?*26;v6dyfcb=>2_QkjJ!|2TAU)A@Q#ei58sQL8b3s@Xa z4%ZuLGFPaN*`lQgnLUI?!J6PGjRG zc|D!Nwf-!buqmlETJ-OZhnK(zvg(LB-}WLWP09z!{i`hI8tdKLulSTT4XmP{b``dJ zpXYYyd9xNGV{r8O$GkX<@#hmJ4XiL2W9C;#Y!a_Lmpqu!fI3O0y$QX`vuX<_l>y32 z8riiEylW$;dErE^Dnl(5h)A1ZBdnT1P!Bq8!~0!pb9qjzpo33cNN~a-MS_4i6FEJ6V`~t?$DTH1bLdHU?&$ zXDq*ZC@N`k05cI~PbE{CA{oVU8tJ{XbAxJMbB}iGOFLaAD6!Y2C!~?qaUOKO&6)8F zIVAI}A%?;RrP*paJ}x+wlAFnuySwoKF*4ds6^+i+2}bS$t_j~QmyS#`(++diHH@{^ zkch)wpWu|)V%b%%)r^t|*PP>qHs`6OD!wzQY3=Pa0G~WJ2kh~_>#%iCpnxOY(*7u) zvG2HRY_r-Ic(3gujeBa?AsBUdTO*=%Hv<8Weg|cf+gKiylp?r_m*kIi9*VjB@Vr6h z!VjV8sia(jrJm_o9;Q!gUuhXPtnn~mM6^MiGuaa|xs~Z8B>_1OP2_EiT3MF!RR+nD z=;UG7pb>f1dF? zkb`iJ$oX~H4+3qL8FPqp)U`P9H;Qg>wDqzUU2nRUcw zb1f;SA5ZUmmF%qOncrCEek0O>{rj|!K8VhfK)-&rcAC}~r-(JPB0P#gBqhxdVVVk{ z^k<;S)@b&iKqcnU74;QT%2%$a>D1;fs(vNh37*3mJBufFJhQnYHX);;VL4NkC;>L@ z4R_#sW**&Rm-mr8m*2z5*w%{ksOWuaENskzmY1ACPGnC2SMezjV zcD=z`-jjc7Oemt+uKyY{(rh4qj*AaO%Wv-?&2E1IMOWEpIngG17P!qXJ6O%T4NjYG zDY1KJ4VPvMqwvSO6flmANS@ABxSM~IQo7Gqj+@X(B>kR zqiGk(YZDoL31$B%r}AGAIgz;jHjGTy0pcen_v-3dW3xrvf0N_q*=f#Mpi^VnA9tCn zINuF|M@D(^M2LbySdAp@4Lk>Us@uKY9Fnc&ZQo+3&SMGfahv&grCZ{)9Ct2%o4Nv$ zZy=UIxIw!&ShXVku7kQ=qiUm}EiV>KulxJYR^ zHBwaYmgkV)w;@cj=-IZu3a-mK5ywXr{=h-lSH}@o`<5-_|Fwu6IW`$6im!psu1kv< z+B;?|nmLK94hVE$QdP=7W6U!c2E0fbXpta8JO{$;(ujiEDdz$Qr%lWij3HB#dw3PY z<@8!Z{u*;QSX^X;BsoeSw*2Bgy&Ey!V(|5=`rdi|QQBzmaKcTm1lmvM$2U@x@tY(R zT?%%JXSb0IUA3CZCENjStRF?&$LyLgTE6z%&dWSJP;PBmlORtr`L#j%41E708PL?o zck%d&bVcs&?i5Ns*~J&k!ix5XcKrk_u|x|u*4#GR5es-8;36I@1hYHPTO0YfSn;cz z|Jt(W%wmSH``q0p@)>QLt_$P{wjsiHXY+h0&^*!%^^4Hoz+DB}8i+QaepcY36>G~V zddT?z@ovZTx2NlaqjYlEt=0XpLx&6qM^)duZiy^GWV41P74%0$4o=2=?^lebX8#WVNuW`rH)zO?cc&n$eri)Yu+b(X1!5N z$D&Uqk;Mb`_5Hvjd4ihjIaJiCv)A`OLy@6dfrN0s zni6y)*USod;CPN_W$MQSQIvl#i?An|PER|^A?Sr66Yl4oYypuwES?#H8!qQ7o6X2E zH%N@u0!`LE<;_C~VQwn=p}x->1+vZ^q7SpQXVQYjEj2JGU}W$+0yxb}XtsBg5;2L3 z@B_I!0chv*b!h%5gW^_eX`pB@s$d%TzgcF#6Zi7~pr*hcDp-B_rD6n~EqF~0RW4

L;R8wKtH~Y}mI!H*e(LVd0-`*syjw)S=ymYy#KkP%*5U*d(qo z9L!;u(XDkbxRvGX5#)z{`-V?FYlh9Or4D4t0BKGThfMN8UBgh8>r4HDQUZ2ETc#c+ zFJGy~h8?S4`bz?oCMa%6nG>hNgqd-@b(8GrkrDMiL=V$BWtVmk;KbQfDb_%>!%nTxffzjJWO z0n~Qv)viZd{sg?r1Ba!xtJ+j(6cSCANxcSNvhyg2EreWNH9SfAC>iqHF6DW%T_>5jroK%|P+F~rv}HRONAuao5J2C9ddwBT{MX)}()VaCv8*B@6JqPdBggNWyeA6# z#|y;j>(O(O!A0LESn3K=Dmx0x+UwGK0Qe0<8@qZ>lc{b@4g9Dn?R2@bPyN}z{_}sX6udCma{9UE%&37eC4Gh7#GCKN>k(06u%q(1R zeOW~lgIG4oY8IaOK(7A8MlBA6MY1Y@5{MVq%4u$`$i{id_}hBXlanqInHlK6 zH0|$(9}fl*LWZ_v`#8W8j7NH;QCs`M^zq3HBRBDp1)!VUvFT#ylW`1SC3>QucIqbR zm740`KpAgOw0Nk^hZZ>bW6#k-i^g?)m?zA%I{_g){%}uPziGNCGzCe`magwALK)InjN# zYSLPoXp0dqQ>&7GI@ei?rc|ldXQ%CaqK`4LrBaes3eT?Dn(vJZBsMdSCcD%u@$1{3 z^Ld6ihNT)~Ns-L{Ld^eb_W{o$Ex;n(kmQBV?=%55_B+Hm2H@ z0-j`^qpVC>i{ms?aC00;XNiB~#%?NXStjE)x)B8CbvNMf?SY2C7>EOf`TD9Rs-{AY zqdTnm(Gyt$5qii-Q9`G2ySRi12-KZIr?c3BEK$!!um!u=M{ybjX->(KL)4#akd|lQ zlDqbb)Jvc@oyPL~{G`y$aS||Rs2COCfI^z9Z%r%$VY(Ku7oUl#O3kgoJkH4E>9grD zu%LU~9C9`oRanyJNp+~ewn_If)>I8!Nja)1h3rPvnD&WEBy+PF)Ov?Ru99!~3)s_! zuanfyqr^4ypl7P>g1CZ$7G-~k{_lQkv`GoLkR0&dhlEHJ%7?skDCLlZjg+$KK+z5~ zrp3SZuRhk>z-g9;BokW1s2LkLwO1?758iZ#J1l}Yn4%04emRL$$Db8@v>?n83JP9Y zDNZxP@ps;jI}$py&{oPk3_6MCFL{M^B4)Ta5%J`!5oPA-(`DEvNSpa0%yuW78q>E? z_KJ9g@y|>)dGvfSbh7HqaP{DGbM+_ZHRyn z`McWi>devgoffO>^=_MqpRt?aw(w`4NgYvQR~o^6F7JzYfljdyjaq>)Ibc{|R4hwk zUBD0&f_CP(rPj~TuRKuFPxei9j)S(lv41JZrbvXwARvo8rUw7xgUqde%hVX-n3hJZ zA6`C@WtvpZiOG~)w#~(6?n`aaa~I~%Ad4l6^nh@~aOf?JS1(p`n7QT>^MYj`i|ghz zc{@|=A%Yi+%5ke*w!%h$uG19NmvtzQjVGTnqN_-apygx_W3M!)jCKGLBOIr^7Z!=g{xoe=23d zOt%p&IFT`o&f;Qv7etl9cEdco+s)J z<0E%CCNNCeCSc|za7Z8AHF%cO1Rdhc;5}+rc=TAPaj(Ir#!s>!bLRFD?h~d$WWZJ{ z;Qwu+kw#~!a3LOu0qPHUjv+!d`_n0S`!j17=B~E6sTV)QV>@3$-)D~Rh^H*>$e~o{ zL$wVb>kE)DF%GkkmS*IdVhVOXQ)^Z+&o`BglcOrCGZ>{eTn`cl(s(JQC5#8*5YMY7 z9b|Y*{E`4#%MLwQwo;_jkHIfj}5}t*&!f=>;cUW9y&% zN^uN)E6Wtk<(7?33$_cze*Vcl2}hcTw*13s;!8=U!dYl%tmaRF8jxRu&ZsY5lKCODk>qfsFLy}cDBR!XF47XOLO>(7=l3uPIQz3qp#XRm<9w^a5x{(vZgGQoytVTd=aB2ahW zYoMc^3A~66R@XU6Q4jQs5L$503b;sn(7BL-$G<@Q+B?3m7 z?y~T}^o>4K1l=BcI)3mJgXX17xV}&{s~<99TJW)CCsM z3?KqRi1MDG`Pdng=0~%t_3>vK}-aYb&B@%`|lA9CR`fUox8EOKFsOR?_8q z1Eze+Fb|=)$;hoGWS`CrX(Yyqp9aUR^{e_8@+F}ln<67|Ar|>Ea_*{pI?IHOb7ON!(v4ge^% zRMV0dODxn^e#W7GL`k@cWSI}!NJcomu%0nR$gZ#0)CbdKup8hma?}T~SsHi;8(AAH z4wrE>yJ8;7Ei%{092oU7der6qc67jt9uH2XmNJ`4;%D$YT=XN-;1$J`{#1^jGlC) z$W?R4r+xDqqA+6COjier!wrPhR*w zw^~wTo)r@NTZWU8&z}#(!=l8oI}#@x;_F`R!L3jsEUGEC63y8nQtIm^#uA2@cf| zPfmDuZ1Cxh5Sg%K0P1DJMu{!ae2D7jqVJUq3X%&9cADTKMzUEP?M$}+T^ z*6)#OQrz}ZEpme|GP@D(_&)J8(NcrHbSXPBKtEH|=^$ZV0IDy7_-JMy(zlax52z)w zg4da!%jUEV$P5pl^#ZhSo9l-*>TB7A)9iyMxYeBDs@YsOld0pg*(=g6m7gx2I^693 zsuk4IGte9~Y4&41(Is?Z%BOy;^F6~Z{5v3~R%yhQKyvdtPlKp!?U1Y4 z#O2MzTkJ?Uo0g{fV7n3RU?3J&wI((ewNmwWP4#0hcIT8>E`Wi^$G{?l$sn@qrMV!| zZf(WGtisx}`O=H!y3uuA^i%hr$&{9*!d36O4-=+=HiLIdg+qg7smHx}J-sfjCxefD zdNcf}nJr=g6I}vm`V;+LKRm+IX6-7{37Bt44!xfUFB!mu`hPJHR_6p1Ah0|WP^zmV zSG}2V50{-Gxd8?~MnhY%+x_LeOpkXr=FG!+$#};1jKf@U0PCr<6-s;WgpBdsi52H4 z*EBGaKSI=8=g{ay@SVD6G}I$r?@;enk#D-@R+4Y3=8p#7FYG7Tce8i)kCczMMiLG9 z1Dz>T(J5=kDQlrp)4k-?&Frho9^cqHxtNS|3ZgYz9zgqTG=+67qF6rdIbMmbD)h>^ zN=tsR%8hs2pnKLiO;4t#8uqC*?bn|tccza7y*V8R{~_|NiHARduWTc{*W;kV*R3a_ zVtUIIiF1X4k~O85_wBdHKP?mG=Pj*_k@vj{U@E2{@vUY?d=;m^QiEAhT5 zx!Yrl0Z`-XNA@SGa7v2GL&NH&!`#vN!#{AS7Ky;~4WR+aX`lOrn2L~9i>S%RZT6~4- zIm9j-%$^Uw3nfC)?E#Zw5886WSZYmY**G57){DFOjjSCn5ZYH>A{WJI*2h!+UdQAd z`LUj*J}|YszP)fxq(^WCg3Zl`vV?uV z1md*EJp%M(i+MhoTS2t5LukMdm0ZJjEsGz$nDywFHEiLnm|gYpH3!5rVaPv;?VebV zSUnpNzV^F0S++iL+o~0Q*{wWVJ+6=d;-nYbXzg=~dp){g2@Xf$xl)G(y}Wv|dWIE# z;d>li!V|uL?4|pu*BY0o9TWO<+x3%+uky*$JnFDsf*8jcupV{Y@fwRF)^XW1KFKyU zApNw5`TcXAlQBp9h0Xu4Wtekmkn_764R4Ba^Cs{YPP zDTzL{s#aSqTb&oH%IO-foLpE43o}QfFXBK$4`LIraaJU2f~WrdRRk>v9)`hcv0C;l zPnPGUd!Q|o2$&{pP4$A*_vD6AE|g(NYA_MAjnmqvXp>zU&vxvnOzqeQwP~4}B!a&^ zjG1~Ufa2+I%dCN94-cW{e?ZiQZ+20w2qrv=&l?CPXo@O_F9!dA-h8ml delta 41749 zcmY(KQ*$m1kgb#K*tTukwr$&X^2WA}9Xr{vZQHhOoo}kn#mq(jg6^vB^{mxP5#Wu% z;0YW6E-rq4Xct#!Gb1}_&y8zcDF@P4w4PIqCne!Iw3}#>OGHAZgzVh*VXpEr36w!6 z+juyLvAWsUw&dYT`&d#R_=ZBO$mxTJx3pWj>cMyjf5W$Yf@)698w`bCydD4Wme;p; z;_t5BxA!uRaDlH-f;K(SRq6Y|?9o9bh5<`}1A+t<#%PmTILB%4E|WR0iS*v7F9YjO zOBj=vFpo)0ubC|G!Rht45|h6K-K)enC?CaxOGB^o8|u4`cT`Ua(gBBMk&@pHkzJ06 z%M)j+(sp;VM}t7DY@;H=U^zsMVAY3ir@UBi`n#1zR{ApUk=KF38K6`x4L@VFV3;2; z)NZ93^AgVi0HZn%d>po!jwMlcj&C&5?;KQW8}_oF3Q04MPP3T8qRiDma}UxK28moUf%9&RYPv3ak>N9_)sK*_)wP zhAHXe!#B`WOA9b^)dm!%3j_#?V~PO~SgP|G0c~>MrmX;yoO%oH znwyjO*2ey;f$kIP=lAo$&38p(q^zU;k3xL@u3c?3_mL@-u#flsZ#vM!lgrJ`b)x#= zVEkvq>ai;p{5DMZS;&3ctYX9l9NYs-)}dn9o&0U-VY_>Wq}>8sk>~^|%piPN;w()a zP&j6t1`bZen&*r94l;t&(`bO2Ce;wPH5L#3PQ;X__zV_TwB0{Z4g8wP(tPrU`;)PxNxyAw ziUXF@p!8)bGu8~*1$9#W40pxV8a>C&Wkq2H^#DU81MRdTD|nte`2sMn3;N^i@AFl&r1O_oqbA?mBx(iQl+{AKJ-5N_!#g9<3r6cDxi_ncEejxN&B z-)W$60;36JEb3_7RSclwxLRI1Urm{zqLDA4K(P?rC|hA&FG?nzA$%oA)NM;@>k;Ko z!92%^OzM1O3qFo7^&O9mQ9=_W52O3Xv~lH{L{2_P~()k1}tpu z_)$9`Rb`H1b6`s~F~p1clXvSo2$2Lng4~ppnP5{gEN~e(Xv`3_u zvy4ok!5n$+-$#sT0`Apj%+}2fg65u5JHvfJ(_z7X=EEi-#}xDns-Qn6)e6(-ZHFiE)< zi@)WuG|c6hQP9EFLkQpLpMUTrvgzbdLcWHE|NC;^RNrwT39}pI3p7_hY>{#4wCilP zvqBFgneweU?sl1mPdd&Vf00CkOR{PdLQ=U?ZGMm2tOSHc|3PZxix?aj8U@6v5opFIhZ``~eM*Y>L^mCP<8j^6FR_OQc$XlB}%u$CtV66sN*W_L_R- z6g5AGnPnCC13N>7h%%=pDar47zwM+_eGq(7K6jSIdI?iNPbK8Ab<*{22CqfgRt^wB zv`lslPXM?{c)cQ3vV$9_GiG}%H`Ii?v350+Be2aCZY>p5hT7C=D>$+<_ba%%`cF$y z@e66veGLuzqIn^1=#d~aY*He815Llzl$fMM8Ky+*bqq{6NQpqb<|IR#)Km)OIp401`t|4pMU+{8C+t7%f)bRH#k1l$cL% z{z+iNOVstfagglCF<_o84i2;aVhP10O==laRwA>##kL~j59FHivW#30tZuRSaiokY zfk1)kv3uYlV|%NVSV=I+S-}b=30$d2=19iF3)kXykAL?_YK20lLpj?1Q=QrKg~Dw{ zfLoT@{VBSYD0X7>VX_Q3+}g3l@-ZlhNX)DQSZn{%ZRQ)fQ8VMOr}r!Je*~ED-ZQ)( zOtqZinue^^($Iem)P~k4vX-rEK*c3e;Ug} ze9|z=j|b3ZJYdI2t)D(kVGjQ~*^#jp{zHmK=?o)m*>|JOUbgRZJ*|SBUb_o;dGy43mYxAE&?bmpi>)i268ok0!SE~kNBV^av%*jys zpLiI!0y>G!;O5?njORaOEBa%&B%SYCnMby~>JXoRE@+Kdo0Sod%1)+XSUxjI=BX4m z*4>gwGs{M@avRK&{qrh3*LRBu5_M5+5)5)bpmC}`#x%dYVXG9hANOh97Mn}Nvu z6nW~Hwe&W1CT_KX!r~iPQ#I~!Z9t1ofRHwLJ7}YRUjnrVG47}QHh(V}02lT2aSD}| zVCmUENrp~ZGyNWa6Uf_mUmdzr9yX;w8_Mel)iWJlQrmnn5*1Z9G&KuopOJ4Mf^rn&^5v{zM@>B0{ zJ9h+ffV~lZ2ji{ZSj$GUnjrh@$0WJgpKf`6(Ur3!-x|XoeGO`x!fw32vB9>gPYxud*v%jr23qfHNj(IS`bL4oo%*(8h~~)aVP{eqk3mi z7lr+lCJZw}Ga(i1B?ztmuAhF(mur>j2*evsW$A@3B>m5RER=clAa`Lk;nTM}iBVjw zrL~X0KaU>WpcD3+`m8=}KE3-0*YW!`Q`R}zdsB6_`4QlSu)-6c0IZ%UU;huGW18 z!R9vY7~?RcR;V&alH);cT@DV*D*sO#)~&_S(lW*sxLKCw%?N$Ms6n;6^c8-fPLmt` zpU_`XA3hPIUj1`J_!m@g_m za1aB--E?_+n&Z7q)k8PuD8cvQrfbdBb*wpX;et2jd;a4y#R^!bZPg_R z_`H3(8gMtwN*wIvtJ0x=FjqFBVP8@qsnqjcXzc2HG0&*#5B*FU_KjYl(t~6&PzRFO zwf%T4-WlZh{^ZQN&Pqf)KZ3-r`WFn0Ek#r7XUgV}vWc+n3L^dm1QIb*C@2rrYWh5? z%6-WdUP)nI84f!*1Twgf=5qxlMC2Rk&9?3e9DrHs7aLFEPeh@)_Q{pX3~mXmoJxH{ z34?_PIKG`VX7p1{%*PhDyNA=uohweEdd1hqQe^@yVp+$ScJVh;gTlNj;gNM*Ey!^) ztd57*kB@&Vy{=awMH6> zB>-%4nq7dJ+dk_{lB(UX4*3Z3&L%@yBTLtU#4+E7%vlh67xpenG~8pGi5ocA}}q+BQ~*O6j2nT2p2uV|v@s z<8C_08OI?ExiucMpxJ$YJY7|X)R9j7+?*4 z5*C|wVpD0}lp3A-Tmy;&%hD z;_DzNURl4cAP3Ev;SVz=YiByEzZc>Va+zJ@IoxiaJgrPw?(DVvop)k`D(5+ zmsT6~>txf=j>hrkrkv&OM%xb?L0= z`ZRtP)yT?Hc1GQC&aJrwt$UJNy*rqeU+=0X5oK15(=h#)GMP^g0YjfdMLh>h=tqg3 zj`}>Xt+U8(V8RMF?2I+64A2lqVTW>CR<^^E3(x)*#}eSkI{Q=>U)-~XJOJ8)$Dx~E z#gr?$bA~z0?fAgou>N&*GuE?}pon@4McE94RJ(~0m^ARAB6&+`&$j>_e1{kpP44{A zHuA~#_Vnpjb%OTph0^C?lw+@Ad$)>J2cS3H=#Qx!CO%dP z`4O+FAZg9uccn!)YyCu>;~h4Fc646Aad#)^V${Ex!9*&JsPM5E-cPFc30octWUe80 z7XgW_>ZS0)M9C}lPW>#E%C=i+*n_^T0?S46M~z43%>0~#T`?6!=INtZZseHe|(cTpuLb`|-1$YAD zE=fHlv`pfhy%5RNkVrY?B2TcFTpcRbXag-L+b&tIMSa#cT}>QIiiGHO*^b~6)`%Z7 zB!g18eE25V3KHFoK}FlyjJEc{0t|j`05)LZCeUW~rvG7&$vuo>O$)OJ1 z7L-xN{7LyK+wnZzDqjk>knY8U$Va`~hZ6XoeFnQHbqLp|!Yg+kRfK@w$JXTjeEzGQ z?b{x}N0_@dL%;U**UgWWM-`RAK8D%Y+V=PPCr?R2%0F$Y?{ZXLJbO4T{N|`$MJkAJ zf}7vA&q2Rs;)#{C63|mvuEf}SP7ogHm5;H7xq4Pk@^?>b5{SF6Mc!13FP>gA!5tv4 zL#ZF-sAUyMDRFP=e2my!Y5>{xV!Fy-w*`JcT-ofg+E<$nWkOK>r-MbJj%E}2r8|Ka zRa3pIz9HV%=nX^X?L)_N9NuxCk%RKVF04);cY18rlt&vF;|KjYiT8b_7y?{hp7K}A z#BKE7haf2i&t6Lu#-&qv3KK&XQ)JvMC*592SbZK$jBzILy>IJOwITkSv4D_JsyNa76*GcQ3i3MLya=+R3b$~b z>0Sr%N$salOsWA}boEwIR6^}hIwxuFiFC_^=OR*d5R`bcrs^d8q#?|I_mPz2`k)i~ zrYsLA5m751@?J`eYo{p|!&%G52sxsOAfsEV`|C}HklBt# zs5mR(5}7Pi#w;dk6EmF4`kj+2Y7BCc;Zz4#4Fpl1U_fmQw=CUQ?O2-&FviRQjBDey z|DdKB!qq%REh);q|h?4SnBxEL)FUuEK6BchB#V_3qy!n{RqhmE4PrwLlO&)A>{GsKQQ z+&~=rXOIK5fW{S21ceG`gWdxH^V{6r+FE$s;E&BGlCEb?JbmPy;LitK@AGH{4E8N} zZ)FsYNKYoD)^){kgnoAk^X1?gU<3FaCINiUb9^%f{EcvP-_)2e%-nHBcKC_dBt&UE zXoyCF9Lh3tp-f6})?CO>jqstsBhpZ6vqbgaCU|HLxnmgo^{XZ|(j;0XN3>hPsQ8m$P z<^^3)0GvM)AwN!wE>7*8LRv4*xDw2p?o=HF!(eYqJ>`qVSptRdw920a)JZ5k{bZON z6Gx~9ZCs1XBq31!tAJL3OjElqmi4RtpbWWNpQo9hVKRtGx{yT!K~JUdY1F8T8lHGa zTIM|c3h0f2bvxex+pW9=<^XO@4Rn5uFok|j5#;K(@a3JFCte!`>7`7t#jH*>7I>`q zTJP>GV}zWfC3bo6I1tGhX1b4dx|um>2N)Yir&kgOhl(6}ZB`e6K59!AO=um?w)f(n zVB7|BE60@j8!Inpf}Vnz2+bwWNB=2H+oj!~WYg;d39=S$5Af=bNEt@d^%ta`)o zRBHj9hGCRkySPn}#tHl;eGONe3s%S$Ps6(LkFB8|?b5&r?JLOZXnRxKx^AM{sFU9*U;-b+rfTlUuU5G-->wrzh@b< zUwwbY4|Z>O!ij7PCNxa5v3Kk(GVu~H7X^sAaODURt>J_U-w^BEJar~}X;g~h znn6!nLR)Nr+8q8VBp(|yBu`lXlor_Fa|7E-9@vZ<91IKWj#By&p6NZ6%}^W% zu5CRnAUwyx_3#hQrDI#O`G|7yJX{*FK<;oh8@CG}xM(Yh35LVi=Al+8W33AlE!(F! z%ue=9!OE=N#hXYThDw($ILWtmrZhA`=XhERZ^!2k)wwUf3wF@~o<5HF1fe`SwmF-Q z&}M!r*mMl=h;gs1a*~!6TQtpQ9)+Abj>&Tu!BqoWji@5F*2ONo@!LhcWZgmNZei3 znC$4UG@Qch$vVFyXOUj#pU}K8uQVP_wU$)CvNfM9L0d_MWq(!i0HXOOxKTS(^3G-d zWOcX@*HzQbYixo8ooCBmDK$y!;VLlcc;Po=@}bVqPjn+;b!=Bz$$O_Gb5j1#;91C9kwoBnsBq|pA%k)UOc{+`fZO1KR?Xw zU@^Q3Tfvw0vYRJLQ>PdU*AQ0{^!A}Mj$RUh-BNIzmiX&G@4KIqm29{L@ii;tQ1#=b zO7T{h#+kUVkzQ1xijjQl&VRq=Vq@$8c1_7A1rD54m|G**;@L^dUGAxgN!&W+GXa6` zuh~&J66W)gcTEP1yzjD#@2i1fThAVBK9k*z`D*P(DZePq(`jUEI*+uA^09S$Azlco zu*Vq#$pUG^5G``PU6E^9_RBJDeT89-d|aGBasV523+Wg?9~--)7+6b@Z|e-ekPP&E zNdsN)ceP}SxzX)f7A)ozeqkaL+Gp!g$)TU<%=($3o^JP~_H^6Vl+gL3Yynzj4B%__T4H~ct?RlEm6O61c6 z-Pj&gi4=ZbDt^08wn?b>?@A^$$gYH+-*;ocz&+JntueeyFYfW$`0W*WZS~+zCWc*} zg5yNC{jB--tqx}-?%S#dmS4#u;6^PCM~}_gee18r8)_-g{NUQrYz@H?K^UZg2Qb1O!RR64b-#^!%B|Vntv{bulSuDf}y|kvCjOLUP2Zlhnf!-Gn z(>JLPgY$_iTJ&TN5~~@I7~CWlk`6A)O_;AiSvTnF=X--1_bghAdsd=s5`N!QL1Taf zryJL;Zav*sr8jHOa!owpCRw*!NBA9M^!-{PKEOe|kFj{xj{3c6kQ=-6IGpnSV~)x6 z%aD5KM(so}#bN%2D-`GcP6*R}QER97w~$)v-fZGdV0i1B9FGssH7#(;=2Z+h7nYi- zy7M(^(#mu-frLPJGpkgy`EaMQ5?mn79VB_;2|tub|Gz5Qxm6?gRe zDF{GnOxkFBN#HPo_C0_?6mJA#uL5C-)C!=$gLJsbyF zU(1ioP{+^(nUu3_A!a<%Zt6fUnNoFNBpd06{V^ zJ`XT!BOkuyAAmUam`)6>xEkF|*g!zLW`GTc4Df zp%)~Ze3Hie&;}i;=fBL8emXwQjFKl|gh)iNP=U_UU_kw4{bqTKNz%0D*FBZw33_JQ zaZLRu?9yF+@2^SBH)tw8V%@z=+M8gCJ*7IHJ1u^!*V@SIzu?p_+sq`uVvIESip`6;U#CR8YHqmJAP=12I=uDciem3 z=LBRC1z=Zo2oi!*#6&FO@XV^>$jyNQ+k#xoXI0lOa-W~eDol~+1v5QQ01^;hyD1xCr#^Z`Y<>V++tKx>$FrGk1 zB#Z6yUkh@jis#IU>r^Ii${Io6P>T^C77HIumWSVrknaqSC^#n;#)!%J5qzL@t7Zzj zB`Li94bx3LM$eJ#1C=qJLXcIEW*Q$G+|c9BxezzA%pv8nO?KlD2`dDsA2}sL@)tEX zfQ9{ki^>0S)Nx0#x=%G^?L|OsJ+9Rn1pDBP|Fo0MPd@~*h)_q7tf#?_j@Qk8TGw4% zXi-xCHCiVeK0mKL-b~l%es>yVXZf+f8RQ6SAD;(gf7|H#MCR8W&Q5;a#%GUnvbeOq zzn*BzHZC--F|wRzN;Lw!x)X)v2Q-!RzE)3evwQ7(e0$!nXhzsGL|o`NW^WfCZHvmZ zpp7_Y5q5dAE!Ykvc6>j-7;vh{PaF|S;dk#py;g3-8NckBdT%!Xh+6^$??Y|J$H3F~ zc=*Pjo-7O;EHe~x!K{NYyhin)k%0Hwm>55)nTc*^jGIJ=DQ5r%{~|3p?f2|#r22wZ zsh1p$ZdRrpB^txBE3*iAJ%2x<)O&v5;hk9!eor4X!KF zftIF#9$MQ#N$zY1ldcl=%%mqDQ}(mz(bd7!rbpe6j|ZD3p-QsyR$G@~35rNbaylLE zftF}!wzTRO1S5czUrb>xc}Q4Ev%5fG_06=JZ4W3rNBNE4d5^vJ2vHOcg&+D^qS)5V z=hOJ};KI6Tt3>m2c^?9Ic-HeyysLG$I8n?GB|nBeY1qfVrvu&k^L?3Z5u6}wD;=eD zYU}pGmO^V{2An#Z|UP_HIeNM>*)8!=<15I-RMED(;4v`75`hjCSUlOS zmb-~@G3ND>k&VO@vYE_Y&(G>XUj9&oFLxV8V9GSTGvJo4YZE*F=bl0*xW<`tNl&S+gxba+DsyZIb2e+-fARGOaG&LC+p zE7;|FX0?4GtL2h^*)gAVRD=$Ark_EHKV5?8=%f7K6o34OQGA<1Y;*||ZI!iUc8*c6 z%gu>Tvu@r8vCjM9#n3G;U!w1+Em9P6jQ|+1Z^w>=GG-&bv)Tjff6Of}c#Z(*Pju`Lvzz~lKa^O-S)4EYsciBdtvHf?JlmGI9+Gx%__(*kR zT-c0|6DAo@Fx}Db^PGPtqGeM8pB#X>=SEDsT*PMc{4KIeend~-<7vRzo{U*X{^nJy zPOZ2eRt#y!T;nT~JRxOWCIC$Hsr%#U#({C$K$VSG%SSmk6yXr7=~oGb_<^7nrolTH z&Z&nD9C~~Ct{is1S&^}!h|)?Y80OFzvm|VU7H)GV{3551bwio&eTHsU7a9(4xPe$z z!7b;2OhObhhKlhGLAOGnphu8*3Eh;#;(-b7e_S0fBWpiGNOsC#ECCEVw~Gx+^Xc?( zweI^T^^dHwLu#?bf{KPTIgR6!uodwKuqgkAq)a5vn^jLTPH;{2sh?slhk;I*YtRdL zTeZVaV5r;Kt%?Bo}p19yTGElqczB~_ME}TTODh#y#Ie!chXdSYF{2#isJHz%LckQF?o7-a z+Amp1iEK%JDlhI4NcSTq1IZtrMo5)8%vF{z5B%WPg+AIVLy;3HsyzIDxQO>1qxw~` zEva8>fotB6%+vp@aU6bw6}$BI%0C;QnNozxF(5&j;NY6cwy9i|q>Sl*>2e3d5OB#C2NHJb5&zuwajV}LBsY=_6oNZ}wSL>u1C|6QQ`V22;#P>%HG@RN2bRiy;&KC= z@X`=>Eq4eTyDXxPI&+r(5j7hOY~c<^?*d50bWvDbx$z{t%o?hWDwBBziM!WIk{&d# z6o-a|Xi3!FH_oOe;xyd359kKWa~7pDr(RwU)6xV>3o|-~9t#?2Q`pK#LEtRd3+t$7 z2gh5ZPwkDNQMsBu?JekcT<#mT{)NPFfT##@YS)L#LfMqR*W#13!7Z4vfgN_^S@ zSoDVGuN7A)H%D}gR^yf8c~y?etk}v=q1TrbqD548Zmp@-e}3%~WcBjuq|58|xD+)n zvk)*((wO-%(M{4!zkBK?G~QXGd1DsA|0yqpmx_Y@b zOoG*bc*D7=mrPt-v9eV3_`e5`O_K$)3=MtuKa>>N_?P6gT5LXQdUmlkSDP807mOs_ zS@{LYl^>8TPjyR?l}a;WOgIDsL^Q2Ca?@LWH+W?0{jSH&d5}*g14HVQ4DUrLw8r)A ze_oys7BT(JqVJp51BzF%cZSLuwktAh*{91(MY2u;x`6v zgrEN0A3x~o{__?TZ4qN<$tSEsnQjNh@NvtL7?80kC1u!x>BNXiiklMU;^s)bP~#`> za|XQQP$5>VLM_#;LNViLO2AY@P#8{s37lSqIE7$Jj%q;}E5wDHhyfuw7nflUa0-M8 zXPB;@#1(qwQe?@VL=1wq(6ytKXkLz#v4|aiA~+Ki4}Z@8I!AtgD$RBJ`uJK#bOb1 zrY2p^amDB`^sL}YVE`=($#~r#4lgK{&%`&asi)8}y>&p8owhdZ!woyVQXlVhm-UKr zWYX?Vj6rblqWIQRUB6ORJ-u2>_7O4DXy+h3F`JtxwlJFx7^U2p+QGudmp2{hwvV$r zn|JlOwO953hqP_N$7@$hi+6+{5sw|5E+%=VDpR9O+_9iTIMsDNNTw_?$RZe~35 zzlGy#WTLX752KD$j!JrotzZhnN;#~g|4v;Aqx1bh>QI~Ww>2v;Ua)0lQ{mSov4c+H z!?1^D+%i}5Q7>PRG-d%rcG!kw+y@8S`F(?WrEl&S=p9X^wizu~4b(IE0J>E6>)LDi zzxL4QdW-?Og z7IV^_uJM4AsTpf}J(H|Fb5ZnZ!|PyVgD_JMco7_002oYV1ow9dnzQ`oN9ie|SCw7u z)QveWFVD+E>9`X36;;b$N&IleOFH>Y5nmS!c}E!99)z`XY}IB4sIgyDW)1JgL+`R? z);l5YpZ>VW+(U*X_S%=S7ahmWqQ`?(jNa#V*5ENRtZ%U|&_+&XOG`MyJqF;|KCXdpZxrPma>C!GAs`bfa(7rfd4K_(pI?t3DpxpoHMdQ_1M+G zR}9roOvl+f!Si5Gs~=QX<;eW__d!V3XY;Xaq{3JH(`(E+H%|tF^aUEPOW#Frmi#iH z!$+_C8*$!y{LZ5P8In(MajNq9xk>E%gnaO@dt4COB!401wO`Ea{|@NF#1eJX?(o0g zo!m_$NEhIv+x4wm3NqnzH230Oe6)5F#;zHBaQ*lUPEkJ?l{_c*1Mmibg|;p=rrvi)qYTN`GwVxO3f(|z@y(${3d0!vk-#y-=n?z zPLFq_%g~q>8tZxAjr3Bj3R9)jY845^EQCU#0&Y%|_xDbN8w+E|E0h8_;Ih%+l@9H? z>>f={Jsh32%Ie{?6+V2_maNUag5Ye0cWwKUpaAo%P6G>P2>?A~!Pj_E_1~R90Vrk9n$()^UtbS_TzI6ec~+cW|F`3xg5_u;gKiR$-A z3RbN~U9`cdWL`uM*JuT$pchPflhnGjcT}vYBBA6^TsAf5?n=@*jb*7;)n2Awq5QCr z6%|mm)23i07Grz9)A8*kA4MQXnARh{>$^_3dGJ+119pV|=~Ki?AHhaugo&{~TZwqv z=joKAH~`wLC@{47t5WHQ4a|=u<-v(fvRUxC1Nli?L<*FQb}#Xud55U~$=}!oQ(;M=dGf=)_1)`|6_DR=QfXn0qX7naUt~(mO zaLEZA&(%w3v|lEC3r;l)^+PNJI#utl`{x==KOhMl#+)Ztdo2pwYr@$l%h)+RymwZIJ~ORiZ8w}y7UziTZ653qzZq_m6ooU^fCIJ_ zrCSGDU~Uw|I3eT;{!pt(ELrTNU07Bw7s97NmuZi$B9^i-u_)?5tcC0UQEu}nph7dV z8cs_r7ECG#%Xs(f)CIM3dt!+P1*e%efyzwU1J^K5$OfB^sJ=w=xL}b3zxKhmZTVJ) zaBLSHmNn*c~Rn9dEXP2o=Vzb|u{P@S@Zll}HP(4r~?FErWE zI3Fh+blI!)7^R<)?@vgR%Uf_f6WipyzDrr%YFcyQ5Pc-E+O#$0 zG6shlXr>wHle?6702b{IXmQ`*VP;pGV2hjYrPXY)C3ldNYTS}yfwK}`*afHrT1H%R ziR>;6!ClijKven#Z_L1dqeV7vNXCp$g^dOD1Q7BSghR|_f(+%ouUMB5OD=k{g1s)nqicy)bZ z1kk1sv33kr9Er|Ac8lP~+R7jUlyT+kL?VeYQtNZIgD~lqkR>L)(mY7Gp1xEe<3=6o z&rx-ph1&6~{Fy1F+o0wdQrlK(om072Tr~)4K0ijJdv_-bE+6#01uU&8qzg(W74(Y;#*+IF1hTR%o>p z4P}&1<(zj~7u=Q3)F-20*9(SRJ8r*NKHPD$jRQ z%jDe1s~_9K=(N}I5TXyWy~xumb?Pg5X}`v7#irnz)IS4_v>xVj2Myn@xlTTx?VJ)^ z2ESO-Mm`2=I77K}$5Ar@?x%)TN4s%`@H0c7SixqUoK<|pH+e<}1Ag}ZUSW=Cy|QZ0 zR{cDAt+hvWd;mY5S|~rb0#0T~!yB*I*B}>Nvwxk~p^hx|H0`%^tmQvj*erHjHU9Ku z+HhufukPR@nHi1DyK*$9u`Sx$c^8(+h0b?q>o{J7>RsXvqEy|n4;F9h&cAT(s$2CH zPdrClKT1p7hsrSK6klo0ZJGDG`kHU+n=N#BU*-m#)B_9-IJ5nNo}pfb-2i`!!LECq z_YGDwAF=K<`zZZe*yMq_U}Y-*VA7_55awGGEZ{Lh7*T|H$$XVCz&@V7yT92x4;OBw z;iYJKdlD4*##*QMC)WgIf^Z03U?Lkp*LRlx@UZ^w3c3Ify#|}Ty4PRrpr+~79Uc&_ zxk_`t!2=pzQcrdQZB6cJv7sn#TQzj5tN~8W8;72xU8SNjMj$t##YXoAW!Ddj2+7gE z@2WGw8W1)RveKb$e)av3#pc}p=Jn(`B@aYc@*=x4cB2`GSqh9uzfv`nz~rXy zODBwsNjAm+mV<$=6l&% zg~IEkrmiep*K?4v+LE=$bS&*F+kQHnExYi&)b}lLyvW;kkQye_`=g1GjBwVLwu}$3 zNC1f&|B?4ho5KX%GhM!|UKJG*bhfL4k(`%HP>^6kwG$Q zPU5KetEcb?#L({Hw;X!rUi@IYbo9V0R9M{caxwxQsmjD@i4HkLsMt?;Ym#DfV0x?Q z<)S#H?$#DpkFZ-WkA2=ife!g+1{GFWK~<6ZThKkqc1mz(9IaL%HK0UZK$;$K>1D}dY0 zH*mT}MEIY18}#Yak02e}NL{;&8MQ>K!B;|i2r6c`t^%+g?tZOv9EkP8Qn8voxH^mC zZ>xXbKiOb5l{xaoyGO1$4B6#UH@d2_4F$__sMMC1BP2gJz>1`V*cyXp>K>+x?0;&< znZ<927I=aGxfgnxRHL4sw8~oK{J`&9CYQT&z&IkA5+NL`pHUrR$A< z+rt=%q9uJOJ`)Y1L}1ghYEIe=b2R7Zva zxo>$q=9oLh`9jBf5RMFk<6-^!#qBUhCd9b3OE>8=*~t4v8`Rs}7%^4(c>S0C{0@0sTGJ4BWs2xB;o#kze=PK%B>?wNKI4Lb8uFo3jvtn@@?_ljQX)muk7Igf9~3E9*kxBBdttOJ^MSYk1Ch^c7k5fLlm4GV>Rbq<8@?%?x+V8t@@iL za;K%3pfQ2cg`1|p&mqANM5Ao-QSzN6CxDrY@&8D>WgVX6|D@f_Hx2m~T&Y~p-sHwc zpIS?0j#hJKiQ%hwkzk2n+!es#4({LjBA-WV)uW!B+^lOd==~cI$l2m8g%|RZK^|>R z^;0y$uYA6L(XM}C@9pC*!}DVuLz%#@-UkYwb>H6ku{XC&_GmjmjI!$K{P>}cf@sfz zCp^!dQV1ISwIW@1Q&^l_W~Nps&b31(5w6YxSz?X z!Vw*6QEvsaH{bzyyi(2h7FO`gT!pd$g&uIrAen(bZ+*ZkRMa|IdGn1+7GJO=p+p{c z6Tx!|+~UOI1JmfqeCx6?okPyttu)C~i59-JJJ=b#JPaAkGyR)l#T+Q=ZM+8eVh2sU z;2V6eB$z(JN<#eXih&*rk5*Bw+qJS_o|Pik2<{%PAfISRs&Nh%Y-D9g;{+V5{2}o{nkoEr#_G;nfSY(Emm5#fPV7MjI!F; z=FLXC3AIr^Y+Yr`lB^K>=bL>3w2wltIlwaN;c)O!Hh<}WHcbp)AAeS!&-c|cZG`tL z0Rkv*>YxVTd^P}ijvibtRRH6iq>4*fntT{EvUv10o;xlj#Y=f*k^VaKlYVEkDDlRA zh#*v+imbZx;jzeeMckdniMc7*q~C{W)ssMill&P9?)iuW%LIxufEM=k#u<#>nP6rV z{27%-JV+pMF26etZ{Z#1KVN)0jdOMh9YXG!GcOHL)*EO?`zI>dvi@+a5(ZuJf=I}( zM9VoqW=f!RE81BzJ99?}k`M$dqiwE+%dtK<+MvD6C@UM>B>MyvKEQRq9dg6Emv`hQe?Q*dTov~9=g*tTuk zwr$(|V*5)vHafO#yJMqc+sWy3Z{4c1w7B8nVJ9!m0e^$x?=OV(TqrTq5mdz0JCJS~{EIRjVY%QOi;He4{mM#Ac1IYlUrxGf%UksL#eLG{D3o0x2s(L>2ty<% ziRTWdTb8`ZR@3{?-G?@O5?ky2u+$Y4oQN})YjctpS+~@;Da4~Q2(1#(hm_&x=wTyX?l#J2Tbpx+JgR>?YUjZP(lD)RZdlW zo~VZcAz{D~*FevABP*2T>iaA~1O>Jq$hr3BPk_U!PLW(MKLj{3^s% znYZAKkkTrv)a$mC>flnZ%tBxj%WHI0OiNm#m7a4gaIBb$u{0`1#sI65Kj8aiN*d8^ zBrS?5Ahv9qRMbAyF-EJ$wbsQ`XUWZgczzcJDE7{IZm62OKom8;th71;5vwMIT8n$YxF3#ba0VKHx9OI+}D6OV#Dk_yi- z+08n>x`jWb8N;#{Mymk|eo~nd1yE5R-U=dQaYm?F{jym*wFQB;Pg7BGE!0E#FzeaX zHHfVX&OQ6{7}q1b2g7|W=i4-AZe2fQJ@_1z6U$bXAUA~VWLzSNeVW0O*+V%sGPgPcN;c;pyn zB!!!z%-S`<0=(76TTxtVWq0iT2J27a7@i-YHV3B3%yvnpK^Ck3@Tw9FuNTmKDZss! z!>&9I+mCt5iZ|W3ffgv#n*4F}tQ_MC>k+72ZFi>bg0TUqTy6vtuw5pQG>&?na+B@N zosON}f*;;GJ)-g9#f@Q?yBJ&WNJ}cit24}OyTiy&1$cF64E9GwC@|4`HIA{x+A#1_ z(%jhN@*bUrzo!Ou*;zr;k2n}yYla)co{_{&D8)9idp%i(_c5P`tC=OBF{{&AN-+Ao64@@^9S}SWDP!`BxBb7 z!IfaT2H<*HcHz-xJob71&Ln2o_KrqsP`rJvTw@ALoiz}Av~Yv-6bZ3**qFdH7W6nf z$7d+8Mc_3|NaLUX!)~=rb?HON;L4p~_5p3?j9DPAUvv5=?<3mYV(2k@84BU7*dD}5i-m8s$;q=$IAQe(KNFKB<( z_WnlcQ^bw^exQVU(76|U;;FehrTMR&aA}%EG(~9#D9QylA}wWd!z_91Iij}u z--Sj~6=bXRa2*)sQVGZDO30&RZDJQ8;*Jzm+2~r9R8gvYeXFzM_HBPkDQbBhEx?+3 zaF75LEj2QB0@AKd?%&_zd>k9nDNr{+Gr%S_R63gt3jZvrfnLh|fZsbZW4FKM!yMkU zXJjeWJk~v>dpTbv?2syFPH0`BsO?Xz6pF4G@o{0Cx`n6Dp%JzUB55m^4;R00@yp)t zyYpB>G-tSAhp+Gn_}X?gs@J8;I@E+{xj zp1_JDSmif7;E#Cgo6`fOBRrE^>+iXM5pR|q$Gn6y!@x>nuL&?S&Yyg(tK|I@B_aG! z8PY8N18|J=W?U%tpw7RPVzv$VaBHK+msf_82_nAl6~DpsSY~VgFU!(stx*dP!U@DS zBLGl@iwUcm+T+g4Izoq#Cxumc-E3e!$@@^jkoD%QaW*DsbFP)}>OT*Bve+jnGgj!N zg~addM*x62^mWWNX`tDF>yEX}Dcv&!i^Y~$*hb~OIM7?p{Hjq|@i<_3{{jsF ztc5kUn0JwwvHt(0mOgOWHbe+%`1%Xo*N%EP;lu$kW_)wq7=N}f0R=Fg+McoA1?sWaOHrzVhVzJy&kJF>J|lH%JCxEx6h z;+!0X4p>N>)(!*r}YdEud(vC zsa>fqx|9;L(#7X4Q0`t)QB{vO=bZ5J=-}3Zhd|sq+n0tr15xXHb^9ctP z4?_=v-%QG&5DG%&BH}OlJF)F>fZQf>vgRf-YJ-NXU?iXOz0}P6L#x8$bh~xk=fZ1h z2J-Jt_J`KAmezav?{|oA^3!D?O*A)vmGi%wpcpg2^luYB(YQP47J8-8iu0>QjJNbP zhF$F?Tl1}>9(P-JI2>>4!Y~5H$sCW|M`okDnB*Bh95*z0&xYHXn@fD(>(HZ6^>Gfu zlFG6wEM0caG(2+i>?f}pyOM71yR--st_e5nZP%I7zvd1B&h(cLbvvOli*6D(Uu#K` z=`nuKUC(Z`Q|;(6VU(eQc(zyF{?X$N4vG&PHU4XvcBTv_E@K`J=FcY`v|D2jBsU9- z`j{l071gKPspxA9dSklagTUxv3on%fxlczyWiAgM&E69H!NDaY0&wYco(Vi^B{UPp zAL9AYXr##i>m#WeAy*{?oqfHZ!MP?A_~O`Babw_3!emrOdSq12Xrjc+k!nQCe~6Y% zO~NvYLa(z!l=4gla$s0w|2>0c$8kb&(4-(&Hv;qX)ANxA^59^ zp|50KNcOsdb1i~%Lp!6tVe!=3#90deP-fsfQt#`sj$ zp8Q0qO>teKAJ(u=6BG0}{EQ9r&w^29spQKbIEz-gO}jHT=?T@}vsXLx zj7E$>&5n*q&bpiZmdqmr>)dVTME{O?KCpn&jdk{g{1XfopiO*xRH?D97{Q*+MggAZKGznfb)Ip`BbV&LtD$b5QrW6=<8tgn{R0%jIi;MJC9VBm z+Me3H_##9(48O^XVqwHu??&acS{A0`7wFPo0y8KCA~Ue^hWKyHGOb+#Ca;Hez1S#U zv$I{3ub;C9kz>M~@GSp*uOnZrJ}sUem;9R*Y8;G!r#jZSUyE<$I2}=Q67dc-g(~d! zb#@e?Rf5`kJCEo!4pleas7uudCVO1QPFQG%5Uqt57qMayCjEg9WAZ%~?akfY_7PpG z%Ik+KqB?XZs)H_RHYYk#>{ENB?Ueg)HY16fO}d=1CzJH_w_cmo;gFk2TDwa}_%3`_ zCwIaCd!ZP3n}9V`7~H!6_Tupb)lS0J87V5CsW}rD&7s=ifl#Mkq-0V9n}OK(@(l@i zL|esaT9wd}bYkS3?Gg+IOJahp2ZBli%_tJCc*`W(!*4cKC0jG`J?Je*r;Rfu(We`) zc|?{QD(hjkV`;15ED7qpafZMVqn|8AgOrK@N+M#fV-!Qfyzu~CXtZZG*1`sy@%U%C z+q>t=GOhs0uq9$-!a`w0?zQbd?EW8HKVS6i*8Ys{NYR(Ttz5@y=yiD2`{jOJbfKrK zcXwPp&YZ@CZC=oi_gs0AVUBUn+5Ez+HkrRze3dfgc@*Ul1bDiwFW%&jkUk$TBs8u8 z`d7$24gtrEFMyql9SZvhQK)xBxP5;zzVk9Ol$gm>H4h)gcA;TB&Jg(Y{%{+f1Y=5p zKPndX7vzWBPiz}6D;$0ho`O`S-9t0=tvPbwWQcl7p}dAO3fpvMY570G6dBIfh#m&s z2N;Y(qmpxavZm(cS>yeLKOq#;nJ&QqS}p|R7g>SHbstC+CFo-jqduyGVUvO{QgwVuCkWVRaeya23378LlcVEuJdeh{Z0$~mXMG*1Qvm)QM zlo*l>@qNxnkVMvPSfiA?i<&Bs3yIw+!INn-#g2rZ2&F=8K@NS@3|g_Xr+<#vR6lm` z_&bk-jRd)cJ4bH^FJ572--!m(&gR~RL`Ez`&ztN+)GV#D8kS1!JZLyRpr9a}fVT<% ztS0{nZ`4<+w;5zY+ImFiEKM=d4e~$^H##tqSyv$!Z5&@%Lmx^`Jp5Nrq&Tp7vfFsO zpS`XdDynshz|0J{W#U16>{CX^XADT(xU58BVPz)kr)v|j zv0THfwAKUrH9mhgw`K0xF~qW&aRWI1GtAaknsf+d`q_0%>$h1^;iYEMi6UaNCO#2#3HAgg zVq#UCEJlp?@*XbqJSVze&u^-Wn))D@>W1X6HI3^wCreJJ;M0OvFepruTuemf>5CB{ zz#>}MU~x1d5qcuC(eUa=cb-X*=U|^|MpU|N%ul9&@m;FP`FktuZ z1!#n30k5By?X^aJs!iZvwVg;wM%-&Za%xCAnmAUP9i=n96H9E7xs6crs56MSIy!jj z%Zl$dn*MBW2D$ey9}B$q-COfP%!C9+koUTo)wS_F4?|!MAGPm>+V7n9$K~NlQO}SU zyJ|sx)Kx4w?dvqyYXw|;4Rxnb90zjekvA3yS>)A<8Dk2K@R(98Q(>@~jfvN)LYag1$!4ioIv!&3sNk~uxl zRCz<7Jaw!gej$r$g0u|ydiQ>P&4OKW3z2ZHibg&fHgm#k2v^z!dJ4(YX$>>auu@9I$BDYJ3Bas=(P%Ws5@k41#U66?lS&G3Z zOVi1cN(C#5jrqB7REKDe8v^a7u1}_o4xBw05S%)sZBD*32l6`Wg z#;%PcYnv8!VE8QI#9VMK7O-|0+Hiw*dba8prb`OIZUi|a3CA6j&p^$C6=#l_{>fGY zhQf)xIvH=gNB#rLOU=AahQO%4Q*edn=b%nTAB!VDH@vyO)=R%Z6K!@>uF5wIz4Jkp zELe^WYyMGCM}QrI?!m!lj6aP&Q_AesJT)w#ZnsHZBntKWmOXS4;|`EuJYE_Bm(SR8 z?h$fxGS~&x!}CLaURpljiQ5Fxuts&=j7XQUp2MBtV4?>JUBZgLDWP*t3NNO5vPJD$ z`q3Z}(IL}j|FC~JS@?k}DduNj{P?p_MtziSqwPHKtqN%HhJnj98q_yX88Q}^18a)b z_(K`IF6L+E6>3uX^+hzPq}F2TIf3p8^yFG)2H&zO6DY3bS6CUNWaliy4NOZ&*q<`I zF|_5^zxvVI>*3T8dKje@%ec^_y3A{zH2Gkb)CH{$L$dC+eFJR?K| zpbKCc6anb{%IoLW&DMB|v*cV^$L)hrSj1uzE9VM*`&4dy#@oJv-)Zsh7vf&+ziQ&b zX?KwK8wtq2P=?OC3Npl9F zL}KAhNhWU-k!mZkCij^7%^^yc3bMdG0H(+4SFnBGKVSZvDFUNKe4=bxNv9xXt!E2A zH5h=Eu7g^{Z?g&K@lZ&T`{ecf*dq%J4Bu$h6F;k#v~P-Qk-F5m!aS#oI@Q%0IuxB@ z*xkv!se8BnC9eGG4bc>g;ehWpiS&SkY0G>N-V!1}YafR2nf+!=4Qhn&uy+ zNvYBmoU(?{SW{Nqg_bCXDM%W)S0D*9b?(^fDZl~gfE5N}8n;vx9!_oxHTqV1EQXA& z4%1*$-DP|>Cw)aciCJHnLhT2#f>g_h)(N=ZBo{Kh4F?0A%c8A@O)0-t#PINdWZwEl zp}o?RsNThVu%vUG)!LiT^0pZcN&Vf0js3D$`Taxp-T_#jh6>clr2=6A(kJ2oP|6k! zkW^&Dh6=WHcA-Lt@})?lM7VCQB~{i35n4WOb1yA*)Y4>V5g$gem5U+_tWTX-8d0I~ z5oCa55appe4JX56^aaDI8@PnXR=xr|SsRwl{&>CXrE&;F^#IV!VUl891N}4eJ1&h5 z&O~$YK!Ig%peNjg!h+iivdxaRLZPZppuX;6y~^zJ*M8(W7^yeGDEVwmXcCt;!<;ie z(Kyy5G3{h_{;zKz+@f>N69J*;(=#mHgFdfEQ0_=STeL>k^S75MD!R>lww^rmIQS|n zz(TM?F0j5D6&RJr2F#7af`V~(f&F>=r?ESZ85FEV`~q0tf!fGX(4hIR`v1ql>#qf) zg{s+^i)8^x4{i><{4*y?16eJxqtF%whmW*h5K8NeMiR zMs_mNUcaJw`O87EvUSq1(*JB-NB13`z~`F3zvS_$9gs^2{5zGwz2n{c{J!m3n169W z6(+LB1_1P{42Ivdt@w5M!Jm`bi+(j^eZaaPnFS?_hgIPdA!<%Ac&u&-dyn+jFrpH= zn|^FvaLm;zrI0L*d+-T@6FJDG^U{msWY~&_D7}4b!HP_io2u7Hd}`C{Ib7shyMF#Oz&~49`%xmEhMQi~F2z(Lbx zO7Gst(|bcS@7omIM;i8U99eJGgYuO!n8owOtVK2($V1tG_VF_BMczeF8|LOv*nH6P zB{eK`1#u6{W33_AerK1K)D)?zVr_YM#PCh$Yy+VA|gkn4?d&AK#!6L z6zcCfUFq54=eZrGTsbz?VFlCK)r306hF~8zkbM&iY2JE!Cl%&865K|KU4Jc&y%fuR-VcA>2K(&e2hEQir+o9-e>Q6?UNeC3)1r6yBi%Gi{ zPR0rM-WnDAL2&7~JlGTGse+N!VgIF8qU+O%vp=bv=A1MI%uKp}}= z>{o~e!6b=@=|wXCdXC}i@sEKXwma`BlBe`iBEC4GtngvPC4a3i=7KV4D3@H*g|$B0 z^8Ph7%}v#))R)P3(KeTp_x84}s1ezrx@P#1&utBH_UC@!b+)JWAD@3V!#1@PbE*;J zjkse)ZG#4*)~Vzz5x4>sQ264YW(#Y*u8oVaop_)CTLY#q# zbTT`$$op6F`D%|@RNs>EH+K5)lh^8{^Dg{1*N>-?!ZYR&ve(k$K*0Pi1!CCz#CF0S z6#LKtbN)!bNM0>MSx;8_!Z<6o7L2Ks-?KDmc50T0PUZ@u1%;a#|9ouoz$mA0e< zrwIu`yy2JgKc%OO8^BSt5o^ljkFfFR<>V?JGFRLv4R}0eLg7K2!w49?JASZCPps&=(dL%6C-91*DQIhJ$+Kv^kSDv~0uq$GV^3t*brI|} zluxglvXR}aXa=oYXBX$7l4aoB;>OEtSFJa43z>er5J1cW(66vCb$0Tts$2V;#bb!- zX2NsQcfEcZ1LY!^<0`d_F?x-t6*_VULp-KJRKvhduw7nGwQZn6;e>E2y+xRg>xJdL zzb;OFyE}rvAB68No{Hh?8pKU$I0zrDST${BSGCow*;jwCSYS0H7WB>)K=StS`85$J|03ez3kN=k?WWI{d3-0Ne_lmXz&6H>ji9=7cxI33;2d-NLN9xu^VybD)p;q5Z z&&%RtH;3XMtpweQb+v8yK3-4R7u*;{;rd8NRrBoUQ^w$)8TiaMtVcfUZ6Ni}3}B@R z!&*#%6~!*@Rfr@ogsM#{hR;--j)iY~*_P1p3b=Y7TNJ+Y&G`Ea0(0KmH~NZu(@&G% z2FbNgL!T(wxJi9s3wP)Sh{|rR{@t*!If?lf>a6BuxvuWyq^@rHq^4$hnPGwX8wD~u zDWzNV-CK~C6IHSA>q80!m65r0Nh>J$b?IuX0y9tZMZ852&7~cl*cWA}b54Hl^icb1 z0I+l8M3NQYhb~=7{>oF%k&uAkN5%={Sh{VCv&$ucQ$|rRQjMjWr`qr8k;taxT!5D# zmDzJHw4Z1F$;n?*H)n*xRYW}=`_r#5{--u#o`|H^%;IYSJU(^OVK55|y*Rc>cJrJ{ zZ4x;g9K8Rtbq#TKS4DpEj($sbmsThSHsH^%uejB#c@H@Eo9z07mTEM$gfs#17wP@M zye|FCw%k!i0qO6-{P-jRT8(fr=+=c1-N%pOl7f3v@qDc-TfWa(1otOLxtE^fi-NbN z);!id!R*3!?w2$!J8(OnpuovANOI^F_1dpv_3xS9?VZ@1ZaLpxtLYz%=8n7{Wq|NU zFSOXcC@!#=Rj}G&TcH@kFt#yp5DDS7!0wjH&lUyfJo=aLw-UMo%=fEr77;y9j^_pU zY3_5+>rdI2;)DC}KOgOI;q90Ed!ErdPb-PJ!i!PojsPWC2ANmrp>ko?FOUHI??0Ak z1mRFSSn!Smx;uhDC_y^aDkzENi~$I)&}G*BfvaNXI};UEM%d(ppkkPA? z?ifC}3FIR1{C%uUkn&m9w16(X;5~b%oD*TYcU8e4>tsmEed>$y-=zb`8~_4t|JS!z zvx11jiSpy7l^FkDmqDi64wcr7-uQZJs_T!IE4pH6qtT zRH^OvT3)XYQpna5wDj8}Wnuz9~E!m^gT9O3;*NaGRDA6;O zq1;_j?|i8aVLogB3c%-696SthraDZcVNUqyCFHE0u_f{PuK~(}4>ROh(?7~HGk1lB z3oHelSqK6E+B~wSxJDsCA}sLZ)}fYhznJ?y*+l;+*)+;vjjhyWoNmKNyJ-3h9sAxE zH-nO5ZeO^|w|ytQT@K@-VU-)UncBC(O&f{RBoTKJw{e^RFu;?9*GcBqZ71vS)5+kg zCg)n~A)wy7-r~(Gg@tcVMzcjpRVk2zKTdw0P}drH-GI`mbVP@z@!Lf%%sGwUNazCgCjwY#G#2ZLRk4jUwXZe=2QN zFHjMys0nfV0N5rMss{6sV7H60Ox@YJS4_XkS*SP*&@&6AY5KvFzE=9L)M4%|JMaHg zxc@yq@0MLRDckcz4p*zu6_o*tBLCx7@|#pxDDz&_$PMLZa+_}h>rV|pF`UV#De$N4 zTsuD~TaqRA1p)|9i<4VjM8vp2w9^T0SHgTip3CXvGeA)vSuFtyg)|_6Y6_M089NC} zVZ*mOO;vUXG4nxInQYcOwDu%z8FHb*a~UJkGg_WP);PL!Hi?{G5jhs!s}wst7mbV1 z8)d%ui^m|a zSAo}Z7a+A|`*U$pg)ZP&@e8WX5uE%3Xlyg{AS3>HhsV0ybli0N%BMHl!_9#C| zfPvYo4uDO1UuJo+~<5o+Ni0Gw}1T(i_7qtPdlo#$Vkr9Qw1GM*P8g+^@@T zZWwO>)k#(IK0UmjsPk|<3bNnf;qFH3$8bO0tA})TN!c1tm6a#{x^Q6kX0P=z_P^YZgysUJZB;9_j5|2_eQ3RSY%ZxhG9o4l9=@)z zWb!kGMt9mI80(hxBWTAxq0B5+P#SB7DUR9kQ1q5gCrt&`-6hV*89IL)UHkGebb;Us z$+ojFVJw&gD+h_kmi>l5eB(;J_?nB8mVlHy&YA@}j9!RJXluRG%(LVLNADO>&26$)jpr!)Kp0RIQF2@Lj6`u(4BFx_Y1cTP40sh1pn`=gTZHiTrg(GtVPRp_->z{y*e{Q+Sl}_^J$W>%1}lnVIF_8VxSgxXbpAq zdOB%ZfyY`rRnJ9b3Mo+vm&|i^RoIJp01Ft@E@DSjLs^-gYAcL6w7(1m?kt)9fw8pf zhUgbBWb1>nTRd`xD;xEaRT>eZg{)c&papWWD}?veH~I~V+o`FsUd2EUp}-wIHI`|N zVQ!ZRiMxvo9`vXrM5A(8Ma_U|Kyih%5>c zlLDIDrc21}e7h$65At(JjsIOoYbAjJ!AAgPVM`On1R(^}*_;OwMYk9zIp^Bu)MyAJ zBPop_Ate$E1(NZ?xTd4Uf+v=h?Uq>KS2RDByXSl4w=^%>o#vQz$;(YolVPqTrUgbu~(1 z(RT%Iat8s5zzxW#AYrymjAP*lt{JYJJnL8Vbr)GOO}4BUAwf$8h7Y zJcCObtPgk&wYS2;_zSDeQS3RziUMbn?vl|?zr3hl_}`!Hr#%Cot8-F_+Xvmooe_+< z+uXr~%2(3f6@1sS_N1of+YPl{B)fHzI}m0e^c+cv@bvpVDvY}SGHO~2$-e7gvj#1F!W=snx?JfHz}ur1MHid|Jx>s`q$ zb_BEN8h_j&!b)s7Cfhv%{VRBwN!NIPbh@?GpD^$aaK4F6TAHReCIfJhGrculkO6V%|0S zUDnbXeQxS(uHOCg46c%Pd|z4VjxPx6dvYj9`9; z!u^cwbacBn?Wc;7$$Tb>&?f>YEhwAV9P|J;YvE;%aO;#YI$Y}sIfYUFDFAnk+2zcx zIA&oE_}?;B?feBe=Br0VO4K(oLA4uX)P7Y^3dTfL^%{I$$D zurzBnYAUm$NU9MNu5b?>n|=%?r1sMO2Sa)oiUTZs;EE%_psoS3n~u-gTB+czF^>rf zHzbqa&#}&@Zef8)$xVwlPM?{MREuua4=+GVaqpQ_jMEwJdR=hGUNXe$kN(j^0CWiFv*!7 z=OjK%@EudhgkbG^@6nPz6_HD?;dHR908G^*9#l;myPLrm8S zu*b7*r7%>%%v%5rS+8X_&nOKS7A0oVGgn3q#9e1qQ+Z?y2WV{^;-*O%blH%?O`ngH zH<3`g$Rb$OB~K@C!Y!c~T;?G_4#i}L7yg^(0kcXs)4bL??PbD;N6X+}QcLx*Zb9Xj zs*5u+_LNtom&)5Mz>L_64X-)C;D-`A?yeXa5v2#{F&Ll(rF<%CoC@ZprD`;qN2NMK zdi^%u60mNzW>`&nEkg+N?~0*i!ly#P2>G{p@IgEsGJh^k)SNxy8=m_lZufH3$da=~ zBL|o%Pte7asjD&;a~DtmiJ_rZCBg5z6%ik@=8>V%vu#Ct?#6&XoMRh zQk^}b+RT6t7f8h69;#8qT5c%GU8*YH&tXQv?r48RClyRfUzyIY24P)OO~X5jTMIt8 z**;<}aD#qUy>LsRoQCoj0C7PBJQ2ZB+vme2%64o)XbS)9_R$VSSC`EiZ%kD zduhGrmZ!6TwRuVpySggVTkwl8C{sc{5D6p18{7A?zaqSO>hn17WI~zNj12svN&Ui7 z6LliR>VJt~ws5$0+7u*4lLr=1yNHYxQc?_#tA3U?rL)Y-l;$xe`wC3GoImG!P=Teu zP(cCGKJy?Zi3u6Mw|vd48Vr=EqY|h4Lof~ZH)N-IAtLYK7n$T{pn5?1`$S{)3rO`_AAA2 z7y6HQj6R-mfRajETx%!O1}q#Vmk2Q>|64eKG~y-u4IdV9O!7jGJ!h-`z-TOd#rP^f z^tUxyfqZJYWdD?6BTKK}y%N^0EG30HJu(!tV6Lob{A zt7?)G^K(*+bK{}ILS@r6F|IF#X|7+zK2Nq+qq}%!;2-t%m?Wji+uWSW9c38j2d1!0W#~NA zlM*jvE_7;r*eL`v9^C3ClPZu@vg+&_!Jz>dzUccVe@#s-!zbFXJv+s5;E^d?E9(f{ ziaptKskSNjDTBzEJI|>SFR(;eO=rVv6vXud8bj?vTpsyC#P zqoWV=sXfATo>;$EQ(8;QjXEIh+4ROYjY9iy&wwj_rr(f$IH|wo`B&yqJMyJqLtDpx zqMM`{_@gW+m5A|dYTxZJEqu~684DDYu87IX+9;m6Yve*N^}n86f_3d)MP35f-&+Jn zS|lP}hK`;?eFkhclyknAb{x^)=kFYmHBcJMu2s1fC*)(cKbd0to7 zu0Gv(=d=_M#A0jUw8dgUp_>7+ETTUp$f<3thv=JxhYwhPI-r}mx#uF3)UR#@NtnVd zu-!Z%e{|iO8fpeR67Q)z%#$EOH=z>U@Bk}mrfl|9a4rMX+(D8C%<&jlWJ)~mzPgt6a zVnZRECsm)Wvuh84%WjOqE6yJ!<$pu?P}PW@8iG2ue$Z3;@^Gi=6#fHIbK2F>Io5Hm zRAH{^AG(IViD=O6A^OGt)D}YN+;*(}O~Z`;*++~|_`sV>S(Q2f{m-km<@Pd7c z@=AiXgx!D1^Q2&a`>y?oNzb=UDhH&-uyeID2p35$Uef~1TkZcu=UD6ufS4ua3sQPZ zRLV39BKtikfAyJ|PP})H+km2$QN4VO?p(>H>;vD)Tf45;cf5k$WM(++ILXiZoNxQw zGpYYgeEkFy@k=5hS(#mqFYO}772DoWf5;==R%deV;Xww(!YVLih*Uv%Hn&;Vn zVIbNZ-JA?QJbadOCQLZ$b-7@+wT>oK?lr`f&GFUsLZ7zE;uKSQSdZFta5ER%qQvNb z{`O`RMA69^&+{o#ceV7?pEt1ApD(Ej7txKiEiVPITXg&Nhrb3G%TDmFI8V;l2rd#j z)BnN5f?KmImOj44pa8R;GM7*lr}n(y=jl28v`^5+x*aYul=ARd1`8%Q1r;|L0Ir#wbx|+ z&0GiwrsR$#a8i3qk*ld=;%qGId(<+MTJl(SJl)T4j_}Ypys37E7e`c?wbVRADDIG< zca@FKL~Whm6t6N&Y3gaS$HiPP<*#=1pkF!0ZJNO=?nkr0F3(&KxucbZhm@t4R7g!x zTgFK1kXdpCBIK%e&jtY6k6TPFqwGK_iuM5W{8m-y&*U(;QHBZy-=EoJ@W1CYVEzdN z+4|L5zK9mo?=a{ml*gr!NAs(g^|+|_Q3VTt7vrN+zQL_wtforS$fFdFE5L(W>c%eCPqs+h3kv&N==!Pn*`Z$(T^x0|#8$3dctzIWi2f zzY?)!TTW&l9UnpqT7I*7)L|ex&#cl%@qH!rZy#93874#!gf3+>T|_AoAF-yV&yKYN zm=v#m5{SAffNwJ;vGs`}T*TTNHK7!J2ctWDZA(8VXymwr^aRRBa$=#5*|`H0mhm7v zA?@_=IBQ@jT#UR&ShlGcRwWjUQ>-GGv$E80*XTe6+YCNWMZpjs4yPsO3>yAo z(dRU6dEbVqOwZsDXsl_@t(5>I8h#83rSu$4NqB?aU{GyJPh+_+%^)cw~nZr zwnAN2iMQ@Hh69Y;o$KGYmZL7cyXDat8O1sV><|=NB@eiR!@O`DWs zpu4=v4$fqk*4LenGsZ-IU1O__qMAxs$m)FH4q}`bWNH`z|t<8mdw9OA}P`@U4_<;$CP9dJp{p- zY0b+%nO4q3F9BQMWUhZdegsN`_njhOY%i)0nU5`r+o;%T1q+T0?ivHcO724J%eJEo z)-@=7MyJ>LQ~laID8K>S$y5{Ms+hf!4&GuYj1@$!cmU8Bpe|2k^!^l|Tw|>}c$VSi zSbnnEE_tM|#qWP(#V_E@&k-2l`7GyBGqxl@d%|U+B1gpQevb~LE>5Ggx?g0L<=GS_-`Ub+&{Gvnr2H4WJjjF(nS1fYPpT`4pJNrlstHx7_^ZZ8m5_Qv2cEU z=bEqwI70)3@RPqLsInf)0ZOZf^a2fzKQ`sOQ6cj`gFknwo<&dlw8=ph=gu>1PeqS< z>lOcV_Wk#9Ep4Hk995DO+sxLWC;+=4`LD^LdD+I-V-gK0JngGYWtBJ4mW z$AWPocSk4gRbEm*vtZVTd;X>Ll|lIhCmzL8^>{pb&^NIV|B=k2tuVjS=E$AO!d@tr z!fh@upKtIxS4qo^j>VT7c#mLilz1MW4=o=_;SkgvGB}VrF~+4XRVPAf2-w}d8NvH} zt-V8ka%l~`>`Y-k(N?IeBF^TLKYspsk6o9nX1 z(`&A~>&jK0>sQ%pVm0!eN|k&^kR~7`Ol|+aT|3hDRI0)Q!YFdE#4VF3J8CjDgPKXr zqGnTbDSK)jHJ@5Yy-7JzOR2XhXKDrYF14C+r#z_lskPKPYCW}q+DL7t{3w5aDu7Z@ zAygO@PDM~kDw5hwsVEHMv+yJNoCJj(o{#T1KRi%yGQEi1+tk7==@thQR6 zw6?ciYkhv~%&}jM+ihcJbI|601vpzF?QdFPQjDkt);xhV#zJPh^g!1Bx`f7x){^c#?t{F%+I?JJ`2keIeswzR9UiaB#@3r>%a|0G zD{jxbFBf-w%rQ=3^`GSSu&wo|gUtA=RD+^-#Bt1l_+Vd@&UyqM`sn2SZkSrY6BABp zp`DyW$Ig9!4pyBr7BpXfOIwvFH9i}qrgFO!K`9|Dre*;Jb};jo=fAcdPxWlz_hl=q zL)oQni?A45Tz9;6@<}f^Uqg_9-@&irv3LSrjo$?;JQ3s%zd7H=uZJk9qwhL@pMXnu zzd7+)@9|@69XaB6h}MFy*5-R1BK;rb*OFPyn@>%xkK zoYfNTACt8rP?6DNHcC6xD1pOs{+JAN=!zziD5T9}S2{E!Z5RK~_atFOsfRdl9el=* zyqlymhR*_|f4nzj4e6e!B7SHjet2b1diM^Y87dx%NufKsUA5R4v;1B&= z#akg5CjuXsW(-f8q}m-H`EA#P?+R*D9V{x&+gHpbdiW$H?=?-$j?0RXC04{&bv{1} zM*36|lC+|GI;bX#j)SvOShxsg;fWj$9Eo5BUq6t46lG{ZAPp0~134@r0mNo&u{FuG z*+SA28pP-DL~P0rA@oaqn%F=y*t_Ulz<#_HvqvZ)P~DA^IA5WN{*OGt3+T4cz)F7> zi)PQk4A11_u@7gxIM>_K*2*!S!6@l4*GE@2>CtHqZ%Uc2^}b8qerL^%{wLS3UB7yA zz1JXrG@`WNr?nyJIYD<^=c8l(6mpM9`z1A~K16|{V>lOF3sEpZ5{ME=PS)3OkA|Nm z1af!+_|eP}$k|B8qNqdq*pk#7v~<9XPSj&b?}GM$q;P{n1fFJ$hH?nJuiv{a65<6L zY1dA&PW+Tk+KCj3gQ$#$Kk2fAsEfyQOu9mUQ8I`Bq?1F*vu-^cpVH;slIp$|(Uhb;`a=%ukq@)ntZtnSR_k_O*sEK6ff-8)MS`ugA+}>f z%9h-aZ5(4xl_n@^wy|^Haa!zO6B_S;;T&5^jl_Yaa##m=x^lpl5(eIux z9>)KE`CE>0mUqW}x~YXL-V;zipIt?#??L<0`NT|Ap3F0LDJkW-^+;B*ue@MCmy_S4 zIm!Oece9^o7C5!1wZ*q`b!CkO2l7lKiyAWavsDdcZ9L&;W9d3SKkp3zAD`iWm}`aS zj~zRGw%2DP2NcFqKPUg`)18hUt1jv6tmP`3N>Cr$4`Q^4e@WV+2Rcv?r{}g0QJYFg zY8n--Q4wguzE}aiuRcvs4<|s=K$6PP0t-Vg6(MW|j#iE0IE!I=9D+N`3GZ8 z2}wX1(g*j9N-Jt02~(rmJ%jy!hST)cGaCqbwnE2fg{{Mb*}=UbTw}XYZ;A0`JZ2Pu zd@?i&3P`&2)r3CcVg+y}D7iUXCo&1`#Feh)7cHAzP1 zBrf(EBr-33g)bqdtH`W=kVyZs>FBc6z9F%(9HZ+*7298>(aV0Lmhc8$u8%BvllOyF zG}HF%R?n$3<;pc2j@dDPL?giI3vmx=8Q< z_xcv-CiBUs(1Xupm(iD{C%B6TI?C>`x4Y83D)=%V z&7(4-HBUOJ4ke@Y6ZiTw;b4l_T_0V6Q_@d?;)F2z!Rdk-gPXOG*DzQ1qVAggwF z1op?axCC3AnBTRTTXE+0`bM_7u=WEo#D_%5{0GUKc^kbz>o(jMJsimmWKq zOqZc9g#sx^YcjTzB$J`=xTN2 zzh!XJ%LncqL}uF)Qm-CuqWn{#;8 z|Bu5$@6t7Y?MT_lGk?}cqo@Fx8w#)4#f4Y3CLCg~G=AR0;|1`mbc^p=2W};V-lUoS zACA4h)c_ors{>;l`j&AJ5(EorZF29DP4C6IdvS*(%$*_Z8ySH4H+7GZn3we~(!x^Y(v#nQ}`(%XTiHWp8h8wP{suZuKr!uV{E|D@j1A_WGjo&Jr&C zur}d;DhsAJzv~|%t}k>ykiYRuge)ObIwEISIy5~!lwTovDRj=j#DOGliNOiZhIn|B zB4Im96zP%tB_UPf;{_ziH-L?_8r3O6bED%qkKA>s8uo2;x`~H0SRyf$uO;{Bmon_H z3k_oCJDg3={xZ1l*58iy>*a|+5P4e%DP-h-D&g+1Hj$13Bz{cGUA?_Gx}E+92anq& zBv>8ogTZ>G*Hao>KbZII*@GXznuDk9l0oq}Bu_7o)~2sBi3V*tQo~*$)exFFIo~mP z*D`dllY@r;{K~;4^lY8F@5qt;t=$~svn#I-RkOo#)e0{qs!)M#hv9dFHo)x$Jp`|R z^sfb#)}|WV@nmrSA8=>x2>ByxpE|UtI@0fLd~wgl*yS1i6LaaTuDw-h-Lr&=wv2yQ z-LMCk_ovd#qv8G^k_;QD5(n+CL~U$K4eBl-H-u{Ohjn_Zs-gvRt*8+;--FG@Lbfnv zxHy^_fY3@~5x$K@a9i9uN{5&kP}@#_g9JYkhm?^L&DJ!agQX;I;jy@Vm}1kv5hAoU zrLZs!7Iwfw=8>>pUqJQCS~iLaXg_>E-7~{GS4kF@dDRED#c_E#rmEs*RLqt&qOL;zCfTScZP8o1_bVVD^_8&oF>Ey!3WcfCb~TFQ zu!N3N6PGlg8vZ1`8J`m8(tDftoJc-mdL=HqRKcbaWTX%qw_|Q^46)H0PLU0k!Ho?K ziseV%uOo9@!u?`RL4M+4_HcfGiME<=`Yx|Lzn)lD=-rLY3ervE%c_ap-T9UE1^gxG z7q<@vYmnA#T3`$If-Psr#T~Yg9LX3~UfXFr9u@%&GsK5!gE|@xWj$))ou@7TMchpG zHuRe%gHLu37yrXFnBy5_`(crIJIyq`yboqW{*6Xq{C%+Q$JSzH)CgIBn+?uFQae`b z2{wL218QxWVKZKR4_sg~Bl~apv=WEx*_j!a5vEVznzXu%8hgmFZTi2HtAi%x=nrv0 zD#}JOqj9&&1O?EB!XiMz%UBNBy>^mrY8ku(A~1DlI6bIdc;)rKK6Nw9uOCEP5)7 z^7KbxZ9$QxWmnz4g5tcIy!~en?zbEx)Q|s%^xt@&DC-JkZe(+l#=;JlVLbso4cPO| z@xb|44gfPJ2UwHA!V|Z?Jpmjve>5^aJ_>Vma%Ev{3V7PZTibHmI1+u|ui)DzHRS&fDi~YP9HS7o3zZc zJd-*TLS)k6zsi)%w6>WRnN=Ls%xjn7Kj&IzGULMF4v}#waA{CFlc9+#f8{gjGp=)+ zDTUJJnc*2X5(DBs<1rfgTdgvK`yIz5=LK~Mk zkzsO_xWS^`IVxR-frYgw6-tk>ax5mN@p1!E@r!fFEYZxl%B12{!DL36Rrib`GtQ^V!*l6&XcQlz$nMW<7VR8W&e`xKP1T8c*kR*km zVhL#snR`IQi39+a$b*Um5EuvmJ5oN?45k#cm@tc`ng^tUq$m72(Lo6C1_(e3`mL#^ z0W8fdEMVq@vLKd#NYG6H9lVj;`HU7SHH{dkXmg$^63wFr-K!uf{U*&B@RBsAu>|Ev zg)~0_p&Y5)2JZR{e+vOG^4~=&{!ahwQBQwqQq>gNtH67JIQc>OXn%k7RY#5>mO6e>GGhCTnb5*h&^ugA(hj zxvPXf6ef|Nnl=%!39hz0bZ&D)ZA2Q)`-V@<^^LE{QyUFXfOVxb4yXEaRxqa1pjt-SO4| zQStCm&NdA-GX)C(r+h|BvIq zLB!>1g3$Ye#b7+Wz}G!%2@L66_>kX5*YQW^)n|EK{fN*r?ce<;jv-cKgxSCX1%fQg zs&(0qe@aLGIog)rVGVHeB%EnlNBAb#Df&$xjMV=}kG%W){?P37p_S9@O9ZT@54ptf zWknk*ChR#Iw4t49okhc9E3($u=~`6n5{M|6%h=}NY#Yd+%8?Ydq%t8!$TU3j*skHh zE1zTiGHh9l!hIzX%#^meyi!`CpYtNixB4T6f5$vNY9+7Frr`H0NN@5I+1xZDY3W`A z*EfFpkt@}lIvRKc0b0L2gZzcL);PTJ81cXT>%t?>nZK~QuT~h zeI^AS~yoZ*));(cS_;80YJ?dgbEHv;GeH(9ZJalm?TzLa7slIn8 ze|#T-T9|{H5N3rQ2$g!u={`cG9%9aZDHmu6foyL3il!Dp9-zA`-rLvOHJ<$IR)e-)QuN+W4XLvE!F(@K0{e9s|**RWZ}BIHMt zRXs%i2zjL=sc0mZ_nZo0df&;O1=VR7MX^0PdEIl8g2p^HD;1Psk#wN15vfBY9YjKO z-osod6-f-m01`+|OR+t&UUUW0*qjWg33`G9MvN2Ig^x8Ve&u@xq6agLn=~;Ee|Z&9 zY0)$K*36FLHfHD?j-OJkufY7TPnC1{#gR;~J6^E#edsNWPM~6gArNXoZnn!0_l;Hbf3k2>=7& z#BfJ|f+M(@gcgw6*c_$ok=+3Qe+0={qV33H(H7M`BYofy+;Q4A#33jzw1Pu4oUpV_ zVY&mAY+x^aZ*))58oBRV-a|qrR*6EWBmWgBgpNcZQXFn23Q^_@twbFI$KO(phr$A) zjtCPFG>(oH@HGp9rbq(+0Gh5$<_iC0+3aziRwC5|B4e~?PYeeX=z zmp0lV9T&(|+@30IPn=m&$zTxB+hR;5f@PvBhz^i#93t#n6|95}mPzG;_N}5M4|)eh zQwl67-G^RbGhjg43^tUeymqk^w92La#7b|Vt-;=*T>=5Mi`EQ?B8e4rCuo1FpgTp} zbF?`u83+k83n*~{8^wbTf5`@DGQ-#kf|h8YPJ{HAj;6o^VdT@Y<*dOJkhVv!p zPc5WQ+8#Jd4g3S8U1M@xxhZ29Y-+5AOO>aW zu}Rl^_+a&QH2zx0!mS20@e8xMi#WWhnpM?p&g80lT%5(%=caX6e|5J$Ky9#~cu^X5 zDg{rDtFR-h1JmT;oI*j=Ga@++l@RerlA=8N%sPZ(6SMp#*+LhcH{lqUZDa*!T!rx! zhVc%oHfFW)Zdc*$YMsIfSXO-;*O%bbfVE{?HD}mWp;WyZxD}|8Na1i6@o-KS8CO@X z#l|aZ6KOZ((Wwkwe*iBf=!Q4H5A5CXE^!tev2P>vGGwJp3_HF4vJ{q*~MQCuEPK2JmNeKDFZX5Sy}4yT`r z$L!T?SjS|nEQbe62lSAVl6!Y`hf9ML)Zg}!Qap}Pa*y@~p z2#-NR&Zj1CbqCv>pVL^#&NFJI`VJmeuv19&9vLM>w^z@L(Z!e8XZP#HhtY8H1?RgB zL+w%cy@SIb+rjC8tUEpbz}ekrM~I3?=ev`OagpWha6Gt}X9nYb9~U8vo=oS(FZuGH z&TIdklxurAf4D5#@uMDxqj5oxBg$+F5G@w;dCxKRZjDSlk@2is$fZ5 z1Z+5f5YvzeiCKxW={qDGro#ef%S}==lV$JY(fgB+D`CVo$)eWDVs1$mlafW1WHGu% z7L$-g*2n_y4`d0CXy_3UJhBHQODldUS#+H&wgXw(f5z`h7P~9uSChWXjIBaY6)B1y$+yOvS{T~vPkk)JiQP=?{hrx0o00LN)=nDitj*`w(&c& zMQ*Ugc4y1(@$>zU$B8ssGAv@bjZ%9mzM)JF6MJK9}nhV*abZ&E?`{P9@}RJ>@~wP;FUf`?BDFq zQWrH%BzmY})6LYrKY0J_?Co+Z`z@Z!Vm&Yv_X-T*(YKw;M%A2sNw$_=x(n|Pee5s+W$AzLxT-Y*vXH&c3?%O-K`}@0xryp1DB*33<)sGYF!J*9W!xwjh(=SuL zcvJdu8tETIr1?Yg<=h%LRRd110q1yyoJ$ZVO%hBjA>VQ??<3fLDdnD*TxYs7@~frI?vDCp5gRxLDxj+yt}QLT(M3 zt%2s&K=ZZeE3q~&kIwc_f8PXGwSjMTt6n^`*F?JIUPZvQzE^!6zfYw%K4ISy{L&@e z7CxV+@Oi!lpGx4<33~7L_THSmq=2gdJh!dT)i#B$@MBBd;(IqKZeG-JOT*GMY^hdr zYuscFH`Q+BAXkOki`TE7J$)0*LepOLTQhTc6XW3A?Sw5$xk`(de|b{qbfcLIp(D*! z%!(nqydEz`SL5&Od^9^B7a{bgfHH>oWjq-#sQL2xd=+Je`d~Jj{#AzkRQ)tvd|{K~ zBD}5~gHpVEJUSnYC({M{Jf2QxAz=_|@xD)RKxTs>^+_~Z!?~M}czFE!*VC6tPAP)A z4IgnmK`if;Al9Zme?eS(nt%-&0}J}Yug%BVfjD^i4lfq!iZyo$GTNX_9maaDSa!)3 zw+!FaVN@HTg}*~+@%sJQ*^A@Kd)*pq)OxO1iT_Dt;mgRvN!1WpG?9kiDavR8GU1G> zXN*;sjB)emohf5BD07FW3%~9@J3jusl;y=@dyf~?dd5rLe=g%?amrMb;dbf1$dod4 z614@R^mrT3)@`7S^fx?Q;5R&6Xy-}`f+<((NiWqU>D4@XXRf#nuDI@8dHL#O_wB1v zmThK7)q1u|-|JNft2!kvmRxz*#6=5uDNX8WF5M-~)jWFV*Q@dknk>)4{{l(94Co4F zZe(+Ga%Ev{lgq*hm$iWb8bV|>HD)t7IXN?9H!w3WGGR1jHaBE6IWaS2VP$1CGh#kG zWHmKrGdMXpGh;U}GcYn?G-Wn7WHdQ3Gh|_9Wi>Nmml1*iR)5J$m0ScukK3X$CP^q@iXI25P!$qyNQ1#?lok=UHAUblkT--*=6^E z&|lo^$Vlp$_dgtZz+FWq$K5?-s?fcjZ292sC0ong8_1q-?hKi3aBn2Dm)yC~6Yfpn te;#u0k^5x3JSLCJ6Y`|&CU2Gk{{i**T{)L5gaIB3Fg7;|B_%~qMhd_r3vd7c diff --git a/spec/tinyspec.tex b/spec/tinyspec.tex index 911ee82c03b97..c541657a657a7 100644 --- a/spec/tinyspec.tex +++ b/spec/tinyspec.tex @@ -50,9 +50,11 @@ \subsection*{Source Ops \normalfont\small--- leaf nodes} \toprule \textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ \midrule -\op{Param} & () & slot, dtype, size?, device?, addrspace? & +\op{Param} & () & \texttt{ParamArg} & Placeholder with flat storage of $\mathrm{size}$ elements. Substituted in \op{Call}. \\[4pt] -\op{Buffer} & () & slot, dtype, size, device, addrspace & +\op{Returned} & () & \texttt{ParamArg} & + Call output buffer placeholder; slot is its call argument index. \\[4pt] +\op{Buffer} & () & \texttt{ParamArg} & Concrete buffer slot with flat storage of $\mathrm{size}$ elements. \\ \op{Const} & () & value, dtype & A scalar constant with shape $(\ )$. \\ @@ -62,7 +64,21 @@ \subsection*{Source Ops \normalfont\small--- leaf nodes} \end{tabular} \smallskip -\textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. +\texttt{ParamArg} contains slot, dtype, concrete size (or \textsc{null} for a scalar), value bounds, alignment, name, addrspace, device, volatility, and optional image shape. \textbf{addrspace} is \texttt{GLOBAL}, \texttt{LOCAL}, or \texttt{REG}. + +%% ============================================================ +\subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus} + +\begin{tabular}{@{}l l l l@{}} +\toprule +\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ +\midrule +\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in body with $a_k$. \\ +\bottomrule +\end{tabular} + +\smallskip +A value \op{Call} is void: its \op{Sink} body stores to \op{Param}s bound positionally to \op{Returned} arguments; output $a_k$ is \op{After}$(a_k, \op{Call})$. %% ============================================================ \subsection*{{\color{movgreen}Movement Ops} \normalfont\small--- no arithmetic; view, indexing, and reinterpretation only} @@ -95,19 +111,6 @@ \subsection*{{\color{reducered}Reduce Ops} \normalfont\small--- remove axes} \bottomrule \end{tabular} -%% ============================================================ -\subsection*{{\color{callblue}Call Ops} \normalfont\small--- function abstraction, like the lambda calculus} - -\begin{tabular}{@{}l l l l@{}} -\toprule -\textbf{Op} & \textbf{src} & \textbf{arg} & \textbf{Semantics} \\ -\midrule -\op{Call} & (body, $a_0$, $a_1$, \ldots) & --- & Substitute each \op{Param} $k$ in body with $a_k$. \\ -\op{Tuple} & $(v_0, v_1, \ldots)$ & --- & Pack values; required as a value-producing \op{Call} body. \\ -\op{GetTuple} & $(T,)$ & idx & Extract element at idx from a \op{Tuple}. \\ -\bottomrule -\end{tabular} - %% ============================================================ \subsection*{{\color{loadred}Load Ops} \normalfont\small--- can change device or addrspace} @@ -257,7 +260,8 @@ \subsection*{Derived Properties} \midrule \op{Buffer} & from arg & from arg ($\mathrm{size}$) & from arg & dtype range \\ \op{Const} & from arg & $()$ & \textsc{null} & $[v, v]$ \\ -\op{Param} & from arg & from arg ($\mathrm{size}$) & from arg & from src or dtype range \\[3pt] +\op{Param} & from arg & from arg ($\mathrm{size}$) & from arg & from src or dtype range \\ +\op{Returned} & from arg & from arg ($\mathrm{size}$) & from arg & dtype range \\[3pt] Movement ops & $\mathrm{src}[0].\mathrm{dtype}$ & (see op) & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Unshard} & $\mathrm{src}[0].\mathrm{dtype}$ & $\mathrm{src}[0]$, each $a_k \times n_k$ & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Reduce} & $\mathrm{src}[0].\mathrm{dtype}$ & remove first $n$ axes & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\[3pt] @@ -271,7 +275,7 @@ \subsection*{Derived Properties} Other binary & $\mathrm{src}[0].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & dtype range \\ \op{CmpLt}, \op{CmpNe} & bool & broadcast & $\mathrm{src}[0].\mathrm{device}$ & from intervals \\ \op{Where} & $\mathrm{src}[1].\mathrm{dtype}$ & broadcast & $\mathrm{src}[0].\mathrm{device}$ & $[\min(b,c),\, \max(B,C)]$ \\[3pt] -\op{Call} & $\mathrm{src}[0].\mathrm{dtype}$ & substitute \op{Param} shapes & $\mathrm{src}[1].\mathrm{device}$ & dtype range \\ +\op{Call} & void & --- & first non-null src device & --- \\ \op{Range} & index & $()$ & \textsc{null} & $[0,\, n{-}1]$ \\ \op{Index} & $\mathrm{src}[0].\mathrm{dtype}$ & remaining dims & $\mathrm{src}[0].\mathrm{device}$ & $\mathrm{src}[0]$ \\ \op{Store} & void & $()$ & $\mathrm{src}[0].\mathrm{device}$ & --- \\ @@ -432,14 +436,15 @@ \subsection*{{\color{callblue}The \texttt{@function} Decorator} \normalfont\smal \begin{enumerate}[leftmargin=1.5em, itemsep=2pt] \item \textbf{Extracts inputs}: walks all arguments to find every Tensor, deduplicates by identity. - \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from the result. - \item \textbf{Parameterizes}: replaces each input UOp with a \op{Param}$(k)$ placeholder. - \item \textbf{Wraps the body} in a \op{Tuple} (even for single returns) and creates\\ - \op{Call}(\op{Tuple}(body), $x$, $y$). - \item \textbf{Returns} the result via \op{GetTuple}$(0)$, or one \op{GetTuple} per element for tuple returns. + \item \textbf{Runs the function} lazily (no device execution), building a UOp graph from each returned value. + \item \textbf{Parameterizes inputs}: replaces each input UOp with a positional \op{Param}$(k)$ placeholder. + \item \textbf{Parameterizes outputs}: for each returned value $v_i$, creates an output \op{Param}$(m+i)$ and a matching \op{Returned}$(m+i)$, where $m$ is the number of inputs. + \item \textbf{Builds the call}: stores every $v_i$ into its output parameter and creates\\ + \op{Call}(\op{Sink}(\op{Store}(\op{Param}$(m)$, $v_0$), \ldots), $x$, $y$, \op{Returned}$(m)$, \ldots). + \item \textbf{Returns values}: exposes each result as \op{After}(\op{Returned}$(m+i)$, \op{Call}). \end{enumerate} -The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers. At schedule time, the value-producing \op{Call} is resolved by substituting each \op{Param}$(k)$ back with its corresponding argument $a_k$, or its body is lowered into a \op{Sink} (opaque) if it is to be compiled as a reusable kernel. +The result is a reusable graph fragment: the body contains only \op{Param} references, not concrete buffers, and a single call can return any number of values. At schedule time, an ordinary value-producing \op{Call} is inlined by positional \op{Param} substitution and each output \op{After} resolves to the value stored in the body. A precompiled call instead materializes real output buffers in the \op{Returned} slots and lowers the body to an opaque call that writes them. %% ============================================================ \subsection*{Lowering Pipeline \normalfont\small--- from Tensor graph to machine code} diff --git a/test/null/test_viz.py b/test/null/test_viz.py index 998d9b9efc836..fae51d311e754 100644 --- a/test/null/test_viz.py +++ b/test/null/test_viz.py @@ -228,8 +228,7 @@ def test_enter_calls_rewrite(self): with save_viz() as viz: inner = UOp.const(3) call = UOp(Ops.CALL, src=(UOp(Ops.SINK, src=(inner,)),)) - func = UOp(Ops.CALL, src=(UOp(Ops.TUPLE, src=(call,)),)) - graph_rewrite(func, TrackedPatternMatcher(pm.patterns), enter_calls=True) + graph_rewrite(call, TrackedPatternMatcher(pm.patterns), enter_calls=True) details = list(viz.get_details(0, 0)) self.assertTrue(details[-1]["change"], "viz replay should detect change inside CALL") diff --git a/test/unit/test_call.py b/test/unit/test_call.py index d681802746fce..db509e7cedf50 100644 --- a/test/unit/test_call.py +++ b/test/unit/test_call.py @@ -1,6 +1,6 @@ import unittest import numpy as np -from tinygrad import Tensor, function +from tinygrad import Tensor, function, Device from tinygrad.dtype import dtypes from tinygrad.uop.ops import UOp, Ops @@ -223,8 +223,8 @@ def f(x:Tensor, scale:UOp) -> Tensor: return x * scale a = Tensor.ones(3) x = f(a, UOp.variable("scale_a", 1, 100).bind(2)) y = f(a, UOp.variable("scale_b", 1, 100).bind(3)) - fx = next(u for u in x.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) - fy = next(u for u in y.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) + fx = next(u for u in x.uop.toposort() if u.op is Ops.CALL and u.num_returned) + fy = next(u for u in y.uop.toposort() if u.op is Ops.CALL and u.num_returned) self.assertEqual(fx.src[0].key, fy.src[0].key) np.testing.assert_equal(x.numpy(), [2, 2, 2]) np.testing.assert_equal(y.numpy(), [3, 3, 3]) @@ -251,9 +251,9 @@ def f(x:Tensor) -> Tensor: return x + Tensor.full(x.shape, -1.0) a = Tensor.empty(4, 8) b = Tensor.empty(4, 8) r0, r1 = f(a), f(b) - # find the value-producing call nodes - c0 = next(u for u in r0.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) - c1 = next(u for u in r1.uop.toposort() if u.op is Ops.CALL and u.src[0].op is Ops.TUPLE) + # find the call nodes + c0 = next(u for u in r0.uop.toposort() if u.op is Ops.CALL and u.num_returned) + c1 = next(u for u in r1.uop.toposort() if u.op is Ops.CALL and u.num_returned) # the function bodies (src[0]) should have identical keys self.assertEqual(c0.src[0].key, c1.src[0].key) @@ -276,6 +276,54 @@ def f(x:Tensor) -> Tensor: return x + 1 out = f(a) + 2 np.testing.assert_allclose(out.numpy(), np.arange(8, dtype=np.float32).reshape(4, 2) + 3) +class TestArgOrder(unittest.TestCase): + """RETURNED placeholders can appear anywhere in a call's srcs: slots are src positions, nothing reorders""" + def make_intersperse_call(self, x, precompile=False): + # call with sources (body, returned(slot=0), input(slot=1)): the input is the input, the output binds the RETURNED + dev = x.device if isinstance(x.device, str) else (x.device or (Device.DEFAULT,))[0] + r0 = UOp.returned(0, x.dtype, x.shape, device=dev) + o0 = UOp.param(0, x.dtype, x.shape, dev) + p1 = UOp.param(1, x.dtype, x.shape, dev) + from tinygrad.uop.ops import CallInfo + return UOp(Ops.CALL, src=(UOp.sink(o0.store(p1.reshape(x.shape) * 2)), r0, x.uop), + arg=CallInfo(None, 't', precompile, False, None)) + + def test_intersperse_returned(self): + x = Tensor.arange(3, dtype=dtypes.int).realize() + call = self.make_intersperse_call(x) + out = Tensor(call.returned_outputs[0], device=x.device) + 1 + np.testing.assert_equal(out.numpy(), [1, 3, 5]) + + def test_intersperse_returned_precompile(self): + x = Tensor.arange(3, dtype=dtypes.int).realize() + call = self.make_intersperse_call(x, precompile=True) + # the transform must preserve the RETURNED's src position: its placeholder is at src 1, the input stays at src 2 + from tinygrad.tensor import transform_precompiled_call + new = transform_precompiled_call(call) + new_call = new.src[0].src[1].src[1] + # the out buffer takes the RETURNED's position (src 1), the input value keeps its position (src 2) + self.assertEqual(new_call.src[1].op, Ops.BUFFER) + self.assertEqual(new_call.src[1].arg.size, 3) + self.assertEqual(new_call.src[2].op, Ops.ADD) + # the body binds positionally: store dest at slot 0 (the RETURNED's position), input param at slot 1 + store = [u for u in new_call.src[0].toposort(enter_calls=False) if u.op is Ops.STORE][0] + self.assertEqual(store.src[0].arg.slot, 0) + self.assertEqual([u.arg.slot for u in store.src[1].toposort(enter_calls=False) if u.op is Ops.PARAM], [1]) + + def test_intersperse_returned_gradient(self): + x = Tensor([1.0, 2.0, 3.0]).realize() + x.requires_grad = True + dev = x.device if isinstance(x.device, str) else (x.device or (Device.DEFAULT,))[0] + r0 = UOp.returned(0, dtypes.float, x.shape, device=dev) + o0 = UOp.param(0, dtypes.float, x.shape, dev) + p1 = UOp.param(1, dtypes.float, x.shape, dev) + from tinygrad.uop.ops import CallInfo + body = UOp.sink(o0.store(p1.reshape(x.shape) * p1.reshape(x.shape))) + call = UOp(Ops.CALL, src=(body, r0, x.uop), arg=CallInfo(None, 't', False, False, None)) + y = Tensor(call.returned_outputs[0], device=x.device) + y.sum().backward() + np.testing.assert_equal(x.grad.numpy(), [2, 4, 6]) + class TestCallMultiSharded(unittest.TestCase): # TODO: multi-output + sharded needs per-device CALL execution, which requires reworking how MULTI propagates through TUPLE bodies def test_tuple_sharded(self): diff --git a/test/unit/test_function.py b/test/unit/test_function.py index ec1cd33a15601..0ebcaef776d01 100644 --- a/test/unit/test_function.py +++ b/test/unit/test_function.py @@ -15,6 +15,16 @@ def f(a:Tensor, b:Tensor) -> Tensor: return a+b b = Tensor([4,5,6]) np.testing.assert_equal(f(a,b).numpy(), [5,7,9]) + def test_two_return(self, precompile=False): + @function(precompile=precompile) + def f(a:Tensor, b:Tensor) -> tuple[Tensor, Tensor]: + return (a+b, (a+b)*2) + a = Tensor([1,2,3]) + b = Tensor([4,5,6]) + c = f(a,b) + np.testing.assert_equal((c[0]+c[1]).numpy(), [5*3,7*3,9*3]) + def test_two_return_precompiled(self): self.test_two_return(True) + def test_simple_same(self): @function def f(a:Tensor, b:Tensor) -> Tensor: return a+b @@ -174,13 +184,13 @@ def f(a:Tensor, b:Tensor, c:Tensor) -> Tensor: return a + c # b is unused def test_name(self): @function def f(a:Tensor) -> Tensor: return a + 1 - assert f(Tensor([1])).uop.src[0].arg.name.endswith("f") + assert f(Tensor([1])).uop.src[1].arg.name.endswith("f") def test_method_name(self): class Foo: @function def __call__(self, x:Tensor) -> Tensor: return x + 1 - assert Foo()(Tensor([1])).uop.src[0].arg.name.endswith("Foo.__call__") + assert Foo()(Tensor([1])).uop.src[1].arg.name.endswith("Foo.__call__") def test_callable_instance(self): class Foo: @@ -189,7 +199,7 @@ def __call__(self, x:Tensor) -> Tensor: return x + self.w foo = Foo() f = function(foo, allow_implicit=True) np.testing.assert_equal(f(Tensor([1,2,3])).numpy(), [11,22,33]) - assert f(Tensor([1,2,3])).uop.src[0].arg.name.endswith("Foo") + assert f(Tensor([1,2,3])).uop.src[1].arg.name.endswith("Foo") def test_iadd(self): @function diff --git a/tinygrad/function.py b/tinygrad/function.py index 813fcd2ec83a6..4ba84d7c0b09a 100644 --- a/tinygrad/function.py +++ b/tinygrad/function.py @@ -59,7 +59,7 @@ def __call__(self, *args, **kwargs) -> ReturnType: if isinstance(ret, Tensor): uret = ret.uop elif isinstance(ret, tuple) and all(isinstance(x, Tensor) for x in ret): - uret = UOp.maketuple(*[x.uop for x in ret]) + uret = UOp.sink(*[x.uop for x in ret]) else: raise RuntimeError(f"function return type {type(ret)} not supported") @@ -78,16 +78,17 @@ def __call__(self, *args, **kwargs) -> ReturnType: buf_strs = '\n '.join(f"{i}: dtype={b.dtype}, size={b.max_numel()}, device={b.device}" for i,b in enumerate(implicit_buffers)) raise RuntimeError(f"function {name} has {len(implicit_buffers)} implicit buffer(s), but allow_implicit=False\n {buf_strs}") - fret = uret.call(*call_uops, grad_fxn=self.grad_fxn, name=name, precompile=self.precompile, - precompile_backward=self.precompile_backward) + fret = UOp.call_outputs(uret.src if isinstance(ret, tuple) else (uret,), *call_uops, grad_fxn=self.grad_fxn, name=name, + precompile=self.precompile, precompile_backward=self.precompile_backward) if DEBUG >= 2: print(" "*_function.depth+f"function {uret.key.hex()[:8]} in {(time.perf_counter()-st)*1000:8.2f} ms: {name}") + outs = fret.returned_outputs if isinstance(ret, tuple): - return cast(ReturnType, tuple(Tensor(fret.gettuple(i)) for i in range(len(ret)))) + return cast(ReturnType, tuple(Tensor(o) for o in outs)) else: - return cast(ReturnType, Tensor(fret.gettuple(0))) + return cast(ReturnType, Tensor(outs[0])) # overload signatures support both @function and @function(precompile=True) syntax @overload diff --git a/tinygrad/mixin/gradient.py b/tinygrad/mixin/gradient.py index 0d154e7451591..93e5d213bb89c 100644 --- a/tinygrad/mixin/gradient.py +++ b/tinygrad/mixin/gradient.py @@ -16,7 +16,8 @@ def reduce_gradient(ctx:UOp, ret:UOp, op:Ops): def _compact_params(body:UOp, all_args:tuple[UOp, ...]) -> tuple[UOp, tuple[UOp, ...]]: """Remove unused PARAMs from body and return compacted (body, args).""" - used = sorted({p.arg.slot: p for p in body.toposort() if p.op is Ops.PARAM}.items()) + # NOTE: don't enter nested calls, their PARAMs are lexical params of the subprogram + used = sorted({p.arg.slot: p for p in body.toposort(enter_calls=False) if p.op is Ops.PARAM}.items()) body = body.substitute({p: p.replace(arg=dataclasses.replace(p.arg, slot=j)) for j,(_, p) in enumerate(used)}, walk=True) return body, tuple(all_args[i] for i,_ in used) @@ -24,32 +25,44 @@ def call_gradient(ctx:UOp, k:UOp, needed:set[int]) -> tuple[UOp|None, ...]: fxn, args = k.src[0], k.src[1:] if k.arg.grad_fxn is not None: # put const on a device, also TODO why do we still have NOOP... - def on_dev(g, i): return g.clone(device=k.device if fxn.op is Ops.TUPLE else args[i].device) if g.device is None else g - if ctx.op is Ops.TUPLE: + def on_dev(g, i): return g.clone(device=args[i].device) if g.device is None else g + # grads align with the call's src positions (None for the body and for RETURNED outputs, wherever they are) + def arg_grads(g): + git = iter(g) + return (None,) + tuple(next(git) if a.unsharded_base.op is not Ops.RETURNED else None for a in k.src[1:]) + if ctx.op is Ops.SINK: real = [on_dev(g, i) for i,g in enumerate(ctx.src) if g.op is not Ops.NOOP] - return (None,) + (k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) - return (None,) + k.arg.grad_fxn(on_dev(ctx, 0), k) - assert fxn.op is Ops.TUPLE, f"expected TUPLE body for gradient, got {fxn.op}" + return arg_grads(k.arg.grad_fxn(*real, call=k) if len(real) > 1 else k.arg.grad_fxn(real[0], k)) + return arg_grads(k.arg.grad_fxn(on_dev(ctx, 0), k)) + # the RETURNED inputs are the call outputs: their positions in the args get the output gradients from the AFTER rule + assert fxn.op is Ops.SINK and k.num_returned, f"expected a CALL with RETURNED inputs or a grad_fxn, got {fxn.op}" + ret_pos = [i for i, a in enumerate(args) if a.unsharded_base.op is Ops.RETURNED] + # the body stores the outputs into output PARAMs: the values are the stored values in slot order + values = UOp.sink(*[st.src[1] for st in fxn.src if st.op is Ops.STORE]) params = {x.arg.slot:x for x in fxn.toposort(enter_calls=False) if x.op == Ops.PARAM} # grads are collected at the flat param storage: reshape to each arg's view (max view shrunk to symbolic) def shaped_grad(grad:UOp, i:int) -> UOp: a = args[i] return grad.view_as(a.shard_shape, a.axis) if a.axis is not None and isinstance(a.device, tuple) else grad.view_as(a._shape) - grad_args = ctx.src - root_grad = UOp(Ops.TUPLE, src=tuple(UOp(Ops.NOOP) if g.op is Ops.NOOP else - g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args))) - grads = compute_gradient(fxn, root_grad, set(params.values())) + grad_args = tuple(ctx.src[i] for i in ret_pos) + root_grad = UOp.sink(*[UOp(Ops.NOOP) if g.op is Ops.NOOP else + g if g.device is None else g.param_like(len(args)+i) for i,g in enumerate(grad_args)]) + grads = compute_gradient(values, root_grad, set(params.values())) # for precompiled calls, substitute forward outputs with params so intermediates aren't recomputed - fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(fxn.src)} if k.arg.precompile else {} - fwd_outs = tuple(k.gettuple(i) for i in range(len(fxn.src))) if k.arg.precompile else () + fwd_subs = {src: src.param_like(len(args)+len(grad_args)+i) for i, src in enumerate(values.src)} if k.arg.precompile else {} + fwd_outs = k.returned_outputs if k.arg.precompile else () # collect needed gradient bodies, compact unused params, create a single backward CALL grad_bodies = [(i, shaped_grad(grads[p], i)) for i in needed if (p:=params.get(i)) is not None and p in grads] - bwd_body = UOp.maketuple(*(gb for _, gb in grad_bodies)).substitute(fwd_subs, walk=True) + bwd_body = UOp.sink(*[gb for _, gb in grad_bodies]).substitute(fwd_subs, walk=True) bwd_body = renumber_invalid_outputs(bwd_body) + # NOTE: args includes the RETURNED inputs so the param slots above line up; they are unused and compacted away bwd_body, compact_args = _compact_params(bwd_body, (*args, *grad_args, *fwd_outs)) - bwd_call = bwd_body.call(*compact_args, name=(k.arg.name or "")+"_backward", precompile=k.arg.precompile_backward) + bwd_outs = UOp.call_outputs(bwd_body.src, *compact_args, name=(k.arg.name or "")+"_backward", + precompile=k.arg.precompile_backward).returned_outputs gb_map = {i: idx for idx, (i, _) in enumerate(grad_bodies)} - return (None,) + tuple(bwd_call.gettuple(gb_map[i]) if i in gb_map else None for i in range(len(args))) + # align gradients with the original source positions: None at RETURNED positions, gradients elsewhere + ret_set = set(ret_pos) + return (None,) + tuple(None if i in ret_set else (bwd_outs[gb_map[i]] if i in gb_map else None) for i in range(len(args))) # ctx is grad_output pm_gradient = PatternMatcher([ @@ -80,9 +93,9 @@ def shaped_grad(grad:UOp, i:int) -> UOp: (UPat(Ops.STACK, name="ret"), lambda ctx, ret: tuple(ctx[i] for i in range(len(ret.src)))), (UPat(Ops.COPY, name="ret"), lambda ctx, ret: (ctx.copy_to_device(ret.src[0].device),)), (UPat(Ops.UNSHARD, name="ret"), lambda ctx, ret: ctx.shard(ret.device, ret.axis).src), - (UPat(Ops.TUPLE), lambda ctx: ctx.src), + (UPat(Ops.SINK), lambda ctx: ctx.src), (UPat(Ops.AFTER, src=(UPat.var("d"), UPat(Ops.CALL, name="k"))), lambda ctx, d, k: - (ctx, UOp.maketuple(*(ctx if i == k.src.index(d)-1 else UOp(Ops.NOOP) for i in range(len(k.src)-1))))), + (ctx, UOp.sink(*([ctx if i == k.src.index(d)-1 else UOp(Ops.NOOP) for i in range(len(k.src)-1)])))), # clone/assign gradient passes through to val (UPat(Ops.AFTER, src=(UPat(), UPat(Ops.STORE))), lambda ctx: (None, ctx)), (UPat(Ops.STORE, src=(UPat(), UPat())), lambda ctx: (None, ctx)), @@ -102,17 +115,8 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp grads: dict[UOp, UOp] = {root: root_grad} for t0 in reversed(walk): if t0 not in grads or grads[t0].op is Ops.NOOP: continue - # GETTUPLE: accumulate gradient into a TUPLE UOp on the value-producing call, process when we hit the call - if t0.op is Ops.GETTUPLE: - k = t0.src[0] # the call - assert k.op is Ops.CALL and k.src[0].op is Ops.TUPLE - n_outputs = len(k.src[0].src) - prev = grads[k].src if k in grads else tuple(UOp(Ops.NOOP) for _ in range(n_outputs)) - grads[k] = UOp.maketuple(*(prev[i] + grads[t0] if i == t0.arg and prev[i].op is not Ops.NOOP else - grads[t0] if i == t0.arg else prev[i] for i in range(n_outputs))) - continue # CALL: pass needed param set so backward only computes required gradients - # (value-producing calls use implicit TUPLE gradient or grad_fxn; opaque calls require an explicit grad_fxn) + # (calls with RETURNED inputs use the implicit body gradient or grad_fxn; opaque CALLs require an explicit grad_fxn) if t0.op is Ops.CALL: needed = {i for i, arg in enumerate(t0.src[1:]) if arg in targets or in_target_path.get(arg, False)} lgrads:tuple[UOp|None, ...]|None = call_gradient(grads[t0], t0, needed) @@ -126,9 +130,9 @@ def compute_gradient(root:UOp, root_grad:UOp, targets:set[UOp]) -> dict[UOp, UOp if k._shape is not None and v._shape is not None and k._shape != v._shape: v = v.cast(sum_acc_dtype(v.dtype))._rop(Ops.ADD, broadcast_axes(k.shape, v.shape)).reshape(k.shape).cast(v.dtype) if k in grads and grads[k].op is not Ops.NOOP: - if v.op is Ops.TUPLE and grads[k].op is Ops.TUPLE: - grads[k] = UOp.maketuple(*(p + n if (p.op is not Ops.NOOP and n.op is not Ops.NOOP) else - n if p.op is Ops.NOOP else p for p, n in zip(grads[k].src, v.src))) + if v.op is Ops.SINK and grads[k].op is Ops.SINK: + grads[k] = UOp.sink(*[p + n if (p.op is not Ops.NOOP and n.op is not Ops.NOOP) else + n if p.op is Ops.NOOP else p for p, n in zip(grads[k].src, v.src)]) else: grads[k] = grads[k] + v else: grads[k] = v if len(forward_metadata:=all_metadata.get(t0, ())): diff --git a/tinygrad/nn/__init__.py b/tinygrad/nn/__init__.py index a74a4c295fb3d..aae4600cc2b32 100644 --- a/tinygrad/nn/__init__.py +++ b/tinygrad/nn/__init__.py @@ -305,7 +305,7 @@ def __call__(self, x:Tensor) -> Tensor: from tinygrad.uop.ops import UOp, KernelInfo, Ops, AxisType def _embedding_bwd(grad_emb:UOp, call:UOp) -> tuple: - weight, idx = call.src[1:] + weight, idx = (a for a in call.src[1:] if a.unsharded_base.op is not Ops.RETURNED) is_vocab_sharded = isinstance(weight.device, tuple) and weight.axis == 0 # for multi-device: replicate grad_emb and idx on all devices if isinstance(weight.device, tuple): diff --git a/tinygrad/schedule/multi.py b/tinygrad/schedule/multi.py index b8b377c2f9dd0..0cfff99529860 100644 --- a/tinygrad/schedule/multi.py +++ b/tinygrad/schedule/multi.py @@ -269,16 +269,12 @@ def passthrough_multi(root:UOp, multi:UOp): return UOp(root.op, src=new_src, arg=root.arg).unshard(multi.arg, multi.src[1:]) def rewrite_into_function(call:UOp): - if call.arg.precompile: return None + if call.arg is None or call.arg.precompile: return None + # the call body is a plain parametric program: multi rewrites it like anything else (the output PARAM dests sub-view per + # shard through the normal store rules), and all srcs (args and RETURNEDs) become their per-shard views new_body = graph_rewrite(call.src[0], multi_pm, name="subcall") - new_args = tuple(a.src[0] if a.op is Ops.UNSHARD else a for a in call.src[1:]) - # after multi resolution, TUPLE elements may be UNSHARD — strip UNSHARD from body, create per-shard call, wrap each GETTUPLE in its own UNSHARD - assert new_body.op is Ops.TUPLE - if any(s.op is Ops.UNSHARD for s in new_body.src): - shard_call = call.replace(src=(UOp.maketuple(*[s.src[0] if s.op is Ops.UNSHARD else s for s in new_body.src]),)+new_args) - return UOp.maketuple(*[shard_call.gettuple(i).unshard(s.arg, s.src[1:]) if s.op is Ops.UNSHARD else shard_call.gettuple(i) - for i, s in enumerate(new_body.src)]) - return call.replace(src=(new_body,)+new_args) + assert new_body.op is Ops.SINK + return call.replace(src=(new_body,) + tuple(a.src[0] if a.op is Ops.UNSHARD else a for a in call.src[1:])) # NOTE: this is the same pattern as unrolled ranges multi_pm = PatternMatcher([ @@ -297,16 +293,12 @@ def rewrite_into_function(call:UOp): (UPat(Ops.ALLREDUCE, src=(UPat(Ops.UNSHARD, name="multi"),), name="red"), lambda multi,red: multi.src[0].allreduce(*red.arg).unshard(multi.arg, multi.src[1:])), - # resolve TUPLE+GETTUPLE (needed in multi) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - # rewrite into FUNCTION calls explicitly for UNSHARD (value-producing) - # rewrite into value-producing calls explicitly for UNSHARD - (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="call"), rewrite_into_function), + # rewrite value-producing calls explicitly for UNSHARD + (UPat(Ops.CALL, name="call"), lambda call: rewrite_into_function(call) if call.num_returned else None), (UPat((Ops.CALL, Ops.AFTER), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root", allow_any_len=True), passthrough_multi), # just strip the UNSHARD from non-value-producing CALLs (custom kernels, etc.) — value-producing CALLs are handled by rewrite_into_function (UPat(Ops.CALL, dtype=dtypes.void, name="root", custom_early_reject=set([Ops.UNSHARD])), lambda root: - UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg) - if root.src[0].op is not Ops.TUPLE else None), + UOp(root.op, src=tuple(x.src[0] if x.op is Ops.UNSHARD else x for x in root.src), arg=root.arg) if root.num_returned == 0 else None), (UPat((Ops.CAST, Ops.BITCAST, Ops.CONTIGUOUS, Ops.DETACH, Ops.CONTIGUOUS_BACKWARD), src=(UPat(Ops.UNSHARD, name="multi"), ), name="root"), passthrough_multi), # STORE of a sharded value into an unsharded dest (e.g. a fragment into a full output tile) diff --git a/tinygrad/schedule/prepare.py b/tinygrad/schedule/prepare.py index 037afb6df7424..95f019dc200bb 100644 --- a/tinygrad/schedule/prepare.py +++ b/tinygrad/schedule/prepare.py @@ -1,7 +1,7 @@ import itertools from tinygrad.dtype import dtypes, to_dtype from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp -from tinygrad.uop.ops import graph_rewrite, rewrite_group, ParamArg, identity_element +from tinygrad.uop.ops import graph_rewrite, rewrite_group, ParamArg, identity_element, resolve_returned_after from tinygrad.uop.movement import mop_cleanup from tinygrad.helpers import prod, getenv, all_int, DEBUG, SPLIT_REDUCEOP, OPENPILOT_HACKS, FLOAT16, argsort from tinygrad.schedule.indexing import apply_movement_op @@ -93,6 +93,7 @@ def resolve_function(c:UOp, allow_param_mismatch=True) -> UOp|None: params: list[UOp] = [] graph_rewrite(c.src[0], pm_gather_params, bottom_up=True, ctx=params, name="gather params") params = sorted(params, key=lambda x: x.arg.slot) + # the RETURNED inputs bind positionally to the output PARAMs, just like the args bind to the input PARAMs args = c.src[1:] # NOTE: this isn't really needed. it's okay if there's unused args in the function @@ -129,11 +130,11 @@ def expand_bitcast(bc:UOp) -> UOp|None: return parts[0].stack(*parts[1:], dim=-1).flatten(-2).cast(new_uint).bitcast(bc.dtype) earliest_rewrites = mop_cleanup+PatternMatcher([ - # resolve value-producing calls (inline the body) - (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="c"), resolve_function), + # resolve calls with RETURNED inputs (inline the body) + (UPat(Ops.CALL, name="c"), lambda c: resolve_function(c) if c.num_returned else None), - # resolve TUPLE+GETTUPLE - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), + # resolve AFTER on RETURNED (call outputs) + (UPat(Ops.AFTER, src=(UPat(name="r"), UPat(Ops.SINK, name="t")), allow_any_len=True), resolve_returned_after), # resolve allreduce (must be bottom up) (UPat(Ops.ALLREDUCE, src=(UPat.var("buf"),), name="red"), create_allreduce_function), diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 4daf6c21a59ba..4bed307068ea9 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -8,6 +8,7 @@ from tinygrad.helpers import all_int, getenv, fetch, Metadata, TRACEMETA, TracingKey from tinygrad.helpers import cpu_profile, suppress_finalizing, disable_gc, VIZ, pluralize from tinygrad.uop.ops import UOp, Ops, sint, all_metadata, Variable, ConstLike, UPat, PatternMatcher, GroupOp, ParamArg, graph_rewrite, rewrite_group +from tinygrad.uop.ops import resolve_returned_after from tinygrad.mixin.rand import RandMixin from tinygrad.schedule import create_linear_with_vars from tinygrad.device import Buffer, canonicalize_device @@ -106,14 +107,15 @@ def _precompiled_output_redirect(s:UOp, t:UOp) -> UOp|None: return None def transform_precompiled_call(c:UOp) -> UOp|None: - if c.arg is None or not c.arg.precompile: return None - input_buffers = tuple(x.contiguous() if x.op is not Ops.AFTER else x for x in c.src[1:]) + if c.arg is None or not c.arg.precompile or c.num_returned == 0: return None + assert c.src[0].op is Ops.SINK, "precompiled call bodies are SINKs of stores into the output PARAMs" + # the RETURNED srcs are the call outputs (slots are src positions) + ret_pos = [p for p,a in enumerate(c.src[1:]) if a.unsharded_base.op is Ops.RETURNED] + srcs = tuple(st.src[1] for st in c.src[0].src if st.op is Ops.STORE) # add the outputs to the call - srcs = c.src[0].src - resolved = [c.gettuple(i) for i in range(len(srcs))] - outs = tuple(r.empty_like() for r in resolved) - targets = [o.param_like(len(c.src)-1+i).shrink_to(s.shape) for i,(o,s) in enumerate(zip(outs, srcs))] + outs = tuple(c.src[1+p].empty_like() for p in ret_pos) + targets = [o.param_like(p).shrink_to(s.shape) for p,o,s in zip(ret_pos, outs, srcs)] subs:dict[UOp, UOp] = {} items:list[UOp] = [] @@ -129,23 +131,28 @@ def transform_precompiled_call(c:UOp) -> UOp|None: items.append(t.after(t.store(s.after(*after_deps)))) fxn = UOp.sink(*(x.substitute(subs) for x in items)) - # body switches from TUPLE to SINK, so the node becomes an opaque CALL - new_call = UOp(Ops.CALL, src=(fxn, *input_buffers, *outs), arg=c.arg) + # all bodies are SINKs now, the node just becomes an opaque CALL: outs take the RETURNEDs' places; afters on real + # buffers are the input storage, afters on RETURNED placeholders have no storage yet, materialize them + rmap = dict(zip(ret_pos, outs)) + new_call = UOp(Ops.CALL, src=(fxn, *[rmap.get(i, a if a.has_buffer_identity(after_ok=True) else a.contiguous()) + for i, a in enumerate(c.src[1:])]), arg=c.arg) rets = tuple(o.after(new_call) for o in outs) # if the CALL has symbolic shapes, shrink the max-sized output to the actual symbolic shape - # NOTE: must use the resolved shapes of the CALL (which substitutes PARAMs with external args), not raw body shapes - rets = tuple(r.shrink_to(rs.shape) for r,rs in zip(rets, resolved)) + # NOTE: must use the resolved shapes of the RETURNED placeholders (which substitute PARAMs with external args), not raw body shapes + rets = tuple(r.shrink_to(rs.shape) for r,rs in zip(rets, (c.src[1+p] for p in ret_pos))) - return UOp.maketuple(*rets) + # the AFTER outputs resolve against this: stores of each real output into its RETURNED placeholder + return UOp.sink(*[c.src[1+p].store(v) for p, v in zip(ret_pos, rets)]) # NOTE: adding rules to here is bad. these all need to run before the schedule cache pm_early_transform_tensor_graph = PatternMatcher([ - # transform precompiled value-producing calls into opaque CALLs (body becomes SINK with stores) - (UPat(Ops.CALL, src=(UPat(Ops.TUPLE),), allow_any_len=True, name="c"), transform_precompiled_call), + # transform precompiled value-producing calls into opaque CALLs (outputs become real buffers) + (UPat(Ops.CALL, name="c"), transform_precompiled_call), + + # resolve AFTER on RETURNED placeholders (for precompiled calls) + (UPat(Ops.AFTER, src=(UPat(name="r"), UPat(Ops.SINK, name="t")), allow_any_len=True), resolve_returned_after), - # resolve TUPLE+GETTUPLE (for precompiled calls) - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), # fold MOPS+BITCAST over BUFFER into SHRINK when movement ops collapse to contiguous range (UPat((Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BITCAST}, name="src"),), name="c"), contiguous_mops_to_view), @@ -176,6 +183,8 @@ def transform_precompiled_call(c:UOp) -> UOp|None: def finalize_after(ctx:AllocCtx, x:UOp): # bound Variables are call inputs, not assigns: they stay in the graph and pm_replace_buf turns them into call args if x.is_bound_var: return None + # AFTER on a RETURNED placeholder is a call output, not an assign: it's inlined when the call is resolved + if x.src[0].unsharded_base.op is Ops.RETURNED: return None # untagged: record as an assign for the call body if x.tag is None: ctx.assigns.append(x) @@ -227,6 +236,12 @@ def transform_to_call(big_sink:UOp) -> tuple[UOp, dict[UOp, UOp]]: # this rewrite is "read-only", it adds simple things to buffer_map and may sink things on big_sink, bottom_up # this is the only one where we have to be careful to not break the tensor graph big_sink = graph_rewrite(big_sink, add_tags, ctx=ctx, bottom_up=True, name="number the uops") + # final outputs of value calls materialize with fresh storage (precompiled calls don't: transform gives them real buffers) + def materialize_finals(u:UOp): + if u.op is not Ops.AFTER or u.src[0].unsharded_base.op is not Ops.RETURNED: return u + if u.src[1].op is Ops.CALL and (u.src[1].arg is not None and u.src[1].arg.precompile) and u.src[1].num_returned: return u + return u.rtag(None).contiguous(tag=u.tag) + big_sink = big_sink.replace(src=tuple(materialize_finals(u) for u in big_sink.src)) # here we can break the tensor graph. this is the only place you need to maintain numbered tags big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, ctx=ctx, name="early transform tensor graph") @@ -383,7 +398,7 @@ def as_param(self, slot:int): def call(self, *lst:Tensor, fxn:Tensor|UOp, grad_fxn:Callable|None=None) -> Tensor: fret = fxn._uop.call(*[t.uop for t in (self,)+lst], grad_fxn=grad_fxn) - return Tensor(fret.gettuple(0)) + return Tensor(fret.returned_outputs[0]) def custom_kernel(self, *lst:Tensor, fxn:Callable, grad_fxn:Callable|None=None) -> list[Tensor]: """ diff --git a/tinygrad/uop/__init__.py b/tinygrad/uop/__init__.py index abe700810da2f..060d5671ff806 100644 --- a/tinygrad/uop/__init__.py +++ b/tinygrad/uop/__init__.py @@ -23,7 +23,7 @@ class Ops(FastEnum): # uops that aren't rendered NOOP = auto(); REWRITE_ERROR = auto() - # CALL is a kernel invocation; a CALL with a TUPLE body is value-producing (and gradient-able), the rest are opaque + # CALL is a kernel invocation; calls with RETURNED inputs are value-producing (and gradient-able), the rest are opaque PARAM = auto(); CALL = auto() # renderer @@ -37,8 +37,8 @@ class Ops(FastEnum): # vector creation / item selection STACK = auto() - # tuple/gettuple for function with multiple returns - TUPLE = auto(); GETTUPLE = auto() + # RETURNED is a placeholder for a buffer a call writes and returns: it's an input to the call and you AFTER on it + RETURNED = auto() # hcq specific GETADDR = auto() diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index c0907ad6df9bf..196c0851337b5 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -120,7 +120,7 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: match op: case Ops.STORE | Ops.LINEAR | Ops.SINK | Ops.PROGRAM | Ops.SOURCE | \ Ops.END | Ops.BARRIER | Ops.GROUP | Ops.IF | Ops.ENDIF | Ops.NOOP | \ - Ops.TUPLE | Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: + Ops.CUSTOM_FUNCTION | Ops.REWRITE_ERROR | Ops.PYLITERAL: # always void return dtypes.void case Ops.CALL: @@ -156,10 +156,6 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: case Ops.WMMA: # WMMA output dtype is the accumulator dtype (src[2]) return src[2].dtype - case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a CALL body) - in_tuple = src[0].src[0] if src[0].op is Ops.CALL else src[0] - return in_tuple.src[arg].dtype case Ops.GETADDR: return dtypes.uint64 case Ops.THREEFRY: @@ -170,8 +166,8 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: if not all(dtypes.is_int(x.dtype) or x.base.is_invalid for x in src): raise RuntimeError(f"shift operands must be int, got {[x.dtype for x in src]}") return src[0].dtype - case Ops.BUFFER | Ops.PARAM: - assert isinstance(arg, ParamArg), "BUFFER/PARAM must have ParamArg" + case Ops.BUFFER | Ops.PARAM | Ops.RETURNED: + assert isinstance(arg, ParamArg), f"{op} must have ParamArg" return arg.dtype case Ops.BINARY: return dtypes.uint8 @@ -191,6 +187,11 @@ def dtype_from_uop(op:Ops, src:tuple[UOp,...], arg:Any) -> DType: if op in GroupOp.Movement: return src[0].dtype raise RuntimeError(f"no dtype for {op} with arg {arg}") +class _LegacyTupleValues: + """legacy compatibility shim: TUPLE is gone, a tuple-of-values just holds the values until they are called""" + def __init__(self, srcs:tuple[UOp, ...]): self.srcs = srcs + def call(self, *args:UOp, **kwargs) -> UOp: return UOp.call_outputs(self.srcs, *args, **kwargs) + class UOpMetaClass(type): ucache:dict[tuple, weakref.ReferenceType[UOp]] = {} def __call__(cls, op:Ops, src:tuple[UOp,...]=tuple(), arg:Any=None, tag:Any=None, @@ -332,7 +333,7 @@ def _shape(self) -> tuple[sint, ...]|None: match self.op: # late ops don't have shape case Ops.IF | Ops.BARRIER | Ops.SINK | Ops.REWRITE_ERROR | Ops.ENDIF | Ops.GROUP | \ - Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE | Ops.TUPLE: + Ops.LINEAR | Ops.PROGRAM | Ops.SOURCE: return None # a void CALL has no shape, the return value of a CALL has the shape of its dtype @@ -352,17 +353,6 @@ def _shape(self) -> tuple[sint, ...]|None: case Ops.NOOP: return self.src[0]._shape if len(self.src) >= 1 else None - case Ops.GETTUPLE: - # GETTUPLE extracts from a TUPLE (possibly through a CALL body) - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.CALL else self.src[0] - assert in_tuple.op is Ops.TUPLE - inner_shape = in_tuple.src[self.arg]._shape - if inner_shape is None: return None - # if through a CALL, substitute internal PARAMs in the shape with corresponding args - if self.src[0].op is Ops.CALL: - return tuple(graph_rewrite(s, _pm_resolve_params, self.src[0].src[1:], walk=True) if isinstance(s, UOp) else s for s in inner_shape) - return inner_shape - case Ops.INDEX: shp:list[sint] = [] for s in self.src[1:]: shp.extend(list(s.shape)) @@ -378,8 +368,8 @@ def _shape(self) -> tuple[sint, ...]|None: case Ops.GETADDR: return () case Ops.RANGE | Ops.SPECIAL: return () case Ops.BINARY: return (len(self.arg),) - case Ops.BUFFER | Ops.PARAM: - # PARAM/BUFFER don't have a shape input, they have a size in the arg: int gives shape (size,), None gives () + case Ops.BUFFER | Ops.PARAM | Ops.RETURNED: + # these don't have a shape input, they have a size in the arg: int gives shape (size,), None gives () if (img:=self.arg.image) is not None: return (img[0], img[1], 4) return () if self.arg.size is None else (self.arg.size,) case Ops.CUSTOM | Ops.CUSTOMI: @@ -421,7 +411,8 @@ def _shape(self) -> tuple[sint, ...]|None: match self.op: case Ops.RESHAPE: if not all(x >= 0 for x in self.marg): raise ValueError(f"shape can't contain negative numbers {self.marg}") - if prod(ps) != prod(self.marg): raise ValueError(f"bad reshape: {ps} -> {self.marg}") + # with symbolic views prod equality can be true at runtime but unprovable, only reject provably unequal products + if resolve(prod(ps) != prod(self.marg), False): raise ValueError(f"bad reshape: {ps} -> {self.marg}") return self.marg case Ops.EXPAND: return tuple(self.marg) + ps @@ -548,12 +539,29 @@ def trace_num(self): def sink(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument return UOp(Ops.SINK, src=tuple([x for x in srcs if x is not None]), **kwargs) - def maketuple(*srcs:UOp): # pylint: disable=no-self-argument - return UOp(Ops.TUPLE, src=srcs) + @staticmethod + def returned(slot:int, dtype:DType, shape:tuple[sint, ...]|sint|None=None, device=None, axis:int|None=None) -> UOp: + """create a RETURNED placeholder for a buffer a call writes and returns: it's an input to the call and you AFTER on it + like a normal buffer. its slot is its position among the call's srcs, which is its identity (identical slots merge) + like PARAM, the arg only stores the concrete max size: a shape is a view (RESHAPE/SHRINK/UNSHARD) on the flat placeholder""" + if isinstance(shape, (int, UOp)): shape = (shape,) + # multi-device values have a per-shard sized storage wrapped in UNSHARD: the sharding lives in the graph, not the arg + if shape is None or len(shape) == 0: return UOp(Ops.RETURNED, arg=ParamArg(slot, dtype, None, device=device)) + shp = tuple(s//len(device) if (i == axis and isinstance(device, tuple)) else s for i,s in enumerate(shape)) + ret = UOp(Ops.RETURNED, arg=ParamArg(slot, dtype, prod(to_max_shape(shp)), device=device)) + return ret.view_as(shp, axis) + @property + def num_returned(self) -> int: return sum(x.unsharded_base.op is Ops.RETURNED for x in self.src[1:]) + @property + def returned_outputs(self) -> tuple[UOp, ...]: + """the outputs of a value-producing call: an AFTER on each RETURNED input, usable like a normal buffer""" + return tuple(x.after(self) for x in self.src[1:] if x.unsharded_base.op is Ops.RETURNED) + # legacy compatibility: TUPLE/GETTUPLE are gone. a tuple of values called is call_outputs, gettuple is returned_outputs[i] + @staticmethod + def maketuple(*srcs:UOp) -> _LegacyTupleValues: return _LegacyTupleValues(srcs) def gettuple(self, idx:int) -> UOp: - in_tuple = self.src[0] if self.op is Ops.CALL else self - assert in_tuple.op is Ops.TUPLE, f"gettuple requires CALL or TUPLE source, got {self.op}" - return UOp(Ops.GETTUPLE, src=(self,), arg=idx) + assert self.op is Ops.CALL and self.num_returned, f"gettuple requires a CALL with RETURNED outputs, got {self.op}" + return self.returned_outputs[idx] def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None]), **kwargs) @@ -700,10 +708,6 @@ def axis(self) -> int|None: if self.op is Ops.UNSHARD: if len(self.arg) != 1: raise RuntimeError(f"UOp is sharded on multiple axes {self.arg}, use .sharding") return self.arg[0] - # GETTUPLE: axis comes from the specific TUPLE element, not src[0] - if self.op is Ops.GETTUPLE: - in_tuple = self.src[0].src[0] if self.src[0].op is Ops.CALL else self.src[0] - return in_tuple.src[self.arg].axis if in_tuple.op is Ops.TUPLE else None if self.op is Ops.PARAM: return None # NOTE: they all have to share an axis, we always choose [-1]. src axes are right-aligned into the output shape if self.op in GroupOp.ALU.union({Ops.STACK}): @@ -850,7 +854,7 @@ def clone(self, device=None) -> UOp: return ret.after(ret.store(src.cast(ret.dtype))) @recursive_property def device(self) -> str|tuple[str, ...]|None: - if self.op is Ops.PARAM: return self.arg.device + if self.op in (Ops.PARAM, Ops.RETURNED): return self.arg.device if self.op is Ops.STAGE: return self.arg.device if self.op is Ops.AFTER: return self.src[0].device if self.op is Ops.MSELECT: @@ -870,7 +874,7 @@ def is_virtual(self) -> bool: return self.device is None or self.dtype in dtypes.weaks @recursive_property def addrspace(self) -> AddrSpace|None: - if self.op is Ops.PARAM: return self.arg.addrspace + if self.op in (Ops.PARAM, Ops.RETURNED): return self.arg.addrspace if self.op is Ops.BUFFER: return self.arg.addrspace if self.op in {Ops.SPECIAL, Ops.RANGE}: return AddrSpace.ALU if self.op is Ops.LOAD: return AddrSpace.ALU # LOAD brings things into the ALU @@ -884,7 +888,7 @@ def addrspace(self) -> AddrSpace|None: return None @property def buf_uop(self) -> UOp: - if self.op in {Ops.BUFFER, Ops.PARAM}: return self + if self.op in {Ops.BUFFER, Ops.PARAM, Ops.RETURNED}: return self if self.op is Ops.MSELECT: return self.src[0].buf_uop.mselect(self.arg) if self.op is Ops.MSTACK: return UOp(Ops.MSTACK, src=tuple(x.buf_uop for x in self.src)) if self.base.op is Ops.AFTER: return self.base.src[0].buf_uop.base @@ -1196,7 +1200,7 @@ def view_as(self:UOp, shape:tuple[sint, ...], axis:int|None=None) -> UOp: @staticmethod def custom_function(name:str, *src:UOp) -> UOp: return UOp(Ops.CUSTOM_FUNCTION, src=src, arg=name) - # opaque bodies stay as plain CALLs; value-producing bodies get a TUPLE body on the CALL + # opaque bodies are just CALLs; value-producing bodies become CALLs with RETURNED placeholders as extra inputs _OPAQUE_CALL_BODIES = {Ops.SINK, Ops.PROGRAM, Ops.LINEAR, Ops.COPY, Ops.CUSTOM_FUNCTION} def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None, name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: @@ -1206,9 +1210,27 @@ def call(self, *srcs:UOp, ret_dtype:DType|None=None, grad_fxn:Callable|None=None f"ranges {self.ranges} are leaking out of the call in {self.pyrender()}" if self.op in UOp._OPAQUE_CALL_BODIES: return UOp(Ops.CALL, src=(self,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) - # value-producing bodies are always wrapped in TUPLE so the CALL dtype is always void - body = self if self.op is Ops.TUPLE else UOp.maketuple(self) - return UOp(Ops.CALL, src=(body,)+srcs, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) + # value-producing bodies delegate to call_outputs with a single output + return UOp.call_outputs((self,), *srcs, grad_fxn=grad_fxn, name=name, precompile=precompile, + precompile_backward=precompile_backward, aux=aux) + + @staticmethod + def call_outputs(values:tuple[UOp, ...], *srcs:UOp, grad_fxn:Callable|None=None, + name:str|None=None, precompile:bool=False, precompile_backward:bool=False, aux:Any=None) -> UOp: + """call a body producing the given values: the body stores into output PARAMs, and the outputs are RETURNED + placeholders that are inputs to the call (you AFTER on them like normal buffers). the RETURNEDs are bound to the + output PARAMs positionally wherever the call is resolved, just like the args are bound to the input PARAMs""" + # the device defaults to the first device in the values or args, like srcs-based device resolution + default_dev = next((x.device for x in itertools.chain(values, srcs) if x.device is not None), None) + # the RETURNED storage has the resolved shape: substitute internal PARAMs in the shapes with corresponding args + def returned(o:UOp, i:int) -> UOp: + return UOp.returned(len(srcs)+i, o.dtype, None if (shp:=o._shape) is None else + tuple(graph_rewrite(s, _pm_resolve_params, srcs, walk=True) if isinstance(s, UOp) else s for s in shp), + dev if (dev:=o.device) is not None else default_dev, o.axis if isinstance(o.device, tuple) else None) + rets = tuple(returned(o, i) for i, o in enumerate(values)) + # the body only knows PARAMs: the output PARAMs get the slots right after the input PARAM slots + body = UOp.sink(*[v.param_like(len(srcs)+i).store(v) for i, v in enumerate(values)]) + return UOp(Ops.CALL, src=(body,)+srcs+rets, arg=CallInfo(grad_fxn, name, precompile, precompile_backward, aux)) def custom_kernel(*srcs:UOp, fxn:Callable, grad_fxn:Callable|None=None) -> list[UOp]: placeholders = [UOp.placeholder_like(s, slot=i) for i,s in enumerate(srcs)] kernel = fxn(*placeholders).call(*srcs, grad_fxn=grad_fxn) @@ -1680,7 +1702,8 @@ def walk_rewrite(self, root:UOp) -> UOp: continue # no rewrite, process children then come back to rebuild stack.append((n, True)) - if not self.enter_calls and n.op is Ops.CALL and n.src[0].op in UOp._OPAQUE_CALL_BODIES|{Ops.TUPLE}: + # calls with RETURNED inputs are always inlined into the enclosing graph, their bodies are never rewritten separately + if n.op is Ops.CALL and (n.num_returned or (not self.enter_calls and n.src[0].op in UOp._OPAQUE_CALL_BODIES)): self.replace[n.src[0]] = n.src[0] for x in reversed(n.src): if x not in self.replace: stack.append((x, False)) @@ -1718,11 +1741,10 @@ def unified_rewrite(self, root:UOp) -> UOp: if n in waitlist: stack.extend(waitlist.pop(n)) continue stack.append((n, 1, new_n)) - # NOTE: CALLs are handled as a special case. - # The function that is called is not included in the graph_rewrite. - # If you want to graph_rewrite a call, you can - # A CALL of an address is not a body, its srcs are regular dataflow - if not self.enter_calls and new_n.op is Ops.CALL and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES|{Ops.TUPLE}: + # NOTE: CALLs are handled as a special case: the call body is not included in the graph_rewrite (a CALL of an + # address is not a body, its srcs are regular dataflow). calls with RETURNED inputs are always inlined into the + # enclosing graph, their bodies are never rewritten separately + if new_n.op is Ops.CALL and (new_n.num_returned or (not self.enter_calls and new_n.src[0].op in UOp._OPAQUE_CALL_BODIES)): self.replace[new_n.src[0]] = new_n.src[0] for x in reversed(new_n.src): if x in on_stack: continue @@ -1772,6 +1794,13 @@ def to_max_shape(shape:tuple[sint, ...]) -> tuple[int, ...]: return tuple(int(x. _substitute = PatternMatcher([(UPat(tuple(Ops), name="x"), lambda ctx,x: ctx.get(x,None))]) _pm_resolve_params = PatternMatcher([(UPat(Ops.PARAM, name="p"), lambda ctx,p: ctx[p.arg.slot])]) + +def resolve_returned_after(r:UOp, t:UOp) -> UOp|None: + """AFTER on a RETURNED placeholder extracts the call output value: the value of its matching store in a SINK body + (called from patterns that bind t to a SINK)""" + vals = [st.src[1] for st in t.src if st.op is Ops.STORE and st.src[0].unsharded_base is r.unsharded_base] \ + if r.unsharded_base.op is Ops.RETURNED else [] + return vals[0] if len(vals) == 1 else None remove_all_tags = PatternMatcher([(UPat(GroupOp.All, name="x"), lambda x: x.replace(tag=None) if x.tag is not None else None)]) def gate_kernel_sink(x:UOp) -> bool: diff --git a/tinygrad/uop/spec.py b/tinygrad/uop/spec.py index ae8b3bc785ca1..03ed5b22091a0 100644 --- a/tinygrad/uop/spec.py +++ b/tinygrad/uop/spec.py @@ -93,9 +93,9 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base # GROUP of stores (or groups, or NOOPs) (UPat(Ops.GROUP, dtypes.void, src=UPat((Ops.GROUP, Ops.STORE, Ops.NOOP, Ops.INS, Ops.END))), lambda: True), - # AFTER on Movement Op, PARAM, BUFFER, CONTIGUOUS, or another AFTER + # AFTER on Movement Op, PARAM, BUFFER, CONTIGUOUS, RETURNED, or another AFTER (UPat(Ops.AFTER, src=(UPat(GroupOp.Movement.union({Ops.PARAM, Ops.BUFFER, Ops.CONTIGUOUS, Ops.INDEX, - Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS})),), + Ops.AFTER, Ops.UNSHARD, Ops.BITCAST, Ops.INS, Ops.RETURNED})),), allow_any_len=True), lambda: True), # CUSTOM (inline and non inline): the arg is the source string and the dtype it produces, void for a bare statement @@ -131,8 +131,6 @@ def matches_dtype(x:UOp, dtype:DType) -> bool: return x.dtype == dtype or x.base def is_device(d): return isinstance(d, str) or (isinstance(d, tuple) and all(isinstance(s, str) for s in d)) -def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < len(t.src) - # these ops can exist in tensor but not programs. example: movement spec_tensor = PatternMatcher([ (UPat((Ops.SIN, Ops.LOG2, Ops.EXP2, Ops.SQRT, Ops.RECIPROCAL), src=(UPat(),), name="u"), @@ -152,11 +150,8 @@ def valid_gettuple(g:UOp, t:UOp): return isinstance(g.arg, int) and 0 <= g.arg < # CALL (UPat(Ops.CALL, dtypes.void, src=(UPat((Ops.SINK, Ops.LINEAR, Ops.PROGRAM, Ops.COPY, Ops.CUSTOM_FUNCTION)),), allow_any_len=True), lambda: True), - # value-producing CALLs and TUPLEs must have void dtype, GETTUPLE can only appear on CALL or TUPLE - (UPat(Ops.CALL, dtypes.void, src=(UPat(Ops.TUPLE),), allow_any_len=True), lambda: True), - (UPat(Ops.TUPLE, dtypes.void), lambda: True), - (UPat(Ops.GETTUPLE, src=(UPat(Ops.CALL, src=(UPat(Ops.TUPLE, name="t"),), allow_any_len=True),), name="g"), valid_gettuple), - (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), valid_gettuple), + # RETURNED is a placeholder for a buffer a call writes and returns: it has a size in the arg, no shape input + (UPat(Ops.RETURNED, src=(), name="x"), lambda x: isinstance(x.arg, ParamArg)), # SPECIAL is index before index lowering. custom_kernel currently has this (UPat(Ops.SPECIAL, src=(UPat(dtype=dtypes.weakint),), name="s"), lambda s: isinstance(s.arg, str)), diff --git a/tinygrad/viz/serve.py b/tinygrad/viz/serve.py index beeff6fc7dcda..7d421e1a5b97d 100755 --- a/tinygrad/viz/serve.py +++ b/tinygrad/viz/serve.py @@ -51,7 +51,7 @@ def stream_json(self, source:Generator): Ops.WMMA: "#efefc0", Ops.UNSHARD: "#f6ccff", Ops.INS: "#eec4ff", **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", Ops.BUFFER: "#B0BDFF", Ops.GETADDR: "#9DB1F0", Ops.COPY: "#a040a0", Ops.CUSTOM_FUNCTION: "#bf71b6", - Ops.CALL: "#00B7C8", Ops.PARAM: "#14686F", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", + Ops.CALL: "#00B7C8", Ops.PARAM: "#14686F", Ops.RETURNED: "#C07788", Ops.SOURCE: "#c0c0c0", Ops.BINARY: "#404040", Ops.LINEAR: "#7DF4FF", Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D", Ops.STAGE: "#AC640D", Ops.REWRITE_ERROR: "#1a1b26", Ops.AFTER: "#8A7866", Ops.END: "#524C46"} From 32bbf82d5b4a031757f8c2cac24e5f0849542cc5 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 31 Aug 2026 10:23:39 -0700 Subject: [PATCH 298/306] Revert "ci: use fetch to download libraries" (#17863) --- .github/actions/setup-tinygrad/action.yml | 27 ++++++++--------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/actions/setup-tinygrad/action.yml b/.github/actions/setup-tinygrad/action.yml index e30bc8a470eca..372f227e30db7 100644 --- a/.github/actions/setup-tinygrad/action.yml +++ b/.github/actions/setup-tinygrad/action.yml @@ -248,10 +248,10 @@ runs: if: inputs.amd == 'true' && runner.os == 'macOS' shell: bash run: | - sudo "$VIRTUAL_ENV/bin/python" -c " - from tinygrad.helpers import fetch - fetch('https://github.com/tinygrad/amdcomgr_dylib/releases/download/v7.2.0/libamd_comgr.dylib', name='/usr/local/lib/libamd_comgr.dylib', - sha256='7712fbe4fcb9fcdea49aeac989876448df975ce0a8ce7c9b15b55c15e7a05935').chmod(0o644)" + sudo mkdir -p /usr/local/lib + curl -s -H "Authorization: token $GH_TOKEN" curl -s https://api.github.com/repos/tinygrad/amdcomgr_dylib/releases/latest | \ + jq -r '.assets[] | select(.name == "libamd_comgr.dylib").browser_download_url' | \ + sudo xargs curl -fL -o /usr/local/lib/libamd_comgr.dylib # **** CUDA **** - name: Install CUDA @@ -269,11 +269,8 @@ runs: if: inputs.ocelot == 'true' shell: bash run: | - sudo "$VIRTUAL_ENV/bin/python" -c " - from tinygrad.helpers import fetch - fetch('https://github.com/tinygrad/gpuocelot/releases/download/v0.1.0/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', - name='/usr/local/lib/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', - sha256='${{ runner.os == 'Linux' && 'a24705276a9a187111371465987b3258f8836ef512a34266e3075bc4714e125a' || '5106c998c795a36dec79eb7b2aae324a93d1338236d36eeaae232649ec457663' }}').chmod(0o644)" + sudo mkdir -p /usr/local/lib + sudo curl --output-dir /usr/local/lib -fLO https://github.com/tinygrad/gpuocelot/releases/download/v0.1.0/libgpuocelot.${{ runner.os == 'Linux' && 'so' || 'dylib' }} # **** WebGPU **** @@ -281,11 +278,8 @@ runs: if: inputs.webgpu == 'true' shell: bash run: | - sudo "$VIRTUAL_ENV/bin/python" -c " - from tinygrad.helpers import fetch - fetch('https://github.com/wpmed92/pydawn/releases/download/v0.1.6/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', - name='/usr/local/lib/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }}', - sha256='${{ runner.os == 'Linux' && 'cf36091d266a32c9d5080f14662de44cece241987939713282ea0ff558db81c6' || '7e87c7acefda8b6af1a1c5debfedcf62958311284b8fd8d9bcf93e312e6636e3' }}').chmod(0o644)" + sudo mkdir -p /usr/local/lib + sudo curl --output-dir /usr/local/lib -fLO https://github.com/wpmed92/pydawn/releases/download/v0.1.6/libwebgpu_dawn.${{ runner.os == 'Linux' && 'so' || 'dylib' }} # **** LLVM **** @@ -299,10 +293,7 @@ runs: if: inputs.opencl == 'true' shell: bash run: | - sudo "$VIRTUAL_ENV/bin/python" -c " - from tinygrad.helpers import fetch - fetch('https://github.com/sirhcm/tinymesa/releases/download/rusticl-v1/libRusticlOpenCL.so.1.0.0', name='/usr/lib/libRusticlOpenCL.so', - sha256='d4f48566d8fd33f6cdd8ef6de35a71966e8a8517e6f68ff3c52dbb43765a2513').chmod(0o644)" + sudo curl -fL https://github.com/sirhcm/tinymesa/releases/download/rusticl-v1/libRusticlOpenCL.so.1.0.0 -o /usr/lib/libRusticlOpenCL.so sudo mkdir -p /etc/OpenCL/vendors echo "/usr/lib/libRusticlOpenCL.so" | sudo tee /etc/OpenCL/vendors/rusticl.icd echo "RUSTICL_ENABLE=llvmpipe" >> "$GITHUB_ENV" From 8f4bd50dddafce21db52cde89f26ce5f2ea26236 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 31 Aug 2026 13:31:03 -0700 Subject: [PATCH 299/306] openpilot: no NOLOCALS=1 (kimi) (#17762) --- .github/workflows/benchmark.yml | 18 +++++++----------- tinygrad/codegen/opt/heuristic.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c1a75ecf5dd11..6b1d495f974f3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -458,31 +458,27 @@ jobs: - version: '0.11.0' model: vision url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - timing: 17 + timing: 18 - version: '0.11.0' model: policy url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx - timing: 3.2 + timing: 3.4 - version: '0.11.0' model: dmonitoring url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx - timing: 11 + timing: 12.5 - version: '0.11.2' model: supercombo url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - timing: 26 + timing: 28 - dev: QCOM:IR3 version: '0.11.2' model: supercombo - timing: 41 + timing: 29 - version: '0.11.2' model: dmonitoring url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 - timing: 11 - # IR3 dmonitoring is slightly slower - - dev: QCOM:IR3 - model: dmonitoring - timing: 12 + timing: 12.5 fail-fast: false name: openpilot ${{ matrix.version }} compile3 ${{ matrix.model }} (DEV=${{ matrix.dev }}) runs-on: [self-hosted, Linux, comma] @@ -506,7 +502,7 @@ jobs: - name: reset process replay run: test/external/process_replay/reset.py - name: compile - run: FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }} + run: FLOAT16=1 IMAGE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }} - name: run pickle run: BENCHMARK_LOG="${BENCHMARK_LOG}_run_pickle" RUN_PICKLE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py - name: Run process replay tests diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index f78dcb2b78590..9ff87f9c094f0 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -78,7 +78,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: return k # are we grouping? (requires local shape support) - if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if NOLOCALS else 2048), False): + if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if NOLOCALS or k.ren.target.device == "QCOM" else 2048), False): for axis, sz in itertools.product((0, 1, 2), (16,)): try: k.apply_opt(Opt(OptOps.GROUPTOP, axis, sz)) @@ -162,6 +162,19 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: if k.ren.has_local: if NOLOCALS: k.apply_opt(Opt(OptOps.NOLOCALS)) + elif k.ren.target.device == "QCOM": + # for openpilot: use 32..128 threads per workgroup, at most 8 on the innermost axis + # apply innermost global axes first so the leading hardware local dims hold the trailing global axes, like gidx + workgroup = 1 + opts: list[tuple[int, int]] = [] + for axis in [a for a in k.axes_of(AxisType.GLOBAL, AxisType.WEAK) if k.rngs[a].src[0].op is Ops.CONST][-3:][::-1]: + if (sz:=max(x for x in range(1, min(int(k.full_shape[axis]), 128 // workgroup if opts else 8) + 1) if int(k.full_shape[axis]) % x == 0)) > 1: + opts.append((axis, sz)) + workgroup *= sz + if opts and workgroup < 32: # fill at least one wave: grow the innermost local as much as possible + axis, sz = opts[0] + opts[0] = axis, max(x for x in range(1, min(int(k.full_shape[axis]), 128 * sz // workgroup) + 1) if int(k.full_shape[axis]) % x == 0) + for axis, sz in opts: k.apply_opt(Opt(OptOps.LOCAL, axis, sz)) else: # prioritize making expand axes local local_axis_ranking = [(any(k.rngs[axis] not in b.src[1].get_idx().backward_slice for b in k.bufs), axis) \ From 4197f74231c96d768e530ddb063bf538065eeeed Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:49:33 -0700 Subject: [PATCH 300/306] delete CPU threading support (#17866) * delete CPU threading support * more removals * fixes --- .github/workflows/benchmark.yml | 4 +- examples/mlperf/dataloader.py | 10 ++-- extra/export_model.py | 3 +- extra/onnx_helpers.py | 4 +- spec/tinyspec.pdf | Bin 99412 -> 99291 bytes spec/tinyspec.tex | 2 - test/backend/test_jit.py | 15 +----- test/device/test_hcq2.py | 10 ++++ test/opt/test_kernel_opts.py | 14 ----- tinygrad/codegen/__init__.py | 4 +- tinygrad/codegen/gpudims.py | 7 ++- tinygrad/codegen/opt/__init__.py | 2 +- tinygrad/codegen/opt/heuristic.py | 13 ----- tinygrad/codegen/opt/postrange.py | 10 +--- tinygrad/codegen/opt/search.py | 1 - tinygrad/engine/jit.py | 4 +- tinygrad/helpers.py | 4 +- tinygrad/renderer/__init__.py | 2 - tinygrad/renderer/cstyle.py | 6 +-- tinygrad/renderer/isa/x86.py | 6 +-- tinygrad/renderer/llvmir.py | 6 +-- tinygrad/runtime/graph/hcq.py | 1 - tinygrad/runtime/ops_cpu.py | 85 +++++------------------------- tinygrad/runtime/ops_dsp.py | 1 - tinygrad/runtime/support/hcq2.py | 9 +++- tinygrad/uop/ops.py | 16 +++--- 26 files changed, 64 insertions(+), 175 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6b1d495f974f3..2a9ff721725e6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -648,6 +648,6 @@ jobs: - name: Checkout Code uses: actions/checkout@v6 - name: Speed Test - run: DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py - name: Speed Test (BEAM=2) - run: BEAM=2 DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py + run: BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py diff --git a/examples/mlperf/dataloader.py b/examples/mlperf/dataloader.py index 275bcd4aef7c4..3f5db0c74abd2 100644 --- a/examples/mlperf/dataloader.py +++ b/examples/mlperf/dataloader.py @@ -5,7 +5,7 @@ import numpy as np from tinygrad import dtypes, Tensor -from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, NUM_CPU_THREADS +from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX, CPU_COUNT from tinygrad.nn.state import TensorIO ### ResNet @@ -131,7 +131,7 @@ def receive_batch(): else: X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name}") Y = [None] * (batch_size*BATCH_COUNT) - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): p = Process(target=loader_process, args=(q_in, q_out, X, seed)) p.daemon = True p.start() @@ -212,7 +212,7 @@ def batch_load_train_bert(BS:int, seed:int|None=None): rng.shuffle(fs) train_files.append(fs.pop(0)) - cycle_length = min(NUM_CPU_THREADS.value, len(train_files)) + cycle_length = min(CPU_COUNT, len(train_files)) assert cycle_length > 0, "cycle_length must be greater than 0" dataset = InterleavedDataset(train_files, cycle_length) @@ -301,7 +301,7 @@ def shuffle_indices(file_indices, seed=None): X = Tensor.empty(*sz, dtype=dtypes.float32, device=f"disk:/dev/shm/{shm_name_x}") Y = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name_y}") - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process(target=load_unet3d_data, args=(preprocessed_dataset_dir, seed, queue_in, queue_out, X, Y)) proc.daemon = True proc.start() @@ -437,7 +437,7 @@ def shuffle_indices(indices, seed): dataset_iter = iter(image_ids) try: - for _ in range(NUM_CPU_THREADS.value): + for _ in range(CPU_COUNT): proc = Process( target=load_retinanet_data, args=(base_dir, val, queue_in, queue_out, imgs, boxes, labels), diff --git a/extra/export_model.py b/extra/export_model.py index f5d5a09a840db..a6baf20bc771c 100644 --- a/extra/export_model.py +++ b/extra/export_model.py @@ -241,8 +241,7 @@ def export_model_webgpu(functions, statements, bufs, weight_names, input_names, def export_model(model, target:str, *inputs, model_name: Optional[str] = "model", stream_weights=False): assert Device.DEFAULT in EXPORT_SUPPORTED_DEVICE, f"only {', '.join(EXPORT_SUPPORTED_DEVICE)} are supported" - # NOTE: NUM_CPU_THREADS=1, since export does not support threading - with Context(JIT=2, NUM_CPU_THREADS=1): linear, output_bufs = jit_model(model, *inputs) + with Context(JIT=2): linear, output_bufs = jit_model(model, *inputs) functions, statements, bufs, bufs_to_save = compile_net(linear, output_bufs) state = get_state_dict(model) weight_names = {(id(b), b.offset, b.size, b.dtype): name for name, x in state.items() if (b:=x.uop.base.realized) is not None} diff --git a/extra/onnx_helpers.py b/extra/onnx_helpers.py index c11a8d41f8353..a0a0363d4b475 100644 --- a/extra/onnx_helpers.py +++ b/extra/onnx_helpers.py @@ -1,12 +1,12 @@ from tinygrad import Tensor -from tinygrad.helpers import NUM_CPU_THREADS +import os from tinygrad.tensor import _to_np_dtype from tinygrad.nn.onnx import OnnxRunner, OnnxValue import numpy as np import onnxruntime as ort ort_options = ort.SessionOptions() ort_options.log_severity_level = 3 -ort_options.intra_op_num_threads = NUM_CPU_THREADS.value +ort_options.intra_op_num_threads = os.cpu_count() or 1 def get_example_inputs(graph_inputs:dict[str, OnnxValue], config={}): """ diff --git a/spec/tinyspec.pdf b/spec/tinyspec.pdf index 5e8ebe58b3b28885d72f51e27f594eda73dd59ed..7dec94f0b04948bfb83d1914651ec732e868e6c6 100644 GIT binary patch delta 13174 zcmai(LwKMKu%+X4Y}>ZgvD2|_+x}wP?AW$#ckGUBbd33DR&!T(TTh*HwsqcGih$Y- zhpK0%1K4;tIDh_xb#ZkzGqQvA+_=(}cEKCN=snRO=Bii5f9@ToS6{DKQ|sWFpSceN z5fgVJ-%Ofvy1#hn(JxIu8xT=JA&{j_7&a_TZ|l4189fe1*kS%S!K33$d!JbBoB86X z?HRt9di3V`{xV|Ia>%^R2JC8&hMSF?hptSq0S)&D@1o}1YL~yM(cfDib`Q(P4;I-9 zy}Q1_JTc7H50^g$=eW#`Wn|jC++W_pSY~dO+jjWhr$)~reYS!t4lMS6vm7{3jP0z1 zyw?ul(F}r1$mwz`5i*12?UD2U@J_$kHb@1giXixKqy%#$&z=MDziyep(g{f+xJ2ap@B-q-~7gn3`b@q}gD~Vg!Lb6MKpQ6wV*B43Y4W zqMk^YCfIIh;rRs-WsS_h1oD3FA@+CH!-P%j+#P6 z^N(B_8tZWRex)OdHBhT79#t^m<<3Ny5+}3O3y&W=_OW(3IW%cpQVBKpvieR5(Q23( zcYW(Ue#_VPeYsxbm%j8RRc4T3-9r1N`b+jWREumM(jCAy9m^w@gjFMz6b50#)8hB+ z{>|}jZp&{dP~A$q{W?}KV97AOH%4dF_WkvBuAOm}fRx|np7k&Yof`zAH0AhLY+qT> z#)gH+2aU}Je+pZf`YV&`+uY9Cp(p)dg3rc;vMkoY0@zqcxAS6F45u{rKQAT0-ZH(g#yd67 zmU)_`gkBbNxpHpFf?q8}1K*K}x>d*ELS_p+gaXD6lZ+Q4LjJZPoD0~Iq2F*g+Y6wG z&kneLIusp`m0Wc7?Ic2MkOyME;)==mITBtQ*AopL4GVd&N$T=6J_w`C{(xkfi4vPm zwE}B+A3rh3tBq^Yg)OM|iZYV0$j|0JAi=+;-eLVHAl;x^zX)7F(RW<1%kmSfx+A&g zoCCf#cmFv2f{wglUq$=zFbC5bLJy0&|MrfdaZAqH;IdoWwFmD`Dw%CEAefWLc^Qx) z?9`SQt1%4{=QG8GefzK{6eo1}`a6}|fCSvApFeFJ4-^uG?StArp2@z>pa*9*C@H3S z%dF9#H%I)JjApupm*4ZlpU@xVG{%?#A%TDt9~mhv8hvP(n+R4wCk9+eK!TH9Vso5W zX=Q{jdgd5LW>;RG!`MSWy?mu@@HH5zLu(hyp%BS9yh9qS2wP!Jj@klYst7jXi5Nww zO%>W&cqYT4g_WC0kvN64ow+lAAFiaN3`yUyDFxOoMDzmP&B?u{yn|`QjzEFb**74h zoTr?I+N2ki;Gj3yAc>BdtfbzB8;<`=g4NzDbaIvLvWkPVh_0~9o#T7KKDgoE1SZv7 zC$-V{WQV|PjH-&xcqh&+m!0ylSYS#=#zordhCdAUgy8uj1?W_kUwX2l#5ipy#e1*8 z!_QL`r}N;Byt%qsj?GezFDnMH^etdkhv{cIQb_*EC{G)r9p}HNg9NO-KNBLi#f&ub z7gyjXT~1uG(7F~Qd?`?nBeWFspCB+V`e{)bZzdwn2AGY+F^Z_hN8u&G1)+lSS(nWx z%jHNr1Os7>P-rDv)lM`sK5X=SPAn`3FZ8DQw=&*ZNE54*9K^Wi2*%PbBXa=hUlqn> zh(u}D1Yk-+Lao!Kr2hV*^7=kz%H#U7;ORhDz>D;5s8Wq5E+({m$b-R&=x*+rcDdom zX`L{NV~~y3xzlaa#I3{L4?TEA5*PeI1ivL#X=L-}8u7i$LoiNEO1Ln!2F#N9U?sv- zLJA|Ot%GU4%s4ezK5vLcpAMLV7AST(iusuz2Pxn|{|DsYDU&;LCbWJj7L|&+rsxHt zBe2zY3pDY1?s+uv)cW?aYk#6?F8(>(mrSE2buRcqubbzZA1V25APmC_cD4AKu`7?{ zAJt>`q%zLD4I?^lRZbs-<1eEPY-;g77B263?Ng00n}u#d+LjMgYEQree^>|(hx}$< zopK9Oc9SU9u7UkW_TgAs*iU=95G_~f6DiiL^)<^da$KPpimPkz_ z4s^yM54E6AWOlw;4wcmv-QwxY5WveEyv7en33v@9uW3k52h$UrI)G~^tBdp+GfY05 zj&6iBsBgZMNr{XTrQP9)qjs`FZT)>eLZv{D)R?}lmcbxx*!vsUWEFMB{h2L`y60lO zZL7CU%S?ss`%`qvq2Qm^L`m`Ps*Qfz)p1LClJy~bC-WcoGg!?0=*b-0iAybWVpn%B z#3RMGLO9S930!1ltsM`xbf00lQyh@Fyp!1r5Z%rX7>63>x7%%Ul&T3cbymZVE!pDb z!T|<1_pIr|_9_y9g(Re?;h0Z?GRYo)V{sz31AKzuN_-Qm{Wn)`MB?~!;E!kHk$O&|k=HjM5;Ouzpp#$v-uteI)r8npH*HU9`c?+5W3!B^gX(!@AHdy zS+)QT72no7ay`b*F^T0&p`u8!XhjYN}Z-HO5* zTd7)|>DB^JA;lq$>!mxx9dbQdPPQ3mXF(t!ZA|y`Z&YSf-Nf}OuMjA;^eT&+T(4A5 z0?lO?SGyPs$olZ+U@T!XDle)r4mwkW?N{ha5}0yyzbtR`c=II%m9Az?HDv0jc`qqA9A>}>U=r&$@4`4zZ)zHjX6?Dx`@V#gL#F+%G*sd z+jJ#kR!bJyl+VwyK7S+GwCy)pQ2H3=2~ddbXeM_wcGg!r{!C~7V~<)E6YZv*g51tk zK3Zfd+rXZ!5P2R25)SWX(GhE~rnmbb|tDG(j^cLQwB}>xYEHA&FEE9i8(}Ec5(l(M9_nTSDyS+X>*9T}aW1gxm zV$~7_R3a4A)*wEmpW9d?!?av(q(hxKlJR8`5~Ed!!8D)fmZB=vcF34;5r6^V?MuF0 z)SnuiW33HSE?o*NcE6zfxAqJuxgT}Zo zCb)my;~0b~Goc4#$SNVyTso6W^8c)D8N4num-_W^GA^w#0)x_-CD zt4-JJY?Eyn^r+7vS3^Lt7H@yS&}|@dk0TY z&7a63n=)_$#`V^LI6G}^`ul56#`#lNi$dBJR56-s`!R#>pdyaJCxh?8L}wihvo0-0 zR+k{YOHySW$zm2&7eFmwAw-vgu?ER$Rb;`hZ?4v!!lqJtpj1ZV^9NK*hjY|jR)KPS z_kdd#zCIRmITNIVMA9Z$@8K#{;p}NMrjmtEx&1ba_Ryz2!!NQ?V#nmI4 z9mL^YIT7s_eD^uiZ&R$7X8{Vg@`$ENRB0Qp=xS~l%G za)!iwaT2YSs3PJ`E2-`a->(NznHz*q7?kcE*VW)M(0xi(rfNy>zqa_kes==T6iMSw zSJl8BFP~sIWN85XccO8#k~QDW?xOIhWzyppIVz_mU-D9$xvZ7rg_|!J>)0m6^bu_S zdsig?15KXvbzSY{>#olK*#J*CLa6D3*Fph|jXT*aY6mnnSv*<_fKJLW0~L|Xg3%(k zPCS)jbHY58ZYSp=e#{r99o)c#5SW4JLBFQuF5j(Hb~Hv@AXY8iwE*+L@Ktiu7oB4k zRv6HmDFcJ%XvC&`%-=1OKnjC7$yaPK7ihQ9jF0@cTPqBN!C097NfefZBEVTnlK4ux zx67%$U^fR;)OT`m@R)k?#0BLSXB-$lV{LZ+Ei!(8ftiI~c{b6+oaX@?9REKt@Y96R zL?3It1}zQ$rMRJFD)$`m9%i}ZIWP7nLIea&+)0>IILyrv`^gW3+7vG6@}0FDqOg68 zLIP)ryr^(K-4hK1Ew%%$Nnsh z`@Z{72c;C^MO`0XUeIZ|nhC*%x-ox>|nR{-S3!A`M_ZSibxm*^o&B8jl_jw4Xf z6uY!6aZVXeq7hu|a=5452a@SDFqWJ(f2L6tL=ag5q5 zfkal?KN7V+beqE;BX!Yvw`iObpE^S0@V2+GI=O4~PFlTkV_DT{w#hW4KEYFZg$oin zzc*Wo?6^8er@2x{iC~Ft=b>7&s6Ot_~M%}vcy>>nrB zhT5lJsFK|%c_>$Z;ViNpn8#djWSgg1y6^ZFbqs#od7j*TgrG)yVH&U|_Yc5V0D#8B zqA^5G{T+_&(n*u*e(*Z9a@1N-HFXO0z`+Zzk%zake>pinC}gpV=ayj)vvj0$Q~OeN zb+c(zJ*MhRZ?R2QLM0g#q;qT^oDt79w;ncju*guBBEoZ0t-UZUVjtY*@LJLrYAKCKM{W9KFFZi8_rUQ0SnoBn$;tP?)XLJI3Z~UdUne59U{@U5Vc7<=E@1 z+xz-QRB*ZK{mkp1XP$0G=xm2Phmm4leK-I0WsL==EVD~dyxBuH*$rGp=*&B=22}C) zv4-h^7a^t;5~qKB_h;+Sddy7;WRy87!BO}SRE(dVFe`csC#gG8n}^g4z)$Fsk_M8- zfF)v>(qmA_73SyF^uCr{ITxG5r;~A;APTzNbR!bZ4kM5E%O*m{_BdkDzWB2JJ5ScX z0jt3{zrhy9c7XX=yD;j8+*L{)^5lLF;c51HQ zF)sPI&PD&Z_HrJl)xh=WKD4ThsZMJ$t46Ues}?gLD2A{}Z3`t5RkM3{*;P|WgA*-3 zfipb!ba(m@`PzUeLL_=Iu6~!@H6G88Q@RK#?>)-EkJ%RC>~}e_za^=E0OWU899E1K z-+Z{N43d+1h4q&S(6U<&RhtlLVwC#gNG|hBqwrSO-M&<>IB(c}J&zgb_~(a9{_}&L zJBF_Bz!yYnG1~H~Q-5ToqKl@kyS6!Jb{EnSB^~@`kjJuJvLOp+zxo;u0CEVQ+WkA^X+;Vdd0P0;b5{@#x?-fPWD}XJdG@e96&X_5PmXa-11%aw=30%1oSFuXoTYR zU3N`OV=g4cFU*R^X`MwqGWBjbTwfnTfFb3Dyk)l+%6h!;V7CxEjWO8E6E^x&f=qLQ z1D;a+OnX-n7&??B4mBeWSUV-Nl~3TzQ}C7uzD4~VgwSCW=3f&U3E3PFTQrJdOPR$o z;Y(9+uAegOeK+#hw+F#guo?VW=}WH341k(*mx9Ct{5Ny50yS55^aq#`NpX#zu4+wb zxtzCg8E0FUE&DysTz~OB-4EVkZ7jIDkd(+|5dK;NaA0#2S;@Ex2oG3!9&-G|mEs7( ztmp(oy!*TnK@+5cDMHNaic>#vI?{K?_9u3Kv5gUX&=Euz##x%k=JS}I<)h@5A0@3o zRt^+1=Zbp9nLmcHKoD|W?;zjab%(aQAZbzg%Trwmv#uMd&JGy3z8Wn}>_I~14p4`V z0P{4!z@dJ6THM@O6r;0(4UmURiTtC>(%-9mj_(d@q=!yvaPrxm!K`f-6XDl}9OWm* z^UMMc#4lHhiDUvFB~mW9w|-lAwQ2dq=Ux$7cY_MRLeE4EPc{M3p=vKg`%g_EChD`? z_)-`};s_2|)f%@` zdr@t9!DZ{w++TBxS(2H&Rb$~1YWn3rY5bt1eF_jIAbc`o~&Sg%Rpvslj z=0Yxb5zXc0<70a#;1_6Ci{cUb+c7-p*c^7 z4?t!(kjQ{)7~aWN-H#*jQ1L}o>>S^_WCGNV zufbpH2ap#aF|6v@IqjilSOREetQN1qrq!0{<0705h^G`r{u5u$Pf~@g?O`Dl;^NNJ z9+g|z=u|K`PvQb^AwzzJ{eUH_FCEU;eEs4CHhwi0iv@IC?#~rXl$+^&a;0kH;>BDp z-cLb2iPhIbR53J)gozyj{d4wQ$t>t2FLw7mmymE3R5()d3pX(vLOJDAZQwkkYWL`s zy&eAehZ%I$y*7E}eUN<|lRak78BWsKRtsu=3b(kh`M)r+w^h3_4Z!XeHY(N(XNE+z z<9&vI^F&~D-eD1ALoJDl$3at)?TX3XBd<%5WM_OI`+Lu~NRNgd|C~#klF2GR6+#Uv zedG_?c9>p+jVr{@IG=N}ehgh)?SgPOdW^2ZXO*`4Y?WsTro`@=@^AW3S)Oo0fBWU| zUPitVeV#QXe*@<-HekCaiSAkK$zGkz?G6rs-G-!S857O+cQUNCAse(p|MVQ35YnT+ zKNM!QN#xq-^521hf3wL%bq0N27g;`HiY zW!@mh1SLcLp_a7Fm%vAe>Qv3oxQBJCga@k6%k~3xWk&q*1bLm|)XY$)BYMK`PsR)| zD0QaJK=3M}^~%PiL7A%fri-UA6C&G4*@lUVk=)I74yxVRa+Rph_{~N zZsw<)N%An3uGeyIqK>4#Gd4=E&y~g*g4bD|5kmqEu~THDCgmSt@?{+%q13BBOI^(w zf>32I%T3y`@o2^sp5E`UPoPh<-e^Pef;j{K;krMR%e&t%f^cf{2PuXjuRlBDLfKp0 z_MLck1jrR2HNHl)pcWut=^OHMH&Hbi$!M;rOPyi+t?vbOur~g2o?$L6s;74GLydmc zz+)w61*8n7eKzF2z>umJb1m69;U6Ea(!G7GUTlejm(Yn-m5asP7R141Xu*iBs<4*& zwv=BBeq~+a`kfJuZP`wmGZV~8n?Z~35KUR1B;ZUOh0L02rv=1oSs(2^?9sl)4X$ig zJ6!bs?#3ND@9xe*j#R&QTvJX7K(7qjj}k4hT~l%!&?!b)!w`CA=Zngal7uD z=Baukb-Ei9V`v|H+Jf2%q5_r7FUCv7eY(FVER%VJy_~IQA1->u(zczOWi2j^o{bs(8 zgHoOtU;gMpxg#YMmHJ3YiVhKSMoZhud~aCXoo z)ly0i@nW^NVc(55xg4#oDD^3}7*~wV0>IiV38@wdM{4-0e%t=9*->}Xk&|1kJ;IGfe+#6{zewBY5z0C2T(JzzmQ zx>IHXvRC3ox@V>G-14s`gJsawvr*U2yrDU4nmq|#(&l;1fEz>5EtJ++lw1kkl}bzG zJ*vTcXHs(Vme^BAoB;-Yj7kY*+tFv=_KiaLw9=skM9oS5JtRxGa(At&QB$xDJi&3t zsmJl;j_3$Ufko1BTW<7I@6-)t?C1oSRz_PfvbY<@4b!P@ zTwX$}YGuQ32gK{5E+|*QsO+b~@Em#-sSQB>dh+2M2Jia2ZtIj6Ni~nY14iz z`HBIFFx*HQ&uFWfVT5gjGxAag6yS^l#`I59^9(5W28fM+I&$9CO zBZvw6lr_b82APJg?@%P<$f}MFGbi~Ir`8o1%B56k9gVpb$F&75xi+i$NyT3?B0S;K zxX&uu0#f^{Zmzl|P;Tn@8&8*jokp@QB-Dx;T8m_aZvyt76#1Gk1IW&>h=`Iw(q?&@ zapE}Kzl(Tj@9C9v&+~IarJsW3(Hy?7lLCf9x5{eEe9*Ky9Kjr^r>m=n zoGHxP@D{APKhupK?SN-??Pc2|cL4jipMK_H^ONg%f4w@>oyd)!i6vM3NrZ;)s@F0_ z+#08^75*;+9et6O2@AMKM+FKihZ3HRTxV6(LrY#^=b*95;_TaJD8*~bD>NhFPea}~ zhm046yKj)b1&)>EfkCu%BLPr;#02h>?1R9EI(!heP-yf2J8$b103~1t{s&SS(XsJB zlSKXO?KAFW$icUG`&>&N;_)|hj2Re!LPhsOUNLM=q^KjON-*ea>y*qQv>NrawZS5L z=6_RiyKr|?Yopg1WvyT%5u+sds*;)T=_p_40aNZ%(4+Cw#T6q zU_$tV+J8b-%Zq;)4kq=VKrJL+Zh5nl)^nf1=5!Dv@XQPUfOJ_88J zc$^ft$0cu!G{6X~r4W;oag~$h0;5KU6ivC~YbzMgDXkZvQzn?vHE=tr!6uOzA}c3_ zLBpU z)&o?Tvioc6WxJP513YnVfw=VFy9nu?yP`@ znveI3!9yS2FP-D@wf|jQ{`61APD{YkB~he%Dr|*rBw}hrWLzp|WPM~OGOk4Rl%D}i z*AEqhY}n)gP3g_*mjzfIwAMt_iVzs*H11yekMHCy7$7_GSAJc&hAs>p|F22zc9_@U!sXg?pT*jD(vMt0iGc#CQY%9W9jN1ffvJ|ZwIkbEHh=Z5 z9e*XNAN1RKMz;j&WOeaz-o&*bMktee%F61TV^>O$rDvp)K0YKHbK_qA>U%n^fl(@? z?%K@5_>USqnJVehmnP4TsE~f%9v5Z^yz+jyxC(su{^NN6gn3W={(7{k7RZuEgo@su z7y$YpL<26!_eIT8{u*88y?nE~qK7&-qtJh@reH31*%Nib+a4pDshqp- zeM2cQ*Y0~)xD>5^`%x4K2@9c>w!CO2Kua)k;@Rp8O@OQ@3$a)%>4~Px@L7uFeg0h} zIsYjNSh~I?9J9Znf>xc0lCT(S&4)$7xB#vm}a*&oL3Kocg}MG-!`Yy#eKVfrIFX}d9T~6~ zeuLNkGE@hF-1C-s?^P8jQZtDB&5~k>UA%2iD_n0BGe60ql0NvS3gg!aPU&Ykh972!Ee29uDuBE(a+2J?SN z$H6xfAv`5fQ%v?rVk5)ZDYrXrx2LjRdU$Gy+0|` zVtfXkGD|E!J{Yd<6t#txP>eu=M&}#B{Gnk7GE||Bp>d4@BiNJJzvaAcjHR1{NT*)Q z-%#r{WW1?P-nvLq2>rX zDJ2-%RXH$gSEKKVjd$${L((^pJgIY|->MXCs+2#}(_7(l${uR3%#9X(@y1i`>~y~% zSmK2<$12e%Q@%B=fn99SWin!4g?FJ%snh^(1LJ6QYXZ3JG|W*+B4er4_~Mmxbxvn;`=8@#vP#5tqE&{LwS5VTDI2*!9}{!)tdQ zbbApl?FQjk+|%dL`T6wVok%~oJ69ipa_hRAfze?!7za=jeCISk*N{w}TTJjWh&zs6 zZ=>oOIxE|^`ZDLFH!qqvN-iR~Kbha(REcr*6JV^R&NW{fE%-u;dL6NKjaB8s=TlVn zz6JfAbwo6XBeiy0F`?5k?s0;(ly1UXR_59hESm8FuiP`{j;g$Qkv?*IFI(O8esUB( z?|26p-31`%T`67n1eXwR4r~N4c<{RP4{p!JC!oeNx>Kbekea;ic{=jU!M}ZSGKwGy zWX?wCt=;FDw*Yf8+xkegQSKk*qkERRf3(CpQo+QcAqjNY%X?TC39g!&Li|q5zI>fo z2nNx;?&-le@r5F+)}SJFP)`;;*6cD6;ppkxq=9+4w4Qj$8-<^>At;;V(b`ne9b{T^ zGMH*fBd54paw3>oWN;#VxsI0NO623lM9K6uFD0h)ZjEb*rdAhybXu2CTB;t6_@CbG z?|TFxJel&Pn@3SP2Qpi(UyK?F5~{ibcg|=U<+D+A;|eOZ#Qd9yu7on#oA7&{WNMcT zs=(cqe+gty3Qg}Q`fOKw>hX0K>WBlfxChnbE7J+nPOpq+7Z(9in^5w7IY~jJcZ4kH zm)*}1LpvGIAYY=Np|cie5ZP0hd89G@XZCbDTMTg}ID;W{x+@)pe2ygz54CcUm(@xm zu4R)Bb2~jgO?{}LcJmb}Csr^zTfvz5>OiCAkxHXfl=>hSSi9dNWI0A(3T>-A#!euu z-KU3n^p;`t)oj}KLHavUJepPOq2{o+A>D8=KAc8PoOo0UT^xS&*uH4?OUA+dJb^4* z2#k)HKr4ox@#g3(pNbhh<}n0q2SBs3+$&-iL`hF*hfH-T)CTXn_bow=uKt8 zg+Vn>rGK_}GMYaOm}VF(PMoyAbVq2gB~kSe?o2=QMX$5x$w&F#t+Idfzcr1Yj9~2I zDo}ad#vJv?2E(l3H$qotJb0Acw3&P8qLuV zD}0)f1@9Nu)y{va+uo~1_{Vs^!e2)o&#e>UqhI;+;-MdyUMUDru^bs+^?}|zqQf#C z!~l&a?4$45Z)!WGCNbl1T+VIXtM}k%rr90Fvc{eLPWlxz(YyR%WE8*6PFq!mwe&-q zGn>@C=-@R<|6#Wdgx;n3l?Rb|3MSjL-lnSUAJ10;gP)(7ucPui)fjjKF087p&3*_&Rm_U>k=)9X_fL(Trse()5NPCEoD*HX`nn zvC?N98lXKYr(VP7RSuM`?hYwQ1`(}d_%r$c^z!NsA9(V&i;|q#24&AJmy|YV;Hw5% zGLIx~^;<<@cY_cF7Y1NyPGI~w{?q5zfS#^^m}2;54AMR76Xo9IeFhcG`r8=h^+3O` z!Sb%i^B>_wmNgO7oz=PC!Wm3f>>XdU>%~%zH`Q7cRgnras-4vKd!>P$yy7}$kkK9L zWPiV8R!=62D!Ui$Piz?9i0*QubBhYHRFrI#?}0~8rv4QcNo4&#iNFCv~< z&Bak$IP6;Bzgn6Q^xMI%md~=W=8F8XgdTJb@ND++g1T}82(Hw$eUqt1^)50 zXZ6+Kp)7d;_U}%OnSaY+y`J2;^1|Uk?a{Wrhk%%JVk_qkmvZEEjYNcxl?mqc$Umi! zmTyL&ta3#h1#?EUSnUwk=29Bkj}8ZYRXROz^C3g;bPu(`x{YMLljGPamV%o2)oHq~ z)7hW(+Vq!mD#HQsOZ$(Xudh`HwBFmeHcM~%4XEEh^7LSOW@vIHNs}chLZ3$3JecVq zj@k~3x}JMXw$F_mPLpt^PY&Zp_xt|KOPg|Mrf9X&Lio0+a;?&o zg4A*boRR~(oSID`oHGX=-Q6~P#%7li$btW!+P|2V%@(UVo@aHP@N}KFJMQ0RCayQR zO6WIPwg}WNy7bv<}qPoWj14HG3MapH0LtoFy%Bi<>KLC=44@GG5X2>zip(W1c9mn z^L(~5P9P(9F13|~@v;??Rca2543&gEb^;sLO7Vqi%bjAw#CO}j}SbPJl;>=cw3 z#%%1Aq$mEzZH!Y1xMo z9pWpKyd7$#X2DxDZBynQvaOSt>y*#Hq`Q|8A*munxv+k%>&e8B*nJV!VTakALl>MiE! zRs7R0F*dBCvc-8eg(3@wm!B>XF0W1<&f8PE`;}?g4D&iX^8*I+n`l4FLfX-QK8&!j zo@uh{WPMCjB)o(B)9`fV{kzb1ew@fSv!XfF`IA-O zS={}RfUX$q8;?4;&-I=NPk~c?qPOg>9MA6+JyG9`k5!D}bob}uKPJ~B!#BXS-Ef}D zwG6Any_b8)*P*^hnrmmaNS~`T6NQS}^>#t{*3|gh&9!`E!E#TI?c3QkWj5ay-S{=w zbL5|sZMuo+@^{nIzH7C^60U3Sb?WDFKj%JvYQ94hXJUj8=!qEJfOPs0P(oNv79Lm% K3ULJq*#85_bDB&5 delta 13330 zcmaibQ*fXSuxyeIHrB@4Xk**9v$1X4Uu@g9ZQItyw(Z=%?!6D^?YvFZ%*#}F)zox# zuZKV`hCS?f|6LfQ~m*$(C9;unp<`_uf#!gSG4==AhYfSD3f^9Ls?xWCfCO!{N zw~wCStGoKoCmuXlzQ6RD)NRvlG5~AZ!@)*&nWn%a=%yT24Y~OS*n_<6( zbYy!u@SS!5hob+7sI(Tl0zN%h&Mu)(h7J0miS3^vtmml{6hyU3{@s?e;f&NRc zhP9!LMMC_BjiG1eh$yM=d)WM>yJ#GR1(*~}SNNj&qI6jo@e-+oztqG)MXOIcrM(FG zh-^E9&K&2|ga{9VsY!kKEA#!+=lS=BHXQosV_{Qm5XM^IZDCsA-{G9@8-|wnou4@1 z$LYesgM^(>5e$*-;XW3LBI!^h^?8I6tJi9D09^%f+eC~=qNqfs8ookF588JDG3 zq7XJiq$un2S6W7lbq}^m=@?bmC&^!=3uWvSZCC*9U#`pPGp`R2+{O?hewU9XF(0c( zQ__6Bz3DU6jfI1#hFDQ0>H1}FMy~+|dfC&{)B4MV>)^*pRn}4GjX=q7{zb-2 zwz$^CnKg~ck*&c^xfvd`N6%-8jHU_zLCWJcA+{rMAFtc5|w_o1ZF@6EytA#!~BL%#`V&p zugjhmPFLBrSC}yz#jo$l44{Lxtz~QC_8_Z?Az@*-rGd}-bfTFXv(y`viO+cf3C^b8 zxe4B>L59@RG$Ftuzs;phLnhdAF3iuCRK%?!949nQ&>;XYbdX>$7ZxN~gM7?xMh-{B z=4ijb|$rmHjDZ`{E zYsrnkJ;N0EwzfNIDGnHPO+SYYd@TXj6hw!Bw*UMLr*cC{TcxmDnX-fAiYu6^+aZt@ z$8q7G%xztt8KyD;8RTp1maS#HAZP(y~O7^{fp4!ITSIFfKWG+U}I1fSG7wLWAU5fa1@A z-OSQLJ8NgKclw5)59%r#tHx52vMK546pZ~7af*YL%*q3>Y&`{!!%r>ziPI?R#}2Yz z6rTiu_}SlaMT2H3#&ta_ z@A{#!23pD09tWEkCgP<<^m-e}51EWa4u#*w(?{)xP0cw!MDI+qc**qC8?GHt#(%1d zv>0vMu5==L2xq>TT^-*?NKG}HM*$iUy`(L`{c;52*$={#Hv8!+p$wU>Px=E9cBgtI z?lLLK2aosOSDTHgdC(R0+ovJ{fjUWODL#I%?v$dvq_5Q2jg%njpTvoQYOngIdS<(b z34gdQ-C54|UBzniD*J>f*etaq@3#;pWVNIx*?D2H$UEb8mWA$`9NTcbHO0A zjgHaS^4jB(Ch{E%w|nUq$hfBA3s%R$_sz*Bt(Oam$OIL#3{3Uao9DJ0DV} z&eEK4&wa7OFRavkE2)n>yo6*ytqjmI+Ys;ipWbuA%1^ASEWsp!)`PleJ0JWg(XF`V z`O&u0ZO}#d2%j(nZ zy`0P7;?d4iwWACZ$z%0bla*Cvrh=zQ@DuPlB0Z0egM0t_Kx)kl-RN6`T#cRKuG@Ua z!TyB0X}6C9K+F6TI2tlWI6{RgYr!#d8z0SEgi81|p!a=a1CeEL6mO)%yp9tzSPBuO z^rkqP9wbx-Sr~*4=V5!PBD9TFvo^8Nu?A^>Od8hxD2zyuP^rFXoJ&i>tGRF*)C`)5=7PnvX-g0Xm#8gW9z&m9lUS*83=*h zJ=BeQs|*|PVhcsHb!er(GTMAb8Srk>oZ&g_5I1EEDaqin($f{=o!=;sYchL@_I0um zC}7W!OKyo)6v-y{6bjnNdO3Ml)GfELIFuW|1-zG|ro84xV^x&pW`Kw{v*O^k+hKA3 zw>(lHvcRI(_%8?)ejKzM0v~(!G|AGO;a3O8t>n8xzL4}0^;R;979~wFhiMhUbz+;% z=9Uk`TYil>O<^asX4k>!Adfecu)S}@$seUlb5NI7kl|~l3)Jgh*5_04%2C{^yML6R7>bzn?z}xe7ef=`YjGXY4vgw5`AqlBOzw8Ve;ggw{D;6eH=m) zK?%H3UTQTzIU}RGO}{-t2~6&0v=sYe!;&_E5BOQ`^-);K&F#Mm&UeDF0#+P`7sD_TW$83=xW zx0Xe>cA}4OQ;kXAlYo<59oeoIM?8`W3U&4gGugk1Dcv3|EstkP^{&c%5%pX z2qT>9Ko@`;=?A;Ko{{VBh_rP{UH>T!`|T@|x-b=-$5Z`yppu4vv;0<`EFt z!LSGp_CQoR4K|15YzALik>B|s>tdyVs0lJ(86VY6UXNQMYVPk$6Uv@|z1*qogc{v|19Lc5j=iw#2h2SLyu~LW@Y>yay<_1GY|&yWn-G$WT!v$Y0zPDG8&#VYQy;tfB#$j z6s~TBJY=UN*A;O-Ehq3hm`W;m7-ql%8Lw{1*>z5CrxAP8>!zA8%Tfa(v~&0m@QXy4 zl1ifrPpr)*4=pRyTq9|rQhTr`TAK44!p*jFTm9Q-_hx!4zV;McB%B$ zlQ(KlI;BIk^Rf8$PQU>&idn2Qcpi5+LasVuf%~Ba_@zMHtT; ziT<@X=H;x=21?QQf;A0X5gebP?;(R8R}92Gl_Ui4jbz$s!X~^>~|p5D)&-B z%%Oj+y%`yAEB+&vZfOQRv+{U*)nPL^_G{X`!TnI^g|M+jH#Tr_lwvQQjv0Wq(YTzZ z<(#n_Iv%2`|1iJ42!|=eY}Ob?%;X7O$7#W0dl#6$yQA}kR$M)@@VGY#zwJ#zpV7|kp$qPFgzTauQW z(r*dDht{bA)150@Bv2~dw+XI8)mmtWfb9_s?E#}>$xHlF7B@tn2L<8E;z2KcBb{e4 z<#DrCi;sHVXBqGXhblXnOB@gl$H`bq8ugKM*ukN_Wq$I7XHwnG#TMR}`@}&)#M`<4jI&C43|_5?9wmPtNhKMEe7aQm=YONqo%8pP{R? zsx~Q}P`0rnr2)zo?xB)cs8!i}l<#u&WcbYj*KUDem)y%DB@swr`GcYO+Q__N#r@K; z#a|YbEK%tK9Ewb3>WjE%*5hd!{9QtAic+PAi!f(A;{kVtTO}%gElZmgIW2ga;?^Wz z{{o^F<#65o_1l`E= zgzMvPGRxwkgb)#BDA0VzSO1G?TyDGy5pn1zKAQLyboEh_jHhlKUAxkp{oyQ7<#y+=Q6+T>3%} zajW-p@R*HdoE#Kg)ArR%9r+(g0p=Mf#w161k#MwH$eH-#1{Rd^^ytMOA9pFuf3b9A zKB~EmtuZ@#y=l=6&oic7r+K*$s@af8c3p4HxZz?3scsxiY z9Pm$X$=igPm;dD{a>5N?0`Np1X|#IlP}88)kv?C3(08umEF`zlX%@OYAg_?NIlT5a zaiye8?pht6e`A(l_=QK}x9E8AN5!*qf5zC;XH{De@)KI)*Ax!aGYz}~U1VY8Q|yy{ z<6E8|y$uwQjxQwqZzG=G6@;4!$|ML=Bnb4My8AZ|0+&e-d7H*ifcv^rSFIQwr-d(- z(Vm+cIZvVJX&Uygq#2np7Zs>_Sw|qGjQ()ZCYQli84K zi3y2u1SwNRM}zO}K>dC4*2h%=Yb?n64dM6!T97lvl$E29jOPmbL5Q}>?wqZDMhR*! z{>uR0(oOTv;D}y~Kj{Pa=w-Yo16AwHal3t&=* z5w`Nlw9ahJY4S!I8UM-c>^dd7rMbmEQuU`K$GG#{5{;+}XaBmQXv?zg7uR)l`_kvd zQB=mtO!ebc5g;0g-+Xon%`Jv4i{%ABnIqW8VBS)Qz!FR;Z-sjtd`<`{DhZ~5(r09d z_gwg`qxd^StY0Oa1M>C!)a{q!YN{sK)K=0HTwX$y7Gp5~dIE|$Ola>yq&h)x@9|MU z1`(k`ZOU15g7@CB)3{5aJgy)b_lJZkt{^AAT+3`w9>9$7Em0B_=ZPoE$y-9jHOjO9 z#`Q>_$-g>qvoGYRUu37?;S4INaWkFFvGFDmza^nLTKFHw!-^pUn6yQl?s^dtC*=ZV z|3emSo$29oCN6nh9lfx^w%lg_=lm`WPv&AoG^QTkm?x_d&O-d8z9lMM^uj8Ub;7mB zvO5DZP$xmZKcRbdUTsdVJV16uDYO2KePigjAe6vSWuU1H5@9`Th+Z>@--CkH@b%Ez zT$UXpVDDWQ929@_C!Sw3(+*9G@AH6?x*OYlsr%ERQ$Gq+1=B@q%UsPn4NDZd$UmTw z;;Xm)K?WjElYKaxQciKvTHmzuoN=b-PbFnGU@ENSp=csqD6LRNDYc(^pv2w(qYE5j=J6&9DcOt9h5v*B=gVnUwU!4OZG`ao72>C74JFNwAM~4 zfJ2&-&F%iRXTSZ3D~~DF(*7cse&DcYWc{x%;8Dw23hT_ELm=|#u0~k%VHN}i|-uS*RYBU8P0ru=?|aA;BhL_GC}+ zI6dJA+H0jzlQ1Wq5rz97hR3}{1aqb@}OYlu2QQC2IOfGx0# zqK5M=7HKjK^FFFpM;;iURnM7gDh62$NzpbiHVi|6pN3L!lvdODYESM5e;pz6_m@OxdC$V;3g;(*HssUm0m>i>S3J$e z`TALEU#tT9?5fZxDuJXFZMaDafY_goEK{S=g9w(8OI_GkKqgnYs;XU^v!wD1Z#QTj zee67r(Ba(rfzX(QoRaZeMZ6f;vNPC)>6v|Ti)O$TR$kY^c%QBc0Q*GOHs2t1kKcxw zFf0BX$|63dRiae2XQwTddx+!;!s_~jw0I=_(HLJyxl{iudZbxj?g9%3h?3jcho9Sl z#*eD9%XFkl@W^*vSh2TUa2=dB*;Zun${a4q5<=vQbIzw5mvdBo!Yg&_-C8pCP`yR# z9vzZe*3IN&;g3LO6b)e7xaFYOM~_apwVV8)$D8cE2r<>0X}e73^k{Ew_t^5j+l<0# zvC(yPJqxO-!nfKNj+M#=p3g!u6Xb<(iWqsxXlPPkvyT(-b8OR;kTPl#%Pbr#&Z3j> zImRB@xg?}BX<7Sbgb>}kvZ-?5%8<1R<+KP4p+i`{%gFr}g-^wAeh(v1ci{68l6rRa ztTWrd9em32aqTu|FH);A9*jHBmtX7!!XO~Nd%#A5!TyUN?hUvAxT@Q|Tit(@sN!l6O zv3-4BbmiT)t9prbh8T4+S`K0=HuYp>?3HEvg%T!hxSjH3ujMbtNjD)=~Wc-&gpXXhXGHb1}b>aV2^=N z+f;(UcCz_^!D(YNc_Yx2q#hoHFj?K!;3p$id-Kcm;6w+hqm~)YvxgC*ZCW4SD)!Ec z?~+FS=TlA^Mf`&_J{&^@Y2Qgak>w!AI2LP(kbjm_Ie5Fkt<{T2`-piK{_-j%(H^>$u8a?MkeHYs*O`_@ntHgvF5h9 zju^n>5DV^bF^I*U#>&v!*^*EBVrJWpEt3Jn_GfpW@K59&>Mnc-ZfgPzH)fCLe2rs` z5Z`dUO{{-FTLZx+SVcJ&a*>v_g1f9Y5a)VAb9c5eI7%&x(OTUfGjv1?dtAldbw^-< zPcmmvTuyULVDD(e`*lsHWJ77QISkQw#jPW6ss2}99%mZc8zTmfL_)mCn!1WxZCC_< ziA>wH@9wGK3Uv2vc3+#9)d_= zj1W+n5jUobbV|mMEyLf9a8cA$DuAqllrid#UO6^fd!2hv!)?^>6;?Jn4e<9cMgSRx z^;$7@_&k>pvDI9eoJBIk?3FrZjzqjD6afnCF378}d`0+Rf;L)#|q#_zw#$A%rM-}{UG6(|1O5i=xAg$glXdFeFMJxx6} zQs+v+oYm7g@cL_UkM`0q2a^-k!zbMaJzQpcFgVI7%tunL)_W-Rq|mK-BP&M4XrRvz z`C!-!>US4eJo_|eRGEpkqkOS)$O06$EY&W@+kUt_N&`nFwg0rpk^hJ{StR!ALucjw zAv70s{!sTI<|U)eb-j|~$#R*z(-72Y^Pl=XDNbg&5!{w#ZxqFA7mW-4h~Hx-&&_w^ z^&@Sc@(SHDJR&}(UMyn#p~-8apnp7Hw7woC2Z6hgeS)zrKe@7_ptQX%wFiLNG_ba< z_b{I7Mq|~tU3K#Fa_jx$A9vvcv_=gR>(S2At9vD~y0?BQvF596Iq&V|484y*N8XB; zJWoe|?E#69RNQH}@snt_1!>8uEBfNPUSU;xJrEr>R%9x}aG{T}Tk1a>_aAlidXm_c zyFPtZLshv!*j7hJzfiN2_kh{OOAhFN$f6)C#;isw9D>b46W<8cKbR@RfY1mgdHi_7 zrS&q(J4=#rE)u@BUX-N7%LE2m8tA5jy|9zPKs?ZpmMm|37~Js)_f!fiABa9)IU$55 zUJ`#4;|FFPR6P>b0rUh9M5Iogc-;~cy<0G&o$0nDsaqpgt0uEJE^W0wqf|3DpOW*V zU)*60Echn~sA?Hu2lB}dtoXBE#yL8))HER?heLO zHnt&Glu``Ks@Y!XjSV0)HHsp+(kS-r+nM)%137^tA7f0G$bu$hJiXC1mNd~=myVdE zjJ7&znz+evq+v?4*Pj(M(t_c}E}S+Z-;;cmev7m;VJeE%NXE*x2Z-kg*RW!?Q&tmtm@L*vnIt9<> zFaj7O-wYx1_b`rQ)$>ywlO%>Hepth=%t9q~?H8&QgKs&G<@)+cqL^XEqs@}j$wL)T zUjJ@l0Hh4DJVga4y+#6yi(28!26CX-gU3yx-b2PpFA>Ut0_~Y+jBxy$wZXS+f=IUP)2z4?O(to5TELguy zXm;@$K(}H=;>bOH{XncHVZfM8NI|h0!a;)7YS0T<_nP;r5LxBZ1_mRaI{@3N{j~i`1I7I$zg6a0sk$5c zmjkT}g(>y@GfAUsaK67uT>H08j8IRgC{_DmO-NshZ%%{Z=-&AGMPclHy4}buXvl^G^fhhnqUm!zMG$%eXKjj<>W*9Oofm6yVy}~ zhe*3+KK~GQ+e)!_0{smT$`zaIk!nI7J^Ce?Syc~hrdw`lU&WHiRoL&ab`Gue(oAjQ zWw?%LLJ5zdb`}-UIKwFy#7Rv`BEP*#FyO*J73+FFEQ8Z2ByL{Lg!dV6de)QEqA!e{ z>`}z|xH&3JW_#g%akS(6bibv4eg%Gx0>B&dGQJEMRDj6s!49D7-`D5d?^u#n&lUNJ z`kgZz9S|yI?LYe-Fr>%bHF%!g1Rm@}>osayaQsrB{;1BY%15*)eeU`l<{hd`pwC>( z@ArG6ky3lPU@;Deu0P~D0SVFQPy55upHaIwf4##=vGgq#)A=6qHGA?vIA#7o3Z}9U zqGj+>pAV0QdIV&`FVD(0Md$DSq|m5hSZFF8Cq+_Fqcu!xxEUl2p!8HsjUNxhBwSEQ zJWTfzpAiRIOAkY`Q%j@DFkchEc#^Rh7E1j@G_n5h9qe{{1VG4Xc3sFwEjsELSyk{U z#?tbxuKZ~(vuJc&v{@|j^-JoBKh`+1;TujBTTV0)$^?+lnao}T)Iom*o>N@DD?Uc5 zc$0cSCd$(n&sNq@FI#V|REETNFZ1Zfz0CHf;w#q?{lRt%?~a1UMO4m&k=ybGEx*P? z*%r&h9L^;QRLO(fE6H|fkC87c3A##g@C!y7Wn5QfZv1)I>p}v$6$gHpN?ypIAZ|FH z0DEm_76N>wMNZ^re3_s%%xka%|{0=WBs_+Q? zLcWs}Q&f%im)1EGRj`VY)Op5zQUZcD|NNpti_}LBT(A_LMUsGuL&|!l zM^_X<8KCzCS4^O67OTME@ncS&1GDhY+VMr+x06TaT`qkYe})rAoM6T_H^81&=C3>S z(brbXco!Y4u5*;680Z(qGv}P+cb0Oeb|&E-hX$G1Kht3)1`({PH^gRp2)>8&BBH`| zUId=$f74_Lqu60g#|@sLQoff8)fb3l_JhXP6w-=BS-NWl*|+*$Db!Ji;^zJWf$R<2 zjRt`bn@tTwk6lv>R(dkq1o;LZQ_sINgOC3EtAy51$Zo`Z4JkyAo7j5?$(f8=<}Y=t zpCoH-$`;P%-$LSN?+H)H@?dI(pYE9e1PqWzq!0=hQY=i!`6@`E5HgKJopmd(#=nTH z58@vz%lGRAj~v`k5ib98S!Vh$#p`ncdss@H&HN~~zV(e;nC6ix~u z-s3*yv@DtlSu=H+ks6}>dcGWKf4{I?EH38ysz_=xB{d2wwRAXidNWK3r4rX_nk>-k zKjmGDb_B*rLTV*0^Lk-GDLz*8Iyi2nSJk(eCjkc96cK?1vc#J%JNuy2=M-LgY-XdT zK|rFS{yWTifgE2tRMC9DKT?Qyey%q&c-mo4kzdROong&kwf0wRlXR^i2Z)*+sgZ0t z{^|mJ2@|nuHMjyhZh&8=!;&gJu?P_Dw01&o^VmNZp=emSfmOBDSUhayrbO)>DblVp z5TdL*GNgN++NvD#r<=Y)WpG!yc`IcdcS`wE3wLZ~J@!g@tIAdWoLWscF@meepG;FF zHEF5XTy3=?7U?@u+(kIkY}i^N+~J++oIYG;W7WDoh%%i;A8U!V-k;e*-wOybw9;Q1 zE@f?YK|7LNVyKZmH0pvv#5O*m_GT#>u zsx%Hd?qSa+7_$CS9J<i9*TuVieYplOxVZOeb{lgW|~5N`ru#?CT2??56M}*$~CO@m=@X8 zK&gsw`6%t%G1Hz%=5s56<`4PX&|xp?q@6}T{ZsGnzGnn@+fotq!?`IUeBq&x46tQ| z_QnQZAMgyb^ zL?k2V6=o+H@R|$cvH5UaNpx+ph)e$D=C6+UA)8ncXR?X8nvf2HErHa8Q=@k6>Q24(q5W@Lb7wy@J)+$#n(eXtTbSSt!*mXo6GK396deWh;pyE{ml+^-AgW^0mX&_DMCr zmWGz{ut}pI9eB=#FHNG>cuk~^??jVJ`QG4tgIxS?K{U;h@N530<}a=W5u4f}7t@KW z+lkMZkuYXW4Yk2`L#n|5bo6Qs40H;`YIY5^6HgYWb{NX2io+%01Npe{T0cV{RLg!E{<3I?*kfB zoXFX2LVjZ%TyUCGy6$wU-l>{10_SD7!pk*H=A0F?F)h=@)+p)@``7K1Y+8*29bBQs3efsjEV+ovJkD zma1HN#`U}BoKkhAYpNk%TT^H5U)|`x<8|k?0ein8((Q@o`+yJT5uTfI{DP0|SArrM zi)8T&`GMkf#rLnB&xre$iL#59R=SACUU_bE`arSmW;z^Y$0yRNCjagN3^1YR)OQCQ zuTw$I7+8^%*4m=?X{pQL4%_S9tLtn0Y;Uc$1Q4KGnNPM7DkPpKknRe-M7P-Mw)kNm zV2%cm;_62ZCaN%t3(G=6>!m{7)-koyh*>)UPsHo*ot=}{5X6aG=ZwO*^Spc|L{5bp zYcd#m2TAM=oLym1wuJcZ!^tVbZ1Tpulv$DG0^xt8l2)hPOn+Iq0n3h7$K|CDu%07~ z(!s2Su!j%=M4cXPVvIo>P6!Ll=}c?D;i$G=%++^f{bUi(uJQ_@Fj}KNj%;lMjcw%b zMyA@p)XK)r;st>&?lnGSP97*nm$K{l?H|{0y4`@|4uqV_f#))(h(E6@;9W{ZTmvN* zH>WHwc1v+PBhEqF5jk=?_}BWiUAh?`3XZNIGGu28vFZ^BR3J_2M4^V;&fqctUf&FV ztk$@@zphLX*AFvGkaiXjbqIpu8ptFQ|L7)ExmVG>Q>zin zH$%LSepg3})>lp&)dJ|f%JY9G<>IkYOKntk*+sqX-H^CPqc9vP!vdb3J()ej@-sN@ z$5$|T@A!67{S@ns%M=dr{W#u7}LEcA@m=G5mDN6@_%w!-NG>d^ue-2itx zW+d5y$d@~J)^xw0!03=m&W`USq)>UnYGbe?8^P_(%foTQ%2=8rxHAs#-WzD0*DPoq z>m}4+(>nO%6sFoN0h^7EDb(H$tlNu2UbUKI&hG;F@kRa09JC2q(;=EoS7*#6QM27^ zWR1*jK0d%%%e_X^FiHd9qf*%eP}E-->vMblY(!G0iw|6*88?EY*@-U6MJlJD?jLP& z{T;TFKf-T;<-6`?=l!ZmJX|vHT!FRt9F*F;9GoynDI*`aoVQG-Fg7EzfE7LUz;Qii zl^1=}q{VpoPvO1PG8dpfl>D5Pa^`rjxawk3-GEfpa;@(4aja*3f0Ch?V3iYF`p+lf zGpkBidG?>z>R@~yU#rnKWxmj7vZw>sk>z>o^<3F1;Lk+YO;4BveDweO^SS*%u?1Nf z=$RNy^qCA<4fPF}^$iV-*_hcm^bOb;jP;oqnAmu^|F?;hR6kHddlQ^G+t|pC#aqWCxlXgG;Lyov8YX~=E0vfndR{;hmOjWjsyHQO%vk; z?%&U2Chqe1aWr?OfK`&ayI?aiIq2eV+sIZ}V=jfaO;}@O1?3}B(&KRi_u$cA?LcxoRFQ9@vsND9J-B zI&Y|5A5+g1WYvu$#(EnQM0>@_o{0HERderAb0O+Ee+9VP0qm3!2p!5WGHZLR?n`3^ z21cxu5_dYOP*c~vyUvLTMFS^tTDr4NYGNIq(20uexjL-}XWloH{$neWE!nmdR06Wi zj4(O#XBY$SR}0OsFb!8Uh9s|Zyu6yWim;r4tM*x7lVz|6-Bsv>!8M&)I@^>_U1?uO zLM5^tF)qo@zJ`dYWg|rB+pgwM#@nW;5a3cnn2}(54)?j#m3+~8eueNk(G`6`=lUGn zeWgS2bqwiRutDW`mmH~#mInd1fF1#wzdGXcif#_0ctd;Y6z4J`hP`sh12xj7h{ Sz;Lj$G1J44l8VTQ!u%h0{@j=V diff --git a/spec/tinyspec.tex b/spec/tinyspec.tex index c541657a657a7..1a31a03eb8608 100644 --- a/spec/tinyspec.tex +++ b/spec/tinyspec.tex @@ -23,7 +23,6 @@ \definecolor{axblue}{HTML}{1565C0} % GLOBAL \definecolor{axcyan}{HTML}{00838F} % LOCAL \definecolor{axbrcyan}{HTML}{00ACC1} % WARP -\definecolor{axbrblue}{HTML}{42A5F5} % THREAD \definecolor{axwhite}{HTML}{616161} % LOOP (gray on white paper) \definecolor{axred}{HTML}{C62828} % REDUCE \definecolor{axbrred}{HTML}{E53935} % GROUP_REDUCE @@ -307,7 +306,6 @@ \subsection*{Kernel Optimizations (OptOps) \normalfont\small--- schedule-level t {\color{axblue}\texttt{GLOBAL}} & \texttt{g} & --- & --- & GPU global workgroup dimension. \\ {\color{axcyan}\texttt{LOCAL}} & \texttt{l} & g, L & inner & Workgroup local dimension (shared memory). \\ {\color{axbrcyan}\texttt{WARP}} & \texttt{w} & \multicolumn{2}{l}{(created by \op{TC})} & Warp-level lanes for tensor cores. \\ -{\color{axbrblue}\texttt{THREAD}} & \texttt{t} & g & outer & CPU thread parallelism. \\ {\color{axwhite}\texttt{LOOP}} & \texttt{L} & --- & --- & Generic sequential loop (initial state). \\ {\color{axred}\texttt{REDUCE}} & \texttt{R} & --- & --- & Reduction axis. \\ {\color{axbrred}\texttt{GROUP\_REDUCE}} & \texttt{G} & R & inner/outer & Shared-memory group reduction. \\ diff --git a/test/backend/test_jit.py b/test/backend/test_jit.py index a9c34ad678644..920cfa2f3d5d6 100644 --- a/test/backend/test_jit.py +++ b/test/backend/test_jit.py @@ -4,7 +4,7 @@ from test.helpers import assert_jit_cache_len, call_is_graph, not_support_multi_device, needs_second_gpu, KernelCountException from test.unit.test_jit import _simple_test -from tinygrad import Tensor, Variable, TinyJit, Device, dtypes +from tinygrad import Tensor, TinyJit, Device, dtypes from tinygrad.engine.jit import graph_class from tinygrad.helpers import JIT, DEV, GlobalCounters, HCQ2 from tinygrad.uop.ops import Ops @@ -16,19 +16,6 @@ def test_simple_jit(self): def add(a, b): return (a+b).realize() _simple_test(add) - @unittest.skipUnless(Device.DEFAULT == "CPU", "core_id is a CPU runtimevar") - def test_hcq_core_id_runtimevar_merge(self): - N = 262144 - @TinyJit - def f(x, st): - y = (x + 1).contiguous().realize() - z = x.shrink(((st, st + N),)).contiguous().realize() - return y, z - x = Tensor.arange(2*N).clone().realize() - for _ in range(3): y, z = f(x, Variable("a", 0, N).bind(0)) - self.assertEqual(y.shape, (2*N,)) - self.assertEqual(z.shape, (N,)) - def test_jit_input_view(self): @TinyJit def f(x): return (x[2:5].contiguous() + 1).realize() diff --git a/test/device/test_hcq2.py b/test/device/test_hcq2.py index 364d02f0ec739..002be545720c8 100644 --- a/test/device/test_hcq2.py +++ b/test/device/test_hcq2.py @@ -12,6 +12,16 @@ def test_copy_without_copy_queue(self): with patch.object(Device[Device.DEFAULT], "has_copy_queue", False): np.testing.assert_equal(Tensor(np.arange(61, dtype=np.float32)).to(Device.DEFAULT).contiguous().realize().numpy(), np.arange(61)) + @unittest.skipIf(Device.DEFAULT == "CPU", "ping-pong needs a non-CPU hcq2 device") + def test_cpu_device_ping_pong(self): + # CPU submits run inline, so alternating dependencies must be submitted in schedule order to avoid blocking the host submitter. + x = Tensor.ones(16, device="CPU").contiguous().realize() + a = (x + 1).contiguous() + b = (a.to(Device.DEFAULT).contiguous() + 1).contiguous() + c = (b.to("CPU").contiguous() + 1).contiguous() + out = (c.to(Device.DEFAULT).contiguous() + 1).contiguous().realize() + np.testing.assert_equal(out.numpy(), np.full(16, 5)) + @unittest.skipIf(Device.DEFAULT == "CPU", "staged copies need a non-CPU hcq2 device") def test_staged_copy_slot_reuse(self): # chunks of a staged copy rotate through the staging buffer slots, many rotations must stay bit-exact in both directions diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index 0c347bef212f0..44150cb8f6054 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -333,20 +333,6 @@ def test_arange_opts(self): #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=1, arg=4)], # noqa: E501 ]) - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_threads, "test requires threads") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.global_max is not None and - Device[Device.DEFAULT].renderer.global_max[0] > 1, "test requires multicore") - def test_thread_opts(self): - a = Tensor.rand(4, 4, 4, 4) - b = Tensor.rand(4, 4, 4) - r = (b.sqrt() + ((a+1).sum(axis=3).exp())) - helper_linearizer_opt(r, [ - [Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.THREAD, 0, 2), Opt(OptOps.UNROLL, 0, 2)], - ] + [[Opt(OptOps.THREAD, 0, 4)] if Device[Device.DEFAULT].renderer.global_max[0] >= 4 else []] - + [[Opt(OptOps.THREAD, 0, 8)] if Device[Device.DEFAULT].renderer.global_max[0] >= 8 else []]) - def test_double_sum_group(self): a = Tensor.rand(4, 4, 4) r = a.sum((1, 2)).sum() diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index 4f01a8eca58d9..d555254dca3a1 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -1,7 +1,7 @@ from dataclasses import replace, dataclass import itertools, functools from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC -from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT, TracingKey, Context, panic +from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const from tinygrad.uop.render import pyrender @@ -497,7 +497,7 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: # config affects generated programs and cache keys; context also carries compile-only behavior to workers to_program_config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, - DEFAULT_FLOAT, DEFAULT_INT, NUM_CPU_THREADS, TC_SELECT, TC_OPT) + DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT) to_program_context = (*to_program_config, SPEC, DEBUG) def to_program_key(ast:UOp, renderer:Renderer) -> tuple: return (ast.key, type(renderer), renderer.target, *[x.value for x in to_program_config]) diff --git a/tinygrad/codegen/gpudims.py b/tinygrad/codegen/gpudims.py index 10f45d89c89c9..849061f4b0dce 100644 --- a/tinygrad/codegen/gpudims.py +++ b/tinygrad/codegen/gpudims.py @@ -1,6 +1,6 @@ import math from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, KernelInfo, ssimplify, AxisType -from tinygrad.dtype import dtypes, AddrSpace +from tinygrad.dtype import AddrSpace from tinygrad.renderer import Renderer def _dim_max(d:sint) -> int: return d if isinstance(d, int) else int(d.vmax) @@ -47,7 +47,7 @@ def add_gpudims(ctx:Renderer, s:UOp): all_ranges = {x.arg[0:-1]:x for x in s_topo if x.op is Ops.RANGE} # extract global/local dims - global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.GLOBAL, AxisType.THREAD)]) + global_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] is AxisType.GLOBAL]) local_dims = sorted([x.arg[0:-1] for x in all_ranges.values() if x.arg[-1] in (AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) if not global_dims and not local_dims: return None @@ -57,8 +57,7 @@ def add_gpudims(ctx:Renderer, s:UOp): # get the idxs ki: KernelInfo = s.arg - if ctx.has_threads: idxs = [UOp.variable("core_id", 0, int(global_shape[0])-1, dtypes.int, param=True).cast(dtypes.weakint)] - elif ki.dont_use_locals: + if ki.dont_use_locals: assert not local_dims, "can't use locals if there's no local dims" idxs = get_grouped_dims("idx", global_shape, ctx.global_max, reverse=True) else: diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index fb4b84ddee184..0618cb4d27678 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -4,7 +4,7 @@ from dataclasses import dataclass class OptOps(Enum): - TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto(); THREAD = auto() # noqa: E702 + TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto() # noqa: E702 GROUP = auto(); GROUPTOP = auto(); NOLOCALS = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index 9ff87f9c094f0..c0e70443cb906 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -191,17 +191,4 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: k.apply_opt(Opt(OptOps.LOCAL, axis, local_sz)) if will_delete_shape: deleted_shape += 1 - # **** threading **** - - if k.ren.has_threads and k.ren.global_max is not None: - for threads in [32,16,12,8,6,5,4,3,2]: - # Skip if too many threads. Heuristic: use about 128K ops per thread - if threads > k.ren.global_max[0] or resolve(prod(k.full_shape) // (128 << 10) < threads): continue - for axis in k.axes_of(AxisType.WEAK): - if k.full_shape[axis] % threads == 0: - try: k.apply_opt(Opt(OptOps.THREAD, axis, threads)) - except KernelOptError: pass - break - if k.applied_opts and k.applied_opts[-1].op is OptOps.THREAD: break - return k diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index cda176f7e73a8..f15cad181c101 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -133,7 +133,7 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): opt_to_at = { OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, OptOps.UNROLL: AxisType.UNROLL, OptOps.GROUP: AxisType.GROUP_REDUCE, - OptOps.GROUPTOP: AxisType.GROUP_REDUCE, OptOps.THREAD: AxisType.THREAD} + OptOps.GROUPTOP: AxisType.GROUP_REDUCE} ret = None if opt.op in opt_to_at: @@ -160,16 +160,11 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): if opt.op is OptOps.LOCAL: check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") - if opt.op is OptOps.THREAD: - check(self.ren is not None and self.ren.has_threads, "target does not support threads") - check(self.ren is not None and self.ren.global_max is not None and amt <= self.ren.global_max[0], "too many threads") - check(all(x is not AxisType.THREAD for x in self.axis_types), "already threaded") - check(rng in self._globalizable_rngs(), "can't apply range to this dim") if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") - ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op in {OptOps.GROUPTOP, OptOps.THREAD}) + ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op is OptOps.GROUPTOP) elif opt.op is OptOps.TC: check(len(self.applied_opts) == 0, "tensor core opts must be first") # TODO: remove the need for this by having warps check(opt.axis is not None, "tensor core opts must have an axis") @@ -183,7 +178,6 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): elif opt.op is OptOps.PADTO: check(rng.src[0].op is Ops.CONST, "only pad const axes") check(rng.arg[-1] not in {AxisType.UPCAST, AxisType.UNROLL}, "cannot pad upcasted") # TODO: why is this wrong? - check(rng.arg[-1] is not AxisType.THREAD, "cannot pad thread") new_sz = round_up(int(rng.vmax+1), cast(int, opt.arg)) check(rng.vmax+1 > new_sz//4, "pad adds more than quadruple the work") replaced_rng = UOp.range(new_sz, *rng.arg, dtype=rng.dtype) diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index cd7bc59284a5b..9d3d7d375340d 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -22,7 +22,6 @@ # covers resnet kernels (3 global * 3 reduce) actions += [Opt(op=OptOps.TC, axis=axis, arg=(-1, getenv("TC_OPT", 2), getenv("TC", 1))) for axis in range(9)] actions += [Opt(op=OptOps.SWAP, axis=axis_0, arg=axis_1) for axis_0 in range(5) for axis_1 in range(axis_0+1, 5)] -actions += [Opt(op=OptOps.THREAD, axis=axis, arg=amt) for amt in [2,3,4,5,8,12,16,24,32,64] for axis in range(3)] if getenv("NOLOCALS"): actions += [Opt(op=OptOps.NOLOCALS)] def get_test_global_size(global_size, max_global_size, var_vals): diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index bfb7f29d3572f..cbb32b62cc238 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -106,13 +106,13 @@ def __init__(self, linear:UOp, input_uops:tuple[UOp, ...]=()): def is_sym_dim(dim) -> bool: return not all(isinstance(d, (int, float)) for d in dim) crs = [(j, self.calls[j][1].arg, self.calls[j][3]) for j in range(len(self.calls)) if self.calls[j][1].op is Ops.PROGRAM] - self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv | p.runtimevars}) + self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv}) self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if d and is_sym_dim(d)) def find_symbolic_dim(dim): return self.symbolic_dims.index(tuple(dim)) if dim is not None and tuple(dim) in self.symbolic_dims else None for j,p,dv in crs: - if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv | p.runtimevars]): + if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv]): self.var_vals_replace[j] = replace global_dim_idx, local_dim_idx = find_symbolic_dim(p.global_size), find_symbolic_dim(p.local_size) if global_dim_idx is not None or local_dim_idx is not None: diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index b3e7fdded8669..a318d5cf8c76f 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -260,13 +260,13 @@ def _get_cpu_count() -> int: if quota != "max": count = min(count, max(1, int(quota) // int(period))) except (FileNotFoundError, ValueError, ZeroDivisionError): pass return count -NUM_CPU_THREADS = ContextVar("NUM_CPU_THREADS", _get_cpu_count()) +CPU_COUNT = _get_cpu_count() NULL_ALLOW_COPYOUT = ContextVar("NULL_ALLOW_COPYOUT", 0) # VIZ implies PROFILE, but you can run PROFILE without VIZ VIZ = ContextVar("VIZ", 0) # this PARALLEL is for BEAM and compilation, it's currently disabled if you are using VIZ # pytest-xdist workers share the CPU budget, explicit PARALLEL still overrides this default -PARALLEL = ContextVar("PARALLEL", NUM_CPU_THREADS.value // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) +PARALLEL = ContextVar("PARALLEL", CPU_COUNT // max(1, getenv("PYTEST_XDIST_WORKER_COUNT", 1)) if VIZ == 0 else 0) PROFILE = ContextVar("PROFILE", abs(VIZ.value)) SPEC = ContextVar("SPEC", 1) # TODO: disable by default due to speed diff --git a/tinygrad/renderer/__init__.py b/tinygrad/renderer/__init__.py index 40682e7ccf64f..15fde7b19310a 100644 --- a/tinygrad/renderer/__init__.py +++ b/tinygrad/renderer/__init__.py @@ -50,7 +50,6 @@ def from_uops(uops:tuple[UOp, ...], ignore_indexing=False) -> Estimates: mults = mults.substitute({x:x.const_like(0) for x in mults.toposort() if x.op is Ops.SPECIAL}) if isinstance(mults, UOp) else mults elif u.op is Ops.END: mults = mult_stack.pop(-1) elif u.op is Ops.SPECIAL: mults *= cast(sint, u.src[0].ssimplify()) # NOTE: we don't push to the mult_stack here, you can't end these - elif u.op is Ops.PARAM and u.arg.addrspace == AddrSpace.ALU and u.expr == 'core_id': mults *= int(u.vmax) + 1 elif u.op is Ops.LOAD and u.src[0].addrspace != AddrSpace.REG: lds += u.max_numel() * u.dtype.itemsize * mults elif u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG: @@ -67,7 +66,6 @@ class Renderer: # TODO: make this generic with a list of supported types supports_float4: bool = True has_local: bool = True - has_threads: bool = False has_shared: bool = True # NOTE: these two should be in (x,y,z) order to match the max_sizes argument in get_grouped_dims global_max: tuple[int, ...]|None = (0x8FFFFFFF,) * (3) # TODO: Ops.SPECIAL int32 indexes right now diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 680bd4608dfdf..c62a69d14a19e 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -4,7 +4,7 @@ from tinygrad.codegen.opt import tc from tinygrad.uop.ops import GroupOp, Ops, UOp, PatternMatcher, UPat, range_str, axis_letters from tinygrad.uop.weak import commit_weak_consts -from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, NUM_CPU_THREADS, IMAGE, FLOAT16, is_image_shape +from tinygrad.helpers import strip_parens, getenv, prod, dedup, Target, IMAGE, FLOAT16, is_image_shape from tinygrad.dtype import dtypes, DType, AddrSpace, truncate, float_to_bf16 from tinygrad.renderer import Renderer @@ -263,9 +263,7 @@ class ClangRenderer(CStyleLanguage): float4_style = ('{', '}') gep_arr_threshold = 0 has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) infinity = "__builtin_inff()" nan = '__builtin_nanf("")' diff --git a/tinygrad/renderer/isa/x86.py b/tinygrad/renderer/isa/x86.py index 042486d3d7c4e..16fcff26cbabc 100644 --- a/tinygrad/renderer/isa/x86.py +++ b/tinygrad/renderer/isa/x86.py @@ -7,7 +7,7 @@ from tinygrad.uop import FastEnum, auto, Ops, GroupOp from tinygrad.uop.ops import UOp, UPat, PatternMatcher, promo_dtype from tinygrad.renderer.isa import ISARenderer, IselContext, Register, PreRegAllocContext, greg -from tinygrad.helpers import getenv, NUM_CPU_THREADS, unwrap, Target +from tinygrad.helpers import unwrap, Target # ***** X86 Ops ***** @@ -791,9 +791,7 @@ def _encode(reg_uop:UOp|None, rm_uop:UOp, idx_uop:UOp|None=None, disp_uop:UOp|No class X86Renderer(ISARenderer): device = "CPU" has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) extra_matcher = extra_matcher pre_isel_matcher = pre_isel_matcher isel_matcher = isel_matcher diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 2b47bb8ea990a..0664d79a17b25 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -5,7 +5,7 @@ from tinygrad.codegen.decomp.transcendental import xexp2, xlog2 from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, GroupOp, range_str from tinygrad.dtype import dtypes, float_to_fp8, DType, truncate, AddrSpace -from tinygrad.helpers import prod, Target, NUM_CPU_THREADS, getenv, OSX +from tinygrad.helpers import prod, Target, OSX def is_volatile(u:UOp) -> bool: return (buf:=u.buf_uop).op is Ops.PARAM and buf.arg.volatile @@ -203,9 +203,7 @@ def _render_kernel(self, uops: list[UOp], prefix:list[str]|None=None) -> tuple[t class CPULLVMRenderer(LLVMRenderer): has_local = False - has_threads = bool(getenv("THREADS", 1)) - @property - def global_max(self): return (NUM_CPU_THREADS.value, 0, 0) # type: ignore[override] + global_max = (1, 0, 0) abi = 'win64cc' if sys.platform == 'win32' else None string_rewrite = base_rewrite def render(self, uops: list[UOp]) -> str: return "\n".join((k:=self._render_kernel(uops))[0] + (k[1], self._render_footer(uops))) diff --git a/tinygrad/runtime/graph/hcq.py b/tinygrad/runtime/graph/hcq.py index 39e68a4430405..3ce29acde6936 100644 --- a/tinygrad/runtime/graph/hcq.py +++ b/tinygrad/runtime/graph/hcq.py @@ -96,7 +96,6 @@ def __init__(self, *args, **kwargs): # set any fixedvars on the device self.device_vars[enqueue_dev] = merge_dicts([self.device_vars.get(enqueue_dev, {}), device_vars]) - if runtime is not None: self.device_vars[enqueue_dev] = merge_dicts([self.device_vars[enqueue_dev], {k: 0 for k in ast.arg.runtimevars}]) if runtime is not None: enqueue_queue = self.comp_queues[enqueue_dev] diff --git a/tinygrad/runtime/ops_cpu.py b/tinygrad/runtime/ops_cpu.py index 048c23ce117bc..ae9967a3d53fb 100644 --- a/tinygrad/runtime/ops_cpu.py +++ b/tinygrad/runtime/ops_cpu.py @@ -1,6 +1,6 @@ from __future__ import annotations -import platform, sys, os, ctypes, functools, mmap, threading, array, struct, time -from dataclasses import dataclass, replace +import platform, sys, ctypes, functools, mmap, array, struct, time +from dataclasses import replace from typing import cast, Callable from tinygrad.helpers import to_mv, from_mv, OSX, WIN, Context, mv_address, suppress_finalizing, unwrap, data64_le from tinygrad.device import Buffer, BufferSpec, TinyELF, Program, Device @@ -19,10 +19,10 @@ from tinygrad.dtype import AddrSpace from tinygrad.uop.ops import KernelInfo, Ops, UPat, PatternMatcher -MAX_ARGS, CMD_SIZE, RING_SLOTS, FUNCS = 63, 64, (16 << 10), (() if WIN else ('clock_gettime', 'sem_wait', 'sem_post')) +MAX_ARGS, CMD_SIZE, FUNCS = 63, 64, (() if WIN else ('clock_gettime',)) # ***************** -# 1. workers +# 1. signal programs def signal_prog(): val = UOp.param(1, dtypes.int, (), vmin_vmax=(0, dtypes.int.max), name="value", addrspace=AddrSpace.ALU) @@ -40,21 +40,6 @@ def timestamp_prog(): val = ts.after(call)[0].load() * 1_000_000_000 + ts.after(call)[1].load() return UOp.param(0, dtypes.uint64, 1)[0].store(val) -def worker_prog(): - ring = UOp.param(0, dtypes.uint64, RING_SLOTS * CMD_SIZE, volatile=True) - wait, done = UOp.param(1, dtypes.uint64, 1, volatile=True), UOp.param(2, dtypes.uint64, 1, volatile=True) - sem, cur = UOp.param(3, dtypes.uint64, 1), UOp.range(2**64-1, 0, dtype=dtypes.uint64) # sem is unused on windows, it has to come last - - # spin on windows, sem_wait to sleep on posix - if WIN: ready = (v:=wait.after(lw:=UOp.loop(1), cur)[0].load()).end(lw, v <= cur) - else: ready = (rv:=wait.after(lw:=UOp.loop(1), cur)[0].load().call(sem.after(cur)[0], ret_dtype=dtypes.int)).end(lw, rv != 0) - - entry = [ring.after(ready).index((cur % RING_SLOTS) * CMD_SIZE + i).load() for i in range(CMD_SIZE)] - return done.after(entry[0].call(*entry[1:], ret_dtype=dtypes.void)).index(0).store(cur + 1).end(cur) - -@dataclass -class CPUWorker: ring:Buffer; put:Buffer; sem:Buffer; sys:Buffer; done:Buffer; thread:threading.Thread # noqa: E702 - # ***************** # 2. queue encoders @@ -67,10 +52,7 @@ def cpu_cmd(devs:tuple[str, ...], prog, *args:UOp) -> UOp: def cpu_exec(ctx, call:UOp, prg:UOp) -> UOp: devs = ctx.devs args = [get_call_arg_uops(call)[i].getaddr(devs) for i in prg.arg.globals] + [v.cast(dtypes.uint64) for v in get_call_var_uops(call, prg)] - if (core:=prg.arg.runtimevars.get('core_id')) is None: return cpu_cmd(devs, prg, *args) - - la = [cpu_cmd(devs,prg,*args[:(cid:=(len(prg.arg.globals)+core))],UOp.const(t,dtypes.uint64),*args[cid+1:]) for t in range(prg.arg.global_size[0])] - return UOp(Ops.LINEAR, src=tuple(la)) + return cpu_cmd(devs, prg, *args) pm_cpu_opsel = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), cpu_exec), @@ -85,25 +67,11 @@ def cpu_exec(ctx, call:UOp, prg:UOp) -> UOp: ]) def cpu_submit(ctx, cmdbuf:UOp) -> UOp: - # copy the cmd entries into the worker ring and post the semaphore once per entry - assert ctx.nbytes % (CMD_SIZE * 8) == 0 and ctx.nbytes // (CMD_SIZE * 8) < RING_SLOTS, f"submit of {ctx.nbytes} bytes doesn't fit the ring" - devs, cnt, cb = ctx.devs, hcq_size_var(cmdbuf) // (CMD_SIZE * 8), cmdbuf.bitcast(dtypes.uint64) - ring, put, done, sem = (make_buf(devs, tag=f"{ctx.queue}_{n}") for n in ("ring", "put", "done", "sem")) - - # submits are serialized on the submitter, so they can bump put without atomics - ran = done.after(l:=UOp.loop(10)).index(0).load() - room = ran.end(l, put.index(0).load() - ran > (RING_SLOTS - cnt).cast(ran.dtype)) # wait until cnt entries fit in the ring - base = ((put.after(room, cmdbuf).index(0).load() % RING_SLOTS) * CMD_SIZE).cast(dtypes.int) - e = UOp.range(cnt, 11, dtype=dtypes.int, src=(cmdbuf, ring)) - # the slot is a multiple of CMD_SIZE, so a word can never wrap on its own: take the modulo once per entry, not per word - slot = (base + e*CMD_SIZE) % (RING_SLOTS * CMD_SIZE) - w = UOp.range(CMD_SIZE, 12, dtype=dtypes.int, src=(cmdbuf, ring)) - copy = ring.index(slot + w).store(cb.index(e*CMD_SIZE + w).load()).end(w) - - bumped = put.after(copy.end(e)).index(0).store(put.index(0).load() + cnt.cast(dtypes.uint64)) - if WIN: return make_buf(devs, tag=f"{ctx.queue}_sys").after(bumped).index(0).store(put.after(bumped).index(0).load()) - e = UOp.range(cnt, 13, dtype=dtypes.int, src=(bumped,)) - return make_buf(devs, tag="func:sem_post").after(e).index(0).load().call(sem.after(e).index(0), ret_dtype=dtypes.void).end(e) + # run the cmd entries inline on the submitting thread, the cpu has no worker threads + cb, cnt = cmdbuf.bitcast(dtypes.uint64), hcq_size_var(cmdbuf) // (CMD_SIZE * 8) + e = UOp.range(cnt, 10, dtype=dtypes.int, src=(cmdbuf,)) + entry = [cb.index(e*CMD_SIZE + i).load() for i in range(CMD_SIZE)] + return entry[0].call(*entry[1:], ret_dtype=dtypes.void).end(e) pm_cpu_submit = PatternMatcher([(UPat(Ops.CUSTOM_FUNCTION, arg="submit_cmdbuf", src=(UPat(name="cmdbuf"),)), cpu_submit)]) @@ -119,7 +87,6 @@ def _load(self, lib, base=0): return lib if lib[:4] != libc.ELFMAG.encode() else def __init__(self, dev:CPUDevice, obj:TinyELF): self.dev, self.name, self.signature = dev, obj.name, obj.signature - self.runtimevars = {name:slot for name,slot,*_ in obj.signature if name == 'core_id'} self.lvp = obj.target.renderer == "LVP" if sys.platform == "win32": # mypy doesn't understand when WIN is used here @@ -165,9 +132,7 @@ def __call__(self, *bufs:HCQBuffer, global_size:tuple[int,int,int]=(1,1,1), loca else: args = [*[cast(int, b.va_addr) for b in bufs], *cast(tuple[int, ...], vals)] assert len(args) <= MAX_ARGS, f"CPU programs support at most {MAX_ARGS} arguments, got {len(args)}" - for tid in range(global_size[0]): - if 'core_id' in self.runtimevars: args[self.runtimevars['core_id']] = tid - self.fxn(*[ctypes.c_uint64(x) for x in args]) + self.fxn(*[ctypes.c_uint64(x) for x in args]) return time.perf_counter() - st if wait else None @suppress_finalizing @@ -198,48 +163,22 @@ class CPUDevice(HCQ2Compiled): pm_encode, pm_lower = {"COMPUTE": pm_cpu_opsel, "SUBMIT": pm_cpu_opsel}, {"COMPUTE": pm_cpu_submit, "SUBMIT": pm_cpu_submit} def __init__(self, device:str=""): - self.workers:list[CPUWorker] = [] super().__init__(device, CPUAllocator(self), [ClangRenderer, CPULLVMRenderer, LVPRenderer, X86Renderer], CPUProgram, arch={'amd64':'x86_64', 'aarch64':'arm64'}.get(m:=platform.machine().lower(), m)+",native") self.pm_bufferize = PatternMatcher( - [(UPat(Ops.PARAM, tag=f"{q}_{n}"), lambda ctx, q=q, n=n: getattr(ctx[0].worker(q), n)) - for q in ("COMPUTE:0", "SUBMIT:0") for n in ("ring","put","sem","sys","done")] + [(UPat(Ops.PARAM, tag=f"func:{f}"), lambda ctx, f=f: ctx[0].func_ptr(f)) for f in FUNCS]) + self.pm_bufferize with Context(EMULATED_DTYPES="", TRACK_MATCH_STATS=0): clang = ClangRenderer(replace(self.renderer.target, renderer="CLANG")) self.prgs:dict[Callable, CPUProgram] = {f: CPUProgram(self, do_to_program(f().sink(arg=KernelInfo(f.__name__), tag=1), clang).to_elf()) - for f in (signal_prog, wait_prog, timestamp_prog, worker_prog)} + for f in (signal_prog, wait_prog, timestamp_prog)} def func_ptr(self, name:str) -> Buffer: return self.func_table.view(1, dtypes.uint64, FUNCS.index(name)*8).ensure_allocated() - def synchronize(self, timeout:int|None=None): - for worker in self.workers: - put, done = (getattr(worker, x)._buf.cpu_view().view(fmt='Q') for x in ("put", "done")) - while done[0] < put[0]: self._wait_signal(done, put[0], timeout) - super().synchronize(timeout) - @functools.cached_property def func_table(self) -> Buffer: lib = ctypes.windll.kernel32 if sys.platform == "win32" else libc.dll # type: ignore[attr-defined] (ft:=Buffer(self.device, len(FUNCS), dtypes.uint64, preallocate=True))._buf.cpu_view().view(fmt='Q')[:] = \ array.array('Q', [unwrap(ctypes.cast(getattr(lib, f), ctypes.c_void_p).value) for f in FUNCS]) return ft - - @functools.cache - def worker(self, queue:str) -> CPUWorker: - ring, put, sysbuf, done = (Buffer(self.device, sz, dtypes.uint64, preallocate=True) for sz in (RING_SLOTS*CMD_SIZE, 1, 1, 1)) - addr, hsem = 0, None - - # sem are posix-only - if not WIN: - hsem = libc.sem_open(nm:=f"/tinygrad-{os.getpid()}-{id(ring):x}".encode(), os.O_CREAT|os.O_EXCL, 0o600, 0) # type: ignore[call-arg] - if (addr:=unwrap(ctypes.cast(hsem, ctypes.c_void_p).value)) == ctypes.c_void_p(-1).value or libc.sem_unlink(nm): - raise OSError(ctypes.get_errno(), "semaphore") - sem = Buffer(self.device, 1, dtypes.uint64, options=BufferSpec(external_ptr=addr), preallocate=True) - - worker_args = [ring._buf.va_addr, sysbuf._buf.va_addr if WIN else self.func_ptr('sem_wait')._buf.va_addr, done._buf.va_addr, addr] - (thread:=threading.Thread(target=self.prgs[worker_prog].fxn, daemon=True, args=[ctypes.c_uint64(x) for x in worker_args])).start() - self.workers.append(worker:=CPUWorker(ring, put, sem, sysbuf, done, thread)) - return worker diff --git a/tinygrad/runtime/ops_dsp.py b/tinygrad/runtime/ops_dsp.py index e487f666e7ea6..2de37e791c663 100644 --- a/tinygrad/runtime/ops_dsp.py +++ b/tinygrad/runtime/ops_dsp.py @@ -10,7 +10,6 @@ if getenv("IOCTL"): import extra.dsp.run # noqa: F401 # pylint: disable=unused-import class DSPRenderer(ClangRenderer): - has_threads = False buffer_suffix = " restrict __attribute__((align_value(128)))" kernel_typedef = "__attribute__((noinline)) void" type_map = { **ClangRenderer.type_map, dtypes.uint64: "unsigned long long", dtypes.int64: "long long" } diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index af3ad32a63a54..5c8519b549ba2 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -167,6 +167,11 @@ def _merge_submits(calls:list[UOp]) -> UOp: estimates=sum((c.arg.aux.estimates for c in calls), start=Estimates()).simplify())) def _merge_queues(submits:list[UOp]) -> list[UOp]: + # CPU submits run inline and can block on another queue. Keep multi-queue CPU work in schedule order so every + # producer queue is submitted before a CPU wait; merging by queue can otherwise deadlock alternating dependencies. + keys = [unwrap(get_submit(call)).src[0].arg for call in submits] + if len(set(keys)) > 1 and any(any(d.split(":")[0] == "CPU" for d in devs) for devs, _ in keys): return submits + merged:list[UOp] = [] opened:dict[tuple[tuple[str, ...], str], list[UOp]] = {} # (devs, queue) -> hcq calls in submit order limits:dict[tuple[tuple[str, ...], str], int] = collections.defaultdict(lambda: JIT_BATCH_SIZE.value) @@ -412,8 +417,8 @@ def lower_hcq_call(call:UOp) -> UOp|None: (UPat(Ops.CALL, src=(UPat(Ops.CUSTOM_FUNCTION, arg="hcq", src=(UPat(Ops.SINK),)),), name="call", allow_any_len=True), lower_hcq_call)]) # ***************** -# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 ring: a submit whose cmds call the -# compiled piece programs, so the worker runs the batch in fifo order and the python exec is one ring push +# 6. batch: adjacent hcq calls fold into one submitter on the host SUBMIT:0 queue: a submit whose cmds call the +# compiled piece programs, so the batch runs in fifo order and the python exec is one submitter call def _lane_arg(a:UOp, lane:int) -> UOp: return a.mselect(lane) if len(to_tuple(a.device)) > 1 else a diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 196c0851337b5..486f1dbf37021 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -17,7 +17,7 @@ class AxisType(Enum): def __repr__(self): return str(self) DEVICE = auto(); GLOBAL = auto(); WARP = auto(); LOCAL = auto(); WEAK = auto(); GROUP_REDUCE = auto(); REDUCE = auto(); UPCAST = auto() # noqa: E702 - UNROLL = auto(); THREAD = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 + UNROLL = auto(); PLACEHOLDER = auto(); LOOP = auto() # noqa: E702 @dataclass(frozen=True, order=True) class ParamArg: @@ -39,14 +39,14 @@ def __repr__(self): args = [repr(self.slot), repr(self.dtype)] + ([repr(self.size)] if self.size is not None else []) + \ [f"{k}={v!r}" for k,default in fields if (v:=getattr(self, k)) != default] return f"ParamArg({', '.join(args)})" -axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.THREAD: "t", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", +axis_letters = {AxisType.DEVICE: "d", AxisType.GLOBAL: "g", AxisType.LOCAL: "l", AxisType.WARP: "w", AxisType.WEAK: "L", AxisType.LOOP: "L", AxisType.UPCAST: "u", AxisType.GROUP_REDUCE: "G", AxisType.REDUCE: "R", AxisType.UNROLL: "r"} -axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.THREAD: "BLUE", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", +axis_colors = {AxisType.DEVICE: "green", AxisType.GLOBAL: "blue", AxisType.LOCAL: "cyan", AxisType.WARP: "CYAN", AxisType.WEAK: "WHITE", AxisType.LOOP: "WHITE", AxisType.UPCAST: "yellow", AxisType.GROUP_REDUCE: "RED", AxisType.REDUCE: "red", AxisType.UNROLL: "magenta"} # NOTE: LOCAL and GROUP_REDUCE have the same priority. the order here matters -axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.THREAD: 0, AxisType.GLOBAL: 0, AxisType.WARP: 1, +axis_to_pos = {AxisType.DEVICE: -2, AxisType.WEAK: -1, AxisType.LOOP: -1, AxisType.GLOBAL: 0, AxisType.WARP: 1, AxisType.LOCAL: 2, AxisType.UPCAST: 3, AxisType.GROUP_REDUCE: 2, AxisType.REDUCE: 4, AxisType.UNROLL: 5} range_start = {Ops.STAGE: 1, Ops.REDUCE: 1, Ops.WMMA: 3, Ops.END: 1, Ops.CALL: 1, Ops.LINEAR: 0} @@ -1268,16 +1268,13 @@ class ProgramInfo: @property def function_name(self): return to_function_name(self.name) - @property - def runtimevars(self) -> dict[str, int]: return {v.expr: i for i, v in enumerate(self.vars) if v.expr == 'core_id'} - def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]|None]: global_size = tuple([sym_infer(sz, var_vals) for sz in self.global_size]) # type: ignore[arg-type] local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) if self.local_size is not None else None return global_size, local_size - def vals(self, var_vals:dict[str, int]) -> tuple[int|None, ...]: - try: return tuple(var_vals[k.expr] if k.expr not in self.runtimevars else None for k in self.vars) + def vals(self, var_vals:dict[str, int]) -> tuple[int, ...]: + try: return tuple(var_vals[k.expr] for k in self.vars) except KeyError as e: raise RuntimeError(f"unbound Variable {e} used by {self.function_name}") from None @staticmethod @@ -1298,7 +1295,6 @@ def from_sink(sink:UOp, target:Target=Target()) -> ProgramInfo: if u.arg[0] == 'i': local_size = None special_size = local_size if u.arg[0] == 'l' else global_size if special_size is not None: special_size[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) - if u.op is Ops.PARAM and u in _vars and u.expr == 'core_id': global_size[0] = int(u.vmax) + 1 return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), tuple(local_size) if local_size is not None else None, tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), tuple(sorted(dedup(ins))), target) From 79250d76dee4a1bcf3a31281ad1f3477d2fa86a4 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Mon, 31 Aug 2026 16:43:13 -0700 Subject: [PATCH 301/306] remove NOLOCALS [pr] (#17868) --- extra/hcq2/ops_amd2.py | 2 +- test/backend/test_linearizer_dumb.py | 2 +- tinygrad/codegen/__init__.py | 4 ++-- tinygrad/codegen/gpudims.py | 20 ++++++---------- tinygrad/codegen/opt/__init__.py | 2 +- tinygrad/codegen/opt/heuristic.py | 8 +++---- tinygrad/codegen/opt/postrange.py | 18 +++----------- tinygrad/codegen/opt/search.py | 1 - tinygrad/engine/jit.py | 5 ++-- tinygrad/engine/realize.py | 36 ++-------------------------- tinygrad/helpers.py | 2 +- tinygrad/renderer/cstyle.py | 10 ++++---- tinygrad/renderer/nir.py | 5 +--- tinygrad/runtime/graph/hcq.py | 2 +- tinygrad/runtime/ops_cl.py | 6 ++--- tinygrad/uop/ops.py | 21 +++++++--------- 16 files changed, 40 insertions(+), 104 deletions(-) diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index 8de14a207333b..a6f47142ad552 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -139,7 +139,7 @@ def pm4_program(ctx, call, prg): ins += [wreg(ctx, ctx.gc.regCOMPUTE_RESTART_X, 0, 0, 0), wreg(ctx, ctx.gc.regCOMPUTE_USER_DATA_0, *user_regs), wreg(ctx, ctx.gc.regCOMPUTE_RESOURCE_LIMITS, ctx.gc.regCOMPUTE_RESOURCE_LIMITS.encode(waves_per_sh=getenv("WAVES_PER_SH"))), - wreg(ctx, ctx.gc.regCOMPUTE_START_X, 0, 0, 0, *(info.local_size or (1, 1, 1)), 0, 0), + wreg(ctx, ctx.gc.regCOMPUTE_START_X, 0, 0, 0, *info.local_size, 0, 0), pkt3(ctx, PM4Ops.DISPATCH_DIRECT, *info.global_size, dispatch_init), pkt3(ctx, PM4Ops.EVENT_WRITE, ctx.pm4.EVENT_TYPE(ctx.soc.CS_PARTIAL_FLUSH) | ctx.pm4.EVENT_INDEX(EVENT_INDEX_PARTIAL_FLUSH))] return UOp(Ops.LINEAR, src=tuple(ins)) diff --git a/test/backend/test_linearizer_dumb.py b/test/backend/test_linearizer_dumb.py index 364bcc2578ffb..583c3c64b6918 100644 --- a/test/backend/test_linearizer_dumb.py +++ b/test/backend/test_linearizer_dumb.py @@ -24,7 +24,7 @@ def test_failure_beam_mnist(self): c10 = c9.index((((c3*UOp.const(4704000))+c2)+(c6*UOp.const(784))).valid(UOp.const(True))) c11 = c5.alu(Ops.CMPNE, ((((c3*UOp.const(6000))+c6)+((c7*UOp.const(16))+c8)).alu(Ops.CMPLT, UOp.const(59999)).where(UOp.const(0).cast(dtypes.int), UOp.const(1).cast(dtypes.int)).reduce(c7, c8, arg=Ops.ADD)+UOp.const(-1).cast(dtypes.int))).where(UOp.const(0).cast(dtypes.uchar), c10).reduce(c6, arg=Ops.ADD) c12 = c0.index((((c1*UOp.const(7840))+(c2*UOp.const(10)))+c3).valid(UOp.const(True))).store(c11).end(c1, c2, c3) - ast = c12.sink(arg=KernelInfo(name='test', axis_types=(), dont_use_locals=False, applied_opts=(Opt(op=OptOps.GROUP, axis=1, arg=16),), opts_to_apply=None)) + ast = c12.sink(arg=KernelInfo(name='test', applied_opts=(Opt(op=OptOps.GROUP, axis=1, arg=16),), opts_to_apply=None)) _ = to_program(ast, Device["METAL"].renderer) if __name__ == '__main__': diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index d555254dca3a1..141787c587228 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -1,6 +1,6 @@ from dataclasses import replace, dataclass import itertools, functools -from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC +from tinygrad.helpers import DISABLE_FAST_IDIV, TRANSCENDENTAL, SPEC, DEBUG, VIZ, IMAGE, NOOPT, EMULATED_DTYPES, USE_TC from tinygrad.helpers import ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT, TracingKey, Context, panic from tinygrad.uop.ops import PatternMatcher, graph_rewrite, UOp, Ops, UPat, rewrite_group, KernelInfo, ProgramInfo, GroupOp, AxisType from tinygrad.uop.weak import pm_lower_weak, pm_commit_weak, pm_cast_const @@ -496,7 +496,7 @@ def do_to_program(ast:UOp, renderer:Renderer) -> UOp: return prg # config affects generated programs and cache keys; context also carries compile-only behavior to workers -to_program_config = (NOOPT, EMULATED_DTYPES, NOLOCALS, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, +to_program_config = (NOOPT, EMULATED_DTYPES, USE_TC, IMAGE, DISABLE_FAST_IDIV, TRANSCENDENTAL, ALLOW_TF32, DEFAULT_FLOAT, DEFAULT_INT, TC_SELECT, TC_OPT) to_program_context = (*to_program_config, SPEC, DEBUG) def to_program_key(ast:UOp, renderer:Renderer) -> tuple: diff --git a/tinygrad/codegen/gpudims.py b/tinygrad/codegen/gpudims.py index 849061f4b0dce..f3a403b48e778 100644 --- a/tinygrad/codegen/gpudims.py +++ b/tinygrad/codegen/gpudims.py @@ -1,5 +1,5 @@ import math -from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, KernelInfo, ssimplify, AxisType +from tinygrad.uop.ops import UOp, Ops, sint, PatternMatcher, UPat, ssimplify, AxisType from tinygrad.dtype import AddrSpace from tinygrad.renderer import Renderer @@ -55,18 +55,12 @@ def add_gpudims(ctx:Renderer, s:UOp): global_shape = tuple(ssimplify(all_ranges[r].src[0]) for r in global_dims) local_shape = tuple(ssimplify(all_ranges[r].src[0]) for r in local_dims) - # get the idxs - ki: KernelInfo = s.arg - if ki.dont_use_locals: - assert not local_dims, "can't use locals if there's no local dims" - idxs = get_grouped_dims("idx", global_shape, ctx.global_max, reverse=True) - else: - # define indexes for GPU-like execution - local_idxs = get_grouped_dims("lidx", local_shape, ctx.local_max) - hw_local = [_dim_max(u.src[0]) for u in local_idxs if u.op is Ops.SPECIAL] - global_max = ctx.global_max if ctx.global_prod_max is None else \ - tuple(min(gm, pm//l) for gm,pm,l in zip(ctx.global_max or ctx.global_prod_max, ctx.global_prod_max, hw_local+[1]*3)) - idxs = get_grouped_dims("gidx", global_shape, global_max, reverse=True) + local_idxs + # define indexes for GPU-like execution + local_idxs = get_grouped_dims("lidx", local_shape, ctx.local_max) + hw_local = [_dim_max(u.src[0]) for u in local_idxs if u.op is Ops.SPECIAL] + global_max = ctx.global_max if ctx.global_prod_max is None else \ + tuple(min(gm, pm//l) for gm,pm,l in zip(ctx.global_max or ctx.global_prod_max, ctx.global_prod_max, hw_local+[1]*3)) + idxs = get_grouped_dims("gidx", global_shape, global_max, reverse=True) + local_idxs # apply to multiple ranges subs = {} diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index 0618cb4d27678..d1048b6cb09d9 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -5,7 +5,7 @@ class OptOps(Enum): TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto() # noqa: E702 - GROUP = auto(); GROUPTOP = auto(); NOLOCALS = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 + GROUP = auto(); GROUPTOP = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value @dataclass(frozen=True, order=True) diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index c0e70443cb906..66f506b3c8940 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -1,6 +1,6 @@ import itertools from tinygrad.codegen.opt import Opt, OptOps, KernelOptError -from tinygrad.helpers import getenv, DEBUG, prod, NOLOCALS, TC_OPT, TC_SELECT, USE_TC, IMAGE +from tinygrad.helpers import getenv, DEBUG, prod, TC_OPT, TC_SELECT, USE_TC, IMAGE from tinygrad.uop.ops import Ops, resolve, AxisType from tinygrad.codegen.late.coalesce import image_valid_dims from tinygrad.codegen.opt.postrange import Scheduler @@ -78,7 +78,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: return k # are we grouping? (requires local shape support) - if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if NOLOCALS or k.ren.target.device == "QCOM" else 2048), False): + if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if k.ren.target.device == "QCOM" else 2048), False): for axis, sz in itertools.product((0, 1, 2), (16,)): try: k.apply_opt(Opt(OptOps.GROUPTOP, axis, sz)) @@ -160,9 +160,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: # **** local groups **** if k.ren.has_local: - if NOLOCALS: - k.apply_opt(Opt(OptOps.NOLOCALS)) - elif k.ren.target.device == "QCOM": + if k.ren.target.device == "QCOM": # for openpilot: use 32..128 threads per workgroup, at most 8 on the innermost axis # apply innermost global axes first so the leading hardware local dims hold the trailing global axes, like gidx workgroup = 1 diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index f15cad181c101..5afdde2b2df54 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -14,7 +14,6 @@ class Scheduler: def __init__(self, ast:UOp, ren:Renderer): self.ast, self.ren = ast, ren - self.dont_use_locals = self.ast.arg.dont_use_locals if self.ast.arg is not None else False self.applied_opts = list(self.ast.arg.applied_opts) if self.ast.arg is not None else [] self.opt_range = count(start=max([x.arg[0] for x in self.rngs], default=0)+1) @@ -42,7 +41,6 @@ def shape_str_to_axis(self, nms:list[str]) -> tuple[int, ...]: return tuple([sel def copy(self) -> Scheduler: ret = Scheduler(self.ast, self.ren) - ret.dont_use_locals = self.dont_use_locals ret.applied_opts = self.applied_opts[:] if hasattr(self, 'tensor_core'): ret.tensor_core = self.tensor_core return ret @@ -55,7 +53,7 @@ def get_optimized_ast(self, name_override:str|None=None) -> UOp: special_ops = [colored(str(x.vmax+1), "blue" if x.arg[0] == "g" else "cyan") for x in special_uops] name = k_type + colored('_', 'BLACK').join(['']+special_ops+[colored(x.src[0].render(), color) for x,color in zip(self.rngs, self.colors())]) self.ast = graph_rewrite(self.ast, pm_flatten_range, name="flatten range") - return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts), dont_use_locals=self.dont_use_locals), tag=1) + return self.ast.replace(arg=KernelInfo(name=name, applied_opts=tuple(self.applied_opts)), tag=1) def _output_rngs(self) -> list[UOp]: return flatten([[r for r in UOp.sink(*s.src[1:]).ranges if r.arg[-1] != AxisType.REDUCE] for s in self.ast.src if s.op is Ops.END]) @@ -80,8 +78,7 @@ def colors(self) -> list[str]: globalizible_rngs = self._globalizable_rngs() ret = [] for x,r in zip(self.axis_types, self.rngs): - if self.dont_use_locals and x == AxisType.GLOBAL: ret.append("BLUE") - elif r not in output_rngs and x == AxisType.WEAK: ret.append("BLACK") + if r not in output_rngs and x == AxisType.WEAK: ret.append("BLACK") elif r not in globalizible_rngs and x == AxisType.WEAK: ret.append("white") else: ret.append(axis_colors[x]) return ret @@ -119,12 +116,6 @@ def real_axis(self, op:OptOps, axis:int|None) -> int: except IndexError as e: raise KernelOptError from e def apply_opt(self, opt:Opt, append_opt:bool=True): - if opt.op is OptOps.NOLOCALS: - check(all(x not in {AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE} for x in self.axis_types), "no locals can't have locals") - if append_opt: self.applied_opts.append(opt) - self.dont_use_locals = True - return - if opt.op in {OptOps.LOCAL, OptOps.GROUP, OptOps.GROUPTOP}: check(self.ren.has_local, "locals needed for opt") @@ -140,8 +131,7 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): amt:int = int(rng.vmax+1) if opt.arg == 0 else cast(int, opt.arg) # copied from kernel.py. prevents METAL compiler hangs - if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP} or \ - (self.group_for_reduces and opt.op not in {OptOps.NOLOCALS, OptOps.PADTO})): + if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP} or (self.group_for_reduces and opt.op != OptOps.PADTO)): upcast_local_sz = prod([self.full_shape[a] for a in self.axes_of(AxisType.UPCAST, AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) smem_sz = amt*upcast_local_sz*self.reduceop.dtype.itemsize check(smem_sz <= self.ren.shared_max, f"exceeds maximum shared memory size: needs {smem_sz}, max {self.ren.shared_max}") @@ -158,11 +148,9 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.LOCAL, AxisType.WEAK}, f"upcast is for GLOBAL/LOCAL/LOOP, not {rng.arg[-1]}") if opt.op is OptOps.LOCAL: - check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? - check(not self.dont_use_locals, "can't use locals") check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op is OptOps.GROUPTOP) elif opt.op is OptOps.TC: diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index 9d3d7d375340d..e4d09b6cd02ca 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -22,7 +22,6 @@ # covers resnet kernels (3 global * 3 reduce) actions += [Opt(op=OptOps.TC, axis=axis, arg=(-1, getenv("TC_OPT", 2), getenv("TC", 1))) for axis in range(9)] actions += [Opt(op=OptOps.SWAP, axis=axis_0, arg=axis_1) for axis_0 in range(5) for axis_1 in range(axis_0+1, 5)] -if getenv("NOLOCALS"): actions += [Opt(op=OptOps.NOLOCALS)] def get_test_global_size(global_size, max_global_size, var_vals): test_global_size = [sym_infer(sz, var_vals) for sz in global_size] diff --git a/tinygrad/engine/jit.py b/tinygrad/engine/jit.py index cbb32b62cc238..1ad4c5e7929b1 100644 --- a/tinygrad/engine/jit.py +++ b/tinygrad/engine/jit.py @@ -107,9 +107,9 @@ def is_sym_dim(dim) -> bool: return not all(isinstance(d, (int, float)) for d in crs = [(j, self.calls[j][1].arg, self.calls[j][3]) for j in range(len(self.calls)) if self.calls[j][1].op is Ops.PROGRAM] self.vars = sorted({v.expr for _,p,dv in crs for v in p.vars if v.expr not in dv}) - self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if d and is_sym_dim(d)) + self.symbolic_dims = dedup(tuple(d) for _,p,_ in crs for d in (p.local_size, p.global_size) if is_sym_dim(d)) - def find_symbolic_dim(dim): return self.symbolic_dims.index(tuple(dim)) if dim is not None and tuple(dim) in self.symbolic_dims else None + def find_symbolic_dim(dim:tuple[int,int,int]): return self.symbolic_dims.index(tuple(dim)) if tuple(dim) in self.symbolic_dims else None for j,p,dv in crs: if (replace:=[(i, self.vars.index(v.expr)) for i, v in enumerate(p.vars) if v.expr not in dv]): @@ -117,7 +117,6 @@ def find_symbolic_dim(dim): return self.symbolic_dims.index(tuple(dim)) if dim i global_dim_idx, local_dim_idx = find_symbolic_dim(p.global_size), find_symbolic_dim(p.local_size) if global_dim_idx is not None or local_dim_idx is not None: self.launch_dims_replace[j] = (global_dim_idx, local_dim_idx) - assert p.local_size is not None self.launch_dims_base[j] = (tuple(p.global_size), tuple(p.local_size)) estimates = sum((estimate_uop(call) for call in self.linear.src), Estimates()) diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index e825bcb7f24e8..062ad32fcb725 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -1,14 +1,13 @@ from __future__ import annotations from typing import cast, Iterator, Any, Sequence -import random, itertools, math, weakref, array, decimal +import weakref, array, decimal from dataclasses import dataclass, replace, field -from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, all_int, prod, flatten, Context, to_tuple, tqdm, dedup, to_mv +from tinygrad.helpers import colored, DEBUG, GlobalCounters, ansipad, prod, flatten, Context, to_tuple, tqdm, dedup, to_mv from tinygrad.helpers import BEAM, size_to_str, time_to_str, VALIDATE_WITH_CPU, HCQ2, PROFILE, ProfilePointEvent, cpu_events, perf_counter_us from tinygrad.uop.ops import Ops, PatternMatcher, UOp, UPat, AxisType, sym_infer, graph_rewrite, ProgramInfo from tinygrad.device import Device, Buffer, MultiBuffer, ProfileGraphEntry from tinygrad.renderer import Estimates, Renderer from tinygrad.codegen import to_program, to_program_cache, to_program_key, to_program_context -from tinygrad.codegen.opt.postrange import args_from_ast from tinygrad.engine.worker import get_worker_pool, terminate_worker_pool # **************** Helpers **************** @@ -99,32 +98,6 @@ def track_stats(ctx:ExecContext, call:UOp, st:decimal.Decimal, ets:list[float|No ("" if et is None else f" tm {ptm}/{GlobalCounters.time_sum_s*1e3:9.2f}ms ({flops_str} {mem_str})")) first_run_cache.add(key) -local_size_cache: dict[bytes, tuple[int, ...]] = {} -def optimize_local_size(call:UOp, prg:UOp) -> UOp|None: - device = to_tuple(prg.device)[0] - if prg.arg.local_size is not None or not Device[device].renderer.has_local or not all_int(prg.arg.global_size): return None - - if (local_size:=local_size_cache.get(prg.key)) is None: - # reuse one loaded runtime across candidates, only launch dims vary - (bufs, var_vals), runtime = args_from_ast(prg.src[0], device), get_runtime(device, prg, cache=False) - bufs = [b.allocate() for b in bufs] - def try_exec(local_size): - try: - new_gs = tuple(g//l if g%l == 0 else g/l for g,l in zip(prg.arg.global_size, local_size)) - return runtime(*[bufs[i].get_buf(device) for i in prg.arg.globals], global_size=new_gs, local_size=(*local_size,), - vals=prg.arg.vals(var_vals), wait=True) - except Exception: return float('inf') - - MAX_WORKGROUP = 1024 - local_dims = [[x for x in set([sz, 1, 2, 4, 8, 16, 32, 64, 128, 256, MAX_WORKGROUP]) if x<=sz] for sz in prg.arg.global_size] - local_sizes = [list(x) for x in itertools.product(*local_dims) if prod(x) <= MAX_WORKGROUP] * 2 # try each valid size twice - best_time, best = min([(try_exec(ls), ls) for ls in random.sample(local_sizes, len(local_sizes))]) - assert not math.isinf(best_time), "all optimize_local_size exec failed" - local_size = local_size_cache[prg.key] = tuple(best) - - new_global = tuple(g//l if g%l == 0 else g/l for g,l in zip(prg.arg.global_size, local_size)) - return call.replace(src=(prg.replace(arg=replace(prg.arg, global_size=new_global, local_size=local_size)), *call.src[1:])) - # **************** runtime cache **************** runtime_cache: dict[tuple[bytes, str], Any] = {} @@ -296,10 +269,6 @@ def lower_and_compile(linear:UOp) -> UOp: return linear.substitute({c: c.replace(src=(c.src[0].substitute({a[0]: to_program_cache[keys[c]]}), *c.src[1:])) for c, a in ar.items()}, name="precompile kernels") -pm_optimize_local_size = PatternMatcher([ - (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="prg"),), name="call", allow_any_len=True), optimize_local_size), -]) - pm_exec = PatternMatcher([ (UPat(Ops.CALL, src=(UPat(Ops.COPY, name="ast"),), name="call", allow_any_len=True), exec_copy), (UPat(Ops.CALL, src=(UPat(Ops.PROGRAM, name="ast"),), name="call", allow_any_len=True), exec_kernel), @@ -315,7 +284,6 @@ def compile_linear(linear:UOp, beam:int|None=None, validate=False, input_uops:li if validate: linear = graph_rewrite(linear, pm_validate, name="validate", walk=True) if (beam_val:=BEAM.value if beam is None else beam) >= 1: linear = graph_rewrite(linear, pm_beam, ctx=beam_val, walk=True) linear = lower_and_compile(linear) - linear = graph_rewrite(linear, pm_optimize_local_size, name="optimize local size", walk=True) if HCQ2: linear = hcq_compile(linear, input_uops, bool(PROFILE or DEBUG >= 2) if profile is None else profile) return linear diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index a318d5cf8c76f..94198c3f6a160 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -236,7 +236,7 @@ def target(self, dev:str, **kwargs) -> Target: WINO, CAPTURING, TRACEMETA, NO_COLOR = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("TRACEMETA", 1), ContextVar("NO_COLOR", 0) TRAINING = ContextVar("TRAINING", 0) USE_TC, TC_SELECT, TC_OPT = ContextVar("TC", 1), ContextVar("TC_SELECT", -1), ContextVar("TC_OPT", 0) -TRANSCENDENTAL, NOLOCALS = ContextVar("TRANSCENDENTAL", 1), ContextVar("NOLOCALS", 0) +TRANSCENDENTAL = ContextVar("TRANSCENDENTAL", 1) SPLIT_REDUCEOP, NO_MEMORY_PLANNER, LRU = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("LRU", 1) RING, ALL2ALL, ALLREDUCE_CAST = ContextVar("RING", 1), ContextVar("ALL2ALL", 0), ContextVar("ALLREDUCE_CAST", 1) CACHELEVEL, IGNORE_BEAM_CACHE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index c62a69d14a19e..9c0d4019b5206 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -128,7 +128,7 @@ class CStyleLanguage(Renderer): var_prefix: str = "const " var_suffix: str = "" barrier: str = "" - code_for_workitem: dict[Literal["g", "l", "i"], Callable] = {} + code_for_workitem: dict[Literal["g", "l"], Callable] = {} extra_args: list[str] = [] float4: str|None = None float4_style: tuple[str, str] = ('(', ')') @@ -316,7 +316,7 @@ class OpenCLRenderer(CStyleLanguage): smem_prefix = "__local " barrier = "barrier(CLK_LOCAL_MEM_FENCE);" float4 = "(float4)" - code_for_workitem = {"g": lambda x: f"get_group_id({x})", "l": lambda x: f"get_local_id({x})", "i": lambda x: f"get_global_id({x})"} + code_for_workitem = {"g": lambda x: f"get_group_id({x})", "l": lambda x: f"get_local_id({x})"} type_map = { dtypes.int8: "char", dtypes.uint8: "uchar", dtypes.uint32: "uint", dtypes.uint16: "ushort", dtypes.uint64: "ulong", dtypes.bfloat16: "ushort" } extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast @@ -417,8 +417,7 @@ def __init__(self, target:Target, use_nvcc=False): barrier = "__syncthreads();" float4 = "make_float4" gep_arr_threshold = 8 - code_for_workitem = {"g": lambda x: f"blockIdx.{chr(120+int(x))}", "l": lambda x: f"threadIdx.{chr(120+int(x))}", - "i": lambda x: f"(blockIdx.{chr(120+int(x))}*blockDim.{chr(120+int(x))}+threadIdx.{chr(120+int(x))})"} + code_for_workitem = {"g": lambda x: f"blockIdx.{chr(120+int(x))}", "l": lambda x: f"threadIdx.{chr(120+int(x))}"} code_for_op = { **CStyleLanguage.code_for_op, Ops.TRUNC: lambda x,dtype: f"htrunc({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"trunc({x})", Ops.SIN: lambda x,dtype: f"hsin({x})" if dtype in (dtypes.half, dtypes.bfloat16) else f"sin({x})", @@ -516,8 +515,7 @@ def __init__(self, target:Target, use_hipcc=False): # gfx942 => MI300, gfx1100 = # https://clang.llvm.org/docs/AttributeReference.html#amdgpu-flat-work-group-size # NOTE: this makes hlb_cifar10 twice as fast, there may be more gains in tweaking these parameters kernel_typedef = 'extern "C" __attribute__((global)) void __attribute__((amdgpu_flat_work_group_size(1, {launch_bounds})))' - code_for_workitem = {"g": lambda x: f"__ockl_get_group_id({x})", "l": lambda x: f"__ockl_get_local_id({x})", - "i": lambda x: f"(__ockl_get_group_id({x})*__ockl_get_local_size({x})+__ockl_get_local_id({x}))"} + code_for_workitem = {"g": lambda x: f"__ockl_get_group_id({x})", "l": lambda x: f"__ockl_get_local_id({x})"} code_for_op = {**CStyleLanguage.code_for_op, Ops.TRUNC: _ocml("trunc"), Ops.SIN: _ocml("sin"), Ops.LOG2: _ocml("log2"), Ops.EXP2: _ocml("exp2"), Ops.SQRT: _ocml("sqrt")} smem_prefix = "__attribute__((shared, aligned(16)))" diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index 3dcc4ebf8a62d..2917da319d538 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -92,8 +92,6 @@ def scope(space): return 'global' if space == AddrSpace.GLOBAL else ('shared' if ngid = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_workgroup_id)) nlid = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_local_invocation_id)) -ngsz = nir_instr(nc=3, bs=32)(lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_load_workgroup_size)) -def nid(b): return nalu(b, "iadd", nalu(b, "imul", ngid(b), ngsz(b)), nlid(b)) nbarrier = nir_instr(has_def=False, intrins={"EXECUTION_SCOPE":mesa.SCOPE_WORKGROUP})( lambda b: mesa.nir_intrinsic_instr_create(b.shader, mesa.nir_intrinsic_barrier)) @@ -147,7 +145,7 @@ class NIRRenderer(Renderer): def_rewrite = PatternMatcher([ (UPat.cvar("c").cast(name="x"), lambda ctx,x,c: nimm(ctx.b, c.val, x.dtype)), (UPat(Ops.PARAM, name="x"), lambda ctx,x: ctx.param(ctx.b, x, x.dtype.itemsize if x.addrspace is AddrSpace.ALU else 8)), - (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: nchannel(ctx.b, {'g':ngid, 'l':nlid, 'i': nid}[x.arg[0]](ctx.b), int(x.arg[-1]))), + (UPat(Ops.SPECIAL, name="x"), lambda ctx,x: nchannel(ctx.b, {'g':ngid, 'l':nlid}[x.arg[0]](ctx.b), int(x.arg[-1]))), (UPat(Ops.STORE, src=(UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"),UPat.var("off")), allow_any_len=True), UPat.var("val"))), lambda ctx,buf,off,val: nstore(ctx.b, buf.addrspace, nidx(ctx.b, ctx.r[buf], ctx.r[off], buf.addrspace, buf.dtype.itemsize), ctx.r[val])), (UPat(Ops.LOAD, src=(UPat((Ops.INDEX, Ops.SHRINK), src=(UPat.var("buf"), UPat.var("off")), allow_any_len=True), UPat.var("alt"), @@ -181,7 +179,6 @@ def __del__(self): def param(self, b:mesa.nir_builder, x, sz:int) -> mesa.nir_def: raise NotImplementedError("needs param") def prerender(self, uops:list[UOp]): self.b = mesa.nir_builder_init_simple_shader(mesa.MESA_SHADER_COMPUTE, mesa.nir_shader_compiler_options.from_buffer_copy(self.nir_options), None) - self.b.shader.contents.info.workgroup_size_variable = any([u.op == Ops.SPECIAL and u.arg[0] == 'i' for u in uops]) def postrender(self, uops:list[UOp]): pass def render(self, uops:list[UOp]): diff --git a/tinygrad/runtime/graph/hcq.py b/tinygrad/runtime/graph/hcq.py index 3ce29acde6936..d8aecce5a06fd 100644 --- a/tinygrad/runtime/graph/hcq.py +++ b/tinygrad/runtime/graph/hcq.py @@ -172,7 +172,7 @@ def __init__(self, *args, **kwargs): # Encode main commands based on ji type. if runtime is not None: - enqueue_queue.exec(runtime, self.ji_args[j], ast.arg.global_size or (1,1,1), ast.arg.local_size or (1,1,1)) + enqueue_queue.exec(runtime, self.ji_args[j], ast.arg.global_size, ast.arg.local_size) elif j in self.rdma_deps: dest_queue, dest_deps, dest_out_signal, dest_out_val = self.rdma_deps[j] for sig, val in dest_deps: dest_queue.wait(sig, val) diff --git a/tinygrad/runtime/ops_cl.py b/tinygrad/runtime/ops_cl.py index 33180f2c5c84e..061f1eee2dafb 100644 --- a/tinygrad/runtime/ops_cl.py +++ b/tinygrad/runtime/ops_cl.py @@ -52,7 +52,7 @@ def __del__(self): try: check(cl.clReleaseProgram(self.program)) except (TypeError, AttributeError): pass - def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]|None=None, vals:tuple[int, ...]=(), + def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False, **kw) -> float|None: for i, (_, slot, dt, shape) in enumerate(self.signature): b = bufs[slot] if slot < len(bufs) else getattr(ctypes, f"c_int{dt.bitsize}")(vals[slot-len(bufs)]) @@ -63,10 +63,10 @@ def __call__(self, *bufs:cl.cl_mem, global_size:tuple[int,int,int]=(1,1,1), loca img = checked(cl.clCreateImage(self.dev.context, cl.CL_MEM_READ_WRITE, fmt, desc, None, status:=ctypes.c_int32()), status) check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(img), ctypes.byref(img))) else: check(cl.clSetKernelArg(self.kernel, i, ctypes.sizeof(b), ctypes.byref(b))) - if local_size is not None: global_size = cast(tuple[int,int,int], tuple(int(g*l) for g,l in zip(global_size, local_size))) + global_size = cast(tuple[int,int,int], tuple(int(g*l) for g,l in zip(global_size, local_size))) event = cl.cl_event() if wait else None check(cl.clEnqueueNDRangeKernel(self.dev.queue, self.kernel, len(global_size), None, (ctypes.c_size_t * len(global_size))(*global_size), - (ctypes.c_size_t * len(local_size))(*local_size) if local_size else None, 0, None, event)) + (ctypes.c_size_t * len(local_size))(*local_size), 0, None, event)) if wait: assert event is not None check(cl.clWaitForEvents(1, event)) diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 486f1dbf37021..2de5c4151db49 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1245,8 +1245,6 @@ def to_elf(self) -> TinyELF: @dataclass(frozen=True) class KernelInfo: name: str = "test" # name of the kernel - axis_types: tuple[AxisType, ...] = tuple() - dont_use_locals: bool = False # don't use local indexing applied_opts: tuple = tuple() opts_to_apply: tuple|None = None estimates: Estimates|None = None @@ -1258,7 +1256,7 @@ def function_name(self): return to_function_name(self.name) class ProgramInfo: name: str = "test" global_size: tuple[int|float, ...] = (1, 1, 1) - local_size: tuple[int, ...]|None = None + local_size: tuple[int, ...] = (1, 1, 1) vars: tuple[UOp, ...] = () globals: tuple[int, ...] = () outs: tuple[int, ...] = () @@ -1268,9 +1266,9 @@ class ProgramInfo: @property def function_name(self): return to_function_name(self.name) - def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]|None]: + def launch_dims(self, var_vals:dict[str, int]) -> tuple[tuple[int, ...], tuple[int, ...]]: global_size = tuple([sym_infer(sz, var_vals) for sz in self.global_size]) # type: ignore[arg-type] - local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) if self.local_size is not None else None + local_size = tuple([sym_infer(sz, var_vals) for sz in self.local_size]) return global_size, local_size def vals(self, var_vals:dict[str, int]) -> tuple[int, ...]: @@ -1284,20 +1282,17 @@ def from_sink(sink:UOp, target:Target=Target()) -> ProgramInfo: outs: list[int] = [] ins: list[int] = [] global_size: list[int] = [1, 1, 1] - local_size: list[int]|None = [1, 1, 1] + local_size: list[int] = [1, 1, 1] for u in sink.toposort(): if u.op is Ops.PARAM and u.addrspace == AddrSpace.ALU: _vars.append(u) if u.op is Ops.PARAM and u.addrspace != AddrSpace.ALU: _globals.append(u.arg.slot) if u.op in (Ops.STORE, Ops.LOAD): if (idx:=u.src[0]).op in (Ops.INDEX, Ops.SHRINK) or (u.src[0].op is Ops.CAST and (idx:=u.src[0].src[0]).op is Ops.INDEX): if (buf:=idx.src[0].buf_uop).op is Ops.PARAM: (outs if u.op is Ops.STORE else ins).append(buf.arg.slot) - if u.op is Ops.SPECIAL: - if u.arg[0] == 'i': local_size = None - special_size = local_size if u.arg[0] == 'l' else global_size - if special_size is not None: special_size[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) - return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), - tuple(local_size) if local_size is not None else None, tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), - tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), tuple(sorted(dedup(ins))), target) + if u.op is Ops.SPECIAL: (local_size if u.arg[0] == 'l' else global_size)[int(u.arg[-1])] = cast(int, u.src[0].ssimplify()) + return ProgramInfo(sink.arg.name if isinstance(sink.arg, KernelInfo) else "test", tuple(global_size), tuple(local_size), + tuple(sorted(dedup(_vars), key=lambda v: v.arg.slot)), tuple(sorted(dedup(_globals))), tuple(sorted(dedup(outs))), + tuple(sorted(dedup(ins))), target) @dataclass(frozen=True) class CallInfo: From c12eefe447db188b3cc2ccd57946175e91ed4796 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 31 Aug 2026 20:16:32 -0400 Subject: [PATCH 302/306] run llvmspeed Speed Test (BEAM=2) without cache (#17869) * run llvmspeed Speed Test (BEAM=2) without cache * HEVC too --- .github/workflows/benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2a9ff721725e6..8a96ea94502ea 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -403,7 +403,7 @@ jobs: run: python test/external/external_benchmark_multitensor_allreduce.py - name: HEVC Decode Benchmark if: ${{ matrix.dev == 'NV' }} - run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py + run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py - uses: actions/upload-artifact@v7 if: ${{ matrix.dev != 'AMD' }} with: @@ -619,7 +619,7 @@ jobs: GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit - name: HEVC Decode Benchmark if: ${{ matrix.dev == 'NV' }} - run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py + run: IGNORE_BEAM_CACHE=1 VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py - name: Run 10 MLPerf ResNet50 training steps (1 gpu) if: ${{ matrix.dev == 'NV' }} run: BENCHMARK_LOG=resnet_10steps MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py @@ -642,7 +642,7 @@ jobs: llvmspeed: name: LLVM Speed runs-on: [self-hosted, Linux, tinyboxrandom] - timeout-minutes: 5 + timeout-minutes: 10 if: github.repository_owner == 'tinygrad' steps: - name: Checkout Code @@ -650,4 +650,4 @@ jobs: - name: Speed Test run: DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py - name: Speed Test (BEAM=2) - run: BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py + run: IGNORE_BEAM_CACHE=1 BEAM=2 DEV=CPU:LLVM python3 test/speed/external_test_speed_v_torch.py From 37738a7276597a25cedcd65e0c2dce68ccafd1a4 Mon Sep 17 00:00:00 2001 From: YassineYousfi Date: Mon, 31 Aug 2026 18:39:31 -0700 Subject: [PATCH 303/306] onnx: add tinygrad contiguous custom op (#17871) Co-authored-by: Christopher Milan --- test/external/external_test_onnx_runner.py | 23 +++++++++++++++++++--- tinygrad/nn/onnx.py | 5 +++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/test/external/external_test_onnx_runner.py b/test/external/external_test_onnx_runner.py index 193a30b882500..96b73505a8cfa 100644 --- a/test/external/external_test_onnx_runner.py +++ b/test/external/external_test_onnx_runner.py @@ -13,10 +13,10 @@ def _check_ast_count(desired_count:int, t:Tensor): asts = [call for call in linear.src if call.src[0].op is Ops.SINK] assert len(asts) == desired_count, f"{len(asts)} != {desired_count}" -def build_onnx(nodes, from_disk:bool=True, **kwargs): +def build_onnx(nodes, from_disk:bool=True, opset_imports=None, **kwargs): """Helper to build and return an OnnxRunner from ONNX nodes.""" graph = onnx.helper.make_graph(nodes, 'test', kwargs.get('inputs', []), kwargs.get('outputs', []), kwargs.get('initializers', [])) - model = onnx.helper.make_model(graph) + model = onnx.helper.make_model(graph) if opset_imports is None else onnx.helper.make_model(graph, opset_imports=opset_imports) if from_disk: with tempfile.TemporaryDirectory() as tmpdir: tmp_path = pathlib.Path(tmpdir) @@ -29,6 +29,23 @@ def build_onnx(nodes, from_disk:bool=True, **kwargs): return runner class TestOnnxRunner(unittest.TestCase): + def test_tinygrad_contiguous(self): + runner = build_onnx( + nodes=[ + onnx.helper.make_node('Add', ['inp', 'one'], ['added']), + onnx.helper.make_node('Contiguous', ['added'], ['materialized'], domain='org.tinygrad'), + onnx.helper.make_node('Mul', ['materialized', 'two'], ['output']) + ], + inputs=[onnx.helper.make_tensor_value_info('inp', onnx.TensorProto.FLOAT, (4,))], + outputs=[onnx.helper.make_tensor_value_info('output', onnx.TensorProto.FLOAT, (4,))], + initializers=[ + onnx.helper.make_tensor('one', onnx.TensorProto.FLOAT, (), [1.0]), + onnx.helper.make_tensor('two', onnx.TensorProto.FLOAT, (), [2.0]) + ], + opset_imports=[onnx.helper.make_opsetid('', 13), onnx.helper.make_opsetid('org.tinygrad', 1)], + from_disk=False).to('PYTHON') + _check_ast_count(2, runner({'inp': Tensor.empty(4, device='PYTHON')})['output']) + def _test_const_fold_unary_op(self, from_disk:bool): runner = build_onnx( nodes=[ @@ -162,4 +179,4 @@ def test_metadata_props(self): self.assertEqual(parsed["metadata_props"][1]["value"], "dGVzdA==") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tinygrad/nn/onnx.py b/tinygrad/nn/onnx.py index 8050142b8f497..898dd2f6850a3 100644 --- a/tinygrad/nn/onnx.py +++ b/tinygrad/nn/onnx.py @@ -46,6 +46,7 @@ class Domain(enum.Enum): MICROSOFT_NCHWC = "com.microsoft.nchwc" MICROSOFT_EXPERIMENTAL = "com.microsoft.experimental" PYTORCH_ATEN = "org.pytorch.aten" + TINYGRAD = "org.tinygrad" @classmethod def from_onnx(cls, domain: str | None) -> "Domain": return cls.ONNX if domain is None or domain == "" else cls(domain) @@ -537,6 +538,10 @@ def ___wrapper(R:Tensor, T:int, *inputs:Tensor, **kwargs): return ___wrapper return __decorator + # ***** Tinygrad Custom Ops ***** + def contiguous_1(x:Tensor): return x.contiguous() + Contiguous = {OpSetId(Domain.TINYGRAD, 1):contiguous_1} + # ***** Property/Graph Ops ***** def If(condition:Tensor, else_branch:OnnxRunner, then_branch:OnnxRunner, intermediate_tensors:dict[str, Tensor]): def run_branch(branch:OnnxRunner): From a3bde519d69097c5e33dba9b3c7cc635c56608d8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 31 Aug 2026 21:50:05 -0400 Subject: [PATCH 304/306] merge OptOps.UNROLL and OptOps.UPCAST (#17867) * merge OptOps.UNROLL and OptOps.UPCAST * why are convs so slow? --- docs/developer/speed.md | 2 +- extra/gemm/tinygrad_nv_matmul.py | 2 +- extra/optimization/test_beam_search.py | 4 +- test/backend/test_linearizer.py | 6 +- test/backend/test_quantize_onnx.py | 15 ++--- test/backend/test_rangeify.py | 26 ++++----- test/backend/test_uops.py | 2 +- test/external/external_benchmark_op_conv.py | 1 - test/external/speed_v_theoretical.py | 2 +- test/null/test_linearizer_rewrite.py | 2 +- test/null/test_uops_stats.py | 6 +- test/opt/test_gen_float4.py | 4 +- test/opt/test_kernel_opts.py | 61 ++++++++++----------- test/opt/test_tensor_cores.py | 12 ++-- tinygrad/codegen/opt/__init__.py | 3 +- tinygrad/codegen/opt/heuristic.py | 11 ++-- tinygrad/codegen/opt/postrange.py | 18 +++--- tinygrad/codegen/opt/search.py | 3 +- 18 files changed, 87 insertions(+), 93 deletions(-) diff --git a/docs/developer/speed.md b/docs/developer/speed.md index 460e093b95311..6b6f3cd5b51f5 100644 --- a/docs/developer/speed.md +++ b/docs/developer/speed.md @@ -52,7 +52,7 @@ In `kernel.py` we have a set of `OptOps`, these control the parameters of the sp The main bottleneck in most kernels is accessing memory. In a freshman algorithms class, you'll learn about cache aware matrix multiplication, and this is all forms of that. While the same math is run, the order in which you run it can have large impacts on the speed depending on if the data you are loading. OptOps will change this order. -Memory, even cache, is often much slower than accessing the register file. The amount of times data is used in math is called the "arithmetic intensity". For operations like BS=1 GEMV, the arithmetic intensity is 1, but for GEMMs and convs it can be much higher. OptOps like UPCAST and UNROLL can increase this, but be careful of making them too large, as if there's too much register pressure on the GPU the warp scheduler may not be able to fit many warps, or even worse, it could be spilling to local memory. +Memory, even cache, is often much slower than accessing the register file. The amount of times data is used in math is called the "arithmetic intensity". For operations like BS=1 GEMV, the arithmetic intensity is 1, but for GEMMs and convs it can be much higher. The UPCAST OptOp can increase this, but be careful of making them too large, as if there's too much register pressure on the GPU the warp scheduler may not be able to fit many warps, or even worse, it could be spilling to local memory. 4090s have 1 TB/s of ram bandwidth and ~160 TFLOPS of compute, so you need to use each loaded value ~100 times. The L1 cache has around 40 TB/s of bandwidth, so in order to get full compute utilization you need to use each value ~4 times. diff --git a/extra/gemm/tinygrad_nv_matmul.py b/extra/gemm/tinygrad_nv_matmul.py index adc9a747e8c54..77538f7e742f9 100644 --- a/extra/gemm/tinygrad_nv_matmul.py +++ b/extra/gemm/tinygrad_nv_matmul.py @@ -13,7 +13,7 @@ C = A.matmul(B) if getenv("GEMV"): opts = [ - Opt(op=OptOps.UNROLL, axis=0, amt=8), + Opt(op=OptOps.UPCAST, axis=1, amt=8), Opt(op=OptOps.GROUP, axis=0, amt=32), ] else: diff --git a/extra/optimization/test_beam_search.py b/extra/optimization/test_beam_search.py index 133c779960148..c3527bd45a9ff 100644 --- a/extra/optimization/test_beam_search.py +++ b/extra/optimization/test_beam_search.py @@ -89,7 +89,7 @@ def test_tc_up(self): s.apply_opt(Opt(OptOps.TC, 0, (-1, 0, 1))) up = prod([x for x, t in zip(s.full_shape, s.axis_types) if t in (AxisType.UPCAST, AxisType.UNROLL)]) actions = get_kernel_actions(s, include_0=False, max_up=int(up)) - upcasted = [s for s in actions.values() if any(opt.op in (OptOps.UPCAST, OptOps.UNROLL) for opt in s.applied_opts)] + upcasted = [s for s in actions.values() if any(opt.op is OptOps.UPCAST for opt in s.applied_opts)] assert len(upcasted) > 0, f"expected upcast/unroll actions after TC with max_up={up}, but got none" def test_max_up(self): @@ -98,7 +98,7 @@ def test_max_up(self): s = Scheduler(ast, Device[Device.DEFAULT].renderer) for max_up in (2, 4): actions = get_kernel_actions(s, include_0=False, max_up=max_up) - for up_opts in [s.applied_opts for s in actions.values() if any(opt.op in (OptOps.UPCAST, OptOps.UNROLL) for opt in s.applied_opts)]: + for up_opts in [s.applied_opts for s in actions.values() if any(opt.op is OptOps.UPCAST for opt in s.applied_opts)]: assert len([opt for opt in up_opts if opt.arg > max_up]) == 0 and len([op for op in up_opts if op.arg <= max_up]) > 0 if __name__ == '__main__': diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 2d527140cb60e..6a33edb377910 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -159,7 +159,7 @@ def test_reduce_upcast(self): r = Tensor.conv2d(x,w,padding=1).relu() uops = tuple(to_program(replace_opts(r.schedule_linear().src[-1].src[0], - [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UNROLL, axis=0, arg=0)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UPCAST, axis=1, arg=0)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) accs = [u for u in uops if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG] stores = [u for u in uops if u.op is Ops.STORE] assert len(accs) == 0 # it's removed now @@ -240,7 +240,7 @@ def helper_arg_acc_dtype(c: Tensor, expected_dtype:DType): def test_simple_unroll_no_between_phi_dependencies(self): x, y = Tensor.empty(64, 64), Tensor.empty(64, 64) r = (x@y).relu() - opt = [Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4)] + opt = [Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 0, 4)] ast = helper_linearizer_opt(r, [opt]) # the uops graph is reg BUFFER -> 4x STORE 0.0 -> RANGE -> 4x ALU -> 4x STORE -> ENDRANGE uops = tuple(to_program(replace_opts(ast, opt), renderer=Device[Device.DEFAULT].renderer).src[1].src) @@ -354,7 +354,7 @@ def test_grouped_store_locals_and_globals(self): x, y = Tensor.empty(64, 64), Tensor.empty(64, 64) out = x@y opt = [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), - Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)] # upcast accs in both reduces + Opt(OptOps.UPCAST, 3, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)] # upcast accs in both reduces ast = helper_linearizer_opt(out, opts=[opt]) def get_recursive(uop): return set.union(set(uop.src), [uop], *[get_recursive(v) for v in uop.src]) uops = tuple(to_program(replace_opts(ast, opt), renderer=Device[Device.DEFAULT].renderer).src[1].src) diff --git a/test/backend/test_quantize_onnx.py b/test/backend/test_quantize_onnx.py index 8528caa9c4512..f743ee425b268 100644 --- a/test/backend/test_quantize_onnx.py +++ b/test/backend/test_quantize_onnx.py @@ -98,7 +98,7 @@ def test_prequant_conv2d_1x1(self): X = Tensor(np.random.uniform(0, 255, size=(1, 32, 128, 128)).astype(np.uint8)) W = Tensor(np.random.uniform(0, 255, size=(64, 32, 1, 1)).astype(np.uint8)) out = X.conv2d(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UPCAST, axis=3, arg=4)] sexec(out, opts) def test_prequant_gemm(self): @@ -106,7 +106,7 @@ def test_prequant_gemm(self): X = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(np.uint8)) W = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(np.uint8)) out = X.matmul(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UPCAST, axis=3, arg=4)] sexec(out, opts) # TODO: this has to work @@ -116,7 +116,7 @@ def test_prequant_gemm_intacc_early(self, xi=np.int8, wi=np.int8): W = Tensor(np.random.uniform(0, 255, size=(N,N)).astype(wi)) # this divide is interesting and forces the accumulator to actually be an int out = (X.cast("int").matmul(W.cast("int"))//1000).cast("int8") - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UPCAST, axis=3, arg=4)] sexec(out, opts) def test_prequant_gemm_handcode(self): @@ -200,9 +200,10 @@ def test_prequant_gemm_handcode(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, src) def test_prequant_gemm_intacc_32(self): - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=0), Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UNROLL, axis=0, arg=0)] + opts = [Opt(op=OptOps.UPCAST, axis=1, arg=0), Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=3, arg=0)] self.test_prequant_gemm_intacc(np.uint8, np.int8, N=32, opts=opts) - def test_prequant_gemm_intacc_128(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=128) + def test_prequant_gemm_intacc_128(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=128, + opts=[Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UPCAST, axis=2, arg=4)]) def test_prequant_gemm_intacc_256(self): self.test_prequant_gemm_intacc(np.uint8, np.int8, N=256) def test_prequant_gemm_intacc(self, xi=np.uint8, wi=np.uint8, replace_src=None, N=512, clip=True, opts=None): X = Tensor(m1:=(np.random.uniform(0, 255, size=(N,N)).astype(xi))).realize() @@ -211,7 +212,7 @@ def test_prequant_gemm_intacc(self, xi=np.uint8, wi=np.uint8, replace_src=None, out = (X.int().matmul(W.int())//1000) if clip: out = out.clip(tg_dtype.min, tg_dtype.max) out = out.cast(tg_dtype) - opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] if opts is None else opts + opts = [Opt(op=OptOps.UPCAST, axis=1, arg=128), Opt(op=OptOps.UPCAST, axis=3, arg=4)] if opts is None else opts sexec(out, opts, replace_src, run_count=1) tout = out.numpy() mout = ((m1.astype(np.int32) @ m2.astype(np.int32)) // 1000) @@ -232,7 +233,7 @@ def test_prequant_gemv(self): #out = X.cast(dtypes.int) @ W.cast(dtypes.int) #out = X @ W out = X.matmul(W, dtype=X.dtype) - opts = [Opt(op=OptOps.UPCAST, axis=0, arg=128), Opt(op=OptOps.UNROLL, axis=0, arg=4)] + opts = [Opt(op=OptOps.UPCAST, axis=0, arg=128), Opt(op=OptOps.UPCAST, axis=2, arg=4)] sexec(out, opts) if __name__ == "__main__": diff --git a/test/backend/test_rangeify.py b/test/backend/test_rangeify.py index 2eeba7a749958..78be5e2ba02b1 100644 --- a/test/backend/test_rangeify.py +++ b/test/backend/test_rangeify.py @@ -31,25 +31,25 @@ def test_upcast_01_mismatch(self): self._test((Opt(OptOps.UPCAST, 0, 2), Opt(Opt def test_upcast_02(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 2, 4))) def test_upcast_12(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4))) - def test_unroll_0(self): self._test((Opt(OptOps.UNROLL, 0, 4),)) - def test_unroll_1(self): self._test((Opt(OptOps.UNROLL, 1, 4),)) - def test_unroll_01(self): self._test((Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) + def test_unroll_0(self): self._test((Opt(OptOps.UPCAST, 3, 4),)) + def test_unroll_1(self): self._test((Opt(OptOps.UPCAST, 4, 4),)) + def test_unroll_01(self): self._test((Opt(OptOps.UPCAST, 3, 4), Opt(OptOps.UPCAST, 4, 4))) - def test_upcast_0_unroll_0(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 4))) - def test_upcast_1_unroll_0(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4))) - def test_upcast_2_unroll_0(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 0, 4))) + def test_upcast_0_unroll_0(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 4, 4))) + def test_upcast_1_unroll_0(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 4, 4))) + def test_upcast_2_unroll_0(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 4, 4))) - def test_upcast_0_unroll_1(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 1, 4))) - def test_upcast_1_unroll_1(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 1, 4))) - def test_upcast_2_unroll_1(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 1, 4))) + def test_upcast_0_unroll_1(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 5, 4))) + def test_upcast_1_unroll_1(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 5, 4))) + def test_upcast_2_unroll_1(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 5, 4))) - def test_upcast_1_unroll_1_small(self): self._test((Opt(OptOps.UPCAST, 1, 2), Opt(OptOps.UNROLL, 1, 2))) - def test_upcast_1_unroll_1_rev(self): self._test((Opt(OptOps.UNROLL, 1, 2), Opt(OptOps.UPCAST, 1, 2))) + def test_upcast_1_unroll_1_small(self): self._test((Opt(OptOps.UPCAST, 1, 2), Opt(OptOps.UPCAST, 5, 2))) + def test_upcast_1_unroll_1_rev(self): self._test((Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 1, 2))) def test_upcast_01_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) + self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 5, 4), Opt(OptOps.UPCAST, 6, 4))) def test_upcast_12_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4))) + self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 5, 4), Opt(OptOps.UPCAST, 6, 4))) class TestRangeifyAssign(unittest.TestCase): def test_assign_permuted(self): diff --git a/test/backend/test_uops.py b/test/backend/test_uops.py index 67fd41ddc2db2..48a2c98b7ac55 100644 --- a/test/backend/test_uops.py +++ b/test/backend/test_uops.py @@ -271,7 +271,7 @@ def test_mulacc_unrolled(self): b = Tensor.empty(1024) c = (a*b).sum() ast = c.schedule_linear().src[-1].src[0] - opts_to_apply = [Opt(OptOps.UNROLL, 0, 4)] + opts_to_apply = [Opt(OptOps.UPCAST, 0, 4)] ast = ast.replace(arg=KernelInfo(opts_to_apply=tuple(opts_to_apply))) program = to_program(ast, Device[Device.DEFAULT].renderer) uops = tuple(program.src[1].src) diff --git a/test/external/external_benchmark_op_conv.py b/test/external/external_benchmark_op_conv.py index 9280ef4a050c3..79321e578a4b8 100644 --- a/test/external/external_benchmark_op_conv.py +++ b/test/external/external_benchmark_op_conv.py @@ -81,7 +81,6 @@ def dm_conv_172(): opts = None # JITBEAM=2 - # (Opt(op=OptOps.UPCAST, axis=0, arg=4), Opt(op=OptOps.GROUPTOP, axis=1, arg=32), Opt(op=OptOps.UNROLL, axis=1, arg=4), Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=0, arg=4), Opt(op=OptOps.GROUP, axis=1, arg=0)) return c55.sink(arg=KernelInfo(name="conv", opts_to_apply=opts)) ast = {143: vision_conv_143, 153: vision_conv_153, 172: dm_conv_172}[getenv("NUM", 143)]() diff --git a/test/external/speed_v_theoretical.py b/test/external/speed_v_theoretical.py index 4f09759942674..432d7adef2a2a 100644 --- a/test/external/speed_v_theoretical.py +++ b/test/external/speed_v_theoretical.py @@ -86,7 +86,7 @@ def f(conv, x) -> Tensor: return conv(x).realize() self._compare(tm, tflops, gbs, nv_tflops, nv_gbs, amd_tflops, amd_gbs) # TODO: why are convs so slow?!? - def test_conv_3x3_256_32_32_256_256(self): self._test_conv_3x3(256, 32, 32, 256, 256, nv_tflops=27, amd_tflops=14) + def test_conv_3x3_256_32_32_256_256(self): self._test_conv_3x3(256, 32, 32, 256, 256, nv_tflops=27, amd_tflops=13) # theoretical is nv_tflops=165, amd_tflops=123 def test_gemm_4096(self): self._test_matmul(4096, nv_tflops=110, amd_tflops=65) diff --git a/test/null/test_linearizer_rewrite.py b/test/null/test_linearizer_rewrite.py index 031c86bb53916..1946629912147 100644 --- a/test/null/test_linearizer_rewrite.py +++ b/test/null/test_linearizer_rewrite.py @@ -12,7 +12,7 @@ def test_reduction(self): si = out.schedule_linear().src[-1] opts_to_apply = [] opts_to_apply.append(Opt(OptOps.UPCAST, 0, 4)) - opts_to_apply.append(Opt(OptOps.UNROLL, 0, 4)) + opts_to_apply.append(Opt(OptOps.UPCAST, 2, 4)) ast = si.src[0].replace(arg=KernelInfo(opts_to_apply=tuple(opts_to_apply))) prg = to_program(ast, Device["CPU"].renderer) print(prg.src[2].arg) diff --git a/test/null/test_uops_stats.py b/test/null/test_uops_stats.py index 94dd2d0957855..4fa4700d68c97 100644 --- a/test/null/test_uops_stats.py +++ b/test/null/test_uops_stats.py @@ -190,7 +190,7 @@ def test_gemm(self): @unittest.skip("fails locally on AMD") def test_gemm_tc_unroll_half(self): try: - p = to_program(replace_opts(self.ast_gemm_half, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UNROLL, 0, 2)]), + p = to_program(replace_opts(self.ast_gemm_half, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UPCAST, 4, 2)]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no tensor cores") @@ -199,7 +199,7 @@ def test_gemm_tc_unroll_half(self): def test_gemm_tc_unroll(self): try: - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UNROLL, 0, 2)]), + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UPCAST, 4, 2)]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no tensor cores") @@ -214,7 +214,7 @@ def test_gemm_one_upcasted(self): self.assertEqual(p.src[0].arg.estimates.lds, N*N*N*4 + N*N*N*4//4 + 4*N*N) def test_gemm_upcasted(self): - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4)]), + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 4, 4)]), renderer=Device[Device.DEFAULT].renderer) self.check_gemm(p) self.assertEqual(p.src[0].arg.estimates.lds, 2*N*N*N*4//4 + 4*N*N) diff --git a/test/opt/test_gen_float4.py b/test/opt/test_gen_float4.py index b09aabe50f271..7bdcd6e1f37ba 100644 --- a/test/opt/test_gen_float4.py +++ b/test/opt/test_gen_float4.py @@ -70,7 +70,7 @@ def test_float4_sometimes_unaligned(self): # float4 should be emitted (the reduce axis of size 4 is the float4 axis here) s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UNROLL, axis=0, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=1, arg=4)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) == (0, 0) @@ -84,7 +84,7 @@ def test_float4_multidim_sometimes_unaligned(self): # UPDATE: now we do this fusion s = c.schedule_linear().src[0] - uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UNROLL, axis=0, arg=0)]), + uops = tuple(to_program(replace_opts(s.src[0], [Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.UPCAST, axis=1, arg=0)]), renderer=Device[Device.DEFAULT].renderer).src[1].src) assert TestFloat4.count_float4(uops) in {(0,1), (1,1)} diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index 44150cb8f6054..b480228a7fc3e 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -27,7 +27,7 @@ def test_local_and_grouped_reduce(self): # Checking how it works with locals + grouped reduce [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 64)], # Checking how it works with locals + grouped reduce + upcasts - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.UPCAST, 0, 8), Opt(OptOps.UNROLL, 1, 4)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.UPCAST, 0, 8), Opt(OptOps.UPCAST, 4, 4)], # many local + many group [Opt(OptOps.GROUP, 0, 2)] * 4, [Opt(OptOps.LOCAL, 0, 2)] * 4, @@ -73,15 +73,15 @@ def test_matmul(self): [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.LOCAL, 1, 8)], # Checking how it works with locals [Opt(OptOps.GROUPTOP, 0, 2)], [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 0, 32), Opt(OptOps.UNROLL, 0, 4)], # Checking how it works with grouped_reduce + [Opt(OptOps.GROUPTOP, 0, 32), Opt(OptOps.UPCAST, 2, 4)], # Checking how it works with grouped_reduce [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 32)], [Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 32)], [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 4)], # Checking how it works with local+grouped_reduce # Checking all together - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 4), + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)], # Full global upcast + local - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UPCAST, 0, 8)], + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 8)], ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @@ -98,14 +98,14 @@ def test_double_reduce(self): [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 2)], [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2)], [Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 64)], # Checking how it works with 2 grouped_reduces. - [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2), Opt(OptOps.UNROLL, 0, 4)], - [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UNROLL, 2, 4)], # Checking how it works with 2 grouped_reduces + upcasts. + [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2), Opt(OptOps.UPCAST, 2, 4)], + [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UPCAST, 4, 4)], # Checking how it works with 2 grouped_reduces + upcasts. [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UNROLL, 1, 4)], + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UPCAST, 5, 4)], [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2)], [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), - Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.UNROLL, 1, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. + Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 5, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 0, 2)], # No globals ]) @@ -124,14 +124,14 @@ def test_tensor_core_opts(self): [Opt(OptOps.UPCAST, 0, 4)], [Opt(OptOps.UPCAST, 1, 4)], [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4)], # check upcasts - [Opt(OptOps.UNROLL, 0, 2)], # check unroll - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 2)], # check combo of unroll and local - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 2)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4)], + [Opt(OptOps.UPCAST, 4, 2)], # check unroll + [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 5, 2)], # check combo of unroll and local + [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 6, 2)], + [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 6, 4)], [Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 0, 4)], # check permutations - [Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 0, 4)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 4)], - [Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UNROLL, 0, 4)], + [Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 0, 4)], + [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 5, 2), Opt(OptOps.UPCAST, 1, 4)], + [Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 6, 4)], ], apply_tc=True, atol=atol, rtol=rtol) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") @@ -145,10 +145,10 @@ def test_tensor_core_opts_locals(self): r = a.matmul(b, dtype=dtypes.half) atol, rtol = 0.25, 0.01 helper_linearizer_opt(r, [ - [Opt(OptOps.UNROLL, 0, 0)], # check full unroll of reduce with locals + [Opt(OptOps.UPCAST, 4, 0)], # check full unroll of reduce with locals [Opt(OptOps.LOCAL, 0, 4)], # check local - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.LOCAL, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 0, 4)], + [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 6, 4), Opt(OptOps.LOCAL, 0, 2)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 6, 2), Opt(OptOps.UPCAST, 0, 4)], ], apply_tc=True, atol=atol, rtol=rtol) @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") @@ -170,9 +170,9 @@ def test_tensor_core_opts_group(self): [Opt(OptOps.GROUPTOP, 0, 4)], [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.GROUP, 0, 2)], [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.UNROLL, 0, 4), Opt(OptOps.GROUP, 0, 2)], + [Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.GROUP, 0, 2)], [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UNROLL, 0, 2), Opt(OptOps.UPCAST, 1, 2)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 1, 2)], ], apply_tc=True, atol=atol, rtol=rtol) def test_padto_matmul(self): @@ -197,7 +197,7 @@ def test_padto_upcasted_not_ok(self): helper_linearizer_opt(a@b, [ [Opt(OptOps.UPCAST, 0, 0)], [Opt(OptOps.UPCAST, 1, 0)], - [Opt(OptOps.UNROLL, 0, 0)], + [Opt(OptOps.UPCAST, 2, 0)], [Opt(OptOps.PADTO, 0, 8)], [Opt(OptOps.PADTO, 1, 8)], [Opt(OptOps.PADTO, 2, 8)], @@ -207,7 +207,7 @@ def test_padto_upcasted_not_ok(self): with self.assertRaises(KernelOptError): helper_linearizer_opt(a@b, [[Opt(OptOps.UPCAST, 1, 0), Opt(OptOps.PADTO, 1, 8)]]) with self.assertRaises(KernelOptError): - helper_linearizer_opt(a@b, [[Opt(OptOps.UNROLL, 0, 0), Opt(OptOps.PADTO, 2, 8)]]) + helper_linearizer_opt(a@b, [[Opt(OptOps.UPCAST, 2, 0), Opt(OptOps.PADTO, 2, 8)]]) def test_padto_sum_ok(self): N = 18 @@ -245,7 +245,7 @@ def test_padto_sum_ok(self): def test_padto_group_full_unroll_sum(self): a = Tensor.ones(2, 28, 4096, dtype=dtypes.bfloat16).realize() out = ((a * 0.5).float().square()).sum(axis=(0, 2)) - opts_to_apply = [Opt(OptOps.GROUPTOP, 1, 256), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.UNROLL, 2, 0), Opt(OptOps.UPCAST, 0, 7)] + opts_to_apply = [Opt(OptOps.GROUPTOP, 1, 256), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.UPCAST, 3, 0), Opt(OptOps.UPCAST, 0, 7)] helper_linearizer_opt(out, [opts_to_apply], check_default_opt=False) def test_padto_sum(self): @@ -308,13 +308,13 @@ def test_color_shapes_with_local(self): ([Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("red",32)]), ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 2)], [("blue",16),("blue",32),("cyan",2),("green",2),("red",16)]), # check to ensure local_dims are stable for full UNROLL of the first reduce - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.UNROLL, 0, 0),Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UPCAST, 3, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.UPCAST, 2, 0),Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), # check behavior for full UNROLL on an existing GROUP - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UNROLL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 0),Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 2),Opt(OptOps.UNROLL, 0, 0)], [("blue",32),("blue",32),("red",16),("magenta",2)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UPCAST, 3, 2)], [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UPCAST, 3, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.GROUP, 0, 0),Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.GROUP, 0, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",32),("blue",32),("red",16),("magenta",2)]), ] helper_linearizer_opt(r, [x[0] for x in opts_shapes], color_sizes=[x[1] for x in opts_shapes]) @@ -330,7 +330,6 @@ def test_arange_opts(self): [Opt(op=OptOps.LOCAL, axis=0, arg=8)], [Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0)], #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8)], - #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.UNROLL, axis=1, arg=4)], # noqa: E501 ]) def test_double_sum_group(self): @@ -340,7 +339,7 @@ def test_double_sum_group(self): helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 0, 16)],]) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): - helper_linearizer_opt(r, [[Opt(OptOps.UNROLL, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) + helper_linearizer_opt(r, [[Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) diff --git a/test/opt/test_tensor_cores.py b/test/opt/test_tensor_cores.py index 0d9b5db3d1f31..518e22603095b 100644 --- a/test/opt/test_tensor_cores.py +++ b/test/opt/test_tensor_cores.py @@ -188,9 +188,9 @@ def test_tensor_cores_unroll_phi(self): tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in not in dtypes.fp8s) x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UNROLL, 0, 2)] + opts = [Opt(OptOps.UPCAST, 4, 2)] ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): + for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: assert u.src[-1].src[0].op != Ops.STORE @@ -202,9 +202,9 @@ def test_tensor_cores_unroll_casted_phi(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UNROLL, 0, 2)] + opts = [Opt(OptOps.UPCAST, 4, 2)] ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): + for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) assert u.src[-1].src[0].op != Ops.STORE @@ -218,9 +218,9 @@ def test_tensor_cores_unroll_casted_phi_with_children(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out).relu() - opts = [Opt(OptOps.UNROLL, 0, 2)] + opts = [Opt(OptOps.UPCAST, 4, 2)] ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src): + for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): if u.op is Ops.WMMA: #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) assert u.src[-1].src[0].op != Ops.STORE diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index d1048b6cb09d9..da1f3083f673a 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -4,8 +4,7 @@ from dataclasses import dataclass class OptOps(Enum): - TC = auto(); UPCAST = auto(); UNROLL = auto(); LOCAL = auto() # noqa: E702 - GROUP = auto(); GROUPTOP = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 + TC = auto(); UPCAST = auto(); LOCAL = auto(); GROUP = auto(); GROUPTOP = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value @dataclass(frozen=True, order=True) diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index 66f506b3c8940..63a3b2c063eab 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -52,10 +52,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: unit_stride_axes_mul_4 = [k.rngs.index(c) for c in idx.get_idx().split_uop(Ops.ADD) if c.op is Ops.RANGE and (c.vmax+1)%4 == 0 and c not in idx.get_valid().backward_slice] if len(unit_stride_axes_mul_4): - if (axis:=unit_stride_axes_mul_4[0]) in k.upcastable_dims: - k.apply_opt(Opt(OptOps.UPCAST, axis, 4)) - elif axis in k.unrollable_dims: - k.apply_opt(Opt(OptOps.UNROLL, k.unrollable_dims.index(axis), 4)) + if (axis:=unit_stride_axes_mul_4[0]) in k.upcastable_dims+k.unrollable_dims: k.apply_opt(Opt(OptOps.UPCAST, axis, 4)) # should use matvec - TODO: adjust/tune based on the wide vs tall/large vs small mat MV_BLOCKSIZE, MV_THREADS_PER_ROW, MV_ROWS_PER_THREAD = getenv("MV_BLOCKSIZE", 4), getenv("MV_THREADS_PER_ROW", 8), getenv("MV_ROWS_PER_THREAD", 4) @@ -141,14 +138,14 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: try: if k.unrollable_dims and (k.upcast_size() <= 4 or not k.axes_of(AxisType.UNROLL)) and (k.upcast_size() < 64): if (s:=k.full_shape[k.unrollable_dims[-1]]) <= 32: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, 0)) + k.apply_opt(Opt(OptOps.UPCAST, k.unrollable_dims[-1], 0)) # if it's small, upcast a second reduce dimension too if k.unrollable_dims and s <= 3 and k.full_shape[k.unrollable_dims[-1]] <= 3: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, 0)) + k.apply_opt(Opt(OptOps.UPCAST, k.unrollable_dims[-1], 0)) else: for splits in [4]: if k.full_shape[axis:=k.unrollable_dims[-1]]%splits == 0: - k.apply_opt(Opt(OptOps.UNROLL, len(k.unrollable_dims)-1, splits)) + k.apply_opt(Opt(OptOps.UPCAST, axis, splits)) break except KernelOptError: pass diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index 5afdde2b2df54..085324bedd424 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -11,6 +11,9 @@ from tinygrad.codegen.simplify import pm_flatten_range from tinygrad.renderer import Renderer +upcast_to = {AxisType.GLOBAL: AxisType.UPCAST, AxisType.LOCAL: AxisType.UPCAST, AxisType.WEAK: AxisType.UPCAST, + AxisType.GROUP_REDUCE: AxisType.UNROLL, AxisType.REDUCE: AxisType.UNROLL} + class Scheduler: def __init__(self, ast:UOp, ren:Renderer): self.ast, self.ren = ast, ren @@ -109,7 +112,6 @@ def unrollable_dims(self) -> list[int]: return [i for i in self.axes_of(AxisType def real_axis(self, op:OptOps, axis:int|None) -> int: try: if axis is None or op is OptOps.TC: return -1 - if op is OptOps.UNROLL: return self.unrollable_dims[axis] if op in {OptOps.GROUP, OptOps.GROUPTOP}: return self.axes_of(AxisType.REDUCE)[axis] check(axis < self.shape_len, f"invalid axis on {axis=} {op=} {self.shape_len=}") return axis @@ -122,13 +124,13 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): rng = self.rngs[real_axis] if (real_axis:=self.real_axis(opt.op, opt.axis)) >= 0 else UOp(Ops.NOOP) opt_to_at = { - OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, - OptOps.UNROLL: AxisType.UNROLL, OptOps.GROUP: AxisType.GROUP_REDUCE, + OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, OptOps.GROUP: AxisType.GROUP_REDUCE, OptOps.GROUPTOP: AxisType.GROUP_REDUCE} ret = None if opt.op in opt_to_at: amt:int = int(rng.vmax+1) if opt.arg == 0 else cast(int, opt.arg) + new_type = opt_to_at[opt.op] # copied from kernel.py. prevents METAL compiler hangs if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP} or (self.group_for_reduces and opt.op != OptOps.PADTO)): @@ -141,18 +143,16 @@ def apply_opt(self, opt:Opt, append_opt:bool=True): check(not any(u.arg[-1] in (AxisType.REDUCE, AxisType.UNROLL, AxisType.GROUP_REDUCE) for u in reduce.ranges), "cannot have a GROUP_REDUCE inside another reduce") - if opt.op is OptOps.UNROLL: - check(amt <= 32, "don't unroll more than 32") - check(rng.arg[-1] in {AxisType.GROUP_REDUCE, AxisType.REDUCE}, "unroll is for GROUP_REDUCE/REDUCE") if opt.op is OptOps.UPCAST: - check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") - check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.LOCAL, AxisType.WEAK}, f"upcast is for GLOBAL/LOCAL/LOOP, not {rng.arg[-1]}") + check(rng.arg[-1] in upcast_to, f"upcast is for GLOBAL/LOCAL/LOOP/REDUCE, not {rng.arg[-1]}") + if (new_type:=upcast_to[rng.arg[-1]]) is AxisType.UNROLL: check(amt <= 32, "don't unroll more than 32") + else: check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") if opt.op is OptOps.LOCAL: check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") - ret = self.shift_to(rng, amt, opt_to_at[opt.op], top=opt.op is OptOps.GROUPTOP) + ret = self.shift_to(rng, amt, new_type, top=opt.op is OptOps.GROUPTOP) elif opt.op is OptOps.TC: check(len(self.applied_opts) == 0, "tensor core opts must be first") # TODO: remove the need for this by having warps check(opt.axis is not None, "tensor core opts must have an axis") diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index e4d09b6cd02ca..2cc29d54c6185 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -11,8 +11,7 @@ from tinygrad.codegen import to_program from tinygrad.codegen.opt.postrange import Scheduler -actions = [Opt(op=OptOps.UPCAST, axis=axis, arg=amt) for amt in [0,2,3,4,5,7] for axis in range(8)] -actions += [Opt(op=OptOps.UNROLL, axis=axis, arg=amt) for amt in [0,4,7] for axis in range(5)] +actions = [Opt(op=OptOps.UPCAST, axis=axis, arg=amt) for amt in [0,2,3,4,5,7] for axis in range(10)] actions += [Opt(op=OptOps.LOCAL, axis=axis, arg=amt) for amt in [2,3,4,8,13,16,29] for axis in range(6)] actions += [Opt(op=OptOps.GROUPTOP, axis=axis, arg=amt) for amt in [13,16,28,29,32,49,64,256] for axis in range(3)] actions += [Opt(op=OptOps.GROUP, axis=axis, arg=amt) for amt in [0,4,8,16] for axis in range(3)] From b87159cee1b137c327f901a6aef69f394aa629f6 Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 31 Aug 2026 22:44:42 -0400 Subject: [PATCH 305/306] delete dead OptOps tests (#17872) --- test/backend/test_linearizer.py | 12 ++------ test/backend/test_rangeify.py | 51 ++++----------------------------- test/null/test_uops_stats.py | 8 ------ test/opt/test_kernel_opts.py | 27 ----------------- test/opt/test_tensor_cores.py | 32 ++++++++++----------- 5 files changed, 23 insertions(+), 107 deletions(-) diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 6a33edb377910..64180d6cc535d 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -7,7 +7,7 @@ from tinygrad.tensor import Tensor, _to_np_dtype from tinygrad.engine.realize import run_linear from tinygrad.codegen import to_program -from tinygrad.helpers import Context, flatten, dedup, TC_SELECT, TC_OPT, DEV +from tinygrad.helpers import Context, dedup, TC_SELECT, TC_OPT, DEV from tinygrad.dtype import DType, dtypes, AddrSpace from tinygrad.renderer.ptx import PTXRenderer from tinygrad.renderer.cstyle import CUDARenderer @@ -73,14 +73,6 @@ def test_late_bias_load(self): # assert that there is a global load after the reduce ends assert any(u.addrspace == AddrSpace.GLOBAL for u in load_idxs) - def _test_no_nested_ranges(self, lins, skip=None): - for l in lins: - range_in_acc = flatten([[x for x in u.src if x.op is Ops.RANGE] for u in l.uops if u.op is Ops.BUFFER and u.addrspace is AddrSpace.REG]) - ranges = [u.op for u in l.uops if (u.op is Ops.RANGE and u in range_in_acc) or (u.op is Ops.END and u.src[0] in range_in_acc)] - for i,u in enumerate(ranges): - if skip and i in skip: continue - assert ranges[i-1] != u, f"multireduce nested the ranges! {ranges[i-1], {u}}" - def test_two_nested_range(self): a = Tensor.randn(2, ).realize() out = a.reshape(2, 1).expand(2, 3).sum() @@ -228,7 +220,7 @@ def helper_arg_acc_dtype(c: Tensor, expected_dtype:DType): (dtypes.float, dtypes.float16, dtypes.float16), ) for tensor_dtype, acc_dtype, expected_dtype in tests: - if tensor_dtype in (dts:=Device[Device.DEFAULT].renderer.supported_dtypes()) and acc_dtype in dts and expected_dtype in dts: + if tensor_dtype in (dts:=Device[Device.DEFAULT].renderer.supported_dtypes()) and acc_dtype in dts|{None} and expected_dtype in dts: a, b = Tensor.rand(8, 8, dtype=tensor_dtype), Tensor.rand(8, 8, dtype=tensor_dtype) helper_arg_acc_dtype(a.sum(dtype=acc_dtype), expected_dtype) helper_arg_acc_dtype(a.matmul(b, dtype=acc_dtype), expected_dtype) diff --git a/test/backend/test_rangeify.py b/test/backend/test_rangeify.py index 78be5e2ba02b1..6743275ba524d 100644 --- a/test/backend/test_rangeify.py +++ b/test/backend/test_rangeify.py @@ -1,55 +1,16 @@ import unittest -from tinygrad import Tensor, Device, dtypes, Variable +from tinygrad import Tensor, dtypes, Variable from tinygrad.helpers import Context, GlobalCounters, getenv, DEBUG from tinygrad.uop.ops import graph_rewrite, PatternMatcher, UPat, Ops, UOp -from tinygrad.codegen.opt import OptOps, Opt -from tinygrad.renderer.ptx import PTXRenderer -from tinygrad.renderer.nir import NIRRenderer -@unittest.skipIf(isinstance(Device[Device.DEFAULT].renderer, (NIRRenderer, PTXRenderer)), "broken in LVP and PTX") class TestDoubleMatmul(unittest.TestCase): - def setUp(self): + def test_double_matmul(self): with Context(DEBUG=0): - self.a, self.b, self.c = [Tensor.randn(16, 16).contiguous().realize() for _ in range(3)] - self.ref = (self.a @ self.b @ self.c).realize() - - def _test(self, opts): + a, b, c = [Tensor.randn(16, 16).contiguous().realize() for _ in range(3)] + ref = a.numpy() @ b.numpy() @ c.numpy() with Context(DEBUG=max(2, DEBUG.value)): - out = (self.a @ self.b @ self.c).contiguous(arg=opts).realize() - - with Context(DEBUG=0): - err = (out-self.ref).square() - self.assertLess(err.max().item(), 1e-4) - self.assertLess(err.mean().item(), 1e-6) - - def test_baseline(self): self._test(()) - def test_upcast_0(self): self._test((Opt(OptOps.UPCAST, 0, 4),)) - def test_upcast_1(self): self._test((Opt(OptOps.UPCAST, 1, 4),)) - def test_upcast_2(self): self._test((Opt(OptOps.UPCAST, 2, 4),)) - def test_upcast_01(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4))) - def test_upcast_01_mismatch(self): self._test((Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 1, 4))) - def test_upcast_02(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 2, 4))) - def test_upcast_12(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4))) - - def test_unroll_0(self): self._test((Opt(OptOps.UPCAST, 3, 4),)) - def test_unroll_1(self): self._test((Opt(OptOps.UPCAST, 4, 4),)) - def test_unroll_01(self): self._test((Opt(OptOps.UPCAST, 3, 4), Opt(OptOps.UPCAST, 4, 4))) - - def test_upcast_0_unroll_0(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 4, 4))) - def test_upcast_1_unroll_0(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 4, 4))) - def test_upcast_2_unroll_0(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 4, 4))) - - def test_upcast_0_unroll_1(self): self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 5, 4))) - def test_upcast_1_unroll_1(self): self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 5, 4))) - def test_upcast_2_unroll_1(self): self._test((Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 5, 4))) - - def test_upcast_1_unroll_1_small(self): self._test((Opt(OptOps.UPCAST, 1, 2), Opt(OptOps.UPCAST, 5, 2))) - def test_upcast_1_unroll_1_rev(self): self._test((Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 1, 2))) - - def test_upcast_01_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 5, 4), Opt(OptOps.UPCAST, 6, 4))) - def test_upcast_12_unroll_01(self): - self._test((Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 2, 4), Opt(OptOps.UPCAST, 5, 4), Opt(OptOps.UPCAST, 6, 4))) + out = (a @ b @ c).numpy() + self.assertLess(abs(out-ref).max(), 1e-3) class TestRangeifyAssign(unittest.TestCase): def test_assign_permuted(self): diff --git a/test/null/test_uops_stats.py b/test/null/test_uops_stats.py index 4fa4700d68c97..ff90569350064 100644 --- a/test/null/test_uops_stats.py +++ b/test/null/test_uops_stats.py @@ -245,13 +245,5 @@ def test_reduce(self): self.assertEqual(est.ops, N*N) self.assertEqual(est.mem, N*N*4 + 4) - def test_reduce_group(self): - try: - p = to_program(replace_opts(self.ast_reduce, [Opt(OptOps.GROUP, 0, 50)]), renderer=Device[Device.DEFAULT].renderer) - except KernelOptError: - raise unittest.SkipTest("no locals") - est = p.src[0].arg.estimates - print(p.arg.name, est.ops, est.mem, est.lds) - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index b480228a7fc3e..864f8c288db31 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -151,30 +151,6 @@ def test_tensor_core_opts_locals(self): [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.UPCAST, 6, 2), Opt(OptOps.UPCAST, 0, 4)], ], apply_tc=True, atol=atol, rtol=rtol) - @unittest.skipUnless(Device[Device.DEFAULT].renderer.tensor_cores, "test requires tensor cores") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") - @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_shared, "test requires shared memory") - @unittest.skipUnless(any(tc.dtype_in == tc.dtype_out == dtypes.half for tc in Device[Device.DEFAULT].renderer.tensor_cores), - "test requires tensor cores with accumulation in half") # testing with half suffices. - # NOTE: the METAL test is broken, likely due to a compiler bug. passes on CI with -O0 and with default opt level locally on M3 - @unittest.skipIf(Device.DEFAULT == "METAL", "broken for METAL") - @unittest.skip("feature was removed") - def test_tensor_core_opts_group(self): - N = 128 - Tensor.manual_seed(1552) - a, b = Tensor.rand(N, N, dtype=dtypes.half), Tensor.rand(N, N, dtype=dtypes.half) - r = a.matmul(b, dtype=dtypes.half) - atol, rtol = 0.25, 0.01 - helper_linearizer_opt(r, [ - [Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 4)], - [Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 2), Opt(OptOps.UPCAST, 1, 2)], - ], apply_tc=True, atol=atol, rtol=rtol) - def test_padto_matmul(self): N = 17 Tensor.manual_seed(289) @@ -325,11 +301,8 @@ def test_arange_opts(self): a = Tensor.arange(128).clone() # NOTE: arange no longer has reduce ops available for opt helper_linearizer_opt(a, [ - #[Opt(OptOps.GROUP, 0, 32)], - #[Opt(OptOps.GROUPTOP, 0, 32)], [Opt(op=OptOps.LOCAL, axis=0, arg=8)], [Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0)], - #[Opt(op=OptOps.LOCAL, axis=0, arg=8), Opt(op=OptOps.UPCAST, axis=0, arg=0), Opt(op=OptOps.GROUP, axis=0, arg=8)], ]) def test_double_sum_group(self): diff --git a/test/opt/test_tensor_cores.py b/test/opt/test_tensor_cores.py index 518e22603095b..d5f7e49262985 100644 --- a/test/opt/test_tensor_cores.py +++ b/test/opt/test_tensor_cores.py @@ -188,11 +188,11 @@ def test_tensor_cores_unroll_phi(self): tc = next(tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in not in dtypes.fp8s) x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UPCAST, 4, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UPCAST, 4, 2)] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE @Context(ALLOW_TF32=1) @unittest.skipIf(Device.DEFAULT == "PYTHON", "slow on EMULATED device") @@ -202,12 +202,11 @@ def test_tensor_cores_unroll_casted_phi(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out) - opts = [Opt(OptOps.UPCAST, 4, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UPCAST, 4, 2)] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE @Context(ALLOW_TF32=1) @unittest.skipIf(Device.DEFAULT == "PYTHON", "slow on EMULATED device") @@ -218,12 +217,11 @@ def test_tensor_cores_unroll_casted_phi_with_children(self): tc = [tc for tc in Device[Device.DEFAULT].renderer.tensor_cores if tc.dtype_in != tc.dtype_out and tc.dtype_in not in dtypes.fp8s][0] x, y = Tensor.rand(16, 64, dtype=tc.dtype_in), Tensor.rand(64, 16, dtype=tc.dtype_in) r = x.matmul(y, dtype=tc.dtype_out).relu() - opts = [Opt(OptOps.UPCAST, 4, 2)] - ast = helper_linearizer_opt(r, [opts], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) - for u in tuple(to_program(replace_opts(ast, [Opt(OptOps.UPCAST, 2, 2)]), Device[Device.DEFAULT].renderer).src[1].src): - if u.op is Ops.WMMA: - #assert u.src[-1].dtype == dtypes.float.vec(prod(tc.thread_local_sizes[2])) - assert u.src[-1].src[0].op != Ops.STORE + opts = [Opt(OptOps.TC, 0, (-1, 0, 1)), Opt(OptOps.UPCAST, 4, 2)] + ast = helper_linearizer_opt(r, [opts[1:]], apply_tc=True, atol=3e-2, rtol=1e-3, check_default_opt=False) + wmmas = [u for u in tuple(to_program(replace_opts(ast, opts), Device[Device.DEFAULT].renderer).src[1].src) if u.op is Ops.WMMA] + self.assertGreater(len(wmmas), 0) + for u in wmmas: assert u.src[-1].src[0].op != Ops.STORE if __name__ == '__main__': unittest.main() From cf6fff13aa4dbf61d930c5c6da607ab2954edede Mon Sep 17 00:00:00 2001 From: chenyu Date: Mon, 31 Aug 2026 23:30:42 -0400 Subject: [PATCH 306/306] OptOps.GROUP is OptOps.LOCAL (#17873) --- extra/gemm/tinygrad_nv_matmul.py | 2 +- test/backend/test_linearizer.py | 6 +-- test/backend/test_linearizer_dumb.py | 2 +- test/null/test_uops_stats.py | 2 +- test/opt/test_kernel_opts.py | 75 ++++++++++++++-------------- tinygrad/codegen/opt/__init__.py | 2 +- tinygrad/codegen/opt/heuristic.py | 4 +- tinygrad/codegen/opt/postrange.py | 41 +++++++-------- tinygrad/codegen/opt/search.py | 7 ++- 9 files changed, 69 insertions(+), 72 deletions(-) diff --git a/extra/gemm/tinygrad_nv_matmul.py b/extra/gemm/tinygrad_nv_matmul.py index 77538f7e742f9..afb9d99408674 100644 --- a/extra/gemm/tinygrad_nv_matmul.py +++ b/extra/gemm/tinygrad_nv_matmul.py @@ -14,7 +14,7 @@ if getenv("GEMV"): opts = [ Opt(op=OptOps.UPCAST, axis=1, amt=8), - Opt(op=OptOps.GROUP, axis=0, amt=32), + Opt(op=OptOps.LOCAL, axis=1, amt=32), ] else: opts = [ diff --git a/test/backend/test_linearizer.py b/test/backend/test_linearizer.py index 64180d6cc535d..99ac143df75bd 100644 --- a/test/backend/test_linearizer.py +++ b/test/backend/test_linearizer.py @@ -173,7 +173,7 @@ def test_upcast_with_locals_cpu(self): def test_upcast_with_locals(self): x, y = Tensor.rand(1,128), Tensor.rand(128, 128) r = (x@y).relu() - opts_to_apply = [Opt(op=OptOps.GROUP, axis=0, arg=8), Opt(op=OptOps.LOCAL, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=0, arg=4)] + opts_to_apply = [Opt(op=OptOps.LOCAL, axis=1, arg=8), Opt(op=OptOps.LOCAL, axis=0, arg=4), Opt(op=OptOps.UPCAST, axis=0, arg=4)] program = to_program(replace_opts(r.schedule_linear().src[-1].src[0], opts_to_apply), renderer=Device[Device.DEFAULT].renderer) stores = [u for u in tuple(program.src[1].src) if u.op is Ops.STORE and u.src[0].addrspace != AddrSpace.REG] @@ -345,7 +345,7 @@ def test_grouped_store_values(self): def test_grouped_store_locals_and_globals(self): x, y = Tensor.empty(64, 64), Tensor.empty(64, 64) out = x@y - opt = [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), + opt = [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 3, 8), Opt(OptOps.UPCAST, 3, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)] # upcast accs in both reduces ast = helper_linearizer_opt(out, opts=[opt]) def get_recursive(uop): return set.union(set(uop.src), [uop], *[get_recursive(v) for v in uop.src]) @@ -386,7 +386,7 @@ def test_grouped_store_local_only(self): def test_two_grouped_stores_local(self): # GROUP on both reduces puts two LOCAL buffers in one kernel, and the store to each needs its own barrier a = Tensor.rand(32, 32).realize() - opts = [Opt(OptOps.GROUP, 1, 4), Opt(OptOps.GROUP, 2, 4)] + opts = [Opt(OptOps.LOCAL, 3, 4), Opt(OptOps.LOCAL, 5, 4)] ast = helper_linearizer_opt(single_kernel_softmax(a), [opts]) uops = to_program(replace_opts(ast, opts), renderer=Device[Device.DEFAULT].renderer).src[1].src self.assertEqual(len([u for u in uops if u.op is Ops.BARRIER]), 2) diff --git a/test/backend/test_linearizer_dumb.py b/test/backend/test_linearizer_dumb.py index 583c3c64b6918..63908f24af84d 100644 --- a/test/backend/test_linearizer_dumb.py +++ b/test/backend/test_linearizer_dumb.py @@ -24,7 +24,7 @@ def test_failure_beam_mnist(self): c10 = c9.index((((c3*UOp.const(4704000))+c2)+(c6*UOp.const(784))).valid(UOp.const(True))) c11 = c5.alu(Ops.CMPNE, ((((c3*UOp.const(6000))+c6)+((c7*UOp.const(16))+c8)).alu(Ops.CMPLT, UOp.const(59999)).where(UOp.const(0).cast(dtypes.int), UOp.const(1).cast(dtypes.int)).reduce(c7, c8, arg=Ops.ADD)+UOp.const(-1).cast(dtypes.int))).where(UOp.const(0).cast(dtypes.uchar), c10).reduce(c6, arg=Ops.ADD) c12 = c0.index((((c1*UOp.const(7840))+(c2*UOp.const(10)))+c3).valid(UOp.const(True))).store(c11).end(c1, c2, c3) - ast = c12.sink(arg=KernelInfo(name='test', applied_opts=(Opt(op=OptOps.GROUP, axis=1, arg=16),), opts_to_apply=None)) + ast = c12.sink(arg=KernelInfo(name='test', applied_opts=(Opt(op=OptOps.LOCAL, axis=4, arg=16),), opts_to_apply=None)) _ = to_program(ast, Device["METAL"].renderer) if __name__ == '__main__': diff --git a/test/null/test_uops_stats.py b/test/null/test_uops_stats.py index ff90569350064..315f2a657adb4 100644 --- a/test/null/test_uops_stats.py +++ b/test/null/test_uops_stats.py @@ -230,7 +230,7 @@ def test_gemm_upcasted_locals(self): def test_gemm_group(self): try: - p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.GROUP, 0, 4)]), renderer=Device[Device.DEFAULT].renderer) + p = to_program(replace_opts(self.ast_gemm, [Opt(OptOps.LOCAL, 2, 4)]), renderer=Device[Device.DEFAULT].renderer) except KernelOptError: raise unittest.SkipTest("no locals") SZ = N*N*4 diff --git a/test/opt/test_kernel_opts.py b/test/opt/test_kernel_opts.py index 864f8c288db31..2aed363f26080 100644 --- a/test/opt/test_kernel_opts.py +++ b/test/opt/test_kernel_opts.py @@ -18,20 +18,21 @@ def test_local_and_grouped_reduce(self): [Opt(OptOps.LOCAL, 0, 2)], [Opt(OptOps.LOCAL, 0, 8)], [Opt(OptOps.LOCAL, 0, 16)], # Checking how it works with locals - [Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 0, 64)], # Checking how it works with grouped reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.GROUPTOP, 0, 16)], - [Opt(OptOps.LOCAL, 0, 32), Opt(OptOps.GROUPTOP, 0, 2)], + [Opt(OptOps.GROUPTOP, 1, 2)], + [Opt(OptOps.GROUPTOP, 1, 32)], + [Opt(OptOps.GROUPTOP, 1, 64)], # Checking how it works with grouped reduce + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 2, 2)], + [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.GROUPTOP, 2, 16)], + [Opt(OptOps.LOCAL, 0, 32), Opt(OptOps.GROUPTOP, 2, 2)], # Checking how it works with locals + grouped reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 64)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 2, 64)], # Checking how it works with locals + grouped reduce + upcasts - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.UPCAST, 0, 8), Opt(OptOps.UPCAST, 4, 4)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUPTOP, 2, 2), Opt(OptOps.UPCAST, 0, 8), Opt(OptOps.UPCAST, 4, 4)], # many local + many group - [Opt(OptOps.GROUP, 0, 2)] * 4, + [Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.LOCAL, 2, 2), Opt(OptOps.LOCAL, 3, 2), Opt(OptOps.LOCAL, 4, 2)], [Opt(OptOps.LOCAL, 0, 2)] * 4, - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.GROUP, 0, 2)] * 4, + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 2, 2), Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 4, 2), + Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 6, 2), Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 8, 2)], ]) def test_upcasts(self): @@ -71,17 +72,17 @@ def test_matmul(self): [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4)], [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 32)], [Opt(OptOps.LOCAL, 0, 16), Opt(OptOps.LOCAL, 1, 8)], # Checking how it works with locals - [Opt(OptOps.GROUPTOP, 0, 2)], - [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 0, 32), Opt(OptOps.UPCAST, 2, 4)], # Checking how it works with grouped_reduce - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 0, 4)], # Checking how it works with local+grouped_reduce + [Opt(OptOps.GROUPTOP, 2, 2)], + [Opt(OptOps.GROUPTOP, 2, 32)], + [Opt(OptOps.GROUPTOP, 2, 32), Opt(OptOps.UPCAST, 2, 4)], # Checking how it works with grouped_reduce + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 4, 32)], + [Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 3, 32)], + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 8), Opt(OptOps.GROUPTOP, 4, 4)], # Checking how it works with local+grouped_reduce # Checking all together - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 4), + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 4, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 4), Opt(OptOps.UPCAST, 1, 2)], # Full global upcast + local - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 8)], + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.GROUPTOP, 4, 8), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 0, 8)], ]) @unittest.skipUnless(Device[Device.DEFAULT].renderer.has_local, "test requires locals") @@ -93,20 +94,20 @@ def test_double_reduce(self): r = a.sum(axis=(1,3)) helper_linearizer_opt(r, [ # openCL / DEV=CL is 256 max threads - [Opt(OptOps.GROUPTOP, 0, 2)], [Opt(OptOps.GROUPTOP, 0, 32)], - [Opt(OptOps.GROUPTOP, 1, 2)], [Opt(OptOps.GROUPTOP, 1, 32)], # Checking how it works with 1 grouped_reduce. - [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 2)], - [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2)], - [Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 64)], # Checking how it works with 2 grouped_reduces. - [Opt(OptOps.GROUPTOP, 0, 16), Opt(OptOps.GROUPTOP, 1, 2), Opt(OptOps.UPCAST, 2, 4)], - [Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UPCAST, 4, 4)], # Checking how it works with 2 grouped_reduces + upcasts. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4)], + [Opt(OptOps.GROUPTOP, 2, 2)], [Opt(OptOps.GROUPTOP, 2, 32)], + [Opt(OptOps.GROUPTOP, 3, 2)], [Opt(OptOps.GROUPTOP, 3, 32)], # Checking how it works with 1 grouped_reduce. + [Opt(OptOps.GROUPTOP, 2, 2), Opt(OptOps.GROUPTOP, 4, 2)], + [Opt(OptOps.GROUPTOP, 2, 16), Opt(OptOps.GROUPTOP, 4, 2)], + [Opt(OptOps.GROUPTOP, 2, 4), Opt(OptOps.GROUPTOP, 4, 64)], # Checking how it works with 2 grouped_reduces. + [Opt(OptOps.GROUPTOP, 2, 16), Opt(OptOps.GROUPTOP, 4, 2), Opt(OptOps.UPCAST, 2, 4)], + [Opt(OptOps.GROUPTOP, 2, 2), Opt(OptOps.GROUPTOP, 4, 32), Opt(OptOps.UPCAST, 4, 4)], # Checking how it works with 2 grouped_reduces + upcasts. + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 4, 4), Opt(OptOps.GROUPTOP, 6, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 2), Opt(OptOps.GROUPTOP, 1, 32), Opt(OptOps.UPCAST, 5, 4)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2)], - [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 0, 8), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 4, 2), Opt(OptOps.GROUPTOP, 6, 32), Opt(OptOps.UPCAST, 5, 4)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 4, 8), Opt(OptOps.GROUPTOP, 6, 4), Opt(OptOps.UPCAST, 0, 2)], + [Opt(OptOps.LOCAL, 0, 2), Opt(OptOps.LOCAL, 1, 2), Opt(OptOps.GROUPTOP, 4, 8), Opt(OptOps.GROUPTOP, 6, 4), Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 4, 4), Opt(OptOps.UPCAST, 5, 4)], # Checking how it works with 2 grouped_reduces + upcasts + locals. - [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 0, 4), Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.UPCAST, 0, 2), + [Opt(OptOps.LOCAL, 0, 4), Opt(OptOps.LOCAL, 1, 4), Opt(OptOps.GROUPTOP, 4, 4), Opt(OptOps.GROUPTOP, 6, 4), Opt(OptOps.UPCAST, 0, 2), Opt(OptOps.UPCAST, 0, 2)], # No globals ]) @@ -221,7 +222,7 @@ def test_padto_sum_ok(self): def test_padto_group_full_unroll_sum(self): a = Tensor.ones(2, 28, 4096, dtype=dtypes.bfloat16).realize() out = ((a * 0.5).float().square()).sum(axis=(0, 2)) - opts_to_apply = [Opt(OptOps.GROUPTOP, 1, 256), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.UPCAST, 3, 0), Opt(OptOps.UPCAST, 0, 7)] + opts_to_apply = [Opt(OptOps.GROUPTOP, 2, 256), Opt(OptOps.PADTO, 3, 32), Opt(OptOps.UPCAST, 3, 0), Opt(OptOps.UPCAST, 0, 7)] helper_linearizer_opt(out, [opts_to_apply], check_default_opt=False) def test_padto_sum(self): @@ -282,15 +283,15 @@ def test_color_shapes_with_local(self): r = a@b opts_shapes = [ ([Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("red",32)]), - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 2)], [("blue",16),("blue",32),("cyan",2),("green",2),("red",16)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.LOCAL, 3, 2)], [("blue",16),("blue",32),("cyan",2),("green",2),("red",16)]), # check to ensure local_dims are stable for full UNROLL of the first reduce ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UPCAST, 3, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), ([Opt(OptOps.UPCAST, 2, 0),Opt(OptOps.LOCAL, 0, 2)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), # check behavior for full UNROLL on an existing GROUP - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UPCAST, 3, 2)], [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), - ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.GROUP, 0, 0),Opt(OptOps.UPCAST, 3, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 0),Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), - ([Opt(OptOps.GROUP, 0, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",32),("blue",32),("red",16),("magenta",2)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.LOCAL, 3, 0),Opt(OptOps.UPCAST, 3, 2)], [("blue",16),("blue",32),("cyan",2),("green",16),("magenta",2)]), + ([Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.LOCAL, 3, 0),Opt(OptOps.UPCAST, 3, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.LOCAL, 2, 0),Opt(OptOps.LOCAL, 0, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",16),("blue",32),("cyan",2),("magenta",32)]), + ([Opt(OptOps.LOCAL, 2, 2),Opt(OptOps.UPCAST, 2, 0)], [("blue",32),("blue",32),("red",16),("magenta",2)]), ] helper_linearizer_opt(r, [x[0] for x in opts_shapes], color_sizes=[x[1] for x in opts_shapes]) @@ -315,7 +316,7 @@ def test_double_sum_group(self): helper_linearizer_opt(r, [[Opt(OptOps.UPCAST, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) r = a.sum((1, 2)).sum() with self.assertRaises(KernelOptError): - helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.GROUPTOP, 0, 16)],]) + helper_linearizer_opt(r, [[Opt(OptOps.GROUPTOP, 1, 4), Opt(OptOps.GROUPTOP, 1, 16)],]) if __name__ == '__main__': unittest.main() diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index da1f3083f673a..61737f218493b 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -4,7 +4,7 @@ from dataclasses import dataclass class OptOps(Enum): - TC = auto(); UPCAST = auto(); LOCAL = auto(); GROUP = auto(); GROUPTOP = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 + TC = auto(); UPCAST = auto(); LOCAL = auto(); GROUPTOP = auto(); PADTO = auto(); SWAP = auto() # noqa: E702 def __lt__(self, x:OptOps): return self.value < x.value @dataclass(frozen=True, order=True) diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index 63a3b2c063eab..7c7fb654946f6 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -68,7 +68,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: if DEBUG >= 3: print(f"MATVEC: {k.full_shape=} {first_reduce_rng.render()} {MV_BLOCKSIZE=} {MV_THREADS_PER_ROW=} {MV_ROWS_PER_THREAD=}") try: - if MV_THREADS_PER_ROW > 1: k.apply_opt(Opt(OptOps.GROUP, 0, MV_THREADS_PER_ROW)) + if MV_THREADS_PER_ROW > 1: k.apply_opt(Opt(OptOps.LOCAL, k.axes_of(AxisType.REDUCE)[0], MV_THREADS_PER_ROW)) except KernelOptError: pass if MV_BLOCKSIZE > 1: k.apply_opt(Opt(OptOps.LOCAL, global_idx, MV_BLOCKSIZE)) if MV_ROWS_PER_THREAD > 1: k.apply_opt(Opt(OptOps.UPCAST, global_idx, MV_ROWS_PER_THREAD)) @@ -76,7 +76,7 @@ def hand_coded_optimizations(k:Scheduler) -> Scheduler: # are we grouping? (requires local shape support) if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= (240 if k.ren.target.device == "QCOM" else 2048), False): - for axis, sz in itertools.product((0, 1, 2), (16,)): + for axis, sz in itertools.product(k.axes_of(AxisType.REDUCE)[:3], (16,)): try: k.apply_opt(Opt(OptOps.GROUPTOP, axis, sz)) break diff --git a/tinygrad/codegen/opt/postrange.py b/tinygrad/codegen/opt/postrange.py index 085324bedd424..5ee2ed550177f 100644 --- a/tinygrad/codegen/opt/postrange.py +++ b/tinygrad/codegen/opt/postrange.py @@ -13,6 +13,7 @@ upcast_to = {AxisType.GLOBAL: AxisType.UPCAST, AxisType.LOCAL: AxisType.UPCAST, AxisType.WEAK: AxisType.UPCAST, AxisType.GROUP_REDUCE: AxisType.UNROLL, AxisType.REDUCE: AxisType.UNROLL} +local_to = {AxisType.GLOBAL: AxisType.LOCAL, AxisType.WEAK: AxisType.LOCAL, AxisType.REDUCE: AxisType.GROUP_REDUCE} class Scheduler: def __init__(self, ast:UOp, ren:Renderer): @@ -110,48 +111,44 @@ def unrollable_dims(self) -> list[int]: return [i for i in self.axes_of(AxisType if isinstance(s:=self.full_shape[i], int) and s > 1] def real_axis(self, op:OptOps, axis:int|None) -> int: - try: - if axis is None or op is OptOps.TC: return -1 - if op in {OptOps.GROUP, OptOps.GROUPTOP}: return self.axes_of(AxisType.REDUCE)[axis] - check(axis < self.shape_len, f"invalid axis on {axis=} {op=} {self.shape_len=}") - return axis - except IndexError as e: raise KernelOptError from e + if axis is None or op is OptOps.TC: return -1 + check(0 <= axis < self.shape_len, f"invalid axis on {axis=} {op=} {self.shape_len=}") + return axis def apply_opt(self, opt:Opt, append_opt:bool=True): - if opt.op in {OptOps.LOCAL, OptOps.GROUP, OptOps.GROUPTOP}: + if opt.op in {OptOps.LOCAL, OptOps.GROUPTOP}: check(self.ren.has_local, "locals needed for opt") rng = self.rngs[real_axis] if (real_axis:=self.real_axis(opt.op, opt.axis)) >= 0 else UOp(Ops.NOOP) - opt_to_at = { - OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, OptOps.GROUP: AxisType.GROUP_REDUCE, - OptOps.GROUPTOP: AxisType.GROUP_REDUCE} + opt_to_at = {OptOps.LOCAL: AxisType.LOCAL, OptOps.UPCAST: AxisType.UPCAST, OptOps.GROUPTOP: AxisType.GROUP_REDUCE} ret = None if opt.op in opt_to_at: amt:int = int(rng.vmax+1) if opt.arg == 0 else cast(int, opt.arg) new_type = opt_to_at[opt.op] + if opt.op is OptOps.UPCAST: + check(rng.arg[-1] in upcast_to, f"upcast is for GLOBAL/LOCAL/LOOP/REDUCE, not {rng.arg[-1]}") + if (new_type:=upcast_to[rng.arg[-1]]) is AxisType.UNROLL: check(amt <= 32, "don't unroll more than 32") + else: check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") + if opt.op is OptOps.LOCAL: + check(rng.arg[-1] in local_to, f"local is for GLOBAL/LOOP/REDUCE, not {rng.arg[-1]}") + new_type = local_to[rng.arg[-1]] + if opt.op is OptOps.GROUPTOP: check(rng.arg[-1] is AxisType.REDUCE, "grouptop is for reduce") + if new_type is AxisType.GROUP_REDUCE: + check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? + # copied from kernel.py. prevents METAL compiler hangs - if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP} or (self.group_for_reduces and opt.op != OptOps.PADTO)): + if self.reduceop is not None and (new_type is AxisType.GROUP_REDUCE or (self.group_for_reduces and opt.op != OptOps.PADTO)): upcast_local_sz = prod([self.full_shape[a] for a in self.axes_of(AxisType.UPCAST, AxisType.WARP, AxisType.LOCAL, AxisType.GROUP_REDUCE)]) smem_sz = amt*upcast_local_sz*self.reduceop.dtype.itemsize check(smem_sz <= self.ren.shared_max, f"exceeds maximum shared memory size: needs {smem_sz}, max {self.ren.shared_max}") - if self.reduceop is not None and (opt.op in {OptOps.GROUP, OptOps.GROUPTOP}): + if self.reduceop is not None and new_type is AxisType.GROUP_REDUCE: # We currently dont support a group within another rudece, TODO: fix if-contexts reduce = [u for u in self.ast.backward_slice if u.op is Ops.REDUCE and rng in merge_dicts([r.ranges for r in u.src[1:]])][0] check(not any(u.arg[-1] in (AxisType.REDUCE, AxisType.UNROLL, AxisType.GROUP_REDUCE) for u in reduce.ranges), "cannot have a GROUP_REDUCE inside another reduce") - - if opt.op is OptOps.UPCAST: - check(rng.arg[-1] in upcast_to, f"upcast is for GLOBAL/LOCAL/LOOP/REDUCE, not {rng.arg[-1]}") - if (new_type:=upcast_to[rng.arg[-1]]) is AxisType.UNROLL: check(amt <= 32, "don't unroll more than 32") - else: check((self.ren is not None and self.ren.target.device == "DSP") or amt <= 16, "don't upcast more than 16") - if opt.op is OptOps.LOCAL: - check(rng.arg[-1] in {AxisType.GLOBAL, AxisType.WEAK}, "local is for globals") - if opt.op in {OptOps.GROUP, OptOps.GROUPTOP}: - check(all(x.op is not OptOps.TC for x in self.applied_opts), "no grouping with tensor cores") # TODO: why is this wrong? - check(rng.arg[-1] == AxisType.REDUCE, "group is for reduce") ret = self.shift_to(rng, amt, new_type, top=opt.op is OptOps.GROUPTOP) elif opt.op is OptOps.TC: check(len(self.applied_opts) == 0, "tensor core opts must be first") # TODO: remove the need for this by having warps diff --git a/tinygrad/codegen/opt/search.py b/tinygrad/codegen/opt/search.py index 2cc29d54c6185..4e505e5dfe650 100644 --- a/tinygrad/codegen/opt/search.py +++ b/tinygrad/codegen/opt/search.py @@ -12,11 +12,10 @@ from tinygrad.codegen.opt.postrange import Scheduler actions = [Opt(op=OptOps.UPCAST, axis=axis, arg=amt) for amt in [0,2,3,4,5,7] for axis in range(10)] -actions += [Opt(op=OptOps.LOCAL, axis=axis, arg=amt) for amt in [2,3,4,8,13,16,29] for axis in range(6)] -actions += [Opt(op=OptOps.GROUPTOP, axis=axis, arg=amt) for amt in [13,16,28,29,32,49,64,256] for axis in range(3)] -actions += [Opt(op=OptOps.GROUP, axis=axis, arg=amt) for amt in [0,4,8,16] for axis in range(3)] +actions += [Opt(op=OptOps.LOCAL, axis=axis, arg=amt) for amt in [0,2,3,4,8,13,16,29] for axis in range(8)] +actions += [Opt(op=OptOps.GROUPTOP, axis=axis, arg=amt) for amt in [13,16,28,29,32,49,64,256] for axis in range(8)] if getenv("BEAM_PADTO", 0): actions += [Opt(op=OptOps.PADTO, axis=axis, arg=amt) for amt in [32] for axis in range(7)] -actions += [Opt(op=OptOps.LOCAL, axis=0, arg=32), Opt(op=OptOps.LOCAL, axis=6, arg=2)] +actions += [Opt(op=OptOps.LOCAL, axis=0, arg=32)] actions += [Opt(op=OptOps.TC, axis=0, arg=(-1, 0, getenv("TC", 1)))] # covers resnet kernels (3 global * 3 reduce) actions += [Opt(op=OptOps.TC, axis=axis, arg=(-1, getenv("TC_OPT", 2), getenv("TC", 1))) for axis in range(9)]