Mlp example

src/common/MLP.py

@@ -0,0 +1,61 @@
+import random
+from typing import Any
+
+from common.autograd import Value
+from common.Module import Module
+
+
+class Neuron(Module):  # type: ignore[misc]
+    def __init__(self, nin: int, nonlin: bool = True) -> None:
+        super().__init__()
+        self.w = [Value(random.uniform(-1, 1)) for _ in range(nin)]
+        self.b = Value(0)
+        self.nonlin = nonlin
+
+    def forward(self, x: list[Value]) -> Value:
+        act = sum((wi * xi for wi, xi in zip(self.w, x)), self.b)
+        return act.relu() if self.nonlin else act
+
+    def parameters(self) -> list[Value]:
+        return self.w + [self.b]
+
+    def __repr__(self) -> str:
+        return f"{'ReLU' if self.nonlin else 'Linear'}Neuron({len(self.w)})"
+
+
+class Layer(Module):  # type: ignore[misc]
+    def __init__(self, nin: int, nout: int, **kwargs: Any) -> None:
+        super().__init__()
+        self.neurons = [Neuron(nin, **kwargs) for _ in range(nout)]
+
+    def forward(self, x: list[Value]) -> Value | list[Value]:
+        out = [n(x) for n in self.neurons]
+        return out[0] if len(out) == 1 else out
+
+    def parameters(self) -> list[Value]:
+        return [p for n in self.neurons for p in n.parameters()]
+
+    def __repr__(self) -> str:
+        return f"Layer of [{', '.join(str(n) for n in self.neurons)}]"
+
+
+class MLP(Module):  # type: ignore[misc]
+    def __init__(self, nin: int, nouts: list[int]) -> None:
+        super().__init__()
+        sz = [nin] + nouts
+        self.layers = [
+            Layer(sz[i], sz[i + 1], nonlin=i != len(nouts) - 1)
+            for i in range(len(nouts))
+        ]
+
+    def forward(self, x: list[Value]) -> Any:
+        current: Any = x
+        for layer in self.layers:
+            current = layer(current)
+        return current
+
+    def parameters(self) -> list[Value]:
+        return [p for layer in self.layers for p in layer.parameters()]
+
+    def __repr__(self) -> str:
+        return f"MLP of [{', '.join(str(layer) for layer in self.layers)}]"

src/common/MLPexample.py

@@ -0,0 +1,74 @@
+import random
+
+from common.autograd import Value
+from common.MLP import MLP
+
+
+# Helper to convert autograd Value nodes to plain Python floats for printing.
+def to_scalar(v: Value) -> float:
+    return float(v.data)
+
+
+def main() -> None:
+    # Phase 1: Define the supervised training data.
+    # XOR truth table: [x1, x2] -> y
+    dataset: list[tuple[list[float], float]] = [
+        ([0.0, 0.0], 0.0),
+        ([0.0, 1.0], 1.0),
+        ([1.0, 0.0], 1.0),
+        ([1.0, 1.0], 0.0),
+    ]
+
+    # Phase 2: Initialize model + training hyperparameters.
+    random.seed(42)
+    model = MLP(nin=2, nouts=[3, 1])  # 2 inputs -> hidden layer(3) -> output(1)
+    # Start from smaller weights for more stable early training.
+    for p in model.parameters():
+        p.data *= 0.1
+    lr = 0.01  # learning rate for optimizer updates
+    epochs = 3000  # full passes through the dataset
+
+    # Phase 3: Train with full-batch gradient descent.
+    # Optimizer: manual SGD-style parameter update (p.data -= lr * p.grad).
+    # Loss: sum of squared errors over all samples (MSE-style objective without averaging).
+
+    for epoch in range(epochs):
+        # Reset epoch loss accumulator.
+        loss = Value(0.0)
+        for x_raw, y_raw in dataset:
+            # Forward phase: wrap raw inputs/target into Value nodes for autograd.
+            x = [Value(x_raw[0]), Value(x_raw[1])]
+            y = Value(y_raw)
+            pred = model(x)
+            # Loss phase: accumulate squared error for this sample.
+            loss = loss + (pred - y) ** 2
+
+        # Backward phase: clear old gradients, then compute fresh gradients.
+        for p in model.parameters():
+            p.grad = 0
+        loss.backward()
+        # Optimization phase: gradient clipping + SGD update step.
+        for p in model.parameters():
+            p.grad = p.grad.clip(-1.0, 1.0)
+            p.data -= lr * p.grad
+
+        # Monitoring phase: print training loss periodically.
+        if epoch % 500 == 0 or epoch == epochs - 1:
+            print(f"epoch={epoch:4d} loss={to_scalar(loss):.6f}")
+
+    # Phase 4: Evaluate trained model on the XOR dataset.
+    print("\nXOR predictions:")
+    for x_raw, y_raw in dataset:
+        x = [Value(x_raw[0]), Value(x_raw[1])]
+        pred = model(x)
+        pred_value = to_scalar(pred)
+        # Convert regression output to binary class using 0.5 threshold.
+        pred_label = 1 if pred_value > 0.5 else 0
+        print(f"x={x_raw} target={int(y_raw)} pred={pred_value:.4f} class={pred_label}")
+
+
+if __name__ == "__main__":
+    main()
+
+    # ACHIVES 100 percent accuracry yay
+    # NOTE the gradient clipping was a must though because the grads were blowing up.

src/common/Module.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Generator, Union
+from typing import Any, Generator
 
 from common.autograd import Value
 
@@ -32,10 +32,12 @@ def forward(self, x):
     .. note::
         As per the example above, an ``__init__()`` call to the parent class
         must be made before assignment on the child."""
+
     def __init__(self) -> None:
-        self.params: dict[str, Value | "Module"] = {} # real param store: name -> Value or Module
-
-
+        self.params: dict[
+            str, Value | "Module"
+        ] = {}  # real param store: name -> Value or Module
+
     @abstractmethod
     def forward(self, *args: Any, **kwargs: Any) -> Value:
         """Compute the forward pass for this module."""
@@ -66,7 +68,7 @@ def __setattr__(self, key: str, value: Any) -> None:
         if isinstance(value, (Value, Module)):
             self.params[key] = value
 
-    def parameters(self) -> Generator[Value,None,None]:
+    def parameters(self) -> Generator[Value, None, None]:
         # recursively collects parameters and yields them (use a Generator)
         for key, param in self.params.items():
             if isinstance(param, Module):

src/common/autograd.py

@@ -155,7 +155,7 @@ def _backward() -> None:
         return out
 
     def relu(self) -> "Value":
-        out = Value(0 if self.data.any() < 0 else self.data, (self,), "ReLU")
+        out = Value(np.where(self.data < 0, 0, self.data), (self,), "ReLU")
 
         def _backward() -> None:
             self.grad += (out.data > 0) * out.grad