diff --git a/src/common/MLP.py b/src/common/MLP.py new file mode 100644 index 0000000..28fbbd1 --- /dev/null +++ b/src/common/MLP.py @@ -0,0 +1,61 @@ +import random +from typing import Any + +from common.autograd import Value +from common.Module import Module + + +class Neuron(Module): # type: ignore[misc] + def __init__(self, nin: int, nonlin: bool = True) -> None: + super().__init__() + self.w = [Value(random.uniform(-1, 1)) for _ in range(nin)] + self.b = Value(0) + self.nonlin = nonlin + + def forward(self, x: list[Value]) -> Value: + act = sum((wi * xi for wi, xi in zip(self.w, x)), self.b) + return act.relu() if self.nonlin else act + + def parameters(self) -> list[Value]: + return self.w + [self.b] + + def __repr__(self) -> str: + return f"{'ReLU' if self.nonlin else 'Linear'}Neuron({len(self.w)})" + + +class Layer(Module): # type: ignore[misc] + def __init__(self, nin: int, nout: int, **kwargs: Any) -> None: + super().__init__() + self.neurons = [Neuron(nin, **kwargs) for _ in range(nout)] + + def forward(self, x: list[Value]) -> Value | list[Value]: + out = [n(x) for n in self.neurons] + return out[0] if len(out) == 1 else out + + def parameters(self) -> list[Value]: + return [p for n in self.neurons for p in n.parameters()] + + def __repr__(self) -> str: + return f"Layer of [{', '.join(str(n) for n in self.neurons)}]" + + +class MLP(Module): # type: ignore[misc] + def __init__(self, nin: int, nouts: list[int]) -> None: + super().__init__() + sz = [nin] + nouts + self.layers = [ + Layer(sz[i], sz[i + 1], nonlin=i != len(nouts) - 1) + for i in range(len(nouts)) + ] + + def forward(self, x: list[Value]) -> Any: + current: Any = x + for layer in self.layers: + current = layer(current) + return current + + def parameters(self) -> list[Value]: + return [p for layer in self.layers for p in layer.parameters()] + + def __repr__(self) -> str: + return f"MLP of [{', '.join(str(layer) for layer in self.layers)}]" diff --git a/src/common/MLPexample.py b/src/common/MLPexample.py new file mode 100644 index 0000000..18f56dd --- /dev/null +++ b/src/common/MLPexample.py @@ -0,0 +1,74 @@ +import random + +from common.autograd import Value +from common.MLP import MLP + + +# Helper to convert autograd Value nodes to plain Python floats for printing. +def to_scalar(v: Value) -> float: + return float(v.data) + + +def main() -> None: + # Phase 1: Define the supervised training data. + # XOR truth table: [x1, x2] -> y + dataset: list[tuple[list[float], float]] = [ + ([0.0, 0.0], 0.0), + ([0.0, 1.0], 1.0), + ([1.0, 0.0], 1.0), + ([1.0, 1.0], 0.0), + ] + + # Phase 2: Initialize model + training hyperparameters. + random.seed(42) + model = MLP(nin=2, nouts=[3, 1]) # 2 inputs -> hidden layer(3) -> output(1) + # Start from smaller weights for more stable early training. + for p in model.parameters(): + p.data *= 0.1 + lr = 0.01 # learning rate for optimizer updates + epochs = 3000 # full passes through the dataset + + # Phase 3: Train with full-batch gradient descent. + # Optimizer: manual SGD-style parameter update (p.data -= lr * p.grad). + # Loss: sum of squared errors over all samples (MSE-style objective without averaging). + + for epoch in range(epochs): + # Reset epoch loss accumulator. + loss = Value(0.0) + for x_raw, y_raw in dataset: + # Forward phase: wrap raw inputs/target into Value nodes for autograd. + x = [Value(x_raw[0]), Value(x_raw[1])] + y = Value(y_raw) + pred = model(x) + # Loss phase: accumulate squared error for this sample. + loss = loss + (pred - y) ** 2 + + # Backward phase: clear old gradients, then compute fresh gradients. + for p in model.parameters(): + p.grad = 0 + loss.backward() + # Optimization phase: gradient clipping + SGD update step. + for p in model.parameters(): + p.grad = p.grad.clip(-1.0, 1.0) + p.data -= lr * p.grad + + # Monitoring phase: print training loss periodically. + if epoch % 500 == 0 or epoch == epochs - 1: + print(f"epoch={epoch:4d} loss={to_scalar(loss):.6f}") + + # Phase 4: Evaluate trained model on the XOR dataset. + print("\nXOR predictions:") + for x_raw, y_raw in dataset: + x = [Value(x_raw[0]), Value(x_raw[1])] + pred = model(x) + pred_value = to_scalar(pred) + # Convert regression output to binary class using 0.5 threshold. + pred_label = 1 if pred_value > 0.5 else 0 + print(f"x={x_raw} target={int(y_raw)} pred={pred_value:.4f} class={pred_label}") + + +if __name__ == "__main__": + main() + + # ACHIVES 100 percent accuracry yay + # NOTE the gradient clipping was a must though because the grads were blowing up. diff --git a/src/common/Module.py b/src/common/Module.py index c1c3464..99c5a6e 100644 --- a/src/common/Module.py +++ b/src/common/Module.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, Generator, Union +from typing import Any, Generator from common.autograd import Value @@ -32,10 +32,12 @@ def forward(self, x): .. note:: As per the example above, an ``__init__()`` call to the parent class must be made before assignment on the child.""" + def __init__(self) -> None: - self.params: dict[str, Value | "Module"] = {} # real param store: name -> Value or Module - - + self.params: dict[ + str, Value | "Module" + ] = {} # real param store: name -> Value or Module + @abstractmethod def forward(self, *args: Any, **kwargs: Any) -> Value: """Compute the forward pass for this module.""" @@ -66,7 +68,7 @@ def __setattr__(self, key: str, value: Any) -> None: if isinstance(value, (Value, Module)): self.params[key] = value - def parameters(self) -> Generator[Value,None,None]: + def parameters(self) -> Generator[Value, None, None]: # recursively collects parameters and yields them (use a Generator) for key, param in self.params.items(): if isinstance(param, Module): diff --git a/src/common/autograd.py b/src/common/autograd.py index b4870de..a1b27c7 100644 --- a/src/common/autograd.py +++ b/src/common/autograd.py @@ -155,7 +155,7 @@ def _backward() -> None: return out def relu(self) -> "Value": - out = Value(0 if self.data.any() < 0 else self.data, (self,), "ReLU") + out = Value(np.where(self.data < 0, 0, self.data), (self,), "ReLU") def _backward() -> None: self.grad += (out.data > 0) * out.grad