From f34f7b6afea86511b34e9e50ec5cf3cff43a9d07 Mon Sep 17 00:00:00 2001 From: Insop <1240382+insop@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:18:34 -0700 Subject: [PATCH 1/4] Add test code for Python implementation of transformer --- For more details, open the [Copilot Workspace session](https://copilot-workspace-dev.githubnext.com/insop/transformer_simple?shareId=XXXX-XXXX-XXXX-XXXX). --- README.md | 10 +++++++ src/python/test_transformer.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/python/test_transformer.py diff --git a/README.md b/README.md index a72d9ea..432adc9 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,16 @@ python ./experiments/classify.py --random-seed=1234 --num-epochs=1 --tiny ``` +# Tests + +## Python implementation tests +To run the tests for the Python implementation of the transformer, execute the following command: + +``` +cd src/python +python -m unittest test_transformer.py +``` + # TODO: - main Makefile for build library and c executable - add config load diff --git a/src/python/test_transformer.py b/src/python/test_transformer.py new file mode 100644 index 0000000..76fb889 --- /dev/null +++ b/src/python/test_transformer.py @@ -0,0 +1,50 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import unittest +from transformer_simple import SelfAttention_naive, MultiHeadAttention_naive, TransformerBlock_naive + +class TestSelfAttentionNaive(unittest.TestCase): + def setUp(self): + self.dim_emb = 4 + self.dim_internal = 3 + self.heads = 8 + self.n_seq = 2 + self.dtype = torch.float32 + self.model = SelfAttention_naive(self.dim_emb, self.dim_internal, self.heads, dtype=self.dtype) + self.x = torch.ones([1, self.n_seq, self.dim_emb], dtype=self.dtype) + + def test_forward(self): + output = self.model(self.x) + self.assertEqual(output.shape, (1, self.n_seq, self.dim_internal)) + +class TestMultiHeadAttentionNaive(unittest.TestCase): + def setUp(self): + self.dim_emb = 4 + self.dim_internal = 3 + self.heads = 8 + self.n_seq = 2 + self.dtype = torch.float32 + self.model = MultiHeadAttention_naive(self.n_seq, self.dim_emb, self.dim_internal, self.heads, dtype=self.dtype) + self.x = torch.ones([1, self.n_seq, self.dim_emb], dtype=self.dtype) + + def test_forward(self): + output = self.model(self.x) + self.assertEqual(output.shape, (1, self.n_seq, self.dim_emb)) + +class TestTransformerBlockNaive(unittest.TestCase): + def setUp(self): + self.dim_emb = 4 + self.dim_internal = 3 + self.heads = 8 + self.n_seq = 2 + self.dtype = torch.float32 + self.model = TransformerBlock_naive(self.n_seq, self.dim_emb, self.dim_internal, self.heads, dtype=self.dtype) + self.x = torch.ones([1, self.n_seq, self.dim_emb], dtype=self.dtype) + + def test_forward(self): + output = self.model(self.x) + self.assertEqual(output.shape, (1, self.n_seq, self.dim_emb)) + +if __name__ == '__main__': + unittest.main() From 979819d7f24aa1705e8309ddd066f7dea7dedb36 Mon Sep 17 00:00:00 2001 From: Insop <1240382+insop@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:20:53 -0700 Subject: [PATCH 2/4] --- src/c/test_transformer.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/c/test_transformer.c diff --git a/src/c/test_transformer.c b/src/c/test_transformer.c new file mode 100644 index 0000000..8265b3c --- /dev/null +++ b/src/c/test_transformer.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include "transformer.h" + +void test_self_attention() { + struct Transformer *trfm = init_transformer(); + assert(trfm); + + Tensor x, z; + x = MatDim(trfm->n_seq, trfm->dim_embedding); + MatFill(x, 4); + + z = self_attention(&trfm->sa[0], x); + + assert(MatRows(z) == trfm->n_seq); + assert(MatCols(z) == trfm->dim_internal); + + MatUnDim(x); + MatUnDim(z); +} + +void test_multi_head_attention() { + struct Transformer *trfm = init_transformer(); + assert(trfm); + + Tensor x, z; + x = MatDim(trfm->n_seq, trfm->dim_embedding); + MatFill(x, 4); + + z = multi_head_attention(trfm, x); + + assert(MatRows(z) == trfm->n_seq); + assert(MatCols(z) == trfm->dim_embedding); + + MatUnDim(x); + MatUnDim(z); +} + +void test_transformer_block() { + struct Transformer *trfm = init_transformer(); + assert(trfm); + + Tensor x, z; + x = MatDim(trfm->n_seq, trfm->dim_embedding); + MatFill(x, 4); + + z = transformer_block(trfm, x); + + assert(MatRows(z) == trfm->n_seq); + assert(MatCols(z) == trfm->dim_embedding); + + MatUnDim(x); + MatUnDim(z); +} + +int main() { + test_self_attention(); + test_multi_head_attention(); + test_transformer_block(); + + printf("All tests passed.\n"); + return 0; +} From 62d4e97fc840132ddc88a3672c83a86a3697b345 Mon Sep 17 00:00:00 2001 From: Insop <1240382+insop@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:25:23 -0700 Subject: [PATCH 3/4] Add instructions for running C implementation tests in README.md * Add a section to mention the new test file for the C implementation of the transformer * Add instructions on how to run the C tests --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 432adc9..283631a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,14 @@ cd src/python python -m unittest test_transformer.py ``` +## C implementation tests +To run the tests for the C implementation of the transformer, execute the following command: + +``` +cd src/c +./test_transformer +``` + # TODO: - main Makefile for build library and c executable - add config load From 33898c74d3e1fa3d17472b2db2181115e58224b7 Mon Sep 17 00:00:00 2001 From: Insop <1240382+insop@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:33:47 -0700 Subject: [PATCH 4/4] --- src/python/transformer_simple.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/python/transformer_simple.py b/src/python/transformer_simple.py index a8ad015..7c47c88 100644 --- a/src/python/transformer_simple.py +++ b/src/python/transformer_simple.py @@ -22,16 +22,16 @@ def init_weight_zavier(x): nn.init.constant_(x.bias, 0) class SelfAttention_naive(nn.Module): - def __init__(self, dim_emb, dim_internal, heads=8, mask=False, dropout=0.0, dtype=torch.float32): - """ - A single self attention block + """ + A single self attention block - :param dim_emb: embedding dimension - :param dim_internal: dimension of internal representation, usually the same as dim_emb - :param head: number of multi head - :param mask + :param dim_emb: embedding dimension + :param dim_internal: dimension of internal representation, usually the same as dim_emb + :param head: number of multi head + :param mask - """ + """ + def __init__(self, dim_emb, dim_internal, heads=8, mask=False, dropout=0.0, dtype=torch.float32): super().__init__() self.dim_emb = dim_emb @@ -69,17 +69,17 @@ def forward(self, x): class MultiHeadAttention_naive(nn.Module): - def __init__(self, n_seq, dim_emb, dim_internal, heads=8, mask=False, dropout=0.0, dtype=torch.float32): - """ - multi head attention block + """ + multi head attention block - :param n_seq: number of token sequence - :param dim_emb: embedding dimension - :param dim_internal: dimension of internal representation, usually the same as dim_emb - :param head: number of multi head - :param mask + :param n_seq: number of token sequence + :param dim_emb: embedding dimension + :param dim_internal: dimension of internal representation, usually the same as dim_emb + :param head: number of multi head + :param mask - """ + """ + def __init__(self, n_seq, dim_emb, dim_internal, heads=8, mask=False, dropout=0.0, dtype=torch.float32): super().__init__() self.n_seq = n_seq @@ -104,10 +104,10 @@ def forward(self, x): class TransformerBlock_naive(nn.Module): + """ + :ff_hidden_mult: number of multiples of embedding for total hidden size + """ def __init__(self, n_seq, dim_emb, dim_internal, heads=8, mask=False, ff_hidden_mult=4, dropout=0.0, dtype=torch.float32): - """ - :ff_hidden_mult: number of multiples of embedding for total hidden size - """ super().__init__() self.mha = MultiHeadAttention_naive(n_seq=n_seq, dim_emb=dim_emb, dim_internal=dim_internal, heads=heads, mask=mask, dropout=dropout, dtype=dtype)