From c404d674cbf4547e1d92bc5303a30eae6d24dd2f Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 12:53:13 -0700 Subject: [PATCH 01/20] Jupyter Testing --- .github/workflows/jupyter.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/jupyter.yml diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml new file mode 100644 index 00000000..958ce7dd --- /dev/null +++ b/.github/workflows/jupyter.yml @@ -0,0 +1,40 @@ +name: Jupyter + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tensorflow + pip install tensorflow_data_validation + pip install tensorflow_hub + pip install tensorflow-model-analysis + + - name: Set up Jupyter + run: | + pip install jupyter + jupyter nbextension enable --py widgetsnbextension --sys-prefix + jupyter nbextension install --py --symlink tensorflow_model_analysis --sys-prefix + jupyter nbextension enable --py tensorflow_model_analysis --sys-prefix + + - name: Test Jupyter Notebooks + run: echo TODO From d8e9488db98656b0ac23a9408738c3aec71cde73 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:00:43 -0700 Subject: [PATCH 02/20] test jupyter workflow --- .github/workflows/jupyter.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index 958ce7dd..56ba2a48 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -2,9 +2,9 @@ name: Jupyter on: push: - branches: [ master ] + branches: [ master, shuklak13-patch-1 ] pull_request: - branches: [ master ] + branches: [ master, shuklak13-patch-1 ] jobs: build: From 41d87a79242c05f80574b7c087d056d62baa9f8c Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:13:11 -0700 Subject: [PATCH 03/20] Python 3.7 --- .github/workflows/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index 56ba2a48..9ad95eb2 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.7, 3.8] steps: - uses: actions/checkout@v2 From a6a9dd3fdaf9246587242bb86637a7e4757e28d6 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:24:13 -0700 Subject: [PATCH 04/20] Run Jupter workflow daily --- .github/workflows/jupyter.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index 958ce7dd..b23ec3d6 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -5,6 +5,8 @@ on: branches: [ master ] pull_request: branches: [ master ] + schedule: + - cron: '0 11 * * *' jobs: build: From 3b2315e60a1d3ba695cde536624b2a8b988404c3 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:25:09 -0700 Subject: [PATCH 05/20] 3.7 only --- .github/workflows/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index 9ad95eb2..e998704a 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8] + python-version: [3.7] steps: - uses: actions/checkout@v2 From c7c61948f21ad2619b861688395b679ecdfdbf01 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:32:21 -0700 Subject: [PATCH 06/20] add jupyter workflow badge to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e3987704..fbe11204 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Fairness Indicators BETA +https://github.com/tensorflow/fairness-indicators/workflows/jupyter/badge.svg + ![Fairness_Indicators](https://raw.githubusercontent.com/tensorflow/fairness-indicators/master/fairness_indicators/images/fairnessIndicators.png) Fairness Indicators is designed to support teams in evaluating, improving, and comparing models for fairness concerns in partnership with the broader Tensorflow toolkit. From dd2f7e8822a62089fb5a3287174ae834b6eefbb5 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:32:54 -0700 Subject: [PATCH 07/20] lowercase workflow name --- .github/workflows/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index b23ec3d6..29d8e4df 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -1,4 +1,4 @@ -name: Jupyter +name: jupyter on: push: From 54e7ce87a92676fe530dc1544c3f4bcf7c6938dd Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 13:33:50 -0700 Subject: [PATCH 08/20] fix badge formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbe11204..398b62bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Fairness Indicators BETA -https://github.com/tensorflow/fairness-indicators/workflows/jupyter/badge.svg +![](https://github.com/tensorflow/fairness-indicators/workflows/jupyter/badge.svg) ![Fairness_Indicators](https://raw.githubusercontent.com/tensorflow/fairness-indicators/master/fairness_indicators/images/fairnessIndicators.png) From cfff48d16265a253416795d1dc5e5707406fc325 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 14:05:01 -0700 Subject: [PATCH 09/20] Create notebook_runner.py --- notebook_runner.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 notebook_runner.py diff --git a/notebook_runner.py b/notebook_runner.py new file mode 100644 index 00000000..686ec6cf --- /dev/null +++ b/notebook_runner.py @@ -0,0 +1,34 @@ +import nbformat +import os + +from nbconvert.preprocessors import ExecutePreprocessor + + +def run_notebook(notebook_path): + nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) + dirname = os.path.dirname(notebook_path) + + with open(notebook_path) as f: + nb = nbformat.read(f, as_version=4) + + proc = ExecutePreprocessor(timeout=600, kernel_name='python3') + proc.allow_errors = True + + proc.preprocess(nb, {'metadata': {'path': '/'}}) + output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name)) + + with open(output_path, mode='wt') as f: + nbformat.write(nb, f) + + errors = [] + for cell in nb.cells: + if 'outputs' in cell: + for output in cell['outputs']: + if output.output_type == 'error': + errors.append(output) + + return nb, errors + +if __name__ == '__main__': + nb, errors = run_notebook('Testing.ipynb') + print(errors) From c21769740f53f46d722ff736d5946e4011dda619 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 14:05:26 -0700 Subject: [PATCH 10/20] Create test_runner.py --- test_runner.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 test_runner.py diff --git a/test_runner.py b/test_runner.py new file mode 100644 index 00000000..62c9f075 --- /dev/null +++ b/test_runner.py @@ -0,0 +1,15 @@ +import unittest + +import notebook_runner + + +class TestNotebook(unittest.TestCase): + + def test_runner(self): + errors = notebook_runner.run_notebook( + 'examples/Fairness_Indicators_Example_Colab.ipynb') + self.assertEqual(errors, []) + + +if __name__ == '__main__': + unittest.main() From a99e4869688accf46de3b0e9ca02bc54414e8832 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 14:08:40 -0700 Subject: [PATCH 11/20] Jupyter test workflow --- .github/workflows/jupyter.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index fd01834f..e65bab90 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -23,9 +23,18 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Install pip run: | python -m pip install --upgrade pip + + - name: Install testing dependencies + run: | + pip install unittest + pip install nbformat + pip install nbconvert + + - name: Install TensorFlow ecosystem + run: | pip install tensorflow pip install tensorflow_data_validation pip install tensorflow_hub @@ -39,4 +48,4 @@ jobs: jupyter nbextension enable --py tensorflow_model_analysis --sys-prefix - name: Test Jupyter Notebooks - run: echo TODO + run: python test_runner.py From 27df085c4f4549d85de2643b80bc0a7e1b8ddba3 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 14:14:57 -0700 Subject: [PATCH 12/20] remove `pip install unittest` unittest is a standard module --- .github/workflows/jupyter.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/jupyter.yml b/.github/workflows/jupyter.yml index e65bab90..af8c49f2 100644 --- a/.github/workflows/jupyter.yml +++ b/.github/workflows/jupyter.yml @@ -27,12 +27,6 @@ jobs: run: | python -m pip install --upgrade pip - - name: Install testing dependencies - run: | - pip install unittest - pip install nbformat - pip install nbconvert - - name: Install TensorFlow ecosystem run: | pip install tensorflow @@ -40,6 +34,11 @@ jobs: pip install tensorflow_hub pip install tensorflow-model-analysis + - name: Install test dependencies + run: | + pip install nbformat + pip install nbconvert + - name: Set up Jupyter run: | pip install jupyter From 385ebb973a80492f33357bb2e24af564150dbc66 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 14:33:33 -0700 Subject: [PATCH 13/20] fix test path --- test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner.py b/test_runner.py index 62c9f075..fdb2dc01 100644 --- a/test_runner.py +++ b/test_runner.py @@ -7,7 +7,7 @@ class TestNotebook(unittest.TestCase): def test_runner(self): errors = notebook_runner.run_notebook( - 'examples/Fairness_Indicators_Example_Colab.ipynb') + 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') self.assertEqual(errors, []) From 0e3d6758824d89d6bc229588c645d5440a60471c Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 15:16:55 -0700 Subject: [PATCH 14/20] remove timeout --- notebook_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook_runner.py b/notebook_runner.py index 686ec6cf..14433fb0 100644 --- a/notebook_runner.py +++ b/notebook_runner.py @@ -11,7 +11,7 @@ def run_notebook(notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) - proc = ExecutePreprocessor(timeout=600, kernel_name='python3') + proc = ExecutePreprocessor(timeout=None, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) From 70a7c78b20a64292b7032435f3cce30228eda980 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 19:46:19 -0700 Subject: [PATCH 15/20] print errors --- notebook_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/notebook_runner.py b/notebook_runner.py index 14433fb0..a7ff41e7 100644 --- a/notebook_runner.py +++ b/notebook_runner.py @@ -26,9 +26,7 @@ def run_notebook(notebook_path): for output in cell['outputs']: if output.output_type == 'error': errors.append(output) + + print(errors[0]) return nb, errors - -if __name__ == '__main__': - nb, errors = run_notebook('Testing.ipynb') - print(errors) From 1d0cceb4007eb274cfc456700bd57edf4d80ee47 Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 20:09:58 -0700 Subject: [PATCH 16/20] print errors --- notebook_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook_runner.py b/notebook_runner.py index a7ff41e7..f065e111 100644 --- a/notebook_runner.py +++ b/notebook_runner.py @@ -27,6 +27,6 @@ def run_notebook(notebook_path): if output.output_type == 'error': errors.append(output) - print(errors[0]) + print(errors) return nb, errors From d0591212aa8e5efe9bf2e32cb595da055184d24a Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 20:37:40 -0700 Subject: [PATCH 17/20] print errors --- notebook_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebook_runner.py b/notebook_runner.py index f065e111..68daf10a 100644 --- a/notebook_runner.py +++ b/notebook_runner.py @@ -27,6 +27,8 @@ def run_notebook(notebook_path): if output.output_type == 'error': errors.append(output) + print('ERRORS') + print(type(errors)) print(errors) return nb, errors From cb66a17efbb9de17f46d4fba28762534d0ed4e9b Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 20:38:09 -0700 Subject: [PATCH 18/20] print errors (test) --- test_runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test_runner.py b/test_runner.py index fdb2dc01..c01edc5e 100644 --- a/test_runner.py +++ b/test_runner.py @@ -8,6 +8,9 @@ class TestNotebook(unittest.TestCase): def test_runner(self): errors = notebook_runner.run_notebook( 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') + print('ERRORS (test)') + print(type(errors)) + print(errors) self.assertEqual(errors, []) From 6c2a5a15480c3aff6ea1503dba3c53540880681b Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 20:57:38 -0700 Subject: [PATCH 19/20] return errors --- notebook_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook_runner.py b/notebook_runner.py index 68daf10a..730aeeae 100644 --- a/notebook_runner.py +++ b/notebook_runner.py @@ -31,4 +31,4 @@ def run_notebook(notebook_path): print(type(errors)) print(errors) - return nb, errors + return errors From a0b2b15165b3c2bc34b3e5a9699d6a71f16840ac Mon Sep 17 00:00:00 2001 From: Karan Shukla Date: Wed, 15 Apr 2020 21:24:56 -0700 Subject: [PATCH 20/20] all the testz --- test_runner.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/test_runner.py b/test_runner.py index c01edc5e..dad5c691 100644 --- a/test_runner.py +++ b/test_runner.py @@ -5,13 +5,40 @@ class TestNotebook(unittest.TestCase): - def test_runner(self): + def test_example(self): errors = notebook_runner.run_notebook( 'fairness_indicators/examples/Fairness_Indicators_Example_Colab.ipynb') - print('ERRORS (test)') - print(type(errors)) - print(errors) - self.assertEqual(errors, []) + self.assertEmpty(errors) + + def test_facessd(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Facessd_Fairness_Indicators_Example_Colab.ipynb') + self.assertEmpty(errors) + + def test_lineage(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_Lineage_Case_Study.ipynb') + self.assertEmpty(errors) + + def test_tb(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_TensorBoard_Plugin_Example_Colab.ipynb') + self.assertEmpty(errors) + + def test_tfco_celeba(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_TFCO_CelebA_Case_Study.ipynb') + self.assertEmpty(errors) + + def test_tfco_wiki(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_TFCO_Wiki_Case_Study.ipynb') + self.assertEmpty(errors) + + def test_tfhub(self): + errors = notebook_runner.run_notebook( + 'fairness_indicators/examples/Fairness_Indicators_on_TF_Hub_Text_Embeddings.ipynb') + self.assertEmpty(errors) if __name__ == '__main__':