From 33367c81d9e0245dcf102ad6d9ba61f8037e5322 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 10:26:46 -0400 Subject: [PATCH 1/8] ci(sec): add pip audit and detect secrets in precommit and ci Signed-off-by: whatsacomputertho --- .github/workflows/doc.yaml | 2 +- .github/workflows/sec.yaml | 10 ++ .github/workflows/test.yaml | 2 +- .pre-commit-config.yaml | 11 ++ .secrets.baseline | 222 ++++++++++++++++++++++++++++++++++++ Makefile | 11 ++ requirements.sec.txt | 1 + tox.ini | 5 +- 8 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/sec.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets.baseline create mode 100644 requirements.sec.txt diff --git a/.github/workflows/doc.yaml b/.github/workflows/doc.yaml index 5357a4b..9cdd5de 100644 --- a/.github/workflows/doc.yaml +++ b/.github/workflows/doc.yaml @@ -6,7 +6,7 @@ permissions: contents: write jobs: - docs: + doc: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/sec.yaml b/.github/workflows/sec.yaml new file mode 100644 index 0000000..d79aa97 --- /dev/null +++ b/.github/workflows/sec.yaml @@ -0,0 +1,10 @@ +jobs: + sec: + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: | + make sec-dependencies + - name: Security Scan + run: | + make sec \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 49dc8ed..b4c44ff 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,7 +5,7 @@ on: branches: '**' jobs: - build: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..229f5e4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pypa/pip-audit + rev: v2.9.0 + hooks: + - id: pip-audit + args: ["-r", "requirements.txt"] + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] \ No newline at end of file diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..f8917cb --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,222 @@ +{ + "exclude": { + "files": null, + "lines": null + }, + "generated_at": "2025-04-27T13:48:44Z", + "plugins_used": [ + { + "name": "AWSKeyDetector" + }, + { + "name": "ArtifactoryDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "base64_limit": 4.5, + "name": "Base64HighEntropyString" + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "BoxDetector" + }, + { + "name": "CloudantDetector" + }, + { + "ghe_instance": "github.ibm.com", + "name": "GheDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "hex_limit": 3, + "name": "HexHighEntropyString" + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "keyword_exclude": null, + "name": "KeywordDetector" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "results": { + "README.md": [ + { + "hashed_secret": "df243b639b13750f52c874242c9543dbb772a1d2", + "is_secret": false, + "is_verified": false, + "line_number": 25, + "type": "Secret Keyword", + "verified_result": null + } + ], + "doc/source/index.rst": [ + { + "hashed_secret": "df243b639b13750f52c874242c9543dbb772a1d2", + "is_secret": false, + "is_verified": false, + "line_number": 37, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tests/registryclient_test.py": [ + { + "hashed_secret": "8fa058269d612d026cfd3b9f9db5d1e6f4109ea7", + "is_secret": false, + "is_verified": false, + "line_number": 12, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f953298876f062f1e31ec1a795f2013db8825b00", + "is_secret": false, + "is_verified": false, + "line_number": 14, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "2a950a36e6109e987e5a6303b2aaeafc5c78cfaf", + "is_secret": false, + "is_verified": false, + "line_number": 18, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "a34febb59b638f5eaa9d232f6eed227357cfeffc", + "is_secret": false, + "is_verified": false, + "line_number": 20, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "0ecf76737a7d031c90db96feca95db9fcedd8b4c", + "is_secret": false, + "is_verified": false, + "line_number": 24, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "93a57d5e27f7c0ed8c950d83986366da7b10ae98", + "is_secret": false, + "is_verified": false, + "line_number": 26, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "2121f1d1daf940a357c2ec7538f04c5f8cc979a0", + "is_secret": false, + "is_verified": false, + "line_number": 30, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "f4e9087e6434ec8abaed659126e293ef4d8b24c8", + "is_secret": false, + "is_verified": false, + "line_number": 32, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "0884ba5d1b69ceb6d963c48036b62fb57a661835", + "is_secret": false, + "is_verified": false, + "line_number": 36, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "ac3b246a80af47c4758b21b0a7c58c3421d9777e", + "is_secret": false, + "is_verified": false, + "line_number": 38, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "86b685dabe1ad6bb5ba579c743c20503823992fb", + "is_secret": false, + "is_verified": false, + "line_number": 42, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "7f12808de3d4b2d893ce1dc145a05568f4e6b95a", + "is_secret": false, + "is_verified": false, + "line_number": 44, + "type": "Secret Keyword", + "verified_result": null + } + ], + "tests/registryclientmock.py": [ + { + "hashed_secret": "ad39374651568c2292d4a354ebfff23364bad68f", + "is_secret": false, + "is_verified": false, + "line_number": 80, + "type": "Secret Keyword", + "verified_result": null + }, + { + "hashed_secret": "206c80413b9a96c1312cc346b7d2517b84463edd", + "is_secret": false, + "is_verified": false, + "line_number": 82, + "type": "Secret Keyword", + "verified_result": null + } + ] + }, + "version": "0.13.1+ibm.62.dss", + "word_list": { + "file": null, + "hash": null + } +} diff --git a/Makefile b/Makefile index d40a634..b114ad5 100644 --- a/Makefile +++ b/Makefile @@ -34,3 +34,14 @@ doc-dependencies: doc: $(PYTHON) -m sphinx.ext.apidoc -o ./doc/source/image . "tests/*" $(PYTHON) -m sphinx ./doc/source ./doc/sphinx + +########## +# Security recipes +# +# Install required dependencies for the sec recipe +sec-dependencies: + $(PYTHON) -m pip install -r requirements.sec.txt + +# Security scan locally and in CI +sec: + $(PYTHON) -m detect_secrets.pre_commit_hook --baseline .secrets.baseline diff --git a/requirements.sec.txt b/requirements.sec.txt new file mode 100644 index 0000000..43957a4 --- /dev/null +++ b/requirements.sec.txt @@ -0,0 +1 @@ +detect-secrets \ No newline at end of file diff --git a/tox.ini b/tox.ini index ad2b1d1..9e824f4 100644 --- a/tox.ini +++ b/tox.ini @@ -6,5 +6,8 @@ deps = pytest pytest-mock pytest-cov + pip-audit # the command used to run the tests for the given python version -commands = pytest -vv --cov --cov-config=.coveragerc --cov-report=term --cov-report=json \ No newline at end of file +commands = + pytest -vv --cov --cov-config=.coveragerc --cov-report=term --cov-report=json + pip-audit -r requirements.txt \ No newline at end of file From 87473a46e5c9b79e4c73b2c4421ec6a8ac272a96 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 10:44:05 -0400 Subject: [PATCH 2/8] fix(ci): move pip-audit to sec recipe, fix sec workflow Signed-off-by: whatsacomputertho --- .github/workflows/sec.yaml | 6 ++++++ Makefile | 1 + requirements.sec.txt | 3 ++- tox.ini | 5 +---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sec.yaml b/.github/workflows/sec.yaml index d79aa97..84f8702 100644 --- a/.github/workflows/sec.yaml +++ b/.github/workflows/sec.yaml @@ -1,3 +1,9 @@ +name: Test + +on: + push: + branches: '**' + jobs: sec: steps: diff --git a/Makefile b/Makefile index b114ad5..4bb82d8 100644 --- a/Makefile +++ b/Makefile @@ -45,3 +45,4 @@ sec-dependencies: # Security scan locally and in CI sec: $(PYTHON) -m detect_secrets.pre_commit_hook --baseline .secrets.baseline + $(PYTHON) -m pip_audit -r requirements.txt diff --git a/requirements.sec.txt b/requirements.sec.txt index 43957a4..1b4b20e 100644 --- a/requirements.sec.txt +++ b/requirements.sec.txt @@ -1 +1,2 @@ -detect-secrets \ No newline at end of file +detect-secrets +pip-audit \ No newline at end of file diff --git a/tox.ini b/tox.ini index 9e824f4..ad2b1d1 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,5 @@ deps = pytest pytest-mock pytest-cov - pip-audit # the command used to run the tests for the given python version -commands = - pytest -vv --cov --cov-config=.coveragerc --cov-report=term --cov-report=json - pip-audit -r requirements.txt \ No newline at end of file +commands = pytest -vv --cov --cov-config=.coveragerc --cov-report=term --cov-report=json \ No newline at end of file From cfcebb61bb2b0c22b90976fb5e60e89f995f62a6 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 10:45:30 -0400 Subject: [PATCH 3/8] fix(ci): rename sec recipe Signed-off-by: whatsacomputertho --- .github/workflows/sec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sec.yaml b/.github/workflows/sec.yaml index 84f8702..22c3d73 100644 --- a/.github/workflows/sec.yaml +++ b/.github/workflows/sec.yaml @@ -1,4 +1,4 @@ -name: Test +name: Sec on: push: From 4ed4f22582025b3ad00028c7b5f80ec37c61de7a Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 10:47:32 -0400 Subject: [PATCH 4/8] fix(ci): add runs-on to sec workflow, fix doc workflow Signed-off-by: whatsacomputertho --- .github/workflows/doc.yaml | 2 +- .github/workflows/sec.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/doc.yaml b/.github/workflows/doc.yaml index 9cdd5de..7ab37c1 100644 --- a/.github/workflows/doc.yaml +++ b/.github/workflows/doc.yaml @@ -1,6 +1,6 @@ name: Doc -on: [push, pull_request, workflow_dispatch] +on: [push, workflow_dispatch] permissions: contents: write diff --git a/.github/workflows/sec.yaml b/.github/workflows/sec.yaml index 22c3d73..ce98818 100644 --- a/.github/workflows/sec.yaml +++ b/.github/workflows/sec.yaml @@ -6,6 +6,7 @@ on: jobs: sec: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install Dependencies From 7a02798cb6772c353a59d74fac578d7d76d05666 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 11:03:15 -0400 Subject: [PATCH 5/8] fix(ci): attempt to give detect-secrets write access in ci Signed-off-by: whatsacomputertho --- .github/workflows/sec.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sec.yaml b/.github/workflows/sec.yaml index ce98818..9e15b93 100644 --- a/.github/workflows/sec.yaml +++ b/.github/workflows/sec.yaml @@ -4,6 +4,9 @@ on: push: branches: '**' +permissions: + contents: write + jobs: sec: runs-on: ubuntu-latest From cc9da26ac2fe03d4984239ed6bab0cf021c34662 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 11:05:15 -0400 Subject: [PATCH 6/8] debug(ci): enable verbose logging in detect-secrets Signed-off-by: whatsacomputertho --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4bb82d8..51ff08a 100644 --- a/Makefile +++ b/Makefile @@ -44,5 +44,5 @@ sec-dependencies: # Security scan locally and in CI sec: - $(PYTHON) -m detect_secrets.pre_commit_hook --baseline .secrets.baseline + $(PYTHON) -m detect_secrets.pre_commit_hook --baseline .secrets.baseline -v $(PYTHON) -m pip_audit -r requirements.txt From e9bbf4e1f93cc6496c78e2c2ee2100cedc2d8e31 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 11:12:05 -0400 Subject: [PATCH 7/8] fix(ci): re-generate secrets baseline using latest detect-secrets Signed-off-by: whatsacomputertho --- .secrets.baseline | 196 ++++++++++++++++++++-------------------------- 1 file changed, 83 insertions(+), 113 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index f8917cb..1433dd2 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,42 +1,37 @@ { - "exclude": { - "files": null, - "lines": null - }, - "generated_at": "2025-04-27T13:48:44Z", + "version": "1.5.0", "plugins_used": [ { - "name": "AWSKeyDetector" + "name": "ArtifactoryDetector" }, { - "name": "ArtifactoryDetector" + "name": "AWSKeyDetector" }, { "name": "AzureStorageKeyDetector" }, { - "base64_limit": 4.5, - "name": "Base64HighEntropyString" + "name": "Base64HighEntropyString", + "limit": 4.5 }, { "name": "BasicAuthDetector" }, - { - "name": "BoxDetector" - }, { "name": "CloudantDetector" }, { - "ghe_instance": "github.ibm.com", - "name": "GheDetector" + "name": "DiscordBotTokenDetector" }, { "name": "GitHubTokenDetector" }, { - "hex_limit": 3, - "name": "HexHighEntropyString" + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 }, { "name": "IbmCloudIamDetector" @@ -44,12 +39,15 @@ { "name": "IbmCosHmacDetector" }, + { + "name": "IPPublicDetector" + }, { "name": "JwtTokenDetector" }, { - "keyword_exclude": null, - "name": "KeywordDetector" + "name": "KeywordDetector", + "keyword_exclude": "" }, { "name": "MailchimpDetector" @@ -57,9 +55,18 @@ { "name": "NpmDetector" }, + { + "name": "OpenAIDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, { "name": "SlackDetector" }, @@ -72,151 +79,114 @@ { "name": "StripeDetector" }, + { + "name": "TelegramBotTokenDetector" + }, { "name": "TwilioKeyDetector" } ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], "results": { - "README.md": [ - { - "hashed_secret": "df243b639b13750f52c874242c9543dbb772a1d2", - "is_secret": false, - "is_verified": false, - "line_number": 25, - "type": "Secret Keyword", - "verified_result": null - } - ], - "doc/source/index.rst": [ - { - "hashed_secret": "df243b639b13750f52c874242c9543dbb772a1d2", - "is_secret": false, - "is_verified": false, - "line_number": 37, - "type": "Secret Keyword", - "verified_result": null - } - ], "tests/registryclient_test.py": [ { - "hashed_secret": "8fa058269d612d026cfd3b9f9db5d1e6f4109ea7", - "is_secret": false, - "is_verified": false, - "line_number": 12, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "f953298876f062f1e31ec1a795f2013db8825b00", - "is_secret": false, "is_verified": false, "line_number": 14, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false }, { - "hashed_secret": "2a950a36e6109e987e5a6303b2aaeafc5c78cfaf", - "is_secret": false, - "is_verified": false, - "line_number": 18, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "a34febb59b638f5eaa9d232f6eed227357cfeffc", - "is_secret": false, "is_verified": false, "line_number": 20, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false }, { - "hashed_secret": "0ecf76737a7d031c90db96feca95db9fcedd8b4c", - "is_secret": false, - "is_verified": false, - "line_number": 24, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "93a57d5e27f7c0ed8c950d83986366da7b10ae98", - "is_secret": false, "is_verified": false, "line_number": 26, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false }, { - "hashed_secret": "2121f1d1daf940a357c2ec7538f04c5f8cc979a0", - "is_secret": false, - "is_verified": false, - "line_number": 30, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "f4e9087e6434ec8abaed659126e293ef4d8b24c8", - "is_secret": false, "is_verified": false, "line_number": 32, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false }, { - "hashed_secret": "0884ba5d1b69ceb6d963c48036b62fb57a661835", - "is_secret": false, - "is_verified": false, - "line_number": 36, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "ac3b246a80af47c4758b21b0a7c58c3421d9777e", - "is_secret": false, "is_verified": false, "line_number": 38, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false }, { - "hashed_secret": "86b685dabe1ad6bb5ba579c743c20503823992fb", - "is_secret": false, - "is_verified": false, - "line_number": 42, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclient_test.py", "hashed_secret": "7f12808de3d4b2d893ce1dc145a05568f4e6b95a", - "is_secret": false, "is_verified": false, "line_number": 44, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false } ], "tests/registryclientmock.py": [ { - "hashed_secret": "ad39374651568c2292d4a354ebfff23364bad68f", - "is_secret": false, - "is_verified": false, - "line_number": 80, "type": "Secret Keyword", - "verified_result": null - }, - { + "filename": "tests/registryclientmock.py", "hashed_secret": "206c80413b9a96c1312cc346b7d2517b84463edd", - "is_secret": false, "is_verified": false, "line_number": 82, - "type": "Secret Keyword", - "verified_result": null + "is_secret": false } ] }, - "version": "0.13.1+ibm.62.dss", - "word_list": { - "file": null, - "hash": null - } + "generated_at": "2025-04-27T15:11:17Z" } From c9a3cec92ef5a50f7c9e7b8171d9fd851be1d985 Mon Sep 17 00:00:00 2001 From: whatsacomputertho Date: Sun, 27 Apr 2025 11:18:20 -0400 Subject: [PATCH 8/8] feat(precommit): install pre commit hooks in sec-dependencies recipe Signed-off-by: whatsacomputertho --- Makefile | 1 + requirements.sec.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 51ff08a..7aa4fbe 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ doc: # Install required dependencies for the sec recipe sec-dependencies: $(PYTHON) -m pip install -r requirements.sec.txt + $(PYTHON) -m pre_commit install # Security scan locally and in CI sec: diff --git a/requirements.sec.txt b/requirements.sec.txt index 1b4b20e..b54c600 100644 --- a/requirements.sec.txt +++ b/requirements.sec.txt @@ -1,2 +1,3 @@ +pre-commit detect-secrets pip-audit \ No newline at end of file