Website Guardian: 1 issue(s) found #1011
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Threat Intelligence | ||
| on: | ||
| schedule: | ||
| - cron: '0 3 * * *' # Daily at 3AM UTC | ||
| workflow_dispatch: | ||
| jobs: | ||
| fetch-intel: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Pull threat intel feeds | ||
| run: | | ||
| mkdir -p threat-intel | ||
| # ── URLhaus (abuse.ch) — Malicious URLs ── | ||
| echo "=== URLhaus ===" | ||
| curl -sL --max-time 30 "https://urlhaus.abuse.ch/downloads/csv_recent/" -o /tmp/urlhaus.csv 2>&1 | tail -1 | ||
| if [ -s /tmp/urlhaus.csv ]; then | ||
| tail -n +9 /tmp/urlhaus.csv | head -10000 | awk -F, '{print $2}' | grep -v '^$' > threat-intel/urlhaus-urls.txt | ||
| echo " $(wc -l < threat-intel/urlhaus-urls.txt) URLs" | ||
| fi | ||
| # ── PhishTank — Phishing URLs ── | ||
| echo "=== PhishTank ===" | ||
| curl -sL --max-time 30 "https://data.phishtank.com/data/online-valid.csv" -o /tmp/phishtank.csv 2>&1 | tail -1 | ||
| if [ -s /tmp/phishtank.csv ]; then | ||
| awk -F, 'NR>1{print $5}' /tmp/phishtank.csv | grep -v '^$' | head -5000 > threat-intel/phishtank-urls.txt | ||
| echo " $(wc -l < threat-intel/phishtank-urls.txt) URLs" | ||
| fi | ||
| # ── OpenPhish — Phishing URLs ── | ||
| echo "=== OpenPhish ===" | ||
| curl -sL --max-time 30 "https://openphish.com/feed.txt" -o threat-intel/openphish-urls.txt 2>&1 | tail -1 | ||
| echo " $(wc -l < threat-intel/openphish-urls.txt) URLs" | ||
| # ── MalwareBazaar — Malware hashes ── | ||
| echo "=== MalwareBazaar ===" | ||
| curl -sL --max-time 30 -X POST "https://mb-api.abuse.ch/api/v1/" \ | ||
| -d "query=get_recent&selector=100" \ | ||
| -o /tmp/malwarebazaar.json 2>&1 | tail -1 | ||
| if [ -s /tmp/malwarebazaar.json ]; then | ||
| python3 -c " | ||
| import json | ||
| with open('/tmp/malwarebazaar.json') as f: | ||
| data = json.load(f) | ||
| hashes = set() | ||
| for d in data.get('data', []): | ||
| h = d.get('sha256_hash', '') | ||
| if h: hashes.add(h) | ||
| h = d.get('sha1_hash', '') | ||
| if h: hashes.add(h) | ||
| h = d.get('md5_hash', '') | ||
| if h: hashes.add(h) | ||
| with open('threat-intel/malwarebazaar-hashes.txt', 'w') as f: | ||
| f.write('\n'.join(sorted(hashes))) | ||
| print(f' {len(hashes)} hashes') | ||
| " 2>&1 | ||
| fi | ||
| # ── CISA Known Exploited Vulnerabilities ── | ||
| echo "=== CISA KEV ===" | ||
| curl -sL --max-time 30 "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" \ | ||
| -o /tmp/cisa-kev.json 2>&1 | tail -1 | ||
| if [ -s /tmp/cisa-kev.json ]; then | ||
| python3 -c " | ||
| import json | ||
| with open('/tmp/cisa-kev.json') as f: | ||
| data = json.load(f) | ||
| vendors = set() | ||
| for vuln in data.get('vulnerabilities', []): | ||
| v = vuln.get('vendorProject', '') | ||
| if v: vendors.add(v.lower()) | ||
| with open('threat-intel/cisa-vendors.txt', 'w') as f: | ||
| f.write('\n'.join(sorted(vendors))) | ||
| print(f' {len(vendors)} affected vendors tracked') | ||
| " 2>&1 | ||
| fi | ||
| # ── AlienVault OTX (pulse feed - no API key needed for limited) ── | ||
| echo "=== AlienVault OTX ===" | ||
| curl -sL --max-time 30 "https://otx.alienvault.com/api/v1/pulses/subscribed?limit=20" \ | ||
| -o /tmp/otx.json 2>&1 | tail -1 | ||
| if [ -s /tmp/otx.json ]; then | ||
| python3 -c " | ||
| import json | ||
| with open('/tmp/otx.json') as f: | ||
| data = json.load(f) | ||
| indicators = set() | ||
| for pulse in data.get('results', []): | ||
| for io in pulse.get('indicators', []): | ||
| ind = io.get('indicator', '') | ||
| if ind: indicators.add(ind) | ||
| with open('threat-intel/otx-indicators.txt', 'w') as f: | ||
| f.write('\n'.join(sorted(indicators))) | ||
| print(f' {len(indicators)} indicators') | ||
| " 2>&1 | ||
| fi | ||
| # ── URLScan.io recent malicious ── | ||
| echo "=== URLScan.io ===" | ||
| curl -sL --max-time 30 "https://urlscan.io/api/v1/search/?q=task.time:%3Enow-24h+AND+verdict:malicious&size=50" \ | ||
| -o /tmp/urlscan.json 2>&1 | tail -1 | ||
| if [ -s /tmp/urlscan.json ]; then | ||
| python3 -c " | ||
| import json | ||
| with open('/tmp/urlscan.json') as f: | ||
| data = json.load(f) | ||
| urls = set() | ||
| for r in data.get('results', []): | ||
| url = r.get('page', {}).get('url', '') | ||
| if url: urls.add(url) | ||
| domain = r.get('page', {}).get('domain', '') | ||
| if domain: urls.add(domain) | ||
| with open('threat-intel/urlscan-malicious.txt', 'w') as f: | ||
| f.write('\n'.join(sorted(urls))) | ||
| print(f' {len(urls)} malicious URLs/domains') | ||
| " 2>&1 | ||
| fi | ||
| # ── Merge all URL lists into one ── | ||
| cat threat-intel/*-urls.txt threat-intel/*-indicators.txt 2>/dev/null \ | ||
| | tr '[:upper:]' '[:lower:]' \ | ||
| | sed 's/^https\?:\/\///' \ | ||
| | sed 's/\/.*$//' \ | ||
| | grep -v '^$' \ | ||
| | sort -u > threat-intel/all-blocked-domains.txt | ||
| echo "=== Merged ===" | ||
| echo " $(wc -l < threat-intel/all-blocked-domains.txt) unique blocked domains" | ||
| # ── Generate status report ── | ||
| python3 << 'PYEOF' | ||
| import json, os | ||
| report = { | ||
| "updated": os.popen('date -u +%Y-%m-%dT%H:%M:%SZ').read().strip(), | ||
| "sources": {}, | ||
| "total_blocked_domains": 0 | ||
| } | ||
| for f in os.listdir('threat-intel'): | ||
| path = os.path.join('threat-intel', f) | ||
| if os.path.isfile(path): | ||
| try: | ||
| with open(path) as fh: | ||
| count = sum(1 for _ in fh) | ||
| report["sources"][f.replace('.txt','').replace('.csv','')] = count | ||
| except: | ||
| pass | ||
| if os.path.exists('threat-intel/all-blocked-domains.txt'): | ||
| with open('threat-intel/all-blocked-domains.txt') as fh: | ||
| report["total_blocked_domains"] = sum(1 for _ in fh) | ||
| with open('threat-intel/report.json', 'w') as f: | ||
| json.dump(report, f, indent=2) | ||
| print(json.dumps(report, indent=2)) | ||
| PYEOF | ||
| - name: Upload to R2 | ||
| env: | ||
| CF_ACCOUNT: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | ||
| CF_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | ||
| run: | | ||
| for f in threat-intel/*; do | ||
| name=$(basename "$f") | ||
| echo "Uploading $name..." | ||
| curl -s -X PUT \ | ||
| "https://api.cloudflare.com/client/v4/accounts/$CF_ACCOUNT/r2/buckets/acreetionos-hosting/objects/threat-intel/$name" \ | ||
| -H "Authorization: Bearer $CF_TOKEN" \ | ||
| --data-binary "@$f" \ | ||
| -H "Content-Type: text/plain" > /dev/null | ||
| done | ||
| echo "Upload complete" | ||
| - name: Commit threat intel status | ||
| run: | | ||
| cp threat-intel/report.json threat-intel-report.json 2>/dev/null || true | ||
| git config user.name "AcreetionOS Threat Intel" | ||
| git config user.email "threat-intel@acreetionos.org" | ||
| git add threat-intel-report.json -f | ||
| git diff --cached --quiet || git commit -m "Update threat intelligence report [skip ci]" | ||
| git push origin main | ||
