From 9f08e6190e1c98ee2b83bfed8f6bdd09ca605d21 Mon Sep 17 00:00:00 2001 From: grumo35 Date: Mon, 31 Jan 2022 11:24:04 +0100 Subject: [PATCH 01/32] Update __init__.py Added basic proxy support from environment variables, need to add this option in parameter of patchman cli. --- util/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/__init__.py b/util/__init__.py index ddbe63eb..bc669c1e 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see +from os import getenv import sys import requests @@ -33,6 +34,11 @@ from progressbar import Bar, ETA, Percentage, ProgressBar from patchman.signals import error_message +http_proxy=getenv('http_proxy') +proxies = { + 'http': http_proxy, + 'https': http_proxy, +} if ProgressBar.__dict__.get('maxval'): pbar2 = False @@ -130,7 +136,7 @@ def get_url(url): """ res = None try: - res = requests.get(url, stream=True) + res = requests.get(url,proxies=proxies, stream=True) except requests.exceptions.Timeout: error_message.send(sender=None, text='Timeout - {0!s}'.format(url)) except requests.exceptions.TooManyRedirects: From cd577bf3fdfb77f23b44f1ca0283009dd01041ef Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:48:33 -0400 Subject: [PATCH 02/32] Update __init__.py --- util/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index f27e232a..457e6782 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -29,24 +29,23 @@ from time import time from tqdm import tqdm -from patchman.signals import error_message, info_message, debug_message - -http_proxy = getenv('http_proxy') -http_proxy = getenv('https_proxy') -proxies = { - 'http': http_proxy, - 'https': https_proxy, -} - from django.utils.timezone import make_aware from django.utils.dateparse import parse_datetime from django.conf import settings +from patchman.signals import error_message, info_message, debug_message pbar = None verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') +http_proxy = getenv('http_proxy') +https_proxy = getenv('https_proxy') +proxies = { + 'http': http_proxy, + 'https': https_proxy, +} + def get_verbosity(): """ Get the global verbosity level From 80a2d60e60d9b8d947d33f5b45966c59d0158a7a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Sat, 22 Mar 2025 00:49:38 -0400 Subject: [PATCH 03/32] Update __init__.py --- util/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/__init__.py b/util/__init__.py index 457e6782..10ee21f7 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -14,13 +14,13 @@ # # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from os import getenv import requests import bz2 import magic import zlib import lzma +import os from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -39,8 +39,8 @@ verbose = None Checksum = Enum('Checksum', 'md5 sha sha1 sha256 sha512') -http_proxy = getenv('http_proxy') -https_proxy = getenv('https_proxy') +http_proxy = os.getenv('http_proxy') +https_proxy = os.getenv('https_proxy') proxies = { 'http': http_proxy, 'https': https_proxy, @@ -122,7 +122,7 @@ def get_url(url, headers={}, params={}): response = None try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') - response = requests.get(url, headers=headers, params=params, stream=True, proxies = proxies, timeout=30) + response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response From d44a2ce5d9ff30abb6caf3037307706cdbc4c5d5 Mon Sep 17 00:00:00 2001 From: "furlongm@gmail.com" Date: Wed, 23 Apr 2025 17:35:03 -0400 Subject: [PATCH 04/32] handle duplicate CVSSes better --- security/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/security/models.py b/security/models.py index a847ab02..b60acb47 100644 --- a/security/models.py +++ b/security/models.py @@ -125,19 +125,20 @@ def add_cvss_score(self, vector_string, score=None, severity=None, version=None) score = cvss_score.base_score if not severity: severity = cvss_score.severities()[0] - existing = self.cvss_scores.filter(version=version, vector_string=vector_string) - if existing: - cvss = existing.first() - else: + try: cvss, created = CVSS.objects.get_or_create( version=version, vector_string=vector_string, score=score, severity=severity, ) - cvss.score = score - cvss.severity = severity - cvss.save() + except CVSS.MultipleObjectsReturned: + matching_cvsses = CVSS.objects.filter( + version=version, + vector_string=vector_string, + ) + cvss = matching_cvsses.first() + matching_cvsses.exclude(id=cvss.id).delete() self.cvss_scores.add(cvss) def fetch_cve_data(self, fetch_nist_data=False, sleep_secs=6): From bf626c90dd697dfd7bc41f542ae55c88af296e2a Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 8 Apr 2025 15:49:31 -0400 Subject: [PATCH 05/32] reduce max charfield length for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index 5655f8b0..c22d1727 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=65535, null=True)), + ('description', models.CharField(blank=True, max_length=21844, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=65535)), + ('description', models.CharField(max_length=21844)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 382cd29ad3aeba4481fa4d14e38ce75bcff37874 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 00:05:34 -0400 Subject: [PATCH 06/32] further reduce charfield size for mysql --- security/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/migrations/0001_initial.py b/security/migrations/0001_initial.py index c22d1727..5f922c9a 100644 --- a/security/migrations/0001_initial.py +++ b/security/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cwe_id', models.CharField(max_length=255, unique=True)), ('name', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(blank=True, max_length=21844, null=True)), + ('description', models.CharField(blank=True, max_length=255, null=True)), ], ), migrations.CreateModel( @@ -36,7 +36,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('cve_id', models.CharField(max_length=255, unique=True)), ('title', models.CharField(blank=True, max_length=255, null=True)), - ('description', models.CharField(max_length=21844)), + ('description', models.CharField(max_length=255)), ('reserved_date', models.DateTimeField(blank=True, null=True)), ('published_date', models.DateTimeField(blank=True, null=True)), ('rejected_date', models.DateTimeField(blank=True, null=True)), From 20a42edbe72459f2ddb70c438e283262f5a9df95 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 29 Apr 2025 16:10:52 -0400 Subject: [PATCH 07/32] reduce URLField max_length to 765 --- errata/migrations/0001_initial.py | 2 +- security/migrations/0005_reference_cve_references.py | 2 +- security/models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/errata/migrations/0001_initial.py b/errata/migrations/0001_initial.py index 85fe88b4..d02a7dc8 100644 --- a/errata/migrations/0001_initial.py +++ b/errata/migrations/0001_initial.py @@ -19,7 +19,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('er_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], ), migrations.CreateModel( diff --git a/security/migrations/0005_reference_cve_references.py b/security/migrations/0005_reference_cve_references.py index 97251add..f94cf7d5 100644 --- a/security/migrations/0005_reference_cve_references.py +++ b/security/migrations/0005_reference_cve_references.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('ref_type', models.CharField(max_length=255)), - ('url', models.URLField(max_length=2000)), + ('url', models.URLField(max_length=765)), ], options={ 'unique_together': {('ref_type', 'url')}, diff --git a/security/models.py b/security/models.py index b60acb47..9c097eed 100644 --- a/security/models.py +++ b/security/models.py @@ -29,7 +29,7 @@ class Reference(models.Model): ref_type = models.CharField(max_length=255) - url = models.URLField(max_length=2000) + url = models.URLField(max_length=765) class Meta: unique_together = ['ref_type', 'url'] From bf5478cf5d5374b693d2641d526db1a1e02f47e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 15:45:31 +0000 Subject: [PATCH 08/32] Bump django from 4.2.20 to 4.2.21 Bumps [django](https://github.com/django/django) from 4.2.20 to 4.2.21. - [Commits](https://github.com/django/django/compare/4.2.20...4.2.21) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.21 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index dca4fe03..2f72fae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.20 +Django==4.2.21 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 57e5c0d4861b7c80d081e1b85a8b48f9bd1cb332 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jun 2025 23:28:39 +0000 Subject: [PATCH 09/32] Bump django from 4.2.21 to 4.2.22 Bumps [django](https://github.com/django/django) from 4.2.21 to 4.2.22. - [Commits](https://github.com/django/django/compare/4.2.21...4.2.22) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.22 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..3418b5d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.21 +Django==4.2.22 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 6a45e90ed63efadd4e1e7c246d174feceb91c439 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:47:24 +0000 Subject: [PATCH 10/32] Bump requests from 2.32.3 to 2.32.4 Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.3...v2.32.4) --- updated-dependencies: - dependency-name: requests dependency-version: 2.32.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f72fae5..955c7bbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 chardet==5.2.0 -requests==2.32.3 +requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 django-filter==25.1 From 56748532f099b6bd1dfaf202e55e4a160dc9fa3f Mon Sep 17 00:00:00 2001 From: vtalos Date: Thu, 17 Jul 2025 19:33:36 +0300 Subject: [PATCH 11/32] Remove unused dependency 'chardet' from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 088f0870..a39eb8cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ django-bootstrap3==23.1 python-debian==1.0.1 defusedxml==0.7.1 PyYAML==6.0.2 -chardet==5.2.0 requests==2.32.4 colorama==0.4.6 djangorestframework==3.15.2 From 94fcb04694d204c32d4098fe9981f255eb4a843e Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 5 Aug 2025 20:15:16 -0400 Subject: [PATCH 12/32] get_or_create_module only returns module --- modules/utils.py | 4 ++-- repos/repo_types/yum.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 817a610c..f56a0f62 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -23,7 +23,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): """ Get or create a module object - Returns the module and a boolean for created + Returns the module """ created = False m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) @@ -46,7 +46,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): arch=m_arch, repo=repo, ) - return module, created + return module def get_matching_modules(name, stream, version, context, arch): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index d08c7393..7ac85816 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -91,7 +91,7 @@ def extract_module_metadata(data, url, repo): packages.add(package) from modules.utils import get_or_create_module - module, created = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) + module = get_or_create_module(m_name, m_stream, m_version, m_context, arch, repo) package_ids = [] for package in packages: From 1480468f32ed5386c678e5fa0972eaf4c708c879 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 02:19:45 +0000 Subject: [PATCH 13/32] Bump django from 4.2.22 to 4.2.24 Bumps [django](https://github.com/django/django) from 4.2.22 to 4.2.24. - [Commits](https://github.com/django/django/compare/4.2.22...4.2.24) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.24 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a39eb8cc..9d2baa9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.22 +Django==4.2.24 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From b6162963e7811081eb7a4f60ef04a594d23d5866 Mon Sep 17 00:00:00 2001 From: Will Furnell Date: Fri, 12 Sep 2025 13:33:45 +0100 Subject: [PATCH 14/32] Package types are in the Package class --- packages/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models.py b/packages/models.py index f4c9c59e..74a83c0c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -195,11 +195,11 @@ def __str__(self): rel = f'-{self.release}' else: rel = '' - if self.packagetype == self.GENTOO: + if self.packagetype == Package.GENTOO: return f'{self.category}/{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype in [self.DEB, self.ARCH]: + elif self.packagetype in [Package.DEB, Package.ARCH]: return f'{self.name}_{epo}{self.version}{rel}_{self.arch}.{self.get_packagetype_display()}' - elif self.packagetype == self.RPM: + elif self.packagetype == Package.RPM: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' else: return f'{self.name}-{epo}{self.version}{rel}-{self.arch}.{self.get_packagetype_display()}' From 3676e78cd0afbd0467241f8fb62f54626f2508fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:00:50 +0000 Subject: [PATCH 15/32] Bump django from 4.2.24 to 4.2.25 Bumps [django](https://github.com/django/django) from 4.2.24 to 4.2.25. - [Commits](https://github.com/django/django/compare/4.2.24...4.2.25) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.25 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d2baa9e..9d277d38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.24 +Django==4.2.25 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From 1c260013c8cdb1bb5913a1151e37bde309ac438d Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:54:33 -0400 Subject: [PATCH 16/32] bump redis --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d277d38..2f264c9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ python-magic==0.4.27 gitpython==3.1.44 tenacity==8.2.3 celery==5.4.0 -redis==5.2.1 +redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 From 0f5445448421f17efb66d07d3f511b6327732440 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 10:59:40 -0400 Subject: [PATCH 17/32] Update license in common.py --- util/templatetags/common.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 6737c438..2aea1e5e 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -1,12 +1,10 @@ -# Copyright 2010 VPAC -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see import re From ce9f4f07dc176cfdab072f425acfdd69e9b728eb Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:11:25 -0400 Subject: [PATCH 18/32] fix licenses --- hosts/templatetags/report_alert.py | 7 +++---- setup.py | 2 +- util/filterspecs.py | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index 3a3e3a9a..a28c5058 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -1,11 +1,10 @@ -# Copyright 2016-2021 Marcus Furlong +# Copyright 2016-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -13,7 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from datetime import timedelta diff --git a/setup.py b/setup.py index d8249a67..6ec6d974 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2013-2021 Marcus Furlong +# Copyright 2013-2025 Marcus Furlong # # This file is part of Patchman. # diff --git a/util/filterspecs.py b/util/filterspecs.py index 1c845ff3..722b45df 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -1,12 +1,11 @@ # Copyright 2010 VPAC -# Copyright 2014-2021 Marcus Furlong +# Copyright 2014-2025 Marcus Furlong # # This file is part of Patchman. # # Patchman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# the Free Software Foundation, version 3 only. # # Patchman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -14,7 +13,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with Patchman If not, see . +# along with Patchman. If not, see from django.utils.safestring import mark_safe from django.db.models.query import QuerySet From c651c3f222ec7c53af47cfaedfcb5d7254b47710 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 3 Oct 2025 11:57:40 -0400 Subject: [PATCH 19/32] use GPL-3.0-only for debian copyright --- debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index ab051037..5202ff0e 100644 --- a/debian/copyright +++ b/debian/copyright @@ -6,7 +6,7 @@ Source: https://github.com/furlongm/patchman Files: * Copyright: 2011-2012 VPAC http://www.vpac.org 2013-2021 Marcus Furlong -License: GPL-3.0 +License: GPL-3.0-only This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 3 only. From fb9b56c453f1c93b617b2b2d98e8bb4db005c8f7 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Mon, 20 Oct 2025 20:47:26 -0400 Subject: [PATCH 20/32] fix tag handling Signed-off-by: Marcus Furlong --- hosts/models.py | 2 +- hosts/utils.py | 19 +++++++++++++++++-- sbin/patchman | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hosts/models.py b/hosts/models.py index a6c451b5..5b7b3979 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -85,7 +85,7 @@ def show(self): text += f'Packages : {self.get_num_packages()}\n' text += f'Repos : {self.get_num_repos()}\n' text += f'Updates : {self.get_num_updates()}\n' - text += f'Tags : {self.tags}\n' + text += f'Tags : {" ".join(self.tags.slugs())}\n' text += f'Needs reboot : {self.reboot_required}\n' text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' diff --git a/hosts/utils.py b/hosts/utils.py index b328129f..f07d5d1e 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -18,8 +18,9 @@ from socket import gethostbyaddr, gaierror, herror from django.db import transaction, IntegrityError +from taggit.models import Tag -from patchman.signals import error_message +from patchman.signals import error_message, info_message def update_rdns(host): @@ -62,7 +63,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.osvariant = osvariant host.domain = domain host.lastreport = report.created - host.tags = report.tags + host.tags.set(report.tags.split(','), clear=True) if report.reboot == 'True': host.reboot_required = True else: @@ -73,3 +74,17 @@ def get_or_create_host(report, arch, osvariant, domain): if host: host.check_rdns() return host + + +def clean_tags(): + """ Delete Tags that have no Host + """ + tags = Tag.objects.filter( + host__isnull=True, + ) + tlen = tags.count() + if tlen == 0: + info_message.send(sender=None, text='No orphaned Tags found.') + else: + info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + tags.delete() diff --git a/sbin/patchman b/sbin/patchman index 9cc6048e..c0911434 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -34,6 +34,7 @@ from errata.utils import mark_errata_security_updates, enrich_errata, \ scan_package_updates_for_affected_packages from errata.tasks import update_errata from hosts.models import Host +from hosts.utils import clean_tags from modules.utils import clean_modules from packages.utils import clean_packages, clean_packageupdates, clean_packagenames from repos.models import Repository @@ -362,6 +363,7 @@ def dbcheck(remove_duplicates=False): clean_repos() clean_modules() clean_packageupdates() + clean_tags() def collect_args(): From eee56752f2dcfdaa9e63361bb20f7aa5b99138fa Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Tue, 21 Oct 2025 22:35:58 -0400 Subject: [PATCH 21/32] fix some flake8-bugbear bugs Signed-off-by: Marcus Furlong --- patchman/urls.py | 2 +- repos/repo_types/gentoo.py | 2 +- repos/repo_types/rpm.py | 2 +- util/__init__.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/patchman/urls.py b/patchman/urls.py index 337d6b63..ee786566 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -44,7 +44,7 @@ router.register(r'package', package_views.PackageViewSet) router.register(r'package-update', package_views.PackageUpdateViewSet) router.register(r'cve', security_views.CVEViewSet) -router.register(r'reference', security_views.ReferenceViewSet), +router.register(r'reference', security_views.ReferenceViewSet) router.register(r'erratum', errata_views.ErratumViewSet) router.register(r'repo', repo_views.RepositoryViewSet) router.register(r'mirror', repo_views.MirrorViewSet) diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 94df139a..8e4198d9 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -226,7 +226,7 @@ def extract_gentoo_overlay_ebuilds(t): """ Extract ebuilds from a Gentoo overlay tarball """ extracted_ebuilds = {} - for root, dirs, files in os.walk(t): + for root, _, files in os.walk(t): for name in files: if fnmatch(name, '*.ebuild'): package_name = root.replace(t + '/', '') diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index aa3354c7..d9501cde 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -69,7 +69,7 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(mirrorlist=False, refresh=True, enabled=True) - for i, mirror in enumerate(enabled_mirrors): + for mirror in enabled_mirrors: res = find_mirror_url(mirror.url, formats) if not res: mirror.fail() diff --git a/util/__init__.py b/util/__init__.py index a56ed3b6..4958dfb0 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -107,10 +107,14 @@ def fetch_content(response, text='', ljust=35): wait=wait_exponential(multiplier=1, min=1, max=10), reraise=False, ) -def get_url(url, headers={}, params={}): +def get_url(url, headers=None, params=None): """ Perform a http GET on a URL. Return None on error. """ response = None + if not headers: + headers = {} + if not params: + params = {} try: debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, timeout=30) From 2bcd4dac0bd7dd81f4ebe4aa483532aec04a8650 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:45 -0400 Subject: [PATCH 22/32] fix package filter list for errata --- packages/views.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/views.py b/packages/views.py index cd53fa6e..c55a6c72 100644 --- a/packages/views.py +++ b/packages/views.py @@ -62,9 +62,16 @@ def package_list(request): if 'affected_by_errata' in request.GET: affected_by_errata = request.GET['affected_by_errata'] == 'true' if affected_by_errata: - packages = packages.filter(erratum__isnull=False) + packages = packages.filter(affected_by_erratum__isnull=False) else: - packages = packages.filter(erratum__isnull=True) + packages = packages.filter(affected_by_erratum__isnull=True) + + if 'provides_fix_in_erratum' in request.GET: + provides_fix_in_erratum = request.GET['provides_fix_in_erratum'] == 'true' + if provides_fix_in_erratum: + packages = packages.filter(provides_fix_in_erratum__isnull=False) + else: + packages = packages.filter(provides_fix_in_erratum__isnull=True) if 'installed_on_hosts' in request.GET: installed_on_hosts = request.GET['installed_on_hosts'] == 'true' @@ -102,6 +109,8 @@ def package_list(request): filter_list = [] filter_list.append(Filter(request, 'Affected by Errata', 'affected_by_errata', {'true': 'Yes', 'false': 'No'})) + filter_list.append(Filter(request, 'Provides Fix in Errata', 'provides_fix_in_erratum', + {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Installed on Hosts', 'installed_on_hosts', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Available in Repos', 'available_in_repos', {'true': 'Yes', 'false': 'No'})) filter_list.append(Filter(request, 'Package Type', 'packagetype', Package.PACKAGE_TYPES)) From 856f41f9f7cdc8fbebb0c6c3685a014526efdc36 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:29 -0400 Subject: [PATCH 23/32] add support for zstd compression in deb and rpm repos fixes: #698 Signed-off-by: Marcus Furlong --- debian/control | 2 +- repos/repo_types/deb.py | 7 ++++++- repos/repo_types/rpm.py | 2 ++ requirements.txt | 1 + setup.cfg | 1 + util/__init__.py | 17 +++++++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 67026269..7bfe320e 100644 --- a/debian/control +++ b/debian/control @@ -20,7 +20,7 @@ Depends: ${misc:Depends}, python3 (>= 3.11), python3-django (>= 4.2), python3-requests, python3-colorama, python3-magic, python3-humanize, python3-yaml, libapache2-mod-wsgi-py3, apache2, sqlite3, celery, python3-celery, python3-django-celery-beat, redis-server, - python3-redis, python3-git, python3-django-taggit + python3-redis, python3-git, python3-django-taggit, python3-zstandard Suggests: python3-mysqldb, python3-psycopg2, python3-pymemcache, memcached Description: Django-based patch status monitoring tool for linux systems. . diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 25d8eba7..1d3607c5 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -71,7 +71,12 @@ def refresh_deb_repo(repo): are and then fetches and extracts packages from those files. """ - formats = ['Packages.xz', 'Packages.bz2', 'Packages.gz', 'Packages'] + formats = [ + 'Packages.xz', + 'Packages.bz2', + 'Packages.gz', + 'Packages', + ] ts = get_datetime_now() enabled_mirrors = repo.mirror_set.filter(refresh=True, enabled=True) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index d9501cde..93d47007 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -57,10 +57,12 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): which type of repo it is, then refreshes the mirrors """ formats = [ + 'repodata/repomd.xml.zst', 'repodata/repomd.xml.xz', 'repodata/repomd.xml.bz2', 'repodata/repomd.xml.gz', 'repodata/repomd.xml', + 'suse/repodata/repomd.xml.zst', 'suse/repodata/repomd.xml.xz', 'suse/repodata/repomd.xml.bz2', 'suse/repodata/repomd.xml.gz', diff --git a/requirements.txt b/requirements.txt index 2f264c9b..788f4241 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ redis==6.4.0 django-celery-beat==2.7.0 tqdm==4.67.1 cvss==3.4 +zstandard==0.25.0 diff --git a/setup.cfg b/setup.cfg index 7af9ccb0..b1d5ee4e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ requires = /usr/bin/python3 python3-importlib-metadata python3-cvss python3-redis + python3-zstandard redis celery python3-django-celery-beat diff --git a/util/__init__.py b/util/__init__.py index ac6f8f1b..4a3f9caa 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -21,6 +21,11 @@ import zlib import lzma import os +try: + # python 3.14+ - can also remove the dependency at that stage + from compression import zstd +except ImportError: + import zstandard as zstd from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 @@ -202,6 +207,16 @@ def unxz(contents): error_message.send(sender=None, text='lzma: ' + e) +def unzstd(contents): + """ unzstd contents in memory and return the data + """ + try: + zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() + return zstddata + except zstd.ZstdError as e: + error_message.send(sender=None, text='zstd: ' + e) + + def extract(data, fmt): """ Extract the contents based on mimetype or file ending. Return the unmodified data if neither mimetype nor file ending matches, otherwise @@ -214,6 +229,8 @@ def extract(data, fmt): m = magic.open(magic.MAGIC_MIME) m.load() mime = m.buffer(data).split(';')[0] + if mime == 'application/zstd' or fmt.endswith('zst'): + return unzstd(data) if mime == 'application/x-xz' or fmt.endswith('xz'): return unxz(data) elif mime == 'application/x-bzip2' or fmt.endswith('bz2'): From 8d9da89ebccd367de221b891ad02f99b205d9c19 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:41 -0400 Subject: [PATCH 24/32] simplify logging --- arch/utils.py | 10 ++--- errata/models.py | 12 ++--- errata/sources/distros/arch.py | 5 ++- errata/sources/distros/centos.py | 5 ++- errata/sources/distros/debian.py | 7 +-- errata/sources/distros/rocky.py | 11 ++--- errata/sources/distros/ubuntu.py | 7 +-- errata/sources/repos/yum.py | 5 ++- errata/tasks.py | 2 +- errata/utils.py | 9 ++-- hosts/models.py | 14 +++--- hosts/tasks.py | 4 +- hosts/utils.py | 8 ++-- modules/utils.py | 8 ++-- packages/utils.py | 26 +++++------ patchman/receivers.py | 21 ++++----- patchman/signals.py | 8 ++-- reports/models.py | 10 ++--- reports/tasks.py | 4 +- reports/utils.py | 5 ++- repos/models.py | 18 ++++---- repos/repo_types/arch.py | 11 ++--- repos/repo_types/deb.py | 11 ++--- repos/repo_types/gentoo.py | 15 ++++--- repos/repo_types/rpm.py | 8 ++-- repos/repo_types/yast.py | 5 ++- repos/repo_types/yum.py | 23 +++++----- repos/utils.py | 37 ++++++++-------- sbin/patchman | 76 ++++++++++++++++---------------- security/models.py | 8 ++-- util/__init__.py | 24 +++++----- util/logging.py | 42 ++++++++++++++++++ 32 files changed, 258 insertions(+), 201 deletions(-) create mode 100644 util/logging.py diff --git a/arch/utils.py b/arch/utils.py index 1498fdec..04d0b350 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from arch.models import PackageArchitecture, MachineArchitecture -from patchman.signals import info_message +from util.logging import info_message def clean_package_architectures(): @@ -24,9 +24,9 @@ def clean_package_architectures(): parches = PackageArchitecture.objects.filter(package__isnull=True) plen = parches.count() if plen == 0: - info_message.send(sender=None, text='No orphaned PackageArchitectures found.') + info_message(text='No orphaned PackageArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures') + info_message(text=f'Removing {plen} orphaned PackageArchitectures') parches.delete() @@ -39,9 +39,9 @@ def clean_machine_architectures(): ) mlen = marches.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned MachineArchitectures found.') + info_message(text='No orphaned MachineArchitectures found.') else: - info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures') + info_message(text=f'Removing {mlen} orphaned MachineArchitectures') marches.delete() diff --git a/errata/models.py b/errata/models.py index b10daf4d..cfc9bd0d 100644 --- a/errata/models.py +++ b/errata/models.py @@ -25,7 +25,7 @@ from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from patchman.signals import error_message +from util.logging import error_message from util import get_url @@ -70,7 +70,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -84,7 +84,7 @@ def scan_for_security_updates(self): try: affected_update.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete() @@ -93,7 +93,7 @@ def fetch_osv_dev_data(self): osv_dev_url = f'https://api.osv.dev/v1/vulns/{self.name}' res = get_url(osv_dev_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.name} - {osv_dev_url}') + error_message(text=f'404 - Skipping {self.name} - {osv_dev_url}') return data = res.content osv_dev_json = json.loads(data) @@ -102,7 +102,7 @@ def fetch_osv_dev_data(self): def parse_osv_dev_data(self, osv_dev_json): name = osv_dev_json.get('id') if name != self.name: - error_message.send(sender=None, text=f'Erratum name mismatch - {self.name} != {name}') + error_message(text=f'Erratum name mismatch - {self.name} != {name}') return related = osv_dev_json.get('related') if related: @@ -155,7 +155,7 @@ def add_cve(self, cve_id): """ Add a CVE to an Erratum object """ if not cve_id.startswith('CVE') or not cve_id.split('-')[1].isdigit(): - error_message.send(sender=None, text=f'Not a CVE ID: {cve_id}') + error_message(text=f'Not a CVE ID: {cve_id}') return self.cves.add(get_or_create_cve(cve_id)) diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 40d0dada..87c6c47a 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,7 +20,8 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from packages.models import Package from packages.utils import find_evr, get_matching_packages, get_or_create_package from util import get_url, fetch_content @@ -99,7 +100,7 @@ def process_arch_erratum(advisory, osrelease): add_arch_erratum_references(e, advisory) add_arch_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_arch_linux_osrelease(): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index eefb2b88..d2722a6b 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -20,7 +20,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import parse_package_string, get_or_create_package -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type @@ -34,7 +35,7 @@ def update_centos_errata(): if actual_checksum != expected_checksum: e = 'CEFS checksum mismatch, skipping CentOS errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) else: if data: parse_centos_errata(bunzip2(data)) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 93ae2bd5..1ae919e4 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -27,7 +27,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, find_evr -from patchman.signals import error_message, pbar_start, pbar_update, warning_message +from util.logging import error_message, warning_message +from patchman.signals import pbar_start, pbar_update from util import get_url, fetch_content, get_setting_of_type, extract DSCs = {} @@ -217,7 +218,7 @@ def process_debian_erratum(erratum, accepted_codenames): for package in packages: process_debian_erratum_fixed_packages(e, package) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def parse_debian_erratum_package(line, accepted_codenames): @@ -249,7 +250,7 @@ def fetch_debian_dsc_package_list(package, version): """ Fetch the package list from a DSC file for a given source package/version """ if not DSCs.get(package) or not DSCs[package].get(version): - warning_message.send(sender=None, text=f'No DSC found for {package} {version}') + warning_message(text=f'No DSC found for {package} {version}') return source_url = DSCs[package][version]['url'] res = get_url(source_url) diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 693d7b0c..16d4d12c 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -25,7 +25,8 @@ from packages.models import Package from packages.utils import parse_package_string, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, info_message, error_message +from util import get_url, fetch_content +from util.logging import info_message, error_message def update_rocky_errata(concurrent_processing=True): @@ -50,16 +51,16 @@ def check_rocky_errata_endpoint_health(rocky_errata_api_host): health = json.loads(data) if health.get('status') == 'ok': s = f'Rocky Errata API healthcheck OK: {rocky_errata_healthcheck_url}' - info_message.send(sender=None, text=s) + info_message(text=s) return True else: s = f'Rocky Errata API healthcheck FAILED: {rocky_errata_healthcheck_url}' - error_message.send(sender=None, text=s) + error_message(text=s) return False except Exception as e: s = f'Rocky Errata API healthcheck exception occured: {rocky_errata_healthcheck_url}\n' s += str(e) - error_message.send(sender=None, text=s) + error_message(text=s) return False @@ -194,7 +195,7 @@ def process_rocky_erratum(advisory): add_rocky_erratum_oses(e, advisory) add_rocky_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_rocky_erratum_references(e, advisory): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 7f50962c..d1ce7cc5 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -28,7 +28,8 @@ from packages.models import Package from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from patchman.signals import error_message, pbar_start, pbar_update +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update def update_ubuntu_errata(concurrent_processing=False): @@ -45,7 +46,7 @@ def update_ubuntu_errata(concurrent_processing=False): else: e = 'Ubuntu USN DB checksum mismatch, skipping Ubuntu errata parsing\n' e += f'{actual_checksum} (actual) != {expected_checksum} (expected)' - error_message.send(sender=None, text=e) + error_message(text=e) def fetch_ubuntu_usn_db(): @@ -126,7 +127,7 @@ def process_usn(usn_id, advisory, accepted_releases): add_ubuntu_erratum_references(e, usn_id, advisory) add_ubuntu_erratum_packages(e, advisory) except Exception as exc: - error_message.send(sender=None, text=exc) + error_message(text=exc) def add_ubuntu_erratum_osreleases(e, affected_releases, accepted_releases): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index dfeed879..f361d10e 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -23,7 +23,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from patchman.signals import pbar_start, pbar_update, error_message +from util.logging import error_message +from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url @@ -38,7 +39,7 @@ def extract_updateinfo(data, url, concurrent_processing=True): elen = root.__len__() updates = root.findall('update') except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing updateinfo file from {url} : {e}') + error_message(text=f'Error parsing updateinfo file from {url} : {e}') if concurrent_processing: extract_updateinfo_concurrently(updates, elen) else: diff --git a/errata/tasks.py b/errata/tasks.py index fe53b415..f1d6eeee 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -22,7 +22,7 @@ from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from patchman.signals import error_message +from util.logging import error_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type diff --git a/errata/utils.py b/errata/utils.py index d8099db4..a8d8d424 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -21,7 +21,8 @@ from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from patchman.signals import pbar_start, pbar_update, warning_message +from util.logging import warning_message +from patchman.signals import pbar_start, pbar_update def get_or_create_erratum(name, e_type, issue_date, synopsis): @@ -36,16 +37,16 @@ def get_or_create_erratum(name, e_type, issue_date, synopsis): days_delta = abs(e.issue_date.date() - issue_date_tz.date()).days updated = False if e.e_type != e_type: - warning_message.send(sender=None, text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') + warning_message(text=f'Updating {name} type `{e.e_type}` -> `{e_type}`') e.e_type = e_type updated = True if days_delta > 1: text = f'Updating {name} issue date `{e.issue_date.date()}` -> `{issue_date_tz.date()}`' - warning_message.send(sender=None, text=text) + warning_message(text=text) e.issue_date = issue_date_tz updated = True if e.synopsis != synopsis: - warning_message.send(sender=None, text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') + warning_message(text=f'Updating {name} synopsis `{e.synopsis}` -> `{synopsis}`') e.synopsis = synopsis updated = True if updated: diff --git a/hosts/models.py b/hosts/models.py index 5b7b3979..650544dc 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -34,7 +34,7 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from patchman.signals import info_message +from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo @@ -90,7 +90,7 @@ def show(self): text += f'Updated at : {self.updated_at}\n' text += f'Host repos : {self.host_repos_only}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def get_absolute_url(self): return reverse('hosts:host_detail', args=[self.hostname]) @@ -114,13 +114,13 @@ def check_rdns(self): if self.check_dns: update_rdns(self) if self.hostname.lower() == self.reversedns.lower(): - info_message.send(sender=None, text='Reverse DNS matches') + info_message(text='Reverse DNS matches') else: text = 'Reverse DNS mismatch found: ' text += f'{self.hostname} != {self.reversedns}' - info_message.send(sender=None, text=text) + info_message(text=text) else: - info_message.send(sender=None, text='Reverse DNS check disabled') + info_message(text='Reverse DNS check disabled') def clean_reports(self): """ Remove all but the last 3 reports for a host @@ -131,7 +131,7 @@ def clean_reports(self): for report in Report.objects.filter(host=self).order_by('-created')[3:]: report.delete() if rlen > 0: - info_message.send(sender=None, text=f'{self.hostname}: removed {rlen} old reports') + info_message(text=f'{self.hostname}: removed {rlen} old reports') def get_host_repo_packages(self): if self.host_repos_only: @@ -163,7 +163,7 @@ def process_update(self, package, highest_package): security = True update = get_or_create_package_update(oldpackage=package, newpackage=highest_package, security=security) self.updates.add(update) - info_message.send(sender=None, text=f'{update}') + info_message(text=f'{update}') return update.id def find_updates(self): diff --git a/hosts/tasks.py b/hosts/tasks.py index 2fdce96f..1643901d 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -20,7 +20,7 @@ from hosts.models import Host from util import get_datetime_now -from patchman.signals import info_message +from util.logging import info_message @shared_task @@ -78,4 +78,4 @@ def find_all_host_updates_homogenous(): phost.updated_at = ts phost.save() updated_hosts.append(phost) - info_message.send(sender=None, text=f'Added the same updates to {phost}') + info_message(text=f'Added the same updates to {phost}') diff --git a/hosts/utils.py b/hosts/utils.py index f07d5d1e..d6e663cf 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -20,7 +20,7 @@ from django.db import transaction, IntegrityError from taggit.models import Tag -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message def update_rdns(host): @@ -70,7 +70,7 @@ def get_or_create_host(report, arch, osvariant, domain): host.reboot_required = False host.save() except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) if host: host.check_rdns() return host @@ -84,7 +84,7 @@ def clean_tags(): ) tlen = tags.count() if tlen == 0: - info_message.send(sender=None, text='No orphaned Tags found.') + info_message(text='No orphaned Tags found.') else: - info_message.send(sender=None, text=f'{tlen} orphaned Tags found.') + info_message(text=f'{tlen} orphaned Tags found.') tags.delete() diff --git a/modules/utils.py b/modules/utils.py index f56a0f62..05c57c80 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,7 +15,7 @@ # along with Patchman. If not, see from django.db import IntegrityError -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message from modules.models import Module from arch.models import PackageArchitecture @@ -37,7 +37,7 @@ def get_or_create_module(name, stream, version, context, arch, repo): repo=repo, ) except IntegrityError as e: - error_message.send(sender=None, text=e) + error_message(text=e) module = Module.objects.get( name=name, stream=stream, @@ -73,7 +73,7 @@ def clean_modules(): ) mlen = modules.count() if mlen == 0: - info_message.send(sender=None, text='No orphaned Modules found.') + info_message(text='No orphaned Modules found.') else: - info_message.send(sender=None, text=f'{mlen} orphaned Modules found.') + info_message(text=f'{mlen} orphaned Modules found.') modules.delete() diff --git a/packages/utils.py b/packages/utils.py index 9b098225..f00f6710 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -22,7 +22,7 @@ from arch.models import PackageArchitecture from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString -from patchman.signals import error_message, info_message, warning_message +from util.logging import error_message, info_message, warning_message def convert_package_to_packagestring(package): @@ -141,7 +141,7 @@ def parse_redhat_package_string(pkg_str): name, epoch, ver, rel, dist, arch = m.groups() else: e = f'Error parsing package string: "{pkg_str}"' - error_message.send(sender=None, text=e) + error_message(text=e) return if dist: rel = f'{rel}.{dist}' @@ -195,7 +195,7 @@ def get_or_create_package(name, epoch, version, release, arch, p_type): package = packages.first() # TODO this should handle gentoo package categories too, otherwise we may be deleting packages # that should be kept - warning_message.send(sender=None, text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') + warning_message(text=f'Deleting duplicate packages: {packages.exclude(id=package.id)}') packages.exclude(id=package.id).delete() return package @@ -218,10 +218,10 @@ def get_or_create_package_update(oldpackage, newpackage, security): except MultipleObjectsReturned: e = 'Error: MultipleObjectsReturned when attempting to add package \n' e += f'update with oldpackage={oldpackage} | newpackage={newpackage}:' - error_message.send(sender=None, text=e) + error_message(text=e) updates = PackageUpdate.objects.filter(oldpackage=oldpackage, newpackage=newpackage) for update in updates: - error_message.send(sender=None, text=str(update)) + error_message(text=str(update)) return try: if update: @@ -281,13 +281,13 @@ def clean_packageupdates(): for update in package_updates: if update.host_set.count() == 0: text = f'Removing unused PackageUpdate {update}' - info_message.send(sender=None, text=text) + info_message(text=text) update.delete() for duplicate in package_updates: if update.oldpackage == duplicate.oldpackage and update.newpackage == duplicate.newpackage and \ update.security == duplicate.security and update.id != duplicate.id: text = f'Removing duplicate PackageUpdate: {update}' - info_message.send(sender=None, text=text) + info_message(text=text) for host in duplicate.host_set.all(): host.updates.remove(duplicate) host.updates.add(update) @@ -307,12 +307,12 @@ def clean_packages(remove_duplicates=False): ) plen = packages.count() if plen == 0: - info_message.send(sender=None, text='No orphaned Packages found.') + info_message(text='No orphaned Packages found.') else: - info_message.send(sender=None, text=f'Removing {plen} orphaned Packages') + info_message(text=f'Removing {plen} orphaned Packages') packages.delete() if remove_duplicates: - info_message.send(sender=None, text='Checking for duplicate Packages...') + info_message(text='Checking for duplicate Packages...') for package in Package.objects.all(): potential_duplicates = Package.objects.filter( name=package.name, @@ -326,7 +326,7 @@ def clean_packages(remove_duplicates=False): if potential_duplicates.count() > 1: for dupe in potential_duplicates: if dupe.id != package.id: - info_message.send(sender=None, text=f'Removing duplicate Package {dupe}') + info_message(text=f'Removing duplicate Package {dupe}') dupe.delete() @@ -336,7 +336,7 @@ def clean_packagenames(): names = PackageName.objects.filter(package__isnull=True) nlen = names.count() if nlen == 0: - info_message.send(sender=None, text='No orphaned PackageNames found.') + info_message(text='No orphaned PackageNames found.') else: - info_message.send(sender=None, text=f'Removing {nlen} orphaned PackageNames') + info_message(text=f'Removing {nlen} orphaned PackageNames') names.delete() diff --git a/patchman/receivers.py b/patchman/receivers.py index 5ec32cdd..8d8893ca 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -21,7 +21,8 @@ from django.dispatch import receiver from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update, \ + info_message_s, warning_message_s, error_message_s, debug_message_s from django.conf import settings @@ -47,36 +48,36 @@ def pbar_update_receiver(**kwargs): update_pbar(index) -@receiver(info_message) -def print_info_message(sender=None, **kwargs): - """ Receiver to print an info message, no color +@receiver(info_message_s) +def print_info_message(**kwargs): + """ Receiver to handle an info message, no color """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.RESET_ALL + Fore.RESET + text) -@receiver(warning_message) +@receiver(warning_message_s) def print_warning_message(**kwargs): - """ Receiver to print a warning message in yellow text + """ Receiver to handle a warning message, yellow text """ text = str(kwargs.get('text')) if get_verbosity(): tqdm.write(Style.BRIGHT + Fore.YELLOW + text) -@receiver(error_message) +@receiver(error_message_s) def print_error_message(**kwargs): - """ Receiver to print an error message in red text + """ Receiver to handle an error message, red text """ text = str(kwargs.get('text')) if text: tqdm.write(Style.BRIGHT + Fore.RED + text) -@receiver(debug_message) +@receiver(debug_message_s) def print_debug_message(**kwargs): - """ Receiver to print a debug message in blue, if verbose and DEBUG are set + """ Receiver to handle a debug message, blue text if verbose and DEBUG are set """ text = str(kwargs.get('text')) if get_verbosity() and settings.DEBUG and text: diff --git a/patchman/signals.py b/patchman/signals.py index 917a48e4..799b9c98 100644 --- a/patchman/signals.py +++ b/patchman/signals.py @@ -19,7 +19,7 @@ pbar_start = Signal() pbar_update = Signal() -info_message = Signal() -warning_message = Signal() -error_message = Signal() -debug_message = Signal() +info_message_s = Signal() +warning_message_s = Signal() +error_message_s = Signal() +debug_message_s = Signal() diff --git a/reports/models.py b/reports/models.py index 6818ea23..d529804b 100644 --- a/reports/models.py +++ b/reports/models.py @@ -19,7 +19,7 @@ from django.urls import reverse from hosts.utils import get_or_create_host -from patchman.signals import error_message, info_message +from util.logging import error_message, info_message class Report(models.Model): @@ -97,11 +97,11 @@ def process(self, find_updates=True, verbose=False): """ Process a report and extract os, arch, domain, packages, repos etc """ if not self.os or not self.kernel or not self.arch: - error_message.send(sender=None, text=f'Error: OS, kernel or arch not sent with report {self.id}') + error_message(text=f'Error: OS, kernel or arch not sent with report {self.id}') return if self.processed: - info_message.send(sender=None, text=f'Report {self.id} has already been processed') + info_message(text=f'Report {self.id} has already been processed') return from reports.utils import get_arch, get_os, get_domain @@ -111,7 +111,7 @@ def process(self, find_updates=True, verbose=False): host = get_or_create_host(self, arch, osvariant, domain) if verbose: - info_message.send(sender=None, text=f'Processing report {self.id} - {self.host}') + info_message(text=f'Processing report {self.id} - {self.host}') from reports.utils import process_packages, process_repos, process_updates, process_modules process_repos(report=self, host=host) @@ -124,5 +124,5 @@ def process(self, find_updates=True, verbose=False): if find_updates: if verbose: - info_message.send(sender=None, text=f'Finding updates for report {self.id} - {self.host}') + info_message(text=f'Finding updates for report {self.id} - {self.host}') host.find_updates() diff --git a/reports/tasks.py b/reports/tasks.py index db9e4103..fe294e8d 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -21,7 +21,7 @@ from hosts.models import Host from reports.models import Report -from util import info_message +from util.logging import info_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -48,5 +48,5 @@ def clean_reports_with_no_hosts(): for report in Report.objects.filter(processed=True): if not Host.objects.filter(hostname=report.host).exists(): text = f'Deleting report {report.id} for Host `{report.host}` as the host no longer exists' - info_message.send(sender=None, text=text) + info_message(text=text) report.delete() diff --git a/reports/utils.py b/reports/utils.py index 641f90df..76b6e09c 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -26,7 +26,8 @@ from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant from packages.models import Package, PackageCategory from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.models import Repository, Mirror, MirrorPackage from repos.utils import get_or_create_repo @@ -93,7 +94,7 @@ def process_packages(report, host): host.packages.add(package) else: if pkg_str[0].lower() != 'gpg-pubkey': - info_message.send(sender=None, text=f'No package returned for {pkg_str}') + info_message(text=f'No package returned for {pkg_str}') pbar_update.send(sender=None, index=i + 1) for package in host.packages.all(): diff --git a/repos/models.py b/repos/models.py index 181a103d..a1db2a93 100644 --- a/repos/models.py +++ b/repos/models.py @@ -26,7 +26,7 @@ from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from patchman.signals import info_message, warning_message, error_message +from util.logging import info_message, warning_message, error_message class Repository(models.Model): @@ -72,7 +72,7 @@ def show(self): text += f'arch: {self.arch}\n' text += 'Mirrors:' - info_message.send(sender=None, text=text) + info_message(text=text) for mirror in self.mirror_set.all(): mirror.show() @@ -99,10 +99,10 @@ def refresh(self, force=False): refresh_gentoo_repo(self) else: text = f'Error: unknown repo type for repo {self.id}: {self.repotype}' - error_message.send(sender=None, text=text) + error_message(text=text) else: text = 'Repo requires authentication, not updating' - warning_message.send(sender=None, text=text) + warning_message(text=text) def refresh_errata(self, force=False): """ Refresh errata metadata for all of a repos mirrors @@ -168,7 +168,7 @@ def show(self): text = f' {self.id} : {self.url}\n' text += ' last updated: ' text += f'{self.timestamp} checksum: {self.packages_checksum}\n' - info_message.send(sender=None, text=text) + info_message(text=text) def fail(self): """ Records that the mirror has failed @@ -178,10 +178,10 @@ def fail(self): """ if self.repo.auth_required: text = f'Mirror requires authentication, not updating - {self.url}' - warning_message.send(sender=None, text=text) + warning_message(text=text) return text = f'No usable mirror found at {self.url}' - error_message.send(sender=None, text=text) + error_message(text=text) default_max_mirror_failures = 28 max_mirror_failures = get_setting_of_type( setting_name='MAX_MIRROR_FAILURES', @@ -191,11 +191,11 @@ def fail(self): self.fail_count = self.fail_count + 1 if max_mirror_failures == -1: text = f'Mirror has failed {self.fail_count} times, but MAX_MIRROR_FAILURES=-1, not disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) elif self.fail_count > max_mirror_failures: self.refresh = False text = f'Mirror has failed {self.fail_count} times (max={max_mirror_failures}), disabling refresh' - error_message.send(sender=None, text=text) + error_message(text=text) self.last_access_ok = False self.save() diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 6e85b153..09719428 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,7 +18,8 @@ from io import BytesIO from packages.models import PackageString -from patchman.signals import info_message, warning_message, pbar_start, pbar_update +from util.logging import info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages from util import get_datetime_now, get_checksum, Checksum @@ -34,7 +35,7 @@ def refresh_arch_repo(repo): for i, mirror in enumerate(enabled_mirrors): if i >= max_mirrors: text = f'{max_mirrors} Mirrors already refreshed (max={max_mirrors}), skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) break res = find_mirror_url(mirror.url, [fname]) @@ -42,7 +43,7 @@ def refresh_arch_repo(repo): continue mirror_url = res.url text = f'Found Arch Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -54,7 +55,7 @@ def refresh_arch_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum @@ -111,5 +112,5 @@ def extract_arch_packages(data): packagetype='A') packages.add(package) else: - info_message.send(sender=None, text='No Packages found in Repo') + info_message(text='No Packages found in Repo') return packages diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index 1d3607c5..c6c26d78 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -19,7 +19,8 @@ from debian.debian_support import Version from packages.models import PackageString -from patchman.signals import error_message, pbar_start, pbar_update, info_message, warning_message +from util.logging import error_message, info_message, warning_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url from util import get_datetime_now, get_checksum, Checksum, extract @@ -30,7 +31,7 @@ def extract_deb_packages(data, url): try: extracted = extract(data, url).decode('utf-8') except UnicodeDecodeError as e: - error_message.send(sender=None, text=f'Skipping {url} : {e}') + error_message(text=f'Skipping {url} : {e}') return package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) @@ -61,7 +62,7 @@ def extract_deb_packages(data, url): packagetype='D') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages @@ -86,7 +87,7 @@ def refresh_deb_repo(repo): continue mirror_url = res.url text = f'Found deb Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) package_data = fetch_mirror_data( mirror=mirror, @@ -98,7 +99,7 @@ def refresh_deb_repo(repo): computed_checksum = get_checksum(package_data, Checksum.sha1) if mirror.packages_checksum == computed_checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue else: mirror.packages_checksum = computed_checksum diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index 8e4198d9..e440f0d5 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -26,7 +26,8 @@ from packages.models import PackageString from packages.utils import find_evr -from patchman.signals import info_message, warning_message, error_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid @@ -56,7 +57,7 @@ def refresh_gentoo_main_repo(repo): if mirror.packages_checksum == checksum: text = 'Mirror checksum has not changed, not refreshing Package metadata' - warning_message.send(sender=None, text=text) + warning_message(text=text) continue res = get_url(mirror.url) @@ -70,7 +71,7 @@ def refresh_gentoo_main_repo(repo): mirror.fail() continue extracted = extract(data, mirror.url) - info_message.send(sender=None, text=f'Found Gentoo Repo - {mirror.url}') + info_message(text=f'Found Gentoo Repo - {mirror.url}') computed_checksum = get_checksum(data, Checksum.md5) if not mirror_checksum_is_valid(computed_checksum, checksum, mirror, 'package'): @@ -165,7 +166,7 @@ def get_gentoo_overlay_mirrors(repo_name): if element.text.startswith('http'): mirrors.append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_overlays_url}: {e}') + error_message(text=f'Error parsing {gentoo_overlays_url}: {e}') return mirrors @@ -199,7 +200,7 @@ def get_gentoo_mirror_urls(): if element.get('protocol') == 'http': mirrors[name]['urls'].append(element.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing {gentoo_distfiles_url}: {e}') + error_message(text=f'Error parsing {gentoo_distfiles_url}: {e}') mirror_urls = [] # for now, ignore region data and choose MAX_MIRRORS mirrors at random for _, v in mirrors.items(): @@ -274,7 +275,7 @@ def extract_gentoo_packages_from_ebuilds(extracted_ebuilds): ) packages.add(package) plen = len(packages) - info_message.send(sender=None, text=f'Extracted {plen} Packages', plen=plen) + info_message(text=f'Extracted {plen} Packages', plen=plen) return packages @@ -282,7 +283,7 @@ def extract_gentoo_overlay_packages(mirror): """ Extract packages from gentoo overlay repo """ t = tempfile.mkdtemp() - info_message.send(sender=None, text=f'Extracting Gentoo packages from {mirror.url}') + info_message(text=f'Extracting Gentoo packages from {mirror.url}') git.Repo.clone_from(mirror.url, t, depth=1) packages = set() extracted_ebuilds = extract_gentoo_overlay_ebuilds(t) diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 93d47007..51661809 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,7 +16,7 @@ from django.db.models import Q -from patchman.signals import info_message, warning_message +from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data @@ -47,7 +47,7 @@ def max_mirrors_refreshed(repo, checksum, ts): have_checksum_and_ts = repo.mirror_set.filter(mirrors_q).count() if have_checksum_and_ts >= max_mirrors: text = f'{max_mirrors} Mirrors already have this checksum and timestamp, skipping further refreshes' - warning_message.send(sender=None, text=text) + warning_message(text=text) return True return False @@ -87,11 +87,11 @@ def refresh_rpm_repo_mirrors(repo, errata_only=False): if mirror_url.endswith('content'): text = f'Found yast rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yast_repo(mirror, repo_data) else: text = f'Found yum rpm Repo - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) refresh_yum_repo(mirror, repo_data, mirror_url, errata_only) if mirror.last_access_ok: mirror.timestamp = ts diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index 0ef54358..bf594040 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,7 +17,8 @@ import re from packages.models import PackageString -from patchman.signals import pbar_start, pbar_update, info_message +from util.logging import info_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -65,5 +66,5 @@ def extract_yast_packages(data): packagetype='R') packages.add(package) else: - info_message.send(sender=None, text='No packages found in repo') + info_message(text='No packages found in repo') return packages diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index 7ac85816..bc0fbc4b 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -22,7 +22,8 @@ from errata.sources.repos.yum import extract_updateinfo from packages.models import Package, PackageString from packages.utils import get_or_create_package, parse_package_string -from patchman.signals import warning_message, error_message, pbar_start, pbar_update +from util.logging import warning_message, error_message +from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract @@ -50,7 +51,7 @@ def get_repomd_url(mirror_url, data, url_type='primary'): checksum = grandchild.text checksum_type = grandchild.attrib.get('type') except ElementTree.ParseError as e: - error_message.send(sender=None, text=(f'Error parsing repomd from {mirror_url}: {e}')) + error_message(text=(f'Error parsing repomd from {mirror_url}: {e}')) if not location: return None, None, None url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location @@ -65,7 +66,7 @@ def extract_module_metadata(data, url, repo): try: modules_yaml = yaml.safe_load_all(extracted) except yaml.YAMLError as e: - error_message.send(sender=None, text=f'Error parsing modules.yaml: {e}') + error_message(text=f'Error parsing modules.yaml: {e}') mlen = len(re.findall(r'---', yaml.dump(extracted.decode()))) pbar_start.send(sender=None, ptext=f'Extracting {mlen} Modules ', plen=mlen) @@ -150,10 +151,10 @@ def extract_yum_packages(data, url): i += 1 else: text = f'Error parsing Package: {name} {epoch} {version} {release} {arch}' - error_message.send(sender=None, text=text) + error_message(text=text) elem.clear() except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing yum primary.xml from {url}: {e}') + error_message(text=f'Error parsing yum primary.xml from {url}: {e}') return packages @@ -162,7 +163,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='updateinfo') if not url: - warning_message.send(sender=None, text=f'No Errata metadata found in {mirror_url}') + warning_message(text=f'No Errata metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -177,7 +178,7 @@ def refresh_repomd_updateinfo(mirror, data, mirror_url): if mirror.errata_checksum and mirror.errata_checksum == checksum: text = 'Mirror Errata checksum has not changed, skipping Erratum refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.errata_checksum = checksum @@ -191,7 +192,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='modules') if not url: - warning_message.send(sender=None, text=f'No Module metadata found in {mirror_url}') + warning_message(text=f'No Module metadata found in {mirror_url}') return data = fetch_mirror_data( mirror=mirror, @@ -206,7 +207,7 @@ def refresh_repomd_modules(mirror, data, mirror_url): if mirror.modules_checksum and mirror.modules_checksum == checksum: text = 'Mirror Modules checksum has not changed, skipping Module refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.modules_checksum = checksum @@ -220,7 +221,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): """ url, checksum, checksum_type = get_repomd_url(mirror_url, data, url_type='primary') if not url: - warning_message.send(sender=None, text=f'No Package metadata found in {mirror_url}') + warning_message(text=f'No Package metadata found in {mirror_url}') data = fetch_mirror_data( mirror=mirror, url=url, @@ -234,7 +235,7 @@ def refresh_repomd_primary(mirror, data, mirror_url): if mirror.packages_checksum and mirror.packages_checksum == checksum: text = 'Mirror Packages checksum has not changed, skipping Package refresh' - warning_message.send(sender=None, text=text) + warning_message(text=text) return else: mirror.packages_checksum = checksum diff --git a/repos/utils.py b/repos/utils.py index 49b5d07f..c11c41ad 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -26,7 +26,8 @@ from packages.models import Package from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from patchman.signals import info_message, warning_message, error_message, debug_message, pbar_start, pbar_update +from util.logging import info_message, warning_message, error_message, debug_message +from patchman.signals import pbar_start, pbar_update def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -77,7 +78,7 @@ def update_mirror_packages(mirror, packages): package = convert_packagestring_to_package(strpackage) mirror_package, c = MirrorPackage.objects.get_or_create(mirror=mirror, package=package) except Package.MultipleObjectsReturned: - error_message.send(sender=None, text=f'Duplicate Package found in {mirror}: {strpackage}') + error_message(text=f'Duplicate Package found in {mirror}: {strpackage}') def find_mirror_url(stored_mirror_url, formats): @@ -89,7 +90,7 @@ def find_mirror_url(stored_mirror_url, formats): if mirror_url.endswith(f): mirror_url = mirror_url[:-len(f)] mirror_url = f"{mirror_url.rstrip('/')}/{fmt}" - debug_message.send(sender=None, text=f'Checking for Mirror at {mirror_url}') + debug_message(text=f'Checking for Mirror at {mirror_url}') try: res = get_url(mirror_url) except RetryError: @@ -133,7 +134,7 @@ def get_metalink_urls(url): if greatgreatgrandchild.attrib.get('protocol') in ['https', 'http']: metalink_urls.append(greatgreatgrandchild.text) except ElementTree.ParseError as e: - error_message.send(sender=None, text=f'Error parsing metalink {url}: {e}') + error_message(text=f'Error parsing metalink {url}: {e}') return metalink_urls @@ -152,12 +153,12 @@ def get_mirrorlist_urls(url): return mirror_urls = re.findall(r'^http[s]*://.*$|^ftp://.*$', data.decode('utf-8'), re.MULTILINE) if mirror_urls: - debug_message.send(sender=None, text=f'Found mirrorlist: {url}') + debug_message(text=f'Found mirrorlist: {url}') return mirror_urls else: - debug_message.send(sender=None, text=f'Not a mirrorlist: {url}') + debug_message(text=f'Not a mirrorlist: {url}') except Exception as e: - error_message.send(sender=None, text=f'Error attempting to parse a mirrorlist: {e} {url}') + error_message(text=f'Error attempting to parse a mirrorlist: {e} {url}') def add_mirrors_from_urls(repo, mirror_urls): @@ -172,7 +173,7 @@ def add_mirrors_from_urls(repo, mirror_urls): existing = repo.mirror_set.filter(q).count() if existing >= max_mirrors: text = f'{existing} Mirrors already exist (max={max_mirrors}), not adding more' - warning_message.send(sender=None, text=text) + warning_message(text=text) break from repos.models import Mirror # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? @@ -180,7 +181,7 @@ def add_mirrors_from_urls(repo, mirror_urls): m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) if c: text = f'Added Mirror - {mirror_url}' - info_message.send(sender=None, text=text) + info_message(text=text) def check_for_mirrorlists(repo): @@ -193,7 +194,7 @@ def check_for_mirrorlists(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found mirrorlist - {mirror.url}') + info_message(text=f'Found mirrorlist - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -210,7 +211,7 @@ def check_for_metalinks(repo): mirror.mirrorlist = True mirror.last_access_ok = True mirror.save() - info_message.send(sender=None, text=f'Found metalink - {mirror.url}') + info_message(text=f'Found metalink - {mirror.url}') add_mirrors_from_urls(repo, mirror_urls) @@ -249,9 +250,9 @@ def mirror_checksum_is_valid(computed, provided, mirror, metadata_type): """ if not computed or computed != provided: text = f'Checksum failed for mirror {mirror.id}, not refreshing {metadata_type} metadata' - error_message.send(sender=None, text=text) + error_message(text=text) text = f'Found checksum: {computed}\nExpected checksum: {provided}' - error_message.send(sender=None, text=text) + error_message(text=text) mirror.last_access_ok = False mirror.fail() return False @@ -296,9 +297,9 @@ def clean_repos(): repos = Repository.objects.filter(mirror__isnull=True) rlen = repos.count() if rlen == 0: - info_message.send(sender=None, text='No Repositories with zero Mirrors found.') + info_message(text='No Repositories with zero Mirrors found.') else: - info_message.send(sender=None, text=f'Removing {rlen} empty Repositories.') + info_message(text=f'Removing {rlen} empty Repositories.') repos.delete() @@ -309,13 +310,13 @@ def remove_mirror_trailing_slashes(): mirrors = Mirror.objects.filter(url__endswith='/') mlen = mirrors.count() if mlen == 0: - info_message.send(sender=None, text='No Mirrors with trailing slashes found.') + info_message(text='No Mirrors with trailing slashes found.') else: - info_message.send(sender=None, text=f'Removing trailing slashes from {mlen} Mirrors.') + info_message(text=f'Removing trailing slashes from {mlen} Mirrors.') for mirror in mirrors: mirror.url = mirror.url.rstrip('/') try: mirror.save() except IntegrityError: - warning_message.send(sender=None, text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') + warning_message(text=f'Deleting duplicate Mirror {mirror.id}: {mirror.url}') mirror.delete() diff --git a/sbin/patchman b/sbin/patchman index c0911434..47d89b06 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -43,7 +43,7 @@ from reports.models import Report from reports.tasks import clean_reports_with_no_hosts from security.utils import update_cves, update_cwes from util import set_verbosity, get_datetime_now -from patchman.signals import info_message +from util.logging import info_message def get_host(host=None, action='Performing action'): @@ -64,7 +64,7 @@ def get_host(host=None, action='Performing action'): matches = Host.objects.filter(hostname__startswith=host).count() text = f'{matches} Hosts match hostname "{host}"' - info_message.send(sender=None, text=text) + info_message(text=text) return host_obj @@ -84,7 +84,7 @@ def get_hosts(hosts=None, action='Performing action'): host_objs.append(host_obj) else: text = f'{action} for all Hosts\n' - info_message.send(sender=None, text=text) + info_message(text=text) host_objs = Host.objects.all() return host_objs @@ -107,7 +107,7 @@ def get_repos(repo=None, action='Performing action', only_enabled=False): else: repos = Repository.objects.all() - info_message.send(sender=None, text=text) + info_message(text=text) return repos @@ -118,9 +118,9 @@ def refresh_repos(repo=None, force=False): repos = get_repos(repo, 'Refreshing metadata', True) for repo in repos: text = f'Repository {repo.id} : {repo}' - info_message.send(sender=None, text=text) + info_message(text=text) repo.refresh(force) - info_message.send(sender=None, text='') + info_message(text='') def list_repos(repos=None): @@ -161,10 +161,10 @@ def host_updates_alt(host=None): hosts = get_hosts(host, 'Finding updates') ts = get_datetime_now() for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) if host not in updated_hosts: host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') host.updated_at = ts host.save() @@ -200,10 +200,10 @@ def host_updates_alt(host=None): phost.save() updated_hosts.append(phost) text = f'Added the same updates to {phost}' - info_message.send(sender=None, text=text) + info_message(text=text) else: text = 'Updates already added in this run' - info_message.send(sender=None, text=text) + info_message(text=text) def host_updates(host=None): @@ -211,9 +211,9 @@ def host_updates(host=None): """ hosts = get_hosts(host, 'Finding updates') for host in hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.find_updates() - info_message.send(sender=None, text='') + info_message(text='') def diff_hosts(hosts): @@ -236,47 +236,47 @@ def diff_hosts(hosts): repo_diff_AB = reposA.difference(reposB) repo_diff_BA = reposB.difference(reposA) - info_message.send(sender=None, text=f'+ {hostA.hostname}') - info_message.send(sender=None, text=f'- {hostB.hostname}') + info_message(text=f'+ {hostA.hostname}') + info_message(text=f'- {hostB.hostname}') if hostA.os != hostB.os: - info_message.send(sender=None, text='\nOperating Systems') - info_message.send(sender=None, text=f'+ {hostA.os}') - info_message.send(sender=None, text=f'- {hostB.os}') + info_message(text='\nOperating Systems') + info_message(text=f'+ {hostA.os}') + info_message(text=f'- {hostB.os}') else: - info_message.send(sender=None, text='\nNo OS differences') + info_message(text='\nNo OS differences') if hostA.arch != hostB.arch: - info_message.send(sender=None, text='\nArchitecture') - info_message.send(sender=None, text=f'+ {hostA.arch}') - info_message.send(sender=None, text=f'- {hostB.arch}') + info_message(text='\nArchitecture') + info_message(text=f'+ {hostA.arch}') + info_message(text=f'- {hostB.arch}') else: - info_message.send(sender=None, text='\nNo Architecture differences') + info_message(text='\nNo Architecture differences') if hostA.kernel != hostB.kernel: - info_message.send(sender=None, text='\nKernels') - info_message.send(sender=None, text=f'+ {hostA.kernel}') - info_message.send(sender=None, text=f'- {hostB.kernel}') + info_message(text='\nKernels') + info_message(text=f'+ {hostA.kernel}') + info_message(text=f'- {hostB.kernel}') else: - info_message.send(sender=None, text='\nNo Kernel differences') + info_message(text='\nNo Kernel differences') if len(package_diff_AB) != 0 or len(package_diff_BA) != 0: - info_message.send(sender=None, text='\nPackages') + info_message(text='\nPackages') for package in package_diff_AB: - info_message.send(sender=None, text=f'+ {package}') + info_message(text=f'+ {package}') for package in package_diff_BA: - info_message.send(sender=None, text=f'- {package}') + info_message(text=f'- {package}') else: - info_message.send(sender=None, text='\nNo Package differences') + info_message(text='\nNo Package differences') if len(repo_diff_AB) != 0 or len(repo_diff_BA) != 0: - info_message.send(sender=None, text='\nRepositories') + info_message(text='\nRepositories') for repo in repo_diff_AB: - info_message.send(sender=None, text=f'+ {repo}') + info_message(text=f'+ {repo}') for repo in repo_diff_BA: - info_message.send(sender=None, text=f'- {repo}') + info_message(text=f'- {repo}') else: - info_message.send(sender=None, text='\nNo Repo differences') + info_message(text='\nNo Repo differences') def delete_hosts(hosts=None): @@ -286,7 +286,7 @@ def delete_hosts(hosts=None): matching_hosts = get_hosts(hosts) for host in matching_hosts: text = f'Deleting host: {host.hostname}:' - info_message.send(sender=None, text=text) + info_message(text=text) host.delete() @@ -300,7 +300,7 @@ def toggle_host_hro(hosts=None, host_repos_only=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} host_repos_only') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.host_repos_only = host_repos_only host.save() @@ -315,7 +315,7 @@ def toggle_host_check_dns(hosts=None, check_dns=True): if hosts: matching_hosts = get_hosts(hosts, f'{toggle} check_dns') for host in matching_hosts: - info_message.send(sender=None, text=str(host)) + info_message(text=str(host)) host.check_dns = check_dns host.save() @@ -347,7 +347,7 @@ def process_reports(host=None, force=False): text = 'Processing Reports for all Hosts' reports = Report.objects.filter(processed=force).order_by('created') - info_message.send(sender=None, text=text) + info_message(text=text) for report in reports: report.process(find_updates=False) diff --git a/security/models.py b/security/models.py index 9c097eed..7f674a0b 100644 --- a/security/models.py +++ b/security/models.py @@ -152,7 +152,7 @@ def fetch_mitre_cve_data(self): mitre_cve_url = f'https://cveawg.mitre.org/api/cve/{self.cve_id}' res = get_url(mitre_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {mitre_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} MITRE data') cve_json = json.loads(data) @@ -162,7 +162,7 @@ def fetch_osv_dev_cve_data(self): osv_dev_cve_url = f'https://api.osv.dev/v1/vulns/{self.cve_id}' res = get_url(osv_dev_cve_url) if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {osv_dev_cve_url}') return data = fetch_content(res, f'Fetching {self.cve_id} OSV data') cve_json = json.loads(data) @@ -186,7 +186,7 @@ def fetch_nist_cve_data(self): res = get_url(nist_cve_url) data = fetch_content(res, f'Fetching {self.cve_id} NIST data') if res.status_code == 404: - error_message.send(sender=None, text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') + error_message(text=f'404 - Skipping {self.cve_id} - {nist_cve_url}') cve_json = json.loads(data) self.parse_nist_cve_data(cve_json) @@ -197,7 +197,7 @@ def parse_nist_cve_data(self, cve_json): cve = vulnerability.get('cve') cve_id = cve.get('id') if cve_id != self.cve_id: - error_message.send(sender=None, text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') + error_message(text=f'CVE ID mismatch - {self.cve_id} != {cve_id}') return metrics = cve.get('metrics') for metric, score_data in metrics.items(): diff --git a/util/__init__.py b/util/__init__.py index 4a3f9caa..c3dfd6bd 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -38,7 +38,7 @@ from django.utils.dateparse import parse_datetime from django.conf import settings -from patchman.signals import error_message, info_message, debug_message +from util.logging import error_message, info_message, debug_message pbar = None verbose = None @@ -109,7 +109,7 @@ def fetch_content(response, text='', ljust=35): data += chunk return data else: - info_message.send(sender=None, text=text) + info_message(text=text) return response.content @@ -128,16 +128,16 @@ def get_url(url, headers=None, params=None): if not params: params = {} try: - debug_message.send(sender=None, text=f'Trying {url} headers:{headers} params:{params}') + debug_message(text=f'Trying {url} headers:{headers} params:{params}') response = requests.get(url, headers=headers, params=params, stream=True, proxies=proxies, timeout=30) - debug_message.send(sender=None, text=f'{response.status_code}: {response.headers}') + debug_message(text=f'{response.status_code}: {response.headers}') if response.status_code in [403, 404]: return response response.raise_for_status() except requests.exceptions.TooManyRedirects: - error_message.send(sender=None, text=f'Too many redirects - {url}') + error_message(text=f'Too many redirects - {url}') except ConnectionError: - error_message.send(sender=None, text=f'Connection error - {url}') + error_message(text=f'Connection error - {url}') return response @@ -180,7 +180,7 @@ def gunzip(contents): wbits = zlib.MAX_WBITS | 32 return zlib.decompress(contents, wbits) except zlib.error as e: - error_message.send(sender=None, text='gunzip: ' + str(e)) + error_message(text='gunzip: ' + str(e)) def bunzip2(contents): @@ -191,10 +191,10 @@ def bunzip2(contents): return bzip2data except IOError as e: if e == 'invalid data stream': - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) except ValueError as e: if e == "couldn't find end of stream": - error_message.send(sender=None, text='bunzip2: ' + e) + error_message(text='bunzip2: ' + e) def unxz(contents): @@ -204,7 +204,7 @@ def unxz(contents): xzdata = lzma.decompress(contents) return xzdata except lzma.LZMAError as e: - error_message.send(sender=None, text='lzma: ' + e) + error_message(text='lzma: ' + e) def unzstd(contents): @@ -214,7 +214,7 @@ def unzstd(contents): zstddata = zstd.ZstdDecompressor().stream_reader(contents).read() return zstddata except zstd.ZstdError as e: - error_message.send(sender=None, text='zstd: ' + e) + error_message(text=f'zstd: {e}') def extract(data, fmt): @@ -253,7 +253,7 @@ def get_checksum(data, checksum_type): checksum = get_md5(data) else: text = f'Unknown checksum type: {checksum_type}' - error_message.send(sender=None, text=text) + error_message(text=text) return checksum diff --git a/util/logging.py b/util/logging.py new file mode 100644 index 00000000..dd79d296 --- /dev/null +++ b/util/logging.py @@ -0,0 +1,42 @@ +# Copyright 2025 Marcus Furlong +# +# This file is part of Patchman. +# +# Patchman is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 only. +# +# Patchman is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Patchman. If not, see + +from datetime import datetime + +from patchman.signals import info_message_s +from patchman.signals import warning_message_s +from patchman.signals import error_message_s +from patchman.signals import debug_message_s + + +def info_message(text): + ts = datetime.now() + info_message_s.send(sender=None, text=text, ts=ts) + + +def warning_message(text): + ts = datetime.now() + warning_message_s.send(sender=None, text=text, ts=ts) + + +def debug_message(text): + ts = datetime.now() + debug_message_s.send(sender=None, text=text, ts=ts) + + +def error_message(text): + ts = datetime.now() + error_message_s.send(sender=None, text=text, ts=ts) From 00fbd6e42581ffedf93cdfedd47154cb6ffdeea2 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 29 Oct 2025 23:34:50 -0400 Subject: [PATCH 25/32] use redis for caching and use locks for tasks --- errata/tasks.py | 68 ++++++++++++++++++++-------------- etc/patchman/local_settings.py | 16 +++----- reports/tasks.py | 15 +++++++- repos/tasks.py | 17 ++++++++- security/tasks.py | 33 ++++++++++++++--- 5 files changed, 101 insertions(+), 48 deletions(-) diff --git a/errata/tasks.py b/errata/tasks.py index f1d6eeee..d9aaf3d4 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -16,13 +16,15 @@ from celery import shared_task +from django.core.cache import cache + from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.centos import update_centos_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message +from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type @@ -44,34 +46,44 @@ def update_yum_repo_errata(repo_id=None, force=False): def update_errata(erratum_type=None, force=False, repo=None): """ Update all distros errata """ - errata_os_updates = [] - erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] - erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] - if erratum_type: - if erratum_type not in erratum_types: - error_message.send(sender=None, text=f'Erratum type `{erratum_type}` not in {erratum_types}') - else: - errata_os_updates = erratum_type + lock_key = 'update_errata_lock' + # lock will expire after 48 hours + lock_expire = 60 * 60 * 48 + + if cache.add(lock_key, 'true', lock_expire): + try: + errata_os_updates = [] + erratum_types = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian', 'centos'] + erratum_type_defaults = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + if erratum_type: + if erratum_type not in erratum_types: + error_message(text=f'Erratum type `{erratum_type}` not in {erratum_types}') + else: + errata_os_updates = erratum_type + else: + errata_os_updates = get_setting_of_type( + setting_name='ERRATA_OS_UPDATES', + setting_type=list, + default=erratum_type_defaults, + ) + if 'yum' in errata_os_updates: + update_yum_repo_errata(repo_id=repo, force=force) + if 'arch' in errata_os_updates: + update_arch_errata() + if 'alma' in errata_os_updates: + update_alma_errata() + if 'rocky' in errata_os_updates: + update_rocky_errata() + if 'debian' in errata_os_updates: + update_debian_errata() + if 'ubuntu' in errata_os_updates: + update_ubuntu_errata() + if 'centos' in errata_os_updates: + update_centos_errata() + finally: + cache.delete(lock_key) else: - errata_os_updates = get_setting_of_type( - setting_name='ERRATA_OS_UPDATES', - setting_type=list, - default=erratum_type_defaults, - ) - if 'yum' in errata_os_updates: - update_yum_repo_errata(repo_id=repo, force=force) - if 'arch' in errata_os_updates: - update_arch_errata() - if 'alma' in errata_os_updates: - update_alma_errata() - if 'rocky' in errata_os_updates: - update_rocky_errata() - if 'debian' in errata_os_updates: - update_debian_errata() - if 'ubuntu' in errata_os_updates: - update_ubuntu_errata() - if 'centos' in errata_os_updates: - update_centos_errata() + warning_message('Already updating Errata, skipping task.') @shared_task diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..9e7ca21b 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -44,22 +44,16 @@ # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False +# Set the default timeout to e.g. 30 seconds to enable UI caching +# Note that the UI results may be out of date for this amount of time CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://127.0.0.1:6379', + 'TIMEOUT': 0, } } -# Uncomment to enable redis caching for e.g. 30 seconds -# Note that the UI results may be out of date for this amount of time -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.redis.RedisCache', -# 'LOCATION': 'redis://127.0.0.1:6379', -# 'TIMEOUT': 30, -# } -# } - from datetime import timedelta # noqa from celery.schedules import crontab # noqa CELERY_BEAT_SCHEDULE = { diff --git a/reports/tasks.py b/reports/tasks.py index fe294e8d..d2a47e8f 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -17,11 +17,12 @@ from celery import shared_task +from django.core.cache import cache from django.db.utils import OperationalError from hosts.models import Host from reports.models import Report -from util.logging import info_message +from util.logging import info_message, warning_message @shared_task(bind=True, autoretry_for=(OperationalError,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @@ -29,7 +30,17 @@ def process_report(self, report_id): """ Task to process a single report """ report = Report.objects.get(id=report_id) - report.process() + lock_key = f'process_report_lock_{report_id}' + # lock will expire after 1 hour + lock_expire = 60 * 60 + + if cache.add(lock_key, 'true', lock_expire): + try: + report.process() + finally: + cache.delete(lock_key) + else: + warning_message(f'Already processing report {report_id}, skipping task.') @shared_task diff --git a/repos/tasks.py b/repos/tasks.py index 39098fa8..436da82a 100644 --- a/repos/tasks.py +++ b/repos/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from repos.models import Repository +from util.logging import warning_message @shared_task @@ -32,5 +35,15 @@ def refresh_repos(force=False): """ Refresh metadata for all enabled repos """ repos = Repository.objects.filter(enabled=True) - for repo in repos: - refresh_repo.delay(repo.id, force) + lock_key = 'refresh_repos_lock' + # lock will expire after 1 day + lock_expire = 60 * 60 * 24 + + if cache.add(lock_key, 'true', lock_expire): + try: + for repo in repos: + refresh_repo.delay(repo.id, force) + finally: + cache.delete(lock_key) + else: + warning_message('Already refreshing repos, skipping task.') diff --git a/security/tasks.py b/security/tasks.py index a04bb1c8..ce60df83 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -16,7 +16,10 @@ from celery import shared_task +from django.core.cache import cache + from security.models import CVE, CWE +from util.logging import warning_message @shared_task @@ -31,8 +34,18 @@ def update_cve(cve_id): def update_cves(): """ Task to update all CVEs """ - for cve in CVE.objects.all(): - update_cve.delay(cve.id) + lock_key = 'update_cves_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cve in CVE.objects.all(): + update_cve.delay(cve.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CVEs, skipping task.') @shared_task @@ -45,7 +58,17 @@ def update_cwe(cwe_id): @shared_task def update_cwes(): - """ Task to update all CWEa + """ Task to update all CWEs """ - for cwe in CWE.objects.all(): - update_cwe.delay(cwe.id) + lock_key = 'update_cwes_lock' + # lock will expire after 1 week + lock_expire = 60 * 60 * 168 + + if cache.add(lock_key, 'true', lock_expire): + try: + for cwe in CWE.objects.all(): + update_cwe.delay(cwe.id) + finally: + cache.delete(lock_key) + else: + warning_message('Already updating CWEs, skipping task.') From fd6f9aadf43f8e988967bbdc2efe0b3dc1665f94 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Fri, 18 Apr 2025 18:38:04 -0400 Subject: [PATCH 26/32] add errata source options to config file --- errata/sources/distros/debian.py | 2 +- errata/sources/distros/ubuntu.py | 2 +- etc/patchman/local_settings.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 1ae919e4..ece3754d 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -264,7 +264,7 @@ def get_accepted_debian_codenames(): """ Get acceptable Debian OS codenames Can be overridden by specifying DEBIAN_CODENAMES in settings """ - default_codenames = ['bookworm', 'bullseye'] + default_codenames = ['bookworm', 'trixie'] accepted_codenames = get_setting_of_type( setting_name='DEBIAN_CODENAMES', setting_type=list, diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index d1ce7cc5..6fafb40a 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -203,7 +203,7 @@ def get_accepted_ubuntu_codenames(): """ Get acceptable Ubuntu OS codenames Can be overridden by specifying UBUNTU_CODENAMES in settings """ - default_codenames = ['focal', 'jammy', 'noble'] + default_codenames = ['jammy', 'noble'] accepted_codenames = get_setting_of_type( setting_name='UBUNTU_CODENAMES', setting_type=list, diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 181c4c4d..ab93c89d 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -41,6 +41,18 @@ # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14 +# list of errata sources to update, remove unwanted ones to improve performance +ERRATA_OS_UPDATES = ['yum', 'rocky', 'alma', 'arch', 'ubuntu', 'debian'] + +# list of Alma Linux releases to update +ALMA_RELEASES = [8, 9, 10] + +# list of Debian Linux releases to update +DEBIAN_CODENAMES = ['bookworm', 'trixie'] + +# list of Ubuntu Linux releases to update +UBUNTU_CODENAMES = ['jammy', 'noble'] + # Whether to run patchman under the gunicorn web server RUN_GUNICORN = False From f06729e89aafa10910a421a21761d60d54345896 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Wed, 14 May 2025 22:39:29 -0400 Subject: [PATCH 27/32] remove daily cronjob in favour of patchman-celery --- debian/python3-patchman.cron.daily | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 debian/python3-patchman.cron.daily diff --git a/debian/python3-patchman.cron.daily b/debian/python3-patchman.cron.daily deleted file mode 100644 index d4752f75..00000000 --- a/debian/python3-patchman.cron.daily +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -/usr/bin/patchman -a -q From a59a23b5e26b3e2c6436c140055bf55a1b47c712 Mon Sep 17 00:00:00 2001 From: Marcus Furlong Date: Thu, 30 Oct 2025 22:02:44 -0400 Subject: [PATCH 28/32] add isort check --- .github/workflows/lint-and-test.yml | 4 +++ arch/admin.py | 3 ++- arch/serializers.py | 2 +- arch/utils.py | 2 +- arch/views.py | 7 ++--- domains/admin.py | 1 + errata/admin.py | 1 + errata/models.py | 7 +++-- errata/sources/distros/alma.py | 2 +- errata/sources/distros/arch.py | 10 ++++--- errata/sources/distros/centos.py | 7 ++--- errata/sources/distros/debian.py | 8 +++--- errata/sources/distros/rocky.py | 12 +++++---- errata/sources/distros/ubuntu.py | 13 ++++++--- errata/sources/repos/yum.py | 4 +-- errata/tasks.py | 7 +++-- errata/utils.py | 4 +-- errata/views.py | 7 +++-- etc/patchman/local_settings.py | 4 ++- hooks/yum/patchman.py | 1 + hooks/zypper/patchman.py | 3 ++- hosts/admin.py | 1 + hosts/migrations/0001_initial.py | 3 ++- hosts/migrations/0002_initial.py | 2 +- .../0004_remove_host_tags_host_tags.py | 3 ++- hosts/migrations/0006_migrate_to_tz_aware.py | 1 + hosts/migrations/0007_alter_host_tags.py | 2 +- hosts/models.py | 3 ++- hosts/tasks.py | 1 - hosts/templatetags/report_alert.py | 2 +- hosts/utils.py | 4 +-- hosts/views.py | 23 ++++++++-------- modules/admin.py | 1 + modules/migrations/0001_initial.py | 2 +- modules/utils.py | 4 +-- modules/views.py | 7 +++-- operatingsystems/admin.py | 3 ++- operatingsystems/forms.py | 4 +-- operatingsystems/migrations/0002_initial.py | 2 +- operatingsystems/migrations/0003_os_arch.py | 2 +- operatingsystems/serializers.py | 2 +- operatingsystems/views.py | 17 +++++++----- packages/admin.py | 1 + packages/migrations/0001_initial.py | 2 +- .../migrations/0002_auto_20250207_1319.py | 2 +- packages/serializers.py | 2 +- packages/utils.py | 4 ++- packages/views.py | 13 ++++----- patchman/__init__.py | 3 +-- patchman/celery.py | 3 ++- patchman/receivers.py | 16 +++++------ patchman/urls.py | 3 +-- patchman/wsgi.py | 3 +-- reports/admin.py | 1 + .../migrations/0004_migrate_to_tz_aware.py | 1 + reports/models.py | 6 +++-- reports/tasks.py | 1 - reports/utils.py | 13 ++++++--- reports/views.py | 19 ++++++------- repos/admin.py | 3 ++- repos/forms.py | 7 +++-- repos/migrations/0001_initial.py | 2 +- repos/migrations/0003_migrate_to_tz_aware.py | 1 + repos/models.py | 9 +++---- repos/repo_types/arch.py | 9 ++++--- repos/repo_types/deb.py | 9 ++++--- repos/repo_types/gentoo.py | 16 +++++++---- repos/repo_types/rpm.py | 7 +++-- repos/repo_types/yast.py | 2 +- repos/repo_types/yum.py | 5 ++-- repos/serializers.py | 2 +- repos/tasks.py | 1 - repos/utils.py | 18 +++++++++---- repos/views.py | 27 ++++++++++--------- sbin/patchman | 21 +++++++++------ security/admin.py | 2 +- security/models.py | 4 +-- security/tasks.py | 1 - security/views.py | 11 ++++---- setup.py | 3 ++- util/__init__.py | 23 +++++++++------- util/filterspecs.py | 5 ++-- util/logging.py | 8 +++--- util/tasks.py | 4 ++- util/templatetags/common.py | 7 +++-- util/views.py | 6 ++--- 86 files changed, 292 insertions(+), 212 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 20b221aa..436698be 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -28,6 +28,10 @@ jobs: pip install flake8 flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 . --count --max-line-length=120 --show-source --statistics + - name: Check isort + run: | + pip install isort + isort --check --profile=django . - name: Set secret key run: ./sbin/patchman-set-secret-key - name: Test with django diff --git a/arch/admin.py b/arch/admin.py index 624a3720..5224711c 100644 --- a/arch/admin.py +++ b/arch/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from arch.models import PackageArchitecture, MachineArchitecture + +from arch.models import MachineArchitecture, PackageArchitecture admin.site.register(PackageArchitecture) admin.site.register(MachineArchitecture) diff --git a/arch/serializers.py b/arch/serializers.py index 5319e796..a5765128 100644 --- a/arch/serializers.py +++ b/arch/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture class PackageArchitectureSerializer(serializers.HyperlinkedModelSerializer): diff --git a/arch/utils.py b/arch/utils.py index 04d0b350..3db6ac70 100644 --- a/arch/utils.py +++ b/arch/utils.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from arch.models import PackageArchitecture, MachineArchitecture +from arch.models import MachineArchitecture, PackageArchitecture from util.logging import info_message diff --git a/arch/views.py b/arch/views.py index 56a2a150..21f6b7c7 100644 --- a/arch/views.py +++ b/arch/views.py @@ -16,9 +16,10 @@ from rest_framework import viewsets -from arch.models import PackageArchitecture, MachineArchitecture -from arch.serializers import PackageArchitectureSerializer, \ - MachineArchitectureSerializer +from arch.models import MachineArchitecture, PackageArchitecture +from arch.serializers import ( + MachineArchitectureSerializer, PackageArchitectureSerializer, +) class PackageArchitectureViewSet(viewsets.ModelViewSet): diff --git a/domains/admin.py b/domains/admin.py index 2ef883e3..5cb0fee3 100644 --- a/domains/admin.py +++ b/domains/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from domains.models import Domain admin.site.register(Domain) diff --git a/errata/admin.py b/errata/admin.py index 88190ff6..ac4b8a50 100644 --- a/errata/admin.py +++ b/errata/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from errata.models import Erratum diff --git a/errata/models.py b/errata/models.py index cfc9bd0d..8c21bcfa 100644 --- a/errata/models.py +++ b/errata/models.py @@ -16,17 +16,16 @@ import json -from django.db import models +from django.db import IntegrityError, models from django.urls import reverse -from django.db import IntegrityError +from errata.managers import ErratumManager from packages.models import Package, PackageUpdate from packages.utils import find_evr, get_matching_packages -from errata.managers import ErratumManager from security.models import CVE, Reference from security.utils import get_or_create_cve, get_or_create_reference -from util.logging import error_message from util import get_url +from util.logging import error_message class Erratum(models.Model): diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index e0f2d4ae..0091b8bf 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -22,8 +22,8 @@ from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string -from util import get_url, fetch_content, get_setting_of_type from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_setting_of_type, get_url def update_alma_errata(concurrent_processing=True): diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index 87c6c47a..e22de403 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -20,11 +20,13 @@ from django.db import connections from operatingsystems.utils import get_or_create_osrelease -from util.logging import error_message -from patchman.signals import pbar_start, pbar_update from packages.models import Package -from packages.utils import find_evr, get_matching_packages, get_or_create_package -from util import get_url, fetch_content +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, +) +from patchman.signals import pbar_start, pbar_update +from util import fetch_content, get_url +from util.logging import error_message def update_arch_errata(concurrent_processing=False): diff --git a/errata/sources/distros/centos.py b/errata/sources/distros/centos.py index d2722a6b..8f4aa4a1 100644 --- a/errata/sources/distros/centos.py +++ b/errata/sources/distros/centos.py @@ -15,14 +15,15 @@ # along with Patchman. If not, see import re + from defusedxml import ElementTree from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package -from util.logging import error_message +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import bunzip2, get_url, fetch_content, get_sha1, get_setting_of_type +from util import bunzip2, fetch_content, get_setting_of_type, get_sha1, get_url +from util.logging import error_message def update_centos_errata(): diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index ece3754d..8025b1bf 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -18,18 +18,18 @@ import csv import re from datetime import datetime -from debian.deb822 import Dsc from io import StringIO +from debian.deb822 import Dsc from django.db import connections from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, find_evr -from util.logging import error_message, warning_message +from packages.utils import find_evr, get_or_create_package from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content, get_setting_of_type, extract +from util import extract, fetch_content, get_setting_of_type, get_url +from util.logging import error_message, warning_message DSCs = {} diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 16d4d12c..2d805985 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -14,19 +14,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import json import concurrent.futures -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +import json from django.db import connections from django.db.utils import OperationalError +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import parse_package_string, get_or_create_package +from packages.utils import get_or_create_package, parse_package_string from patchman.signals import pbar_start, pbar_update -from util import get_url, fetch_content -from util.logging import info_message, error_message +from util import fetch_content, get_url +from util.logging import error_message, info_message def update_rocky_errata(concurrent_processing=True): diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 6fafb40a..5616331f 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -16,8 +16,8 @@ import concurrent.futures import csv -import os import json +import os from io import StringIO from urllib.parse import urlparse @@ -26,10 +26,15 @@ from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package -from packages.utils import get_or_create_package, parse_package_string, find_evr, get_matching_packages -from util import get_url, fetch_content, get_sha256, bunzip2, get_setting_of_type -from util.logging import error_message +from packages.utils import ( + find_evr, get_matching_packages, get_or_create_package, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update +from util import ( + bunzip2, fetch_content, get_setting_of_type, get_sha256, get_url, +) +from util.logging import error_message def update_ubuntu_errata(concurrent_processing=False): diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index f361d10e..8b6732c4 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -16,17 +16,17 @@ import concurrent.futures from io import BytesIO -from defusedxml import ElementTree +from defusedxml import ElementTree from django.db import connections from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package -from util.logging import error_message from patchman.signals import pbar_start, pbar_update from security.models import Reference from util import extract, get_url +from util.logging import error_message def extract_updateinfo(data, url, concurrent_processing=True): diff --git a/errata/tasks.py b/errata/tasks.py index d9aaf3d4..8ded3a79 100644 --- a/errata/tasks.py +++ b/errata/tasks.py @@ -15,19 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache -from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.alma import update_alma_errata -from errata.sources.distros.debian import update_debian_errata +from errata.sources.distros.arch import update_arch_errata from errata.sources.distros.centos import update_centos_errata +from errata.sources.distros.debian import update_debian_errata from errata.sources.distros.rocky import update_rocky_errata from errata.sources.distros.ubuntu import update_ubuntu_errata -from util.logging import error_message, warning_message from repos.models import Repository from security.tasks import update_cves, update_cwes from util import get_setting_of_type +from util.logging import error_message, warning_message @shared_task diff --git a/errata/utils.py b/errata/utils.py index a8d8d424..e0a5e01b 100644 --- a/errata/utils.py +++ b/errata/utils.py @@ -18,11 +18,11 @@ from django.db import connections -from util import tz_aware_datetime from errata.models import Erratum from packages.models import PackageUpdate -from util.logging import warning_message from patchman.signals import pbar_start, pbar_update +from util import tz_aware_datetime +from util.logging import warning_message def get_or_create_erratum(name, e_type, issue_date, synopsis): diff --git a/errata/views.py b/errata/views.py index 42d12f71..8e1c0b2f 100644 --- a/errata/views.py +++ b/errata/views.py @@ -14,16 +14,15 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from operatingsystems.models import OSRelease from errata.models import Erratum from errata.serializers import ErratumSerializer +from operatingsystems.models import OSRelease from util.filterspecs import Filter, FilterBar diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index f7b98cf8..40dd2efd 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -66,8 +66,10 @@ } } -from datetime import timedelta # noqa +from datetime import timedelta # noqa + from celery.schedules import crontab # noqa + CELERY_BEAT_SCHEDULE = { 'process_all_unprocessed_reports': { 'task': 'reports.tasks.process_reports', diff --git a/hooks/yum/patchman.py b/hooks/yum/patchman.py index 343144eb..52f9cc8b 100644 --- a/hooks/yum/patchman.py +++ b/hooks/yum/patchman.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see import os + from yum.plugins import TYPE_CORE requires_api_version = '2.1' diff --git a/hooks/zypper/patchman.py b/hooks/zypper/patchman.py index 14781565..d9d478f3 100755 --- a/hooks/zypper/patchman.py +++ b/hooks/zypper/patchman.py @@ -18,8 +18,9 @@ # # zypp system plugin for patchman -import os import logging +import os + from zypp_plugin import Plugin diff --git a/hosts/admin.py b/hosts/admin.py index 8a42e8cc..43bf31da 100644 --- a/hosts/admin.py +++ b/hosts/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from hosts.models import Host, HostRepo diff --git a/hosts/migrations/0001_initial.py b/hosts/migrations/0001_initial.py index 43366684..0037e094 100644 --- a/hosts/migrations/0001_initial.py +++ b/hosts/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.db import migrations, models + try: import tagging.fields has_tagging = True diff --git a/hosts/migrations/0002_initial.py b/hosts/migrations/0002_initial.py index cc59a70e..6c453c49 100644 --- a/hosts/migrations/0002_initial.py +++ b/hosts/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/hosts/migrations/0004_remove_host_tags_host_tags.py b/hosts/migrations/0004_remove_host_tags_host_tags.py index 84e7affe..bf03a84e 100644 --- a/hosts/migrations/0004_remove_host_tags_host_tags.py +++ b/hosts/migrations/0004_remove_host_tags_host_tags.py @@ -1,8 +1,9 @@ # Generated by Django 4.2.18 on 2025-02-04 23:37 +import taggit.managers from django.apps import apps from django.db import migrations -import taggit.managers + try: import tagging # noqa except ImportError: diff --git a/hosts/migrations/0006_migrate_to_tz_aware.py b/hosts/migrations/0006_migrate_to_tz_aware.py index e36bbf1f..c14ea50b 100644 --- a/hosts/migrations/0006_migrate_to_tz_aware.py +++ b/hosts/migrations/0006_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Host = apps.get_model('hosts', 'Host') for host in Host.objects.all(): diff --git a/hosts/migrations/0007_alter_host_tags.py b/hosts/migrations/0007_alter_host_tags.py index 3858b847..3910a06f 100644 --- a/hosts/migrations/0007_alter_host_tags.py +++ b/hosts/migrations/0007_alter_host_tags.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.19 on 2025-02-28 19:53 -from django.db import migrations import taggit.managers +from django.db import migrations class Migration(migrations.Migration): diff --git a/hosts/models.py b/hosts/models.py index 650544dc..8ea5e3d5 100644 --- a/hosts/models.py +++ b/hosts/models.py @@ -24,6 +24,7 @@ from version_utils.rpm import labelCompare except ImportError: from rpm import labelCompare + from taggit.managers import TaggableManager from arch.models import MachineArchitecture @@ -34,9 +35,9 @@ from operatingsystems.models import OSVariant from packages.models import Package, PackageUpdate from packages.utils import get_or_create_package_update -from util.logging import info_message from repos.models import Repository from repos.utils import find_best_repo +from util.logging import info_message class Host(models.Model): diff --git a/hosts/tasks.py b/hosts/tasks.py index 1643901d..226652f6 100755 --- a/hosts/tasks.py +++ b/hosts/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.db.models import Count from hosts.models import Host diff --git a/hosts/templatetags/report_alert.py b/hosts/templatetags/report_alert.py index a28c5058..48d8f966 100644 --- a/hosts/templatetags/report_alert.py +++ b/hosts/templatetags/report_alert.py @@ -17,9 +17,9 @@ from datetime import timedelta from django.template import Library -from django.utils.html import format_html from django.templatetags.static import static from django.utils import timezone +from django.utils.html import format_html from util import get_setting_of_type diff --git a/hosts/utils.py b/hosts/utils.py index d6e663cf..44441f9b 100644 --- a/hosts/utils.py +++ b/hosts/utils.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from socket import gethostbyaddr, gaierror, herror +from socket import gaierror, gethostbyaddr, herror -from django.db import transaction, IntegrityError +from django.db import IntegrityError, transaction from taggit.models import Tag from util.logging import error_message, info_message diff --git a/hosts/views.py b/hosts/views.py index 0fc83ffa..8f20ab19 100644 --- a/hosts/views.py +++ b/hosts/views.py @@ -15,24 +15,23 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages - -from taggit.models import Tag +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets +from taggit.models import Tag -from util.filterspecs import Filter, FilterBar -from hosts.models import Host, HostRepo -from domains.models import Domain from arch.models import MachineArchitecture -from operatingsystems.models import OSVariant, OSRelease -from reports.models import Report +from domains.models import Domain from hosts.forms import EditHostForm -from hosts.serializers import HostSerializer, HostRepoSerializer +from hosts.models import Host, HostRepo +from hosts.serializers import HostRepoSerializer, HostSerializer +from operatingsystems.models import OSRelease, OSVariant +from reports.models import Report +from util.filterspecs import Filter, FilterBar @login_required diff --git a/modules/admin.py b/modules/admin.py index 33b94d20..9cf21e6c 100644 --- a/modules/admin.py +++ b/modules/admin.py @@ -15,6 +15,7 @@ # along with Patchman. If not, see from django.contrib import admin + from modules.models import Module admin.site.register(Module) diff --git a/modules/migrations/0001_initial.py b/modules/migrations/0001_initial.py index 12a8e278..9c27d425 100644 --- a/modules/migrations/0001_initial.py +++ b/modules/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/modules/utils.py b/modules/utils.py index 05c57c80..0d669478 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -15,10 +15,10 @@ # along with Patchman. If not, see from django.db import IntegrityError -from util.logging import error_message, info_message -from modules.models import Module from arch.models import PackageArchitecture +from modules.models import Module +from util.logging import error_message, info_message def get_or_create_module(name, stream, version, context, arch, repo): diff --git a/modules/views.py b/modules/views.py index b897a709..2d017220 100644 --- a/modules/views.py +++ b/modules/views.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - -from rest_framework import viewsets, permissions +from django.shortcuts import get_object_or_404, render +from rest_framework import permissions, viewsets from modules.models import Module from modules.serializers import ModuleSerializer diff --git a/operatingsystems/admin.py b/operatingsystems/admin.py index 15f5e200..4884b5eb 100644 --- a/operatingsystems/admin.py +++ b/operatingsystems/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from operatingsystems.models import OSVariant, OSRelease + +from operatingsystems.models import OSRelease, OSVariant class OSReleaseAdmin(admin.ModelAdmin): diff --git a/operatingsystems/forms.py b/operatingsystems/forms.py index 548a7d88..fa319182 100644 --- a/operatingsystems/forms.py +++ b/operatingsystems/forms.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ModelForm, ModelMultipleChoiceField -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant from repos.models import Repository diff --git a/operatingsystems/migrations/0002_initial.py b/operatingsystems/migrations/0002_initial.py index 517a3f9a..04cbb411 100644 --- a/operatingsystems/migrations/0002_initial.py +++ b/operatingsystems/migrations/0002_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/migrations/0003_os_arch.py b/operatingsystems/migrations/0003_os_arch.py index 2778ca3f..4d0e0f93 100644 --- a/operatingsystems/migrations/0003_os_arch.py +++ b/operatingsystems/migrations/0003_os_arch.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:02 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/operatingsystems/serializers.py b/operatingsystems/serializers.py index 8418c720..be178909 100644 --- a/operatingsystems/serializers.py +++ b/operatingsystems/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from operatingsystems.models import OSVariant, OSRelease +from operatingsystems.models import OSRelease, OSVariant class OSVariantSerializer(serializers.HyperlinkedModelSerializer): diff --git a/operatingsystems/views.py b/operatingsystems/views.py index 2b696f92..6009f119 100644 --- a/operatingsystems/views.py +++ b/operatingsystems/views.py @@ -15,19 +15,22 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse - from rest_framework import viewsets from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from operatingsystems.forms import AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm -from operatingsystems.serializers import OSVariantSerializer, OSReleaseSerializer +from operatingsystems.forms import ( + AddOSVariantToOSReleaseForm, AddReposToOSReleaseForm, CreateOSReleaseForm, +) +from operatingsystems.models import OSRelease, OSVariant +from operatingsystems.serializers import ( + OSReleaseSerializer, OSVariantSerializer, +) @login_required diff --git a/packages/admin.py b/packages/admin.py index 979ba779..bc4b1aaa 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from packages.models import Package, PackageName, PackageUpdate diff --git a/packages/migrations/0001_initial.py b/packages/migrations/0001_initial.py index 07e7bcb0..bfea3e1a 100644 --- a/packages/migrations/0001_initial.py +++ b/packages/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/migrations/0002_auto_20250207_1319.py b/packages/migrations/0002_auto_20250207_1319.py index 1563d139..4c744203 100644 --- a/packages/migrations/0002_auto_20250207_1319.py +++ b/packages/migrations/0002_auto_20250207_1319.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.25 on 2025-02-07 13:19 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/packages/serializers.py b/packages/serializers.py index 902cb3e0..b6e3fb83 100644 --- a/packages/serializers.py +++ b/packages/serializers.py @@ -16,7 +16,7 @@ from rest_framework import serializers -from packages.models import PackageName, Package, PackageUpdate +from packages.models import Package, PackageName, PackageUpdate class PackageNameSerializer(serializers.HyperlinkedModelSerializer): diff --git a/packages/utils.py b/packages/utils.py index f00f6710..87395ff6 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -21,7 +21,9 @@ from django.db import IntegrityError, transaction from arch.models import PackageArchitecture -from packages.models import PackageName, Package, PackageUpdate, PackageCategory, PackageString +from packages.models import ( + Package, PackageCategory, PackageName, PackageString, PackageUpdate, +) from util.logging import error_message, info_message, warning_message diff --git a/packages/views.py b/packages/views.py index c55a6c72..413faee0 100644 --- a/packages/views.py +++ b/packages/views.py @@ -15,17 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar -from packages.models import PackageName, Package, PackageUpdate from arch.models import PackageArchitecture -from packages.serializers import PackageNameSerializer, PackageSerializer, PackageUpdateSerializer +from packages.models import Package, PackageName, PackageUpdate +from packages.serializers import ( + PackageNameSerializer, PackageSerializer, PackageUpdateSerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/patchman/__init__.py b/patchman/__init__.py index af122cc6..321dd7e5 100644 --- a/patchman/__init__.py +++ b/patchman/__init__.py @@ -14,10 +14,9 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from .receivers import * # noqa - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app +from .receivers import * # noqa __all__ = ('celery_app',) diff --git a/patchman/celery.py b/patchman/celery.py index 3c58edc5..c47f994d 100644 --- a/patchman/celery.py +++ b/patchman/celery.py @@ -15,10 +15,11 @@ # along with Patchman. If not, see import os + from celery import Celery os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa +from django.conf import settings # noqa app = Celery('patchman') app.config_from_object('django.conf:settings', namespace='CELERY') diff --git a/patchman/receivers.py b/patchman/receivers.py index 8d8893ca..19312ed5 100644 --- a/patchman/receivers.py +++ b/patchman/receivers.py @@ -15,16 +15,16 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from colorama import init, Fore, Style -from tqdm import tqdm - +from colorama import Fore, Style, init +from django.conf import settings from django.dispatch import receiver +from tqdm import tqdm -from util import create_pbar, update_pbar, get_verbosity -from patchman.signals import pbar_start, pbar_update, \ - info_message_s, warning_message_s, error_message_s, debug_message_s - -from django.conf import settings +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, pbar_start, pbar_update, + warning_message_s, +) +from util import create_pbar, get_verbosity, update_pbar init(autoreset=True) diff --git a/patchman/urls.py b/patchman/urls.py index ee786566..2ae64f56 100644 --- a/patchman/urls.py +++ b/patchman/urls.py @@ -15,12 +15,11 @@ # You should have received a copy of the GNU General Public License # along with If not, see -from django.conf.urls import include, handler404, handler500 # noqa from django.conf import settings +from django.conf.urls import handler404, handler500, include # noqa from django.contrib import admin from django.urls import path from django.views import static - from rest_framework import routers from arch import views as arch_views diff --git a/patchman/wsgi.py b/patchman/wsgi.py index 9a9b4b7f..16f02d5a 100644 --- a/patchman/wsgi.py +++ b/patchman/wsgi.py @@ -19,7 +19,6 @@ from django.core.wsgi import get_wsgi_application os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') # noqa -from django.conf import settings # noqa - +from django.conf import settings # noqa application = get_wsgi_application() diff --git a/reports/admin.py b/reports/admin.py index a37ec4d0..66e0bf5b 100644 --- a/reports/admin.py +++ b/reports/admin.py @@ -16,6 +16,7 @@ # along with Patchman. If not, see from django.contrib import admin + from reports.models import Report diff --git a/reports/migrations/0004_migrate_to_tz_aware.py b/reports/migrations/0004_migrate_to_tz_aware.py index 98176510..20510dcd 100644 --- a/reports/migrations/0004_migrate_to_tz_aware.py +++ b/reports/migrations/0004_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Report = apps.get_model('reports', 'Report') for report in Report.objects.all(): diff --git a/reports/models.py b/reports/models.py index d529804b..f1e0f6f3 100644 --- a/reports/models.py +++ b/reports/models.py @@ -104,7 +104,7 @@ def process(self, find_updates=True, verbose=False): info_message(text=f'Report {self.id} has already been processed') return - from reports.utils import get_arch, get_os, get_domain + from reports.utils import get_arch, get_domain, get_os arch = get_arch(self.arch) osvariant = get_os(self.os, arch) domain = get_domain(self.domain) @@ -113,7 +113,9 @@ def process(self, find_updates=True, verbose=False): if verbose: info_message(text=f'Processing report {self.id} - {self.host}') - from reports.utils import process_packages, process_repos, process_updates, process_modules + from reports.utils import ( + process_modules, process_packages, process_repos, process_updates, + ) process_repos(report=self, host=host) process_modules(report=self, host=host) process_packages(report=self, host=host) diff --git a/reports/tasks.py b/reports/tasks.py index d2a47e8f..07fb3004 100755 --- a/reports/tasks.py +++ b/reports/tasks.py @@ -16,7 +16,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from django.db.utils import OperationalError diff --git a/reports/utils.py b/reports/utils.py index 76b6e09c..8b12f046 100644 --- a/reports/utils.py +++ b/reports/utils.py @@ -23,13 +23,18 @@ from domains.models import Domain from hosts.models import HostRepo from modules.utils import get_or_create_module -from operatingsystems.utils import get_or_create_osrelease, get_or_create_osvariant +from operatingsystems.utils import ( + get_or_create_osrelease, get_or_create_osvariant, +) from packages.models import Package, PackageCategory -from packages.utils import find_evr, get_or_create_package, get_or_create_package_update, parse_package_string -from util.logging import info_message +from packages.utils import ( + find_evr, get_or_create_package, get_or_create_package_update, + parse_package_string, +) from patchman.signals import pbar_start, pbar_update -from repos.models import Repository, Mirror, MirrorPackage +from repos.models import Mirror, MirrorPackage, Repository from repos.utils import get_or_create_repo +from util.logging import info_message def process_repos(report, host): diff --git a/reports/views.py b/reports/views.py index ccef1bb2..f247deb2 100644 --- a/reports/views.py +++ b/reports/views.py @@ -15,20 +15,21 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -from django.http import HttpResponse, Http404 -from django.views.decorators.csrf import csrf_exempt -from django.shortcuts import get_object_or_404, render, redirect +from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.contrib import messages from django.db.utils import OperationalError +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) -from util.filterspecs import Filter, FilterBar from reports.models import Report +from util.filterspecs import Filter, FilterBar @retry( diff --git a/repos/admin.py b/repos/admin.py index bea87567..a516ff8a 100644 --- a/repos/admin.py +++ b/repos/admin.py @@ -16,7 +16,8 @@ # along with Patchman. If not, see from django.contrib import admin -from repos.models import Repository, Mirror, MirrorPackage + +from repos.models import Mirror, MirrorPackage, Repository class MirrorAdmin(admin.ModelAdmin): diff --git a/repos/forms.py b/repos/forms.py index 0800a5c3..9cb66897 100644 --- a/repos/forms.py +++ b/repos/forms.py @@ -15,10 +15,13 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.forms import ModelForm, ModelMultipleChoiceField, TextInput, Form, ModelChoiceField, ValidationError from django.contrib.admin.widgets import FilteredSelectMultiple +from django.forms import ( + Form, ModelChoiceField, ModelForm, ModelMultipleChoiceField, TextInput, + ValidationError, +) -from repos.models import Repository, Mirror +from repos.models import Mirror, Repository class EditRepoForm(ModelForm): diff --git a/repos/migrations/0001_initial.py b/repos/migrations/0001_initial.py index a99f6878..1ae96a98 100644 --- a/repos/migrations/0001_initial.py +++ b/repos/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-12-11 22:15 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/repos/migrations/0003_migrate_to_tz_aware.py b/repos/migrations/0003_migrate_to_tz_aware.py index dddd78ba..38e30488 100644 --- a/repos/migrations/0003_migrate_to_tz_aware.py +++ b/repos/migrations/0003_migrate_to_tz_aware.py @@ -1,6 +1,7 @@ from django.db import migrations from django.utils import timezone + def make_datetimes_tz_aware(apps, schema_editor): Mirror = apps.get_model('repos', 'Mirror') for mirror in Mirror.objects.all(): diff --git a/repos/models.py b/repos/models.py index a1db2a93..9b9082af 100644 --- a/repos/models.py +++ b/repos/models.py @@ -20,13 +20,12 @@ from arch.models import MachineArchitecture from packages.models import Package -from util import get_setting_of_type - -from repos.repo_types.deb import refresh_deb_repo -from repos.repo_types.rpm import refresh_rpm_repo, refresh_repo_errata from repos.repo_types.arch import refresh_arch_repo +from repos.repo_types.deb import refresh_deb_repo from repos.repo_types.gentoo import refresh_gentoo_repo -from util.logging import info_message, warning_message, error_message +from repos.repo_types.rpm import refresh_repo_errata, refresh_rpm_repo +from util import get_setting_of_type +from util.logging import error_message, info_message, warning_message class Repository(models.Model): diff --git a/repos/repo_types/arch.py b/repos/repo_types/arch.py index 09719428..390b321d 100644 --- a/repos/repo_types/arch.py +++ b/repos/repo_types/arch.py @@ -18,10 +18,13 @@ from io import BytesIO from packages.models import PackageString -from util.logging import info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import get_max_mirrors, fetch_mirror_data, find_mirror_url, update_mirror_packages -from util import get_datetime_now, get_checksum, Checksum +from repos.utils import ( + fetch_mirror_data, find_mirror_url, get_max_mirrors, + update_mirror_packages, +) +from util import Checksum, get_checksum, get_datetime_now +from util.logging import info_message, warning_message def refresh_arch_repo(repo): diff --git a/repos/repo_types/deb.py b/repos/repo_types/deb.py index c6c26d78..33d1f2c4 100644 --- a/repos/repo_types/deb.py +++ b/repos/repo_types/deb.py @@ -15,14 +15,17 @@ # along with Patchman. If not, see import re + from debian.deb822 import Packages from debian.debian_support import Version from packages.models import PackageString -from util.logging import error_message, info_message, warning_message from patchman.signals import pbar_start, pbar_update -from repos.utils import fetch_mirror_data, update_mirror_packages, find_mirror_url -from util import get_datetime_now, get_checksum, Checksum, extract +from repos.utils import ( + fetch_mirror_data, find_mirror_url, update_mirror_packages, +) +from util import Checksum, extract, get_checksum, get_datetime_now +from util.logging import error_message, info_message, warning_message def extract_deb_packages(data, url): diff --git a/repos/repo_types/gentoo.py b/repos/repo_types/gentoo.py index e440f0d5..a0584618 100644 --- a/repos/repo_types/gentoo.py +++ b/repos/repo_types/gentoo.py @@ -14,22 +14,28 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import git import os import shutil import tarfile import tempfile -from defusedxml import ElementTree from fnmatch import fnmatch from io import BytesIO from pathlib import Path +import git +from defusedxml import ElementTree + from packages.models import PackageString from packages.utils import find_evr -from util.logging import info_message, warning_message, error_message from patchman.signals import pbar_start, pbar_update -from repos.utils import add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages -from util import extract, get_url, get_datetime_now, get_checksum, Checksum, fetch_content, response_is_valid +from repos.utils import ( + add_mirrors_from_urls, mirror_checksum_is_valid, update_mirror_packages, +) +from util import ( + Checksum, extract, fetch_content, get_checksum, get_datetime_now, get_url, + response_is_valid, +) +from util.logging import error_message, info_message, warning_message def refresh_gentoo_main_repo(repo): diff --git a/repos/repo_types/rpm.py b/repos/repo_types/rpm.py index 51661809..5ffbb708 100644 --- a/repos/repo_types/rpm.py +++ b/repos/repo_types/rpm.py @@ -16,11 +16,14 @@ from django.db.models import Q -from util.logging import info_message, warning_message from repos.repo_types.yast import refresh_yast_repo from repos.repo_types.yum import refresh_yum_repo -from repos.utils import check_for_metalinks, check_for_mirrorlists, find_mirror_url, get_max_mirrors, fetch_mirror_data +from repos.utils import ( + check_for_metalinks, check_for_mirrorlists, fetch_mirror_data, + find_mirror_url, get_max_mirrors, +) from util import get_datetime_now +from util.logging import info_message, warning_message def refresh_repo_errata(repo): diff --git a/repos/repo_types/yast.py b/repos/repo_types/yast.py index bf594040..e37b9934 100644 --- a/repos/repo_types/yast.py +++ b/repos/repo_types/yast.py @@ -17,10 +17,10 @@ import re from packages.models import PackageString -from util.logging import info_message from patchman.signals import pbar_start, pbar_update from repos.utils import fetch_mirror_data, update_mirror_packages from util import extract +from util.logging import info_message def refresh_yast_repo(mirror, data): diff --git a/repos/repo_types/yum.py b/repos/repo_types/yum.py index bc0fbc4b..1e96db39 100644 --- a/repos/repo_types/yum.py +++ b/repos/repo_types/yum.py @@ -15,17 +15,18 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from repos.models import Repository diff --git a/repos/utils.py b/repos/utils.py index c11c41ad..29e9cdb3 100644 --- a/repos/utils.py +++ b/repos/utils.py @@ -17,17 +17,24 @@ import re from io import BytesIO -from defusedxml import ElementTree -from tenacity import RetryError +from defusedxml import ElementTree from django.db import IntegrityError from django.db.models import Q +from tenacity import RetryError from packages.models import Package -from packages.utils import convert_package_to_packagestring, convert_packagestring_to_package -from util import get_url, fetch_content, response_is_valid, extract, get_checksum, Checksum, get_setting_of_type -from util.logging import info_message, warning_message, error_message, debug_message +from packages.utils import ( + convert_package_to_packagestring, convert_packagestring_to_package, +) from patchman.signals import pbar_start, pbar_update +from util import ( + Checksum, extract, fetch_content, get_checksum, get_setting_of_type, + get_url, response_is_valid, +) +from util.logging import ( + debug_message, error_message, info_message, warning_message, +) def get_or_create_repo(r_name, r_arch, r_type, r_id=None): @@ -176,6 +183,7 @@ def add_mirrors_from_urls(repo, mirror_urls): warning_message(text=text) break from repos.models import Mirror + # FIXME: maybe we should store the mirrorlist url with full path to repomd.xml? # that is what metalink urls return now m, c = Mirror.objects.get_or_create(repo=repo, url=mirror_url.rstrip('/').replace('repodata/repomd.xml', '')) diff --git a/repos/views.py b/repos/views.py index 199c834e..1f0c2bfa 100644 --- a/repos/views.py +++ b/repos/views.py @@ -15,24 +15,27 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render, redirect -from django.http import HttpResponse -from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.urls import reverse -from django.db.models import Q from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db import IntegrityError - +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse from rest_framework import viewsets -from util.filterspecs import Filter, FilterBar +from arch.models import MachineArchitecture from hosts.models import HostRepo -from repos.models import Repository, Mirror, MirrorPackage from operatingsystems.models import OSRelease -from arch.models import MachineArchitecture -from repos.forms import EditRepoForm, LinkRepoForm, CreateRepoForm, EditMirrorForm -from repos.serializers import RepositorySerializer, MirrorSerializer, MirrorPackageSerializer +from repos.forms import ( + CreateRepoForm, EditMirrorForm, EditRepoForm, LinkRepoForm, +) +from repos.models import Mirror, MirrorPackage, Repository +from repos.serializers import ( + MirrorPackageSerializer, MirrorSerializer, RepositorySerializer, +) +from util.filterspecs import Filter, FilterBar @login_required diff --git a/sbin/patchman b/sbin/patchman index 47d89b06..c415abec 100755 --- a/sbin/patchman +++ b/sbin/patchman @@ -17,32 +17,37 @@ # along with Patchman. If not, see +import argparse import os import sys -import argparse +from django import setup as django_setup from django.core.exceptions import MultipleObjectsReturned from django.db.models import Count -from django import setup as django_setup os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'patchman.settings') from django.conf import settings # noqa + django_setup() from arch.utils import clean_architectures -from errata.utils import mark_errata_security_updates, enrich_errata, \ - scan_package_updates_for_affected_packages from errata.tasks import update_errata +from errata.utils import ( + enrich_errata, mark_errata_security_updates, + scan_package_updates_for_affected_packages, +) from hosts.models import Host from hosts.utils import clean_tags from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames -from repos.models import Repository -from repos.utils import clean_repos +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from reports.models import Report from reports.tasks import clean_reports_with_no_hosts +from repos.models import Repository +from repos.utils import clean_repos from security.utils import update_cves, update_cwes -from util import set_verbosity, get_datetime_now +from util import get_datetime_now, set_verbosity from util.logging import info_message diff --git a/security/admin.py b/security/admin.py index 196a9468..aedeaea9 100644 --- a/security/admin.py +++ b/security/admin.py @@ -15,8 +15,8 @@ # along with Patchman. If not, see from django.contrib import admin -from security.models import CWE, CVSS, CVE, Reference +from security.models import CVE, CVSS, CWE, Reference admin.site.register(CWE) admin.site.register(CVSS) diff --git a/security/models.py b/security/models.py index 7f674a0b..0f848260 100644 --- a/security/models.py +++ b/security/models.py @@ -16,14 +16,14 @@ import json import re -from cvss import CVSS2, CVSS3, CVSS4 from time import sleep +from cvss import CVSS2, CVSS3, CVSS4 from django.db import models from django.urls import reverse from security.managers import CVEManager -from util import get_url, fetch_content, tz_aware_datetime, error_message +from util import error_message, fetch_content, get_url, tz_aware_datetime class Reference(models.Model): diff --git a/security/tasks.py b/security/tasks.py index ce60df83..7bff4149 100644 --- a/security/tasks.py +++ b/security/tasks.py @@ -15,7 +15,6 @@ # along with Patchman. If not, see from celery import shared_task - from django.core.cache import cache from security.models import CVE, CWE diff --git a/security/views.py b/security/views.py index 58a686b5..c9e606a6 100644 --- a/security/views.py +++ b/security/views.py @@ -14,17 +14,18 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q - +from django.shortcuts import get_object_or_404, render from rest_framework import viewsets -from packages.models import Package from operatingsystems.models import OSRelease +from packages.models import Package from security.models import CVE, CWE, Reference -from security.serializers import CVESerializer, CWESerializer, ReferenceSerializer +from security.serializers import ( + CVESerializer, CWESerializer, ReferenceSerializer, +) from util.filterspecs import Filter, FilterBar diff --git a/setup.py b/setup.py index 6ec6d974..8e18eaf9 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ # along with Patchman. If not, see import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup with open('VERSION.txt', 'r', encoding='utf_8') as v: version = v.readline().strip() diff --git a/util/__init__.py b/util/__init__.py index c3dfd6bd..c6c9aa0d 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -15,30 +15,35 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -import requests import bz2 -import magic -import zlib import lzma import os +import zlib + +import magic +import requests + try: # python 3.14+ - can also remove the dependency at that stage from compression import zstd except ImportError: import zstandard as zstd + from datetime import datetime, timezone from enum import Enum from hashlib import md5, sha1, sha256, sha512 -from requests.exceptions import HTTPError, Timeout, ConnectionError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential from time import time -from tqdm import tqdm -from django.utils.timezone import make_aware -from django.utils.dateparse import parse_datetime from django.conf import settings +from django.utils.dateparse import parse_datetime +from django.utils.timezone import make_aware +from requests.exceptions import ConnectionError, HTTPError, Timeout +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, wait_exponential, +) +from tqdm import tqdm -from util.logging import error_message, info_message, debug_message +from util.logging import debug_message, error_message, info_message pbar = None verbose = None diff --git a/util/filterspecs.py b/util/filterspecs.py index 722b45df..eac0f747 100644 --- a/util/filterspecs.py +++ b/util/filterspecs.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.utils.safestring import mark_safe -from django.db.models.query import QuerySet from operator import itemgetter +from django.db.models.query import QuerySet +from django.utils.safestring import mark_safe + def get_query_string(qs): new_qs = [f'{k}={v}' for k, v in list(qs.items())] diff --git a/util/logging.py b/util/logging.py index dd79d296..cb00ccce 100644 --- a/util/logging.py +++ b/util/logging.py @@ -14,12 +14,12 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see + from datetime import datetime -from patchman.signals import info_message_s -from patchman.signals import warning_message_s -from patchman.signals import error_message_s -from patchman.signals import debug_message_s +from patchman.signals import ( + debug_message_s, error_message_s, info_message_s, warning_message_s, +) def info_message(text): diff --git a/util/tasks.py b/util/tasks.py index f650e3e2..bd76bac6 100644 --- a/util/tasks.py +++ b/util/tasks.py @@ -18,7 +18,9 @@ from arch.utils import clean_architectures from modules.utils import clean_modules -from packages.utils import clean_packages, clean_packageupdates, clean_packagenames +from packages.utils import ( + clean_packagenames, clean_packages, clean_packageupdates, +) from repos.utils import clean_repos, remove_mirror_trailing_slashes diff --git a/util/templatetags/common.py b/util/templatetags/common.py index 2aea1e5e..674e1721 100644 --- a/util/templatetags/common.py +++ b/util/templatetags/common.py @@ -15,16 +15,15 @@ # along with Patchman. If not, see import re - -from humanize import naturaltime from datetime import datetime, timedelta from urllib.parse import urlencode +from django.core.paginator import Paginator from django.template import Library from django.template.loader import get_template -from django.utils.html import format_html from django.templatetags.static import static -from django.core.paginator import Paginator +from django.utils.html import format_html +from humanize import naturaltime from util import get_setting_of_type diff --git a/util/views.py b/util/views.py index b66db6b0..fd003bca 100644 --- a/util/views.py +++ b/util/views.py @@ -17,16 +17,16 @@ from datetime import datetime, timedelta -from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site from django.db.models import F +from django.shortcuts import render from hosts.models import Host -from operatingsystems.models import OSVariant, OSRelease -from repos.models import Repository, Mirror +from operatingsystems.models import OSRelease, OSVariant from packages.models import Package from reports.models import Report +from repos.models import Mirror, Repository from util import get_setting_of_type From 5fa1ef0e3d7484275238b761b05d07782e08a2f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 00:40:08 +0000 Subject: [PATCH 29/32] Bump django from 4.2.25 to 4.2.26 Bumps [django](https://github.com/django/django) from 4.2.25 to 4.2.26. - [Commits](https://github.com/django/django/compare/4.2.25...4.2.26) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.26 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 788f4241..f67fc2ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.25 +Django==4.2.26 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From aac552d7cf22ee1f6037a9604813cc3bfa870571 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:57:03 +0000 Subject: [PATCH 30/32] Bump django from 4.2.26 to 4.2.27 Bumps [django](https://github.com/django/django) from 4.2.26 to 4.2.27. - [Commits](https://github.com/django/django/compare/4.2.26...4.2.27) --- updated-dependencies: - dependency-name: django dependency-version: 4.2.27 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f67fc2ce..08ce4573 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==4.2.26 +Django==4.2.27 django-taggit==4.0.0 django-extensions==3.2.3 django-bootstrap3==23.1 From bdfafb115e5952760c90fd9b7712f3b4611a9e26 Mon Sep 17 00:00:00 2001 From: Maurice Smiley Date: Thu, 11 Dec 2025 22:45:47 +0000 Subject: [PATCH 31/32] Fix duplicate insert error in get_or_create_module by removing repo from uniqueness filter and handling IntegrityError --- modules/utils.py | 59 ++++++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/modules/utils.py b/modules/utils.py index 0d669478..da44ac45 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -14,46 +14,50 @@ # You should have received a copy of the GNU General Public License # along with Patchman. If not, see -from django.db import IntegrityError +from django.db import IntegrityError, DatabaseError, transaction +from patchman.signals import error_message -from arch.models import PackageArchitecture from modules.models import Module -from util.logging import error_message, info_message - +from arch.models import PackageArchitecture def get_or_create_module(name, stream, version, context, arch, repo): - """ Get or create a module object - Returns the module - """ created = False - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + with transaction.atomic(): + m_arch, _ = PackageArchitecture.objects.get_or_create(name=arch) try: - module, created = Module.objects.get_or_create( - name=name, - stream=stream, - version=version, - context=context, - arch=m_arch, - repo=repo, - ) + with transaction.atomic(): + module, created = Module.objects.get_or_create( + name=name, + stream=stream, + version=version, + context=context, + arch=m_arch, + # keep repo in create, but not in uniqueness filter + defaults={'repo': repo}, + ) except IntegrityError as e: - error_message(text=e) + error_message.send(sender=None, text=e) + # Lookup only on the unique fields module = Module.objects.get( name=name, stream=stream, version=version, context=context, arch=m_arch, - repo=repo, ) - return module + created = False + except DatabaseError as e: + error_message.send(sender=None, text=e) + module = None + return module, created def get_matching_modules(name, stream, version, context, arch): """ Return modules that match name, stream, version, context, and arch, regardless of repo """ - m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) + with transaction.atomic(): + m_arch, c = PackageArchitecture.objects.get_or_create(name=arch) modules = Module.objects.filter( name=name, stream=stream, @@ -62,18 +66,3 @@ def get_matching_modules(name, stream, version, context, arch): arch=m_arch, ) return modules - - -def clean_modules(): - """ Delete modules that have no host or no repo - """ - modules = Module.objects.filter( - host__isnull=True, - repo__isnull=True, - ) - mlen = modules.count() - if mlen == 0: - info_message(text='No orphaned Modules found.') - else: - info_message(text=f'{mlen} orphaned Modules found.') - modules.delete() From ac8a4bc7ee3defe5067104ecf6f3d68f7528a80f Mon Sep 17 00:00:00 2001 From: Maurice Smiley Date: Thu, 11 Dec 2025 22:47:40 +0000 Subject: [PATCH 32/32] Adjust Module field lengths and unique_together for MySQL compatibility --- modules/models.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/modules/models.py b/modules/models.py index 931a41c3..989b2ec9 100644 --- a/modules/models.py +++ b/modules/models.py @@ -24,10 +24,18 @@ class Module(models.Model): - name = models.CharField(max_length=255) - stream = models.CharField(max_length=255) - version = models.CharField(max_length=255) - context = models.CharField(max_length=255) +# name = models.CharField(max_length=255) +# stream = models.CharField(max_length=255) +# version = models.CharField(max_length=255) +# context = models.CharField(max_length=255) +# arch = models.ForeignKey(PackageArchitecture, on_delete=models.CASCADE) +# repo = models.ForeignKey(Repository, on_delete=models.CASCADE) +# packages = models.ManyToManyField(Package, blank=True) + + name = models.CharField(max_length=150) + stream = models.CharField(max_length=150) + version = models.CharField(max_length=150) + context = models.CharField(max_length=150) arch = models.ForeignKey(PackageArchitecture, on_delete=models.CASCADE) repo = models.ForeignKey(Repository, on_delete=models.CASCADE) packages = models.ManyToManyField(Package, blank=True) @@ -35,8 +43,9 @@ class Module(models.Model): class Meta: verbose_name = 'Module' verbose_name_plural = 'Modules' - unique_together = ['name', 'stream', 'version', 'context', 'arch'] - ordering = ['name', 'stream'] + #unique_together = ('name', 'stream', 'version', 'context', 'arch',) + unique_together = (("name", "stream", "version", "context"),) + ordering = ('name', 'stream',) def __str__(self): return f'{self.name}-{self.stream}-{self.version}-{self.version}-{self.context}'