From 9c376c70fb7a0c5ac9688f5ff4351f247730ea98 Mon Sep 17 00:00:00 2001 From: Sofya Anokhovskaya Date: Tue, 27 May 2025 18:44:09 +0300 Subject: [PATCH 1/4] feat: compare index --- apps/staff/services/diploma_export.py | 74 +++++++++++++-------------- apps/users/utils.py | 31 +++++------ 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/apps/staff/services/diploma_export.py b/apps/staff/services/diploma_export.py index 91e24c512..d0e3aa313 100644 --- a/apps/staff/services/diploma_export.py +++ b/apps/staff/services/diploma_export.py @@ -18,24 +18,24 @@ class ElectronicDiplomaExportService: """ Service for exporting student data for electronic diplomas. - + This service handles the preparation and export of student data for electronic diplomas, including personal information and course grades. """ - + @staticmethod def get_student_profiles(site, graduated_year: int) -> QuerySet: """ Get student profiles for electronic diplomas export with optimized prefetching. """ - + return StudentProfile.objects.filter( site_id=site.id, type=StudentTypes.REGULAR, - status__in=[StudentStatuses.REINSTATED, ""], + status=StudentStatuses.WILL_GRADUATE, year_of_admission__in=[graduated_year-2, graduated_year-3, graduated_year-4] ).select_related( - 'user', + 'user', 'branch', 'user__yandex_data' ).prefetch_related( @@ -47,7 +47,7 @@ def get_student_profiles(site, graduated_year: int) -> QuerySet: Prefetch( 'user__enrollment_set', queryset=Enrollment.objects.filter( - is_deleted=False, + is_deleted=False, grade__in=GradeTypes.satisfactory_grades ).select_related('course', 'course__meta_course'), to_attr='prefetched_enrollments' @@ -55,7 +55,7 @@ def get_student_profiles(site, graduated_year: int) -> QuerySet: # Prefetch academic_discipline as it's a many-to-many relationship 'academic_disciplines' ) - + @staticmethod def get_meta_courses_data(student_profiles: Iterable[StudentProfile]) -> Tuple[Dict[str, str], List[str], Dict[str, str]]: """ @@ -63,44 +63,44 @@ def get_meta_courses_data(student_profiles: Iterable[StudentProfile]) -> Tuple[D """ # Collect all meta course IDs from student enrollments meta_course_ids = { - enrollment.course.meta_course_id - for profile in student_profiles + enrollment.course.meta_course_id + for profile in student_profiles for enrollment in getattr(profile.user, 'prefetched_enrollments', []) - if any(e.course.is_visible_in_certificates for e in getattr(profile.user, 'prefetched_enrollments', []) + if any(e.course.is_visible_in_certificates for e in getattr(profile.user, 'prefetched_enrollments', []) if e.course.meta_course_id == enrollment.course.meta_course_id) } - - meta_courses = {} + + meta_courses = [] courses_headers = [] header_to_index = {} - + # Get meta courses and create headers for mc in MetaCourse.objects.filter(id__in=meta_course_ids): - meta_courses[mc.name] = mc.index - + meta_courses.append(mc.index) + # Generate header and add to headers list header = f"{mc.index}:evaluation" courses_headers.append(header) header_to_index[header] = mc.index - + return meta_courses, courses_headers, header_to_index - + @staticmethod - def prepare_student_data(student_profiles: Iterable[StudentProfile], meta_courses: Dict[str, str], + def prepare_student_data(student_profiles: Iterable[StudentProfile], meta_courses: Dict[str, str], graduated_year: int) -> Tuple[List[Dict[str, Any]], Set[str]]: """ Prepare student data for CSV export. """ courses_with_grades = set() student_data = [] - + for profile in student_profiles: user = profile.user - + # Get courses with grades for this student course_results = profile.get_courses_grades(meta_courses) courses_with_grades.update(course_results.keys()) - + # Prepare base data for the student base_data = [ user.yandex_login, @@ -118,47 +118,47 @@ def prepare_student_data(student_profiles: Iterable[StudentProfile], meta_course profile.academic_discipline if profile.academic_discipline else '', profile.get_passed_courses_total(), ] - + student_data.append({ 'base_data': base_data, 'course_results': course_results }) - + return student_data, courses_with_grades - + @staticmethod - def create_csv_response(student_data: List[Dict[str, Any]], courses_headers: List[str], + def create_csv_response(student_data: List[Dict[str, Any]], courses_headers: List[str], header_to_index: Dict[str, str], graduated_year: int) -> HttpResponse: """ Create CSV response with student data. """ response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="export_for_electronic_diplomas_{graduated_year}.csv"' - + writer = csv.writer(response) - + # Write header row writer.writerow([ - 'yauid', 'last_name', 'first_name', 'middle_name', 'birth_date', - 'snils', 'citizenship', 'sex', 'study_period_from', 'study_period_to', + 'yauid', 'last_name', 'first_name', 'middle_name', 'birth_date', + 'snils', 'citizenship', 'sex', 'study_period_from', 'study_period_to', 'number', 'issue_date', 'frdo_qual', 'control_mc_sum', *courses_headers ]) - + # Write data for each student for data in student_data: row_data = data['base_data'].copy() course_results = data['course_results'] - + # Add grades for each course in the headers for header in courses_headers: course_index = header_to_index[header] grade = course_results.get(course_index, "") row_data.append(grade.lower()) - + writer.writerow(row_data) - + return response - + @classmethod def generate_export(cls, site, graduated_year: int) -> HttpResponse: """ @@ -166,12 +166,12 @@ def generate_export(cls, site, graduated_year: int) -> HttpResponse: """ # Get student profiles with optimized prefetching student_profiles = cls.get_student_profiles(site, graduated_year) - + # Get meta courses data and generate headers meta_courses, courses_headers, header_to_index = cls.get_meta_courses_data(student_profiles) - + # Prepare student data student_data, _ = cls.prepare_student_data(student_profiles, meta_courses, graduated_year) - + # Create CSV response return cls.create_csv_response(student_data, courses_headers, header_to_index, graduated_year) diff --git a/apps/users/utils.py b/apps/users/utils.py index 43f6b80d1..8fc3a8846 100644 --- a/apps/users/utils.py +++ b/apps/users/utils.py @@ -15,52 +15,47 @@ def get_passed_courses_total(profile: "StudentProfile"): for course in shad_records: if course.grade in GradeTypes.satisfactory_grades: shad += 1 - + online = profile.user.onlinecourserecord_set.count() - + regular = 0 if hasattr(profile.user, 'prefetched_enrollments'): enrollments = profile.user.prefetched_enrollments else: enrollments = profile.user.enrollment_set.filter(is_deleted=False) - + if enrollments: for enrollment in enrollments: if enrollment.grade in GradeTypes.satisfactory_grades: regular += 1 - + return shad + online + regular - + def get_courses_grades(profile: "StudentProfile", meta_courses): """ Returns a dictionary mapping course indexes to grade displays for all courses where there is at least one grade. """ result = {} - + # Process regular course enrollments if hasattr(profile.user, 'prefetched_enrollments'): enrollments = profile.user.prefetched_enrollments else: enrollments = profile.user.enrollment_set.filter(is_deleted=False) - + if enrollments: for enrollment in enrollments: if enrollment.grade in GradeTypes.satisfactory_grades: # Try to find the course index by name - course_name = enrollment.course.meta_course.name if hasattr(enrollment.course, 'meta_course') else enrollment.course.name - if course_name in meta_courses and meta_courses[course_name]: - result[meta_courses[course_name]] = enrollment.grade_display.lower() - else: - result[course_name] = enrollment.grade_display.lower() - + course_index = enrollment.course.meta_course.index if hasattr(enrollment.course, 'meta_course') else enrollment.course.index + result[course_index] = enrollment.grade_display.lower() + # Process SHAD courses for course in profile.user.shadcourserecord_set.all(): if course.grade in GradeTypes.satisfactory_grades: # Try to find the course index by name - if course.name in meta_courses and meta_courses[course.name]: - result[meta_courses[course.name]] = course.grade_display.lower() - else: - result[course.name] = course.grade_display.lower() - + if course.index in meta_courses: + result[course.index] = course.grade_display.lower() + return result From 817992d8e1acef2c90ea433f306c560c775edce7 Mon Sep 17 00:00:00 2001 From: Sofya Anokhovskaya Date: Tue, 27 May 2025 18:54:19 +0300 Subject: [PATCH 2/4] feat: + 1 year for fast --- apps/staff/services/diploma_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/staff/services/diploma_export.py b/apps/staff/services/diploma_export.py index d0e3aa313..0a98673a8 100644 --- a/apps/staff/services/diploma_export.py +++ b/apps/staff/services/diploma_export.py @@ -33,7 +33,7 @@ def get_student_profiles(site, graduated_year: int) -> QuerySet: site_id=site.id, type=StudentTypes.REGULAR, status=StudentStatuses.WILL_GRADUATE, - year_of_admission__in=[graduated_year-2, graduated_year-3, graduated_year-4] + year_of_admission__in=[graduated_year-1, graduated_year-2, graduated_year-3, graduated_year-4] ).select_related( 'user', 'branch', From bcefadd5feb761f8b2cbd00085804aabe1446c0b Mon Sep 17 00:00:00 2001 From: Sofya Anokhovskaya Date: Tue, 27 May 2025 19:13:11 +0300 Subject: [PATCH 3/4] feat: tests --- .../tests/test_diploma_export_service.py | 153 ++++++------------ apps/users/utils.py | 7 - 2 files changed, 45 insertions(+), 115 deletions(-) diff --git a/apps/staff/tests/test_diploma_export_service.py b/apps/staff/tests/test_diploma_export_service.py index 908cff9bd..732eee2dd 100644 --- a/apps/staff/tests/test_diploma_export_service.py +++ b/apps/staff/tests/test_diploma_export_service.py @@ -31,22 +31,22 @@ def setup_method(self): current_year = 2025 self.graduated_year = current_year curriculum_year = current_year - 2 - + # Create semester for the courses self.semester = SemesterFactory(year=curriculum_year, type=SemesterTypes.AUTUMN) - + # Create meta courses with indexes self.meta_course1 = MetaCourseFactory(index="CS101") self.meta_course2 = MetaCourseFactory(index="CS102") - + # Create a site object for testing from django.contrib.sites.models import Site self.site = Site.objects.create(domain='example.com', name='Test Site') - + # Create student profiles with Yandex data and user details self.student_profile1 = StudentProfileFactory( year_of_curriculum=curriculum_year, - status="", # Empty status means studying in progress + status=StudentStatuses.WILL_GRADUATE, # Empty status means studying in progress year_of_admission=curriculum_year, type=StudentTypes.REGULAR, user__birth_date="2000-01-01", @@ -54,10 +54,10 @@ def setup_method(self): site=self.site, branch__site=self.site # Ensure branch is associated with the same site ) - + self.student_profile2 = StudentProfileFactory( year_of_curriculum=curriculum_year, - status=StudentStatuses.REINSTATED, + status=StudentStatuses.WILL_GRADUATE, year_of_admission=curriculum_year, type=StudentTypes.REGULAR, user__birth_date="2000-02-02", @@ -65,11 +65,11 @@ def setup_method(self): site=self.site, branch__site=self.site # Ensure branch is associated with the same site ) - + # Create Yandex user data for the students self.yandex_data1 = YandexUserDataFactory(user=self.student_profile1.user) self.yandex_data2 = YandexUserDataFactory(user=self.student_profile2.user) - + # Create courses self.course1 = CourseFactory( meta_course=self.meta_course1, @@ -81,7 +81,7 @@ def setup_method(self): semester=self.semester, is_visible_in_certificates=True ) - + # Create enrollments with passing grades self.enrollment1 = EnrollmentFactory( course=self.course1, @@ -105,7 +105,7 @@ def test_get_student_profiles(self): student_profiles = ElectronicDiplomaExportService.get_student_profiles( self.site, self.graduated_year ) - + # Check that the correct student profiles are returned assert len(student_profiles) == 2 assert self.student_profile1 in student_profiles @@ -115,7 +115,7 @@ def test_get_meta_courses_data(self): """Test the get_meta_courses_data method.""" # Get student profiles student_profiles = [self.student_profile1, self.student_profile2] - + # Set prefetched_enrollments on the user objects self.student_profile1.user.prefetched_enrollments = [ self.enrollment1, self.enrollment2 @@ -123,24 +123,22 @@ def test_get_meta_courses_data(self): self.student_profile2.user.prefetched_enrollments = [ self.enrollment3 ] - + # Call the method meta_courses, courses_headers, header_to_index = ElectronicDiplomaExportService.get_meta_courses_data( student_profiles ) - + # Check the meta courses assert len(meta_courses) == 2 - assert self.meta_course1.name in meta_courses - assert self.meta_course2.name in meta_courses - assert meta_courses[self.meta_course1.name] == self.meta_course1.index - assert meta_courses[self.meta_course2.name] == self.meta_course2.index - + assert self.meta_course1.index in meta_courses + assert self.meta_course2.index in meta_courses + # Check the courses headers assert len(courses_headers) == 2 assert f"{self.meta_course1.index}:evaluation" in courses_headers assert f"{self.meta_course2.index}:evaluation" in courses_headers - + # Check the header to index mapping assert len(header_to_index) == 2 assert header_to_index[f"{self.meta_course1.index}:evaluation"] == self.meta_course1.index @@ -150,7 +148,7 @@ def test_prepare_student_data(self): """Test the prepare_student_data method.""" # Get student profiles student_profiles = [self.student_profile1, self.student_profile2] - + # Set prefetched_enrollments on the user objects self.student_profile1.user.prefetched_enrollments = [ self.enrollment1, self.enrollment2 @@ -158,41 +156,41 @@ def test_prepare_student_data(self): self.student_profile2.user.prefetched_enrollments = [ self.enrollment3 ] - + # Convert birth_date strings to datetime objects if they're strings import datetime if isinstance(self.student_profile1.user.birth_date, str): self.student_profile1.user.birth_date = datetime.datetime.strptime(self.student_profile1.user.birth_date, '%Y-%m-%d').date() if isinstance(self.student_profile2.user.birth_date, str): self.student_profile2.user.birth_date = datetime.datetime.strptime(self.student_profile2.user.birth_date, '%Y-%m-%d').date() - + # Get meta courses data meta_courses, _, _ = ElectronicDiplomaExportService.get_meta_courses_data( student_profiles ) - + # Call the method student_data, courses_with_grades = ElectronicDiplomaExportService.prepare_student_data( student_profiles, meta_courses, self.graduated_year ) - + # Check the student data assert len(student_data) == 2 - + # Check the first student's data assert student_data[0]['base_data'][0] == self.yandex_data1.login assert student_data[0]['base_data'][1] == self.student_profile1.user.last_name assert student_data[0]['base_data'][2] == self.student_profile1.user.first_name - + # Check the second student's data assert student_data[1]['base_data'][0] == self.yandex_data2.login assert student_data[1]['base_data'][1] == self.student_profile2.user.last_name assert student_data[1]['base_data'][2] == self.student_profile2.user.first_name - + # Check the course results assert len(student_data[0]['course_results']) == 2 assert len(student_data[1]['course_results']) == 1 - + # Check the courses with grades assert len(courses_with_grades) == 2 assert self.meta_course1.index in courses_with_grades @@ -202,7 +200,7 @@ def test_create_csv_response(self): """Test the create_csv_response method.""" # Get student profiles student_profiles = [self.student_profile1, self.student_profile2] - + # Set prefetched_enrollments on the user objects self.student_profile1.user.prefetched_enrollments = [ self.enrollment1, self.enrollment2 @@ -210,56 +208,56 @@ def test_create_csv_response(self): self.student_profile2.user.prefetched_enrollments = [ self.enrollment3 ] - + # Convert birth_date strings to datetime objects if they're strings import datetime if isinstance(self.student_profile1.user.birth_date, str): self.student_profile1.user.birth_date = datetime.datetime.strptime(self.student_profile1.user.birth_date, '%Y-%m-%d').date() if isinstance(self.student_profile2.user.birth_date, str): self.student_profile2.user.birth_date = datetime.datetime.strptime(self.student_profile2.user.birth_date, '%Y-%m-%d').date() - + # Get meta courses data meta_courses, courses_headers, header_to_index = ElectronicDiplomaExportService.get_meta_courses_data( student_profiles ) - + # Get student data student_data, _ = ElectronicDiplomaExportService.prepare_student_data( student_profiles, meta_courses, self.graduated_year ) - + # Call the method response = ElectronicDiplomaExportService.create_csv_response( student_data, courses_headers, header_to_index, self.graduated_year ) - + # Check the response assert isinstance(response, HttpResponse) assert response['Content-Type'] == 'text/csv' assert f'attachment; filename="export_for_electronic_diplomas_{self.graduated_year}.csv"' in response['Content-Disposition'] - + # Parse the CSV content content = response.content.decode('utf-8') reader = csv.reader(io.StringIO(content)) rows = list(reader) - + # Check the header row assert rows[0] == [ - 'yauid', 'last_name', 'first_name', 'middle_name', 'birth_date', - 'snils', 'citizenship', 'sex', 'study_period_from', 'study_period_to', - 'number', 'issue_date', 'frdo_qual', 'control_mc_sum', + 'yauid', 'last_name', 'first_name', 'middle_name', 'birth_date', + 'snils', 'citizenship', 'sex', 'study_period_from', 'study_period_to', + 'number', 'issue_date', 'frdo_qual', 'control_mc_sum', f'{self.meta_course1.index}:evaluation', f'{self.meta_course2.index}:evaluation' ] - + # Check the data rows assert len(rows) == 3 # Header + 2 students - + # Check the first student's data assert rows[1][0] == self.yandex_data1.login assert rows[1][1] == self.student_profile1.user.last_name assert rows[1][2] == self.student_profile1.user.first_name - + # Check the second student's data assert rows[2][0] == self.yandex_data2.login assert rows[2][1] == self.student_profile2.user.last_name @@ -272,18 +270,18 @@ def test_generate_export(self): patch.object(ElectronicDiplomaExportService, 'get_meta_courses_data') as mock_get_meta_courses_data, \ patch.object(ElectronicDiplomaExportService, 'prepare_student_data') as mock_prepare_student_data, \ patch.object(ElectronicDiplomaExportService, 'create_csv_response') as mock_create_csv_response: - + # Set up the mocks mock_get_student_profiles.return_value = [self.student_profile1, self.student_profile2] mock_get_meta_courses_data.return_value = ({}, [], {}) mock_prepare_student_data.return_value = ([], set()) mock_create_csv_response.return_value = HttpResponse() - + # Call the method response = ElectronicDiplomaExportService.generate_export( self.site, self.graduated_year ) - + # Check that the mocks were called with the correct arguments mock_get_student_profiles.assert_called_once_with(self.site, self.graduated_year) mock_get_meta_courses_data.assert_called_once_with(mock_get_student_profiles.return_value) @@ -298,67 +296,6 @@ def test_generate_export(self): mock_get_meta_courses_data.return_value[2], self.graduated_year ) - + # Check the response assert response == mock_create_csv_response.return_value - - def test_with_shad_courses(self): - """Test the service with SHAD course records.""" - # Create SHAD course record for the first student - shad_course = SHADCourseRecordFactory( - student=self.student_profile1.user, - name="SHAD Course", - grade=GradeTypes.EXCELLENT, - semester=self.semester - ) - - # Get student profiles - student_profiles = ElectronicDiplomaExportService.get_student_profiles( - self.site, self.graduated_year - ) - - # Set prefetched_enrollments on the user objects - for profile in student_profiles: - if profile.id == self.student_profile1.id: - profile.user.prefetched_enrollments = [ - self.enrollment1, self.enrollment2 - ] - # Convert birth_date string to datetime object if it's a string - import datetime - if isinstance(profile.user.birth_date, str): - profile.user.birth_date = datetime.datetime.strptime(profile.user.birth_date, '%Y-%m-%d').date() - elif profile.id == self.student_profile2.id: - profile.user.prefetched_enrollments = [ - self.enrollment3 - ] - # Convert birth_date string to datetime object if it's a string - import datetime - if isinstance(profile.user.birth_date, str): - profile.user.birth_date = datetime.datetime.strptime(profile.user.birth_date, '%Y-%m-%d').date() - - # Get meta courses data - meta_courses, courses_headers, header_to_index = ElectronicDiplomaExportService.get_meta_courses_data( - student_profiles - ) - - # Get student data - student_data, _ = ElectronicDiplomaExportService.prepare_student_data( - student_profiles, meta_courses, self.graduated_year - ) - - # Create CSV response - response = ElectronicDiplomaExportService.create_csv_response( - student_data, courses_headers, header_to_index, self.graduated_year - ) - - # Parse the CSV content - content = response.content.decode('utf-8') - reader = csv.reader(io.StringIO(content)) - rows = list(reader) - - # Check that the SHAD course is included in the control_mc_sum - # The first student should have 3 courses (2 regular + 1 SHAD) - assert int(rows[1][13]) == 3 - - # The second student should have 1 course - assert int(rows[2][13]) == 1 diff --git a/apps/users/utils.py b/apps/users/utils.py index 8fc3a8846..a010d20d5 100644 --- a/apps/users/utils.py +++ b/apps/users/utils.py @@ -51,11 +51,4 @@ def get_courses_grades(profile: "StudentProfile", meta_courses): course_index = enrollment.course.meta_course.index if hasattr(enrollment.course, 'meta_course') else enrollment.course.index result[course_index] = enrollment.grade_display.lower() - # Process SHAD courses - for course in profile.user.shadcourserecord_set.all(): - if course.grade in GradeTypes.satisfactory_grades: - # Try to find the course index by name - if course.index in meta_courses: - result[course.index] = course.grade_display.lower() - return result From 2d67600893080bbad7785c7be5df606d94d465cf Mon Sep 17 00:00:00 2001 From: Sofya Anokhovskaya Date: Tue, 27 May 2025 20:19:41 +0300 Subject: [PATCH 4/4] feat: new status filter --- apps/staff/services/diploma_export.py | 6 ++---- apps/staff/tests/test_diploma_export_service.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/staff/services/diploma_export.py b/apps/staff/services/diploma_export.py index 0a98673a8..cbfefb050 100644 --- a/apps/staff/services/diploma_export.py +++ b/apps/staff/services/diploma_export.py @@ -6,7 +6,7 @@ from typing import Dict, List, Set, Tuple, Any, Iterable from django.http import HttpResponse -from django.db.models import Prefetch, QuerySet +from django.db.models import Prefetch, QuerySet, Q from courses.models import MetaCourse from learning.models import Enrollment @@ -29,11 +29,9 @@ def get_student_profiles(site, graduated_year: int) -> QuerySet: Get student profiles for electronic diplomas export with optimized prefetching. """ - return StudentProfile.objects.filter( + return StudentProfile.objects.filter( Q(status=StudentStatuses.WILL_GRADUATE) | Q(status=StudentStatuses.GRADUATE, graduation_year=graduated_year), site_id=site.id, type=StudentTypes.REGULAR, - status=StudentStatuses.WILL_GRADUATE, - year_of_admission__in=[graduated_year-1, graduated_year-2, graduated_year-3, graduated_year-4] ).select_related( 'user', 'branch', diff --git a/apps/staff/tests/test_diploma_export_service.py b/apps/staff/tests/test_diploma_export_service.py index 732eee2dd..8e3317fab 100644 --- a/apps/staff/tests/test_diploma_export_service.py +++ b/apps/staff/tests/test_diploma_export_service.py @@ -57,13 +57,14 @@ def setup_method(self): self.student_profile2 = StudentProfileFactory( year_of_curriculum=curriculum_year, - status=StudentStatuses.WILL_GRADUATE, + status=StudentStatuses.GRADUATE, year_of_admission=curriculum_year, type=StudentTypes.REGULAR, user__birth_date="2000-02-02", user__gender=GenderTypes.FEMALE, site=self.site, - branch__site=self.site # Ensure branch is associated with the same site + branch__site=self.site, # Ensure branch is associated with the same site + graduation_year=current_year ) # Create Yandex user data for the students