From ddc107ea0b104ea18a3d8de37928cf1171c79ab8 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 8 May 2018 10:00:01 +0100 Subject: [PATCH 01/23] Import for explanation purposes --- radar/exporter/__main__.py | 41 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index a9cdcb84c..c8d52edbf 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -5,6 +5,8 @@ from ConfigParser import ConfigParser import os +import sqlite3 + from cornflake import fields, serializers from cornflake.sqlalchemy_orm import ReferenceField import tablib @@ -51,7 +53,7 @@ def parse_config(config_parser): def main(): # Note: xls doesn't support timezones - formats = ['csv', 'xlsx'] + formats = ['csv', 'xlsx', 'sqlite'] book_formats = ['xlsx'] argument_parser = argparse.ArgumentParser() @@ -101,7 +103,42 @@ def main(): is_dir = os.path.isdir(args.dest) - if args.format in book_formats: + if args.format == 'sqlite': + + # TODO: This will crash if the file already exists + connection = sqlite3.connect(args.dest) + cursor = connection.cursor() + + for name, exporter in exporters: + columns = exporter._columns + resultset = exporter._query + + # Build Table + sqlstring = """ + CREATE TABLE """ + name + """ + ( + """ + " TEXT,\n".join([row[0] for row in columns]) + """ TEXT + ) + """ + + print(sqlstring) + cursor.execute(sqlstring) + + # Populate Table + for row in resultset: + + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len([c[1](row) for c in columns]) * ['']) + """? ) + """ + print(sqlstring) + try: + cursor.execute(sqlstring, [c[1](row) for c in columns]) + except: + print [c[1](row) for c in columns] + raise + + elif args.format in book_formats: if is_dir: for dataset in datasets: dest = os.path.join(args.dest, '%s.%s' % (dataset.title, args.format)) From d3e19f1318c3ae8a481dcdcd63d5fe72840c5fd5 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 8 May 2018 17:47:46 +0100 Subject: [PATCH 02/23] Initial working version --- radar/exporter/__main__.py | 45 ++++++++++++++----------- radar/exporter/exporters.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 19 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index c8d52edbf..d7a7f14ae 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -110,34 +110,41 @@ def main(): cursor = connection.cursor() for name, exporter in exporters: + + # Change so SQLite is happy as a table name + name = name.replace("-", "_") + columns = exporter._columns - resultset = exporter._query + + # And so SQLite is happy as a column name + column_row = list() + for x in [row[0] for row in columns]: + column_row.append(x.replace('-', '_')) # Build Table sqlstring = """ CREATE TABLE """ + name + """ ( - """ + " TEXT,\n".join([row[0] for row in columns]) + """ TEXT + """ + " TEXT,\n".join(column_row) + """ TEXT ) """ - - print(sqlstring) cursor.execute(sqlstring) - - # Populate Table - for row in resultset: - - sqlstring = """ - INSERT INTO """ + name + """ - VALUES (""" + "?,".join(len([c[1](row) for c in columns]) * ['']) + """? ) - """ - print(sqlstring) - try: - cursor.execute(sqlstring, [c[1](row) for c in columns]) - except: - print [c[1](row) for c in columns] - raise - + + resultset = exporter.plain_rows + if resultset is not None: + for insert_row in resultset: + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len(column_row) * ['']) + """? ) + """ + try: + cursor.execute(sqlstring, insert_row) + except Exception: + print(insert_row) + raise + + connection.commit() + elif args.format in book_formats: if is_dir: for dataset in datasets: diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index b01a43c3e..c3c23eb69 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -172,6 +172,24 @@ def get_meta_columns(): ] +def query_to_plain_rows(query, columns): + if query is None: + return None + else: + output_list = list() + for row in query: + insert_row = list() + for x in [c[1](row) for c in columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + output_list.append(insert_row) + + return output_list + + class Exporter(object): def __init__(self, config): self.config = config @@ -189,6 +207,10 @@ def run(self): def dataset(self): return query_to_dataset(self._query, self._columns) + @property + def plain_rows(self): + return query_to_plain_rows(self._query, self._columns) + @register('patients') class PatientExporter(Exporter): @@ -796,6 +818,51 @@ def dataset(self): return dataset +@register('resultslist') +class ResultListExporter(Exporter): + def run(self): + q = queries.get_results(self.config) + + self._columns = [ + column('patient_id'), + column('source_group'), + column('source_type'), + column('date'), + column('pv_code'), + column('value') + ] + self._query = q + + @property + def plain_rows(self): + output_list = list() + + for result in self._query: + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + output_row.append(result.observation.pv_code) + output_row.append(result.value) + + output_list.append(output_row) + + # Add Row for Calculated eGFR, if present + if result.observation.pv_code == 'CREATININE': + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + output_row.append('EGFR') + output_row.append(result.egfr_calculated) + + output_list.append(output_row) + + return output_list + + @register('observations') class ObservationExporter(Exporter): def run(self): From 71c038270b18a3f1c7b9387f416c76ea2c0f129f Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Mon, 14 May 2018 09:17:24 +0100 Subject: [PATCH 03/23] Working Nurture Extract --- radar/exporter/__main__.py | 21 ++------------ radar/exporter/exporters.py | 58 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index d7a7f14ae..2c61e06a7 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -106,6 +106,7 @@ def main(): if args.format == 'sqlite': # TODO: This will crash if the file already exists + # Annoyingly only after its finished querying connection = sqlite3.connect(args.dest) cursor = connection.cursor() @@ -114,29 +115,13 @@ def main(): # Change so SQLite is happy as a table name name = name.replace("-", "_") - columns = exporter._columns - - # And so SQLite is happy as a column name - column_row = list() - for x in [row[0] for row in columns]: - column_row.append(x.replace('-', '_')) - - # Build Table - sqlstring = """ - CREATE TABLE """ + name + """ - ( - """ + " TEXT,\n".join(column_row) + """ TEXT - ) - """ + sqlstring = exporter.get_create_table_string(name) cursor.execute(sqlstring) + sqlstring = exporter.get_insert_string(name) resultset = exporter.plain_rows if resultset is not None: for insert_row in resultset: - sqlstring = """ - INSERT INTO """ + name + """ - VALUES (""" + "?,".join(len(column_row) * ['']) + """? ) - """ try: cursor.execute(sqlstring, insert_row) except Exception: diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index c3c23eb69..9ec266d53 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -211,6 +211,32 @@ def dataset(self): def plain_rows(self): return query_to_plain_rows(self._query, self._columns) + def get_create_table_string(self, name): + columns = self._columns + + # Make Column Names SQL Compatible + column_row = list() + for x in [row[0] for row in columns]: + column_row.append(x.replace('-', '_')) + + # Build Query + sqlstring = """ + CREATE TABLE """ + name + """ + ( + """ + " TEXT,\n".join(column_row) + """ TEXT + ) + """ + + return sqlstring + + def get_insert_string(self, name): + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len(self._columns) * ['']) + """? ) + """ + + return sqlstring + @register('patients') class PatientExporter(Exporter): @@ -407,7 +433,9 @@ def run(self): ] self._columns.extend(get_meta_columns()) q = queries.get_patient_diagnoses(self.config) + self._patient_diagnoses_query = q primary_diagnoses = queries.get_primary_diagnoses(self.config) + self._primary_diagnoses_query = primary_diagnoses primary = make_dataset(self._columns) comorbidity = make_dataset(self._columns) @@ -442,6 +470,36 @@ class PrimaryDiagnosisExporter(DiagnosisExporter): def dataset(self): return self._primary + @property + def plain_rows(self): + if self._patient_diagnoses_query is None: + return None + else: + output_list = list() + for patient_diagnosis in self._patient_diagnoses_query: + insert_row = list() + for x in [c[1](patient_diagnosis) for c in self._columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + diagnosis = patient_diagnosis.diagnosis + + if diagnosis and diagnosis.codes: + for code in diagnosis.codes: + if code.system == 'ERA-EDTA PRD': + insert_row[5] = code.code + elif code.system == 'ICD-10': + insert_row[6] = code.code + elif code.system == 'SNOMED CT': + insert_row[7] = code.code + + if diagnosis in self._primary_diagnoses_query: + output_list.append(insert_row) + + return output_list + @register('comorbidities') class ComorbiditiesExporter(DiagnosisExporter): From 1a7ed25b92254b93cabdc3707cb904d9e26435f5 Mon Sep 17 00:00:00 2001 From: Rapolas Kaselis Date: Mon, 14 May 2018 13:24:46 +0100 Subject: [PATCH 04/23] filter primary diagnoses --- radar/exporter/queries.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/radar/exporter/queries.py b/radar/exporter/queries.py index 8b7814434..45cb3bf69 100644 --- a/radar/exporter/queries.py +++ b/radar/exporter/queries.py @@ -5,7 +5,13 @@ from radar.database import db from radar.models.codes import Code from radar.models.consultants import Consultant, GroupConsultant -from radar.models.diagnoses import Diagnosis, DiagnosisCode, GroupDiagnosis, PatientDiagnosis +from radar.models.diagnoses import ( + Diagnosis, + DiagnosisCode, + GroupDiagnosis, + GROUP_DIAGNOSIS_TYPE, + PatientDiagnosis, +) from radar.models.dialysis import Dialysis from radar.models.family_histories import FamilyHistory, FamilyHistoryRelative from radar.models.fetal_ultrasounds import FetalUltrasound @@ -224,6 +230,7 @@ def get_primary_diagnoses(config): """Return query to get primary diagnoses for data_group.""" q = db.session.query(Diagnosis).filter(GroupDiagnosis.group == config['data_group']) q = q.join(GroupDiagnosis, GroupDiagnosis.diagnosis_id == Diagnosis.id) + q = q.filter(GroupDiagnosis.type == GROUP_DIAGNOSIS_TYPE.PRIMARY) return q From 892a8e56d598aa82eb1e89c83cc664ded0509372 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Mon, 14 May 2018 14:54:20 +0100 Subject: [PATCH 05/23] Change code to keep our egfrs distinct from unit supplied ones --- radar/exporter/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index 9ec266d53..447b5c444 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -913,7 +913,8 @@ def plain_rows(self): output_row.append(result.source_group.code) output_row.append(result.source_type) output_row.append(result.date) - output_row.append('EGFR') + # Keep distinct from unit supplied eGFRs + output_row.append('EGFR_RDR') output_row.append(result.egfr_calculated) output_list.append(output_row) From 59b1a74d3a6c19343eecec8256f1a9ef5d70c02d Mon Sep 17 00:00:00 2001 From: Rapolas Kaselis Date: Thu, 17 May 2018 08:18:37 +0100 Subject: [PATCH 06/23] flake8 --- radar/exporter/__main__.py | 1 - radar/exporter/queries.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 2c61e06a7..d0c0995d9 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -4,7 +4,6 @@ except ImportError: from ConfigParser import ConfigParser import os - import sqlite3 from cornflake import fields, serializers diff --git a/radar/exporter/queries.py b/radar/exporter/queries.py index 747dcd3ba..7d4dd3e2a 100644 --- a/radar/exporter/queries.py +++ b/radar/exporter/queries.py @@ -8,8 +8,8 @@ from radar.models.diagnoses import ( Diagnosis, DiagnosisCode, - GroupDiagnosis, GROUP_DIAGNOSIS_TYPE, + GroupDiagnosis, PatientDiagnosis, ) from radar.models.dialysis import Dialysis From bcf8d326c9f97f4731277fd55dedc03d9969cab1 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 8 May 2018 10:00:01 +0100 Subject: [PATCH 07/23] Import for explanation purposes --- radar/exporter/__main__.py | 41 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index a9cdcb84c..c8d52edbf 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -5,6 +5,8 @@ from ConfigParser import ConfigParser import os +import sqlite3 + from cornflake import fields, serializers from cornflake.sqlalchemy_orm import ReferenceField import tablib @@ -51,7 +53,7 @@ def parse_config(config_parser): def main(): # Note: xls doesn't support timezones - formats = ['csv', 'xlsx'] + formats = ['csv', 'xlsx', 'sqlite'] book_formats = ['xlsx'] argument_parser = argparse.ArgumentParser() @@ -101,7 +103,42 @@ def main(): is_dir = os.path.isdir(args.dest) - if args.format in book_formats: + if args.format == 'sqlite': + + # TODO: This will crash if the file already exists + connection = sqlite3.connect(args.dest) + cursor = connection.cursor() + + for name, exporter in exporters: + columns = exporter._columns + resultset = exporter._query + + # Build Table + sqlstring = """ + CREATE TABLE """ + name + """ + ( + """ + " TEXT,\n".join([row[0] for row in columns]) + """ TEXT + ) + """ + + print(sqlstring) + cursor.execute(sqlstring) + + # Populate Table + for row in resultset: + + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len([c[1](row) for c in columns]) * ['']) + """? ) + """ + print(sqlstring) + try: + cursor.execute(sqlstring, [c[1](row) for c in columns]) + except: + print [c[1](row) for c in columns] + raise + + elif args.format in book_formats: if is_dir: for dataset in datasets: dest = os.path.join(args.dest, '%s.%s' % (dataset.title, args.format)) From fe99f938058a59e15cc057ebdec01e6a55f1828e Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 8 May 2018 17:47:46 +0100 Subject: [PATCH 08/23] Initial working version --- radar/exporter/__main__.py | 45 ++++++++++++++----------- radar/exporter/exporters.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 19 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index c8d52edbf..d7a7f14ae 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -110,34 +110,41 @@ def main(): cursor = connection.cursor() for name, exporter in exporters: + + # Change so SQLite is happy as a table name + name = name.replace("-", "_") + columns = exporter._columns - resultset = exporter._query + + # And so SQLite is happy as a column name + column_row = list() + for x in [row[0] for row in columns]: + column_row.append(x.replace('-', '_')) # Build Table sqlstring = """ CREATE TABLE """ + name + """ ( - """ + " TEXT,\n".join([row[0] for row in columns]) + """ TEXT + """ + " TEXT,\n".join(column_row) + """ TEXT ) """ - - print(sqlstring) cursor.execute(sqlstring) - - # Populate Table - for row in resultset: - - sqlstring = """ - INSERT INTO """ + name + """ - VALUES (""" + "?,".join(len([c[1](row) for c in columns]) * ['']) + """? ) - """ - print(sqlstring) - try: - cursor.execute(sqlstring, [c[1](row) for c in columns]) - except: - print [c[1](row) for c in columns] - raise - + + resultset = exporter.plain_rows + if resultset is not None: + for insert_row in resultset: + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len(column_row) * ['']) + """? ) + """ + try: + cursor.execute(sqlstring, insert_row) + except Exception: + print(insert_row) + raise + + connection.commit() + elif args.format in book_formats: if is_dir: for dataset in datasets: diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index fe8009b13..09806d186 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -182,6 +182,24 @@ def modified_user(x): ] +def query_to_plain_rows(query, columns): + if query is None: + return None + else: + output_list = list() + for row in query: + insert_row = list() + for x in [c[1](row) for c in columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + output_list.append(insert_row) + + return output_list + + class Exporter(object): def __init__(self, config): self.config = config @@ -199,6 +217,10 @@ def run(self): def dataset(self): return query_to_dataset(self._query, self._columns) + @property + def plain_rows(self): + return query_to_plain_rows(self._query, self._columns) + @register('patients') class PatientExporter(Exporter): @@ -812,6 +834,51 @@ def dataset(self): return dataset +@register('resultslist') +class ResultListExporter(Exporter): + def run(self): + q = queries.get_results(self.config) + + self._columns = [ + column('patient_id'), + column('source_group'), + column('source_type'), + column('date'), + column('pv_code'), + column('value') + ] + self._query = q + + @property + def plain_rows(self): + output_list = list() + + for result in self._query: + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + output_row.append(result.observation.pv_code) + output_row.append(result.value) + + output_list.append(output_row) + + # Add Row for Calculated eGFR, if present + if result.observation.pv_code == 'CREATININE': + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + output_row.append('EGFR') + output_row.append(result.egfr_calculated) + + output_list.append(output_row) + + return output_list + + @register('observations') class ObservationExporter(Exporter): def run(self): From f70fe48348a1870a45d48a156bd6c306d525598c Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Mon, 14 May 2018 09:17:24 +0100 Subject: [PATCH 09/23] Working Nurture Extract --- radar/exporter/__main__.py | 21 ++------------ radar/exporter/exporters.py | 58 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index d7a7f14ae..2c61e06a7 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -106,6 +106,7 @@ def main(): if args.format == 'sqlite': # TODO: This will crash if the file already exists + # Annoyingly only after its finished querying connection = sqlite3.connect(args.dest) cursor = connection.cursor() @@ -114,29 +115,13 @@ def main(): # Change so SQLite is happy as a table name name = name.replace("-", "_") - columns = exporter._columns - - # And so SQLite is happy as a column name - column_row = list() - for x in [row[0] for row in columns]: - column_row.append(x.replace('-', '_')) - - # Build Table - sqlstring = """ - CREATE TABLE """ + name + """ - ( - """ + " TEXT,\n".join(column_row) + """ TEXT - ) - """ + sqlstring = exporter.get_create_table_string(name) cursor.execute(sqlstring) + sqlstring = exporter.get_insert_string(name) resultset = exporter.plain_rows if resultset is not None: for insert_row in resultset: - sqlstring = """ - INSERT INTO """ + name + """ - VALUES (""" + "?,".join(len(column_row) * ['']) + """? ) - """ try: cursor.execute(sqlstring, insert_row) except Exception: diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index 09806d186..e7b628795 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -221,6 +221,32 @@ def dataset(self): def plain_rows(self): return query_to_plain_rows(self._query, self._columns) + def get_create_table_string(self, name): + columns = self._columns + + # Make Column Names SQL Compatible + column_row = list() + for x in [row[0] for row in columns]: + column_row.append(x.replace('-', '_')) + + # Build Query + sqlstring = """ + CREATE TABLE """ + name + """ + ( + """ + " TEXT,\n".join(column_row) + """ TEXT + ) + """ + + return sqlstring + + def get_insert_string(self, name): + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len(self._columns) * ['']) + """? ) + """ + + return sqlstring + @register('patients') class PatientExporter(Exporter): @@ -423,7 +449,9 @@ def run(self): ] self._columns.extend(get_meta_columns(self.config)) q = queries.get_patient_diagnoses(self.config) + self._patient_diagnoses_query = q primary_diagnoses = queries.get_primary_diagnoses(self.config) + self._primary_diagnoses_query = primary_diagnoses primary = make_dataset(self._columns) comorbidity = make_dataset(self._columns) @@ -458,6 +486,36 @@ class PrimaryDiagnosisExporter(DiagnosisExporter): def dataset(self): return self._primary + @property + def plain_rows(self): + if self._patient_diagnoses_query is None: + return None + else: + output_list = list() + for patient_diagnosis in self._patient_diagnoses_query: + insert_row = list() + for x in [c[1](patient_diagnosis) for c in self._columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + diagnosis = patient_diagnosis.diagnosis + + if diagnosis and diagnosis.codes: + for code in diagnosis.codes: + if code.system == 'ERA-EDTA PRD': + insert_row[5] = code.code + elif code.system == 'ICD-10': + insert_row[6] = code.code + elif code.system == 'SNOMED CT': + insert_row[7] = code.code + + if diagnosis in self._primary_diagnoses_query: + output_list.append(insert_row) + + return output_list + @register('comorbidities') class ComorbiditiesExporter(DiagnosisExporter): From 38b55385afbf217140959dc8385a573830338fd4 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Mon, 14 May 2018 14:54:20 +0100 Subject: [PATCH 10/23] Change code to keep our egfrs distinct from unit supplied ones --- radar/exporter/exporters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index e7b628795..76960b337 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -929,7 +929,8 @@ def plain_rows(self): output_row.append(result.source_group.code) output_row.append(result.source_type) output_row.append(result.date) - output_row.append('EGFR') + # Keep distinct from unit supplied eGFRs + output_row.append('EGFR_RDR') output_row.append(result.egfr_calculated) output_list.append(output_row) From 7a1d7d70a6d59b92012501408874194285e33afc Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Wed, 23 May 2018 14:08:38 +0100 Subject: [PATCH 11/23] Escape various characters for table/column names --- radar/exporter/__main__.py | 5 ++++- radar/exporter/exporters.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 2c61e06a7..90a8869c0 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -105,7 +105,7 @@ def main(): if args.format == 'sqlite': - # TODO: This will crash if the file already exists + # NOTE: This will crash if the file already exists # Annoyingly only after its finished querying connection = sqlite3.connect(args.dest) cursor = connection.cursor() @@ -114,6 +114,9 @@ def main(): # Change so SQLite is happy as a table name name = name.replace("-", "_") + name = name.replace(" ", "_") + name = name.replace(":", "_") + name = name.replace(">", "_") sqlstring = exporter.get_create_table_string(name) cursor.execute(sqlstring) diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index 76960b337..b084d3f19 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -226,8 +226,10 @@ def get_create_table_string(self, name): # Make Column Names SQL Compatible column_row = list() - for x in [row[0] for row in columns]: - column_row.append(x.replace('-', '_')) + for column_name in [row[0] for row in columns]: + for character in ("-", " ", ":", ">", "/", "<", "(", ")"): + column_name = column_name.replace(character, "_") + column_row.append(column_name) # Build Query sqlstring = """ @@ -891,6 +893,10 @@ def dataset(self): return dataset + @property + def plain_rows(self): + raise NotImplementedError + @register('resultslist') class ResultListExporter(Exporter): From f1f59562bd4472a6e19325003c5ae87d3469b7ad Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Wed, 23 May 2018 14:20:53 +0100 Subject: [PATCH 12/23] Remove duplicate import --- radar/exporter/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index a38985ebd..0ae965c94 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -3,9 +3,8 @@ from configparser import ConfigParser except ImportError: from ConfigParser import ConfigParser -import os -import sqlite3 +import os import sqlite3 from cornflake import fields, serializers From 993c2a799da344af8f2a2a25e347d3f58663ba87 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Wed, 23 May 2018 16:55:25 +0100 Subject: [PATCH 13/23] Raise NotImplementedError --- radar/exporter/__main__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 0ae965c94..98d4d6ab9 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -118,16 +118,25 @@ def main(): name = name.replace(":", "_") name = name.replace(">", "_") + resultset = exporter.plain_rows + sqlstring = exporter.get_create_table_string(name) - cursor.execute(sqlstring) + try: + cursor.execute(sqlstring) + except: + print(exporter._columns) + print(sqlstring) + raise sqlstring = exporter.get_insert_string(name) - resultset = exporter.plain_rows + if resultset is not None: for insert_row in resultset: try: cursor.execute(sqlstring, insert_row) except Exception: + print(exporter._columns) + print(sqlstring) print(insert_row) raise From fc3c569721dacad4e54b02b618be31018d2e046a Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Wed, 23 May 2018 17:20:29 +0100 Subject: [PATCH 14/23] Travis being picky about imports --- radar/exporter/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 8b2489560..7c03ce3f2 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -5,8 +5,8 @@ from ConfigParser import ConfigParser import os -import sqlite3 import socket +import sqlite3 from cornflake import fields, serializers from cornflake.sqlalchemy_orm import ReferenceField @@ -170,7 +170,7 @@ def main(): connection.commit() - elif args.format in book_formats: + elif args.format in book_formats: if is_dir: for dataset in datasets: From 86ebb888489c96ca4a88c49d4d4912f60a0de709 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Wed, 23 May 2018 17:30:18 +0100 Subject: [PATCH 15/23] Travis being picky about lines --- radar/exporter/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 7c03ce3f2..db5ee9395 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -130,7 +130,6 @@ def main(): is_dir = os.path.isdir(args.dest) - if args.format == 'sqlite': # NOTE: This will crash if the file already exists From 64b97a12286bd375c0cec09269ce29fdce4474ae Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 6 Dec 2018 14:35:00 +0000 Subject: [PATCH 16/23] Try to reduce memory usage --- radar/exporter/__main__.py | 17 ++++++++--------- radar/exporter/exporters.py | 7 ++----- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index db5ee9395..272a24db3 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -157,15 +157,14 @@ def main(): sqlstring = exporter.get_insert_string(name) - if resultset is not None: - for insert_row in resultset: - try: - cursor.execute(sqlstring, insert_row) - except Exception: - print(exporter._columns) - print(sqlstring) - print(insert_row) - raise + for insert_row in resultset: + try: + cursor.execute(sqlstring, insert_row) + except Exception: + print(exporter._columns) + print(sqlstring) + print(insert_row) + raise connection.commit() diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index df4979d2c..64908b904 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -915,7 +915,6 @@ def run(self): @property def plain_rows(self): - output_list = list() for result in self._query: output_row = list() @@ -926,7 +925,7 @@ def plain_rows(self): output_row.append(result.observation.pv_code) output_row.append(result.value) - output_list.append(output_row) + yield output_row # Add Row for Calculated eGFR, if present if result.observation.pv_code == 'CREATININE': @@ -939,9 +938,7 @@ def plain_rows(self): output_row.append('EGFR_RDR') output_row.append(result.egfr_calculated) - output_list.append(output_row) - - return output_list + yield output_row @register('observations') From f9d42885f868864f5be93fcc7abc356cac1519cc Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 6 Dec 2018 15:32:26 +0000 Subject: [PATCH 17/23] Try commiting each row to reduce memory --- radar/exporter/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index d3b1ec9fb..d3be76aea 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -176,7 +176,7 @@ def main(): print(insert_row) raise - connection.commit() + connection.commit() elif args.format in book_formats: From 2cb850387680b3d71086963a1eb39558e3465c69 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 6 Dec 2018 15:57:59 +0000 Subject: [PATCH 18/23] Try re-jigging things to reduce memory --- radar/exporter/__main__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index d3be76aea..1bc63bc2a 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -128,6 +128,14 @@ def main(): datasets = [] error = False + is_dir = os.path.isdir(args.dest) + + if args.format == 'sqlite': + # NOTE: This will crash if the file already exists + # Annoyingly only after its finished querying + connection = sqlite3.connect(args.dest) + cursor = connection.cursor() + # Export data for name, exporter in exporters: print('Exporting {0}...'.format(name)) @@ -138,16 +146,9 @@ def main(): dataset.title = name datasets.append(dataset) - is_dir = os.path.isdir(args.dest) - - if args.format == 'sqlite': - - # NOTE: This will crash if the file already exists - # Annoyingly only after its finished querying - connection = sqlite3.connect(args.dest) - cursor = connection.cursor() + if args.format == 'sqlite': - for name, exporter in exporters: + print('Saving to SQLite {0}...'.format(name)) # Change so SQLite is happy as a table name name = name.replace("-", "_") @@ -178,7 +179,7 @@ def main(): connection.commit() - elif args.format in book_formats: + if args.format in book_formats: if is_dir: for dataset in datasets: From dabbe0ba4e962e3e1e77b19fa20021a52039c59f Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 6 Dec 2018 15:59:09 +0000 Subject: [PATCH 19/23] Try re-jigging things to reduce memory --- radar/exporter/__main__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 1bc63bc2a..3d069821c 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -140,11 +140,6 @@ def main(): for name, exporter in exporters: print('Exporting {0}...'.format(name)) exporter.run() - dataset = exporter.dataset - if name == 'nurtureckd': - name = 'visits' - dataset.title = name - datasets.append(dataset) if args.format == 'sqlite': @@ -178,6 +173,12 @@ def main(): raise connection.commit() + else: + dataset = exporter.dataset + if name == 'nurtureckd': + name = 'visits' + dataset.title = name + datasets.append(dataset) if args.format in book_formats: From 231b5ad024bf2fd28e2fd844aedf140cc5e73293 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 11 Dec 2018 13:18:36 +0000 Subject: [PATCH 20/23] radar/exporter/exporters.py --- radar/exporter/__main__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 3d069821c..be27be56c 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -151,8 +151,6 @@ def main(): name = name.replace(":", "_") name = name.replace(">", "_") - resultset = exporter.plain_rows - sqlstring = exporter.get_create_table_string(name) try: cursor.execute(sqlstring) @@ -163,7 +161,7 @@ def main(): sqlstring = exporter.get_insert_string(name) - for insert_row in resultset: + for insert_row in exporter.plain_rows: try: cursor.execute(sqlstring, insert_row) except Exception: From 8baff3ff1faef2ba3769a12024a496b01ca7438d Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Tue, 11 Dec 2018 13:19:05 +0000 Subject: [PATCH 21/23] Try to improve performance by using sqlalchemy yield --- radar/exporter/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index 1e197c4f3..52ba9b1f8 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -972,7 +972,7 @@ def run(self): @property def plain_rows(self): - for result in self._query: + for result in self._query.yield_per(1000): output_row = list() output_row.append(result.patient_id) output_row.append(result.source_group.code) From 83bb58d73f626aff3dfbd5e8c0c90fec4cae27c1 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 21 Nov 2019 14:17:20 +0000 Subject: [PATCH 22/23] Update egfr_calculated to match master --- radar/models/results.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/radar/models/results.py b/radar/models/results.py index f87ac7db2..551737c8b 100644 --- a/radar/models/results.py +++ b/radar/models/results.py @@ -240,32 +240,35 @@ def value_label_or_value(self): @property def egfr_calculated(self): - - if self.observation.short_name.lower() != 'creatinine' or not self.value: - return '' + if self.observation.short_name.lower() != "creatinine" or not self.value: + return "" creat88 = self.value / 88.4 egfr = 0 black_adj = 1 - if self.patient.ethnicity and self.patient.ethnicity.code in ('M', 'N', 'P'): + ethnicity = self.patient.available_ethnicity + if ethnicity and ethnicity.code in ("M", "N", "P"): black_adj = 1.159 - months = self.patient.to_age(date(self.date.year, self.date.month, self.date.day)) + months = self.patient.to_age( + date(self.date.year, self.date.month, self.date.day) + ) if not months: - return '' + return "" years_old = months // 12 - age_adj = 0.993**years_old - - if self.patient.is_female and creat88 > 0.7: - egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-1.209)) - elif self.patient.is_female and creat88 <= 0.7: - egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-0.329)) - elif self.patient.is_male and creat88 > 0.7: - egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-1.209)) - elif self.patient.is_male and creat88 <= 0.7: - egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-0.411)) + age_adj = 0.993 ** years_old + is_female = self.patient.radar_gender == GENDER_FEMALE + is_male = self.patient.radar_gender == GENDER_MALE + if is_female and creat88 > 0.7: + egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-1.209)) + elif is_female and creat88 <= 0.7: + egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-0.329)) + elif is_male and creat88 > 0.7: + egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-1.209)) + elif is_male and creat88 <= 0.7: + egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-0.411)) return egfr From 6226040236fbaf1a501fd66e91ad8e2feea2dce0 Mon Sep 17 00:00:00 2001 From: George Swinnerton Date: Thu, 21 Nov 2019 14:32:30 +0000 Subject: [PATCH 23/23] Add missing import --- radar/models/results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radar/models/results.py b/radar/models/results.py index 551737c8b..89d9cb4bf 100644 --- a/radar/models/results.py +++ b/radar/models/results.py @@ -20,6 +20,7 @@ from radar.database import db from radar.models.common import MetaModelMixin, patient_id_column, patient_relationship, uuid_pk_column from radar.models.logs import log_changes +from radar.models.patient_codes import GENDER_FEMALE, GENDER_MALE from radar.models.types import EnumType from radar.utils import pairwise