diff --git a/radar/exporter/__main__.py b/radar/exporter/__main__.py index 25186bc51..be27be56c 100644 --- a/radar/exporter/__main__.py +++ b/radar/exporter/__main__.py @@ -3,8 +3,10 @@ from configparser import ConfigParser except ImportError: from ConfigParser import ConfigParser + import os import socket +import sqlite3 from cornflake import fields, serializers from cornflake.sqlalchemy_orm import ReferenceField @@ -87,7 +89,7 @@ def log_data_export(config, sections): def main(): # Note: xls doesn't support timezones - formats = ['csv', 'xlsx'] + formats = ['csv', 'xlsx', 'sqlite'] book_formats = ['xlsx'] argument_parser = argparse.ArgumentParser() @@ -124,21 +126,60 @@ def main(): exporters.append((name, exporter)) datasets = [] + error = False + + is_dir = os.path.isdir(args.dest) + + if args.format == 'sqlite': + # NOTE: This will crash if the file already exists + # Annoyingly only after its finished querying + connection = sqlite3.connect(args.dest) + cursor = connection.cursor() # Export data for name, exporter in exporters: print('Exporting {0}...'.format(name)) exporter.run() - dataset = exporter.dataset - if name == 'nurtureckd': - name = 'visits' - dataset.title = name - datasets.append(dataset) - is_dir = os.path.isdir(args.dest) + if args.format == 'sqlite': + + print('Saving to SQLite {0}...'.format(name)) + + # Change so SQLite is happy as a table name + name = name.replace("-", "_") + name = name.replace(" ", "_") + name = name.replace(":", "_") + name = name.replace(">", "_") + + sqlstring = exporter.get_create_table_string(name) + try: + cursor.execute(sqlstring) + except: + print(exporter._columns) + print(sqlstring) + raise + + sqlstring = exporter.get_insert_string(name) + + for insert_row in exporter.plain_rows: + try: + cursor.execute(sqlstring, insert_row) + except Exception: + print(exporter._columns) + print(sqlstring) + print(insert_row) + raise + + connection.commit() + else: + dataset = exporter.dataset + if name == 'nurtureckd': + name = 'visits' + dataset.title = name + datasets.append(dataset) - error = False if args.format in book_formats: + if is_dir: for dataset in datasets: dest = os.path.join(args.dest, '%s.%s' % (dataset.title, args.format)) diff --git a/radar/exporter/exporters.py b/radar/exporter/exporters.py index 7414fcdb9..52ba9b1f8 100644 --- a/radar/exporter/exporters.py +++ b/radar/exporter/exporters.py @@ -191,6 +191,24 @@ def modified_user(x): ] +def query_to_plain_rows(query, columns): + if query is None: + return None + else: + output_list = list() + for row in query: + insert_row = list() + for x in [c[1](row) for c in columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + output_list.append(insert_row) + + return output_list + + class Exporter(object): def __init__(self, config): self.config = config @@ -208,6 +226,38 @@ def run(self): def dataset(self): return query_to_dataset(self._query, self._columns) + @property + def plain_rows(self): + return query_to_plain_rows(self._query, self._columns) + + def get_create_table_string(self, name): + columns = self._columns + + # Make Column Names SQL Compatible + column_row = list() + for column_name in [row[0] for row in columns]: + for character in ("-", " ", ":", ">", "/", "<", "(", ")"): + column_name = column_name.replace(character, "_") + column_row.append(column_name) + + # Build Query + sqlstring = """ + CREATE TABLE """ + name + """ + ( + """ + " TEXT,\n".join(column_row) + """ TEXT + ) + """ + + return sqlstring + + def get_insert_string(self, name): + sqlstring = """ + INSERT INTO """ + name + """ + VALUES (""" + "?,".join(len(self._columns) * ['']) + """? ) + """ + + return sqlstring + @register('patients') class PatientExporter(Exporter): @@ -436,7 +486,9 @@ def run(self): ] self._columns.extend(get_meta_columns(self.config)) q = queries.get_patient_diagnoses(self.config) + self._patient_diagnoses_query = q primary_diagnoses = queries.get_primary_diagnoses(self.config) + self._primary_diagnoses_query = primary_diagnoses primary = make_dataset(self._columns) comorbidity = make_dataset(self._columns) @@ -471,6 +523,36 @@ class PrimaryDiagnosisExporter(DiagnosisExporter): def dataset(self): return self._primary + @property + def plain_rows(self): + if self._patient_diagnoses_query is None: + return None + else: + output_list = list() + for patient_diagnosis in self._patient_diagnoses_query: + insert_row = list() + for x in [c[1](patient_diagnosis) for c in self._columns]: + if getattr(x, 'code', None): + insert_row.append(x.code) + else: + insert_row.append(x) + + diagnosis = patient_diagnosis.diagnosis + + if diagnosis and diagnosis.codes: + for code in diagnosis.codes: + if code.system == 'ERA-EDTA PRD': + insert_row[5] = code.code + elif code.system == 'ICD-10': + insert_row[6] = code.code + elif code.system == 'SNOMED CT': + insert_row[7] = code.code + + if diagnosis in self._primary_diagnoses_query: + output_list.append(insert_row) + + return output_list + @register('comorbidities') class ComorbiditiesExporter(DiagnosisExporter): @@ -867,6 +949,53 @@ def dataset(self): return dataset + @property + def plain_rows(self): + raise NotImplementedError + + +@register('resultslist') +class ResultListExporter(Exporter): + def run(self): + q = queries.get_results(self.config) + + self._columns = [ + column('patient_id'), + column('source_group'), + column('source_type'), + column('date'), + column('pv_code'), + column('value') + ] + self._query = q + + @property + def plain_rows(self): + + for result in self._query.yield_per(1000): + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + output_row.append(result.observation.pv_code) + output_row.append(result.value) + + yield output_row + + # Add Row for Calculated eGFR, if present + if result.observation.pv_code == 'CREATININE': + output_row = list() + output_row.append(result.patient_id) + output_row.append(result.source_group.code) + output_row.append(result.source_type) + output_row.append(result.date) + # Keep distinct from unit supplied eGFRs + output_row.append('EGFR_RDR') + output_row.append(result.egfr_calculated) + + yield output_row + @register('observations') class ObservationExporter(Exporter): diff --git a/radar/models/results.py b/radar/models/results.py index f87ac7db2..89d9cb4bf 100644 --- a/radar/models/results.py +++ b/radar/models/results.py @@ -20,6 +20,7 @@ from radar.database import db from radar.models.common import MetaModelMixin, patient_id_column, patient_relationship, uuid_pk_column from radar.models.logs import log_changes +from radar.models.patient_codes import GENDER_FEMALE, GENDER_MALE from radar.models.types import EnumType from radar.utils import pairwise @@ -240,32 +241,35 @@ def value_label_or_value(self): @property def egfr_calculated(self): - - if self.observation.short_name.lower() != 'creatinine' or not self.value: - return '' + if self.observation.short_name.lower() != "creatinine" or not self.value: + return "" creat88 = self.value / 88.4 egfr = 0 black_adj = 1 - if self.patient.ethnicity and self.patient.ethnicity.code in ('M', 'N', 'P'): + ethnicity = self.patient.available_ethnicity + if ethnicity and ethnicity.code in ("M", "N", "P"): black_adj = 1.159 - months = self.patient.to_age(date(self.date.year, self.date.month, self.date.day)) + months = self.patient.to_age( + date(self.date.year, self.date.month, self.date.day) + ) if not months: - return '' + return "" years_old = months // 12 - age_adj = 0.993**years_old - - if self.patient.is_female and creat88 > 0.7: - egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-1.209)) - elif self.patient.is_female and creat88 <= 0.7: - egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-0.329)) - elif self.patient.is_male and creat88 > 0.7: - egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-1.209)) - elif self.patient.is_male and creat88 <= 0.7: - egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-0.411)) + age_adj = 0.993 ** years_old + is_female = self.patient.radar_gender == GENDER_FEMALE + is_male = self.patient.radar_gender == GENDER_MALE + if is_female and creat88 > 0.7: + egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-1.209)) + elif is_female and creat88 <= 0.7: + egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-0.329)) + elif is_male and creat88 > 0.7: + egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-1.209)) + elif is_male and creat88 <= 0.7: + egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-0.411)) return egfr