Nurture dashboard exporter

radar/exporter/__main__.py

@@ -3,8 +3,10 @@
     from configparser import ConfigParser
 except ImportError:
     from ConfigParser import ConfigParser
+
 import os
 import socket
+import sqlite3
 
 from cornflake import fields, serializers
 from cornflake.sqlalchemy_orm import ReferenceField
@@ -87,7 +89,7 @@ def log_data_export(config, sections):
 
 def main():
     # Note: xls doesn't support timezones
-    formats = ['csv', 'xlsx']
+    formats = ['csv', 'xlsx', 'sqlite']
     book_formats = ['xlsx']
 
     argument_parser = argparse.ArgumentParser()
@@ -124,21 +126,60 @@ def main():
             exporters.append((name, exporter))
 
         datasets = []
+        error = False
+
+        is_dir = os.path.isdir(args.dest)
+
+        if args.format == 'sqlite':
+            # NOTE: This will crash if the file already exists
+            # Annoyingly only after its finished querying
+            connection = sqlite3.connect(args.dest)
+            cursor = connection.cursor()
 
         # Export data
         for name, exporter in exporters:
             print('Exporting {0}...'.format(name))
             exporter.run()
-            dataset = exporter.dataset
-            if name == 'nurtureckd':
-                name = 'visits'
-            dataset.title = name
-            datasets.append(dataset)
 
-        is_dir = os.path.isdir(args.dest)
+            if args.format == 'sqlite':
+
+                print('Saving to SQLite {0}...'.format(name))
+
+                # Change so SQLite is happy as a table name
+                name = name.replace("-", "_")
+                name = name.replace(" ", "_")
+                name = name.replace(":", "_")
+                name = name.replace(">", "_")
+
+                sqlstring = exporter.get_create_table_string(name)
+                try:
+                    cursor.execute(sqlstring)
+                except:
+                    print(exporter._columns)
+                    print(sqlstring)
+                    raise
+
+                sqlstring = exporter.get_insert_string(name)
+
+                for insert_row in exporter.plain_rows:
+                    try:
+                        cursor.execute(sqlstring, insert_row)
+                    except Exception:
+                        print(exporter._columns)
+                        print(sqlstring)
+                        print(insert_row)
+                        raise
+
+                    connection.commit()
+            else:
+                dataset = exporter.dataset
+                if name == 'nurtureckd':
+                    name = 'visits'
+                dataset.title = name
+                datasets.append(dataset)
 
-        error = False
         if args.format in book_formats:
+
             if is_dir:
                 for dataset in datasets:
                     dest = os.path.join(args.dest, '%s.%s' % (dataset.title, args.format))

radar/exporter/exporters.py

@@ -191,6 +191,24 @@ def modified_user(x):
     ]
 
 
+def query_to_plain_rows(query, columns):
+    if query is None:
+        return None
+    else:
+        output_list = list()
+        for row in query:
+            insert_row = list()
+            for x in [c[1](row) for c in columns]:
+                if getattr(x, 'code', None):
+                    insert_row.append(x.code)
+                else:
+                    insert_row.append(x)
+
+            output_list.append(insert_row)
+
+        return output_list
+
+
 class Exporter(object):
     def __init__(self, config):
         self.config = config
@@ -208,6 +226,38 @@ def run(self):
     def dataset(self):
         return query_to_dataset(self._query, self._columns)
 
+    @property
+    def plain_rows(self):
+        return query_to_plain_rows(self._query, self._columns)
+
+    def get_create_table_string(self, name):
+        columns = self._columns
+
+        # Make Column Names SQL Compatible
+        column_row = list()
+        for column_name in [row[0] for row in columns]:
+            for character in ("-", " ", ":", ">", "/", "<", "(", ")"):
+                column_name = column_name.replace(character, "_")
+            column_row.append(column_name)
+
+        # Build Query
+        sqlstring = """
+        CREATE TABLE """ + name + """
+        (
+        """ + " TEXT,\n".join(column_row) + """ TEXT
+        )
+        """
+
+        return sqlstring
+
+    def get_insert_string(self, name):
+        sqlstring = """
+        INSERT INTO """ + name + """
+        VALUES (""" + "?,".join(len(self._columns) * ['']) + """?  )
+        """
+
+        return sqlstring
+
 
 @register('patients')
 class PatientExporter(Exporter):
@@ -436,7 +486,9 @@ def run(self):
         ]
         self._columns.extend(get_meta_columns(self.config))
         q = queries.get_patient_diagnoses(self.config)
+        self._patient_diagnoses_query = q
         primary_diagnoses = queries.get_primary_diagnoses(self.config)
+        self._primary_diagnoses_query = primary_diagnoses
 
         primary = make_dataset(self._columns)
         comorbidity = make_dataset(self._columns)
@@ -471,6 +523,36 @@ class PrimaryDiagnosisExporter(DiagnosisExporter):
     def dataset(self):
         return self._primary
 
+    @property
+    def plain_rows(self):
+        if self._patient_diagnoses_query is None:
+            return None
+        else:
+            output_list = list()
+            for patient_diagnosis in self._patient_diagnoses_query:
+                insert_row = list()
+                for x in [c[1](patient_diagnosis) for c in self._columns]:
+                    if getattr(x, 'code', None):
+                        insert_row.append(x.code)
+                    else:
+                        insert_row.append(x)
+
+                diagnosis = patient_diagnosis.diagnosis
+
+                if diagnosis and diagnosis.codes:
+                    for code in diagnosis.codes:
+                        if code.system == 'ERA-EDTA PRD':
+                            insert_row[5] = code.code
+                        elif code.system == 'ICD-10':
+                            insert_row[6] = code.code
+                        elif code.system == 'SNOMED CT':
+                            insert_row[7] = code.code
+
+                if diagnosis in self._primary_diagnoses_query:
+                    output_list.append(insert_row)
+
+            return output_list
+
 
 @register('comorbidities')
 class ComorbiditiesExporter(DiagnosisExporter):
@@ -867,6 +949,53 @@ def dataset(self):
 
         return dataset
 
+    @property
+    def plain_rows(self):
+        raise NotImplementedError
+
+
+@register('resultslist')
+class ResultListExporter(Exporter):
+    def run(self):
+        q = queries.get_results(self.config)
+
+        self._columns = [
+            column('patient_id'),
+            column('source_group'),
+            column('source_type'),
+            column('date'),
+            column('pv_code'),
+            column('value')
+        ]
+        self._query = q
+
+    @property
+    def plain_rows(self):
+
+        for result in self._query.yield_per(1000):
+            output_row = list()
+            output_row.append(result.patient_id)
+            output_row.append(result.source_group.code)
+            output_row.append(result.source_type)
+            output_row.append(result.date)
+            output_row.append(result.observation.pv_code)
+            output_row.append(result.value)
+
+            yield output_row
+
+            # Add Row for Calculated eGFR, if present
+            if result.observation.pv_code == 'CREATININE':
+                output_row = list()
+                output_row.append(result.patient_id)
+                output_row.append(result.source_group.code)
+                output_row.append(result.source_type)
+                output_row.append(result.date)
+                # Keep distinct from unit supplied eGFRs
+                output_row.append('EGFR_RDR')
+                output_row.append(result.egfr_calculated)
+
+                yield output_row
+
 
 @register('observations')
 class ObservationExporter(Exporter):

radar/models/results.py

@@ -20,6 +20,7 @@
 from radar.database import db
 from radar.models.common import MetaModelMixin, patient_id_column, patient_relationship, uuid_pk_column
 from radar.models.logs import log_changes
+from radar.models.patient_codes import GENDER_FEMALE, GENDER_MALE
 from radar.models.types import EnumType
 from radar.utils import pairwise
 
@@ -240,32 +241,35 @@ def value_label_or_value(self):
 
     @property
     def egfr_calculated(self):
-
-        if self.observation.short_name.lower() != 'creatinine' or not self.value:
-            return ''
+        if self.observation.short_name.lower() != "creatinine" or not self.value:
+            return ""
 
         creat88 = self.value / 88.4
         egfr = 0
         black_adj = 1
 
-        if self.patient.ethnicity and self.patient.ethnicity.code in ('M', 'N', 'P'):
+        ethnicity = self.patient.available_ethnicity
+        if ethnicity and ethnicity.code in ("M", "N", "P"):
             black_adj = 1.159
 
-        months = self.patient.to_age(date(self.date.year, self.date.month, self.date.day))
+        months = self.patient.to_age(
+            date(self.date.year, self.date.month, self.date.day)
+        )
         if not months:
-            return ''
+            return ""
 
         years_old = months // 12
-        age_adj = 0.993**years_old
-
-        if self.patient.is_female and creat88 > 0.7:
-            egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-1.209))
-        elif self.patient.is_female and creat88 <= 0.7:
-            egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-0.329))
-        elif self.patient.is_male and creat88 > 0.7:
-            egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-1.209))
-        elif self.patient.is_male and creat88 <= 0.7:
-            egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-0.411))
+        age_adj = 0.993 ** years_old
+        is_female = self.patient.radar_gender == GENDER_FEMALE
+        is_male = self.patient.radar_gender == GENDER_MALE
+        if is_female and creat88 > 0.7:
+            egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-1.209))
+        elif is_female and creat88 <= 0.7:
+            egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-0.329))
+        elif is_male and creat88 > 0.7:
+            egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-1.209))
+        elif is_male and creat88 <= 0.7:
+            egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-0.411))
 
         return egfr