Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
ddc107e
Import for explanation purposes
George-D-S May 8, 2018
d3e19f1
Initial working version
George-D-S May 8, 2018
71c0382
Working Nurture Extract
George-D-S May 14, 2018
1a7ed25
filter primary diagnoses
May 14, 2018
892a8e5
Change code to keep our egfrs distinct from unit supplied ones
George-D-S May 14, 2018
798ebfa
Merge branch 'master' into nurture-dashboard-exporter
George-D-S May 16, 2018
59b1a74
flake8
May 17, 2018
bcf8d32
Import for explanation purposes
George-D-S May 8, 2018
fe99f93
Initial working version
George-D-S May 8, 2018
f70fe48
Working Nurture Extract
George-D-S May 14, 2018
38b5538
Change code to keep our egfrs distinct from unit supplied ones
George-D-S May 14, 2018
7a1d7d7
Escape various characters for table/column names
George-D-S May 23, 2018
cd4e471
Merge Changes
George-D-S May 23, 2018
f1f5956
Remove duplicate import
George-D-S May 23, 2018
993c2a7
Raise NotImplementedError
George-D-S May 23, 2018
b6b8a54
Merge branch 'master' into nurture-dashboard-exporter
George-D-S May 23, 2018
fc3c569
Travis being picky about imports
George-D-S May 23, 2018
86ebb88
Travis being picky about lines
George-D-S May 23, 2018
d36c552
Merge branch 'master' into nurture-dashboard-exporter
Sep 13, 2018
64b97a1
Try to reduce memory usage
George-D-S Dec 6, 2018
dbb718e
Merge branch 'nurture-dashboard-exporter' of https://github.com/renal…
George-D-S Dec 6, 2018
f9d4288
Try commiting each row to reduce memory
George-D-S Dec 6, 2018
2cb8503
Try re-jigging things to reduce memory
George-D-S Dec 6, 2018
dabbe0b
Try re-jigging things to reduce memory
George-D-S Dec 6, 2018
231b5ad
radar/exporter/exporters.py
George-D-S Dec 11, 2018
8baff3f
Try to improve performance by using sqlalchemy yield
George-D-S Dec 11, 2018
83bb58d
Update egfr_calculated to match master
George-D-S Nov 21, 2019
6226040
Add missing import
George-D-S Nov 21, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 49 additions & 8 deletions radar/exporter/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
from configparser import ConfigParser
except ImportError:
from ConfigParser import ConfigParser

import os
import socket
import sqlite3

from cornflake import fields, serializers
from cornflake.sqlalchemy_orm import ReferenceField
Expand Down Expand Up @@ -87,7 +89,7 @@ def log_data_export(config, sections):

def main():
# Note: xls doesn't support timezones
formats = ['csv', 'xlsx']
formats = ['csv', 'xlsx', 'sqlite']
book_formats = ['xlsx']

argument_parser = argparse.ArgumentParser()
Expand Down Expand Up @@ -124,21 +126,60 @@ def main():
exporters.append((name, exporter))

datasets = []
error = False

is_dir = os.path.isdir(args.dest)

if args.format == 'sqlite':
# NOTE: This will crash if the file already exists
# Annoyingly only after its finished querying
connection = sqlite3.connect(args.dest)
cursor = connection.cursor()

# Export data
for name, exporter in exporters:
print('Exporting {0}...'.format(name))
exporter.run()
dataset = exporter.dataset
if name == 'nurtureckd':
name = 'visits'
dataset.title = name
datasets.append(dataset)

is_dir = os.path.isdir(args.dest)
if args.format == 'sqlite':

print('Saving to SQLite {0}...'.format(name))

# Change so SQLite is happy as a table name
name = name.replace("-", "_")
name = name.replace(" ", "_")
name = name.replace(":", "_")
name = name.replace(">", "_")

sqlstring = exporter.get_create_table_string(name)
try:
cursor.execute(sqlstring)
except:
print(exporter._columns)
print(sqlstring)
raise

sqlstring = exporter.get_insert_string(name)

for insert_row in exporter.plain_rows:
try:
cursor.execute(sqlstring, insert_row)
except Exception:
print(exporter._columns)
print(sqlstring)
print(insert_row)
raise

connection.commit()
else:
dataset = exporter.dataset
if name == 'nurtureckd':
name = 'visits'
dataset.title = name
datasets.append(dataset)

error = False
if args.format in book_formats:

if is_dir:
for dataset in datasets:
dest = os.path.join(args.dest, '%s.%s' % (dataset.title, args.format))
Expand Down
129 changes: 129 additions & 0 deletions radar/exporter/exporters.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,24 @@ def modified_user(x):
]


def query_to_plain_rows(query, columns):
if query is None:
return None
else:
output_list = list()
for row in query:
insert_row = list()
for x in [c[1](row) for c in columns]:
if getattr(x, 'code', None):
insert_row.append(x.code)
else:
insert_row.append(x)

output_list.append(insert_row)

return output_list


class Exporter(object):
def __init__(self, config):
self.config = config
Expand All @@ -208,6 +226,38 @@ def run(self):
def dataset(self):
return query_to_dataset(self._query, self._columns)

@property
def plain_rows(self):
return query_to_plain_rows(self._query, self._columns)

def get_create_table_string(self, name):
columns = self._columns

# Make Column Names SQL Compatible
column_row = list()
for column_name in [row[0] for row in columns]:
for character in ("-", " ", ":", ">", "/", "<", "(", ")"):
column_name = column_name.replace(character, "_")
column_row.append(column_name)

# Build Query
sqlstring = """
CREATE TABLE """ + name + """
(
""" + " TEXT,\n".join(column_row) + """ TEXT
)
"""

return sqlstring

def get_insert_string(self, name):
sqlstring = """
INSERT INTO """ + name + """
VALUES (""" + "?,".join(len(self._columns) * ['']) + """? )
"""

return sqlstring


@register('patients')
class PatientExporter(Exporter):
Expand Down Expand Up @@ -436,7 +486,9 @@ def run(self):
]
self._columns.extend(get_meta_columns(self.config))
q = queries.get_patient_diagnoses(self.config)
self._patient_diagnoses_query = q
primary_diagnoses = queries.get_primary_diagnoses(self.config)
self._primary_diagnoses_query = primary_diagnoses

primary = make_dataset(self._columns)
comorbidity = make_dataset(self._columns)
Expand Down Expand Up @@ -471,6 +523,36 @@ class PrimaryDiagnosisExporter(DiagnosisExporter):
def dataset(self):
return self._primary

@property
def plain_rows(self):
if self._patient_diagnoses_query is None:
return None
else:
output_list = list()
for patient_diagnosis in self._patient_diagnoses_query:
insert_row = list()
for x in [c[1](patient_diagnosis) for c in self._columns]:
if getattr(x, 'code', None):
insert_row.append(x.code)
else:
insert_row.append(x)

diagnosis = patient_diagnosis.diagnosis

if diagnosis and diagnosis.codes:
for code in diagnosis.codes:
if code.system == 'ERA-EDTA PRD':
insert_row[5] = code.code
elif code.system == 'ICD-10':
insert_row[6] = code.code
elif code.system == 'SNOMED CT':
insert_row[7] = code.code

if diagnosis in self._primary_diagnoses_query:
output_list.append(insert_row)

return output_list


@register('comorbidities')
class ComorbiditiesExporter(DiagnosisExporter):
Expand Down Expand Up @@ -867,6 +949,53 @@ def dataset(self):

return dataset

@property
def plain_rows(self):
raise NotImplementedError


@register('resultslist')
class ResultListExporter(Exporter):
def run(self):
q = queries.get_results(self.config)

self._columns = [
column('patient_id'),
column('source_group'),
column('source_type'),
column('date'),
column('pv_code'),
column('value')
]
self._query = q

@property
def plain_rows(self):

for result in self._query.yield_per(1000):
output_row = list()
output_row.append(result.patient_id)
output_row.append(result.source_group.code)
output_row.append(result.source_type)
output_row.append(result.date)
output_row.append(result.observation.pv_code)
output_row.append(result.value)

yield output_row

# Add Row for Calculated eGFR, if present
if result.observation.pv_code == 'CREATININE':
output_row = list()
output_row.append(result.patient_id)
output_row.append(result.source_group.code)
output_row.append(result.source_type)
output_row.append(result.date)
# Keep distinct from unit supplied eGFRs
output_row.append('EGFR_RDR')
output_row.append(result.egfr_calculated)

yield output_row


@register('observations')
class ObservationExporter(Exporter):
Expand Down
36 changes: 20 additions & 16 deletions radar/models/results.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from radar.database import db
from radar.models.common import MetaModelMixin, patient_id_column, patient_relationship, uuid_pk_column
from radar.models.logs import log_changes
from radar.models.patient_codes import GENDER_FEMALE, GENDER_MALE
from radar.models.types import EnumType
from radar.utils import pairwise

Expand Down Expand Up @@ -240,32 +241,35 @@ def value_label_or_value(self):

@property
def egfr_calculated(self):

if self.observation.short_name.lower() != 'creatinine' or not self.value:
return ''
if self.observation.short_name.lower() != "creatinine" or not self.value:
return ""

creat88 = self.value / 88.4
egfr = 0
black_adj = 1

if self.patient.ethnicity and self.patient.ethnicity.code in ('M', 'N', 'P'):
ethnicity = self.patient.available_ethnicity
if ethnicity and ethnicity.code in ("M", "N", "P"):
black_adj = 1.159

months = self.patient.to_age(date(self.date.year, self.date.month, self.date.day))
months = self.patient.to_age(
date(self.date.year, self.date.month, self.date.day)
)
if not months:
return ''
return ""

years_old = months // 12
age_adj = 0.993**years_old

if self.patient.is_female and creat88 > 0.7:
egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-1.209))
elif self.patient.is_female and creat88 <= 0.7:
egfr = age_adj * black_adj * 144 * ((creat88/0.7)**(-0.329))
elif self.patient.is_male and creat88 > 0.7:
egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-1.209))
elif self.patient.is_male and creat88 <= 0.7:
egfr = age_adj * black_adj * 141 * ((creat88/0.9)**(-0.411))
age_adj = 0.993 ** years_old
is_female = self.patient.radar_gender == GENDER_FEMALE
is_male = self.patient.radar_gender == GENDER_MALE
if is_female and creat88 > 0.7:
egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-1.209))
elif is_female and creat88 <= 0.7:
egfr = age_adj * black_adj * 144 * ((creat88 / 0.7) ** (-0.329))
elif is_male and creat88 > 0.7:
egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-1.209))
elif is_male and creat88 <= 0.7:
egfr = age_adj * black_adj * 141 * ((creat88 / 0.9) ** (-0.411))

return egfr

Expand Down