diff --git a/openfn/README.md b/openfn/README.md new file mode 100644 index 00000000..ef3c6117 --- /dev/null +++ b/openfn/README.md @@ -0,0 +1,99 @@ +# OpenFn Microservice + +A demo package using OpenFn Microservice, OpenHIM and HAPI FHIR using Instant OpenHIE. + +The `microservice` project.yaml file is located here: [`openfn/docker/config/project.yaml`](./docker/config/project.yaml). + +## Getting Started + +### Docker + +To start up the service + +```sh +yarn +yarn docker:build +yarn docker:instant init -t docker openfn +``` + +To tear down the deployment use the opposing command: + +```bash +yarn docker:instant down -t docker openfn +``` + +To start up the service after a tear down, use the following command: + +```bash +yarn docker:instant up -t docker openfn +``` + +To completely remove all package components use the following option: + +```bash +yarn docker:instant destroy -t docker openfn +``` + +### Kubernetes + +To start up the service + +```sh +yarn +yarn docker:build +yarn docker:instant init -t k8s openfn +``` + +To tear down the deployment use the opposing command: + +```bash +yarn docker:instant down -t k8s openfn +``` + +To start up the service after a tear down, use the following command: + +```bash +yarn docker:instant up -t k8s openfn +``` + +To completely remove all package components use the following option: + +```bash +yarn docker:instant destroy -t k8s openfn +``` + +## How data gets to HAPI FHIR + +Using the example payload [commcare_sample.json](./fixtures/commcare_sample.json) +we send that to the configured OpenFn Microservice. + +Microservice is configurated to run a job based on the shape of the incoming +payload _see [project.yaml](./docker/config/project.yaml)_. + +The job `commcare-to-him` will match against this message and will be invoked +performing the following actions: + +- creates a payload in the FHIR standard containing + - a Encounter resource that contains (`contained` resource field) a Patient resource +- sends the payload to OpenHIM +- which in turn sends the payload to HAPI FHIR + + +### In Docker + +* URL - `http://localhost:4001` + +### In Kubernetes + +Use the host ip or DNS, and the port `4001`. If the deployment has been done to a [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/) cluster, get the external ip and port for the `openfn-service` service by running the following command + +```sh +kubectl get services +``` + +## Notes + +- The HAPI FHIR service runs on port `3447` +- The OpenHIM channel that we go through is on port `5001`. + The API is identical, with the exception of a required `Authorization` header. + diff --git a/openfn/docker/compose.sh b/openfn/docker/compose.sh new file mode 100755 index 00000000..3a48c3a8 --- /dev/null +++ b/openfn/docker/compose.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +composeFilePath=$(dirname $(readlink -f $0)) + +# Create a volume and copy the project configuration in before starting the stack. +docker volume create microservice_him_hapi +docker container create --name dummy -v microservice_him_hapi:/mnt tianon/true +docker cp $composeFilePath/config/project.yaml dummy:/mnt/ +docker rm dummy + +if [ "$1" == "init" ]; then + docker-compose -p openfn \ + -f "$composeFilePath"/docker-compose.yml \ + -f "$composeFilePath"/docker-compose.dev.yml \ + -f "$composeFilePath"/docker-compose.config.yml \ + up -d +elif [ "$1" == "up" ]; then + docker-compose -p openfn \ + -f "$composeFilePath"/docker-compose.yml \ + -f "$composeFilePath"/docker-compose.dev.yml \ + -f "$composeFilePath"/docker-compose.config.yml \ + up -d +elif [ "$1" == "down" ]; then + docker-compose -p openfn \ + -f "$composeFilePath"/docker-compose.yml \ + -f "$composeFilePath"/docker-compose.dev.yml \ + -f "$composeFilePath"/docker-compose.config.yml \ + stop +elif [ "$1" == "destroy" ]; then + docker-compose -p openfn \ + -f "$composeFilePath"/docker-compose.yml \ + -f "$composeFilePath"/docker-compose.dev.yml \ + -f "$composeFilePath"/docker-compose.config.yml \ + down +else + echo "Valid options are: init, up, down, or destroy" +fi diff --git a/openfn/docker/config/project.yaml b/openfn/docker/config/project.yaml new file mode 100644 index 00000000..7c9eecfd --- /dev/null +++ b/openfn/docker/config/project.yaml @@ -0,0 +1,167 @@ +jobs: + create-patient: + expression: > + post('Patient', { + body: dataValue('$'), + headers: { + 'Content-Type': 'application/json', + Authorization: 'Custom test', + 'If-None-Exist': state => + `identifier=${dataValue('identifier[0].value')(state)}`, + }, + }); + adaptor: '@openfn/language-http' + trigger: patient-message + credential: openhim + + create-organization: + expression: > + post('Organization', { + body: dataValue('$'), + headers: { + 'Content-Type': 'application/json', + Authorization: 'Custom test', + 'If-None-Exist': state => + `identifier=${dataValue('identifier[0].value')(state)}`, + }, + }); + adaptor: '@openfn/language-http' + trigger: organization-message + credential: openhim + + commcare-to-openhim: + expression: > + post("Patient", { + body: fields( + field("resourceType", "Patient"), + field("id", dataValue("$form.patient_data.patient_id")), + field("identifier", [ + { + use: "usual", + type: { + coding: [ + { + system: "http://terminology.hl7.org/CodeSystem/v2-0203", + code: "MR", + }, + ], + }, + system: "urn:oid:0.1.2.3.4.5.6.7", + value: dataValue("$form.patient_data.patient_id"), + }, + ]), + field("name", [ + { + use: "official", + family: dataValue("$form.patient_data.age.last_name"), + given: [dataValue("$form.patient_data.age.first_name")], + }, + ]), + field("gender", dataValue("$form.patient_data.age.gender")), + field("birthDate", dataValue("$form.patient_data.age.birthDate")) + ), + headers: { + "Content-Type": "application/json", + Authorization: "Custom test", + "If-None-Exist": (state) => + `identifier=${dataValue("$form.patient_data.patient_id")(state)}`, + }, + }); + + post("Encounter", { + body: fields( + field("resourceType", "Encounter"), + field("status", "finished"), + field( + "subject", + field("reference", (state) => `Patient/${dataValue("id")(state)}`) + ) + ), + headers: { + "Content-Type": "application/json", + Authorization: "Custom test", + }, + }); + adaptor: '@openfn/language-http' + trigger: commcare-message + credential: openhim + + kobo-to-openhim: + expression: > + post("Patient", { + body: fields( + field("resourceType", "Patient"), + field("id", dataValue("$body.Godata_ID")), + field("identifier", [ + { + use: "usual", + type: { + coding: [ + { + system: "http://terminology.hl7.org/CodeSystem/v2-0203", + code: "MR", + }, + ], + }, + system: "urn:oid:0.1.2.3.4.5.6.7", + value: dataValue("$body.Godata_ID"), + }, + ]), + field("name", [ + { + use: "official", + text: dataValue("$body.Patient_name"), + }, + ]), + field("gender", dataValue("$body.Sex")), + field("birthDate", (state) => { + const year = + Intl.DateTimeFormat("en", { year: "numeric" }).format() - + Number(dataValue("$body.Age_in_year")(state)); + return `${year}-01-01`; + }) + ), + headers: { + "Content-Type": "application/json", + Authorization: "Custom test", + "If-None-Exist": (state) => + `identifier=${dataValue("$body.Godata_ID")(state)}`, + }, + }); + + post("Encounter", { + body: fields( + field("resourceType", "Encounter"), + field("status", "finished"), + field( + "subject", + field( + "reference", + (state) => `Patient/${dataValue("id")(state)}` + ) + ) + ), + headers: { + "Content-Type": "application/json", + Authorization: "Custom test", + }, + }); + + adaptor: '@openfn/language-http' + trigger: kobo-message + credential: openhim + + +triggers: + patient-message: + criteria: '{"resourceType":"Patient"}' + organization-message: + criteria: '{"resourceType":"Organization"}' + kobo-message: + criteria: '{"form":"covid19-registration"}' + commcare-message: + criteria: '{"app_id":"aa"}' + +credentials: + openhim: + baseUrl: "http://openhim-core:5001/fhir/" \ No newline at end of file diff --git a/openfn/docker/docker-compose.config.yml b/openfn/docker/docker-compose.config.yml new file mode 100644 index 00000000..7c8d76c6 --- /dev/null +++ b/openfn/docker/docker-compose.config.yml @@ -0,0 +1,18 @@ +version: "3.3" + +services: + openfn: + environment: + PROJECT_DIR: /opt/app/project + # MIX_ENV: prod + volumes: + - microservice_him_hapi:/opt/app/project + +volumes: + microservice_him_hapi: + external: true + +networks: + default: + external: + name: instant_default \ No newline at end of file diff --git a/openfn/docker/docker-compose.dev.yml b/openfn/docker/docker-compose.dev.yml new file mode 100644 index 00000000..9a683b0f --- /dev/null +++ b/openfn/docker/docker-compose.dev.yml @@ -0,0 +1,6 @@ +version: '3.3' + +services: + openfn: + ports: + - '4001:4001' \ No newline at end of file diff --git a/openfn/docker/docker-compose.yml b/openfn/docker/docker-compose.yml new file mode 100644 index 00000000..2cc78864 --- /dev/null +++ b/openfn/docker/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.3' + +services: + openfn: + image: openfn/microservice:v0.3.3 + container_name: openfn_him_hapi + environment: + HOST_URL: localhost + PORT: 4001 + ENDPOINT_STYLE: async \ No newline at end of file diff --git a/openfn/fixtures/commcare_sample.json b/openfn/fixtures/commcare_sample.json new file mode 100644 index 00000000..254f9a39 --- /dev/null +++ b/openfn/fixtures/commcare_sample.json @@ -0,0 +1,43 @@ +{ + "app_id": "aa", + "archived": false, + "attachments": { + "form.xml": { + "content_type": "text/xml", + "length": 11528, + "url": "https://www.commcarehq.org/a/x.xml" + } + }, + "form": { + "#type": "data", + "@name": "Register New Patient", + "@uiVersion": "1", + "@version": "1", + "patient_data": { + "age": { + "age_at_registration_days": "181", + "gender": "female", + "birthDate": "2020-10-11", + "first_name": "test", + "last_name": "case" + }, + "locations": { + "city": "k", + "clinic_code": "test_clinic", + "country": "Kenya" + }, + "patient_id": "test_clinic-0009-01" + }, + "case": { + "@case_id": "xxx", + "@date_modified": "2021-01-21T07:08:19.431000Z", + "@user_id": "aaa", + "@xmlns": "http://commcarehq.org/case/transaction/v2", + "create": { + "case_name": "test case", + "case_type": "patient", + "owner_id": "aaa" + } + } +} +} \ No newline at end of file diff --git a/openfn/fixtures/koboCaseRegistration.json b/openfn/fixtures/koboCaseRegistration.json new file mode 100644 index 00000000..b00f890b --- /dev/null +++ b/openfn/fixtures/koboCaseRegistration.json @@ -0,0 +1,45 @@ +{ + "__query_params": {}, + "body": { + "Age_in_year": "32", + "Case_ID": "PVT6546163", + "Covid_19_suspected_criteria/HF_visited": "no", + "Covid_19_suspected_criteria/Specimen_Collected": "not_collected", + "Covid_19_suspected_criteria/Symptoms": "difficulty_breathing", + "Covid_19_suspected_criteria/Travel_history": "no", + "Covid_19_suspected_criteria/contact_with_case": "no", + "Covid_19_suspected_criteria/risk_factors": "asthma", + "Date": "2020-12-20", + "Godata_ID": "C19-930020123", + "HF_Name": "375", + "Patient_name": "Aleksa Krolls", + "Sample_Classification": "n_id", + "Sex": "female", + "UID": "375", + "__version__": "vkHKMmRAxMM5TbxYeJxRrZ", + "_attachments": [], + "_geolocation": [ null, null ], + "_id": 145623988, + "_notes": [], + "_status": "submitted_via_web", + "_submission_time": "2020-12-15T14:56:25", + "_submitted_by": null, + "_tags": [], + "_uuid": "5e6fbde5-27ce-4829-8615-546c40aaec61", + "_validation_status": {}, + "_version_": "vDV7SMYHxkcCmzQSGFVSUS", + "_version__001": "vyWA2rz4Lz2ee5hMH6xEax", + "_version__002": "vzMPKoBxnNGuXLgUkzcoE6", + "_version__003": "vGxbxgVz3DRChZXUJuAFjz", + "_xform_id_string": "afSehzJiWwktDGYucGJq2X", + "end": "2020-12-15T09:56:14.066-05:00", + "formhub/uuid": "4778c297728b4303807a8647038c5011", + "meta/instanceID": "uuid:5e6fbde5-27ce-4829-8615-546c40aaec61", + "patient_address/Nationality": "fdmn", + "patient_address/Upazilla": "teknaf", + "patient_address/teknaf_Camp": "camp_23", + "sample_type_001": "ari_ili", + "start": "2020-12-15T09:49:44.452-05:00" + }, + "form": "covid19-registration" +} diff --git a/openfn/fixtures/organization.json b/openfn/fixtures/organization.json new file mode 100644 index 00000000..7eef6626 --- /dev/null +++ b/openfn/fixtures/organization.json @@ -0,0 +1,27 @@ +{ + "resourceType": "Organization", + "id": "1", + "text": { + "status": "generated", + "div": "
\n \n

Gastroenterology @ Acme Hospital. ph: +1 555 234 3523, email: \n gastro@acme.org\n

\n \n
" + }, + "identifier": [ + { + "system": "http://www.acme.org.au/units", + "value": "Gastro" + } + ], + "name": "Gastroenterology", + "telecom": [ + { + "system": "phone", + "value": "+1 555 234 3523", + "use": "mobile" + }, + { + "system": "email", + "value": "gastro@acme.org", + "use": "work" + } + ] +} diff --git a/openfn/fixtures/patient.json b/openfn/fixtures/patient.json new file mode 100644 index 00000000..6045b9bf --- /dev/null +++ b/openfn/fixtures/patient.json @@ -0,0 +1,165 @@ +{ + "resourceType": "Patient", + "id": "example", + "text": { + "status": "generated", + "div": "
\n\t\t\t\n\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\t\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t
NamePeter James \n Chalmers ("Jim")\n
Address534 Erewhon, Pleasantville, Vic, 3999
ContactsHome: unknown. Work: (03) 5555 6473
IdMRN: 12345 (Acme Healthcare)
\n\t\t
" + }, + "identifier": [ + { + "use": "usual", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0203", + "code": "MR" + } + ] + }, + "system": "urn:oid:1.2.36.146.595.217.0.1", + "value": "12345", + "period": { + "start": "2001-05-06" + }, + "assigner": { + "display": "Acme Healthcare" + } + } + ], + "active": true, + "name": [ + { + "use": "official", + "family": "Chalmers", + "given": [ + "Peter", + "James" + ] + }, + { + "use": "usual", + "given": [ + "Jim" + ] + }, + { + "use": "maiden", + "family": "Windsor", + "given": [ + "Peter", + "James" + ], + "period": { + "end": "2002" + } + } + ], + "telecom": [ + { + "use": "home" + }, + { + "system": "phone", + "value": "(03) 5555 6473", + "use": "work", + "rank": 1 + }, + { + "system": "phone", + "value": "(03) 3410 5613", + "use": "mobile", + "rank": 2 + }, + { + "system": "phone", + "value": "(03) 5555 8834", + "use": "old", + "period": { + "end": "2014" + } + } + ], + "gender": "male", + "birthDate": "1974-12-25", + "_birthDate": { + "extension": [ + { + "url": "http://hl7.org/fhir/StructureDefinition/patient-birthTime", + "valueDateTime": "1974-12-25T14:35:45-05:00" + } + ] + }, + "deceasedBoolean": false, + "address": [ + { + "use": "home", + "type": "both", + "text": "534 Erewhon St PeasantVille, Rainbow, Vic 3999", + "line": [ + "534 Erewhon St" + ], + "city": "PleasantVille", + "district": "Rainbow", + "state": "Vic", + "postalCode": "3999", + "period": { + "start": "1974-12-25" + } + } + ], + "contact": [ + { + "relationship": [ + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/v2-0131", + "code": "N" + } + ] + } + ], + "name": { + "family": "du Marché", + "_family": { + "extension": [ + { + "url": "http://hl7.org/fhir/StructureDefinition/humanname-own-prefix", + "valueString": "VV" + } + ] + }, + "given": [ + "Bénédicte" + ] + }, + "telecom": [ + { + "system": "phone", + "value": "+33 (237) 998327" + } + ], + "address": { + "use": "home", + "type": "both", + "line": [ + "534 Erewhon St" + ], + "city": "PleasantVille", + "district": "Rainbow", + "state": "Vic", + "postalCode": "3999", + "period": { + "start": "1974-12-25" + } + }, + "gender": "female", + "period": { + "start": "2012" + } + } + ], + "managingOrganization": { + "reference": "Organization/1" + } +} \ No newline at end of file diff --git a/openfn/instant.json b/openfn/instant.json new file mode 100644 index 00000000..e2631bbc --- /dev/null +++ b/openfn/instant.json @@ -0,0 +1,8 @@ +{ + "id": "openfnHimHapi", + "name": "OpenFn Microservice", + "description": "", + "type": "infrastructure", + "version": "v0.0.0", + "dependencies": ["core"] +} \ No newline at end of file diff --git a/openfn/kubernetes/README.md b/openfn/kubernetes/README.md new file mode 100644 index 00000000..ff4d2578 --- /dev/null +++ b/openfn/kubernetes/README.md @@ -0,0 +1,10 @@ +# Kubernetes + + +To start use the following: + +```bash +./main/k8s.sh init +# to remove +./main/k8s.sh destroy +``` diff --git a/openfn/kubernetes/main/k8s.sh b/openfn/kubernetes/main/k8s.sh new file mode 100755 index 00000000..de2f857c --- /dev/null +++ b/openfn/kubernetes/main/k8s.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +currentDir=$(dirname $(readlink -f $0)) + +if [ "$1" == "init" ]; then + + kubectl apply -f $currentDir/manifest.yaml --wait + + # Copy the example project.yaml to the created PVC + IMAGE=openfn/microservice:v0.3.3 + VOL_MOUNTS='[{"mountPath": "/opt/app/project", "name": "openfn-microservice-data"}]' + VOLS='[{"name": "openfn-microservice-data", "persistentVolumeClaim": {"claimName": "openfn-microservice-data"}}]' + SUFFIX=$(date +%s | shasum | base64 | fold -w 10 | head -1 | tr '[:upper:]' '[:lower:]') + + cd $currentDir/../../docker/config + + tar cf - project.yaml | kubectl run -n openfn \ + -i --rm --restart=Never \ + --image=${IMAGE} \ + openfn-microservice-${SUFFIX} \ + --overrides "{ + \"spec\": { + \"hostNetwork\": true, + \"containers\":[ + { + \"args\": [\"sh\", \"-c\", \"tar xvf - -C /opt/app/project\"], + \"stdin\": true, + \"name\": \"openfn-microservice\", + \"image\": \"${IMAGE}\", + \"volumeMounts\": ${VOL_MOUNTS} + } + ], + \"volumes\": ${VOLS} + } }" + + + +elif [ "$1" == "up" ]; then + + kubectl apply -f $currentDir/manifest.yaml + +elif [ "$1" == "down" ]; then + + kubectl delete -f $currentDir/manifest.yaml + +elif [ "$1" == "destroy" ]; then + + kubectl delete -f $currentDir/manifest.yaml + +else + echo "Valid options are: init, up, down, or destroy" +fi diff --git a/openfn/kubernetes/main/manifest.yaml b/openfn/kubernetes/main/manifest.yaml new file mode 100644 index 00000000..a7cbb420 --- /dev/null +++ b/openfn/kubernetes/main/manifest.yaml @@ -0,0 +1,121 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: openfn +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: openfn-microservice-data + namespace: openfn + labels: + app: openfn-microservice + needed_early: t +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +status: {} + + +# kubectl -n openfn get pods -o name +# tar cf - ../../docker/example/volume/* | kubectl exec -i -n openfn -- tar xf - -C /tmp/bar +# --overrides='{ "apiVersion": "v1", "spec": { "volumes": { "name": "openfn-microservice-data", "persistentVolumeClaim": "claimName": "openfn-microservice-data" } }' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: microservice-deployment + namespace: openfn +spec: + replicas: 1 + selector: + matchLabels: + app: openfn-microservice + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: openfn-microservice + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openfn-microservice + topologyKey: 'kubernetes.io/hostname' + containers: + - image: openfn/microservice:v0.3.3 + name: openfn-microservice + resources: + requests: + cpu: '250m' + memory: '256Mi' + limits: + memory: '768Mi' + ports: + - containerPort: 4001 + protocol: TCP + env: + - name: HOST_URL + value: "localhost" + - name: PORT + value: "4001" + - name: ENDPOINT_STYLE + value: "async" + - name: PROJECT_DIR + value: "/opt/app/project" + # command: ['./prod/rel/open_fn/bin/open_fn'] + # args: ['start'] + volumeMounts: + - mountPath: /opt/app/project + name: openfn-microservice-data + terminationGracePeriodSeconds: 60 + volumes: + - name: openfn-microservice-data + persistentVolumeClaim: + claimName: openfn-microservice-data +--- +apiVersion: v1 +kind: Service +metadata: + name: openfn-service + namespace: openfn + annotations: + cloud.google.com/app-protocols: '{"service-https-port":"HTTPS"}' + cloud.google.com/neg: '{"ingress": true}' # Creates a NEG after an Ingress is created +spec: + ports: + - name: service-https + port: 443 + protocol: TCP + targetPort: 4001 + selector: + app: openfn-microservice + type: NodePort +--- +# apiVersion: networking.k8s.io/v1beta1 +# kind: Ingress +# metadata: +# name: openfn-ingress +# annotations: +# kubernetes.io/ingress.global-static-ip-name: staging-managed-ip +# ingress.gcp.kubernetes.io/pre-shared-cert: 'staging' +# kubernetes.io/ingress.allow-http: 'false' +# spec: +# backend: +# serviceName: openfn-service +# servicePort: service-https +# ---