diff --git a/APISyncExternalModule.php b/APISyncExternalModule.php index 9b7574c..8b8b149 100644 --- a/APISyncExternalModule.php +++ b/APISyncExternalModule.php @@ -1,4 +1,5 @@ "; + public const UPDATE = 'update'; + public const DELETE = 'delete'; + public const TRANSLATION_TABLE_CELL = ""; - const EXPORT_CANCELLED_MESSAGE = 'Export cancelled.'; + public const EXPORT_CANCELLED_MESSAGE = 'Export cancelled.'; - const DATA_VALUES_MAX_LENGTH = (2^16) - 1; - const MAX_LOG_QUERY_PERIOD = 7; + public const DATA_VALUES_MAX_LENGTH = (2 ^ 16) - 1; + public const MAX_LOG_QUERY_PERIOD = 7; private $settingPrefix; private $cachedSettings; private $allFieldNames = []; - function cron($cronInfo){ + public function cron($cronInfo) { /** * We know 2g is required to prevent exports from crashing on the SAMMC project. * This was set to 4g somewhat arbitrarily. Hopefully that will cover many potential future use cases. @@ -39,19 +41,17 @@ function cron($cronInfo){ $cronName = $cronInfo['cron_name']; - foreach($this->framework->getProjectsWithModuleEnabled() as $localProjectId){ + foreach ($this->framework->getProjectsWithModuleEnabled() as $localProjectId) { // This automatically associates all log statements with this project. $_GET['pid'] = $localProjectId; $this->settingPrefix = substr($cronName, 0, -1); // remove the 's' - if($cronName === 'exports'){ + if ($cronName === 'exports') { $this->handleExports(); - } - else if($cronName === 'imports'){ + } elseif ($cronName === 'imports') { $this->handleImports(); - } - else{ + } else { throw new Exception("Unsupported cron name: $cronName"); } } @@ -62,19 +62,19 @@ function cron($cronInfo){ return "The \"{$cronInfo['cron_description']}\" cron job completed successfully."; } - private function areAnyEmpty($array){ + private function areAnyEmpty($array) { $filteredArray = array_filter($array); return count($array) != count($filteredArray); } - private function handleExports(){ + private function handleExports() { /** * This amount of time was chosen semi-arbitrarily. * A time limit less than the cron max run time of 24 hours is important. * Ideally the cron wouldn't only run for a few minutes max, but VUMC Project 111585 has batches that last a couple of hours. * We might as well set this high, at least until we can justify including sub-batches in the export progress. */ - $twentyHours = 60*60*20; + $twentyHours = 60 * 60 * 20; set_time_limit($twentyHours); // In case the previous export was cancelled, or the button pushed when an export wasn't active. @@ -84,32 +84,30 @@ private function handleExports(){ $firstServer = $servers[0] ?? null; $firstProject = $firstServer['export-projects'][0] ?? null; - if($this->areAnyEmpty([ + if ($this->areAnyEmpty([ $firstServer['export-redcap-url'] ?? null, $firstProject['export-api-key'] ?? null, $firstProject['export-project-name'] ?? null - ])){ + ])) { return; } - try{ + try { $this->export($servers); - } - catch(\Exception $e){ - if($e->getMessage() === self::EXPORT_CANCELLED_MESSAGE){ + } catch (\Exception $e) { + if ($e->getMessage() === self::EXPORT_CANCELLED_MESSAGE) { // No reason to report this exception since this is an expected use case. - } - else{ + } else { $this->handleException($e); } } } - function getIdentifiers(){ + public function getIdentifiers() { $fields = REDCap::getDataDictionary($this->getProjectId(), 'array'); $fieldNames = []; - foreach($fields as $fieldName=>$details){ - if($details['identifier'] === 'y'){ + foreach ($fields as $fieldName => $details) { + if ($details['identifier'] === 'y') { $fieldNames[] = $fieldName; } } @@ -118,25 +116,25 @@ function getIdentifiers(){ } // This method can be removed once it makes it into a REDCap version. - private function getLogTable(){ + private function getLogTable() { $result = $this->query('select log_event_table from redcap_projects where project_id = ?', $this->getProjectId()); $table = $result->fetch_assoc()['log_event_table']; $prefix = 'redcap_log_event'; $number = explode($prefix, $table)[1]; $verifiedTable = $prefix; - if(!empty($number)){ + if (!empty($number)) { $verifiedTable .= (int)$number; } - if($table !== $verifiedTable){ + if ($table !== $verifiedTable) { throw new \Exception('An error occurred while generating the verified log table name.'); } return $verifiedTable; } - private function getLogIndexHint($logTable){ + private function getLogIndexHint($logTable) { $result = $this->query("show variables like 'version'", []); $versionParts = explode('.', $result->fetch_assoc()['Value']); @@ -148,7 +146,7 @@ private function getLogIndexHint($logTable){ $row = $result->fetch_assoc(); $indexName = db_escape($row['Key_name']) ?? null; - if($indexName === null){ + if ($indexName === null) { return ''; } @@ -159,7 +157,7 @@ private function getLogIndexHint($logTable){ return " use index ($indexName) "; } - private function getLastExportedLogId(){ + private function getLastExportedLogId() { $logTable = $this->getLogTable(); $result = $this->query( @@ -174,7 +172,7 @@ private function getLastExportedLogId(){ ", [ $this->getProjectId(), - (new DateTime)->modify('-' . self::MAX_LOG_QUERY_PERIOD . ' days')->format('YmdHis') + (new DateTime())->modify('-' . self::MAX_LOG_QUERY_PERIOD . ' days')->format('YmdHis') ] ); @@ -188,7 +186,7 @@ private function getLastExportedLogId(){ return max($weekOldId, $lastExportedId); } - private function getLatestLogId(){ + private function getLatestLogId() { $result = $this->query(" select log_event_id from " . $this->getLogTable() . " @@ -200,9 +198,9 @@ private function getLatestLogId(){ return $row['log_event_id']; } - private function getAllFieldNames(){ + private function getAllFieldNames() { $pid = $this->getProjectId(); - if(!isset($this->allFieldNames[$pid])){ + if (!isset($this->allFieldNames[$pid])) { $dictionary = REDCap::getDataDictionary($pid, 'array'); $this->allFieldNames[$pid] = array_keys($dictionary); } @@ -210,10 +208,10 @@ private function getAllFieldNames(){ return $this->allFieldNames[$pid]; } - private function addBatchesSinceLastExport($specificFieldsBatchBuilder, $allFieldsBatchBuilder, $recordIds){ + private function addBatchesSinceLastExport($specificFieldsBatchBuilder, $allFieldsBatchBuilder, $recordIds) { $recordIds = array_flip($recordIds); - $lastExportedLogId = $this->getLastExportedLogId(); + $lastExportedLogId = $this->getLastExportedLogId(); $result = $this->query(" select log_event_id, pk, event, data_values from " . $this->getLogTable() . " @@ -228,17 +226,16 @@ private function addBatchesSinceLastExport($specificFieldsBatchBuilder, $allFiel $lastExportedLogId ]); - while($row = $result->fetch_assoc()){ + while ($row = $result->fetch_assoc()) { $recordId = $row['pk']; - if(!isset($recordIds[$recordId])){ + if (!isset($recordIds[$recordId])) { continue; } $fields = $this->getChangedFieldNamesForLogRow($row['data_values'], $this->getAllFieldNames()); - if(empty($fields)){ + if (empty($fields)) { $batchBuilder = $allFieldsBatchBuilder; - } - else{ + } else { $batchBuilder = $specificFieldsBatchBuilder; } @@ -246,8 +243,8 @@ private function addBatchesSinceLastExport($specificFieldsBatchBuilder, $allFiel } } - function getChangedFieldNamesForLogRow($dataValues, $allFieldNames){ - if(strlen($dataValues) === self::DATA_VALUES_MAX_LENGTH){ + public function getChangedFieldNamesForLogRow($dataValues, $allFieldNames) { + if (strlen($dataValues) === self::DATA_VALUES_MAX_LENGTH) { // The data_values column was maxed out, so all changes were not included. // Return an empty array, which will cause all fields to be synced. return []; @@ -262,7 +259,7 @@ function getChangedFieldNamesForLogRow($dataValues, $allFieldNames){ return array_intersect($allFieldNames, $matches[1]); } - private function export($servers){ + private function export($servers) { /** * WHEN MODIFYING EXPORT BEHAVIOR * If further export permissions tweaks are made, Paul recommended selecting @@ -275,8 +272,8 @@ private function export($servers){ $recordIdFieldName = $this->getRecordIdField(); $exportProgress = $this->getProjectSetting('export-progress'); - if($exportProgress === null){ - if(!$this->isTimeToRunExports()){ + if ($exportProgress === null) { + if (!$this->isTimeToRunExports()) { return; } @@ -292,7 +289,8 @@ private function export($servers){ $allFieldsBatchBuilder = new BatchBuilder($this->getExportBatchSize()); $latestLogId = $this->getLatestLogId(); - $allRecordsIds = array_column(REDCap::getData($this->getProjectId(), + $allRecordsIds = array_column(REDCap::getData( + $this->getProjectId(), 'json-array', null, $recordIdFieldName, @@ -312,19 +310,18 @@ private function export($servers){ ), $recordIdFieldName); $exportAllRecords = $this->getProjectSetting('export-all-records') === true; - if($exportAllRecords){ + if ($exportAllRecords) { $this->removeProjectSetting('export-all-records'); - foreach($allRecordsIds as $recordId){ + foreach ($allRecordsIds as $recordId) { // An empty fields array will cause all fields to be pulled. $allFieldsBatchBuilder->addEvent($latestLogId, $recordId, 'UPDATE', []); } - } - else{ + } else { $this->addBatchesSinceLastExport($specificFieldsBatchBuilder, $allFieldsBatchBuilder, $allRecordsIds); } $batches = $this->mergeBatches($specificFieldsBatchBuilder, $allFieldsBatchBuilder); - if(empty($batches)){ + if (empty($batches)) { /** * No recent changes exist to sync. * Update the last exported log ID to whatever the latest ID across all projects is @@ -335,8 +332,7 @@ private function export($servers){ $this->setProjectSetting('last-exported-log-id', $latestLogId); return; } - } - else{ + } else { // Continue an export in progress $this->removeProjectSetting('export-progress'); @@ -347,16 +343,16 @@ private function export($servers){ $maxSubBatchSize = $this->getExportSubBatchSize(); $excludedFieldNames = []; - if($this->getProjectSetting('export-exclude-identifiers') === true){ + if ($this->getProjectSetting('export-exclude-identifiers') === true) { $excludedFieldNames = $this->getIdentifiers(); } - for($i=0; $iisCronRunningTooLong()){ + for ($i = 0; $i < count($batches); $i++) { + if ($this->isCronRunningTooLong()) { $remainingBatches = array_splice($batches, $i); $this->setProjectSetting('export-progress', serialize([ - $startingBatchIndex+$i, + $startingBatchIndex + $i, $remainingBatches ])); @@ -370,7 +366,7 @@ private function export($servers){ $recordIds = $batch->getRecordIds(); $fieldsByRecord = $batch->getFieldsByRecord(); - $batchText = "batch " . ($startingBatchIndex+$i+1) . " of " . ($startingBatchIndex+count($batches)); + $batchText = "batch " . ($startingBatchIndex + $i + 1) . " of " . ($startingBatchIndex + count($batches)); $this->log("Preparing to export {$type}s for $batchText", [ 'details' => json_encode([ @@ -379,10 +375,10 @@ private function export($servers){ ], JSON_PRETTY_PRINT) ]); - if($type === self::UPDATE){ + if ($type === self::UPDATE) { $fields = $batch->getFields(); - if(!empty($fields)){ + if (!empty($fields)) { $fields[] = $recordIdFieldName; } @@ -406,28 +402,28 @@ private function export($servers){ $subBatchData = []; $subBatchSize = 0; $subBatchNumber = 1; - for($rowIndex=0; $rowIndexgetFields() as $field){ - if( + foreach ($batch->getFields() as $field) { + if ( $field !== $recordIdFieldName && !isset($fieldsByRecord[$row[$recordIdFieldName]][$field]) - ){ + ) { // This field didn't change for this record, so don't include it in the export. unset($row[$field]); } } - foreach($excludedFieldNames as $excludedFieldName){ + foreach ($excludedFieldNames as $excludedFieldName) { unset($row[$excludedFieldName]); } $rowSize = strlen(json_encode($row)); $spaceLeftInSubBatch = $maxSubBatchSize - $subBatchSize; - if($rowSize > $spaceLeftInSubBatch){ - if($subBatchSize === 0){ + if ($rowSize > $spaceLeftInSubBatch) { + if ($subBatchSize === 0) { $this->log("The export failed because the sub-batch size setting is not large enough to handle the data in the details of this log message.", [ 'details' => json_encode($row, JSON_PRETTY_PRINT) ]); @@ -444,16 +440,14 @@ private function export($servers){ $subBatchData[] = $row; $subBatchSize += $rowSize; - $isLastRow = $rowIndex === count($data)-1; - if($isLastRow){ + $isLastRow = $rowIndex === count($data) - 1; + if ($isLastRow) { $this->exportSubBatch($servers, $type, $subBatchData, $subBatchNumber, $subBatchSize); } } - } - else if($type === self::DELETE){ + } elseif ($type === self::DELETE) { $this->exportSubBatch($servers, $type, $recordIds, 1, 0); - } - else{ + } else { throw new Exception("Unsupported export type: $type"); } @@ -462,27 +456,27 @@ private function export($servers){ } } - function mergeBatches($builder1, $builder2){ + public function mergeBatches($builder1, $builder2) { $batches = array_merge($builder1->getBatches(), $builder2->getBatches()); - usort($batches, function($a, $b){ + usort($batches, function ($a, $b) { return $a->getLastLogId() - $b->getLastLogId(); }); return $batches; } - private function isCronRunningTooLong(){ + private function isCronRunningTooLong() { return time() >= $_SERVER['REQUEST_TIME_FLOAT'] + 55; } - function logDetails($message, $details){ + public function logDetails($message, $details) { $parts = str_split($details, 65535); $params = [ 'details' => array_shift($parts) ]; $n = 2; - foreach($parts as $part){ + foreach ($parts as $part) { $params["details$n"] = $part; $n++; } @@ -490,14 +484,14 @@ function logDetails($message, $details){ return $this->log($message, $params); } - function getProjects($server){ + public function getProjects($server) { $fieldListSettingName = $this->getPrefixedSettingName("field-list"); $projects = $server[$this->getPrefixedSettingName("projects")]; $incorrectlyLocatedFieldLists = $projects[$fieldListSettingName] ?? null; - if($incorrectlyLocatedFieldLists !== null){ + if ($incorrectlyLocatedFieldLists !== null) { // Recover from a getSubSettings() bug which was fixed in framework version 9. - foreach ($projects as $i=>&$project) { + foreach ($projects as $i => &$project) { $project[$fieldListSettingName] = $incorrectlyLocatedFieldLists[$i] ?? null; } @@ -507,8 +501,8 @@ function getProjects($server){ return $projects; } - private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBatchSize){ - $subBatchSize = round($subBatchSize/1024/1024, 1) . ' MB'; + private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBatchSize) { + $subBatchSize = round($subBatchSize / 1024 / 1024, 1) . ' MB'; $recordIdFieldName = $this->getRecordIdField(); foreach ($servers as $server) { @@ -516,7 +510,7 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat $logUrl = $this->formatURLForLogs($url); foreach ($this->getProjects($server) as $project) { - $getProjectExportMessage = function($action) use ($type, $subBatchNumber, $logUrl, $project, $subBatchSize){ + $getProjectExportMessage = function ($action) use ($type, $subBatchNumber, $logUrl, $project, $subBatchSize) { return "
$action exporting $type sub-batch $subBatchNumber ($subBatchSize) to the following project at $logUrl:
" . $project['export-project-name'] . "
@@ -530,12 +524,11 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat $args = ['content' => 'record']; $prepped_data = []; - if($type === self::UPDATE){ + if ($type === self::UPDATE) { $prepped_data = $this->prepareData($project, $data, $recordIdFieldName); $args['overwriteBehavior'] = 'overwrite'; $args['data'] = json_encode($prepped_data, JSON_PRETTY_PRINT); - } - else if($type === self::DELETE){ + } elseif ($type === self::DELETE) { $recordIdPrefix = $project['export-record-id-prefix']; if ($recordIdPrefix) { foreach ($data as &$rId) { @@ -548,7 +541,7 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat } $results = $this->apiRequest($url, $apiKey, $args); - if (($type === self::UPDATE) && ($project['export-files'] ?? FALSE)) { + if (($type === self::UPDATE) && ($project['export-files'] ?? false)) { # import is from the perspective of the remote server $recordIds = []; foreach ($data as $row) { @@ -564,7 +557,7 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat ['details' => json_encode($results, JSON_PRETTY_PRINT)] ); - if($this->isExportCancelled()){ + if ($this->isExportCancelled()) { $this->log(self::EXPORT_CANCELLED_MESSAGE); throw new \Exception(self::EXPORT_CANCELLED_MESSAGE); } @@ -572,17 +565,17 @@ private function exportSubBatch($servers, $type, $data, $subBatchNumber, $subBat } } - function isExportCancelled(){ + public function isExportCancelled() { return $this->getProjectSetting('export-cancelled') === true; } - function setExportCancelled($value){ + public function setExportCancelled($value) { return $this->setProjectSetting('export-cancelled', $value); } - private function getExportBatchSize(){ + private function getExportBatchSize() { $size = (int) $this->getProjectSetting('export-batch-size'); - if(!$size){ + if (!$size) { // A size of 100 caused our 4g memory limit to be reached on VUMC project 111585. $size = 50; } @@ -590,9 +583,9 @@ private function getExportBatchSize(){ return $size; } - private function getExportSubBatchSize(){ + private function getExportSubBatchSize() { $size = $this->getProjectSetting('export-sub-batch-size'); - if($size === null){ + if ($size === null) { /** * A 7MB limit was added semi-arbitrarily. We know requests greater than 16MB were truncated * and returning an empty error message when OSHU was attempting to push to Vanderbilt. @@ -602,18 +595,18 @@ private function getExportSubBatchSize(){ } // Return the size in bytes - return $size*1024*1024; + return $size * 1024 * 1024; } - function clearExportQueue(){ + public function clearExportQueue() { $this->setProjectSetting('last-exported-log-id', $this->getLatestLogId()); } - private function getImportServers(){ + private function getImportServers() { $servers = $this->framework->getSubSettings('servers'); $importTimes = $this->getProjectSetting('last-import-time'); - for($i=0; $i