From 4e31920d0851817f35ad8e2a2a1ac5778ed005b4 Mon Sep 17 00:00:00 2001 From: Daniel Neugebauer Date: Wed, 23 Sep 2015 15:55:18 +0200 Subject: [PATCH 1/2] associate DAM and FAL records via DAM instead of FAL Reason is issue #62 - DAM may have multiple records for the same file. Since we correlate DAM and FAL by file path, we need to associate at least one DAM record to each migrated FAL record, thus the old way of using one field sys_file._migrateddamuid was incomplete as it only allowed associating at most one DAM record to each FAL record. Note that we may have different metadata for the same file as a result. Before these changes, only the metadata of the last file having been migrated was used. Now, all metadata will be processed which may lead to errors. This commit is only intended to record the change in columns and does not handle that issue yet. --- .../Controller/DamMigrationCommandController.php | 8 ++++---- Classes/Helper/DatabaseHelper.php | 6 +++--- .../Service/MigrateCategoryRelationsService.php | 4 ++-- Classes/Service/MigrateLinksService.php | 6 +++--- Classes/Service/MigrateMetadataService.php | 4 ++-- Classes/Service/MigrateRelationsService.php | 10 ++++++---- Classes/Service/MigrateRteMediaTagService.php | 6 +++--- Classes/Service/MigrateService.php | 14 +++++++------- Documentation/CommandReference/Index.rst | 2 +- Documentation/UserManual/Index.rst | 6 +++--- ext_tables.sql | 10 ++-------- 11 files changed, 36 insertions(+), 40 deletions(-) diff --git a/Classes/Controller/DamMigrationCommandController.php b/Classes/Controller/DamMigrationCommandController.php index 60947cb..8e801fb 100644 --- a/Classes/Controller/DamMigrationCommandController.php +++ b/Classes/Controller/DamMigrationCommandController.php @@ -52,7 +52,7 @@ public function migrateDamRecordsToStorageCommand($storageUid = 1, $recordLimit /** * Migrates all DAM records to FAL. - * A database field "_migrateddamuid" connects each FAL record to the original DAM record. + * A database field "_migratedfaluid" connects each original DAM record to the matching FAL record. * * @param int $storageUid The UID of the storage (default: 1 Do not modify if you are unsure.) * @param int $recordLimit The amount of records to process in a single run. You can set this value if you have memory constraints. @@ -373,9 +373,9 @@ public function migrateDamFrontendPluginsCommand() { } else { $newFalRecords = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( - 'uid, _migrateddamuid', - 'sys_file', - '_migrateddamuid IN (' . $plugin['tx_damdownloadlist_records'] . ')' + '_migratedfaluid AS uid', + 'tx_dam', + 'tx_dam.uid IN (' . $plugin['tx_damdownloadlist_records'] . ') AND _migratedfaluid > 0' ); // update ctype for dam_frontend_pi2 plugin diff --git a/Classes/Helper/DatabaseHelper.php b/Classes/Helper/DatabaseHelper.php index 3019770..8d95553 100644 --- a/Classes/Helper/DatabaseHelper.php +++ b/Classes/Helper/DatabaseHelper.php @@ -52,9 +52,9 @@ public static function getInstance() { */ public function getAllMigratedFalRecords() { return $GLOBALS['TYPO3_DB']->exec_SELECTgetRows( - 'uid, _migrateddamuid AS damuid', - 'sys_file', - '_migrateddamuid>0', + 'sys_file.uid AS uid, tx_dam.uid AS damuid', + 'tx_dam JOIN sys_file ON sys_file._migratedfaluid = tx_dam.uid', + '', // where '', // group by '', // order by '', // limit diff --git a/Classes/Service/MigrateCategoryRelationsService.php b/Classes/Service/MigrateCategoryRelationsService.php index b1c0ad9..88f5a59 100644 --- a/Classes/Service/MigrateCategoryRelationsService.php +++ b/Classes/Service/MigrateCategoryRelationsService.php @@ -100,8 +100,8 @@ public function execute() { protected function getCategoryRelationsWhereSysCategoryExists() { $rows = $this->database->exec_SELECTgetRows( 'MM.*, SM.uid as sys_metadata_uid, SC.uid as sys_category_uid', - 'tx_dam_mm_cat MM, sys_file SF, sys_category SC, sys_file_metadata SM', - 'SC._migrateddamcatuid = MM.uid_foreign AND SF._migrateddamuid = MM.uid_local AND SM.file = SF.uid' + 'tx_dam_mm_cat MM, tx_dam D, sys_category SC, sys_file_metadata SM', + 'SC._migrateddamcatuid = MM.uid_foreign AND D.uid = MM.uid_local AND SM.file = D._migratedfaluid' ); if ($rows === NULL) { throw new \Exception('SQL-Error in getCategoryRelationsWhereSysCategoryExists()', 1382968725); diff --git a/Classes/Service/MigrateLinksService.php b/Classes/Service/MigrateLinksService.php index f48d0b0..8f470b4 100644 --- a/Classes/Service/MigrateLinksService.php +++ b/Classes/Service/MigrateLinksService.php @@ -188,9 +188,9 @@ protected function findMigratedFileUIDForDAMRecord($mediaUID) { // query database $res = $this->database->exec_SELECTgetSingleRow( - 'uid', - 'sys_file', - '`_migrateddamuid` = ' . $this->database->fullQuoteStr((int)$mediaUID, 'sys_file', false) + '_migratedfaluid AS uid', + 'tx_dam', + 'tx_dam.uid = ' . $this->database->fullQuoteStr((int)$mediaUID, 'tx_dam', false) ); // use negative value if not found diff --git a/Classes/Service/MigrateMetadataService.php b/Classes/Service/MigrateMetadataService.php index a75f73e..067f4fa 100644 --- a/Classes/Service/MigrateMetadataService.php +++ b/Classes/Service/MigrateMetadataService.php @@ -163,9 +163,9 @@ public function execute() { */ protected function execSelectMigratedSysFilesQuery() { return $this->database->exec_SELECTquery( - 'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, f._migrateddamuid AS dam_uid, d.*', + 'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, d.uid AS dam_uid, d.*', 'sys_file f, sys_file_metadata m, tx_dam d', - 'm.file=f.uid AND f._migrateddamuid=d.uid AND f._migrateddamuid > 0' + 'm.file=f.uid AND d._migratedfaluid=f.uid' ); } diff --git a/Classes/Service/MigrateRelationsService.php b/Classes/Service/MigrateRelationsService.php index cc8078e..e2fc654 100644 --- a/Classes/Service/MigrateRelationsService.php +++ b/Classes/Service/MigrateRelationsService.php @@ -315,9 +315,9 @@ protected function getPidOfForeignRecord(array $damRelation) { } /** - * After a migration of tx_dam -> sys_file the col _migrateddamuid is - * filled with dam uid Now we can search in dam relations for dam records - * which have already been migrated to sys_file + * After a migration of tx_dam -> sys_file the col _migratedfaluid is + * filled with FAL uid. Now we can search in DAM relations for DAM records + * which have already been migrated to sys_file. * * @return \mysqli_result */ @@ -339,8 +339,10 @@ protected function execSelectDamReferencesWhereSysFileExists() { return $this->database->exec_SELECTquery( $selectFields, 'tx_dam_mm_ref + JOIN tx_dam ON + tx_dam.uid = tx_dam_mm_ref.uid_local JOIN sys_file ON - sys_file._migrateddamuid = tx_dam_mm_ref.uid_local + sys_file.uid = tx_dam._migratedfaluid JOIN sys_file_metadata ON sys_file.uid = sys_file_metadata.file ', diff --git a/Classes/Service/MigrateRteMediaTagService.php b/Classes/Service/MigrateRteMediaTagService.php index 1af8b58..46a275e 100644 --- a/Classes/Service/MigrateRteMediaTagService.php +++ b/Classes/Service/MigrateRteMediaTagService.php @@ -59,9 +59,9 @@ public function execute($table, $field) { /** @var PreparedStatement $getSysFileUidStatement */ $getSysFileUidStatement = $this->database->prepare_SELECTquery( - 'uid', - 'sys_file', - '_migrateddamuid = :migrateddamuid' + '_migratedfaluid AS uid', + 'tx_dam', + 'tx_dam.uid = :migrateddamuid AND _migratedfaluid > 0' ); foreach ($records as $rec) { $originalContent = $rec[$field]; diff --git a/Classes/Service/MigrateService.php b/Classes/Service/MigrateService.php index 8d193c9..5e1aebc 100644 --- a/Classes/Service/MigrateService.php +++ b/Classes/Service/MigrateService.php @@ -30,7 +30,7 @@ /** * Service to Migrate Records * Finds all DAM records that have not been migrated yet - * and adds a DB field "_migrateddamuid" to each FAL record + * and adds a DB field "_migratedfaluid" to each DAM record * to connect the DAM and FAL DB records * * currently it only works for files within the fileadmin @@ -165,7 +165,7 @@ protected function isValidDirectory(array $damRecord) { public function countNotMigratedDamRecordsQuery() { $row = $this->database->exec_SELECTgetSingleRow( 'COUNT(*) as total', - 'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)', + 'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)', 'sys_file.uid IS NULL AND tx_dam.deleted = 0 AND tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND @@ -182,7 +182,7 @@ public function countNotMigratedDamRecordsQuery() { protected function execSelectNotMigratedDamRecordsQuery() { return $this->database->exec_SELECTquery( 'tx_dam.*', - 'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)', + 'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)', 'sys_file.uid IS NULL AND tx_dam.deleted = 0 AND tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND @@ -228,11 +228,11 @@ protected function migrateFileFromDamToFal(array $damRecord, \TYPO3\CMS\Core\Res $this->createArrayForUpdateInsertSysFileRecord($damRecord) ); - // add the migrated uid of the DAM record to the FAL record + // add the uid of the FAL record to the original DAM record $this->database->exec_UPDATEquery( - 'sys_file', - 'uid = ' . $fileObject->getUid(), - array('_migrateddamuid' => $damRecord['uid']) + 'tx_dam', + 'uid = ' . $damRecord['uid'], + array('_migratedfaluid' => $fileObject->getUid()) ); } } diff --git a/Documentation/CommandReference/Index.rst b/Documentation/CommandReference/Index.rst index 82362eb..517ddb6 100644 --- a/Documentation/CommandReference/Index.rst +++ b/Documentation/CommandReference/Index.rst @@ -147,7 +147,7 @@ dam_falmigration:dammigration:migratedamrecords **Migrates all DAM records to FAL.** -A database field "_migrateddamuid" connects each FAL record to the original DAM record. +A database field "_migratedfaluid" connects each original DAM record to the matching FAL record. diff --git a/Documentation/UserManual/Index.rst b/Documentation/UserManual/Index.rst index 3364a27..0037f01 100644 --- a/Documentation/UserManual/Index.rst +++ b/Documentation/UserManual/Index.rst @@ -34,9 +34,9 @@ The available migration tasks can be found under the *extbase* cliKey: "_migrated/dam" is created and files are copied and indexed. dammigration:migratedamrecords Migrates all DAM records to FAL. A - DB field "_migrateddamuid" connects - each FAL record to the original DAM - record. + DB field "_migratedfaluid" connects + each original DAM record to the + matching FAL record. dammigration:migratedammetadata Migrates DAM metadata to FAL metadata. Searches for all migrated sys_file records that do not have any diff --git a/ext_tables.sql b/ext_tables.sql index c3796a3..cf88327 100644 --- a/ext_tables.sql +++ b/ext_tables.sql @@ -1,11 +1,3 @@ -# -# Table structure for table 'sys_file' -# -CREATE TABLE sys_file ( - _migrateddamuid int(11) unsigned DEFAULT '0' NOT NULL, - KEY migratedRecords (_migrateddamuid) -); - # # Table structure for table 'sys_file_collection' # @@ -26,6 +18,8 @@ CREATE TABLE sys_category ( # CREATE TABLE tx_dam ( _missingfile tinyint(4) unsigned DEFAULT '0' NOT NULL, + _migratedfaluid int(11) unsigned DEFAULT '0' NOT NULL, + KEY migratedRecords (_migratedfaluid), KEY deletedRecords (uid, deleted), KEY missingFiles (_missingfile) ); From 841a2011f4142b1e39b420388692adb2abb9ed1a Mon Sep 17 00:00:00 2001 From: Daniel Neugebauer Date: Wed, 23 Sep 2015 17:20:30 +0200 Subject: [PATCH 2/2] warn about duplicate DAM metadata for same file on metadata migration If multiple metadata for same files is encountered, we now print a warning prompting the user to check migrated metadata of all listed files (message includes file path and UIDs of sys_file and tx_dam records). This is for issue #62 again but still doesn't attempt to solve the conflict, we leave it up to the user to check for any possible errors. --- Classes/Service/MigrateMetadataService.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Classes/Service/MigrateMetadataService.php b/Classes/Service/MigrateMetadataService.php index 067f4fa..a70d449 100644 --- a/Classes/Service/MigrateMetadataService.php +++ b/Classes/Service/MigrateMetadataService.php @@ -135,6 +135,10 @@ public function execute() { $this->isInstalledFileMetadata = ExtensionManagementUtility::isLoaded('filemetadata'); $this->isInstalledMedia = ExtensionManagementUtility::isLoaded('media'); + // keep track of duplicate metadata to report to user + $processedFileIds = array(); + $duplicateMetadataFileIds = array(); + while ($record = $this->database->sql_fetch_assoc($res)) { $this->database->exec_UPDATEquery( 'sys_file_metadata', @@ -150,9 +154,27 @@ public function execute() { $this->controller->message(number_format(100 * ($counter / $total), 1) . '% of ' . $total . ' id: ' . $record['file_uid']); $this->amountOfMigratedRecords++; $counter++; + + // remember which sys_file records got data from which tx_dam records + $processedFileIds[$record['file_uid']]['damIds'][] = $record['dam_uid']; + $processedFileIds[$record['file_uid']]['path'] = rtrim($record['file_path'], '/').'/'.$record['file_name']; + if (count($processedFileIds[$record['file_uid']]['damIds']) > 1) { + // remember any duplicates we encountered + $duplicateMetadataFileIds[$record['file_uid']] = ''; + } } $this->database->sql_free_result($res); + // warn user about duplicate metadata + if (count($duplicateMetadataFileIds) > 0) { + $msg = PHP_EOL.'Multiple DAM file metadata has been found for the same files. You should verify metadata for the following files manually:'.PHP_EOL; + foreach ($duplicateMetadataFileIds as $fileId => $_a) { + $info = $processedFileIds[$fileId]; + $msg .= ' sys_file uid '.$fileId.', tx_dam uids '.implode(', ', $info['damIds']).', path '.$info['path'].PHP_EOL; + } + $this->controller->warningMessage($msg); + } + return $this->getResultMessage(); }