Fix duplicate metadata

Classes/Controller/DamMigrationCommandController.php

@@ -52,7 +52,7 @@ public function migrateDamRecordsToStorageCommand($storageUid = 1, $recordLimit
 
 	/**
 	 * Migrates all DAM records to FAL.
-	 * A database field "_migrateddamuid" connects each FAL record to the original DAM record.
+	 * A database field "_migratedfaluid" connects each original DAM record to the matching FAL record.
 	 *
 	 * @param int $storageUid The UID of the storage (default: 1 Do not modify if you are unsure.)
 	 * @param int $recordLimit The amount of records to process in a single run. You can set this value if you have memory constraints.
@@ -373,9 +373,9 @@ public function migrateDamFrontendPluginsCommand() {
 				}
 				else {
 					$newFalRecords = $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
-						'uid, _migrateddamuid',
-						'sys_file',
-						'_migrateddamuid IN (' . $plugin['tx_damdownloadlist_records'] . ')'
+						'_migratedfaluid AS uid',
+						'tx_dam',
+						'tx_dam.uid IN (' . $plugin['tx_damdownloadlist_records'] . ') AND _migratedfaluid > 0'
 					);
 
 					// update ctype for dam_frontend_pi2 plugin

Classes/Helper/DatabaseHelper.php

@@ -52,9 +52,9 @@ public static function getInstance() {
 	 */
 	public function getAllMigratedFalRecords() {
 		return $GLOBALS['TYPO3_DB']->exec_SELECTgetRows(
-			'uid, _migrateddamuid AS damuid',
-			'sys_file',
-			'_migrateddamuid>0',
+			'sys_file.uid AS uid, tx_dam.uid AS damuid',
+			'tx_dam JOIN sys_file ON sys_file._migratedfaluid = tx_dam.uid',
+			'', // where
 			'',	// group by
 			'', // order by
 			'', // limit

Classes/Service/MigrateCategoryRelationsService.php

@@ -100,8 +100,8 @@ public function execute() {
 	protected function getCategoryRelationsWhereSysCategoryExists() {
 		$rows = $this->database->exec_SELECTgetRows(
 			'MM.*, SM.uid as sys_metadata_uid, SC.uid as sys_category_uid',
-			'tx_dam_mm_cat MM, sys_file SF, sys_category SC, sys_file_metadata SM',
-			'SC._migrateddamcatuid = MM.uid_foreign AND SF._migrateddamuid = MM.uid_local AND SM.file = SF.uid'
+			'tx_dam_mm_cat MM, tx_dam D, sys_category SC, sys_file_metadata SM',
+			'SC._migrateddamcatuid = MM.uid_foreign AND D.uid = MM.uid_local AND SM.file = D._migratedfaluid'
 		);
 		if ($rows === NULL) {
 			throw new \Exception('SQL-Error in getCategoryRelationsWhereSysCategoryExists()', 1382968725);

Classes/Service/MigrateLinksService.php

@@ -188,9 +188,9 @@ protected function findMigratedFileUIDForDAMRecord($mediaUID) {
 
 		// query database
 		$res = $this->database->exec_SELECTgetSingleRow(
-			'uid',
-			'sys_file',
-			'`_migrateddamuid` = ' . $this->database->fullQuoteStr((int)$mediaUID, 'sys_file', false)
+			'_migratedfaluid AS uid',
+			'tx_dam',
+			'tx_dam.uid = ' . $this->database->fullQuoteStr((int)$mediaUID, 'tx_dam', false)
 		);
 
 		// use negative value if not found

Classes/Service/MigrateMetadataService.php

@@ -135,6 +135,10 @@ public function execute() {
 		$this->isInstalledFileMetadata = ExtensionManagementUtility::isLoaded('filemetadata');
 		$this->isInstalledMedia = ExtensionManagementUtility::isLoaded('media');
 
+		// keep track of duplicate metadata to report to user
+		$processedFileIds = array();
+		$duplicateMetadataFileIds = array();
+
 		while ($record = $this->database->sql_fetch_assoc($res)) {
 			$this->database->exec_UPDATEquery(
 				'sys_file_metadata',
@@ -150,9 +154,27 @@ public function execute() {
 			$this->controller->message(number_format(100 * ($counter / $total), 1) . '% of ' . $total . ' id: ' . $record['file_uid']);
 			$this->amountOfMigratedRecords++;
 			$counter++;
+
+			// remember which sys_file records got data from which tx_dam records
+			$processedFileIds[$record['file_uid']]['damIds'][] = $record['dam_uid'];
+			$processedFileIds[$record['file_uid']]['path'] = rtrim($record['file_path'], '/').'/'.$record['file_name'];
+			if (count($processedFileIds[$record['file_uid']]['damIds']) > 1) {
+				// remember any duplicates we encountered
+				$duplicateMetadataFileIds[$record['file_uid']] = '';
+			}
 		}
 		$this->database->sql_free_result($res);
 
+		// warn user about duplicate metadata
+		if (count($duplicateMetadataFileIds) > 0) {
+			$msg = PHP_EOL.'Multiple DAM file metadata has been found for the same files. You should verify metadata for the following files manually:'.PHP_EOL;
+			foreach ($duplicateMetadataFileIds as $fileId => $_a) {
+				$info = $processedFileIds[$fileId];
+				$msg .= ' sys_file uid '.$fileId.', tx_dam uids '.implode(', ', $info['damIds']).', path '.$info['path'].PHP_EOL;
+			}
+			$this->controller->warningMessage($msg);
+		}
+
 		return $this->getResultMessage();
 	}
 
@@ -163,9 +185,9 @@ public function execute() {
 	 */
 	protected function execSelectMigratedSysFilesQuery() {
 		return $this->database->exec_SELECTquery(
-			'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, f._migrateddamuid AS dam_uid, d.*',
+			'DISTINCT m.uid AS metadata_uid, f.uid as file_uid, d.uid AS dam_uid, d.*',
 			'sys_file f, sys_file_metadata m, tx_dam d',
-			'm.file=f.uid AND f._migrateddamuid=d.uid AND f._migrateddamuid > 0'
+			'm.file=f.uid AND d._migratedfaluid=f.uid'
 		);
 	}
 

Classes/Service/MigrateRelationsService.php

@@ -315,9 +315,9 @@ protected function getPidOfForeignRecord(array $damRelation) {
     }
 
 	/**
-	 * After a migration of tx_dam -> sys_file the col _migrateddamuid is
-	 * filled with dam uid Now we can search in dam relations for dam records
-	 * which have already been migrated to sys_file
+	 * After a migration of tx_dam -> sys_file the col _migratedfaluid is
+	 * filled with FAL uid. Now we can search in DAM relations for DAM records
+	 * which have already been migrated to sys_file.
 	 *
 	 * @return \mysqli_result
 	 */
@@ -339,8 +339,10 @@ protected function execSelectDamReferencesWhereSysFileExists() {
 		return $this->database->exec_SELECTquery(
 			$selectFields,
 			'tx_dam_mm_ref
+			JOIN tx_dam ON
+				tx_dam.uid = tx_dam_mm_ref.uid_local
 			JOIN sys_file ON
-				sys_file._migrateddamuid = tx_dam_mm_ref.uid_local
+				sys_file.uid = tx_dam._migratedfaluid
 			JOIN sys_file_metadata ON
 				sys_file.uid = sys_file_metadata.file
 				',

Classes/Service/MigrateRteMediaTagService.php

@@ -59,9 +59,9 @@ public function execute($table, $field) {
 
 		/** @var PreparedStatement $getSysFileUidStatement */
 		$getSysFileUidStatement = $this->database->prepare_SELECTquery(
-			'uid',
-			'sys_file',
-			'_migrateddamuid = :migrateddamuid'
+			'_migratedfaluid AS uid',
+			'tx_dam',
+			'tx_dam.uid = :migrateddamuid AND _migratedfaluid > 0'
 		);
 		foreach ($records as $rec) {
 			$originalContent = $rec[$field];

Classes/Service/MigrateService.php

@@ -30,7 +30,7 @@
 /**
  * Service to Migrate Records
  * Finds all DAM records that have not been migrated yet
- * and adds a DB field "_migrateddamuid" to each FAL record
+ * and adds a DB field "_migratedfaluid" to each DAM record
  * to connect the DAM and FAL DB records
  *
  * currently it only works for files within the fileadmin
@@ -165,7 +165,7 @@ protected function isValidDirectory(array $damRecord) {
 	public function countNotMigratedDamRecordsQuery() {
 		$row = $this->database->exec_SELECTgetSingleRow(
 			'COUNT(*) as total',
-			'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)',
+			'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)',
 			'sys_file.uid IS NULL AND
 			 tx_dam.deleted = 0 AND
 			 tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND
@@ -182,7 +182,7 @@ public function countNotMigratedDamRecordsQuery() {
 	protected function execSelectNotMigratedDamRecordsQuery() {
 		return $this->database->exec_SELECTquery(
 			'tx_dam.*',
-			'tx_dam LEFT JOIN sys_file ON (tx_dam.uid = sys_file._migrateddamuid)',
+			'tx_dam LEFT JOIN sys_file ON (tx_dam._migratedfaluid = sys_file.uid)',
 			'sys_file.uid IS NULL AND
 			 tx_dam.deleted = 0 AND
 			 tx_dam.file_path LIKE "' . $this->storageBasePath . '%" AND
@@ -228,11 +228,11 @@ protected function migrateFileFromDamToFal(array $damRecord, \TYPO3\CMS\Core\Res
 				$this->createArrayForUpdateInsertSysFileRecord($damRecord)
 			);
 
-			// add the migrated uid of the DAM record to the FAL record
+			// add the uid of the FAL record to the original DAM record
 			$this->database->exec_UPDATEquery(
-				'sys_file',
-				'uid = ' . $fileObject->getUid(),
-				array('_migrateddamuid' => $damRecord['uid'])
+				'tx_dam',
+				'uid = ' . $damRecord['uid'],
+				array('_migratedfaluid' => $fileObject->getUid())
 			);
 		}
 	}

Documentation/CommandReference/Index.rst

@@ -147,7 +147,7 @@ dam_falmigration:dammigration:migratedamrecords
 
 **Migrates all DAM records to FAL.**
 
-A database field "_migrateddamuid" connects each FAL record to the original DAM record.
+A database field "_migratedfaluid" connects each original DAM record to the matching FAL record.
 
 
 

Documentation/UserManual/Index.rst

@@ -34,9 +34,9 @@ The available migration tasks can be found under the *extbase* cliKey:
 	                                           "_migrated/dam" is created and files
 	                                           are copied and indexed.
 	  dammigration:migratedamrecords           Migrates all DAM records to FAL. A
-	                                           DB field "_migrateddamuid" connects
-	                                           each FAL record to the original DAM
-	                                           record.
+                                               DB field "_migratedfaluid" connects
+                                               each original DAM record to the
+                                               matching FAL record.
 	  dammigration:migratedammetadata          Migrates DAM metadata to FAL
 	                                           metadata. Searches for all migrated
 	                                           sys_file records that do not have any

ext_tables.sql

@@ -1,11 +1,3 @@
-#
-# Table structure for table 'sys_file'
-#
-CREATE TABLE sys_file (
-	_migrateddamuid int(11) unsigned DEFAULT '0' NOT NULL,
-	KEY migratedRecords (_migrateddamuid)
-);
-
 #
 # Table structure for table 'sys_file_collection'
 #
@@ -26,6 +18,8 @@ CREATE TABLE sys_category (
 #
 CREATE TABLE tx_dam (
 	_missingfile tinyint(4) unsigned DEFAULT '0' NOT NULL,
+	_migratedfaluid int(11) unsigned DEFAULT '0' NOT NULL,
+	KEY migratedRecords (_migratedfaluid),
 	KEY deletedRecords (uid, deleted),
 	KEY missingFiles (_missingfile)
 );