Checksum verification optimization

flutter/lib/benchmark/state.dart

@@ -46,6 +46,8 @@ class BenchmarkState extends ChangeNotifier {
   // null - downloading/waiting; false - running; true - done
   bool? _doneRunning;
 
+  ResourceLoadingStatus get loadingStatus => resourceManager.status;
+
   // Only if [state] == [BenchmarkStateEnum.downloading]
   String get loadingPath => resourceManager.loadingPath;
 
@@ -147,10 +149,6 @@ class BenchmarkState extends ChangeNotifier {
       modes: [taskRunner.perfMode, taskRunner.accuracyMode],
       benchmarks: selectedBenchmarks,
     );
-    final allResources = _benchmarkStore.listResources(
-      modes: [taskRunner.perfMode, taskRunner.accuracyMode],
-      benchmarks: allBenchmarks,
-    );
     try {
       final selectedBenchmarkIds = selectedBenchmarks
           .map((e) => e.benchmarkSettings.benchmarkId)
@@ -163,12 +161,6 @@ class BenchmarkState extends ChangeNotifier {
         downloadMissing: downloadMissing,
       );
       print('Finished loading resources with downloadMissing=$downloadMissing');
-      // We still need to load all resources after download selected resources.
-      await resourceManager.handleResources(
-        resources: allResources,
-        purgeOldCache: false,
-        downloadMissing: false,
-      );
       error = null;
       stackTrace = null;
       taskConfigFailedToLoad = false;
@@ -261,7 +253,8 @@ class BenchmarkState extends ChangeNotifier {
   }
 
   Future<void> runBenchmarks() async {
-    assert(resourceManager.done, 'Resource manager is not done.');
+    assert(resourceManager.status == ResourceLoadingStatus.done,
+        'Resource manager is not done.');
     assert(_doneRunning != false, '_doneRunning is false');
     _store.previousExtendedResult = '';
     _doneRunning = false;

flutter/lib/l10n/app_en.arb

@@ -137,6 +137,7 @@
   "resourceErrorCurrentConfig": "Current task config file: ",
   "resourceErrorRereadConfig": "I have fixed current config, try again",
   "resourceErrorFail": "Config failed to load",
+  "resourceVerifying": "Verifying",
   "resourceStatusNotDownloaded": "Resources Not Downloaded",
 
   "listScreenTitleMain": "Results",

flutter/lib/resources/cache_manager.dart

@@ -10,6 +10,7 @@ class CacheManager {
   late final FileCacheHelper fileCacheHelper;
   late final ArchiveCacheHelper archiveCacheHelper;
   Map<String, String> _resourcesMap = {};
+  List<String> newDownloads = [];
 
   CacheManager(this.loadedResourcesDir) {
     fileCacheHelper = FileCacheHelper(loadedResourcesDir);
@@ -30,40 +31,46 @@ class CacheManager {
 
   Future<void> deleteLoadedResources(List<String> excludes,
       [int atLeastDaysOld = 0]) async {
-    final directory = Directory(loadedResourcesDir);
-
-    // can't use 'await for' here because we delete files,
-    // and 'await for' on stream from directory.list() throws exceptions when file is missing
-    for (final file in await directory.list(recursive: true).toList()) {
-      // skip files in already deleted folders
-      if (!await file.exists()) continue;
-      final relativePath = file.path
-          .replaceAll('\\', '/')
-          .substring(loadedResourcesDir.length + 1);
-      var keep = false;
-      for (var resource in excludes) {
-        // relativePath.startsWith(resource): if we want to preserve a folder resource
-        // resource.startsWith(relativePath): if we want to preserve a file resource
-        //   for example:
-        //   we are checking folder 'github.com'
-        //   resource is 'github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/ade20k'
-        if (relativePath.startsWith(resource) ||
-            resource.startsWith(relativePath)) {
-          keep = true;
-          break;
-        }
+    final Map<String, String> resourcesToDelete = Map.from(_resourcesMap);
+    final deletionTime = DateTime.now();
+
+    for (var key in excludes) {
+      resourcesToDelete.remove(key);
+    }
+
+    for (var resource in resourcesToDelete.keys.toList()) {
+      String resourcePath = resourcesToDelete[resource] ?? '';
+      var stat = FileStat.statSync(resourcePath);
+      if (stat.type == FileSystemEntityType.notFound ||
+          (atLeastDaysOld > 0 &&
+              deletionTime.difference(stat.modified).inDays < atLeastDaysOld)) {
+        resourcesToDelete.remove(resource);
+        continue;
       }
-      if (keep) continue;
-      if (atLeastDaysOld > 0) {
-        var stat = await file.stat();
-        if (DateTime.now().difference(stat.modified).inDays < atLeastDaysOld) {
-          continue;
+      try {
+        switch (stat.type) {
+          case FileSystemEntityType.file:
+          case FileSystemEntityType.link:
+            File(resourcePath).deleteSync(recursive: true);
+            break;
+          case FileSystemEntityType.directory:
+            Directory(resourcePath).deleteSync(recursive: true);
+            break;
+          default:
+            break;
         }
+        _resourcesMap.remove(resource);
+      } on FileSystemException catch (e) {
+        if (e.osError?.errorCode == 2) {
+          // TODO might need to be changed for Windows
+          _resourcesMap.remove(resource);
+        } else
+          rethrow;
       }
-      await file.delete(recursive: true);
     }
   }
 
+  // NOTE this does not remove the files that were extracted from the archive, just the archive itself.
   Future<void> deleteArchives(List<String> resources) async {
     for (final resource in resources) {
       final archivePath = getArchive(resource);
@@ -79,6 +86,8 @@ class CacheManager {
       if (filePath == null) continue;
       final file = File(filePath);
       if (await file.exists()) await file.delete();
+      _resourcesMap.remove(
+          resource); // Update the resource map to reflect the deleted file
       print('Deleted resource $resource stored at ${file.path}');
     }
   }
@@ -96,12 +105,14 @@ class CacheManager {
     required void Function(double, String) onProgressUpdate,
     required bool purgeOldCache,
     required bool downloadMissing,
+    bool overrideMap = false,
   }) async {
     final resourcesToDownload = <String>[];
-    _resourcesMap = {};
+    final resourcesMap = <String, String>{};
+    newDownloads.clear();
 
     for (final resource in urls) {
-      if (_resourcesMap.containsKey(resource)) continue;
+      if (resourcesMap.containsKey(resource)) continue;
       if (resourcesToDownload.contains(resource)) continue;
 
       String path;
@@ -112,7 +123,7 @@ class CacheManager {
       }
 
       if (path != '') {
-        _resourcesMap[resource] = path;
+        resourcesMap[resource] = path;
         continue;
       }
 
@@ -121,7 +132,12 @@ class CacheManager {
       continue;
     }
     if (downloadMissing) {
-      await _download(resourcesToDownload, onProgressUpdate);
+      await _download(resourcesToDownload, onProgressUpdate, resourcesMap);
+    }
+    if (overrideMap) {
+      _resourcesMap = resourcesMap;
+    } else {
+      _resourcesMap.addAll(resourcesMap);
     }
     if (purgeOldCache) {
       await purgeOutdatedCache(_oldFilesAgeInDays);
@@ -133,17 +149,19 @@ class CacheManager {
   }
 
   Future<void> _download(
-    List<String> urls,
-    void Function(double, String) onProgressUpdate,
-  ) async {
+      List<String> urls,
+      void Function(double, String) onProgressUpdate,
+      Map<String, String> resourcesMap // Passed by reference
+      ) async {
+    newDownloads = urls;
     var progress = 0.0;
     for (var url in urls) {
       progress += 0.1 / urls.length;
       onProgressUpdate(progress, url);
       if (isResourceAnArchive(url)) {
-        _resourcesMap[url] = await archiveCacheHelper.get(url, true);
+        resourcesMap[url] = await archiveCacheHelper.get(url, true);
       } else {
-        _resourcesMap[url] = await fileCacheHelper.get(url, true);
+        resourcesMap[url] = await fileCacheHelper.get(url, true);
       }
       progress += 0.9 / urls.length;
       onProgressUpdate(progress, url);

flutter/lib/resources/resource_manager.dart

@@ -11,6 +11,12 @@ import 'package:mlperfbench/resources/result_manager.dart';
 import 'package:mlperfbench/resources/utils.dart';
 import 'package:mlperfbench/store.dart';
 
+enum ResourceLoadingStatus {
+  done,
+  loading,
+  verifying,
+}
+
 class ResourceManager {
   static const _dataPrefix = 'local://';
   static const _loadedResourcesDirName = 'loaded_resources';
@@ -19,7 +25,8 @@ class ResourceManager {
   final VoidCallback _onUpdate;
   final Store store;
 
-  bool _done = false;
+  // We start out as loading to prevent benchmarks from running before handleResources() is called.
+  ResourceLoadingStatus _status = ResourceLoadingStatus.loading;
   String _loadingPath = '';
   double _loadingProgress = 0.0;
 
@@ -31,7 +38,9 @@ class ResourceManager {
 
   ResourceManager(this._onUpdate, this.store);
 
-  bool get done => _done;
+  ResourceLoadingStatus get status {
+    return _status;
+  }
 
   double get loadingProgress {
     return _loadingProgress;
@@ -101,7 +110,7 @@ class ResourceManager {
   }) async {
     _loadingPath = '';
     _loadingProgress = 0.001;
-    _done = false;
+    _status = ResourceLoadingStatus.loading;
     _onUpdate();
     try {
       var internetResources = <Resource>[];
@@ -130,24 +139,33 @@ class ResourceManager {
         throw 'A network error has occurred. Please make sure you are connected to the internet.';
       }
 
-      if (downloadMissing) {
-        // If the files are downloaded from the internet, validate its checksum and delete corrupted files.
-        final checksumFailed = await validateResourcesChecksum(resources);
-        if (checksumFailed.isNotEmpty) {
-          final checksumFailedPathString =
-              checksumFailed.map((e) => '\n${e.path}').join();
-          final checksumFailedPaths =
-              checksumFailed.map((e) => e.path).toList();
-          await cacheManager.deleteFiles(checksumFailedPaths);
-          throw 'Checksum validation failed for: $checksumFailedPathString. \nPlease download the missing files again.';
-        }
+      // Make sure to only checksum resources that were downloaded in the step above
+      final newResources = internetResources
+          .where((element) => cacheManager.newDownloads.contains(element.path))
+          .toList();
+
+      _status = ResourceLoadingStatus.verifying;
+      final checksumFailed = await validateResourcesChecksum(
+        newResources,
+        (double currentProgress, String currentPath) {
+          _loadingProgress = currentProgress;
+          _loadingPath = currentPath;
+          _onUpdate();
+        },
+      );
+      if (checksumFailed.isNotEmpty) {
+        final checksumFailedPathString =
+            checksumFailed.map((e) => '\n${e.path}').join();
+        final checksumFailedPaths = checksumFailed.map((e) => e.path).toList();
+        await cacheManager.deleteFiles(checksumFailedPaths);
+        throw 'Checksum validation failed for: $checksumFailedPathString. \nPlease download the missing files again.';
       }
       // delete downloaded archives to free up disk space
       await cacheManager.deleteArchives(internetPaths);
     } finally {
       _loadingPath = '';
       _loadingProgress = 1.0;
-      _done = true;
+      _status = ResourceLoadingStatus.done;
       _onUpdate();
     }
   }
@@ -209,6 +227,7 @@ class ResourceManager {
     return result;
   }
 
+
   // Similar to [validateResourcesExist], but returns one result for all resources provided
   Future<bool> validateAllResourcesExist(List<Resource> resources) async {
     for (Resource r in resources) {
@@ -222,10 +241,14 @@ class ResourceManager {
     return true;
   }
 
-  Future<List<Resource>> validateResourcesChecksum(
-      List<Resource> resources) async {
+  Future<List<Resource>> validateResourcesChecksum(List<Resource> resources,
+      void Function(double, String) onProgressUpdate) async {
     final checksumFailedResources = <Resource>[];
+    double progress = 0.0;
     for (final resource in resources) {
+      progress += 0.1 / resources.length;
+      onProgressUpdate(progress, resource.path);
+
       final md5Checksum = resource.md5Checksum;
       if (md5Checksum.isEmpty) continue;
       String? localPath;
@@ -238,6 +261,9 @@ class ResourceManager {
       if (!await isChecksumMatched(localPath, md5Checksum)) {
         checksumFailedResources.add(resource);
       }
+
+      progress += 0.9 / resources.length;
+      onProgressUpdate(progress, resource.path);
     }
     return checksumFailedResources;
   }

flutter/lib/resources/validation_helper.dart

@@ -36,13 +36,14 @@ class ValidationHelper {
     return errorDescription; // + missing.mapIndexed((i, element) => '\n${i + 1}) $element').join();
   }
 
+  // TODO progress could be added here for verification dialog
   Future<String> validateChecksum(String errorDescription) async {
     final resources = benchmarkStore.listResources(
       modes: selectedRunModes,
       benchmarks: benchmarkStore.activeBenchmarks,
     );
     final checksumFailed =
-        await resourceManager.validateResourcesChecksum(resources);
+        await resourceManager.validateResourcesChecksum(resources, (_,__){});
     if (checksumFailed.isEmpty) return '';
     final mismatchedPaths = checksumFailed.map((e) => '\n${e.path}').join();
     return errorDescription + mismatchedPaths;

flutter/lib/ui/settings/resources_screen.dart

@@ -1,6 +1,7 @@
 import 'package:flutter/material.dart';
 
 import 'package:bot_toast/bot_toast.dart';
+import 'package:mlperfbench/resources/resource_manager.dart';
 import 'package:provider/provider.dart';
 
 import 'package:mlperfbench/benchmark/benchmark.dart';
@@ -189,7 +190,9 @@ class _ResourcesScreen extends State<ResourcesScreen> {
       crossAxisAlignment: CrossAxisAlignment.start,
       children: [
         Text(
-          l10n.resourceDownloading,
+          state.loadingStatus == ResourceLoadingStatus.loading
+              ? l10n.resourceDownloading
+              : l10n.resourceVerifying,
           maxLines: 1,
           style: const TextStyle(fontSize: 12),
         ),