diff --git a/FileSyncApp/FileSyncApp.csproj b/FileSyncApp/FileSyncApp.csproj index 0ef6a67..0636e23 100644 --- a/FileSyncApp/FileSyncApp.csproj +++ b/FileSyncApp/FileSyncApp.csproj @@ -2,16 +2,16 @@ Exe - net8.0 + net10.0 - - - + + + - - + + diff --git a/FileSyncApp/Properties/launchSettings.json b/FileSyncApp/Properties/launchSettings.json new file mode 100644 index 0000000..dc72117 --- /dev/null +++ b/FileSyncApp/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "FileSyncApp": { + "commandName": "Project", + "commandLineArgs": "Verbose" + } + } +} \ No newline at end of file diff --git a/FileSyncApp/buildcommand.cmd b/FileSyncApp/buildcommand.cmd new file mode 100644 index 0000000..8ff5b57 --- /dev/null +++ b/FileSyncApp/buildcommand.cmd @@ -0,0 +1,2 @@ +dotnet publish -o pub-winx64 -c Release -p:PublishSingleFile=true -p:PublishTrimmed=false -p:Version=0.0.19-dev -r win-x64 --self-contained .\FileSyncApp.csproj +dotnet publish -o pub-linux64 -c Release -p:PublishSingleFile=true -p:PublishTrimmed=false -p:Version=0.0.19-dev -r linux-x64 --self-contained .\FileSyncApp.csproj diff --git a/FileSyncApp/genswu.sh b/FileSyncApp/genswu.sh new file mode 100644 index 0000000..4ba634f --- /dev/null +++ b/FileSyncApp/genswu.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# Set variables +APP_VERSION="0.0.19-dev" +APP_NAME="FileSyncApp" +APP_DIR="/opt/$APP_NAME" +SERVICE_FILE="$APP_NAME.service" +SW_DESCRIPTION="sw-description" +OUTPUT_DIR="FileSyncAppPackage" +PUBLISH_DIR="publish" + +dotnet publish -o $PUBLISH_DIR -c Release -p:PublishSingleFile=true -p:PublishTrimmed=false -p:Version=$APP_VERSION -r linux-x64 --self-contained ./FileSyncApp.csproj +# Create directory structure + +mkdir -p ${OUTPUT_DIR} +mount -t ramfs ramfs ${OUTPUT_DIR} + +# Create systemd service file +cat < ${OUTPUT_DIR}/${SERVICE_FILE} +[Unit] +Description=FileSyncApp - Datei-Synchronisation +After=network.target + +[Service] +ExecStart=$APP_DIR/$APP_NAME +WorkingDirectory=$APP_DIR +Restart=on-failure +User=root +Environment=DOTNET_EnableDiagnostics=0 + +[Install] +WantedBy=multi-user.target +EOL + +# Create sw-description filex +cat < ${OUTPUT_DIR}/${SW_DESCRIPTION} +software = +{ + version = "$APP_VERSION"; + description = "FileSyncApp Deployment"; + bootloader_transaction_marker = false; + bootloader_state_marker = false; + hardware-compatibility: [ "#RE:.*" ]; + files: ( + { + filename = "$APP_NAME"; + path = "$APP_DIR/$APP_NAME"; + }, + { + filename = "$APP_NAME.service"; + path = "/etc/systemd/system/$SERVICE_FILE"; + } + ); + scripts: ( + { + filename = "update.sh"; + type = "shellscript"; + } + ); + preinstall = " || true && mkdir -p /opt/$APP_NAME"; + postinstall = "systemctl daemon-reexec && systemctl daemon-reload && systemctl enable --now $APP_NAME.service"; +}; +EOL +cat <<\EOFUPDATE > ${OUTPUT_DIR}/update.sh +#!/bin/sh + +if [ $# -lt 1 ]; then + exit 0; +fi +if [ $1 = "preinst" ]; then + systemctl stop FileSyncApp.service + mkdir -p /opt/FileSyncApp + echo "PREINST -> directory created" +fi + +if [ $1 = "postinst" ]; then + chmod +x /opt/FileSyncApp/FileSyncApp + systemctl daemon-reexec + systemctl daemon-reload + systemctl enable --now FileSyncApp.service +fi + +EOFUPDATE + +# Copy the compiled binary to the package directory +cp ${PUBLISH_DIR}/$APP_NAME ${OUTPUT_DIR} + +# Create CPIO archive +cd $OUTPUT_DIR +cpio -H crc -o < <(printf '%s\n' sw-description; find . ! -name sw-description -type f | sort) > ../$APP_NAME-$APP_VERSION.swu +cd .. +umount ${OUTPUT_DIR} +rm -r ${OUTPUT_DIR} + +echo "SWUpdate package created: $APP_NAME-$APP_VERSION.swu" diff --git a/FileSyncAppConfigEditor/FileSyncAppConfigEditor.csproj b/FileSyncAppConfigEditor/FileSyncAppConfigEditor.csproj index 5c8aafa..ce3abe3 100644 --- a/FileSyncAppConfigEditor/FileSyncAppConfigEditor.csproj +++ b/FileSyncAppConfigEditor/FileSyncAppConfigEditor.csproj @@ -2,14 +2,14 @@ WinExe - net8.0-windows + net10.0-windows enable true enable - + diff --git a/FileSyncAppWin/FileSyncAppWin.csproj b/FileSyncAppWin/FileSyncAppWin.csproj index 82030c1..af45f33 100644 --- a/FileSyncAppWin/FileSyncAppWin.csproj +++ b/FileSyncAppWin/FileSyncAppWin.csproj @@ -2,7 +2,7 @@ WinExe - net8.0-windows + net10.0-windows true enable diff --git a/FileSyncAppWin/Program.cs b/FileSyncAppWin/Program.cs index f22328d..87aef88 100644 --- a/FileSyncAppWin/Program.cs +++ b/FileSyncAppWin/Program.cs @@ -20,6 +20,7 @@ internal static class Program [STAThread] static void Main(string[] args) { + _args = args; FileSyncApp.Program.ConfigureLogger(args.FirstOrDefault()); logger = FileSyncApp.Program.LoggerFactory.CreateLogger("FileSyncAppWin"); diff --git a/FileSyncAppWin/Properties/launchSettings.json b/FileSyncAppWin/Properties/launchSettings.json new file mode 100644 index 0000000..a6b346c --- /dev/null +++ b/FileSyncAppWin/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "FileSyncAppWin": { + "commandName": "Project", + "commandLineArgs": "Verbose" + } + } +} \ No newline at end of file diff --git a/FileSyncAppWin/buildcommand.cmd b/FileSyncAppWin/buildcommand.cmd index fd0057f..f480527 100644 --- a/FileSyncAppWin/buildcommand.cmd +++ b/FileSyncAppWin/buildcommand.cmd @@ -1 +1 @@ -dotnet publish -o pub -c Release -p:PublishSingleFile=true -p:PublishTrimmed=false -p:Version=0.0.16 -r win-x86 --self-contained .\FileSyncAppWin.csproj +dotnet publish -o pub -c Release -p:PublishSingleFile=true -p:PublishTrimmed=false -p:Version=0.0.20 -r win-x64 --self-contained .\FileSyncAppWin.csproj diff --git a/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs b/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs index 9bb4f0a..1ad96f2 100644 --- a/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs +++ b/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; namespace FileSyncLibNet.AccessProviders { @@ -12,6 +13,9 @@ internal class FileIoAccessProvider : IAccessProvider public string AccessPath { get; private set; } private readonly RemoteState remoteState; private readonly ILogger logger; + private const int DefaultBandwidthLimit = 100 * 1024 * 1024; // 100 MB/s + private int bandwidthLimit = DefaultBandwidthLimit; + public FileIoAccessProvider(ILogger logger, string stateFilename) { this.logger = logger; @@ -38,7 +42,7 @@ public FileInfo2 GetFileInfo(string path) return new FileInfo2(path, fi.Exists) { - LastWriteTime = fi.Exists ? fi.LastWriteTime : DateTime.MinValue, + LastWriteTime = fi.Exists ? new DateTime(fi.LastWriteTime.Ticks) : DateTime.MinValue, Length = fi.Exists ? fi.Length : 0 }; } @@ -79,9 +83,28 @@ public void WriteFile(FileInfo2 file, Stream content) { var realFilename = Path.Combine(AccessPath, file.Name); Directory.CreateDirectory(Path.GetDirectoryName(realFilename)); - using (var stream = File.Create(realFilename)) + if (bandwidthLimit == DefaultBandwidthLimit) + { + using (var stream = File.Create(realFilename)) + { + content.CopyTo(stream); + } + } + else { - content.CopyTo(stream); + using (var stream = new FileStream(realFilename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.None)) + { + int sleepIntervalMs = 50; // Fixed sleep interval + int bufferSize = bandwidthLimit / (1000 / sleepIntervalMs); // Calculate buffer size based on bandwidth limit and interval + byte[] buffer = new byte[bufferSize]; + int bytesRead; + + while ((bytesRead = content.Read(buffer, 0, buffer.Length)) > 0) + { + stream.Write(buffer, 0, bytesRead); + Thread.Sleep(sleepIntervalMs); // Sleep for the fixed interval + } + } } File.SetLastWriteTime(realFilename, file.LastWriteTime); remoteState?.SetFileInfo(realFilename, file); diff --git a/FileSyncLibNet/FileSyncJob/FileSyncJob.cs b/FileSyncLibNet/FileSyncJob/FileSyncJob.cs index 967fd9b..5101716 100644 --- a/FileSyncLibNet/FileSyncJob/FileSyncJob.cs +++ b/FileSyncLibNet/FileSyncJob/FileSyncJob.cs @@ -110,6 +110,7 @@ private void RunJobInterlocked() } catch (Exception exc) { + options.Logger.LogError(exc, "JobError {0}", JobName); JobError?.Invoke(this, new FileSyncJobEventArgs(JobName, FileSyncJobStatus.Error, exc)); } finally diff --git a/FileSyncLibNet/FileSyncLibNet.csproj b/FileSyncLibNet/FileSyncLibNet.csproj index 64d1e15..93ef71e 100644 --- a/FileSyncLibNet/FileSyncLibNet.csproj +++ b/FileSyncLibNet/FileSyncLibNet.csproj @@ -12,11 +12,11 @@ A library to easily backup or sync 2 folders either once or in a given interval. - + - - - + + + diff --git a/FileSyncLibNet/SyncProviders/AbstractProvider.cs b/FileSyncLibNet/SyncProviders/AbstractProvider.cs index eb5388b..4b809ae 100644 --- a/FileSyncLibNet/SyncProviders/AbstractProvider.cs +++ b/FileSyncLibNet/SyncProviders/AbstractProvider.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using System; using System.Diagnostics; +using System.Threading.Tasks; namespace FileSyncLibNet.SyncProviders { @@ -130,48 +131,106 @@ public override void SyncSourceToDest() pattern: JobOptions.SearchPattern, recursive: JobOptions.Recursive, subfolders: JobOptions.Subfolders); - foreach (var sourceFile in sourceFiles) + + bool parallel = false; + if (parallel) { - var remoteFile = DestinationAccess.GetFileInfo(sourceFile.Name); - bool copy = !remoteFile.Exists || remoteFile.Length != sourceFile.Length || remoteFile.LastWriteTime != sourceFile.LastWriteTime; - if (copy) + Parallel.ForEach(sourceFiles, (sourceFile) => { - if (createDestinationDir) + var remoteFile = DestinationAccess.GetFileInfo(sourceFile.Name); + bool copy = !remoteFile.Exists || remoteFile.Length != sourceFile.Length || remoteFile.LastWriteTime != sourceFile.LastWriteTime; + if (copy) { + if (createDestinationDir) + { + try + { + DestinationAccess.CreateDirectory(""); + } + catch (Exception ex) { logger?.LogError(ex, "exception creating destination directory {A}", jobOptions.DestinationPath); } + createDestinationDir = false; + } try { - DestinationAccess.CreateDirectory(""); + logger.LogDebug("Copy {A}", sourceFile.Name); + using (var sourceStream = SourceAccess.GetStream(sourceFile)) + { + DestinationAccess.WriteFile(sourceFile, sourceStream); + } + //file.copy + copied++; + if (jobOptions.DeleteSourceAfterBackup) + { + SourceAccess.Delete(sourceFile); + } + } + catch (Exception exc) + { + error_occured = true; + logger.LogError(exc, "Exception copying {A}", sourceFile); } - catch (Exception ex) { logger?.LogError(ex, "exception creating destination directory {A}", jobOptions.DestinationPath); } - createDestinationDir = false; } - try + else { - logger.LogDebug("Copy {A}", sourceFile.Name); - using (var sourceStream = SourceAccess.GetStream(sourceFile)) + + skipped++; + if (skipped % 1000 == 0) { - DestinationAccess.WriteFile(sourceFile, sourceStream); + logger.LogTrace("Skip {A} {B}", skipped, sourceFile); } - //file.copy - copied++; - if (jobOptions.DeleteSourceAfterBackup) + //logger.LogTrace("Skip {A} {B}", skipped, sourceFile); + } + }); + } + else + { + foreach (var sourceFile in sourceFiles) + { + var remoteFile = DestinationAccess.GetFileInfo(sourceFile.Name); + bool copy = !remoteFile.Exists || remoteFile.Length != sourceFile.Length || remoteFile.LastWriteTime != sourceFile.LastWriteTime; + if (copy) + { + if (createDestinationDir) { - SourceAccess.Delete(sourceFile); + try + { + DestinationAccess.CreateDirectory(""); + } + catch (Exception ex) { logger?.LogError(ex, "exception creating destination directory {A}", jobOptions.DestinationPath); } + createDestinationDir = false; + } + try + { + logger.LogDebug("Copy {A}", sourceFile.Name); + using (var sourceStream = SourceAccess.GetStream(sourceFile)) + { + DestinationAccess.WriteFile(sourceFile, sourceStream); + } + //file.copy + copied++; + if (jobOptions.DeleteSourceAfterBackup) + { + SourceAccess.Delete(sourceFile); + } + } + catch (Exception exc) + { + error_occured = true; + logger.LogError(exc, "Exception copying {A}", sourceFile); } } - catch (Exception exc) + else { - error_occured = true; - logger.LogError(exc, "Exception copying {A}", sourceFile); + + skipped++; + if (skipped % 1000 == 0) + { + logger.LogTrace("Skip {A} {B}", skipped, sourceFile); + } + //logger.LogTrace("Skip {A} {B}", skipped, sourceFile); } - } - else - { - skipped++; - logger.LogTrace("Skip {A}", sourceFile); } - } if (!error_occured) { diff --git a/FileSyncLibNet/SyncProviders/FileIOProvider.cs b/FileSyncLibNet/SyncProviders/FileIOProvider.cs index 58dcd87..faa7870 100644 --- a/FileSyncLibNet/SyncProviders/FileIOProvider.cs +++ b/FileSyncLibNet/SyncProviders/FileIOProvider.cs @@ -50,7 +50,6 @@ public override void SyncSourceToDest() var old = _fi.Count(); _fi = _fi.Where(x => x.LastWriteTime > (LastRun - jobOptions.Interval)).ToList(); skipped += old-_fi.Count(); - LastRun = DateTimeOffset.Now; } foreach (FileInfo f in _fi) { @@ -83,6 +82,8 @@ public override void SyncSourceToDest() } } } + if (jobOptions.RememberLastSync) + LastRun = DateTimeOffset.Now; sw.Stop(); logger.LogInformation("{A} files copied, {B} files skipped in {C}s", copied, skipped, sw.ElapsedMilliseconds / 1000.0); } diff --git a/FileSyncLibNet/SyncProviders/ScpProvider.cs b/FileSyncLibNet/SyncProviders/ScpProvider.cs index 808fee6..2e431f6 100644 --- a/FileSyncLibNet/SyncProviders/ScpProvider.cs +++ b/FileSyncLibNet/SyncProviders/ScpProvider.cs @@ -89,7 +89,6 @@ public override void SyncSourceToDest() var old = _fi.Count(); _fi = _fi.Where(x => x.LastWriteTime > (LastRun - jobOptions.Interval)).ToList(); skipped += old - _fi.Count(); - LastRun = DateTimeOffset.Now; } foreach (FileInfo f in _fi) { @@ -136,6 +135,8 @@ public override void SyncSourceToDest() } } } + if (jobOptions.RememberLastSync) + LastRun = DateTimeOffset.Now; sw.Stop(); logger.LogInformation("{A} files copied, {B} files skipped in {C}s", copied, skipped, sw.ElapsedMilliseconds / 1000.0); } diff --git a/FileSyncLibNet/SyncProviders/SmbLibProvider.cs b/FileSyncLibNet/SyncProviders/SmbLibProvider.cs index 40573bf..363b138 100644 --- a/FileSyncLibNet/SyncProviders/SmbLibProvider.cs +++ b/FileSyncLibNet/SyncProviders/SmbLibProvider.cs @@ -84,7 +84,7 @@ public override void SyncSourceToDest() if (jobOptions.RememberLastSync) { _fi = _fi.Where(x => x.LastWriteTime > (LastRun - jobOptions.Interval)); - LastRun = DateTimeOffset.Now; + } if (jobOptions.SyncDeleted) { @@ -100,12 +100,12 @@ public override void SyncSourceToDest() { bool copy = false; var relativeFilename = f.FullName.Substring(Path.GetFullPath(jobOptions.SourcePath).Length); - var remotefile = Path.Combine(DestinationPath, relativeFilename.TrimStart('\\', '/')).Replace('/', '\\'); + var remotefile = Path.Combine(DestinationPath, relativeFilename.TrimStart('\\', '/')).Replace('/', '\\').TrimStart('/'); var exists = FileExists(remotefile, out long size); copy = !exists || size != f.Length; if (copy) { - logger.LogDebug("Copy {A}", relativeFilename); + logger.LogDebug("Copy {A} to {B}", f.FullName, remotefile); try { WriteFile(f.FullName, remotefile); @@ -148,7 +148,7 @@ public override void SyncSourceToDest() } Directory.CreateDirectory(jobOptions.DestinationPath); - foreach(var subfolder in jobOptions.Subfolders) + foreach (var subfolder in jobOptions.Subfolders) Directory.CreateDirectory(Path.Combine(jobOptions.DestinationPath, subfolder)); DirectoryInfo _di = new DirectoryInfo(jobOptions.DestinationPath); foreach (var dir in JobOptions.Subfolders.Count > 0 ? _di.GetDirectories() : new[] { _di }) @@ -163,7 +163,6 @@ public override void SyncSourceToDest() var remoteFiles = ListFiles(SourcePath, dir.Name, JobOptions.Recursive, jobOptions.RememberLastSync ? LastRun - jobOptions.Interval : DateTime.MinValue, out int skippedByTimestamp); skipped += skippedByTimestamp; - LastRun = DateTimeOffset.Now; if (jobOptions.SyncDeleted) { foreach (var file in remoteFiles) @@ -212,6 +211,7 @@ public override void SyncSourceToDest() } sw.Stop(); + LastRun = DateTimeOffset.Now; logger.LogInformation("{A} files copied, {B} files skipped in {C}s", copied, skipped, sw.ElapsedMilliseconds / 1000.0); } @@ -247,7 +247,7 @@ public void ConnectToShare(string server, string shareName, string domain, strin } - + } public void Dispose() @@ -308,7 +308,15 @@ public void WriteFile(string localFilePath, string remoteFilePath) string createpath = ""; for (int i = 0; i < paths.Length - 1; i++) { + createpath = Path.Combine(createpath, paths[i]); + + createpath = createpath.Replace('/', '\\'); + var exists = DirExists(createpath); + + if (exists) + continue; + logger.LogDebug("Create path {A}", createpath); status = fileStore.CreateFile(out fileHandle, out fileStatus, createpath, AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, FileAttributes.Normal, ShareAccess.None, CreateDisposition.FILE_OPEN_IF, CreateOptions.FILE_DIRECTORY_FILE, null); if (status == NTStatus.STATUS_SUCCESS) { @@ -385,7 +393,7 @@ List ListFiles(string sourcePath, string sourcesubPath, bool recurse, Da { try { - retval.AddRange(ListFiles(sourcePath,Path.Combine(sourcesubPath, file.FileName).Trim('\\'), recurse, maxAge, out int moreSkipped)); + retval.AddRange(ListFiles(sourcePath, Path.Combine(sourcesubPath, file.FileName).Trim('\\'), recurse, maxAge, out int moreSkipped)); skipped += moreSkipped; } catch { } @@ -410,6 +418,23 @@ List ListFiles(string sourcePath, string sourcesubPath, bool recurse, Da return retval; } + bool DirExists(string filepathFromShare) + { + object directoryHandle; + FileStatus fileStatus; + var status = fileStore.CreateFile(out directoryHandle, out fileStatus, filepathFromShare.Trim('\\'), AccessMask.GENERIC_READ, FileAttributes.Normal, ShareAccess.Read | ShareAccess.Write, CreateDisposition.FILE_OPEN, CreateOptions.FILE_DIRECTORY_FILE, null); + if (status == NTStatus.STATUS_SUCCESS) + { + status = fileStore.GetFileInformation(out FileInformation result, directoryHandle, FileInformationClass.FileStandardInformation); + + status = fileStore.CloseFile(directoryHandle); + return true; + } + + return false; + + } + bool FileExists(string filepathFromShare, out long size) { object directoryHandle; @@ -430,19 +455,23 @@ void SetFileAttributes(string filepathFromShare, DateTime lastWriteTime, DateTim { object directoryHandle; FileStatus fileStatus; - var status = fileStore.CreateFile(out directoryHandle, out fileStatus, filepathFromShare.Trim('\\'), AccessMask.GENERIC_READ, FileAttributes.Normal, ShareAccess.Read | ShareAccess.Write, CreateDisposition.FILE_OPEN, CreateOptions.FILE_NON_DIRECTORY_FILE, null); + //var status = fileStore.CreateFile(out directoryHandle, out fileStatus, filepathFromShare.Trim('\\'), AccessMask.GENERIC_READ, FileAttributes.Normal, ShareAccess.Read | ShareAccess.Write, CreateDisposition.FILE_OPEN, CreateOptions.FILE_NON_DIRECTORY_FILE, null); + var status = fileStore.CreateFile(out directoryHandle, out fileStatus, filepathFromShare.Trim('\\'), AccessMask.GENERIC_READ | AccessMask.GENERIC_WRITE | AccessMask.SYNCHRONIZE, FileAttributes.Normal, ShareAccess.None, CreateDisposition.FILE_OPEN, CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, null); if (status == NTStatus.STATUS_SUCCESS) { - + status = fileStore.GetFileInformation(out FileInformation result, directoryHandle, FileInformationClass.FileBasicInformation); - (result as FileBasicInformation).LastWriteTime=lastWriteTime; - (result as FileBasicInformation).CreationTime=createTime; - (result as FileBasicInformation).ChangeTime=modifiedTime; - (result as FileBasicInformation).LastAccessTime=accessTime; + (result as FileBasicInformation).LastWriteTime = lastWriteTime; + (result as FileBasicInformation).CreationTime = createTime; + (result as FileBasicInformation).ChangeTime = modifiedTime; + (result as FileBasicInformation).LastAccessTime = accessTime; status = fileStore.SetFileInformation(directoryHandle, result); + if (status != NTStatus.STATUS_SUCCESS) + throw new Exception("unable to set attributes - status " + status); status = fileStore.CloseFile(directoryHandle); } - throw new Exception("unable to set attributes - status " + status); + if (status != NTStatus.STATUS_SUCCESS) + throw new Exception("unable to set attributes - status " + status); } void GetFileAttributes(string filepathFromShare, out DateTime lastWriteTime, out DateTime createTime, out DateTime modifiedTime, out DateTime accessTime)