From 19087887abc7a5dc1a80bad07f4f0f5cbbf262d6 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 07:15:47 +0100 Subject: [PATCH 1/8] fix startup problems and add exception handling of config.json --- FileSyncApp/Program.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/FileSyncApp/Program.cs b/FileSyncApp/Program.cs index 1fd608f..dbce2f1 100644 --- a/FileSyncApp/Program.cs +++ b/FileSyncApp/Program.cs @@ -25,7 +25,7 @@ public class Program public static Dictionary Jobs = new Dictionary(); public static void Main(string[] args) { - ConfigureLogger(args.FirstOrDefault()); + ConfigureLogger(args?.FirstOrDefault()); log = LoggerFactory.CreateLogger("FileSyncAppMain"); if (null != args && args.Length > 0) { @@ -50,6 +50,7 @@ static void RunProgram() ContractResolver = new IgnorePropertyResolver(new string[] { "Logger" }), TypeNameHandling = TypeNameHandling.Auto, }; + if (!File.Exists("config.json")) { log.LogInformation("Config file {A} not found, creating a new one", "config.json"); @@ -96,10 +97,19 @@ static void RunProgram() var json = JsonConvert.SerializeObject(jobOptions, Formatting.Indented, jsonSettings); File.WriteAllText("config.json", json); } - log.LogInformation("reading config file {A}", "config.json"); - var readJobOptions = JsonConvert.DeserializeObject>(File.ReadAllText("config.json"), jsonSettings); - //List Jobs = new List(); + log.LogInformation("reading config file {A}", "config.json"); + Dictionary readJobOptions=new Dictionary(); + try + { + readJobOptions = JsonConvert.DeserializeObject>(File.ReadAllText("config.json"), jsonSettings); + } + catch (Exception exc) + { + log.LogCritical(exc, "exception reading config file {A}", "config.json"); + return; + } + foreach (var jobOption in readJobOptions) { From 389379b7b5774f3ddd9b519dee21376210af1b11 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 09:48:04 +0100 Subject: [PATCH 2/8] Initialize MainForm with args in Program.cs --- FileSyncAppWin/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FileSyncAppWin/Program.cs b/FileSyncAppWin/Program.cs index 8056c59..66f8b9f 100644 --- a/FileSyncAppWin/Program.cs +++ b/FileSyncAppWin/Program.cs @@ -52,7 +52,7 @@ static void Main(string[] args) Application.SetUnhandledExceptionMode(UnhandledExceptionMode.CatchException); AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException; ApplicationConfiguration.Initialize(); - mainForm = new MainForm(); + mainForm = new MainForm(args); Application.Run(mainForm); } catch (Exception exception) From 67d05df0cffae232c31cc435f241478377e2bc06 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 09:52:24 +0100 Subject: [PATCH 3/8] fix date initialization to correctly set January 1, 1970 --- FileSyncLibNet/SyncProviders/AbstractProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FileSyncLibNet/SyncProviders/AbstractProvider.cs b/FileSyncLibNet/SyncProviders/AbstractProvider.cs index d0c77bd..5cf095e 100644 --- a/FileSyncLibNet/SyncProviders/AbstractProvider.cs +++ b/FileSyncLibNet/SyncProviders/AbstractProvider.cs @@ -58,11 +58,11 @@ public override void SyncSourceToDest() bool createDestinationDir = true; int copied = 0; int skipped = 0; - DateTimeOffset minimumLastWriteTime = new DateTimeOffset(1970, 0, 0, 0, 0, 0, TimeSpan.Zero); + DateTimeOffset minimumLastWriteTime = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero); if (jobOptions.RememberLastSync) { if (LastRun.ToUnixTimeMilliseconds() == 0) - LastRun = jobOptions.MaxAge < jobOptions.Interval ? new DateTimeOffset(1970, 0, 0, 0, 0, 0, TimeSpan.Zero) : DateTimeOffset.Now - jobOptions.MaxAge; + LastRun = jobOptions.MaxAge < jobOptions.Interval ? new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero) : DateTimeOffset.Now - jobOptions.MaxAge; minimumLastWriteTime = LastRun - jobOptions.Interval - jobOptions.Interval; } else From 0850fbc070c239ac0ca0959a6b12d8164c1f066b Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 09:52:51 +0100 Subject: [PATCH 4/8] add string[] args to MainForm --- FileSyncAppWin/MainForm.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FileSyncAppWin/MainForm.cs b/FileSyncAppWin/MainForm.cs index cf7eeab..faec8ca 100644 --- a/FileSyncAppWin/MainForm.cs +++ b/FileSyncAppWin/MainForm.cs @@ -4,13 +4,13 @@ public partial class MainForm : Form { Thread consoleThread; - public MainForm() + public MainForm(string[] args) { InitializeComponent(); this.FormClosing += (s, e) => { FileSyncApp.Program.keepRunning = false; consoleThread?.Join(10_000); }; consoleThread = new Thread(() => { - FileSyncApp.Program.Main(null); + FileSyncApp.Program.Main(args); }); FileSyncApp.Program.JobsReady += (s, e) => { From 106e8dec4cd9ef648bca215702a678f1cd298493 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 13:44:50 +0100 Subject: [PATCH 5/8] Add olderFiles param to GetFiles methods --- .../AccessProviders/FileIoAccessProvider.cs | 14 ++++++++------ FileSyncLibNet/AccessProviders/IAccessProvider.cs | 2 +- .../AccessProviders/ScpAccessProvider.cs | 10 +++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs b/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs index a1bfb60..9bb4f0a 100644 --- a/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs +++ b/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs @@ -17,7 +17,7 @@ public FileIoAccessProvider(ILogger logger, string stateFilename) this.logger = logger; if (!string.IsNullOrEmpty(stateFilename)) remoteState = new RemoteState(stateFilename); - + } public void UpdateAccessPath(string accessPath) { @@ -31,7 +31,7 @@ public void CreateDirectory(string path) public FileInfo2 GetFileInfo(string path) { - if(null!=remoteState) + if (null != remoteState) return remoteState.GetFileInfo(Path.Combine(AccessPath, path)); var fi = new FileInfo(Path.Combine(AccessPath, path)); @@ -43,7 +43,7 @@ public FileInfo2 GetFileInfo(string path) }; } - public List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null) + public List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null, bool olderFiles = false) { DirectoryInfo _di = new DirectoryInfo(AccessPath); List ret_val = new List(); @@ -57,10 +57,12 @@ public List GetFiles(DateTime minimumLastWriteTime, string pattern, s searchPattern: pattern, searchOption: recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly); - - ret_val.AddRange(_fi.Select(x => new FileInfo2(x.FullName.Substring(AccessPath.Length + 1), x.Exists) { Length = x.Length, LastWriteTime = x.LastWriteTime }).ToList()); + ret_val.AddRange(_fi.Where(x => (olderFiles ? x.LastWriteTime <= minimumLastWriteTime : x.LastWriteTime >= minimumLastWriteTime)) + .Select(x => new FileInfo2(x.FullName.Substring(AccessPath.Length + 1), x.Exists) { Length = x.Length, LastWriteTime = x.LastWriteTime }) + .ToList()); } - catch (Exception exc) { + catch (Exception exc) + { logger?.LogError(exc, "exception in GetFiles for path {A}, dir {B}", path, dir); } } diff --git a/FileSyncLibNet/AccessProviders/IAccessProvider.cs b/FileSyncLibNet/AccessProviders/IAccessProvider.cs index 11aad77..90055fc 100644 --- a/FileSyncLibNet/AccessProviders/IAccessProvider.cs +++ b/FileSyncLibNet/AccessProviders/IAccessProvider.cs @@ -9,7 +9,7 @@ internal interface IAccessProvider { void CreateDirectory(string path); void UpdateAccessPath(string path); - List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null); + List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null, bool olderFiles=false); FileInfo2 GetFileInfo(string path); void Delete(FileInfo2 fileInfo); Stream GetStream(FileInfo2 file); diff --git a/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs b/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs index 3eb5ff9..d747df9 100644 --- a/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs +++ b/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs @@ -17,7 +17,7 @@ internal class ScpAccessProvider : IAccessProvider SftpClient ftpClient; private readonly RemoteState remoteState; public string AccessPath { get; private set; } - + private readonly ILogger logger; public ScpAccessProvider(NetworkCredential credentials, ILogger logger, string stateFilename) { @@ -137,7 +137,7 @@ public bool MatchesPattern(string fileName, string pattern) return Regex.IsMatch(fileName, regexPattern, RegexOptions.IgnoreCase); } - public List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null) + public List GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List subfolders = null, bool olderFiles = false) { //add try catch for non existent folders EnsureConnected(); @@ -156,7 +156,7 @@ public List GetFiles(DateTime minimumLastWriteTime, string pattern, s try { var files = ftpClient.ListDirectory(basePath); - var sepChar="/"; + var sepChar = "/"; if (recursive) { foreach (var folder in files.Where(x => x.IsDirectory && !(x.Name == ".") && !(x.Name == ".."))) @@ -165,8 +165,8 @@ public List GetFiles(DateTime minimumLastWriteTime, string pattern, s ret_val.AddRange(GetFiles(minimumLastWriteTime, pattern, subPath, recursive)); } } - ret_val.AddRange(files.Where(x => MatchesPattern(x.Name, pattern)).Where(x => x.LastWriteTime >= minimumLastWriteTime && !x.IsDirectory).Select(x => - new FileInfo2($"{(AccessPath.Length + 1 MatchesPattern(x.Name, pattern)).Where(x => (olderFiles ? x.LastWriteTime <= minimumLastWriteTime : x.LastWriteTime >= minimumLastWriteTime) && !x.IsDirectory).Select(x => + new FileInfo2($"{(AccessPath.Length + 1 < basePath.Length ? (basePath.Substring(AccessPath.Length + 1)) + sepChar : string.Empty)}{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList()); } catch (Exception exc) { From 68ef57b824a8057e805e46276199e204dffe9309 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 13:45:27 +0100 Subject: [PATCH 6/8] Remove RememberRemoteState --- FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs | 1 - FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs | 6 +----- FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs b/FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs index f2aae4b..a8bd0b8 100644 --- a/FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs +++ b/FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs @@ -14,7 +14,6 @@ public class FileSyncJobOptions : FileJobOptionsBase, IFileSyncJobOptions public bool SyncDeleted { get; set; } = false; public bool DeleteSourceAfterBackup { get; set; } = false; public bool RememberLastSync { get; set; } = true; - public bool RememberRemoteState { get; set; } = false; public TimeSpan MaxAge { get; set; } public FileSyncJobOptions() diff --git a/FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs b/FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs index b94680a..ae72e69 100644 --- a/FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs +++ b/FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs @@ -58,11 +58,7 @@ public IFileSyncJobOptionsBuilderSetProperties RememberLastSync(bool rememberLas jobOptions.RememberLastSync = rememberLastSync; return this; } - public IFileSyncJobOptionsBuilderSetProperties RememberRemoteState(bool rememberRemoteState) - { - jobOptions.RememberRemoteState = rememberRemoteState; - return this; - } + public IFileSyncJobOptionsBuilderSetProperties WithMaxAge(TimeSpan maxAge) { jobOptions.MaxAge = maxAge; diff --git a/FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs b/FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs index e51ee75..dd1e13a 100644 --- a/FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs +++ b/FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs @@ -9,7 +9,6 @@ public interface IFileSyncJobOptions : IFileJobOptions bool DeleteSourceAfterBackup { get; set; } bool SyncDeleted { get; set; } bool RememberLastSync { get; set; } - bool RememberRemoteState { get; set; } TimeSpan MaxAge { get; set; } } } \ No newline at end of file From a0e538245d4c83c8df3b719c961894d3cf862f42 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 13:46:04 +0100 Subject: [PATCH 7/8] implement clean job handling in AbstractProvider --- .../SyncProviders/AbstractProvider.cs | 70 +++++++++++++++---- 1 file changed, 58 insertions(+), 12 deletions(-) diff --git a/FileSyncLibNet/SyncProviders/AbstractProvider.cs b/FileSyncLibNet/SyncProviders/AbstractProvider.cs index 5cf095e..943e8e0 100644 --- a/FileSyncLibNet/SyncProviders/AbstractProvider.cs +++ b/FileSyncLibNet/SyncProviders/AbstractProvider.cs @@ -1,5 +1,6 @@ using FileSyncLibNet.AccessProviders; using FileSyncLibNet.Commons; +using FileSyncLibNet.FileCleanJob; using FileSyncLibNet.FileSyncJob; using Microsoft.Extensions.Logging; using System; @@ -14,32 +15,77 @@ internal class AbstractProvider : ProviderBase public AbstractProvider(IFileJobOptions jobOptions) : base(jobOptions) { - if (!(JobOptions is IFileSyncJobOptions syncJobOptions)) - throw new ArgumentException("this instance has no information about syncing files, it has type " + JobOptions.GetType().ToString()); - if (syncJobOptions.SourcePath.StartsWith("scp:")) + if (!(JobOptions is IFileCleanJobOptions) && !(JobOptions is IFileSyncJobOptions)) { - SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename: null); + throw new ArgumentException("this instance has no information about syncing files or cleaning files, it has type " + JobOptions.GetType().ToString()); } - else + string stateFilename = null; + if (JobOptions is IFileSyncJobOptions syncJobOptions) { - SourceAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename: null); + if (syncJobOptions.SourcePath.StartsWith("scp:")) + { + SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, JobOptions.Logger, stateFilename: null); + } + else + { + SourceAccess = new FileIoAccessProvider(JobOptions.Logger, stateFilename: null); + } + SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath); + stateFilename = null; } - SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath); - string stateFilename = syncJobOptions.RememberRemoteState ? syncJobOptions.GetHashedName() : null; - if (syncJobOptions.DestinationPath.StartsWith("scp:")) + if (JobOptions.DestinationPath.StartsWith("scp:")) { - DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename); + DestinationAccess = new ScpAccessProvider(JobOptions.Credentials, JobOptions.Logger, stateFilename); } else { - DestinationAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename); + DestinationAccess = new FileIoAccessProvider(JobOptions.Logger, stateFilename); } } public override void DeleteFiles() { - throw new NotImplementedException(); + //Use from file io provider and adapt to use access provider + + if (!(JobOptions is IFileCleanJobOptions jobOptions)) + throw new ArgumentException("this instance has no information about deleting files, it has type " + JobOptions.GetType().ToString()); + try + { + string formattedDestinationPath = string.Format(jobOptions.DestinationPath, DateTime.Now); + DestinationAccess.UpdateAccessPath(formattedDestinationPath); + } + catch (Exception ex) { logger?.LogError(ex, "exception formatting destination accesspath with DateTime.Now as 0 arg {A}", jobOptions.DestinationPath); } + + try + { + var minimumLastWriteTime = DateTimeOffset.Now - jobOptions.MaxAge; + var sourceFiles = DestinationAccess.GetFiles(minimumLastWriteTime: minimumLastWriteTime.DateTime, + pattern: JobOptions.SearchPattern, + recursive: JobOptions.Recursive, + subfolders: JobOptions.Subfolders, + olderFiles: true); + int fileCount = 0; + foreach (var fileToDelete in sourceFiles) + { + logger.LogDebug("deleting file {A}", fileToDelete.Name); + + try + { + DestinationAccess.Delete(fileToDelete); + + fileCount++; + } + catch (Exception ex) { logger.LogError(ex, "exception deleting file {A}", fileToDelete.Name); } + logger.LogDebug("deleted {A} files", fileCount); + } + + + } + catch (Exception exc) + { + logger.LogError(exc, "TimerCleanup_Tick threw an exception"); + } } public override void SyncSourceToDest() From a241cf33a3cab0ee001e32dfcdc2269f74bd2270 Mon Sep 17 00:00:00 2001 From: Andreas Kueffel Date: Wed, 15 Jan 2025 14:31:10 +0100 Subject: [PATCH 8/8] track job completion and handle initial full sync --- FileSyncApp/Program.cs | 9 ++++++--- FileSyncLibNet/FileSyncJob/FileSyncJob.cs | 1 + FileSyncLibNet/SyncProviders/AbstractProvider.cs | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/FileSyncApp/Program.cs b/FileSyncApp/Program.cs index dbce2f1..ab27a16 100644 --- a/FileSyncApp/Program.cs +++ b/FileSyncApp/Program.cs @@ -99,7 +99,7 @@ static void RunProgram() } log.LogInformation("reading config file {A}", "config.json"); - Dictionary readJobOptions=new Dictionary(); + Dictionary readJobOptions = new Dictionary(); try { readJobOptions = JsonConvert.DeserializeObject>(File.ReadAllText("config.json"), jsonSettings); @@ -109,17 +109,20 @@ static void RunProgram() log.LogCritical(exc, "exception reading config file {A}", "config.json"); return; } - + foreach (var jobOption in readJobOptions) { - jobOption.Value.Logger = LoggerFactory.CreateLogger(jobOption.Key); Jobs.Add(jobOption.Key, FileSyncJob.CreateJob(jobOption.Value)); } JobsReady?.Invoke(null, EventArgs.Empty); + Dictionary jobsDone = new Dictionary(); foreach (var job in Jobs) { + jobsDone.Add(job.Value, false); + job.Value.JobFinished += (s, e) => { jobsDone[(IFileJob)s] = true; if (jobsDone.All(x => x.Value)) if (!File.Exists("fullsync.done")) File.Create("fullsync.done"); }; job.Value.StartJob(); + } log.LogInformation("Press Ctrl+C to exit"); while (keepRunning) diff --git a/FileSyncLibNet/FileSyncJob/FileSyncJob.cs b/FileSyncLibNet/FileSyncJob/FileSyncJob.cs index 7a9441b..967fd9b 100644 --- a/FileSyncLibNet/FileSyncJob/FileSyncJob.cs +++ b/FileSyncLibNet/FileSyncJob/FileSyncJob.cs @@ -18,6 +18,7 @@ public class FileSyncJob : IFileJob private readonly Timer timer; private readonly ISyncProvider syncProvider; private volatile bool v_jobRunning = false; + public static bool InitialFullSync { get; set; } = false; private FileSyncJob(IFileJobOptions fileSyncJobOptions) { diff --git a/FileSyncLibNet/SyncProviders/AbstractProvider.cs b/FileSyncLibNet/SyncProviders/AbstractProvider.cs index 943e8e0..eb5388b 100644 --- a/FileSyncLibNet/SyncProviders/AbstractProvider.cs +++ b/FileSyncLibNet/SyncProviders/AbstractProvider.cs @@ -117,6 +117,11 @@ public override void SyncSourceToDest() DateTimeOffset.MinValue : DateTimeOffset.Now - jobOptions.MaxAge - jobOptions.Interval; } + if (!System.IO.File.Exists("fullsync.done")) + { + logger.LogWarning("fullsync.done not found, syncing all files for initial run"); + minimumLastWriteTime = DateTimeOffset.MinValue; + } bool error_occured = false; try