remove rememberremotestate and fix maxage handling, implement deleting for abstract provider

FileSyncApp/Program.cs

@@ -25,7 +25,7 @@ public class Program
         public static Dictionary<string, IFileJob> Jobs = new Dictionary<string, IFileJob>();
         public static void Main(string[] args)
         {
-            ConfigureLogger(args.FirstOrDefault());
+            ConfigureLogger(args?.FirstOrDefault());
             log = LoggerFactory.CreateLogger("FileSyncAppMain");
             if (null != args && args.Length > 0)
             {
@@ -50,6 +50,7 @@ static void RunProgram()
                 ContractResolver = new IgnorePropertyResolver(new string[] { "Logger" }),
                 TypeNameHandling = TypeNameHandling.Auto,
             };
+
             if (!File.Exists("config.json"))
             {
                 log.LogInformation("Config file {A} not found, creating a new one", "config.json");
@@ -96,20 +97,32 @@ static void RunProgram()
                 var json = JsonConvert.SerializeObject(jobOptions, Formatting.Indented, jsonSettings);
                 File.WriteAllText("config.json", json);
             }
+
             log.LogInformation("reading config file {A}", "config.json");
-            var readJobOptions = JsonConvert.DeserializeObject<Dictionary<string, IFileJobOptions>>(File.ReadAllText("config.json"), jsonSettings);
-            //List<IFileJob> Jobs = new List<IFileJob>();
+            Dictionary<string, IFileJobOptions> readJobOptions = new Dictionary<string, IFileJobOptions>();
+            try
+            {
+                readJobOptions = JsonConvert.DeserializeObject<Dictionary<string, IFileJobOptions>>(File.ReadAllText("config.json"), jsonSettings);
+            }
+            catch (Exception exc)
+            {
+                log.LogCritical(exc, "exception reading config file {A}", "config.json");
+                return;
+            }
 
             foreach (var jobOption in readJobOptions)
             {
-
                 jobOption.Value.Logger = LoggerFactory.CreateLogger(jobOption.Key);
                 Jobs.Add(jobOption.Key, FileSyncJob.CreateJob(jobOption.Value));
             }
             JobsReady?.Invoke(null, EventArgs.Empty);
+            Dictionary<IFileJob, bool> jobsDone = new Dictionary<IFileJob, bool>();
             foreach (var job in Jobs)
             {
+                jobsDone.Add(job.Value, false);
+                job.Value.JobFinished += (s, e) => { jobsDone[(IFileJob)s] = true; if (jobsDone.All(x => x.Value)) if (!File.Exists("fullsync.done")) File.Create("fullsync.done"); };
                 job.Value.StartJob();
+
             }
             log.LogInformation("Press Ctrl+C to exit");
             while (keepRunning)

FileSyncAppWin/MainForm.cs

@@ -4,13 +4,13 @@ public partial class MainForm : Form
     {
         Thread consoleThread;
 
-        public MainForm()
+        public MainForm(string[] args)
         {
             InitializeComponent();
             this.FormClosing += (s, e) => { FileSyncApp.Program.keepRunning = false; consoleThread?.Join(10_000); };
             consoleThread = new Thread(() =>
             {
-                FileSyncApp.Program.Main(null);
+                FileSyncApp.Program.Main(args);
             });
             FileSyncApp.Program.JobsReady += (s, e) =>
             {

FileSyncAppWin/Program.cs

@@ -52,7 +52,7 @@ static void Main(string[] args)
                 Application.SetUnhandledExceptionMode(UnhandledExceptionMode.CatchException);
                 AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
                 ApplicationConfiguration.Initialize();
-                mainForm = new MainForm();
+                mainForm = new MainForm(args);
                 Application.Run(mainForm);
             }
             catch (Exception exception)

FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs

@@ -17,7 +17,7 @@ public FileIoAccessProvider(ILogger logger, string stateFilename)
             this.logger = logger;
             if (!string.IsNullOrEmpty(stateFilename))
                 remoteState = new RemoteState(stateFilename);
-            
+
         }
         public void UpdateAccessPath(string accessPath)
         {
@@ -31,7 +31,7 @@ public void CreateDirectory(string path)
 
         public FileInfo2 GetFileInfo(string path)
         {
-            if(null!=remoteState)
+            if (null != remoteState)
                 return remoteState.GetFileInfo(Path.Combine(AccessPath, path));
 
             var fi = new FileInfo(Path.Combine(AccessPath, path));
@@ -43,7 +43,7 @@ public FileInfo2 GetFileInfo(string path)
             };
         }
 
-        public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null)
+        public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null, bool olderFiles = false)
         {
             DirectoryInfo _di = new DirectoryInfo(AccessPath);
             List<FileInfo2> ret_val = new List<FileInfo2>();
@@ -57,10 +57,12 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s
                         searchPattern: pattern,
                         searchOption: recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly);
 
-
-                    ret_val.AddRange(_fi.Select(x => new FileInfo2(x.FullName.Substring(AccessPath.Length + 1), x.Exists) { Length = x.Length, LastWriteTime = x.LastWriteTime }).ToList());
+                    ret_val.AddRange(_fi.Where(x => (olderFiles ? x.LastWriteTime <= minimumLastWriteTime : x.LastWriteTime >= minimumLastWriteTime))
+                        .Select(x => new FileInfo2(x.FullName.Substring(AccessPath.Length + 1), x.Exists) { Length = x.Length, LastWriteTime = x.LastWriteTime })
+                        .ToList());
                 }
-                catch (Exception exc) {
+                catch (Exception exc)
+                {
                     logger?.LogError(exc, "exception in GetFiles for path {A}, dir {B}", path, dir);
                 }
             }

FileSyncLibNet/AccessProviders/IAccessProvider.cs

@@ -9,7 +9,7 @@ internal interface IAccessProvider
     {
         void CreateDirectory(string path);
         void UpdateAccessPath(string path);
-        List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null);
+        List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null, bool olderFiles=false);
         FileInfo2 GetFileInfo(string path);
         void Delete(FileInfo2 fileInfo);
         Stream GetStream(FileInfo2 file);

FileSyncLibNet/AccessProviders/ScpAccessProvider.cs

@@ -17,7 +17,7 @@ internal class ScpAccessProvider : IAccessProvider
         SftpClient ftpClient;
         private readonly RemoteState remoteState;
         public string AccessPath { get; private set; }
-        
+
         private readonly ILogger logger;
         public ScpAccessProvider(NetworkCredential credentials, ILogger logger, string stateFilename)
         {
@@ -137,7 +137,7 @@ public bool MatchesPattern(string fileName, string pattern)
             return Regex.IsMatch(fileName, regexPattern, RegexOptions.IgnoreCase);
         }
 
-        public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null)
+        public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, string path = null, bool recursive = false, List<string> subfolders = null, bool olderFiles = false)
         {
             //add try catch for non existent folders
             EnsureConnected();
@@ -156,7 +156,7 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s
                 try
                 {
                     var files = ftpClient.ListDirectory(basePath);
-                    var sepChar="/";
+                    var sepChar = "/";
                     if (recursive)
                     {
                         foreach (var folder in files.Where(x => x.IsDirectory && !(x.Name == ".") && !(x.Name == "..")))
@@ -165,8 +165,8 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s
                             ret_val.AddRange(GetFiles(minimumLastWriteTime, pattern, subPath, recursive));
                         }
                     }
-                    ret_val.AddRange(files.Where(x => MatchesPattern(x.Name, pattern)).Where(x => x.LastWriteTime >= minimumLastWriteTime && !x.IsDirectory).Select(x =>
-                    new FileInfo2($"{(AccessPath.Length + 1<basePath.Length?(basePath.Substring(AccessPath.Length+1))+sepChar:string.Empty)}{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());
+                    ret_val.AddRange(files.Where(x => MatchesPattern(x.Name, pattern)).Where(x => (olderFiles ? x.LastWriteTime <= minimumLastWriteTime : x.LastWriteTime >= minimumLastWriteTime) && !x.IsDirectory).Select(x =>
+                    new FileInfo2($"{(AccessPath.Length + 1 < basePath.Length ? (basePath.Substring(AccessPath.Length + 1)) + sepChar : string.Empty)}{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());
                 }
                 catch (Exception exc)
                 {

FileSyncLibNet/FileSyncJob/FileSyncJob.cs

@@ -18,6 +18,7 @@ public class FileSyncJob : IFileJob
         private readonly Timer timer;
         private readonly ISyncProvider syncProvider;
         private volatile bool v_jobRunning = false;
+        public static bool InitialFullSync { get; set; } = false;
 
         private FileSyncJob(IFileJobOptions fileSyncJobOptions)
         {

FileSyncLibNet/FileSyncJob/FileSyncJobOptions.cs

@@ -14,7 +14,6 @@ public class FileSyncJobOptions : FileJobOptionsBase, IFileSyncJobOptions
         public bool SyncDeleted { get; set; } = false;
         public bool DeleteSourceAfterBackup { get; set; } = false;
         public bool RememberLastSync { get; set; } = true;
-        public bool RememberRemoteState { get; set; } = false;
         public TimeSpan MaxAge { get; set; }
 
         public FileSyncJobOptions()

FileSyncLibNet/FileSyncJob/FileSyncJobOptionsBuilder.cs

@@ -58,11 +58,7 @@ public IFileSyncJobOptionsBuilderSetProperties RememberLastSync(bool rememberLas
             jobOptions.RememberLastSync = rememberLastSync;
             return this;
         }
-        public IFileSyncJobOptionsBuilderSetProperties RememberRemoteState(bool rememberRemoteState)
-        {
-            jobOptions.RememberRemoteState = rememberRemoteState;
-            return this;
-        }
+
         public IFileSyncJobOptionsBuilderSetProperties WithMaxAge(TimeSpan maxAge)
         {
             jobOptions.MaxAge = maxAge;

FileSyncLibNet/FileSyncJob/IFileSyncJobOptions.cs

@@ -9,7 +9,6 @@ public interface IFileSyncJobOptions : IFileJobOptions
         bool DeleteSourceAfterBackup { get; set; }
         bool SyncDeleted { get; set; }
         bool RememberLastSync { get; set; }
-        bool RememberRemoteState { get; set; }
         TimeSpan MaxAge { get; set; }
     }
 }

FileSyncLibNet/SyncProviders/AbstractProvider.cs

@@ -1,5 +1,6 @@
 using FileSyncLibNet.AccessProviders;
 using FileSyncLibNet.Commons;
+using FileSyncLibNet.FileCleanJob;
 using FileSyncLibNet.FileSyncJob;
 using Microsoft.Extensions.Logging;
 using System;
@@ -14,32 +15,77 @@ internal class AbstractProvider : ProviderBase
 
         public AbstractProvider(IFileJobOptions jobOptions) : base(jobOptions)
         {
-            if (!(JobOptions is IFileSyncJobOptions syncJobOptions))
-                throw new ArgumentException("this instance has no information about syncing files, it has type " + JobOptions.GetType().ToString());
-            if (syncJobOptions.SourcePath.StartsWith("scp:"))
+            if (!(JobOptions is IFileCleanJobOptions) && !(JobOptions is IFileSyncJobOptions))
             {
-                SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename: null);
+                throw new ArgumentException("this instance has no information about syncing files or cleaning files, it has type " + JobOptions.GetType().ToString());
             }
-            else
+            string stateFilename = null;
+            if (JobOptions is IFileSyncJobOptions syncJobOptions)
             {
-                SourceAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename: null);
+                if (syncJobOptions.SourcePath.StartsWith("scp:"))
+                {
+                    SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, JobOptions.Logger, stateFilename: null);
+                }
+                else
+                {
+                    SourceAccess = new FileIoAccessProvider(JobOptions.Logger, stateFilename: null);
+                }
+                SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath);
+                stateFilename = null;
             }
-            SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath);
-            string stateFilename = syncJobOptions.RememberRemoteState ? syncJobOptions.GetHashedName() : null;
-            if (syncJobOptions.DestinationPath.StartsWith("scp:"))
+            if (JobOptions.DestinationPath.StartsWith("scp:"))
             {
-                DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename);
+                DestinationAccess = new ScpAccessProvider(JobOptions.Credentials, JobOptions.Logger, stateFilename);
             }
             else
             {
-                DestinationAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename);
+                DestinationAccess = new FileIoAccessProvider(JobOptions.Logger, stateFilename);
             }
 
         }
 
         public override void DeleteFiles()
         {
-            throw new NotImplementedException();
+            //Use from file io provider and adapt to use access provider
+
+            if (!(JobOptions is IFileCleanJobOptions jobOptions))
+                throw new ArgumentException("this instance has no information about deleting files, it has type " + JobOptions.GetType().ToString());
+            try
+            {
+                string formattedDestinationPath = string.Format(jobOptions.DestinationPath, DateTime.Now);
+                DestinationAccess.UpdateAccessPath(formattedDestinationPath);
+            }
+            catch (Exception ex) { logger?.LogError(ex, "exception formatting destination accesspath with DateTime.Now as 0 arg {A}", jobOptions.DestinationPath); }
+
+            try
+            {
+                var minimumLastWriteTime = DateTimeOffset.Now - jobOptions.MaxAge;
+                var sourceFiles = DestinationAccess.GetFiles(minimumLastWriteTime: minimumLastWriteTime.DateTime,
+                                                       pattern: JobOptions.SearchPattern,
+                                                       recursive: JobOptions.Recursive,
+                                                       subfolders: JobOptions.Subfolders,
+                                                       olderFiles: true);
+                int fileCount = 0;
+                foreach (var fileToDelete in sourceFiles)
+                {
+                    logger.LogDebug("deleting file {A}", fileToDelete.Name);
+
+                    try
+                    {
+                        DestinationAccess.Delete(fileToDelete);
+
+                        fileCount++;
+                    }
+                    catch (Exception ex) { logger.LogError(ex, "exception deleting file {A}", fileToDelete.Name); }
+                    logger.LogDebug("deleted {A} files", fileCount);
+                }
+
+
+            }
+            catch (Exception exc)
+            {
+                logger.LogError(exc, "TimerCleanup_Tick threw an exception");
+            }
         }
 
         public override void SyncSourceToDest()
@@ -58,11 +104,11 @@ public override void SyncSourceToDest()
             bool createDestinationDir = true;
             int copied = 0;
             int skipped = 0;
-            DateTimeOffset minimumLastWriteTime = new DateTimeOffset(1970, 0, 0, 0, 0, 0, TimeSpan.Zero);
+            DateTimeOffset minimumLastWriteTime = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero);
             if (jobOptions.RememberLastSync)
             {
                 if (LastRun.ToUnixTimeMilliseconds() == 0)
-                    LastRun = jobOptions.MaxAge < jobOptions.Interval ? new DateTimeOffset(1970, 0, 0, 0, 0, 0, TimeSpan.Zero) : DateTimeOffset.Now - jobOptions.MaxAge;
+                    LastRun = jobOptions.MaxAge < jobOptions.Interval ? new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero) : DateTimeOffset.Now - jobOptions.MaxAge;
                 minimumLastWriteTime = LastRun - jobOptions.Interval - jobOptions.Interval;
             }
             else
@@ -71,6 +117,11 @@ public override void SyncSourceToDest()
                     DateTimeOffset.MinValue :
                     DateTimeOffset.Now - jobOptions.MaxAge - jobOptions.Interval;
             }
+            if (!System.IO.File.Exists("fullsync.done"))
+            {
+                logger.LogWarning("fullsync.done not found, syncing all files for initial run");
+                minimumLastWriteTime = DateTimeOffset.MinValue;
+            }
 
             bool error_occured = false;
             try