From f03515d0306b1f7a7e87433c1085a5dbea27ae29 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 2 Mar 2019 18:36:31 +0000 Subject: [PATCH 1/3] Make the DataReporterSettingBase class abstract --- .../DataReporter/DataReporterSettingsBase.cs | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/CumulusMX.Extensions/DataReporter/DataReporterSettingsBase.cs b/CumulusMX.Extensions/DataReporter/DataReporterSettingsBase.cs index df123e24..fe941e9d 100644 --- a/CumulusMX.Extensions/DataReporter/DataReporterSettingsBase.cs +++ b/CumulusMX.Extensions/DataReporter/DataReporterSettingsBase.cs @@ -2,32 +2,17 @@ namespace CumulusMX.Extensions.DataReporter { - public class DataReporterSettingsBase : IDataReporterSettings + public abstract class DataReporterSettingsBase : IDataReporterSettings { - public int GetValue(string key, int defaultValue) - { - throw new System.NotImplementedException(); - } + public abstract int GetValue(string key, int defaultValue); - public string GetValue(string key, string defaultValue) - { - throw new System.NotImplementedException(); - } + public abstract string GetValue(string key, string defaultValue); - public double GetValue(string key, double defaultValue) - { - throw new System.NotImplementedException(); - } + public abstract double GetValue(string key, double defaultValue); - public byte[] GetValue(string key, byte[] defaultValue) - { - throw new System.NotImplementedException(); - } + public abstract byte[] GetValue(string key, byte[] defaultValue); - public bool GetValue(string key, bool defaultValue) - { - throw new System.NotImplementedException(); - } + public abstract bool GetValue(string key, bool defaultValue); public Setting this[string key] => new Setting(GetValue(key,string.Empty)); } From 1be4e4b629d0da8cf52a110533acb32c94791a24 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 2 Mar 2019 22:50:18 +0000 Subject: [PATCH 2/3] Make DataReporterSettings GetValue method override - for compatibility with abstract base class. --- CumulusMX/DataReporter/DataReporterSettings.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CumulusMX/DataReporter/DataReporterSettings.cs b/CumulusMX/DataReporter/DataReporterSettings.cs index de07cb58..6197f739 100644 --- a/CumulusMX/DataReporter/DataReporterSettings.cs +++ b/CumulusMX/DataReporter/DataReporterSettings.cs @@ -17,22 +17,27 @@ internal DataReporterSettings(IniFile sourceFile, string sectionName) _sectionName = sectionName; } - public int GetValue(string key, int defaultValue) + public override int GetValue(string key, int defaultValue) { return _sourceFile.GetValue(_sectionName, key, defaultValue); } - public string GetValue(string key, string defaultValue) + public override string GetValue(string key, string defaultValue) { return _sourceFile.GetValue(_sectionName, key, defaultValue); } - public double GetValue(string key, double defaultValue) + public override double GetValue(string key, double defaultValue) { return _sourceFile.GetValue(_sectionName, key, defaultValue); } - public byte[] GetValue(string key, byte[] defaultValue) + public override byte[] GetValue(string key, byte[] defaultValue) + { + return _sourceFile.GetValue(_sectionName, key, defaultValue); + } + + public override bool GetValue(string key, bool defaultValue) { return _sourceFile.GetValue(_sectionName, key, defaultValue); } From 6ff8f3d91bd5a2d574c169a41e6a57964064f256 Mon Sep 17 00:00:00 2001 From: David Baker Date: Sat, 2 Mar 2019 23:00:25 +0000 Subject: [PATCH 3/3] Concrete implementation of the IWeatherDataStatistic and IStatistic classes. Desparately need unit tests. --- CumulusMX.Extensions/Station/IStatistic.cs | 2 +- .../Station/IWeatherDataStatistics.cs | 2 +- CumulusMX/CumulusMX.csproj | 1 + CumulusMX/Data/MaxMinAverageUnit.cs | 119 +++++++++++ CumulusMX/Data/StatisticUnit.cs | 185 ++++++++++++++++++ CumulusMX/Data/WeatherDataStatistics.cs | 28 +++ 6 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 CumulusMX/Data/MaxMinAverageUnit.cs create mode 100644 CumulusMX/Data/StatisticUnit.cs create mode 100644 CumulusMX/Data/WeatherDataStatistics.cs diff --git a/CumulusMX.Extensions/Station/IStatistic.cs b/CumulusMX.Extensions/Station/IStatistic.cs index fac92ccb..6b06934f 100644 --- a/CumulusMX.Extensions/Station/IStatistic.cs +++ b/CumulusMX.Extensions/Station/IStatistic.cs @@ -7,7 +7,7 @@ namespace CumulusMX.Extensions.Station { public interface IStatistic { - void Add(TBase sample); + void Add(DateTime timestamp, TBase sample); TBase Latest { get; } TBase OneHourMaximum { get; } TBase ThreeHourMaximum { get; } diff --git a/CumulusMX.Extensions/Station/IWeatherDataStatistics.cs b/CumulusMX.Extensions/Station/IWeatherDataStatistics.cs index caaffb3a..3212c6cf 100644 --- a/CumulusMX.Extensions/Station/IWeatherDataStatistics.cs +++ b/CumulusMX.Extensions/Station/IWeatherDataStatistics.cs @@ -20,7 +20,7 @@ public interface IWeatherDataStatistics IStatistic RainRate { get; set; } IStatistic Rain { get; set; } IStatistic SolarRadiation { get; set; } - IStatistic UvIndex { get; set; } + IStatistic UvIndex { get; set; } Dictionary> Extra { get; set; } diff --git a/CumulusMX/CumulusMX.csproj b/CumulusMX/CumulusMX.csproj index 383930fc..fd6e8c16 100644 --- a/CumulusMX/CumulusMX.csproj +++ b/CumulusMX/CumulusMX.csproj @@ -3,6 +3,7 @@ Exe netcoreapp2.2 + 7.3 diff --git a/CumulusMX/Data/MaxMinAverageUnit.cs b/CumulusMX/Data/MaxMinAverageUnit.cs new file mode 100644 index 00000000..3f80c6a3 --- /dev/null +++ b/CumulusMX/Data/MaxMinAverageUnit.cs @@ -0,0 +1,119 @@ +using System; +using System.Dynamic; +using Autofac.Core.Activators.Reflection; +using UnitsNet; + +namespace CumulusMX.Data +{ + internal class MaxMinAverageUnit + where TUnitType : Enum where TBase : IComparable, IQuantity + { + private int _count = 0; + private int _nonZero = 0; + private TBase _minimum; + private TBase _maximum; + + public TBase Minimum + { + get + { + if (_count == 0) + throw new InvalidOperationException("An empty set has no minimum."); + return _minimum; + } + private set + { + _minimum = value; + } + + } + + public TBase Maximum + { + get + { + if (_count == 0) + throw new InvalidOperationException("An empty set has no maximum."); + return _maximum; + } + private set + { + _maximum = value; + } + + } + + public TBase Average + { + get + { + if (_count == 0) + throw new DivideByZeroException("Cannot get the average of 0 values."); + if (!_averageValid) CalculateAverage(); + return _average; + } + } + + public TBase Total + { + get + { + if (!_unitTotalValid) CreateTotal(); + return _unitTotal; + } + } + + public int NonZero => _nonZero; + + private TBase _average; + private bool _averageValid = false; + private double _total; + private TBase _unitTotal; + private bool _unitTotalValid = false; + + private readonly TUnitType _itemOne; + private readonly TBase _zeroQuantity; + + public MaxMinAverageUnit() + { + _itemOne = (TUnitType) Enum.ToObject(typeof(TUnitType), 1); + _zeroQuantity = (TBase)Activator.CreateInstance(typeof(TBase), 0, _itemOne); + Reset(); + } + + public void Reset() + { + _count = 0; + _total = 0; + _averageValid = false; + _unitTotal = _zeroQuantity; + _unitTotalValid = true; + _nonZero = 0; + } + + public void AddValue(TBase newValue) + { + _count++; + _total += newValue.As(_itemOne); + if (newValue.CompareTo(Maximum) > 0 || _count == 1) Maximum = newValue; + if (newValue.CompareTo(Minimum) < 0 || _count == 1) Minimum = newValue; + _averageValid = false; + _unitTotalValid = false; + if (newValue.CompareTo(_zeroQuantity) != 0) + _nonZero++; + } + + private void CalculateAverage() + { + double avgValue = _total / _count; + _average = (TBase) Activator.CreateInstance(typeof(TBase), avgValue, _itemOne); + _averageValid = true; + } + + private void CreateTotal() + { + _unitTotal = (TBase)Activator.CreateInstance(typeof(TBase), _total, _itemOne); + _unitTotalValid = true; + } + } +} \ No newline at end of file diff --git a/CumulusMX/Data/StatisticUnit.cs b/CumulusMX/Data/StatisticUnit.cs new file mode 100644 index 00000000..79282ba3 --- /dev/null +++ b/CumulusMX/Data/StatisticUnit.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using CumulusMX.Extensions.Station; +using UnitsNet; +using Unosquare.Swan; + +namespace CumulusMX.Data +{ + public class StatisticUnit : IStatistic where TBase : IComparable, IQuantity where TUnitType : Enum + { + private Dictionary _sampleHistory = new Dictionary(); + private DateTime _lastSampleTime = DateTime.Now; + private TBase _lastValue; + private MaxMinAverageUnit _day; + private MaxMinAverageUnit _yesterday; + private MaxMinAverageUnit _month; + private MaxMinAverageUnit _lastMonth; + private MaxMinAverageUnit _year; + private MaxMinAverageUnit _lastYear; + private double _oneHourChange; + private double _threeHourChange; + private double _24HourTotal; + private TBase _oneHourMaximum; + private TBase _threeHourMaximum; + + private TimeSpan _dayNonZero = TimeSpan.Zero; + private TimeSpan _monthNonZero = TimeSpan.Zero; + private TimeSpan _yearNonZero = TimeSpan.Zero; + + private readonly TUnitType _itemOne; + private readonly TBase _zeroQuantity; + + public StatisticUnit() + { + _itemOne = (TUnitType)Enum.ToObject(typeof(TUnitType), 1); + _zeroQuantity = (TBase)Activator.CreateInstance(typeof(TBase), 0, _itemOne); + + } + + /// + /// Adds a new sample to the statistics set, and updates aggregates. This will be called a lot, it needs to be fast. + /// + /// The DateTime when the observation was taken. + /// The observed value. + public void Add(DateTime timestamp,TBase sample) + { + if (timestamp.Year > _lastSampleTime.Year) + ResetYearValues(); + if (timestamp.Year * 12 + timestamp.Month > _lastSampleTime.Year * 12 + _lastSampleTime.Month) + ResetMonthValues(); + if (timestamp.DayOfYear != _lastSampleTime.DayOfYear) + ResetDayValues(); + + _sampleHistory.Add(timestamp,sample); + _lastValue = sample; + + _day.AddValue(sample); + _month.AddValue(sample); + _year.AddValue(sample); + + // Update rolling 24 hour total + var rolledOff = _sampleHistory + .Where(x => x.Key >= _lastSampleTime.AddHours(-24) && x.Key < timestamp.AddHours(-24)); + + foreach (var oldValue in rolledOff) + { + _24HourTotal -= oldValue.Value.As(_itemOne); + } + _24HourTotal += sample.As(_itemOne); + + UpdateMaximumAndChange(3, sample, _lastSampleTime, timestamp, ref _threeHourChange, ref _threeHourMaximum); + UpdateMaximumAndChange(1, sample, _lastSampleTime, timestamp, ref _oneHourChange, ref _oneHourMaximum); + + if (sample.CompareTo(_zeroQuantity) != 0) + { + var newSpan = timestamp - _lastSampleTime; + _dayNonZero += newSpan; + _monthNonZero += newSpan; + _yearNonZero += newSpan; + } + + _lastSampleTime = timestamp; + } + + private void UpdateMaximumAndChange(int months, TBase sample, DateTime lastSample, DateTime thisSample, ref double change, ref TBase maximum) + { + var oldSamples = _sampleHistory.Where(x => x.Key >= lastSample.AddHours(-1 * months)).OrderBy(x => x.Key); + if (!oldSamples.Any()) + return; + + change = sample.As(_itemOne) - oldSamples.First().Value.As(_itemOne); + + var rolledOff = _sampleHistory + .Where(x => x.Key >= lastSample.AddHours(-1*months) && x.Key < thisSample.AddHours(-1*months)); + + if (sample.CompareTo(maximum) > 0) + maximum = sample; + else + { + bool maxInvalid = false; + foreach (var oldValue in rolledOff) + { + if (oldValue.Value.Equals(maximum)) + maxInvalid = true; + } + + if (!maxInvalid) return; + + var historyWindow = _sampleHistory + .Where(x => x.Key > thisSample.AddHours(-1 * months) && x.Key <= thisSample); + maximum = sample; + foreach (var oldSample in historyWindow) + { + if (oldSample.Value.CompareTo(maximum) > 0) + maximum = oldSample.Value; + } + } + } + + private void ResetDayValues() + { + _yesterday = _day.DeepClone(); + _day.Reset(); + _dayNonZero = TimeSpan.Zero; + } + + private void ResetMonthValues() + { + _lastMonth = _month.DeepClone(); + _month.Reset(); + _monthNonZero = TimeSpan.Zero; + } + + private void ResetYearValues() + { + _lastYear = _year.DeepClone(); + _year.Reset(); + _yearNonZero = TimeSpan.Zero; + } + + public TBase Latest => _lastValue; + + public TBase OneHourMaximum => _oneHourMaximum; + + public TBase ThreeHourMaximum => _threeHourMaximum; + + public TBase OneHourChange => (TBase)Activator.CreateInstance(typeof(TBase), _oneHourChange, _itemOne); + + public TBase ThreeHourChange => (TBase)Activator.CreateInstance(typeof(TBase), _threeHourChange, _itemOne); + + public TBase DayMaximum => _day.Maximum; + + public TBase DayMinimum => _day.Minimum; + + public TBase DayAverage => _day.Average; + + public TBase MonthMaximum => _month.Maximum; + + public TBase MonthMinimum => _month.Minimum; + + public TBase MonthAverage => _month.Average; + + public TBase YearMaximum => _year.Maximum; + + public TBase YearMinimum => _year.Minimum; + + public TBase YearAverage => _year.Average; + + public TBase Last24hTotal => (TBase)Activator.CreateInstance(typeof(TBase), _24HourTotal, _itemOne); + + public TBase DayTotal => _day.Total; + + public TBase MonthTotal => _month.Total; + + public TBase YearTotal => _year.Total; + + public TimeSpan DayNonZero => _dayNonZero; + + public TimeSpan MonthNonZero => _monthNonZero; + + public TimeSpan YearNonZero => _yearNonZero; + } +} diff --git a/CumulusMX/Data/WeatherDataStatistics.cs b/CumulusMX/Data/WeatherDataStatistics.cs new file mode 100644 index 00000000..6637ec1c --- /dev/null +++ b/CumulusMX/Data/WeatherDataStatistics.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; +using CumulusMX.Extensions.Station; +using UnitsNet; +using UnitsNet.Units; + +namespace CumulusMX.Data +{ + public class WeatherDataStatistics : IWeatherDataStatistics + { + public IStatistic IndoorTemperature { get; set; } = new StatisticUnit(); + public IStatistic OutdoorTemperature { get; set; } = new StatisticUnit(); + public IStatistic IndoorHumidity { get; set; } = new StatisticUnit(); + public IStatistic OutdoorHumidity { get; set; } = new StatisticUnit(); + public IStatistic WindGust { get; set; } = new StatisticUnit(); + public IStatistic WindSpeed { get; set; } = new StatisticUnit(); + public IStatistic WindBearing { get; set; } = new StatisticUnit(); + public IStatistic Pressure { get; set; } = new StatisticUnit(); + public IStatistic AltimeterPressure { get; set; } = new StatisticUnit(); + public IStatistic OutdoorDewpoint { get; set; } = new StatisticUnit(); + public IStatistic RainRate { get; set; } = new StatisticUnit(); + public IStatistic Rain { get; set; } = new StatisticUnit(); + public IStatistic SolarRadiation { get; set; } = new StatisticUnit(); + public IStatistic UvIndex { get; set; } + public Dictionary> Extra { get; set; } = new Dictionary>(); + } +}