changes to table schema only happen when user specifies option now (#38)

* changes to table schema only happen when user specifies option now

* updated unit tests to reflect new adx option

Author: namita-prakash

Source/RealTimeKql/CommandLineParsing/CommandLineParser.cs

@@ -486,10 +486,10 @@ private void GetAllSubcommands()
                 "Azure Data Explorer (ADX) database name. eg, --adxdatabase=TestDb", true),
                 new Option("adxtable", "atb",
                 "Azure Data Explorer (ADX) table name. eg, --adxtable=OutputTable", true),
-                new Option("adxresettable", "art",
-                "The existing data in the destination table is dropped before new data is logged.", false, true),
+                new Option("adxcreatereset", "acr",
+                "If table doesn't exist, it is created. If table exists, data in table is dropped before new data is logged. eg, --adxcreatereset", false, true),
                 new Option("adxdirectingest", "adi",
-                "Default upload to ADX is using queued ingest. Use this option to do a direct ingest to ADX.", false, true)
+                "Default upload to ADX is using queued ingest. Use this option to do a direct ingest to ADX. eg, --adxdirectingest", false, true)
             };
 
             var adx = new Subcommand("adx", "Ingest output to Azure Data Explorer", null, adxOptions);

Source/RealTimeKql/Program.cs

@@ -116,7 +116,7 @@ static AdxOutput GetAdxOutput(List<Option> opts)
             string cluster = "";
             string database = "";
             string table = "";
-            bool resetTable = false;
+            bool createOrResetTable = false;
             bool directIngest = false;
 
             foreach(var opt in opts)
@@ -144,8 +144,8 @@ static AdxOutput GetAdxOutput(List<Option> opts)
                     case "adxdirectingest":
                         directIngest = opt.WasSet;
                         break;
-                    case "adxresettable":
-                        resetTable = opt.WasSet;
+                    case "adxcreatereset":
+                        createOrResetTable = opt.WasSet;
                         break;
                 }
             }
@@ -157,7 +157,7 @@ static AdxOutput GetAdxOutput(List<Option> opts)
                 cluster,
                 database,
                 table,
-                resetTable,
+                createOrResetTable,
                 directIngest);
         }
 

Source/RealTimeKqlLibrary/Output/AdxOutput.cs

@@ -16,7 +16,7 @@ public class AdxOutput : IOutput
         public AutoResetEvent Completed { get; private set; }
 
         private readonly string _table;
-        private readonly bool _resetTable;
+        private readonly bool _createOrResetTable;
 
         private readonly KustoConnectionStringBuilder kscbAdmin;
         private readonly KustoConnectionStringBuilder kscbIngest;
@@ -40,11 +40,11 @@ public AdxOutput(
             string cluster,
             string database,
             string table,
-            bool resetTable=false,
+            bool createOrResetTable=false,
             bool directIngest=false)
         {
             _table = table;
-            _resetTable = resetTable;
+            _createOrResetTable = createOrResetTable;
 
             _batchSize = 10000;
             _flushDuration = TimeSpan.FromMilliseconds(5);
@@ -104,7 +104,7 @@ public void OutputAction(IDictionary<string, object> obj)
                 _fields = obj.Keys.ToArray();
             }
 
-            if (kscbAdmin != null && _initializeTable == false)
+            if (kscbAdmin != null && _initializeTable == false && _createOrResetTable == true)
             {
                 CreateOrResetTable(obj);
                 _initializeTable = true;
@@ -196,12 +196,8 @@ private void CreateOrResetTable(IDictionary<string, object> value)
         {
             using (var admin = KustoClientFactory.CreateCslAdminProvider(kscbAdmin))
             {
-                if (_resetTable)
-                {
-                    string dropTable = CslCommandGenerator.GenerateTableDropCommand(_table, true);
-                    admin.ExecuteControlCommand(dropTable);
-                }
-
+                string dropTable = CslCommandGenerator.GenerateTableDropCommand(_table, true);
+                admin.ExecuteControlCommand(dropTable);
                 CreateMergeKustoTable(admin, value);
             }
         }

Source/RealTimeKqlTests/CommandLineParserTest.cs

@@ -68,7 +68,7 @@ public void Parse_ValidOutputs_ReturnTrue(params string[] args)
 
         [Theory]
         [InlineData("etw", "tcp", "json", "file.json")]
-        [InlineData("etw", "tcp", "adx", "-ad=test.com", "-aclid=val", "-akey=val", "-acl=cluster", "-adb=database", "-atb=table", "-art", "-adi")]
+        [InlineData("etw", "tcp", "adx", "-ad=test.com", "-aclid=val", "-akey=val", "-acl=cluster", "-adb=database", "-atb=table", "-acr", "-adi")]
         public void Parse_ValidOutputsWithOptions_ReturnTrue(params string[] args)
         {
             var c = new CommandLineParser(args);