fix: Changed fluentd config to prevent excess logging from the daemonset pods (#174)

* fix: Changed fluentd config to prevent excess logging from the daemonset pods

* Fix formatting in systemd conf

Author: bmonkman

templates/kubernetes/terraform/modules/kubernetes/logging/cloudwatch/files/host.conf

@@ -66,4 +66,4 @@
         retry_forever true
     </buffer>
     </match>
-</label>
+</label>

templates/kubernetes/terraform/modules/kubernetes/logging/cloudwatch/files/systemd.conf

@@ -2,13 +2,17 @@
     @type systemd
     @id in_systemd_kubelet
     @label @systemd
-    filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+    matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
     <entry>
-    field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
-    field_map_strict true
+      field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
+      field_map_strict true
     </entry>
+    <storage>
+      @type local
+      persistent true
+      path /var/log/fluentd-journald-kubelet.pos
+    </storage>
     path /var/log/journal
-    pos_file /var/log/fluentd-journald-kubelet.pos
     read_from_head true
     tag kubelet.service
 </source>
@@ -17,13 +21,17 @@
     @type systemd
     @id in_systemd_kubeproxy
     @label @systemd
-    filters [{ "_SYSTEMD_UNIT": "kubeproxy.service" }]
+    matches [{ "_SYSTEMD_UNIT": "kubeproxy.service" }]
     <entry>
-    field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
-    field_map_strict true
+        field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
+        field_map_strict true
     </entry>
+    <storage>
+      @type local
+      persistent true
+      path /var/log/fluentd-journald-kubeproxy.pos
+    </storage>
     path /var/log/journal
-    pos_file /var/log/fluentd-journald-kubeproxy.pos
     read_from_head true
     tag kubeproxy.service
 </source>
@@ -32,13 +40,17 @@
     @type systemd
     @id in_systemd_docker
     @label @systemd
-    filters [{ "_SYSTEMD_UNIT": "docker.service" }]
+    matches [{ "_SYSTEMD_UNIT": "docker.service" }]
     <entry>
-    field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
-    field_map_strict true
+        field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"}
+        field_map_strict true
     </entry>
+    <storage>
+      @type local
+      persistent true
+      path /var/log/fluentd-journald-docker.pos
+    </storage>
     path /var/log/journal
-    pos_file /var/log/fluentd-journald-docker.pos
     read_from_head true
     tag docker.service
 </source>
@@ -72,4 +84,4 @@
         retry_forever true
     </buffer>
     </match>
-</label>
+</label>

templates/kubernetes/terraform/modules/kubernetes/logging/cloudwatch/fluentd.tf

@@ -28,8 +28,8 @@ resource "kubernetes_config_map" "cluster_info" {
 
 resource "kubernetes_service_account" "fluentd" {
   metadata {
-    name        = "fluentd"
-    namespace   = "amazon-cloudwatch"
+    name      = "fluentd"
+    namespace = "amazon-cloudwatch"
     annotations = {
       "eks.amazonaws.com/role-arn" = module.iam_assumable_role_fluentd.this_iam_role_arn
     }
@@ -178,6 +178,10 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" {
               }
             }
           }
+          env {
+            name  = "FLUENTD_OPT"
+            value = "-q"
+          }
           resources {
             limits {
               memory = "200Mi"
@@ -217,7 +221,7 @@ resource "kubernetes_daemonset" "fluentd_cloudwatch" {
         }
 
         security_context {
-          fs_group =  65534
+          fs_group = 65534
         }
 
         termination_grace_period_seconds = 30