added Full Discoveries to give the endpoint a complete Update

Author: CodeShellDev

.github/templates/README.template.md

@@ -56,7 +56,22 @@ If no Key is provided ServDiscovery will leave out the Authorization Header.
 
 ### DISCOVERY_INTERVAL
 
-The Discovery Interval sets the Interval of which ServDiscovery will update the Endpoint, etc.
+The Discovery Interval sets the Interval (in seconds) of which ServDiscovery will update the provided Endpoint.
+
+Default: `60`
+
+### FULL_DISCOVERY_INTERVAL
+
+Sets the Interval for Full Disoveries.
+This tells ServDiscovery to send a Full Update of all the activate Containers in the `added` JSON Key.
+
+> [!IMPORTANT]
+> Must be set to a **fraction of** DISCOVERY_INTERVAL.
+> (example: `DISCOVERY_INTERVAL * 2`)
+> `FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL` must result in an int.
+> `FULL_DISCOVERY_INTERVAL` must be bigger than `DISCOVERY_INTERVAL`.
+
+Default: **Disabled**
 
 ## Contributing
 

.gitignore

@@ -1,2 +1,3 @@
 .env
-.venv
+.venv
+*.lua

app.py

@@ -29,6 +29,9 @@
 ENDPOINT_KEY = os.getenv("ENDPOINT_KEY")
 
 DISCOVERY_INTERVAL = os.getenv("DISCOVERY_INTERVAL")
+FULL_DISCOVERY_INTERVAL = os.getenv("FULL_DISCOVERY_INTERVAL")
+
+fullDiscoveryRatio = 0
 
 dockerClient = DockerClient(base_url="unix://var/run/docker.sock")
 
@@ -60,19 +63,25 @@ def getHostsFromLabels(labels):
 
     return hosts
 
-def updateEnabledContainers():
+def getEnabledContainers():
+    newContainers = []
+
+    try:
+        newContainers = dockerClient.containers.list(filters={"label": "discovery.enable=true"})
+    except Exception as e:
+        logger.error(f"Error fetching containers: {str(e)}")
+
+    return newContainers
+
+def getDiscovery():
     diffs = {
         "removed": [],
         "added": []
     }
 
     global containers
-    newContainers = []
 
-    try:
-        newContainers = dockerClient.containers.list(filters={"label": "discovery.enable=true"})
-    except Exception as e:
-        logger.error(f"Error fetching containers: {str(e)}")
+    newContainers = getEnabledContainers()
 
     containerDiff = getDiff(containers, newContainers)
 
@@ -124,6 +133,23 @@ def updateEnabledContainers():
 
     return diffs
 
+def getFullDiscovery():
+    diffs = {
+        "removed": [],
+        "added": []
+    }
+
+    newContainers = getEnabledContainers()
+
+    logger.info(f"Found {newContainers.__len__()} Containers")
+
+    for container in newContainers:
+        hosts = getHostsFromLabels(container.labels)
+
+        diffs.get("added").extend(hosts)
+
+    return diffs
+
 def cleanDiff(diff):
     removed, added = set(diff.get("removed")), set(diff.get("added"))
 
@@ -163,21 +189,49 @@ def sendDiffToEndpoint(diff):
 #   thread = threading.Thread(target=main, daemon=True)
 #   thread.start()
 
+def doDiscovery():
+    logger.debug("Starting Discovery")
+
+    globalDiff = getDiscovery()
+
+    logger.debug("Cleaning Diff")
+
+    return cleanDiff(globalDiff)
+
+def doFullDiscovery():
+    logger.debug("Starting Full Discovery")
+
+    globalDiff = getFullDiscovery()
+
+    return globalDiff
+
 def main():
+    i = 0
     while True:
-        logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...")
+        isFullDiscovery = False
 
-        sleep(DISCOVERY_INTERVAL)
-        
-        logger.info("Starting Discovery")
+        if i == fullDiscoveryRatio:
+            isFullDiscovery = True
+            i = 0
 
-        globalDiff = updateEnabledContainers()
+        if not isFullDiscovery:
+            logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...")
+        else:
+            logger.info(f"Starting Full Discover in {DISCOVERY_INTERVAL}...")
 
-        logger.debug("Cleaning Diff")
+        sleep(DISCOVERY_INTERVAL)
 
-        globalDiff = cleanDiff(globalDiff)
+        globalDiff = {
+            "removed": [],
+            "added": []
+        }
 
-        # Check if there is actually any Diff
+        if not isFullDiscovery:
+            globalDiff = doDiscovery()
+        else:
+            globalDiff = doFullDiscovery()
+
+        # Check if there are any Changes
         if globalDiff.get("removed").__len__() + globalDiff.get("added").__len__() <= 0:
             logger.debug("No Changes were made, skipping...")
             continue
@@ -188,6 +242,8 @@ def main():
 
         logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}")
 
+        i += 1
+
 if __name__ == '__main__':
     logger.setLevel(level=LOG_LEVEL)
 
@@ -203,6 +259,17 @@ def main():
         logger.warning(f"No DISCOVERY_INTERVAL set, using 30sec as default")
         DISCOVERY_INTERVAL = 30
 
+    if not FULL_DISCOVERY_INTERVAL:
+        logger.warning(f"No FULL_DISCOVERY_INTERVAL set, disregarding Full Discoveries")
+        FULL_DISCOVERY_INTERVAL = 0
+    elif FULL_DISCOVERY_INTERVAL % DISCOVERY_INTERVAL != 0:
+        logger.warning(f"FULL_DISCOVERY_INTERVAL is not a valid Integer, rounding to nearest Int")
+        FULL_DISCOVERY_INTERVAL = round(FULL_DISCOVERY_INTERVAL)
+
+    if FULL_DISCOVERY_INTERVAL != 0:
+        fullDiscoveryRatio = FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL
+        logger.debug(f"FULL_DISCOVERY_INTERVAL ratio set to {fullDiscoveryRatio}")
+
     if not ENDPOINT_KEY or ENDPOINT_KEY == "":
         logger.warning(f"No ENDPOINT_KEY set, requests may be denied")
 

docker-compose.yaml

@@ -5,6 +5,7 @@ services:
     environment:
       ENDPOINT: https://mydomain.com/ENDPOINT
       ENDPOINT_KEY: MY_VERY_SECURE_KEY
+      ALIVE_UPDATE_INTERVAL: 60
       SERVER_NAME: server-1
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock