diff --git a/deploy/connectivityserver.yaml b/deploy/connectivityserver.yaml index 8388d37..c957b45 100644 --- a/deploy/connectivityserver.yaml +++ b/deploy/connectivityserver.yaml @@ -54,16 +54,25 @@ spec: requests: memory: 32Mi cpu: 100m + startupProbe: + httpGet: + path: /ready + port: http + failureThreshold: 30 + periodSeconds: 10 livenessProbe: - tcpSocket: + httpGet: + path: /live port: http initialDelaySeconds: 15 periodSeconds: 20 readinessProbe: - tcpSocket: + httpGet: + path: /ready port: http - initialDelaySeconds: 15 - periodSeconds: 20 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: diff --git a/src/connectivityserver/connectionflask.py b/src/connectivityserver/connectionflask.py index b164662..16ea84c 100755 --- a/src/connectivityserver/connectionflask.py +++ b/src/connectivityserver/connectionflask.py @@ -17,309 +17,372 @@ from flask import Flask, abort, make_response, request # Some functions exit with an abort(NNN) instead of return so don't complain! -#ruff: noqa RET503 +# ruff: noqa RET503 -partitions={} -partlock=Lock() +partitions = {} +partlock = Lock() -if 'CONNECTION_FLASK_DEBUG' in os.environ: - debug_level=int(os.environ['CONNECTION_FLASK_DEBUG']) +if "CONNECTION_FLASK_DEBUG" in os.environ: + debug_level = int(os.environ["CONNECTION_FLASK_DEBUG"]) else: - debug_level=1 + debug_level = 1 + def convert_log_level(log_level): - if log_level == 0: - return logging.WARNING - if log_level == 1: + if log_level == 0: + return logging.WARNING + if log_level == 1: + return logging.INFO + if log_level == 2: + return logging.DEBUG return logging.INFO - if log_level == 2: - return logging.DEBUG - return logging.INFO -logging.basicConfig(level=convert_log_level(debug_level), format='%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s') + +logging.basicConfig( + level=convert_log_level(debug_level), + format="%(asctime)s %(levelname)s %(filename)s:%(funcName)s:%(lineno)d %(message)s", +) log = logging.getLogger(__name__) -if 'ENTRY_TTL' in os.environ: - ttl=int(os.environ['ENTRY_TTL']) +if "ENTRY_TTL" in os.environ: + ttl = int(os.environ["ENTRY_TTL"]) else: - ttl=10 -entry_ttl=timedelta(seconds=ttl) + ttl = 10 +entry_ttl = timedelta(seconds=ttl) + +last_stats = datetime.now() +npublishes = 0 +nlookups = 0 +lookup_time = timedelta(0) +publish_time = timedelta() +maxpartitions = 0 +maxentries = {} + +app = Flask(__name__) +appstarted = False + + +@app.before_first_request +def mark_started(): + global appstarted + appstarted = True -last_stats=datetime.now() -npublishes=0 -nlookups=0 -lookup_time=timedelta(0) -publish_time=timedelta() -maxpartitions=0 -maxentries={} -app=Flask(__name__) +@app.route("/live", methods=["GET"]) +def live(): + return "OK", 200 + + +@app.route("/ready", methods=["GET"]) +def ready(): + if appstarted: + return "OK", 200 + return "Not Ready", 503 + @app.route("/") def dump(): - now=datetime.now() - dstream=StringIO() - dstream.write('

Dump of configuration dictionary

') - dstream.write("

Active partitions

") - if len(partitions)>0: - pad=' style="padding-left: 1em;padding-right: 1em"' - dstream.write(f'' - f'Partition' - f'Entries') - for p in partitions: - dstream.write(f'{p}' - f'{len(partitions[p])}') - dstream.write("
") - dstream.write(f'

Partitions

') - for p in partitions: - store=partitions[p] - dstream.write(f'

{p}

') - dstream.write(f'' - f'' - f'Name' - f'Connection' - f'' - f'' - f'uri' - f'data_type' - f'capacity' - f'connection_type' - f'time' - f'') - format_cell=lambda value,strike: f'{value}' if strike else f'{value}' - for k,v in store.items(): - expired = now-v.time >= entry_ttl - dstream.write(f'{format_cell(k,expired)}' - f'{format_cell(v.uri,expired)}' - f'{format_cell(v.data_type,expired)}' - f'{format_cell(v.capacity,expired)}' - f'{format_cell(v.connection_type,expired)}' - f'{format_cell(v.time,expired)}') - dstream.write("
") - else: - dstream.write("None

") - dstream.write("

Server statistics

") - stats_to_html(dstream) - dstream.seek(0) - return dstream.read() + now = datetime.now() + dstream = StringIO() + dstream.write("

Dump of configuration dictionary

") + dstream.write("

Active partitions

") + if len(partitions) > 0: + pad = ' style="padding-left: 1em;padding-right: 1em"' + dstream.write( + f'' + f'Partition' + f"Entries" + ) + for p in partitions: + dstream.write( + f"{p}{len(partitions[p])}" + ) + dstream.write("
") + dstream.write(f"

Partitions

") + for p in partitions: + store = partitions[p] + dstream.write(f"

{p}

") + dstream.write( + f'' + f'' + f'Name' + f'Connection' + f"" + f'' + f"uri" + f"data_type" + f"capacity" + f"connection_type" + f"time" + f"" + ) + format_cell = lambda value, strike: ( + f'{value}' if strike else f"{value}" + ) + for k, v in store.items(): + expired = now - v.time >= entry_ttl + dstream.write( + f"{format_cell(k, expired)}" + f"{format_cell(v.uri, expired)}" + f"{format_cell(v.data_type, expired)}" + f"{format_cell(v.capacity, expired)}" + f"{format_cell(v.connection_type, expired)}" + f"{format_cell(v.time, expired)}" + ) + dstream.write("
") + else: + dstream.write("None

") + dstream.write("

Server statistics

") + stats_to_html(dstream) + dstream.seek(0) + return dstream.read() + def stats_to_html(dstream): - dstream.write(f"

Since {last_stats}

") - if npublishes>0: - avg_pub=publish_time/npublishes - else: - avg_pub=timedelta() - dstream.write(f"

{npublishes} calls to publish in total time {publish_time} " - f"(average {avg_pub.microseconds} µs per call)

") - if nlookups>0: - avg_lookup=lookup_time/nlookups - else: - avg_lookup=timedelta() - dstream.write(f"

{nlookups} calls to lookup in total time {lookup_time} " - f"(average {avg_lookup.microseconds} µs per call)

") - dstream.write(f"

Maximum number of partitions active = {maxpartitions}

") - for part in maxentries: - dstream.write(f"

Maximum entries in partition {part} = {maxentries[part]}

") + dstream.write(f"

Since {last_stats}

") + if npublishes > 0: + avg_pub = publish_time / npublishes + else: + avg_pub = timedelta() + dstream.write( + f"

{npublishes} calls to publish in total time {publish_time} " + f"(average {avg_pub.microseconds} µs per call)

" + ) + if nlookups > 0: + avg_lookup = lookup_time / nlookups + else: + avg_lookup = timedelta() + dstream.write( + f"

{nlookups} calls to lookup in total time {lookup_time} " + f"(average {avg_lookup.microseconds} µs per call)

" + ) + dstream.write(f"

Maximum number of partitions active = {maxpartitions}

") + for part in maxentries: + dstream.write( + f"

Maximum entries in partition {part} = {maxentries[part]}

" + ) + @app.route("/stats") def dumpStats(): - dstream=StringIO() - dstream.write('

Connection server statistics

') - stats_to_html(dstream) - dstream.seek(0) - return dstream.read() + dstream = StringIO() + dstream.write("

Connection server statistics

") + stats_to_html(dstream) + dstream.seek(0) + return dstream.read() + @app.route("/resetStats") def resetStats(): - stats = dumpStats() + stats = dumpStats() + + global \ + last_stats, \ + npublishes, \ + nlookups, \ + lookup_time, \ + publish_time, \ + maxpartitions, \ + maxentries - global last_stats,npublishes,nlookups,lookup_time,publish_time,maxpartitions,maxentries + last_stats = datetime.now() + npublishes = 0 + nlookups = 0 + lookup_time = timedelta(0) + publish_time = timedelta() + maxpartitions = 0 + maxentries = {} - last_stats=datetime.now() - npublishes=0 - nlookups=0 - lookup_time=timedelta(0) - publish_time=timedelta() - maxpartitions=0 - maxentries={} + return stats - return stats @app.route("/resetService") def reset(): - global partitions - partitions={} - return resetStats() + global partitions + partitions = {} + return resetStats() -@app.route("/publish",methods=['POST']) +@app.route("/publish", methods=["POST"]) def publish(): - # Store multiple connection ids and corresponding uris in a - # dictionary associated with the appropriate partition. - timestamp=datetime.now() - js=json.loads(request.data) - - log.debug(f"{js=}") - part=js['partition'] - - log.info( - f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..." - ) - - partlock.acquire() - if part in partitions: - store=partitions[part] - else: - store={} - partitions[part]=store - global maxpartitions - if len(partitions)>maxpartitions: - maxpartitions=len(partitions) - if part not in maxentries: - maxentries[part]=0 - - Connection=namedtuple( - 'Connection',['uri','data_type','capacity','connection_type','time']) - - for connection in js['connections']: - - if 'uid' in connection and 'uri' in connection: - uid=connection['uid'] - now=datetime.now() - log.info(f"uid={uid}") - - store[uid]=Connection( - uri=connection['uri'], - connection_type=connection['connection_type'], - data_type=connection['data_type'], - capacity=connection['capacity'] if 'capacity' in connection else 0, # Backwards compatibility - time=timestamp - ) - - now=datetime.now() - elapsed=now-timestamp - - log.debug(f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections") - - global npublishes, publish_time - publish_time+=elapsed - npublishes+=1 - - if len(store)>maxentries[part]: - maxentries[part]=len(store) - - partlock.release() - return 'OK' - -@app.route("/retract-partition",methods=['POST']) -def retract_partition(): - if len(request.data) == 0: - abort(400) + # Store multiple connection ids and corresponding uris in a + # dictionary associated with the appropriate partition. + timestamp = datetime.now() + js = json.loads(request.data) - js=json.loads(request.data) - log.debug(f"request=[{js}]") + log.debug(f"{js=}") + part = js["partition"] - if 'partition' not in js: - abort(400) + log.info( + f"{len(js['connections'])} connections in partition {part} from {request.remote_addr} uri={js['connections'][0]['uri']}..." + ) - part=js['partition'] - partlock.acquire() + partlock.acquire() + if part in partitions: + store = partitions[part] + else: + store = {} + partitions[part] = store + global maxpartitions + if len(partitions) > maxpartitions: + maxpartitions = len(partitions) + if part not in maxentries: + maxentries[part] = 0 + + Connection = namedtuple( + "Connection", ["uri", "data_type", "capacity", "connection_type", "time"] + ) - if part in partitions: - partitions.pop(part) - partlock.release() - return 'OK' - partlock.release() - abort(404) + for connection in js["connections"]: + if "uid" in connection and "uri" in connection: + uid = connection["uid"] + now = datetime.now() + log.info(f"uid={uid}") + + store[uid] = Connection( + uri=connection["uri"], + connection_type=connection["connection_type"], + data_type=connection["data_type"], + capacity=( + connection["capacity"] if "capacity" in connection else 0 + ), # Backwards compatibility + time=timestamp, + ) + + now = datetime.now() + elapsed = now - timestamp + + log.debug( + f"Took {elapsed.microseconds} us to add {len(js['connections'])} connections" + ) + + global npublishes, publish_time + publish_time += elapsed + npublishes += 1 + + if len(store) > maxentries[part]: + maxentries[part] = len(store) -@app.route("/retract",methods=['POST']) -def retract(): - if len(request.data) == 0: - abort(400) - - js=json.loads(request.data) - good=True - part=js['partition'] - partlock.acquire() - if part not in partitions: partlock.release() - return make_response(f"Partition {part} not found", 404) - - store=partitions[part] - if 'connections' not in js: - abort(400) - for con in js['connections']: - id=con['connection_id'] - if id in store: - store.pop(id) - else: - log.info(f"Could not find connection_id <{id}>") - good=False - if len(store)==0: - # We've deleted the last entry in this partition so delete the - # partition as well - partitions.pop(part) - partlock.release() - if good: - return 'OK' - abort(404) - -@app.route("/getconnection/",methods=['POST','GET']) -def get_connection(part): - if len(request.data) == 0: - abort(400) + return "OK" - # Find connection uris that correspond to the connection id pattern - # in the request. The pattern is treated as a regular expression. - now=datetime.now() - js=json.loads(request.data) - log.debug(f"{js=}") +@app.route("/retract-partition", methods=["POST"]) +def retract_partition(): + if len(request.data) == 0: + abort(400) - if 'uid_regex' in js and 'data_type' in js: - log.info( - f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}" - ) + js = json.loads(request.data) + log.debug(f"request=[{js}]") + + if "partition" not in js: + abort(400) - result=[] - regex=re.compile(js['uid_regex']) - dt=js['data_type'] + part = js["partition"] partlock.acquire() if part in partitions: - store=partitions[part] - matched=[] - for uid,con in store.items(): - if regex.fullmatch(uid) and con.data_type==dt and now-con.time") + good = False + if len(store) == 0: + # We've deleted the last entry in this partition so delete the + # partition as well + partitions.pop(part) partlock.release() - log.info(f"Partition {part} not found") + if good: + return "OK" abort(404) - else: - abort(400) + +@app.route("/getconnection/", methods=["POST", "GET"]) +def get_connection(part): + if len(request.data) == 0: + abort(400) + + # Find connection uris that correspond to the connection id pattern + # in the request. The pattern is treated as a regular expression. + + now = datetime.now() + js = json.loads(request.data) + log.debug(f"{js=}") + + if "uid_regex" in js and "data_type" in js: + log.info( + f"Searching for connections matching uid_regex<{js['uid_regex']}> and data_type {js['data_type']}" + ) + + result = [] + regex = re.compile(js["uid_regex"]) + dt = js["data_type"] + partlock.acquire() + + if part in partitions: + store = partitions[part] + matched = [] + for uid, con in store.items(): + if ( + regex.fullmatch(uid) + and con.data_type == dt + and now - con.time < entry_ttl + ): + matched.append((uid, con)) + partlock.release() + # We should now be able to construct JSON string while other threads + # have access to the partition dict + for uid, con in matched: + result.append( + "{" + f'"uid":"{uid}",' + f'"uri":"{con.uri}",' + f'"connection_type":{con.connection_type},' + f'"data_type":"{con.data_type}",' + f'"capacity":{con.capacity}' + "}" + ) + + td = datetime.now() - now + log.debug( + f"Lookup took {td.microseconds} us to find {len(result)} connections" + ) + global nlookups, lookup_time + # Should we have the lock while updating statistics? It doesn't + # really matter if the stats aren't entirely accurate. + nlookups += 1 + lookup_time += td + + return "[" + ",".join(result) + "]" + + partlock.release() + log.info(f"Partition {part} not found") + abort(404) + else: + abort(400) diff --git a/tests/test_connectivityservice.py b/tests/test_connectivityservice.py index ec31065..7c8e4a7 100644 --- a/tests/test_connectivityservice.py +++ b/tests/test_connectivityservice.py @@ -3,22 +3,22 @@ import connectivityserver.connectionflask as cf + @pytest.fixture() def app(): - yield cf.app + yield cf.app @pytest.fixture() def client(app): return app.test_client() + @pytest.fixture() def runner(app): return app.test_cli_runner() - - con = json.loads("""{ "connections":[ { @@ -38,100 +38,116 @@ def runner(app): }""") - def test_noconnection(client): - resp = client.get("/getconnection/bad/con") - assert resp.status_code == 404 + resp = client.get("/getconnection/bad/con") + assert resp.status_code == 404 + def test_publish(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 + resp = client.post("/publish", json=con) + assert resp.status_code == 200 + def test_lookup(client): - query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 200 + query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 200 - rjson = json.loads(resp.data) - assert len(rjson) == 2 - assert rjson[0]["uid"] == "DRO-000-tp_to_trigger" - assert rjson[1]["uid"] == "DRO-001-tp_to_trigger" + rjson = json.loads(resp.data) + assert len(rjson) == 2 + assert rjson[0]["uid"] == "DRO-000-tp_to_trigger" + assert rjson[1]["uid"] == "DRO-001-tp_to_trigger" - query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 200 - rjson = json.loads(resp.data) - assert len(rjson) == 0 + query = json.loads("""{"uid_regex":"DUMMY.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 200 + rjson = json.loads(resp.data) + assert len(rjson) == 0 def test_retract(client): - resp = client.post("/retract") - assert resp.status_code == 400 + resp = client.post("/retract") + assert resp.status_code == 400 - retraction = json.loads("""{"partition":"ccTest", + retraction = json.loads("""{"partition":"ccTest", "connections":[{"connection_id":"DRO-000-tp_to_trigger"}, {"connection_id":"DRO-001-tp_to_trigger"}] }""") - resp = client.post("/retract", json=retraction) - assert resp.status_code == 200 + resp = client.post("/retract", json=retraction) + assert resp.status_code == 200 - query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") - resp = client.post("/getconnection/ccTest", json=query) - assert resp.status_code == 404 + query = json.loads("""{"uid_regex":"DRO.*", "data_type":"TPSet"}""") + resp = client.post("/getconnection/ccTest", json=query) + assert resp.status_code == 404 - # Second time should fail - resp = client.post("/retract", json=retraction) - assert resp.status_code == 404 + # Second time should fail + resp = client.post("/retract", json=retraction) + assert resp.status_code == 404 def test_retract_partition(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 + resp = client.post("/publish", json=con) + assert resp.status_code == 200 - resp = client.post("/retract-partition") - assert resp.status_code == 400 + resp = client.post("/retract-partition") + assert resp.status_code == 400 - retraction = json.loads("""{"partition":"ccTest"}""") - resp = client.post("/retract-partition", json=retraction) - assert resp.status_code == 200 + retraction = json.loads("""{"partition":"ccTest"}""") + resp = client.post("/retract-partition", json=retraction) + assert resp.status_code == 200 - # Second time should fail - resp = client.post("/retract-partition", json=retraction) - assert resp.status_code == 404 + # Second time should fail + resp = client.post("/retract-partition", json=retraction) + assert resp.status_code == 404 def test_dump(client): - resp = client.get("/") - assert b"Dump" in resp.data + resp = client.get("/") + assert b"Dump" in resp.data + def test_stats(client): - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" not in resp.data - - resp = client.get("/resetStats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" not in resp.data - - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

Connection server statistics" in resp.data - assert b"

0 calls to publish" in resp.data - + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" not in resp.data + + resp = client.get("/resetStats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" not in resp.data + + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

Connection server statistics" in resp.data + assert b"

0 calls to publish" in resp.data + def test_reset(client): - resp = client.post("/publish", json=con) - assert resp.status_code == 200 + resp = client.post("/publish", json=con) + assert resp.status_code == 200 + + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

0 calls to publish" not in resp.data + + resp = client.get("/resetService") + assert resp.status_code == 200 + + resp = client.get("/stats") + assert resp.status_code == 200 + assert b"

0 calls to publish" in resp.data + - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

0 calls to publish" not in resp.data +def test_live(client): + resp = client.get("/live") + assert resp.status_code == 200 + assert b"OK" in resp.data - resp = client.get("/resetService") - assert resp.status_code == 200 - resp = client.get("/stats") - assert resp.status_code == 200 - assert b"

0 calls to publish" in resp.data +def test_ready(client): + resp = client.post("/publish", json=con) + assert resp.status_code == 200 + resp = client.get("/ready") + assert resp.status_code == 200 + assert b"OK" in resp.data