From a4e152c17d226cb613a7b684bb4ff215a7e2e131 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 10:31:46 -0400 Subject: [PATCH 1/3] Defer disarm until after successful client notification It is theoretically possible for a client to get disconnected right in the middle. In such a scenario, err on the side of letting the mechanism stay armed for the sake of a retry being possible. --- confluent_server/confluent/credserver.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/credserver.py b/confluent_server/confluent/credserver.py index c569bc4d..390179f8 100644 --- a/confluent_server/confluent/credserver.py +++ b/confluent_server/confluent/credserver.py @@ -127,14 +127,15 @@ class CredServer(object): if hmacval != hmac.new(hmackey, etok, hashlib.sha256).digest(): client.close() return - cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}, 'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} - if hmackey and apiarmed != 'continuous': - self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) - if apiarmed == 'continuous': - del cfgupdate[nodename]['deployment.apiarmed'] + cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}}} self.cfm.set_node_attributes(cfgupdate) client.recv(2) # drain end of message client.send(b'\x05\x00') # report success + if hmackey and apiarmed != 'continuous': + self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken']) + if apiarmed != 'continuous': + tokclear = {nodename: {'deployment.sealedapikey': '', 'deployment.apiarmed': ''}} + self.cfm.set_node_attributes(tokclear) finally: client.close() From f68f9f46939ed115b501b125212359777cf045c6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 11:07:11 -0400 Subject: [PATCH 2/3] Make syncfile step robust or pause If syncfiles fails, keep it retrying. Also, slow down sync checking to avoid hammering the system. Further, randomized delay to spread highly synchronized requestors. Block attempts to do multiple concurrent syncfile runs. --- .../profiles/default/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../suse15/profiles/hpc/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/server/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- .../profiles/default/scripts/syncfileclient | 18 +++++++++++++++++- confluent_server/confluent/syncfiles.py | 2 ++ 13 files changed, 176 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient index 8d37d43a..cca0f57d 100644 --- a/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,5 @@ #!/usr/bin/python +import time import importlib import tempfile import json @@ -223,6 +224,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: + time.sleep(2) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient index 8d37d43a..02dbcc4d 100644 --- a/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el7/profiles/default/scripts/syncfileclient @@ -5,6 +5,7 @@ import json import os import shutil import pwd +import time import grp try: from importlib.machinery import SourceFileLoader @@ -223,6 +224,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: + time.sleep(2) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index f7d4c0b4..088fa9f7 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -1,4 +1,6 @@ #!/usr/bin/python3 +import random +import time import subprocess import importlib import tempfile @@ -227,9 +229,14 @@ def synchronize(): myips.append(addr) data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) + if status >= 300: + sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) + sys.stderr.write(repr(rsp)) + return status if status == 202: lastrsp = '' while status != 204: + time.sleep(1+(2*random.random(a))) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') @@ -277,10 +284,19 @@ def synchronize(): os.chmod(fname, int(opts[fname][opt], 8)) if uid != -1 or gid != -1: os.chown(fname, uid, gid) + return status finally: shutil.rmtree(tmpdir) shutil.rmtree(appendoncedir) if __name__ == '__main__': - synchronize() + status = 202 + while status not in (204, 200): + try: + status = synchronize() + except Exception as e: + sys.stderr.write(str(e)) + status = 300 + if status not in (204, 200): + time.sleep((random.random()*3)+2) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 94b74eea..df5574e3 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -289,6 +289,8 @@ syncrunners = {} def start_syncfiles(nodename, cfg, suffixes, principals=[]): peerip = None + if nodename in syncrunners: + return '503 Synchronization already in progress ' if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] From 33271451d711d3bc2a26a10cf22381d948e8aed2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 13:17:19 -0400 Subject: [PATCH 3/3] Support SHA384 if used as fingerprint --- confluent_server/confluent/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/util.py b/confluent_server/confluent/util.py index 96d2291b..462ec930 100644 --- a/confluent_server/confluent/util.py +++ b/confluent_server/confluent/util.py @@ -168,7 +168,7 @@ def cert_matches(fingerprint, certificate): return algo(certificate).digest() == fingerprint algo, _, fp = fingerprint.partition('$') newfp = None - if algo in ('sha512', 'sha256'): + if algo in ('sha512', 'sha256', 'sha384'): newfp = get_fingerprint(certificate, algo) return newfp and fingerprint == newfp