From 02f301b5d08eaa20a9e02a4c1bd39be019f073a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 13:41:27 -0400 Subject: [PATCH 1/9] Fix mistakes in syncfileclient change --- .../el8-diskless/profiles/default/scripts/syncfileclient | 3 ++- confluent_osdeploy/el8/profiles/default/scripts/syncfileclient | 3 ++- .../el9-diskless/profiles/default/scripts/syncfileclient | 3 ++- .../genesis/profiles/default/scripts/syncfileclient | 3 ++- .../suse15-diskless/profiles/default/scripts/syncfileclient | 3 ++- confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 3 ++- .../suse15/profiles/server/scripts/syncfileclient | 3 ++- .../profiles/default/scripts/syncfileclient | 3 ++- .../ubuntu20.04/profiles/default/scripts/syncfileclient | 3 ++- .../ubuntu22.04/profiles/default/scripts/syncfileclient | 3 ++- 10 files changed, 20 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index 088fa9f7..ac60f5f7 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -9,6 +9,7 @@ import os import shutil import pwd import grp +import sys from importlib.machinery import SourceFileLoader try: apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() @@ -236,7 +237,7 @@ def synchronize(): if status == 202: lastrsp = '' while status != 204: - time.sleep(1+(2*random.random(a))) + time.sleep(1+(2*random.random())) status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') if not isinstance(rsp, str): rsp = rsp.decode('utf8') From 8ca9a44476de8da95894d7a7ed3324232f12bc05 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 14:27:00 -0400 Subject: [PATCH 2/9] Provide more interesting response body to syncfileclient --- confluent_server/confluent/selfservice.py | 4 ++-- confluent_server/confluent/syncfiles.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 3d7feebb..a166e0fb 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -517,8 +517,8 @@ def handle_request(env, start_response): pals = get_extra_names(nodename, cfg, myip) result = syncfiles.start_syncfiles( nodename, cfg, json.loads(reqbody), pals) - start_response(result, ()) - yield '' + start_response(result[0], ()) + yield result[1] return if 'GET' == operation: status, output = syncfiles.get_syncresult(nodename) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index df5574e3..ed99fedf 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -290,7 +290,7 @@ syncrunners = {} def start_syncfiles(nodename, cfg, suffixes, principals=[]): peerip = None if nodename in syncrunners: - return '503 Synchronization already in progress ' + return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] @@ -313,13 +313,13 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): raise Exception('Cannot perform syncfiles without profile assigned') synclist = '/var/lib/confluent/public/os/{}/syncfiles'.format(profile) if not os.path.exists(synclist): - return '200 OK' # not running + return '200 OK', 'No synclist' # not running sl = SyncList(synclist, nodename, cfg) if not (sl.appendmap or sl.mergemap or sl.replacemap or sl.appendoncemap): - return '200 OK' # the synclist has no actual entries + return '200 OK', 'Empty synclist' # the synclist has no actual entries syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) - return '202 Queued' # backgrounded + return '202 Queued', 'Background synchronization initiated' # backgrounded def get_syncresult(nodename): if nodename not in syncrunners: From 67b3c48dc9b322817f2a556e7afdf463be4c1ee8 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 14:58:38 -0400 Subject: [PATCH 3/9] Clean up error output on syncfileclient execution --- .../el8-diskless/profiles/default/scripts/syncfileclient | 2 ++ confluent_osdeploy/el8/profiles/default/scripts/syncfileclient | 2 ++ .../el9-diskless/profiles/default/scripts/syncfileclient | 2 ++ .../genesis/profiles/default/scripts/syncfileclient | 2 ++ .../suse15-diskless/profiles/default/scripts/syncfileclient | 2 ++ confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 2 ++ .../suse15/profiles/server/scripts/syncfileclient | 2 ++ .../profiles/default/scripts/syncfileclient | 2 ++ .../ubuntu20.04/profiles/default/scripts/syncfileclient | 2 ++ .../ubuntu22.04/profiles/default/scripts/syncfileclient | 2 ++ 10 files changed, 20 insertions(+) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index ac60f5f7..237c443d 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -298,6 +298,8 @@ if __name__ == '__main__': status = synchronize() except Exception as e: sys.stderr.write(str(e)) + sys.stderr.write('\n') + sys.stderr.flush() status = 300 if status not in (204, 200): time.sleep((random.random()*3)+2) From 1da27083cc8a64510009331b8853fcbc8a804c0f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:08:56 -0400 Subject: [PATCH 4/9] Another cleanup of syncfileclient output --- .../el8-diskless/profiles/default/scripts/syncfileclient | 4 +++- .../el8/profiles/default/scripts/syncfileclient | 4 +++- .../el9-diskless/profiles/default/scripts/syncfileclient | 4 +++- .../genesis/profiles/default/scripts/syncfileclient | 4 +++- .../suse15-diskless/profiles/default/scripts/syncfileclient | 4 +++- confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient | 4 +++- .../suse15/profiles/server/scripts/syncfileclient | 4 +++- .../profiles/default/scripts/syncfileclient | 4 +++- .../ubuntu20.04/profiles/default/scripts/syncfileclient | 4 +++- .../ubuntu22.04/profiles/default/scripts/syncfileclient | 4 +++- 10 files changed, 30 insertions(+), 10 deletions(-) diff --git a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient index 237c443d..5f2efc5e 100644 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient @@ -232,7 +232,9 @@ def synchronize(): status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) if status >= 300: sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(repr(rsp)) + sys.stderr.write(rsp.decode('utf8')) + sys.stderr.write('\n') + sys.stderr.flush() return status if status == 202: lastrsp = '' From eaffb342b2af2878c1a8aaad00c79b7873d23f74 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:19:38 -0400 Subject: [PATCH 5/9] Reap stale sync runners after a minute dead If the client never claims the result, delete the sync task. --- confluent_server/confluent/syncfiles.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index ed99fedf..1f739ba1 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -285,9 +285,10 @@ def mkpathorlink(source, destination, appendexist=False): syncrunners = {} - +cleaner = None def start_syncfiles(nodename, cfg, suffixes, principals=[]): + global cleaner peerip = None if nodename in syncrunners: return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) @@ -319,7 +320,26 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): return '200 OK', 'Empty synclist' # the synclist has no actual entries syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) - return '202 Queued', 'Background synchronization initiated' # backgrounded + if not cleaner: + cleaner = eventlet.spawn(cleanit) + return '202 Queued', 'Background synchronization initiated' # backgrounded + + +def cleanit(): + toreap = {} + while True: + for nn in list(syncrunners): + if syncrunners[nn].dead: + if nn in toreap: + syncrunners[nn].wait() + del syncrunners[nn] + del toreap[nn] + else: + toreap[nn] = 1 + elif nn is in toreap: + del toreap[nn] + eventlet.sleep(30) + def get_syncresult(nodename): if nodename not in syncrunners: From 8fb889ba736609b6ee70be835836d8691e91f4ff Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:27:20 -0400 Subject: [PATCH 6/9] Correct syntax error --- confluent_server/confluent/syncfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 1f739ba1..068d1ae4 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -336,7 +336,7 @@ def cleanit(): del toreap[nn] else: toreap[nn] = 1 - elif nn is in toreap: + elif nn in toreap: del toreap[nn] eventlet.sleep(30) From 01722c18c4e1e99d56e4bd465eb8005a69a1ae58 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 15:40:40 -0400 Subject: [PATCH 7/9] Fix location of idle sleep in syncfiles cleaner --- confluent_server/confluent/syncfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 068d1ae4..9c96e533 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -338,7 +338,7 @@ def cleanit(): toreap[nn] = 1 elif nn in toreap: del toreap[nn] - eventlet.sleep(30) + eventlet.sleep(30) def get_syncresult(nodename): From ceaf641c1a31cd03da1ecab66557bfb6d244d73f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 9 Apr 2024 16:18:24 -0400 Subject: [PATCH 8/9] Keep reap loop going on error --- confluent_server/confluent/syncfiles.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index 9c96e533..f1e638f8 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -331,7 +331,11 @@ def cleanit(): for nn in list(syncrunners): if syncrunners[nn].dead: if nn in toreap: - syncrunners[nn].wait() + try: + syncrunners[nn].wait() + except Exception as e: + print(repr(e)) + pass del syncrunners[nn] del toreap[nn] else: From 8e5ee6c9d8490d0275d9e84a118048d1e783744b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 10 Apr 2024 13:54:06 -0400 Subject: [PATCH 9/9] Make orphaned sync runner retire on new sync request --- confluent_server/confluent/syncfiles.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/syncfiles.py b/confluent_server/confluent/syncfiles.py index f1e638f8..16cf4c49 100644 --- a/confluent_server/confluent/syncfiles.py +++ b/confluent_server/confluent/syncfiles.py @@ -290,8 +290,6 @@ cleaner = None def start_syncfiles(nodename, cfg, suffixes, principals=[]): global cleaner peerip = None - if nodename in syncrunners: - return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) if 'myips' in suffixes: targips = suffixes['myips'] del suffixes['myips'] @@ -318,6 +316,11 @@ def start_syncfiles(nodename, cfg, suffixes, principals=[]): sl = SyncList(synclist, nodename, cfg) if not (sl.appendmap or sl.mergemap or sl.replacemap or sl.appendoncemap): return '200 OK', 'Empty synclist' # the synclist has no actual entries + if nodename in syncrunners: + if syncrunners[nodename].dead: + syncrunners[nodename].wait() + else: + return '503 Synchronization already in progress', 'Synchronization already in progress for {}'.format(nodename) syncrunners[nodename] = eventlet.spawn( sync_list_to_node, sl, nodename, suffixes, peerip) if not cleaner: