From 6f484aab53a74a2f84677c9683623c26433e5dd6 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 9 Jun 2022 15:49:06 -0400 Subject: [PATCH 1/3] Allow restore to replace unsupported format Going from python 2 to python 3, the dbm format goes from the default to unsupported. This allows a python3 confluentdbutil restore to handle a python2 dump of unsupported format. --- confluent_server/bin/confluentdbutil | 3 ++ .../confluent/config/configmanager.py | 35 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil index be35e5aa..0f3148e7 100755 --- a/confluent_server/bin/confluentdbutil +++ b/confluent_server/bin/confluentdbutil @@ -69,7 +69,10 @@ if args[0] == 'restore': if options.interactivepassword: password = getpass.getpass('Enter password to restore backup: ') try: + cfm.statelessmode = True cfm.restore_db_from_directory(dumpdir, password) + cfm.statelessmode = False + cfm.ConfigManager.wait_for_sync(True) if owner != 0: for targdir in os.walk('/etc/confluent'): os.chown(targdir[0], owner, group) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 58f4bc59..f1a4192e 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -2604,7 +2604,13 @@ class ConfigManager(object): with _dirtylock: dirtyglobals = copy.deepcopy(_cfgstore['dirtyglobals']) del _cfgstore['dirtyglobals'] - globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384) # 0600 + try: + globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384) # 0600 + except dbm.error: + if not fullsync: + raise + os.remove(os.path.join(cls._cfgdir, "globals")) + globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384) # 0600 try: for globalkey in dirtyglobals: if globalkey in _cfgstore['globals']: @@ -2617,8 +2623,15 @@ class ConfigManager(object): globalf.close() if fullsync or 'collectivedirty' in _cfgstore: if len(_cfgstore.get('collective', ())) > 1: - collectivef = dbm.open(os.path.join(cls._cfgdir, "collective"), - 'c', 384) + try: + collectivef = dbm.open(os.path.join(cls._cfgdir, 'collective'), + 'c', 384) + except dbm.error: + if not fullsync: + raise + os.remove(os.path.join(cls._cfgdir, 'collective')) + collectivef = dbm.open(os.path.join(cls._cfgdir, 'collective'), + 'c', 384) try: if fullsync: colls = _cfgstore['collective'] @@ -2645,7 +2658,13 @@ class ConfigManager(object): currdict = _cfgstore['main'] for category in currdict: _mkpath(pathname) - dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 + try: + dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 + except dbm.error: + if not fullsync: + raise + os.remove(os.path.join(pathname, category)) + dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 try: for ck in currdict[category]: dbf[ck] = cPickle.dumps(currdict[category][ck], protocol=cPickle.HIGHEST_PROTOCOL) @@ -2665,7 +2684,13 @@ class ConfigManager(object): currdict = _cfgstore['tenant'][tenant] for category in dkdict: _mkpath(pathname) - dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 + try: + dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 + except dbm.error: + if not fullsync: + raise + os.remove(os.path.join(pathname, category)) + dbf = dbm.open(os.path.join(pathname, category), 'c', 384) # 0600 try: for ck in dkdict[category]: if ck not in currdict[category]: From db5c31030d3276ff03720c404c1cc8645f0312df Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 9 Jun 2022 16:23:35 -0400 Subject: [PATCH 2/3] Migrate DB on start If python2 db format detected, use python2 to dump to text, then python3 to restore to get the python3 native version --- confluent_server/bin/confluentdbutil | 1 + .../confluent/config/configmanager.py | 2 ++ confluent_server/confluent/main.py | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/confluent_server/bin/confluentdbutil b/confluent_server/bin/confluentdbutil index 0f3148e7..25a5acf8 100755 --- a/confluent_server/bin/confluentdbutil +++ b/confluent_server/bin/confluentdbutil @@ -69,6 +69,7 @@ if args[0] == 'restore': if options.interactivepassword: password = getpass.getpass('Enter password to restore backup: ') try: + cfm.init(True) cfm.statelessmode = True cfm.restore_db_from_directory(dumpdir, password) cfm.statelessmode = False diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index f1a4192e..68a8c94a 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -565,6 +565,8 @@ def _load_dict_from_dbm(dpath, tdb): currdict[tks] = cPickle.loads(dbe[tk]) # nosec tk = dbe.nextkey(tk) except dbm.error: + if os.path.exists(tdb): + raise return diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index 5f35b2c8..2d26df8d 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -29,6 +29,10 @@ import atexit import confluent.auth as auth import confluent.config.conf as conf import confluent.config.configmanager as configmanager +try: + import anydbm as dbm +except ModuleNotFoundError: + import dbm import confluent.consoleserver as consoleserver import confluent.core as confluentcore import confluent.httpapi as httpapi @@ -62,8 +66,10 @@ import os import glob import signal import socket +import subprocess import time import traceback +import tempfile import uuid @@ -232,8 +238,20 @@ def sanity_check(): assure_ownership('/etc/confluent/srvcert.pem') +def migrate_db(): + tdir = tempfile.mkdtemp() + subprocess.check_call(['python2', '/opt/confluent/bin/confluentdbutil', 'dump', '-u', tdir]) + subprocess.check_call(['python3', '/opt/confluent/bin/confluentdbutil', 'restore', '-u', tdir]) + subprocess.check_call(['rm', '-rf', tdir]) + configmanager.init() + + def run(args): setlimits() + try: + configmanager.ConfigManager(None) + except dbm.error: + migrate_db() try: signal.signal(signal.SIGUSR1, dumptrace) except AttributeError: From 13d4d1dd98e1b4b077d9672e0881fe22d86f0f1b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 10 Jun 2022 11:19:21 -0400 Subject: [PATCH 3/3] Ensure that python3 will execute before doing migrate If python3 is not the executable name, prevent the restoration attempt from happening. --- confluent_server/confluent/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index 2d26df8d..207d3783 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -240,6 +240,7 @@ def sanity_check(): def migrate_db(): tdir = tempfile.mkdtemp() + subprocess.check_call(['python3', '-c', 'pass']) subprocess.check_call(['python2', '/opt/confluent/bin/confluentdbutil', 'dump', '-u', tdir]) subprocess.check_call(['python3', '/opt/confluent/bin/confluentdbutil', 'restore', '-u', tdir]) subprocess.check_call(['rm', '-rf', tdir])