diff --git a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/el8/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/el9-diskless/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient b/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/suse15-diskless/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient b/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient deleted file mode 100644 index 5f2efc5e..00000000 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/python3 -import random -import time -import subprocess -import importlib -import tempfile -import json -import os -import shutil -import pwd -import grp -import sys -from importlib.machinery import SourceFileLoader -try: - apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module() -except FileNotFoundError: - apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module() - - -def partitionhostsline(line): - comment = '' - try: - cmdidx = line.index('#') - comment = line[cmdidx:] - line = line[:cmdidx].strip() - except ValueError: - pass - if not line: - return '', [], comment - ipaddr, names = line.split(maxsplit=1) - names = names.split() - return ipaddr, names, comment - -class HostMerger(object): - def __init__(self): - self.byip = {} - self.byname = {} - self.sourcelines = [] - self.targlines = [] - - def read_source(self, sourcefile): - with open(sourcefile, 'r') as hfile: - self.sourcelines = hfile.read().split('\n') - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - for x in range(len(self.sourcelines)): - line = self.sourcelines[x] - currip, names, comment = partitionhostsline(line) - if currip: - self.byip[currip] = x - for name in names: - self.byname[name] = x - - def read_target(self, targetfile): - with open(targetfile, 'r') as hfile: - lines = hfile.read().split('\n') - if not lines[-1]: - lines = lines[:-1] - for y in range(len(lines)): - line = lines[y] - currip, names, comment = partitionhostsline(line) - if currip in self.byip: - x = self.byip[currip] - if self.sourcelines[x] is None: - # have already consumed this enntry - continue - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - continue - for name in names: - if name in self.byname: - x = self.byname[name] - if self.sourcelines[x] is None: - break - self.targlines.append(self.sourcelines[x]) - self.sourcelines[x] = None - break - else: - self.targlines.append(line) - - def write_out(self, targetfile): - while not self.targlines[-1]: - self.targlines = self.targlines[:-1] - if not self.targlines: - break - while not self.sourcelines[-1]: - self.sourcelines = self.sourcelines[:-1] - if not self.sourcelines: - break - with open(targetfile, 'w') as hosts: - for line in self.targlines: - hosts.write(line + '\n') - for line in self.sourcelines: - if line is not None: - hosts.write(line + '\n') - - -class CredMerger: - def __init__(self): - try: - with open('/etc/login.defs', 'r') as ldefs: - defs = ldefs.read().split('\n') - except FileNotFoundError: - defs = [] - lkup = {} - self.discardnames = {} - self.shadowednames = {} - for line in defs: - try: - line = line[:line.index('#')] - except ValueError: - pass - keyval = line.split() - if len(keyval) < 2: - continue - lkup[keyval[0]] = keyval[1] - self.uidmin = int(lkup.get('UID_MIN', 1000)) - self.uidmax = int(lkup.get('UID_MAX', 60000)) - self.gidmin = int(lkup.get('GID_MIN', 1000)) - self.gidmax = int(lkup.get('GID_MAX', 60000)) - self.shadowlines = None - - def read_passwd(self, source, targfile=False): - self.read_generic(source, self.uidmin, self.uidmax, targfile) - - def read_group(self, source, targfile=False): - self.read_generic(source, self.gidmin, self.gidmax, targfile) - - def read_generic(self, source, minid, maxid, targfile): - if targfile: - self.targdata = [] - else: - self.sourcedata = [] - with open(source, 'r') as inputfile: - for line in inputfile.read().split('\n'): - try: - name, _, uid, _ = line.split(':', 3) - uid = int(uid) - except ValueError: - continue - if targfile: - if uid < minid or uid > maxid: - self.targdata.append(line) - else: - self.discardnames[name] = 1 - else: - if name[0] in ('+', '#', '@'): - self.sourcedata.append(line) - elif uid >= minid and uid <= maxid: - self.sourcedata.append(line) - - def read_shadow(self, source): - self.shadowlines = [] - try: - with open(source, 'r') as inshadow: - for line in inshadow.read().split('\n'): - try: - name, _ = line.split(':' , 1) - except ValueError: - continue - if name in self.discardnames: - continue - self.shadowednames[name] = 1 - self.shadowlines.append(line) - except FileNotFoundError: - return - - def write_out(self, outfile): - with open(outfile, 'w') as targ: - for line in self.targdata: - targ.write(line + '\n') - for line in self.sourcedata: - targ.write(line + '\n') - if outfile == '/etc/passwd': - if self.shadowlines is None: - self.read_shadow('/etc/shadow') - with open('/etc/shadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':', 1) - if name[0] in ('+', '#', '@'): - continue - if name in self.shadowednames: - continue - shadout.write(name + ':!:::::::\n') - if outfile == '/etc/group': - if self.shadowlines is None: - self.read_shadow('/etc/gshadow') - with open('/etc/gshadow', 'w') as shadout: - for line in self.shadowlines: - shadout.write(line + '\n') - for line in self.sourcedata: - name, _ = line.split(':' , 1) - if name in self.shadowednames: - continue - shadout.write(name + ':!::\n') - -def appendonce(basepath, filename): - with open(filename, 'rb') as filehdl: - thedata = filehdl.read() - targname = filename.replace(basepath, '') - try: - with open(targname, 'rb') as filehdl: - targdata = filehdl.read() - except IOError: - targdata = b'' - if thedata in targdata: - return - with open(targname, 'ab') as targhdl: - targhdl.write(thedata) - -def synchronize(): - tmpdir = tempfile.mkdtemp() - appendoncedir = tempfile.mkdtemp() - try: - ac = apiclient.HTTPSClient() - myips = [] - ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n') - for line in ipaddrs: - isa = line.split() - if len(isa) < 3 or isa[1] != b'UP': - continue - for addr in isa[2:]: - if addr.startswith(b'fe80::') or addr.startswith(b'169.254'): - continue - addr = addr.split(b'/')[0] - if not isinstance(addr, str): - addr = addr.decode('utf8') - myips.append(addr) - data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips}) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data) - if status >= 300: - sys.stderr.write("Error starting syncfiles - {}:\n".format(status)) - sys.stderr.write(rsp.decode('utf8')) - sys.stderr.write('\n') - sys.stderr.flush() - return status - if status == 202: - lastrsp = '' - while status != 204: - time.sleep(1+(2*random.random())) - status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles') - if not isinstance(rsp, str): - rsp = rsp.decode('utf8') - if status == 200: - lastrsp = rsp - pendpasswd = os.path.join(tmpdir, 'etc/passwd') - if os.path.exists(pendpasswd): - cm = CredMerger() - cm.read_passwd(pendpasswd, targfile=False) - cm.read_passwd('/etc/passwd', targfile=True) - cm.write_out('/etc/passwd') - pendgroup = os.path.join(tmpdir, 'etc/group') - if os.path.exists(pendgroup): - cm = CredMerger() - cm.read_group(pendgroup, targfile=False) - cm.read_group('/etc/group', targfile=True) - cm.write_out('/etc/group') - pendhosts = os.path.join(tmpdir, 'etc/hosts') - if os.path.exists(pendhosts): - cm = HostMerger() - cm.read_source(pendhosts) - cm.read_target('/etc/hosts') - cm.write_out('/etc/hosts') - for dirn in os.walk(appendoncedir): - for filen in dirn[2]: - appendonce(appendoncedir, os.path.join(dirn[0], filen)) - if lastrsp: - lastrsp = json.loads(lastrsp) - opts = lastrsp.get('options', {}) - for fname in opts: - uid = -1 - gid = -1 - for opt in opts[fname]: - if opt == 'owner': - try: - uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid - except KeyError: - uid = opts[fname][opt]['id'] - elif opt == 'group': - try: - gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid - except KeyError: - gid = opts[fname][opt]['id'] - elif opt == 'permissions': - os.chmod(fname, int(opts[fname][opt], 8)) - if uid != -1 or gid != -1: - os.chown(fname, uid, gid) - return status - finally: - shutil.rmtree(tmpdir) - shutil.rmtree(appendoncedir) - - -if __name__ == '__main__': - status = 202 - while status not in (204, 200): - try: - status = synchronize() - except Exception as e: - sys.stderr.write(str(e)) - sys.stderr.write('\n') - sys.stderr.flush() - status = 300 - if status not in (204, 200): - time.sleep((random.random()*3)+2)