Correct the equality message in better messagesw

Some versions start manifesting nvme devnames with 'c', which
are to be used to interact with multipath to have raw devices
backing a traditional nvme device.

confluent_client/bin/nodeapply

@@ -102,9 +102,9 @@ def run():
         cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
         if currprocs < concurrentprocs:
             currprocs += 1
-            run_cmdv(node, cmdv, all, pipedesc)
+            run_cmdv(sshnode, cmdv, all, pipedesc)
         else:
-            pendingexecs.append((node, cmdv))        
+            pendingexecs.append((sshnode, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
     rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodeattrib

@@ -22,6 +22,7 @@ import optparse
 import os
 import signal
 import sys
+import shlex
 
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
@@ -53,6 +54,8 @@ argparser.add_option('-p', '--prompt', action='store_true',
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Prompt if trying to set attributes on more '
                           'than specified number of nodes')
+argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
+                     default=False, help='set attributes using a batch file')
 (options, args) = argparser.parse_args()
 
 
@@ -109,6 +112,23 @@ elif options.clear or options.environment or options.prompt:
     sys.stderr.write('Attribute names required with specified options\n')
     argparser.print_help()
     exitcode = 400
+
+elif options.set:
+    arglist = [noderange]
+    showtype='current'
+    argfile = open(options.set, 'r')
+    argset = argfile.readline()
+    while argset:
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            arglist += shlex.split(argset)
+        argset = argfile.readline() 
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
 if exitcode != 0:
     sys.exit(exitcode)
 

confluent_client/bin/nodedeploy

@@ -90,17 +90,6 @@ def main(args):
         if 'error' in rsp:
             sys.stderr.write(rsp['error'] + '\n')
             sys.exit(1)
-    if not args.clear and args.network and not args.prepareonly:
-        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
-                                        bootmode='uefi',
-                                        persistent=False,
-                                        errnodes=errnodes)
-        if errnodes:
-            sys.stderr.write(
-                'Unable to set boot device for following nodes: {0}\n'.format(
-                    ','.join(errnodes)))
-            return 1
-        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.clear:
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
@@ -120,7 +109,7 @@ def main(args):
                 for profname in profnames:
                     sys.stderr.write('    ' + profname + '\n')
             else:
-                sys.stderr.write('No deployment profiles available, try osdeploy fiimport or imgutil capture\n')
+                sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n')
             sys.exit(1)
         armonce(args.noderange, c)
         setpending(args.noderange, args.profile, c)
@@ -166,6 +155,17 @@ def main(args):
             else:
                 print('{0}: {1}{2}'.format(node, profile, armed))
         sys.exit(0)
+    if not args.clear and args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.network and not args.prepareonly:
         return rc
     return 0

confluent_client/bin/nodersync

@@ -48,6 +48,8 @@ def run():
                           'prompting if over the threshold')   
     argparser.add_option('-l', '--loginname', type='str',
                          help='Username to use when connecting, defaults to current user.')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for rsync')
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -63,10 +65,20 @@ def run():
     c = client.Command()
     cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
-    if options.loginname:
-        cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath
+    
+    targname = options.substitutename
+    if targname and '{' in targname:
+        targname =  targname + ':'
+    elif targname:
+        targname = '{node}' + targname + ':'
     else:
-        cmdstr += ' {node}:' + targpath
+        targname = '{node}:'
+
+    if options.loginname:
+        cmdstr += ' {}@'.format(options.loginname) + targname + targpath
+    else:
+        cmdstr += ' {}'.format(targname) + targpath
+    
 
     currprocs = 0
     all = set([])

confluent_client/bin/stats

@@ -16,13 +16,10 @@
 # limitations under the License.
 
 import argparse
+import base64
 import csv
-import fcntl
 import io
 import numpy as np
-
-import os
-import subprocess
 import sys
 
 try:
@@ -35,7 +32,31 @@ except ImportError:
     pass
 
 
-def plot(gui, output, plotdata, bins):
+def iterm_draw(data):
+    databuf = data.getbuffer()
+    datalen = len(databuf)
+    data = base64.b64encode(databuf).decode('utf8')
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;size={}:'.format(datalen))
+    sys.stdout.write(data)
+    sys.stdout.write('\a')
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+
+def kitty_draw(data):
+    data = base64.b64encode(data.getbuffer())
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+    sys.stdout.write('\n')
+
+
+def plot(gui, output, plotdata, bins, fmt):
     import matplotlib as mpl
     if gui and mpl.get_backend() == 'agg':
         sys.stderr.write('Error: No GUI backend available and -g specified!\n')
@@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
-        writer = DumbWriter()
-        writer.draw(tdata)
+        if fmt == 'sixel':
+            writer = DumbWriter()
+            writer.draw(tdata)
+        elif fmt == 'kitty':
+            kitty_draw(tdata)
+        elif fmt == 'iterm':
+            iterm_draw(tdata)
     return n, bins
 
 def textplot(plotdata, bins):
@@ -81,7 +107,8 @@ histogram = False
 aparser = argparse.ArgumentParser(description='Quick access to common statistics')
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
-aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
+aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
@@ -138,7 +165,7 @@ while data:
     data = list(csv.reader([data], delimiter=delimiter))[0]
 n = None
 if args.g or args.o or args.x:
-    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f)
 if args.t:
     n, bins = textplot(plotdata, bins=args.b)
 print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -7,7 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
-`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`
+`nodeattrib <noderange> -s <attributes.batch> ...` 
 
 ## DESCRIPTION
 
@@ -53,12 +54,18 @@ to a blank value will allow masking a group defined attribute with an empty valu
 * `-e`, `--environment`:
   Set specified attributes based on exported environment variable of matching name. 
   Environment variable names may be lower case or all upper case. 
-  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE)
   
 * `-p`, `--prompt`:
   Request interactive prompting to provide values rather than the command line
   or environment variables.
-  
+
+* `-s`, `--set`:
+  Set attributes using a batch file rather than the command line. The attributes in the batch file 
+  can be specified as one line of key=value pairs simmilar to command line or each attribute can
+  be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch
+  file syntax.   
+ 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Prompt if trying to set attributes on more than
   specified number of nodes.
@@ -116,6 +123,25 @@ to a blank value will allow masking a group defined attribute with an empty valu
     `d1: net.pxe.switch: pxeswitch1`  
     `d1: net.switch:`  
 
+* Setting attributes using a batch file with syntax similar to command line:
+    `# cat nodeattributes.batch`  
+    `# power`
+    `power.psu1.outlet=3 power.psu1.pdu=pdu2`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: 3`
+    `n41: pdu2`
+
+* Setting attributes using a batch file with syntax where each attribute is in its own line:
+    `# cat nodeattributes.batch`  
+    `# management`
+    `custom.mgt.switch=switch_main`
+    `custom.mgt.switch.port=swp4`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: switch_main`
+    `n41: swp4`
+
+  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodersync.ronn

@@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes.
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 
+
+* `-s`, `--substitutename`:
+  'Use a different name other than the nodename for rsync'
   
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to run rsync to, prompting if over the

confluent_osdeploy/buildrpm-aarch64

@@ -1,3 +1,4 @@
+cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
@@ -29,4 +30,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
 rm -rf el8bin
-rpmbuild -ba confluent_osdeploy-aarch64.spec
+podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec
+

confluent_osdeploy/common/profile/scripts/confignet

@@ -112,9 +112,10 @@ def get_interface_name(iname, settings):
     return None
 
 class NetplanManager(object):
-    def __init__(self):
+    def __init__(self, deploycfg):
         self.cfgbydev = {}
         self.read_connections()
+        self.deploycfg = deploycfg
 
     def read_connections(self):
         for plan in glob.glob('/etc/netplan/*.y*ml'):
@@ -150,23 +151,46 @@ class NetplanManager(object):
         needcfgapply = False
         for devname in devnames:
             needcfgwrite = False
-            if stgs['ipv4_method'] == 'static':
+            # ipv6_method missing at uconn...
+            if stgs.get('ipv6_method', None) == 'static':
+                curraddr = stgs['ipv6_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            if stgs.get('ipv4_method', None) == 'static':
                 curraddr = stgs['ipv4_address']
                 currips = self.getcfgarrpath([devname, 'addresses'])
                 if curraddr not in currips:
                     needcfgwrite = True
                     currips.append(curraddr)
-            gwaddr = stgs.get('ipv4_gateway', None)
-            if gwaddr:
-                cfgroutes = self.getcfgarrpath([devname, 'routes'])
-                for rinfo in cfgroutes:
-                    if rinfo.get('via', None) == gwaddr:
-                        break
-                else:
+            gws = []
+            gws.append(stgs.get('ipv4_gateway', None))
+            gws.append(stgs.get('ipv6_gateway', None))
+            for gwaddr in gws:
+                if gwaddr:
+                    cfgroutes = self.getcfgarrpath([devname, 'routes'])
+                    for rinfo in cfgroutes:
+                        if rinfo.get('via', None) == gwaddr:
+                            break
+                    else:
+                        needcfgwrite = True
+                        cfgroutes.append({'via': gwaddr, 'to': 'default'})
+            dnsips = self.deploycfg.get('nameservers', [])
+            dnsdomain = self.deploycfg.get('dnsdomain', '')
+            if dnsips:
+                currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses'])
+                for dnsip in dnsips:
+                    if dnsip and dnsip not in currdnsips:
+                        needcfgwrite = True
+                        currdnsips.append(dnsip)
+            if dnsdomain:
+                currdnsdomain = self.getcfgarrpath([devname, 'nameservers', 'search'])
+                if dnsdomain not in currdnsdomain:
                     needcfgwrite = True
-                    cfgroutes.append({'via': gwaddr, 'to': 'default'})
+                    currdnsdomain.append(dnsdomain)
             if needcfgwrite:
-                needcfgaply = True
+                needcfgapply = True
                 newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
                 with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout:
                     planout.write(yaml.dump(newcfg))
@@ -321,7 +345,7 @@ class NetworkManager(object):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
             subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
-        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
         if bondcfg:
             args = []
             for parm in bondcfg:
@@ -355,7 +379,7 @@ class NetworkManager(object):
             for arg in cmdargs:
                 cargs.append(arg)
                 cargs.append(cmdargs[arg])
-            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team', 'con-name', cname, 'connection.interface-name', cname, 'team.runner', stgs['team_mode']] + cargs)
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
             for iface in cfg['interfaces']:
                 self.add_team_member(cname, iface)
             subprocess.check_call(['nmcli', 'c', 'u', cname])
@@ -394,6 +418,7 @@ if __name__ == '__main__':
     myaddrs = apiclient.get_my_addresses()
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
+    dc = None
     for srv in srvs:
         try:
             s = socket.create_connection((srv, 443))
@@ -411,8 +436,27 @@ if __name__ == '__main__':
                     curridx = addr[-1]
         if curridx in doneidxs:
             continue
-        status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+        for tries in (1, 2, 3):
+            try:
+                status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+                break
+            except Exception:
+                if tries == 3:
+                    raise
+                time.sleep(1)
+                continue
         nc = json.loads(nc)
+        if not dc:
+            for tries in (1, 2, 3):
+                try:
+                    status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+                    break
+                except Exception:
+                    if tries == 3:
+                        raise
+                    time.sleep(1)
+                    continue
+            dc = json.loads(dc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:
             for iname in iname.split(','):
@@ -439,7 +483,7 @@ if __name__ == '__main__':
             del netname_to_interfaces['default']
     rm_tmp_llas(tmpllas)
     if os.path.exists('/usr/sbin/netplan'):
-        nm = NetplanManager()
+        nm = NetplanManager(dc)
     if os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes)
     elif os.path.exists('/usr/sbin/wicked'):

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,13 @@ import struct
 import sys
 import subprocess
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +60,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -166,6 +176,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -224,7 +236,8 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    # do this safer, unit s print might bomb
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +271,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk , volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +281,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -189,8 +189,15 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+	linktype="infiniband"
+else
+	linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
+
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -199,9 +206,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -13,6 +13,12 @@ import subprocess
 
 bootuuid = None
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -202,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if 'nvme' in targblock and targblock[-1] == 'p':
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -295,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -305,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/el8/profiles/default/kickstart

@@ -33,15 +33,7 @@
 reboot
 
 %packages
-@^minimal-environment
-#-kernel-uek # This can opt out of the UEK for the relevant distribution
-bind-utils
-chrony
-pciutils
-python3
-rsync
-tar
--iwl*-firmware
+%include /tmp/pkglist
 %include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end

confluent_osdeploy/el8/profiles/default/packagelist

@@ -0,0 +1,9 @@
+@^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
+bind-utils
+chrony
+pciutils
+python3
+rsync
+tar
+-iwl*-firmware

confluent_osdeploy/el8/profiles/default/partitioning

@@ -0,0 +1,4 @@
+clearpart --all --initlabel
+ignoredisk --only-use %%INSTALLDISK%%
+autopart --nohome %%LUKSHOOK%%
+

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -87,6 +87,7 @@ done
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 touch /tmp/cryptpkglist
+touch /tmp/pkglist
 touch /tmp/addonpackages
 if [ "$cryptboot" == "tpm2" ]; then
 	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
@@ -102,15 +103,18 @@ confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_pro
 run_remote pre.custom
 run_remote_parts pre.d
 confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart -o /tmp/kickstart.base
+if grep '^%include /tmp/pkglist' /tmp/kickstart.* > /dev/null; then
+    confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/packagelist -o /tmp/pkglist
+fi
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || touch /tmp/installdisk
 if [ ! -e /tmp/installdisk ]; then
     run_remote_python getinstalldisk
 fi
+confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/partitioning -o /tmp/partitioning.template
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk
 if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
-    echo clearpart --all --initlabel >> /tmp/partitioning
-    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
-    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+    INSTALLDISK=$(cat /tmp/installdisk)
+    sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning
     dd if=/dev/zero of=/dev/$(cat /tmp/installdisk) bs=1M count=1 >& /dev/null
     vgchange -a n >& /dev/null
 fi

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -154,8 +154,14 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+        linktype="infiniband"
+else
+        linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -164,9 +170,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -13,6 +13,12 @@ import subprocess
 
 bootuuid = None
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -202,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -295,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -305,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -174,6 +174,8 @@ dnsdomain=${dnsdomain#dnsdomain: }
 echo search $dnsdomain >> /etc/resolv.conf
 echo -n "Initializing ssh..."
 ssh-keygen -A
+mkdir -p /usr/share/empty.sshd
+rm /etc/ssh/ssh_host_dsa_key*
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent

@@ -1,4 +1,5 @@
 if ! grep console= /proc/cmdline > /dev/null; then
+    mkdir -p /custom-installation
     /opt/confluent/bin/autocons > /custom-installation/autocons.info
     cons=$(cat /custom-installation/autocons.info)
     if [ ! -z "$cons" ]; then

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/firstboot.sh

@@ -24,8 +24,6 @@ done
 if [ ! -f /etc/confluent/firstboot.ran ]; then
     touch /etc/confluent/firstboot.ran
 
-    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
-
     run_remote firstboot.custom
     # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
     run_remote_parts firstboot.d

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py

@@ -13,6 +13,13 @@ import subprocess
 
 bootuuid = None
 
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -199,6 +206,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -292,7 +301,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -302,10 +311,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,286 @@
+#!/usr/bin/python3
+import subprocess
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -74,8 +74,11 @@ if [ -e /sys/firmware/efi ]; then
     fi
 fi
 cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -2,7 +2,10 @@
 echo "Confluent first boot is running"
 HOME=$(getent passwd $(whoami)|cut -d: -f 6)
 export HOME
-seems a potentially relevant thing to put i... by Jarrod Johnson
+(
+exec >> /target/var/log/confluent/confluent-firstboot.log
+exec 2>> /target/var/log/confluent/confluent-firstboot.log
+chmod 600 /target/var/log/confluent/confluent-firstboot.log
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
@@ -18,7 +21,10 @@ done
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions
-
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export confluent_mgr confluent_profile
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -8,7 +8,6 @@ chmod go-rwx /etc/confluent/*
 for i in /custom-installation/ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
 done
-
 cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
 cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
 sshconf=/target/etc/ssh/ssh_config
@@ -19,10 +18,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-
+cp /etc/confluent/functions /target/etc/confluent/functions
+source /etc/confluent/functions
+mkdir -p /target/var/log/confluent
+cp /var/log/confluent/* /target/var/log/confluent/
+(
+exec >> /target/var/log/confluent/confluent-post.log
+exec 2>> /target/var/log/confluent/confluent-post.log
+chmod 600 /target/var/log/confluent/confluent-post.log
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions
-source /target/etc/confluent/functions
 chmod +x /target/etc/confluent/firstboot.sh
 cp /tmp/allnodes /target/root/.shosts
 cp /tmp/allnodes /target/etc/ssh/shosts.equiv
@@ -78,10 +82,13 @@ cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/co
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
 chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 
 run_remote_config post
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
 
 umount /target/sys /target/dev /target/proc
-
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 deploycfg=/custom-installation/confluent/confluent.deploycfg
+mkdir -p /var/log/confluent
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp /custom-installation/confluent/confluent.info /custom-installation/confluent/confluent.apikey /etc/confluent/
+cat /custom-installation/tls/*.pem >> /etc/confluent/ca.pem
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin
+cp $deploycfg /etc/confluent/
+(
+exec >> /var/log/confluent/confluent-pre.log
+exec 2>> /var/log/confluent/confluent-pre.log
+chmod 600 /var/log/confluent/confluent-pre.log
 
 cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
 if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
@@ -23,7 +34,17 @@ echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
 systemctl restart sshd
+mkdir -p /etc/confluent
+export nodename confluent_profile confluent_mgr
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+run_remote_parts pre.d
 curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
-python3 /custom-installation/getinstalldisk
+if [ ! -e /tmp/installdisk ]; then
+    curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+    python3 /custom-installation/getinstalldisk
+fi
 sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml
+) &
+tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console
+

confluent_server/bin/confluent_selfcheck

@@ -15,6 +15,7 @@ import confluent.sshutil as sshutil
 import confluent.certutil as certutil
 import confluent.client as client
 import confluent.config.configmanager as configmanager
+import confluent.netutil as netutil
 import eventlet.green.subprocess as subprocess
 import tempfile
 import shutil
@@ -244,7 +245,7 @@ if __name__ == '__main__':
                 allok = False
                 uuidok = True  # not really, but suppress the spurious error
             dnsdomain = rsp.get('dns.domain', {}).get('value', '')
-            if ',' in dnsdomain or ' ' in dnsdomain:
+            if dnsdomain and (',' in dnsdomain or ' ' in dnsdomain):
                 allok = False
                 emprint(f'{args.node} has a dns.domain that appears to be a search instead of singular domain')
                 uuidok = True  # not really, but suppress the spurious error
@@ -269,9 +270,28 @@ if __name__ == '__main__':
                     switch_value = rsp[key].get('value',None)
                     if switch_value and switch_value not in valid_nodes:
                         emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).')
+        print(f"Checking network configuration for {args.node}")
+        cfg = configmanager.ConfigManager(None)
+        bootablev4nics = []
+        bootablev6nics = []
+        for nic in glob.glob("/sys/class/net/*/ifindex"):
+            idx = int(open(nic, "r").read())
+            nicname = nic.split('/')[-2]
+            ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx)
+            if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp':
+                bootablev4nics.append(nicname)
+            if ncfg['ipv6_address']:
+                bootablev6nics.append(nicname)
+        if bootablev4nics:
+            print("{} appears to have network configuration suitable for IPv4 deployment via: {}".format(args.node, ",".join(bootablev4nics)))
+        elif bootablev6nics:
+            print('{} appears to have networking configuration suitable for IPv6 deployment via: {}'.format(args.node, ",".join(bootablev6nics)))
+        else:
+            emprint(f"{args.node} may not have any viable IP network configuration (check name resolution (DNS or hosts file) "
+                    "and/or net.*ipv4_address, and verify that the deployment serer addresses and subnet mask/prefix length are accurate)")
         if not uuidok and not macok:
             allok = False
-            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work')
+            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work (Example resolution: nodeinventory {args.node} -s)')
         if allok:
             print(f'No issues detected with attributes of {args.node}')
         fprint("Checking name resolution: ")

confluent_server/bin/osdeploy

@@ -72,6 +72,12 @@ def main(args):
         return rebase(cmdset.profile)
     ap.print_help()
 
+def symlinkp(src, trg):
+    try:
+        os.symlink(src, trg)
+    except Exception as e:
+        if e.errno != 17:
+            raise
 
 def initialize_genesis():
     if not os.path.exists('/opt/confluent/genesis/x86_64/boot/kernel'):
@@ -89,30 +95,33 @@ def initialize_genesis():
         return retval[1]
     retcode = 0
     try:
+        util.mkdirp('/var/lib/confluent', 0o755)
         if hasconfluentuser:
+            os.chown('/var/lib/confluent', hasconfluentuser.pw_uid, -1)
             os.setgid(hasconfluentuser.pw_gid)
             os.setuid(hasconfluentuser.pw_uid)
         os.umask(0o22)
-        os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755)
-        os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755)
-        os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI',
+        util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755)
+        util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755)
+        symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/BOOTX64.EFI')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/grubx64.efi')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/initramfs/distribution',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/initramfs/distribution',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/distribution')
-        os.symlink('/var/lib/confluent/public/site/initramfs.cpio',
+        symlinkp('/var/lib/confluent/public/site/initramfs.cpio',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/site.cpio')
-        os.symlink('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio',
+        symlinkp('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/addons.cpio')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/kernel',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/kernel',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/kernel')
-        shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/',
-                        '/var/lib/confluent/public/os/genesis-x86_64/ansible/')
-        shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/',
-                        '/var/lib/confluent/public/os/genesis-x86_64/scripts/')
-        shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml',
-                       '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml')
+        if not os.path.exists('/var/lib/confluent/public/os/genesis-x86_64/ansible/'):
+            shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/',
+                            '/var/lib/confluent/public/os/genesis-x86_64/ansible/')
+            shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/',
+                            '/var/lib/confluent/public/os/genesis-x86_64/scripts/')
+            shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml',
+                            '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml')
     except Exception as e:
         sys.stderr.write(str(e) + '\n')
         retcode = 1
@@ -373,9 +382,14 @@ def initialize(cmdset):
         for rsp in c.read('/uuid'):
             uuid = rsp.get('uuid', {}).get('value', None)
             if uuid:
-                with open('confluent_uuid', 'w') as uuidout:
-                    uuidout.write(uuid)
-                    uuidout.write('\n')
+                oum = os.umask(0o11)
+                try:
+                    with open('confluent_uuid', 'w') as uuidout:
+                        uuidout.write(uuid)
+                        uuidout.write('\n')
+                    os.chmod('confluent_uuid', 0o644)
+                finally:
+                    os.umask(oum)
     totar.append('confluent_uuid')
     topack.append('confluent_uuid')
     if os.path.exists('ssh'):
@@ -403,7 +417,17 @@ def initialize(cmdset):
         if res:
             sys.stderr.write('Error occurred while packing site initramfs')
             sys.exit(1)
-    os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio')
+    oum = os.umask(0o22)
+    try:
+        os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio')
+        os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644)
+    finally:
+        os.umask(oum)
+    oum = os.umask(0o22)
+    try:
+        os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644)
+    finally:
+        os.umask(oum)
     if cmdset.g:
         updateboot('genesis-x86_64')
     if totar:
@@ -411,6 +435,11 @@ def initialize(cmdset):
         tarcmd = ['tar', '-czf', tmptarname] + totar
         subprocess.check_call(tarcmd)
         os.rename(tmptarname, '/var/lib/confluent/public/site/initramfs.tgz')
+        oum = os.umask(0o22)
+        try:
+            os.chmod('/var/lib/confluent/public/site/initramfs.tgz', 0o644)
+        finally:
+            os.umask(0o22)
     os.chdir(opath)
     print('Site initramfs content packed successfully')
 
@@ -421,6 +450,9 @@ def initialize(cmdset):
 
 
 def updateboot(profilename):
+    if not os.path.exists('/var/lib/confluent/public/site/initramfs.cpio'):
+        emprint('Must generate site content first (TLS (-t) and/or SSH (-s))')
+        return 1
     c = client.Command()
     for rsp in c.update('/deployment/profiles/{0}'.format(profilename),
             {'updateboot': 1}):

confluent_server/builddeb

@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -49,6 +49,13 @@ if ! grep wheezy /etc/os-release; then
 fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
+cat > debian/postinst << EOF
+if ! getent passwd confluent > /dev/null; then 
+    useradd -r confluent -d /var/lib/confluent -s /usr/sbin/nologin
+    mkdir -p /etc/confluent
+    chown confluent /etc/confluent
+fi
+EOF
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
 #echo 'Provides: python-'$DPKGNAME >> debian/control
 #echo 'Conflicts: python-'$DPKGNAME >> debian/control

confluent_server/confluent/certutil.py

@@ -95,27 +95,29 @@ def assure_tls_ca():
         os.makedirs(os.path.dirname(fname))
     except OSError as e:
         if e.errno != 17:
+            os.seteuid(ouid)
             raise
+    try:
+        shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
+        hv, _ = util.run(
+            ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
+        if not isinstance(hv, str):
+            hv = hv.decode('utf8')
+        hv = hv.strip()
+        hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
+        certname = '{0}.pem'.format(collective.get_myname())
+        for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
+            currname = os.path.join('/var/lib/confluent/public/site/tls/', currname)
+            if currname.endswith('.0'):
+                try:
+                    realname = os.readlink(currname)
+                    if realname == certname:
+                        os.unlink(currname)
+                except OSError:
+                    pass
+        os.symlink(certname, hashname)
     finally:
         os.seteuid(ouid)
-    shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
-    hv, _ = util.run(
-        ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
-    if not isinstance(hv, str):
-        hv = hv.decode('utf8')
-    hv = hv.strip()
-    hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
-    certname = '{0}.pem'.format(collective.get_myname())
-    for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
-        currname = os.path.join('/var/lib/confluent/public/site/tls/', currname)
-        if currname.endswith('.0'):
-            try:
-                realname = os.readlink(currname)
-                if realname == certname:
-                    os.unlink(currname)
-            except OSError:
-                pass
-    os.symlink(certname, hashname)
 
 def substitute_cfg(setting, key, val, newval, cfgfile, line):
     if key.strip() == setting:

confluent_server/confluent/collective/manager.py

@@ -259,6 +259,9 @@ def get_myname():
         mycachedname[1] = time.time()
         return myname
 
+def in_collective():
+    return bool(list(cfm.list_collective()))
+
 def handle_connection(connection, cert, request, local=False):
     global currentleader
     global retrythread
@@ -713,6 +716,7 @@ def become_leader(connection):
     if reassimilate is not None:
         reassimilate.kill()
     reassimilate = eventlet.spawn(reassimilate_missing)
+    cfm._ready = True
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
 

confluent_server/confluent/config/attributes.py

@@ -371,7 +371,7 @@ node = {
                         'the managed node.  If not specified, then console '
                         'is disabled.  "ipmi" should be specified for most '
                         'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/configmanager.py

@@ -119,6 +119,7 @@ _cfgstore = None
 _pendingchangesets = {}
 _txcount = 0
 _hasquorum = True
+_ready = False
 
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
@@ -830,6 +831,9 @@ _oldcfgstore = None
 _oldtxcount = 0
 
 
+def config_is_ready():
+    return _ready
+
 def rollback_clear():
     global _cfgstore
     global _txcount
@@ -847,6 +851,8 @@ def clear_configuration():
     global _txcount
     global _oldcfgstore
     global _oldtxcount
+    global _ready
+    _ready = False
     stop_leading()
     stop_following()
     _oldcfgstore = _cfgstore
@@ -857,6 +863,7 @@ def clear_configuration():
 def commit_clear():
     global _oldtxcount
     global _oldcfgstore
+    global _ready
     # first, copy over old non-key globals, as those are
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
@@ -876,6 +883,7 @@ def commit_clear():
                 pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
+    _ready = True
 
 cfgleader = None
 
@@ -1273,6 +1281,7 @@ class ConfigManager(object):
     def __init__(self, tenant, decrypt=False, username=None):
         self.clientfiles = {}
         global _cfgstore
+        self.inrestore = False
         with _initlock:
             if _cfgstore is None:
                 init()
@@ -2089,6 +2098,10 @@ class ConfigManager(object):
     def _notif_attribwatchers(self, nodeattrs):
         if self.tenant not in self._attribwatchers:
             return
+        if self.inrestore:
+            # Do not stir up attribute watchers during a collective join or DB restore,
+            # it's too hectic of a time to react
+            return
         notifdata = {}
         attribwatchers = self._attribwatchers[self.tenant]
         for node in nodeattrs:
@@ -2471,6 +2484,13 @@ class ConfigManager(object):
         #TODO: wait for synchronization to suceed/fail??)
 
     def _load_from_json(self, jsondata, sync=True):
+        self.inrestore = True
+        try:
+            self._load_from_json_backend(jsondata, sync=True)
+        finally:
+            self.inrestore = False
+
+    def _load_from_json_backend(self, jsondata, sync=True):
         """Load fresh configuration data from jsondata
 
         :param jsondata: String of jsondata
@@ -2939,9 +2959,9 @@ def get_globals():
         bkupglobals[globvar] = _cfgstore['globals'][globvar]
     return bkupglobals
 
-
 def init(stateless=False):
     global _cfgstore
+    global _ready
     if stateless:
         _cfgstore = {}
         return
@@ -2949,6 +2969,9 @@ def init(stateless=False):
         ConfigManager._read_from_path()
     except IOError:
         _cfgstore = {}
+    members = list(list_collective())
+    if len(members) < 2:
+        _ready = True
 
 
 if __name__ == '__main__':

confluent_server/confluent/consoleserver.py

@@ -49,7 +49,6 @@ _handled_consoles = {}
 
 _tracelog = None
 _bufferdaemon = None
-_bufferlock = None
 
 try:
     range = xrange
@@ -62,39 +61,38 @@ def chunk_output(output, n):
         yield output[i:i + n]
 
 def get_buffer_output(nodename):
-    out = _bufferdaemon.stdin
-    instream = _bufferdaemon.stdout
+    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+    out.connect("\x00confluent-vtbuffer")
     if not isinstance(nodename, bytes):
         nodename = nodename.encode('utf8')
     outdata = bytearray()
-    with _bufferlock:
-        out.write(struct.pack('I', len(nodename)))
-        out.write(nodename)
-        out.flush()
-        select.select((instream,), (), (), 30)
-        while not outdata or outdata[-1]:
-            try:
-                chunk = os.read(instream.fileno(), 128)
-            except IOError:
-                chunk = None
-            if chunk:
-                outdata.extend(chunk)
-            else:
-                select.select((instream,), (), (), 0)
-        return bytes(outdata[:-1])
+    out.send(struct.pack('I', len(nodename)))
+    out.send(nodename)
+    select.select((out,), (), (), 30)
+    while not outdata or outdata[-1]:
+        try:
+            chunk = os.read(out.fileno(), 128)
+        except IOError:
+            chunk = None
+        if chunk:
+            outdata.extend(chunk)
+        else:
+            select.select((out,), (), (), 0)
+    return bytes(outdata[:-1])
 
 
 def send_output(nodename, output):
     if not isinstance(nodename, bytes):
         nodename = nodename.encode('utf8')
-    with _bufferlock:
-        _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29)))
-        _bufferdaemon.stdin.write(nodename)
-        _bufferdaemon.stdin.flush()
-        for chunk in chunk_output(output, 8192):
-            _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29)))
-            _bufferdaemon.stdin.write(chunk)
-            _bufferdaemon.stdin.flush()
+    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+    out.connect("\x00confluent-vtbuffer")
+    out.send(struct.pack('I', len(nodename) | (1 << 29)))
+    out.send(nodename)
+    for chunk in chunk_output(output, 8192):
+        out.send(struct.pack('I', len(chunk) | (2 << 29)))
+        out.send(chunk)
 
 def _utf8_normalize(data, decoder):
     # first we give the stateful decoder a crack at the byte stream,
@@ -600,15 +598,10 @@ def _start_tenant_sessions(cfm):
 def initialize():
     global _tracelog
     global _bufferdaemon
-    global _bufferlock
-    _bufferlock = semaphore.Semaphore()
     _tracelog = log.Logger('trace')
     _bufferdaemon = subprocess.Popen(
-        ['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE)
-    fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL)
-    fcntl.fcntl(_bufferdaemon.stdout.fileno(),
-                fcntl.F_SETFL, fl | os.O_NONBLOCK)
+        ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL)
 
 def start_console_sessions():
     configmodule.hook_new_configmanagers(_start_tenant_sessions)

confluent_server/confluent/core.py

@@ -446,6 +446,7 @@ def _init_core():
                 },
             },
         },
+        'layout': PluginRoute({'handler': 'layout'}),
         'media': {
             'uploads': PluginCollection({
                 'pluginattrs': ['hardwaremanagement.method'],
@@ -480,6 +481,20 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'normalized': {
+                    'inlet_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'average_cpu_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'total_power': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
                 'energy': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',

confluent_server/confluent/discovery/core.py

@@ -648,6 +648,8 @@ def detected_models():
 
 
 def _recheck_nodes(nodeattribs, configmanager):
+    if not cfm.config_is_ready():
+        return
     if rechecklock.locked():
         # if already in progress, don't run again
         # it may make sense to schedule a repeat, but will try the easier and less redundant way first
@@ -766,6 +768,9 @@ def eval_detected(info):
 def detected(info):
     global rechecker
     global rechecktime
+    if not cfm.config_is_ready():
+        # drop processing of discovery data while configmanager is 'down'
+        return
     # later, manual and CMM discovery may act on SN and/or UUID
     for service in info['services']:
         if service in nodehandlers:
@@ -1429,7 +1434,12 @@ def discover_node(cfg, handler, info, nodename, manual):
                 newnodeattribs['pubkeys.tls_hardwaremanager'] = \
                     util.get_fingerprint(handler.https_cert, 'sha256')
             if newnodeattribs:
-                cfg.set_node_attributes({nodename: newnodeattribs})
+                currattrs = cfg.get_node_attributes(nodename, newnodeattribs)
+                for checkattr in newnodeattribs:
+                    checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                    if checkval != newnodeattribs[checkattr]:
+                        cfg.set_node_attributes({nodename: newnodeattribs})
+                        break
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
             if nodeconfig:
@@ -1508,7 +1518,12 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
                 if info['hwaddr'] != oldhwaddr:
                     attribs[newattrname] = info['hwaddr']
         if attribs:
-            cfg.set_node_attributes({nodename: attribs})
+            currattrs = cfg.get_node_attributes(nodename, attribs)
+            for checkattr in attribs:
+                checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                if checkval != attribs[checkattr]:
+                    cfg.set_node_attributes({nodename: attribs})
+                    break
     if info['uuid'] in known_pxe_uuids:
         return True
     if uuid_is_valid(info['uuid']):
@@ -1597,7 +1612,10 @@ def remotescan():
     mycfm = cfm.ConfigManager(None)
     myname = collective.get_myname()
     for remagent in get_subscriptions():
-        affluent.renotify_me(remagent, mycfm, myname)
+        try:
+            affluent.renotify_me(remagent, mycfm, myname)
+        except Exception as e:
+            log.log({'error': 'Unexpected problem asking {} for discovery notifications'.format(remagent)})
 
 
 def blocking_scan():
@@ -1637,7 +1655,7 @@ def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
     #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
-    remotescan()
+    eventlet.spawn(remotescan)
 
 
 nodes_by_fprint = {}

confluent_server/confluent/discovery/handlers/xcc.py

@@ -247,6 +247,10 @@ class NodeHandler(immhandler.NodeHandler):
             if rsp.status == 200:
                 pwdchanged = True
                 password = newpassword
+                wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+                if '_csrf_token' in wc.cookies:
+                    wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+                wc.grab_json_response_with_status('/api/providers/logout')
             else:
                 if rspdata.get('locktime', 0) > 0:
                     raise LockedUserException(
@@ -280,6 +284,7 @@ class NodeHandler(immhandler.NodeHandler):
                 rsp.read()
                 if rsp.status != 200:
                     return (None, None)
+                wc.grab_json_response_with_status('/api/providers/logout')
                 self._currcreds = (username, newpassword)
                 wc.set_basic_credentials(username, newpassword)
                 pwdchanged = True
@@ -434,6 +439,7 @@ class NodeHandler(immhandler.NodeHandler):
                     '/api/function',
                     {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)})
                 if status == 200 and rsp.get('return', 0) == 13:
+                    wc.grab_json_response('/api/providers/logout')
                     wc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
                     status = 503
                     while status != 200:
@@ -442,10 +448,13 @@ class NodeHandler(immhandler.NodeHandler):
                             {'UserName': username}, method='PATCH')
                         if status != 200:
                             rsp = json.loads(rsp)
-                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError'):
-                                eventlet.sleep(10)
+                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'):
+                                eventlet.sleep(4)
                             else:
                                 break
+                    self.tmppasswd = None
+                    self._currcreds = (username, passwd)
+                    return
             self.tmppasswd = None
         wc.grab_json_response('/api/providers/logout')
         self._currcreds = (username, passwd)
@@ -632,3 +641,4 @@ def remote_nodecfg(nodename, cfm):
     info = {'addresses': [ipaddr]}
     nh = NodeHandler(info, cfm)
     nh.config(nodename)
+

confluent_server/confluent/discovery/protocols/pxe.py

@@ -587,7 +587,10 @@ def get_deployment_profile(node, cfg, cfd=None):
         return None
     candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None)
     if candmgrs:
-        candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+        try:
+            candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+        except Exception: # fallback to unverified noderange
+            candmgrs = noderange.NodeRange(candmgrs).nodes
         if collective.get_myname() not in candmgrs:
             return None
     return profile

confluent_server/confluent/discovery/protocols/slp.py

@@ -246,11 +246,11 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
             try:
                 net4.sendto(data, ('239.255.255.253', 427))
             except socket.error as se:
-                # On occasion, multicasting may be disabled
-                # tolerate this scenario and move on
-                if se.errno != 101:
-                    raise
-            net4.sendto(data, (bcast, 427))
+                pass
+            try:
+                net4.sendto(data, (bcast, 427))
+            except socket.error as se:
+                pass
 
 
 def _grab_rsps(socks, rsps, interval, xidmap, deferrals):

confluent_server/confluent/firmwaremanager.py

@@ -53,7 +53,7 @@ def execupdate(handler, filename, updateobj, type, owner, node, datfile):
                 return
         if type == 'ffdc' and os.path.isdir(filename):
             filename += '/' + node
-        if 'type' == 'ffdc':
+        if type == 'ffdc':
             errstr = False
             if os.path.exists(filename):
                 errstr = '{0} already exists on {1}, cannot overwrite'.format(

confluent_server/confluent/httpapi.py

@@ -1084,7 +1084,7 @@ def serve(bind_host, bind_port):
         pass  # we gave it our best shot there
     try:
         eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                             debug=False, socket_timeout=60)
+                             debug=False, socket_timeout=60, keepalive=False)
     except TypeError:
         # Older eventlet in place, skip arguments it does not understand
         eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)

confluent_server/confluent/main.py

@@ -326,7 +326,7 @@ def run(args):
             break
         except Exception:
             eventlet.sleep(0.5)
-    disco.start_detection()
+    eventlet.spawn_n(disco.start_detection)
     eventlet.sleep(1)
     consoleserver.start_console_sessions()
     while 1:

confluent_server/confluent/messages.py

@@ -92,6 +92,7 @@ def msg_deserialize(packed):
         return cls(*m[1:])
     raise Exception("Unknown shenanigans")
 
+
 class ConfluentMessage(object):
     apicode = 200
     readonly = False
@@ -254,6 +255,21 @@ class ConfluentNodeError(object):
         raise Exception('{0}: {1}'.format(self.node, self.error))
 
 
+class Generic(ConfluentMessage):
+
+    def __init__(self, data):
+        self.data = data
+
+    def json(self):
+        return json.dumps(self.data)
+    
+    def raw(self):
+        return self.data
+    
+    def html(self):
+        return json.dumps(self.data)
+
+
 class ConfluentResourceUnavailable(ConfluentNodeError):
     apicode = 503
 

confluent_server/confluent/netutil.py

@@ -25,6 +25,9 @@ import eventlet.support.greendns
 import os
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
+eventlet.support.greendns.resolver.clear()
+eventlet.support.greendns.resolver._resolver.lifetime = 1
+
 def msg_align(len):
     return (len + 3) & ~3
 
@@ -332,19 +335,57 @@ def get_full_net_config(configmanager, node, serverip=None):
     if serverip:
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
+    defaultnic = {}
+    ppool = eventlet.greenpool.GreenPool(64)
     if None in attribs:
-        nm.process_attribs(None, attribs[None])
+        ppool.spawn(nm.process_attribs, None, attribs[None])
         del attribs[None]
     for netname in sorted(attribs):
-        nm.process_attribs(netname, attribs[netname])
+        ppool.spawn(nm.process_attribs, netname, attribs[netname])
+    ppool.waitall()
     retattrs = {}
     if None in nm.myattribs:
         retattrs['default'] = nm.myattribs[None]
         add_netmask(retattrs['default'])
         del nm.myattribs[None]
+    else:
+        nnc = get_nic_config(configmanager, node, serverip=serverip)
+        if nnc.get('ipv4_address', None):
+            defaultnic['ipv4_address'] = '{}/{}'.format(nnc['ipv4_address'], nnc['prefix'])
+        if nnc.get('ipv4_gateway', None):
+            defaultnic['ipv4_gateway'] = nnc['ipv4_gateway']
+        if nnc.get('ipv4_method', None):
+            defaultnic['ipv4_method'] = nnc['ipv4_method']
+        if nnc.get('ipv6_address', None):
+            defaultnic['ipv6_address'] = '{}/{}'.format(nnc['ipv6_address'], nnc['ipv6_prefix'])
+        if nnc.get('ipv6_method', None):
+            defaultnic['ipv6_method'] = nnc['ipv6_method']
     retattrs['extranets'] = nm.myattribs
     for attri in retattrs['extranets']:
         add_netmask(retattrs['extranets'][attri])
+        if retattrs['extranets'][attri].get('ipv4_address', None) == defaultnic.get('ipv4_address', 'NOPE'):
+            defaultnic = {}
+        if retattrs['extranets'][attri].get('ipv6_address', None) == defaultnic.get('ipv6_address', 'NOPE'):
+            defaultnic = {}
+    if defaultnic:
+        retattrs['default'] = defaultnic
+        add_netmask(retattrs['default'])
+        ipv4addr = defaultnic.get('ipv4_address', None)
+        if ipv4addr and '/' in ipv4addr:
+            ipv4bytes = socket.inet_pton(socket.AF_INET, ipv4addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv4bytes, addr[2]):
+                    defaultnic['current_nic'] = True
+        ipv6addr = defaultnic.get('ipv6_address', None)
+        if ipv6addr and '/' in ipv6addr:
+            ipv6bytes = socket.inet_pton(socket.AF_INET6, ipv6addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET6:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv6bytes, addr[2]):
+                    defaultnic['current_nic'] = True
     return retattrs
 
 

confluent_server/confluent/networking/lldp.py

@@ -381,9 +381,10 @@ def list_info(parms, requestedparameter):
                 break
         else:
             candidate = info[requestedparameter]
-            candidate = candidate.strip()
-            if candidate != '':
-                results.add(_api_sanitize_string(candidate))
+            if candidate:
+                candidate = candidate.strip()
+                if candidate != '':
+                    results.add(_api_sanitize_string(candidate))
     return [msg.ChildCollection(x + suffix) for x in util.natural_sort(results)]
 
 def _handle_neighbor_query(pathcomponents, configmanager):

confluent_server/confluent/networking/macmap.py

@@ -49,10 +49,11 @@ import eventlet.green.select as select
 
 import eventlet.green.socket as socket
 
-
+import confluent.collective.manager as collective
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
+import confluent.noderange as noderange
 import confluent.util as util
 from eventlet.greenpool import GreenPool
 import eventlet.green.subprocess as subprocess
@@ -502,10 +503,21 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'collective.managercandidates', 'net*.switch', 'net*.switchport'))
         switches = set([])
+        incollective = collective.in_collective()
+        if incollective:
+            mycollectivename = collective.get_myname()
         for node in nodelocations:
             cfg = nodelocations[node]
+            if incollective:
+                candmgrs = cfg.get('collective.managercandidates', {}).get('value', None)
+                if candmgrs:
+                    candmgrs = noderange.NodeRange(candmgrs, configmanager).nodes
+                    if mycollectivename not in candmgrs:
+                        # do not think about trying to find nodes that we aren't possibly
+                        # supposed to be a manager for in a collective
+                        continue
             if cfg.get('type', {}).get('value', None) == 'switch':
                 switches.add(node)
             for attr in cfg:

confluent_server/confluent/noderange.py

@@ -55,6 +55,145 @@ def humanify_nodename(nodename):
     return [int(text) if text.isdigit() else text.lower()
             for text in re.split(numregex, nodename)]
 
+def unnumber_nodename(nodename):
+    # stub out numbers
+    chunked = ["{}" if text.isdigit() else text.lower()
+            for text in re.split(numregex, nodename)]
+    return chunked
+
+def getnumbers_nodename(nodename):
+    return [x for x in re.split(numregex, nodename) if x.isdigit()]
+    
+
+class Bracketer(object):
+    __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens']
+
+    def __init__(self, nodename):
+        self.sequences = []
+        self.numlens = []
+        realnodename = nodename
+        if ':' in nodename:
+            realnodename = nodename.split(':', 1)[0]
+        self.count = len(getnumbers_nodename(realnodename))
+        self.nametmpl = unnumber_nodename(realnodename)
+        for n in range(self.count):
+            self.sequences.append(None)
+            self.numlens.append([0, 0])
+        self.diffn = None
+        self.tokens = []
+        self.extend(nodename)
+        if self.count == 0:
+            self.tokens = [nodename]
+
+    def extend(self, nodeorseq):
+        # can only differentiate a single number
+        endname = None
+        endnums = None
+        if ':' in nodeorseq:
+            nodename, endname = nodeorseq.split(':', 1)
+        else:
+            nodename = nodeorseq
+        txtnums = getnumbers_nodename(nodename)
+        nums = [int(x) for x in txtnums]
+        for n in range(self.count):
+            # First pass to see if we have exactly one different number
+            padto = len(txtnums[n])
+            needpad = (padto != len('{}'.format(nums[n])))
+            if self.sequences[n] is None:
+                # We initialize to text pieces, 'currstart', and 'prev' number
+                self.sequences[n] = [[], nums[n], nums[n]]
+                self.numlens[n] = [len(txtnums[n]), len(txtnums[n])]
+            elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto:
+                continue  # new nodename has no new number, keep going
+            else: # if self.sequences[n][2] != nums[n] or :
+                if self.diffn is not None and (n != self.diffn or
+                        (padto < self.numlens[n][1]) or
+                        (needpad and padto != self.numlens[n][1])):
+                    self.flush_current()
+                    self.sequences[n] = [[], nums[n], nums[n]]
+                    self.numlens[n] = [padto, padto]
+                self.diffn = n
+        for n in range(self.count):
+            padto = len(txtnums[n])
+            needpad = (padto != len('{}'.format(nums[n])))
+            if self.sequences[n] is None:
+                # We initialize to text pieces, 'currstart', and 'prev' number
+                self.sequences[n] = [[], nums[n], nums[n]]
+                self.numlens[n] = [len(txtnums[n]), len(txtnums[n])]
+            elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto:
+                continue  # new nodename has no new number, keep going
+            else: # if self.sequences[n][2] != nums[n] or :
+                if self.diffn is not None and (n != self.diffn or
+                        (padto < self.numlens[n][1]) or
+                        (needpad and padto != self.numlens[n][1])):
+                    self.flush_current()
+                    self.sequences[n] = [[], nums[n], nums[n]]
+                    self.numlens[n] = [padto, padto]
+                    self.diffn = None
+                else:
+                    self.diffn = n
+                if self.sequences[n][2] == (nums[n] - 1):
+                    self.sequences[n][2] = nums[n]
+                    self.numlens[n][1] = padto
+                elif self.sequences[n][2] < (nums[n] - 1):
+                    if self.sequences[n][2] != self.sequences[n][1]:
+                        fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                    else:
+                        fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                    self.sequences[n][1] = nums[n]
+                    self.numlens[n][0] = padto
+                self.sequences[n][2] = nums[n]
+                self.numlens[n][1] = padto
+
+    def flush_current(self):
+        txtfields = []
+        if self.sequences and self.sequences[0] is not None:
+            for n in range(self.count):
+                if self.sequences[n][1] == self.sequences[n][2]:
+                    fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                else:
+                    fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                txtfield = ','.join(self.sequences[n][0])
+                if txtfield.isdigit():
+                    txtfields.append(txtfield)
+                else:
+                    txtfields.append('[{}]'.format(txtfield))
+            self.tokens.append(''.join(self.nametmpl).format(*txtfields))
+        self.sequences = []
+        for n in range(self.count):
+            self.sequences.append(None)
+
+    @property
+    def range(self):
+        if self.sequences:
+            self.flush_current()
+        return ','.join(self.tokens)
+
+
+def group_elements(elems):
+    """ Take the specefied elements and chunk them according to text similarity
+    """
+    prev = None
+    currchunk = []
+    chunked_elems = [currchunk]
+    for elem in elems:
+        elemtxt = unnumber_nodename(elem)
+        if not prev:
+            prev = elemtxt
+            currchunk.append(elem)
+            continue
+        if prev == elemtxt:
+            currchunk.append(elem)
+        else:
+            currchunk = [elem]
+            chunked_elems.append(currchunk)
+            prev = elemtxt
+    return chunked_elems
+
 
 class ReverseNodeRange(object):
     """Abbreviate a set of nodes to a shorter noderange representation
@@ -71,7 +210,8 @@ class ReverseNodeRange(object):
     @property
     def noderange(self):
         subsetgroups = []
-        for group in self.cfm.get_groups(sizesort=True):
+        allgroups = self.cfm.get_groups(sizesort=True)
+        for group in allgroups:
             if lastnoderange:
                 for nr in lastnoderange:
                     if lastnoderange[nr] - self.nodes:
@@ -88,7 +228,39 @@ class ReverseNodeRange(object):
                 self.nodes -= nl
                 if not self.nodes:
                     break
-        return ','.join(sorted(subsetgroups) + sorted(self.nodes))
+        # then, analyze sequentially identifying matching alpha subsections
+        # then try out noderange from beginning to end
+        # we need to know discontinuities, which are either:
+        # nodes that appear in the noderange that are not in the nodes
+        # nodes that do not exist at all (we need a noderange modification 
+        # that returns non existing nodes)
+        ranges = []
+        try:
+            subsetgroups.sort(key=humanify_nodename)
+            groupchunks = group_elements(subsetgroups)
+            for gc in groupchunks:
+                if not gc:
+                    continue
+                bracketer = Bracketer(gc[0])
+                for chnk in gc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            subsetgroups.sort()
+            ranges.extend(subsetgroups)
+        try:
+            nodes = sorted(self.nodes, key=humanify_nodename)
+            nodechunks = group_elements(nodes)
+            for nc in nodechunks:
+                if not nc:
+                    continue
+                bracketer = Bracketer(nc[0])
+                for chnk in nc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            ranges.extend(sorted(self.nodes))
+        return ','.join(ranges)
 
 
 
@@ -230,12 +402,16 @@ class NodeRange(object):
     def _expandstring(self, element, filternodes=None):
         prefix = ''
         if element[0][0] in ('/', '~'):
+            if self.purenumeric:
+                raise Exception('Regular expression not supported within "[]"')
             element = ''.join(element)
             nameexpression = element[1:]
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
             return set(self.cfm.filter_nodenames(nameexpression, filternodes))
         elif '=' in element[0] or '!~' in element[0]:
+            if self.purenumeric:
+                raise Exception('Equality/Inequality operators (=, !=, =~, !~) are invalid within "[]"')
             element = ''.join(element)
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
@@ -295,3 +471,29 @@ class NodeRange(object):
         if self.cfm is None:
             return set([element])
         raise Exception(element + ' not a recognized node, group, or alias')
+
+if __name__ == '__main__':
+    cases = [
+        (['r3u4', 'r5u6'], 'r3u4,r5u6'),  # should not erroneously gather
+        (['r3u4s1', 'r5u6s3'], 'r3u4s1,r5u6s3'),  # should not erroneously gather
+        (['r3u4s1', 'r3u4s2', 'r5u4s3'], 'r3u4s[1:2],r5u4s3'),  # should not erroneously gather
+        (['r3u4', 'r3u5', 'r3u6', 'r3u9', 'r4u1'], 'r3u[4:6,9],r4u1'),
+        (['n01', 'n2', 'n03'], 'n01,n2,n03'),
+        (['n7', 'n8', 'n09', 'n10', 'n11', 'n12', 'n13', 'n14', 'n15', 'n16',
+          'n17', 'n18', 'n19', 'n20'], 'n[7:8],n[09:20]')
+    ]
+    for case in cases:
+        gc = case[0]
+        bracketer = Bracketer(gc[0])
+        for chnk in gc[1:]:
+            bracketer.extend(chnk)
+        br = bracketer.range
+        resnodes = NodeRange(br).nodes
+        if set(resnodes) != set(gc):
+            print('FAILED: ' + repr(sorted(gc)))
+            print('RESULT: ' + repr(sorted(resnodes)))
+            print('EXPECTED: ' + repr(case[1]))
+            print('ACTUAL: ' + br)
+
+
+

confluent_server/confluent/plugins/console/openbmc.py

@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import pyghmi.util.webclient as webclient
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    bmc = bmc.split('/', 1)[0]
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+         
+
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        wc = webclient.SecureHTTPConnection(self.origbmc, 443, verifycallback=kv)
+        rsp = wc.grab_json_response_with_status('/login', {'data': [self.username.decode('utf8'), self.password.decode("utf8")]}, headers={'Content-Type': 'application/json'})
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION']))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)

confluent_server/confluent/plugins/hardwaremanagement/cooltera.py

@@ -210,10 +210,12 @@ def xml2stateinfo(statdata):
     stateinfo = []
     sensornames = sorted([x.tag for x in statdata])
     themodel = None
-    for model in sensorsbymodel:
-        if sensorsbymodel[model] == sensornames:
+    for model in sorted(sensorsbymodel):
+        if all([x in sensornames for x in sensorsbymodel[model]]):
             themodel = model
             break
+    else:
+        print(repr(sensornames))
     thesensors = _thesensors[themodel]
 #['mode', 't1', 't2a', 't2b', 't2c', 't2', 't5', 't3', 't4', 'dw', 't3', 'rh', 'setpoint', 'secflow', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'sdp', 'valve', 'valve2', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'alarms', 'dt', 'p3state', 'duty']
     for tagname in thesensors:

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -861,6 +861,23 @@ class IpmiHandler(object):
             resourcename = sensor['name']
             self.ipmicmd.sensormap[simplify_name(resourcename)] = resourcename
 
+    def read_normalized(self, sensorname):
+        readings = None
+        if sensorname == 'average_cpu_temp':
+            cputemp = self.ipmicmd.get_average_processor_temperature()
+            readings = [cputemp]
+        elif sensorname == 'inlet_temp':
+            inltemp = self.ipmicmd.get_inlet_temperature()
+            readings = [inltemp]
+        elif sensorname == 'total_power':
+            sensor = EmptySensor('Total Power')
+            sensor.states = []
+            sensor.units = 'W'
+            sensor.value = self.ipmicmd.get_system_power_watts()
+            readings = [sensor]
+        if readings:
+            self.output.put(msg.SensorReadings(readings, name=self.node))
+
     def read_sensors(self, sensorname):
         if sensorname == 'all':
             sensors = self.ipmicmd.get_sensor_descriptions()
@@ -1157,6 +1174,8 @@ class IpmiHandler(object):
         if len(self.element) < 3:
             return
         self.sensorcategory = self.element[2]
+        if self.sensorcategory == 'normalized':
+            return self.read_normalized(self.element[-1])
         # list sensors per category
         if len(self.element) == 3 and self.element[-2] == 'hardware':
             if self.sensorcategory == 'leds':

confluent_server/confluent/plugins/hardwaremanagement/redfish.py

@@ -712,6 +712,23 @@ class IpmiHandler(object):
             resourcename = sensor['name']
             self.sensormap[simplify_name(resourcename)] = resourcename
 
+    def read_normalized(self, sensorname):
+        readings = None
+        if sensorname == 'average_cpu_temp':
+            cputemp = self.ipmicmd.get_average_processor_temperature()
+            readings = [cputemp]
+        elif sensorname == 'inlet_temp':
+            inltemp = self.ipmicmd.get_inlet_temperature()
+            readings = [inltemp]
+        elif sensorname == 'total_power':
+            sensor = EmptySensor('Total Power')
+            sensor.states = []
+            sensor.units = 'W'
+            sensor.value = self.ipmicmd.get_system_power_watts()
+            readings = [sensor]
+        if readings:
+            self.output.put(msg.SensorReadings(readings, name=self.node))
+
     def read_sensors(self, sensorname):
         if sensorname == 'all':
             sensors = self.ipmicmd.get_sensor_descriptions()
@@ -1012,6 +1029,8 @@ class IpmiHandler(object):
         if len(self.element) < 3:
             return
         self.sensorcategory = self.element[2]
+        if self.sensorcategory == 'normalized':
+            return self.read_normalized(self.element[-1])
         # list sensors per category
         if len(self.element) == 3 and self.element[-2] == 'hardware':
             if self.sensorcategory == 'leds':

confluent_server/confluent/plugins/info/layout.py

@@ -0,0 +1,100 @@
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.core as core
+import confluent.messages as msg
+
+def retrieve(nodes, element, configmanager, inputdata):
+    locationinfo = configmanager.get_node_attributes(nodes,
+            (u'enclosure.manager', u'enclosure.bay', u'location.rack',
+             u'location.row', u'location.u', u'location.height'))
+    enclosuremap = {}
+    rackmap = {}
+    allnodedata = {}
+    needenclosures = set([])
+    locatednodes = set([])
+    for node in locationinfo:
+        nodeinfo = locationinfo[node]
+        rack = nodeinfo.get(u'location.rack', {}).get('value', '')
+        u = nodeinfo.get(u'location.u', {}).get('value', None)
+        row = nodeinfo.get(u'location.row', {}).get('value', '')
+        enclosure = nodeinfo.get(u'enclosure.manager', {}).get('value', None)
+        bay = nodeinfo.get(u'enclosure.bay', {}).get('value', None)
+        height = nodeinfo.get(u'location.height', {}).get('value', None)
+        if enclosure:
+            if enclosure not in enclosuremap:
+                enclosuremap[enclosure] = {}
+            enclosuremap[enclosure][bay] = node
+            if u:
+                if row not in rackmap:
+                    rackmap[row] = {}
+                if rack not in rackmap[row]:
+                    rackmap[row][rack] = {}
+                rackmap[row][rack][u] = {'node': enclosure, 'children': enclosuremap[enclosure]}
+                allnodedata[enclosure] = rackmap[row][rack][u]
+                if height:
+                    allnodedata[enclosure]['height'] = height
+            else: # need to see if enclosure lands in the map naturally or need to pull it
+                needenclosures.add(enclosure)
+        elif u:
+            if row not in rackmap:
+                rackmap[row] = {}
+            if rack not in rackmap[row]:
+                rackmap[row][rack] = {}
+            rackmap[row][rack][u] = {'node': node}
+            allnodedata[node] = rackmap[row][rack][u]
+            if height:
+                allnodedata[node]['height'] = height
+            locatednodes.add(node)
+    cfgenc = needenclosures - locatednodes
+    locationinfo = configmanager.get_node_attributes(cfgenc, (u'location.rack', u'location.row', u'location.u', u'location.height'))
+    for enclosure in locationinfo:
+        nodeinfo = locationinfo[enclosure]
+        rack = nodeinfo.get(u'location.rack', {}).get('value', '')
+        u = nodeinfo.get(u'location.u', {}).get('value', None)
+        row = nodeinfo.get(u'location.row', {}).get('value', '')
+        height = nodeinfo.get(u'location.height', {}).get('value', None)
+        if u:
+            allnodedata[enclosure] = {'node': enclosure, 'children': enclosuremap[enclosure]}
+            if height:
+                allnodedata[enclosure]['height'] = height
+            if row not in rackmap:
+                rackmap[row] = {}
+            if rack not in rackmap[row]:
+                rackmap[row][rack] = {}
+            rackmap[row][rack][u] = allnodedata[enclosure]
+    results = {
+        'errors': [],
+        'locations': rackmap,
+    }
+    for enclosure in enclosuremap:
+        if enclosure not in allnodedata:
+            results['errors'].append('Enclosure {} is missing required location information'.format(enclosure))
+        else:
+            allnodedata[enclosure]['children'] = enclosuremap[enclosure]
+    needheight = set([])
+    for node in allnodedata:
+        if 'height' not in allnodedata[node]:
+            needheight.add(node)
+    needheight = ','.join(needheight)
+    if needheight:
+        for rsp in core.handle_path(
+            '/noderange/{0}/description'.format(needheight),
+            'retrieve', configmanager,
+            inputdata=None):
+                kvp = rsp.kvpairs
+                for node in kvp:
+                    allnodedata[node]['height'] = kvp[node]['height']
+    yield msg.Generic(results)
+

confluent_server/confluent/selfservice.py

@@ -19,6 +19,12 @@ import json
 import os
 import time
 import yaml
+try:
+    from yaml import CSafeDumper as SafeDumper
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+    from yaml import SafeDumper
 import confluent.discovery.protocols.ssdp as ssdp
 import eventlet
 webclient = eventlet.import_patched('pyghmi.util.webclient')
@@ -31,7 +37,20 @@ currtzvintage = None
 
 
 def yamldump(input):
-    return yaml.safe_dump(input, default_flow_style=False)
+    return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False)
+
+def yamlload(input):
+    return yaml.load(input, Loader=SafeLoader)
+
+def listdump(input):
+    # special case yaml for flat dumb list
+    # this is about 25x faster than doing full yaml dump even with CSafeDumper
+    # with a 17,000 element list
+    retval = ''
+    for entry in input:
+        retval += '- ' + entry + '\n'
+    return retval
+
 
 def get_extra_names(nodename, cfg, myip=None):
     names = set([])
@@ -402,10 +421,13 @@ def handle_request(env, start_response):
                 yield node + '\n'
         else:
             start_response('200 OK', (('Content-Type', retype),))
-            yield dumper(list(util.natural_sort(nodes)))
+            if retype == 'application/yaml':
+                yield listdump(list(util.natural_sort(nodes)))
+            else:
+                yield dumper(list(util.natural_sort(nodes)))
     elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody:
         try:
-            reqbody = yaml.safe_load(reqbody)
+            reqbody = yamlload(reqbody)
         except Exception:
             reqbody = None
         cfgmod = reqbody.get('configmod', 'unspecified')
@@ -419,7 +441,7 @@ def handle_request(env, start_response):
         start_response('200 Ok', ())
         yield 'complete'
     elif env['PATH_INFO'] == '/self/updatestatus' and reqbody:
-        update = yaml.safe_load(reqbody)
+        update = yamlload(reqbody)
         statusstr = update.get('state', None)
         statusdetail = update.get('state_detail', None)
         didstateupdate = False
@@ -522,7 +544,7 @@ def handle_request(env, start_response):
             '/var/lib/confluent/public/os/{0}/scripts/{1}')
         if slist:
             start_response('200 OK', (('Content-Type', 'application/yaml'),))
-            yield yaml.safe_dump(util.natural_sort(slist), default_flow_style=False)
+            yield yamldump(util.natural_sort(slist))
         else:
             start_response('200 OK', ())
             yield ''

confluent_server/confluent/sockapi.py

@@ -141,6 +141,8 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             if 'collective' in response:
                 return collective.handle_connection(connection, cert,
                                             response['collective'])
+            while not configmanager.config_is_ready():
+                eventlet.sleep(1)
             if 'dispatch' in response:
                 dreq = tlvdata.recvall(connection, response['dispatch']['length'])
                 return pluginapi.handle_dispatch(connection, cert, dreq,

confluent_server/confluent/sshutil.py

@@ -98,14 +98,15 @@ def initialize_ca():
          preexec_fn=normalize_uid)
     ouid = normalize_uid()
     try:
-        os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755)
-    except OSError as e:
-        if e.errno != 17:
-            raise
+        try:
+            os.makedirs('/var/lib/confluent/public/site/ssh/', mode=0o755)
+        except OSError as e:
+            if e.errno != 17:
+                raise
+        cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname)
+        shutil.copy('/etc/confluent/ssh/ca.pub', cafilename)
     finally:
         os.seteuid(ouid)
-    cafilename = '/var/lib/confluent/public/site/ssh/{0}.ca'.format(myname)
-    shutil.copy('/etc/confluent/ssh/ca.pub', cafilename)
     #    newent = '@cert-authority * ' + capub.read()
 
 
@@ -129,11 +130,21 @@ def prep_ssh_key(keyname):
             ap.write('#!/bin/sh\necho $CONFLUENT_SSH_PASSPHRASE\nrm {0}\n'.format(askpass))
         os.chmod(askpass, 0o700)
         os.environ['CONFLUENT_SSH_PASSPHRASE'] = get_passphrase()
+        olddisplay = os.environ.get('DISPLAY', None)
+        oldaskpass = os.environ.get('SSH_ASKPASS', None)
         os.environ['DISPLAY'] = 'NONE'
         os.environ['SSH_ASKPASS'] = askpass
-        with open(os.devnull, 'wb') as devnull:
-            subprocess.check_output(['ssh-add', keyname], stdin=devnull, stderr=devnull)
-        del os.environ['CONFLUENT_SSH_PASSPHRASE']
+        try:
+            with open(os.devnull, 'wb') as devnull:
+                subprocess.check_output(['ssh-add', keyname], stdin=devnull, stderr=devnull)
+        finally:
+            del os.environ['CONFLUENT_SSH_PASSPHRASE']
+            del os.environ['DISPLAY']
+            del os.environ['SSH_ASKPASS']
+            if olddisplay:
+                os.environ['DISPLAY'] = olddisplay
+            if oldaskpass:
+                os.environ['SSH_ASKPASS'] = oldaskpass
         ready_keys[keyname] = 1
     finally:
         adding_key = False
@@ -175,6 +186,14 @@ def initialize_root_key(generate, automation=False):
         if os.path.exists('/etc/confluent/ssh/automation'):
             alreadyexist = True
         else:
+            ouid = normalize_uid()
+            try:
+                os.makedirs('/etc/confluent/ssh', mode=0o700)
+            except OSError as e:
+                if e.errno != 17:
+                    raise
+            finally:
+                os.seteuid(ouid)
             subprocess.check_call(
                 ['ssh-keygen', '-t', 'ed25519',
                 '-f','/etc/confluent/ssh/automation', '-N', get_passphrase(),

confluent_server/confluent/syncfiles.py

@@ -212,6 +212,8 @@ def sync_list_to_node(sl, node, suffixes, peerip=None):
                     unreadablefiles.append(filename.replace(targdir, ''))
         if unreadablefiles:
             raise Exception("Syncing failed due to unreadable files: " + ','.join(unreadablefiles))
+        elif b'Permission denied, please try again.' in e.stderr:
+            raise Exception('Syncing failed due to authentication error, is the confluent automation key not set up (osdeploy initialize -a) or is there some process replacing authorized_keys on the host?')
         else:
             raise
     finally:

confluent_server/confluent/util.py

@@ -29,9 +29,9 @@ import struct
 import eventlet.green.subprocess as subprocess
 
 
-def mkdirp(path):
+def mkdirp(path, mode=0o777):
     try:
-        os.makedirs(path)
+        os.makedirs(path, mode)
     except OSError as e:
         if e.errno != 17:
             raise
@@ -42,7 +42,7 @@ def run(cmd):
     stdout, stderr = process.communicate()
     retcode = process.poll()
     if retcode:
-        raise subprocess.CalledProcessError(retcode, process.args, output=stdout)
+        raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr)
     return stdout, stderr
 
 

confluent_server/setup.py.tmpl

@@ -19,6 +19,7 @@ setup(
               'confluent/plugins/hardwaremanagement/',
               'confluent/plugins/deployment/',
               'confluent/plugins/console/',
+              'confluent/plugins/info/',
               'confluent/plugins/shell/',
               'confluent/collective/',
               'confluent/plugins/configuration/'],

confluent_vtbufferd/NOTICE

@@ -22,3 +22,16 @@ modification, are permitted provided that the following conditions are met:
   * Neither the name of the copyright holder nor the
     names of contributors may be used to endorse or promote products
     derived from this software without specific prior written permission.
+
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS,
+* COPYRIGHT HOLDERS, OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

confluent_vtbufferd/vtbufferd.c

@@ -1,8 +1,14 @@
+#include <asm-generic/socket.h>
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <locale.h>
 #include <unistd.h>
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <sys/un.h>
+#include <fcntl.h>
 #include "tmt.h"
 #define HASHSIZE 2053
 #define MAXNAMELEN 256
@@ -10,13 +16,17 @@
 struct terment {
     struct terment *next;
     char *name;
+    int fd;
     TMT *vt;
 };
 
 #define SETNODE 1
 #define WRITE 2
 #define READBUFF 0
+#define CLOSECONN 3
+#define MAXEVTS 16
 static struct terment *buffers[HASHSIZE];
+static char* nodenames[HASHSIZE];
 
 unsigned long hash(char *str)
 /* djb2a */
@@ -37,10 +47,13 @@ TMT *get_termentbyname(char *name) {
     return NULL;
 }
 
-TMT *set_termentbyname(char *name) {
+TMT *set_termentbyname(char *name, int fd) {
     struct terment *ret;
     int idx;
 
+    if (nodenames[fd] == NULL) {
+        nodenames[fd] = strdup(name);
+    }
     idx = hash(name);
     for (ret = buffers[idx]; ret != NULL; ret = ret->next)
         if (strcmp(name, ret->name) == 0)
@@ -48,12 +61,13 @@ TMT *set_termentbyname(char *name) {
     ret = (struct terment *)malloc(sizeof(*ret));
     ret->next = buffers[idx];
     ret->name = strdup(name);
+    ret->fd = fd;
     ret->vt = tmt_open(31, 100, NULL, NULL, L"→←↑↓■◆▒°±▒┘┐┌└┼⎺───⎽├┤┴┬│≤≥π≠£•");
     buffers[idx] = ret;
     return ret->vt;
 }
 
-void dump_vt(TMT* outvt) {
+void dump_vt(TMT* outvt, int outfd) {
     const TMTSCREEN *out = tmt_screen(outvt);
     const TMTPOINT *curs = tmt_cursor(outvt);
     int line, idx, maxcol, maxrow;
@@ -67,9 +81,10 @@ void dump_vt(TMT* outvt) {
     tmt_color_t fg = TMT_COLOR_DEFAULT;
     tmt_color_t bg = TMT_COLOR_DEFAULT;
     wchar_t sgrline[30];
+    char strbuffer[128];
     size_t srgidx = 0;
     char colorcode = 0;
-    wprintf(L"\033c");
+    write(outfd, "\033c", 2);
     maxcol = 0;
     maxrow = 0;
     for (line = out->nline - 1; line >= 0; --line) {
@@ -148,60 +163,136 @@ void dump_vt(TMT* outvt) {
             }
             if (sgrline[0] != 0) {
                 sgrline[wcslen(sgrline) - 1] = 0;  // Trim last ;
-                wprintf(L"\033[%lsm", sgrline);
+
+                snprintf(strbuffer, sizeof(strbuffer), "\033[%lsm", sgrline);
+                write(outfd, strbuffer, strlen(strbuffer));
+                write(outfd, "\033[]", 3);
             }
-            wprintf(L"%lc", out->lines[line]->chars[idx].c);
+            snprintf(strbuffer, sizeof(strbuffer), "%lc", out->lines[line]->chars[idx].c);
+            write(outfd, strbuffer, strlen(strbuffer));
         }
         if (line < maxrow)
-            wprintf(L"\r\n");
+            write(outfd, "\r\n", 2);
     }
-    fflush(stdout);
-    wprintf(L"\x1b[%ld;%ldH", curs->r + 1, curs->c + 1);
-    fflush(stdout);
+    //fflush(stdout);
+    snprintf(strbuffer, sizeof(strbuffer), "\x1b[%ld;%ldH", curs->r + 1, curs->c + 1);
+    write(outfd, strbuffer, strlen(strbuffer));
+    //fflush(stdout);
+}
+
+int handle_traffic(int fd) {
+    int cmd, length;
+    char currnode[MAXNAMELEN];
+    char cmdbuf[MAXDATALEN];
+    char *nodename;
+    TMT *currvt = NULL;
+    TMT *outvt = NULL;
+    length = read(fd, &cmd, 4);
+    if (length <= 0) {
+        return 0;
+    }
+    length = cmd & 536870911;
+    cmd = cmd >> 29;
+    if (cmd == SETNODE) {
+        cmd = read(fd, currnode, length);
+        currnode[length] = 0;
+        if (cmd < 0)
+            return 0;
+        currvt = set_termentbyname(currnode, fd);
+    } else if (cmd == WRITE) {
+        if (currvt == NULL) {
+            nodename = nodenames[fd];
+            currvt = set_termentbyname(nodename, fd);
+        }
+        cmd = read(fd, cmdbuf, length);
+        cmdbuf[length] = 0;
+        if (cmd < 0)
+            return 0;
+        tmt_write(currvt, cmdbuf, length);
+    } else if (cmd == READBUFF) {
+        cmd = read(fd, cmdbuf, length);
+        cmdbuf[length] = 0;
+        if (cmd < 0)
+            return 0;
+        outvt = get_termentbyname(cmdbuf);
+        if (outvt != NULL)
+            dump_vt(outvt, fd);
+        length = write(fd, "\x00", 1);
+        if (length < 0)
+            return 0;
+    } else if (cmd == CLOSECONN) {
+        return 0;
+    }
+    return 1;
 }
 
 int main(int argc, char* argv[]) {
-    int cmd, length;
     setlocale(LC_ALL, "");
-    char cmdbuf[MAXDATALEN];
-    char currnode[MAXNAMELEN];
-    TMT *currvt = NULL;
-    TMT *outvt = NULL;
+    struct sockaddr_un addr;
+    int numevts;
+    int status;
+    int poller;
+    int n;
+    socklen_t len;
+    int ctlsock, currsock;
+    socklen_t addrlen;
+    struct ucred ucr;
+
+    struct epoll_event epvt, evts[MAXEVTS];
     stdin = freopen(NULL, "rb", stdin);
     if (stdin == NULL) {
         exit(1);
     }
+    memset(&addr, 0, sizeof(struct sockaddr_un));
+    addr.sun_family = AF_UNIX;
+    strncpy(addr.sun_path + 1, argv[1], sizeof(addr.sun_path) - 2); // abstract namespace socket
+    ctlsock = socket(AF_UNIX, SOCK_STREAM, 0);
+    status = bind(ctlsock, (const struct sockaddr*)&addr, sizeof(sa_family_t) + strlen(argv[1]) + 1); //sizeof(struct sockaddr_un));
+    if (status < 0) {
+        perror("Unable to open unix socket - ");
+        exit(1);
+    }
+    listen(ctlsock, 128);
+    poller = epoll_create(1);
+    memset(&epvt, 0, sizeof(struct epoll_event));
+    epvt.events = EPOLLIN;
+    epvt.data.fd = ctlsock;
+    if (epoll_ctl(poller, EPOLL_CTL_ADD, ctlsock, &epvt) < 0) {
+        perror("Unable to poll the socket");
+        exit(1);
+    }
+    // create a unix domain socket for accepting, each connection is only allowed to either read or write, not both
     while (1) {
-        length = fread(&cmd, 4, 1, stdin);
-        if (length < 0)
-            continue;
-        length = cmd & 536870911;
-        cmd = cmd >> 29;
-        if (cmd == SETNODE) {
-            cmd = fread(currnode, 1, length, stdin);
-            currnode[length] = 0;
-            if (cmd < 0)
-                continue;
-            currvt = set_termentbyname(currnode);
-        } else if (cmd == WRITE) {
-            if (currvt == NULL)
-                currvt = set_termentbyname("");
-            cmd = fread(cmdbuf, 1, length, stdin);
-            cmdbuf[length] = 0;
-            if (cmd < 0)
-                continue;
-            tmt_write(currvt, cmdbuf, length);
-        } else if (cmd == READBUFF) {
-            cmd = fread(cmdbuf, 1, length, stdin);
-            cmdbuf[length] = 0;
-            if (cmd < 0)
-                continue;
-            outvt = get_termentbyname(cmdbuf);
-            if (outvt != NULL)
-                dump_vt(outvt);
-            length = write(1, "\x00", 1);
-            if (length < 0)
-                continue;
+        numevts = epoll_wait(poller, evts, MAXEVTS, -1);
+        if (numevts < 0) {
+            perror("Failed wait");
+            exit(1);
+        }
+        for (n = 0; n < numevts; ++n) {
+            if (evts[n].data.fd == ctlsock) {
+                currsock = accept(ctlsock, (struct sockaddr *) &addr, &addrlen);
+                len = sizeof(ucr);
+                getsockopt(currsock, SOL_SOCKET, SO_PEERCRED, &ucr, &len);
+                if (ucr.uid != getuid()) { // block access for other users
+                    close(currsock);
+                    continue;
+                }
+                memset(&epvt, 0, sizeof(struct epoll_event));
+                epvt.events = EPOLLIN;
+                epvt.data.fd = currsock;
+                epoll_ctl(poller, EPOLL_CTL_ADD, currsock, &epvt);
+            } else {
+                if (!handle_traffic(evts[n].data.fd)) {
+                    epoll_ctl(poller, EPOLL_CTL_DEL, evts[n].data.fd, NULL);
+                    close(evts[n].data.fd);
+                    if (nodenames[evts[n].data.fd] != NULL) {
+                        free(nodenames[evts[n].data.fd]);
+                        nodenames[evts[n].data.fd] = NULL;
+                    }
+                }
+            }
         }
     }
 }
+
+

imgutil/imgutil

@@ -23,7 +23,10 @@ import subprocess
 import sys
 import tempfile
 import time
-import yaml
+try:
+    import yaml
+except ImportError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -58,13 +61,27 @@ FALLOC_FL_PUNCH_HOLE = 2
 numregex = re.compile('([0-9]+)')
 
 def get_partition_info():
+    with open('/proc/self/mountinfo') as procinfo:
+        mountinfo = procinfo.read()
+    capmounts = set([])
+    for entry in mountinfo.split('\n'):
+        if not entry:
+            continue
+        firstinf, lastinf = entry.split(' - ')
+        root, mount = firstinf.split()[3:5]
+        filesystem = lastinf.split()[0]
+        if root != '/':
+            continue
+        if filesystem not in ('ext3', 'ext4', 'xfs', 'btrfs', 'vfat'):
+            continue
+        capmounts.add(mount)
     with open('/proc/mounts') as procmounts:
         mountinfo = procmounts.read()
     for entry in mountinfo.split('\n'):
         if not entry:
             continue
         dev, mount, fs, flags = entry.split()[:4]
-        if fs not in ('ext3', 'ext4', 'xfs', 'btrfs', 'vfat'):
+        if mount not in capmounts:
             continue
         fsinfo = os.statvfs(mount)
         partinfo = {
@@ -196,7 +213,13 @@ def capture_remote(args):
     finfo = subprocess.check_output(['ssh', targ, 'python3', '/run/imgutil/capenv/imgutil', 'getfingerprint']).decode('utf8')
     finfo = json.loads(finfo)
     if finfo['oscategory'] not in ('el8', 'el9', 'ubuntu20.04', 'ubuntu22.04'):
-        raise Exception('Not yet supported for capture: ' + repr(finfo))
+        sys.stderr.write('Not yet supported for capture: ' + repr(finfo) + '\n')
+        sys.exit(1)
+    unmet = finfo.get('unmetprereqs', [])
+    if unmet:
+        for cmd in unmet:
+            sys.stderr.write(cmd + '\n')
+        sys.exit(1)
     oscat = finfo['oscategory']
     subprocess.check_call(['ssh', '-o', 'LogLevel=QUIET', '-t', targ, 'python3', '/run/imgutil/capenv/imgutil', 'capturelocal'])
     utillib = __file__.replace('bin/imgutil', 'lib/imgutil')        
@@ -437,10 +460,12 @@ def get_mydir(oscategory):
 class OsHandler(object):
     def __init__(self, name, version, arch, args):
         self.name = name
+        self._interactive = True
         self.version = version
         self.arch = arch
         self.sourcepath = None
         self.osname = '{}-{}-{}'.format(name, version, arch)
+        self.captureprereqs = []
         try:
             pkglist = args.packagelist
         except AttributeError:
@@ -464,13 +489,16 @@ class OsHandler(object):
         except AttributeError:
             self.addrepos = []
 
+    def set_interactive(self, shouldbeinteractive):
+        self._interactive = shouldbeinteractive
+
     def get_json(self):
         odata = [self.oscategory, self.version, self.arch, self.name]
         for idx in range(len(odata)):
             if not isinstance(odata[idx], str):
                 odata[idx] = odata[idx].decode('utf8')
         info = {'oscategory': odata[0],
-                'version': odata[1], 'arch': odata[2], 'name': odata[3]}
+                'version': odata[1], 'arch': odata[2], 'name': odata[3], 'unmetprereqs': self.captureprereqs}
         return json.dumps(info)
 
     def prep_root_premount(self, args):
@@ -577,7 +605,10 @@ class SuseHandler(OsHandler):
         cmd = ['chmod', 'a+x']
         cmd.extend(glob.glob(os.path.join(targdir, '*')))
         subprocess.check_call(cmd)
-        subprocess.check_call(['zypper', '-R', self.targpath, 'install'] + self.zyppargs)
+        if self._interactive:
+            subprocess.check_call(['zypper', '-R', self.targpath, 'install'] + self.zyppargs)
+        else:
+            subprocess.check_call(['zypper', '-n', '-R', self.targpath, 'install'] + self.zyppargs)
         os.symlink('/usr/lib/systemd/system/sshd.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service'))
         if os.path.exists(os.path.join(self.targpath, 'sbin/mkinitrd')):
             args.cmd = ['mkinitrd']
@@ -587,12 +618,21 @@ class SuseHandler(OsHandler):
 
 
 class DebHandler(OsHandler):
-    def __init__(self, name, version, arch, args, codename):
+    def __init__(self, name, version, arch, args, codename, hostpath):
         self.includepkgs = []
         self.targpath = None
         self.codename = codename
         self.oscategory = name + version
         super().__init__(name, version, arch, args)
+        needpkgs = []
+        if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
+            needpkgs.append('tpm2-tools')
+        lfuses = glob.glob(os.path.join(hostpath, '/lib/*/libfuse.so.2'))
+        if not lfuses:
+            needpkgs.append('libfuse2')
+        if needpkgs:
+            needapt = 'Missing packages needed in target for capture, to add required packages: apt install ' + ' '.join(needpkgs)
+            self.captureprereqs.append(needapt)
 
     def add_pkglists(self):
         self.includepkgs.extend(self.list_packages())
@@ -620,11 +660,27 @@ class DebHandler(OsHandler):
 
 
 class ElHandler(OsHandler):
-    def __init__(self, name, version, arch, args):
+    def __init__(self, name, version, arch, args, hostpath='/'):
         self.oscategory = 'el{0}'.format(version.split('.')[0])
         self.yumargs = []
         super().__init__(name, version, arch, args)
-
+        needpkgs = []
+        if not hostpath:
+            return
+        if not os.path.exists(os.path.join(hostpath, 'usr/bin/tpm2_getcap')):
+            needpkgs.append('tpm2-tools')
+        lfuses = glob.glob(os.path.join(hostpath, '/usr/lib64/libfuse.so.2'))
+        if not lfuses:
+            needpkgs.append('fuse-libs')
+        if not os.path.exists(os.path.join(hostpath, '/usr/bin/ipcalc')):
+            needpkgs.append('ipcalc')
+        if not os.path.exists(os.path.join(hostpath, 'usr/sbin/dhclient')):
+            needpkgs.append('dhcp-client')
+        if not os.path.exists(os.path.join(hostpath, 'usr/sbin/mount.nfs')):
+            needpkgs.append('nfs-utils')
+        if needpkgs:
+            needapt = 'Missing packages needed in target for capture, to add required packages: dnf install ' + ' '.join(needpkgs)
+            self.captureprereqs.append(needapt)
 
     def add_pkglists(self):
         self.yumargs.extend(self.list_packages())
@@ -657,7 +713,10 @@ class ElHandler(OsHandler):
         cmd = ['chmod', 'a+x']
         cmd.extend(glob.glob(os.path.join(targdir, '*')))
         subprocess.check_call(cmd)
-        subprocess.check_call(['yum'] + self.yumargs)
+        if self._interactive:
+            subprocess.check_call(['yum'] + self.yumargs)
+        else:
+            subprocess.check_call(['yum', '-y'] + self.yumargs)
         with open('/proc/mounts') as mountinfo:
             for line in mountinfo.readlines():
                 if line.startswith('selinuxfs '):
@@ -794,6 +853,7 @@ def main():
     buildp.add_argument('-a', '--addpackagelist', action='append', default=[],
                         help='A list of additional packages to include, may be specified multiple times')
     buildp.add_argument('-s', '--source', help='Directory to pull installation from, typically a subdirectory of /var/lib/confluent/distributions. By default, the repositories for the build system are used.')
+    buildp.add_argument('-y', '--non-interactive', help='Avoid prompting for confirmation', action='store_true')
     buildp.add_argument('-v', '--volume',
         help='Directory to make available in the build environment.  -v / will '
         'cause it to be mounted in image as /run/external/, -v /:/run/root '
@@ -996,7 +1056,7 @@ def fingerprint_source_el(files, sourcepath, args):
             if arch == 'noarch':
                 prodinfo = open(os.path.join(sourcepath, '.discinfo')).read()
                 arch = prodinfo.split('\n')[2]
-            return ElHandler(osname, ver, arch, args)
+            return ElHandler(osname, ver, arch, args, None)
     return None
 
 
@@ -1048,7 +1108,7 @@ def fingerprint_host_el(args, hostpath='/'):
     osname = osname.replace('-release', '').replace('-', '_')
     if osname == 'centos_linux':
         osname = 'centos'
-    return ElHandler(osname, version, os.uname().machine, args)
+    return ElHandler(osname, version, os.uname().machine, args, hostpath)
 
 
 def fingerprint_host_deb(args, hostpath='/'):
@@ -1072,7 +1132,7 @@ def fingerprint_host_deb(args, hostpath='/'):
     except IOError:
         pass
     if osname:
-        return DebHandler(osname, vers, os.uname().machine, args, codename)
+        return DebHandler(osname, vers, os.uname().machine, args, codename, hostpath)
 
 
 def fingerprint_host_suse(args, hostpath='/'):
@@ -1128,6 +1188,8 @@ def build_root(args):
             sys.stderr.write(
                 'Unable to recognize build system os\n')
         sys.exit(1)
+    if args.non_interactive:
+        oshandler.set_interactive(False)
     oshandler.set_target(args.scratchdir)
     oshandler.add_pkglists()
     for dirname in ('proc', 'sys', 'dev', 'run'):

misc/swraid

@@ -1,6 +1,6 @@
 DEVICES="/dev/sda /dev/sdb"
 RAIDLEVEL=1
-mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null && exit 0
+mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null || (
 lvm vgchange -a n
 mdadm -S -s
 NUMDEVS=$(for dev in $DEVICES; do
@@ -14,5 +14,6 @@ mdadm -C /dev/md/raid $DEVICES -n $NUMDEVS -e 1.0 -l $RAIDLEVEL
 # shut and restart array to prime things for anaconda
 mdadm -S -s
 mdadm --assemble --scan
+)
 readlink /dev/md/raid|sed -e 's/.*\///' > /tmp/installdisk