Move up creation of ca.pem

ssh module was pausing input for the
entire websocket while doing the simple 'write' operation.

Change to background the actual
logon processing,
rather than blocking what should be a fairly trivial write operation.

confluent_client/bin/dir2img

@@ -21,17 +21,17 @@ def create_image(directory, image, label=None):
             currsz = (currsz // 512 +1) * 512
             datasz += currsz
     datasz += ents * 32768
-    datasz = datasz // 4096 + 1
+    datasz = datasz // 16384 + 1
     with open(image, 'wb') as imgfile:
-        imgfile.seek(datasz * 4096 - 1)
+        imgfile.seek(datasz * 16384 - 1)
         imgfile.write(b'\x00')
     if label:
         subprocess.check_call(['mformat', '-i', image, '-v', label,
                                '-r', '16', '-d', '1', '-t', str(datasz),
-                               '-s', '4','-h', '2', '::'])
+                               '-s', '16','-h', '2', '::'])
     else:
         subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
-                                str(datasz), '-s', '4','-h', '2', '::'])
+                                str(datasz), '-s', '16','-h', '2', '::'])
     # Some clustered filesystems will have the lock from mformat
     # linger after close (mformat doesn't unlock)
     # do a blocking wait for shared lock and then explicitly

confluent_client/bin/nodeapply

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+devnull = None
+
+def run():
+    global devnull
+    devnull = open(os.devnull, 'rb')
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-k', '--security', action='store_true',
+                         help='Update SSH setup')
+    argparser.add_option('-F', '--sync', action='store_true',
+                         help='Run the syncfiles associated with the currently completed OS profile on the noderange')
+    argparser.add_option('-P', '--scripts',
+                         help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                    help='Specify a maximum number of '
+                        'nodes to run remote ssh command to, '
+                        'prompting if over the threshold')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    #argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    c.stop_if_noderange_over(args[0], options.maxnodes)
+    nodemap = {}
+    cmdparms = []
+    nodes = []
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
+    
+    cmdstorun = []
+    if options.security:
+        cmdstorun.append(['run_remote', 'setupssh'])
+    if options.sync:
+        cmdstorun.append(['run_remote_python', 'syncfileclient'])
+    if options.scripts:
+        for script in options.scripts.split(','):
+            cmdstorun.append(['run_remote', script])
+    if not cmdstorun:
+        argparser.print_help()
+        sys.exit(1)
+    idxbynode = {}
+    cmdvbase = ['bash', '/etc/confluent/functions']
+    for sshnode in nodes:
+        idxbynode[sshnode] = 1
+        cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(node, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((node, cmdv))        
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if idxbynode[node] < len(cmdstorun):
+                                cmdv =  ['ssh', sshnode] + cmdvbase + cmdstorun[idxbynode[node]]
+                                idxbynode[node] += 1
+                                run_cmdv(node, cmdv, all, pipedesc)
+                            elif pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                line = line.lstrip('\x08')
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodeconfig

@@ -90,6 +90,12 @@ cfgpaths = {
     'bmc.ipv4_gateway': (
         'configuration/management_controller/net_interfaces/management',
         'ipv4_gateway'),
+    'bmc.static_ipv6_addresses': (
+        'configuration/management_controller/net_interfaces/management',
+        'static_v6_addresses'),
+    'bmc.static_ipv6_gateway': (
+        'configuration/management_controller/net_interfaces/management',
+        'static_v6_gateway'),
     'bmc.hostname': (
         'configuration/management_controller/hostname', 'hostname'),
 }

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -23,13 +23,17 @@ path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
+
 import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
+import time
+import socket
+import re
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] node",
+    usage="Usage: %prog [options] <noderange> [kill][-- [passthroughoptions]]",
     epilog="Command sequences are available while connected to a console, hit "
            "ctrl-'e', then release ctrl, then 'c', then '?' for a full list. "
            "For example, ctrl-'e', then 'c', then '.' will exit the current "
@@ -38,6 +42,10 @@ argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                      help='Enter log replay mode instead of showing a live console')
+
+argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
+                     help= 'Dump log in stdout with timestamps')
+
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -58,10 +66,26 @@ argparser.add_option('-w','--windowed', action='store_true', default=False,
                           '--shell-type login".  If the NODECONSOLE_WINDOWED_COMMAND '
                           'environment variable isn\'t set, xterm will be used by'
                           'default.')
+
 (options, args) = argparser.parse_args()
+
+pass_through_args = []
+killcon = False
+try:
+    noderange = args[0]
+    if len(args) > 1:
+        if args[1] == 'kill':
+            killcon = True
+        pass_through_args = args[1:]
+        args = args[:1]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+
 if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
+
 if options.log:
     logname = args[0]
     if not os.path.exists(logname) and logname[0] != '/':
@@ -71,14 +95,105 @@ if options.log:
         sys.exit(1)
     logreader.replay_to_console(logname)
     sys.exit(0)
-#added functionality for wcons
 
-if options.windowed:
+if options.Timestamp:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.dump_to_console(logname)
+    sys.exit(0)
+
+def kill(noderange):
+    sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
     if not envstring:
-        envlist=["xterm", "-e"]
+        envstring = 'xterm'
+
+    nodes = []
+    for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
+        node = res.get('item', {}).get('href', '/').replace('/', '')
+        if not node:
+            sys.stderr.write(res.get('error', repr(res)) + '\n')
+            sys.exit(1)
+        nodes.append(node) 
+
+    for node in nodes:
+        s=socket.socket(socket.AF_UNIX)
+        winid=None
+        try:
+            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
+            wintr=win.communicate()[0]
+            for line in wintr.decode('utf-8').split('\n'):
+                if 'console: {0}'.format(node) in line or 'confetty' in line:
+                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
+                    winid = win_obj[0]
+        except:
+            print("Error: cannot retrieve window id of node {}".format(node))
+
+        if winid:
+            ps_data=subprocess.Popen(['xkill', '-id', winid ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    sys.exit(0)
+
+def handle_geometry(envlist, sizegeometry, side_pad=0, top_pad=0, first=False):    
+    if '-geometry' in envlist:
+        g_index = envlist.index('-geometry')
+    elif '-g' in envlist:
+        g_index = envlist.index('-g')
+    else:
+        g_index = 0
+    if g_index:   
+        if first:
+             envlist[g_index+1] = '{0}+{1}+{2}'.format(envlist[g_index+1],side_pad, top_pad)
+        else:
+            envlist[g_index+1] = '{0}+{1}+{2}'.format(sizegeometry,side_pad, top_pad)
+    else:
+        envlist.insert(1, '-geometry')
+        envlist.insert(2, '{0}+{1}+{2}'.format(sizegeometry,side_pad, top_pad))
+        g_index = 1
+    return envlist     
+
+# add funcltionality to close/kill all open consoles
+if killcon:
+    kill(noderange)
+    
+#added functionality for wcons
+if options.windowed:
+    result=subprocess.Popen(['xwininfo', '-root'], stdout=subprocess.PIPE)
+    rootinfo=result.communicate()[0]
+    result.wait()
+    for line in rootinfo.decode('utf-8').split('\n'):
+        if 'Width' in line:
+            screenwidth = int(line.split(':')[1])
+        if 'Height' in line:
+            screenheight = int(line.split(':')[1])
+
+    envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
+    if not envstring:
+        sizegeometry='100x31'
+        corrected_x, corrected_y = (13,84)
+        envlist = handle_geometry(['xterm'] + pass_through_args + ['-e'],sizegeometry, first=True)
+        #envlist=['xterm', '-bg', 'black', '-fg', 'white', '-geometry', '{sizegeometry}+0+0'.format(sizegeometry=sizegeometry), '-e']
     else:
         envlist=os.environ.get('NODECONSOLE_WINDOWED_COMMAND').split(' ')
+        if envlist[0] == 'xterm':
+            if '-geometry' in envlist:
+                g_index = envlist.index('-geometry')
+            elif '-g' in envlist:
+                g_index = envlist.index('-g')
+            else:
+                g_index = 0
+            if g_index:    
+                envlist[g_index+1] = envlist[g_index+1] + '+0+0'
+
+            else:
+                envlist.insert(1, '-geometry')
+                envlist.insert(2, '100x31+0+0')
+                g_index = 1
+            
     nodes = []
     sess = client.Command()
     for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
@@ -87,9 +202,87 @@ if options.windowed:
             sys.stderr.write(res.get('error', repr(res)) + '\n')
             sys.exit(1)
         nodes.append(node)
+
+    if options.tile and not envlist[0] == 'xterm':
+        sys.stderr.write('[ERROR] UNSUPPORTED OPTIONS. \nWindowed and tiled consoles are only supported when using xterm \n')
+        sys.exit(1)
+    firstnode=nodes[0]
+    nodes.pop(0)
+    with open(os.devnull, 'wb') as devnull:
+        xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(firstnode), '-m', '5', 'start', '/nodes/{0}/console/session'.format(firstnode) ] , stdin=devnull)
+        time.sleep(2)
+        s=socket.socket(socket.AF_UNIX)
+        winid=''
+        try:
+            s.connect('/tmp/controlpath-{firstnode}'.format(firstnode=firstnode))
+            s.recv(64)
+            s.send(b'GETWINID')
+            winid=s.recv(64).decode('utf-8')
+
+        except:
+            time.sleep(2)
+            # try to get id of first panel/xterm window using name
+            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
+            wintr=win.communicate()[0]
+            for line in wintr.decode('utf-8').split('\n'):
+                if 'console: {firstnode}'.format(firstnode=firstnode) in line or 'confetty' in line:
+                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
+                    winid = win_obj[0]
+    if winid:
+        firstnode_window=subprocess.Popen(['xwininfo', '-id', '{winid}'.format(winid=winid)], stdout=subprocess.PIPE)
+        xinfo=firstnode_window.communicate()[0]
+        xinfl = xinfo.decode('utf-8').split('\n')
+        for line in xinfl:
+            if 'Absolute upper-left X:' in line:
+                side_pad = int(line.split(':')[1])
+            elif 'Absolute upper-left Y:' in line:
+                top_pad = int(line.split(':')[1])
+
+            elif 'Width:' in line:
+                window_width = int(line.split(':')[1])
+            elif 'Height' in line:
+                window_height = int(line.split(':')[1])
+            elif '-geometry' in line:
+                l = re.split(' |x|-|\+', line)
+                l_nosp = [ele for ele in l if ele.strip()]
+                wmxo = int(l_nosp[1])
+                wmyo = int(l_nosp[2])
+                sizegeometry = str(wmxo) + 'x' + str(wmyo)
+            else:
+                pass
+
+        window_width += side_pad*2
+        window_height += side_pad+top_pad
+        screenwidth -= wmxo
+        screenheight -= wmyo
+        currx = window_width
+        curry = 0
+        maxcol = int(screenwidth/window_width)
+
     for node in sortutil.natural_sort(nodes):
+        if options.tile and envlist[0] == 'xterm':
+            corrected_x = currx
+            corrected_y = curry
+            xgeometry = '{0}+{1}+{2}'.format(sizegeometry, corrected_x, corrected_y)
+            currx += window_width
+            if currx >= screenwidth:
+                currx=0
+                curry += window_height
+                if curry > screenheight:
+                    curry =top_pad
+            if not envstring:        
+                envlist= handle_geometry(envlist, sizegeometry, corrected_x, corrected_y)
+            else:
+                if g_index:    
+                    envlist[g_index+1] = xgeometry
+        elif envlist[0] == 'xterm':
+            envlist=handle_geometry(envlist, sizegeometry, side_pad, top_pad)
+            side_pad+=(side_pad+1)
+            top_pad+=(top_pad+30)
+        else:
+            pass 
         with open(os.devnull, 'wb') as devnull:
-            subprocess.Popen(envlist + [confettypath, '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)], stdin=devnull)
+            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node), '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
     sys.exit(0)
 #end of wcons
 if options.tile:
@@ -103,8 +296,13 @@ if options.tile:
             sys.exit(1)
         nodes.append(node)
     initial = True
+    in_tmux = False
     pane = 0
     sessname = 'nodeconsole_{0}'.format(os.getpid())
+    if os.environ.get("TMUX"):
+        initial = False
+        in_tmux = True
+        subprocess.call(['tmux', 'rename-session', sessname])
     for node in sortutil.natural_sort(nodes):
         panename = '{0}:{1}'.format(sessname, pane)
         if initial:
@@ -125,7 +323,8 @@ if options.tile:
             pane += 1
     subprocess.call(['tmux', 'select-pane', '-t', sessname])
     subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
+    if not in_tmux:
+        os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
              '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodedeploy

@@ -49,7 +49,7 @@ def armonce(nr, cli):
 
 
 def setpending(nr, profile, cli):
-    args = {'deployment.pendingprofile': profile}
+    args = {'deployment.pendingprofile': profile, 'deployment.state': '', 'deployment.state_detail': ''}
     if not profile.startswith('genesis-'):
         args['deployment.stagedprofile'] = ''
         args['deployment.profile'] = ''
@@ -132,7 +132,7 @@ def main(args):
                 if node not in databynode:
                     databynode[node] = {}
                 for attr in dbn[node]:
-                    if attr in ('deployment.pendingprofile', 'deployment.apiarmed', 'deployment.stagedprofile', 'deployment.profile'):
+                    if attr in ('deployment.pendingprofile', 'deployment.apiarmed', 'deployment.stagedprofile', 'deployment.profile', 'deployment.state', 'deployment.state_detail'):
                         databynode[node][attr] = dbn[node][attr].get('value', '')
         for node in sortutil.natural_sort(databynode):
             profile = databynode[node].get('deployment.pendingprofile', '')
@@ -153,7 +153,18 @@ def main(args):
                 armed = ' (node authentication armed)'
             else:
                 armed = ''
-            print('{0}: {1}{2}'.format(node, profile, armed))
+            stateinfo = ''
+            deploymentstate = databynode[node].get('deployment.state', '')
+            if deploymentstate:
+                statedetails = databynode[node].get('deployment.state_detail', '')
+                if statedetails:
+                    stateinfo = '{}: {}'.format(deploymentstate, statedetails)
+                else:
+                    stateinfo = deploymentstate
+            if stateinfo:
+                print('{0}: {1} ({2})'.format(node, profile, stateinfo))
+            else:
+                print('{0}: {1}{2}'.format(node, profile, armed))
         sys.exit(0)
     if args.network and not args.prepareonly:
         return rc

confluent_client/bin/nodeeventlog

@@ -17,6 +17,7 @@
 
 import codecs
 from datetime import datetime as dt
+from datetime import timedelta
 import optparse
 import os
 import signal
@@ -41,7 +42,18 @@ argparser = optparse.OptionParser(
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to clear if clearing log, '
-                          'prompting if over the threshold')            
+                          'prompting if over the threshold')   
+argparser.add_option('-l', '--lines', type='int',
+                     help='return the last <n> entries  '
+                          'for each node in the eventlog. '
+                          )
+argparser.add_option('-t', '--timeframe', type='string',
+                     help='return entries within a specified timeframe  '
+                          'for each node in the eventlog. This will return '
+                          'entries from the last hours or days. '
+                          '1h would be one hour, 4d would be four days. '
+                          'format <num>h or <num>d'
+                          )  
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -89,12 +101,34 @@ def format_event(evt):
         msg = ''
     return ' '.join(retparts) + msg
 
-
 if deletemode:
     func = session.delete
     session.stop_if_noderange_over(noderange, options.maxnodes)
 else:
     func = session.read
+
+if options.timeframe:
+    try:
+        delta = int(options.timeframe[:-1])
+    except ValueError:
+        argparser.print_help()
+        sys.exit(1)
+    if options.timeframe[-1].lower() == 'd':
+        tdelta = timedelta(days=delta)
+    elif options.timeframe[-1].lower() == 'h':
+        tdelta = timedelta(hours=delta)
+    else:
+        argparser.print_help()
+        sys.exit(1)
+    timeframe = dt.now() - tdelta
+
+event_dict = {}
+nodes = []
+for res in session.read('/noderange/{0}/nodes/'.format(args[0])):
+    node = res.get('item', {}).get('href', '/').replace('/', '')
+    nodes.append(node)
+    event_dict[node] = []
+
 for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
     if 'error' in rsp:
         sys.stderr.write(rsp['error'] + '\n')
@@ -107,6 +141,31 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
                sys.stderr.write('{0}: {1}\n'.format(node, thisdata['error']))
                exitcode |= 1
             if 'events' in thisdata:
-               evtdata = thisdata['events']
-               for evt in evtdata:
-                   print('{0}: {1}'.format(node, format_event(evt)))
+                evtdata = thisdata['events']
+                if options.lines:
+                    event_dict[node].extend(evtdata)
+                else:
+                    for evt in evtdata:
+                        if options.timeframe:
+                            # check if line is in timeframe 
+                            if 'timestamp' in evt and evt['timestamp'] is not None:
+                                display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
+                                if display > timeframe:                          
+                                    print('{0}: {1}'.format(node, format_event(evt))) 
+                        else:
+                            print('{0}: {1}'.format(node, format_event(evt))) 
+
+if options.lines:
+    for node in nodes:
+        evtdata_list = event_dict[node]
+        if len(evtdata_list) != 0:
+            if len(evtdata_list) > options.lines:
+                evtdata_list = evtdata_list[-abs(options.lines):]
+            for evt in evtdata_list:
+                if options.timeframe:
+                    if 'timestamp' in evt and evt['timestamp'] is not None:
+                        display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
+                        if display > timeframe:                          
+                            print('{0}: {1}'.format(node, format_event(evt)))
+                else:
+                    print('{0}: {1}'.format(node, format_event(evt)))

confluent_client/bin/nodeinventory

@@ -53,6 +53,8 @@ def print_mem_info(node, prefix, meminfo):
         memdescfmt += '3-{1} '
     elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
+    elif 'DDR5' in meminfo['memory_type']:
+        memdescfmt += '5-{1} '
     elif 'DCPMM' in meminfo['memory_type']:
         memdescfmt = '{0}GB {1} '
         meminfo['module_type'] = 'DCPMM'
@@ -99,6 +101,7 @@ usedprefixes = set([])
 argparser = optparse.OptionParser(
     usage="Usage: %prog <noderange> [serial|model|uuid|mac]")
 argparser.add_option('-j', '--json', action='store_true', help='Output JSON')
+argparser.add_option('-s', '--store', action='store_true', help='Store serial, model, and uuid into id.serial, id.model, and id.uuid')
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -126,17 +129,36 @@ if len(args) > 1:
 try:
     if options.json:
         databynode = {}
+    if options.store and len(args) <= 1:
+        url = '/noderange/{0}/inventory/hardware/all/system'
+    pushattribs = {}
     session = client.Command()
     for res in session.read(url.format(noderange)):
         printerror(res)
         if 'databynode' not in res:
             continue
         for node in res['databynode']:
+            if options.store and node not in pushattribs:
+                pushattribs[node] = {}
             printerror(res['databynode'][node], node)
             if 'inventory' not in res['databynode'][node]:
                 continue
             for inv in res['databynode'][node]['inventory']:
                 prefix = inv['name']
+                if options.store and prefix == 'System':
+                    currinfo = inv.get('information', {})
+                    curruuid = currinfo.get('UUID', '')
+                    if curruuid:
+                        curruuid = curruuid.lower()
+                        pushattribs[node]['id.uuid'] = curruuid
+                    currserial = currinfo.get('Serial Number', '')
+                    if currserial:
+                        currserial = currserial.strip()
+                        pushattribs[node]['id.serial'] = currserial
+                    currmodelnum = currinfo.get('Model', '')
+                    if currmodelnum:
+                        currmodelnum = currmodelnum.strip()
+                        pushattribs[node]['id.model'] = currmodelnum
                 idx = 2
                 while (node, prefix) in usedprefixes:
                     prefix = '{0} {1}'.format(inv['name'], idx)
@@ -148,7 +170,7 @@ try:
                             if node not in databynode:
                                 databynode[node] = {}
                             databynode[node][prefix] = inv
-                        else:
+                        elif not options.store:
                             print('{0}: {1}: Not Present'.format(node, prefix))
                     continue
                 info = inv['information']
@@ -179,12 +201,18 @@ try:
                             databynode[node] = {}
                         databynode[node][prefix] = inv
                         break
-                    print(u'{0}: {1} {2}: {3}'.format(node, prefix,
-                                                     pretty(datum),
-                                                     info[datum]))
+                    elif not options.store:
+                        print(u'{0}: {1} {2}: {3}'.format(node, prefix,
+                                                         pretty(datum),
+                                                         info[datum]))
     if options.json:
         print(json.dumps(databynode, sort_keys=True, indent=4,
                          separators=(',', ': ')))
+    if pushattribs:
+        for node in pushattribs:
+            for rsp in session.update('/nodes/{0}/attributes/current'.format(node), pushattribs[node]):
+                if 'error' in rsp:
+                    sys.stderr.write(rsp['error'] + '\n')
 except KeyboardInterrupt:
     print('')
 sys.exit(exitcode)

confluent_client/bin/nodesensors

@@ -37,6 +37,12 @@ class hybridcsv(csv.excel):
     lineterminator = '\n'
 
 
+def floatformat(num):
+    fm = u'{:.5f}'.format(num).rstrip('0')
+    if fm[-1:] == u'.':
+        return fm + u'0'
+    return fm
+
 csv.register_dialect('hybrid', hybridcsv)
 
 import confluent.client as client
@@ -135,7 +141,7 @@ def sensorpass(showout=True, appendtime=False):
                         if sensedata['value'] is None:
                             showval = ''
                         elif isinstance(sensedata['value'], float):
-                            showval = u' {0:.5f} '.format(sensedata['value'])
+                            showval = u' {0} '.format(floatformat(sensedata['value']))
                         else:
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):
@@ -191,6 +197,8 @@ def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
                             datum = ','.join([datum, healthstates])
                         else:
                             datum = healthstates
+                if isinstance(datum, float):
+                    datum = floatformat(datum)
                 rowdata.append(datum)
             except KeyError:
                 rowdata.append('N/A')

confluent_client/confluent/logreader.py

@@ -296,5 +296,47 @@ def replay_to_console(txtfile):
             sys.exit(1)
     _replay_to_console(txtfile, binfile)
 
+def dump_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    elif '.' not in txtfile:
+        if '/' not in txtfile:
+            txtfile = os.getcwd() + '/' + txtfile
+        sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(txtfile + '.cbl'))
+        sys.exit(1)
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(binfile))
+            sys.exit(1)
+    replay = LogReplay(txtfile, binfile)
+    quitit = False
+    writeout('\x1b[2J\x1b[;H')
+    prepend = ''
+    try:
+        while not quitit:
+            newdata, delay = replay.get_output(False)
+            if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
+                writeout(newdata.replace(b'\r\n', '\r\n[ {0} ] '.format(time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))).encode('utf8')))
+                newdata = ''
+                try:
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                except IOError:
+                    pass
+            else:
+                break
+    except Exception:
+        writeout('\x1b[m\x1b[?25h\n')
+        raise
+    writeout('\x1b[m\x1b[?25h\n')
+
 if __name__ == '__main__':
     replay_to_console(sys.argv[1])

confluent_client/doc/man/nodeconsole.ronn

@@ -2,7 +2,7 @@ nodeconsole(8) -- Open a console to a confluent node
 =====================================================
 
 ## SYNOPSIS
-`nodeconsole [options] <noderange>`
+`nodeconsole [options] <noderange> [kill][-- [passthroughoptions]]`
 
 ## DESCRIPTION
 
@@ -16,6 +16,9 @@ will initiate an automatic retry interval that is randomized between 2 and 4 min
 The reopen escape sequence below requests an immediate retry, as does connecting
 a new session.
 
+When a windowed console is open the `nodeconsole <noderange> kill` command will kill the
+console process which will result in the console window closing. 
+
 ## OPTIONS
 
 * `-t`, `--tile`:
@@ -26,6 +29,11 @@ a new session.
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+
+  * `-T`, `--Timestamp`:
+ Dump the log with Timpstamps on the current, local log in /var/log/confluent/consoles.
+  If in collective mode, this only makes sense to use on the current collective
+  manager at this time.
   
 * `-w`, `--windowed`:
   Open terminal windows for each node.  The
@@ -86,3 +94,14 @@ keystroke will be interpreted as a command.  The following commands are availabl
   Get a list of supported commands
 * `<ent>`:
   Hit enter to skip entering a command at the escape prompt.
+
+
+## PASSTHROUGH OPTIONS
+
+While opening a windowed console with xterm or any other console of choice. The 
+nodeconsole command gives capality to specify passthrough options targeted at 
+the console. All options after the -- will be parsed the console program. For 
+example, opening a windowed console using xterm with a black background. 
+`nodeconconsole -w n1 -- -bg black`  
+
+

confluent_client/doc/man/nodeeventlog.ronn

@@ -15,6 +15,15 @@ noderange.
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to clear if clearing log, prompting if 
   over the threshold 
+
+* `-l LINES`, `--lines=LINES`:
+    return the last <n> entries for each node in the eventlog. 
+
+* `-t TIMEFRAME`, `--timeframe=TIMEFRAME`:
+   return entries within a specified timeframe for each node's event log. 
+   This will return entries from the last <n> hours or days. 1h would be 
+   entries from with the last one hour. 
+                           
   
 * `-h`, `--help`:
   Show help message and exit  

confluent_osdeploy/buildrpm

@@ -11,13 +11,24 @@ mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
 cd -
 mkdir -p el9bin/opt/confluent/bin
 mkdir -p el9bin/stateless-bin
-podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuilder make -C /buildutils
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuild make -C /buildutils
 cd utils
-mv confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
-mv start_root urlmount ../el9bin/stateless-bin/
+cp confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
+cp start_root urlmount ../el9bin/stateless-bin/
+make clean
+cd ..
+mkdir -p el8bin/opt/confluent/bin
+mkdir -p el8bin/stateless-bin
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t el7build make -C /buildutils
+cd utils
+cp confluent_imginfo copernicus clortho autocons ../el8bin/opt/confluent/bin
+cp start_root urlmount ../el8bin/stateless-bin/
 make clean
 cd ..
 tar Jcvf confluent_el9bin.tar.xz el9bin/
+tar Jcvf confluent_el8bin.tar.xz el8bin/
+mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
+rm -rf el8bin
 rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/buildrpm-aarch64

@@ -0,0 +1,32 @@
+VERSION=`git describe|cut -d- -f 1`
+NUMCOMMITS=`git describe|cut -d- -f 2`
+if [ "$NUMCOMMITS" != "$VERSION"  ]; then
+    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+fi
+sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy-aarch64.spec.tmpl > confluent_osdeploy-aarch64.spec
+cd ..
+cp ../LICENSE .
+tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy
+mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
+cd -
+mkdir -p el9bin/opt/confluent/bin
+mkdir -p el9bin/stateless-bin
+mkdir -p el8bin/opt/confluent/bin
+mkdir -p el8bin/stateless-bin
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t el9builder make -C /buildutils
+cd utils
+mv confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
+mv start_root urlmount ../el9bin/stateless-bin/
+cd ..
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t el8builder make -C /buildutils
+cd utils
+mv confluent_imginfo copernicus clortho autocons ../el8bin/opt/confluent/bin
+mv start_root urlmount ../el8bin/stateless-bin/
+cd ..
+tar Jcvf confluent_el9bin.tar.xz el9bin/
+tar Jcvf confluent_el8bin.tar.xz el8bin/
+mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
+mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
+rm -rf el9bin
+rm -rf el8bin
+rpmbuild -ba confluent_osdeploy-aarch64.spec

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -21,6 +21,10 @@ class InvalidApiKey(Exception):
 def msg_align(len):
     return (len + 3) & ~3
 
+try:
+    _protoparm = ssl.PROTOCOL_TLS_CLIENT
+except AttributeError:    
+    _protoparm = ssl.PROTOCOL_SSLv23
 cryptname = ctypes.util.find_library('crypt')
 if not cryptname:
     if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
@@ -253,6 +257,7 @@ class HTTPSClient(client.HTTPConnection, object):
                     self.stdheaders['CONFLUENT_NODENAME'] = node
                 if line.startswith('MANAGER:') and not host:
                     host = line.split(' ')[1]
+                    self.hosts.append(host)
                     if not plainhost:
                         plainhost = host
                 if line.startswith('EXTMGRINFO:'):
@@ -304,6 +309,10 @@ class HTTPSClient(client.HTTPConnection, object):
     def check_connections(self):
         foundsrv = None
         hosts = self.hosts
+        ctx = ssl.SSLContext(_protoparm)
+        ctx.load_verify_locations('/etc/confluent/ca.pem')
+        ctx.verify_mode = ssl.CERT_REQUIRED
+        ctx.check_hostname = True
         for timeo in (0.1, 5):
             for host in hosts:
                 try:
@@ -311,10 +320,16 @@ class HTTPSClient(client.HTTPConnection, object):
                     psock = socket.socket(addrinf[0])
                     psock.settimeout(timeo)
                     psock.connect(addrinf[4])
+                    chost = host.split('%', 1)[0]
+                    ctx.wrap_socket(psock, server_hostname=chost)
                     foundsrv = host
                     psock.close()
                     break
-                except OSError:
+                except (OSError, socket.error):
+                    continue
+                except ssl.SSLError:
+                    continue
+                except ssl.CertificateError:
                     continue
             else:
                 continue
@@ -342,11 +357,12 @@ class HTTPSClient(client.HTTPConnection, object):
                 raise Exception('Unable to reach any hosts')
         return foundsrv
 
-    def connect(self):
+    def connect(self, tries=3):
         addrinf = socket.getaddrinfo(self.host, self.port)[0]
         psock = socket.socket(addrinf[0])
+        psock.settimeout(15)
         psock.connect(addrinf[4])
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         host = self.host.split('%', 1)[0]
         if '[' not in host and ':' in host:
@@ -364,6 +380,10 @@ class HTTPSClient(client.HTTPConnection, object):
                 self.errout.write(errmsg)
                 self.errout.flush()
             sys.exit(1)
+        except socket.timeout:
+            if not tries:
+                raise
+            return self.connect(tries=tries-1)
 
     def grab_url(self, url, data=None, returnrsp=False):
         return self.grab_url_with_status(url, data, returnrsp)[1]

confluent_osdeploy/common/profile/scripts/confignet

@@ -296,6 +296,10 @@ class NetworkManager(object):
                 subprocess.check_call(['nmcli', 'c', 'u', u])
             else:
                 subprocess.check_call(['nmcli', 'c', 'add', 'type', self.devtypes[iname], 'con-name', cname, 'connection.interface-name', iname] + cargs)
+                self.read_connections()
+                u = self.uuidbyname.get(cname, None)
+                if u:
+                    subprocess.check_call(['nmcli', 'c', 'u', u])
 
 
 
@@ -312,7 +316,10 @@ if __name__ == '__main__':
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
     for srv in srvs:
-        s = socket.create_connection((srv, 443))
+        try:
+            s = socket.create_connection((srv, 443))
+        except socket.error:
+            continue
         myname = s.getsockname()
         s.close()
         if len(myname) == 4:
@@ -344,6 +351,13 @@ if __name__ == '__main__':
                     else:
                         netname_to_interfaces[uname] = {'interfaces': set([iname]), 'settings': nc['extranets'][netname]}
         doneidxs.add(curridx)
+    if 'default' in netname_to_interfaces:
+        for netn in netname_to_interfaces:
+            if netn == 'default':
+                continue
+            netname_to_interfaces['default']['interfaces'] -= netname_to_interfaces[netn]['interfaces']
+        if not netname_to_interfaces['default']['interfaces']:
+            del netname_to_interfaces['default']
     rm_tmp_llas(tmpllas)
     if os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes)

confluent_osdeploy/common/profile/scripts/setupssh

@@ -0,0 +1,31 @@
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    rm $certfile
+    confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
+done
+TMPDIR=$(mktemp -d)
+cd $TMPDIR
+confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz
+tar xf initramfs.tgz
+for ca in ssh/*.ca; do
+	LINE=$(cat $ca)
+	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
+	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
+done
+for pubkey in ssh/*.*pubkey; do
+	LINE=$(cat $pubkey)
+	cp -af /root/.ssh/authorized_keys /root/.ssh/authorized_keys.new
+	grep -v "$LINE" /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.new
+	echo "$LINE" >> /root/.ssh/authorized_keys.new
+	mv /root/.ssh/authorized_keys.new /root/.ssh/authorized_keys
+done
+confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' > /etc/ssh/shosts.equiv
+cat /etc/ssh/shosts.equiv > /root/.shosts
+cd -
+rm -rf $TMPDIR

confluent_osdeploy/confluent_osdeploy-aarch64.spec.tmpl

@@ -0,0 +1,91 @@
+Name: confluent_osdeploy-aarch64
+Version: #VERSION#
+Release: 1
+Summary: OS Deployment support for confluent
+
+License: Apache2
+URL: https://hpc.lenovo.com/
+Source0: confluent_osdeploy.tar.xz
+Source1: confluent_el9bin.tar.xz
+Source2: confluent_el8bin.tar.xz
+BuildArch: noarch
+Requires: confluent_ipxe mtools tar
+BuildRoot: /tmp
+
+%description
+This contains support utilities for enabling deployment of aarch64 architecture systems
+
+
+%define debug_package %{nil}
+
+%prep
+%setup -n confluent_osdeploy -a 2 -a 1
+
+%build
+mkdir -p opt/confluent/bin
+mkdir -p stateless-bin
+cp -a el8bin/* .
+ln -s el8 el9
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
+    mkdir ${os}out
+    cd ${os}out
+    if [ -d ../${os}bin ]; then 
+        cp -a ../${os}bin/opt .
+    else
+        cp -a ../opt .
+    fi
+    cp -a ../${os}/initramfs/* .
+    cp -a ../common/initramfs/* .
+    find . | cpio -H newc -o > ../addons.cpio
+    mv ../addons.cpio .
+    cd ..
+done
+for os in el7 el8 suse15 el9 ubuntu20.04; do
+    mkdir ${os}disklessout
+    cd ${os}disklessout
+    if [ -d ../${os}bin ]; then
+        cp -a ../${os}bin/opt .
+    else
+        cp -a ../opt .
+    fi
+    cp -a ../${os}-diskless/initramfs/* .
+    cp -a ../common/initramfs/* .
+    if [ -d ../${os}bin ]; then 
+        cp -a ../${os}bin/stateless-bin/* opt/confluent/bin
+    else
+        cp -a ../stateless-bin/* opt/confluent/bin
+    fi
+    find . | cpio -H newc -o > ../addons.cpio
+    mv ../addons.cpio .
+    cd ..
+done
+mkdir esxi7out
+cd esxi7out
+cp -a ../opt .
+cp -a ../esxi7/initramfs/* .
+cp -a ../common/initramfs/* .
+chmod +x bin/* opt/confluent/bin/*
+tar zcvf ../addons.tgz *
+mv ../addons.tgz .
+cd ..
+cp -a esxi7out esxi6out
+cp -a esxi7 esxi6
+cp -a esxi7out esxi8out
+cp -a esxi7 esxi8
+
+%install
+mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+#cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
+    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
+    cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs/aarch64/
+    if [ -d ${os}disklessout ]; then
+        mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs/aarch64/
+        cp ${os}disklessout/addons.* %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs/aarch64/
+    fi
+done
+find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} +
+
+%files
+/opt/confluent/lib/osdeploy
+#%license /opt/confluent/share/licenses/confluent_osdeploy/LICENSE

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -7,6 +7,7 @@ License: Apache2
 URL: https://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
 Source1: confluent_el9bin.tar.xz
+Source2: confluent_el8bin.tar.xz
 BuildArch: noarch
 Requires: confluent_ipxe mtools tar
 BuildRoot: /tmp
@@ -18,24 +19,22 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 %define debug_package %{nil}
 
 %prep
-%setup -n confluent_osdeploy -a 1
+%setup -n confluent_osdeploy -a 2 -a 1
 
 %build
-mkdir -p opt/confluent/bin
-mkdir -p stateless-bin
-cd utils
-make all
-cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
-cp start_root urlmount ../stateless-bin/
-cd ..
+#cd utils
+#make all
+#cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
+#cp start_root urlmount ../stateless-bin/
+#cd ..
 ln -s el8 el9
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreos el9; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}/initramfs/* .
     cp -a ../common/initramfs/* .
@@ -49,14 +48,14 @@ for os in el7 el8 suse15 el9 ubuntu20.04; do
     if [ -d ../${os}bin ]; then
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}-diskless/initramfs/* .
     cp -a ../common/initramfs/* .
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/stateless-bin/* opt/confluent/bin
     else
-        cp -a ../stateless-bin/* opt/confluent/bin
+        cp -a ../el8bin/stateless-bin/* opt/confluent/bin
     fi
     find . | cpio -H newc -o > ../addons.cpio
     mv ../addons.cpio .
@@ -64,7 +63,7 @@ for os in el7 el8 suse15 el9 ubuntu20.04; do
 done
 mkdir esxi7out
 cd esxi7out
-cp -a ../opt .
+cp -a ../el8bin/opt .
 cp -a ../esxi7/initramfs/* .
 cp -a ../common/initramfs/* .
 chmod +x bin/* opt/confluent/bin/*
@@ -79,7 +78,7 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/debian/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -69,7 +69,7 @@ if [ "$v4cfgmeth" = "static" ]; then
     for namesrv in "$namesrvs"; do
         setdebopt netcfg/get_nameservers $namesrv string
     done
-elif [ "$vpcfgmeth" = "dhcp" ]; then
+elif [ "$v4cfgmeth" = "dhcp" ]; then
     setdebopt netcfg/disable_dhcp false boolean
     setdebopt netcfg/confirm_static false boolean
 fi

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -7,41 +7,111 @@ if [ -f /tmp/dd_disk ]; then
         fi
     done
 fi
+oum=$(umask)
+umask 0077
 mkdir -p /etc/confluent
-cat /tls/*.pem > /etc/confluent/ca.pem
-TRIES=0
 echo -n > /etc/confluent/confluent.info
-cd /sys/class/net
-while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    echo "NODENAME: "$nodename > /etc/confluent/confluent.info
+    for dsrv in $deploysrvs; do
+        echo 'MANAGER: '$dsrv >> /etc/confluent/confluent.info
+    done
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    usedhcp=0
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            usedhcp=1
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            TESTSRV=$(python /opt/confluent/bin/apiclient -c 2> /dev/null)
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip -4 addr flush dev $NICGUESS
+            if [ ! -z "$TESTSRV" ]; then
+                mgr=$TESTSRV
+                ifname=$NICGUESS
+                break
+            fi
+        fi
+    done
+fi
+TRIES=0
+if [ "$usedhcp" = 1 ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+elif [ -z "$ifname" ]; then
+    cd /sys/class/net
+    while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    cd /
+    grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+    echo -n "" > /etc/cmdline.d/01-confluent.conf
+else
+    echo -n ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none > /etc/cmdline.d/01-confluent.conf
+fi
 echo -n "" > /tmp/confluent.initq
 # restart cmdline
-echo -n "" > /etc/cmdline.d/01-confluent.conf
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 #TODO: blkid --label <whatever> to find mounted api
-
-oum=$(umask)
 python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-ifidx=$(cat /tmp/confluent.ifidx)
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-echo $ifname > /tmp/net.ifaces
-
+if [ -z "$ifname"  ]; then
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+if [ ! -z "$ifname" ]; then
+    echo $ifname > /tmp/net.ifaces
+fi
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 hostname=$nodename
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
     hostname=$hostname.$dnsdomain
 fi
-mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
-mgr=${mgr#deploy_server: }
+if [ -z "$mgr" ]; then
+    mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+    mgr=${mgr#deploy_server: }
+fi
 profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 profilename=${profilename#profile: }
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
@@ -65,21 +135,23 @@ kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
 root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
 export kickstart
 export root
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
-autoconfigmethod=${autoconfigmethod#ipv4_method: }
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
+if [ -z "$autoconfigmethod" ]; then
+    autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+    else
+        v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
     fi
-    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#ipv4_netmask: }
-    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
 nameserversec=0
 while read -r entry; do

confluent_osdeploy/el7/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,8 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +55,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +130,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +146,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -184,6 +220,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +261,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -366,6 +403,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +418,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
 . /lib/dracut-lib.sh
+setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' &
 udevadm trigger
 udevadm trigger --type=devices --action=add
 udevadm settle
@@ -40,7 +41,7 @@ mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
 TRIES=5
-while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 2
     TRIES=$((TRIES - 1))
 done
@@ -55,21 +56,49 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
     autoconfigmethod=$(grep ^ipv4_method: $tcfg)
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    if [ "$autoconfigmethod" = "dhcp" ]; then
-        /usr/libexec/nm-initrd-generator ip=:dhcp
-    else
-        v4addr=$(grep ^ipv4_address: $tcfg)
-        v4addr=${v4addr#ipv4_address: }
-        v4addr=${v4addr%/*}
-        v4gw=$(grep ^ipv4_gateway: $tcfg)
-        v4gw=${v4gw#ipv4_gateway: }
-        if [ "$v4gw" = "null" ]; then
-            v4gw=""
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            DETECTED=0
+            for dsrv in $deploysrvs; do
+                if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then
+                    rm /run/NetworkManager/system-connections/*
+                    /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none
+                    DETECTED=1
+                    ifname=$NICGUESS
+                    break
+                fi
+            done
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip addr flush dev $NICGUESS
+            if [ $DETECTED = 1 ]; then
+                break
+            fi
         fi
-        v4nm=$(grep ipv4_netmask: $tcfg)
-        v4nm=${v4nm#ipv4_netmask: }
-        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
-    fi
+    done
     NetworkManager --configure-and-quit=initrd --no-daemon
     hmackeyfile=/tmp/cnflnthmackeytmp
     echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
@@ -85,10 +114,10 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
     done
     for deployer in $deploysrvs; do
-        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+        if curl --capath /tls/ -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
             cp $passfile /etc/confluent/confluent.apikey
             confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+            curl --capath /tls/ -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
             confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
             confluent_profile=${confluent_profile#profile: }
             mgr=$deployer
@@ -100,7 +129,9 @@ cd /sys/class/net
 if ! grep MANAGER: /etc/confluent/confluent.info; then
     confluentsrv=$(getarg confluent)
     if [ ! -z "$confluentsrv" ]; then
+        mgr=$confluentsrv
         if [[ "$confluentsrv" = *":"* ]]; then
+            mgr="[$mgr]"
             confluenthttpsrv=[$confluentsrv]
             /usr/libexec/nm-initrd-generator ip=:dhcp6
         else
@@ -112,7 +143,7 @@ if ! grep MANAGER: /etc/confluent/confluent.info; then
         for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
             myids=$myids"/mac="$mac
         done
-        myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+        myname=$(curl --capath /tls/ -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
         if [ ! -z "$myname" ]; then
             echo NODENAME: $myname > /etc/confluent/confluent.info
             echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
@@ -146,15 +177,17 @@ fi
 while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg; do
 	sleep 10
 done
-ifidx=$(cat /tmp/confluent.ifidx)
-if [ ! -z "$ifidx" ]; then
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-    ifname=${ifname%@*}
-    echo $ifname > /tmp/net.ifaces
-else
-    ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
-    ifname=$(cat /tmp/net.ifaces)
+ifidx=$(cat /tmp/confluent.ifidx 2> /dev/null)
+if [ -z "$ifname" ]; then
+    if [ ! -z "$ifidx" ]; then
+        ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+        ifname=${ifname%:}
+        ifname=${ifname%@*}
+        echo $ifname > /tmp/net.ifaces
+    else
+        ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
+        ifname=$(cat /tmp/net.ifaces)
+    fi
 fi
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)

confluent_osdeploy/el8/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -2,6 +2,10 @@
 sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
-mkdir -p $2/boot/efi/boot && \
-ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+mkdir -p $2/boot/efi/boot
+if [ -e $1/EFI/BOOT/BOOTAA64.EFI ]; then
+    ln -s $1/EFI/BOOT/BOOTAA64.EFI $1/EFI/BOOT/grubaa64.efi $2/boot/efi/boot/
+else
+    ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+fi
 

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as
@@ -33,16 +35,21 @@ export nodename confluent_mgr confluent_profile
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
 chmod 600 /var/log/confluent/confluent-firstboot.log
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+    confluentpython /root/confignet
+    rm /root/confignet
+fi
+
+
 while ! ping -c 1 $confluent_pingtarget >& /dev/null; do
 	sleep 1
 done
 
+
 if [ ! -f /etc/confluent/firstboot.ran ]; then
     touch /etc/confluent/firstboot.ran
 
-    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
-    run_remote_python confignet
-
     run_remote firstboot.custom
     # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
     run_remote_parts firstboot.d

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -47,5 +47,8 @@ run_remote_parts post.d
 
 # Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
 run_remote_config post.d
+cd /root
+fetch_remote confignet
+cd -
 curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/updatestatus
 kill $logshowpid

confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,8 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +55,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +130,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +146,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -184,6 +220,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +261,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -366,6 +403,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +418,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -94,6 +94,7 @@ while ! grep NODENAME /etc/confluent/confluent.info; do
     fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+cat /tls/*.pem > /etc/confluent/ca.pem
 mgr=$(python /opt/confluent/bin/apiclient -c)
 APIKEY=$(cat /etc/confluent/confluent.apikey)
 
@@ -108,7 +109,6 @@ while [ -z "$APIKEY" ]; do
     (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
     APIKEY=$(cat /etc/confluent/confluent.apikey)
 done
-cat /tls/*.pem > /etc/confluent/ca.pem
 /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -33,6 +33,8 @@ for info in vswinfo.split('\n'):
     upinfo = uplinkmatch.match(info)
     if upinfo:
         vmnic = upinfo.group(1)
+        if vmnic and 'vusb0' not in vmnic:
+            break
 try:
     with open('/tmp/confluentident/cnflnt.jsn') as identin:
         identcfg = json.load(identin)

confluent_osdeploy/genesis/profiles/default/ansible/onboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -161,7 +161,7 @@ else
     echo -e "BOOTPROTO='static'\nSTARTMODE='auto'" >> /run/confluent/ifcfg-$ifname
     echo "IPADDR='$v4addr/$v4nm'" >> /run/confluent/ifcfg-$ifname
     if [ ! -z "$v4gw" ]; then
-        echo defafult $v4gw - $ifname > /run/confluent/ifroute-$ifname
+        echo default $v4gw - $ifname > /run/confluent/ifroute-$ifname
     fi
 fi
 

confluent_osdeploy/suse15-diskless/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -77,6 +77,7 @@ if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null && [
 	sed -e s'/$/ 'console=${autocons#*/dev/}/ /proc/cmdline > /etc/fakecmdline
 	mount -o bind /etc/fakecmdline /proc/cmdline
 	echo "ConsoleDevice: ${autocons%,*}" >> /etc/linuxrc.d/01-confluent
+	echo "Textmode: 1" >> /etc/linuxrc.d/01-confluent
 fi
 
 tz=$(grep timezone: /etc/confluent/confluent.deploycfg | awk '{print $2}')

confluent_osdeploy/suse15/profiles/hpc/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/suse15/profiles/server/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -0,0 +1,205 @@
+reverse_uuid() {
+    echo $1 | head -c 8 | tail -c 2
+    echo $1 | head -c 6 | tail -c 2
+    echo $1 | head -c 4 | tail -c 2
+    echo $1 | head -c 2 | tail -c 2
+    echo $1 | head -c 13 | tail -c 2
+    echo $1 | head -c 11 | tail -c 2
+    echo $1 | head -c 18 | tail -c 2
+    echo $1 | head -c 16 | tail -c 2
+    echo $1 | tail -c 18 | sed -e s/-//
+}
+setdebopt() {
+    debconf-set $1 $2
+    echo d-i $1 $3 $2 >> /preseed.cfg
+}
+
+mkdir -p /etc/confluent
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) up
+done
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) down
+    udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+    ip link set $(basename $i) up
+done
+cp -a /tls/* /etc/ssl/certs/
+mkdir -p /etc/confluent
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|cut -f 2 -d ' ')
+    echo NODENAME: $nodename > /etc/confluent/confluent.info
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: $tcfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress ${v4addr%/*} string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            for dsrv in $deploysrvs; do
+                if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    setdebopt netcfg/choose_interface $NIC select
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done 
+        #TODO: nameservers
+    elif [ "$v4cfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            udhcpc $NICGUESS
+        done
+        for dsrv in $deploysrvs; do
+            if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                deploysrvs=$dsrv
+            fi
+        done
+    fi
+    mgr=$deploysrvs
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|cut -d ' ' -f 2) > $hmackeyfile
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" --post-file=$passcrypt https://$mgr/confluent-api/self/registerapikey -O - --quiet
+    cp $passfile /etc/confluent/confluent.apikey
+    nic=$NIC
+else
+    dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
+    dhcpid=$(mktemp)
+    mkdir -p /etc/confluent
+    cp /tls/* /etc/ssl/certs/
+    cat /tls/*.pem >> /etc/confluent/ca.pem
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
+        ip link set $nic up
+    done
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+        if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+            /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+            if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+                break
+            fi
+        fi
+        ip -4 flush dev $nic
+    done
+    mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    /opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+fi
+apikey=$(cat /etc/confluent/confluent.apikey)
+cd /etc/confluent
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg
+cd -
+predir=$(mktemp -d)
+cd $predir
+cp /etc/confluent/deploycfg /etc/confluent/confluent.deploycfg
+profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for namesrv in "$namesrvs"; do
+    setdebopt netcfg/get_nameservers $namesrv string
+done
+rootpass=$(grep ^rootpassword: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //')
+if [ "$rootpass" = null ] || [ -z "$rootpass" ]; then
+    setdebopt passwd/root-login false boolean
+else
+    setdebopt passwd/root-login true boolean
+    setdebopt passwd/root-password-crypted $rootpass string
+fi
+setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //') string
+ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for ntpsrv in "$ntpsrvs"; do
+    setdebopt clock-setup/ntp true boolean
+    setdebopt clock-setup/ntp-server $ntpsrv string
+done
+setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select
+domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+if [ ! -z "$domainname" ] && [ "$domainname" != "null" ]; then
+    setdebopt netcfg/get_domain $domainname string
+fi
+
+
+
+
+wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
+chmod u+x pre.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh
+chmod u+x prechroot.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/post.sh
+chmod u+x post.sh
+wget https://$mgr/confluent-public/os/$profile/preseed.cfg
+cat preseed.cfg >> /preseed.cfg
+echo $mgr > /etc/confluent/deployer
+setdebopt auto-install/enable true boolean
+setdebopt partman/early_command  $predir/pre.sh string
+setdebopt preseed/late_command $predir/prechroot.sh string
+mv $predir/post.sh /tmp/
+cd -
+ip -4 a flush dev $nic
+setdebopt netcfg/choose_interface $nic select
+setdebopt netcfg/get_hostname $nodename string
+setdebopt mirror/protocol https string
+setdebopt mirror/country manual string
+setdebopt mirror/https/hostname $mgr string
+setdebopt mirror/https/directory /confluent-public/os/$profile/distribution string
+setdebopt mirror/protocol https string
+setdebopt mirror/https/proxy "" string
+setdebopt apt-setup/security_host $mgr string
+if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
+    if [ "$v4cfgmeth" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress $v4addr string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+        for namesrv in "$namesrvs"; do
+            setdebopt netcfg/get_nameservers $namesrv string
+        done
+    elif [ "$vpcfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+    fi
+fi

confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/linux $2/boot/kernel && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/initrd.gz $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/* $2/boot/efi/boot
+

confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg

@@ -0,0 +1,21 @@
+d-i anna/choose_modules string openssh-server-udeb
+d-i partman-auto/method string regular
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-auto/expert_recipe_file string /tmp/partitionfile
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i passwd/make-user boolean false
+d-i clock-setup/utc boolean true
+d-i apt-setup/multiverse boolean false
+d-i apt-setup/universe boolean false
+d-i apt-setup/backports boolean false
+d-i apt-setup/updates boolean false
+d-i grub-installer/only_debian boolean true
+tasksel tasksel/first multiselect standard
+d-i pkgsel/include string openssh-server curl
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean false
+d-i finish-install/reboot_in_progress note

confluent_osdeploy/ubuntu18.04/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+#cp -a /etc/confluent/ssh/* /etc/ssh/
+#systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+source /etc/confluent/functions
+
+run_remote_parts firstboot.d
+run_remote_config firstboot.d
+systemctl disable firstboot
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions

@@ -0,0 +1,208 @@
+#!/bin/bash
+function test_mgr() {
+    whost=$1
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+mkdir -p /run/sshd
+mkdir -p /root/.ssh
+cat /tmp/ssh/*pubkey >> /root/.ssh/authorized_keys
+cat /tmp/ssh/*.ca | sed -e s/^/'@cert-authority * '/ >> /etc/ssh/ssh_known_hosts
+chmod 700 /etc/confluent
+chmod go-rwx /etc/confluent/*
+sshconf=/etc/ssh/ssh_config
+if [ -d /etc/ssh/ssh_config.d/ ]; then
+    sshconf=/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+mkdir -p /opt/confluent/bin
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+systemctl enable firstboot
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+source /etc/confluent/functions
+python3 /opt/confluent/bin/apiclient /confluent-api/self/nodelist | sed -e s/'^- //' > /tmp/allnodes
+cp /tmp/allnodes /root/.shosts
+cp /tmp/allnodes /etc/ssh/shosts.equiv
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ')
+    sed -i "s/#NTP=/NTP=$ntps/" /etc/systemd/timesyncd.conf
+fi
+textcons=$(grep ^textconsole: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /tmp/autocons.info ]; then
+        cons=$(cat /tmp/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /etc/default/grub
+        updategrub=1
+    fi
+fi
+kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
+fi
+
+if [ 1 = $updategrub ]; then
+    update-grub
+fi
+
+if [ -e /sys/firmware/efi ]; then
+    bootnum=$(efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//)
+    if [ ! -z "$bootnum" ]; then
+        currboot=$(efibootmgr | grep ^BootOrder: | awk '{print $2}')
+        nextboot=$(echo $currboot| awk -F, '{print $1}')
+        [ "$nextboot" = "$bootnum" ] || efibootmgr -o $bootnum,$currboot
+        efibootmgr -D
+    fi
+fi
+run_remote_python syncfileclient
+run_remote_parts post.d
+run_remote_config post
+
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+## Use the following option to add additional boot parameters for the
+## installed system (if supported by the bootloader installer).
+## Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string [from profile.yaml]
+deploycfg=/etc/confluent/confluent.deploycfg
+mgr=$(cat /etc/confluent/deployer)
+
+cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+cat > /usr/lib/live-installer.d/confluent-certs << EOF
+#!/bin/sh
+cp /tls/* /target/etc/ssl/certs/
+cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt
+EOF
+chmod a+x /usr/lib/live-installer.d/confluent-certs
+mkdir -p /.ssh/
+cat /ssh/*pubkey > /.ssh/authorized_keys
+mkdir -p /etc/ssh
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|cut -d ' ' -f 2)
+apikey=$(cat /etc/confluent/confluent.apikey)
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=$(echo $pubkey | sed -e s/.pub/-cert.pub/)
+    keyfile=${pubkey%.pub}
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" --post-file=$pubkey https://$mgr/confluent-api/self/sshcert -O $certfile --quiet
+    echo HostKey $keyfile >> /etc/ssh/sshd_config
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config
+done
+if [ -e /tmp/installdisk ]; then
+    instdisk=$(cat /tmp/installdisk)
+else
+    for blockdev in $(ls /sys/class/block/); do
+        shortname=$(basename $blockdev)
+        if [ "$shortname" != "${shortname%loop*}" ]; then
+            continue
+        fi
+        udevadm info --query=property /dev/$shortname |grep DEVTYPE=disk > /dev/null || continue # ignore partitions
+        udevadm info --query=property /dev/$shortname |grep DM_NAME > /dev/null && continue # not a real disk
+        sz=$(cat /sys/block/$shortname/size 2> /dev/null)
+        [ -z "$sz" ] && continue
+        [ $sz -lt 1048576 ] && continue # Too small
+        [ -z "$firstdisk" ] && firstdisk=$shortname
+        if udevadm info --query=property /dev/$shortname|grep ID_MODEL=| sed -e s/' '/_/g | grep -iE '(thinksystem_m.2|m.2_nvme_2-bay_raid_kit)' > /dev/null; then
+            instdisk=$shortname
+            break
+        fi
+        if udevadm info --query=property /dev/$shortname|grep MD_CONTAINER=imsm; then
+            sraid=$sortname
+        else
+            drv=$(udevadm info -a /dev/sdb|grep DRIVERS==|grep -Ev '""|"sd"' | sed -e s/.*=// -e s/'"'//g)
+            if [ "ahci" = "$drv" -a -z "$onbdisk" ]; then
+                onbdisk=$shortname
+            elif [ "megaraid" = "$drv" -a -z "$rdisk" ]; then
+                rdisk=$shortname
+            fi
+        fi
+    done
+fi
+if [ -z "$instdisk" ]; then
+    if [ ! -z "$sraid"]; then
+        instdisk=$sraid
+    elif [ ! -z "$onbdisk" ]; then
+        instdisk=$onbdisk
+    elif [ ! -z "$rdisk" ]; then
+        instdisk=$rdisk
+    else
+        instdisk=$firstdisk
+    fi
+fi
+if [ ! -z "$instdisk" ]; then
+    debconf-set partman-auto/disk /dev/$instdisk
+fi
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config
+echo IgnoreRhosts no >> /etc/ssh/sshd_config
+echo sshd:x:1:1::/run/sshd:/bin/false >> /etc/passwd
+/usr/sbin/sshd
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist -O /tmp/allnodes --quiet
+#kill -HUP $(ps | grep -v grep | grep /usr/sbin/sshd | sed -e 's/^ *//'|cut -d ' ' -f 1)
+#curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk
+#python3 /tmp/getinstalldisk
+#sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+mount -o bind /sys /target/sys
+mount -o bind /dev /target/dev
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /proc /target/proc
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /run /target/run
+cp -a /etc/confluent /target/etc/confluent
+cp -a /opt/confluent /target/opt/confluent
+mv /tmp/post.sh /target/tmp/
+cp -a /ssh /tls /target/tmp
+cat /tls/*.pem >> /target/etc/confluent/ca.pem
+cp -a /etc/ssh/ssh_host_* /target/etc/ssh/
+grep HostCertificate /etc/ssh/sshd_config  >> /target/etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+kill -HUP $(ps |grep -v grep|grep sshd|grep /usr|sed -e s/' root.*//')
+chroot /target bash /tmp/post.sh

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+seems a potentially relevant thing to put i... by Jarrod Johnson
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -11,32 +11,48 @@ MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| aw
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
 umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile | sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+seems a potentially relevant thing to put i... by Jarrod Johnson
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions

@@ -1,10 +1,10 @@
 #!/bin/bash
 function test_mgr() {
     whost=$1
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    if curl -s https://${whost}/confluent-api/ > /dev/null; then
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
         return 0
     fi
     return 1
@@ -63,10 +63,10 @@ fetch_remote() {
     set_confluent_vars
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
 }
 
@@ -174,10 +174,10 @@ run_remote_python() {
     cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     whost=$confluent_mgr
-    if [[ "$whost" == *:* ]]; then
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
         whost="[$whost]"
     fi
-    curl -f -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     confluentpython $*
     retcode=$?

confluent_osdeploy/utils/clortho.c

@@ -18,7 +18,8 @@
 
 #define MAXPACKET 1024
 
-static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./";
+static const char b64alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 unsigned char* genpasswd(int len) {
         unsigned char * passwd;
@@ -56,10 +57,10 @@ char * b64e(uint8_t * data, uint32_t datalen) {
         currchunk[1] = remaining > 1 ? currptr[1] : 0;
         currchunk[2] = remaining > 2 ? currptr[2] : 0;
         currptr += 3;
-        currout[0] = cryptalpha[currchunk[0] >> 2];
-        currout[1] = cryptalpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
-        currout[2] = remaining > 1 ? cryptalpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
-        currout[3] = remaining > 2 ? cryptalpha[currchunk[2] & 0x3f] : '=';
+        currout[0] = b64alpha[currchunk[0] >> 2];
+        currout[1] = b64alpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
+        currout[2] = remaining > 1 ? b64alpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
+        currout[3] = remaining > 2 ? b64alpha[currchunk[2] & 0x3f] : '=';
         remaining -= 3;
         currout += 4;
     }

confluent_osdeploy/utils/urlmount3.c

@@ -0,0 +1,348 @@
+/*
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2021 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+#define FUSE_USE_VERSION 30
+#include <fuse3/fuse.h>
+#include <curl/curl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <pthread.h>
+#include <unistd.h>
+
+CURL *curl;
+
+char curlerror[CURL_ERROR_SIZE];
+curl_off_t filesize;
+
+typedef struct downloadbuffer {
+    char *response;
+    size_t completed;
+    size_t total;
+} downloadbuffer;
+
+#define MAX_FILE_LEN 1024
+#define MAX_URL_PATHS 512
+static char filename[MAX_FILE_LEN + 1];
+static int urlidx, newidx;
+static char* urls[MAX_URL_PATHS + 1];
+
+
+void *http_rechecker(void *argp) {
+    CURL *checkurl;
+    int tmpidx, tmpval;
+    tmpidx = open("/dev/urandom", O_RDONLY);
+    if (tmpidx <= 0 || read(tmpidx, (char*)&tmpval, 4) < 0)
+        tmpval = time(NULL) & 0xffffffff;
+    if (tmpidx >= 0)
+        close(tmpidx);
+    srand(tmpval);
+    checkurl = curl_easy_init();
+    if (curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Error setting timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Error setting nobody\n");
+        exit(1);
+    }
+    while (1) {
+        sleep(25 + tmpval % 10);  // Spread out retries across systems
+        tmpidx = 0;
+        while (tmpidx < urlidx && tmpidx < newidx && urls[tmpidx] != NULL) {
+            if (curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]) != CURLE_OK) {
+                tmpidx++;
+                continue;
+            }
+            if (curl_easy_perform(checkurl) == CURLE_OK)
+                newidx = tmpidx;
+            else
+                tmpidx++;
+        }
+    }
+}
+
+static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
+                        off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags frf)
+{
+    if (strcmp(path, "/") != 0)  // We don't support subdirs
+        return -ENOENT;
+    filler(buf, ".", NULL, 0, 0);
+    filler(buf, "..", NULL, 0, 0);
+    filler(buf, filename + 1, NULL, 0, 0);
+    return 0;
+}
+
+size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) {
+    size_t amount;
+    amount = size * nmemb;
+    if (userdata->total < amount + userdata->completed) return 0;
+    memcpy(&(userdata->response[userdata->completed]), data, amount);
+    userdata->completed += amount;
+    return amount;
+}
+
+static int http_read(const char *path, char *buf, size_t size, off_t offset,
+                     struct fuse_file_info *fi) {
+    char headbuffer[512];
+    double dldbl = 0.0;
+    int startidx;
+    int reconnecting = 0;
+    FILE* fd;
+    startidx = urlidx;
+    memset(buf, 0, size);
+    curl_off_t downloaded;
+    //Would be needed for multithread, however preferring to conserve
+    //filehandles rather than go multithread
+    // Some comparisons showed that the threaded performance boost doesn't
+    // do even offset the overhead of the new curl handles, so better
+    // to use single threaded curl overall for now
+    //CURL *tmpcurl = curl_easy_duphandle(curl);
+    downloadbuffer dlbuf;
+    dlbuf.response = buf;
+    dlbuf.completed = 0;
+    dlbuf.total = size;
+    fd = NULL;
+
+    if (strcmp(path, filename) != 0) return -ENOENT;
+    memset(headbuffer, 0, 512);
+    if (offset >= filesize) return 0;
+    if (offset + size - 1 >= filesize) size = filesize - offset - 1;
+    snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
+    if (curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer) != CURLE_OK) {
+        fprintf(stderr, "Error setting range\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&dlbuf) != CURLE_OK) {
+        fprintf(stderr, "Error setting writedata\n");
+        exit(1);
+    }
+    if (newidx < MAX_URL_PATHS) {
+        reconnecting = 1;
+        urlidx = newidx;
+        newidx = MAX_URL_PATHS;
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        // if fail, carry on and take the error in curl_easy_perform instead
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        reconnecting = 1;
+        fd = fopen("/dev/kmsg", "w+");
+        dlbuf.completed = 0;
+        fprintf(fd, "<4>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
+        fclose(fd);
+        urlidx++;
+        if (urlidx > MAX_URL_PATHS)
+            urlidx = 0;
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == startidx) {
+            fd = fopen("/dev/kmsg", "w+");
+            fprintf(fd, "<1>urlmount: All connections to source are down\n");
+            fclose(fd);
+            sleep(10);
+        }
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        if (urlidx > MAX_URL_PATHS) {
+            fprintf(stderr, "Maximum url path exceeded\n");
+            exit(1);
+        }
+        // ignore, let the curl_easy_perform get the error
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    if (reconnecting) {
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Successfully connected to %s\n", urls[urlidx]);
+        fclose(fd);
+    }
+    curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl);
+    downloaded = round(dldbl);
+    //Would be needed for multithread
+    //curl_easy_cleanup(tmpcurl);
+    return downloaded;
+}
+
+static int http_open(const char *path, struct fuse_file_info *fi) {
+    if (strcmp(path, filename) != 0)
+        return -ENOENT;
+
+    if ((fi->flags & 3) != O_RDONLY)
+        return -EACCES;
+
+    return 0;
+}
+
+static void* http_init(struct fuse_conn_info *conn, struct fuse_config *cfg) {
+    // Because we fork, we need to redo curl
+    // or else suffer the wrath of NSS TLS
+    pthread_t tid;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    pthread_create(&tid, NULL, http_rechecker, NULL);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl error buffor\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl failonerror\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing url\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl fill buffer\n");
+        exit(1);
+    }
+    return NULL;
+}
+
+static int http_getattr(const char *path, struct stat *st, struct fuse_file_info *ffi) {
+    memset(st, 0, sizeof(struct stat));
+
+    if (strcmp(path, "/") == 0) {
+        st->st_mode = S_IFDIR | 0555;
+        st->st_nlink = 2;
+    } else if (strcmp(path, filename) == 0) {
+        st->st_mode = S_IFREG | 0444;
+        st->st_nlink = 1;
+        st->st_size = filesize; // TODO: fix with curl HEAD
+    } else
+        return -ENOENT;
+    return 0;
+}
+
+
+static const struct fuse_operations http_ops = {
+    .getattr = http_getattr,
+    .readdir = http_readdir,
+    .read = http_read,
+    .open = http_open,
+    .init = http_init,
+};
+
+int main(int argc, char* argv[]) {
+    char *tmp;
+    double fsize;
+    unsigned int i;
+    int j;
+    j = 0;
+    memset(urls, 0, 32*sizeof(char*));
+    urlidx = 0;
+    newidx = MAX_URL_PATHS;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Unable to set error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Unable to set fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L) != CURLE_OK) {
+        fprintf(stderr, "Unable to setup curl timeout\n");
+        exit(1);
+    }
+    memset(filename, 0, MAX_FILE_LEN);
+    for (i=0; i < argc; i++) {
+        if (strstr(argv[i], ":") > 0) {
+            if (j < MAX_URL_PATHS) {
+                urls[j] = argv[i];
+                tmp = strrchr(urls[j++], '/');
+                strncpy(filename, tmp, MAX_FILE_LEN);
+            }
+            //Request single threaded mode, as curl would need more
+            // filehandles for multithread
+            argv[i] = "-s";
+        }
+    }
+    if (filename[0] == 0) {
+        fprintf(stderr, "No URL given in arguments\n");
+        exit(1);
+    }
+    for (i=0; urls[i] != NULL; i++) {
+        printf("Registering mount path: %s\n", urls[i]);
+    }
+    j = urlidx;
+    printf("Connecting to %s\n", urls[urlidx]);
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Unable to set url\n");
+    }
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Failure setting no body\n");
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == j) {
+            fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n");
+            exit(1);
+        }
+        printf("Connecting to %s\n", urls[urlidx]);
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+            fprintf(stderr, "Unable to set url\n");
+        }
+    }
+    printf("Successfully connected to %s\n", urls[urlidx]);
+    if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize) != CURLE_OK) {
+        fprintf(stderr, "Failed getting content length\n");
+        exit(1);
+    }
+    filesize = round(fsize);
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 0) != CURLE_OK) {
+        fprintf(stderr, "Failed setting nobody\n");
+        exit(1);
+    }
+    if (filesize < 1) {
+        fprintf(stderr, "Unable to reach designated URL\n");
+        exit(1);
+    }
+    if (!curl) {
+        fprintf(stderr, "Unable to initialize CURL!\n");
+        exit(1);
+    }
+    curl_easy_cleanup(curl);
+    curl_global_cleanup();
+    fuse_main(argc, argv, &http_ops, NULL);
+}

confluent_server/bin/collective

@@ -52,11 +52,12 @@ def make_certificate():
     os.umask(umask)
 
 
-def show_invitation(name):
+def show_invitation(name, nonvoting=False):
     if not os.path.exists('/etc/confluent/srvcert.pem'):
         make_certificate()
     s = client.Command().connection
-    tlvdata.send(s, {'collective': {'operation': 'invite', 'name': name}})
+    role = 'nonvoting' if nonvoting else None
+    tlvdata.send(s, {'collective': {'operation': 'invite', 'name': name, 'role': role}})
     invite = tlvdata.recv(s)['collective']
     if 'error' in invite:
         sys.stderr.write(invite['error'] + '\n')
@@ -104,16 +105,19 @@ def show_collective():
         return
     if 'quorum' in res['collective']:
         print('Quorum: {0}'.format(res['collective']['quorum']))
-    print('Leader: {0}'.format(res['collective']['leader']))
+    leadernonvoting = res['collective']['leader'] in res['collective'].get('nonvoting', ())
+    print('Leader: {0}{1}'.format(res['collective']['leader'], ' (non-voting)' if leadernonvoting else ''))
     if 'active' in res['collective']:
         if res['collective']['active']:
             print('Active collective members:')
             for member in sortutil.natural_sort(res['collective']['active']):
-                print('    {0}'.format(member))
+                nonvoter = member in res['collective'].get('nonvoting', ())
+                print('    {0}{1}'.format(member, ' (nonvoting)' if nonvoter else ''))
         if res['collective']['offline']:
             print('Offline collective members:')
             for member in sortutil.natural_sort(res['collective']['offline']):
-                print('    {0}'.format(member))
+                nonvoter = member in res['collective'].get('nonvoting', ())
+                print('    {0}{1}'.format(member, ' (nonvoting)' if nonvoter else ''))
     else:
         print('Run collective show on leader for more data')
 
@@ -128,6 +132,7 @@ def main():
                                     'collective member.  Run collective invite -h for more information')
     ic.add_argument('name', help='Name of server to invite to join the '
                                  'collective')
+    ic.add_argument('-n', help='Join as a non-voting member, do not have this member contribute to quorum', action='store_true')
     dc = sp.add_parser('delete', help='Delete a member of a collective')
     dc.add_argument('name', help='Name of server to delete from collective')
     jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
@@ -138,7 +143,7 @@ def main():
     if cmdset.command == 'gencert':
         make_certificate()
     elif cmdset.command == 'invite':
-        show_invitation(cmdset.name)
+        show_invitation(cmdset.name, cmdset.n)
     elif cmdset.command == 'join':
         join_collective(cmdset.server, cmdset.i)
     elif cmdset.command == 'show':

confluent_server/bin/confluent_selfcheck

@@ -15,7 +15,7 @@ import confluent.sshutil as sshutil
 import confluent.certutil as certutil
 import confluent.client as client
 import confluent.config.configmanager as configmanager
-import subprocess
+import eventlet.green.subprocess as subprocess
 import tempfile
 import shutil
 import eventlet.green.socket as socket
@@ -237,11 +237,17 @@ if __name__ == '__main__':
         allok = True
         uuidok = False
         macok = False
+        valid_nodes = [node['item']['href'][:-1] for node in sess.read('/nodes/')] #get all valid nodes
         for rsp in sess.read(f'/nodes/{args.node}/attributes/all'):
             if rsp.get('errorcode', None) == 404:
                 emprint(f'There is no node named "{args.node}"')
                 allok = False
                 uuidok = True  # not really, but suppress the spurious error
+            dnsdomain = rsp.get('dns.domain', {}).get('value', '')
+            if ',' in dnsdomain or ' ' in dnsdomain:
+                allok = False
+                emprint(f'{args.node} has a dns.domain that appears to be a search instead of singular domain')
+                uuidok = True  # not really, but suppress the spurious error
             uuid = rsp.get('id.uuid', {}).get('value', None)
             if uuid:
                 uuidok = True
@@ -254,6 +260,15 @@ if __name__ == '__main__':
                     mac = rsp[key].get('value', None)
                     if mac:
                         macok = True
+            #adding new code to check if the response is something like net.<something>switch
+            for key in rsp:
+                if ((key.startswith('net.') and key.endswith('switch')) 
+                        or (key.startswith('power.') and key.endswith('pdu'))
+                        or (key.startswith('enclosure.') and key.endswith('manager'))
+                        ):
+                    switch_value = rsp[key].get('value',None)
+                    if switch_value and switch_value not in valid_nodes:
+                        emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).')
         if not uuidok and not macok:
             allok = False
             emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work')

confluent_server/bin/osdeploy

@@ -217,14 +217,24 @@ def install_tftp_content():
     else:
         emprint(
             'Detected {0} as tftp directory, but unable to determine tftp service, ensure that a tftp server is installed and enabled manually'.format(tftplocation))
+    otftplocation = tftplocation
     tftplocation = '{0}/confluent/x86_64'.format(tftplocation)
     try:
         os.makedirs(tftplocation)
     except OSError as e:
         if e.errno != 17:
             raise
+    armtftplocation = '{0}/confluent/aarch64'.format(otftplocation)
+    try:
+        os.makedirs(armtftplocation)
+    except OSError as e:
+        if e.errno != 17:
+            raise
     shutil.copy('/opt/confluent/lib/ipxe/ipxe.efi', tftplocation)
     shutil.copy('/opt/confluent/lib/ipxe/ipxe.kkpxe', tftplocation)
+    if os.path.exists('/opt/confluent/lib/ipxe/ipxe-aarch64.efi'):
+        shutil.copy('/opt/confluent/lib/ipxe/ipxe-aarch64.efi', os.path.join(armtftplocation, 'ipxe.efi'))
+
 
 
 def initialize(cmdset):
@@ -285,13 +295,13 @@ def initialize(cmdset):
             try:
                 subprocess.check_call(['systemctl', 'try-restart', 'httpd'])
                 print('HTTP server has been restarted if it was running')
-            except subprocess.CalledProcessError:
+            except Exception:
                 emprint('New HTTPS certificates generated, restart the web server manually')
         elif os.path.exists('/usr/lib/systemd/system/apache2.service'):
             try:
                 subprocess.check_call(['systemctl', 'try-restart', 'apache2'])
                 print('HTTP server has been restarted if it was running')
-            except subprocess.CalledProcessError:
+            except Exception:
                 emprint('New HTTPS certificates generated, restart the web server manually')
         else:
             emprint('New HTTPS certificates generated, restart the web server manually')

confluent_server/confluent/auth.py

@@ -41,6 +41,7 @@ try:
 except ImportError:
     pass
 import time
+import yaml
 
 _pamservice = 'confluent'
 _passcache = {}
@@ -115,6 +116,47 @@ class PromptsNeeded(Exception):
     def __init__(self, prompts):
         self.prompts = prompts
 
+#add function to change _allowedbyrole and _deniedbyrole vars.
+def add_roles(_allowed,_denied):
+    # function to parse the roles and the files. If there are modifications to be done to the roles, items will be added to dictionaries.
+    # If there are no moodifications done to one of the roles, it continues to the next
+    # Opening YAML file and reading the custom roles
+    with open("/etc/confluent/authorization.yaml","r") as stream:
+        loaded_file = yaml.safe_load(stream)
+        try:
+            allowed_loaded = loaded_file["allowedbyrole"]
+            for role in allowed_loaded:
+                if role not in configmanager._validroles:
+                    configmanager._validroles.append(role)
+        except:
+            pass
+        try:
+            denied_loaded = loaded_file["deniedbyrole"]
+        except:
+            pass
+ 
+        try:
+            _allowed.update(allowed_loaded)
+        except NameError:
+            pass
+        try: 
+            _denied.update(denied_loaded)
+        except NameError:
+            pass
+        return
+                
+    
+def check_for_yaml():
+    #checking if the file exists
+    if os.path.exists("/etc/confluent/authorization.yaml"):
+        add_roles(_allowedbyrole,_deniedbyrole)
+
+        return "Custom auth. file detected in /etc/confluent, updated roles accordingly"
+    else:
+        return "No custom auth. file. Continuing as normal"
+    
+
+        
 def _get_usertenant(name, tenant=False):
     """_get_usertenant
 

confluent_server/confluent/collective/invites.py

@@ -22,12 +22,13 @@ import hmac
 import os
 pending_invites = {}
 
-def create_server_invitation(servername):
+def create_server_invitation(servername, role):
     servername = servername.encode('utf-8')
     randbytes = (3 - ((len(servername) + 2) % 3)) % 3 + 64
     invitation = os.urandom(randbytes)
-    pending_invites[servername] = invitation
-    return base64.b64encode(servername + b'@' + invitation)
+    pending_invites[servername] = {'invitation': invitation, 'role': role}
+    invite = servername + b'@' + invitation
+    return base64.b64encode(invite)
 
 def create_client_proof(invitation, mycert, peercert):
     return hmac.new(invitation, peercert + mycert, hashlib.sha256).digest()
@@ -42,6 +43,8 @@ def check_client_proof(servername, mycert, peercert, proof):
     if servername not in pending_invites:
         return False
     invitation = pending_invites[servername]
+    role = invitation['role']
+    invitation = invitation['invitation']
     validproof = hmac.new(invitation, mycert + peercert, hashlib.sha256
                           ).digest()
     if proof == validproof:
@@ -54,7 +57,7 @@ def check_client_proof(servername, mycert, peercert, proof):
         # Now to generate an answer...., reverse the cert order so our answer
         # is different, but still proving things
         return hmac.new(invitation, peercert + mycert, hashlib.sha256
-                          ).digest()
+                          ).digest(), role
     # The given proof did not verify the invitation
-    return False
+    return False, None
 

confluent_server/confluent/collective/manager.py

@@ -61,14 +61,20 @@ class ContextBool(object):
 
 connecting = ContextBool()
 leader_init = ContextBool()
+enrolling = ContextBool()
 
-def connect_to_leader(cert=None, name=None, leader=None, remote=None):
+def connect_to_leader(cert=None, name=None, leader=None, remote=None, isretry=False):
     global currentleader
     global follower
+    ocert = cert
+    oname = name
+    oleader = leader
+    oremote = remote
     if leader is None:
         leader = currentleader
-    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
-             'subsystem': 'collective'})
+    if not isretry:
+        log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+                 'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader, remote)
     except Exception as e:
@@ -79,9 +85,11 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     with connecting:
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
+            if not banner:
+                return
             vers = banner.split()[2]
-            if vers not in (b'v2', b'v3'):
-                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
+            if vers != b'v4':
+                raise Exception('This instance only supports protocol 4, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -98,7 +106,12 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                                 '{0}'.format(leader),
                         'subsystem': 'collective'})
                     return False
+                if 'waitinline' in keydata:
+                    eventlet.sleep(0.3)
+                    return connect_to_leader(cert, name, leader, None, isretry=True)
                 if 'leader' in keydata:
+                    if keydata['leader'] == None:
+                        return None
                     log.log(
                         {'info': 'Prospective leader {0} has redirected this '
                                  'member to {1}'.format(leader, keydata['leader']),
@@ -138,7 +151,9 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                         remote.close()
                     except Exception:
                         pass
-                    raise Exception("Error doing initial DB transfer")
+                    log.log({'error': 'Retrying connection, error during initial sync', 'subsystem': 'collective'})
+                    return connect_to_leader(ocert, oname, oleader, None)
+                    raise Exception("Error doing initial DB transfer")  # bad ssl write retry
                 dbjson += ndata
             cfm.clear_configuration()
             try:
@@ -146,7 +161,7 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                 for c in colldata:
                     cfm._true_add_collective_member(c, colldata[c]['address'],
                                                     colldata[c]['fingerprint'],
-                                                    sync=False)
+                                                    sync=False, role=colldata[c].get('role', None))
                 for globvar in globaldata:
                     cfm.set_global(globvar, globaldata[globvar], False)
                 cfm._txcount = dbi.get('txcount', 0)
@@ -264,7 +279,11 @@ def handle_connection(connection, cert, request, local=False):
                 return
             if follower is not None:
                 linfo = cfm.get_collective_member_by_address(currentleader)
-                remote = socket.create_connection((currentleader, 13001), 15)
+                try:
+                    remote = socket.create_connection((currentleader, 13001), 15)
+                except Exception:
+                    cfm.stop_following()
+                    return
                 remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                          keyfile='/etc/confluent/privkey.pem',
                                          certfile='/etc/confluent/srvcert.pem')
@@ -321,7 +340,8 @@ def handle_connection(connection, cert, request, local=False):
             #TODO(jjohnson2): Cannot do the invitation if not the head node, the certificate hand-carrying
             #can't work in such a case.
             name = request['name']
-            invitation = invites.create_server_invitation(name)
+            role = request.get('role', '')
+            invitation = invites.create_server_invitation(name, role)
             tlvdata.send(connection,
                          {'collective': {'invitation': invitation}})
             connection.close()
@@ -390,34 +410,44 @@ def handle_connection(connection, cert, request, local=False):
             eventlet.spawn_n(connect_to_leader, rsp['collective'][
                 'fingerprint'], name)
     if 'enroll' == operation:
-        #TODO(jjohnson2): error appropriately when asked to enroll, but the master is elsewhere
-        mycert = util.get_certificate_from_file('/etc/confluent/srvcert.pem')
-        proof = base64.b64decode(request['hmac'])
-        myrsp = invites.check_client_proof(request['name'], mycert,
-                                           cert, proof)
-        if not myrsp:
-            tlvdata.send(connection, {'error': 'Invalid token'})
-            connection.close()
-            return
-        if not list(cfm.list_collective()):
-            # First enrollment of a collective, since the collective doesn't
-            # quite exist, then set initting false to let the enrollment action
-            # drive this particular initialization
-            initting = False
-        myrsp = base64.b64encode(myrsp)
-        fprint = util.get_fingerprint(cert)
-        myfprint = util.get_fingerprint(mycert)
-        cfm.add_collective_member(get_myname(),
-                                  connection.getsockname()[0], myfprint)
-        cfm.add_collective_member(request['name'],
-                                  connection.getpeername()[0], fprint)
-        myleader = get_leader(connection)
-        ldrfprint = cfm.get_collective_member_by_address(
-            myleader)['fingerprint']
-        tlvdata.send(connection,
-                     {'collective': {'approval': myrsp,
-                                     'fingerprint': ldrfprint,
-                                     'leader': get_leader(connection)}})
+        with enrolling:
+            cfm.check_quorum()
+            mycert = util.get_certificate_from_file('/etc/confluent/srvcert.pem')
+            proof = base64.b64decode(request['hmac'])
+            myrsp, role = invites.check_client_proof(request['name'], mycert,
+                                                     cert, proof)
+            if not myrsp:
+                tlvdata.send(connection, {'error': 'Invalid token'})
+                connection.close()
+                return
+            if not list(cfm.list_collective()):
+                # First enrollment of a collective, since the collective doesn't
+                # quite exist, then set initting false to let the enrollment action
+                # drive this particular initialization
+                initting = False
+            myrsp = base64.b64encode(myrsp)
+            fprint = util.get_fingerprint(cert)
+            myfprint = util.get_fingerprint(mycert)
+            iam = cfm.get_collective_member(get_myname())
+            if not iam:
+                cfm.add_collective_member(get_myname(),
+                                          connection.getsockname()[0], myfprint)
+            cfm.add_collective_member(request['name'],
+                                    connection.getpeername()[0], fprint, role)
+            myleader = get_leader(connection)
+            ldrfprint = cfm.get_collective_member_by_address(
+                myleader)['fingerprint']
+            tlvdata.send(connection,
+                        {'collective': {'approval': myrsp,
+                                        'fingerprint': ldrfprint,
+                                        'leader': get_leader(connection)}})
+            havequorum = False
+            while not havequorum:
+                try:
+                    cfm.check_quorum()
+                    havequorum = True
+                except exc.DegradedCollective:
+                    eventlet.sleep(0.1)
     if 'assimilate' == operation:
         drone = request['name']
         droneinfo = cfm.get_collective_member(drone)
@@ -519,6 +549,11 @@ def handle_connection(connection, cert, request, local=False):
                                       'backoff': True})
             connection.close()
             return
+        if leader_init.active:
+            tlvdata.send(connection, {'error': 'Servicing a connection',
+                                      'waitinline': True})
+            connection.close()
+            return
         if myself != get_leader(connection):
             tlvdata.send(
                 connection,
@@ -534,7 +569,7 @@ def handle_connection(connection, cert, request, local=False):
                           'txcount': cfm._txcount})
             log.log({'info': 'Connecting to leader due to superior '
                              'transaction count', 'subsystem': 'collective'})
-            connection.close()
+            connection.close()  # well this won't work
             if not connect_to_leader(
                 None, None, connection.getpeername()[0]):
                 if retrythread is None:
@@ -551,9 +586,15 @@ def handle_connection(connection, cert, request, local=False):
             tlvdata.send(connection, cfm._cfgstore['collective'])
             tlvdata.send(connection, {'confluent_uuid': cfm.get_global('confluent_uuid')}) # cfm.get_globals())
             cfgdata = cfm.ConfigManager(None)._dump_to_json()
-            tlvdata.send(connection, {'txcount': cfm._txcount,
-                                      'dbsize': len(cfgdata)})
-            connection.sendall(cfgdata)
+            try:
+                tlvdata.send(connection, {'txcount': cfm._txcount,
+                            'dbsize': len(cfgdata)})
+                connection.sendall(cfgdata)
+            except Exception:
+                try:
+                    connection.close()
+                finally:
+                    return None
         #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
         # so far unused anyway
         connection.settimeout(90)
@@ -575,9 +616,12 @@ def populate_collinfo(collinfo):
     activemembers = set(cfm.cfgstreams)
     activemembers.add(iam)
     collinfo['offline'] = []
+    collinfo['nonvoting'] = []
     for member in cfm.list_collective():
         if member not in activemembers:
             collinfo['offline'].append(member)
+        if cfm.get_collective_member(member).get('role', None) == 'nonvoting':
+            collinfo['nonvoting'].append(member)
 
 
 def try_assimilate(drone, followcount, remote):
@@ -639,7 +683,11 @@ def get_leader(connection):
 
 def retire_as_leader(newleader=None):
     global currentleader
+    global reassimilate
     cfm.stop_leading(newleader)
+    if reassimilate is not None:
+        reassimilate.kill()
+        reassimilate = None
     currentleader = None
 
 def become_leader(connection):
@@ -647,6 +695,10 @@ def become_leader(connection):
     global follower
     global retrythread
     global reassimilate
+    if cfm.get_collective_member(get_myname()).get('role', None) == 'nonvoting':
+        log.log({'info': 'Refraining from being leader of collective (nonvoting)',
+            'subsystem': 'collective'})
+        return False
     log.log({'info': 'Becoming leader of collective',
              'subsystem': 'collective'})
     if follower is not None:
@@ -658,15 +710,20 @@ def become_leader(connection):
         retrythread = None
     currentleader = connection.getsockname()[0]
     skipaddr = connection.getpeername()[0]
+    if reassimilate is not None:
+        reassimilate.kill()
+    reassimilate = eventlet.spawn(reassimilate_missing)
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
-        if reassimilate is not None:
-            reassimilate.kill()
-        reassimilate = eventlet.spawn(reassimilate_missing)
+
 
 def reassimilate_missing():
     eventlet.sleep(30)
-    while cfm.cfgstreams and _assimilate_missing():
+    while True:
+        try:
+            _assimilate_missing()
+        except Exception as e:
+            cfm.logException()
         eventlet.sleep(30)
 
 def _assimilate_missing(skipaddr=None):
@@ -780,6 +837,8 @@ def start_collective():
         for member in sorted(list(cfm.list_collective())):
             if member == myname:
                 continue
+            if cfm.get_collective_member(member).get('role', None) == 'nonvoting':
+                continue
             if cfm.cfgleader is None:
                 cfm.stop_following(True)
             ldrcandidate = cfm.get_collective_member(member)['address']

confluent_server/confluent/config/attributes.py

@@ -233,6 +233,12 @@ node = {
                         'a stateless profile would be both after first boot.')
 
     },
+    'deployment.state': {
+        'description': ('Profiles may push more specific state, for example, it may set the state to "failed" or "succeded"'),
+    },
+    'deployment.state_detail': {
+        'description': ('Detailed state information as reported by an OS profile, when available'),
+    },
     'deployment.useinsecureprotocols': {
         'description': ('What phase(s) of boot are permitted to use insecure protocols '
                         '(TFTP and HTTP without TLS.  By default, only HTTPS is used.  However '

confluent_server/confluent/config/configmanager.py

@@ -59,13 +59,21 @@ except ModuleNotFoundError:
 import ast
 import base64
 from binascii import hexlify
+import os
+import sys
+
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.config.attributes as allattributes
 import confluent.config.conf as conf
-import confluent.log
 import confluent.noderange as noderange
 import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
+import confluent.log
 import copy
 import crypt
 try:
@@ -77,18 +85,18 @@ import eventlet
 import eventlet.event as event
 import eventlet.green.select as select
 import eventlet.green.threading as gthread
+import eventlet.green.subprocess as subprocess
 import fnmatch
 import hashlib
 import json
 import msgpack
 import operator
-import os
 import random
 import re
 import string
 import struct
-import sys
 import threading
+import time
 import traceback
 try:
     unicode
@@ -119,7 +127,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
-_validroles = ('Administrator', 'Operator', 'Monitor', 'Stub')
+_validroles = ['Administrator', 'Operator', 'Monitor', 'Stub']
 
 membership_callback = None
 
@@ -320,7 +328,7 @@ def _rpc_set_group_attributes(tenant, attribmap, autocreate):
 def check_quorum():
     if isinstance(cfgleader, bool):
         raise exc.DegradedCollective()
-    if (not cfgleader) and len(cfgstreams) < (len(_cfgstore.get('collective', {})) // 2):
+    if (not cfgleader) and (not has_quorum()):
         # the leader counts in addition to registered streams
         raise exc.DegradedCollective()
     if cfgleader and not _hasquorum:
@@ -348,9 +356,9 @@ def exec_on_followers(fnname, *args):
     pushes = eventlet.GreenPool()
     # Check health of collective prior to attempting
     for _ in pushes.starmap(
-        _push_rpc, [(cfgstreams[s], b'') for s in cfgstreams]):
+            _push_rpc, [(cfgstreams[s]['stream'], b'') for s in cfgstreams]):
         pass
-    if len(cfgstreams) < (len(_cfgstore['collective']) // 2):
+    if not has_quorum():
         # the leader counts in addition to registered streams
         raise exc.DegradedCollective()
     exec_on_followers_unconditional(fnname, *args)
@@ -363,7 +371,7 @@ def exec_on_followers_unconditional(fnname, *args):
     payload = msgpack.packb({'function': fnname, 'args': args,
                              'txcount': _txcount}, use_bin_type=False)
     for _ in pushes.starmap(
-            _push_rpc, [(cfgstreams[s], payload) for s in cfgstreams]):
+            _push_rpc, [(cfgstreams[s]['stream'], payload) for s in cfgstreams]):
         pass
 
 
@@ -421,7 +429,10 @@ def _push_rpc(stream, payload):
                 pass
             if membership_callback:
                 membership_callback()
-            stream.close()
+            try:
+                stream.close()
+            except Exception:
+                pass
 
 
 def decrypt_value(cryptvalue,
@@ -615,6 +626,40 @@ def set_global(globalname, value, sync=True):
     if sync:
         ConfigManager._bg_sync_to_file()
 
+mycachedname = [None, 0]
+def get_myname():
+    if mycachedname[1] > time.time() - 15:
+        return mycachedname[0]
+    try:
+        with open('/etc/confluent/cfg/myname', 'r') as f:
+            mycachedname[0] = f.read().strip()
+            mycachedname[1] = time.time()
+            return mycachedname[0]
+    except IOError:
+        myname = socket.gethostname().split('.')[0]
+        with open('/etc/confluent/cfg/myname', 'w') as f:
+            f.write(myname)
+        mycachedname[0] = myname
+        mycachedname[1] = time.time()
+        return myname
+
+def has_quorum():
+    voters = 0
+    for follower in cfgstreams:
+        if cfgstreams[follower].get('role', None) != 'nonvoting':
+            voters += 1
+    iam = get_collective_member(get_myname())
+    myrole = None
+    if iam:
+        myrole = iam.get('role', None)
+    if myrole != 'nonvoting':
+        voters += 1
+    allvoters = 0
+    for ghost in list_collective():
+        if get_collective_member(ghost).get('role', None) != 'nonvoting':
+            allvoters += 1
+    return voters > allvoters // 2
+    
 cfgstreams = {}
 def relay_slaved_requests(name, listener):
     global cfgleader
@@ -622,21 +667,21 @@ def relay_slaved_requests(name, listener):
     pushes = eventlet.GreenPool()
     if name not in _followerlocks:
         _followerlocks[name] = gthread.RLock()
+    meminfo = get_collective_member(name)
     with _followerlocks[name]:
         try:
             stop_following()
             if name in cfgstreams:
                 try:
-                    cfgstreams[name].close()
+                    cfgstreams[name]['stream'].close()
                 except Exception:
                     pass
                 del cfgstreams[name]
                 if membership_callback:
                     membership_callback()
-            cfgstreams[name] = listener
+            cfgstreams[name] = {'stream': listener, 'role': meminfo.get('role', None)}
             lh = StreamHandler(listener)
-            _hasquorum = len(cfgstreams) >= (
-                    len(_cfgstore['collective']) // 2)
+            _hasquorum = has_quorum()
             _newquorum = None
             while _hasquorum != _newquorum:
                 if _newquorum is not None:
@@ -644,10 +689,9 @@ def relay_slaved_requests(name, listener):
                 payload = msgpack.packb({'quorum': _hasquorum}, use_bin_type=False)
                 for _ in pushes.starmap(
                         _push_rpc,
-                        [(cfgstreams[s], payload) for s in cfgstreams]):
+                        [(cfgstreams[s]['stream'], payload) for s in cfgstreams]):
                     pass
-                _newquorum = len(cfgstreams) >= (
-                        len(_cfgstore['collective']) // 2)
+                _newquorum = has_quorum()
             _hasquorum = _newquorum
             if _hasquorum and _pending_collective_updates:
                 apply_pending_collective_updates()
@@ -656,7 +700,9 @@ def relay_slaved_requests(name, listener):
                 if name not in cfgstreams:
                     raise Exception("Unexpected loss of node in followers: " + name)
                 sz = struct.unpack('!Q', msg)[0]
-                if sz != 0:
+                if sz == 0:
+                    _push_rpc(listener, b'')
+                else:
                     rpc = b''
                     while len(rpc) < sz:
                         nrpc = listener.recv(sz - len(rpc))
@@ -693,12 +739,11 @@ def relay_slaved_requests(name, listener):
             except KeyError:
                 pass  # May have already been closed/deleted...
             if cfgstreams:
-                _hasquorum = len(cfgstreams) >= (
-                        len(_cfgstore['collective']) // 2)
+                _hasquorum = has_quorum()
                 payload = msgpack.packb({'quorum': _hasquorum}, use_bin_type=False)
                 for _ in pushes.starmap(
                         _push_rpc,
-                        [(cfgstreams[s], payload) for s in cfgstreams]):
+                        [(cfgstreams[s]['stream'], payload) for s in cfgstreams]):
                     pass
             if membership_callback:
                 membership_callback()
@@ -707,6 +752,16 @@ def relay_slaved_requests(name, listener):
                 return False
             return True
 
+lastheartbeat = None
+def check_leader():
+    _push_rpc(cfgleader, b'')
+    tries = 0
+    while tries < 30:
+        eventlet.sleep(0.1)
+        tries += 1
+        if lastheartbeat and lastheartbeat > (confluent.util.monotonic_time() - 3):
+            return True
+    raise Exception("Leader has disappeared")
 
 class StreamHandler(object):
     def __init__(self, sock):
@@ -728,10 +783,13 @@ class StreamHandler(object):
                     res = _push_rpc(self.sock, b'')  # nulls are a keepalive
                     if not res:
                         return None
+                    #TODO: this test can work fine even if the other end is
+                    # gone, go to a more affirmative test to more quickly
+                    # detect outage to peer
                     self.keepalive = confluent.util.monotonic_time() + 20
             self.expiry = confluent.util.monotonic_time() + 60
             msg = self.sock.recv(8)
-        except Exception:
+        except Exception as e:
             msg = None
         return msg
 
@@ -756,8 +814,8 @@ def stop_leading(newleader=None):
     for stream in list(cfgstreams):
         try:
             if rpcpayload is not None:
-                _push_rpc(cfgstreams[stream], rpcpayload)
-            cfgstreams[stream].close()
+                _push_rpc(cfgstreams[stream]['stream'], rpcpayload)
+            cfgstreams[stream]['stream'].close()
         except Exception:
             pass
         try:
@@ -825,6 +883,7 @@ cfgleader = None
 def follow_channel(channel):
     global _txcount
     global _hasquorum
+    global lastheartbeat
     try:
         stop_leading()
         stop_following(channel)
@@ -832,7 +891,9 @@ def follow_channel(channel):
         msg = lh.get_next_msg()
         while msg:
             sz = struct.unpack('!Q', msg)[0]
-            if sz != 0:
+            if sz == 0:
+                lastheartbeat = confluent.util.monotonic_time()
+            else:
                 rpc = b''
                 while len(rpc) < sz:
                     nrpc = channel.recv(sz - len(rpc))
@@ -876,12 +937,12 @@ def follow_channel(channel):
     return {}
 
 
-def add_collective_member(name, address, fingerprint):
+def add_collective_member(name, address, fingerprint, role=None):
     if cfgleader:
-        return exec_on_leader('add_collective_member', name, address, fingerprint)
+        return exec_on_leader('add_collective_member', name, address, fingerprint, role)
     if cfgstreams:
-        exec_on_followers('_true_add_collective_member', name, address, fingerprint)
-    _true_add_collective_member(name, address, fingerprint)
+        exec_on_followers('_true_add_collective_member', name, address, fingerprint, True, role)
+    _true_add_collective_member(name, address, fingerprint, role=role)
 
 def del_collective_member(name):
     if cfgleader and not isinstance(cfgleader, bool):
@@ -934,7 +995,7 @@ def apply_pending_collective_updates():
         del _pending_collective_updates[name]
 
 
-def _true_add_collective_member(name, address, fingerprint, sync=True):
+def _true_add_collective_member(name, address, fingerprint, sync=True, role=None):
     name = confluent.util.stringify(name)
     if _cfgstore is None:
         init(not sync)  # use not sync to avoid read from disk
@@ -942,6 +1003,8 @@ def _true_add_collective_member(name, address, fingerprint, sync=True):
         _cfgstore['collective'] = {}
     _cfgstore['collective'][name] = {'name': name, 'address': address,
                                      'fingerprint': fingerprint}
+    if role:
+        _cfgstore['collective'][name]['role'] = role
     with _dirtylock:
         if 'collectivedirty' not in _cfgstore:
             _cfgstore['collectivedirty'] = set([])
@@ -1515,6 +1578,8 @@ class ConfigManager(object):
         groupname = confluent.util.stringify(groupname)
         if groupname in self._cfgstore['usergroups']:
             raise Exception("Duplicate groupname requested")
+        if role is None:
+            role = 'Administrator'
         for candrole in _validroles:
             if candrole.lower().startswith(role.lower()):
                 role = candrole
@@ -1625,6 +1690,8 @@ class ConfigManager(object):
         name = confluent.util.stringify(name)
         if name in self._cfgstore['users']:
             raise Exception("Duplicate username requested")
+        if role is None:
+            role = 'Administrator'
         for candrole in _validroles:
             if candrole.lower().startswith(role.lower()):
                 role = candrole
@@ -2162,9 +2229,6 @@ class ConfigManager(object):
         self._bg_sync_to_file()
 
     def add_node_attributes(self, attribmap):
-        for node in attribmap:
-            if 'groups' not in attribmap[node]:
-                attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
 
     def rename_nodes(self, renamemap):
@@ -2296,6 +2360,8 @@ class ConfigManager(object):
                         '"{0}" is not a valid node name'.format(node))
             if autocreate is False and node not in self._cfgstore['nodes']:
                 raise ValueError("node {0} does not exist".format(node))
+            if 'groups' not in attribmap[node] and node not in self._cfgstore['nodes']:
+                attribmap[node]['groups'] = []
             for attrname in list(attribmap[node]):
                 if attrname in _attraliases:
                     truename = _attraliases[attrname]
@@ -2502,6 +2568,15 @@ class ConfigManager(object):
                        data.
 
         """
+        with open(os.devnull, 'w+') as devnull:
+            worker = subprocess.Popen(
+                [sys.executable, __file__, '-r' if redact else ''],
+                        stdin=devnull, stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+            stdout, stderr = worker.communicate()
+        return stdout
+
+    def _real_dump_to_json(self, redact=None):
         dumpdata = {}
         for confarea in _config_areas:
             if confarea not in self._cfgstore:
@@ -2833,9 +2908,9 @@ def dump_db_to_directory(location, password, redact=None, skipkeys=False):
         with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
             cfgfile.write(_dump_keys(password))
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
+    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write('\n')
+        cfgfile.write(b'\n')
     if 'collective' in _cfgstore:
         with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
             cfgfile.write(json.dumps(_cfgstore['collective']))
@@ -2876,6 +2951,11 @@ def init(stateless=False):
         _cfgstore = {}
 
 
+if __name__ == '__main__':
+    redact=None
+    if '-r' in sys.argv:
+        redact=True
+    sys.stdout.write(ConfigManager(None)._real_dump_to_json(redact))
 # some unit tests worth implementing:
 # set group attribute on lower priority group, result is that node should not
 # change

confluent_server/confluent/core.py

@@ -66,7 +66,7 @@ import eventlet.semaphore as semaphore
 import itertools
 import msgpack
 import os
-import socket
+import eventlet.green.socket as socket
 import struct
 import sys
 

confluent_server/confluent/credserver.py

@@ -79,14 +79,14 @@ class CredServer(object):
                 hmackey = hmackey.get(nodename, {}).get('secret.selfapiarmtoken', {}).get('value', None)
             elif tlv[1]:
                 client.recv(tlv[1])
+            apimats = self.cfm.get_node_attributes(nodename,
+                    ['deployment.apiarmed', 'deployment.sealedapikey'])
+            apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get(
+                    'value', None)
             if not hmackey:
                 if not address_is_somewhat_trusted(peer[0], nodename, self.cfm):
                     client.close()
                     return
-                apimats = self.cfm.get_node_attributes(nodename,
-                    ['deployment.apiarmed', 'deployment.sealedapikey'])
-                apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get(
-                    'value', None)
                 if not apiarmed:
                     if apimats.get(nodename, {}).get(
                         'deployment.sealedapikey', {}).get('value', None):

confluent_server/confluent/discovery/core.py

@@ -66,6 +66,7 @@ import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 import confluent.discovery.protocols.ssdp as ssdp
+#import confluent.discovery.protocols.mdns as mdns
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
 import confluent.discovery.handlers.cpstorage as cpstorage
@@ -116,6 +117,8 @@ nodehandlers = {
     'pxe-client': pxeh,
     'onie-switch': None,
     'cumulus-switch': None,
+    'affluent-switch': None,
+    #'openbmc': None,
     'service:io-device.Lenovo:management-module': None,
     'service:thinkagile-storage': cpstorage,
     'service:lenovo-tsm': tsm,
@@ -127,7 +130,9 @@ servicenames = {
     'cumulus-switch': 'cumulus-switch',
     'service:lenovo-smm': 'lenovo-smm',
     'service:lenovo-smm2': 'lenovo-smm2',
+    'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
+    #'openbmc': 'openbmc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
     'service:thinkagile-storage': 'thinkagile-storagebmc',
@@ -140,6 +145,7 @@ servicebyname = {
     'cumulus-switch': 'cumulus-switch',
     'lenovo-smm': 'service:lenovo-smm',
     'lenovo-smm2': 'service:lenovo-smm2',
+    'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
     'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2',
     'lenovo-switch': 'service:io-device.Lenovo:management-module',
@@ -223,6 +229,7 @@ def _printable_ip(sa):
 
 def send_discovery_datum(info):
     addresses = info.get('addresses', [])
+    addresses = util.natural_sort(addresses)
     if info['handler'] == pxeh:
         enrich_pxe_info(info)
     yield msg.KeyValueData({'nodename': info.get('nodename', '')})
@@ -356,6 +363,14 @@ def show_info(mac):
     for i in send_discovery_datum(known_info[mac]):
         yield i
 
+def dump_discovery():
+    infobymac = {}
+    for mac in known_info:
+        infobymac[mac] = {}
+        for i in send_discovery_datum(known_info[mac]):
+            for kn in i.kvpairs:
+                infobymac[mac][kn] = i.kvpairs[kn]
+    yield msg.KeyValueData(infobymac)
 
 list_info = {
     'by-node': list_matching_nodes,
@@ -599,11 +614,14 @@ def handle_read_api_request(pathcomponents):
     # starting at 2 are parameters to previous index
     if pathcomponents == ['discovery', 'rescan']:
         return (msg.KeyValueData({'scanning': bool(scanner)}),)
+    if pathcomponents == ['discovery', 'alldata']:
+        return dump_discovery()
     subcats, queryparms, indexof, coll = _parameterize_path(pathcomponents[1:])
     if len(pathcomponents) == 1:
         dirlist = [msg.ChildCollection(x + '/') for x in sorted(list(subcats))]
         dirlist.append(msg.ChildCollection('rescan'))
         dirlist.append(msg.ChildCollection('autosense'))
+        dirlist.append(msg.ChildCollection('alldata'))
         dirlist.append(msg.ChildCollection('subscriptions/'))
         return dirlist
     if not coll:
@@ -1143,6 +1161,38 @@ def get_nodename_from_enclosures(cfg, info):
     return nodename
 
 
+def search_smms_by_cert(currsmm, cert, cfg):
+    neighs = []
+    cv = util.TLSCertVerifier(
+        cfg, currsmm, 'pubkeys.tls_hardwaremanager').verify_cert
+    try:
+        cd = cfg.get_node_attributes(currsmm, ['hardwaremanagement.manager',
+                                               'pubkeys.tls_hardwaremanager'])
+        smmaddr = cd.get(currsmm, {}).get('hardwaremanagement.manager', {}).get('value', None)
+        wc = webclient.SecureHTTPConnection(currsmm, verifycallback=cv)
+        neighs = wc.grab_json_response('/scripts/neighdata.json')
+    except Exception:
+        return None
+    for neigh in neighs:
+        fprint = neigh.get('sha384', None)
+        if fprint and fprint.endswith('AA=='):
+            fprint = fprint[:-4]
+        if fprint and util.cert_matches(fprint, cert):
+            port = neigh.get('port', None)
+            if port is not None:
+                bay = port + 1
+                nl = list(
+                    cfg.filter_node_attributes('enclosure.manager=' + currsmm))
+                nl = list(
+                    cfg.filter_node_attributes('enclosure.bay={}'.format(bay), nl))
+                if len(nl) == 1:
+                    return currsmm, bay, nl[0]
+                return currsmm, bay, None
+    exnl = list(cfg.filter_node_attributes('enclosure.extends=' + currsmm))
+    if len(exnl) == 1:
+        return search_smms_by_cert(exnl[0], cert, cfg)
+
+
 def eval_node(cfg, handler, info, nodename, manual=False):
     try:
         handler.probe()  # unicast interrogation as possible to get more data
@@ -1177,6 +1227,14 @@ def eval_node(cfg, handler, info, nodename, manual=False):
         # The specified node is an enclosure (has nodes mapped to it), but
         # what we are talking to is *not* an enclosure
         # might be ambiguous, need to match chassis-uuid as well..
+        match = search_smms_by_cert(nodename, handler.https_cert, cfg)
+        if match:
+            info['verfied'] = True
+            info['enclosure.bay'] = match[1]
+            if match[2]:
+                if not discover_node(cfg, handler, info, match[2], manual):
+                    pending_nodes[match[2]] = info
+                return
         if 'enclosure.bay' not in info:
             unknown_info[info['hwaddr']] = info
             info['discostatus'] = 'unidentified'
@@ -1577,6 +1635,7 @@ def stop_autosense():
 
 def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
     remotescan()
 

confluent_server/confluent/discovery/protocols/mdns.py

@@ -0,0 +1,449 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Documented somewhat at
+# http://buildingskb.schneider-electric.com/view.php?AID=15197
+
+# Here is the payload of an SSDP 'announce', sent to the multicast v4/v6 1900
+# NOTIFY * HTTP/1.1
+# HOST: 239.255.255.250:1900
+# CACHE-CONTROL: max-age=1800
+# AL: https://172.30.254.151:8080/redfish/v1
+# SERVER: Linux/3.14.28-ltsi Redfish/1.0
+# NT: urn:dmtf-org:service:redfish-rest:1
+# USN: uuid:00000000-0000-0000-0005-000000000001::urn:dmtf-org:service:redfish-rest:1
+# NTS: ssdp:alive
+
+
+import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
+import confluent.neighutil as neighutil
+import confluent.noderange as noderange
+import confluent.util as util
+import confluent.log as log
+import confluent.netutil as netutil
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import eventlet.greenpool as gp
+import os
+import time
+import struct
+import traceback
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+mcastv4addr = '224.0.0.251'
+mcastv6addr = 'ff02::fb'
+
+mdns6mcast = socket.inet_pton(socket.AF_INET6, mcastv6addr)
+
+def name_to_qname(name):
+    nameparts = name.split('.')
+    qname = b''
+    for namepart in name.split('.'):
+        namepart = namepart.encode('utf8')
+        qname += bytes(bytearray([len(namepart)])) + namepart
+    qname += b'\x00'
+    return qname
+
+PTR = 12
+SRV = 33
+
+def _makequery(name):
+    return struct.pack('!HHHHHH', 
+            0, # transaction id
+            0, # flags, stdard query
+            1, # query count
+            0, # answers
+            0, # authorities
+            0) + \
+        name_to_qname(name) + \
+        struct.pack('!HH', 
+            PTR,
+            (1 << 15) | 1)  # Unicast response
+
+#listsrvs = _makequery('_services._dns-sd._udp.local')  # to get all possible services
+listobmccons = _makequery('_obmc_console._tcp.local')
+
+
+def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler):
+    if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']:
+        peerbymacaddress[mac]['addresses'].append(peer)
+    else:
+        sdata = _mdns_to_dict(rsp)
+        if not sdata:
+            return 0
+        peerdata = {
+            'hwaddr': mac,
+            'addresses': [peer],
+            'services': ['openbmc'],
+            'urls': '/redfish/v1/'
+        }
+        if sdata.get('ttl', 0) == 0:
+            if byehandler:
+                eventlet.spawn_n(check_fish_handler, byehandler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+            return 1
+        if handler:
+            eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+        return 2
+        
+def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer):
+    retdata = check_fish(('/redfish/v1/', peerdata))
+    if retdata:
+        known_peers.add(peer)
+        newmacs.add(mac)
+        peerbymacaddress[mac] = retdata
+        machandlers[mac] = handler
+
+
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
+    """Watch for unsolicited mDNS answers
+
+    The handler shall be called on any service coming online.
+    byehandler is called whenever a system advertises that it is departing.
+    If no byehandler is specified, byebye messages are ignored.  The handler is
+    given (as possible), the mac address, a list of viable sockaddrs to reference
+    the peer, and the notification type (e.g.
+    'urn:dmtf-org:service:redfish-rest:1'
+
+    :param handler:  A handler for online notifications from network
+    :param byehandler: Optional handler for devices going off the network
+    """
+    # Normally, I like using v6/v4 agnostic socket. However, since we are
+    # dabbling in multicast wizardry here, such sockets can cause big problems,
+    # so we will have two distinct sockets
+    # TTL=0 is a wthdrawal, otherwise an announce
+    tracelog = log.Logger('trace')
+    net4, net6 = get_sockets()
+    net6.bind(('', 5353))
+    net4.bind(('', 5353))
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                    event=log.Events.stacktrace)
+    known_peers = set([])
+    recent_peers = set([])
+    for ifidx in util.list_interface_indexes():
+        v6grp = mdns6mcast + struct.pack('=I', ifidx)
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
+    for i4 in util.list_ips():
+        mdns4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
+                     socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            mdns4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
+    peerbymacaddress = {}
+    while True:
+        try:
+            newmacs = set([])
+            deferrednotifies = []
+            machandlers = {}
+            r = select.select((net4, net6), (), (), 60)
+            if r:
+                r = r[0]
+            recent_peers = set([])
+            while r and len(deferrednotifies) < 256:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    if peer in recent_peers:
+                        continue
+                    mac = neighutil.get_hwaddr(peer[0])
+                    if not mac:
+                        probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                        try:
+                            s.sendto(b'\x00', probepeer)
+                        except Exception:
+                            continue
+                        deferrednotifies.append((peer, rsp))
+                    datum = _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+                    if datum == 2:
+                        recent_peers.add(peer)
+                r = select.select((net4, net6), (), (), 1.5)
+                if r:
+                    r = r[0]
+            if deferrednotifies:
+                eventlet.sleep(2.2)
+            for peerrsp in deferrednotifies:
+                peer, rsp = peerrsp
+                mac = neighutil.get_hwaddr(peer[0])
+                if not mac:
+                    continue
+                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
+
+
+def _get_svrip(peerdata):
+    for addr in peerdata['addresses']:
+        if addr[0].startswith('fe80::'):
+            if '%' not in addr[0]:
+                return addr[0] + '%{0}'.format(addr[3])
+            return addr[0]
+    return peerdata['addresses'][0][0]
+
+def get_sockets():
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    return net4, net6
+        
+def active_scan(handler, protocol=None):
+    net4, net6 = get_sockets()
+    for idx in util.list_interface_indexes():
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
+                        idx)
+        try:
+            net6.sendto(listobmccons, (mcastv6addr, 5353, 0, 0))
+        except socket.error:
+            # ignore interfaces without ipv6 multicast causing error
+                pass
+    for i4 in util.list_ips():
+        if 'broadcast' not in i4:
+            continue
+        addr = i4['addr']
+        net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                        socket.inet_aton(addr))
+        try:
+            net4.sendto(listobmccons, (mcastv4addr, 5353))
+        except socket.error as se:
+            if se.errno != 101 and se.errno != 1:
+                raise
+    deadline = util.monotonic_time() + 2
+    r, _, _ = select.select((net4, net6), (), (), 2)
+    peerdata = {}
+    deferparse = []
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            if not neighutil.get_hwaddr(peer[0]):
+                probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                try:
+                    s.sendto(b'\x00', probepeer)
+                except Exception:
+                    continue
+                deferparse.append((rsp, peer))
+                continue
+            _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
+    if deferparse:
+        eventlet.sleep(2.2)
+    for dp in deferparse:
+        rsp, peer = dp
+        _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+    querypool = gp.GreenPool()
+    pooltargs = []
+    for nid in peerdata:
+        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
+        pooltargs.append(('/redfish/v1/', peerdata[nid]))
+        # For now, don't interrogate generic redfish bmcs
+        # This is due to a need to deduplicate from some supported SLP
+        # targets (IMM, TSM, others)
+        # activate this else once the core filters/merges duplicate uuid
+        # or we drop support for those devices
+        #else:
+        #    pooltargs.append(('/redfish/v1/', peerdata[nid]))
+    for pi in querypool.imap(check_fish, pooltargs):
+        if pi is not None:
+            handler(pi)
+
+def check_fish(urldata, port=443, verifycallback=None):
+    if not verifycallback:
+        verifycallback = lambda x: True
+    url, data = urldata
+    try:
+        wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5)
+        peerinfo = wc.grab_json_response(url)
+    except socket.error:
+        return None
+    if url == '/DeviceDescription.json':
+        try:
+            peerinfo = peerinfo[0]
+            myuuid = peerinfo['node-uuid'].lower()
+            if '-' not in myuuid:
+                myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]])
+            data['uuid'] = myuuid
+            data['attributes'] = peerinfo
+            data['services'] = ['lenovo-xcc']
+            return data
+        except (IndexError, KeyError):
+            return None
+            url = '/redfish/v1/'
+            peerinfo = wc.grab_json_response('/redfish/v1/')
+    if url == '/redfish/v1/':
+        if 'UUID' in peerinfo:
+            if 'services' not in data:
+                data['services'] = ['service:redfish-bmc']
+            else:
+                data['services'].append('service:redfish-bmc')
+            data['uuid'] = peerinfo['UUID'].lower()
+            return data
+    return None
+
+def extract_qname(view, reply):
+    name = ''
+    idx = 1
+    if isinstance(view[0], int):
+        currlen = view[0]
+    else:
+        currlen = ord(view[0])
+    while currlen:
+        if currlen == 192:
+            name += extract_qname(reply[view[1]:], reply)[1] + '.'
+            view = view[2:]
+            idx += 1
+            if name:
+                return idx, name[:-1]
+            return idx, ''
+        else:
+            name += view[1:currlen + 1].tobytes().decode('utf8') + '.'
+            view = view[currlen + 1:]
+            idx += currlen + 1
+        if not view:
+            break  # some contexts don't null terminate
+        if isinstance(view[0], int):
+            currlen = view[0]
+        else:
+            currlen = ord(view[0])
+    if name:
+        return idx, name[:-1]
+    return idx, ''
+
+
+
+def _mdns_to_dict(rsp):
+    txid, flags, quests, answers, arr, morerr = struct.unpack('!HHHHHH', rsp[:12])
+    rv = memoryview(rsp[12:])
+    rspv = memoryview(rsp)
+    retval = {}
+    while quests:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass = struct.unpack('!HH', rv[:4]) 
+        quests -= 1
+        rv = rv[4:]
+    while answers:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass, ttl, dlen = struct.unpack('!HHIH', rv[:10])
+        if 0 and typ == 12:  # PTR, we don't need for now...
+            adata = extract_qname(rv[10:], rspv)
+            if 'ptrs' not in retval:
+                retval['ptrs'] = [{'name': adata, 'ttl': ttl}]
+            else:
+                retval['ptrs'].append({'name': adata, 'ttl': ttl})
+        if typ == 33:
+            portnum = struct.unpack('!H', rv[14:16])[0]
+            retval['protoname'] = name.split('.', 1)[1]
+            retval['portnumber'] = portnum
+            retval['ttl'] = ttl
+        rv = rv[dlen + 10:]
+        answers -= 1
+    return retval
+
+
+def _parse_mdns(peer, rsp, peerdata, srvname):
+    parsed = _mdns_to_dict(rsp)
+    if not parsed:
+        return
+    if parsed.get('ttl', 0) == 0:
+        return
+    nid = peer[0]
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    if nid in peerdata:
+        peerdatum = peerdata[nid]
+        if peer not in peerdatum['addresses']:
+            peerdatum['addresses'].append(peer)
+    else:
+        peerdatum = {
+            'addresses': [peer],
+            'hwaddr': mac,
+            'services': [srvname]
+        }
+        if srvname == '_obmc_console._tcp.local':
+            peerdatum['services'] = ['openbmc']
+            peerdatum['urls'] = ['/redfish/v1/']
+        peerdata[nid] = peerdatum
+    
+
+def _parse_ssdp(peer, rsp, peerdata):
+    nid = peer[0]
+    mac = None
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    headlines = rsp.split(b'\r\n')
+    try:
+        _, code, _ = headlines[0].split(b' ', 2)
+    except ValueError:
+        return
+    if code == b'200':
+        if nid in peerdata:
+            peerdatum = peerdata[nid]
+            if peer not in peerdatum['addresses']:
+                peerdatum['addresses'].append(peer)
+        else:
+            peerdatum = {
+                'addresses': [peer],
+                'hwaddr': mac,
+            }
+            peerdata[nid] = peerdatum
+        for headline in headlines[1:]:
+            if not headline:
+                continue
+            header, _, value = headline.partition(b':')
+            header = header.strip().decode('utf8')
+            value = value.strip().decode('utf8')
+            if header == 'AL' or header == 'LOCATION':
+                value = value[value.index('://')+3:]
+                value = value[value.index('/'):]
+                if 'urls' not in peerdatum:
+                    peerdatum['urls'] = [value]
+                elif value not in peerdatum['urls']:
+                    peerdatum['urls'].append(value)
+            elif header == 'ST':
+                if 'services' not in peerdatum:
+                    peerdatum['services'] = [value]
+                elif value not in peerdatum['services']:
+                    peerdatum['services'].append(value)
+            elif header == 'USN':
+                peerdatum['usn'] = value
+            elif header == 'MODELNAME':
+                peerdatum['modelname'] = value
+
+
+from pprint import pprint
+if __name__ == '__main__':
+    def printit(rsp):
+        print(repr(rsp))
+    snoop(pprint)

confluent_server/confluent/discovery/protocols/pxe.py

@@ -297,90 +297,96 @@ def proxydhcp(handler, nodeguess):
     msg.msg_namelen = ctypes.sizeof(clientaddr)
     cfg = cfm.ConfigManager(None)
     while True:
-        ready = select.select([net4011], [], [], None)
-        if not ready or not ready[0]:
-            continue
-        i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
-        #nb, client = net4011.recvfrom_into(rq)
-        if i < 240:
-            continue
-        rqv = memoryview(rq)[:i]
-        client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
-        _, level, typ = struct.unpack('QII', cmsgarr[:16])
-        if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
-            idx, recv = struct.unpack('II', cmsgarr[16:24])
-            recv = ipfromint(recv)
         try:
-            optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
-        except ValueError:
-            continue
-        hwlen = rq[2]
-        opts, disco = opts_to_dict(rq, optidx, 3)
-        disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
-        node = None
-        if disco.get('hwaddr', None) in macmap:
-            node = macmap[disco['hwaddr']]
-        elif disco.get('uuid', None) in uuidmap:
-            node = uuidmap[disco['uuid']]
-        myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
-        skiplogging = True
-        netaddr = disco['hwaddr']
-        if time.time() > ignoredisco.get(netaddr, 0) + 90:
-            skiplogging = False
-            ignoredisco[netaddr] = time.time()
-        if not myipn:
-            info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                            'architecture': disco['arch'],
-                            'netinfo': {'ifidx': idx, 'recvip': recv},
-                            'services': ('pxe-client',)}
-            if not skiplogging:
-                handler(info)
-        if not node:
-            if not myipn and not skiplogging:
-                log.log(
-                        {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
-                            disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
-                )})
-            continue
-        profile = None
-        if not myipn:
-            myipn = socket.inet_aton(recv)
-            profile = get_deployment_profile(node, cfg)
-            if profile:
-                log.log({
-                    'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
-            else:
+            ready = select.select([net4011], [], [], None)
+            if not ready or not ready[0]:
+                continue
+            i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
+            #nb, client = net4011.recvfrom_into(rq)
+            if i < 240:
+                continue
+            rqv = memoryview(rq)[:i]
+            client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
+            _, level, typ = struct.unpack('QII', cmsgarr[:16])
+            if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+                idx, recv = struct.unpack('II', cmsgarr[16:24])
+                recv = ipfromint(recv)
+            try:
+                optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
+            except ValueError:
+                continue
+            hwlen = rq[2]
+            opts, disco = opts_to_dict(rq, optidx, 3)
+            disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
+            node = None
+            if disco.get('hwaddr', None) in macmap:
+                node = macmap[disco['hwaddr']]
+            elif disco.get('uuid', None) in uuidmap:
+                node = uuidmap[disco['uuid']]
+            myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
+            skiplogging = True
+            netaddr = disco['hwaddr']
+            if time.time() > ignoredisco.get(netaddr, 0) + 90:
+                skiplogging = False
+                ignoredisco[netaddr] = time.time()
+            if not myipn:
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                                'architecture': disco['arch'],
+                                'netinfo': {'ifidx': idx, 'recvip': recv},
+                                'services': ('pxe-client',)}
                 if not skiplogging:
-                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    handler(info)
+            if not node:
+                if not myipn and not skiplogging:
+                    log.log(
+                            {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
+                                disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
+                    )})
                 continue
-        if opts.get(77, None) == b'iPXE':
-            if not profile:
+            profile = None
+            if not myipn:
+                myipn = socket.inet_aton(recv)
                 profile = get_deployment_profile(node, cfg)
-            if not profile:
-                log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                if profile:
+                    log.log({
+                        'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
+                else:
+                    if not skiplogging:
+                        log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+            if opts.get(77, None) == b'iPXE':
+                if not profile:
+                    profile = get_deployment_profile(node, cfg)
+                if not profile:
+                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+                myip = socket.inet_ntoa(myipn)
+                bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
+            elif disco['arch'] == 'uefi-x64':
+                bootfile = b'confluent/x86_64/ipxe.efi'
+            elif disco['arch'] == 'bios-x86':
+                bootfile = b'confluent/x86_64/ipxe.kkpxe'
+            elif disco['arch'] == 'uefi-aarch64':
+                bootfile = b'confluent/aarch64/ipxe.efi'
+            if len(bootfile) > 127:
+                log.log(
+                    {'info': 'Boot offer cannot be made to {0} as the '
+                    'profile name "{1}" is {2} characters longer than is supported '
+                    'for this boot method.'.format(
+                        node, profile, len(bootfile) - 127)})
                 continue
-            myip = socket.inet_ntoa(myipn)
-            bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
-        elif disco['arch'] == 'uefi-x64':
-            bootfile = b'confluent/x86_64/ipxe.efi'
-        elif disco['arch'] == 'bios-x86':
-            bootfile = b'confluent/x86_64/ipxe.kkpxe'
-        if len(bootfile) > 127:
-            log.log(
-                {'info': 'Boot offer cannot be made to {0} as the '
-                'profile name "{1}" is {2} characters longer than is supported '
-                'for this boot method.'.format(
-                    node, profile, len(bootfile) - 127)})
-            continue
-        rpv[:240] = rqv[:240].tobytes()
-        rpv[0:1] = b'\x02'
-        rpv[108:108 + len(bootfile)] = bootfile
-        rpv[240:243] = b'\x35\x01\x05'
-        rpv[243:249] = b'\x36\x04' + myipn
-        rpv[20:24] = myipn
-        rpv[249:268] = b'\x61\x11' + opts[97]
-        rpv[268:280] = b'\x3c\x09PXEClient\xff'
-        net4011.sendto(rpv[:281], client)
+            rpv[:240] = rqv[:240].tobytes()
+            rpv[0:1] = b'\x02'
+            rpv[108:108 + len(bootfile)] = bootfile
+            rpv[240:243] = b'\x35\x01\x05'
+            rpv[243:249] = b'\x36\x04' + myipn
+            rpv[20:24] = myipn
+            rpv[249:268] = b'\x61\x11' + opts[97]
+            rpv[268:280] = b'\x3c\x09PXEClient\xff'
+            net4011.sendto(rpv[:281], client)
+        except Exception as e:
+            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                            event=log.Events.stacktrace)
 
 
 def start_proxydhcp(handler, nodeguess=None):

confluent_server/confluent/discovery/protocols/slp.py

@@ -63,6 +63,9 @@ def _parse_slp_header(packet):
                                            bytes(b'\x00' + packet[7:14]))
     parsed['lang'] = packet[14:14 + langlen].decode('utf-8')
     parsed['payload'] = packet[14 + langlen:]
+    errcode = struct.unpack('!H', packet[14 + langlen:16 + langlen])[0]
+    if errcode != 0:
+        return None
     if offset:
         parsed['offset'] = 14 + langlen
         parsed['extoffset'] = offset

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -58,7 +58,7 @@ smsg = ('M-SEARCH * HTTP/1.1\r\n'
 
 def active_scan(handler, protocol=None):
     known_peers = set([])
-    for scanned in scan(['urn:dmtf-org:service:redfish-rest:1']):
+    for scanned in scan(['urn:dmtf-org:service:redfish-rest:1', 'urn::service:affluent']):
         for addr in scanned['addresses']:
             if addr in known_peers:
                 break
@@ -258,10 +258,15 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                         seconds = int(currtime)
                                         msecs = int(currtime * 1000 % 1000)
                                         reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(node, seconds, msecs)
-                                        if '%' in peer[0]:
-                                            iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
-                                            reply += 'MGTIFACE: {0}\r\n'.format(
-                                                peer[0].split('%', 1)[1])
+                                        theip = peer[0].split('%', 1)[0]
+                                        if netutil.ip_on_same_subnet(theip, 'fe80::', 64):
+                                            if '%' in peer[0]:
+                                                ifidx = peer[0].split('%', 1)[1]
+                                                iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
+                                            else:
+                                                ifidx = '{}'.format(peer[-1])
+                                                iface = peer[-1]
+                                            reply += 'MGTIFACE: {0}\r\n'.format(ifidx)
                                             ncfg = netutil.get_nic_config(
                                                 cfg, node, ifidx=iface)
                                             if ncfg.get('matchesnodename', None):
@@ -345,12 +350,15 @@ def _find_service(service, target):
             try:
                 net4.sendto(msg, (mcastv4addr, 1900))
             except socket.error as se:
-                if se.errno != 101:
+                if se.errno != 101 and se.errno != 1:
                     raise
             msg = smsg.format(bcast, service)
             if not isinstance(msg, bytes):
                 msg = msg.encode('utf8')
-            net4.sendto(msg, (bcast, 1900))
+            try:
+                net4.sendto(msg, (bcast, 1900))
+            except socket.error:
+                pass
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
     deadline = util.monotonic_time() + 4
@@ -381,6 +389,24 @@ def _find_service(service, target):
     querypool = gp.GreenPool()
     pooltargs = []
     for nid in peerdata:
+        if peerdata[nid].get('services', [None])[0] == 'urn::service:affluent:1':
+            peerdata[nid]['attributes'] = {
+                'type': 'affluent-switch',
+            }
+            peerdata[nid]['services'] = ['affluent-switch']
+            mya = peerdata[nid]['attributes']
+            usn = peerdata[nid]['usn']
+            idinfo = usn.split('::')
+            for idi in idinfo:
+                key, val = idi.split(':', 1)
+                if key == 'uuid':
+                    peerdata[nid]['uuid'] = val
+                elif key == 'serial':
+                    mya['enclosure-serial-number'] = [val]
+                elif key == 'model':
+                    mya['enclosure-machinetype-model'] = [val]
+            yield peerdata[nid]
+            continue
         if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
             continue
         if '/DeviceDescription.json' in peerdata[nid]['urls']:
@@ -466,6 +492,10 @@ def _parse_ssdp(peer, rsp, peerdata):
                     peerdatum['services'] = [value]
                 elif value not in peerdatum['services']:
                     peerdatum['services'].append(value)
+            elif header == 'USN':
+                peerdatum['usn'] = value
+            elif header == 'MODELNAME':
+                peerdatum['modelname'] = value
 
 
 

confluent_server/confluent/forwarder.py

@@ -119,7 +119,7 @@ def get_port(addr, clientip, sessionid):
                 newsock.bind(('::', newport, 0, 0))
                 newsock.listen(50)
                 break
-            except OSError as e:
+            except (socket.error, OSError) as e:
                 if e.errno == 98:
                     newport += 1
                     continue

confluent_server/confluent/httpapi.py

@@ -288,12 +288,16 @@ def _authorize_request(env, operation, reqbody):
             authdata = auth.authorize(name, element=element, operation=operation)
         else:
             element = None
-    if (not authdata) and 'HTTP_COOKIE' in env:
-        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
-        if cidx >= 0:
-            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
-            sessid = sessionid
+    if not authdata:
+        if 'HTTP_CONFLUENTSESSION' in env:
+            sessionid = env['HTTP_CONFLUENTSESSION']
             sessid = sessionid
+        elif 'HTTP_COOKIE' in env:
+            cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+            if cidx >= 0:
+                sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+                sessid = sessionid
+        if sessionid:
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):
                     if env['PATH_INFO'] == '/sessions/current/logout':
@@ -483,6 +487,8 @@ def wsock_handler(ws):
                         elif clientmsg[0] == '!':
                             msg = json.loads(clientmsg[1:])
                             action = msg.get('operation', None)
+                            if not action:
+                                action = msg.get('action', None)
                             targ = msg.get('target', None)
                             if targ:
                                 authdata = auth.authorize(name, targ, operation=action)
@@ -520,6 +526,13 @@ def wsock_handler(ws):
                                             datacallback=datacallback,
                                             width=width, height=height)
                                     myconsoles[clientsessid] = consession
+                            elif action == 'resize':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].resize(
+                                    width=msg['width'], height=msg['height'])
+                            if action == 'break':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].send_break()
                             elif action == 'stop':
                                 sessid = '{0}'.format(msg.get('sessid', None))
                                 if sessid in myconsoles:

confluent_server/confluent/main.py

@@ -306,6 +306,7 @@ def run(args):
         except AttributeError:
             pass  # Windows...
         os.umask(oumask)
+    auth.check_for_yaml()
     collective.startup()
     consoleserver.initialize()
     http_bind_host, http_bind_port = _get_connector_config('http')

confluent_server/confluent/messages.py

@@ -1682,16 +1682,19 @@ class NetworkConfiguration(ConfluentMessage):
     desc = 'Network configuration'
 
     def __init__(self, name=None, ipv4addr=None, ipv4gateway=None,
-                 ipv4cfgmethod=None, hwaddr=None):
+                 ipv4cfgmethod=None, hwaddr=None, staticv6addrs=(), staticv6gateway=None):
         self.myargs = (name, ipv4addr, ipv4gateway, ipv4cfgmethod, hwaddr)
         self.notnode = name is None
         self.stripped = False
+        v6addrs = ','.join(staticv6addrs)
 
         kvpairs = {
             'ipv4_address': {'value': ipv4addr},
             'ipv4_gateway': {'value': ipv4gateway},
             'ipv4_configuration': {'value': ipv4cfgmethod},
             'hw_addr': {'value': hwaddr},
+            'static_v6_addresses': {'value': v6addrs},
+            'static_v6_gateway': {'value': staticv6gateway}
         }
         if self.notnode:
             self.kvpairs = kvpairs

confluent_server/confluent/netutil.py

@@ -198,8 +198,9 @@ class NetManager(object):
             ipv4addr = attribs.get('ipv4_address', None)
             if ipv4addr:
                 try:
-                    for ai in socket.getaddrinfo(ipv4addr, 0, socket.AF_INET, socket.SOCK_STREAM):
-                        ipv4addr = ai[-1][0]
+                    luaddr = ipv4addr.split('/', 1)[0]
+                    for ai in socket.getaddrinfo(luaddr, 0, socket.AF_INET, socket.SOCK_STREAM):
+                        ipv4addr.replace(luaddr, ai[-1][0])
                 except socket.gaierror:
                     pass
             else:
@@ -661,6 +662,8 @@ def addresses_match(addr1, addr2):
     :param addr2:
     :return: True if the given addresses refer to the same thing
     """
+    if '%' in addr1 or '%' in addr2:
+        return False
     for addrinfo in socket.getaddrinfo(addr1, 0, 0, socket.SOCK_STREAM):
         rootaddr1 = socket.inet_pton(addrinfo[0], addrinfo[4][0])
         if addrinfo[0] == socket.AF_INET6 and rootaddr1[:12] == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff':

confluent_server/confluent/networking/macmap.py

@@ -31,7 +31,14 @@
 # this module will provide mac to switch and full 'ifName' label
 # This functionality is restricted to the null tenant
 
+import os
+import sys
+
 if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
     import confluent.config.configmanager as cfm
     import confluent.snmputil as snmp
 
@@ -43,8 +50,6 @@ import eventlet.green.select as select
 import eventlet.green.socket as socket
 
 
-import os
-import sys
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
@@ -208,7 +213,6 @@ def _offload_map_switch(switch, password, user):
 
 def _start_offloader():
     global _offloader
-    os.environ['PYTHONPATH'] = ':'.join(sys.path)
     _offloader = subprocess.Popen(
         [sys.executable, __file__, '-o'], bufsize=0, stdin=subprocess.PIPE,
         stdout=subprocess.PIPE)
@@ -220,7 +224,10 @@ def _start_offloader():
 
 
 def _recv_offload():
-    upacker = msgpack.Unpacker(encoding='utf8')
+    try:
+        upacker = msgpack.Unpacker(encoding='utf8')
+    except TypeError:
+        upacker = msgpack.Unpacker(raw=False, strict_map_key=False)
     instream = _offloader.stdout.fileno()
     while True:
         select.select([_offloader.stdout], [], [])
@@ -601,7 +608,7 @@ def handle_read_api_request(pathcomponents, configmanager):
     elif len(pathcomponents) == 2:
         if pathcomponents[-1] == 'macs':
             return [msg.ChildCollection(x) for x in (# 'by-node/',
-                                                     'by-mac/', 'by-switch/',
+                                                     'alldata', 'by-mac/', 'by-switch/',
                                                      'rescan')]
         elif pathcomponents[-1] == 'neighbors':
             return [msg.ChildCollection('by-switch/')]
@@ -616,6 +623,8 @@ def handle_read_api_request(pathcomponents, configmanager):
         elif len(pathcomponents) == 4:
             macaddr = pathcomponents[-1].replace('-', ':')
             return dump_macinfo(macaddr)
+    elif pathcomponents[2] == 'alldata':
+        return [msg.KeyValueData(_apimacmap)]
     elif pathcomponents[2] == 'by-mac':
         if len(pathcomponents) == 3:
             return [msg.ChildCollection(x.replace(':', '-'))
@@ -687,7 +696,10 @@ def rescan(cfg):
 
 if __name__ == '__main__':
     if len(sys.argv) > 1 and sys.argv[1] == '-o':
-        upacker = msgpack.Unpacker(encoding='utf8')
+        try:
+            upacker = msgpack.Unpacker(encoding='utf8')
+        except TypeError:
+            upacker = msgpack.Unpacker(raw=False, strict_map_key=False)
         currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
         fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl | os.O_NONBLOCK)
 
@@ -712,4 +724,4 @@ if __name__ == '__main__':
         print("Mac to location lookup table: -------------------")
         print(repr(_macmap))
         print("switch to fdb lookup table: -------------------")
-        print(repr(_macsbyswitch))
+        print(repr(_macsbyswitch))

confluent_server/confluent/noderange.py

@@ -186,7 +186,7 @@ class NodeRange(object):
         for idx in range(len(leftbits)):
             if leftbits[idx] == rightbits[idx]:
                 finalfmt += leftbits[idx]
-            elif leftbits[idx][0] in pp.alphas:
+            elif leftbits[idx][0] in pp.alphas or rightbits[idx][0] in pp.alphas:
                 # if string portion unequal, not going to work
                 return self.failorreturn(seqrange)
             else:

confluent_server/confluent/osimage.py

@@ -1,6 +1,4 @@
 #!/usr/bin/python
-import confluent.exceptions as exc
-import confluent.messages as msg
 import eventlet
 import eventlet.green.select as select
 import eventlet.green.subprocess as subprocess
@@ -14,6 +12,14 @@ import shutil
 import sys
 import time
 import yaml
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
+
+import confluent.exceptions as exc
+import confluent.messages as msg
 
 COPY = 1
 EXTRACT = 2
@@ -376,6 +382,20 @@ def check_ubuntu(isoinfo):
         if not isinstance(arch, str):
             arch = arch.decode('utf8')
         major = '.'.join(ver.split('.', 2)[:2])
+        if 'install/hwe-netboot/ubuntu-installer/amd64/linux' in isoinfo[0]:
+            # debian-installer style amd64
+            return {
+                'name': 'ubuntu-{0}-{1}'.format(ver, arch),
+                'method': EXTRACT,
+                'category': 'ubuntu{0}'.format(major)}
+        elif 'efi/boot/bootaa64.efi' in isoinfo[0]:
+            exlist = ['casper/vmlinuz', 'casper/initrd',
+                    'efi/boot/bootaa64.efi', 'efi/boot/grubaa64.efi'
+                    ]
+        else:
+            exlist = ['casper/vmlinuz', 'casper/initrd',
+                    'efi/boot/bootx64.efi', 'efi/boot/grubx64.efi'
+                    ]
         return {'name': 'ubuntu-{0}-{1}'.format(ver, arch),
                 'method': EXTRACT|COPY,
                 'extractlist': ['casper/vmlinuz', 'casper/initrd',
@@ -765,10 +785,15 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname,
             yout.write('# This manifest enables rebase to know original source of profile data and if any customizations have been done\n')
             manifestdata = {'distdir': srcname, 'disthashes': hmap}
             yout.write(yaml.dump(manifestdata, default_flow_style=False))
-        for initrd in os.listdir('{0}/initramfs'.format(defprofile)):
-            fullpath = '{0}/initramfs/{1}'.format(defprofile, initrd)
+        initrds = ['{0}/initramfs/{1}'.format(defprofile, initrd) for initrd in os.listdir('{0}/initramfs'.format(defprofile))]
+        if os.path.exists('{0}/initramfs/{1}'.format(defprofile, arch)):
+            initrds.extend(['{0}/initramfs/{1}/{2}'.format(defprofile, arch, initrd) for initrd in os.listdir('{0}/initramfs/{1}'.format(defprofile, arch))])
+        for fullpath in initrds:
+            initrd = os.path.basename(fullpath)
             if os.path.isdir(fullpath):
                 continue
+            if os.path.exists('{0}/boot/initramfs/{1}'.format(dirname, initrd)):
+                os.remove('{0}/boot/initramfs/{1}'.format(dirname, initrd))
             os.symlink(fullpath,
                        '{0}/boot/initramfs/{1}'.format(dirname, initrd))
         os.symlink(
@@ -792,11 +817,17 @@ class MediaImporter(object):
             raise Exception('`osdeploy initialize` must be executed before importing any media')
         self.profiles = []
         medfile = None
+        self.medfile = None
         if cfm and media in cfm.clientfiles:
-            medfile = cfm.clientfiles[media]
+            self.medfile = cfm.clientfiles[media]
+            medfile = self.medfile
         else:
             medfile = open(media, 'rb')
-        identity = fingerprint(medfile)
+        try:
+            identity = fingerprint(medfile)
+        finally:
+            if not self.medfile:
+                medfile.close()
         if not identity:
             raise exc.InvalidArgumentException('Unsupported Media')
         self.percent = 0.0
@@ -824,7 +855,6 @@ class MediaImporter(object):
             del importing[importkey]
             raise Exception('{0} already exists'.format(self.targpath))
         self.filename = os.path.abspath(media)
-        self.medfile = medfile
         self.error = ''
         self.importer = eventlet.spawn(self.importmedia)
 
@@ -837,8 +867,8 @@ class MediaImporter(object):
         return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles, 'error': self.error}
 
     def importmedia(self):
-        os.environ['PYTHONPATH'] = ':'.join(sys.path)
-        os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
+        if self.medfile:
+            os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
         with open(os.devnull, 'w') as devnull:
             self.worker = subprocess.Popen(
                 [sys.executable, __file__, self.filename, '-b'],
@@ -911,3 +941,4 @@ if __name__ == '__main__':
         sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd))
     else:
         sys.exit(import_image(sys.argv[1], callback=printit))
+

confluent_server/confluent/plugins/hardwaremanagement/affluent.py

@@ -49,7 +49,8 @@ class WebClient(object):
             results.put(msg.ConfluentTargetInvalidCredentials(self.node, 'Unable to authenticate'))
             return {}
         elif status != 200:
-            results.put(msg.ConfluentNodeError(self.node, 'Unknown error: ' + rsp + ' while retrieving ' + url))
+            #must be str not bytes
+            results.put(msg.ConfluentNodeError(self.node, 'Unknown error: {} while retrieving {}'.format(rsp, url)))
             return {}
         return rsp
     
@@ -192,4 +193,4 @@ def retrieve_health(configmanager, creds, node, results, element):
     hinfo = wc.fetch('/affluent/health', results) 
     if hinfo:
         results.put(msg.HealthSummary(hinfo.get('health', 'unknown'), name=node))
-        results.put(msg.SensorReadings(hinfo.get('sensors', []), name=node))
+        results.put(msg.SensorReadings(hinfo.get('sensors', []), name=node))

confluent_server/confluent/plugins/hardwaremanagement/cooltera.py

@@ -0,0 +1,266 @@
+# Copyright 2022 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from xml.etree.ElementTree import fromstring as rfromstring
+import pyghmi.util.webclient as wc
+import confluent.util as util
+import confluent.messages as msg
+import confluent.exceptions as exc
+import eventlet.green.time as time
+import eventlet.green.socket as socket
+import eventlet.greenpool as greenpool
+import eventlet
+try:
+    import Cookie
+    httplib = eventlet.import_patched('httplib')
+except ImportError:
+    httplib = eventlet.import_patched('http.client')
+    import http.cookies as Cookie
+
+sensorsbymodel = {
+    'FS1350': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve', 'valve2'],
+    'FS600': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'pdp', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+    'RM100': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps2', 'ps3', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+}
+_thesensors = {
+    'RM100': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS600': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS1350': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'pumpspeed3': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+        'ps4': ('Primary loop return pressure', 'bar'),
+    },
+}
+
+
+def fromstring(inputdata):
+    if isinstance(inputdata, bytes):
+        cmpstr = b'!entity'
+    else:
+        cmpstr = '!entity'
+    if cmpstr in inputdata.lower():
+        raise Exception('!ENTITY not supported in this interface')
+    # The measures above should filter out the risky facets of xml
+    # We don't need sophisticated feature support
+    return rfromstring(inputdata)  # nosec
+
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+pdupool = greenpool.GreenPool(128)
+
+class WebResponse(httplib.HTTPResponse):
+    def _check_close(self):
+        return True
+
+class WebConnection(wc.SecureHTTPConnection):
+    response_class = WebResponse
+    def __init__(self, host, secure, verifycallback):
+        if secure:
+            port = 443
+        else:
+            port = 80
+        wc.SecureHTTPConnection.__init__(self, host, port, verifycallback=verifycallback)
+        self.secure = secure
+        self.cookies = {}
+
+    def connect(self):
+        if self.secure:
+            return super(WebConnection, self).connect()
+        addrinfo = socket.getaddrinfo(self.host, self.port)[0]
+        # workaround problems of too large mtu, moderately frequent occurance
+        # in this space
+        plainsock = socket.socket(addrinfo[0])
+        plainsock.settimeout(self.mytimeout)
+        try:
+            plainsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
+        except socket.error:
+            pass
+        plainsock.connect(addrinfo[4])
+        self.sock = plainsock
+    
+    def getresponse(self):
+        try:
+            rsp = super(WebConnection, self).getresponse()
+            try:
+                hdrs = [x.split(':', 1) for x in rsp.msg.headers]
+            except AttributeError:
+                hdrs = rsp.msg.items()
+            for hdr in hdrs:
+                if hdr[0] == 'Set-Cookie':
+                    c = Cookie.BaseCookie(hdr[1])
+                    for k in c:
+                        self.cookies[k] = c[k].value
+        except httplib.BadStatusLine:
+            self.broken = True
+            raise
+        return rsp
+
+    def request(self, method, url, body=None):
+        headers = {}
+        if body:
+            headers['Content-Length'] = len(body)
+        cookies = []
+        for cookie in self.cookies:
+            cookies.append('{0}={1}'.format(cookie, self.cookies[cookie]))
+        headers['Cookie'] = ';'.join(cookies)
+        headers['Host'] = 'pdu.cluster.net'
+        headers['Accept'] = '*/*'
+        headers['Accept-Language'] = 'en-US,en;q=0.9'
+        headers['Connection'] = 'close'
+        headers['Referer'] = 'http://pdu.cluster.net/setting_admin.htm'
+        return super(WebConnection, self).request(method, url, body, headers)
+
+    def grab_response(self, url, body=None, method=None):
+        if method is None:
+            method = 'GET' if body is None else 'POST'
+        if body:
+            self.request(method, url, body)
+        else:
+            self.request(method, url)
+        rsp = self.getresponse()
+        body = rsp.read()
+        return body, rsp.status
+
+class CoolteraClient(object):
+    def __init__(self, cdu, configmanager):
+        self.node = cdu
+        self.configmanager = configmanager
+        self._wc = None
+
+    @property
+    def wc(self):
+        if self._wc:
+            return self._wc
+        targcfg = self.configmanager.get_node_attributes(self.node,
+                                            ['hardwaremanagement.manager'],
+                                            decrypt=True)
+        targcfg = targcfg.get(self.node, {})
+        target = targcfg.get(
+            'hardwaremanagement.manager', {}).get('value', None)
+        if not target:
+            target = self.node
+        target = target.split('/', 1)[0]
+        cv = util.TLSCertVerifier(
+            self.configmanager, self.node,
+            'pubkeys.tls_hardwaremanager').verify_cert
+        self._wc = WebConnection(target, secure=True, verifycallback=cv)
+        return self._wc
+
+    
+def xml2stateinfo(statdata):
+    statdata = fromstring(statdata)
+    stateinfo = []
+    sensornames = sorted([x.tag for x in statdata])
+    themodel = None
+    for model in sensorsbymodel:
+        if sensorsbymodel[model] == sensornames:
+            themodel = model
+            break
+    thesensors = _thesensors[themodel]
+#['mode', 't1', 't2a', 't2b', 't2c', 't2', 't5', 't3', 't4', 'dw', 't3', 'rh', 'setpoint', 'secflow', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'sdp', 'valve', 'valve2', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'alarms', 'dt', 'p3state', 'duty']
+    for tagname in thesensors:
+        label, units = thesensors[tagname]
+        val = statdata.find(tagname).text.replace(units, '').strip()
+        stateinfo.append({
+            'name': label,
+            'value': val,
+            'units': units.replace('degC', '°C'),
+            'type': 'Temperature',
+        })
+    return stateinfo
+
+    
+_sensors_by_node = {}
+def read_sensors(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+        for sensor in sensors:
+            yield msg.ChildCollection(simplify_name(sensors[sensor][0]))
+        return     
+    if category in ('leds, fans'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        cc = CoolteraClient(node, configmanager)
+        cc.wc.request('GET', '/status.xml')
+        rsp = cc.wc.getresponse()
+        statdata = rsp.read()
+        statinfo = xml2stateinfo(statdata)
+        _sensors_by_node[node] = (statinfo, time.time() + 1)
+        sn = _sensors_by_node.get(node, None)
+    if sn:
+        yield msg.SensorReadings(sn[0], name=node)
+
+
+def retrieve(nodes, element, configmanager, inputdata):
+    if element[0] == 'sensors':
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(read_sensors, element, node, configmanager)
+        for rsp in gp:
+            for datum in rsp:
+                yield datum
+        return

confluent_server/confluent/plugins/hardwaremanagement/deltapdu.py

@@ -199,11 +199,12 @@ def update(nodes, element, configmanager, inputdata):
     if 'outlets' not in element:
         yield msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return
+    timeout = 4
     for node in nodes:
         gc = PDUClient(node, configmanager)
         newstate = inputdata.powerstate(node)
         gc.set_outlet(element[-1], newstate)
         gc.logout()
-    eventlet.sleep(2)
+    eventlet.sleep(timeout)
     for res in retrieve(nodes, element, configmanager, inputdata):
         yield res