Correct the equality message in better messagesw

Some versions start manifesting nvme devnames with 'c', which
are to be used to interact with multipath to have raw devices
backing a traditional nvme device.

confluent_client/bin/nodeapply

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+devnull = None
+
+def run():
+    global devnull
+    devnull = open(os.devnull, 'rb')
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-k', '--security', action='store_true',
+                         help='Update SSH setup')
+    argparser.add_option('-F', '--sync', action='store_true',
+                         help='Run the syncfiles associated with the currently completed OS profile on the noderange')
+    argparser.add_option('-P', '--scripts',
+                         help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                    help='Specify a maximum number of '
+                        'nodes to run remote ssh command to, '
+                        'prompting if over the threshold')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    #argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    c.stop_if_noderange_over(args[0], options.maxnodes)
+    nodemap = {}
+    cmdparms = []
+    nodes = []
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
+    
+    cmdstorun = []
+    if options.security:
+        cmdstorun.append(['run_remote', 'setupssh'])
+    if options.sync:
+        cmdstorun.append(['run_remote_python', 'syncfileclient'])
+    if options.scripts:
+        for script in options.scripts.split(','):
+            cmdstorun.append(['run_remote', script])
+    if not cmdstorun:
+        argparser.print_help()
+        sys.exit(1)
+    idxbynode = {}
+    cmdvbase = ['bash', '/etc/confluent/functions']
+    for sshnode in nodes:
+        idxbynode[sshnode] = 1
+        cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(sshnode, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((sshnode, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if idxbynode[node] < len(cmdstorun):
+                                cmdv =  ['ssh', sshnode] + cmdvbase + cmdstorun[idxbynode[node]]
+                                idxbynode[node] += 1
+                                run_cmdv(node, cmdv, all, pipedesc)
+                            elif pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                line = line.lstrip('\x08')
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodeattrib

@@ -22,6 +22,7 @@ import optparse
 import os
 import signal
 import sys
+import shlex
 
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
@@ -53,6 +54,8 @@ argparser.add_option('-p', '--prompt', action='store_true',
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Prompt if trying to set attributes on more '
                           'than specified number of nodes')
+argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
+                     default=False, help='set attributes using a batch file')
 (options, args) = argparser.parse_args()
 
 
@@ -109,6 +112,23 @@ elif options.clear or options.environment or options.prompt:
     sys.stderr.write('Attribute names required with specified options\n')
     argparser.print_help()
     exitcode = 400
+
+elif options.set:
+    arglist = [noderange]
+    showtype='current'
+    argfile = open(options.set, 'r')
+    argset = argfile.readline()
+    while argset:
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            arglist += shlex.split(argset)
+        argset = argfile.readline() 
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
 if exitcode != 0:
     sys.exit(exitcode)
 

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -23,6 +23,7 @@ path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
+
 import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
@@ -41,6 +42,10 @@ argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                      help='Enter log replay mode instead of showing a live console')
+
+argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
+                     help= 'Dump log in stdout with timestamps')
+
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -91,6 +96,16 @@ if options.log:
     logreader.replay_to_console(logname)
     sys.exit(0)
 
+if options.Timestamp:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.dump_to_console(logname)
+    sys.exit(0)
+
 def kill(noderange):
     sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
@@ -106,12 +121,20 @@ def kill(noderange):
         nodes.append(node) 
 
     for node in nodes:
-        ps_data=subprocess.Popen(['ps', '-auxww' ], stdout=subprocess.PIPE)
-        wintr=ps_data.communicate()[0]
-        for line in wintr.decode('utf-8').split('\n'):
-            if confettypath in line and envstring in line and node in line:
-                pid_line = [x for x in line.split(' ') if x != '']
-                ps_data=subprocess.Popen(['kill', '-9', pid_line[1] ], stdout=subprocess.PIPE)
+        s=socket.socket(socket.AF_UNIX)
+        winid=None
+        try:
+            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
+            wintr=win.communicate()[0]
+            for line in wintr.decode('utf-8').split('\n'):
+                if 'console: {0}'.format(node) in line or 'confetty' in line:
+                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
+                    winid = win_obj[0]
+        except:
+            print("Error: cannot retrieve window id of node {}".format(node))
+
+        if winid:
+            ps_data=subprocess.Popen(['xkill', '-id', winid ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     sys.exit(0)
 
@@ -220,7 +243,7 @@ if options.windowed:
             elif 'Height' in line:
                 window_height = int(line.split(':')[1])
             elif '-geometry' in line:
-                l = re.split(' |x|\+', line)
+                l = re.split(' |x|-|\+', line)
                 l_nosp = [ele for ele in l if ele.strip()]
                 wmxo = int(l_nosp[1])
                 wmyo = int(l_nosp[2])
@@ -259,7 +282,7 @@ if options.windowed:
         else:
             pass 
         with open(os.devnull, 'wb') as devnull:
-            xopen=subprocess.Popen(envlist + [confettypath, '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
+            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node), '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
     sys.exit(0)
 #end of wcons
 if options.tile:
@@ -273,8 +296,13 @@ if options.tile:
             sys.exit(1)
         nodes.append(node)
     initial = True
+    in_tmux = False
     pane = 0
     sessname = 'nodeconsole_{0}'.format(os.getpid())
+    if os.environ.get("TMUX"):
+        initial = False
+        in_tmux = True
+        subprocess.call(['tmux', 'rename-session', sessname])
     for node in sortutil.natural_sort(nodes):
         panename = '{0}:{1}'.format(sessname, pane)
         if initial:
@@ -295,7 +323,8 @@ if options.tile:
             pane += 1
     subprocess.call(['tmux', 'select-pane', '-t', sessname])
     subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
+    if not in_tmux:
+        os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
-             '/nodes/{0}/console/session'.format(args[0]))
+             '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodedeploy

@@ -90,17 +90,6 @@ def main(args):
         if 'error' in rsp:
             sys.stderr.write(rsp['error'] + '\n')
             sys.exit(1)
-    if not args.clear and args.network and not args.prepareonly:
-        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
-                                        bootmode='uefi',
-                                        persistent=False,
-                                        errnodes=errnodes)
-        if errnodes:
-            sys.stderr.write(
-                'Unable to set boot device for following nodes: {0}\n'.format(
-                    ','.join(errnodes)))
-            return 1
-        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.clear:
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
@@ -120,7 +109,7 @@ def main(args):
                 for profname in profnames:
                     sys.stderr.write('    ' + profname + '\n')
             else:
-                sys.stderr.write('No deployment profiles available, try osdeploy fiimport or imgutil capture\n')
+                sys.stderr.write('No deployment profiles available, try osdeploy import or imgutil capture\n')
             sys.exit(1)
         armonce(args.noderange, c)
         setpending(args.noderange, args.profile, c)
@@ -166,6 +155,17 @@ def main(args):
             else:
                 print('{0}: {1}{2}'.format(node, profile, armed))
         sys.exit(0)
+    if not args.clear and args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.network and not args.prepareonly:
         return rc
     return 0

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -15,13 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__author__ = 'jjohnson2,alin37'
+__author__ = 'jjohnson2,alin37,wgrzeda'
 
 import optparse
 import os
 import signal
 import sys
 
+
+
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 except AttributeError:
@@ -39,9 +41,12 @@ def main():
           "   or: %prog [options] noderange <nodeattribute>...")
     argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
+    argparser.add_option('-d', '--delim', metavar="STRING", default = "\n",
+                     help='Delimiter separating the values')
     (options, args) = argparser.parse_args()
     noderange=""
     nodelist=""
+    list = []
     try:
         noderange = args[0]
         nodelist = '/noderange/{0}/nodes/'.format(noderange)
@@ -61,7 +66,9 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print(res['item']['href'].replace('/', ''))
+                elem=(res['item']['href'].replace('/', ''))
+                list.append(elem)
+    print(options.delim.join(list))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodersync

@@ -48,6 +48,8 @@ def run():
                           'prompting if over the threshold')   
     argparser.add_option('-l', '--loginname', type='str',
                          help='Username to use when connecting, defaults to current user.')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for rsync')
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -63,10 +65,20 @@ def run():
     c = client.Command()
     cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
-    if options.loginname:
-        cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath
+    
+    targname = options.substitutename
+    if targname and '{' in targname:
+        targname =  targname + ':'
+    elif targname:
+        targname = '{node}' + targname + ':'
     else:
-        cmdstr += ' {node}:' + targpath
+        targname = '{node}:'
+
+    if options.loginname:
+        cmdstr += ' {}@'.format(options.loginname) + targname + targpath
+    else:
+        cmdstr += ' {}'.format(targname) + targpath
+    
 
     currprocs = 0
     all = set([])

confluent_client/bin/stats

@@ -16,13 +16,10 @@
 # limitations under the License.
 
 import argparse
+import base64
 import csv
-import fcntl
 import io
 import numpy as np
-
-import os
-import subprocess
 import sys
 
 try:
@@ -35,7 +32,31 @@ except ImportError:
     pass
 
 
-def plot(gui, output, plotdata, bins):
+def iterm_draw(data):
+    databuf = data.getbuffer()
+    datalen = len(databuf)
+    data = base64.b64encode(databuf).decode('utf8')
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;size={}:'.format(datalen))
+    sys.stdout.write(data)
+    sys.stdout.write('\a')
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+
+def kitty_draw(data):
+    data = base64.b64encode(data.getbuffer())
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+    sys.stdout.write('\n')
+
+
+def plot(gui, output, plotdata, bins, fmt):
     import matplotlib as mpl
     if gui and mpl.get_backend() == 'agg':
         sys.stderr.write('Error: No GUI backend available and -g specified!\n')
@@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
-        writer = DumbWriter()
-        writer.draw(tdata)
+        if fmt == 'sixel':
+            writer = DumbWriter()
+            writer.draw(tdata)
+        elif fmt == 'kitty':
+            kitty_draw(tdata)
+        elif fmt == 'iterm':
+            iterm_draw(tdata)
     return n, bins
 
 def textplot(plotdata, bins):
@@ -81,7 +107,8 @@ histogram = False
 aparser = argparse.ArgumentParser(description='Quick access to common statistics')
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
-aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
+aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
@@ -138,7 +165,7 @@ while data:
     data = list(csv.reader([data], delimiter=delimiter))[0]
 n = None
 if args.g or args.o or args.x:
-    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f)
 if args.t:
     n, bins = textplot(plotdata, bins=args.b)
 print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))

confluent_client/confluent/logreader.py

@@ -296,5 +296,47 @@ def replay_to_console(txtfile):
             sys.exit(1)
     _replay_to_console(txtfile, binfile)
 
+def dump_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    elif '.' not in txtfile:
+        if '/' not in txtfile:
+            txtfile = os.getcwd() + '/' + txtfile
+        sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(txtfile + '.cbl'))
+        sys.exit(1)
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(binfile))
+            sys.exit(1)
+    replay = LogReplay(txtfile, binfile)
+    quitit = False
+    writeout('\x1b[2J\x1b[;H')
+    prepend = ''
+    try:
+        while not quitit:
+            newdata, delay = replay.get_output(False)
+            if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
+                writeout(newdata.replace(b'\r\n', '\r\n[ {0} ] '.format(time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))).encode('utf8')))
+                newdata = ''
+                try:
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                except IOError:
+                    pass
+            else:
+                break
+    except Exception:
+        writeout('\x1b[m\x1b[?25h\n')
+        raise
+    writeout('\x1b[m\x1b[?25h\n')
+
 if __name__ == '__main__':
     replay_to_console(sys.argv[1])

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -7,7 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
-`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`
+`nodeattrib <noderange> -s <attributes.batch> ...` 
 
 ## DESCRIPTION
 
@@ -53,12 +54,18 @@ to a blank value will allow masking a group defined attribute with an empty valu
 * `-e`, `--environment`:
   Set specified attributes based on exported environment variable of matching name. 
   Environment variable names may be lower case or all upper case. 
-  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE)
   
 * `-p`, `--prompt`:
   Request interactive prompting to provide values rather than the command line
   or environment variables.
-  
+
+* `-s`, `--set`:
+  Set attributes using a batch file rather than the command line. The attributes in the batch file 
+  can be specified as one line of key=value pairs simmilar to command line or each attribute can
+  be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch
+  file syntax.   
+ 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Prompt if trying to set attributes on more than
   specified number of nodes.
@@ -116,6 +123,25 @@ to a blank value will allow masking a group defined attribute with an empty valu
     `d1: net.pxe.switch: pxeswitch1`  
     `d1: net.switch:`  
 
+* Setting attributes using a batch file with syntax similar to command line:
+    `# cat nodeattributes.batch`  
+    `# power`
+    `power.psu1.outlet=3 power.psu1.pdu=pdu2`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: 3`
+    `n41: pdu2`
+
+* Setting attributes using a batch file with syntax where each attribute is in its own line:
+    `# cat nodeattributes.batch`  
+    `# management`
+    `custom.mgt.switch=switch_main`
+    `custom.mgt.switch.port=swp4`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: switch_main`
+    `n41: swp4`
+
+  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeconsole.ronn

@@ -29,6 +29,11 @@ console process which will result in the console window closing.
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+
+  * `-T`, `--Timestamp`:
+ Dump the log with Timpstamps on the current, local log in /var/log/confluent/consoles.
+  If in collective mode, this only makes sense to use on the current collective
+  manager at this time.
   
 * `-w`, `--windowed`:
   Open terminal windows for each node.  The

confluent_client/doc/man/nodelist.ronn

@@ -4,7 +4,7 @@ nodelist(8) -- List confluent nodes and their attributes
 ## SYNOPSIS
 
 `nodelist <noderange>`  
-`nodelist <noderange> [-b] <nodeattribute>...`  
+`nodelist <noderange> [-b] [-d] {string} <nodeattribute>...`  
 
 ## DESCRIPTION
 
@@ -24,7 +24,8 @@ all attributes that begin with `net.` and end with `switch`.
 
 * `-b`, `--blame`:
   Annotate inherited and expression based attributes to show their base value.
-
+* `-d`, `--delim`:
+  Choose a delimiter to separat the values. Default - ENTER.
 ## EXAMPLES
 * Listing matching nodes of a simple noderange:
   `# nodelist n1-n4`  

confluent_client/doc/man/nodersync.ronn

@@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes.
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 
+
+* `-s`, `--substitutename`:
+  'Use a different name other than the nodename for rsync'
   
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to run rsync to, prompting if over the

confluent_osdeploy/buildrpm-aarch64

@@ -1,3 +1,4 @@
+cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
@@ -29,4 +30,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
 rm -rf el8bin
-rpmbuild -ba confluent_osdeploy-aarch64.spec
+podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec
+

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -21,6 +21,10 @@ class InvalidApiKey(Exception):
 def msg_align(len):
     return (len + 3) & ~3
 
+try:
+    _protoparm = ssl.PROTOCOL_TLS_CLIENT
+except AttributeError:    
+    _protoparm = ssl.PROTOCOL_SSLv23
 cryptname = ctypes.util.find_library('crypt')
 if not cryptname:
     if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
@@ -223,7 +227,7 @@ def get_apikey(nodename, hosts, errout=None):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None):
+    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False):
         self.phmac = phmac
         self.errout = None
         if errout:
@@ -287,7 +291,7 @@ class HTTPSClient(client.HTTPConnection, object):
         if self.phmac:
             with open(phmac, 'r') as hmacin:
                 self.stdheaders['CONFLUENT_CRYPTHMAC'] = hmacin.read()
-        else:
+        elif not checkonly:
             self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
@@ -305,7 +309,7 @@ class HTTPSClient(client.HTTPConnection, object):
     def check_connections(self):
         foundsrv = None
         hosts = self.hosts
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         ctx.verify_mode = ssl.CERT_REQUIRED
         ctx.check_hostname = True
@@ -321,7 +325,7 @@ class HTTPSClient(client.HTTPConnection, object):
                     foundsrv = host
                     psock.close()
                     break
-                except OSError:
+                except (OSError, socket.error):
                     continue
                 except ssl.SSLError:
                     continue
@@ -353,11 +357,12 @@ class HTTPSClient(client.HTTPConnection, object):
                 raise Exception('Unable to reach any hosts')
         return foundsrv
 
-    def connect(self):
+    def connect(self, tries=3):
         addrinf = socket.getaddrinfo(self.host, self.port)[0]
         psock = socket.socket(addrinf[0])
+        psock.settimeout(15)
         psock.connect(addrinf[4])
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         host = self.host.split('%', 1)[0]
         if '[' not in host and ':' in host:
@@ -375,6 +380,10 @@ class HTTPSClient(client.HTTPConnection, object):
                 self.errout.write(errmsg)
                 self.errout.flush()
             sys.exit(1)
+        except socket.timeout:
+            if not tries:
+                raise
+            return self.connect(tries=tries-1)
 
     def grab_url(self, url, data=None, returnrsp=False):
         return self.grab_url_with_status(url, data, returnrsp)[1]
@@ -459,7 +468,7 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    client = HTTPSClient(usejson, errout=errout, phmac=phmac)
+    client = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
     if waitfor:
         status = 201
         while status != waitfor:

confluent_osdeploy/common/profile/scripts/confignet

@@ -8,6 +8,10 @@ import sys
 import time
 import shlex
 import subprocess
+try:
+    import yaml
+except ImportError:
+    pass
 try:
     from importlib.machinery import SourceFileLoader
     def load_source(mod, path):
@@ -107,6 +111,104 @@ def get_interface_name(iname, settings):
         return iname
     return None
 
+class NetplanManager(object):
+    def __init__(self, deploycfg):
+        self.cfgbydev = {}
+        self.read_connections()
+        self.deploycfg = deploycfg
+
+    def read_connections(self):
+        for plan in glob.glob('/etc/netplan/*.y*ml'):
+            with open(plan) as planfile:
+                planinfo = yaml.safe_load(planfile)
+                if not planinfo:
+                    continue
+            nicinfo = planinfo.get('network', {}).get('ethernets', {})
+            for devname in nicinfo:
+                if devname == 'lo':
+                    continue
+                if 'gateway4' in nicinfo[devname]:
+                    # normalize deprecated syntax on read in
+                    gw4 = nicinfo[devname]['gateway4']
+                    del nicinfo[devname]['gateway4']
+                    routeinfo = nicinfo[devname].get('routes', [])
+                    for ri in routeinfo:
+                        if ri.get('via', None) == gw4 and ri.get('to', None) in ('default', '0.0.0.0/0', '0/0'):
+                            break
+                    else:
+                        routeinfo.append({
+                            'to': 'default',
+                            'via': gw4
+                        })
+                    nicinfo[devname]['routes'] = routeinfo
+                self.cfgbydev[devname] = nicinfo[devname]
+    
+    def apply_configuration(self, cfg):
+        devnames = cfg['interfaces']
+        if len(devnames) != 1:
+            raise Exception('Multi-nic team/bonds not yet supported')
+        stgs = cfg['settings']
+        needcfgapply = False
+        for devname in devnames:
+            needcfgwrite = False
+            # ipv6_method missing at uconn...
+            if stgs.get('ipv6_method', None) == 'static':
+                curraddr = stgs['ipv6_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            if stgs.get('ipv4_method', None) == 'static':
+                curraddr = stgs['ipv4_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            gws = []
+            gws.append(stgs.get('ipv4_gateway', None))
+            gws.append(stgs.get('ipv6_gateway', None))
+            for gwaddr in gws:
+                if gwaddr:
+                    cfgroutes = self.getcfgarrpath([devname, 'routes'])
+                    for rinfo in cfgroutes:
+                        if rinfo.get('via', None) == gwaddr:
+                            break
+                    else:
+                        needcfgwrite = True
+                        cfgroutes.append({'via': gwaddr, 'to': 'default'})
+            dnsips = self.deploycfg.get('nameservers', [])
+            dnsdomain = self.deploycfg.get('dnsdomain', '')
+            if dnsips:
+                currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses'])
+                for dnsip in dnsips:
+                    if dnsip and dnsip not in currdnsips:
+                        needcfgwrite = True
+                        currdnsips.append(dnsip)
+            if dnsdomain:
+                currdnsdomain = self.getcfgarrpath([devname, 'nameservers', 'search'])
+                if dnsdomain not in currdnsdomain:
+                    needcfgwrite = True
+                    currdnsdomain.append(dnsdomain)
+            if needcfgwrite:
+                needcfgapply = True
+                newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
+                with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout:
+                    planout.write(yaml.dump(newcfg))
+        if needcfgapply:
+            subprocess.call(['netplan', 'apply'])
+
+    def getcfgarrpath(self, devpath):
+        currptr = self.cfgbydev
+        for k in devpath[:-1]:
+            if k not in currptr:
+                currptr[k] = {}
+            currptr = currptr[k]
+        if devpath[-1] not in currptr:
+            currptr[devpath[-1]] = []
+        return currptr[devpath[-1]]
+        
+
+
 class WickedManager(object):
     def __init__(self):
         self.teamidx = 0
@@ -243,7 +345,7 @@ class NetworkManager(object):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
             subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
-        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
         if bondcfg:
             args = []
             for parm in bondcfg:
@@ -253,6 +355,7 @@ class NetworkManager(object):
 
     def apply_configuration(self, cfg):
         cmdargs = {}
+        cmdargs['connection.autoconnect'] = 'yes'
         stgs = cfg['settings']
         cmdargs['ipv6.method'] = stgs.get('ipv6_method', 'link-local')
         if stgs.get('ipv6_address', None):
@@ -276,7 +379,7 @@ class NetworkManager(object):
             for arg in cmdargs:
                 cargs.append(arg)
                 cargs.append(cmdargs[arg])
-            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team', 'con-name', cname, 'connection.interface-name', cname, 'team.runner', stgs['team_mode']] + cargs)
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
             for iface in cfg['interfaces']:
                 self.add_team_member(cname, iface)
             subprocess.check_call(['nmcli', 'c', 'u', cname])
@@ -315,8 +418,12 @@ if __name__ == '__main__':
     myaddrs = apiclient.get_my_addresses()
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
+    dc = None
     for srv in srvs:
-        s = socket.create_connection((srv, 443))
+        try:
+            s = socket.create_connection((srv, 443))
+        except socket.error:
+            continue
         myname = s.getsockname()
         s.close()
         if len(myname) == 4:
@@ -329,8 +436,27 @@ if __name__ == '__main__':
                     curridx = addr[-1]
         if curridx in doneidxs:
             continue
-        status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+        for tries in (1, 2, 3):
+            try:
+                status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+                break
+            except Exception:
+                if tries == 3:
+                    raise
+                time.sleep(1)
+                continue
         nc = json.loads(nc)
+        if not dc:
+            for tries in (1, 2, 3):
+                try:
+                    status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+                    break
+                except Exception:
+                    if tries == 3:
+                        raise
+                    time.sleep(1)
+                    continue
+            dc = json.loads(dc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:
             for iname in iname.split(','):
@@ -356,6 +482,8 @@ if __name__ == '__main__':
         if not netname_to_interfaces['default']['interfaces']:
             del netname_to_interfaces['default']
     rm_tmp_llas(tmpllas)
+    if os.path.exists('/usr/sbin/netplan'):
+        nm = NetplanManager(dc)
     if os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes)
     elif os.path.exists('/usr/sbin/wicked'):

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -42,7 +42,7 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreo
     mv ../addons.cpio .
     cd ..
 done
-for os in el7 el8 suse15 el9 ubuntu20.04; do
+for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,13 @@ import struct
 import sys
 import subprocess
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +60,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -166,6 +176,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -224,7 +236,8 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    # do this safer, unit s print might bomb
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +271,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk , volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +281,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -125,4 +125,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -7,41 +7,127 @@ if [ -f /tmp/dd_disk ]; then
         fi
     done
 fi
-mkdir -p /etc/confluent
-cat /tls/*.pem > /etc/confluent/ca.pem
-TRIES=0
-echo -n > /etc/confluent/confluent.info
-cd /sys/class/net
-while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
-echo -n "" > /tmp/confluent.initq
-# restart cmdline
-echo -n "" > /etc/cmdline.d/01-confluent.conf
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-#TODO: blkid --label <whatever> to find mounted api
-
+shutdownnic=""
 oum=$(umask)
-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-ifidx=$(cat /tmp/confluent.ifidx)
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-echo $ifname > /tmp/net.ifaces
+umask 0077
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
 
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/hmackeyfile
+    passfile=/etc/confluent/confluent.apikey
+    passcrypt=/tmp/passcrypt
+    hmacfile=/tmp/hmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile;
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo "NODENAME: "$nodename > /etc/confluent/confluent.info
+    for dsrv in $deploysrvs; do
+        echo 'MANAGER: '$dsrv >> /etc/confluent/confluent.info
+    done
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    sleep 10
+    usedhcp=0
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            usedhcp=1
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            TESTSRV=$(python /opt/confluent/bin/apiclient -c 2> /dev/null)
+            if [ ! -z "$TESTSRV" ]; then
+                python /opt/confluent/bin/apiclient -p $hmacfile /confluent-api/self/registerapikey $passcrypt
+                mgr=$TESTSRV
+                ifname=$NICGUESS
+                shutdownnic=$ifname
+                break
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip -4 addr flush dev $NICGUESS
+        fi
+    done
+fi
+TRIES=0
+if [ "$usedhcp" = 1 ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+elif [ -z "$ifname" ]; then
+    cd /sys/class/net
+    while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    cd /
+    grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+    echo -n "" > /etc/cmdline.d/01-confluent.conf
+else
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none > /etc/cmdline.d/01-confluent.conf
+fi
+python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+if [ ! -z "$shutdownnic" ]; then
+    if [ ! -z "$v4gw" ]; then
+        ip route del default via $v4gw
+    fi
+    ip -4 addr flush dev $shutdownnic
+fi
+# restart cmdline
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+echo -n "" > /tmp/confluent.initq
+if [ -z "$ifname"  ]; then
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+if [ ! -z "$ifname" ]; then
+    echo $ifname > /tmp/net.ifaces
+fi
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 hostname=$nodename
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
     hostname=$hostname.$dnsdomain
 fi
-mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
-mgr=${mgr#deploy_server: }
+if [ -z "$mgr" ]; then
+    mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+    mgr=${mgr#deploy_server: }
+fi
 profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 profilename=${profilename#profile: }
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
@@ -65,21 +151,23 @@ kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
 root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
 export kickstart
 export root
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
-autoconfigmethod=${autoconfigmethod#ipv4_method: }
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
+if [ -z "$autoconfigmethod" ]; then
+    autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+    else
+        v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
     fi
-    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#ipv4_netmask: }
-    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
 nameserversec=0
 while read -r entry; do

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -189,8 +189,15 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+	linktype="infiniband"
+else
+	linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
+
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -199,9 +206,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,14 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +61,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +136,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +152,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -166,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if 'nvme' in targblock and targblock[-1] == 'p':
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -184,6 +228,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +269,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
@@ -366,6 +411,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +426,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
 . /lib/dracut-lib.sh
+setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' &
 udevadm trigger
 udevadm trigger --type=devices --action=add
 udevadm settle
@@ -40,7 +41,7 @@ mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
 TRIES=5
-while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 2
     TRIES=$((TRIES - 1))
 done
@@ -55,21 +56,49 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
     autoconfigmethod=$(grep ^ipv4_method: $tcfg)
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    if [ "$autoconfigmethod" = "dhcp" ]; then
-        /usr/libexec/nm-initrd-generator ip=:dhcp
-    else
-        v4addr=$(grep ^ipv4_address: $tcfg)
-        v4addr=${v4addr#ipv4_address: }
-        v4addr=${v4addr%/*}
-        v4gw=$(grep ^ipv4_gateway: $tcfg)
-        v4gw=${v4gw#ipv4_gateway: }
-        if [ "$v4gw" = "null" ]; then
-            v4gw=""
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            DETECTED=0
+            for dsrv in $deploysrvs; do
+                if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then
+                    rm /run/NetworkManager/system-connections/*
+                    /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none
+                    DETECTED=1
+                    ifname=$NICGUESS
+                    break
+                fi
+            done
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip addr flush dev $NICGUESS
+            if [ $DETECTED = 1 ]; then
+                break
+            fi
         fi
-        v4nm=$(grep ipv4_netmask: $tcfg)
-        v4nm=${v4nm#ipv4_netmask: }
-        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
-    fi
+    done
     NetworkManager --configure-and-quit=initrd --no-daemon
     hmackeyfile=/tmp/cnflnthmackeytmp
     echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
@@ -85,10 +114,10 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
     done
     for deployer in $deploysrvs; do
-        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+        if curl --capath /tls/ -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
             cp $passfile /etc/confluent/confluent.apikey
             confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+            curl --capath /tls/ -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
             confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
             confluent_profile=${confluent_profile#profile: }
             mgr=$deployer
@@ -114,7 +143,7 @@ if ! grep MANAGER: /etc/confluent/confluent.info; then
         for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
             myids=$myids"/mac="$mac
         done
-        myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+        myname=$(curl --capath /tls/ -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
         if [ ! -z "$myname" ]; then
             echo NODENAME: $myname > /etc/confluent/confluent.info
             echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
@@ -149,14 +178,16 @@ while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self
 	sleep 10
 done
 ifidx=$(cat /tmp/confluent.ifidx 2> /dev/null)
-if [ ! -z "$ifidx" ]; then
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-    ifname=${ifname%@*}
-    echo $ifname > /tmp/net.ifaces
-else
-    ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
-    ifname=$(cat /tmp/net.ifaces)
+if [ -z "$ifname" ]; then
+    if [ ! -z "$ifidx" ]; then
+        ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+        ifname=${ifname%:}
+        ifname=${ifname%@*}
+        echo $ifname > /tmp/net.ifaces
+    else
+        ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
+        ifname=$(cat /tmp/net.ifaces)
+    fi
 fi
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)

confluent_osdeploy/el8/profiles/default/kickstart

@@ -33,15 +33,7 @@
 reboot
 
 %packages
-@^minimal-environment
-#-kernel-uek # This can opt out of the UEK for the relevant distribution
-bind-utils
-chrony
-pciutils
-python3
-rsync
-tar
--iwl*-firmware
+%include /tmp/pkglist
 %include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end

confluent_osdeploy/el8/profiles/default/packagelist

@@ -0,0 +1,9 @@
+@^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
+bind-utils
+chrony
+pciutils
+python3
+rsync
+tar
+-iwl*-firmware

confluent_osdeploy/el8/profiles/default/partitioning

@@ -0,0 +1,4 @@
+clearpart --all --initlabel
+ignoredisk --only-use %%INSTALLDISK%%
+autopart --nohome %%LUKSHOOK%%
+

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -87,6 +87,7 @@ done
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 touch /tmp/cryptpkglist
+touch /tmp/pkglist
 touch /tmp/addonpackages
 if [ "$cryptboot" == "tpm2" ]; then
 	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
@@ -102,15 +103,18 @@ confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_pro
 run_remote pre.custom
 run_remote_parts pre.d
 confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart -o /tmp/kickstart.base
+if grep '^%include /tmp/pkglist' /tmp/kickstart.* > /dev/null; then
+    confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/packagelist -o /tmp/pkglist
+fi
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || touch /tmp/installdisk
 if [ ! -e /tmp/installdisk ]; then
     run_remote_python getinstalldisk
 fi
+confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/partitioning -o /tmp/partitioning.template
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk
 if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
-    echo clearpart --all --initlabel >> /tmp/partitioning
-    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
-    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+    INSTALLDISK=$(cat /tmp/installdisk)
+    sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning
     dd if=/dev/zero of=/dev/$(cat /tmp/installdisk) bs=1M count=1 >& /dev/null
     vgchange -a n >& /dev/null
 fi

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -154,8 +154,14 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+        linktype="infiniband"
+else
+        linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -164,9 +170,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,14 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +61,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +136,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +152,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -166,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -184,6 +228,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +269,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
@@ -366,6 +411,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +426,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -94,6 +94,7 @@ while ! grep NODENAME /etc/confluent/confluent.info; do
     fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+cat /tls/*.pem > /etc/confluent/ca.pem
 mgr=$(python /opt/confluent/bin/apiclient -c)
 APIKEY=$(cat /etc/confluent/confluent.apikey)
 
@@ -108,7 +109,6 @@ while [ -z "$APIKEY" ]; do
     (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
     APIKEY=$(cat /etc/confluent/confluent.apikey)
 done
-cat /tls/*.pem > /etc/confluent/ca.pem
 /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -174,6 +174,8 @@ dnsdomain=${dnsdomain#dnsdomain: }
 echo search $dnsdomain >> /etc/resolv.conf
 echo -n "Initializing ssh..."
 ssh-keygen -A
+mkdir -p /usr/share/empty.sshd
+rm /etc/ssh/ssh_host_dsa_key*
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -161,7 +161,7 @@ else
     echo -e "BOOTPROTO='static'\nSTARTMODE='auto'" >> /run/confluent/ifcfg-$ifname
     echo "IPADDR='$v4addr/$v4nm'" >> /run/confluent/ifcfg-$ifname
     if [ ! -z "$v4gw" ]; then
-        echo defafult $v4gw - $ifname > /run/confluent/ifroute-$ifname
+        echo default $v4gw - $ifname > /run/confluent/ifroute-$ifname
     fi
 fi
 

confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh

@@ -138,4 +138,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -41,15 +41,6 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
     autoconfigmethod=$(grep ^ipv4_method: $tcfg)
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
-        for dsrv in $deploysrvs; do
-            if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
-                deploysrvs=$dsrv
-                NIC=$NICGUESS
-            fi
-        done
-    done
-    deploysrvs=$dsrv
     if [ "$autoconfigmethod" = "static" ]; then
         setdebopt netcfg/disable_dhcp true boolean
         v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
@@ -212,5 +203,3 @@ if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         setdebopt netcfg/confirm_static false boolean
     fi
 fi
-
-

confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg

@@ -7,8 +7,6 @@ d-i partman/confirm_write_new_label boolean true
 d-i partman/choose_partition select finish
 d-i partman/confirm boolean true
 d-i partman/confirm_nooverwrite boolean true
-d-i preseed/late_command string sleep 86400
-d-i partman/early_command string sleep 86400
 d-i passwd/make-user boolean false
 d-i clock-setup/utc boolean true
 d-i apt-setup/multiverse boolean false

confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent

@@ -1,4 +1,5 @@
 if ! grep console= /proc/cmdline > /dev/null; then
+    mkdir -p /custom-installation
     /opt/confluent/bin/autocons > /custom-installation/autocons.info
     cons=$(cat /custom-installation/autocons.info)
     if [ ! -z "$cons" ]; then

confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent

@@ -107,7 +107,7 @@ if [ "$v6meth" = static ]; then
         ip route add default via $v6gw
     fi
 fi
-v4meth=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 if [ "$v4meth" = static ]; then
     v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg | awk '{print $2}')
     v4prefix=$(grep ^prefix: /etc/confluent/confluent.deploycfg | awk '{print $2}')

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+(
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+    sleep 1
+done
+
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    touch /etc/confluent/firstboot.ran
+
+    run_remote firstboot.custom
+    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+    run_remote_parts firstboot.d
+
+    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+    run_remote_config firstboot.d
+fi
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service
+rm /etc/confluent/firstboot.ran
+) &
+tail --pid  $! -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,95 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1,428 @@
+#!/usr/bin/python3
+import glob
+import json
+import os
+import re
+import time
+import shutil
+import socket
+import stat
+import struct
+import sys
+import subprocess
+
+bootuuid = None
+
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+def get_next_part_meta(img, imgsize):
+    if img.tell() == imgsize:
+        return None
+    pathlen = struct.unpack('!H', img.read(2))[0]
+    mountpoint = img.read(pathlen).decode('utf8')
+    jsonlen = struct.unpack('!I', img.read(4))[0]
+    metadata = json.loads(img.read(jsonlen).decode('utf8'))
+    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip filesystem type
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip orig devname (redundant with json)
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip padding
+    nextlen = struct.unpack('!Q', img.read(8))[0]
+    img.seek(nextlen, 1)  # go to next section
+    return metadata
+
+def get_multipart_image_meta(img):
+    img.seek(0, 2)
+    imgsize = img.tell()
+    img.seek(16)
+    seekamt = img.read(1)
+    img.seek(struct.unpack('B', seekamt)[0], 1)
+    partinfo = get_next_part_meta(img, imgsize)
+    while partinfo:
+        yield partinfo
+        partinfo = get_next_part_meta(img, imgsize)
+
+def get_image_metadata(imgpath):
+    with open(imgpath, 'rb') as img:
+        header = img.read(16)
+        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
+            for md in get_multipart_image_meta(img):
+                yield md
+        else:
+            raise Exception('Installation from single part image not supported')
+
+class PartedRunner():
+    def __init__(self, disk):
+        self.disk = disk
+
+    def run(self, command, check=True):
+        command = command.split()
+        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
+
+def fixup(rootdir, vols):
+    devbymount = {}
+    for vol in vols:
+        devbymount[vol['mount']] = vol['targetdisk']
+    fstabfile = os.path.join(rootdir, 'etc/fstab')
+    with open(fstabfile) as tfile:
+        fstab = tfile.read().split('\n')
+    while not fstab[0]:
+        fstab = fstab[1:]
+    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
+        os.unlink(os.path.join(rootdir, '.autorelabel'))
+    with open(fstabfile, 'w') as tfile:
+        for tab in fstab:
+            entry = tab.split()
+            if tab.startswith('#ORIGFSTAB#'):
+                if entry[1] in devbymount:
+                    targetdev = devbymount[entry[1]]
+                    if targetdev.startswith('/dev/localstorage/'):
+                        entry[0] = targetdev
+                    else:
+                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
+                        uuid = uuid.strip()
+                        entry[0] = 'UUID={}'.format(uuid)
+                elif entry[2] == 'swap':
+                    entry[0] = '/dev/mapper/localstorage-swap'
+                entry[0] = entry[0].ljust(42)
+                entry[1] = entry[1].ljust(16)
+                entry[3] = entry[3].ljust(28)
+                tab = '\t'.join(entry)
+            tfile.write(tab + '\n')
+    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
+        nameout.write(socket.gethostname() + '\n')
+    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
+    policy = None
+    if os.path.exists(selinuxconfig):
+        with open(selinuxconfig) as cfgin:
+            sec = cfgin.read().split('\n')
+            for l in sec:
+                l = l.split('#', 1)[0]
+                if l.startswith('SELINUXTYPE='):
+                    _, policy = l.split('=')
+    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
+        os.unlink(sshkey)
+    for sshkey in glob.glob('/etc/ssh/*_key*'):
+        newkey = os.path.join(rootdir, sshkey[1:])
+        shutil.copy2(sshkey, newkey)
+        finfo = os.stat(sshkey)
+        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
+
+    # Will use confignet to handle networking for ubuntu
+    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
+    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    if policy:
+        sys.stdout.write('Applying SELinux labeling...')
+        sys.stdout.flush()
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
+        sys.stdout.write('Done\n')
+        sys.stdout.flush()
+    for metafs in ('proc', 'sys', 'dev'):
+        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
+        defgrub = defgrubin.read().split('\n')
+    with open(grubsyscfg, 'w') as defgrubout:
+        for gline in defgrub:
+            gline = gline.split()
+            newline = []
+            for ent in gline:
+                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    continue
+                newline.append(ent)
+            defgrubout.write(' '.join(newline) + '\n')
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                elif 'search.fs_uuid ' in tgrubcfg[0] and 'root' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search.fs_uuid ' + bootuuid + ' root'
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        # must fixup root@d2:/boot/efi/EFI# cat ubuntu/grub.cfg ... uuid
+        subprocess.check_call(['chroot', rootdir, 'grub-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
+    newroot = None
+    with open('/etc/shadow') as shadowin:
+        shents = shadowin.read().split('\n')
+        for shent in shents:
+            shent = shent.split(':')
+            if not shent:
+                continue
+            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
+                newroot = shent[1]
+    if newroot:
+        shlines = None
+        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
+            shlines = oshadow.read().split('\n')
+        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
+            for line in shlines:
+                if line.startswith('root:'):
+                    line = line.split(':')
+                    line[1] = newroot
+                    line = ':'.join(line)
+                oshadow.write(line + '\n')
+    partnum = None
+    targblock = None
+    for vol in vols:
+        if vol['mount'] == '/boot/efi':
+            targdev = vol['targetdisk']
+            partnum = re.search('(\d+)$', targdev).group(1)
+            targblock = re.search('(.*)\d+$', targdev).group(1)
+    if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
+        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
+        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
+        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    #other network interfaces
+
+
+def had_swap():
+    with open('/etc/fstab') as tabfile:
+        tabs = tabfile.read().split('\n')
+        for tab in tabs:
+            tab = tab.split()
+            if len(tab) < 3:
+                continue
+            if tab[2] == 'swap':
+                return True
+    return False
+
+def install_to_disk(imgpath):
+    global bootuuid
+    lvmvols = {}
+    deftotsize = 0
+    mintotsize = 0
+    deflvmsize = 0
+    minlvmsize = 0
+    biggestsize = 0
+    biggestfs = None
+    plainvols = {}
+    allvols = []
+    swapsize = 0
+    if had_swap():
+        with open('/proc/meminfo') as meminfo:
+            swapsize = meminfo.read().split('\n')[0]
+        swapsize = int(swapsize.split()[1])
+        if swapsize < 2097152:
+            swapsize = swapsize * 2
+        elif swapsize > 8388608 and swapsize < 67108864:
+            swapsize = swapsize * 0.5
+        elif swapsize >= 67108864:
+            swapsize = 33554432
+        swapsize = int(swapsize * 1024)
+    deftotsize = swapsize
+    mintotsize = swapsize
+    for fs in get_image_metadata(imgpath):
+        allvols.append(fs)
+        deftotsize += fs['initsize']
+        mintotsize += fs['minsize']
+        if fs['initsize'] > biggestsize:
+            biggestfs = fs
+            biggestsize = fs['initsize']
+        if fs['device'].startswith('/dev/mapper'):
+            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
+            deflvmsize += fs['initsize']
+            minlvmsize += fs['minsize']
+        else:
+            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+    with open('/tmp/installdisk') as diskin:
+        instdisk = diskin.read()
+    instdisk = '/dev/' + instdisk
+    parted = PartedRunner(instdisk)
+    dinfo = parted.run('unit s print', check=False)
+    dinfo = dinfo.split('\n')
+    sectors = 0
+    sectorsize = 0
+    for inf in dinfo:
+        if inf.startswith('Disk {0}:'.format(instdisk)):
+            _, sectors = inf.split(': ')
+            sectors = int(sectors.replace('s', ''))
+        if inf.startswith('Sector size (logical/physical):'):
+            _, sectorsize = inf.split(':')
+            sectorsize = sectorsize.split('/')[0]
+            sectorsize = sectorsize.replace('B', '')
+            sectorsize = int(sectorsize)
+    # for now, only support resizing/growing the largest partition
+    minexcsize = deftotsize - biggestfs['initsize']
+    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
+    minsectors = mintotsize // sectorsize
+    if sectors < (minsectors + 65536):
+        raise Exception('Disk too small to fit image')
+    biggestsectors = sectors - (minexcsize // sectorsize)
+    biggestsize = sectorsize * biggestsectors
+    parted.run('mklabel gpt')
+    curroffset = 2048
+    for volidx in sorted(plainvols):
+        vol = plainvols[volidx]
+        if vol is not biggestfs:
+            size = vol['initsize'] // sectorsize
+        else:
+            size = biggestsize // sectorsize
+        size += 2047 - (size % 2048)
+        end = curroffset + size
+        if end > sectors:
+            end = sectors
+        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
+        vol['targetdisk'] = get_partname(instdisk, volidx)
+        curroffset += size + 1
+    if not lvmvols:
+        if swapsize:
+            swapsize = swapsize // sectorsize
+            swapsize += 2047 - (size % 2048)
+            end = curroffset + swapsize
+            if end > sectors:
+                end = sectors
+            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
+    else:
+        parted.run('mkpart lvm {}s 100%'.format(curroffset))
+        lvmpart = get_partname(instdisk, volidx + 1)
+        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
+        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
+        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
+        vginfo = vginfo.split('\n')
+        pesize = 0
+        pes = 0
+        for infline in vginfo:
+            infline = infline.split()
+            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
+                pesize = int(infline[2])
+            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
+                pes = int(infline[4])
+        takeaway = swapsize // pesize
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                continue
+            takeaway += vol['initsize'] // pesize
+            takeaway += 1
+        biggestextents = pes - takeaway
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                extents = biggestextents
+            else:
+                extents = vol['initsize'] // pesize
+                extents += 1
+            if vol['mount'] == '/':
+                lvname = 'root'
+            else:
+                lvname = vol['mount'].replace('/', '_')
+            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
+            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
+        if swapsize:
+            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
+            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
+        os.makedirs('/run/imginst/targ')
+        for vol in allvols:
+            with open(vol['targetdisk'], 'wb') as partition:
+                partition.write(b'\x00' * 1 * 1024 * 1024)
+            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
+            source = vol['mount'].replace('/', '_')
+            source = '/run/imginst/sources/' + source
+            blankfsstat = os.statvfs('/run/imginst/targ')
+            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
+            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
+            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
+                stillrunning = copier.poll()
+                lastprogress = 0.0
+                while stillrunning is None:
+                    currfsstat = os.statvfs('/run/imginst/targ')
+                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
+                    currused -= blankused
+                    with open('/proc/meminfo') as meminf:
+                        for line in meminf.read().split('\n'):
+                            if line.startswith('Dirty:'):
+                                _, dirty, _ = line.split()
+                                dirty = int(dirty) * 1024
+                    progress = (currused - dirty) / vol['minsize']
+                    if progress < lastprogress:
+                        progress = lastprogress
+                    if progress > 0.99:
+                        progress = 0.99
+                    lastprogress = progress
+                    progress = progress * 100
+                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                    sys.stdout.flush()
+                    time.sleep(0.5)
+                    stillrunning = copier.poll()
+                if stillrunning != 0:
+                    raise Exception("Error copying volume")
+                with subprocess.Popen(['sync']) as syncrun:
+                    stillrunning = syncrun.poll()
+                    while stillrunning is None:
+                        with open('/proc/meminfo') as meminf:
+                            for line in meminf.read().split('\n'):
+                                if line.startswith('Dirty:'):
+                                    _, dirty, _ = line.split()
+                                    dirty = int(dirty) * 1024
+                        progress = (vol['minsize'] - dirty) / vol['minsize']
+                        if progress < lastprogress:
+                            progress = lastprogress
+                        if progress > 0.99:
+                            progress = 0.99
+                        lastprogress = progress
+                        progress = progress * 100
+                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                        sys.stdout.flush()
+                        time.sleep(0.5)
+                        stillrunning = syncrun.poll()
+                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
+            subprocess.check_call(['umount', '/run/imginst/targ'])
+        for vol in allvols:
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
+        fixup('/run/imginst/targ', allvols)
+
+
+if __name__ == '__main__':
+    install_to_disk(os.environ['mountsrc'])
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh

@@ -127,11 +127,13 @@ chmod +x /sysroot/opt/confluent/bin/onboot.sh
 cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
 ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
 cp /etc/confluent/functions /sysroot/etc/confluent/functions
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 if grep installtodisk /proc/cmdline > /dev/null; then
     . /etc/confluent/functions
     run_remote installimage
     exec reboot -f
 fi
-mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
-ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 exec /opt/confluent/bin/start_root

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/installimage

@@ -0,0 +1,48 @@
+#!/bin/bash
+. /etc/confluent/functions
+# the image will be used to deploy itself
+# provide both access to image (for parsing metadata)
+# and existing mounts of image (to take advantage of caching)
+mount -o bind /sys /sysroot/sys
+mount -o bind /dev /sysroot/dev
+mount -o bind /proc /sysroot/proc
+mount -o bind /run /sysroot/run
+
+
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /sysroot/run/imginst/sources/_
+    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
+        srcname=${srcmount#/dev/mapper/mproot}
+        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
+        mkdir -p /sysroot/run/imginst/sources/$srcname
+        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
+    done
+fi
+cd /sysroot/run
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+if [ ! -f /sysroot/tmp/installdisk ]; then
+    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
+    while [ ! -f /sysroot/tmp/installdisk ]; do
+        sleep 1
+    done
+fi
+lvm vgchange -a n
+udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
+chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
+chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
+echo "Port 22" >> /etc/ssh/sshd_config
+echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
+echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+cat /tls/*.pem > /sysroot/run/imginst/targ/usr/local/share/ca-certificates/confluent.crt
+chroot /sysroot/run/imginst/targ update-ca-certificates
+
+chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
+chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# This script is executed 'chrooted' into a cloned disk target before rebooting
+#
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+tail -f /var/log/confluent/confluent-post.log > /dev/console &
+logshowpid=$!
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+mkdir -p /opt/confluent/bin
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+systemctl enable firstboot
+run_remote_python syncfileclient
+run_remote_python confignet
+run_remote post.custom
+# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+
+kill $logshowpid
+
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,286 @@
+#!/usr/bin/python3
+import subprocess
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/hooks/casper-bottom.sh

@@ -9,33 +9,49 @@ MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|'
 MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}')
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
-umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+umask 077
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile| sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -74,8 +74,11 @@ if [ -e /sys/firmware/efi ]; then
     fi
 fi
 cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 

confluent_osdeploy/ubuntu22.04-diskless

@@ -0,0 +1 @@
+ubuntu20.04-diskless

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -6,7 +6,7 @@ mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
 TRIES=5
-while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 2
     TRIES=$((TRIES - 1))
 done
@@ -24,12 +24,33 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
     . /scripts/functions
     if [ "$autoconfigmethod" = "static" ]; then
-        TRYNIC=$(ip link|grep -v LOOPBACK|grep LOWER_UP|head -n 1|awk '{print $2}'|sed -e s/:$//)
         MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
         MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
         MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
-        ipconfig -d $MYIP::$MYGW:$MYNM::$TRYNIC
-        echo $TRYNIC > /tmp/autodetectnic
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
     else
         configure_networking
     fi

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -2,7 +2,10 @@
 echo "Confluent first boot is running"
 HOME=$(getent passwd $(whoami)|cut -d: -f 6)
 export HOME
-seems a potentially relevant thing to put i... by Jarrod Johnson
+(
+exec >> /target/var/log/confluent/confluent-firstboot.log
+exec 2>> /target/var/log/confluent/confluent-firstboot.log
+chmod 600 /target/var/log/confluent/confluent-firstboot.log
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
@@ -18,7 +21,10 @@ done
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions
-
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export confluent_mgr confluent_profile
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk

@@ -3,6 +3,8 @@ import os
 
 class DiskInfo(object):
     def __init__(self, devname):
+        if devname.startswith('nvme') and 'c' in devname:
+            raise Exception("Skipping multipath devname")
         self.name = devname
         self.wwn = None
         self.path = None

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -8,7 +8,6 @@ chmod go-rwx /etc/confluent/*
 for i in /custom-installation/ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
 done
-
 cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
 cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
 sshconf=/target/etc/ssh/ssh_config
@@ -19,10 +18,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-
+cp /etc/confluent/functions /target/etc/confluent/functions
+source /etc/confluent/functions
+mkdir -p /target/var/log/confluent
+cp /var/log/confluent/* /target/var/log/confluent/
+(
+exec >> /target/var/log/confluent/confluent-post.log
+exec 2>> /target/var/log/confluent/confluent-post.log
+chmod 600 /target/var/log/confluent/confluent-post.log
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions
-source /target/etc/confluent/functions
 chmod +x /target/etc/confluent/firstboot.sh
 cp /tmp/allnodes /target/root/.shosts
 cp /tmp/allnodes /target/etc/ssh/shosts.equiv
@@ -74,12 +78,17 @@ if [ -e /sys/firmware/efi ]; then
     fi
 fi
 cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 
 run_remote_config post
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
 
 umount /target/sys /target/dev /target/proc
-
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 deploycfg=/custom-installation/confluent/confluent.deploycfg
+mkdir -p /var/log/confluent
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp /custom-installation/confluent/confluent.info /custom-installation/confluent/confluent.apikey /etc/confluent/
+cat /custom-installation/tls/*.pem >> /etc/confluent/ca.pem
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin
+cp $deploycfg /etc/confluent/
+(
+exec >> /var/log/confluent/confluent-pre.log
+exec 2>> /var/log/confluent/confluent-pre.log
+chmod 600 /var/log/confluent/confluent-pre.log
 
 cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
 if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
@@ -23,7 +34,17 @@ echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
 systemctl restart sshd
+mkdir -p /etc/confluent
+export nodename confluent_profile confluent_mgr
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+run_remote_parts pre.d
 curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
-python3 /custom-installation/getinstalldisk
+if [ ! -e /tmp/installdisk ]; then
+    curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+    python3 /custom-installation/getinstalldisk
+fi
 sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml
+) &
+tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console
+

confluent_server/bin/confluent_selfcheck

@@ -15,6 +15,7 @@ import confluent.sshutil as sshutil
 import confluent.certutil as certutil
 import confluent.client as client
 import confluent.config.configmanager as configmanager
+import confluent.netutil as netutil
 import eventlet.green.subprocess as subprocess
 import tempfile
 import shutil
@@ -237,14 +238,14 @@ if __name__ == '__main__':
         allok = True
         uuidok = False
         macok = False
-        valid_nodes = [node['item']['href'][:-1] for node in c.read('/nodes/')] #get all valiv nodes
+        valid_nodes = [node['item']['href'][:-1] for node in sess.read('/nodes/')] #get all valid nodes
         for rsp in sess.read(f'/nodes/{args.node}/attributes/all'):
             if rsp.get('errorcode', None) == 404:
                 emprint(f'There is no node named "{args.node}"')
                 allok = False
                 uuidok = True  # not really, but suppress the spurious error
             dnsdomain = rsp.get('dns.domain', {}).get('value', '')
-            if ',' in dnsdomain or ' ' in dnsdomain:
+            if dnsdomain and (',' in dnsdomain or ' ' in dnsdomain):
                 allok = False
                 emprint(f'{args.node} has a dns.domain that appears to be a search instead of singular domain')
                 uuidok = True  # not really, but suppress the spurious error
@@ -262,15 +263,35 @@ if __name__ == '__main__':
                         macok = True
             #adding new code to check if the response is something like net.<something>switch
             for key in rsp:
-                if key.startswith('net.')and key.endswith('switch'):
+                if ((key.startswith('net.') and key.endswith('switch')) 
+                        or (key.startswith('power.') and key.endswith('pdu'))
+                        or (key.startswith('enclosure.') and key.endswith('manager'))
+                        ):
                     switch_value = rsp[key].get('value',None)
-                    if switch_value in valid_nodes:
-                        print(f'{switch_value} is valid.')
-                    else:
-                        emprint(f'{switch_value} is not valid.')
+                    if switch_value and switch_value not in valid_nodes:
+                        emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).')
+        print(f"Checking network configuration for {args.node}")
+        cfg = configmanager.ConfigManager(None)
+        bootablev4nics = []
+        bootablev6nics = []
+        for nic in glob.glob("/sys/class/net/*/ifindex"):
+            idx = int(open(nic, "r").read())
+            nicname = nic.split('/')[-2]
+            ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx)
+            if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp':
+                bootablev4nics.append(nicname)
+            if ncfg['ipv6_address']:
+                bootablev6nics.append(nicname)
+        if bootablev4nics:
+            print("{} appears to have network configuration suitable for IPv4 deployment via: {}".format(args.node, ",".join(bootablev4nics)))
+        elif bootablev6nics:
+            print('{} appears to have networking configuration suitable for IPv6 deployment via: {}'.format(args.node, ",".join(bootablev6nics)))
+        else:
+            emprint(f"{args.node} may not have any viable IP network configuration (check name resolution (DNS or hosts file) "
+                    "and/or net.*ipv4_address, and verify that the deployment serer addresses and subnet mask/prefix length are accurate)")
         if not uuidok and not macok:
             allok = False
-            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work')
+            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work (Example resolution: nodeinventory {args.node} -s)')
         if allok:
             print(f'No issues detected with attributes of {args.node}')
         fprint("Checking name resolution: ")

confluent_server/bin/osdeploy

@@ -72,6 +72,12 @@ def main(args):
         return rebase(cmdset.profile)
     ap.print_help()
 
+def symlinkp(src, trg):
+    try:
+        os.symlink(src, trg)
+    except Exception as e:
+        if e.errno != 17:
+            raise
 
 def initialize_genesis():
     if not os.path.exists('/opt/confluent/genesis/x86_64/boot/kernel'):
@@ -89,30 +95,33 @@ def initialize_genesis():
         return retval[1]
     retcode = 0
     try:
+        util.mkdirp('/var/lib/confluent', 0o755)
         if hasconfluentuser:
+            os.chown('/var/lib/confluent', hasconfluentuser.pw_uid, -1)
             os.setgid(hasconfluentuser.pw_gid)
             os.setuid(hasconfluentuser.pw_uid)
         os.umask(0o22)
-        os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755)
-        os.makedirs('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755)
-        os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI',
+        util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot', 0o755)
+        util.mkdirp('/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs', 0o755)
+        symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/BOOTX64.EFI',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/BOOTX64.EFI')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/efi/boot/grubx64.efi',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/efi/boot/grubx64.efi')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/initramfs/distribution',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/initramfs/distribution',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/distribution')
-        os.symlink('/var/lib/confluent/public/site/initramfs.cpio',
+        symlinkp('/var/lib/confluent/public/site/initramfs.cpio',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/site.cpio')
-        os.symlink('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio',
+        symlinkp('/opt/confluent/lib/osdeploy/genesis/initramfs/addons.cpio',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/addons.cpio')
-        os.symlink('/opt/confluent/genesis/x86_64/boot/kernel',
+        symlinkp('/opt/confluent/genesis/x86_64/boot/kernel',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/kernel')
-        shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/',
-                        '/var/lib/confluent/public/os/genesis-x86_64/ansible/')
-        shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/',
-                        '/var/lib/confluent/public/os/genesis-x86_64/scripts/')
-        shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml',
-                       '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml')
+        if not os.path.exists('/var/lib/confluent/public/os/genesis-x86_64/ansible/'):
+            shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/',
+                            '/var/lib/confluent/public/os/genesis-x86_64/ansible/')
+            shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/',
+                            '/var/lib/confluent/public/os/genesis-x86_64/scripts/')
+            shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml',
+                            '/var/lib/confluent/public/os/genesis-x86_64/profile.yaml')
     except Exception as e:
         sys.stderr.write(str(e) + '\n')
         retcode = 1
@@ -373,9 +382,14 @@ def initialize(cmdset):
         for rsp in c.read('/uuid'):
             uuid = rsp.get('uuid', {}).get('value', None)
             if uuid:
-                with open('confluent_uuid', 'w') as uuidout:
-                    uuidout.write(uuid)
-                    uuidout.write('\n')
+                oum = os.umask(0o11)
+                try:
+                    with open('confluent_uuid', 'w') as uuidout:
+                        uuidout.write(uuid)
+                        uuidout.write('\n')
+                    os.chmod('confluent_uuid', 0o644)
+                finally:
+                    os.umask(oum)
     totar.append('confluent_uuid')
     topack.append('confluent_uuid')
     if os.path.exists('ssh'):
@@ -403,7 +417,17 @@ def initialize(cmdset):
         if res:
             sys.stderr.write('Error occurred while packing site initramfs')
             sys.exit(1)
-    os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio')
+    oum = os.umask(0o22)
+    try:
+        os.rename(tmpname, '/var/lib/confluent/public/site/initramfs.cpio')
+        os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644)
+    finally:
+        os.umask(oum)
+    oum = os.umask(0o22)
+    try:
+        os.chmod('/var/lib/confluent/public/site/initramfs.cpio', 0o644)
+    finally:
+        os.umask(oum)
     if cmdset.g:
         updateboot('genesis-x86_64')
     if totar:
@@ -411,6 +435,11 @@ def initialize(cmdset):
         tarcmd = ['tar', '-czf', tmptarname] + totar
         subprocess.check_call(tarcmd)
         os.rename(tmptarname, '/var/lib/confluent/public/site/initramfs.tgz')
+        oum = os.umask(0o22)
+        try:
+            os.chmod('/var/lib/confluent/public/site/initramfs.tgz', 0o644)
+        finally:
+            os.umask(0o22)
     os.chdir(opath)
     print('Site initramfs content packed successfully')
 
@@ -421,6 +450,9 @@ def initialize(cmdset):
 
 
 def updateboot(profilename):
+    if not os.path.exists('/var/lib/confluent/public/site/initramfs.cpio'):
+        emprint('Must generate site content first (TLS (-t) and/or SSH (-s))')
+        return 1
     c = client.Command()
     for rsp in c.update('/deployment/profiles/{0}'.format(profilename),
             {'updateboot': 1}):

confluent_server/builddeb

@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -49,6 +49,13 @@ if ! grep wheezy /etc/os-release; then
 fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
+cat > debian/postinst << EOF
+if ! getent passwd confluent > /dev/null; then 
+    useradd -r confluent -d /var/lib/confluent -s /usr/sbin/nologin
+    mkdir -p /etc/confluent
+    chown confluent /etc/confluent
+fi
+EOF
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
 #echo 'Provides: python-'$DPKGNAME >> debian/control
 #echo 'Conflicts: python-'$DPKGNAME >> debian/control

confluent_server/confluent/auth.py

@@ -125,6 +125,9 @@ def add_roles(_allowed,_denied):
         loaded_file = yaml.safe_load(stream)
         try:
             allowed_loaded = loaded_file["allowedbyrole"]
+            for role in allowed_loaded:
+                if role not in configmanager._validroles:
+                    configmanager._validroles.append(role)
         except:
             pass
         try:
@@ -145,7 +148,7 @@ def add_roles(_allowed,_denied):
     
 def check_for_yaml():
     #checking if the file exists
-    if exists("/etc/confluent/authorization.yaml"):
+    if os.path.exists("/etc/confluent/authorization.yaml"):
         add_roles(_allowedbyrole,_deniedbyrole)
 
         return "Custom auth. file detected in /etc/confluent, updated roles accordingly"

confluent_server/confluent/certutil.py

@@ -95,27 +95,29 @@ def assure_tls_ca():
         os.makedirs(os.path.dirname(fname))
     except OSError as e:
         if e.errno != 17:
+            os.seteuid(ouid)
             raise
+    try:
+        shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
+        hv, _ = util.run(
+            ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
+        if not isinstance(hv, str):
+            hv = hv.decode('utf8')
+        hv = hv.strip()
+        hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
+        certname = '{0}.pem'.format(collective.get_myname())
+        for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
+            currname = os.path.join('/var/lib/confluent/public/site/tls/', currname)
+            if currname.endswith('.0'):
+                try:
+                    realname = os.readlink(currname)
+                    if realname == certname:
+                        os.unlink(currname)
+                except OSError:
+                    pass
+        os.symlink(certname, hashname)
     finally:
         os.seteuid(ouid)
-    shutil.copy2('/etc/confluent/tls/cacert.pem', fname)
-    hv, _ = util.run(
-        ['openssl', 'x509', '-in', '/etc/confluent/tls/cacert.pem', '-hash', '-noout'])
-    if not isinstance(hv, str):
-        hv = hv.decode('utf8')
-    hv = hv.strip()
-    hashname = '/var/lib/confluent/public/site/tls/{0}.0'.format(hv)
-    certname = '{0}.pem'.format(collective.get_myname())
-    for currname in os.listdir('/var/lib/confluent/public/site/tls/'):
-        currname = os.path.join('/var/lib/confluent/public/site/tls/', currname)
-        if currname.endswith('.0'):
-            try:
-                realname = os.readlink(currname)
-                if realname == certname:
-                    os.unlink(currname)
-            except OSError:
-                pass
-    os.symlink(certname, hashname)
 
 def substitute_cfg(setting, key, val, newval, cfgfile, line):
     if key.strip() == setting:

confluent_server/confluent/collective/manager.py

@@ -63,7 +63,7 @@ connecting = ContextBool()
 leader_init = ContextBool()
 enrolling = ContextBool()
 
-def connect_to_leader(cert=None, name=None, leader=None, remote=None):
+def connect_to_leader(cert=None, name=None, leader=None, remote=None, isretry=False):
     global currentleader
     global follower
     ocert = cert
@@ -72,8 +72,9 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     oremote = remote
     if leader is None:
         leader = currentleader
-    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
-             'subsystem': 'collective'})
+    if not isretry:
+        log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+                 'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader, remote)
     except Exception as e:
@@ -84,9 +85,11 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     with connecting:
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
+            if not banner:
+                return
             vers = banner.split()[2]
-            if vers not in (b'v2', b'v3'):
-                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
+            if vers != b'v4':
+                raise Exception('This instance only supports protocol 4, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -103,7 +106,12 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                                 '{0}'.format(leader),
                         'subsystem': 'collective'})
                     return False
+                if 'waitinline' in keydata:
+                    eventlet.sleep(0.3)
+                    return connect_to_leader(cert, name, leader, None, isretry=True)
                 if 'leader' in keydata:
+                    if keydata['leader'] == None:
+                        return None
                     log.log(
                         {'info': 'Prospective leader {0} has redirected this '
                                  'member to {1}'.format(leader, keydata['leader']),
@@ -144,7 +152,7 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                     except Exception:
                         pass
                     log.log({'error': 'Retrying connection, error during initial sync', 'subsystem': 'collective'})
-                    return connect_to_leader(ocert, oname, oleader, oremote)
+                    return connect_to_leader(ocert, oname, oleader, None)
                     raise Exception("Error doing initial DB transfer")  # bad ssl write retry
                 dbjson += ndata
             cfm.clear_configuration()
@@ -251,6 +259,9 @@ def get_myname():
         mycachedname[1] = time.time()
         return myname
 
+def in_collective():
+    return bool(list(cfm.list_collective()))
+
 def handle_connection(connection, cert, request, local=False):
     global currentleader
     global retrythread
@@ -271,7 +282,11 @@ def handle_connection(connection, cert, request, local=False):
                 return
             if follower is not None:
                 linfo = cfm.get_collective_member_by_address(currentleader)
-                remote = socket.create_connection((currentleader, 13001), 15)
+                try:
+                    remote = socket.create_connection((currentleader, 13001), 15)
+                except Exception:
+                    cfm.stop_following()
+                    return
                 remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                          keyfile='/etc/confluent/privkey.pem',
                                          certfile='/etc/confluent/srvcert.pem')
@@ -537,6 +552,11 @@ def handle_connection(connection, cert, request, local=False):
                                       'backoff': True})
             connection.close()
             return
+        if leader_init.active:
+            tlvdata.send(connection, {'error': 'Servicing a connection',
+                                      'waitinline': True})
+            connection.close()
+            return
         if myself != get_leader(connection):
             tlvdata.send(
                 connection,
@@ -552,7 +572,7 @@ def handle_connection(connection, cert, request, local=False):
                           'txcount': cfm._txcount})
             log.log({'info': 'Connecting to leader due to superior '
                              'transaction count', 'subsystem': 'collective'})
-            connection.close()
+            connection.close()  # well this won't work
             if not connect_to_leader(
                 None, None, connection.getpeername()[0]):
                 if retrythread is None:
@@ -569,9 +589,15 @@ def handle_connection(connection, cert, request, local=False):
             tlvdata.send(connection, cfm._cfgstore['collective'])
             tlvdata.send(connection, {'confluent_uuid': cfm.get_global('confluent_uuid')}) # cfm.get_globals())
             cfgdata = cfm.ConfigManager(None)._dump_to_json()
-            tlvdata.send(connection, {'txcount': cfm._txcount,
-                                      'dbsize': len(cfgdata)})
-            connection.sendall(cfgdata)
+            try:
+                tlvdata.send(connection, {'txcount': cfm._txcount,
+                            'dbsize': len(cfgdata)})
+                connection.sendall(cfgdata)
+            except Exception:
+                try:
+                    connection.close()
+                finally:
+                    return None
         #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
         # so far unused anyway
         connection.settimeout(90)
@@ -660,7 +686,11 @@ def get_leader(connection):
 
 def retire_as_leader(newleader=None):
     global currentleader
+    global reassimilate
     cfm.stop_leading(newleader)
+    if reassimilate is not None:
+        reassimilate.kill()
+        reassimilate = None
     currentleader = None
 
 def become_leader(connection):
@@ -668,6 +698,10 @@ def become_leader(connection):
     global follower
     global retrythread
     global reassimilate
+    if cfm.get_collective_member(get_myname()).get('role', None) == 'nonvoting':
+        log.log({'info': 'Refraining from being leader of collective (nonvoting)',
+            'subsystem': 'collective'})
+        return False
     log.log({'info': 'Becoming leader of collective',
              'subsystem': 'collective'})
     if follower is not None:
@@ -679,15 +713,21 @@ def become_leader(connection):
         retrythread = None
     currentleader = connection.getsockname()[0]
     skipaddr = connection.getpeername()[0]
+    if reassimilate is not None:
+        reassimilate.kill()
+    reassimilate = eventlet.spawn(reassimilate_missing)
+    cfm._ready = True
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
-        if reassimilate is not None:
-            reassimilate.kill()
-        reassimilate = eventlet.spawn(reassimilate_missing)
+
 
 def reassimilate_missing():
     eventlet.sleep(30)
-    while cfm.cfgstreams and _assimilate_missing():
+    while True:
+        try:
+            _assimilate_missing()
+        except Exception as e:
+            cfm.logException()
         eventlet.sleep(30)
 
 def _assimilate_missing(skipaddr=None):
@@ -801,6 +841,8 @@ def start_collective():
         for member in sorted(list(cfm.list_collective())):
             if member == myname:
                 continue
+            if cfm.get_collective_member(member).get('role', None) == 'nonvoting':
+                continue
             if cfm.cfgleader is None:
                 cfm.stop_following(True)
             ldrcandidate = cfm.get_collective_member(member)['address']

confluent_server/confluent/config/attributes.py

@@ -371,7 +371,7 @@ node = {
                         'the managed node.  If not specified, then console '
                         'is disabled.  "ipmi" should be specified for most '
                         'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/configmanager.py

@@ -59,13 +59,21 @@ except ModuleNotFoundError:
 import ast
 import base64
 from binascii import hexlify
+import os
+import sys
+
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.config.attributes as allattributes
 import confluent.config.conf as conf
-import confluent.log
 import confluent.noderange as noderange
 import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
+import confluent.log
 import copy
 import crypt
 try:
@@ -77,17 +85,16 @@ import eventlet
 import eventlet.event as event
 import eventlet.green.select as select
 import eventlet.green.threading as gthread
+import eventlet.green.subprocess as subprocess
 import fnmatch
 import hashlib
 import json
 import msgpack
 import operator
-import os
 import random
 import re
 import string
 import struct
-import sys
 import threading
 import time
 import traceback
@@ -112,6 +119,7 @@ _cfgstore = None
 _pendingchangesets = {}
 _txcount = 0
 _hasquorum = True
+_ready = False
 
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
@@ -120,7 +128,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
-_validroles = ('Administrator', 'Operator', 'Monitor', 'Stub')
+_validroles = ['Administrator', 'Operator', 'Monitor', 'Stub']
 
 membership_callback = None
 
@@ -422,7 +430,10 @@ def _push_rpc(stream, payload):
                 pass
             if membership_callback:
                 membership_callback()
-            stream.close()
+            try:
+                stream.close()
+            except Exception:
+                pass
 
 
 def decrypt_value(cryptvalue,
@@ -690,7 +701,9 @@ def relay_slaved_requests(name, listener):
                 if name not in cfgstreams:
                     raise Exception("Unexpected loss of node in followers: " + name)
                 sz = struct.unpack('!Q', msg)[0]
-                if sz != 0:
+                if sz == 0:
+                    _push_rpc(listener, b'')
+                else:
                     rpc = b''
                     while len(rpc) < sz:
                         nrpc = listener.recv(sz - len(rpc))
@@ -740,6 +753,16 @@ def relay_slaved_requests(name, listener):
                 return False
             return True
 
+lastheartbeat = None
+def check_leader():
+    _push_rpc(cfgleader, b'')
+    tries = 0
+    while tries < 30:
+        eventlet.sleep(0.1)
+        tries += 1
+        if lastheartbeat and lastheartbeat > (confluent.util.monotonic_time() - 3):
+            return True
+    raise Exception("Leader has disappeared")
 
 class StreamHandler(object):
     def __init__(self, sock):
@@ -761,10 +784,13 @@ class StreamHandler(object):
                     res = _push_rpc(self.sock, b'')  # nulls are a keepalive
                     if not res:
                         return None
+                    #TODO: this test can work fine even if the other end is
+                    # gone, go to a more affirmative test to more quickly
+                    # detect outage to peer
                     self.keepalive = confluent.util.monotonic_time() + 20
             self.expiry = confluent.util.monotonic_time() + 60
             msg = self.sock.recv(8)
-        except Exception:
+        except Exception as e:
             msg = None
         return msg
 
@@ -805,6 +831,9 @@ _oldcfgstore = None
 _oldtxcount = 0
 
 
+def config_is_ready():
+    return _ready
+
 def rollback_clear():
     global _cfgstore
     global _txcount
@@ -822,6 +851,8 @@ def clear_configuration():
     global _txcount
     global _oldcfgstore
     global _oldtxcount
+    global _ready
+    _ready = False
     stop_leading()
     stop_following()
     _oldcfgstore = _cfgstore
@@ -832,6 +863,7 @@ def clear_configuration():
 def commit_clear():
     global _oldtxcount
     global _oldcfgstore
+    global _ready
     # first, copy over old non-key globals, as those are
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
@@ -851,6 +883,7 @@ def commit_clear():
                 pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
+    _ready = True
 
 cfgleader = None
 
@@ -858,6 +891,7 @@ cfgleader = None
 def follow_channel(channel):
     global _txcount
     global _hasquorum
+    global lastheartbeat
     try:
         stop_leading()
         stop_following(channel)
@@ -865,7 +899,9 @@ def follow_channel(channel):
         msg = lh.get_next_msg()
         while msg:
             sz = struct.unpack('!Q', msg)[0]
-            if sz != 0:
+            if sz == 0:
+                lastheartbeat = confluent.util.monotonic_time()
+            else:
                 rpc = b''
                 while len(rpc) < sz:
                     nrpc = channel.recv(sz - len(rpc))
@@ -1245,6 +1281,7 @@ class ConfigManager(object):
     def __init__(self, tenant, decrypt=False, username=None):
         self.clientfiles = {}
         global _cfgstore
+        self.inrestore = False
         with _initlock:
             if _cfgstore is None:
                 init()
@@ -2061,6 +2098,10 @@ class ConfigManager(object):
     def _notif_attribwatchers(self, nodeattrs):
         if self.tenant not in self._attribwatchers:
             return
+        if self.inrestore:
+            # Do not stir up attribute watchers during a collective join or DB restore,
+            # it's too hectic of a time to react
+            return
         notifdata = {}
         attribwatchers = self._attribwatchers[self.tenant]
         for node in nodeattrs:
@@ -2443,6 +2484,13 @@ class ConfigManager(object):
         #TODO: wait for synchronization to suceed/fail??)
 
     def _load_from_json(self, jsondata, sync=True):
+        self.inrestore = True
+        try:
+            self._load_from_json_backend(jsondata, sync=True)
+        finally:
+            self.inrestore = False
+
+    def _load_from_json_backend(self, jsondata, sync=True):
         """Load fresh configuration data from jsondata
 
         :param jsondata: String of jsondata
@@ -2540,6 +2588,15 @@ class ConfigManager(object):
                        data.
 
         """
+        with open(os.devnull, 'w+') as devnull:
+            worker = subprocess.Popen(
+                [sys.executable, __file__, '-r' if redact else ''],
+                        stdin=devnull, stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+            stdout, stderr = worker.communicate()
+        return stdout
+
+    def _real_dump_to_json(self, redact=None):
         dumpdata = {}
         for confarea in _config_areas:
             if confarea not in self._cfgstore:
@@ -2871,9 +2928,9 @@ def dump_db_to_directory(location, password, redact=None, skipkeys=False):
         with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
             cfgfile.write(_dump_keys(password))
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
+    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write('\n')
+        cfgfile.write(b'\n')
     if 'collective' in _cfgstore:
         with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
             cfgfile.write(json.dumps(_cfgstore['collective']))
@@ -2902,9 +2959,9 @@ def get_globals():
         bkupglobals[globvar] = _cfgstore['globals'][globvar]
     return bkupglobals
 
-
 def init(stateless=False):
     global _cfgstore
+    global _ready
     if stateless:
         _cfgstore = {}
         return
@@ -2912,8 +2969,16 @@ def init(stateless=False):
         ConfigManager._read_from_path()
     except IOError:
         _cfgstore = {}
+    members = list(list_collective())
+    if len(members) < 2:
+        _ready = True
 
 
+if __name__ == '__main__':
+    redact=None
+    if '-r' in sys.argv:
+        redact=True
+    sys.stdout.write(ConfigManager(None)._real_dump_to_json(redact))
 # some unit tests worth implementing:
 # set group attribute on lower priority group, result is that node should not
 # change

confluent_server/confluent/consoleserver.py

@@ -49,7 +49,6 @@ _handled_consoles = {}
 
 _tracelog = None
 _bufferdaemon = None
-_bufferlock = None
 
 try:
     range = xrange
@@ -62,39 +61,38 @@ def chunk_output(output, n):
         yield output[i:i + n]
 
 def get_buffer_output(nodename):
-    out = _bufferdaemon.stdin
-    instream = _bufferdaemon.stdout
+    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+    out.connect("\x00confluent-vtbuffer")
     if not isinstance(nodename, bytes):
         nodename = nodename.encode('utf8')
     outdata = bytearray()
-    with _bufferlock:
-        out.write(struct.pack('I', len(nodename)))
-        out.write(nodename)
-        out.flush()
-        select.select((instream,), (), (), 30)
-        while not outdata or outdata[-1]:
-            try:
-                chunk = os.read(instream.fileno(), 128)
-            except IOError:
-                chunk = None
-            if chunk:
-                outdata.extend(chunk)
-            else:
-                select.select((instream,), (), (), 0)
-        return bytes(outdata[:-1])
+    out.send(struct.pack('I', len(nodename)))
+    out.send(nodename)
+    select.select((out,), (), (), 30)
+    while not outdata or outdata[-1]:
+        try:
+            chunk = os.read(out.fileno(), 128)
+        except IOError:
+            chunk = None
+        if chunk:
+            outdata.extend(chunk)
+        else:
+            select.select((out,), (), (), 0)
+    return bytes(outdata[:-1])
 
 
 def send_output(nodename, output):
     if not isinstance(nodename, bytes):
         nodename = nodename.encode('utf8')
-    with _bufferlock:
-        _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29)))
-        _bufferdaemon.stdin.write(nodename)
-        _bufferdaemon.stdin.flush()
-        for chunk in chunk_output(output, 8192):
-            _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29)))
-            _bufferdaemon.stdin.write(chunk)
-            _bufferdaemon.stdin.flush()
+    out = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    out.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
+    out.connect("\x00confluent-vtbuffer")
+    out.send(struct.pack('I', len(nodename) | (1 << 29)))
+    out.send(nodename)
+    for chunk in chunk_output(output, 8192):
+        out.send(struct.pack('I', len(chunk) | (2 << 29)))
+        out.send(chunk)
 
 def _utf8_normalize(data, decoder):
     # first we give the stateful decoder a crack at the byte stream,
@@ -600,15 +598,10 @@ def _start_tenant_sessions(cfm):
 def initialize():
     global _tracelog
     global _bufferdaemon
-    global _bufferlock
-    _bufferlock = semaphore.Semaphore()
     _tracelog = log.Logger('trace')
     _bufferdaemon = subprocess.Popen(
-        ['/opt/confluent/bin/vtbufferd'], bufsize=0, stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE)
-    fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL)
-    fcntl.fcntl(_bufferdaemon.stdout.fileno(),
-                fcntl.F_SETFL, fl | os.O_NONBLOCK)
+        ['/opt/confluent/bin/vtbufferd', 'confluent-vtbuffer'], bufsize=0, stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL)
 
 def start_console_sessions():
     configmodule.hook_new_configmanagers(_start_tenant_sessions)

confluent_server/confluent/core.py

@@ -66,7 +66,7 @@ import eventlet.semaphore as semaphore
 import itertools
 import msgpack
 import os
-import socket
+import eventlet.green.socket as socket
 import struct
 import sys
 
@@ -446,6 +446,7 @@ def _init_core():
                 },
             },
         },
+        'layout': PluginRoute({'handler': 'layout'}),
         'media': {
             'uploads': PluginCollection({
                 'pluginattrs': ['hardwaremanagement.method'],
@@ -480,6 +481,20 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'normalized': {
+                    'inlet_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'average_cpu_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'total_power': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
                 'energy': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',

confluent_server/confluent/discovery/core.py

@@ -66,6 +66,7 @@ import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 import confluent.discovery.protocols.ssdp as ssdp
+#import confluent.discovery.protocols.mdns as mdns
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
 import confluent.discovery.handlers.cpstorage as cpstorage
@@ -117,6 +118,7 @@ nodehandlers = {
     'onie-switch': None,
     'cumulus-switch': None,
     'affluent-switch': None,
+    #'openbmc': None,
     'service:io-device.Lenovo:management-module': None,
     'service:thinkagile-storage': cpstorage,
     'service:lenovo-tsm': tsm,
@@ -130,6 +132,7 @@ servicenames = {
     'service:lenovo-smm2': 'lenovo-smm2',
     'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
+    #'openbmc': 'openbmc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
     'service:thinkagile-storage': 'thinkagile-storagebmc',
@@ -226,6 +229,7 @@ def _printable_ip(sa):
 
 def send_discovery_datum(info):
     addresses = info.get('addresses', [])
+    addresses = util.natural_sort(addresses)
     if info['handler'] == pxeh:
         enrich_pxe_info(info)
     yield msg.KeyValueData({'nodename': info.get('nodename', '')})
@@ -644,6 +648,8 @@ def detected_models():
 
 
 def _recheck_nodes(nodeattribs, configmanager):
+    if not cfm.config_is_ready():
+        return
     if rechecklock.locked():
         # if already in progress, don't run again
         # it may make sense to schedule a repeat, but will try the easier and less redundant way first
@@ -762,6 +768,9 @@ def eval_detected(info):
 def detected(info):
     global rechecker
     global rechecktime
+    if not cfm.config_is_ready():
+        # drop processing of discovery data while configmanager is 'down'
+        return
     # later, manual and CMM discovery may act on SN and/or UUID
     for service in info['services']:
         if service in nodehandlers:
@@ -1229,7 +1238,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
             info['enclosure.bay'] = match[1]
             if match[2]:
                 if not discover_node(cfg, handler, info, match[2], manual):
-                    pending_nodes[match[2]] = nodename
+                    pending_nodes[match[2]] = info
                 return
         if 'enclosure.bay' not in info:
             unknown_info[info['hwaddr']] = info
@@ -1425,7 +1434,12 @@ def discover_node(cfg, handler, info, nodename, manual):
                 newnodeattribs['pubkeys.tls_hardwaremanager'] = \
                     util.get_fingerprint(handler.https_cert, 'sha256')
             if newnodeattribs:
-                cfg.set_node_attributes({nodename: newnodeattribs})
+                currattrs = cfg.get_node_attributes(nodename, newnodeattribs)
+                for checkattr in newnodeattribs:
+                    checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                    if checkval != newnodeattribs[checkattr]:
+                        cfg.set_node_attributes({nodename: newnodeattribs})
+                        break
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
             if nodeconfig:
@@ -1504,7 +1518,12 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
                 if info['hwaddr'] != oldhwaddr:
                     attribs[newattrname] = info['hwaddr']
         if attribs:
-            cfg.set_node_attributes({nodename: attribs})
+            currattrs = cfg.get_node_attributes(nodename, attribs)
+            for checkattr in attribs:
+                checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                if checkval != attribs[checkattr]:
+                    cfg.set_node_attributes({nodename: attribs})
+                    break
     if info['uuid'] in known_pxe_uuids:
         return True
     if uuid_is_valid(info['uuid']):
@@ -1593,7 +1612,10 @@ def remotescan():
     mycfm = cfm.ConfigManager(None)
     myname = collective.get_myname()
     for remagent in get_subscriptions():
-        affluent.renotify_me(remagent, mycfm, myname)
+        try:
+            affluent.renotify_me(remagent, mycfm, myname)
+        except Exception as e:
+            log.log({'error': 'Unexpected problem asking {} for discovery notifications'.format(remagent)})
 
 
 def blocking_scan():
@@ -1631,8 +1653,9 @@ def stop_autosense():
 
 def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
-    remotescan()
+    eventlet.spawn(remotescan)
 
 
 nodes_by_fprint = {}

confluent_server/confluent/discovery/handlers/xcc.py

@@ -247,6 +247,10 @@ class NodeHandler(immhandler.NodeHandler):
             if rsp.status == 200:
                 pwdchanged = True
                 password = newpassword
+                wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+                if '_csrf_token' in wc.cookies:
+                    wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+                wc.grab_json_response_with_status('/api/providers/logout')
             else:
                 if rspdata.get('locktime', 0) > 0:
                     raise LockedUserException(
@@ -280,6 +284,7 @@ class NodeHandler(immhandler.NodeHandler):
                 rsp.read()
                 if rsp.status != 200:
                     return (None, None)
+                wc.grab_json_response_with_status('/api/providers/logout')
                 self._currcreds = (username, newpassword)
                 wc.set_basic_credentials(username, newpassword)
                 pwdchanged = True
@@ -434,6 +439,7 @@ class NodeHandler(immhandler.NodeHandler):
                     '/api/function',
                     {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,,,'.format(uid, username)})
                 if status == 200 and rsp.get('return', 0) == 13:
+                    wc.grab_json_response('/api/providers/logout')
                     wc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
                     status = 503
                     while status != 200:
@@ -442,10 +448,13 @@ class NodeHandler(immhandler.NodeHandler):
                             {'UserName': username}, method='PATCH')
                         if status != 200:
                             rsp = json.loads(rsp)
-                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError'):
-                                eventlet.sleep(10)
+                            if rsp.get('error', {}).get('code', 'Unknown') in ('Base.1.8.GeneralError', 'Base.1.12.GeneralError', 'Base.1.14.GeneralError'):
+                                eventlet.sleep(4)
                             else:
                                 break
+                    self.tmppasswd = None
+                    self._currcreds = (username, passwd)
+                    return
             self.tmppasswd = None
         wc.grab_json_response('/api/providers/logout')
         self._currcreds = (username, passwd)
@@ -632,3 +641,4 @@ def remote_nodecfg(nodename, cfm):
     info = {'addresses': [ipaddr]}
     nh = NodeHandler(info, cfm)
     nh.config(nodename)
+

confluent_server/confluent/discovery/protocols/mdns.py

@@ -0,0 +1,449 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Documented somewhat at
+# http://buildingskb.schneider-electric.com/view.php?AID=15197
+
+# Here is the payload of an SSDP 'announce', sent to the multicast v4/v6 1900
+# NOTIFY * HTTP/1.1
+# HOST: 239.255.255.250:1900
+# CACHE-CONTROL: max-age=1800
+# AL: https://172.30.254.151:8080/redfish/v1
+# SERVER: Linux/3.14.28-ltsi Redfish/1.0
+# NT: urn:dmtf-org:service:redfish-rest:1
+# USN: uuid:00000000-0000-0000-0005-000000000001::urn:dmtf-org:service:redfish-rest:1
+# NTS: ssdp:alive
+
+
+import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
+import confluent.neighutil as neighutil
+import confluent.noderange as noderange
+import confluent.util as util
+import confluent.log as log
+import confluent.netutil as netutil
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import eventlet.greenpool as gp
+import os
+import time
+import struct
+import traceback
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+mcastv4addr = '224.0.0.251'
+mcastv6addr = 'ff02::fb'
+
+mdns6mcast = socket.inet_pton(socket.AF_INET6, mcastv6addr)
+
+def name_to_qname(name):
+    nameparts = name.split('.')
+    qname = b''
+    for namepart in name.split('.'):
+        namepart = namepart.encode('utf8')
+        qname += bytes(bytearray([len(namepart)])) + namepart
+    qname += b'\x00'
+    return qname
+
+PTR = 12
+SRV = 33
+
+def _makequery(name):
+    return struct.pack('!HHHHHH', 
+            0, # transaction id
+            0, # flags, stdard query
+            1, # query count
+            0, # answers
+            0, # authorities
+            0) + \
+        name_to_qname(name) + \
+        struct.pack('!HH', 
+            PTR,
+            (1 << 15) | 1)  # Unicast response
+
+#listsrvs = _makequery('_services._dns-sd._udp.local')  # to get all possible services
+listobmccons = _makequery('_obmc_console._tcp.local')
+
+
+def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler):
+    if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']:
+        peerbymacaddress[mac]['addresses'].append(peer)
+    else:
+        sdata = _mdns_to_dict(rsp)
+        if not sdata:
+            return 0
+        peerdata = {
+            'hwaddr': mac,
+            'addresses': [peer],
+            'services': ['openbmc'],
+            'urls': '/redfish/v1/'
+        }
+        if sdata.get('ttl', 0) == 0:
+            if byehandler:
+                eventlet.spawn_n(check_fish_handler, byehandler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+            return 1
+        if handler:
+            eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+        return 2
+        
+def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer):
+    retdata = check_fish(('/redfish/v1/', peerdata))
+    if retdata:
+        known_peers.add(peer)
+        newmacs.add(mac)
+        peerbymacaddress[mac] = retdata
+        machandlers[mac] = handler
+
+
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
+    """Watch for unsolicited mDNS answers
+
+    The handler shall be called on any service coming online.
+    byehandler is called whenever a system advertises that it is departing.
+    If no byehandler is specified, byebye messages are ignored.  The handler is
+    given (as possible), the mac address, a list of viable sockaddrs to reference
+    the peer, and the notification type (e.g.
+    'urn:dmtf-org:service:redfish-rest:1'
+
+    :param handler:  A handler for online notifications from network
+    :param byehandler: Optional handler for devices going off the network
+    """
+    # Normally, I like using v6/v4 agnostic socket. However, since we are
+    # dabbling in multicast wizardry here, such sockets can cause big problems,
+    # so we will have two distinct sockets
+    # TTL=0 is a wthdrawal, otherwise an announce
+    tracelog = log.Logger('trace')
+    net4, net6 = get_sockets()
+    net6.bind(('', 5353))
+    net4.bind(('', 5353))
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                    event=log.Events.stacktrace)
+    known_peers = set([])
+    recent_peers = set([])
+    for ifidx in util.list_interface_indexes():
+        v6grp = mdns6mcast + struct.pack('=I', ifidx)
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
+    for i4 in util.list_ips():
+        mdns4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
+                     socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            mdns4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
+    peerbymacaddress = {}
+    while True:
+        try:
+            newmacs = set([])
+            deferrednotifies = []
+            machandlers = {}
+            r = select.select((net4, net6), (), (), 60)
+            if r:
+                r = r[0]
+            recent_peers = set([])
+            while r and len(deferrednotifies) < 256:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    if peer in recent_peers:
+                        continue
+                    mac = neighutil.get_hwaddr(peer[0])
+                    if not mac:
+                        probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                        try:
+                            s.sendto(b'\x00', probepeer)
+                        except Exception:
+                            continue
+                        deferrednotifies.append((peer, rsp))
+                    datum = _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+                    if datum == 2:
+                        recent_peers.add(peer)
+                r = select.select((net4, net6), (), (), 1.5)
+                if r:
+                    r = r[0]
+            if deferrednotifies:
+                eventlet.sleep(2.2)
+            for peerrsp in deferrednotifies:
+                peer, rsp = peerrsp
+                mac = neighutil.get_hwaddr(peer[0])
+                if not mac:
+                    continue
+                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
+
+
+def _get_svrip(peerdata):
+    for addr in peerdata['addresses']:
+        if addr[0].startswith('fe80::'):
+            if '%' not in addr[0]:
+                return addr[0] + '%{0}'.format(addr[3])
+            return addr[0]
+    return peerdata['addresses'][0][0]
+
+def get_sockets():
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    return net4, net6
+        
+def active_scan(handler, protocol=None):
+    net4, net6 = get_sockets()
+    for idx in util.list_interface_indexes():
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
+                        idx)
+        try:
+            net6.sendto(listobmccons, (mcastv6addr, 5353, 0, 0))
+        except socket.error:
+            # ignore interfaces without ipv6 multicast causing error
+                pass
+    for i4 in util.list_ips():
+        if 'broadcast' not in i4:
+            continue
+        addr = i4['addr']
+        net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                        socket.inet_aton(addr))
+        try:
+            net4.sendto(listobmccons, (mcastv4addr, 5353))
+        except socket.error as se:
+            if se.errno != 101 and se.errno != 1:
+                raise
+    deadline = util.monotonic_time() + 2
+    r, _, _ = select.select((net4, net6), (), (), 2)
+    peerdata = {}
+    deferparse = []
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            if not neighutil.get_hwaddr(peer[0]):
+                probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                try:
+                    s.sendto(b'\x00', probepeer)
+                except Exception:
+                    continue
+                deferparse.append((rsp, peer))
+                continue
+            _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
+    if deferparse:
+        eventlet.sleep(2.2)
+    for dp in deferparse:
+        rsp, peer = dp
+        _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+    querypool = gp.GreenPool()
+    pooltargs = []
+    for nid in peerdata:
+        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
+        pooltargs.append(('/redfish/v1/', peerdata[nid]))
+        # For now, don't interrogate generic redfish bmcs
+        # This is due to a need to deduplicate from some supported SLP
+        # targets (IMM, TSM, others)
+        # activate this else once the core filters/merges duplicate uuid
+        # or we drop support for those devices
+        #else:
+        #    pooltargs.append(('/redfish/v1/', peerdata[nid]))
+    for pi in querypool.imap(check_fish, pooltargs):
+        if pi is not None:
+            handler(pi)
+
+def check_fish(urldata, port=443, verifycallback=None):
+    if not verifycallback:
+        verifycallback = lambda x: True
+    url, data = urldata
+    try:
+        wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5)
+        peerinfo = wc.grab_json_response(url)
+    except socket.error:
+        return None
+    if url == '/DeviceDescription.json':
+        try:
+            peerinfo = peerinfo[0]
+            myuuid = peerinfo['node-uuid'].lower()
+            if '-' not in myuuid:
+                myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]])
+            data['uuid'] = myuuid
+            data['attributes'] = peerinfo
+            data['services'] = ['lenovo-xcc']
+            return data
+        except (IndexError, KeyError):
+            return None
+            url = '/redfish/v1/'
+            peerinfo = wc.grab_json_response('/redfish/v1/')
+    if url == '/redfish/v1/':
+        if 'UUID' in peerinfo:
+            if 'services' not in data:
+                data['services'] = ['service:redfish-bmc']
+            else:
+                data['services'].append('service:redfish-bmc')
+            data['uuid'] = peerinfo['UUID'].lower()
+            return data
+    return None
+
+def extract_qname(view, reply):
+    name = ''
+    idx = 1
+    if isinstance(view[0], int):
+        currlen = view[0]
+    else:
+        currlen = ord(view[0])
+    while currlen:
+        if currlen == 192:
+            name += extract_qname(reply[view[1]:], reply)[1] + '.'
+            view = view[2:]
+            idx += 1
+            if name:
+                return idx, name[:-1]
+            return idx, ''
+        else:
+            name += view[1:currlen + 1].tobytes().decode('utf8') + '.'
+            view = view[currlen + 1:]
+            idx += currlen + 1
+        if not view:
+            break  # some contexts don't null terminate
+        if isinstance(view[0], int):
+            currlen = view[0]
+        else:
+            currlen = ord(view[0])
+    if name:
+        return idx, name[:-1]
+    return idx, ''
+
+
+
+def _mdns_to_dict(rsp):
+    txid, flags, quests, answers, arr, morerr = struct.unpack('!HHHHHH', rsp[:12])
+    rv = memoryview(rsp[12:])
+    rspv = memoryview(rsp)
+    retval = {}
+    while quests:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass = struct.unpack('!HH', rv[:4]) 
+        quests -= 1
+        rv = rv[4:]
+    while answers:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass, ttl, dlen = struct.unpack('!HHIH', rv[:10])
+        if 0 and typ == 12:  # PTR, we don't need for now...
+            adata = extract_qname(rv[10:], rspv)
+            if 'ptrs' not in retval:
+                retval['ptrs'] = [{'name': adata, 'ttl': ttl}]
+            else:
+                retval['ptrs'].append({'name': adata, 'ttl': ttl})
+        if typ == 33:
+            portnum = struct.unpack('!H', rv[14:16])[0]
+            retval['protoname'] = name.split('.', 1)[1]
+            retval['portnumber'] = portnum
+            retval['ttl'] = ttl
+        rv = rv[dlen + 10:]
+        answers -= 1
+    return retval
+
+
+def _parse_mdns(peer, rsp, peerdata, srvname):
+    parsed = _mdns_to_dict(rsp)
+    if not parsed:
+        return
+    if parsed.get('ttl', 0) == 0:
+        return
+    nid = peer[0]
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    if nid in peerdata:
+        peerdatum = peerdata[nid]
+        if peer not in peerdatum['addresses']:
+            peerdatum['addresses'].append(peer)
+    else:
+        peerdatum = {
+            'addresses': [peer],
+            'hwaddr': mac,
+            'services': [srvname]
+        }
+        if srvname == '_obmc_console._tcp.local':
+            peerdatum['services'] = ['openbmc']
+            peerdatum['urls'] = ['/redfish/v1/']
+        peerdata[nid] = peerdatum
+    
+
+def _parse_ssdp(peer, rsp, peerdata):
+    nid = peer[0]
+    mac = None
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    headlines = rsp.split(b'\r\n')
+    try:
+        _, code, _ = headlines[0].split(b' ', 2)
+    except ValueError:
+        return
+    if code == b'200':
+        if nid in peerdata:
+            peerdatum = peerdata[nid]
+            if peer not in peerdatum['addresses']:
+                peerdatum['addresses'].append(peer)
+        else:
+            peerdatum = {
+                'addresses': [peer],
+                'hwaddr': mac,
+            }
+            peerdata[nid] = peerdatum
+        for headline in headlines[1:]:
+            if not headline:
+                continue
+            header, _, value = headline.partition(b':')
+            header = header.strip().decode('utf8')
+            value = value.strip().decode('utf8')
+            if header == 'AL' or header == 'LOCATION':
+                value = value[value.index('://')+3:]
+                value = value[value.index('/'):]
+                if 'urls' not in peerdatum:
+                    peerdatum['urls'] = [value]
+                elif value not in peerdatum['urls']:
+                    peerdatum['urls'].append(value)
+            elif header == 'ST':
+                if 'services' not in peerdatum:
+                    peerdatum['services'] = [value]
+                elif value not in peerdatum['services']:
+                    peerdatum['services'].append(value)
+            elif header == 'USN':
+                peerdatum['usn'] = value
+            elif header == 'MODELNAME':
+                peerdatum['modelname'] = value
+
+
+from pprint import pprint
+if __name__ == '__main__':
+    def printit(rsp):
+        print(repr(rsp))
+    snoop(pprint)

confluent_server/confluent/discovery/protocols/pxe.py

@@ -297,92 +297,96 @@ def proxydhcp(handler, nodeguess):
     msg.msg_namelen = ctypes.sizeof(clientaddr)
     cfg = cfm.ConfigManager(None)
     while True:
-        ready = select.select([net4011], [], [], None)
-        if not ready or not ready[0]:
-            continue
-        i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
-        #nb, client = net4011.recvfrom_into(rq)
-        if i < 240:
-            continue
-        rqv = memoryview(rq)[:i]
-        client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
-        _, level, typ = struct.unpack('QII', cmsgarr[:16])
-        if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
-            idx, recv = struct.unpack('II', cmsgarr[16:24])
-            recv = ipfromint(recv)
         try:
-            optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
-        except ValueError:
-            continue
-        hwlen = rq[2]
-        opts, disco = opts_to_dict(rq, optidx, 3)
-        disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
-        node = None
-        if disco.get('hwaddr', None) in macmap:
-            node = macmap[disco['hwaddr']]
-        elif disco.get('uuid', None) in uuidmap:
-            node = uuidmap[disco['uuid']]
-        myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
-        skiplogging = True
-        netaddr = disco['hwaddr']
-        if time.time() > ignoredisco.get(netaddr, 0) + 90:
-            skiplogging = False
-            ignoredisco[netaddr] = time.time()
-        if not myipn:
-            info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                            'architecture': disco['arch'],
-                            'netinfo': {'ifidx': idx, 'recvip': recv},
-                            'services': ('pxe-client',)}
-            if not skiplogging:
-                handler(info)
-        if not node:
-            if not myipn and not skiplogging:
-                log.log(
-                        {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
-                            disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
-                )})
-            continue
-        profile = None
-        if not myipn:
-            myipn = socket.inet_aton(recv)
-            profile = get_deployment_profile(node, cfg)
-            if profile:
-                log.log({
-                    'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
-            else:
+            ready = select.select([net4011], [], [], None)
+            if not ready or not ready[0]:
+                continue
+            i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
+            #nb, client = net4011.recvfrom_into(rq)
+            if i < 240:
+                continue
+            rqv = memoryview(rq)[:i]
+            client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
+            _, level, typ = struct.unpack('QII', cmsgarr[:16])
+            if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+                idx, recv = struct.unpack('II', cmsgarr[16:24])
+                recv = ipfromint(recv)
+            try:
+                optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
+            except ValueError:
+                continue
+            hwlen = rq[2]
+            opts, disco = opts_to_dict(rq, optidx, 3)
+            disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
+            node = None
+            if disco.get('hwaddr', None) in macmap:
+                node = macmap[disco['hwaddr']]
+            elif disco.get('uuid', None) in uuidmap:
+                node = uuidmap[disco['uuid']]
+            myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
+            skiplogging = True
+            netaddr = disco['hwaddr']
+            if time.time() > ignoredisco.get(netaddr, 0) + 90:
+                skiplogging = False
+                ignoredisco[netaddr] = time.time()
+            if not myipn:
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                                'architecture': disco['arch'],
+                                'netinfo': {'ifidx': idx, 'recvip': recv},
+                                'services': ('pxe-client',)}
                 if not skiplogging:
-                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    handler(info)
+            if not node:
+                if not myipn and not skiplogging:
+                    log.log(
+                            {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
+                                disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
+                    )})
                 continue
-        if opts.get(77, None) == b'iPXE':
-            if not profile:
+            profile = None
+            if not myipn:
+                myipn = socket.inet_aton(recv)
                 profile = get_deployment_profile(node, cfg)
-            if not profile:
-                log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                if profile:
+                    log.log({
+                        'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
+                else:
+                    if not skiplogging:
+                        log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+            if opts.get(77, None) == b'iPXE':
+                if not profile:
+                    profile = get_deployment_profile(node, cfg)
+                if not profile:
+                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+                myip = socket.inet_ntoa(myipn)
+                bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
+            elif disco['arch'] == 'uefi-x64':
+                bootfile = b'confluent/x86_64/ipxe.efi'
+            elif disco['arch'] == 'bios-x86':
+                bootfile = b'confluent/x86_64/ipxe.kkpxe'
+            elif disco['arch'] == 'uefi-aarch64':
+                bootfile = b'confluent/aarch64/ipxe.efi'
+            if len(bootfile) > 127:
+                log.log(
+                    {'info': 'Boot offer cannot be made to {0} as the '
+                    'profile name "{1}" is {2} characters longer than is supported '
+                    'for this boot method.'.format(
+                        node, profile, len(bootfile) - 127)})
                 continue
-            myip = socket.inet_ntoa(myipn)
-            bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
-        elif disco['arch'] == 'uefi-x64':
-            bootfile = b'confluent/x86_64/ipxe.efi'
-        elif disco['arch'] == 'bios-x86':
-            bootfile = b'confluent/x86_64/ipxe.kkpxe'
-        elif disco['arch'] == 'uefi-aarch64':
-            bootfile = b'confluent/aarch64/ipxe.efi'
-        if len(bootfile) > 127:
-            log.log(
-                {'info': 'Boot offer cannot be made to {0} as the '
-                'profile name "{1}" is {2} characters longer than is supported '
-                'for this boot method.'.format(
-                    node, profile, len(bootfile) - 127)})
-            continue
-        rpv[:240] = rqv[:240].tobytes()
-        rpv[0:1] = b'\x02'
-        rpv[108:108 + len(bootfile)] = bootfile
-        rpv[240:243] = b'\x35\x01\x05'
-        rpv[243:249] = b'\x36\x04' + myipn
-        rpv[20:24] = myipn
-        rpv[249:268] = b'\x61\x11' + opts[97]
-        rpv[268:280] = b'\x3c\x09PXEClient\xff'
-        net4011.sendto(rpv[:281], client)
+            rpv[:240] = rqv[:240].tobytes()
+            rpv[0:1] = b'\x02'
+            rpv[108:108 + len(bootfile)] = bootfile
+            rpv[240:243] = b'\x35\x01\x05'
+            rpv[243:249] = b'\x36\x04' + myipn
+            rpv[20:24] = myipn
+            rpv[249:268] = b'\x61\x11' + opts[97]
+            rpv[268:280] = b'\x3c\x09PXEClient\xff'
+            net4011.sendto(rpv[:281], client)
+        except Exception as e:
+            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                            event=log.Events.stacktrace)
 
 
 def start_proxydhcp(handler, nodeguess=None):
@@ -583,7 +587,10 @@ def get_deployment_profile(node, cfg, cfd=None):
         return None
     candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None)
     if candmgrs:
-        candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+        try:
+            candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+        except Exception: # fallback to unverified noderange
+            candmgrs = noderange.NodeRange(candmgrs).nodes
         if collective.get_myname() not in candmgrs:
             return None
     return profile

confluent_server/confluent/discovery/protocols/slp.py

@@ -63,6 +63,9 @@ def _parse_slp_header(packet):
                                            bytes(b'\x00' + packet[7:14]))
     parsed['lang'] = packet[14:14 + langlen].decode('utf-8')
     parsed['payload'] = packet[14 + langlen:]
+    errcode = struct.unpack('!H', packet[14 + langlen:16 + langlen])[0]
+    if errcode != 0:
+        return None
     if offset:
         parsed['offset'] = 14 + langlen
         parsed['extoffset'] = offset
@@ -243,11 +246,11 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
             try:
                 net4.sendto(data, ('239.255.255.253', 427))
             except socket.error as se:
-                # On occasion, multicasting may be disabled
-                # tolerate this scenario and move on
-                if se.errno != 101:
-                    raise
-            net4.sendto(data, (bcast, 427))
+                pass
+            try:
+                net4.sendto(data, (bcast, 427))
+            except socket.error as se:
+                pass
 
 
 def _grab_rsps(socks, rsps, interval, xidmap, deferrals):

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -258,10 +258,15 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                         seconds = int(currtime)
                                         msecs = int(currtime * 1000 % 1000)
                                         reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(node, seconds, msecs)
-                                        if '%' in peer[0]:
-                                            iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
-                                            reply += 'MGTIFACE: {0}\r\n'.format(
-                                                peer[0].split('%', 1)[1])
+                                        theip = peer[0].split('%', 1)[0]
+                                        if netutil.ip_on_same_subnet(theip, 'fe80::', 64):
+                                            if '%' in peer[0]:
+                                                ifidx = peer[0].split('%', 1)[1]
+                                                iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
+                                            else:
+                                                ifidx = '{}'.format(peer[-1])
+                                                iface = peer[-1]
+                                            reply += 'MGTIFACE: {0}\r\n'.format(ifidx)
                                             ncfg = netutil.get_nic_config(
                                                 cfg, node, ifidx=iface)
                                             if ncfg.get('matchesnodename', None):

confluent_server/confluent/firmwaremanager.py

@@ -53,7 +53,7 @@ def execupdate(handler, filename, updateobj, type, owner, node, datfile):
                 return
         if type == 'ffdc' and os.path.isdir(filename):
             filename += '/' + node
-        if 'type' == 'ffdc':
+        if type == 'ffdc':
             errstr = False
             if os.path.exists(filename):
                 errstr = '{0} already exists on {1}, cannot overwrite'.format(

confluent_server/confluent/forwarder.py

@@ -119,7 +119,7 @@ def get_port(addr, clientip, sessionid):
                 newsock.bind(('::', newport, 0, 0))
                 newsock.listen(50)
                 break
-            except OSError as e:
+            except (socket.error, OSError) as e:
                 if e.errno == 98:
                     newport += 1
                     continue

confluent_server/confluent/httpapi.py

@@ -288,12 +288,16 @@ def _authorize_request(env, operation, reqbody):
             authdata = auth.authorize(name, element=element, operation=operation)
         else:
             element = None
-    if (not authdata) and 'HTTP_COOKIE' in env:
-        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
-        if cidx >= 0:
-            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
-            sessid = sessionid
+    if not authdata:
+        if 'HTTP_CONFLUENTSESSION' in env:
+            sessionid = env['HTTP_CONFLUENTSESSION']
             sessid = sessionid
+        elif 'HTTP_COOKIE' in env:
+            cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+            if cidx >= 0:
+                sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+                sessid = sessionid
+        if sessionid:
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):
                     if env['PATH_INFO'] == '/sessions/current/logout':
@@ -483,6 +487,8 @@ def wsock_handler(ws):
                         elif clientmsg[0] == '!':
                             msg = json.loads(clientmsg[1:])
                             action = msg.get('operation', None)
+                            if not action:
+                                action = msg.get('action', None)
                             targ = msg.get('target', None)
                             if targ:
                                 authdata = auth.authorize(name, targ, operation=action)
@@ -520,6 +526,13 @@ def wsock_handler(ws):
                                             datacallback=datacallback,
                                             width=width, height=height)
                                     myconsoles[clientsessid] = consession
+                            elif action == 'resize':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].resize(
+                                    width=msg['width'], height=msg['height'])
+                            if action == 'break':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].send_break()
                             elif action == 'stop':
                                 sessid = '{0}'.format(msg.get('sessid', None))
                                 if sessid in myconsoles:
@@ -1071,7 +1084,7 @@ def serve(bind_host, bind_port):
         pass  # we gave it our best shot there
     try:
         eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                             debug=False, socket_timeout=60)
+                             debug=False, socket_timeout=60, keepalive=False)
     except TypeError:
         # Older eventlet in place, skip arguments it does not understand
         eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)

confluent_server/confluent/main.py

@@ -306,6 +306,7 @@ def run(args):
         except AttributeError:
             pass  # Windows...
         os.umask(oumask)
+    auth.check_for_yaml()
     collective.startup()
     consoleserver.initialize()
     http_bind_host, http_bind_port = _get_connector_config('http')
@@ -325,7 +326,7 @@ def run(args):
             break
         except Exception:
             eventlet.sleep(0.5)
-    disco.start_detection()
+    eventlet.spawn_n(disco.start_detection)
     eventlet.sleep(1)
     consoleserver.start_console_sessions()
     while 1:

confluent_server/confluent/messages.py

@@ -92,6 +92,7 @@ def msg_deserialize(packed):
         return cls(*m[1:])
     raise Exception("Unknown shenanigans")
 
+
 class ConfluentMessage(object):
     apicode = 200
     readonly = False
@@ -254,6 +255,21 @@ class ConfluentNodeError(object):
         raise Exception('{0}: {1}'.format(self.node, self.error))
 
 
+class Generic(ConfluentMessage):
+
+    def __init__(self, data):
+        self.data = data
+
+    def json(self):
+        return json.dumps(self.data)
+    
+    def raw(self):
+        return self.data
+    
+    def html(self):
+        return json.dumps(self.data)
+
+
 class ConfluentResourceUnavailable(ConfluentNodeError):
     apicode = 503
 

confluent_server/confluent/netutil.py

@@ -25,6 +25,9 @@ import eventlet.support.greendns
 import os
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
+eventlet.support.greendns.resolver.clear()
+eventlet.support.greendns.resolver._resolver.lifetime = 1
+
 def msg_align(len):
     return (len + 3) & ~3
 
@@ -332,19 +335,57 @@ def get_full_net_config(configmanager, node, serverip=None):
     if serverip:
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
+    defaultnic = {}
+    ppool = eventlet.greenpool.GreenPool(64)
     if None in attribs:
-        nm.process_attribs(None, attribs[None])
+        ppool.spawn(nm.process_attribs, None, attribs[None])
         del attribs[None]
     for netname in sorted(attribs):
-        nm.process_attribs(netname, attribs[netname])
+        ppool.spawn(nm.process_attribs, netname, attribs[netname])
+    ppool.waitall()
     retattrs = {}
     if None in nm.myattribs:
         retattrs['default'] = nm.myattribs[None]
         add_netmask(retattrs['default'])
         del nm.myattribs[None]
+    else:
+        nnc = get_nic_config(configmanager, node, serverip=serverip)
+        if nnc.get('ipv4_address', None):
+            defaultnic['ipv4_address'] = '{}/{}'.format(nnc['ipv4_address'], nnc['prefix'])
+        if nnc.get('ipv4_gateway', None):
+            defaultnic['ipv4_gateway'] = nnc['ipv4_gateway']
+        if nnc.get('ipv4_method', None):
+            defaultnic['ipv4_method'] = nnc['ipv4_method']
+        if nnc.get('ipv6_address', None):
+            defaultnic['ipv6_address'] = '{}/{}'.format(nnc['ipv6_address'], nnc['ipv6_prefix'])
+        if nnc.get('ipv6_method', None):
+            defaultnic['ipv6_method'] = nnc['ipv6_method']
     retattrs['extranets'] = nm.myattribs
     for attri in retattrs['extranets']:
         add_netmask(retattrs['extranets'][attri])
+        if retattrs['extranets'][attri].get('ipv4_address', None) == defaultnic.get('ipv4_address', 'NOPE'):
+            defaultnic = {}
+        if retattrs['extranets'][attri].get('ipv6_address', None) == defaultnic.get('ipv6_address', 'NOPE'):
+            defaultnic = {}
+    if defaultnic:
+        retattrs['default'] = defaultnic
+        add_netmask(retattrs['default'])
+        ipv4addr = defaultnic.get('ipv4_address', None)
+        if ipv4addr and '/' in ipv4addr:
+            ipv4bytes = socket.inet_pton(socket.AF_INET, ipv4addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv4bytes, addr[2]):
+                    defaultnic['current_nic'] = True
+        ipv6addr = defaultnic.get('ipv6_address', None)
+        if ipv6addr and '/' in ipv6addr:
+            ipv6bytes = socket.inet_pton(socket.AF_INET6, ipv6addr.split('/')[0])
+            for addr in nm.myaddrs:
+                if addr[0] != socket.AF_INET6:
+                    continue
+                if ipn_on_same_subnet(addr[0], addr[1], ipv6bytes, addr[2]):
+                    defaultnic['current_nic'] = True
     return retattrs
 
 

confluent_server/confluent/networking/lldp.py

@@ -381,9 +381,10 @@ def list_info(parms, requestedparameter):
                 break
         else:
             candidate = info[requestedparameter]
-            candidate = candidate.strip()
-            if candidate != '':
-                results.add(_api_sanitize_string(candidate))
+            if candidate:
+                candidate = candidate.strip()
+                if candidate != '':
+                    results.add(_api_sanitize_string(candidate))
     return [msg.ChildCollection(x + suffix) for x in util.natural_sort(results)]
 
 def _handle_neighbor_query(pathcomponents, configmanager):

confluent_server/confluent/networking/macmap.py

@@ -31,7 +31,14 @@
 # this module will provide mac to switch and full 'ifName' label
 # This functionality is restricted to the null tenant
 
+import os
+import sys
+
 if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
     import confluent.config.configmanager as cfm
     import confluent.snmputil as snmp
 
@@ -42,12 +49,11 @@ import eventlet.green.select as select
 
 import eventlet.green.socket as socket
 
-
-import os
-import sys
+import confluent.collective.manager as collective
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
+import confluent.noderange as noderange
 import confluent.util as util
 from eventlet.greenpool import GreenPool
 import eventlet.green.subprocess as subprocess
@@ -208,7 +214,6 @@ def _offload_map_switch(switch, password, user):
 
 def _start_offloader():
     global _offloader
-    os.environ['PYTHONPATH'] = ':'.join(sys.path)
     _offloader = subprocess.Popen(
         [sys.executable, __file__, '-o'], bufsize=0, stdin=subprocess.PIPE,
         stdout=subprocess.PIPE)
@@ -498,10 +503,21 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'collective.managercandidates', 'net*.switch', 'net*.switchport'))
         switches = set([])
+        incollective = collective.in_collective()
+        if incollective:
+            mycollectivename = collective.get_myname()
         for node in nodelocations:
             cfg = nodelocations[node]
+            if incollective:
+                candmgrs = cfg.get('collective.managercandidates', {}).get('value', None)
+                if candmgrs:
+                    candmgrs = noderange.NodeRange(candmgrs, configmanager).nodes
+                    if mycollectivename not in candmgrs:
+                        # do not think about trying to find nodes that we aren't possibly
+                        # supposed to be a manager for in a collective
+                        continue
             if cfg.get('type', {}).get('value', None) == 'switch':
                 switches.add(node)
             for attr in cfg:

confluent_server/confluent/noderange.py

@@ -55,6 +55,145 @@ def humanify_nodename(nodename):
     return [int(text) if text.isdigit() else text.lower()
             for text in re.split(numregex, nodename)]
 
+def unnumber_nodename(nodename):
+    # stub out numbers
+    chunked = ["{}" if text.isdigit() else text.lower()
+            for text in re.split(numregex, nodename)]
+    return chunked
+
+def getnumbers_nodename(nodename):
+    return [x for x in re.split(numregex, nodename) if x.isdigit()]
+    
+
+class Bracketer(object):
+    __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens']
+
+    def __init__(self, nodename):
+        self.sequences = []
+        self.numlens = []
+        realnodename = nodename
+        if ':' in nodename:
+            realnodename = nodename.split(':', 1)[0]
+        self.count = len(getnumbers_nodename(realnodename))
+        self.nametmpl = unnumber_nodename(realnodename)
+        for n in range(self.count):
+            self.sequences.append(None)
+            self.numlens.append([0, 0])
+        self.diffn = None
+        self.tokens = []
+        self.extend(nodename)
+        if self.count == 0:
+            self.tokens = [nodename]
+
+    def extend(self, nodeorseq):
+        # can only differentiate a single number
+        endname = None
+        endnums = None
+        if ':' in nodeorseq:
+            nodename, endname = nodeorseq.split(':', 1)
+        else:
+            nodename = nodeorseq
+        txtnums = getnumbers_nodename(nodename)
+        nums = [int(x) for x in txtnums]
+        for n in range(self.count):
+            # First pass to see if we have exactly one different number
+            padto = len(txtnums[n])
+            needpad = (padto != len('{}'.format(nums[n])))
+            if self.sequences[n] is None:
+                # We initialize to text pieces, 'currstart', and 'prev' number
+                self.sequences[n] = [[], nums[n], nums[n]]
+                self.numlens[n] = [len(txtnums[n]), len(txtnums[n])]
+            elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto:
+                continue  # new nodename has no new number, keep going
+            else: # if self.sequences[n][2] != nums[n] or :
+                if self.diffn is not None and (n != self.diffn or
+                        (padto < self.numlens[n][1]) or
+                        (needpad and padto != self.numlens[n][1])):
+                    self.flush_current()
+                    self.sequences[n] = [[], nums[n], nums[n]]
+                    self.numlens[n] = [padto, padto]
+                self.diffn = n
+        for n in range(self.count):
+            padto = len(txtnums[n])
+            needpad = (padto != len('{}'.format(nums[n])))
+            if self.sequences[n] is None:
+                # We initialize to text pieces, 'currstart', and 'prev' number
+                self.sequences[n] = [[], nums[n], nums[n]]
+                self.numlens[n] = [len(txtnums[n]), len(txtnums[n])]
+            elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto:
+                continue  # new nodename has no new number, keep going
+            else: # if self.sequences[n][2] != nums[n] or :
+                if self.diffn is not None and (n != self.diffn or
+                        (padto < self.numlens[n][1]) or
+                        (needpad and padto != self.numlens[n][1])):
+                    self.flush_current()
+                    self.sequences[n] = [[], nums[n], nums[n]]
+                    self.numlens[n] = [padto, padto]
+                    self.diffn = None
+                else:
+                    self.diffn = n
+                if self.sequences[n][2] == (nums[n] - 1):
+                    self.sequences[n][2] = nums[n]
+                    self.numlens[n][1] = padto
+                elif self.sequences[n][2] < (nums[n] - 1):
+                    if self.sequences[n][2] != self.sequences[n][1]:
+                        fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                    else:
+                        fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                    self.sequences[n][1] = nums[n]
+                    self.numlens[n][0] = padto
+                self.sequences[n][2] = nums[n]
+                self.numlens[n][1] = padto
+
+    def flush_current(self):
+        txtfields = []
+        if self.sequences and self.sequences[0] is not None:
+            for n in range(self.count):
+                if self.sequences[n][1] == self.sequences[n][2]:
+                    fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                else:
+                    fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                txtfield = ','.join(self.sequences[n][0])
+                if txtfield.isdigit():
+                    txtfields.append(txtfield)
+                else:
+                    txtfields.append('[{}]'.format(txtfield))
+            self.tokens.append(''.join(self.nametmpl).format(*txtfields))
+        self.sequences = []
+        for n in range(self.count):
+            self.sequences.append(None)
+
+    @property
+    def range(self):
+        if self.sequences:
+            self.flush_current()
+        return ','.join(self.tokens)
+
+
+def group_elements(elems):
+    """ Take the specefied elements and chunk them according to text similarity
+    """
+    prev = None
+    currchunk = []
+    chunked_elems = [currchunk]
+    for elem in elems:
+        elemtxt = unnumber_nodename(elem)
+        if not prev:
+            prev = elemtxt
+            currchunk.append(elem)
+            continue
+        if prev == elemtxt:
+            currchunk.append(elem)
+        else:
+            currchunk = [elem]
+            chunked_elems.append(currchunk)
+            prev = elemtxt
+    return chunked_elems
+
 
 class ReverseNodeRange(object):
     """Abbreviate a set of nodes to a shorter noderange representation
@@ -71,7 +210,8 @@ class ReverseNodeRange(object):
     @property
     def noderange(self):
         subsetgroups = []
-        for group in self.cfm.get_groups(sizesort=True):
+        allgroups = self.cfm.get_groups(sizesort=True)
+        for group in allgroups:
             if lastnoderange:
                 for nr in lastnoderange:
                     if lastnoderange[nr] - self.nodes:
@@ -88,7 +228,39 @@ class ReverseNodeRange(object):
                 self.nodes -= nl
                 if not self.nodes:
                     break
-        return ','.join(sorted(subsetgroups) + sorted(self.nodes))
+        # then, analyze sequentially identifying matching alpha subsections
+        # then try out noderange from beginning to end
+        # we need to know discontinuities, which are either:
+        # nodes that appear in the noderange that are not in the nodes
+        # nodes that do not exist at all (we need a noderange modification 
+        # that returns non existing nodes)
+        ranges = []
+        try:
+            subsetgroups.sort(key=humanify_nodename)
+            groupchunks = group_elements(subsetgroups)
+            for gc in groupchunks:
+                if not gc:
+                    continue
+                bracketer = Bracketer(gc[0])
+                for chnk in gc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            subsetgroups.sort()
+            ranges.extend(subsetgroups)
+        try:
+            nodes = sorted(self.nodes, key=humanify_nodename)
+            nodechunks = group_elements(nodes)
+            for nc in nodechunks:
+                if not nc:
+                    continue
+                bracketer = Bracketer(nc[0])
+                for chnk in nc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            ranges.extend(sorted(self.nodes))
+        return ','.join(ranges)
 
 
 
@@ -230,12 +402,16 @@ class NodeRange(object):
     def _expandstring(self, element, filternodes=None):
         prefix = ''
         if element[0][0] in ('/', '~'):
+            if self.purenumeric:
+                raise Exception('Regular expression not supported within "[]"')
             element = ''.join(element)
             nameexpression = element[1:]
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
             return set(self.cfm.filter_nodenames(nameexpression, filternodes))
         elif '=' in element[0] or '!~' in element[0]:
+            if self.purenumeric:
+                raise Exception('Equality/Inequality operators (=, !=, =~, !~) are invalid within "[]"')
             element = ''.join(element)
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
@@ -295,3 +471,29 @@ class NodeRange(object):
         if self.cfm is None:
             return set([element])
         raise Exception(element + ' not a recognized node, group, or alias')
+
+if __name__ == '__main__':
+    cases = [
+        (['r3u4', 'r5u6'], 'r3u4,r5u6'),  # should not erroneously gather
+        (['r3u4s1', 'r5u6s3'], 'r3u4s1,r5u6s3'),  # should not erroneously gather
+        (['r3u4s1', 'r3u4s2', 'r5u4s3'], 'r3u4s[1:2],r5u4s3'),  # should not erroneously gather
+        (['r3u4', 'r3u5', 'r3u6', 'r3u9', 'r4u1'], 'r3u[4:6,9],r4u1'),
+        (['n01', 'n2', 'n03'], 'n01,n2,n03'),
+        (['n7', 'n8', 'n09', 'n10', 'n11', 'n12', 'n13', 'n14', 'n15', 'n16',
+          'n17', 'n18', 'n19', 'n20'], 'n[7:8],n[09:20]')
+    ]
+    for case in cases:
+        gc = case[0]
+        bracketer = Bracketer(gc[0])
+        for chnk in gc[1:]:
+            bracketer.extend(chnk)
+        br = bracketer.range
+        resnodes = NodeRange(br).nodes
+        if set(resnodes) != set(gc):
+            print('FAILED: ' + repr(sorted(gc)))
+            print('RESULT: ' + repr(sorted(resnodes)))
+            print('EXPECTED: ' + repr(case[1]))
+            print('ACTUAL: ' + br)
+
+
+

confluent_server/confluent/osimage.py

@@ -1,9 +1,8 @@
 #!/usr/bin/python
-import confluent.exceptions as exc
-import confluent.messages as msg
 import eventlet
 import eventlet.green.select as select
 import eventlet.green.subprocess as subprocess
+from fnmatch import fnmatch
 import glob
 import logging
 logging.getLogger('libarchive').addHandler(logging.NullHandler())
@@ -14,6 +13,14 @@ import shutil
 import sys
 import time
 import yaml
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
+
+import confluent.exceptions as exc
+import confluent.messages as msg
 
 COPY = 1
 EXTRACT = 2
@@ -147,6 +154,14 @@ def update_boot_esxi(profiledir, profile, label):
          '{0}/boot.img'.format(profiledir), profname], preexec_fn=relax_umask)
 
 
+def find_glob(loc, fileglob):
+    for cdir, _, fs in os.walk(loc):
+        for f in fs:
+            if fnmatch(f, fileglob):
+                return os.path.join(cdir, f)
+    return None
+
+
 def update_boot_linux(profiledir, profile, label):
     profname = os.path.basename(profiledir)
     kernelargs = profile.get('kernelargs', '')
@@ -164,7 +179,11 @@ def update_boot_linux(profiledir, profile, label):
     for initramfs in initrds:
         grubcfg += " /initramfs/{0}".format(initramfs)
     grubcfg += "\n}\n"
-    with open(profiledir + '/boot/efi/boot/grub.cfg', 'w') as grubout:
+    # well need to honor grubprefix path if different
+    grubcfgpath = find_glob(profiledir + '/boot', 'grub.cfg')
+    if not grubcfgpath:
+        grubcfgpath = profiledir + '/boot/efi/boot/grub.cfg'
+    with open(grubcfgpath, 'w') as grubout:
         grubout.write(grubcfg)
     ipxeargs = kernelargs
     for initramfs in initrds:
@@ -861,7 +880,6 @@ class MediaImporter(object):
         return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles, 'error': self.error}
 
     def importmedia(self):
-        os.environ['PYTHONPATH'] = ':'.join(sys.path)
         if self.medfile:
             os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
         with open(os.devnull, 'w') as devnull:
@@ -936,3 +954,4 @@ if __name__ == '__main__':
         sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd))
     else:
         sys.exit(import_image(sys.argv[1], callback=printit))
+

confluent_server/confluent/plugins/console/openbmc.py

@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import pyghmi.util.webclient as webclient
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    bmc = bmc.split('/', 1)[0]
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+         
+
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        wc = webclient.SecureHTTPConnection(self.origbmc, 443, verifycallback=kv)
+        rsp = wc.grab_json_response_with_status('/login', {'data': [self.username.decode('utf8'), self.password.decode("utf8")]}, headers={'Content-Type': 'application/json'})
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/console0'.format(self.bmc), host=bmc, cookie='XSRF-TOKEN={0}; SESSION={1}'.format(wc.cookies['XSRF-TOKEN'], wc.cookies['SESSION']))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)

confluent_server/confluent/plugins/hardwaremanagement/cooltera.py

@@ -0,0 +1,268 @@
+# Copyright 2022 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from xml.etree.ElementTree import fromstring as rfromstring
+import pyghmi.util.webclient as wc
+import confluent.util as util
+import confluent.messages as msg
+import confluent.exceptions as exc
+import eventlet.green.time as time
+import eventlet.green.socket as socket
+import eventlet.greenpool as greenpool
+import eventlet
+try:
+    import Cookie
+    httplib = eventlet.import_patched('httplib')
+except ImportError:
+    httplib = eventlet.import_patched('http.client')
+    import http.cookies as Cookie
+
+sensorsbymodel = {
+    'FS1350': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve', 'valve2'],
+    'FS600': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'pdp', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+    'RM100': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps2', 'ps3', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+}
+_thesensors = {
+    'RM100': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS600': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS1350': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'pumpspeed3': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+        'ps4': ('Primary loop return pressure', 'bar'),
+    },
+}
+
+
+def fromstring(inputdata):
+    if isinstance(inputdata, bytes):
+        cmpstr = b'!entity'
+    else:
+        cmpstr = '!entity'
+    if cmpstr in inputdata.lower():
+        raise Exception('!ENTITY not supported in this interface')
+    # The measures above should filter out the risky facets of xml
+    # We don't need sophisticated feature support
+    return rfromstring(inputdata)  # nosec
+
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+pdupool = greenpool.GreenPool(128)
+
+class WebResponse(httplib.HTTPResponse):
+    def _check_close(self):
+        return True
+
+class WebConnection(wc.SecureHTTPConnection):
+    response_class = WebResponse
+    def __init__(self, host, secure, verifycallback):
+        if secure:
+            port = 443
+        else:
+            port = 80
+        wc.SecureHTTPConnection.__init__(self, host, port, verifycallback=verifycallback)
+        self.secure = secure
+        self.cookies = {}
+
+    def connect(self):
+        if self.secure:
+            return super(WebConnection, self).connect()
+        addrinfo = socket.getaddrinfo(self.host, self.port)[0]
+        # workaround problems of too large mtu, moderately frequent occurance
+        # in this space
+        plainsock = socket.socket(addrinfo[0])
+        plainsock.settimeout(self.mytimeout)
+        try:
+            plainsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
+        except socket.error:
+            pass
+        plainsock.connect(addrinfo[4])
+        self.sock = plainsock
+    
+    def getresponse(self):
+        try:
+            rsp = super(WebConnection, self).getresponse()
+            try:
+                hdrs = [x.split(':', 1) for x in rsp.msg.headers]
+            except AttributeError:
+                hdrs = rsp.msg.items()
+            for hdr in hdrs:
+                if hdr[0] == 'Set-Cookie':
+                    c = Cookie.BaseCookie(hdr[1])
+                    for k in c:
+                        self.cookies[k] = c[k].value
+        except httplib.BadStatusLine:
+            self.broken = True
+            raise
+        return rsp
+
+    def request(self, method, url, body=None):
+        headers = {}
+        if body:
+            headers['Content-Length'] = len(body)
+        cookies = []
+        for cookie in self.cookies:
+            cookies.append('{0}={1}'.format(cookie, self.cookies[cookie]))
+        headers['Cookie'] = ';'.join(cookies)
+        headers['Host'] = 'pdu.cluster.net'
+        headers['Accept'] = '*/*'
+        headers['Accept-Language'] = 'en-US,en;q=0.9'
+        headers['Connection'] = 'close'
+        headers['Referer'] = 'http://pdu.cluster.net/setting_admin.htm'
+        return super(WebConnection, self).request(method, url, body, headers)
+
+    def grab_response(self, url, body=None, method=None):
+        if method is None:
+            method = 'GET' if body is None else 'POST'
+        if body:
+            self.request(method, url, body)
+        else:
+            self.request(method, url)
+        rsp = self.getresponse()
+        body = rsp.read()
+        return body, rsp.status
+
+class CoolteraClient(object):
+    def __init__(self, cdu, configmanager):
+        self.node = cdu
+        self.configmanager = configmanager
+        self._wc = None
+
+    @property
+    def wc(self):
+        if self._wc:
+            return self._wc
+        targcfg = self.configmanager.get_node_attributes(self.node,
+                                            ['hardwaremanagement.manager'],
+                                            decrypt=True)
+        targcfg = targcfg.get(self.node, {})
+        target = targcfg.get(
+            'hardwaremanagement.manager', {}).get('value', None)
+        if not target:
+            target = self.node
+        target = target.split('/', 1)[0]
+        cv = util.TLSCertVerifier(
+            self.configmanager, self.node,
+            'pubkeys.tls_hardwaremanager').verify_cert
+        self._wc = WebConnection(target, secure=True, verifycallback=cv)
+        return self._wc
+
+    
+def xml2stateinfo(statdata):
+    statdata = fromstring(statdata)
+    stateinfo = []
+    sensornames = sorted([x.tag for x in statdata])
+    themodel = None
+    for model in sorted(sensorsbymodel):
+        if all([x in sensornames for x in sensorsbymodel[model]]):
+            themodel = model
+            break
+    else:
+        print(repr(sensornames))
+    thesensors = _thesensors[themodel]
+#['mode', 't1', 't2a', 't2b', 't2c', 't2', 't5', 't3', 't4', 'dw', 't3', 'rh', 'setpoint', 'secflow', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'sdp', 'valve', 'valve2', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'alarms', 'dt', 'p3state', 'duty']
+    for tagname in thesensors:
+        label, units = thesensors[tagname]
+        val = statdata.find(tagname).text.replace(units, '').strip()
+        stateinfo.append({
+            'name': label,
+            'value': val,
+            'units': units.replace('degC', '°C'),
+            'type': 'Temperature',
+        })
+    return stateinfo
+
+    
+_sensors_by_node = {}
+def read_sensors(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+        for sensor in sensors:
+            yield msg.ChildCollection(simplify_name(sensors[sensor][0]))
+        return     
+    if category in ('leds, fans'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        cc = CoolteraClient(node, configmanager)
+        cc.wc.request('GET', '/status.xml')
+        rsp = cc.wc.getresponse()
+        statdata = rsp.read()
+        statinfo = xml2stateinfo(statdata)
+        _sensors_by_node[node] = (statinfo, time.time() + 1)
+        sn = _sensors_by_node.get(node, None)
+    if sn:
+        yield msg.SensorReadings(sn[0], name=node)
+
+
+def retrieve(nodes, element, configmanager, inputdata):
+    if element[0] == 'sensors':
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(read_sensors, element, node, configmanager)
+        for rsp in gp:
+            for datum in rsp:
+                yield datum
+        return

confluent_server/confluent/plugins/hardwaremanagement/deltapdu.py

@@ -199,11 +199,12 @@ def update(nodes, element, configmanager, inputdata):
     if 'outlets' not in element:
         yield msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return
+    timeout = 4
     for node in nodes:
         gc = PDUClient(node, configmanager)
         newstate = inputdata.powerstate(node)
         gc.set_outlet(element[-1], newstate)
         gc.logout()
-    eventlet.sleep(2)
+    eventlet.sleep(timeout)
     for res in retrieve(nodes, element, configmanager, inputdata):
         yield res

confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py

@@ -25,6 +25,10 @@ import hashlib
 import json
 import time
 
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
 #eaton uses 'eval' rather than json, massage it to be valid json
 def sanitize_json(data):
     if not isinstance(data, str):
@@ -131,6 +135,43 @@ class WebConnection(wc.SecureHTTPConnection):
         body = rsp.read()
         return body, rsp.status
 
+_sensors_by_node = {}
+def get_sensor_data(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    readings = []
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+    if category in ('leds, fans', 'temperature'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        gc = PDUClient(node, configmanager)
+        try:
+            sdata = gc.get_sensor_data()
+        finally:
+            gc.logout()
+        _sensors_by_node[node] = [sdata, time.time() + 1]
+        sn = _sensors_by_node.get(node, None)
+    for outlet in sn[0]:
+        for sensename in sn[0][outlet]:
+            myname = 'Outlet {0} {1}'.format(outlet, sensename)
+            measurement = sn[0][outlet][sensename]
+            if name == 'all' or simplify_name(myname) == name:
+                readings.append({
+                    'name': myname,
+                    'value': float(measurement['value']),
+                    'units': measurement['units'],
+                    'type': measurement['type'],
+                })
+    if justnames:
+        for reading in readings:
+            yield msg.ChildCollection(simplify_name(reading['name']))
+    else:
+        yield msg.SensorReadings(readings, name=node)
 
 
 class PDUClient(object):
@@ -231,6 +272,28 @@ class PDUClient(object):
             if outdata[0] == outlet:
                 return 'on' if outdata[3] else 'off'
         return
+    
+    def get_sensor_data(self):
+        rsp = self.do_request('cgi_pdu_outlets')
+        data = sanitize_json(rsp[0])
+        data = json.loads(data)
+        data = data['data'][0]
+        sdata = {}
+        for outdata in data:
+            outsense = {}
+            outletname = outdata[0][0]
+            outsense['Energy'] = {
+                'value': float(outdata[11] / 1000),
+                'units': 'kwh',
+                'type': 'Energy'
+            }
+            outsense['Power'] = {
+                'value': float(outdata[4]),
+                'units': 'w',
+                'type': 'Power',
+            }
+            sdata[outletname] = outsense
+        return sdata
 
     def set_outlet(self, outlet, state):
         rsp = self.do_request('cgi_pdu_outlets')
@@ -247,7 +310,12 @@ class PDUClient(object):
             idx += 1
 
 def retrieve(nodes, element, configmanager, inputdata):
-    if 'outlets' not in element:
+    if element[0] == 'sensors':
+        for node in nodes:
+            for res in get_sensor_data(element, node, configmanager):
+                yield res
+        return
+    elif 'outlets' not in element:
         for node in nodes:
             yield  msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return

confluent_server/confluent/plugins/hardwaremanagement/geist.py

@@ -17,6 +17,7 @@ import confluent.util as util
 import confluent.messages as msg
 import confluent.exceptions as exc
 import eventlet.green.time as time
+import eventlet
 import eventlet.greenpool as greenpool
 
 def simplify_name(name):
@@ -25,6 +26,20 @@ def simplify_name(name):
 
 pdupool = greenpool.GreenPool(128)
 
+def data_by_type(indata):
+    databytype = {}
+    for keyname in indata:
+        obj = indata[keyname]
+        objtype = obj.get('type', None)
+        if not objtype:
+            continue
+        if objtype in databytype:
+            raise Exception("Multiple instances of type {} not yet supported".format(objtype))
+        databytype[objtype] = obj
+        obj['keyname'] = keyname
+    return databytype
+
+
 class GeistClient(object):
     def __init__(self, pdu, configmanager):
         self.node = pdu
@@ -90,19 +105,25 @@ class GeistClient(object):
     def get_outlet(self, outlet):
         rsp = self.wc.grab_json_response('/api/dev')
         rsp = rsp['data']
-        if len(rsp) != 1:
+        dbt = data_by_type(rsp)
+        if 't3hd' in dbt:
+            del dbt['t3hd']
+        if len(dbt) != 1:
             raise Exception('Multiple PDUs not supported per pdu')
-        pduname = list(rsp)[0]
-        outlet = rsp[pduname]['outlet'][str(int(outlet) - 1)]
+        pdutype = list(dbt)[0]
+        outlet = dbt[pdutype]['outlet'][str(int(outlet) - 1)]
         state = outlet['state'].split('2')[-1]
         return state
 
     def set_outlet(self, outlet, state):
         rsp = self.wc.grab_json_response('/api/dev')
-        if len(rsp['data']) != 1:
+        dbt = data_by_type(rsp['data'])
+        if 't3hd' in dbt:
+            del dbt['t3hd']
+        if len(dbt) != 1:
             self.logout()
             raise Exception('Multiple PDUs per endpoint not supported')
-        pdu = list(rsp['data'])[0]
+        pdu = dbt[list(dbt)[0]]['keyname']
         outlet = int(outlet) - 1
         rsp = self.wc.grab_json_response(
             '/api/dev/{0}/outlet/{1}'.format(pdu, outlet),
@@ -126,6 +147,12 @@ def process_measurement(keyname, name, enttype, entname, measurement, readings,
         if category not in ('all',):
             return
         readtype = 'Voltage'
+    elif measurement['type'] == 'temperature':
+        readtype = 'Temperature'
+    elif measurement['type'] == 'dewpoint':
+        readtype = 'Dewpoint'
+    elif measurement['type'] == 'humidity':
+        readtype = 'Humidity'
     else:
         return
     myname = entname + ' ' + readtype
@@ -164,21 +191,20 @@ def read_sensors(element, node, configmanager):
         adev = gc.wc.grab_json_response('/api/dev')
         _sensors_by_node[node] = (adev, time.time() + 1)
         sn = _sensors_by_node.get(node, None)
-    if len(sn[0]['data']) != 1:
-        raise Exception('Unable to support multiple pdus at an ip')
-    print(repr(element))
+    dbt = data_by_type(sn[0]['data'])
     readings = []
-    for pduid in sn[0]['data']:
-        datum = sn[0]['data'][pduid]
+    for datatype in dbt:        
+        datum = dbt[datatype]
         process_measurements(name, category, datum['entity'], 'entity', readings)
-        process_measurements(name, category, datum['outlet'], 'outlet', readings)
+        if 'outlet' in datum:
+            process_measurements(name, category, datum['outlet'], 'outlet', readings)
     if justnames:
         for reading in readings:
             yield msg.ChildCollection(simplify_name(reading['name']))
     else:
         yield msg.SensorReadings(readings, name=node)
 
-def get_outlet(node, configmanager):
+def get_outlet(node, configmanager, element):
     gc = GeistClient(node, configmanager)
     state = gc.get_outlet(element[-1])
     return msg.PowerState(node=node, state=state)
@@ -193,7 +219,7 @@ def retrieve(nodes, element, configmanager, inputdata):
     if 'outlets' in element:
         gp = greenpool.GreenPile(pdupool)
         for node in nodes:
-            gp.spawn(get_outlet, node, configmanager)
+            gp.spawn(get_outlet, node, configmanager, element)
         for res in gp:
             yield res
            
@@ -226,5 +252,6 @@ def update(nodes, element, configmanager, inputdata):
         gc = GeistClient(node, configmanager)
         newstate = inputdata.powerstate(node)
         gc.set_outlet(element[-1], newstate)
+    eventlet.sleep(1)
     for res in retrieve(nodes, element, configmanager, inputdata):
         yield res