Draft example of an alternative of setting filename in initial pass

In at least one ARM UEFI, the PXE support is non-compliant,
and absolutely requires the filename field to be populated.

That said, the same firmware fails to let iPXE function, so
we have other issues with this firmware.

Very preliminary testing indicates that PXE compliant firmware
ignores the filename in the offer and proceeds to proxyDHCP.
The non-compliant firmware ignores the PXEClient guidance
and just consumes the filename.

In theory, this just means that the compliant firmware inflicts
a bit of redundant effort on our part, as we must sort out the filename
to offer twice (once for DHCP, then again for proxyDHCP).  It may aid
non compliant firmware to proceed in more limited circumstances.

confluent_client/bin/nodeapply

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+devnull = None
+
+def run():
+    global devnull
+    devnull = open(os.devnull, 'rb')
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-k', '--security', action='store_true',
+                         help='Update SSH setup')
+    argparser.add_option('-F', '--sync', action='store_true',
+                         help='Run the syncfiles associated with the currently completed OS profile on the noderange')
+    argparser.add_option('-P', '--scripts',
+                         help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                    help='Specify a maximum number of '
+                        'nodes to run remote ssh command to, '
+                        'prompting if over the threshold')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    #argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    c.stop_if_noderange_over(args[0], options.maxnodes)
+    nodemap = {}
+    cmdparms = []
+    nodes = []
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
+    
+    cmdstorun = []
+    if options.security:
+        cmdstorun.append(['run_remote', 'setupssh'])
+    if options.sync:
+        cmdstorun.append(['run_remote_python', 'syncfileclient'])
+    if options.scripts:
+        for script in options.scripts.split(','):
+            cmdstorun.append(['run_remote', script])
+    if not cmdstorun:
+        argparser.print_help()
+        sys.exit(1)
+    idxbynode = {}
+    cmdvbase = ['bash', '/etc/confluent/functions']
+    for sshnode in nodes:
+        idxbynode[sshnode] = 1
+        cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(node, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((node, cmdv))        
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if idxbynode[node] < len(cmdstorun):
+                                cmdv =  ['ssh', sshnode] + cmdvbase + cmdstorun[idxbynode[node]]
+                                idxbynode[node] += 1
+                                run_cmdv(node, cmdv, all, pipedesc)
+                            elif pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                line = line.lstrip('\x08')
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodeattrib

@@ -22,6 +22,7 @@ import optparse
 import os
 import signal
 import sys
+import shlex
 
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
@@ -53,6 +54,8 @@ argparser.add_option('-p', '--prompt', action='store_true',
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Prompt if trying to set attributes on more '
                           'than specified number of nodes')
+argparser.add_option('-s', '--set', dest='set', metavar='settings.batch',
+                     default=False, help='set attributes using a batch file')
 (options, args) = argparser.parse_args()
 
 
@@ -109,6 +112,23 @@ elif options.clear or options.environment or options.prompt:
     sys.stderr.write('Attribute names required with specified options\n')
     argparser.print_help()
     exitcode = 400
+
+elif options.set:
+    arglist = [noderange]
+    showtype='current'
+    argfile = open(options.set, 'r')
+    argset = argfile.readline()
+    while argset:
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            arglist += shlex.split(argset)
+        argset = argfile.readline() 
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    exitcode=client.updateattrib(session,arglist,nodetype, noderange, options, None)
 if exitcode != 0:
     sys.exit(exitcode)
 

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -23,6 +23,7 @@ path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
+
 import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
@@ -41,6 +42,10 @@ argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                      help='Enter log replay mode instead of showing a live console')
+
+argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
+                     help= 'Dump log in stdout with timestamps')
+
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -91,6 +96,16 @@ if options.log:
     logreader.replay_to_console(logname)
     sys.exit(0)
 
+if options.Timestamp:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.dump_to_console(logname)
+    sys.exit(0)
+
 def kill(noderange):
     sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
@@ -106,12 +121,20 @@ def kill(noderange):
         nodes.append(node) 
 
     for node in nodes:
-        ps_data=subprocess.Popen(['ps', '-auxww' ], stdout=subprocess.PIPE)
-        wintr=ps_data.communicate()[0]
-        for line in wintr.decode('utf-8').split('\n'):
-            if confettypath in line and envstring in line and node in line:
-                pid_line = [x for x in line.split(' ') if x != '']
-                ps_data=subprocess.Popen(['kill', '-9', pid_line[1] ], stdout=subprocess.PIPE)
+        s=socket.socket(socket.AF_UNIX)
+        winid=None
+        try:
+            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
+            wintr=win.communicate()[0]
+            for line in wintr.decode('utf-8').split('\n'):
+                if 'console: {0}'.format(node) in line or 'confetty' in line:
+                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
+                    winid = win_obj[0]
+        except:
+            print("Error: cannot retrieve window id of node {}".format(node))
+
+        if winid:
+            ps_data=subprocess.Popen(['xkill', '-id', winid ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     sys.exit(0)
 
@@ -220,7 +243,7 @@ if options.windowed:
             elif 'Height' in line:
                 window_height = int(line.split(':')[1])
             elif '-geometry' in line:
-                l = re.split(' |x|\+', line)
+                l = re.split(' |x|-|\+', line)
                 l_nosp = [ele for ele in l if ele.strip()]
                 wmxo = int(l_nosp[1])
                 wmyo = int(l_nosp[2])
@@ -259,7 +282,7 @@ if options.windowed:
         else:
             pass 
         with open(os.devnull, 'wb') as devnull:
-            xopen=subprocess.Popen(envlist + [confettypath, '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
+            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node), '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
     sys.exit(0)
 #end of wcons
 if options.tile:
@@ -273,8 +296,13 @@ if options.tile:
             sys.exit(1)
         nodes.append(node)
     initial = True
+    in_tmux = False
     pane = 0
     sessname = 'nodeconsole_{0}'.format(os.getpid())
+    if os.environ.get("TMUX"):
+        initial = False
+        in_tmux = True
+        subprocess.call(['tmux', 'rename-session', sessname])
     for node in sortutil.natural_sort(nodes):
         panename = '{0}:{1}'.format(sessname, pane)
         if initial:
@@ -295,7 +323,8 @@ if options.tile:
             pane += 1
     subprocess.call(['tmux', 'select-pane', '-t', sessname])
     subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
+    if not in_tmux:
+        os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
-             '/nodes/{0}/console/session'.format(args[0]))
+             '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -15,13 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__author__ = 'jjohnson2,alin37'
+__author__ = 'jjohnson2,alin37,wgrzeda'
 
 import optparse
 import os
 import signal
 import sys
 
+
+
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 except AttributeError:
@@ -39,9 +41,12 @@ def main():
           "   or: %prog [options] noderange <nodeattribute>...")
     argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
+    argparser.add_option('-d', '--delim', metavar="STRING", default = "\n",
+                     help='Delimiter separating the values')
     (options, args) = argparser.parse_args()
     noderange=""
     nodelist=""
+    list = []
     try:
         noderange = args[0]
         nodelist = '/noderange/{0}/nodes/'.format(noderange)
@@ -61,7 +66,9 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print(res['item']['href'].replace('/', ''))
+                elem=(res['item']['href'].replace('/', ''))
+                list.append(elem)
+    print(options.delim.join(list))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodersync

@@ -48,6 +48,8 @@ def run():
                           'prompting if over the threshold')   
     argparser.add_option('-l', '--loginname', type='str',
                          help='Username to use when connecting, defaults to current user.')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for rsync')
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -63,10 +65,20 @@ def run():
     c = client.Command()
     cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
-    if options.loginname:
-        cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath
+    
+    targname = options.substitutename
+    if targname and '{' in targname:
+        targname =  targname + ':'
+    elif targname:
+        targname = '{node}' + targname + ':'
     else:
-        cmdstr += ' {node}:' + targpath
+        targname = '{node}:'
+
+    if options.loginname:
+        cmdstr += ' {}@'.format(options.loginname) + targname + targpath
+    else:
+        cmdstr += ' {}'.format(targname) + targpath
+    
 
     currprocs = 0
     all = set([])

confluent_client/bin/nodesensors

@@ -141,7 +141,7 @@ def sensorpass(showout=True, appendtime=False):
                         if sensedata['value'] is None:
                             showval = ''
                         elif isinstance(sensedata['value'], float):
-                            showval = floatformat(sensedata['value'])
+                            showval = u' {0} '.format(floatformat(sensedata['value']))
                         else:
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):

confluent_client/bin/stats

@@ -16,13 +16,10 @@
 # limitations under the License.
 
 import argparse
+import base64
 import csv
-import fcntl
 import io
 import numpy as np
-
-import os
-import subprocess
 import sys
 
 try:
@@ -35,7 +32,31 @@ except ImportError:
     pass
 
 
-def plot(gui, output, plotdata, bins):
+def iterm_draw(data):
+    databuf = data.getbuffer()
+    datalen = len(databuf)
+    data = base64.b64encode(databuf).decode('utf8')
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;size={}:'.format(datalen))
+    sys.stdout.write(data)
+    sys.stdout.write('\a')
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+
+def kitty_draw(data):
+    data = base64.b64encode(data.getbuffer())
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+    sys.stdout.write('\n')
+
+
+def plot(gui, output, plotdata, bins, fmt):
     import matplotlib as mpl
     if gui and mpl.get_backend() == 'agg':
         sys.stderr.write('Error: No GUI backend available and -g specified!\n')
@@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
-        writer = DumbWriter()
-        writer.draw(tdata)
+        if fmt == 'sixel':
+            writer = DumbWriter()
+            writer.draw(tdata)
+        elif fmt == 'kitty':
+            kitty_draw(tdata)
+        elif fmt == 'iterm':
+            iterm_draw(tdata)
     return n, bins
 
 def textplot(plotdata, bins):
@@ -81,7 +107,8 @@ histogram = False
 aparser = argparse.ArgumentParser(description='Quick access to common statistics')
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
-aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
+aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
@@ -138,7 +165,7 @@ while data:
     data = list(csv.reader([data], delimiter=delimiter))[0]
 n = None
 if args.g or args.o or args.x:
-    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f)
 if args.t:
     n, bins = textplot(plotdata, bins=args.b)
 print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))

confluent_client/confluent/logreader.py

@@ -296,5 +296,47 @@ def replay_to_console(txtfile):
             sys.exit(1)
     _replay_to_console(txtfile, binfile)
 
+def dump_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    elif '.' not in txtfile:
+        if '/' not in txtfile:
+            txtfile = os.getcwd() + '/' + txtfile
+        sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(txtfile + '.cbl'))
+        sys.exit(1)
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(binfile))
+            sys.exit(1)
+    replay = LogReplay(txtfile, binfile)
+    quitit = False
+    writeout('\x1b[2J\x1b[;H')
+    prepend = ''
+    try:
+        while not quitit:
+            newdata, delay = replay.get_output(False)
+            if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
+                writeout(newdata.replace(b'\r\n', '\r\n[ {0} ] '.format(time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))).encode('utf8')))
+                newdata = ''
+                try:
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                except IOError:
+                    pass
+            else:
+                break
+    except Exception:
+        writeout('\x1b[m\x1b[?25h\n')
+        raise
+    writeout('\x1b[m\x1b[?25h\n')
+
 if __name__ == '__main__':
     replay_to_console(sys.argv[1])

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -7,7 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
-`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`
+`nodeattrib <noderange> -s <attributes.batch> ...` 
 
 ## DESCRIPTION
 
@@ -53,12 +54,18 @@ to a blank value will allow masking a group defined attribute with an empty valu
 * `-e`, `--environment`:
   Set specified attributes based on exported environment variable of matching name. 
   Environment variable names may be lower case or all upper case. 
-  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE)
   
 * `-p`, `--prompt`:
   Request interactive prompting to provide values rather than the command line
   or environment variables.
-  
+
+* `-s`, `--set`:
+  Set attributes using a batch file rather than the command line. The attributes in the batch file 
+  can be specified as one line of key=value pairs simmilar to command line or each attribute can
+  be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch
+  file syntax.   
+ 
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Prompt if trying to set attributes on more than
   specified number of nodes.
@@ -116,6 +123,25 @@ to a blank value will allow masking a group defined attribute with an empty valu
     `d1: net.pxe.switch: pxeswitch1`  
     `d1: net.switch:`  
 
+* Setting attributes using a batch file with syntax similar to command line:
+    `# cat nodeattributes.batch`  
+    `# power`
+    `power.psu1.outlet=3 power.psu1.pdu=pdu2`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: 3`
+    `n41: pdu2`
+
+* Setting attributes using a batch file with syntax where each attribute is in its own line:
+    `# cat nodeattributes.batch`  
+    `# management`
+    `custom.mgt.switch=switch_main`
+    `custom.mgt.switch.port=swp4`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: switch_main`
+    `n41: swp4`
+
+  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeconsole.ronn

@@ -29,6 +29,11 @@ console process which will result in the console window closing.
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+
+  * `-T`, `--Timestamp`:
+ Dump the log with Timpstamps on the current, local log in /var/log/confluent/consoles.
+  If in collective mode, this only makes sense to use on the current collective
+  manager at this time.
   
 * `-w`, `--windowed`:
   Open terminal windows for each node.  The

confluent_client/doc/man/nodelist.ronn

@@ -4,7 +4,7 @@ nodelist(8) -- List confluent nodes and their attributes
 ## SYNOPSIS
 
 `nodelist <noderange>`  
-`nodelist <noderange> [-b] <nodeattribute>...`  
+`nodelist <noderange> [-b] [-d] {string} <nodeattribute>...`  
 
 ## DESCRIPTION
 
@@ -24,7 +24,8 @@ all attributes that begin with `net.` and end with `switch`.
 
 * `-b`, `--blame`:
   Annotate inherited and expression based attributes to show their base value.
-
+* `-d`, `--delim`:
+  Choose a delimiter to separat the values. Default - ENTER.
 ## EXAMPLES
 * Listing matching nodes of a simple noderange:
   `# nodelist n1-n4`  

confluent_client/doc/man/nodersync.ronn

@@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes.
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 
+
+* `-s`, `--substitutename`:
+  'Use a different name other than the nodename for rsync'
   
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to run rsync to, prompting if over the

confluent_osdeploy/buildrpm

@@ -11,13 +11,24 @@ mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
 cd -
 mkdir -p el9bin/opt/confluent/bin
 mkdir -p el9bin/stateless-bin
-podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuilder make -C /buildutils
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuild make -C /buildutils
 cd utils
-mv confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
-mv start_root urlmount ../el9bin/stateless-bin/
+cp confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
+cp start_root urlmount ../el9bin/stateless-bin/
+make clean
+cd ..
+mkdir -p el8bin/opt/confluent/bin
+mkdir -p el8bin/stateless-bin
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t el7build make -C /buildutils
+cd utils
+cp confluent_imginfo copernicus clortho autocons ../el8bin/opt/confluent/bin
+cp start_root urlmount ../el8bin/stateless-bin/
 make clean
 cd ..
 tar Jcvf confluent_el9bin.tar.xz el9bin/
+tar Jcvf confluent_el8bin.tar.xz el8bin/
+mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
+rm -rf el8bin
 rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/buildrpm-aarch64

@@ -1,3 +1,4 @@
+cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
@@ -29,4 +30,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
 rm -rf el8bin
-rpmbuild -ba confluent_osdeploy-aarch64.spec
+podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec
+

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -21,6 +21,10 @@ class InvalidApiKey(Exception):
 def msg_align(len):
     return (len + 3) & ~3
 
+try:
+    _protoparm = ssl.PROTOCOL_TLS_CLIENT
+except AttributeError:    
+    _protoparm = ssl.PROTOCOL_SSLv23
 cryptname = ctypes.util.find_library('crypt')
 if not cryptname:
     if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
@@ -223,7 +227,7 @@ def get_apikey(nodename, hosts, errout=None):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None):
+    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False):
         self.phmac = phmac
         self.errout = None
         if errout:
@@ -287,7 +291,7 @@ class HTTPSClient(client.HTTPConnection, object):
         if self.phmac:
             with open(phmac, 'r') as hmacin:
                 self.stdheaders['CONFLUENT_CRYPTHMAC'] = hmacin.read()
-        else:
+        elif not checkonly:
             self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
@@ -305,7 +309,7 @@ class HTTPSClient(client.HTTPConnection, object):
     def check_connections(self):
         foundsrv = None
         hosts = self.hosts
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         ctx.verify_mode = ssl.CERT_REQUIRED
         ctx.check_hostname = True
@@ -321,7 +325,7 @@ class HTTPSClient(client.HTTPConnection, object):
                     foundsrv = host
                     psock.close()
                     break
-                except OSError:
+                except (OSError, socket.error):
                     continue
                 except ssl.SSLError:
                     continue
@@ -353,11 +357,12 @@ class HTTPSClient(client.HTTPConnection, object):
                 raise Exception('Unable to reach any hosts')
         return foundsrv
 
-    def connect(self):
+    def connect(self, tries=3):
         addrinf = socket.getaddrinfo(self.host, self.port)[0]
         psock = socket.socket(addrinf[0])
+        psock.settimeout(15)
         psock.connect(addrinf[4])
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         host = self.host.split('%', 1)[0]
         if '[' not in host and ':' in host:
@@ -375,6 +380,10 @@ class HTTPSClient(client.HTTPConnection, object):
                 self.errout.write(errmsg)
                 self.errout.flush()
             sys.exit(1)
+        except socket.timeout:
+            if not tries:
+                raise
+            return self.connect(tries=tries-1)
 
     def grab_url(self, url, data=None, returnrsp=False):
         return self.grab_url_with_status(url, data, returnrsp)[1]
@@ -459,7 +468,7 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    client = HTTPSClient(usejson, errout=errout, phmac=phmac)
+    client = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
     if waitfor:
         status = 201
         while status != waitfor:

confluent_osdeploy/common/profile/scripts/confignet

@@ -8,6 +8,10 @@ import sys
 import time
 import shlex
 import subprocess
+try:
+    import yaml
+except ImportError:
+    pass
 try:
     from importlib.machinery import SourceFileLoader
     def load_source(mod, path):
@@ -107,6 +111,104 @@ def get_interface_name(iname, settings):
         return iname
     return None
 
+class NetplanManager(object):
+    def __init__(self, deploycfg):
+        self.cfgbydev = {}
+        self.read_connections()
+        self.deploycfg = deploycfg
+
+    def read_connections(self):
+        for plan in glob.glob('/etc/netplan/*.y*ml'):
+            with open(plan) as planfile:
+                planinfo = yaml.safe_load(planfile)
+                if not planinfo:
+                    continue
+            nicinfo = planinfo.get('network', {}).get('ethernets', {})
+            for devname in nicinfo:
+                if devname == 'lo':
+                    continue
+                if 'gateway4' in nicinfo[devname]:
+                    # normalize deprecated syntax on read in
+                    gw4 = nicinfo[devname]['gateway4']
+                    del nicinfo[devname]['gateway4']
+                    routeinfo = nicinfo[devname].get('routes', [])
+                    for ri in routeinfo:
+                        if ri.get('via', None) == gw4 and ri.get('to', None) in ('default', '0.0.0.0/0', '0/0'):
+                            break
+                    else:
+                        routeinfo.append({
+                            'to': 'default',
+                            'via': gw4
+                        })
+                    nicinfo[devname]['routes'] = routeinfo
+                self.cfgbydev[devname] = nicinfo[devname]
+    
+    def apply_configuration(self, cfg):
+        devnames = cfg['interfaces']
+        if len(devnames) != 1:
+            raise Exception('Multi-nic team/bonds not yet supported')
+        stgs = cfg['settings']
+        needcfgapply = False
+        for devname in devnames:
+            needcfgwrite = False
+            # ipv6_method missing at uconn...
+            if stgs.get('ipv6_method', None) == 'static':
+                curraddr = stgs['ipv6_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            if stgs.get('ipv4_method', None) == 'static':
+                curraddr = stgs['ipv4_address']
+                currips = self.getcfgarrpath([devname, 'addresses'])
+                if curraddr not in currips:
+                    needcfgwrite = True
+                    currips.append(curraddr)
+            gws = []
+            gws.append(stgs.get('ipv4_gateway', None))
+            gws.append(stgs.get('ipv6_gateway', None))
+            for gwaddr in gws:
+                if gwaddr:
+                    cfgroutes = self.getcfgarrpath([devname, 'routes'])
+                    for rinfo in cfgroutes:
+                        if rinfo.get('via', None) == gwaddr:
+                            break
+                    else:
+                        needcfgwrite = True
+                        cfgroutes.append({'via': gwaddr, 'to': 'default'})
+            dnsips = self.deploycfg.get('nameservers', [])
+            dnsdomain = self.deploycfg.get('dnsdomain', '')
+            if dnsips:
+                currdnsips = self.getcfgarrpath([devname, 'nameservers', 'addresses'])
+                for dnsip in dnsips:
+                    if dnsip and dnsip not in currdnsips:
+                        needcfgwrite = True
+                        currdnsips.append(dnsip)
+            if dnsdomain:
+                currdnsdomain = self.getcfgarrpath([devname, 'nameservers', 'search'])
+                if dnsdomain not in currdnsdomain:
+                    needcfgwrite = True
+                    currdnsdomain.append(dnsdomain)
+            if needcfgwrite:
+                needcfgapply = True
+                newcfg = {'network': {'version': 2, 'ethernets': {devname: self.cfgbydev[devname]}}}
+                with open('/etc/netplan/{0}-confluentcfg.yaml'.format(devname), 'w') as planout:
+                    planout.write(yaml.dump(newcfg))
+        if needcfgapply:
+            subprocess.call(['netplan', 'apply'])
+
+    def getcfgarrpath(self, devpath):
+        currptr = self.cfgbydev
+        for k in devpath[:-1]:
+            if k not in currptr:
+                currptr[k] = {}
+            currptr = currptr[k]
+        if devpath[-1] not in currptr:
+            currptr[devpath[-1]] = []
+        return currptr[devpath[-1]]
+        
+
+
 class WickedManager(object):
     def __init__(self):
         self.teamidx = 0
@@ -243,7 +345,7 @@ class NetworkManager(object):
                     bondcfg[stg] = deats[stg]
         if member in self.uuidbyname:
             subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
-        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
         if bondcfg:
             args = []
             for parm in bondcfg:
@@ -253,6 +355,7 @@ class NetworkManager(object):
 
     def apply_configuration(self, cfg):
         cmdargs = {}
+        cmdargs['connection.autoconnect'] = 'yes'
         stgs = cfg['settings']
         cmdargs['ipv6.method'] = stgs.get('ipv6_method', 'link-local')
         if stgs.get('ipv6_address', None):
@@ -276,7 +379,7 @@ class NetworkManager(object):
             for arg in cmdargs:
                 cargs.append(arg)
                 cargs.append(cmdargs[arg])
-            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team', 'con-name', cname, 'connection.interface-name', cname, 'team.runner', stgs['team_mode']] + cargs)
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'bond', 'con-name', cname, 'connection.interface-name', cname, 'bond.options', 'mode={}'.format(stgs['team_mode'])] + cargs)
             for iface in cfg['interfaces']:
                 self.add_team_member(cname, iface)
             subprocess.check_call(['nmcli', 'c', 'u', cname])
@@ -315,8 +418,12 @@ if __name__ == '__main__':
     myaddrs = apiclient.get_my_addresses()
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
+    dc = None
     for srv in srvs:
-        s = socket.create_connection((srv, 443))
+        try:
+            s = socket.create_connection((srv, 443))
+        except socket.error:
+            continue
         myname = s.getsockname()
         s.close()
         if len(myname) == 4:
@@ -329,8 +436,27 @@ if __name__ == '__main__':
                     curridx = addr[-1]
         if curridx in doneidxs:
             continue
-        status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+        for tries in (1, 2, 3):
+            try:
+                status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+                break
+            except Exception:
+                if tries == 3:
+                    raise
+                time.sleep(1)
+                continue
         nc = json.loads(nc)
+        if not dc:
+            for tries in (1, 2, 3):
+                try:
+                    status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+                    break
+                except Exception:
+                    if tries == 3:
+                        raise
+                    time.sleep(1)
+                    continue
+            dc = json.loads(dc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:
             for iname in iname.split(','):
@@ -356,6 +482,8 @@ if __name__ == '__main__':
         if not netname_to_interfaces['default']['interfaces']:
             del netname_to_interfaces['default']
     rm_tmp_llas(tmpllas)
+    if os.path.exists('/usr/sbin/netplan'):
+        nm = NetplanManager(dc)
     if os.path.exists('/usr/bin/nmcli'):
         nm = NetworkManager(devtypes)
     elif os.path.exists('/usr/sbin/wicked'):

confluent_osdeploy/common/profile/scripts/setupssh

@@ -0,0 +1,31 @@
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    rm $certfile
+    confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
+done
+TMPDIR=$(mktemp -d)
+cd $TMPDIR
+confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz
+tar xf initramfs.tgz
+for ca in ssh/*.ca; do
+	LINE=$(cat $ca)
+	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
+	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
+done
+for pubkey in ssh/*.*pubkey; do
+	LINE=$(cat $pubkey)
+	cp -af /root/.ssh/authorized_keys /root/.ssh/authorized_keys.new
+	grep -v "$LINE" /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.new
+	echo "$LINE" >> /root/.ssh/authorized_keys.new
+	mv /root/.ssh/authorized_keys.new /root/.ssh/authorized_keys
+done
+confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' > /etc/ssh/shosts.equiv
+cat /etc/ssh/shosts.equiv > /root/.shosts
+cd -
+rm -rf $TMPDIR

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -7,6 +7,7 @@ License: Apache2
 URL: https://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
 Source1: confluent_el9bin.tar.xz
+Source2: confluent_el8bin.tar.xz
 BuildArch: noarch
 Requires: confluent_ipxe mtools tar
 BuildRoot: /tmp
@@ -18,24 +19,22 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 %define debug_package %{nil}
 
 %prep
-%setup -n confluent_osdeploy -a 1
+%setup -n confluent_osdeploy -a 2 -a 1
 
 %build
-mkdir -p opt/confluent/bin
-mkdir -p stateless-bin
-cd utils
-make all
-cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
-cp start_root urlmount ../stateless-bin/
-cd ..
+#cd utils
+#make all
+#cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
+#cp start_root urlmount ../stateless-bin/
+#cd ..
 ln -s el8 el9
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreos el9; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}/initramfs/* .
     cp -a ../common/initramfs/* .
@@ -43,20 +42,20 @@ for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
     mv ../addons.cpio .
     cd ..
 done
-for os in el7 el8 suse15 el9 ubuntu20.04; do
+for os in el7 el8 suse15 el9 ubuntu20.04 ubuntu22.04; do
     mkdir ${os}disklessout
     cd ${os}disklessout
     if [ -d ../${os}bin ]; then
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}-diskless/initramfs/* .
     cp -a ../common/initramfs/* .
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/stateless-bin/* opt/confluent/bin
     else
-        cp -a ../stateless-bin/* opt/confluent/bin
+        cp -a ../el8bin/stateless-bin/* opt/confluent/bin
     fi
     find . | cpio -H newc -o > ../addons.cpio
     mv ../addons.cpio .
@@ -64,7 +63,7 @@ for os in el7 el8 suse15 el9 ubuntu20.04; do
 done
 mkdir esxi7out
 cd esxi7out
-cp -a ../opt .
+cp -a ../el8bin/opt .
 cp -a ../esxi7/initramfs/* .
 cp -a ../common/initramfs/* .
 chmod +x bin/* opt/confluent/bin/*
@@ -79,7 +78,7 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/debian/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -69,7 +69,7 @@ if [ "$v4cfgmeth" = "static" ]; then
     for namesrv in "$namesrvs"; do
         setdebopt netcfg/get_nameservers $namesrv string
     done
-elif [ "$vpcfgmeth" = "dhcp" ]; then
+elif [ "$v4cfgmeth" = "dhcp" ]; then
     setdebopt netcfg/disable_dhcp false boolean
     setdebopt netcfg/confirm_static false boolean
 fi

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,13 @@ import struct
 import sys
 import subprocess
 
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +60,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -166,6 +176,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -224,7 +236,8 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    # do this safer, unit s print might bomb
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +271,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk , volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +281,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -125,4 +125,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -7,41 +7,127 @@ if [ -f /tmp/dd_disk ]; then
         fi
     done
 fi
-mkdir -p /etc/confluent
-cat /tls/*.pem > /etc/confluent/ca.pem
-TRIES=0
-echo -n > /etc/confluent/confluent.info
-cd /sys/class/net
-while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
-echo -n "" > /tmp/confluent.initq
-# restart cmdline
-echo -n "" > /etc/cmdline.d/01-confluent.conf
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-#TODO: blkid --label <whatever> to find mounted api
-
+shutdownnic=""
 oum=$(umask)
-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-ifidx=$(cat /tmp/confluent.ifidx)
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-echo $ifname > /tmp/net.ifaces
+umask 0077
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
 
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/hmackeyfile
+    passfile=/etc/confluent/confluent.apikey
+    passcrypt=/tmp/passcrypt
+    hmacfile=/tmp/hmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile;
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo "NODENAME: "$nodename > /etc/confluent/confluent.info
+    for dsrv in $deploysrvs; do
+        echo 'MANAGER: '$dsrv >> /etc/confluent/confluent.info
+    done
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    sleep 10
+    usedhcp=0
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            usedhcp=1
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            TESTSRV=$(python /opt/confluent/bin/apiclient -c 2> /dev/null)
+            if [ ! -z "$TESTSRV" ]; then
+                python /opt/confluent/bin/apiclient -p $hmacfile /confluent-api/self/registerapikey $passcrypt
+                mgr=$TESTSRV
+                ifname=$NICGUESS
+                shutdownnic=$ifname
+                break
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip -4 addr flush dev $NICGUESS
+        fi
+    done
+fi
+TRIES=0
+if [ "$usedhcp" = 1 ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+elif [ -z "$ifname" ]; then
+    cd /sys/class/net
+    while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    cd /
+    grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+    echo -n "" > /etc/cmdline.d/01-confluent.conf
+else
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none > /etc/cmdline.d/01-confluent.conf
+fi
+python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+if [ ! -z "$shutdownnic" ]; then
+    if [ ! -z "$v4gw" ]; then
+        ip route del default via $v4gw
+    fi
+    ip -4 addr flush dev $shutdownnic
+fi
+# restart cmdline
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+echo -n "" > /tmp/confluent.initq
+if [ -z "$ifname"  ]; then
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+if [ ! -z "$ifname" ]; then
+    echo $ifname > /tmp/net.ifaces
+fi
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 hostname=$nodename
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
     hostname=$hostname.$dnsdomain
 fi
-mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
-mgr=${mgr#deploy_server: }
+if [ -z "$mgr" ]; then
+    mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+    mgr=${mgr#deploy_server: }
+fi
 profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 profilename=${profilename#profile: }
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
@@ -65,21 +151,23 @@ kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
 root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
 export kickstart
 export root
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
-autoconfigmethod=${autoconfigmethod#ipv4_method: }
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
+if [ -z "$autoconfigmethod" ]; then
+    autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+    else
+        v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
     fi
-    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#ipv4_netmask: }
-    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
 nameserversec=0
 while read -r entry; do

confluent_osdeploy/el7/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -189,8 +189,15 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+	linktype="infiniband"
+else
+	linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
+
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -199,9 +206,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,14 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +61,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +136,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +152,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -166,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if 'nvme' in targblock and targblock[-1] == 'p':
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -184,6 +228,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +269,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
@@ -366,6 +411,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +426,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
 . /lib/dracut-lib.sh
+setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' &
 udevadm trigger
 udevadm trigger --type=devices --action=add
 udevadm settle
@@ -40,7 +41,7 @@ mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
 TRIES=5
-while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 2
     TRIES=$((TRIES - 1))
 done
@@ -55,21 +56,49 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
     autoconfigmethod=$(grep ^ipv4_method: $tcfg)
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    if [ "$autoconfigmethod" = "dhcp" ]; then
-        /usr/libexec/nm-initrd-generator ip=:dhcp
-    else
-        v4addr=$(grep ^ipv4_address: $tcfg)
-        v4addr=${v4addr#ipv4_address: }
-        v4addr=${v4addr%/*}
-        v4gw=$(grep ^ipv4_gateway: $tcfg)
-        v4gw=${v4gw#ipv4_gateway: }
-        if [ "$v4gw" = "null" ]; then
-            v4gw=""
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            DETECTED=0
+            for dsrv in $deploysrvs; do
+                if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then
+                    rm /run/NetworkManager/system-connections/*
+                    /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none
+                    DETECTED=1
+                    ifname=$NICGUESS
+                    break
+                fi
+            done
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip addr flush dev $NICGUESS
+            if [ $DETECTED = 1 ]; then
+                break
+            fi
         fi
-        v4nm=$(grep ipv4_netmask: $tcfg)
-        v4nm=${v4nm#ipv4_netmask: }
-        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
-    fi
+    done
     NetworkManager --configure-and-quit=initrd --no-daemon
     hmackeyfile=/tmp/cnflnthmackeytmp
     echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
@@ -85,10 +114,10 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
     done
     for deployer in $deploysrvs; do
-        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+        if curl --capath /tls/ -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
             cp $passfile /etc/confluent/confluent.apikey
             confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+            curl --capath /tls/ -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
             confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
             confluent_profile=${confluent_profile#profile: }
             mgr=$deployer
@@ -114,7 +143,7 @@ if ! grep MANAGER: /etc/confluent/confluent.info; then
         for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
             myids=$myids"/mac="$mac
         done
-        myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+        myname=$(curl --capath /tls/ -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
         if [ ! -z "$myname" ]; then
             echo NODENAME: $myname > /etc/confluent/confluent.info
             echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
@@ -149,14 +178,16 @@ while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self
 	sleep 10
 done
 ifidx=$(cat /tmp/confluent.ifidx 2> /dev/null)
-if [ ! -z "$ifidx" ]; then
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-    ifname=${ifname%@*}
-    echo $ifname > /tmp/net.ifaces
-else
-    ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
-    ifname=$(cat /tmp/net.ifaces)
+if [ -z "$ifname" ]; then
+    if [ ! -z "$ifidx" ]; then
+        ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+        ifname=${ifname%:}
+        ifname=${ifname%@*}
+        echo $ifname > /tmp/net.ifaces
+    else
+        ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
+        ifname=$(cat /tmp/net.ifaces)
+    fi
 fi
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)

confluent_osdeploy/el8/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/kickstart

@@ -33,15 +33,7 @@
 reboot
 
 %packages
-@^minimal-environment
-#-kernel-uek # This can opt out of the UEK for the relevant distribution
-bind-utils
-chrony
-pciutils
-python3
-rsync
-tar
--iwl*-firmware
+%include /tmp/pkglist
 %include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end

confluent_osdeploy/el8/profiles/default/packagelist

@@ -0,0 +1,9 @@
+@^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
+bind-utils
+chrony
+pciutils
+python3
+rsync
+tar
+-iwl*-firmware

confluent_osdeploy/el8/profiles/default/partitioning

@@ -0,0 +1,4 @@
+clearpart --all --initlabel
+ignoredisk --only-use %%INSTALLDISK%%
+autopart --nohome %%LUKSHOOK%%
+

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -87,6 +87,7 @@ done
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 touch /tmp/cryptpkglist
+touch /tmp/pkglist
 touch /tmp/addonpackages
 if [ "$cryptboot" == "tpm2" ]; then
 	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
@@ -102,15 +103,18 @@ confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_pro
 run_remote pre.custom
 run_remote_parts pre.d
 confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart -o /tmp/kickstart.base
+if grep '^%include /tmp/pkglist' /tmp/kickstart.* > /dev/null; then
+    confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/packagelist -o /tmp/pkglist
+fi
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || touch /tmp/installdisk
 if [ ! -e /tmp/installdisk ]; then
     run_remote_python getinstalldisk
 fi
+confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/partitioning -o /tmp/partitioning.template
 grep '^%include /tmp/partitioning' /tmp/kickstart.* > /dev/null || rm /tmp/installdisk
 if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
-    echo clearpart --all --initlabel >> /tmp/partitioning
-    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
-    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+    INSTALLDISK=$(cat /tmp/installdisk)
+    sed -e s/%%INSTALLDISK%%/$INSTALLDISK/ -e s/%%LUKSHOOK%%/$LUKSPARTY/ /tmp/partitioning.template > /tmp/partitioning
     dd if=/dev/zero of=/dev/$(cat /tmp/installdisk) bs=1M count=1 >& /dev/null
     vgchange -a n >& /dev/null
 fi

confluent_osdeploy/el9-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -154,8 +154,14 @@ cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
 EOC
 echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
 echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+linktype=$(ip link show dev ${ifname}|grep link/|awk '{print $1}')
+if [ "$linktype" = link/infiniband ]; then
+        linktype="infiniband"
+else
+        linktype="ethernet"
+fi
+echo type=$linktype >> /run/NetworkManager/system-connections/$ifname.nmconnection
 cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
-type=ethernet
 autoconnect-retries=1
 EOC
 echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
@@ -164,9 +170,6 @@ multi-connect=1
 permissions=
 wait-device-timeout=60000
 
-[ethernet]
-mac-address-blacklist=
-
 EOC
 autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 auto6configmethod=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,14 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +61,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +136,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +152,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -166,6 +208,8 @@ def fixup(rootdir, vols):
             partnum = re.search('(\d+)$', targdev).group(1)
             targblock = re.search('(.*)\d+$', targdev).group(1)
     if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
         shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
         shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
         subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
@@ -184,6 +228,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +269,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -258,7 +303,7 @@ def install_to_disk(imgpath):
         if end > sectors:
             end = sectors
         parted.run('mkpart primary {}s {}s'.format(curroffset, end))
-        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        vol['targetdisk'] = get_partname(instdisk, volidx)
         curroffset += size + 1
     if not lvmvols:
         if swapsize:
@@ -268,10 +313,10 @@ def install_to_disk(imgpath):
             if end > sectors:
                 end = sectors
             parted.run('mkpart swap {}s {}s'.format(curroffset, end))
-            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
     else:
         parted.run('mkpart lvm {}s 100%'.format(curroffset))
-        lvmpart = instdisk + '{}'.format(volidx + 1)
+        lvmpart = get_partname(instdisk, volidx + 1)
         subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
         subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
         vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
@@ -366,6 +411,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +426,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -94,6 +94,7 @@ while ! grep NODENAME /etc/confluent/confluent.info; do
     fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+cat /tls/*.pem > /etc/confluent/ca.pem
 mgr=$(python /opt/confluent/bin/apiclient -c)
 APIKEY=$(cat /etc/confluent/confluent.apikey)
 
@@ -108,7 +109,6 @@ while [ -z "$APIKEY" ]; do
     (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
     APIKEY=$(cat /etc/confluent/confluent.apikey)
 done
-cat /tls/*.pem > /etc/confluent/ca.pem
 /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -174,6 +174,8 @@ dnsdomain=${dnsdomain#dnsdomain: }
 echo search $dnsdomain >> /etc/resolv.conf
 echo -n "Initializing ssh..."
 ssh-keygen -A
+mkdir -p /usr/share/empty.sshd
+rm /etc/ssh/ssh_host_dsa_key*
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}

confluent_osdeploy/genesis/profiles/default/ansible/onboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -161,7 +161,7 @@ else
     echo -e "BOOTPROTO='static'\nSTARTMODE='auto'" >> /run/confluent/ifcfg-$ifname
     echo "IPADDR='$v4addr/$v4nm'" >> /run/confluent/ifcfg-$ifname
     if [ ! -z "$v4gw" ]; then
-        echo defafult $v4gw - $ifname > /run/confluent/ifroute-$ifname
+        echo default $v4gw - $ifname > /run/confluent/ifroute-$ifname
     fi
 fi
 

confluent_osdeploy/suse15-diskless/profiles/default/scripts/imageboot.sh

@@ -138,4 +138,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/suse15/profiles/hpc/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -0,0 +1,205 @@
+reverse_uuid() {
+    echo $1 | head -c 8 | tail -c 2
+    echo $1 | head -c 6 | tail -c 2
+    echo $1 | head -c 4 | tail -c 2
+    echo $1 | head -c 2 | tail -c 2
+    echo $1 | head -c 13 | tail -c 2
+    echo $1 | head -c 11 | tail -c 2
+    echo $1 | head -c 18 | tail -c 2
+    echo $1 | head -c 16 | tail -c 2
+    echo $1 | tail -c 18 | sed -e s/-//
+}
+setdebopt() {
+    debconf-set $1 $2
+    echo d-i $1 $3 $2 >> /preseed.cfg
+}
+
+mkdir -p /etc/confluent
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) up
+done
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) down
+    udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+    ip link set $(basename $i) up
+done
+cp -a /tls/* /etc/ssl/certs/
+mkdir -p /etc/confluent
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|cut -f 2 -d ' ')
+    echo NODENAME: $nodename > /etc/confluent/confluent.info
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: $tcfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress ${v4addr%/*} string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            for dsrv in $deploysrvs; do
+                if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    setdebopt netcfg/choose_interface $NIC select
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done 
+        #TODO: nameservers
+    elif [ "$v4cfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            udhcpc $NICGUESS
+        done
+        for dsrv in $deploysrvs; do
+            if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                deploysrvs=$dsrv
+            fi
+        done
+    fi
+    mgr=$deploysrvs
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|cut -d ' ' -f 2) > $hmackeyfile
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" --post-file=$passcrypt https://$mgr/confluent-api/self/registerapikey -O - --quiet
+    cp $passfile /etc/confluent/confluent.apikey
+    nic=$NIC
+else
+    dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
+    dhcpid=$(mktemp)
+    mkdir -p /etc/confluent
+    cp /tls/* /etc/ssl/certs/
+    cat /tls/*.pem >> /etc/confluent/ca.pem
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
+        ip link set $nic up
+    done
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+        if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+            /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+            if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+                break
+            fi
+        fi
+        ip -4 flush dev $nic
+    done
+    mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    /opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+fi
+apikey=$(cat /etc/confluent/confluent.apikey)
+cd /etc/confluent
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg
+cd -
+predir=$(mktemp -d)
+cd $predir
+cp /etc/confluent/deploycfg /etc/confluent/confluent.deploycfg
+profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for namesrv in "$namesrvs"; do
+    setdebopt netcfg/get_nameservers $namesrv string
+done
+rootpass=$(grep ^rootpassword: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //')
+if [ "$rootpass" = null ] || [ -z "$rootpass" ]; then
+    setdebopt passwd/root-login false boolean
+else
+    setdebopt passwd/root-login true boolean
+    setdebopt passwd/root-password-crypted $rootpass string
+fi
+setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //') string
+ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for ntpsrv in "$ntpsrvs"; do
+    setdebopt clock-setup/ntp true boolean
+    setdebopt clock-setup/ntp-server $ntpsrv string
+done
+setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select
+domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+if [ ! -z "$domainname" ] && [ "$domainname" != "null" ]; then
+    setdebopt netcfg/get_domain $domainname string
+fi
+
+
+
+
+wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
+chmod u+x pre.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh
+chmod u+x prechroot.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/post.sh
+chmod u+x post.sh
+wget https://$mgr/confluent-public/os/$profile/preseed.cfg
+cat preseed.cfg >> /preseed.cfg
+echo $mgr > /etc/confluent/deployer
+setdebopt auto-install/enable true boolean
+setdebopt partman/early_command  $predir/pre.sh string
+setdebopt preseed/late_command $predir/prechroot.sh string
+mv $predir/post.sh /tmp/
+cd -
+ip -4 a flush dev $nic
+setdebopt netcfg/choose_interface $nic select
+setdebopt netcfg/get_hostname $nodename string
+setdebopt mirror/protocol https string
+setdebopt mirror/country manual string
+setdebopt mirror/https/hostname $mgr string
+setdebopt mirror/https/directory /confluent-public/os/$profile/distribution string
+setdebopt mirror/protocol https string
+setdebopt mirror/https/proxy "" string
+setdebopt apt-setup/security_host $mgr string
+if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
+    if [ "$v4cfgmeth" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress $v4addr string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+        for namesrv in "$namesrvs"; do
+            setdebopt netcfg/get_nameservers $namesrv string
+        done
+    elif [ "$vpcfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+    fi
+fi

confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/linux $2/boot/kernel && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/initrd.gz $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/* $2/boot/efi/boot
+

confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg

@@ -0,0 +1,21 @@
+d-i anna/choose_modules string openssh-server-udeb
+d-i partman-auto/method string regular
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-auto/expert_recipe_file string /tmp/partitionfile
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i passwd/make-user boolean false
+d-i clock-setup/utc boolean true
+d-i apt-setup/multiverse boolean false
+d-i apt-setup/universe boolean false
+d-i apt-setup/backports boolean false
+d-i apt-setup/updates boolean false
+d-i grub-installer/only_debian boolean true
+tasksel tasksel/first multiselect standard
+d-i pkgsel/include string openssh-server curl
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean false
+d-i finish-install/reboot_in_progress note

confluent_osdeploy/ubuntu18.04/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+#cp -a /etc/confluent/ssh/* /etc/ssh/
+#systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+source /etc/confluent/functions
+
+run_remote_parts firstboot.d
+run_remote_config firstboot.d
+systemctl disable firstboot
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions

@@ -0,0 +1,208 @@
+#!/bin/bash
+function test_mgr() {
+    whost=$1
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+mkdir -p /run/sshd
+mkdir -p /root/.ssh
+cat /tmp/ssh/*pubkey >> /root/.ssh/authorized_keys
+cat /tmp/ssh/*.ca | sed -e s/^/'@cert-authority * '/ >> /etc/ssh/ssh_known_hosts
+chmod 700 /etc/confluent
+chmod go-rwx /etc/confluent/*
+sshconf=/etc/ssh/ssh_config
+if [ -d /etc/ssh/ssh_config.d/ ]; then
+    sshconf=/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+mkdir -p /opt/confluent/bin
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+systemctl enable firstboot
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+source /etc/confluent/functions
+python3 /opt/confluent/bin/apiclient /confluent-api/self/nodelist | sed -e s/'^- //' > /tmp/allnodes
+cp /tmp/allnodes /root/.shosts
+cp /tmp/allnodes /etc/ssh/shosts.equiv
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ')
+    sed -i "s/#NTP=/NTP=$ntps/" /etc/systemd/timesyncd.conf
+fi
+textcons=$(grep ^textconsole: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /tmp/autocons.info ]; then
+        cons=$(cat /tmp/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /etc/default/grub
+        updategrub=1
+    fi
+fi
+kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
+fi
+
+if [ 1 = $updategrub ]; then
+    update-grub
+fi
+
+if [ -e /sys/firmware/efi ]; then
+    bootnum=$(efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//)
+    if [ ! -z "$bootnum" ]; then
+        currboot=$(efibootmgr | grep ^BootOrder: | awk '{print $2}')
+        nextboot=$(echo $currboot| awk -F, '{print $1}')
+        [ "$nextboot" = "$bootnum" ] || efibootmgr -o $bootnum,$currboot
+        efibootmgr -D
+    fi
+fi
+run_remote_python syncfileclient
+run_remote_parts post.d
+run_remote_config post
+
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+## Use the following option to add additional boot parameters for the
+## installed system (if supported by the bootloader installer).
+## Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string [from profile.yaml]
+deploycfg=/etc/confluent/confluent.deploycfg
+mgr=$(cat /etc/confluent/deployer)
+
+cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+cat > /usr/lib/live-installer.d/confluent-certs << EOF
+#!/bin/sh
+cp /tls/* /target/etc/ssl/certs/
+cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt
+EOF
+chmod a+x /usr/lib/live-installer.d/confluent-certs
+mkdir -p /.ssh/
+cat /ssh/*pubkey > /.ssh/authorized_keys
+mkdir -p /etc/ssh
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|cut -d ' ' -f 2)
+apikey=$(cat /etc/confluent/confluent.apikey)
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=$(echo $pubkey | sed -e s/.pub/-cert.pub/)
+    keyfile=${pubkey%.pub}
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" --post-file=$pubkey https://$mgr/confluent-api/self/sshcert -O $certfile --quiet
+    echo HostKey $keyfile >> /etc/ssh/sshd_config
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config
+done
+if [ -e /tmp/installdisk ]; then
+    instdisk=$(cat /tmp/installdisk)
+else
+    for blockdev in $(ls /sys/class/block/); do
+        shortname=$(basename $blockdev)
+        if [ "$shortname" != "${shortname%loop*}" ]; then
+            continue
+        fi
+        udevadm info --query=property /dev/$shortname |grep DEVTYPE=disk > /dev/null || continue # ignore partitions
+        udevadm info --query=property /dev/$shortname |grep DM_NAME > /dev/null && continue # not a real disk
+        sz=$(cat /sys/block/$shortname/size 2> /dev/null)
+        [ -z "$sz" ] && continue
+        [ $sz -lt 1048576 ] && continue # Too small
+        [ -z "$firstdisk" ] && firstdisk=$shortname
+        if udevadm info --query=property /dev/$shortname|grep ID_MODEL=| sed -e s/' '/_/g | grep -iE '(thinksystem_m.2|m.2_nvme_2-bay_raid_kit)' > /dev/null; then
+            instdisk=$shortname
+            break
+        fi
+        if udevadm info --query=property /dev/$shortname|grep MD_CONTAINER=imsm; then
+            sraid=$sortname
+        else
+            drv=$(udevadm info -a /dev/sdb|grep DRIVERS==|grep -Ev '""|"sd"' | sed -e s/.*=// -e s/'"'//g)
+            if [ "ahci" = "$drv" -a -z "$onbdisk" ]; then
+                onbdisk=$shortname
+            elif [ "megaraid" = "$drv" -a -z "$rdisk" ]; then
+                rdisk=$shortname
+            fi
+        fi
+    done
+fi
+if [ -z "$instdisk" ]; then
+    if [ ! -z "$sraid"]; then
+        instdisk=$sraid
+    elif [ ! -z "$onbdisk" ]; then
+        instdisk=$onbdisk
+    elif [ ! -z "$rdisk" ]; then
+        instdisk=$rdisk
+    else
+        instdisk=$firstdisk
+    fi
+fi
+if [ ! -z "$instdisk" ]; then
+    debconf-set partman-auto/disk /dev/$instdisk
+fi
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config
+echo IgnoreRhosts no >> /etc/ssh/sshd_config
+echo sshd:x:1:1::/run/sshd:/bin/false >> /etc/passwd
+/usr/sbin/sshd
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist -O /tmp/allnodes --quiet
+#kill -HUP $(ps | grep -v grep | grep /usr/sbin/sshd | sed -e 's/^ *//'|cut -d ' ' -f 1)
+#curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk
+#python3 /tmp/getinstalldisk
+#sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+mount -o bind /sys /target/sys
+mount -o bind /dev /target/dev
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /proc /target/proc
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /run /target/run
+cp -a /etc/confluent /target/etc/confluent
+cp -a /opt/confluent /target/opt/confluent
+mv /tmp/post.sh /target/tmp/
+cp -a /ssh /tls /target/tmp
+cat /tls/*.pem >> /target/etc/confluent/ca.pem
+cp -a /etc/ssh/ssh_host_* /target/etc/ssh/
+grep HostCertificate /etc/ssh/sshd_config  >> /target/etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+kill -HUP $(ps |grep -v grep|grep sshd|grep /usr|sed -e s/' root.*//')
+chroot /target bash /tmp/post.sh

confluent_osdeploy/ubuntu20.04-diskless/initramfs/conf/conf.d/confluent

@@ -1,4 +1,5 @@
 if ! grep console= /proc/cmdline > /dev/null; then
+    mkdir -p /custom-installation
     /opt/confluent/bin/autocons > /custom-installation/autocons.info
     cons=$(cat /custom-installation/autocons.info)
     if [ ! -z "$cons" ]; then

confluent_osdeploy/ubuntu20.04-diskless/initramfs/scripts/init-premount/confluent

@@ -107,7 +107,7 @@ if [ "$v6meth" = static ]; then
         ip route add default via $v6gw
     fi
 fi
-v4meth=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+v4meth=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 if [ "$v4meth" = static ]; then
     v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg | awk '{print $2}')
     v4prefix=$(grep ^prefix: /etc/confluent/confluent.deploycfg | awk '{print $2}')

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+(
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+    sleep 1
+done
+
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    touch /etc/confluent/firstboot.ran
+
+    run_remote firstboot.custom
+    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+    run_remote_parts firstboot.d
+
+    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+    run_remote_config firstboot.d
+fi
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service
+rm /etc/confluent/firstboot.ran
+) &
+tail --pid  $! -F /var/log/confluent/confluent-firstboot.log > /dev/console

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,93 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('m.2 nvme 2-bay raid kit', 'thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1,428 @@
+#!/usr/bin/python3
+import glob
+import json
+import os
+import re
+import time
+import shutil
+import socket
+import stat
+import struct
+import sys
+import subprocess
+
+bootuuid = None
+
+
+def get_partname(devname, idx):
+    if devname[-1] in '0123456789':
+        return '{}p{}'.format(devname, idx)
+    else:
+        return '{}{}'.format(devname, idx)
+
+def get_next_part_meta(img, imgsize):
+    if img.tell() == imgsize:
+        return None
+    pathlen = struct.unpack('!H', img.read(2))[0]
+    mountpoint = img.read(pathlen).decode('utf8')
+    jsonlen = struct.unpack('!I', img.read(4))[0]
+    metadata = json.loads(img.read(jsonlen).decode('utf8'))
+    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip filesystem type
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip orig devname (redundant with json)
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip padding
+    nextlen = struct.unpack('!Q', img.read(8))[0]
+    img.seek(nextlen, 1)  # go to next section
+    return metadata
+
+def get_multipart_image_meta(img):
+    img.seek(0, 2)
+    imgsize = img.tell()
+    img.seek(16)
+    seekamt = img.read(1)
+    img.seek(struct.unpack('B', seekamt)[0], 1)
+    partinfo = get_next_part_meta(img, imgsize)
+    while partinfo:
+        yield partinfo
+        partinfo = get_next_part_meta(img, imgsize)
+
+def get_image_metadata(imgpath):
+    with open(imgpath, 'rb') as img:
+        header = img.read(16)
+        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
+            for md in get_multipart_image_meta(img):
+                yield md
+        else:
+            raise Exception('Installation from single part image not supported')
+
+class PartedRunner():
+    def __init__(self, disk):
+        self.disk = disk
+
+    def run(self, command, check=True):
+        command = command.split()
+        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
+
+def fixup(rootdir, vols):
+    devbymount = {}
+    for vol in vols:
+        devbymount[vol['mount']] = vol['targetdisk']
+    fstabfile = os.path.join(rootdir, 'etc/fstab')
+    with open(fstabfile) as tfile:
+        fstab = tfile.read().split('\n')
+    while not fstab[0]:
+        fstab = fstab[1:]
+    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
+        os.unlink(os.path.join(rootdir, '.autorelabel'))
+    with open(fstabfile, 'w') as tfile:
+        for tab in fstab:
+            entry = tab.split()
+            if tab.startswith('#ORIGFSTAB#'):
+                if entry[1] in devbymount:
+                    targetdev = devbymount[entry[1]]
+                    if targetdev.startswith('/dev/localstorage/'):
+                        entry[0] = targetdev
+                    else:
+                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
+                        uuid = uuid.strip()
+                        entry[0] = 'UUID={}'.format(uuid)
+                elif entry[2] == 'swap':
+                    entry[0] = '/dev/mapper/localstorage-swap'
+                entry[0] = entry[0].ljust(42)
+                entry[1] = entry[1].ljust(16)
+                entry[3] = entry[3].ljust(28)
+                tab = '\t'.join(entry)
+            tfile.write(tab + '\n')
+    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
+        nameout.write(socket.gethostname() + '\n')
+    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
+    policy = None
+    if os.path.exists(selinuxconfig):
+        with open(selinuxconfig) as cfgin:
+            sec = cfgin.read().split('\n')
+            for l in sec:
+                l = l.split('#', 1)[0]
+                if l.startswith('SELINUXTYPE='):
+                    _, policy = l.split('=')
+    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
+        os.unlink(sshkey)
+    for sshkey in glob.glob('/etc/ssh/*_key*'):
+        newkey = os.path.join(rootdir, sshkey[1:])
+        shutil.copy2(sshkey, newkey)
+        finfo = os.stat(sshkey)
+        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
+
+    # Will use confignet to handle networking for ubuntu
+    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
+    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    if policy:
+        sys.stdout.write('Applying SELinux labeling...')
+        sys.stdout.flush()
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
+        sys.stdout.write('Done\n')
+        sys.stdout.flush()
+    for metafs in ('proc', 'sys', 'dev'):
+        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
+        defgrub = defgrubin.read().split('\n')
+    with open(grubsyscfg, 'w') as defgrubout:
+        for gline in defgrub:
+            gline = gline.split()
+            newline = []
+            for ent in gline:
+                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    continue
+                newline.append(ent)
+            defgrubout.write(' '.join(newline) + '\n')
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                elif 'search.fs_uuid ' in tgrubcfg[0] and 'root' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search.fs_uuid ' + bootuuid + ' root'
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        # must fixup root@d2:/boot/efi/EFI# cat ubuntu/grub.cfg ... uuid
+        subprocess.check_call(['chroot', rootdir, 'grub-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
+    newroot = None
+    with open('/etc/shadow') as shadowin:
+        shents = shadowin.read().split('\n')
+        for shent in shents:
+            shent = shent.split(':')
+            if not shent:
+                continue
+            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
+                newroot = shent[1]
+    if newroot:
+        shlines = None
+        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
+            shlines = oshadow.read().split('\n')
+        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
+            for line in shlines:
+                if line.startswith('root:'):
+                    line = line.split(':')
+                    line[1] = newroot
+                    line = ':'.join(line)
+                oshadow.write(line + '\n')
+    partnum = None
+    targblock = None
+    for vol in vols:
+        if vol['mount'] == '/boot/efi':
+            targdev = vol['targetdisk']
+            partnum = re.search('(\d+)$', targdev).group(1)
+            targblock = re.search('(.*)\d+$', targdev).group(1)
+    if targblock:
+        if targblock.endswith('p') and 'nvme' in targblock:
+            targblock = targblock[:-1]
+        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
+        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
+        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    #other network interfaces
+
+
+def had_swap():
+    with open('/etc/fstab') as tabfile:
+        tabs = tabfile.read().split('\n')
+        for tab in tabs:
+            tab = tab.split()
+            if len(tab) < 3:
+                continue
+            if tab[2] == 'swap':
+                return True
+    return False
+
+def install_to_disk(imgpath):
+    global bootuuid
+    lvmvols = {}
+    deftotsize = 0
+    mintotsize = 0
+    deflvmsize = 0
+    minlvmsize = 0
+    biggestsize = 0
+    biggestfs = None
+    plainvols = {}
+    allvols = []
+    swapsize = 0
+    if had_swap():
+        with open('/proc/meminfo') as meminfo:
+            swapsize = meminfo.read().split('\n')[0]
+        swapsize = int(swapsize.split()[1])
+        if swapsize < 2097152:
+            swapsize = swapsize * 2
+        elif swapsize > 8388608 and swapsize < 67108864:
+            swapsize = swapsize * 0.5
+        elif swapsize >= 67108864:
+            swapsize = 33554432
+        swapsize = int(swapsize * 1024)
+    deftotsize = swapsize
+    mintotsize = swapsize
+    for fs in get_image_metadata(imgpath):
+        allvols.append(fs)
+        deftotsize += fs['initsize']
+        mintotsize += fs['minsize']
+        if fs['initsize'] > biggestsize:
+            biggestfs = fs
+            biggestsize = fs['initsize']
+        if fs['device'].startswith('/dev/mapper'):
+            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
+            deflvmsize += fs['initsize']
+            minlvmsize += fs['minsize']
+        else:
+            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+    with open('/tmp/installdisk') as diskin:
+        instdisk = diskin.read()
+    instdisk = '/dev/' + instdisk
+    parted = PartedRunner(instdisk)
+    dinfo = parted.run('unit s print', check=False)
+    dinfo = dinfo.split('\n')
+    sectors = 0
+    sectorsize = 0
+    for inf in dinfo:
+        if inf.startswith('Disk {0}:'.format(instdisk)):
+            _, sectors = inf.split(': ')
+            sectors = int(sectors.replace('s', ''))
+        if inf.startswith('Sector size (logical/physical):'):
+            _, sectorsize = inf.split(':')
+            sectorsize = sectorsize.split('/')[0]
+            sectorsize = sectorsize.replace('B', '')
+            sectorsize = int(sectorsize)
+    # for now, only support resizing/growing the largest partition
+    minexcsize = deftotsize - biggestfs['initsize']
+    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
+    minsectors = mintotsize // sectorsize
+    if sectors < (minsectors + 65536):
+        raise Exception('Disk too small to fit image')
+    biggestsectors = sectors - (minexcsize // sectorsize)
+    biggestsize = sectorsize * biggestsectors
+    parted.run('mklabel gpt')
+    curroffset = 2048
+    for volidx in sorted(plainvols):
+        vol = plainvols[volidx]
+        if vol is not biggestfs:
+            size = vol['initsize'] // sectorsize
+        else:
+            size = biggestsize // sectorsize
+        size += 2047 - (size % 2048)
+        end = curroffset + size
+        if end > sectors:
+            end = sectors
+        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
+        vol['targetdisk'] = get_partname(instdisk, volidx)
+        curroffset += size + 1
+    if not lvmvols:
+        if swapsize:
+            swapsize = swapsize // sectorsize
+            swapsize += 2047 - (size % 2048)
+            end = curroffset + swapsize
+            if end > sectors:
+                end = sectors
+            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
+            subprocess.check_call(['mkswap', get_partname(instdisk, volidx + 1)])
+    else:
+        parted.run('mkpart lvm {}s 100%'.format(curroffset))
+        lvmpart = get_partname(instdisk, volidx + 1)
+        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
+        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
+        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
+        vginfo = vginfo.split('\n')
+        pesize = 0
+        pes = 0
+        for infline in vginfo:
+            infline = infline.split()
+            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
+                pesize = int(infline[2])
+            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
+                pes = int(infline[4])
+        takeaway = swapsize // pesize
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                continue
+            takeaway += vol['initsize'] // pesize
+            takeaway += 1
+        biggestextents = pes - takeaway
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                extents = biggestextents
+            else:
+                extents = vol['initsize'] // pesize
+                extents += 1
+            if vol['mount'] == '/':
+                lvname = 'root'
+            else:
+                lvname = vol['mount'].replace('/', '_')
+            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
+            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
+        if swapsize:
+            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
+            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
+        os.makedirs('/run/imginst/targ')
+        for vol in allvols:
+            with open(vol['targetdisk'], 'wb') as partition:
+                partition.write(b'\x00' * 1 * 1024 * 1024)
+            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
+            source = vol['mount'].replace('/', '_')
+            source = '/run/imginst/sources/' + source
+            blankfsstat = os.statvfs('/run/imginst/targ')
+            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
+            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
+            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
+                stillrunning = copier.poll()
+                lastprogress = 0.0
+                while stillrunning is None:
+                    currfsstat = os.statvfs('/run/imginst/targ')
+                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
+                    currused -= blankused
+                    with open('/proc/meminfo') as meminf:
+                        for line in meminf.read().split('\n'):
+                            if line.startswith('Dirty:'):
+                                _, dirty, _ = line.split()
+                                dirty = int(dirty) * 1024
+                    progress = (currused - dirty) / vol['minsize']
+                    if progress < lastprogress:
+                        progress = lastprogress
+                    if progress > 0.99:
+                        progress = 0.99
+                    lastprogress = progress
+                    progress = progress * 100
+                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                    sys.stdout.flush()
+                    time.sleep(0.5)
+                    stillrunning = copier.poll()
+                if stillrunning != 0:
+                    raise Exception("Error copying volume")
+                with subprocess.Popen(['sync']) as syncrun:
+                    stillrunning = syncrun.poll()
+                    while stillrunning is None:
+                        with open('/proc/meminfo') as meminf:
+                            for line in meminf.read().split('\n'):
+                                if line.startswith('Dirty:'):
+                                    _, dirty, _ = line.split()
+                                    dirty = int(dirty) * 1024
+                        progress = (vol['minsize'] - dirty) / vol['minsize']
+                        if progress < lastprogress:
+                            progress = lastprogress
+                        if progress > 0.99:
+                            progress = 0.99
+                        lastprogress = progress
+                        progress = progress * 100
+                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                        sys.stdout.flush()
+                        time.sleep(0.5)
+                        stillrunning = syncrun.poll()
+                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
+            subprocess.check_call(['umount', '/run/imginst/targ'])
+        for vol in allvols:
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
+        fixup('/run/imginst/targ', allvols)
+
+
+if __name__ == '__main__':
+    install_to_disk(os.environ['mountsrc'])
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh

@@ -127,11 +127,13 @@ chmod +x /sysroot/opt/confluent/bin/onboot.sh
 cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
 ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
 cp /etc/confluent/functions /sysroot/etc/confluent/functions
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 if grep installtodisk /proc/cmdline > /dev/null; then
     . /etc/confluent/functions
     run_remote installimage
     exec reboot -f
 fi
-mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
-ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
 exec /opt/confluent/bin/start_root

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/installimage

@@ -0,0 +1,48 @@
+#!/bin/bash
+. /etc/confluent/functions
+# the image will be used to deploy itself
+# provide both access to image (for parsing metadata)
+# and existing mounts of image (to take advantage of caching)
+mount -o bind /sys /sysroot/sys
+mount -o bind /dev /sysroot/dev
+mount -o bind /proc /sysroot/proc
+mount -o bind /run /sysroot/run
+
+
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /sysroot/run/imginst/sources/_
+    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
+        srcname=${srcmount#/dev/mapper/mproot}
+        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
+        mkdir -p /sysroot/run/imginst/sources/$srcname
+        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
+    done
+fi
+cd /sysroot/run
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+if [ ! -f /sysroot/tmp/installdisk ]; then
+    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
+    while [ ! -f /sysroot/tmp/installdisk ]; do
+        sleep 1
+    done
+fi
+lvm vgchange -a n
+udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
+chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
+chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
+echo "Port 22" >> /etc/ssh/sshd_config
+echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
+echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+cat /tls/*.pem > /sysroot/run/imginst/targ/usr/local/share/ca-certificates/confluent.crt
+chroot /sysroot/run/imginst/targ update-ca-certificates
+
+chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
+chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# This script is executed 'chrooted' into a cloned disk target before rebooting
+#
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+tail -f /var/log/confluent/confluent-post.log > /dev/console &
+logshowpid=$!
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+mkdir -p /opt/confluent/bin
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+systemctl enable firstboot
+run_remote_python syncfileclient
+run_remote_python confignet
+run_remote post.custom
+# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+
+kill $logshowpid
+
+

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,286 @@
+#!/usr/bin/python3
+import subprocess
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        myips = []
+        ipaddrs = subprocess.check_output(['ip', '-br', 'a']).split(b'\n')
+        for line in ipaddrs:
+            isa = line.split()
+            if len(isa) < 3 or isa[1] != b'UP':
+                continue
+            for addr in isa[2:]:
+                if addr.startswith(b'fe80::') or addr.startswith(b'169.254'):
+                    continue
+                addr = addr.split(b'/')[0]
+                if not isinstance(addr, str):
+                    addr = addr.decode('utf8')
+                myips.append(addr)
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir, 'myips': myips})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/hooks/casper-bottom.sh

@@ -9,33 +9,49 @@ MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|'
 MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}')
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
-umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+umask 077
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile| sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+seems a potentially relevant thing to put i... by Jarrod Johnson
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -74,8 +74,11 @@ if [ -e /sys/firmware/efi ]; then
     fi
 fi
 cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 

confluent_osdeploy/ubuntu22.04-diskless

@@ -0,0 +1 @@
+ubuntu20.04-diskless

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -11,32 +11,48 @@ MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| aw
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
 umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile | sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/firstboot.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/ansible/post.d/README.txt

@@ -0,0 +1,29 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+(
+exec >> /target/var/log/confluent/confluent-firstboot.log
+exec 2>> /target/var/log/confluent/confluent-firstboot.log
+chmod 600 /target/var/log/confluent/confluent-firstboot.log
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
@@ -15,7 +21,10 @@ done
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
 source /etc/confluent/functions
-
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export confluent_mgr confluent_profile
 run_remote_parts firstboot.d
 run_remote_config firstboot.d
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -8,7 +8,6 @@ chmod go-rwx /etc/confluent/*
 for i in /custom-installation/ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
 done
-
 cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
 cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
 sshconf=/target/etc/ssh/ssh_config
@@ -19,10 +18,15 @@ echo 'Host *' >> $sshconf
 echo '    HostbasedAuthentication yes' >> $sshconf
 echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
-
+cp /etc/confluent/functions /target/etc/confluent/functions
+source /etc/confluent/functions
+mkdir -p /target/var/log/confluent
+cp /var/log/confluent/* /target/var/log/confluent/
+(
+exec >> /target/var/log/confluent/confluent-post.log
+exec 2>> /target/var/log/confluent/confluent-post.log
+chmod 600 /target/var/log/confluent/confluent-post.log
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions
-source /target/etc/confluent/functions
 chmod +x /target/etc/confluent/firstboot.sh
 cp /tmp/allnodes /target/root/.shosts
 cp /tmp/allnodes /target/etc/ssh/shosts.equiv
@@ -74,12 +78,17 @@ if [ -e /sys/firmware/efi ]; then
     fi
 fi
 cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /target/usr/local/share/ca-certificates/confluent.crt
 cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target update-ca-certificates
 chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python confignet"
 chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
 source /target/etc/confluent/functions
 
 run_remote_config post
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
 
 umount /target/sys /target/dev /target/proc
-
+) &
+tail --pid $! -n 0 -F /target/var/log/confluent/confluent-post.log > /dev/console

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 deploycfg=/custom-installation/confluent/confluent.deploycfg
+mkdir -p /var/log/confluent
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp /custom-installation/confluent/confluent.info /custom-installation/confluent/confluent.apikey /etc/confluent/
+cat /custom-installation/tls/*.pem >> /etc/confluent/ca.pem
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin
+cp $deploycfg /etc/confluent/
+(
+exec >> /var/log/confluent/confluent-pre.log
+exec 2>> /var/log/confluent/confluent-pre.log
+chmod 600 /var/log/confluent/confluent-pre.log
 
 cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
 if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
@@ -23,7 +34,17 @@ echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
 systemctl restart sshd
+mkdir -p /etc/confluent
+export nodename confluent_profile confluent_mgr
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+run_remote_parts pre.d
 curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
-curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
-python3 /custom-installation/getinstalldisk
+if [ ! -e /tmp/installdisk ]; then
+    curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+    python3 /custom-installation/getinstalldisk
+fi
 sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml
+) &
+tail --pid $! -n 0 -F /var/log/confluent/confluent-pre.log > /dev/console
+

confluent_osdeploy/utils/clortho.c

@@ -19,6 +19,7 @@
 #define MAXPACKET 1024
 
 static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./";
+static const char b64alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 unsigned char* genpasswd(int len) {
         unsigned char * passwd;
@@ -56,10 +57,10 @@ char * b64e(uint8_t * data, uint32_t datalen) {
         currchunk[1] = remaining > 1 ? currptr[1] : 0;
         currchunk[2] = remaining > 2 ? currptr[2] : 0;
         currptr += 3;
-        currout[0] = cryptalpha[currchunk[0] >> 2];
-        currout[1] = cryptalpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
-        currout[2] = remaining > 1 ? cryptalpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
-        currout[3] = remaining > 2 ? cryptalpha[currchunk[2] & 0x3f] : '=';
+        currout[0] = b64alpha[currchunk[0] >> 2];
+        currout[1] = b64alpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
+        currout[2] = remaining > 1 ? b64alpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
+        currout[3] = remaining > 2 ? b64alpha[currchunk[2] & 0x3f] : '=';
         remaining -= 3;
         currout += 4;
     }

confluent_osdeploy/utils/urlmount3.c

@@ -0,0 +1,348 @@
+/*
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2021 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+#define FUSE_USE_VERSION 30
+#include <fuse3/fuse.h>
+#include <curl/curl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <pthread.h>
+#include <unistd.h>
+
+CURL *curl;
+
+char curlerror[CURL_ERROR_SIZE];
+curl_off_t filesize;
+
+typedef struct downloadbuffer {
+    char *response;
+    size_t completed;
+    size_t total;
+} downloadbuffer;
+
+#define MAX_FILE_LEN 1024
+#define MAX_URL_PATHS 512
+static char filename[MAX_FILE_LEN + 1];
+static int urlidx, newidx;
+static char* urls[MAX_URL_PATHS + 1];
+
+
+void *http_rechecker(void *argp) {
+    CURL *checkurl;
+    int tmpidx, tmpval;
+    tmpidx = open("/dev/urandom", O_RDONLY);
+    if (tmpidx <= 0 || read(tmpidx, (char*)&tmpval, 4) < 0)
+        tmpval = time(NULL) & 0xffffffff;
+    if (tmpidx >= 0)
+        close(tmpidx);
+    srand(tmpval);
+    checkurl = curl_easy_init();
+    if (curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Error setting timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Error setting nobody\n");
+        exit(1);
+    }
+    while (1) {
+        sleep(25 + tmpval % 10);  // Spread out retries across systems
+        tmpidx = 0;
+        while (tmpidx < urlidx && tmpidx < newidx && urls[tmpidx] != NULL) {
+            if (curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]) != CURLE_OK) {
+                tmpidx++;
+                continue;
+            }
+            if (curl_easy_perform(checkurl) == CURLE_OK)
+                newidx = tmpidx;
+            else
+                tmpidx++;
+        }
+    }
+}
+
+static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
+                        off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags frf)
+{
+    if (strcmp(path, "/") != 0)  // We don't support subdirs
+        return -ENOENT;
+    filler(buf, ".", NULL, 0, 0);
+    filler(buf, "..", NULL, 0, 0);
+    filler(buf, filename + 1, NULL, 0, 0);
+    return 0;
+}
+
+size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) {
+    size_t amount;
+    amount = size * nmemb;
+    if (userdata->total < amount + userdata->completed) return 0;
+    memcpy(&(userdata->response[userdata->completed]), data, amount);
+    userdata->completed += amount;
+    return amount;
+}
+
+static int http_read(const char *path, char *buf, size_t size, off_t offset,
+                     struct fuse_file_info *fi) {
+    char headbuffer[512];
+    double dldbl = 0.0;
+    int startidx;
+    int reconnecting = 0;
+    FILE* fd;
+    startidx = urlidx;
+    memset(buf, 0, size);
+    curl_off_t downloaded;
+    //Would be needed for multithread, however preferring to conserve
+    //filehandles rather than go multithread
+    // Some comparisons showed that the threaded performance boost doesn't
+    // do even offset the overhead of the new curl handles, so better
+    // to use single threaded curl overall for now
+    //CURL *tmpcurl = curl_easy_duphandle(curl);
+    downloadbuffer dlbuf;
+    dlbuf.response = buf;
+    dlbuf.completed = 0;
+    dlbuf.total = size;
+    fd = NULL;
+
+    if (strcmp(path, filename) != 0) return -ENOENT;
+    memset(headbuffer, 0, 512);
+    if (offset >= filesize) return 0;
+    if (offset + size - 1 >= filesize) size = filesize - offset - 1;
+    snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
+    if (curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer) != CURLE_OK) {
+        fprintf(stderr, "Error setting range\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&dlbuf) != CURLE_OK) {
+        fprintf(stderr, "Error setting writedata\n");
+        exit(1);
+    }
+    if (newidx < MAX_URL_PATHS) {
+        reconnecting = 1;
+        urlidx = newidx;
+        newidx = MAX_URL_PATHS;
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        // if fail, carry on and take the error in curl_easy_perform instead
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        reconnecting = 1;
+        fd = fopen("/dev/kmsg", "w+");
+        dlbuf.completed = 0;
+        fprintf(fd, "<4>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
+        fclose(fd);
+        urlidx++;
+        if (urlidx > MAX_URL_PATHS)
+            urlidx = 0;
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == startidx) {
+            fd = fopen("/dev/kmsg", "w+");
+            fprintf(fd, "<1>urlmount: All connections to source are down\n");
+            fclose(fd);
+            sleep(10);
+        }
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        if (urlidx > MAX_URL_PATHS) {
+            fprintf(stderr, "Maximum url path exceeded\n");
+            exit(1);
+        }
+        // ignore, let the curl_easy_perform get the error
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    if (reconnecting) {
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Successfully connected to %s\n", urls[urlidx]);
+        fclose(fd);
+    }
+    curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl);
+    downloaded = round(dldbl);
+    //Would be needed for multithread
+    //curl_easy_cleanup(tmpcurl);
+    return downloaded;
+}
+
+static int http_open(const char *path, struct fuse_file_info *fi) {
+    if (strcmp(path, filename) != 0)
+        return -ENOENT;
+
+    if ((fi->flags & 3) != O_RDONLY)
+        return -EACCES;
+
+    return 0;
+}
+
+static void* http_init(struct fuse_conn_info *conn, struct fuse_config *cfg) {
+    // Because we fork, we need to redo curl
+    // or else suffer the wrath of NSS TLS
+    pthread_t tid;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    pthread_create(&tid, NULL, http_rechecker, NULL);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl error buffor\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl failonerror\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing url\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl fill buffer\n");
+        exit(1);
+    }
+    return NULL;
+}
+
+static int http_getattr(const char *path, struct stat *st, struct fuse_file_info *ffi) {
+    memset(st, 0, sizeof(struct stat));
+
+    if (strcmp(path, "/") == 0) {
+        st->st_mode = S_IFDIR | 0555;
+        st->st_nlink = 2;
+    } else if (strcmp(path, filename) == 0) {
+        st->st_mode = S_IFREG | 0444;
+        st->st_nlink = 1;
+        st->st_size = filesize; // TODO: fix with curl HEAD
+    } else
+        return -ENOENT;
+    return 0;
+}
+
+
+static const struct fuse_operations http_ops = {
+    .getattr = http_getattr,
+    .readdir = http_readdir,
+    .read = http_read,
+    .open = http_open,
+    .init = http_init,
+};
+
+int main(int argc, char* argv[]) {
+    char *tmp;
+    double fsize;
+    unsigned int i;
+    int j;
+    j = 0;
+    memset(urls, 0, 32*sizeof(char*));
+    urlidx = 0;
+    newidx = MAX_URL_PATHS;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Unable to set error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Unable to set fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L) != CURLE_OK) {
+        fprintf(stderr, "Unable to setup curl timeout\n");
+        exit(1);
+    }
+    memset(filename, 0, MAX_FILE_LEN);
+    for (i=0; i < argc; i++) {
+        if (strstr(argv[i], ":") > 0) {
+            if (j < MAX_URL_PATHS) {
+                urls[j] = argv[i];
+                tmp = strrchr(urls[j++], '/');
+                strncpy(filename, tmp, MAX_FILE_LEN);
+            }
+            //Request single threaded mode, as curl would need more
+            // filehandles for multithread
+            argv[i] = "-s";
+        }
+    }
+    if (filename[0] == 0) {
+        fprintf(stderr, "No URL given in arguments\n");
+        exit(1);
+    }
+    for (i=0; urls[i] != NULL; i++) {
+        printf("Registering mount path: %s\n", urls[i]);
+    }
+    j = urlidx;
+    printf("Connecting to %s\n", urls[urlidx]);
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Unable to set url\n");
+    }
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Failure setting no body\n");
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == j) {
+            fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n");
+            exit(1);
+        }
+        printf("Connecting to %s\n", urls[urlidx]);
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+            fprintf(stderr, "Unable to set url\n");
+        }
+    }
+    printf("Successfully connected to %s\n", urls[urlidx]);
+    if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize) != CURLE_OK) {
+        fprintf(stderr, "Failed getting content length\n");
+        exit(1);
+    }
+    filesize = round(fsize);
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 0) != CURLE_OK) {
+        fprintf(stderr, "Failed setting nobody\n");
+        exit(1);
+    }
+    if (filesize < 1) {
+        fprintf(stderr, "Unable to reach designated URL\n");
+        exit(1);
+    }
+    if (!curl) {
+        fprintf(stderr, "Unable to initialize CURL!\n");
+        exit(1);
+    }
+    curl_easy_cleanup(curl);
+    curl_global_cleanup();
+    fuse_main(argc, argv, &http_ops, NULL);
+}

confluent_server/bin/confluent_selfcheck

@@ -15,6 +15,7 @@ import confluent.sshutil as sshutil
 import confluent.certutil as certutil
 import confluent.client as client
 import confluent.config.configmanager as configmanager
+import confluent.netutil as netutil
 import eventlet.green.subprocess as subprocess
 import tempfile
 import shutil
@@ -237,13 +238,14 @@ if __name__ == '__main__':
         allok = True
         uuidok = False
         macok = False
+        valid_nodes = [node['item']['href'][:-1] for node in sess.read('/nodes/')] #get all valid nodes
         for rsp in sess.read(f'/nodes/{args.node}/attributes/all'):
             if rsp.get('errorcode', None) == 404:
                 emprint(f'There is no node named "{args.node}"')
                 allok = False
                 uuidok = True  # not really, but suppress the spurious error
             dnsdomain = rsp.get('dns.domain', {}).get('value', '')
-            if ',' in dnsdomain or ' ' in dnsdomain:
+            if dnsdomain and (',' in dnsdomain or ' ' in dnsdomain):
                 allok = False
                 emprint(f'{args.node} has a dns.domain that appears to be a search instead of singular domain')
                 uuidok = True  # not really, but suppress the spurious error
@@ -259,9 +261,37 @@ if __name__ == '__main__':
                     mac = rsp[key].get('value', None)
                     if mac:
                         macok = True
+            #adding new code to check if the response is something like net.<something>switch
+            for key in rsp:
+                if ((key.startswith('net.') and key.endswith('switch')) 
+                        or (key.startswith('power.') and key.endswith('pdu'))
+                        or (key.startswith('enclosure.') and key.endswith('manager'))
+                        ):
+                    switch_value = rsp[key].get('value',None)
+                    if switch_value and switch_value not in valid_nodes:
+                        emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).')
+        print(f"Checking network configuration for {args.node}")
+        cfg = configmanager.ConfigManager(None)
+        bootablev4nics = []
+        bootablev6nics = []
+        for nic in glob.glob("/sys/class/net/*/ifindex"):
+            idx = int(open(nic, "r").read())
+            nicname = nic.split('/')[-2]
+            ncfg = netutil.get_nic_config(cfg, args.node, ifidx=idx)
+            if ncfg['ipv4_address'] or ncfg['ipv4_method'] == 'dhcp':
+                bootablev4nics.append(nicname)
+            if ncfg['ipv6_address']:
+                bootablev6nics.append(nicname)
+        if bootablev4nics:
+            print("{} appears to have network configuration suitable for IPv4 deployment via: {}".format(args.node, ",".join(bootablev4nics)))
+        elif bootablev6nics:
+            print('{} appears to have networking configuration suitable for IPv6 deployment via: {}'.format(args.node, ",".join(bootablev6nics)))
+        else:
+            emprint(f"{args.node} may not have any viable IP network configuration (check name resolution (DNS or hosts file) "
+                    "and/or net.*ipv4_address, and verify that the deployment serer addresses and subnet mask/prefix length are accurate)")
         if not uuidok and not macok:
             allok = False
-            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work')
+            emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work (Example resolution: nodeinventory {args.node} -s)')
         if allok:
             print(f'No issues detected with attributes of {args.node}')
         fprint("Checking name resolution: ")

confluent_server/builddeb

@@ -36,7 +36,7 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
         sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyghmi, python3-paramiko, python3-pysnmp4, python3-libarchive-c, confluent-vtbufferd/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -49,6 +49,13 @@ if ! grep wheezy /etc/os-release; then
 fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
+cat > debian/postinst << EOF
+if ! getent passwd confluent > /dev/null; then 
+    useradd -r confluent -d /var/lib/confluent -s /usr/sbin/nologin
+    mkdir -p /etc/confluent
+    chown confluent /etc/confluent
+fi
+EOF
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
 #echo 'Provides: python-'$DPKGNAME >> debian/control
 #echo 'Conflicts: python-'$DPKGNAME >> debian/control

confluent_server/confluent/auth.py

@@ -41,6 +41,7 @@ try:
 except ImportError:
     pass
 import time
+import yaml
 
 _pamservice = 'confluent'
 _passcache = {}
@@ -115,6 +116,47 @@ class PromptsNeeded(Exception):
     def __init__(self, prompts):
         self.prompts = prompts
 
+#add function to change _allowedbyrole and _deniedbyrole vars.
+def add_roles(_allowed,_denied):
+    # function to parse the roles and the files. If there are modifications to be done to the roles, items will be added to dictionaries.
+    # If there are no moodifications done to one of the roles, it continues to the next
+    # Opening YAML file and reading the custom roles
+    with open("/etc/confluent/authorization.yaml","r") as stream:
+        loaded_file = yaml.safe_load(stream)
+        try:
+            allowed_loaded = loaded_file["allowedbyrole"]
+            for role in allowed_loaded:
+                if role not in configmanager._validroles:
+                    configmanager._validroles.append(role)
+        except:
+            pass
+        try:
+            denied_loaded = loaded_file["deniedbyrole"]
+        except:
+            pass
+ 
+        try:
+            _allowed.update(allowed_loaded)
+        except NameError:
+            pass
+        try: 
+            _denied.update(denied_loaded)
+        except NameError:
+            pass
+        return
+                
+    
+def check_for_yaml():
+    #checking if the file exists
+    if os.path.exists("/etc/confluent/authorization.yaml"):
+        add_roles(_allowedbyrole,_deniedbyrole)
+
+        return "Custom auth. file detected in /etc/confluent, updated roles accordingly"
+    else:
+        return "No custom auth. file. Continuing as normal"
+    
+
+        
 def _get_usertenant(name, tenant=False):
     """_get_usertenant
 

confluent_server/confluent/collective/manager.py

@@ -63,7 +63,7 @@ connecting = ContextBool()
 leader_init = ContextBool()
 enrolling = ContextBool()
 
-def connect_to_leader(cert=None, name=None, leader=None, remote=None):
+def connect_to_leader(cert=None, name=None, leader=None, remote=None, isretry=False):
     global currentleader
     global follower
     ocert = cert
@@ -72,8 +72,9 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     oremote = remote
     if leader is None:
         leader = currentleader
-    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
-             'subsystem': 'collective'})
+    if not isretry:
+        log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+                 'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader, remote)
     except Exception as e:
@@ -84,9 +85,11 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     with connecting:
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
+            if not banner:
+                return
             vers = banner.split()[2]
-            if vers not in (b'v2', b'v3'):
-                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
+            if vers != b'v4':
+                raise Exception('This instance only supports protocol 4, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -103,7 +106,12 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                                 '{0}'.format(leader),
                         'subsystem': 'collective'})
                     return False
+                if 'waitinline' in keydata:
+                    eventlet.sleep(0.3)
+                    return connect_to_leader(cert, name, leader, None, isretry=True)
                 if 'leader' in keydata:
+                    if keydata['leader'] == None:
+                        return None
                     log.log(
                         {'info': 'Prospective leader {0} has redirected this '
                                  'member to {1}'.format(leader, keydata['leader']),
@@ -144,7 +152,7 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                     except Exception:
                         pass
                     log.log({'error': 'Retrying connection, error during initial sync', 'subsystem': 'collective'})
-                    return connect_to_leader(ocert, oname, oleader, oremote)
+                    return connect_to_leader(ocert, oname, oleader, None)
                     raise Exception("Error doing initial DB transfer")  # bad ssl write retry
                 dbjson += ndata
             cfm.clear_configuration()
@@ -251,6 +259,9 @@ def get_myname():
         mycachedname[1] = time.time()
         return myname
 
+def in_collective():
+    return bool(list(cfm.list_collective()))
+
 def handle_connection(connection, cert, request, local=False):
     global currentleader
     global retrythread
@@ -271,7 +282,11 @@ def handle_connection(connection, cert, request, local=False):
                 return
             if follower is not None:
                 linfo = cfm.get_collective_member_by_address(currentleader)
-                remote = socket.create_connection((currentleader, 13001), 15)
+                try:
+                    remote = socket.create_connection((currentleader, 13001), 15)
+                except Exception:
+                    cfm.stop_following()
+                    return
                 remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                          keyfile='/etc/confluent/privkey.pem',
                                          certfile='/etc/confluent/srvcert.pem')
@@ -537,6 +552,11 @@ def handle_connection(connection, cert, request, local=False):
                                       'backoff': True})
             connection.close()
             return
+        if leader_init.active:
+            tlvdata.send(connection, {'error': 'Servicing a connection',
+                                      'waitinline': True})
+            connection.close()
+            return
         if myself != get_leader(connection):
             tlvdata.send(
                 connection,
@@ -552,7 +572,7 @@ def handle_connection(connection, cert, request, local=False):
                           'txcount': cfm._txcount})
             log.log({'info': 'Connecting to leader due to superior '
                              'transaction count', 'subsystem': 'collective'})
-            connection.close()
+            connection.close()  # well this won't work
             if not connect_to_leader(
                 None, None, connection.getpeername()[0]):
                 if retrythread is None:
@@ -569,9 +589,15 @@ def handle_connection(connection, cert, request, local=False):
             tlvdata.send(connection, cfm._cfgstore['collective'])
             tlvdata.send(connection, {'confluent_uuid': cfm.get_global('confluent_uuid')}) # cfm.get_globals())
             cfgdata = cfm.ConfigManager(None)._dump_to_json()
-            tlvdata.send(connection, {'txcount': cfm._txcount,
-                                      'dbsize': len(cfgdata)})
-            connection.sendall(cfgdata)
+            try:
+                tlvdata.send(connection, {'txcount': cfm._txcount,
+                            'dbsize': len(cfgdata)})
+                connection.sendall(cfgdata)
+            except Exception:
+                try:
+                    connection.close()
+                finally:
+                    return None
         #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
         # so far unused anyway
         connection.settimeout(90)
@@ -660,7 +686,11 @@ def get_leader(connection):
 
 def retire_as_leader(newleader=None):
     global currentleader
+    global reassimilate
     cfm.stop_leading(newleader)
+    if reassimilate is not None:
+        reassimilate.kill()
+        reassimilate = None
     currentleader = None
 
 def become_leader(connection):
@@ -668,6 +698,10 @@ def become_leader(connection):
     global follower
     global retrythread
     global reassimilate
+    if cfm.get_collective_member(get_myname()).get('role', None) == 'nonvoting':
+        log.log({'info': 'Refraining from being leader of collective (nonvoting)',
+            'subsystem': 'collective'})
+        return False
     log.log({'info': 'Becoming leader of collective',
              'subsystem': 'collective'})
     if follower is not None:
@@ -679,15 +713,21 @@ def become_leader(connection):
         retrythread = None
     currentleader = connection.getsockname()[0]
     skipaddr = connection.getpeername()[0]
+    if reassimilate is not None:
+        reassimilate.kill()
+    reassimilate = eventlet.spawn(reassimilate_missing)
+    cfm._ready = True
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
-        if reassimilate is not None:
-            reassimilate.kill()
-        reassimilate = eventlet.spawn(reassimilate_missing)
+
 
 def reassimilate_missing():
     eventlet.sleep(30)
-    while cfm.cfgstreams and _assimilate_missing():
+    while True:
+        try:
+            _assimilate_missing()
+        except Exception as e:
+            cfm.logException()
         eventlet.sleep(30)
 
 def _assimilate_missing(skipaddr=None):
@@ -801,6 +841,8 @@ def start_collective():
         for member in sorted(list(cfm.list_collective())):
             if member == myname:
                 continue
+            if cfm.get_collective_member(member).get('role', None) == 'nonvoting':
+                continue
             if cfm.cfgleader is None:
                 cfm.stop_following(True)
             ldrcandidate = cfm.get_collective_member(member)['address']

confluent_server/confluent/config/attributes.py

@@ -371,7 +371,7 @@ node = {
                         'the managed node.  If not specified, then console '
                         'is disabled.  "ipmi" should be specified for most '
                         'systems if console is desired.'),
-        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
+        'validvalues': ('ssh', 'ipmi', 'openbmc', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/configmanager.py

@@ -59,13 +59,21 @@ except ModuleNotFoundError:
 import ast
 import base64
 from binascii import hexlify
+import os
+import sys
+
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.config.attributes as allattributes
 import confluent.config.conf as conf
-import confluent.log
 import confluent.noderange as noderange
 import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
+import confluent.log
 import copy
 import crypt
 try:
@@ -77,17 +85,16 @@ import eventlet
 import eventlet.event as event
 import eventlet.green.select as select
 import eventlet.green.threading as gthread
+import eventlet.green.subprocess as subprocess
 import fnmatch
 import hashlib
 import json
 import msgpack
 import operator
-import os
 import random
 import re
 import string
 import struct
-import sys
 import threading
 import time
 import traceback
@@ -112,6 +119,7 @@ _cfgstore = None
 _pendingchangesets = {}
 _txcount = 0
 _hasquorum = True
+_ready = False
 
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
@@ -120,7 +128,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
-_validroles = ('Administrator', 'Operator', 'Monitor', 'Stub')
+_validroles = ['Administrator', 'Operator', 'Monitor', 'Stub']
 
 membership_callback = None
 
@@ -422,7 +430,10 @@ def _push_rpc(stream, payload):
                 pass
             if membership_callback:
                 membership_callback()
-            stream.close()
+            try:
+                stream.close()
+            except Exception:
+                pass
 
 
 def decrypt_value(cryptvalue,
@@ -690,7 +701,9 @@ def relay_slaved_requests(name, listener):
                 if name not in cfgstreams:
                     raise Exception("Unexpected loss of node in followers: " + name)
                 sz = struct.unpack('!Q', msg)[0]
-                if sz != 0:
+                if sz == 0:
+                    _push_rpc(listener, b'')
+                else:
                     rpc = b''
                     while len(rpc) < sz:
                         nrpc = listener.recv(sz - len(rpc))
@@ -740,6 +753,16 @@ def relay_slaved_requests(name, listener):
                 return False
             return True
 
+lastheartbeat = None
+def check_leader():
+    _push_rpc(cfgleader, b'')
+    tries = 0
+    while tries < 30:
+        eventlet.sleep(0.1)
+        tries += 1
+        if lastheartbeat and lastheartbeat > (confluent.util.monotonic_time() - 3):
+            return True
+    raise Exception("Leader has disappeared")
 
 class StreamHandler(object):
     def __init__(self, sock):
@@ -761,10 +784,13 @@ class StreamHandler(object):
                     res = _push_rpc(self.sock, b'')  # nulls are a keepalive
                     if not res:
                         return None
+                    #TODO: this test can work fine even if the other end is
+                    # gone, go to a more affirmative test to more quickly
+                    # detect outage to peer
                     self.keepalive = confluent.util.monotonic_time() + 20
             self.expiry = confluent.util.monotonic_time() + 60
             msg = self.sock.recv(8)
-        except Exception:
+        except Exception as e:
             msg = None
         return msg
 
@@ -805,6 +831,9 @@ _oldcfgstore = None
 _oldtxcount = 0
 
 
+def config_is_ready():
+    return _ready
+
 def rollback_clear():
     global _cfgstore
     global _txcount
@@ -822,6 +851,8 @@ def clear_configuration():
     global _txcount
     global _oldcfgstore
     global _oldtxcount
+    global _ready
+    _ready = False
     stop_leading()
     stop_following()
     _oldcfgstore = _cfgstore
@@ -832,6 +863,7 @@ def clear_configuration():
 def commit_clear():
     global _oldtxcount
     global _oldcfgstore
+    global _ready
     # first, copy over old non-key globals, as those are
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
@@ -851,6 +883,7 @@ def commit_clear():
                 pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
+    _ready = True
 
 cfgleader = None
 
@@ -858,6 +891,7 @@ cfgleader = None
 def follow_channel(channel):
     global _txcount
     global _hasquorum
+    global lastheartbeat
     try:
         stop_leading()
         stop_following(channel)
@@ -865,7 +899,9 @@ def follow_channel(channel):
         msg = lh.get_next_msg()
         while msg:
             sz = struct.unpack('!Q', msg)[0]
-            if sz != 0:
+            if sz == 0:
+                lastheartbeat = confluent.util.monotonic_time()
+            else:
                 rpc = b''
                 while len(rpc) < sz:
                     nrpc = channel.recv(sz - len(rpc))
@@ -1245,6 +1281,7 @@ class ConfigManager(object):
     def __init__(self, tenant, decrypt=False, username=None):
         self.clientfiles = {}
         global _cfgstore
+        self.inrestore = False
         with _initlock:
             if _cfgstore is None:
                 init()
@@ -2061,6 +2098,10 @@ class ConfigManager(object):
     def _notif_attribwatchers(self, nodeattrs):
         if self.tenant not in self._attribwatchers:
             return
+        if self.inrestore:
+            # Do not stir up attribute watchers during a collective join or DB restore,
+            # it's too hectic of a time to react
+            return
         notifdata = {}
         attribwatchers = self._attribwatchers[self.tenant]
         for node in nodeattrs:
@@ -2201,9 +2242,6 @@ class ConfigManager(object):
         self._bg_sync_to_file()
 
     def add_node_attributes(self, attribmap):
-        for node in attribmap:
-            if 'groups' not in attribmap[node]:
-                attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
 
     def rename_nodes(self, renamemap):
@@ -2335,6 +2373,8 @@ class ConfigManager(object):
                         '"{0}" is not a valid node name'.format(node))
             if autocreate is False and node not in self._cfgstore['nodes']:
                 raise ValueError("node {0} does not exist".format(node))
+            if 'groups' not in attribmap[node] and node not in self._cfgstore['nodes']:
+                attribmap[node]['groups'] = []
             for attrname in list(attribmap[node]):
                 if attrname in _attraliases:
                     truename = _attraliases[attrname]
@@ -2444,6 +2484,13 @@ class ConfigManager(object):
         #TODO: wait for synchronization to suceed/fail??)
 
     def _load_from_json(self, jsondata, sync=True):
+        self.inrestore = True
+        try:
+            self._load_from_json_backend(jsondata, sync=True)
+        finally:
+            self.inrestore = False
+
+    def _load_from_json_backend(self, jsondata, sync=True):
         """Load fresh configuration data from jsondata
 
         :param jsondata: String of jsondata
@@ -2541,6 +2588,15 @@ class ConfigManager(object):
                        data.
 
         """
+        with open(os.devnull, 'w+') as devnull:
+            worker = subprocess.Popen(
+                [sys.executable, __file__, '-r' if redact else ''],
+                        stdin=devnull, stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+            stdout, stderr = worker.communicate()
+        return stdout
+
+    def _real_dump_to_json(self, redact=None):
         dumpdata = {}
         for confarea in _config_areas:
             if confarea not in self._cfgstore:
@@ -2872,9 +2928,9 @@ def dump_db_to_directory(location, password, redact=None, skipkeys=False):
         with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
             cfgfile.write(_dump_keys(password))
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
+    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write('\n')
+        cfgfile.write(b'\n')
     if 'collective' in _cfgstore:
         with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
             cfgfile.write(json.dumps(_cfgstore['collective']))
@@ -2903,9 +2959,9 @@ def get_globals():
         bkupglobals[globvar] = _cfgstore['globals'][globvar]
     return bkupglobals
 
-
 def init(stateless=False):
     global _cfgstore
+    global _ready
     if stateless:
         _cfgstore = {}
         return
@@ -2913,8 +2969,16 @@ def init(stateless=False):
         ConfigManager._read_from_path()
     except IOError:
         _cfgstore = {}
+    members = list(list_collective())
+    if len(members) < 2:
+        _ready = True
 
 
+if __name__ == '__main__':
+    redact=None
+    if '-r' in sys.argv:
+        redact=True
+    sys.stdout.write(ConfigManager(None)._real_dump_to_json(redact))
 # some unit tests worth implementing:
 # set group attribute on lower priority group, result is that node should not
 # change

confluent_server/confluent/core.py

@@ -66,7 +66,7 @@ import eventlet.semaphore as semaphore
 import itertools
 import msgpack
 import os
-import socket
+import eventlet.green.socket as socket
 import struct
 import sys
 
@@ -446,6 +446,7 @@ def _init_core():
                 },
             },
         },
+        'layout': PluginRoute({'handler': 'layout'}),
         'media': {
             'uploads': PluginCollection({
                 'pluginattrs': ['hardwaremanagement.method'],
@@ -480,6 +481,20 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'normalized': {
+                    'inlet_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'average_cpu_temp': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'total_power': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
                 'energy': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',

confluent_server/confluent/discovery/core.py

@@ -66,6 +66,7 @@ import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 import confluent.discovery.protocols.ssdp as ssdp
+#import confluent.discovery.protocols.mdns as mdns
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
 import confluent.discovery.handlers.cpstorage as cpstorage
@@ -117,6 +118,7 @@ nodehandlers = {
     'onie-switch': None,
     'cumulus-switch': None,
     'affluent-switch': None,
+    #'openbmc': None,
     'service:io-device.Lenovo:management-module': None,
     'service:thinkagile-storage': cpstorage,
     'service:lenovo-tsm': tsm,
@@ -130,6 +132,7 @@ servicenames = {
     'service:lenovo-smm2': 'lenovo-smm2',
     'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
+    #'openbmc': 'openbmc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
     'service:thinkagile-storage': 'thinkagile-storagebmc',
@@ -226,6 +229,7 @@ def _printable_ip(sa):
 
 def send_discovery_datum(info):
     addresses = info.get('addresses', [])
+    addresses = util.natural_sort(addresses)
     if info['handler'] == pxeh:
         enrich_pxe_info(info)
     yield msg.KeyValueData({'nodename': info.get('nodename', '')})
@@ -644,6 +648,8 @@ def detected_models():
 
 
 def _recheck_nodes(nodeattribs, configmanager):
+    if not cfm.config_is_ready():
+        return
     if rechecklock.locked():
         # if already in progress, don't run again
         # it may make sense to schedule a repeat, but will try the easier and less redundant way first
@@ -762,6 +768,9 @@ def eval_detected(info):
 def detected(info):
     global rechecker
     global rechecktime
+    if not cfm.config_is_ready():
+        # drop processing of discovery data while configmanager is 'down'
+        return
     # later, manual and CMM discovery may act on SN and/or UUID
     for service in info['services']:
         if service in nodehandlers:
@@ -1229,7 +1238,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
             info['enclosure.bay'] = match[1]
             if match[2]:
                 if not discover_node(cfg, handler, info, match[2], manual):
-                    pending_nodes[match[2]] = nodename
+                    pending_nodes[match[2]] = info
                 return
         if 'enclosure.bay' not in info:
             unknown_info[info['hwaddr']] = info
@@ -1425,7 +1434,12 @@ def discover_node(cfg, handler, info, nodename, manual):
                 newnodeattribs['pubkeys.tls_hardwaremanager'] = \
                     util.get_fingerprint(handler.https_cert, 'sha256')
             if newnodeattribs:
-                cfg.set_node_attributes({nodename: newnodeattribs})
+                currattrs = cfg.get_node_attributes(nodename, newnodeattribs)
+                for checkattr in newnodeattribs:
+                    checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                    if checkval != newnodeattribs[checkattr]:
+                        cfg.set_node_attributes({nodename: newnodeattribs})
+                        break
             log.log({'info': 'Discovered {0} ({1})'.format(nodename,
                                                           handler.devname)})
             if nodeconfig:
@@ -1504,7 +1518,12 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
                 if info['hwaddr'] != oldhwaddr:
                     attribs[newattrname] = info['hwaddr']
         if attribs:
-            cfg.set_node_attributes({nodename: attribs})
+            currattrs = cfg.get_node_attributes(nodename, attribs)
+            for checkattr in attribs:
+                checkval = currattrs.get(nodename, {}).get(checkattr, {}).get('value', None)
+                if checkval != attribs[checkattr]:
+                    cfg.set_node_attributes({nodename: attribs})
+                    break
     if info['uuid'] in known_pxe_uuids:
         return True
     if uuid_is_valid(info['uuid']):
@@ -1593,7 +1612,10 @@ def remotescan():
     mycfm = cfm.ConfigManager(None)
     myname = collective.get_myname()
     for remagent in get_subscriptions():
-        affluent.renotify_me(remagent, mycfm, myname)
+        try:
+            affluent.renotify_me(remagent, mycfm, myname)
+        except Exception as e:
+            log.log({'error': 'Unexpected problem asking {} for discovery notifications'.format(remagent)})
 
 
 def blocking_scan():
@@ -1631,8 +1653,9 @@ def stop_autosense():
 
 def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
-    remotescan()
+    eventlet.spawn(remotescan)
 
 
 nodes_by_fprint = {}

confluent_server/confluent/discovery/protocols/mdns.py

@@ -0,0 +1,449 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Documented somewhat at
+# http://buildingskb.schneider-electric.com/view.php?AID=15197
+
+# Here is the payload of an SSDP 'announce', sent to the multicast v4/v6 1900
+# NOTIFY * HTTP/1.1
+# HOST: 239.255.255.250:1900
+# CACHE-CONTROL: max-age=1800
+# AL: https://172.30.254.151:8080/redfish/v1
+# SERVER: Linux/3.14.28-ltsi Redfish/1.0
+# NT: urn:dmtf-org:service:redfish-rest:1
+# USN: uuid:00000000-0000-0000-0005-000000000001::urn:dmtf-org:service:redfish-rest:1
+# NTS: ssdp:alive
+
+
+import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
+import confluent.neighutil as neighutil
+import confluent.noderange as noderange
+import confluent.util as util
+import confluent.log as log
+import confluent.netutil as netutil
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import eventlet.greenpool as gp
+import os
+import time
+import struct
+import traceback
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+mcastv4addr = '224.0.0.251'
+mcastv6addr = 'ff02::fb'
+
+mdns6mcast = socket.inet_pton(socket.AF_INET6, mcastv6addr)
+
+def name_to_qname(name):
+    nameparts = name.split('.')
+    qname = b''
+    for namepart in name.split('.'):
+        namepart = namepart.encode('utf8')
+        qname += bytes(bytearray([len(namepart)])) + namepart
+    qname += b'\x00'
+    return qname
+
+PTR = 12
+SRV = 33
+
+def _makequery(name):
+    return struct.pack('!HHHHHH', 
+            0, # transaction id
+            0, # flags, stdard query
+            1, # query count
+            0, # answers
+            0, # authorities
+            0) + \
+        name_to_qname(name) + \
+        struct.pack('!HH', 
+            PTR,
+            (1 << 15) | 1)  # Unicast response
+
+#listsrvs = _makequery('_services._dns-sd._udp.local')  # to get all possible services
+listobmccons = _makequery('_obmc_console._tcp.local')
+
+
+def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler):
+    if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']:
+        peerbymacaddress[mac]['addresses'].append(peer)
+    else:
+        sdata = _mdns_to_dict(rsp)
+        if not sdata:
+            return 0
+        peerdata = {
+            'hwaddr': mac,
+            'addresses': [peer],
+            'services': ['openbmc'],
+            'urls': '/redfish/v1/'
+        }
+        if sdata.get('ttl', 0) == 0:
+            if byehandler:
+                eventlet.spawn_n(check_fish_handler, byehandler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+            return 1
+        if handler:
+            eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+        return 2
+        
+def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer):
+    retdata = check_fish(('/redfish/v1/', peerdata))
+    if retdata:
+        known_peers.add(peer)
+        newmacs.add(mac)
+        peerbymacaddress[mac] = retdata
+        machandlers[mac] = handler
+
+
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
+    """Watch for unsolicited mDNS answers
+
+    The handler shall be called on any service coming online.
+    byehandler is called whenever a system advertises that it is departing.
+    If no byehandler is specified, byebye messages are ignored.  The handler is
+    given (as possible), the mac address, a list of viable sockaddrs to reference
+    the peer, and the notification type (e.g.
+    'urn:dmtf-org:service:redfish-rest:1'
+
+    :param handler:  A handler for online notifications from network
+    :param byehandler: Optional handler for devices going off the network
+    """
+    # Normally, I like using v6/v4 agnostic socket. However, since we are
+    # dabbling in multicast wizardry here, such sockets can cause big problems,
+    # so we will have two distinct sockets
+    # TTL=0 is a wthdrawal, otherwise an announce
+    tracelog = log.Logger('trace')
+    net4, net6 = get_sockets()
+    net6.bind(('', 5353))
+    net4.bind(('', 5353))
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                    event=log.Events.stacktrace)
+    known_peers = set([])
+    recent_peers = set([])
+    for ifidx in util.list_interface_indexes():
+        v6grp = mdns6mcast + struct.pack('=I', ifidx)
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
+    for i4 in util.list_ips():
+        mdns4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
+                     socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            mdns4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
+    peerbymacaddress = {}
+    while True:
+        try:
+            newmacs = set([])
+            deferrednotifies = []
+            machandlers = {}
+            r = select.select((net4, net6), (), (), 60)
+            if r:
+                r = r[0]
+            recent_peers = set([])
+            while r and len(deferrednotifies) < 256:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    if peer in recent_peers:
+                        continue
+                    mac = neighutil.get_hwaddr(peer[0])
+                    if not mac:
+                        probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                        try:
+                            s.sendto(b'\x00', probepeer)
+                        except Exception:
+                            continue
+                        deferrednotifies.append((peer, rsp))
+                    datum = _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+                    if datum == 2:
+                        recent_peers.add(peer)
+                r = select.select((net4, net6), (), (), 1.5)
+                if r:
+                    r = r[0]
+            if deferrednotifies:
+                eventlet.sleep(2.2)
+            for peerrsp in deferrednotifies:
+                peer, rsp = peerrsp
+                mac = neighutil.get_hwaddr(peer[0])
+                if not mac:
+                    continue
+                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
+
+
+def _get_svrip(peerdata):
+    for addr in peerdata['addresses']:
+        if addr[0].startswith('fe80::'):
+            if '%' not in addr[0]:
+                return addr[0] + '%{0}'.format(addr[3])
+            return addr[0]
+    return peerdata['addresses'][0][0]
+
+def get_sockets():
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    return net4, net6
+        
+def active_scan(handler, protocol=None):
+    net4, net6 = get_sockets()
+    for idx in util.list_interface_indexes():
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
+                        idx)
+        try:
+            net6.sendto(listobmccons, (mcastv6addr, 5353, 0, 0))
+        except socket.error:
+            # ignore interfaces without ipv6 multicast causing error
+                pass
+    for i4 in util.list_ips():
+        if 'broadcast' not in i4:
+            continue
+        addr = i4['addr']
+        net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                        socket.inet_aton(addr))
+        try:
+            net4.sendto(listobmccons, (mcastv4addr, 5353))
+        except socket.error as se:
+            if se.errno != 101 and se.errno != 1:
+                raise
+    deadline = util.monotonic_time() + 2
+    r, _, _ = select.select((net4, net6), (), (), 2)
+    peerdata = {}
+    deferparse = []
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            if not neighutil.get_hwaddr(peer[0]):
+                probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                try:
+                    s.sendto(b'\x00', probepeer)
+                except Exception:
+                    continue
+                deferparse.append((rsp, peer))
+                continue
+            _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
+    if deferparse:
+        eventlet.sleep(2.2)
+    for dp in deferparse:
+        rsp, peer = dp
+        _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+    querypool = gp.GreenPool()
+    pooltargs = []
+    for nid in peerdata:
+        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
+        pooltargs.append(('/redfish/v1/', peerdata[nid]))
+        # For now, don't interrogate generic redfish bmcs
+        # This is due to a need to deduplicate from some supported SLP
+        # targets (IMM, TSM, others)
+        # activate this else once the core filters/merges duplicate uuid
+        # or we drop support for those devices
+        #else:
+        #    pooltargs.append(('/redfish/v1/', peerdata[nid]))
+    for pi in querypool.imap(check_fish, pooltargs):
+        if pi is not None:
+            handler(pi)
+
+def check_fish(urldata, port=443, verifycallback=None):
+    if not verifycallback:
+        verifycallback = lambda x: True
+    url, data = urldata
+    try:
+        wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5)
+        peerinfo = wc.grab_json_response(url)
+    except socket.error:
+        return None
+    if url == '/DeviceDescription.json':
+        try:
+            peerinfo = peerinfo[0]
+            myuuid = peerinfo['node-uuid'].lower()
+            if '-' not in myuuid:
+                myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]])
+            data['uuid'] = myuuid
+            data['attributes'] = peerinfo
+            data['services'] = ['lenovo-xcc']
+            return data
+        except (IndexError, KeyError):
+            return None
+            url = '/redfish/v1/'
+            peerinfo = wc.grab_json_response('/redfish/v1/')
+    if url == '/redfish/v1/':
+        if 'UUID' in peerinfo:
+            if 'services' not in data:
+                data['services'] = ['service:redfish-bmc']
+            else:
+                data['services'].append('service:redfish-bmc')
+            data['uuid'] = peerinfo['UUID'].lower()
+            return data
+    return None
+
+def extract_qname(view, reply):
+    name = ''
+    idx = 1
+    if isinstance(view[0], int):
+        currlen = view[0]
+    else:
+        currlen = ord(view[0])
+    while currlen:
+        if currlen == 192:
+            name += extract_qname(reply[view[1]:], reply)[1] + '.'
+            view = view[2:]
+            idx += 1
+            if name:
+                return idx, name[:-1]
+            return idx, ''
+        else:
+            name += view[1:currlen + 1].tobytes().decode('utf8') + '.'
+            view = view[currlen + 1:]
+            idx += currlen + 1
+        if not view:
+            break  # some contexts don't null terminate
+        if isinstance(view[0], int):
+            currlen = view[0]
+        else:
+            currlen = ord(view[0])
+    if name:
+        return idx, name[:-1]
+    return idx, ''
+
+
+
+def _mdns_to_dict(rsp):
+    txid, flags, quests, answers, arr, morerr = struct.unpack('!HHHHHH', rsp[:12])
+    rv = memoryview(rsp[12:])
+    rspv = memoryview(rsp)
+    retval = {}
+    while quests:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass = struct.unpack('!HH', rv[:4]) 
+        quests -= 1
+        rv = rv[4:]
+    while answers:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass, ttl, dlen = struct.unpack('!HHIH', rv[:10])
+        if 0 and typ == 12:  # PTR, we don't need for now...
+            adata = extract_qname(rv[10:], rspv)
+            if 'ptrs' not in retval:
+                retval['ptrs'] = [{'name': adata, 'ttl': ttl}]
+            else:
+                retval['ptrs'].append({'name': adata, 'ttl': ttl})
+        if typ == 33:
+            portnum = struct.unpack('!H', rv[14:16])[0]
+            retval['protoname'] = name.split('.', 1)[1]
+            retval['portnumber'] = portnum
+            retval['ttl'] = ttl
+        rv = rv[dlen + 10:]
+        answers -= 1
+    return retval
+
+
+def _parse_mdns(peer, rsp, peerdata, srvname):
+    parsed = _mdns_to_dict(rsp)
+    if not parsed:
+        return
+    if parsed.get('ttl', 0) == 0:
+        return
+    nid = peer[0]
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    if nid in peerdata:
+        peerdatum = peerdata[nid]
+        if peer not in peerdatum['addresses']:
+            peerdatum['addresses'].append(peer)
+    else:
+        peerdatum = {
+            'addresses': [peer],
+            'hwaddr': mac,
+            'services': [srvname]
+        }
+        if srvname == '_obmc_console._tcp.local':
+            peerdatum['services'] = ['openbmc']
+            peerdatum['urls'] = ['/redfish/v1/']
+        peerdata[nid] = peerdatum
+    
+
+def _parse_ssdp(peer, rsp, peerdata):
+    nid = peer[0]
+    mac = None
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    headlines = rsp.split(b'\r\n')
+    try:
+        _, code, _ = headlines[0].split(b' ', 2)
+    except ValueError:
+        return
+    if code == b'200':
+        if nid in peerdata:
+            peerdatum = peerdata[nid]
+            if peer not in peerdatum['addresses']:
+                peerdatum['addresses'].append(peer)
+        else:
+            peerdatum = {
+                'addresses': [peer],
+                'hwaddr': mac,
+            }
+            peerdata[nid] = peerdatum
+        for headline in headlines[1:]:
+            if not headline:
+                continue
+            header, _, value = headline.partition(b':')
+            header = header.strip().decode('utf8')
+            value = value.strip().decode('utf8')
+            if header == 'AL' or header == 'LOCATION':
+                value = value[value.index('://')+3:]
+                value = value[value.index('/'):]
+                if 'urls' not in peerdatum:
+                    peerdatum['urls'] = [value]
+                elif value not in peerdatum['urls']:
+                    peerdatum['urls'].append(value)
+            elif header == 'ST':
+                if 'services' not in peerdatum:
+                    peerdatum['services'] = [value]
+                elif value not in peerdatum['services']:
+                    peerdatum['services'].append(value)
+            elif header == 'USN':
+                peerdatum['usn'] = value
+            elif header == 'MODELNAME':
+                peerdatum['modelname'] = value
+
+
+from pprint import pprint
+if __name__ == '__main__':
+    def printit(rsp):
+        print(repr(rsp))
+    snoop(pprint)

confluent_server/confluent/discovery/protocols/pxe.py

@@ -273,6 +273,29 @@ def opts_to_dict(rq, optidx, expectype=1):
 def ipfromint(numb):
     return socket.inet_ntoa(struct.pack('I', numb))
 
+def get_pxe_bootfile(node, opts, disco, profile, cfg, myip):
+    if opts.get(77, None) == b'iPXE':
+        if not profile:
+            profile = get_deployment_profile(node, cfg)
+        if not profile:
+            log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+            return None
+        bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
+    elif disco['arch'] == 'uefi-x64':
+        bootfile = b'confluent/x86_64/ipxe.efi'
+    elif disco['arch'] == 'bios-x86':
+        bootfile = b'confluent/x86_64/ipxe.kkpxe'
+    elif disco['arch'] == 'uefi-aarch64':
+        bootfile = b'confluent/aarch64/ipxe.efi'
+    if len(bootfile) > 127:
+        log.log(
+            {'info': 'Boot offer cannot be made to {0} as the '
+            'profile name "{1}" is {2} characters longer than is supported '
+            'for this boot method.'.format(
+                node, profile, len(bootfile) - 127)})
+        return None
+    return bootfile
+
 def proxydhcp(handler, nodeguess):
     net4011 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     net4011.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -297,92 +320,80 @@ def proxydhcp(handler, nodeguess):
     msg.msg_namelen = ctypes.sizeof(clientaddr)
     cfg = cfm.ConfigManager(None)
     while True:
-        ready = select.select([net4011], [], [], None)
-        if not ready or not ready[0]:
-            continue
-        i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
-        #nb, client = net4011.recvfrom_into(rq)
-        if i < 240:
-            continue
-        rqv = memoryview(rq)[:i]
-        client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
-        _, level, typ = struct.unpack('QII', cmsgarr[:16])
-        if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
-            idx, recv = struct.unpack('II', cmsgarr[16:24])
-            recv = ipfromint(recv)
         try:
-            optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
-        except ValueError:
-            continue
-        hwlen = rq[2]
-        opts, disco = opts_to_dict(rq, optidx, 3)
-        disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
-        node = None
-        if disco.get('hwaddr', None) in macmap:
-            node = macmap[disco['hwaddr']]
-        elif disco.get('uuid', None) in uuidmap:
-            node = uuidmap[disco['uuid']]
-        myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
-        skiplogging = True
-        netaddr = disco['hwaddr']
-        if time.time() > ignoredisco.get(netaddr, 0) + 90:
-            skiplogging = False
-            ignoredisco[netaddr] = time.time()
-        if not myipn:
-            info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                            'architecture': disco['arch'],
-                            'netinfo': {'ifidx': idx, 'recvip': recv},
-                            'services': ('pxe-client',)}
-            if not skiplogging:
-                handler(info)
-        if not node:
-            if not myipn and not skiplogging:
-                log.log(
-                        {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
-                            disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
-                )})
-            continue
-        profile = None
-        if not myipn:
-            myipn = socket.inet_aton(recv)
-            profile = get_deployment_profile(node, cfg)
-            if profile:
-                log.log({
-                    'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
-            else:
+            ready = select.select([net4011], [], [], None)
+            if not ready or not ready[0]:
+                continue
+            i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
+            #nb, client = net4011.recvfrom_into(rq)
+            if i < 240:
+                continue
+            rqv = memoryview(rq)[:i]
+            client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
+            _, level, typ = struct.unpack('QII', cmsgarr[:16])
+            if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+                idx, recv = struct.unpack('II', cmsgarr[16:24])
+                recv = ipfromint(recv)
+            try:
+                optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
+            except ValueError:
+                continue
+            hwlen = rq[2]
+            opts, disco = opts_to_dict(rq, optidx, 3)
+            disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
+            node = None
+            if disco.get('hwaddr', None) in macmap:
+                node = macmap[disco['hwaddr']]
+            elif disco.get('uuid', None) in uuidmap:
+                node = uuidmap[disco['uuid']]
+            myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
+            skiplogging = True
+            netaddr = disco['hwaddr']
+            if time.time() > ignoredisco.get(netaddr, 0) + 90:
+                skiplogging = False
+                ignoredisco[netaddr] = time.time()
+            if not myipn:
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                                'architecture': disco['arch'],
+                                'netinfo': {'ifidx': idx, 'recvip': recv},
+                                'services': ('pxe-client',)}
                 if not skiplogging:
-                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    handler(info)
+            if not node:
+                if not myipn and not skiplogging:
+                    log.log(
+                            {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
+                                disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
+                    )})
                 continue
-        if opts.get(77, None) == b'iPXE':
-            if not profile:
+            profile = None
+            if not myipn:
+                myipn = socket.inet_aton(recv)
+                myip = socket.inet_ntoa(myipn)
                 profile = get_deployment_profile(node, cfg)
-            if not profile:
-                log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
-                continue
+                if profile:
+                    log.log({
+                        'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
+                else:
+                    if not skiplogging:
+                        log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
             myip = socket.inet_ntoa(myipn)
-            bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
-        elif disco['arch'] == 'uefi-x64':
-            bootfile = b'confluent/x86_64/ipxe.efi'
-        elif disco['arch'] == 'bios-x86':
-            bootfile = b'confluent/x86_64/ipxe.kkpxe'
-        elif disco['arch'] == 'uefi-aarch64':
-            bootfile = b'confluent/aarch64/ipxe.efi'
-        if len(bootfile) > 127:
-            log.log(
-                {'info': 'Boot offer cannot be made to {0} as the '
-                'profile name "{1}" is {2} characters longer than is supported '
-                'for this boot method.'.format(
-                    node, profile, len(bootfile) - 127)})
-            continue
-        rpv[:240] = rqv[:240].tobytes()
-        rpv[0:1] = b'\x02'
-        rpv[108:108 + len(bootfile)] = bootfile
-        rpv[240:243] = b'\x35\x01\x05'
-        rpv[243:249] = b'\x36\x04' + myipn
-        rpv[20:24] = myipn
-        rpv[249:268] = b'\x61\x11' + opts[97]
-        rpv[268:280] = b'\x3c\x09PXEClient\xff'
-        net4011.sendto(rpv[:281], client)
+            bootfile = get_pxe_bootfile(node, opts, disco, profile, cfg, myip)
+            if not bootfile:
+                continue
+            rpv[:240] = rqv[:240].tobytes()
+            rpv[0:1] = b'\x02'
+            rpv[108:108 + len(bootfile)] = bootfile
+            rpv[240:243] = b'\x35\x01\x05'
+            rpv[243:249] = b'\x36\x04' + myipn
+            rpv[20:24] = myipn
+            rpv[249:268] = b'\x61\x11' + opts[97]
+            rpv[268:280] = b'\x3c\x09PXEClient\xff'
+            net4011.sendto(rpv[:281], client)
+        except Exception as e:
+            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                            event=log.Events.stacktrace)
 
 
 def start_proxydhcp(handler, nodeguess=None):
@@ -394,6 +405,7 @@ def snoop(handler, protocol=None, nodeguess=None):
     #prominent
     #TODO(jjohnson2): enable unicast replies. This would suggest either
     # injection into the neigh table before OFFER or using SOCK_RAW.
+    global tracelog
     start_proxydhcp(handler, nodeguess)
     tracelog = log.Logger('trace')
     global attribwatcher
@@ -495,7 +507,7 @@ def process_dhcp6req(handler, rqv, addr, net, cfg, nodeguess):
     if ignoredisco.get(mac, 0) + 90 < time.time():
         ignoredisco[mac] = time.time()
         handler(info)
-    consider_discover(info, req, net, cfg, None, nodeguess, addr)
+    consider_discover(info, req, net, cfg, None, nodeguess, disco, addr)
 
 def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv):
     rq = bytearray(rqv)
@@ -535,7 +547,7 @@ def process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv):
                             and time.time() > ignoredisco.get(netaddr, 0) + 90):
         ignoredisco[netaddr] = time.time()
         handler(info)
-    consider_discover(info, rqinfo, net4, cfg, rqv, nodeguess)
+    consider_discover(info, rqinfo, net4, cfg, rqv, nodeguess, disco)
 
 
 
@@ -590,7 +602,7 @@ def get_deployment_profile(node, cfg, cfd=None):
 
 staticassigns = {}
 myipbypeer = {}
-def check_reply(node, info, packet, sock, cfg, reqview, addr):
+def check_reply(node, info, packet, sock, cfg, reqview, addr, disco):
     httpboot = info['architecture'] == 'uefi-httpboot'
     cfd = cfg.get_node_attributes(node, ('deployment.*', 'collective.managercandidates'))
     profile = get_deployment_profile(node, cfg, cfd)
@@ -607,7 +619,7 @@ def check_reply(node, info, packet, sock, cfg, reqview, addr):
             return
         return reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock)
     else:
-        return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile)
+        return reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, disco)
 
 def reply_dhcp6(node, addr, cfg, packet, cfd, profile, sock):
     myaddrs = netutil.get_my_addresses(addr[-1], socket.AF_INET6)
@@ -691,7 +703,7 @@ def get_my_duid():
     return _myuuid
 
 
-def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile):
+def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile, disco):
     replen = 275  # default is going to be 286
     # while myipn is describing presumed destination, it's really
     # vague in the face of aliases, need to convert to ifidx and evaluate
@@ -729,7 +741,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile):
         log.log({'error': nicerr})
     if niccfg.get('ipv4_broken', False):
         # Received a request over a nic with no ipv4 configured, ignore it
-        log.log({'error': 'Skipping boot reply to {0} due to no viable IPv4 configuration on deployment system'.format(node)})
+        log.log({'error': 'Skipping boot reply to {0} due to no viable IPv4 configuration on deployment system, over nic "{}"'.format(node, info['netinfo']['ifidx'])})
         return
     clipn = None
     if niccfg['ipv4_method'] == 'firmwarenone':
@@ -782,6 +794,7 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile):
     repview[249:255] = b'\x33\x04\x00\x00\x00\xf0'  # fixed short lease time
     repview[255:257] = b'\x61\x11'
     repview[257:274] = packet[97]
+
     # Note that sending PXEClient kicks off the proxyDHCP procedure, ignoring
     # boot filename and such in the DHCP packet
     # we will simply always do it to provide the boot payload in a consistent
@@ -790,6 +803,9 @@ def reply_dhcp4(node, info, packet, cfg, reqview, httpboot, cfd, profile):
         repview[replen - 1:replen + 11] = b'\x3c\x0aHTTPClient'
         replen += 12
     else:
+        bootfile = get_pxe_bootfile(node, packet, disco, profile, cfg, myip)
+        if bootfile:
+            repview[108:108 + len(bootfile)] = bootfile
         repview[replen - 1:replen + 10] = b'\x3c\x09PXEClient'
         replen += 11
     hwlen = bytearray(reqview[2:3].tobytes())[0]
@@ -881,11 +897,11 @@ def ack_request(pkt, rq, info):
     repview[26:28] = struct.pack('!H', datasum)
     send_raw_packet(repview, len(rply), rq, info)
 
-def consider_discover(info, packet, sock, cfg, reqview, nodeguess, addr=None):
+def consider_discover(info, packet, sock, cfg, reqview, nodeguess, disco, addr=None):
     if info.get('hwaddr', None) in macmap and info.get('uuid', None):
-        check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview, addr)
+        check_reply(macmap[info['hwaddr']], info, packet, sock, cfg, reqview, addr, disco)
     elif info.get('uuid', None) in uuidmap:
-        check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr)
+        check_reply(uuidmap[info['uuid']], info, packet, sock, cfg, reqview, addr, disco)
     elif packet.get(53, None) == b'\x03':
         ack_request(packet, reqview, info)
     elif info.get('uuid', None) and info.get('hwaddr', None):