Fix hotplug firmware to use rootfs after boot

Confignet does not set autoconnect with network manager

confluent_client/bin/nodeapply

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+devnull = None
+
+def run():
+    global devnull
+    devnull = open(os.devnull, 'rb')
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-k', '--security', action='store_true',
+                         help='Update SSH setup')
+    argparser.add_option('-F', '--sync', action='store_true',
+                         help='Run the syncfiles associated with the currently completed OS profile on the noderange')
+    argparser.add_option('-P', '--scripts',
+                         help='Re-run specified scripts, with full path under scripts, e.g. post.d/first,firstboot.d/second')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                    help='Specify a maximum number of '
+                        'nodes to run remote ssh command to, '
+                        'prompting if over the threshold')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    #argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    c.stop_if_noderange_over(args[0], options.maxnodes)
+    nodemap = {}
+    cmdparms = []
+    nodes = []
+    for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode |= res.get('errorcode', 1)
+            break
+        node = res['item']['href'][:-1]
+        nodes.append(node)
+    
+    cmdstorun = []
+    if options.security:
+        cmdstorun.append(['run_remote', 'setupssh'])
+    if options.sync:
+        cmdstorun.append(['run_remote_python', 'syncfileclient'])
+    if options.scripts:
+        for script in options.scripts.split(','):
+            cmdstorun.append(['run_remote', script])
+    if not cmdstorun:
+        argparser.print_help()
+        sys.exit(1)
+    idxbynode = {}
+    cmdvbase = ['bash', '/etc/confluent/functions']
+    for sshnode in nodes:
+        idxbynode[sshnode] = 1
+        cmdv = ['ssh', sshnode] + cmdvbase + cmdstorun[0]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(node, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((node, cmdv))        
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        data = client.stringify(data)
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if idxbynode[node] < len(cmdstorun):
+                                cmdv =  ['ssh', sshnode] + cmdvbase + cmdstorun[idxbynode[node]]
+                                idxbynode[node] += 1
+                                run_cmdv(node, cmdv, all, pipedesc)
+                            elif pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                line = line.lstrip('\x08')
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -23,6 +23,7 @@ path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
+
 import confluent.client as client
 import confluent.sortutil as sortutil
 import confluent.logreader as logreader
@@ -41,6 +42,10 @@ argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
 argparser.add_option('-l', '--log', action='store_true', default=False,
                      help='Enter log replay mode instead of showing a live console')
+
+argparser.add_option('-T', '--Timestamp', action='store_true', default=False,
+                     help= 'Dump log in stdout with timestamps')
+
 argparser.add_option('-w','--windowed', action='store_true', default=False,
                      help='Open terminal windows for each node.  The '
                           'environment variable NODECONSOLE_WINDOWED_COMMAND '
@@ -91,6 +96,16 @@ if options.log:
     logreader.replay_to_console(logname)
     sys.exit(0)
 
+if options.Timestamp:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.dump_to_console(logname)
+    sys.exit(0)
+
 def kill(noderange):
     sess = client.Command()
     envstring=os.environ.get('NODECONSOLE_WINDOWED_COMMAND')
@@ -106,12 +121,20 @@ def kill(noderange):
         nodes.append(node) 
 
     for node in nodes:
-        ps_data=subprocess.Popen(['ps', '-auxww' ], stdout=subprocess.PIPE)
-        wintr=ps_data.communicate()[0]
-        for line in wintr.decode('utf-8').split('\n'):
-            if confettypath in line and envstring in line and node in line:
-                pid_line = [x for x in line.split(' ') if x != '']
-                ps_data=subprocess.Popen(['kill', '-9', pid_line[1] ], stdout=subprocess.PIPE)
+        s=socket.socket(socket.AF_UNIX)
+        winid=None
+        try:
+            win=subprocess.Popen(['xwininfo', '-tree', '-root'], stdout=subprocess.PIPE)
+            wintr=win.communicate()[0]
+            for line in wintr.decode('utf-8').split('\n'):
+                if 'console: {0}'.format(node) in line or 'confetty' in line:
+                    win_obj = [ele for ele in line.split(' ') if ele.strip()]
+                    winid = win_obj[0]
+        except:
+            print("Error: cannot retrieve window id of node {}".format(node))
+
+        if winid:
+            ps_data=subprocess.Popen(['xkill', '-id', winid ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     sys.exit(0)
 
@@ -220,7 +243,7 @@ if options.windowed:
             elif 'Height' in line:
                 window_height = int(line.split(':')[1])
             elif '-geometry' in line:
-                l = re.split(' |x|\+', line)
+                l = re.split(' |x|-|\+', line)
                 l_nosp = [ele for ele in l if ele.strip()]
                 wmxo = int(l_nosp[1])
                 wmyo = int(l_nosp[2])
@@ -259,7 +282,7 @@ if options.windowed:
         else:
             pass 
         with open(os.devnull, 'wb') as devnull:
-            xopen=subprocess.Popen(envlist + [confettypath, '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
+            xopen=subprocess.Popen(envlist + [confettypath, '-c', '/tmp/controlpath-{0}'.format(node), '-m', '5', 'start', '/nodes/{0}/console/session'.format(node)] , stdin=devnull)
     sys.exit(0)
 #end of wcons
 if options.tile:
@@ -273,8 +296,13 @@ if options.tile:
             sys.exit(1)
         nodes.append(node)
     initial = True
+    in_tmux = False
     pane = 0
     sessname = 'nodeconsole_{0}'.format(os.getpid())
+    if os.environ.get("TMUX"):
+        initial = False
+        in_tmux = True
+        subprocess.call(['tmux', 'rename-session', sessname])
     for node in sortutil.natural_sort(nodes):
         panename = '{0}:{1}'.format(sessname, pane)
         if initial:
@@ -295,7 +323,8 @@ if options.tile:
             pane += 1
     subprocess.call(['tmux', 'select-pane', '-t', sessname])
     subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
+    if not in_tmux:
+        os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
-             '/nodes/{0}/console/session'.format(args[0]))
+             '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/libexec/platform-python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -15,13 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__author__ = 'jjohnson2,alin37'
+__author__ = 'jjohnson2,alin37,wgrzeda'
 
 import optparse
 import os
 import signal
 import sys
 
+
+
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 except AttributeError:
@@ -39,9 +41,12 @@ def main():
           "   or: %prog [options] noderange <nodeattribute>...")
     argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
+    argparser.add_option('-d', '--delim', metavar="STRING", default = "\n",
+                     help='Delimiter separating the values')
     (options, args) = argparser.parse_args()
     noderange=""
     nodelist=""
+    list = []
     try:
         noderange = args[0]
         nodelist = '/noderange/{0}/nodes/'.format(noderange)
@@ -61,7 +66,9 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print(res['item']['href'].replace('/', ''))
+                elem=(res['item']['href'].replace('/', ''))
+                list.append(elem)
+    print(options.delim.join(list))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodesensors

@@ -141,7 +141,7 @@ def sensorpass(showout=True, appendtime=False):
                         if sensedata['value'] is None:
                             showval = ''
                         elif isinstance(sensedata['value'], float):
-                            showval = floatformat(sensedata['value'])
+                            showval = u' {0} '.format(floatformat(sensedata['value']))
                         else:
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):

confluent_client/confluent/logreader.py

@@ -296,5 +296,47 @@ def replay_to_console(txtfile):
             sys.exit(1)
     _replay_to_console(txtfile, binfile)
 
+def dump_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    elif '.' not in txtfile:
+        if '/' not in txtfile:
+            txtfile = os.getcwd() + '/' + txtfile
+        sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(txtfile + '.cbl'))
+        sys.exit(1)
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file: "{0}"\n'.format(binfile))
+            sys.exit(1)
+    replay = LogReplay(txtfile, binfile)
+    quitit = False
+    writeout('\x1b[2J\x1b[;H')
+    prepend = ''
+    try:
+        while not quitit:
+            newdata, delay = replay.get_output(False)
+            if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
+                writeout(newdata.replace(b'\r\n', '\r\n[ {0} ] '.format(time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))).encode('utf8')))
+                newdata = ''
+                try:
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                except IOError:
+                    pass
+            else:
+                break
+    except Exception:
+        writeout('\x1b[m\x1b[?25h\n')
+        raise
+    writeout('\x1b[m\x1b[?25h\n')
+
 if __name__ == '__main__':
     replay_to_console(sys.argv[1])

confluent_client/doc/man/nodeconsole.ronn

@@ -29,6 +29,11 @@ console process which will result in the console window closing.
   Perform a log reply on the current, local log in /var/log/confluent/consoles.
   If in collective mode, this only makes sense to use on the current collective
   manager at this time.
+
+  * `-T`, `--Timestamp`:
+ Dump the log with Timpstamps on the current, local log in /var/log/confluent/consoles.
+  If in collective mode, this only makes sense to use on the current collective
+  manager at this time.
   
 * `-w`, `--windowed`:
   Open terminal windows for each node.  The

confluent_client/doc/man/nodelist.ronn

@@ -4,7 +4,7 @@ nodelist(8) -- List confluent nodes and their attributes
 ## SYNOPSIS
 
 `nodelist <noderange>`  
-`nodelist <noderange> [-b] <nodeattribute>...`  
+`nodelist <noderange> [-b] [-d] {string} <nodeattribute>...`  
 
 ## DESCRIPTION
 
@@ -24,7 +24,8 @@ all attributes that begin with `net.` and end with `switch`.
 
 * `-b`, `--blame`:
   Annotate inherited and expression based attributes to show their base value.
-
+* `-d`, `--delim`:
+  Choose a delimiter to separat the values. Default - ENTER.
 ## EXAMPLES
 * Listing matching nodes of a simple noderange:
   `# nodelist n1-n4`  

confluent_osdeploy/buildrpm

@@ -11,13 +11,24 @@ mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
 cd -
 mkdir -p el9bin/opt/confluent/bin
 mkdir -p el9bin/stateless-bin
-podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuilder make -C /buildutils
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuild make -C /buildutils
 cd utils
-mv confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
-mv start_root urlmount ../el9bin/stateless-bin/
+cp confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
+cp start_root urlmount ../el9bin/stateless-bin/
+make clean
+cd ..
+mkdir -p el8bin/opt/confluent/bin
+mkdir -p el8bin/stateless-bin
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t el7build make -C /buildutils
+cd utils
+cp confluent_imginfo copernicus clortho autocons ../el8bin/opt/confluent/bin
+cp start_root urlmount ../el8bin/stateless-bin/
 make clean
 cd ..
 tar Jcvf confluent_el9bin.tar.xz el9bin/
+tar Jcvf confluent_el8bin.tar.xz el8bin/
+mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
+rm -rf el8bin
 rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -21,6 +21,10 @@ class InvalidApiKey(Exception):
 def msg_align(len):
     return (len + 3) & ~3
 
+try:
+    _protoparm = ssl.PROTOCOL_TLS_CLIENT
+except AttributeError:    
+    _protoparm = ssl.PROTOCOL_SSLv23
 cryptname = ctypes.util.find_library('crypt')
 if not cryptname:
     if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
@@ -223,7 +227,7 @@ def get_apikey(nodename, hosts, errout=None):
     return apikey
 
 class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None):
+    def __init__(self, usejson=False, port=443, host=None, errout=None, phmac=None, checkonly=False):
         self.phmac = phmac
         self.errout = None
         if errout:
@@ -287,7 +291,7 @@ class HTTPSClient(client.HTTPConnection, object):
         if self.phmac:
             with open(phmac, 'r') as hmacin:
                 self.stdheaders['CONFLUENT_CRYPTHMAC'] = hmacin.read()
-        else:
+        elif not checkonly:
             self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
@@ -305,7 +309,7 @@ class HTTPSClient(client.HTTPConnection, object):
     def check_connections(self):
         foundsrv = None
         hosts = self.hosts
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         ctx.verify_mode = ssl.CERT_REQUIRED
         ctx.check_hostname = True
@@ -321,7 +325,7 @@ class HTTPSClient(client.HTTPConnection, object):
                     foundsrv = host
                     psock.close()
                     break
-                except OSError:
+                except (OSError, socket.error):
                     continue
                 except ssl.SSLError:
                     continue
@@ -353,11 +357,12 @@ class HTTPSClient(client.HTTPConnection, object):
                 raise Exception('Unable to reach any hosts')
         return foundsrv
 
-    def connect(self):
+    def connect(self, tries=3):
         addrinf = socket.getaddrinfo(self.host, self.port)[0]
         psock = socket.socket(addrinf[0])
+        psock.settimeout(15)
         psock.connect(addrinf[4])
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx = ssl.SSLContext(_protoparm)
         ctx.load_verify_locations('/etc/confluent/ca.pem')
         host = self.host.split('%', 1)[0]
         if '[' not in host and ':' in host:
@@ -375,6 +380,10 @@ class HTTPSClient(client.HTTPConnection, object):
                 self.errout.write(errmsg)
                 self.errout.flush()
             sys.exit(1)
+        except socket.timeout:
+            if not tries:
+                raise
+            return self.connect(tries=tries-1)
 
     def grab_url(self, url, data=None, returnrsp=False):
         return self.grab_url_with_status(url, data, returnrsp)[1]
@@ -459,7 +468,7 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    client = HTTPSClient(usejson, errout=errout, phmac=phmac)
+    client = HTTPSClient(usejson, errout=errout, phmac=phmac, checkonly=checkonly)
     if waitfor:
         status = 201
         while status != waitfor:

confluent_osdeploy/common/profile/scripts/confignet

@@ -253,6 +253,7 @@ class NetworkManager(object):
 
     def apply_configuration(self, cfg):
         cmdargs = {}
+        cmdargs['connection.autoconnect'] = 'yes'
         stgs = cfg['settings']
         cmdargs['ipv6.method'] = stgs.get('ipv6_method', 'link-local')
         if stgs.get('ipv6_address', None):
@@ -316,7 +317,10 @@ if __name__ == '__main__':
     srvs, _ = apiclient.scan_confluents()
     doneidxs = set([])
     for srv in srvs:
-        s = socket.create_connection((srv, 443))
+        try:
+            s = socket.create_connection((srv, 443))
+        except socket.error:
+            continue
         myname = s.getsockname()
         s.close()
         if len(myname) == 4:

confluent_osdeploy/common/profile/scripts/setupssh

@@ -0,0 +1,31 @@
+[ -f /lib/confluent/functions ] && . /lib/confluent/functions
+[ -f /etc/confluent/functions ] && . /etc/confluent/functions
+[ -f /opt/confluent/bin/apiclient ] && confapiclient=/opt/confluent/bin/apiclient
+[ -f /etc/confluent/apiclient ] && confapiclient=/etc/confluent/apiclient
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    rm $certfile
+    confluentpython $confapiclient /confluent-api/self/sshcert $pubkey -o $certfile
+done
+TMPDIR=$(mktemp -d)
+cd $TMPDIR
+confluentpython $confapiclient /confluent-public/site/initramfs.tgz -o initramfs.tgz
+tar xf initramfs.tgz
+for ca in ssh/*.ca; do
+	LINE=$(cat $ca)
+	cp -af /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts.new
+	grep -v "$LINE" /etc/ssh/ssh_known_hosts > /etc/ssh/ssh_known_hosts.new
+	echo '@cert-authority *' $LINE >> /etc/ssh/ssh_known_hosts.new
+	mv /etc/ssh/ssh_known_hosts.new /etc/ssh/ssh_known_hosts
+done
+for pubkey in ssh/*.*pubkey; do
+	LINE=$(cat $pubkey)
+	cp -af /root/.ssh/authorized_keys /root/.ssh/authorized_keys.new
+	grep -v "$LINE" /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.new
+	echo "$LINE" >> /root/.ssh/authorized_keys.new
+	mv /root/.ssh/authorized_keys.new /root/.ssh/authorized_keys
+done
+confluentpython $confapiclient /confluent-api/self/nodelist | sed -e 's/^- //' > /etc/ssh/shosts.equiv
+cat /etc/ssh/shosts.equiv > /root/.shosts
+cd -
+rm -rf $TMPDIR

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -7,6 +7,7 @@ License: Apache2
 URL: https://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
 Source1: confluent_el9bin.tar.xz
+Source2: confluent_el8bin.tar.xz
 BuildArch: noarch
 Requires: confluent_ipxe mtools tar
 BuildRoot: /tmp
@@ -18,24 +19,22 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 %define debug_package %{nil}
 
 %prep
-%setup -n confluent_osdeploy -a 1
+%setup -n confluent_osdeploy -a 2 -a 1
 
 %build
-mkdir -p opt/confluent/bin
-mkdir -p stateless-bin
-cd utils
-make all
-cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
-cp start_root urlmount ../stateless-bin/
-cd ..
+#cd utils
+#make all
+#cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
+#cp start_root urlmount ../stateless-bin/
+#cd ..
 ln -s el8 el9
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu18.04 ubuntu20.04 ubuntu22.04 coreos el9; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}/initramfs/* .
     cp -a ../common/initramfs/* .
@@ -49,14 +48,14 @@ for os in el7 el8 suse15 el9 ubuntu20.04; do
     if [ -d ../${os}bin ]; then
         cp -a ../${os}bin/opt .
     else
-        cp -a ../opt .
+        cp -a ../el8bin/opt .
     fi
     cp -a ../${os}-diskless/initramfs/* .
     cp -a ../common/initramfs/* .
     if [ -d ../${os}bin ]; then 
         cp -a ../${os}bin/stateless-bin/* opt/confluent/bin
     else
-        cp -a ../stateless-bin/* opt/confluent/bin
+        cp -a ../el8bin/stateless-bin/* opt/confluent/bin
     fi
     find . | cpio -H newc -o > ../addons.cpio
     mv ../addons.cpio .
@@ -64,7 +63,7 @@ for os in el7 el8 suse15 el9 ubuntu20.04; do
 done
 mkdir esxi7out
 cd esxi7out
-cp -a ../opt .
+cp -a ../el8bin/opt .
 cp -a ../esxi7/initramfs/* .
 cp -a ../common/initramfs/* .
 chmod +x bin/* opt/confluent/bin/*
@@ -79,7 +78,7 @@ cp -a esxi7 esxi8
 %install
 mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
 cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu18.04 ubuntu22.04 esxi6 esxi7 esxi8 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs

confluent_osdeploy/debian/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -69,7 +69,7 @@ if [ "$v4cfgmeth" = "static" ]; then
     for namesrv in "$namesrvs"; do
         setdebopt netcfg/get_nameservers $namesrv string
     done
-elif [ "$vpcfgmeth" = "dhcp" ]; then
+elif [ "$v4cfgmeth" = "dhcp" ]; then
     setdebopt netcfg/disable_dhcp false boolean
     setdebopt netcfg/confirm_static false boolean
 fi

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -125,4 +125,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -7,41 +7,127 @@ if [ -f /tmp/dd_disk ]; then
         fi
     done
 fi
-mkdir -p /etc/confluent
-cat /tls/*.pem > /etc/confluent/ca.pem
-TRIES=0
-echo -n > /etc/confluent/confluent.info
-cd /sys/class/net
-while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
-    TRIES=$((TRIES + 1))
-    for currif in *; do
-        ip link set $currif up
-    done
-    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
-done
-cd /
-grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
-echo -n "" > /tmp/confluent.initq
-# restart cmdline
-echo -n "" > /etc/cmdline.d/01-confluent.conf
-nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
-#TODO: blkid --label <whatever> to find mounted api
-
+shutdownnic=""
 oum=$(umask)
-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-ifidx=$(cat /tmp/confluent.ifidx)
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-echo $ifname > /tmp/net.ifaces
+umask 0077
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
 
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/hmackeyfile
+    passfile=/etc/confluent/confluent.apikey
+    passcrypt=/tmp/passcrypt
+    hmacfile=/tmp/hmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile;
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    echo "NODENAME: "$nodename > /etc/confluent/confluent.info
+    for dsrv in $deploysrvs; do
+        echo 'MANAGER: '$dsrv >> /etc/confluent/confluent.info
+    done
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    sleep 10
+    usedhcp=0
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            usedhcp=1
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            TESTSRV=$(python /opt/confluent/bin/apiclient -c 2> /dev/null)
+            if [ ! -z "$TESTSRV" ]; then
+                python /opt/confluent/bin/apiclient -p $hmacfile /confluent-api/self/registerapikey $passcrypt
+                mgr=$TESTSRV
+                ifname=$NICGUESS
+                shutdownnic=$ifname
+                break
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip -4 addr flush dev $NICGUESS
+        fi
+    done
+fi
+TRIES=0
+if [ "$usedhcp" = 1 ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+elif [ -z "$ifname" ]; then
+    cd /sys/class/net
+    while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    cd /
+    grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+    echo -n "" > /etc/cmdline.d/01-confluent.conf
+else
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none > /etc/cmdline.d/01-confluent.conf
+fi
+python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+if [ ! -z "$shutdownnic" ]; then
+    if [ ! -z "$v4gw" ]; then
+        ip route del default via $v4gw
+    fi
+    ip -4 addr flush dev $shutdownnic
+fi
+# restart cmdline
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+echo -n "" > /tmp/confluent.initq
+if [ -z "$ifname"  ]; then
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+if [ ! -z "$ifname" ]; then
+    echo $ifname > /tmp/net.ifaces
+fi
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 hostname=$nodename
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
     hostname=$hostname.$dnsdomain
 fi
-mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
-mgr=${mgr#deploy_server: }
+if [ -z "$mgr" ]; then
+    mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+    mgr=${mgr#deploy_server: }
+fi
 profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 profilename=${profilename#profile: }
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
@@ -65,21 +151,23 @@ kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
 root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
 export kickstart
 export root
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
-autoconfigmethod=${autoconfigmethod#ipv4_method: }
-if [ "$autoconfigmethod" = "dhcp" ]; then
-    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
-else
-    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
-    v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
-    v4gw=${v4gw#ipv4_gateway: }
-    if [ "$v4gw" = "null" ]; then
-        v4gw=""
+if [ -z "$autoconfigmethod" ]; then
+    autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+    else
+        v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
     fi
-    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
-    v4nm=${v4nm#ipv4_netmask: }
-    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
 nameserversec=0
 while read -r entry; do

confluent_osdeploy/el7/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,8 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +55,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +130,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +146,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -184,6 +220,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +261,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -366,6 +403,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +418,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-trigger/01-confluent.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
 . /lib/dracut-lib.sh
+setsid sh -c 'exec bash <> /dev/tty2 >&0 2>&1' &
 udevadm trigger
 udevadm trigger --type=devices --action=add
 udevadm settle
@@ -40,7 +41,7 @@ mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
 TRIES=5
-while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
     sleep 2
     TRIES=$((TRIES - 1))
 done
@@ -55,21 +56,49 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
     sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
     autoconfigmethod=$(grep ^ipv4_method: $tcfg)
     autoconfigmethod=${autoconfigmethod#ipv4_method: }
-    if [ "$autoconfigmethod" = "dhcp" ]; then
-        /usr/libexec/nm-initrd-generator ip=:dhcp
-    else
-        v4addr=$(grep ^ipv4_address: $tcfg)
-        v4addr=${v4addr#ipv4_address: }
-        v4addr=${v4addr%/*}
-        v4gw=$(grep ^ipv4_gateway: $tcfg)
-        v4gw=${v4gw#ipv4_gateway: }
-        if [ "$v4gw" = "null" ]; then
-            v4gw=""
+     for i in /sys/class/net/*; do
+        ip link set $(basename $i) down
+        udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+        ip link set $(basename $i) up
+    done
+    for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK| awk '{print $2}' | sed -e 's/:$//'); do
+        if [ "$autoconfigmethod" = "dhcp" ]; then
+            /usr/libexec/nm-initrd-generator ip=$NICGUESS:dhcp
+        else
+            v4addr=$(grep ^ipv4_address: $tcfg)
+            v4addr=${v4addr#ipv4_address: }
+            v4plen=${v4addr#*/}
+            v4addr=${v4addr%/*}
+            v4gw=$(grep ^ipv4_gateway: $tcfg)
+            v4gw=${v4gw#ipv4_gateway: }
+            ip addr add dev $NICGUESS $v4addr/$v4plen
+            if [ "$v4gw" = "null" ]; then
+                v4gw=""
+            fi
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            v4nm=$(grep ipv4_netmask: $tcfg)
+            v4nm=${v4nm#ipv4_netmask: }
+            DETECTED=0
+            for dsrv in $deploysrvs; do
+                if curl --capath /tls/ -s --connect-timeout 3 https://$dsrv/confluent-public/ > /dev/null; then
+                    rm /run/NetworkManager/system-connections/*
+                    /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname:$NICGUESS:none
+                    DETECTED=1
+                    ifname=$NICGUESS
+                    break
+                fi
+            done
+            if [ ! -z "$v4gw" ]; then
+                ip route del default via $v4gw
+            fi
+            ip addr flush dev $NICGUESS
+            if [ $DETECTED = 1 ]; then
+                break
+            fi
         fi
-        v4nm=$(grep ipv4_netmask: $tcfg)
-        v4nm=${v4nm#ipv4_netmask: }
-        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
-    fi
+    done
     NetworkManager --configure-and-quit=initrd --no-daemon
     hmackeyfile=/tmp/cnflnthmackeytmp
     echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
@@ -85,10 +114,10 @@ if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
         echo 'MANAGER: '$deploysrv >> /etc/confluent/confluent.info
     done
     for deployer in $deploysrvs; do
-        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+        if curl --capath /tls/ -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
             cp $passfile /etc/confluent/confluent.apikey
             confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
+            curl --capath /tls/ -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg2 > /etc/confluent/confluent.deploycfg
             confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
             confluent_profile=${confluent_profile#profile: }
             mgr=$deployer
@@ -114,7 +143,7 @@ if ! grep MANAGER: /etc/confluent/confluent.info; then
         for mac in $(ip -br link|grep -v LOOPBACK|awk '{print $3}'); do
             myids=$myids"/mac="$mac
         done
-        myname=$(curl -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
+        myname=$(curl --capath /tls/ -sH "CONFLUENT_IDS: $myids" https://$confluenthttpsrv/confluent-api/self/whoami)
         if [ ! -z "$myname" ]; then
             echo NODENAME: $myname > /etc/confluent/confluent.info
             echo MANAGER: $confluentsrv >> /etc/confluent/confluent.info
@@ -149,14 +178,16 @@ while ! confluentpython /opt/confluent/bin/apiclient $errout /confluent-api/self
 	sleep 10
 done
 ifidx=$(cat /tmp/confluent.ifidx 2> /dev/null)
-if [ ! -z "$ifidx" ]; then
-    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-    ifname=${ifname%:}
-    ifname=${ifname%@*}
-    echo $ifname > /tmp/net.ifaces
-else
-    ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
-    ifname=$(cat /tmp/net.ifaces)
+if [ -z "$ifname" ]; then
+    if [ ! -z "$ifidx" ]; then
+        ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+        ifname=${ifname%:}
+        ifname=${ifname%@*}
+        echo $ifname > /tmp/net.ifaces
+    else
+        ip -br a|grep UP|awk '{print $1}' > /tmp/net.ifaces
+        ifname=$(cat /tmp/net.ifaces)
+    fi
 fi
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)

confluent_osdeploy/el8/profiles/default/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/sh
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script is executed on the first boot after install has
 # completed. It is best to edit the middle of the file as

confluent_osdeploy/el9-diskless/profiles/default/scripts/firstboot.sh

@@ -5,6 +5,8 @@
 # noted below so custom commands are executed before
 # the script notifies confluent that install is fully complete.
 
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
 confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')

confluent_osdeploy/el9-diskless/profiles/default/scripts/image2disk.py

@@ -11,6 +11,8 @@ import struct
 import sys
 import subprocess
 
+bootuuid = None
+
 def get_next_part_meta(img, imgsize):
     if img.tell() == imgsize:
         return None
@@ -53,10 +55,13 @@ class PartedRunner():
     def __init__(self, disk):
         self.disk = disk
 
-    def run(self, command):
+    def run(self, command, check=True):
         command = command.split()
         command = ['parted', '-a', 'optimal', '-s', self.disk] + command
-        return subprocess.check_output(command).decode('utf8')
+        if check:
+            return subprocess.check_output(command).decode('utf8')
+        else:
+            return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode('utf8')
 
 def fixup(rootdir, vols):
     devbymount = {}
@@ -125,9 +130,14 @@ def fixup(rootdir, vols):
         sys.stdout.flush()
     for metafs in ('proc', 'sys', 'dev'):
         subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+    if os.path.exists(os.path.join(rootdir, 'etc/lvm/devices/system.devices')):
+        os.remove(os.path.join(rootdir, 'etc/lvm/devices/system.devices'))
+    grubsyscfg = os.path.join(rootdir, 'etc/sysconfig/grub')
+    if not os.path.exists(grubsyscfg):
+        grubsyscfg = os.path.join(rootdir, 'etc/default/grub')
+    with open(grubsyscfg) as defgrubin:
         defgrub = defgrubin.read().split('\n')
-    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+    with open(grubsyscfg, 'w') as defgrubout:
         for gline in defgrub:
             gline = gline.split()
             newline = []
@@ -136,8 +146,34 @@ def fixup(rootdir, vols):
                     continue
                 newline.append(ent)
             defgrubout.write(' '.join(newline) + '\n')
-    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
-    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/').replace('//', '/')
+    grubcfg = grubcfg.split('\n')
+    if not grubcfg[-1]:
+        grubcfg = grubcfg[:-1]
+    if len(grubcfg) == 1:
+        grubcfg = grubcfg[0]
+    else:
+        for gcfg in grubcfg:
+            rgcfg = os.path.join(rootdir, gcfg[1:])  # gcfg has a leading / to get rid of
+            if os.stat(rgcfg).st_size > 256:
+                grubcfg = gcfg
+            else:
+                with open(rgcfg, 'r') as gin:
+                    tgrubcfg = gin.read()
+                tgrubcfg = tgrubcfg.split('\n')
+                if 'search --no-floppy --fs-uuid --set=dev' in tgrubcfg[0]:
+                    tgrubcfg[0] = 'search --no-floppy --fs-uuid --set=dev ' + bootuuid
+                with open(rgcfg, 'w') as gout:
+                    for gcline in tgrubcfg:
+                        gout.write(gcline)
+                        gout.write('\n')
+    try:
+        subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    except Exception as e:
+        print(repr(e))
+        print(rootdir)
+        print(grubcfg)
+        time.sleep(86400)
     newroot = None
     with open('/etc/shadow') as shadowin:
         shents = shadowin.read().split('\n')
@@ -184,6 +220,7 @@ def had_swap():
     return False
 
 def install_to_disk(imgpath):
+    global bootuuid
     lvmvols = {}
     deftotsize = 0
     mintotsize = 0
@@ -224,7 +261,7 @@ def install_to_disk(imgpath):
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk
     parted = PartedRunner(instdisk)
-    dinfo = parted.run('unit s print')
+    dinfo = parted.run('unit s print', check=False)
     dinfo = dinfo.split('\n')
     sectors = 0
     sectorsize = 0
@@ -366,6 +403,14 @@ def install_to_disk(imgpath):
                 sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
                 sys.stdout.write('\n')
                 sys.stdout.flush()
+                if vol['mount'] == '/boot':
+                    tbootuuid = subprocess.check_output(['blkid', vol['targetdisk']])
+                    if b'UUID="' in tbootuuid:
+                        bootuuid = tbootuuid.split(b'UUID="', 1)[1].split(b'"')[0].decode('utf8')
+
+
+
+
             subprocess.check_call(['umount', '/run/imginst/targ'])
         for vol in allvols:
             subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
@@ -373,4 +418,4 @@ def install_to_disk(imgpath):
 
 
 if __name__ == '__main__':
-    install_to_disk(os.environ['mountsrc'])
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el9-diskless/profiles/default/scripts/imageboot.sh

@@ -127,5 +127,7 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el9-diskless/profiles/default/scripts/installimage

@@ -31,6 +31,9 @@ if [ ! -f /sysroot/tmp/installdisk ]; then
 fi
 lvm vgchange -a n
 udevadm control -e
+if [ -f /sysroot/etc/lvm/devices/system.devices ]; then
+    rm /sysroot/etc/lvm/devices/system.devices
+fi
 chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
 chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
 echo "Port 22" >> /etc/ssh/sshd_config

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -94,6 +94,7 @@ while ! grep NODENAME /etc/confluent/confluent.info; do
     fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+cat /tls/*.pem > /etc/confluent/ca.pem
 mgr=$(python /opt/confluent/bin/apiclient -c)
 APIKEY=$(cat /etc/confluent/confluent.apikey)
 
@@ -108,7 +109,6 @@ while [ -z "$APIKEY" ]; do
     (/clortho $node $mgr || /opt/confluent/bin/clortho $node $mgr) > /etc/confluent/confluent.apikey
     APIKEY=$(cat /etc/confluent/confluent.apikey)
 done
-cat /tls/*.pem > /etc/confluent/ca.pem
 /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg

confluent_osdeploy/genesis/profiles/default/ansible/onboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15-diskless/initramfs/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -161,7 +161,7 @@ else
     echo -e "BOOTPROTO='static'\nSTARTMODE='auto'" >> /run/confluent/ifcfg-$ifname
     echo "IPADDR='$v4addr/$v4nm'" >> /run/confluent/ifcfg-$ifname
     if [ ! -z "$v4gw" ]; then
-        echo defafult $v4gw - $ifname > /run/confluent/ifroute-$ifname
+        echo default $v4gw - $ifname > /run/confluent/ifroute-$ifname
     fi
 fi
 

confluent_osdeploy/suse15/profiles/hpc/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt

@@ -1,6 +1,10 @@
 Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
 appropriate phase of the install process.
 
+Alternatively, plays may be placed in /var/lib/confluent/private/os/<profilename>/ansible/<directory>.
+This prevents public clients from being able to read the plays, which is not necessary for them to function,
+and may protect them from divulging material contained in the plays or associated roles.
+
 The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
 requesting the playbooks be executed.
 

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
 
 # This script runs at the end of the final boot, updating status
 exec >> /var/log/confluent/confluent-firstboot.log

confluent_osdeploy/ubuntu18.04/initramfs/lib/debian-installer-startup.d/S25confluentinit

@@ -0,0 +1,205 @@
+reverse_uuid() {
+    echo $1 | head -c 8 | tail -c 2
+    echo $1 | head -c 6 | tail -c 2
+    echo $1 | head -c 4 | tail -c 2
+    echo $1 | head -c 2 | tail -c 2
+    echo $1 | head -c 13 | tail -c 2
+    echo $1 | head -c 11 | tail -c 2
+    echo $1 | head -c 18 | tail -c 2
+    echo $1 | head -c 16 | tail -c 2
+    echo $1 | tail -c 18 | sed -e s/-//
+}
+setdebopt() {
+    debconf-set $1 $2
+    echo d-i $1 $3 $2 >> /preseed.cfg
+}
+
+mkdir -p /etc/confluent
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) up
+done
+TRIES=5
+while [ ! -e /dev/disk ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
+done
+for i in /sys/class/net/*; do
+    ip link set $(basename $i) down
+    udevadm info $i | grep ID_NET_DRIVER=cdc_ether > /dev/null &&  continue
+    ip link set $(basename $i) up
+done
+cp -a /tls/* /etc/ssl/certs/
+mkdir -p /etc/confluent
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|cut -f 2 -d ' ')
+    echo NODENAME: $nodename > /etc/confluent/confluent.info
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: $tcfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress ${v4addr%/*} string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$v4gw" ]; then
+                ip route add default via $v4gw
+            fi
+            for dsrv in $deploysrvs; do
+                if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    setdebopt netcfg/choose_interface $NIC select
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done 
+        #TODO: nameservers
+    elif [ "$v4cfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            udhcpc $NICGUESS
+        done
+        for dsrv in $deploysrvs; do
+            if wget https://$dsrv/confluent-public/ --tries=1 --timeout=1 -O /dev/null > /dev/null 2>&1; then
+                deploysrvs=$dsrv
+            fi
+        done
+    fi
+    mgr=$deploysrvs
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    echo -n $(grep ^apitoken: cnflnt.yml|cut -d ' ' -f 2) > $hmackeyfile
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" --post-file=$passcrypt https://$mgr/confluent-api/self/registerapikey -O - --quiet
+    cp $passfile /etc/confluent/confluent.apikey
+    nic=$NIC
+else
+    dhuuid=$(reverse_uuid $(cat /sys/devices/virtual/dmi/id/product_uuid))
+    dhcpid=$(mktemp)
+    mkdir -p /etc/confluent
+    cp /tls/* /etc/ssl/certs/
+    cat /tls/*.pem >> /etc/confluent/ca.pem
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|cut -d: -f 2|sed -e 's/ //'); do
+        ip link set $nic up
+    done
+    for nic in $(ip link | grep mtu|grep -v LOOPBACK|grep LOWER_UP|cut -d: -f 2|sed -e 's/ //'); do
+        if udhcpc -i $nic -p $dhcpid -t 2 -T 2 -n -x 93:0007 -x 97:00$dhuuid -q; then
+            /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+            if grep ^MANAGER:.*\\. /etc/confluent/confluent.info ; then
+                break
+            fi
+        fi
+        ip -4 flush dev $nic
+    done
+    mgr=$(grep ^MANAGER:.*\\. /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|head -n 1|cut -d: -f 2|sed -e 's/ //')
+    /opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+fi
+apikey=$(cat /etc/confluent/confluent.apikey)
+cd /etc/confluent
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg
+cd -
+predir=$(mktemp -d)
+cd $predir
+cp /etc/confluent/deploycfg /etc/confluent/confluent.deploycfg
+profile=$(grep ^profile: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for namesrv in "$namesrvs"; do
+    setdebopt netcfg/get_nameservers $namesrv string
+done
+rootpass=$(grep ^rootpassword: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //')
+if [ "$rootpass" = null ] || [ -z "$rootpass" ]; then
+    setdebopt passwd/root-login false boolean
+else
+    setdebopt passwd/root-login true boolean
+    setdebopt passwd/root-password-crypted $rootpass string
+fi
+setdebopt time/zone $(grep ^timezone: /etc/confluent/deploycfg|cut -d ' ' -f 2|sed -e 's/ //') string
+ntpsrvs=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+for ntpsrv in "$ntpsrvs"; do
+    setdebopt clock-setup/ntp true boolean
+    setdebopt clock-setup/ntp-server $ntpsrv string
+done
+setdebopt debian-installer/locale $(grep ^locale: /etc/confluent/deploycfg|cut -d ' ' -f 2) select
+domainname=$(grep ^dnsdomain: /etc/confluent/deploycfg|cut -d ' ' -f 2)
+if [ ! -z "$domainname" ] && [ "$domainname" != "null" ]; then
+    setdebopt netcfg/get_domain $domainname string
+fi
+
+
+
+
+wget https://$mgr/confluent-public/os/$profile/scripts/pre.sh
+chmod u+x pre.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh
+chmod u+x prechroot.sh
+wget https://$mgr/confluent-public/os/$profile/scripts/post.sh
+chmod u+x post.sh
+wget https://$mgr/confluent-public/os/$profile/preseed.cfg
+cat preseed.cfg >> /preseed.cfg
+echo $mgr > /etc/confluent/deployer
+setdebopt auto-install/enable true boolean
+setdebopt partman/early_command  $predir/pre.sh string
+setdebopt preseed/late_command $predir/prechroot.sh string
+mv $predir/post.sh /tmp/
+cd -
+ip -4 a flush dev $nic
+setdebopt netcfg/choose_interface $nic select
+setdebopt netcfg/get_hostname $nodename string
+setdebopt mirror/protocol https string
+setdebopt mirror/country manual string
+setdebopt mirror/https/hostname $mgr string
+setdebopt mirror/https/directory /confluent-public/os/$profile/distribution string
+setdebopt mirror/protocol https string
+setdebopt mirror/https/proxy "" string
+setdebopt apt-setup/security_host $mgr string
+if [ ! -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    v4cfgmeth=$(grep ipv4_method: /etc/confluent/deploycfg |cut -d: -f 2|sed -e 's/ //')
+    if [ "$v4cfgmeth" = "static" ]; then
+        setdebopt netcfg/disable_dhcp true boolean
+        v4addr=$(grep ^ipv4_address: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        v4gw=$(grep ^ipv4_gateway: /etc/confluent/deploycfg|cut -d: -f 2| sed -e 's/ //')
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ^ipv4_netmask: /etc/confluent/deploycfg|cut -d: -f 2|sed -e 's/ //')
+        setdebopt netcfg/get_netmask $v4nm string
+        setdebopt netcfg/get_ipaddress $v4addr string
+        setdebopt netcfg/confirm_static true boolean
+        if [ ! -z "$v4gw" ]; then
+            setdebopt netcfg/get_gateway $v4gw string
+        fi
+        namesrvs=$(sed -n '/^nameservers:/,/^[^-]/p' /etc/confluent/deploycfg|grep ^- | cut -d ' ' -f 2|sed -e 's/ //')
+        for namesrv in "$namesrvs"; do
+            setdebopt netcfg/get_nameservers $namesrv string
+        done
+    elif [ "$vpcfgmeth" = "dhcp" ]; then
+        setdebopt netcfg/disable_dhcp false boolean
+        setdebopt netcfg/confirm_static false boolean
+    fi
+fi

confluent_osdeploy/ubuntu18.04/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/linux $2/boot/kernel && \
+ln -s $1/install/hwe-netboot/ubuntu-installer/amd64/initrd.gz $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/* $2/boot/efi/boot
+

confluent_osdeploy/ubuntu18.04/profiles/default/preseed.cfg

@@ -0,0 +1,21 @@
+d-i anna/choose_modules string openssh-server-udeb
+d-i partman-auto/method string regular
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-auto/expert_recipe_file string /tmp/partitionfile
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i passwd/make-user boolean false
+d-i clock-setup/utc boolean true
+d-i apt-setup/multiverse boolean false
+d-i apt-setup/universe boolean false
+d-i apt-setup/backports boolean false
+d-i apt-setup/updates boolean false
+d-i grub-installer/only_debian boolean true
+tasksel tasksel/first multiselect standard
+d-i pkgsel/include string openssh-server curl
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean false
+d-i finish-install/reboot_in_progress note

confluent_osdeploy/ubuntu18.04/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+#cp -a /etc/confluent/ssh/* /etc/ssh/
+#systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+source /etc/confluent/functions
+
+run_remote_parts firstboot.d
+run_remote_config firstboot.d
+systemctl disable firstboot
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/functions

@@ -0,0 +1,208 @@
+#!/bin/bash
+function test_mgr() {
+    whost=$1
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    if curl -gs https://${whost}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    whost=$confluent_mgr
+    if [[ "$whost" == *:* ]] && [[ "$whost" != *[* ]] ; then
+        whost="[$whost]"
+    fi
+    curl -gf -sS $curlargs https://$whost/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/post.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+mkdir -p /run/sshd
+mkdir -p /root/.ssh
+cat /tmp/ssh/*pubkey >> /root/.ssh/authorized_keys
+cat /tmp/ssh/*.ca | sed -e s/^/'@cert-authority * '/ >> /etc/ssh/ssh_known_hosts
+chmod 700 /etc/confluent
+chmod go-rwx /etc/confluent/*
+sshconf=/etc/ssh/ssh_config
+if [ -d /etc/ssh/ssh_config.d/ ]; then
+    sshconf=/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+mkdir -p /opt/confluent/bin
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+systemctl enable firstboot
+python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+source /etc/confluent/functions
+python3 /opt/confluent/bin/apiclient /confluent-api/self/nodelist | sed -e s/'^- //' > /tmp/allnodes
+cp /tmp/allnodes /root/.shosts
+cp /tmp/allnodes /etc/ssh/shosts.equiv
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ')
+    sed -i "s/#NTP=/NTP=$ntps/" /etc/systemd/timesyncd.conf
+fi
+textcons=$(grep ^textconsole: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /tmp/autocons.info ]; then
+        cons=$(cat /tmp/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /etc/default/grub
+        updategrub=1
+    fi
+fi
+kargs=$(python3 /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /etc/default/grub
+fi
+
+if [ 1 = $updategrub ]; then
+    update-grub
+fi
+
+if [ -e /sys/firmware/efi ]; then
+    bootnum=$(efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//)
+    if [ ! -z "$bootnum" ]; then
+        currboot=$(efibootmgr | grep ^BootOrder: | awk '{print $2}')
+        nextboot=$(echo $currboot| awk -F, '{print $1}')
+        [ "$nextboot" = "$bootnum" ] || efibootmgr -o $bootnum,$currboot
+        efibootmgr -D
+    fi
+fi
+run_remote_python syncfileclient
+run_remote_parts post.d
+run_remote_config post
+
+python3 /opt/confluent/bin/apiclient /confluent-api/self/updatestatus  -d 'status: staged'
+

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/pre.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+## Use the following option to add additional boot parameters for the
+## installed system (if supported by the bootloader installer).
+## Note: options passed to the installer will be added automatically.
+#d-i debian-installer/add-kernel-opts string [from profile.yaml]
+deploycfg=/etc/confluent/confluent.deploycfg
+mgr=$(cat /etc/confluent/deployer)
+
+cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+cat > /usr/lib/live-installer.d/confluent-certs << EOF
+#!/bin/sh
+cp /tls/* /target/etc/ssl/certs/
+cat /tls/*.pem >> /target/etc/ssl/certs/ca-certificates.crt
+EOF
+chmod a+x /usr/lib/live-installer.d/confluent-certs
+mkdir -p /.ssh/
+cat /ssh/*pubkey > /.ssh/authorized_keys
+mkdir -p /etc/ssh
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info|cut -d ' ' -f 2)
+apikey=$(cat /etc/confluent/confluent.apikey)
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=$(echo $pubkey | sed -e s/.pub/-cert.pub/)
+    keyfile=${pubkey%.pub}
+    wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" --post-file=$pubkey https://$mgr/confluent-api/self/sshcert -O $certfile --quiet
+    echo HostKey $keyfile >> /etc/ssh/sshd_config
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config
+done
+if [ -e /tmp/installdisk ]; then
+    instdisk=$(cat /tmp/installdisk)
+else
+    for blockdev in $(ls /sys/class/block/); do
+        shortname=$(basename $blockdev)
+        if [ "$shortname" != "${shortname%loop*}" ]; then
+            continue
+        fi
+        udevadm info --query=property /dev/$shortname |grep DEVTYPE=disk > /dev/null || continue # ignore partitions
+        udevadm info --query=property /dev/$shortname |grep DM_NAME > /dev/null && continue # not a real disk
+        sz=$(cat /sys/block/$shortname/size 2> /dev/null)
+        [ -z "$sz" ] && continue
+        [ $sz -lt 1048576 ] && continue # Too small
+        [ -z "$firstdisk" ] && firstdisk=$shortname
+        if udevadm info --query=property /dev/$shortname|grep ID_MODEL=| sed -e s/' '/_/g | grep -iE '(thinksystem_m.2|m.2_nvme_2-bay_raid_kit)' > /dev/null; then
+            instdisk=$shortname
+            break
+        fi
+        if udevadm info --query=property /dev/$shortname|grep MD_CONTAINER=imsm; then
+            sraid=$sortname
+        else
+            drv=$(udevadm info -a /dev/sdb|grep DRIVERS==|grep -Ev '""|"sd"' | sed -e s/.*=// -e s/'"'//g)
+            if [ "ahci" = "$drv" -a -z "$onbdisk" ]; then
+                onbdisk=$shortname
+            elif [ "megaraid" = "$drv" -a -z "$rdisk" ]; then
+                rdisk=$shortname
+            fi
+        fi
+    done
+fi
+if [ -z "$instdisk" ]; then
+    if [ ! -z "$sraid"]; then
+        instdisk=$sraid
+    elif [ ! -z "$onbdisk" ]; then
+        instdisk=$onbdisk
+    elif [ ! -z "$rdisk" ]; then
+        instdisk=$rdisk
+    else
+        instdisk=$firstdisk
+    fi
+fi
+if [ ! -z "$instdisk" ]; then
+    debconf-set partman-auto/disk /dev/$instdisk
+fi
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config
+echo IgnoreRhosts no >> /etc/ssh/sshd_config
+echo sshd:x:1:1::/run/sshd:/bin/false >> /etc/passwd
+/usr/sbin/sshd
+wget --header="CONFLUENT_NODENAME: $nodename" --header="CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/nodelist -O /tmp/allnodes --quiet
+#kill -HUP $(ps | grep -v grep | grep /usr/sbin/sshd | sed -e 's/^ *//'|cut -d ' ' -f 1)
+#curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /tmp/getinstalldisk
+#python3 /tmp/getinstalldisk
+#sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/ubuntu18.04/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+mount -o bind /sys /target/sys
+mount -o bind /dev /target/dev
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /proc /target/proc
+mount -o bind /dev/pts /target/dev/pts
+mount -o bind /run /target/run
+cp -a /etc/confluent /target/etc/confluent
+cp -a /opt/confluent /target/opt/confluent
+mv /tmp/post.sh /target/tmp/
+cp -a /ssh /tls /target/tmp
+cat /tls/*.pem >> /target/etc/confluent/ca.pem
+cp -a /etc/ssh/ssh_host_* /target/etc/ssh/
+grep HostCertificate /etc/ssh/sshd_config  >> /target/etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+kill -HUP $(ps |grep -v grep|grep sshd|grep /usr|sed -e s/' root.*//')
+chroot /target bash /tmp/post.sh

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/imageboot.sh

@@ -134,4 +134,6 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+mv /lib/firmware /lib/firmware-ramfs
+ln -s /sysroot/lib/firmware /lib/firmware
 exec /opt/confluent/bin/start_root

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/hooks/casper-bottom.sh

@@ -9,33 +9,49 @@ MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|'
 MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}')
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
-umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+umask 077
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile| sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu20.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+seems a potentially relevant thing to put i... by Jarrod Johnson
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -11,32 +11,48 @@ MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| aw
 MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
 oum=$(umask)
 umask 077
-chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
-MGR=[$MGR]
 deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+netcfgfile=$deploycfg
+if [ -e /tmp/cnflnthmackeytmp ]; then
+    netcfgfile=/tmp/idnttmp
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    #echo -n $(grep ^apitoken: /tmp/identdata/cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    passfile=/tmp/cnflnttmppassfile
+    passcrypt=/tmp/cnflntcryptfile
+    hmacfile=/tmp/cnflnthmacfile
+    chroot . ln -sf /custom-installation/confluent/bin/clortho custom-installation/confluent/bin/genpasshmac
+    cp $hmackeyfile tmp
+    chroot . custom-installation/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    chroot . curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_CRYPTHMAC: $(cat /root/$hmacfile)" -d @/tmp/cnflntcryptfile  https://$MGR/confluent-api/self/registerapikey
+    cp /root/$passfile /root/custom-installation/confluent/confluent.apikey
+    DEVICE=$(cat /tmp/autodetectnic)
+else
+    chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+    MGR=[$MGR]
+    nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+    nic=$(ip link |grep ^$nic:|awk '{print $2}')
+    DEVICE=${nic%:}
+fi
 if [ -z "$MGTIFACE" ]; then
 	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 else
 	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
 fi
 umask $oum
-nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
-nic=$(ip link |grep ^$nic:|awk '{print $2}')
-DEVICE=${nic%:}
-ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+ipv4m=$(grep ^ipv4_method $netcfgfile|awk '{print$2}')
 . /scripts/functions
 if [ "$ipv4m" = "dhcp" ]; then
     IP=dhcp
     configure_networking
 elif [ "$ipv4m" = "static" ]; then
-    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=$(grep ^ipv4_address: $netcfgfile | sed -e 's!/.*!!')
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=$(grep ^ipv4_gateway: $netcfgfile)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=$(grep ipv4_netmask: $netcfgfile)
     v4nm=${v4nm#ipv4_netmask: }
     dnsdomain=$(grep ^dnsdomain: $deploycfg)
     dnsdomain=${dnsdomain#dnsdomain: }

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -5,10 +5,70 @@ done
 mkdir -p /custom-installation
 cp -a /opt/confluent /custom-installation
 touch /custom-installation/confluent/confluent.info
-while ! grep NODENAME /custom-installation/confluent/confluent.info; do
-    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+TRIES=5
+while [ ! -e /dev/disk/by-label ] && [ $TRIES -gt 0 ]; do
+    sleep 2
+    TRIES=$((TRIES - 1))
 done
-MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=/tmp/idntmnt
+    mkdir -p /tmp/identdata/
+    mkdir -p $tmnt
+    tcfg=/tmp/idnttmp
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cp -a $tmnt/* /tmp/identdata/
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/,/^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    sed -n '/^net_cfgs:/,/^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/,/^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    . /scripts/functions
+    if [ "$autoconfigmethod" = "static" ]; then
+        MYIP=$(grep ^ipv4_address: $tcfg | awk '{print $2}'|sed -e s'!/.*!!')
+        v4addr=$(grep ^ipv4_address: $tcfg|cut -d: -f 2|sed -e 's/ //')
+        MYGW=$(grep ^ipv4_gateway: $tcfg | awk '{print $2}')
+        if [ "$MYGW" = "null" ]; then
+            MYGW=""
+        fi
+        MYNM=$(grep ^ipv4_netmask: $tcfg | awk '{print $2}')
+        for NICGUESS in $(ip link|grep LOWER_UP|grep -v LOOPBACK|cut -d ' ' -f 2 | sed -e 's/:$//'); do
+            ip addr add dev $NICGUESS $v4addr
+            if [ ! -z "$MYGW" ]; then
+                ip route add default via $MYGW
+            fi
+            for dsrv in $deploysrvs; do
+                if openssl s_client -connect $dsrv:443 > /dev/null 2>&1; then
+                    deploysrvs=$dsrv
+                    NIC=$NICGUESS
+                    break
+                fi
+            done
+            if [ -z "$NIC" ]; then
+                ip -4 a flush dev $NICGUESS
+            else
+                break
+            fi
+        done
+        ipconfig -d $MYIP::$MYGW:$MYNM::$NIC
+        echo $NIC > /tmp/autodetectnic
+    else
+        configure_networking
+    fi
+    MGR=$deploysrvs
+    NODENAME=$(grep ^nodename: /tmp/idntmnt/cnflnt.yml | awk '{print $2}')
+    echo "NODENAME: $NODENAME" >> /custom-installation/confluent/confluent.info
+    echo "MANAGER: $MGR" >> /custom-installation/confluent/confluent.info
+    echo "EXTMGRINFO: $MGR||1" >> /custom-installation/confluent/confluent.info
+    hmackeyfile=/tmp/cnflnthmackeytmp
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+else
+    while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+        /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+    done
+    MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+fi
 osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
 cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
 if [ -f /custom-installation/autocons.info ]; then

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 echo "Confluent first boot is running"
+HOME=$(getent passwd $(whoami)|cut -d: -f 6)
+export HOME
+seems a potentially relevant thing to put i... by Jarrod Johnson
 cp -a /etc/confluent/ssh/* /etc/ssh/
 systemctl restart sshd
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')

confluent_osdeploy/utils/clortho.c

@@ -19,6 +19,7 @@
 #define MAXPACKET 1024
 
 static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./";
+static const char b64alpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 unsigned char* genpasswd(int len) {
         unsigned char * passwd;
@@ -56,10 +57,10 @@ char * b64e(uint8_t * data, uint32_t datalen) {
         currchunk[1] = remaining > 1 ? currptr[1] : 0;
         currchunk[2] = remaining > 2 ? currptr[2] : 0;
         currptr += 3;
-        currout[0] = cryptalpha[currchunk[0] >> 2];
-        currout[1] = cryptalpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
-        currout[2] = remaining > 1 ? cryptalpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
-        currout[3] = remaining > 2 ? cryptalpha[currchunk[2] & 0x3f] : '=';
+        currout[0] = b64alpha[currchunk[0] >> 2];
+        currout[1] = b64alpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
+        currout[2] = remaining > 1 ? b64alpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
+        currout[3] = remaining > 2 ? b64alpha[currchunk[2] & 0x3f] : '=';
         remaining -= 3;
         currout += 4;
     }

confluent_osdeploy/utils/urlmount3.c

@@ -0,0 +1,348 @@
+/*
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2021 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+#define FUSE_USE_VERSION 30
+#include <fuse3/fuse.h>
+#include <curl/curl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <pthread.h>
+#include <unistd.h>
+
+CURL *curl;
+
+char curlerror[CURL_ERROR_SIZE];
+curl_off_t filesize;
+
+typedef struct downloadbuffer {
+    char *response;
+    size_t completed;
+    size_t total;
+} downloadbuffer;
+
+#define MAX_FILE_LEN 1024
+#define MAX_URL_PATHS 512
+static char filename[MAX_FILE_LEN + 1];
+static int urlidx, newidx;
+static char* urls[MAX_URL_PATHS + 1];
+
+
+void *http_rechecker(void *argp) {
+    CURL *checkurl;
+    int tmpidx, tmpval;
+    tmpidx = open("/dev/urandom", O_RDONLY);
+    if (tmpidx <= 0 || read(tmpidx, (char*)&tmpval, 4) < 0)
+        tmpval = time(NULL) & 0xffffffff;
+    if (tmpidx >= 0)
+        close(tmpidx);
+    srand(tmpval);
+    checkurl = curl_easy_init();
+    if (curl_easy_setopt(checkurl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(checkurl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Error setting timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(checkurl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Error setting nobody\n");
+        exit(1);
+    }
+    while (1) {
+        sleep(25 + tmpval % 10);  // Spread out retries across systems
+        tmpidx = 0;
+        while (tmpidx < urlidx && tmpidx < newidx && urls[tmpidx] != NULL) {
+            if (curl_easy_setopt(checkurl, CURLOPT_URL, urls[tmpidx]) != CURLE_OK) {
+                tmpidx++;
+                continue;
+            }
+            if (curl_easy_perform(checkurl) == CURLE_OK)
+                newidx = tmpidx;
+            else
+                tmpidx++;
+        }
+    }
+}
+
+static int http_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
+                        off_t offset, struct fuse_file_info *fi, enum fuse_readdir_flags frf)
+{
+    if (strcmp(path, "/") != 0)  // We don't support subdirs
+        return -ENOENT;
+    filler(buf, ".", NULL, 0, 0);
+    filler(buf, "..", NULL, 0, 0);
+    filler(buf, filename + 1, NULL, 0, 0);
+    return 0;
+}
+
+size_t fill_buffer(char *data, size_t size, size_t nmemb, downloadbuffer *userdata) {
+    size_t amount;
+    amount = size * nmemb;
+    if (userdata->total < amount + userdata->completed) return 0;
+    memcpy(&(userdata->response[userdata->completed]), data, amount);
+    userdata->completed += amount;
+    return amount;
+}
+
+static int http_read(const char *path, char *buf, size_t size, off_t offset,
+                     struct fuse_file_info *fi) {
+    char headbuffer[512];
+    double dldbl = 0.0;
+    int startidx;
+    int reconnecting = 0;
+    FILE* fd;
+    startidx = urlidx;
+    memset(buf, 0, size);
+    curl_off_t downloaded;
+    //Would be needed for multithread, however preferring to conserve
+    //filehandles rather than go multithread
+    // Some comparisons showed that the threaded performance boost doesn't
+    // do even offset the overhead of the new curl handles, so better
+    // to use single threaded curl overall for now
+    //CURL *tmpcurl = curl_easy_duphandle(curl);
+    downloadbuffer dlbuf;
+    dlbuf.response = buf;
+    dlbuf.completed = 0;
+    dlbuf.total = size;
+    fd = NULL;
+
+    if (strcmp(path, filename) != 0) return -ENOENT;
+    memset(headbuffer, 0, 512);
+    if (offset >= filesize) return 0;
+    if (offset + size - 1 >= filesize) size = filesize - offset - 1;
+    snprintf(headbuffer, 512, "%ld-%ld", offset, offset + size - 1);
+    if (curl_easy_setopt(curl, CURLOPT_RANGE, headbuffer) != CURLE_OK) {
+        fprintf(stderr, "Error setting range\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&dlbuf) != CURLE_OK) {
+        fprintf(stderr, "Error setting writedata\n");
+        exit(1);
+    }
+    if (newidx < MAX_URL_PATHS) {
+        reconnecting = 1;
+        urlidx = newidx;
+        newidx = MAX_URL_PATHS;
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        // if fail, carry on and take the error in curl_easy_perform instead
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        reconnecting = 1;
+        fd = fopen("/dev/kmsg", "w+");
+        dlbuf.completed = 0;
+        fprintf(fd, "<4>urlmount: error while communicating with %s: %s\n", urls[urlidx], curlerror);
+        fclose(fd);
+        urlidx++;
+        if (urlidx > MAX_URL_PATHS)
+            urlidx = 0;
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == startidx) {
+            fd = fopen("/dev/kmsg", "w+");
+            fprintf(fd, "<1>urlmount: All connections to source are down\n");
+            fclose(fd);
+            sleep(10);
+        }
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Connecting to %s\n", urls[urlidx]);
+        fclose(fd);
+        if (urlidx > MAX_URL_PATHS) {
+            fprintf(stderr, "Maximum url path exceeded\n");
+            exit(1);
+        }
+        // ignore, let the curl_easy_perform get the error
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {}
+    }
+    if (reconnecting) {
+        fd = fopen("/dev/kmsg", "w+");
+        fprintf(fd, "<5>urlmount: Successfully connected to %s\n", urls[urlidx]);
+        fclose(fd);
+    }
+    curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dldbl);
+    downloaded = round(dldbl);
+    //Would be needed for multithread
+    //curl_easy_cleanup(tmpcurl);
+    return downloaded;
+}
+
+static int http_open(const char *path, struct fuse_file_info *fi) {
+    if (strcmp(path, filename) != 0)
+        return -ENOENT;
+
+    if ((fi->flags & 3) != O_RDONLY)
+        return -EACCES;
+
+    return 0;
+}
+
+static void* http_init(struct fuse_conn_info *conn, struct fuse_config *cfg) {
+    // Because we fork, we need to redo curl
+    // or else suffer the wrath of NSS TLS
+    pthread_t tid;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    pthread_create(&tid, NULL, http_rechecker, NULL);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl error buffor\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl failonerror\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl timeout\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing url\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_buffer) != CURLE_OK) {
+        fprintf(stderr, "Failure initializing libcurl fill buffer\n");
+        exit(1);
+    }
+    return NULL;
+}
+
+static int http_getattr(const char *path, struct stat *st, struct fuse_file_info *ffi) {
+    memset(st, 0, sizeof(struct stat));
+
+    if (strcmp(path, "/") == 0) {
+        st->st_mode = S_IFDIR | 0555;
+        st->st_nlink = 2;
+    } else if (strcmp(path, filename) == 0) {
+        st->st_mode = S_IFREG | 0444;
+        st->st_nlink = 1;
+        st->st_size = filesize; // TODO: fix with curl HEAD
+    } else
+        return -ENOENT;
+    return 0;
+}
+
+
+static const struct fuse_operations http_ops = {
+    .getattr = http_getattr,
+    .readdir = http_readdir,
+    .read = http_read,
+    .open = http_open,
+    .init = http_init,
+};
+
+int main(int argc, char* argv[]) {
+    char *tmp;
+    double fsize;
+    unsigned int i;
+    int j;
+    j = 0;
+    memset(urls, 0, 32*sizeof(char*));
+    urlidx = 0;
+    newidx = MAX_URL_PATHS;
+    curl_global_init(CURL_GLOBAL_DEFAULT);
+    curl = curl_easy_init();
+    if (curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curlerror) != CURLE_OK) {
+        fprintf(stderr, "Unable to set error buffer\n");
+        exit(1);
+    }
+    //We want to consider error conditions fatal, rather than
+    //passing error text as data
+    if (curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L) != CURLE_OK) {
+        fprintf(stderr, "Unable to set fail on error\n");
+        exit(1);
+    }
+    if (curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15L) != CURLE_OK) {
+        fprintf(stderr, "Unable to setup curl timeout\n");
+        exit(1);
+    }
+    memset(filename, 0, MAX_FILE_LEN);
+    for (i=0; i < argc; i++) {
+        if (strstr(argv[i], ":") > 0) {
+            if (j < MAX_URL_PATHS) {
+                urls[j] = argv[i];
+                tmp = strrchr(urls[j++], '/');
+                strncpy(filename, tmp, MAX_FILE_LEN);
+            }
+            //Request single threaded mode, as curl would need more
+            // filehandles for multithread
+            argv[i] = "-s";
+        }
+    }
+    if (filename[0] == 0) {
+        fprintf(stderr, "No URL given in arguments\n");
+        exit(1);
+    }
+    for (i=0; urls[i] != NULL; i++) {
+        printf("Registering mount path: %s\n", urls[i]);
+    }
+    j = urlidx;
+    printf("Connecting to %s\n", urls[urlidx]);
+    if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+        fprintf(stderr, "Unable to set url\n");
+    }
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 1) != CURLE_OK) {
+        fprintf(stderr, "Failure setting no body\n");
+    }
+    while (curl_easy_perform(curl) != CURLE_OK) {
+        fprintf(stderr, "urlmount: error while communicating with %s: %s\n", urls[urlidx++], curlerror);
+        if (urls[urlidx] == NULL)
+            urlidx = 0;
+        if (urlidx == j) {
+            fprintf(stderr, "urlmount: Unable to reach any target url, aborting\n");
+            exit(1);
+        }
+        printf("Connecting to %s\n", urls[urlidx]);
+        if (curl_easy_setopt(curl, CURLOPT_URL, urls[urlidx]) != CURLE_OK) {
+            fprintf(stderr, "Unable to set url\n");
+        }
+    }
+    printf("Successfully connected to %s\n", urls[urlidx]);
+    if (curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &fsize) != CURLE_OK) {
+        fprintf(stderr, "Failed getting content length\n");
+        exit(1);
+    }
+    filesize = round(fsize);
+    if (curl_easy_setopt(curl, CURLOPT_NOBODY, 0) != CURLE_OK) {
+        fprintf(stderr, "Failed setting nobody\n");
+        exit(1);
+    }
+    if (filesize < 1) {
+        fprintf(stderr, "Unable to reach designated URL\n");
+        exit(1);
+    }
+    if (!curl) {
+        fprintf(stderr, "Unable to initialize CURL!\n");
+        exit(1);
+    }
+    curl_easy_cleanup(curl);
+    curl_global_cleanup();
+    fuse_main(argc, argv, &http_ops, NULL);
+}

confluent_server/bin/confluent_selfcheck

@@ -237,6 +237,7 @@ if __name__ == '__main__':
         allok = True
         uuidok = False
         macok = False
+        valid_nodes = [node['item']['href'][:-1] for node in sess.read('/nodes/')] #get all valid nodes
         for rsp in sess.read(f'/nodes/{args.node}/attributes/all'):
             if rsp.get('errorcode', None) == 404:
                 emprint(f'There is no node named "{args.node}"')
@@ -259,6 +260,15 @@ if __name__ == '__main__':
                     mac = rsp[key].get('value', None)
                     if mac:
                         macok = True
+            #adding new code to check if the response is something like net.<something>switch
+            for key in rsp:
+                if ((key.startswith('net.') and key.endswith('switch')) 
+                        or (key.startswith('power.') and key.endswith('pdu'))
+                        or (key.startswith('enclosure.') and key.endswith('manager'))
+                        ):
+                    switch_value = rsp[key].get('value',None)
+                    if switch_value and switch_value not in valid_nodes:
+                        emprint(f'{switch_value} is not a valid node name (as referenced by attribute "{key}" of node {args.node}).')
         if not uuidok and not macok:
             allok = False
             emprint(f'{args.node} does not have a uuid or mac address defined in id.uuid or net.*hwaddr, deployment will not work')

confluent_server/confluent/auth.py

@@ -41,6 +41,7 @@ try:
 except ImportError:
     pass
 import time
+import yaml
 
 _pamservice = 'confluent'
 _passcache = {}
@@ -115,6 +116,47 @@ class PromptsNeeded(Exception):
     def __init__(self, prompts):
         self.prompts = prompts
 
+#add function to change _allowedbyrole and _deniedbyrole vars.
+def add_roles(_allowed,_denied):
+    # function to parse the roles and the files. If there are modifications to be done to the roles, items will be added to dictionaries.
+    # If there are no moodifications done to one of the roles, it continues to the next
+    # Opening YAML file and reading the custom roles
+    with open("/etc/confluent/authorization.yaml","r") as stream:
+        loaded_file = yaml.safe_load(stream)
+        try:
+            allowed_loaded = loaded_file["allowedbyrole"]
+            for role in allowed_loaded:
+                if role not in configmanager._validroles:
+                    configmanager._validroles.append(role)
+        except:
+            pass
+        try:
+            denied_loaded = loaded_file["deniedbyrole"]
+        except:
+            pass
+ 
+        try:
+            _allowed.update(allowed_loaded)
+        except NameError:
+            pass
+        try: 
+            _denied.update(denied_loaded)
+        except NameError:
+            pass
+        return
+                
+    
+def check_for_yaml():
+    #checking if the file exists
+    if os.path.exists("/etc/confluent/authorization.yaml"):
+        add_roles(_allowedbyrole,_deniedbyrole)
+
+        return "Custom auth. file detected in /etc/confluent, updated roles accordingly"
+    else:
+        return "No custom auth. file. Continuing as normal"
+    
+
+        
 def _get_usertenant(name, tenant=False):
     """_get_usertenant
 

confluent_server/confluent/collective/manager.py

@@ -63,7 +63,7 @@ connecting = ContextBool()
 leader_init = ContextBool()
 enrolling = ContextBool()
 
-def connect_to_leader(cert=None, name=None, leader=None, remote=None):
+def connect_to_leader(cert=None, name=None, leader=None, remote=None, isretry=False):
     global currentleader
     global follower
     ocert = cert
@@ -72,8 +72,9 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     oremote = remote
     if leader is None:
         leader = currentleader
-    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
-             'subsystem': 'collective'})
+    if not isretry:
+        log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+                 'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader, remote)
     except Exception as e:
@@ -84,9 +85,11 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
     with connecting:
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
+            if not banner:
+                return
             vers = banner.split()[2]
-            if vers not in (b'v2', b'v3'):
-                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
+            if vers != b'v4':
+                raise Exception('This instance only supports protocol 4, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -103,7 +106,12 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                                 '{0}'.format(leader),
                         'subsystem': 'collective'})
                     return False
+                if 'waitinline' in keydata:
+                    eventlet.sleep(0.3)
+                    return connect_to_leader(cert, name, leader, None, isretry=True)
                 if 'leader' in keydata:
+                    if keydata['leader'] == None:
+                        return None
                     log.log(
                         {'info': 'Prospective leader {0} has redirected this '
                                  'member to {1}'.format(leader, keydata['leader']),
@@ -144,7 +152,7 @@ def connect_to_leader(cert=None, name=None, leader=None, remote=None):
                     except Exception:
                         pass
                     log.log({'error': 'Retrying connection, error during initial sync', 'subsystem': 'collective'})
-                    return connect_to_leader(ocert, oname, oleader, oremote)
+                    return connect_to_leader(ocert, oname, oleader, None)
                     raise Exception("Error doing initial DB transfer")  # bad ssl write retry
                 dbjson += ndata
             cfm.clear_configuration()
@@ -271,7 +279,11 @@ def handle_connection(connection, cert, request, local=False):
                 return
             if follower is not None:
                 linfo = cfm.get_collective_member_by_address(currentleader)
-                remote = socket.create_connection((currentleader, 13001), 15)
+                try:
+                    remote = socket.create_connection((currentleader, 13001), 15)
+                except Exception:
+                    cfm.stop_following()
+                    return
                 remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                          keyfile='/etc/confluent/privkey.pem',
                                          certfile='/etc/confluent/srvcert.pem')
@@ -537,6 +549,11 @@ def handle_connection(connection, cert, request, local=False):
                                       'backoff': True})
             connection.close()
             return
+        if leader_init.active:
+            tlvdata.send(connection, {'error': 'Servicing a connection',
+                                      'waitinline': True})
+            connection.close()
+            return
         if myself != get_leader(connection):
             tlvdata.send(
                 connection,
@@ -552,7 +569,7 @@ def handle_connection(connection, cert, request, local=False):
                           'txcount': cfm._txcount})
             log.log({'info': 'Connecting to leader due to superior '
                              'transaction count', 'subsystem': 'collective'})
-            connection.close()
+            connection.close()  # well this won't work
             if not connect_to_leader(
                 None, None, connection.getpeername()[0]):
                 if retrythread is None:
@@ -569,9 +586,15 @@ def handle_connection(connection, cert, request, local=False):
             tlvdata.send(connection, cfm._cfgstore['collective'])
             tlvdata.send(connection, {'confluent_uuid': cfm.get_global('confluent_uuid')}) # cfm.get_globals())
             cfgdata = cfm.ConfigManager(None)._dump_to_json()
-            tlvdata.send(connection, {'txcount': cfm._txcount,
-                                      'dbsize': len(cfgdata)})
-            connection.sendall(cfgdata)
+            try:
+                tlvdata.send(connection, {'txcount': cfm._txcount,
+                            'dbsize': len(cfgdata)})
+                connection.sendall(cfgdata)
+            except Exception:
+                try:
+                    connection.close()
+                finally:
+                    return None
         #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
         # so far unused anyway
         connection.settimeout(90)
@@ -660,7 +683,11 @@ def get_leader(connection):
 
 def retire_as_leader(newleader=None):
     global currentleader
+    global reassimilate
     cfm.stop_leading(newleader)
+    if reassimilate is not None:
+        reassimilate.kill()
+        reassimilate = None
     currentleader = None
 
 def become_leader(connection):
@@ -668,6 +695,10 @@ def become_leader(connection):
     global follower
     global retrythread
     global reassimilate
+    if cfm.get_collective_member(get_myname()).get('role', None) == 'nonvoting':
+        log.log({'info': 'Refraining from being leader of collective (nonvoting)',
+            'subsystem': 'collective'})
+        return False
     log.log({'info': 'Becoming leader of collective',
              'subsystem': 'collective'})
     if follower is not None:
@@ -679,15 +710,20 @@ def become_leader(connection):
         retrythread = None
     currentleader = connection.getsockname()[0]
     skipaddr = connection.getpeername()[0]
+    if reassimilate is not None:
+        reassimilate.kill()
+    reassimilate = eventlet.spawn(reassimilate_missing)
     if _assimilate_missing(skipaddr):
         schedule_rebalance()
-        if reassimilate is not None:
-            reassimilate.kill()
-        reassimilate = eventlet.spawn(reassimilate_missing)
+
 
 def reassimilate_missing():
     eventlet.sleep(30)
-    while cfm.cfgstreams and _assimilate_missing():
+    while True:
+        try:
+            _assimilate_missing()
+        except Exception as e:
+            cfm.logException()
         eventlet.sleep(30)
 
 def _assimilate_missing(skipaddr=None):
@@ -801,6 +837,8 @@ def start_collective():
         for member in sorted(list(cfm.list_collective())):
             if member == myname:
                 continue
+            if cfm.get_collective_member(member).get('role', None) == 'nonvoting':
+                continue
             if cfm.cfgleader is None:
                 cfm.stop_following(True)
             ldrcandidate = cfm.get_collective_member(member)['address']

confluent_server/confluent/config/configmanager.py

@@ -59,13 +59,21 @@ except ModuleNotFoundError:
 import ast
 import base64
 from binascii import hexlify
+import os
+import sys
+
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
 import confluent.config.attributes as allattributes
 import confluent.config.conf as conf
-import confluent.log
 import confluent.noderange as noderange
 import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
+import confluent.log
 import copy
 import crypt
 try:
@@ -77,17 +85,16 @@ import eventlet
 import eventlet.event as event
 import eventlet.green.select as select
 import eventlet.green.threading as gthread
+import eventlet.green.subprocess as subprocess
 import fnmatch
 import hashlib
 import json
 import msgpack
 import operator
-import os
 import random
 import re
 import string
 import struct
-import sys
 import threading
 import time
 import traceback
@@ -120,7 +127,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
-_validroles = ('Administrator', 'Operator', 'Monitor', 'Stub')
+_validroles = ['Administrator', 'Operator', 'Monitor', 'Stub']
 
 membership_callback = None
 
@@ -422,7 +429,10 @@ def _push_rpc(stream, payload):
                 pass
             if membership_callback:
                 membership_callback()
-            stream.close()
+            try:
+                stream.close()
+            except Exception:
+                pass
 
 
 def decrypt_value(cryptvalue,
@@ -690,7 +700,9 @@ def relay_slaved_requests(name, listener):
                 if name not in cfgstreams:
                     raise Exception("Unexpected loss of node in followers: " + name)
                 sz = struct.unpack('!Q', msg)[0]
-                if sz != 0:
+                if sz == 0:
+                    _push_rpc(listener, b'')
+                else:
                     rpc = b''
                     while len(rpc) < sz:
                         nrpc = listener.recv(sz - len(rpc))
@@ -740,6 +752,16 @@ def relay_slaved_requests(name, listener):
                 return False
             return True
 
+lastheartbeat = None
+def check_leader():
+    _push_rpc(cfgleader, b'')
+    tries = 0
+    while tries < 30:
+        eventlet.sleep(0.1)
+        tries += 1
+        if lastheartbeat and lastheartbeat > (confluent.util.monotonic_time() - 3):
+            return True
+    raise Exception("Leader has disappeared")
 
 class StreamHandler(object):
     def __init__(self, sock):
@@ -761,10 +783,13 @@ class StreamHandler(object):
                     res = _push_rpc(self.sock, b'')  # nulls are a keepalive
                     if not res:
                         return None
+                    #TODO: this test can work fine even if the other end is
+                    # gone, go to a more affirmative test to more quickly
+                    # detect outage to peer
                     self.keepalive = confluent.util.monotonic_time() + 20
             self.expiry = confluent.util.monotonic_time() + 60
             msg = self.sock.recv(8)
-        except Exception:
+        except Exception as e:
             msg = None
         return msg
 
@@ -858,6 +883,7 @@ cfgleader = None
 def follow_channel(channel):
     global _txcount
     global _hasquorum
+    global lastheartbeat
     try:
         stop_leading()
         stop_following(channel)
@@ -865,7 +891,9 @@ def follow_channel(channel):
         msg = lh.get_next_msg()
         while msg:
             sz = struct.unpack('!Q', msg)[0]
-            if sz != 0:
+            if sz == 0:
+                lastheartbeat = confluent.util.monotonic_time()
+            else:
                 rpc = b''
                 while len(rpc) < sz:
                     nrpc = channel.recv(sz - len(rpc))
@@ -2201,9 +2229,6 @@ class ConfigManager(object):
         self._bg_sync_to_file()
 
     def add_node_attributes(self, attribmap):
-        for node in attribmap:
-            if 'groups' not in attribmap[node]:
-                attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
 
     def rename_nodes(self, renamemap):
@@ -2335,6 +2360,8 @@ class ConfigManager(object):
                         '"{0}" is not a valid node name'.format(node))
             if autocreate is False and node not in self._cfgstore['nodes']:
                 raise ValueError("node {0} does not exist".format(node))
+            if 'groups' not in attribmap[node] and node not in self._cfgstore['nodes']:
+                attribmap[node]['groups'] = []
             for attrname in list(attribmap[node]):
                 if attrname in _attraliases:
                     truename = _attraliases[attrname]
@@ -2541,6 +2568,15 @@ class ConfigManager(object):
                        data.
 
         """
+        with open(os.devnull, 'w+') as devnull:
+            worker = subprocess.Popen(
+                [sys.executable, __file__, '-r' if redact else ''],
+                        stdin=devnull, stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+            stdout, stderr = worker.communicate()
+        return stdout
+
+    def _real_dump_to_json(self, redact=None):
         dumpdata = {}
         for confarea in _config_areas:
             if confarea not in self._cfgstore:
@@ -2872,9 +2908,9 @@ def dump_db_to_directory(location, password, redact=None, skipkeys=False):
         with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
             cfgfile.write(_dump_keys(password))
             cfgfile.write('\n')
-    with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
+    with open(os.path.join(location, 'main.json'), 'wb') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
-        cfgfile.write('\n')
+        cfgfile.write(b'\n')
     if 'collective' in _cfgstore:
         with open(os.path.join(location, 'collective.json'), 'w') as cfgfile:
             cfgfile.write(json.dumps(_cfgstore['collective']))
@@ -2915,6 +2951,11 @@ def init(stateless=False):
         _cfgstore = {}
 
 
+if __name__ == '__main__':
+    redact=None
+    if '-r' in sys.argv:
+        redact=True
+    sys.stdout.write(ConfigManager(None)._real_dump_to_json(redact))
 # some unit tests worth implementing:
 # set group attribute on lower priority group, result is that node should not
 # change

confluent_server/confluent/core.py

@@ -66,7 +66,7 @@ import eventlet.semaphore as semaphore
 import itertools
 import msgpack
 import os
-import socket
+import eventlet.green.socket as socket
 import struct
 import sys
 

confluent_server/confluent/discovery/core.py

@@ -66,6 +66,7 @@ import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 import confluent.discovery.protocols.ssdp as ssdp
+#import confluent.discovery.protocols.mdns as mdns
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
 import confluent.discovery.handlers.cpstorage as cpstorage
@@ -117,6 +118,7 @@ nodehandlers = {
     'onie-switch': None,
     'cumulus-switch': None,
     'affluent-switch': None,
+    #'openbmc': None,
     'service:io-device.Lenovo:management-module': None,
     'service:thinkagile-storage': cpstorage,
     'service:lenovo-tsm': tsm,
@@ -130,6 +132,7 @@ servicenames = {
     'service:lenovo-smm2': 'lenovo-smm2',
     'affluent-switch': 'affluent-switch',
     'lenovo-xcc': 'lenovo-xcc',
+    #'openbmc': 'openbmc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
     'service:thinkagile-storage': 'thinkagile-storagebmc',
@@ -226,6 +229,7 @@ def _printable_ip(sa):
 
 def send_discovery_datum(info):
     addresses = info.get('addresses', [])
+    addresses = util.natural_sort(addresses)
     if info['handler'] == pxeh:
         enrich_pxe_info(info)
     yield msg.KeyValueData({'nodename': info.get('nodename', '')})
@@ -1229,7 +1233,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
             info['enclosure.bay'] = match[1]
             if match[2]:
                 if not discover_node(cfg, handler, info, match[2], manual):
-                    pending_nodes[match[2]] = nodename
+                    pending_nodes[match[2]] = info
                 return
         if 'enclosure.bay' not in info:
             unknown_info[info['hwaddr']] = info
@@ -1631,6 +1635,7 @@ def stop_autosense():
 
 def start_autosense():
     autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    #autosensors.add(eventlet.spawn(mdns.snoop, safe_detected, mdns))
     autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe, get_node_guess_by_uuid))
     remotescan()
 

confluent_server/confluent/discovery/protocols/mdns.py

@@ -0,0 +1,449 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Documented somewhat at
+# http://buildingskb.schneider-electric.com/view.php?AID=15197
+
+# Here is the payload of an SSDP 'announce', sent to the multicast v4/v6 1900
+# NOTIFY * HTTP/1.1
+# HOST: 239.255.255.250:1900
+# CACHE-CONTROL: max-age=1800
+# AL: https://172.30.254.151:8080/redfish/v1
+# SERVER: Linux/3.14.28-ltsi Redfish/1.0
+# NT: urn:dmtf-org:service:redfish-rest:1
+# USN: uuid:00000000-0000-0000-0005-000000000001::urn:dmtf-org:service:redfish-rest:1
+# NTS: ssdp:alive
+
+
+import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
+import confluent.neighutil as neighutil
+import confluent.noderange as noderange
+import confluent.util as util
+import confluent.log as log
+import confluent.netutil as netutil
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import eventlet.greenpool as gp
+import os
+import time
+import struct
+import traceback
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+mcastv4addr = '224.0.0.251'
+mcastv6addr = 'ff02::fb'
+
+mdns6mcast = socket.inet_pton(socket.AF_INET6, mcastv6addr)
+
+def name_to_qname(name):
+    nameparts = name.split('.')
+    qname = b''
+    for namepart in name.split('.'):
+        namepart = namepart.encode('utf8')
+        qname += bytes(bytearray([len(namepart)])) + namepart
+    qname += b'\x00'
+    return qname
+
+PTR = 12
+SRV = 33
+
+def _makequery(name):
+    return struct.pack('!HHHHHH', 
+            0, # transaction id
+            0, # flags, stdard query
+            1, # query count
+            0, # answers
+            0, # authorities
+            0) + \
+        name_to_qname(name) + \
+        struct.pack('!HH', 
+            PTR,
+            (1 << 15) | 1)  # Unicast response
+
+#listsrvs = _makequery('_services._dns-sd._udp.local')  # to get all possible services
+listobmccons = _makequery('_obmc_console._tcp.local')
+
+
+def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler):
+    if mac in peerbymacaddress and peer not in peerbymacaddress[mac]['addresses']:
+        peerbymacaddress[mac]['addresses'].append(peer)
+    else:
+        sdata = _mdns_to_dict(rsp)
+        if not sdata:
+            return 0
+        peerdata = {
+            'hwaddr': mac,
+            'addresses': [peer],
+            'services': ['openbmc'],
+            'urls': '/redfish/v1/'
+        }
+        if sdata.get('ttl', 0) == 0:
+            if byehandler:
+                eventlet.spawn_n(check_fish_handler, byehandler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+            return 1
+        if handler:
+            eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+        return 2
+        
+def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer):
+    retdata = check_fish(('/redfish/v1/', peerdata))
+    if retdata:
+        known_peers.add(peer)
+        newmacs.add(mac)
+        peerbymacaddress[mac] = retdata
+        machandlers[mac] = handler
+
+
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
+    """Watch for unsolicited mDNS answers
+
+    The handler shall be called on any service coming online.
+    byehandler is called whenever a system advertises that it is departing.
+    If no byehandler is specified, byebye messages are ignored.  The handler is
+    given (as possible), the mac address, a list of viable sockaddrs to reference
+    the peer, and the notification type (e.g.
+    'urn:dmtf-org:service:redfish-rest:1'
+
+    :param handler:  A handler for online notifications from network
+    :param byehandler: Optional handler for devices going off the network
+    """
+    # Normally, I like using v6/v4 agnostic socket. However, since we are
+    # dabbling in multicast wizardry here, such sockets can cause big problems,
+    # so we will have two distinct sockets
+    # TTL=0 is a wthdrawal, otherwise an announce
+    tracelog = log.Logger('trace')
+    net4, net6 = get_sockets()
+    net6.bind(('', 5353))
+    net4.bind(('', 5353))
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                    event=log.Events.stacktrace)
+    known_peers = set([])
+    recent_peers = set([])
+    for ifidx in util.list_interface_indexes():
+        v6grp = mdns6mcast + struct.pack('=I', ifidx)
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
+    for i4 in util.list_ips():
+        mdns4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
+                     socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            mdns4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
+    peerbymacaddress = {}
+    while True:
+        try:
+            newmacs = set([])
+            deferrednotifies = []
+            machandlers = {}
+            r = select.select((net4, net6), (), (), 60)
+            if r:
+                r = r[0]
+            recent_peers = set([])
+            while r and len(deferrednotifies) < 256:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    if peer in recent_peers:
+                        continue
+                    mac = neighutil.get_hwaddr(peer[0])
+                    if not mac:
+                        probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                        try:
+                            s.sendto(b'\x00', probepeer)
+                        except Exception:
+                            continue
+                        deferrednotifies.append((peer, rsp))
+                    datum = _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+                    if datum == 2:
+                        recent_peers.add(peer)
+                r = select.select((net4, net6), (), (), 1.5)
+                if r:
+                    r = r[0]
+            if deferrednotifies:
+                eventlet.sleep(2.2)
+            for peerrsp in deferrednotifies:
+                peer, rsp = peerrsp
+                mac = neighutil.get_hwaddr(peer[0])
+                if not mac:
+                    continue
+                _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byehandler, machandlers, handler)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)
+
+
+def _get_svrip(peerdata):
+    for addr in peerdata['addresses']:
+        if addr[0].startswith('fe80::'):
+            if '%' not in addr[0]:
+                return addr[0] + '%{0}'.format(addr[3])
+            return addr[0]
+    return peerdata['addresses'][0][0]
+
+def get_sockets():
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    return net4, net6
+        
+def active_scan(handler, protocol=None):
+    net4, net6 = get_sockets()
+    for idx in util.list_interface_indexes():
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
+                        idx)
+        try:
+            net6.sendto(listobmccons, (mcastv6addr, 5353, 0, 0))
+        except socket.error:
+            # ignore interfaces without ipv6 multicast causing error
+                pass
+    for i4 in util.list_ips():
+        if 'broadcast' not in i4:
+            continue
+        addr = i4['addr']
+        net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                        socket.inet_aton(addr))
+        try:
+            net4.sendto(listobmccons, (mcastv4addr, 5353))
+        except socket.error as se:
+            if se.errno != 101 and se.errno != 1:
+                raise
+    deadline = util.monotonic_time() + 2
+    r, _, _ = select.select((net4, net6), (), (), 2)
+    peerdata = {}
+    deferparse = []
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            if not neighutil.get_hwaddr(peer[0]):
+                probepeer = (peer[0], struct.unpack('H', os.urandom(2))[0] | 1025) + peer[2:]
+                try:
+                    s.sendto(b'\x00', probepeer)
+                except Exception:
+                    continue
+                deferparse.append((rsp, peer))
+                continue
+            _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
+    if deferparse:
+        eventlet.sleep(2.2)
+    for dp in deferparse:
+        rsp, peer = dp
+        _parse_mdns(peer, rsp, peerdata, '_obmc_console._tcp.local')
+    querypool = gp.GreenPool()
+    pooltargs = []
+    for nid in peerdata:
+        if '/redfish/v1/' not in peerdata[nid].get('urls', ()) and '/redfish/v1' not in peerdata[nid].get('urls', ()):
+            continue
+        pooltargs.append(('/redfish/v1/', peerdata[nid]))
+        # For now, don't interrogate generic redfish bmcs
+        # This is due to a need to deduplicate from some supported SLP
+        # targets (IMM, TSM, others)
+        # activate this else once the core filters/merges duplicate uuid
+        # or we drop support for those devices
+        #else:
+        #    pooltargs.append(('/redfish/v1/', peerdata[nid]))
+    for pi in querypool.imap(check_fish, pooltargs):
+        if pi is not None:
+            handler(pi)
+
+def check_fish(urldata, port=443, verifycallback=None):
+    if not verifycallback:
+        verifycallback = lambda x: True
+    url, data = urldata
+    try:
+        wc = webclient.SecureHTTPConnection(_get_svrip(data), port, verifycallback=verifycallback, timeout=1.5)
+        peerinfo = wc.grab_json_response(url)
+    except socket.error:
+        return None
+    if url == '/DeviceDescription.json':
+        try:
+            peerinfo = peerinfo[0]
+            myuuid = peerinfo['node-uuid'].lower()
+            if '-' not in myuuid:
+                myuuid = '-'.join([myuuid[:8], myuuid[8:12], myuuid[12:16], myuuid[16:20], myuuid[20:]])
+            data['uuid'] = myuuid
+            data['attributes'] = peerinfo
+            data['services'] = ['lenovo-xcc']
+            return data
+        except (IndexError, KeyError):
+            return None
+            url = '/redfish/v1/'
+            peerinfo = wc.grab_json_response('/redfish/v1/')
+    if url == '/redfish/v1/':
+        if 'UUID' in peerinfo:
+            if 'services' not in data:
+                data['services'] = ['service:redfish-bmc']
+            else:
+                data['services'].append('service:redfish-bmc')
+            data['uuid'] = peerinfo['UUID'].lower()
+            return data
+    return None
+
+def extract_qname(view, reply):
+    name = ''
+    idx = 1
+    if isinstance(view[0], int):
+        currlen = view[0]
+    else:
+        currlen = ord(view[0])
+    while currlen:
+        if currlen == 192:
+            name += extract_qname(reply[view[1]:], reply)[1] + '.'
+            view = view[2:]
+            idx += 1
+            if name:
+                return idx, name[:-1]
+            return idx, ''
+        else:
+            name += view[1:currlen + 1].tobytes().decode('utf8') + '.'
+            view = view[currlen + 1:]
+            idx += currlen + 1
+        if not view:
+            break  # some contexts don't null terminate
+        if isinstance(view[0], int):
+            currlen = view[0]
+        else:
+            currlen = ord(view[0])
+    if name:
+        return idx, name[:-1]
+    return idx, ''
+
+
+
+def _mdns_to_dict(rsp):
+    txid, flags, quests, answers, arr, morerr = struct.unpack('!HHHHHH', rsp[:12])
+    rv = memoryview(rsp[12:])
+    rspv = memoryview(rsp)
+    retval = {}
+    while quests:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass = struct.unpack('!HH', rv[:4]) 
+        quests -= 1
+        rv = rv[4:]
+    while answers:
+        idx, name = extract_qname(rv, rspv)
+        rv = rv[idx:]
+        typ, dclass, ttl, dlen = struct.unpack('!HHIH', rv[:10])
+        if 0 and typ == 12:  # PTR, we don't need for now...
+            adata = extract_qname(rv[10:], rspv)
+            if 'ptrs' not in retval:
+                retval['ptrs'] = [{'name': adata, 'ttl': ttl}]
+            else:
+                retval['ptrs'].append({'name': adata, 'ttl': ttl})
+        if typ == 33:
+            portnum = struct.unpack('!H', rv[14:16])[0]
+            retval['protoname'] = name.split('.', 1)[1]
+            retval['portnumber'] = portnum
+            retval['ttl'] = ttl
+        rv = rv[dlen + 10:]
+        answers -= 1
+    return retval
+
+
+def _parse_mdns(peer, rsp, peerdata, srvname):
+    parsed = _mdns_to_dict(rsp)
+    if not parsed:
+        return
+    if parsed.get('ttl', 0) == 0:
+        return
+    nid = peer[0]
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    if nid in peerdata:
+        peerdatum = peerdata[nid]
+        if peer not in peerdatum['addresses']:
+            peerdatum['addresses'].append(peer)
+    else:
+        peerdatum = {
+            'addresses': [peer],
+            'hwaddr': mac,
+            'services': [srvname]
+        }
+        if srvname == '_obmc_console._tcp.local':
+            peerdatum['services'] = ['openbmc']
+            peerdatum['urls'] = ['/redfish/v1/']
+        peerdata[nid] = peerdatum
+    
+
+def _parse_ssdp(peer, rsp, peerdata):
+    nid = peer[0]
+    mac = None
+    mac = neighutil.get_hwaddr(peer[0])
+    if mac:
+        nid = mac
+    headlines = rsp.split(b'\r\n')
+    try:
+        _, code, _ = headlines[0].split(b' ', 2)
+    except ValueError:
+        return
+    if code == b'200':
+        if nid in peerdata:
+            peerdatum = peerdata[nid]
+            if peer not in peerdatum['addresses']:
+                peerdatum['addresses'].append(peer)
+        else:
+            peerdatum = {
+                'addresses': [peer],
+                'hwaddr': mac,
+            }
+            peerdata[nid] = peerdatum
+        for headline in headlines[1:]:
+            if not headline:
+                continue
+            header, _, value = headline.partition(b':')
+            header = header.strip().decode('utf8')
+            value = value.strip().decode('utf8')
+            if header == 'AL' or header == 'LOCATION':
+                value = value[value.index('://')+3:]
+                value = value[value.index('/'):]
+                if 'urls' not in peerdatum:
+                    peerdatum['urls'] = [value]
+                elif value not in peerdatum['urls']:
+                    peerdatum['urls'].append(value)
+            elif header == 'ST':
+                if 'services' not in peerdatum:
+                    peerdatum['services'] = [value]
+                elif value not in peerdatum['services']:
+                    peerdatum['services'].append(value)
+            elif header == 'USN':
+                peerdatum['usn'] = value
+            elif header == 'MODELNAME':
+                peerdatum['modelname'] = value
+
+
+from pprint import pprint
+if __name__ == '__main__':
+    def printit(rsp):
+        print(repr(rsp))
+    snoop(pprint)

confluent_server/confluent/discovery/protocols/pxe.py

@@ -297,92 +297,96 @@ def proxydhcp(handler, nodeguess):
     msg.msg_namelen = ctypes.sizeof(clientaddr)
     cfg = cfm.ConfigManager(None)
     while True:
-        ready = select.select([net4011], [], [], None)
-        if not ready or not ready[0]:
-            continue
-        i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
-        #nb, client = net4011.recvfrom_into(rq)
-        if i < 240:
-            continue
-        rqv = memoryview(rq)[:i]
-        client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
-        _, level, typ = struct.unpack('QII', cmsgarr[:16])
-        if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
-            idx, recv = struct.unpack('II', cmsgarr[16:24])
-            recv = ipfromint(recv)
         try:
-            optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
-        except ValueError:
-            continue
-        hwlen = rq[2]
-        opts, disco = opts_to_dict(rq, optidx, 3)
-        disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
-        node = None
-        if disco.get('hwaddr', None) in macmap:
-            node = macmap[disco['hwaddr']]
-        elif disco.get('uuid', None) in uuidmap:
-            node = uuidmap[disco['uuid']]
-        myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
-        skiplogging = True
-        netaddr = disco['hwaddr']
-        if time.time() > ignoredisco.get(netaddr, 0) + 90:
-            skiplogging = False
-            ignoredisco[netaddr] = time.time()
-        if not myipn:
-            info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
-                            'architecture': disco['arch'],
-                            'netinfo': {'ifidx': idx, 'recvip': recv},
-                            'services': ('pxe-client',)}
-            if not skiplogging:
-                handler(info)
-        if not node:
-            if not myipn and not skiplogging:
-                log.log(
-                        {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
-                            disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
-                )})
-            continue
-        profile = None
-        if not myipn:
-            myipn = socket.inet_aton(recv)
-            profile = get_deployment_profile(node, cfg)
-            if profile:
-                log.log({
-                    'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
-            else:
+            ready = select.select([net4011], [], [], None)
+            if not ready or not ready[0]:
+                continue
+            i = recvmsg(net4011.fileno(), ctypes.pointer(msg), 0)
+            #nb, client = net4011.recvfrom_into(rq)
+            if i < 240:
+                continue
+            rqv = memoryview(rq)[:i]
+            client = (ipfromint(clientaddr.sin_addr.s_addr), socket.htons(clientaddr.sin_port))
+            _, level, typ = struct.unpack('QII', cmsgarr[:16])
+            if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
+                idx, recv = struct.unpack('II', cmsgarr[16:24])
+                recv = ipfromint(recv)
+            try:
+                optidx = rqv.tobytes().index(b'\x63\x82\x53\x63') + 4
+            except ValueError:
+                continue
+            hwlen = rq[2]
+            opts, disco = opts_to_dict(rq, optidx, 3)
+            disco['hwaddr'] = ':'.join(['{0:02x}'.format(x) for x in rq[28:28+hwlen]])
+            node = None
+            if disco.get('hwaddr', None) in macmap:
+                node = macmap[disco['hwaddr']]
+            elif disco.get('uuid', None) in uuidmap:
+                node = uuidmap[disco['uuid']]
+            myipn = myipbypeer.get(rqv[28:28+hwlen].tobytes(), None)
+            skiplogging = True
+            netaddr = disco['hwaddr']
+            if time.time() > ignoredisco.get(netaddr, 0) + 90:
+                skiplogging = False
+                ignoredisco[netaddr] = time.time()
+            if not myipn:
+                info = {'hwaddr': netaddr, 'uuid': disco['uuid'],
+                                'architecture': disco['arch'],
+                                'netinfo': {'ifidx': idx, 'recvip': recv},
+                                'services': ('pxe-client',)}
                 if not skiplogging:
-                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    handler(info)
+            if not node:
+                if not myipn and not skiplogging:
+                    log.log(
+                            {'info': 'No node matches boot attempt from uuid {0} or hardware address {1}'.format(
+                                disco.get('uuid', 'unknown'), disco.get('hwaddr', 'unknown')
+                    )})
                 continue
-        if opts.get(77, None) == b'iPXE':
-            if not profile:
+            profile = None
+            if not myipn:
+                myipn = socket.inet_aton(recv)
                 profile = get_deployment_profile(node, cfg)
-            if not profile:
-                log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                if profile:
+                    log.log({
+                        'info': 'Offering proxyDHCP boot from {0} to {1} ({2})'.format(recv, node, client[0])})
+                else:
+                    if not skiplogging:
+                        log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+            if opts.get(77, None) == b'iPXE':
+                if not profile:
+                    profile = get_deployment_profile(node, cfg)
+                if not profile:
+                    log.log({'info': 'No pending profile for {0}, skipping proxyDHCP reply'.format(node)})
+                    continue
+                myip = socket.inet_ntoa(myipn)
+                bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
+            elif disco['arch'] == 'uefi-x64':
+                bootfile = b'confluent/x86_64/ipxe.efi'
+            elif disco['arch'] == 'bios-x86':
+                bootfile = b'confluent/x86_64/ipxe.kkpxe'
+            elif disco['arch'] == 'uefi-aarch64':
+                bootfile = b'confluent/aarch64/ipxe.efi'
+            if len(bootfile) > 127:
+                log.log(
+                    {'info': 'Boot offer cannot be made to {0} as the '
+                    'profile name "{1}" is {2} characters longer than is supported '
+                    'for this boot method.'.format(
+                        node, profile, len(bootfile) - 127)})
                 continue
-            myip = socket.inet_ntoa(myipn)
-            bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(myip, profile).encode('utf8')
-        elif disco['arch'] == 'uefi-x64':
-            bootfile = b'confluent/x86_64/ipxe.efi'
-        elif disco['arch'] == 'bios-x86':
-            bootfile = b'confluent/x86_64/ipxe.kkpxe'
-        elif disco['arch'] == 'uefi-aarch64':
-            bootfile = b'confluent/aarch64/ipxe.efi'
-        if len(bootfile) > 127:
-            log.log(
-                {'info': 'Boot offer cannot be made to {0} as the '
-                'profile name "{1}" is {2} characters longer than is supported '
-                'for this boot method.'.format(
-                    node, profile, len(bootfile) - 127)})
-            continue
-        rpv[:240] = rqv[:240].tobytes()
-        rpv[0:1] = b'\x02'
-        rpv[108:108 + len(bootfile)] = bootfile
-        rpv[240:243] = b'\x35\x01\x05'
-        rpv[243:249] = b'\x36\x04' + myipn
-        rpv[20:24] = myipn
-        rpv[249:268] = b'\x61\x11' + opts[97]
-        rpv[268:280] = b'\x3c\x09PXEClient\xff'
-        net4011.sendto(rpv[:281], client)
+            rpv[:240] = rqv[:240].tobytes()
+            rpv[0:1] = b'\x02'
+            rpv[108:108 + len(bootfile)] = bootfile
+            rpv[240:243] = b'\x35\x01\x05'
+            rpv[243:249] = b'\x36\x04' + myipn
+            rpv[20:24] = myipn
+            rpv[249:268] = b'\x61\x11' + opts[97]
+            rpv[268:280] = b'\x3c\x09PXEClient\xff'
+            net4011.sendto(rpv[:281], client)
+        except Exception as e:
+            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                            event=log.Events.stacktrace)
 
 
 def start_proxydhcp(handler, nodeguess=None):

confluent_server/confluent/discovery/protocols/slp.py

@@ -63,6 +63,9 @@ def _parse_slp_header(packet):
                                            bytes(b'\x00' + packet[7:14]))
     parsed['lang'] = packet[14:14 + langlen].decode('utf-8')
     parsed['payload'] = packet[14 + langlen:]
+    errcode = struct.unpack('!H', packet[14 + langlen:16 + langlen])[0]
+    if errcode != 0:
+        return None
     if offset:
         parsed['offset'] = 14 + langlen
         parsed['extoffset'] = offset

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -258,10 +258,15 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                         seconds = int(currtime)
                                         msecs = int(currtime * 1000 % 1000)
                                         reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(node, seconds, msecs)
-                                        if '%' in peer[0]:
-                                            iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
-                                            reply += 'MGTIFACE: {0}\r\n'.format(
-                                                peer[0].split('%', 1)[1])
+                                        theip = peer[0].split('%', 1)[0]
+                                        if netutil.ip_on_same_subnet(theip, 'fe80::', 64):
+                                            if '%' in peer[0]:
+                                                ifidx = peer[0].split('%', 1)[1]
+                                                iface = socket.getaddrinfo(peer[0], 0, socket.AF_INET6, socket.SOCK_DGRAM)[0][-1][-1]
+                                            else:
+                                                ifidx = '{}'.format(peer[-1])
+                                                iface = peer[-1]
+                                            reply += 'MGTIFACE: {0}\r\n'.format(ifidx)
                                             ncfg = netutil.get_nic_config(
                                                 cfg, node, ifidx=iface)
                                             if ncfg.get('matchesnodename', None):
@@ -345,12 +350,15 @@ def _find_service(service, target):
             try:
                 net4.sendto(msg, (mcastv4addr, 1900))
             except socket.error as se:
-                if se.errno != 101:
+                if se.errno != 101 and se.errno != 1:
                     raise
             msg = smsg.format(bcast, service)
             if not isinstance(msg, bytes):
                 msg = msg.encode('utf8')
-            net4.sendto(msg, (bcast, 1900))
+            try:
+                net4.sendto(msg, (bcast, 1900))
+            except socket.error:
+                pass
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
     deadline = util.monotonic_time() + 4

confluent_server/confluent/forwarder.py

@@ -119,7 +119,7 @@ def get_port(addr, clientip, sessionid):
                 newsock.bind(('::', newport, 0, 0))
                 newsock.listen(50)
                 break
-            except OSError as e:
+            except (socket.error, OSError) as e:
                 if e.errno == 98:
                     newport += 1
                     continue

confluent_server/confluent/httpapi.py

@@ -288,12 +288,16 @@ def _authorize_request(env, operation, reqbody):
             authdata = auth.authorize(name, element=element, operation=operation)
         else:
             element = None
-    if (not authdata) and 'HTTP_COOKIE' in env:
-        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
-        if cidx >= 0:
-            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
-            sessid = sessionid
+    if not authdata:
+        if 'HTTP_CONFLUENTSESSION' in env:
+            sessionid = env['HTTP_CONFLUENTSESSION']
             sessid = sessionid
+        elif 'HTTP_COOKIE' in env:
+            cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+            if cidx >= 0:
+                sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+                sessid = sessionid
+        if sessionid:
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):
                     if env['PATH_INFO'] == '/sessions/current/logout':
@@ -483,6 +487,8 @@ def wsock_handler(ws):
                         elif clientmsg[0] == '!':
                             msg = json.loads(clientmsg[1:])
                             action = msg.get('operation', None)
+                            if not action:
+                                action = msg.get('action', None)
                             targ = msg.get('target', None)
                             if targ:
                                 authdata = auth.authorize(name, targ, operation=action)
@@ -520,6 +526,13 @@ def wsock_handler(ws):
                                             datacallback=datacallback,
                                             width=width, height=height)
                                     myconsoles[clientsessid] = consession
+                            elif action == 'resize':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].resize(
+                                    width=msg['width'], height=msg['height'])
+                            if action == 'break':
+                                clientsessid = '{0}'.format(msg['sessid'])
+                                myconsoles[clientsessid].send_break()
                             elif action == 'stop':
                                 sessid = '{0}'.format(msg.get('sessid', None))
                                 if sessid in myconsoles:

confluent_server/confluent/main.py

@@ -306,6 +306,7 @@ def run(args):
         except AttributeError:
             pass  # Windows...
         os.umask(oumask)
+    auth.check_for_yaml()
     collective.startup()
     consoleserver.initialize()
     http_bind_host, http_bind_port = _get_connector_config('http')

confluent_server/confluent/networking/macmap.py

@@ -31,7 +31,14 @@
 # this module will provide mac to switch and full 'ifName' label
 # This functionality is restricted to the null tenant
 
+import os
+import sys
+
 if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..', '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
     import confluent.config.configmanager as cfm
     import confluent.snmputil as snmp
 
@@ -43,8 +50,6 @@ import eventlet.green.select as select
 import eventlet.green.socket as socket
 
 
-import os
-import sys
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages as msg
@@ -208,7 +213,6 @@ def _offload_map_switch(switch, password, user):
 
 def _start_offloader():
     global _offloader
-    os.environ['PYTHONPATH'] = ':'.join(sys.path)
     _offloader = subprocess.Popen(
         [sys.executable, __file__, '-o'], bufsize=0, stdin=subprocess.PIPE,
         stdout=subprocess.PIPE)

confluent_server/confluent/noderange.py

@@ -186,7 +186,7 @@ class NodeRange(object):
         for idx in range(len(leftbits)):
             if leftbits[idx] == rightbits[idx]:
                 finalfmt += leftbits[idx]
-            elif leftbits[idx][0] in pp.alphas:
+            elif leftbits[idx][0] in pp.alphas or rightbits[idx][0] in pp.alphas:
                 # if string portion unequal, not going to work
                 return self.failorreturn(seqrange)
             else:

confluent_server/confluent/osimage.py

@@ -1,6 +1,4 @@
 #!/usr/bin/python
-import confluent.exceptions as exc
-import confluent.messages as msg
 import eventlet
 import eventlet.green.select as select
 import eventlet.green.subprocess as subprocess
@@ -14,6 +12,14 @@ import shutil
 import sys
 import time
 import yaml
+if __name__ == '__main__':
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.realpath(os.path.join(path, '..'))
+    if path.startswith('/opt'):
+        sys.path.append(path)
+
+import confluent.exceptions as exc
+import confluent.messages as msg
 
 COPY = 1
 EXTRACT = 2
@@ -376,7 +382,13 @@ def check_ubuntu(isoinfo):
         if not isinstance(arch, str):
             arch = arch.decode('utf8')
         major = '.'.join(ver.split('.', 2)[:2])
-        if 'efi/boot/bootaa64.efi' in isoinfo[0]:
+        if 'install/hwe-netboot/ubuntu-installer/amd64/linux' in isoinfo[0]:
+            # debian-installer style amd64
+            return {
+                'name': 'ubuntu-{0}-{1}'.format(ver, arch),
+                'method': EXTRACT,
+                'category': 'ubuntu{0}'.format(major)}
+        elif 'efi/boot/bootaa64.efi' in isoinfo[0]:
             exlist = ['casper/vmlinuz', 'casper/initrd',
                     'efi/boot/bootaa64.efi', 'efi/boot/grubaa64.efi'
                     ]
@@ -855,7 +867,6 @@ class MediaImporter(object):
         return {'phase': self.phase, 'progress': self.percent, 'profiles': self.profiles, 'error': self.error}
 
     def importmedia(self):
-        os.environ['PYTHONPATH'] = ':'.join(sys.path)
         if self.medfile:
             os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
         with open(os.devnull, 'w') as devnull:
@@ -930,3 +941,4 @@ if __name__ == '__main__':
         sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd))
     else:
         sys.exit(import_image(sys.argv[1], callback=printit))
+

confluent_server/confluent/plugins/hardwaremanagement/cooltera.py

@@ -0,0 +1,266 @@
+# Copyright 2022 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from xml.etree.ElementTree import fromstring as rfromstring
+import pyghmi.util.webclient as wc
+import confluent.util as util
+import confluent.messages as msg
+import confluent.exceptions as exc
+import eventlet.green.time as time
+import eventlet.green.socket as socket
+import eventlet.greenpool as greenpool
+import eventlet
+try:
+    import Cookie
+    httplib = eventlet.import_patched('httplib')
+except ImportError:
+    httplib = eventlet.import_patched('http.client')
+    import http.cookies as Cookie
+
+sensorsbymodel = {
+    'FS1350': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve', 'valve2'],
+    'FS600': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'pdp', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+    'RM100': ['alarms', 'dt', 'duty', 'dw', 'mode', 'p3state', 'primflow', 'ps1', 'ps1a', 'ps2', 'ps3', 'pumpspeed1', 'pumpspeed2', 'rh', 'sdp', 'secflow', 'setpoint', 't1', 't2', 't2a', 't2b', 't2c', 't3', 't3', 't4', 't5', 'valve'],
+}
+_thesensors = {
+    'RM100': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS600': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+    },
+    'FS1350': {
+        't1': ('Primary loop supply temperature', 'degC'),
+        't2': ('Secondary loop supply temperature', 'degC'),
+        't4': ('Secondary loop return temperature', 'degC'),
+        't5': ('Primary loop return temperature', 'degC'),
+        't3': ('Ambient air temperature', 'degC'),
+        'rh': ('Relative Humidity', '%'),
+        'dw': ('Dewpoint', 'degC'),
+        'pumpspeed1': ('Pump 1 Speed', '%'),
+        'pumpspeed2': ('Pump 2 Speed', '%'),
+        'pumpspeed3': ('Pump 2 Speed', '%'),
+        'alarms': ('Number of active alarms', ''),
+        'primflow': ('Input flow rate', 'l/m'),
+        'secflow': ('Output flow rate', 'l/m'),
+        'ps1': ('Secondary loop return pressure', 'bar'),
+        'ps3': ('Primary loop supply pressure', 'bar'),
+        'ps2': ('Secondary loop supply pressure', 'bar'),
+        'ps4': ('Primary loop return pressure', 'bar'),
+    },
+}
+
+
+def fromstring(inputdata):
+    if isinstance(inputdata, bytes):
+        cmpstr = b'!entity'
+    else:
+        cmpstr = '!entity'
+    if cmpstr in inputdata.lower():
+        raise Exception('!ENTITY not supported in this interface')
+    # The measures above should filter out the risky facets of xml
+    # We don't need sophisticated feature support
+    return rfromstring(inputdata)  # nosec
+
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+pdupool = greenpool.GreenPool(128)
+
+class WebResponse(httplib.HTTPResponse):
+    def _check_close(self):
+        return True
+
+class WebConnection(wc.SecureHTTPConnection):
+    response_class = WebResponse
+    def __init__(self, host, secure, verifycallback):
+        if secure:
+            port = 443
+        else:
+            port = 80
+        wc.SecureHTTPConnection.__init__(self, host, port, verifycallback=verifycallback)
+        self.secure = secure
+        self.cookies = {}
+
+    def connect(self):
+        if self.secure:
+            return super(WebConnection, self).connect()
+        addrinfo = socket.getaddrinfo(self.host, self.port)[0]
+        # workaround problems of too large mtu, moderately frequent occurance
+        # in this space
+        plainsock = socket.socket(addrinfo[0])
+        plainsock.settimeout(self.mytimeout)
+        try:
+            plainsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
+        except socket.error:
+            pass
+        plainsock.connect(addrinfo[4])
+        self.sock = plainsock
+    
+    def getresponse(self):
+        try:
+            rsp = super(WebConnection, self).getresponse()
+            try:
+                hdrs = [x.split(':', 1) for x in rsp.msg.headers]
+            except AttributeError:
+                hdrs = rsp.msg.items()
+            for hdr in hdrs:
+                if hdr[0] == 'Set-Cookie':
+                    c = Cookie.BaseCookie(hdr[1])
+                    for k in c:
+                        self.cookies[k] = c[k].value
+        except httplib.BadStatusLine:
+            self.broken = True
+            raise
+        return rsp
+
+    def request(self, method, url, body=None):
+        headers = {}
+        if body:
+            headers['Content-Length'] = len(body)
+        cookies = []
+        for cookie in self.cookies:
+            cookies.append('{0}={1}'.format(cookie, self.cookies[cookie]))
+        headers['Cookie'] = ';'.join(cookies)
+        headers['Host'] = 'pdu.cluster.net'
+        headers['Accept'] = '*/*'
+        headers['Accept-Language'] = 'en-US,en;q=0.9'
+        headers['Connection'] = 'close'
+        headers['Referer'] = 'http://pdu.cluster.net/setting_admin.htm'
+        return super(WebConnection, self).request(method, url, body, headers)
+
+    def grab_response(self, url, body=None, method=None):
+        if method is None:
+            method = 'GET' if body is None else 'POST'
+        if body:
+            self.request(method, url, body)
+        else:
+            self.request(method, url)
+        rsp = self.getresponse()
+        body = rsp.read()
+        return body, rsp.status
+
+class CoolteraClient(object):
+    def __init__(self, cdu, configmanager):
+        self.node = cdu
+        self.configmanager = configmanager
+        self._wc = None
+
+    @property
+    def wc(self):
+        if self._wc:
+            return self._wc
+        targcfg = self.configmanager.get_node_attributes(self.node,
+                                            ['hardwaremanagement.manager'],
+                                            decrypt=True)
+        targcfg = targcfg.get(self.node, {})
+        target = targcfg.get(
+            'hardwaremanagement.manager', {}).get('value', None)
+        if not target:
+            target = self.node
+        target = target.split('/', 1)[0]
+        cv = util.TLSCertVerifier(
+            self.configmanager, self.node,
+            'pubkeys.tls_hardwaremanager').verify_cert
+        self._wc = WebConnection(target, secure=True, verifycallback=cv)
+        return self._wc
+
+    
+def xml2stateinfo(statdata):
+    statdata = fromstring(statdata)
+    stateinfo = []
+    sensornames = sorted([x.tag for x in statdata])
+    themodel = None
+    for model in sensorsbymodel:
+        if sensorsbymodel[model] == sensornames:
+            themodel = model
+            break
+    thesensors = _thesensors[themodel]
+#['mode', 't1', 't2a', 't2b', 't2c', 't2', 't5', 't3', 't4', 'dw', 't3', 'rh', 'setpoint', 'secflow', 'primflow', 'ps1', 'ps1a', 'ps1b', 'ps2', 'ps3', 'ps4', 'ps5a', 'ps5b', 'ps5c', 'sdp', 'valve', 'valve2', 'pumpspeed1', 'pumpspeed2', 'pumpspeed3', 'alarms', 'dt', 'p3state', 'duty']
+    for tagname in thesensors:
+        label, units = thesensors[tagname]
+        val = statdata.find(tagname).text.replace(units, '').strip()
+        stateinfo.append({
+            'name': label,
+            'value': val,
+            'units': units.replace('degC', '°C'),
+            'type': 'Temperature',
+        })
+    return stateinfo
+
+    
+_sensors_by_node = {}
+def read_sensors(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+        for sensor in sensors:
+            yield msg.ChildCollection(simplify_name(sensors[sensor][0]))
+        return     
+    if category in ('leds, fans'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        cc = CoolteraClient(node, configmanager)
+        cc.wc.request('GET', '/status.xml')
+        rsp = cc.wc.getresponse()
+        statdata = rsp.read()
+        statinfo = xml2stateinfo(statdata)
+        _sensors_by_node[node] = (statinfo, time.time() + 1)
+        sn = _sensors_by_node.get(node, None)
+    if sn:
+        yield msg.SensorReadings(sn[0], name=node)
+
+
+def retrieve(nodes, element, configmanager, inputdata):
+    if element[0] == 'sensors':
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(read_sensors, element, node, configmanager)
+        for rsp in gp:
+            for datum in rsp:
+                yield datum
+        return

confluent_server/confluent/plugins/hardwaremanagement/deltapdu.py

@@ -199,11 +199,12 @@ def update(nodes, element, configmanager, inputdata):
     if 'outlets' not in element:
         yield msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return
+    timeout = 4
     for node in nodes:
         gc = PDUClient(node, configmanager)
         newstate = inputdata.powerstate(node)
         gc.set_outlet(element[-1], newstate)
         gc.logout()
-    eventlet.sleep(2)
+    eventlet.sleep(timeout)
     for res in retrieve(nodes, element, configmanager, inputdata):
         yield res

confluent_server/confluent/plugins/hardwaremanagement/eatonpdu.py

@@ -25,6 +25,10 @@ import hashlib
 import json
 import time
 
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
 #eaton uses 'eval' rather than json, massage it to be valid json
 def sanitize_json(data):
     if not isinstance(data, str):
@@ -131,6 +135,43 @@ class WebConnection(wc.SecureHTTPConnection):
         body = rsp.read()
         return body, rsp.status
 
+_sensors_by_node = {}
+def get_sensor_data(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    readings = []
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+    if category in ('leds, fans', 'temperature'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        gc = PDUClient(node, configmanager)
+        try:
+            sdata = gc.get_sensor_data()
+        finally:
+            gc.logout()
+        _sensors_by_node[node] = [sdata, time.time() + 1]
+        sn = _sensors_by_node.get(node, None)
+    for outlet in sn[0]:
+        for sensename in sn[0][outlet]:
+            myname = 'Outlet {0} {1}'.format(outlet, sensename)
+            measurement = sn[0][outlet][sensename]
+            if name == 'all' or simplify_name(myname) == name:
+                readings.append({
+                    'name': myname,
+                    'value': float(measurement['value']),
+                    'units': measurement['units'],
+                    'type': measurement['type'],
+                })
+    if justnames:
+        for reading in readings:
+            yield msg.ChildCollection(simplify_name(reading['name']))
+    else:
+        yield msg.SensorReadings(readings, name=node)
 
 
 class PDUClient(object):
@@ -231,6 +272,28 @@ class PDUClient(object):
             if outdata[0] == outlet:
                 return 'on' if outdata[3] else 'off'
         return
+    
+    def get_sensor_data(self):
+        rsp = self.do_request('cgi_pdu_outlets')
+        data = sanitize_json(rsp[0])
+        data = json.loads(data)
+        data = data['data'][0]
+        sdata = {}
+        for outdata in data:
+            outsense = {}
+            outletname = outdata[0][0]
+            outsense['Energy'] = {
+                'value': float(outdata[11] / 1000),
+                'units': 'kwh',
+                'type': 'Energy'
+            }
+            outsense['Power'] = {
+                'value': float(outdata[4]),
+                'units': 'w',
+                'type': 'Power',
+            }
+            sdata[outletname] = outsense
+        return sdata
 
     def set_outlet(self, outlet, state):
         rsp = self.do_request('cgi_pdu_outlets')
@@ -247,7 +310,12 @@ class PDUClient(object):
             idx += 1
 
 def retrieve(nodes, element, configmanager, inputdata):
-    if 'outlets' not in element:
+    if element[0] == 'sensors':
+        for node in nodes:
+            for res in get_sensor_data(element, node, configmanager):
+                yield res
+        return
+    elif 'outlets' not in element:
         for node in nodes:
             yield  msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return

confluent_server/confluent/plugins/hardwaremanagement/geist.py

@@ -16,6 +16,28 @@ import pyghmi.util.webclient as wc
 import confluent.util as util
 import confluent.messages as msg
 import confluent.exceptions as exc
+import eventlet.green.time as time
+import eventlet
+import eventlet.greenpool as greenpool
+
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+pdupool = greenpool.GreenPool(128)
+
+def data_by_type(indata):
+    databytype = {}
+    for keyname in indata:
+        obj = indata[keyname]
+        objtype = obj.get('type', None)
+        if not objtype:
+            continue
+        if objtype in databytype:
+            raise Exception("Multiple instances of type {} not yet supported".format(objtype))
+        databytype[objtype] = obj
+        obj['keyname'] = keyname
+    return databytype
 
 
 class GeistClient(object):
@@ -83,36 +105,145 @@ class GeistClient(object):
     def get_outlet(self, outlet):
         rsp = self.wc.grab_json_response('/api/dev')
         rsp = rsp['data']
-        if len(rsp) != 1:
+        dbt = data_by_type(rsp)
+        if 't3hd' in dbt:
+            del dbt['t3hd']
+        if len(dbt) != 1:
             raise Exception('Multiple PDUs not supported per pdu')
-        pduname = list(rsp)[0]
-        outlet = rsp[pduname]['outlet'][str(int(outlet) - 1)]
+        pdutype = list(dbt)[0]
+        outlet = dbt[pdutype]['outlet'][str(int(outlet) - 1)]
         state = outlet['state'].split('2')[-1]
         return state
 
     def set_outlet(self, outlet, state):
         rsp = self.wc.grab_json_response('/api/dev')
-        if len(rsp['data']) != 1:
+        dbt = data_by_type(rsp['data'])
+        if 't3hd' in dbt:
+            del dbt['t3hd']
+        if len(dbt) != 1:
             self.logout()
             raise Exception('Multiple PDUs per endpoint not supported')
-        pdu = list(rsp['data'])[0]
+        pdu = dbt[list(dbt)[0]]['keyname']
         outlet = int(outlet) - 1
         rsp = self.wc.grab_json_response(
             '/api/dev/{0}/outlet/{1}'.format(pdu, outlet),
             {'cmd': 'control', 'token': self.token,
             'data': {'action': state, 'delay': False}})
 
+def process_measurement(keyname, name, enttype, entname, measurement, readings, category):
+    if measurement['type'] == 'realPower':
+        if category not in ('all', 'power'):
+            return
+        readtype = 'Real Power'
+    elif measurement['type'] == 'apparentPower':
+        if category not in ('all', 'power'):
+            return
+        readtype = 'Apparent Power'
+    elif measurement['type'] == 'energy':
+        if category not in ('all', 'energy'):
+            return
+        readtype = 'Energy'
+    elif measurement['type'] == 'voltage':
+        if category not in ('all',):
+            return
+        readtype = 'Voltage'
+    elif measurement['type'] == 'temperature':
+        readtype = 'Temperature'
+    elif measurement['type'] == 'dewpoint':
+        readtype = 'Dewpoint'
+    elif measurement['type'] == 'humidity':
+        readtype = 'Humidity'
+    else:
+        return
+    myname = entname + ' ' + readtype
+    if name != 'all' and simplify_name(myname) != name:
+        return
+    readings.append({
+        'name': myname,
+        'value': float(measurement['value']),
+        'units': measurement['units'],
+        'type': readtype.split()[-1]
+    })
+ 
+
+def process_measurements(name, category, measurements, enttype, readings):
+    for measure in util.natural_sort(list(measurements)):
+        measurement = measurements[measure]['measurement']
+        entname = measurements[measure]['name']
+        for measureid in measurement:
+            process_measurement(measure, name, enttype, entname, measurement[measureid], readings, category)
+    
+
+_sensors_by_node = {}
+def read_sensors(element, node, configmanager):
+    category, name = element[-2:]
+    justnames = False
+    if len(element) == 3:
+        # just get names
+        category = name
+        name = 'all'
+        justnames = True
+    if category in ('leds, fans', 'temperature'):
+        return
+    sn = _sensors_by_node.get(node, None)
+    if not sn or sn[1] < time.time():
+        gc = GeistClient(node, configmanager)
+        adev = gc.wc.grab_json_response('/api/dev')
+        _sensors_by_node[node] = (adev, time.time() + 1)
+        sn = _sensors_by_node.get(node, None)
+    dbt = data_by_type(sn[0]['data'])
+    readings = []
+    for datatype in dbt:        
+        datum = dbt[datatype]
+        process_measurements(name, category, datum['entity'], 'entity', readings)
+        if 'outlet' in datum:
+            process_measurements(name, category, datum['outlet'], 'outlet', readings)
+    if justnames:
+        for reading in readings:
+            yield msg.ChildCollection(simplify_name(reading['name']))
+    else:
+        yield msg.SensorReadings(readings, name=node)
+
+def get_outlet(node, configmanager, element):
+    gc = GeistClient(node, configmanager)
+    state = gc.get_outlet(element[-1])
+    return msg.PowerState(node=node, state=state)
+
+def read_firmware(node, configmanager):
+    gc = GeistClient(node, configmanager)
+    adev = gc.wc.grab_json_response('/api/sys')
+    myversion = adev['data']['version']
+    yield msg.Firmware([{'PDU Firmware': {'version': myversion}}], node)
 
 def retrieve(nodes, element, configmanager, inputdata):
-    if 'outlets' not in element:
+    if 'outlets' in element:
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(get_outlet, node, configmanager, element)
+        for res in gp:
+            yield res
+           
+        return
+    elif element[0] == 'sensors':
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(read_sensors, element, node, configmanager)
+        for rsp in gp:
+            for datum in rsp:
+                yield datum
+        return
+    elif '/'.join(element).startswith('inventory/firmware/all'):
+        gp = greenpool.GreenPile(pdupool)
+        for node in nodes:
+            gp.spawn(read_firmware, node, configmanager)
+        for rsp in gp:
+            for datum in rsp:
+                yield datum
+    else:
         for node in nodes:
             yield  msg.ConfluentResourceUnavailable(node, 'Not implemented')
         return
-    for node in nodes:
-        gc = GeistClient(node, configmanager)
-        state = gc.get_outlet(element[-1])
-        yield msg.PowerState(node=node, state=state)
-
+ 
 def update(nodes, element, configmanager, inputdata):
     if 'outlets' not in element:
         yield msg.ConfluentResourceUnavailable(node, 'Not implemented')
@@ -121,5 +252,6 @@ def update(nodes, element, configmanager, inputdata):
         gc = GeistClient(node, configmanager)
         newstate = inputdata.powerstate(node)
         gc.set_outlet(element[-1], newstate)
+    eventlet.sleep(1)
     for res in retrieve(nodes, element, configmanager, inputdata):
         yield res

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -488,6 +488,7 @@ class IpmiHandler(object):
             try:
                 persistent_ipmicmds[(node, tenant)].close_confluent()
                 persistent_ipmicmds[(node, tenant)].ipmi_session._mark_broken()
+                persistent_ipmicmds[(node, tenant)].ipmi_session.logonwaiters = []
             except KeyError:  # was no previous session
                 pass
             try:
@@ -496,7 +497,6 @@ class IpmiHandler(object):
                     userid=connparams['username'],
                     password=connparams['passphrase'], kg=connparams['kg'],
                     port=connparams['port'], onlogon=self.logged)
-
                 ipmisess = persistent_ipmicmds[(node, tenant)].ipmi_session
                 begin = util.monotonic_time()
                 while ((not (ipmisess.broken or self.loggedin)) and

confluent_server/confluent/plugins/shell/ssh.py

@@ -120,6 +120,10 @@ class SshShell(conapi.Console):
         return
 
     def logon(self):
+        self.inputmode = -3
+        eventlet.spawn_n(self.do_logon)
+    
+    def do_logon(self):
         self.ssh = paramiko.SSHClient()
         self.ssh.set_missing_host_key_policy(
                 HostKeyHandler(self.nodeconfig, self.node))
@@ -186,6 +190,8 @@ class SshShell(conapi.Console):
         if self.inputmode == -2:
             self.datacallback(conapi.ConsoleEvent.Disconnect)
             return
+        elif self.inputmode == -3:
+            return
         elif self.inputmode == -1:
             while len(data) and data[0:1] == b'\x7f' and len(self.keyaction):
                 self.datacallback('\b \b')  # erase previously echoed value

confluent_server/confluent/runansible.py

@@ -17,16 +17,18 @@
 
 try:
     import confluent.sshutil as sshutil
+    import eventlet
+    import eventlet.green.subprocess as subprocess
 except ImportError:
     pass
-import eventlet
-import eventlet.green.subprocess as subprocess
+import shutil
 import json
 import msgpack
 import os
 import struct
 import sys
 
+anspypath = None
 running_status = {}
 class PlayRunner(object):
     def __init__(self, playfiles, nodes):
@@ -73,11 +75,22 @@ class PlayRunner(object):
         }
 
     def _really_run_playbooks(self):
+        global anspypath
+        mypath = anspypath
+        if not mypath:
+            ansloc = shutil.which('ansible')
+            if ansloc:
+                with open(ansloc, 'r') as onsop:
+                    shebang = onsop.readline()
+                    anspypath = shebang.strip().replace('#!', '')
+                    mypath = anspypath
+        if not mypath:
+            mypath = sys.executable    
         with open(os.devnull, 'w+') as devnull:
             targnodes = ','.join(self.nodes)
             for playfilename in self.playfiles:
                 worker = subprocess.Popen(
-                    [sys.executable, __file__, targnodes, playfilename],
+                    [mypath, __file__, targnodes, playfilename],
                     stdin=devnull, stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE)
                 stdout, self.stderr = worker.communicate()
@@ -101,7 +114,7 @@ def run_playbooks(playfiles, nodes):
 def print_result(result, state, collector=None):
     output = {
         'task_name': result.task_name,
-        'changed': result._result['changed'],
+        'changed': result._result.get('changed', ''),
     }
     output['state'] = state
     output['warnings'] = result._result.get('warnings', [])
@@ -144,18 +157,30 @@ if __name__ == '__main__':
         diff=False, verbosity=0, remote_user='root')
 
 
-    invlist = sys.argv[1] + ','
     loader = DataLoader()
-    invman = InventoryManager(loader=loader, sources=invlist)
+    invman = None
+    if os.path.exists('/etc/ansible/hosts'):
+        aninv = InventoryManager(loader=loader, sources='/etc/ansible/hosts')
+        anshost = aninv.get_host(sys.argv[1])
+        if not anshost:
+            aninv.add_host(sys.argv[1])
+        invman = aninv
+    if not invman:
+        invlist = sys.argv[1] + ','
+        invman = InventoryManager(loader=loader, sources=invlist)
     varman = VariableManager(loader=loader, inventory=invman)
-
     plays = yaml.safe_load(open(sys.argv[2]))
+    os.chdir(os.path.dirname(sys.argv[2]))
     if isinstance(plays, dict):
         plays = [plays]
-    taskman = TaskQueueManager(inventory=invman, loader=loader, passwords={},
-        variable_manager=varman, stdout_callback=ResultsCollector())
+
     for currplay in plays:
+        taskman = TaskQueueManager(inventory=invman, loader=loader, passwords={},
+            variable_manager=varman, stdout_callback=ResultsCollector())
+
         currplay['hosts'] = sys.argv[1]
+        if 'become' in currplay and 'become_user' not in currplay:
+            del currplay['become']
         play = Play().load(currplay, loader=loader)
         try:
             taskman.run(play)

confluent_server/confluent/selfservice.py

@@ -402,7 +402,7 @@ def handle_request(env, start_response):
                 yield node + '\n'
         else:
             start_response('200 OK', (('Content-Type', retype),))
-            yield dumper(sorted(nodes))
+            yield dumper(list(util.natural_sort(nodes)))
     elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody:
         try:
             reqbody = yaml.safe_load(reqbody)
@@ -469,18 +469,19 @@ def handle_request(env, start_response):
         yield ''
     elif env['PATH_INFO'].startswith('/self/remoteconfig/') and 'POST' == operation:
         scriptcat = env['PATH_INFO'].replace('/self/remoteconfig/', '')
-        slist, profile = get_scriptlist(
-            scriptcat, cfg, nodename,
-            '/var/lib/confluent/public/os/{0}/ansible/{1}')
         playlist = []
-        dirname = '/var/lib/confluent/public/os/{0}/ansible/{1}/'.format(
-            profile, scriptcat)
-        if not os.path.isdir(dirname):
-            dirname = '/var/lib/confluent/public/os/{0}/ansible/{1}.d/'.format(
-                profile, scriptcat)
-        for filename in slist:
-            if filename.endswith('.yaml') or filename.endswith('.yml'):
-                playlist.append(os.path.join(dirname, filename))
+        for privacy in ('public', 'private'):
+            slist, profile = get_scriptlist(
+                scriptcat, cfg, nodename,
+                '/var/lib/confluent/{0}/os/{{0}}/ansible/{{1}}'.format(privacy))
+            dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}/'.format(
+                profile, scriptcat, privacy)
+            if not os.path.isdir(dirname):
+                dirname = '/var/lib/confluent/{2}/os/{0}/ansible/{1}.d/'.format(
+                    profile, scriptcat, privacy)
+            for filename in slist:
+                if filename.endswith('.yaml') or filename.endswith('.yml'):
+                    playlist.append(os.path.join(dirname, filename))
         if playlist:
             runansible.run_playbooks(playlist, [nodename])
             start_response('202 Queued', ())
@@ -569,7 +570,7 @@ def get_scriptlist(scriptcat, cfg, nodename, pathtemplate):
     if not os.path.isdir(target) and os.path.isdir(target + '.d'):
         target = target + '.d'
     try:
-        slist = os.listdir(target)
+        slist = list(util.natural_sort(os.listdir(target)))
     except OSError:
         pass
     return slist, profile

confluent_server/confluent/sockapi.py

@@ -131,7 +131,8 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
                 authenticated = True
         # version 0 == original, version 1 == pickle3 allowed, 2 = pickle forbidden, msgpack allowed
         # v3 - filehandle allowed
-        send_data(connection, "Confluent -- v3 --")
+        # v4 - schema change and keepalive changes
+        send_data(connection, "Confluent -- v4 --")
         while not authenticated:  # prompt for name and passphrase
             send_data(connection, {'authpassed': 0})
             response = tlvdata.recv(connection)

genesis/buildgenesis.sh

@@ -25,6 +25,7 @@ fi
 for lic in $(cat /tmp/tmpliclist); do
     lo=${lic#/usr/share/}
     lo=${lo#licenses/}
+    lo=${lo#doc/}
     fname=$(basename $lo)
     dlo=$(dirname $lo)
     if [[ "$dlo" == *"-lib"* ]]; then
@@ -47,6 +48,8 @@ done
 mkdir -p licenses/ipmitool
 cp /usr/share/doc/ipmitool/COPYING  licenses/ipmitool
 echo %license /opt/confluent/genesis/%{arch}/licenses/ipmitool/COPYING >> confluent-genesis-out.spec
+ln -s /opt/confluent/genesis/%{arch}/licenses/kernel-core licenses/libbpf
+echo /opt/confluent/genesis/%{arch}/licenses/libbpf >> confluent-genesis-out.spec
 cp -f /boot/vmlinuz-$(uname -r) boot/kernel
 cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot
 find /boot/efi -name grubx64.efi -exec cp {} boot/efi/boot/grubx64.efi \;

genesis/confluent-genesis.spec

@@ -1,5 +1,5 @@
 %define arch x86_64
-Version: 3.6.2
+Version: 3.7.1
 Release:  1
 Name: confluent-genesis-%{arch}
 BuildArch: noarch

genesis/getlicenses.py

@@ -36,6 +36,8 @@ for rpm in rpmlist:
     if srpm.startswith('linux-firmware'):
         continue
     for relrpm in srpmtorpm[srpm]:
+        if relrpm.startswith('libss-'):
+            continue
         liclist = runcmd(f'rpm -qL {relrpm}')
         for lic in liclist:
             if not lic:
@@ -53,13 +55,25 @@ manualrpms = [
     'hwdata',
     'snmp',
     'libnl3',
-    'libbpf',  # this is covered by kernel
+    'libbpf',  # this is covered by kernel, but need to make a directory explaining. libbpf is literally just a copy of the kernel source rpm
     'sqlite',  # public domain
     'linux-firmware',  #all pertinent licenses are stripped out
     'xfsprogs',  # manually added by hand below (not in rpm)
     'tmux',  # use the extracttmuxlicenses on the source to generate NOTICE below
 ]
 manuallicenses = [
+    '/usr/share/licenses/lz4/LICENSE.BSD',
+    # cp /usr/share/doc/lz4-libs/LICENSE /usr/share/licenses/lz4/LICENSE.BSD
+    #'lz4-1.8.3]# cp LICENSE  /usr/share/licenses/lz4/LICENSE'
+    # net-snmp has a bundled openssl, but the build does not avail itself of that copy
+    '/usr/share/licenses/perl-libs/LICENSE', # ./dist/ExtUtils-CBuilder/LICENSE from perl srpm
+    '/usr/share/licenses/lz4/LICENSE',
+    '/usr/share/licenses/lm_sensors/COPYING',
+    '/usr/share/doc/libunistring/README',
+    '/usr/share/doc/hostname/COPYRIGHT',
+    '/usr/share/doc/hwdata/COPYING',
+    '/usr/share/doc/zstd/README.md',
+    '/usr/share/doc/hwdata/LICENSE',
     '/usr/share/doc/ipmitool/COPYING',
     '/usr/share/doc/libaio/COPYING',
     '/usr/share/doc/net-snmp/COPYING',
@@ -67,6 +81,30 @@ manuallicenses = [
     '/usr/share/licenses/xfsprogs/GPL-2.0',
     '/usr/share/licenses/xfsprogs/LGPL-2.1',
     '/usr/share/licenses/tmux/NOTICE',
+    '/usr/share/licenses/kernel-extra/exceptions/Linux-syscall-note',
+    '/usr/share/licenses/kernel-extra/other/Apache-2.0',
+    '/usr/share/licenses/kernel-extra/other/CC-BY-SA-4.0',
+    '/usr/share/licenses/kernel-extra/other/CDDL-1.0',
+    '/usr/share/licenses/kernel-extra/other/GPL-1.0',
+    '/usr/share/licenses/kernel-extra/other/Linux-OpenIB',
+    '/usr/share/licenses/kernel-extra/other/MPL-1.1',
+    '/usr/share/licenses/kernel-extra/other/X11',
+    '/usr/share/licenses/kernel-extra/preferred/BSD-2-Clause',
+    '/usr/share/licenses/kernel-extra/preferred/BSD-3-Clause',
+    '/usr/share/licenses/kernel-extra/preferred/BSD-3-Clause-Clear',
+    '/usr/share/licenses/kernel-extra/preferred/GPL-2.0',
+    '/usr/share/licenses/kernel-extra/preferred/LGPL-2.0',
+    '/usr/share/licenses/kernel-extra/preferred/LGPL-2.1',
+    '/usr/share/licenses/kernel-extra/preferred/MIT',
+    '/usr/share/licenses/util-linux/COPYING.GPLv3', # extract from parse-date.c, from srpm
+    '/usr/share/licenses/kmod/tools/COPYING', # GPL not LGPL, must extract from kmod srpm
+    '/usr/share/licenses/krb5-libs/NOTICE', # copy it verbatim from LICENSE, exact same file
+    '/usr/share/doc/less/README',
+    '/usr/share/licenses/libcap-ng-utils/COPYING',
+    '/usr/share/licenses/libdb/copyright', # from libdb, db-5.3.28, lang/sql/odbc/debian/copyright
+    '/usr/share/licenses/libgcrypt/LICENSES.ppc-aes-gcm', # libgcrypt license to carry forward
+    '/usr/share/licenses/libgcrypt/LICENSES', # OCB is covered by several.. OCB license 
+    '/usr/share/licenses/libgcrypt/README'
 ]
 for lic in manuallicenses:
     print(lic)

imgutil/imgutil

@@ -174,7 +174,7 @@ def capture_remote(args):
     subprocess.check_call(['rsync', __file__, '{0}:/run/imgutil/capenv/'.format(targ)])
     finfo = subprocess.check_output(['ssh', targ, 'python3', '/run/imgutil/capenv/imgutil', 'getfingerprint']).decode('utf8')
     finfo = json.loads(finfo)
-    if finfo['oscategory'] != 'el8':
+    if finfo['oscategory'] not in ('el8', 'el9'):
         raise Exception('Not yet supported for capture: ' + repr(finfo))
     oscat = finfo['oscategory']
     subprocess.check_call(['ssh', '-o', 'LogLevel=QUIET', '-t', targ, 'python3', '/run/imgutil/capenv/imgutil', 'capturelocal'])
@@ -549,7 +549,10 @@ class SuseHandler(OsHandler):
         subprocess.check_call(cmd)
         subprocess.check_call(['zypper', '-R', self.targpath, 'install'] + self.zyppargs)
         os.symlink('/usr/lib/systemd/system/sshd.service', os.path.join(self.targpath, 'etc/systemd/system/multi-user.target.wants/sshd.service'))
-        args.cmd = ['mkinitrd']
+        if os.path.exists(os.path.join(self.targpath, 'sbin/mkinitrd')):
+            args.cmd = ['mkinitrd']
+        else:
+            args.cmd = ['dracut', '-f']
         run_constrainedx(fancy_chroot, (args, self.targpath))
 
 
@@ -1008,7 +1011,7 @@ def fingerprint_host_el(args, hostpath='/'):
                     release = v
                 elif k == 'Version':
                     version = v
-        except subprocess.SubprocessError:
+        except (subprocess.SubprocessError, FileNotFoundError):
             return None
     if 'el8' not in release and 'el7' not in release and 'el9' not in release:
         return None
@@ -1206,7 +1209,14 @@ def recursecp(source, targ):
 def pack_image(args):
     outdir = args.profilename
     if '/' in outdir:
-        raise Exception('Full path not supported, supply only the profile name')
+        raise Exception('Full path not supported, supply only the profile name\n')
+    if args.baseprofile:
+        baseprofiledir = args.baseprofile
+        if '/' not in args.baseprofile:
+            baseprofiledir = os.path.join('/var/lib/confluent/public/os', args.baseprofile)
+        if not os.path.exists(baseprofiledir):
+            sys.stderr.write('Specified base profile "{0}" does not seem to exist\n'.format(baseprofiledir))
+            sys.exit(1)
     privdir = os.path.join('/var/lib/confluent/private/os/', outdir)
     outdir = os.path.join('/var/lib/confluent/public/os/', outdir)
     if os.path.exists(outdir):

misc/getusbnicaddr

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+import glob
+import os
+import select
+import socket
+
+
+def scan_nic(nicidx):
+    srvs = {}
+    s6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    s6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    s6.bind(('::', 0))
+    msg = b'M-SEARCH * HTTP/1.1\r\nHOST: [ff02::c]:1900\r\nMAN: "ssdp:discover"\r\nST: urn:dmtf-org:service:redfish-rest:1\r\nMX: 3\r\n\r\n'
+    s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, nicidx)
+    s6.sendto(msg, ('ff02::c', 1900))
+    (rsp, peer) = s6.recvfrom(9000)
+    print('{}%{}'.format(peer[0], nicidx))
+
+if __name__ == '__main__':
+    for nic in glob.glob('/sys/class/net/*'):
+        nic = os.path.basename(nic)
+        try:
+            driver = os.readlink('/sys/class/net/{}/device/driver/module'.format(nic))
+        except:
+            continue
+        if 'cdc_ether' not in driver:
+            continue
+        idx = int(open('/sys/class/net/{}/ifindex'.format(nic)).read())
+        break
+    scan_nic(idx)

misc/getxcccsr.py

@@ -0,0 +1,48 @@
+import argparse
+import io
+import gzip
+import pyghmi.redfish.command as cmd
+import os
+import sys
+
+ap = argparse.ArgumentParser(description='Certificate Generate')
+ap.add_argument('xcc', help='XCC address')
+ap.add_argument('--country', help='Two Letter Country')
+ap.add_argument('--state', help='State or Province')
+ap.add_argument('--city', help='City or Locality')
+ap.add_argument('--org', help='Organization name')
+ap.add_argument('--name', help='Common/Host Name')
+args = ap.parse_args()
+
+c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'],
+                verifycallback=lambda x: True)
+params = [
+        '0', # 'serviceType'
+        args.country,
+        args.state,
+        args.city,
+        args.org,
+        args.name,
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '',
+        '0', # ECDSA cert, 1, 2048 for RSA
+        '256',
+]
+wc = c.oem.wc
+rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_GenKeyAndCSR': ','.join(params)})
+rsp, status = wc.grab_json_response_with_status('/api/dataset', {'CSR_Format': '1'})
+rsp, status = wc.grab_json_response_with_status('/api/function', {'Sec_DownloadCSRANDCert': '0,4,0'})
+wc.request('GET', '/download/{0}'.format(rsp['FileName']))
+rsp = wc.getresponse()
+csr = rsp.read()
+if rsp.getheader('Content-Encoding', None) == 'gzip':
+    csr = gzip.GzipFile(fileobj=io.BytesIO(csr)).read()
+print(csr.decode('utf8'))
+        

misc/installxcccert.py

@@ -0,0 +1,22 @@
+import argparse
+import pyghmi.redfish.command as cmd
+import os
+import sys
+
+ap = argparse.ArgumentParser(description='Certificate Generate')
+ap.add_argument('xcc', help='XCC address')
+ap.add_argument('cert', help='Certificate in PEM format')
+args = ap.parse_args()
+
+c = cmd.Command(args.xcc, os.environ['XCCUSER'], os.environ['XCCPASS'],
+                verifycallback=lambda x: True)
+wc = c.oem.wc
+rawcert = open(args.cert, 'r').read()
+cert = ''
+incert = False
+for line in rawcert.split('\n'):
+    if incert or '-----BEGIN CERTIFICATE-----' in line:
+        incert = True
+        cert += line + '\n'
+res = wc.grab_json_response_with_status('/api/function', {'Sec_ImportCert': '0,1,0,0,,{0}'.format(cert)})
+print(repr(res))