Fix auth handling of the session/info

This should facilitate login.

Further, provide a quick persistence for the
credential test backend

LICENSE

@@ -175,28 +175,3 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

confluent_client/bin/nodeconsole

@@ -25,6 +25,7 @@ if path.startswith('/opt'):
     sys.path.append(path)
 import confluent.client as client
 import confluent.sortutil as sortutil
+import confluent.logreader as logreader
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -35,10 +36,21 @@ argparser = optparse.OptionParser(
            "console")
 argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
+argparser.add_option('-l', '--log', action='store_true', default=False,
+                     help='Enter log replay mode instead of showing a live console')
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
+if options.log:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.replay_to_console(logname)
+    sys.exit(0)
 if options.tile:
     null = open('/dev/null', 'w')
     nodes = []

confluent_client/bin/nodedeploy

@@ -85,6 +85,18 @@ def main(args):
         sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n')
     c = client.Command()
     c.stop_if_noderange_over(args.noderange, args.maxnodes)
+    errnodes = set([])
+    if not args.clear and args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.clear:
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
@@ -139,18 +151,7 @@ def main(args):
                 armed = ''
             print('{0}: {1}{2}'.format(node, profile, armed))
         sys.exit(0)
-    errnodes = set([])
     if args.network and not args.prepareonly:
-        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
-                                        bootmode='uefi',
-                                        persistent=False,
-                                        errnodes=errnodes)
-        if errnodes:
-            sys.stderr.write(
-                'Unable to set boot device for following nodes: {0}\n'.format(
-                    ','.join(errnodes)))
-            return 1
-        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
         return rc
     return 0
 

confluent_client/bin/nodediscover

@@ -48,6 +48,7 @@ columnmapping = {
     'Advertised IP': 'otheripaddrs',
     'Other IP': 'otheripaddrs',
 }
+#TODO: add chassis uuid
 
 
 def print_disco(options, session, currmac, outhandler, columns):

confluent_client/bin/nodeping

@@ -0,0 +1,140 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def run():
+
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for ping')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    nodemap = {}
+    if options.substitutename:
+        subname = options.substitutename
+        if '{' not in subname:
+            subname = '{node}' + subname
+        for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                        {'expression': subname}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                nodemap[node] = ex[node]['value']
+    for rsp in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in rsp:
+            sys.stderr.write(rsp['error'] + '\n')
+            exitcode |= rsp.get('errorcode', 1)
+        if rsp.get('item', {}).get('href', None):
+            node = rsp['item']['href'][:-1]
+            pingnode = nodemap.get(node, node)
+            cmdv = ['ping', '-c', '1', '-W', '1', pingnode]
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if not data:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if ret:
+                                print('{0}: no_ping'.format(node))
+                            else:
+                                print('{0}: ping'.format(node))
+                            if pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodepower

@@ -33,7 +33,7 @@ import confluent.client as client
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog [options] <noderange> "
-          "([status|on|off|shutdown|boot|reset])")
+          "([status|on|off|shutdown|boot|reset|pdu_status|pdu_off|pdu_on])")
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
                      help='Show previous power state')
@@ -51,22 +51,27 @@ except IndexError:
 client.check_globbing(noderange)
 setstate = None
 if len(args) > 1:
+    setstate = args[1]
     if setstate == 'softoff':
         setstate = 'shutdown'
-    elif not args[1] in ('stat', 'state', 'status'):
-        setstate = args[1]
 
-if setstate not in (None, 'on', 'off', 'shutdown', 'boot', 'reset'):
+if setstate not in (None, 'on', 'off', 'shutdown', 'boot', 'reset', 'pdu_status', 'pdu_stat', 'pdu_on', 'pdu_off'):
     argparser.print_help()
     sys.exit(1)
 session = client.Command()
 exitcode = 0
 session.add_precede_key('oldstate')
+powurl = 'state'
+if setstate and setstate.startswith('pdu_'):
+    setstate = setstate.replace('pdu_', '')
+    powurl = 'inlets/all'
+if setstate in ('status', 'state', 'stat'):
+    setstate = None
 
 if options.previous:
     # get previous states
     prev = {}
-    for rsp in session.read("/noderange/{0}/power/state".format(noderange)):
+    for rsp in session.read("/noderange/{0}/power/{1}".format(noderange, powurl)):
         # gets previous (current) states
 
         databynode = rsp["databynode"]
@@ -77,4 +82,7 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))
+def outhandler(node, res):
+    for k in res[node]:
+        client.cprint('{0}: {1}: {2}'.format(node, k.replace('inlet_', ''), res[node][k]))
+sys.exit(session.simple_noderange_command(noderange, '/power/{0}'.format(powurl), setstate, promptover=options.maxnodes, key='state', outhandler=outhandler))

confluent_client/bin/nodeshell

@@ -52,6 +52,8 @@ def run():
                          help='Specify a custom port for ssh')
     argparser.add_option('-s', '--substitutename',
                          help='Use a different name other than the nodename for ssh')
+    argparser.add_option('-x', '--noexpression', action='store_true',
+                         help='Suppress expression expansion of command')
     argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to run remote ssh command to, '
@@ -88,28 +90,39 @@ def run():
             ex = exp.get('databynode', ())
             for node in ex:
                 nodemap[node] = ex[node]['value']
-    for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
-                        {'expression': cmdstr}):
-        if 'error' in exp:
-            sys.stderr.write(exp['error'] + '\n')
-            exitcode |= exp.get('errorcode', 1)
-        ex = exp.get('databynode', ())
-        for node in ex:
-            cmd = ex[node]['value']
-            sshnode = nodemap.get(node, node)
-            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
-                cmd = cmd.encode('utf-8')
-            cmdv = ['ssh']
-            if options.port:
-                cmdv += ['-p', '{0}'.format(options.port)]
-            if options.loginname:
-                cmdv += ['-l', options.loginname]
-            cmdv += [sshnode, cmd]
-            if currprocs < concurrentprocs:
-                currprocs += 1
-                run_cmdv(node, cmdv, all, pipedesc)
-            else:
-                pendingexecs.append((node, cmdv))
+    cmdparms = []
+    if options.noexpression:
+        for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode |= res.get('errorcode', 1)
+                break
+            node = res['item']['href'][:-1]
+            cmdparms.append((node, cmdstr))
+    else:
+        for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                            {'expression': cmdstr}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                cmdparms.append((node, ex[node]['value']))
+    for node, cmd in cmdparms:
+        sshnode = nodemap.get(node, node)
+        if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+            cmd = cmd.encode('utf-8')
+        cmdv = ['ssh']
+        if options.port:
+            cmdv += ['-p', '{0}'.format(options.port)]
+        if options.loginname:
+            cmdv += ['-l', options.loginname]
+        cmdv += [sshnode, cmd]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(node, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((node, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
     rdy, _, _ = select.select(all, [], [], 10)
@@ -149,6 +162,7 @@ def run():
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:
+                    line = line.lstrip('\x08')
                     sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:

confluent_client/confluent/client.py

@@ -208,7 +208,7 @@ class Command(object):
     def add_precede_dict(self, dict):
         self._prevdict = dict
 
-    def handle_results(self, ikey, rc, res, errnodes=None):
+    def handle_results(self, ikey, rc, res, errnodes=None, outhandler=None):
         if 'error' in res:
             if errnodes is not None:
                 errnodes.add(self._currnoderange)
@@ -245,10 +245,12 @@ class Command(object):
                         node, val, self._prevdict[node]))
                 else:
                     cprint('{0}: {1}'.format(node, val))
+            elif outhandler:
+                outhandler(node, res)
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None, errnodes=None, promptover=None, **kwargs):
+                                 key=None, errnodes=None, promptover=None, outhandler=None, **kwargs):
         try:
             self._currnoderange = noderange
             rc = 0
@@ -262,13 +264,13 @@ class Command(object):
             if input is None:
                 for res in self.read('/noderange/{0}/{1}'.format(
                         noderange, resource)):
-                    rc = self.handle_results(ikey, rc, res, errnodes)
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
             else:
                 self.stop_if_noderange_over(noderange, promptover)
                 kwargs[ikey] = input
                 for res in self.update('/noderange/{0}/{1}'.format(
                         noderange, resource), kwargs):
-                    rc = self.handle_results(ikey, rc, res, errnodes)
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
             self._currnoderange = None
             return rc
         except KeyboardInterrupt:

confluent_client/confluent/logreader.py

@@ -10,12 +10,13 @@ import termios
 import tty
 
 def writeout(data):
-    done = False
     if isinstance(data, str) and not isinstance(data, bytes):
         data = data.encode('utf8')
     try:
-        sys.stdout.buffer.write(data)
-        done = True
+        if hasattr(sys.stdout, 'buffer'):
+            sys.stdout.buffer.write(data)
+        else:
+            sys.stdout.write(data)
     except IOError:
         time.sleep(0.1)
         pass
@@ -31,6 +32,7 @@ class LogReplay(object):
         self.priordata = collections.deque([])
         self.laststamp = None
         self.needclear = False
+        self.lasttxt = ''
 
     def _rewind(self, datasize=None):
         curroffset = self.bin.tell() - 16
@@ -99,10 +101,8 @@ class LogReplay(object):
                 if reverse and self.bin.tell() < endoffset:
                     output += txtout
                     continue
-                if b'\x1b[2J' in txtout:
-                    self.cleardata = txtout.split(b'\x1b[2J')
-                    for idx in range(1, len(self.cleardata)):
-                        self.cleardata[idx] = b'\x1b[2J' + self.cleardata[idx]
+                self.paginate(txtout)
+                if self.cleardata:
                     self.clearidx = 0
                     if not self.cleardata[0]:
                         self.cleardata = self.cleardata[1:]
@@ -118,8 +118,26 @@ class LogReplay(object):
                 break
         if endoffset is not None and endoffset >= 0:
             self.bin.seek(endoffset)
+        self.lasttxt = output
         return output, 1
 
+    def paginate(self, txtout):
+        cleardata = [txtout]
+        nextcleardata = []
+        for sep in (b'\x1b[2J', b'\x1b[H\x1b[J'):
+            replacementcleardata = []
+            for txtout in cleardata:
+                nextcleardata = txtout.split(sep)
+                if len(nextcleardata) > 1:
+                    for idx in range(1, len(nextcleardata)):
+                        nextcleardata[idx] = sep + nextcleardata[idx]
+                replacementcleardata.extend(nextcleardata)
+                nextcleardata = []
+            cleardata = replacementcleardata
+        if len(cleardata) > 1:
+            self.cleardata = cleardata
+        return
+
     def begin(self):
         self.needclear = True
         self.bin.seek(0)
@@ -127,8 +145,36 @@ class LogReplay(object):
     def end(self):
         self.bin.seek(0, 2)
 
+    def search(self, searchstr, skipfirst=False):
+        overlap = len(searchstr)
+        firstoffset = self.bin.tell()
+        lastoffset = firstoffset
+        output = self.lasttxt
+        while searchstr not in output:
+            output, delay = self.get_output()
+            if not self.cleardata and lastoffset == self.bin.tell():
+                self.bin.seek(firstoffset)
+                return b'', 0
+            lastoffset = self.bin.tell()
+        if skipfirst:
+            output, delay = self.get_output()
+            while searchstr not in output:
+                output, delay = self.get_output()
+                if not self.cleardata and lastoffset == self.bin.tell():
+                    self.bin.seek(firstoffset)
+                    return b'', 0
+                lastoffset = self.bin.tell()
+        clear = b'\x1b[2J\x1b[H'
+        txtchunks = self.lasttxt.split(searchstr)
+        output = clear + txtchunks[0]
+        txtchunks = txtchunks[1:]
+        while txtchunks:
+            output += b'\x1b[7m' + searchstr + b'\x1b[27m' + txtchunks[0]
+            txtchunks = txtchunks[1:]
+        return output, 0
 
-def main(txtfile, binfile):
+
+def _replay_to_console(txtfile, binfile):
     replay = LogReplay(txtfile, binfile)
     oldtcattr = termios.tcgetattr(sys.stdin.fileno())
     tty.setraw(sys.stdin.fileno())
@@ -137,7 +183,9 @@ def main(txtfile, binfile):
     reverse = False
     skipnext = False
     quitit = False
+    searchstr = None
     writeout('\x1b[2J\x1b[;H')
+    prepend = ''
     try:
         while not quitit:
             if not skipnext:
@@ -145,11 +193,18 @@ def main(txtfile, binfile):
             skipnext = False
             reverse = False
             if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
                 writeout(newdata)
+                newdata = ''
                 writeout('\x1b]0;[Time: {0}]\x07'.format(
                     time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))))
                 try:
-                    sys.stdout.buffer.flush()
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
                 except IOError:
                     pass
             while True:
@@ -173,32 +228,68 @@ def main(txtfile, binfile):
                     break
                 elif myinput.lower() == 'd':
                     writeout('\x1b];{0}\x07'.format(replay.debuginfo()))
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                elif myinput.lower() == '/':
+                    sys.stdout.write('\x1b7\x1b[99999;0H\x1b[2K')
+                    searchstr = ''
+                    nxtchr = '/'
+                    while '\r' not in searchstr:
+                        sys.stdout.write(nxtchr)
+                        sys.stdout.flush()
+                        select.select((sys.stdin,), (), (), 86400)
+                        nxtchr = sys.stdin.read(1)
+                        if nxtchr in ('\x08', '\x7f'):
+                            searchstr = searchstr[:-1]
+                            nxtchr = '\x08 \x08'
+                        else:
+                            searchstr += nxtchr
+                    if not isinstance(searchstr, bytes):
+                        searchstr = searchstr.encode('utf8')
+                    searchstr = searchstr[:-1]
+                    newdata, delay = replay.search(searchstr)
+                    if not newdata:
+                        sys.stdout.write('\x1b[1K\rNo match found')
+                        prepend = b'\x1b[1K\x1b8'
+                    else:
+                        sys.stdout.write('\x1b[1K\x1b8')
                     sys.stdout.flush()
+                    skipnext = True
+                    break
+                elif myinput.lower() == 'n' and searchstr:
+                    newdata, delay = replay.search(searchstr, True)
+                    if not newdata:
+                        sys.stdout.write('\x1b7\x1b[99999;0H\x1b[2KNo more matches found')
+                        sys.stdout.flush()
+                        prepend = b'\x1b[1K\x1b8'
+                    skipnext = True
+                    break
                 else:
                     pass # print(repr(myinput))
     except Exception:
         currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
         fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
         termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
-        writeout('\x1b[m')
+        writeout('\x1b[m\x1b[?25h\n')
         raise
     currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
     fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
     termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
-    writeout('\x1b[m')
+    writeout('\x1b[m\x1b[?25h\n')
+
+def replay_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file\n')
+            sys.exit(1)
+    _replay_to_console(txtfile, binfile)
 
 if __name__ == '__main__':
-    txtfile = sys.argv[1]
-    if len(sys.argv) > 2:
-        binfile = sys.argv[2]
-    else:
-        if os.path.exists(txtfile + '.cbl'):
-            binfile = txtfile + '.cbl'
-        else:
-            fileparts = txtfile.split('.')
-            prefix = '.'.join(fileparts[:-1])
-            binfile = prefix + '.cbl.' + fileparts[-1]
-            if not os.path.exists(binfile):
-                sys.stderr.write('Unable to locate cbl file\n')
-                sys.exit(1)
-    main(txtfile, binfile)
+    replay_to_console(sys.argv[1])

confluent_client/confluent/termhandler.py

@@ -53,13 +53,16 @@ class TermHandler(object):
             connection = None
             try:
                 connection, address = self.socket.accept()
-                connection.sendall("confetty control v1--\n")
+                connection.sendall(b"confetty control v1--\n")
                 cmd = connection.recv(8)
-                if 'GETWINID' == cmd:
-                    connection.sendall(os.environ['WINDOWID'])
+                if b'GETWINID' == cmd:
+                    winid = os.environ['WINDOWID']
+                    if not isinstance(winid, bytes):
+                        winid = winid.encode('utf8')
+                    connection.sendall(winid)
                 connection.close()
             except BaseException:
                 pass
             finally:
                 if connection is not None:
-                    connection.close()
+                    connection.close()

confluent_client/confluent_client.spec.tmpl

@@ -50,4 +50,5 @@ python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUI
 rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
+%license /opt/confluent/share/licenses/confluent_client/LICENSE
 %defattr(-,root,root)

confluent_client/doc/man/nodeping.ronn

@@ -0,0 +1,38 @@
+nodeping(8) -- Pings a node or a noderange.
+==============================
+## SYNOPSIS
+`nodeping [options] noderange`  
+
+## DESCRIPTION  
+**nodeping** is a command that pings the default NIC on a node.
+It can also be used with the `-s` flag to change the ping location to something that is 'non primary'
+
+
+## OPTIONS
+* ` -f` COUNT, `-c` COUNT, --count=COUNT  
+   Number of commands to run at a time  
+* `-h`, `--help`:  
+  Show help message and exit      
+* `-s` SUBSTITUTENAME, --substitutename=SUBSTITUTENAME  
+                        Use a different name other than the nodename for ping  
+   
+## EXAMPLES
+ * Pinging a node :  
+  `# nodeping <node>`  
+  `node : ping`    
+    
+* Pinging a group:  
+  `# nodeping <groupname>`    
+  `Node1 : ping  
+    Node2 : ping  
+    Node3 : ping`  
+
+* Pinging BMC on a node:  
+  `# nodeping -s {bmc} <noderange>`  
+  ` Node-bmc : ping`  
+
+* Fail to ping node:  
+  `# nodeping <node>`  
+  `node : no_ping`  
+ 
+

confluent_client/doc/man/nodepower.ronn

@@ -24,6 +24,9 @@ respond.
 * `reset`:  Request immediate reset of nodes of the noderange.  Nodes that are
 off will not react to this request.
 * `status`:  Behave identically to having no argument passed at all.
+* `pdu_status`: Query state of associated PDU outlets, if configured.
+* `pdu_on`: Energize all PDU outlets associated with the noderange.
+* `pdu_off`: De-energize all PDU outlets associated with the noderange.
 
 ## OPTIONS
 

confluent_client/setup.py.tmpl

@@ -1,7 +1,9 @@
 from setuptools import setup
 import os
 
-data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh'])]
+data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh']),
+              ('/opt/confluent/share/licenses/confluent_client/', ['LICENSE'])
+              ]
 try:
     scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')]
     data_files.append(('/opt/confluent/share/man/man1', ['man/man1/' + x for x in os.listdir('man/man1')]))

confluent_osdeploy/buildrpm

@@ -5,6 +5,7 @@ if [ "$NUMCOMMITS" != "$VERSION"  ]; then
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec
 cd ..
+cp ../LICENSE .
 tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy
 mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
 cd -

confluent_osdeploy/common/profile/scripts/confignet

@@ -154,12 +154,14 @@ class WickedManager(object):
                 sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces'])))
                 return
             if not stgs.get('connection_name', None):
-                stgs['connection_name'] = 'team{0}'.format(self.teamidx)
+                stgs['connection_name'] = 'bond{0}'.format(self.teamidx)
                 self.teamidx += 1
             cname = stgs['connection_name']
             with open('/etc/sysconfig/network/ifcfg-{0}'.format(cname), 'w') as teamout:
                 teamout.write(ipcfg)
-                teamout.write('TEAM_RUNNER={0}\n'.format(stgs['team_mode']))
+                if stgs['team_mode'] == 'lacp':
+                    stgs['team_mode'] = '802.3ad'
+                teamout.write("BONDING_MODULE_OPTS='mode={0} miimon=100'\nBONDING_MASTER=yes\n".format(stgs['team_mode']))
                 idx = 1
                 for iface in cfg['interfaces']:
                     subprocess.call(['wicked', 'ifdown', iface])
@@ -168,7 +170,7 @@ class WickedManager(object):
                         os.remove('/etc/sysconfig/network/ifroute-{0}'.format(iface))
                     except OSError:
                         pass
-                    teamout.write('TEAM_PORT_DEVICE_{0}={1}\n'.format(idx, iface))
+                    teamout.write('BONDING_SLAVE{0}={1}\n'.format(idx, iface))
                     idx += 1
         else:
             cname = list(cfg['interfaces'])[0]
@@ -325,18 +327,20 @@ if __name__ == '__main__':
         nc = json.loads(nc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:
-            if 'default' in netname_to_interfaces:
-                netname_to_interfaces['default']['interfaces'].add(iname)
-            else:
-                netname_to_interfaces['default'] = {'interfaces': set([iname]), 'settings': nc['default']}
+            for iname in iname.split(','):
+                if 'default' in netname_to_interfaces:
+                    netname_to_interfaces['default']['interfaces'].add(iname)
+                else:
+                    netname_to_interfaces['default'] = {'interfaces': set([iname]), 'settings': nc['default']}
         for netname in nc.get('extranets', {}):
             uname = '_' + netname
             iname = get_interface_name(idxmap[curridx], nc['extranets'][netname])
             if iname:
-                if uname in netname_to_interfaces:
-                    netname_to_interfaces[uname]['interfaces'].add(iname)
-                else:
-                    netname_to_interfaces[uname] = {'interfaces': set([iname]), 'settings': nc['extranets'][netname]}
+                for iname in iname.split(','):
+                    if uname in netname_to_interfaces:
+                        netname_to_interfaces[uname]['interfaces'].add(iname)
+                    else:
+                        netname_to_interfaces[uname] = {'interfaces': set([iname]), 'settings': nc['extranets'][netname]}
         doneidxs.add(curridx)
     rm_tmp_llas(tmpllas)
     if os.path.exists('/usr/bin/nmcli'):

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -30,7 +30,7 @@ cp start_root urlmount ../stateless-bin/
 cd ..
 ln -s el8 el9
 ln -s el8-diskless el9-diskless
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 coreos el9; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
     mkdir ${os}out
     cd ${os}out
     if [ -d ../${os}bin ]; then 
@@ -76,7 +76,9 @@ cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
 
 %install
-for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 esxi6 esxi7 coreos; do
+mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
@@ -98,3 +100,4 @@ find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} +
 
 %files
 /opt/confluent/lib/osdeploy
+%license /opt/confluent/share/licenses/confluent_osdeploy/LICENSE

confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh

@@ -5,6 +5,54 @@ umask 0077
 mkdir -p /etc/confluent
 echo -n > /etc/confluent/confluent.info
 umask $oum
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    sed -n '/^net_cfgs:/, /^[^- ]/p' cnflnt.yml |grep '^[ -]'|sed -n '/^-/, /^-/p'|head -n -1 | sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        /usr/libexec/nm-initrd-generator ip=:dhcp
+    else
+        v4addr=$(grep ^ipv4_address: $tcfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: $tcfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: $tcfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
+    fi
+    NetworkManager --configure-and-quit=initrd --no-daemon
+    hmackeyfile=$(mktemp)
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+    passfile=$(mktemp)
+    passcrypt=$(mktemp)
+    hmacfile=$(mktemp)
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    for deployer in $deploysrvs; do
+        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+            cp $passfile /etc/confluent/confluent.apikey
+            confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/profileprivate/pending/config.ign > /config.ign
+            [ -s /config.ign ] || rm /config.ign
+            confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+            confluent_profile=${confluent_profile#profile: }
+            curl -sf https://$deployer/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C /
+            exit 0
+        fi
+    done
+fi
 cd /sys/class/net
 echo "Searching for confluent deployment server...." > /dev/console
 while ! grep ^NODE /etc/confluent/confluent.info; do
@@ -65,7 +113,7 @@ curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_api
 [ -s /config.ign ] || rm /config.ign
 
 umask $oldumask
-autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg)
 autoconfigmethod=${autoconfigmethod#ipv4_method: }
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 confluent_profile=${confluent_profile#profile: }

confluent_osdeploy/el8-diskless/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/opt/confluent/bin/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -123,4 +123,5 @@ if grep installtodisk /proc/cmdline > /dev/null; then
 fi
 mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
 ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
 exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -19,6 +19,9 @@ chmod 600 /var/log/confluent/confluent-onboot.log
 tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
 logshowpid=$!
 
+rpm --import /etc/pki/rpm-gpg/*
+
+run_remote_python add_local_repositories
 run_remote_python syncfileclient
 run_remote_python confignet
 

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -19,5 +19,6 @@ mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
 done
-mkdir -p /sysroot/opt/confuent/bin
+mkdir -p /sysroot/opt/confluent/bin
 cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent/

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -5,7 +5,7 @@ except ImportError:
     import cStringIO
 import imp
 import sys
-apiclient = imp.load_source('apiclient', '/etc/confluent/apiclient')
+apiclient = imp.load_source('apiclient', '/opt/confluent/bin/apiclient')
 repo = None
 server = None
 profile = None

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -88,7 +88,7 @@ fi
 export confluent_mgr confluent_profile nodename
 curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /tmp/functions
 . /tmp/functions
-confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
+confluentpython /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
 run_remote pre.custom
 run_remote_parts pre.d
 confluentpython /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart -o /tmp/kickstart.base

confluent_osdeploy/el8/profiles/default/scripts/prechroot.sh

@@ -10,6 +10,8 @@
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 export confluent_mgr confluent_profile nodename
 cp -a /etc/confluent /mnt/sysimage/etc
+mkdir -p /mnt/sysimage/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /mnt/sysimage/opt/confluent/bin/
 chmod -R og-rwx /mnt/sysimage/etc/confluent
 cp /tmp/functions /mnt/sysimage/etc/confluent/
 hostnamectl set-hostname $nodename

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -43,6 +43,20 @@ mkdir -p /etc/pki/tls/certs
 cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
 TRIES=0
 touch /etc/confluent/confluent.info
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    mkdir -p /media/ident
+    mount /dev/disk/by-label/CNFLNT_IDNT /media/ident
+    if [ -e /media/ident/genesis_bootstrap.sh ]; then
+        exec sh /media/ident/genesis_bootstrap.sh
+    fi
+fi
+if [ -e /dev/disk/by-label/GENESIS-X86 ]; then
+    mkdir -p /media/genesis
+    mount /dev/disk/by-label/GENESIS-X86 /media/genesis
+    if [ -e /media/genesis/genesis_bootstrap.sh ]; then
+        exec sh /media/genesis/genesis_bootstrap.sh
+    fi
+fi
 cd /sys/class/net
 echo -n "Scanning for network configuration..."
 while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
@@ -141,6 +155,23 @@ elif [ "$autoconfigmethod" = "static" ]; then
         ip route add default via $v4gw
     fi
 fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver ${entry#- } >> /etc/resolv.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+echo search $dnsdomain >> /etc/resolv.conf
 echo -n "Initializing ssh..."
 ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host*key.pub; do

confluent_osdeploy/suse15/profiles/hpc/scripts/pre.sh

@@ -38,7 +38,16 @@ if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
     echo '</ntp_servers></ntp-client>' >> /tmp/ntp.cfg
     ntpcfg=$(paste -sd '' /tmp/ntp.cfg)
 fi
+mdadm --assemble --scan
 run_remote_python getinstalldisk
+if grep ^md /tmp/installdisk > /dev/null; then
+   for md in /dev/disk/*/*; do
+	rmd=$(readlink $md)
+	if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
+	    echo ${md#/dev/} > /tmp/installdisk
+	fi
+   done
+fi
 sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!<networking\(.*\)>!'"$ntpcfg"'<networking\1>!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml
 if grep append /tmp/bootloader.xml > /dev/null; then
     sed -i 's@</general>@</general>'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml

confluent_osdeploy/suse15/profiles/hpc/scripts/prechroot.sh

@@ -19,7 +19,7 @@ cp -a /tls/* /mnt/var/lib/ca-certificates/pem
 cp -a /tls/*.pem /mnt/etc/pki/trust/anchors
 cat /tls/*.pem > /mnt/etc/confluent/ca.pem
 mkdir -p /mnt/opt/confluent/bin
-cp /opt/confluent/bin/apiclient /opt/confluent/bin/
+cp /opt/confluent/bin/apiclient /mnt/opt/confluent/bin/
 
 run_remote setupssh.sh
 

confluent_osdeploy/suse15/profiles/server/ansible/firstboot.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/suse15/profiles/server/ansible/post.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/suse15/profiles/server/autoyast

@@ -0,0 +1,148 @@
+<?xml version="1.0"?>
+<!DOCTYPE profile>
+<!--
+This autoyast file will be processed by pre.sh before applying. See pre.sh for
+info on modifying its behavior, and also search and replace '%%' to remove
+dynamic behavior and replace with static configuration.
+--> 
+<profile xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns">
+    <timezone>
+      <hwclock>UTC</hwclock>
+      <!--INSERT:/tmp/timezone-->
+    </timezone>
+    <general>
+      <self_update config:type="boolean">false</self_update>
+      <mode>
+	      <confirm config:type="boolean">false</confirm>
+      </mode>
+    </general>
+    %%IFSLE%%
+    <add-on>
+      <add_on_products config:type="list">
+        <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-basesystem</product><product_dir>/Module-Basesystem</product_dir></listentry>
+  	    <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-server-applications</product><product_dir>/Module-Server-Applications</product_dir></listentry>
+	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>sle-module-containers</product><product_dir>/Module-Containers</product_dir></listentry>
+	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>SLES</product><product_dir>/Product-SLES</product_dir></listentry>
+	      <listentry><media_url/><!--INSERT:/tmp/pkgurl--><product>Legacy-Module</product><product_dir>/Module-Legacy</product_dir></listentry>
+      </add_on_products>
+    </add-on>
+    %%ENDIFSLE%%
+    <!--INSERT:/tmp/bootloader.xml-->
+    <software>
+       %%IFSLE%%
+       <products config:type="list">
+          <product>SLES</product>
+      </products>
+       %%ENDIFSLE%%
+      <patterns config:type="list">
+        <pattern>base</pattern>
+      </patterns>
+      <packages config:type="list">
+	<package>openssh</package>
+	<package>iputils</package>
+        <package>python3</package>
+        <package>openssl</package>
+        <package>chrony</package>
+        <package>rsync</package>
+        <package>screen</package>
+        <package>vim</package>
+        <package>binutils</package>
+        <package>pciutils</package>
+        <package>usbutils</package>
+        <package>nfs-client</package>
+        <package>ethtool</package>
+      </packages>
+    </software>
+  <partitioning config:type="list">
+	  <drive>
+		  <device>%%INSTDISK%%</device>
+		  <initialize config:type="boolean">true</initialize>
+		  <use>all</use>
+                  <partitions config:type="list">
+                         <partition>
+                                 <filesystem config:type="symbol">xfs</filesystem>
+                                 <mount>/</mount>
+                                 <size>max</size>
+                         </partition>
+                         <partition>
+                                 <mount>swap</mount>
+                                 <size>auto</size>
+                         </partition>
+                         <partition>
+                                 <mount>/boot</mount>
+                                 <size>500M</size>
+                         </partition>
+                 </partitions>
+          </drive>
+ </partitioning>
+    <users config:type="list">
+      <user>
+        <username>root</username>
+        <user_password>%%ROOTPASSWORD%%</user_password>
+        <encrypted config:type="boolean">true</encrypted>
+	<!--INSERT:/tmp/rootkeys.xml-->
+      </user>
+    </users>
+    <networking>
+	    <dns>
+		    <hostname>%%NODENAME%%</hostname>
+            </dns>
+	    <keep_install_network config:type="boolean">true</keep_install_network>
+    </networking>
+    <services-manager>
+	    <services>
+		    <enable config:type="list">
+			    <service>sshd</service>
+			</enable>
+		</services>
+	</services-manager>
+    <scripts>
+         <pre-scripts config:type="list">
+        <script>
+          <filename>preinstall.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh
+]]>
+          </source>
+        </script>
+      </pre-scripts>
+         <chroot-scripts config:type="list">
+        <script>
+          <filename>chroot.sh</filename>
+          <source>
+<![CDATA[
+#!/bin/sh
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/prechroot.sh
+. /tmp/prechroot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /mnt/etc/confluent/firstboot.sh
+curl -f $proto://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /mnt/etc/confluent/post.sh
+chmod +x /mnt/etc/confluent/firstboot.sh
+chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
+]]>
+          </source>
+        </script>
+        <script>
+          <chrooted config:type="boolean">true</chrooted>
+          <filename>post.sh</filename>
+          <location>file:///etc/confluent/post.sh</location>
+        </script>
+      </chroot-scripts>
+      <init-scripts config:type="list">
+        <script>
+          <filename>firstboot.sh</filename>
+          <location>file:///etc/confluent/firstboot.sh</location>
+        </script>
+     </init-scripts> 
+
+    </scripts>
+</profile>

confluent_osdeploy/suse15/profiles/server/initprofile.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+discnum=$(basename $1)
+if [ "$discnum" != 1 ]; then exit 0; fi
+if [ -e $2/boot/kernel ]; then exit 0; fi
+profile=$(basename $2)
+
+if [[ $profile =~ ^sle.* ]]; then
+    if ls $1/Product-* >& /dev/null; then
+        ln -s $1 $2/product
+    else
+        ln -s ${1%1}2 $2/product
+    fi
+fi
+sed -i 's/sle 15/SUSE Linux Enterprise 15/; s/opensuse_leap/openSUSE Leap/' $2/profile.yaml
+ln -s $1/boot/x86_64/loader/linux $2/boot/kernel && \
+ln -s $1/boot/x86_64/loader/initrd $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/bootx64.efi $1/EFI/BOOT/grub.efi $2/boot/efi/boot/
+if [[ $profile =~ ^sle.* ]]; then
+	sed -i 's/%%IFSLE%%//;s/%%ENDIFSLE%%//' $2/autoyast
+else
+	sed -i '/%%IFSLE%%/,/%%ENDIFSLE%%/d' $2/autoyast
+fi

confluent_osdeploy/suse15/profiles/server/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet # These arguments are passed to the installer
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.custom

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# This script runs at the end of the final boot
+
+. /etc/confluent/functions
+
+# Custom scripts may go here
+# run_remote example.sh
+# run_remote_python example.py

confluent_osdeploy/suse15/profiles/server/scripts/firstboot.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script runs at the end of the final boot, updating status
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+v6cfg=$(grep ^ipv6_method: /etc/confluent/confluent.deploycfg)
+v6cfg=${v6cfg#ipv6_method: }
+if [ "$v6cfg" = "static" ]; then
+    confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg)
+    confluent_mgr=${confluent_mgr#deploy_server_v6: }
+    confluent_mgr="[$confluent_mgr]"
+else
+    confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+    confluent_mgr=${confluent_mgr#deploy_server: }
+fi
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+. /etc/confluent/functions
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+
+for i in  /etc/ssh/ssh_host*key.pub; do
+    certname=${i/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname
+done
+systemctl restart sshd
+run_remote_python confignet
+run_remote firstboot.custom
+
+# Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+run_remote_parts firstboot.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+run_remote_config firstboot.d
+
+curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -f -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/suse15/profiles/server/scripts/functions

@@ -0,0 +1,196 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/suse15/profiles/server/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/suse15/profiles/server/scripts/post.custom

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# This script runs at the end of install in the installed system
+# but still under the installer kernel.
+
+# This is a good place to run most customizations that do not have any
+# dependency upon the install target kernel being active.
+
+# If there are dependencies on the kernel (drivers or special filesystems)
+# then firstboot.sh would be the script to customize.
+
+. /etc/confluent/functions
+
+# Examples:
+# run_remote script.sh
+# run_remote_python script.py

confluent_osdeploy/suse15/profiles/server/scripts/post.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script runs at the end of install in the installed system
+# but still under the installer kernel.
+
+# This is a good place to run most customizations that do not have any
+# dependency upon the install target kernel being active.
+
+# If there are dependencies on the kernel (drivers or special filesystems)
+# then firstboot.sh would be the script to customize.
+
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+
+chmod 700 /etc/confluent
+chmod og-rwx /etc/confluent/*
+
+
+export confluent_mgr confluent_profile nodename
+. /etc/confluent/functions
+
+# This will induce server side processing of the syncfile contents if
+# present
+run_remote_python syncfileclient
+
+run_remote post.custom
+
+# Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+

confluent_osdeploy/suse15/profiles/server/scripts/pre.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# This script runs before the installer executes, and sets up ssh during install as well
+# as rewriting the autoyast file with any substitutions prior to it being evaluated for real
+
+exec >> /tmp/confluent-pre.log
+exec 2>> /tmp/confluent-pre.log
+chmod 600 /tmp/confluent-pre.log
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+rootpw=$(grep rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
+if [ "$rootpw" = "null" ]; then
+    rootpw="!"
+fi
+cryptboot=$(grep encryptboot: /etc/confluent/confluent.deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+
+mkdir ~/.ssh
+cat /ssh/*pubkey > ~/.ssh/authorized_keys 2>/dev/null
+
+ssh-keygen -A
+for i in  /etc/ssh/ssh_host*key.pub; do
+    certname=${i/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname
+    echo HostKey ${i%.pub} >> /etc/ssh/sshd_config
+    echo HostCertificate $certname >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /tmp/functions
+. /tmp/functions
+ntpcfg=""
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    echo '<ntp-client><ntp_servers config:type="list">' > /tmp/ntp.cfg
+    sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg | sed 1d|sed '$d'| sed -e 's/^- /<ntp_server><address>/' -e 's!$!</address></ntp_server>!' >> /tmp/ntp.cfg
+    echo '</ntp_servers></ntp-client>' >> /tmp/ntp.cfg
+    ntpcfg=$(paste -sd '' /tmp/ntp.cfg)
+fi
+mdadm --assemble --scan
+run_remote_python getinstalldisk
+if grep ^md /tmp/installdisk > /dev/null; then
+   for md in /dev/disk/*/*; do
+	rmd=$(readlink $md)
+	if echo $rmd|grep $(cat /tmp/installdisk)$ > /dev/null; then
+	    echo ${md#/dev/} > /tmp/installdisk
+	fi
+   done
+fi
+sed -e s'!'%%INSTDISK%%'!'/dev/$(cat /tmp/installdisk)'!' -e s'!'%%NODENAME%%'!'$nodename'!' -e 's!<networking\(.*\)>!'"$ntpcfg"'<networking\1>!' -e "s?%%ROOTPASSWORD%%?${rootpw}?" /tmp/profile/autoinst.xml > /tmp/profile/modified.xml
+if grep append /tmp/bootloader.xml > /dev/null; then
+    sed -i 's@</general>@</general>'"$(tr -d '\n' < /tmp/bootloader.xml)"'@' /tmp/profile/modified.xml
+fi
+sed -i 's#root</username>#root</username>'"$(tr -d '\n' < /tmp/rootkeys.xml)"'#' /tmp/profile/modified.xml
+sed -i 's@/hwclock>@/hwclock>'"$(tr -d '\n' < /tmp/timezone)"'@' /tmp/profile/modified.xml
+sed -i 's@<media_url/>@'"$(tr -d '\n' < /tmp/pkgurl)"'@' /tmp/profile/modified.xml

confluent_osdeploy/suse15/profiles/server/scripts/prechroot.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script runs when install is finished, but while the installer
+# is still running, with the to-be-booted system mounted in /mnt
+
+# carry over deployment configuration and api key for OS install action
+confluent_mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^profile: //')
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+export confluent_mgr confluent_profile nodename
+mkdir -p /mnt/etc/confluent
+chmod 700 /mnt/etc/confluent
+cp /tmp/functions /mnt/etc/confluent/
+. /tmp/functions
+cp -a /etc/confluent/* /mnt/etc/confluent/
+cp -a /tls /mnt/etc/confluent/
+cp -a /tls/* /mnt/var/lib/ca-certificates/openssl
+cp -a /tls/* /mnt/var/lib/ca-certificates/pem
+cp -a /tls/*.pem /mnt/etc/pki/trust/anchors
+cat /tls/*.pem > /mnt/etc/confluent/ca.pem
+mkdir -p /mnt/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /mnt/opt/confluent/bin/
+
+run_remote setupssh.sh
+
+echo Port 22 >> /etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+echo Match LocalPort 22 >> /etc/ssh/sshd_config
+echo "    ChrootDirectory /mnt" >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+mkdir -p /mnt/var/log/confluent
+cp /tmp/confluent*log /mnt/var/log/confluent
+

confluent_osdeploy/suse15/profiles/server/scripts/setupssh.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Carry over install-time ssh material into installed system
+mkdir -p /mnt/root/.ssh/
+chmod 700 /mnt/root/.ssh/
+cp /root/.ssh/authorized_keys /mnt/root/.ssh/
+chmd 600 /mnt/root/.ssh/authorized_keys
+cp /etc/ssh/*key* /mnt/etc/ssh/
+for i in /etc/ssh/*-cert.pub; do
+    echo HostCertificate $i >> /mnt/etc/ssh/sshd_config
+done
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /mnt/etc/ssh/ssh_known_hosts
+done
+# Enable ~/.shosts, for the sake of root user, who is forbidden from using shosts.equiv
+echo IgnoreRhosts no >> /mnt/etc/ssh/sshd_config
+echo HostbasedAuthentication yes >> /mnt/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /mnt/etc/ssh/sshd_config
+echo Host '*' >> /mnt/etc/ssh/ssh_config
+echo "    HostbasedAuthentication yes" >> /mnt/etc/ssh/ssh_config
+echo "    EnableSSHKeysign yes" >> /mnt/etc/ssh/ssh_config
+# Limit the attempts of using host key. This prevents client from using 3 or 4
+# authentication attempts through host based attempts
+echo "    HostbasedKeyTypes *ed25519*" >> /mnt/etc/ssh/ssh_config
+
+# In SUSE platform, setuid for ssh-keysign is required for host based,
+# and also must be opted into.
+echo /usr/lib/ssh/ssh-keysign root:root 4711 >> /mnt/etc/permissions.local
+chmod 4711 /mnt/usr/lib/ssh/ssh-keysign
+
+# Download list of nodes from confluent, and put it into shosts.equiv (for most users) and .shosts (for root)
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
+cp /tmp/allnodes /mnt/root/.shosts
+cp /tmp/allnodes /mnt/etc/ssh/shosts.equiv
+

confluent_osdeploy/suse15/profiles/server/scripts/syncfileclient

@@ -0,0 +1,272 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/ubuntu22.04/initramfs/conf/conf.d/confluent

@@ -0,0 +1,8 @@
+if ! grep console= /proc/cmdline > /dev/null; then
+    /opt/confluent/bin/autocons > /custom-installation/autocons.info
+    cons=$(cat /custom-installation/autocons.info)
+    if [ ! -z "$cons" ]; then
+        echo "Auto-detected serial console: $cons" > ${cons%,*}
+    fi
+fi
+echo /scripts/init-premount/confluent >> /scripts/init-premount/ORDER

confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/post.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+deploycfg=/custom-installation/confluent/confluent.deploycfg
+confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}')
+export deploycfg confluent_mgr confluent_profile
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/post.sh
+. /tmp/post.sh

confluent_osdeploy/ubuntu22.04/initramfs/custom-installation/pre.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+deploycfg=/custom-installation/confluent/confluent.deploycfg
+confluent_mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: $deploycfg|awk '{print $2}')
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/pre.sh
+. /tmp/pre.sh

confluent_osdeploy/ubuntu22.04/initramfs/scripts/casper-bottom/99confluent

@@ -0,0 +1,75 @@
+cp -a /custom-installation/ /root/custom-installation
+cd /root
+cat /tls/*.0 >> /root/etc/ssl/certs/ca-certificates.crt
+mkdir -p /root/custom-installation/ssh
+mkdir -p /root/custom-installation/tls
+cp /ssh/* /root/custom-installation/ssh
+cp /tls/* /root/custom-installation/tls
+NODENAME=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}')
+MGR=$(grep ^EXTMGRINFO: /custom-installation/confluent/confluent.info |awk -F'|' '{print $1 " " $3}'|grep " 1$" | head -n 1 | awk '{print $2}')
+MGR=$(grep ^MANAGER: /custom-installation/confluent/confluent.info|head -n 1| awk '{print $2}')
+MGTIFACE=$(grep $MGR /custom-installation/confluent/confluent.info | grep ^EXTMGRINFO: | head -n 1 | awk -F'|' '{print $2}')
+oum=$(umask)
+umask 077
+chroot . custom-installation/confluent/bin/clortho $NODENAME $MGR > /root/custom-installation/confluent/confluent.apikey
+MGR=[$MGR]
+deploycfg=/root/custom-installation/confluent/confluent.deploycfg
+if [ -z "$MGTIFACE" ]; then
+	chroot . usr/bin/curl -f -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
+else
+	chroot . usr/bin/curl -f -H "CONFLUENT_MGTIFACE: $MGTIFACE" -H "CONFLUENT_NODENAME: $NODENAME" -H "CONFLUENT_APIKEY: $(cat /root//custom-installation/confluent/confluent.apikey)" https://${MGR}/confluent-api/self/deploycfg > $deploycfg
+fi
+umask $oum
+nic=$(grep ^MANAGER /custom-installation/confluent/confluent.info|grep fe80::|sed -e s/.*%//|head -n 1)
+nic=$(ip link |grep ^$nic:|awk '{print $2}')
+DEVICE=${nic%:}
+ipv4m=$(grep ^ipv4_method $deploycfg|awk '{print$2}')
+. /scripts/functions
+if [ "$ipv4m" = "dhcp" ]; then
+    IP=dhcp
+    configure_networking
+elif [ "$ipv4m" = "static" ]; then
+    v4addr=$(grep ^ipv4_address: $deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: $deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: $deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    dnsdomain=$(grep ^dnsdomain: $deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    if [ "$dnsdomain" = "null" ]; then dnsdomain=""; fi
+    dns=$(grep -A1 ^nameservers: $deploycfg|head -n 2|tail -n 1|sed -e 's/^- //'|sed -e "s/''//")
+    {
+        echo "DEVICE='$DEVICE'"
+        echo "PROTO='none'"
+        echo "IPV4PROTO='none'"
+        echo "IPV4ADDR='$v4addr'"
+        echo "IPV4NETMASK='$v4nm'"
+        echo "IPV4BROADCAST='$v4nm'"
+        echo "IPV4GATEWAY='$v4gw'"
+        echo "IPV4DNS1='$dns'"
+        echo "HOSTNAME='$NODENAME'"
+        echo "DNSDOMAIN='$dnsdomain'"
+        echo "DOMAINSEARCH='$dnsdomain'"
+    } > "/run/net-$DEVICE.conf"
+    configure_networking
+else
+    IP=off
+fi
+ipv4s=$(grep ^deploy_server $deploycfg|awk '{print $2}')
+osprofile=$(cat /custom-installation/confluent/osprofile)
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) autoinstall ds=nocloud-net;s=https://${ipv4s}/confluent-public/os/${osprofile}/autoinstall/"
+if [ -f /custom-installation/autocons.info ]; then
+    cons=$(cat /custom-installation/autocons.info)
+fi
+if [ ! -z "$cons" ]; then
+    echo "Installation will proceed on graphics console, autoconsole not supported during autoinstall phase" > ${cons%,*}
+    echo "Progress can be checked by using ssh to access and running the screendump command" > ${cons%,*}
+    echo ${cons%,*} > /root/tmp/autoconsdev
+    #fcmdline="$fcmdline console=${cons#/dev/}"
+fi
+echo $fcmdline > /custom-installation/confluent/fakecmdline
+/scripts/casper-bottom/58server_network

confluent_osdeploy/ubuntu22.04/initramfs/scripts/init-premount/confluent

@@ -0,0 +1,31 @@
+cd /sys/class/net
+for nic in *; do
+    ip link set $nic up
+done
+mkdir -p /custom-installation
+cp -a /opt/confluent /custom-installation
+touch /custom-installation/confluent/confluent.info
+while ! grep NODENAME /custom-installation/confluent/confluent.info; do
+    /opt/confluent/bin/copernicus -t > /custom-installation/confluent/confluent.info
+done
+MGR="[$(grep MANAGER: /custom-installation/confluent/confluent.info | head -n 1 | awk '{print $2}')]"
+osprofile=$(sed -e 's/.*osprofile=//' -e 's/ .*//' /proc/cmdline)
+cat /proc/cmdline > /custom-installation/confluent/cmdline.orig
+if [ -f /custom-installation/autocons.info ]; then
+    cons=$(cat /custom-installation/autocons.info)
+fi
+if [ ! -z "$cons" ]; then
+    echo "Preparing to deploy $osprofile from $MGR" > ${cons%,*}
+fi
+echo "Preparing to deploy $osprofile from $MGR"
+echo $osprofile > /custom-installation/confluent/osprofile
+echo URL=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso >> /conf/param.conf
+fcmdline="$(cat /custom-installation/confluent/cmdline.orig) url=http://${MGR}/confluent-public/os/$osprofile/distribution/install.iso"
+if [ ! -z "$cons" ]; then
+    fcmdline="$fcmdline console=${cons#/dev/}"
+fi
+echo $fcmdline > /custom-installation/confluent/fakecmdline
+mount -o bind /custom-installation/confluent/fakecmdline /proc/cmdline
+echo '/scripts/casper-bottom/99confluent "$@"' >> /scripts/casper-bottom/ORDER
+
+

confluent_osdeploy/ubuntu22.04/profiles/default/autoinstall/user-data

@@ -0,0 +1,17 @@
+#cloud-config
+autoinstall:
+  version: 1
+  early-commands:
+    - /custom-installation/pre.sh
+  late-commands:
+    - /custom-installation/post.sh
+  ssh:
+    install-server: true
+  storage:
+    layout:
+      name: lvm
+      match:
+        path: "%%INSTALLDISK%%"
+  user-data:
+    runcmd:
+      - /etc/confluent/firstboot.sh

confluent_osdeploy/ubuntu22.04/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/label: ubuntu/label: Ubuntu/' $2/profile.yaml && \
+ln -s $1/casper/vmlinuz $2/boot/kernel && \
+ln -s $1/casper/initrd $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/boot/* $2/boot/efi/boot
+

confluent_osdeploy/ubuntu22.04/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+echo "Confluent first boot is running"
+cp -a /etc/confluent/ssh/* /etc/ssh/
+systemctl restart sshd
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ ! -z "$rootpw" -a "$rootpw" != "null" ]; then
+	echo root:$rootpw | chpasswd -e
+fi
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+touch /etc/cloud/cloud-init.disabled
+source /etc/confluent/functions
+
+run_remote_parts firstboot.d
+run_remote_config firstboot.d
+curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -X POST -d "status: complete" https://$confluent_mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/functions

@@ -0,0 +1,196 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/post.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+cp -a /root/.ssh /target/root/
+mkdir -p /target/etc/confluent/ssh/sshd_config.d/
+chmod 700 /target/etc/confluent
+cp /custom-installation/confluent/* /target/etc/confluent/
+cp -a /custom-installation/tls /target/etc/confluent/
+chmod go-rwx /etc/confluent/*
+for i in /custom-installation/ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /target/etc/ssh/ssh_known_hosts
+done
+
+cp -a /etc/ssh/ssh_host* /target/etc/confluent/ssh/
+cp -a /etc/ssh/sshd_config.d/confluent.conf /target/etc/confluent/ssh/sshd_config.d/
+sshconf=/target/etc/ssh/ssh_config
+if [ -d /target/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/target/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /target/etc/confluent/functions
+source /target/etc/confluent/functions
+chmod +x /target/etc/confluent/firstboot.sh
+cp /tmp/allnodes /target/root/.shosts
+cp /tmp/allnodes /target/etc/ssh/shosts.equiv
+if grep ^ntpservers: /target/etc/confluent/confluent.deploycfg > /dev/null; then
+    ntps=$(sed -n '/^ntpservers:/,/^[^-]/p' /target/etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd ' ')
+    sed -i "s/#NTP=/NTP=$ntps/" /target/etc/systemd/timesyncd.conf
+fi
+textcons=$(grep ^textconsole: /target/etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
+if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+    cons=""
+    if [ -f /custom-installation/autocons.info ]; then
+        cons=$(cat /custom-installation/autocons.info)
+    fi
+    if [ ! -z "$cons" ]; then
+        sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /target/etc/default/grub
+        updategrub=1
+    fi
+fi
+kargs=$(curl https://$confluent_mgr/confluent-public/os/$confluent_profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub
+fi
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp -a /target/etc/confluent/* /etc/confluent
+mkdir -p /target/opt/confluent/bin
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin/
+cp /custom-installation/confluent/bin/apiclient /target/opt/confluent/bin
+
+mount -o bind /dev /target/dev
+mount -o bind /proc /target/proc
+mount -o bind /sys /target/sys
+if [ 1 = $updategrub ]; then
+    chroot /target update-grub
+fi
+echo "Port 22" >> /etc/ssh/sshd_config
+echo "Port 2222" >> /etc/ssh/sshd_config
+echo "Match LocalPort 22" >> /etc/ssh/sshd_config
+echo "    ChrootDirectory /target" >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+if [ -e /sys/firmware/efi ]; then
+    bootnum=$(chroot /target efibootmgr | grep ubuntu | sed -e 's/ .*//' -e 's/\*//' -e s/Boot//)
+    if [ ! -z "$bootnum" ]; then
+        currboot=$(chroot /target efibootmgr | grep ^BootOrder: | awk '{print $2}')
+        nextboot=$(echo $currboot| awk -F, '{print $1}')
+        [ "$nextboot" = "$bootnum" ] || chroot /target efibootmgr -o $bootnum,$currboot
+        chroot /target efibootmgr -D
+    fi
+fi
+cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post.d"
+source /target/etc/confluent/functions
+
+run_remote_config post
+
+umount /target/sys /target/dev /target/proc
+

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+deploycfg=/custom-installation/confluent/confluent.deploycfg
+
+cryptboot=$(grep encryptboot: $deploycfg|sed -e 's/^encryptboot: //')
+if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "null" ]; then
+   echo "****Encrypted boot requested, but not implemented for this OS, halting install" > /dev/console
+   [ -f '/tmp/autoconsdev' ] && (echo "****Encryptod boot requested, but not implemented for this OS,halting install" >> $(cat /tmp/autoconsdev))
+   while :; do sleep 86400; done
+fi
+
+
+cat /custom-installation/ssh/*pubkey > /root/.ssh/authorized_keys
+nodename=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /custom-installation/confluent/confluent.apikey)
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    keyfile=${pubkey%.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -d @$pubkey https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    echo HostKey $keyfile >> /etc/ssh/sshd_config.d/confluent.conf
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.d/confluent.conf
+done
+echo HostbasedAuthentication yes >> /etc/ssh/sshd_config.d/confluent.conf
+echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.conf
+echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
+systemctl restart sshd
+curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$confluent_mgr/confluent-api/self/nodelist > /tmp/allnodes
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/getinstalldisk > /custom-installation/getinstalldisk
+python3 /custom-installation/getinstalldisk
+sed -i s!%%INSTALLDISK%%!/dev/$(cat /tmp/installdisk)! /autoinstall.yaml

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/syncfileclient

@@ -0,0 +1,272 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/utils/Makefile

@@ -8,7 +8,7 @@ urlmount: urlmount.c
 	$(CC) $(CFLAGS) -o $@ $^  -D_FILE_OFFSET_BITS=64  -lcurl -lm -lfuse -lpthread
 	strip -s $@
 
-clortho: clortho.c
+clortho: clortho.c sha-256.c
 	$(CC) $(CFLAGS) -o $@ $^ -lcrypt
 	strip -s $@
 

confluent_osdeploy/utils/clortho.c

@@ -1,4 +1,6 @@
-/* Copyright 2019 Lenovo */
+/* Copyright 2019-2021 Lenovo */
+
+#include "sha-256.h"
 #include <arpa/inet.h>
 #include <crypt.h>
 #include <net/if.h>
@@ -16,7 +18,7 @@
 
 #define MAXPACKET 1024
 
-static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./";
+static const char cryptalpha[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 unsigned char* genpasswd(int len) {
         unsigned char * passwd;
@@ -37,6 +39,73 @@ unsigned char* genpasswd(int len) {
 
 }
 
+char * b64e(uint8_t * data, uint32_t datalen) {
+    uint8_t * currptr;
+    uint8_t * currout;
+    uint8_t currchunk[4];
+    uint8_t * retval;
+    uint32_t neededlen;
+    int32_t remaining = datalen;
+    neededlen = (datalen - 1) / 3 * 4 + 4;
+    retval = malloc(neededlen + 1);
+    currout = retval;
+    currptr = data;
+    currchunk[3] = 0;
+    while (remaining > 0) {
+        currchunk[0] = currptr[0];
+        currchunk[1] = remaining > 1 ? currptr[1] : 0;
+        currchunk[2] = remaining > 2 ? currptr[2] : 0;
+        currptr += 3;
+        currout[0] = cryptalpha[currchunk[0] >> 2];
+        currout[1] = cryptalpha[(currchunk[0] << 4 | currchunk[1] >> 4) & 0x3f];
+        currout[2] = remaining > 1 ? cryptalpha[(currchunk[1] << 2 | currchunk[2] >> 6) & 0x3f] : '=';
+        currout[3] = remaining > 2 ? cryptalpha[currchunk[2] & 0x3f] : '=';
+        remaining -= 3;
+        currout += 4;
+    }
+    retval[neededlen] = 0;
+    return retval;
+}
+
+int getpasshmac(int argc, char* argv[]) {
+    FILE *outfile;
+    uint8_t *passwd;
+    uint8_t *buffer;
+    uint8_t *tmps;
+    uint8_t *cryptpass;
+    uint8_t hmac[32];
+    uint8_t *hmac64;
+    uint8_t hmackey[64];
+    int hmackeysize;
+    if (argc < 5) {
+        fprintf(stderr, "Usage: %s passfile cryptfile hmacfile hmackey\n", argv[0]);
+        exit(1);
+    }
+    outfile = fopen(argv[4], "r");
+    hmackeysize = fread(hmackey, 1, 64, outfile);
+    fclose(outfile);
+    passwd = genpasswd(48);
+    outfile = fopen(argv[1], "w");
+    buffer = malloc(20);
+    tmps = genpasswd(16);
+    memcpy(buffer, "$5$", 3);
+    memcpy(buffer + 3, tmps, 16);
+    buffer[19] = 0;
+    fwrite(passwd, 1, 48, outfile);
+    fclose(outfile);
+    cryptpass = crypt(passwd, buffer);
+    outfile = fopen(argv[2], "w");
+    fwrite(cryptpass, 1, strlen(cryptpass), outfile);
+    fclose(outfile);
+    hmac_sha256(hmac, cryptpass, strlen(cryptpass), hmackey, hmackeysize);
+    hmac64 = b64e(hmac, 32);
+    outfile = fopen(argv[3], "w");
+    fwrite(hmac64, 1, strlen(hmac64), outfile);
+    fclose(outfile);
+    free(hmac64);
+    free(passwd);
+    free(buffer);
+}
 
 int main(int argc, char* argv[]) {
         int sock, ret;
@@ -52,12 +121,19 @@ int main(int argc, char* argv[]) {
         struct addrinfo *curr;
         struct sockaddr_in net4bind;
         struct sockaddr_in6 net6bind;
+        FILE *hmackeyfile;
+        uint8_t hmackey[64];
+        uint8_t hmac[32];
+        int hmackeysize = 0;
         unsigned char buffer[MAXPACKET];
         memset(&hints, 0, sizeof(struct addrinfo));
         memset(&net4bind, 0, sizeof(struct sockaddr_in));
         memset(&net6bind, 0, sizeof(struct sockaddr_in6));
         memset(&buffer, 0, MAXPACKET);
         memset(&timeout, 0, sizeof(struct timeval));
+        if (strstr(argv[0], "genpasshmac") != NULL) {
+            return getpasshmac(argc, argv);
+        }
         timeout.tv_sec = 10;
         net4bind.sin_port = htons(302);
         net4bind.sin_family = AF_INET;
@@ -77,6 +153,12 @@ int main(int argc, char* argv[]) {
             fprintf(stderr, "Missing node name and manager\n");
             exit(1);
         }
+        if (argc == 4) {
+            hmackeyfile = fopen(argv[3], "r");
+            hmackeysize = fread(hmackey, 1, 64, hmackeyfile);
+            fclose(hmackeyfile);
+            hmac_sha256(hmac, cryptedpass, strlen(cryptedpass), hmackey, hmackeysize);
+        }
         sock = getaddrinfo(argv[2], "13001", &hints, &addrs);
         if (sock != 0) {
                 fprintf(stderr, "Error trying to resolve %s\n", argv[2]);
@@ -118,7 +200,11 @@ int main(int argc, char* argv[]) {
         }
         slen = strlen(argv[1]) & 0xff;
         dprintf(sock, "\x01%c%s", slen, argv[1]);
-        ret = write(sock, "\x00\x00", 2);
+        if (hmackeysize) {
+            ret = write(sock, "\x06\x20", 2);
+            ret = write(sock, hmac, 32);
+        } else
+            ret = write(sock, "\x00\x00", 2);
         memset(buffer, 0, MAXPACKET);
         ret = read(sock, buffer, 2);
         while (buffer[0] != 255) {

confluent_osdeploy/utils/confluent_imginfo.c

@@ -10,7 +10,7 @@ int read_part(FILE* img, long int imgsize) {
     uint16_t shortlength;
     uint32_t length;
     uint64_t longlength;
-    if (fread(&shortlength, 2, 1, img) < 2) {
+    if (fread(&shortlength, 2, 1, img) < 1) {
         fprintf(stderr, "Error reading section\n");
         exit(1);
     }
@@ -21,7 +21,7 @@ int read_part(FILE* img, long int imgsize) {
         exit(1);
     }
     mountpath[shortlength] = 0;
-    if (fread(&length, 4, 1, img) < 4) {
+    if (fread(&length, 4, 1, img) < 1) {
         fprintf(stderr, "Failure reading segment\n");
         exit(1);
     }
@@ -30,19 +30,19 @@ int read_part(FILE* img, long int imgsize) {
         fprintf(stderr, "Error skipping json segment");
         exit(1);
     }
-    if (fread(&longlength, 8, 1, img) < 8) { // minimum size in bytes
+    if (fread(&longlength, 8, 1, img) < 1) { // minimum size in bytes
         fprintf(stderr, "Failure reading segment\n");
         exit(1);
     }
     longlength = be64toh(longlength);
     printf("%ld\t", longlength);
-    if (fread(&longlength, 8, 1, img) < 8) { // default size in bytes
+    if (fread(&longlength, 8, 1, img) < 1) { // default size in bytes
         fprintf(stderr, "Error reading segment\n");
         exit(1);
     }
     longlength = be64toh(longlength);
     printf("%ld\t", longlength);
-    if (fread(&shortlength, 2, 1, img) < 2) { // length of filesystem type
+    if (fread(&shortlength, 2, 1, img) < 1) { // length of filesystem type
         fprintf(stderr, "Error reading segment\n");
         exit(1);
     }
@@ -53,7 +53,7 @@ int read_part(FILE* img, long int imgsize) {
         exit(1);
     }
     fstype[shortlength] = 0;
-    if (fread(&shortlength, 2, 1, img) < 2) { // length of DEVICE
+    if (fread(&shortlength, 2, 1, img) < 1) { // length of DEVICE
         fprintf(stderr, "Error reading segment\n");
         exit(1);
     }
@@ -64,7 +64,7 @@ int read_part(FILE* img, long int imgsize) {
         exit(1);
     }
     devpath[shortlength] = 0;
-    if (fread(&shortlength, 2, 1, img) < 2) {
+    if (fread(&shortlength, 2, 1, img) < 1) {
         fprintf(stderr, "Error reading segment\n");
         exit(1);
     }
@@ -73,7 +73,7 @@ int read_part(FILE* img, long int imgsize) {
         fprintf(stderr, "Failure skipping padding\n");
         exit(1);
     }
-    if (fread(&longlength, 8, 1, img) < 8) {
+    if (fread(&longlength, 8, 1, img) < 1) {
         fprintf(stderr, "Error reading section\n");
         exit(1);
     }

confluent_osdeploy/utils/sha-256.c

@@ -0,0 +1,256 @@
+#include "sha-256.h"
+#include <stdlib.h>
+
+#define TOTAL_LEN_LEN 8
+
+void hmac_sha256(uint8_t* hmac, char* msg, int msglen, char* key, int keylen) {
+    uint8_t *scratch;
+    uint8_t keyprime[SIZE_OF_SHA_256_CHUNK];
+    uint8_t keymod[SIZE_OF_SHA_256_CHUNK];
+    int padneeded;
+    if (keylen > SIZE_OF_SHA_256_CHUNK) {
+        calc_sha_256(keyprime, key, keylen);
+        keylen = SIZE_OF_SHA_256_HASH;
+    } else {
+        memcpy(keyprime, key, keylen);
+    }
+    padneeded = SIZE_OF_SHA_256_CHUNK - keylen;
+    if (padneeded) {
+        memset(keyprime + keylen, 0, padneeded);
+    }
+    for (padneeded=0; padneeded < SIZE_OF_SHA_256_CHUNK; padneeded++) {
+        keymod[padneeded] = keyprime[padneeded] ^ 0x36;
+    }
+    scratch = malloc(SIZE_OF_SHA_256_CHUNK + msglen);
+    memcpy(scratch, keymod, SIZE_OF_SHA_256_CHUNK);
+    memcpy(scratch + SIZE_OF_SHA_256_CHUNK, msg, msglen);
+    calc_sha_256(hmac, scratch, SIZE_OF_SHA_256_CHUNK + msglen);
+    for (padneeded=0; padneeded < SIZE_OF_SHA_256_CHUNK; padneeded++) {
+        keymod[padneeded] = keyprime[padneeded] ^ 0x5c;
+    }
+    free(scratch);
+    scratch = malloc(SIZE_OF_SHA_256_CHUNK + SIZE_OF_SHA_256_HASH);
+    memcpy(scratch, keymod, SIZE_OF_SHA_256_CHUNK);
+    memcpy(scratch + SIZE_OF_SHA_256_CHUNK, hmac, SIZE_OF_SHA_256_HASH);
+    calc_sha_256(hmac, scratch, SIZE_OF_SHA_256_CHUNK + SIZE_OF_SHA_256_HASH);
+    free(scratch);
+}
+
+/*
+ * Comments from pseudo-code at https://en.wikipedia.org/wiki/SHA-2 are reproduced here.
+ * When useful for clarification, portions of the pseudo-code are reproduced here too.
+ */
+
+/*
+ * @brief Rotate a 32-bit value by a number of bits to the right.
+ * @param value The value to be rotated.
+ * @param count The number of bits to rotate by.
+ * @return The rotated value.
+ */
+static inline uint32_t right_rot(uint32_t value, unsigned int count)
+{
+	/*
+	 * Defined behaviour in standard C for all count where 0 < count < 32, which is what we need here.
+	 */
+	return value >> count | value << (32 - count);
+}
+
+/*
+ * @brief Update a hash value under calculation with a new chunk of data.
+ * @param h Pointer to the first hash item, of a total of eight.
+ * @param p Pointer to the chunk data, which has a standard length.
+ *
+ * @note This is the SHA-256 work horse.
+ */
+static inline void consume_chunk(uint32_t *h, const uint8_t *p)
+{
+	unsigned i, j;
+	uint32_t ah[8];
+
+	/* Initialize working variables to current hash value: */
+	for (i = 0; i < 8; i++)
+		ah[i] = h[i];
+
+	/*
+	 * The w-array is really w[64], but since we only need 16 of them at a time, we save stack by
+	 * calculating 16 at a time.
+	 *
+	 * This optimization was not there initially and the rest of the comments about w[64] are kept in their
+	 * initial state.
+	 */
+
+	/*
+	 * create a 64-entry message schedule array w[0..63] of 32-bit words (The initial values in w[0..63]
+	 * don't matter, so many implementations zero them here) copy chunk into first 16 words w[0..15] of the
+	 * message schedule array
+	 */
+	uint32_t w[16];
+
+	/* Compression function main loop: */
+	for (i = 0; i < 4; i++) {
+		for (j = 0; j < 16; j++) {
+			if (i == 0) {
+				w[j] =
+				    (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | (uint32_t)p[3];
+				p += 4;
+			} else {
+				/* Extend the first 16 words into the remaining 48 words w[16..63] of the
+				 * message schedule array: */
+				const uint32_t s0 = right_rot(w[(j + 1) & 0xf], 7) ^ right_rot(w[(j + 1) & 0xf], 18) ^
+						    (w[(j + 1) & 0xf] >> 3);
+				const uint32_t s1 = right_rot(w[(j + 14) & 0xf], 17) ^
+						    right_rot(w[(j + 14) & 0xf], 19) ^ (w[(j + 14) & 0xf] >> 10);
+				w[j] = w[j] + s0 + w[(j + 9) & 0xf] + s1;
+			}
+			const uint32_t s1 = right_rot(ah[4], 6) ^ right_rot(ah[4], 11) ^ right_rot(ah[4], 25);
+			const uint32_t ch = (ah[4] & ah[5]) ^ (~ah[4] & ah[6]);
+
+			/*
+			 * Initialize array of round constants:
+			 * (first 32 bits of the fractional parts of the cube roots of the first 64 primes 2..311):
+			 */
+			static const uint32_t k[] = {
+			    0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
+			    0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
+			    0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
+			    0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+			    0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
+			    0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
+			    0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
+			    0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+			    0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+			    0xc67178f2};
+
+			const uint32_t temp1 = ah[7] + s1 + ch + k[i << 4 | j] + w[j];
+			const uint32_t s0 = right_rot(ah[0], 2) ^ right_rot(ah[0], 13) ^ right_rot(ah[0], 22);
+			const uint32_t maj = (ah[0] & ah[1]) ^ (ah[0] & ah[2]) ^ (ah[1] & ah[2]);
+			const uint32_t temp2 = s0 + maj;
+
+			ah[7] = ah[6];
+			ah[6] = ah[5];
+			ah[5] = ah[4];
+			ah[4] = ah[3] + temp1;
+			ah[3] = ah[2];
+			ah[2] = ah[1];
+			ah[1] = ah[0];
+			ah[0] = temp1 + temp2;
+		}
+	}
+
+	/* Add the compressed chunk to the current hash value: */
+	for (i = 0; i < 8; i++)
+		h[i] += ah[i];
+}
+
+/*
+ * Public functions. See header file for documentation.
+ */
+
+void sha_256_init(struct Sha_256 *sha_256, uint8_t hash[SIZE_OF_SHA_256_HASH])
+{
+	sha_256->hash = hash;
+	sha_256->chunk_pos = sha_256->chunk;
+	sha_256->space_left = SIZE_OF_SHA_256_CHUNK;
+	sha_256->total_len = 0;
+	/*
+	 * Initialize hash values (first 32 bits of the fractional parts of the square roots of the first 8 primes
+	 * 2..19):
+	 */
+	sha_256->h[0] = 0x6a09e667;
+	sha_256->h[1] = 0xbb67ae85;
+	sha_256->h[2] = 0x3c6ef372;
+	sha_256->h[3] = 0xa54ff53a;
+	sha_256->h[4] = 0x510e527f;
+	sha_256->h[5] = 0x9b05688c;
+	sha_256->h[6] = 0x1f83d9ab;
+	sha_256->h[7] = 0x5be0cd19;
+}
+
+void sha_256_write(struct Sha_256 *sha_256, const void *data, size_t len)
+{
+	sha_256->total_len += len;
+
+	const uint8_t *p = data;
+
+	while (len > 0) {
+		/*
+		 * If the input chunks have sizes that are multiples of the calculation chunk size, no copies are
+		 * necessary. We operate directly on the input data instead.
+		 */
+		if (sha_256->space_left == SIZE_OF_SHA_256_CHUNK && len >= SIZE_OF_SHA_256_CHUNK) {
+			consume_chunk(sha_256->h, p);
+			len -= SIZE_OF_SHA_256_CHUNK;
+			p += SIZE_OF_SHA_256_CHUNK;
+			continue;
+		}
+		/* General case, no particular optimization. */
+		const size_t consumed_len = len < sha_256->space_left ? len : sha_256->space_left;
+		memcpy(sha_256->chunk_pos, p, consumed_len);
+		sha_256->space_left -= consumed_len;
+		len -= consumed_len;
+		p += consumed_len;
+		if (sha_256->space_left == 0) {
+			consume_chunk(sha_256->h, sha_256->chunk);
+			sha_256->chunk_pos = sha_256->chunk;
+			sha_256->space_left = SIZE_OF_SHA_256_CHUNK;
+		} else {
+			sha_256->chunk_pos += consumed_len;
+		}
+	}
+}
+
+uint8_t *sha_256_close(struct Sha_256 *sha_256)
+{
+	uint8_t *pos = sha_256->chunk_pos;
+	size_t space_left = sha_256->space_left;
+	uint32_t *const h = sha_256->h;
+
+	/*
+	 * The current chunk cannot be full. Otherwise, it would already have be consumed. I.e. there is space left for
+	 * at least one byte. The next step in the calculation is to add a single one-bit to the data.
+	 */
+	*pos++ = 0x80;
+	--space_left;
+
+	/*
+	 * Now, the last step is to add the total data length at the end of the last chunk, and zero padding before
+	 * that. But we do not necessarily have enough space left. If not, we pad the current chunk with zeroes, and add
+	 * an extra chunk at the end.
+	 */
+	if (space_left < TOTAL_LEN_LEN) {
+		memset(pos, 0x00, space_left);
+		consume_chunk(h, sha_256->chunk);
+		pos = sha_256->chunk;
+		space_left = SIZE_OF_SHA_256_CHUNK;
+	}
+	const size_t left = space_left - TOTAL_LEN_LEN;
+	memset(pos, 0x00, left);
+	pos += left;
+	size_t len = sha_256->total_len;
+	pos[7] = (uint8_t)(len << 3);
+	len >>= 5;
+	int i;
+	for (i = 6; i >= 0; --i) {
+		pos[i] = (uint8_t)len;
+		len >>= 8;
+	}
+	consume_chunk(h, sha_256->chunk);
+	/* Produce the final hash value (big-endian): */
+	int j;
+	uint8_t *const hash = sha_256->hash;
+	for (i = 0, j = 0; i < 8; i++) {
+		hash[j++] = (uint8_t)(h[i] >> 24);
+		hash[j++] = (uint8_t)(h[i] >> 16);
+		hash[j++] = (uint8_t)(h[i] >> 8);
+		hash[j++] = (uint8_t)h[i];
+	}
+	return sha_256->hash;
+}
+
+void calc_sha_256(uint8_t hash[SIZE_OF_SHA_256_HASH], const void *input, size_t len)
+{
+	struct Sha_256 sha_256;
+	sha_256_init(&sha_256, hash);
+	sha_256_write(&sha_256, input, len);
+	(void)sha_256_close(&sha_256);
+}

confluent_osdeploy/utils/sha-256.h

@@ -0,0 +1,104 @@
+#ifndef SHA_256_H
+#define SHA_256_H
+
+#include <stdint.h>
+#include <string.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * @brief Size of the SHA-256 sum. This times eight is 256 bits.
+ */
+#define SIZE_OF_SHA_256_HASH 32
+
+/*
+ * @brief Size of the chunks used for the calculations.
+ *
+ * @note This should mostly be ignored by the user, although when using the streaming API, it has an impact for
+ * performance. Add chunks whose size is a multiple of this, and you will avoid a lot of superfluous copying in RAM!
+ */
+#define SIZE_OF_SHA_256_CHUNK 64
+
+/*
+ * @brief The opaque SHA-256 type, that should be instantiated when using the streaming API.
+ *
+ * @note Although the details are exposed here, in order to make instantiation easy, you should refrain from directly
+ * accessing the fields, as they may change in the future.
+ */
+struct Sha_256 {
+	uint8_t *hash;
+	uint8_t chunk[SIZE_OF_SHA_256_CHUNK];
+	uint8_t *chunk_pos;
+	size_t space_left;
+	size_t total_len;
+	uint32_t h[8];
+};
+
+/*
+ * @brief The simple SHA-256 calculation function.
+ * @param hash Hash array, where the result is delivered.
+ * @param input Pointer to the data the hash shall be calculated on.
+ * @param len Length of the input data, in byte.
+ *
+ * @note If all of the data you are calculating the hash value on is available in a contiguous buffer in memory, this is
+ * the function you should use.
+ *
+ * @note If either of the passed pointers is NULL, the results are unpredictable.
+ */
+void calc_sha_256(uint8_t hash[SIZE_OF_SHA_256_HASH], const void *input, size_t len);
+
+/*
+ * @brief Initialize a SHA-256 streaming calculation.
+ * @param sha_256 A pointer to a SHA-256 structure.
+ * @param hash Hash array, where the result will be delivered.
+ *
+ * @note If all of the data you are calculating the hash value on is not available in a contiguous buffer in memory, this is
+ * where you should start. Instantiate a SHA-256 structure, for instance by simply declaring it locally, make your hash
+ * buffer available, and invoke this function. Once a SHA-256 hash has been calculated (see further below) a SHA-256
+ * structure can be initialized again for the next calculation.
+ *
+ * @note If either of the passed pointers is NULL, the results are unpredictable.
+ */
+void hmac_sha256(uint8_t* hmac, char* msg, int msglen, char* key, int keylen);
+void sha_256_init(struct Sha_256 *sha_256, uint8_t hash[SIZE_OF_SHA_256_HASH]);
+
+/*
+ * @brief Stream more input data for an on-going SHA-256 calculation.
+ * @param sha_256 A pointer to a previously initialized SHA-256 structure.
+ * @param data Pointer to the data to be added to the calculation.
+ * @param len Length of the data to add, in byte.
+ *
+ * @note This function may be invoked an arbitrary number of times between initialization and closing, but the maximum
+ * data length is limited by the SHA-256 algorithm: the total number of bits (i.e. the total number of bytes times
+ * eight) must be representable by a 64-bit unsigned integer. While that is not a practical limitation, the results are
+ * unpredictable if that limit is exceeded.
+ *
+ * @note This function may be invoked on empty data (zero length), although that obviously will not add any data.
+ *
+ * @note If either of the passed pointers is NULL, the results are unpredictable.
+ */
+void sha_256_write(struct Sha_256 *sha_256, const void *data, size_t len);
+
+/*
+ * @brief Conclude a SHA-256 streaming calculation, making the hash value available.
+ * @param sha_256 A pointer to a previously initialized SHA-256 structure.
+ * @return Pointer to the hash array, where the result is delivered.
+ *
+ * @note After this function has been invoked, the result is available in the hash buffer that initially was provided. A
+ * pointer to the hash value is returned for convenience, but you should feel free to ignore it: it is simply a pointer
+ * to the first byte of your initially provided hash array.
+ *
+ * @note If the passed pointer is NULL, the results are unpredictable.
+ *
+ * @note Invoking this function for a calculation with no data (the writing function has never been invoked, or it only
+ * has been invoked with empty data) is legal. It will calculate the SHA-256 value of the empty string.
+ */
+uint8_t *sha_256_close(struct Sha_256 *sha_256);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

confluent_server/bin/osdeploy

@@ -22,6 +22,8 @@ import confluent.util as util
 import confluent.client as client
 import confluent.sshutil as sshutil
 import confluent.certutil as certutil
+import confluent.netutil as netutil
+import socket
 try:
     input = raw_input
 except NameError:
@@ -149,10 +151,13 @@ def local_node_trust_setup():
     neededlines = set([
         'HostbasedAuthentication yes', 'HostbasedUsesNameFromPacketOnly yes',
         'IgnoreRhosts no'])
-    if domain and not myname.endswith(domain):
-        myprincipals.add('{0}.{1}'.format(myname, domain))
-    if domain and '.' in myname and myname.endswith(domain):
-        myprincipals.add(myname.split('.')[0])
+    myshortname = myname.split('.')[0]
+    myprincipals.add(myshortname)
+    if domain:
+        myprincipals.add('{0}.{1}'.format(myshortname, domain))
+    for addr in netutil.get_my_addresses():
+        addr = socket.inet_ntop(addr[0], addr[1])
+        myprincipals.add(addr)
     for pubkey in glob.glob('/etc/ssh/ssh_host_*_key.pub'):
         currpubkey = open(pubkey, 'rb').read()
         cert = sshutil.sign_host_key(currpubkey, myname, myprincipals)
@@ -224,22 +229,22 @@ def initialize(cmdset):
         sys.exit(1)
     if cmdset.i:
         didsomething = True
-        sys.stdout.write('Add root user key to be authorized to log into nodes (-u)? (y/n): ')
+        sys.stdout.write('Add root user key to be authorized to log into nodes (-u)? (y/N): ')
         sys.stdout.flush()
         cmdset.u = input().strip().lower().startswith('y')
-        sys.stdout.write('Initialize a profile to boot Genesis on target systems (a small Linux environment for rescue and staging use)? (y/n): ')
+        sys.stdout.write('Initialize a profile to boot Genesis on target systems (a small Linux environment for rescue and staging use)? (y/N): ')
         cmdset.g = input().strip().lower().startswith('y')
-        sys.stdout.write('Set up an SSH authority to help manage known_hosts and node to node ssh for all users (-s)? (y/n): ')
+        sys.stdout.write('Set up an SSH authority to help manage known_hosts and node to node ssh for all users (-s)? (y/N): ')
         cmdset.s = input().strip().lower().startswith('y')
-        sys.stdout.write('Update global known hosts on this server to trust local CA certificates (-k)? (y/n): ')
+        sys.stdout.write('Update global known hosts on this server to trust local CA certificates (-k)? (y/N): ')
         cmdset.k = input().strip().lower().startswith('y')
-        sys.stdout.write('Allow managed nodes to ssh to this management node without a password (-l)? (y/n): ')
+        sys.stdout.write('Allow managed nodes to ssh to this management node without a password (-l)? (y/N): ')
         cmdset.l = input().strip().lower().startswith('y')
-        sys.stdout.write('Update tftp directory with binaries to support PXE (-p) (y/n): ')
+        sys.stdout.write('Update tftp directory with binaries to support PXE (-p) (y/N): ')
         cmdset.p = input().strip().lower().startswith('y')
-        sys.stdout.write('Initialize confluent ssh user key so confluent can execute remote automation (e.g. Ansible plays) (-a) (y/n): ')
+        sys.stdout.write('Initialize confluent ssh user key so confluent can execute remote automation (e.g. Ansible plays) (-a) (y/N): ')
         cmdset.a = input().strip().lower().startswith('y')
-        sys.stdout.write('Generate new TLS certificates for HTTP, replacing any existing certificate (-t)? (y/n): ')
+        sys.stdout.write('Generate new TLS certificates for HTTP, replacing any existing certificate (-t)? (y/N): ')
         cmdset.t = input().strip().lower().startswith('y')
         if not cmdset.t:
             print(
@@ -250,17 +255,13 @@ def initialize(cmdset):
                 'subject.')
     else:
         didsomething = False
-    if not os.path.exists('/etc/confluent'):
+    if not os.path.exists('/etc/confluent/cfg'):
         sys.stderr.write('Start confluent service prior to initializng OS deployment\n')
         sys.exit(1)
     if cmdset.t or cmdset.s or cmdset.a or cmdset.u:
         neededuid = os.stat('/etc/confluent').st_uid
-        try:
-            os.makedirs('/var/lib/confluent')
-            os.chown('/var/lib/confluent', neededuid, -1)
-        except OSError as e:
-            if e.errno != 17:
-                raise
+        util.mkdirp('/var/lib/confluent')
+        os.chown('/var/lib/confluent', neededuid, -1)
     if cmdset.u:
         didsomething = True
         if not glob.glob('/root/.ssh/*.pub'):

confluent_server/confluent/asynchttp.py

@@ -74,15 +74,19 @@ class AsyncTermRelation(object):
 
 class AsyncSession(object):
 
-    def __init__(self):
+    def __init__(self, wshandler=None):
         self.asyncid = _assign_asyncid(self)
         self.responses = collections.deque()
+        self.wshandler = wshandler
         self._evt = None
         self.termrelations = []
         self.consoles = set([])
-        self.reaper = eventlet.spawn_after(15, self.destroy)
+        if not wshandler:
+            self.reaper = eventlet.spawn_after(15, self.destroy)
 
     def add(self, requestid, rsp):
+        if self.wshandler:
+            self.wshandler(messages.AsyncMessage((requestid, rsp)))
         if self.responses is None:
             return
         self.responses.append((requestid, rsp))
@@ -108,7 +112,7 @@ class AsyncSession(object):
             self._evt = None
         for console in self.consoles:
             _consolesessions[console]['session'].destroy()
-        self.consoles = None
+        self.consoles = set([])
         self.responses = None
         del _asyncsessions[self.asyncid]
 
@@ -159,14 +163,17 @@ def get_async(env, querydict):
     return _asyncsessions[env['HTTP_CONFLUENTASYNCID']]['asyncsession']
 
 
-def handle_async(env, querydict, threadset):
+def handle_async(env, querydict, threadset, wshandler=None):
     global _cleanthread
     # This may be one of two things, a request for a new async stream
     # or a request for next data from async stream
     # httpapi otherwise handles requests an injecting them to queue
     if 'asyncid' not in querydict or not querydict['asyncid']:
         # This is a new request, create a new multiplexer
-        currsess = AsyncSession()
+        currsess = AsyncSession(wshandler)
+        if wshandler:
+            yield currsess
+            return
         yield messages.AsyncSession(currsess.asyncid)
         return
     if  querydict['asyncid'] not in _asyncsessions:
@@ -190,4 +197,4 @@ def handle_async(env, querydict, threadset):
 
 def set_console_sessions(consolesessions):
     global _consolesessions
-    _consolesessions = consolesessions
+    _consolesessions = consolesessions

confluent_server/confluent/auth.py

@@ -162,6 +162,10 @@ def authorize(name, element, tenant=False, operation='create',
         return False
     manager = configmanager.ConfigManager(tenant, username=user)
     userobj = manager.get_user(user)
+    if element and (element.startswith('/sessions/current/webauthn/registered_credentials/') or  element.startswith('/sessions/current/webauthn/validate/')):
+        return userobj, manager, user, tenant, skipuserobj
+    if userobj and userobj.get('role', None) == 'Stub':
+        userobj = None
     if not userobj:
         for group in userutil.grouplist(user):
             userobj = manager.get_usergroup(group)

confluent_server/confluent/config/attributes.py

@@ -534,6 +534,12 @@ node = {
                        'To support this scenario, the switch should be set up to allow independent operation of member ports123654 (e.g. lacp bypass mode or fallback mode).',
         'validvalues': ('lacp', 'loadbalance', 'roundrobin', 'activebackup', 'none')
     },
+    'power.pdu': {
+        'description': 'Specifies the managed PDU associated with a power input on the node'
+    },
+    'power.outlet': {
+        'description': 'Species the outlet identifier on the PDU associoted with a power input on the node'
+    },
 #    'id.modelnumber': {
 #        'description': 'The manufacturer dictated  model number for the node',
 #    },
@@ -549,6 +555,9 @@ node = {
 #    'secret.snmppassword': {
 #        'description': 'The password to use for SNMPv3 access to this node',
 #    },
+    'secret.selfapiarmtoken': {
+        'description': 'A one-time use shared secret to authenticate a node api token',
+    },
     'secret.snmpcommunity': {
         'description': ('SNMPv1 community string, it is highly recommended to'
                         'step up to SNMPv3'),

confluent_server/confluent/config/configmanager.py

@@ -113,7 +113,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
-_validroles = ('Administrator', 'Operator', 'Monitor')
+_validroles = ('Administrator', 'Operator', 'Monitor', 'Stub')
 
 membership_callback = None
 
@@ -485,7 +485,7 @@ def attribute_is_invalid(attrname, attrval):
 
 
 def _get_valid_attrname(attrname):
-    if attrname.startswith('net.'):
+    if attrname.startswith('net.') or attrname.startswith('power.'):
         # For net.* attribtues, split on the dots and put back together
         # longer term we might want a generic approach, but
         # right now it's just net. attributes
@@ -1011,8 +1011,17 @@ class _ExpressionFormat(string.Formatter):
         self._numbers = None
 
     def get_field(self, field_name, args, kwargs):
+        return field_name, field_name
+
+    def format_field(self, val, format_spec):
+        if ']' in format_spec:
+            field_name = val + ':' + format_spec
+            format_spec = ''
+        else:
+            field_name = val
         parsed = ast.parse(field_name)
-        return self._handle_ast_node(parsed.body[0].value), field_name
+        val = self._handle_ast_node(parsed.body[0].value)
+        return format(val, format_spec)
 
     def _handle_ast_node(self, node):
         if isinstance(node, ast.Num):
@@ -1061,6 +1070,34 @@ class _ExpressionFormat(string.Formatter):
             op = self._supported_ops[optype]
             return op(int(self._handle_ast_node(node.left)),
                       int(self._handle_ast_node(node.right)))
+        elif isinstance(node, ast.Index):
+            return self._handle_ast_node(node.value)
+        elif isinstance(node, ast.UnaryOp):
+            if isinstance(node.op, ast.USub):
+                return 0 - self._handle_ast_node(node.operand)
+            else:
+                raise ValueError("Invalid operation in expression")
+        elif isinstance(node, ast.Subscript):
+            strval = self._handle_ast_node(node.value)
+            if isinstance(node.slice, ast.Slice):
+                lower = node.slice.lower
+                if lower is None:
+                    lower = 0
+                else:
+                    lower = self._handle_ast_node(lower)
+                upper = node.slice.upper
+                if upper is None:
+                    return strval[lower:]
+                else:
+                    upper = self._handle_ast_node(upper)
+                    return strval[lower:upper]
+            else:
+                index = self._handle_ast_node(node.slice)
+                return strval[index]
+        elif isinstance(node, ast.Constant):
+            return node.value
+        else:
+            raise ValueError("Unrecognized expression syntax")
 
     def _expand_attribute(self, key):
         if '_expressionkeys' not in self._nodeobj:
@@ -2740,8 +2777,8 @@ def dump_db_to_directory(location, password, redact=None, skipkeys=False):
             cfgfile.write('\n')
     bkupglobals = get_globals()
     if bkupglobals:
-        json.dump(bkupglobals, open(os.path.join(location, 'globals.json'),
-                                    'w'))
+        with open(os.path.join(location, 'globals.json'), 'w') as globout:
+            json.dump(bkupglobals, globout)
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):

confluent_server/confluent/core.py

@@ -382,6 +382,11 @@ def _init_core():
             'pluginattrs': ['hardwaremanagement.method'],
             'default': 'ipmi',
         }),
+        'deployment': {
+            'ident_image': PluginRoute({
+                'handler': 'identimage'
+            })
+        },
         'events': {
             'hardware': {
                 'log': PluginRoute({
@@ -452,6 +457,8 @@ def _init_core():
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
+            'inlets': PluginCollection({'handler': 'pdu'}),
+            'outlets': PluginCollection({'pluginattrs': ['hardwaremanagement.method']}),
             'reseat':  PluginRoute({'handler': 'enclosure'}),
         },
         'sensors': {
@@ -666,6 +673,8 @@ def create_group(inputdata, configmanager):
 def create_node(inputdata, configmanager):
     try:
         nodename = inputdata['name']
+        if ' ' in nodename:
+            raise exc.InvalidArgumentException('Name "{0}" is not supported'.format(nodename))
         del inputdata['name']
         attribmap = {nodename: inputdata}
     except KeyError:

confluent_server/confluent/credserver.py

@@ -21,6 +21,8 @@ import datetime
 import eventlet
 import eventlet.green.socket as socket
 import eventlet.greenpool
+import hashlib
+import hmac
 import os
 import struct
 
@@ -31,6 +33,7 @@ import struct
 # 3, len, token - echo reply
 # 4, len, crypted - crypted apikey
 # 5, 0, accept key
+# 6, len, hmac - hmac of crypted key using shared secret for long-haul support
 # 128, len, len, key - sealed key
 
 class CredServer(object):
@@ -39,9 +42,9 @@ class CredServer(object):
 
     def handle_client(self, client, peer):
         try:
-            if not netutil.address_is_local(peer[0]):
-                client.close()
-                return
+            apiarmed = None
+            hmackey = None
+            hmacval = None
             client.send(b'\xc2\xd1-\xa8\x80\xd8j\xba')
             tlv = bytearray(client.recv(2))
             if tlv[0] != 1:
@@ -49,28 +52,39 @@ class CredServer(object):
                 return
             nodename = util.stringify(client.recv(tlv[1]))
             tlv = bytearray(client.recv(2))  # should always be null
-            apimats = self.cfm.get_node_attributes(nodename,
-                ['deployment.apiarmed', 'deployment.sealedapikey'])
-            apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get(
-                'value', None)
-            if not apiarmed:
-                if apimats.get(nodename, {}).get(
-                    'deployment.sealedapikey', {}).get('value', None):
-                    sealed = apimats[nodename]['deployment.sealedapikey'][
-                        'value']
-                    if not isinstance(sealed, bytes):
-                        sealed = sealed.encode('utf8')
-                    reply = b'\x80' + struct.pack('>H', len(sealed) + 1) + sealed + b'\x00'
-                    client.send(reply)
-                client.close()
-                return
-            if apiarmed not in ('once', 'continuous'):
-                now = datetime.datetime.utcnow()
-                expiry = datetime.datetime.strptime(apiarmed, "%Y-%m-%dT%H:%M:%SZ")
-                if now > expiry:
-                    self.cfm.set_node_attributes({nodename: {'deployment.apiarmed': ''}})
+            onlylocal = True
+            if tlv[0] == 6:
+                hmacval = client.recv(tlv[1])
+                hmackey = self.cfm.get_node_attributes(nodename, ['secret.selfapiarmtoken'], decrypt=True)
+                hmackey = hmackey.get(nodename, {}).get('secret.selfapiarmtoken', {}).get('value', None)
+            elif tlv[1]:
+                client.recv(tlv[1])
+            if not hmackey:
+                if not netutil.address_is_local(peer[0]):
                     client.close()
                     return
+                apimats = self.cfm.get_node_attributes(nodename,
+                    ['deployment.apiarmed', 'deployment.sealedapikey'])
+                apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get(
+                    'value', None)
+                if not apiarmed:
+                    if apimats.get(nodename, {}).get(
+                        'deployment.sealedapikey', {}).get('value', None):
+                        sealed = apimats[nodename]['deployment.sealedapikey'][
+                            'value']
+                        if not isinstance(sealed, bytes):
+                            sealed = sealed.encode('utf8')
+                        reply = b'\x80' + struct.pack('>H', len(sealed) + 1) + sealed + b'\x00'
+                        client.send(reply)
+                    client.close()
+                    return
+                if apiarmed not in ('once', 'continuous'):
+                    now = datetime.datetime.utcnow()
+                    expiry = datetime.datetime.strptime(apiarmed, "%Y-%m-%dT%H:%M:%SZ")
+                    if now > expiry:
+                        self.cfm.set_node_attributes({nodename: {'deployment.apiarmed': ''}})
+                        client.close()
+                        return
             client.send(b'\x02\x20')
             rttoken = os.urandom(32)
             client.send(rttoken)
@@ -88,7 +102,14 @@ class CredServer(object):
                 client.close()
                 return
             echotoken = util.stringify(client.recv(tlv[1]))
+            if hmackey:
+                etok = echotoken.encode('utf8')
+                if hmacval != hmac.new(hmackey, etok, hashlib.sha256).digest():
+                    client.close()
+                    return
             cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': echotoken}, 'deployment.sealedapikey': '', 'deployment.apiarmed': ''}}
+            if hmackey:
+                self.cfm.clear_node_attributes([nodename], ['secret.selfapiarmtoken'])
             if apiarmed == 'continuous':
                 del cfgupdate[nodename]['deployment.apiarmed']
             self.cfm.set_node_attributes(cfgupdate)

confluent_server/confluent/discovery/core.py

@@ -947,7 +947,9 @@ def get_node_by_uuid_or_mac(uuidormac):
 
 def get_nodename_from_enclosures(cfg, info):
     nodename = None
-    cuuid = info.get('attributes', {}).get('chassis-uuid', [None])[0]
+    cuuid = info.get('enclosure.uuid', None)
+    if not cuuid:
+        cuuid = info.get('attributes', {}).get('chassis-uuid', [None])[0]
     if cuuid and cuuid in nodes_by_uuid:
         encl = nodes_by_uuid[cuuid]
         bay = info.get('enclosure.bay', None)

confluent_server/confluent/discovery/handlers/smm.py

@@ -153,7 +153,28 @@ class NodeHandler(bmchandler.NodeHandler):
             rsp = wc.getresponse()
             rspdata = util.stringify(rsp.read())
             if 'renew_account' in rspdata:
-                raise Exception('Configured password has expired')
+                tmppassword = 'Tmp42' + password[5:]
+                tokens = fromstring(rspdata)
+                st2 = tokens.findall('st2')[0].text
+                wc.set_header('ST2', st2)
+                wc.request('POST', '/data/changepwd', 'oripwd={0}&newpwd={1}'.format(password, tmppassword))
+                rsp = wc.getresponse()
+                rspdata = rsp.read().decode('utf8')
+                bdata = 'user={0}&password={1}'.format(username, tmppassword)
+                wc.request('POST', '/data/login', bdata, headers)
+                rsp = wc.getresponse()
+                rspdata = rsp.read().decode('utf8')
+                tokens = fromstring(rspdata)
+                st2 = tokens.findall('st2')[0].text
+                wc.set_header('ST2', st2)
+                rules = 'set=passwordChangeInterval:0,passwordReuseCheckNum:0'
+                wc.request('POST', '/data', rules)
+                wc.getresponse().read()
+                wc.request('POST', '/data/changepwd', 'oripwd={0}&newpwd={1}'.format(tmppassword, password))
+                wc.getresponse().read()
+                wc.request('POST', '/data/login', urlencode(authdata), headers)
+                rsp = wc.getresponse()
+                rspdata = util.stringify(rsp.read())
             if 'authResult>0' not in rspdata:
                 raise Exception('Unknown username/password on SMM')
             tokens = fromstring(rspdata)

confluent_server/confluent/discovery/handlers/xcc.py

@@ -30,6 +30,15 @@ import struct
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 
+def fixuuid(baduuid):
+    # SMM dumps it out in hex
+    uuidprefix = (baduuid[:8], baduuid[9:13], baduuid[14:18])
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]),
+        'hex')
+    a = util.stringify(a)
+    uuid = (a[:8], a[8:12], a[12:16], baduuid[19:23], baduuid[24:])
+    return '-'.join(uuid).lower()
+
 class LockedUserException(Exception):
     pass
 
@@ -94,7 +103,7 @@ class NodeHandler(immhandler.NodeHandler):
         if attrs.get('enclosure-form-factor', None) == 'dense-computing':
             encuuid = attrs.get('chassis-uuid', None)
             if encuuid:
-                self.info['enclosure.uuid'] = encuuid
+                self.info['enclosure.uuid'] = fixuuid(encuuid)
             slot = int(attrs.get('slot', 0))
             if slot != 0:
                 self.info['enclosure.bay'] = slot
@@ -228,6 +237,20 @@ class NodeHandler(immhandler.NodeHandler):
             if '_csrf_token' in wc.cookies:
                 wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
             if rspdata.get('pwchg_required', None) == 'true':
+                if newpassword is None:
+                    # a normal login hit expired condition
+                    tmppassword = 'Tmp42' + password[5:]
+                    wc.request('POST', '/api/function', json.dumps(
+                        {'USER_UserPassChange': '1,{0}'.format(tmppassword)}))
+                    rsp = wc.getresponse()
+                    rsp.read()
+                    # We must step down change interval and reusecycle to restore password
+                    wc.grab_json_response('/api/dataset', {'USER_GlobalMinPassChgInt': '0', 'USER_GlobalMinPassReuseCycle': '0'})
+                    wc.request('POST', '/api/function', json.dumps(
+                        {'USER_UserPassChange': '1,{0}'.format(password)}))
+                    rsp = wc.getresponse()
+                    rsp.read()
+                    return (wc, {})
                 wc.request('POST', '/api/function', json.dumps(
                     {'USER_UserPassChange': '1,{0}'.format(newpassword)}))
                 rsp = wc.getresponse()

confluent_server/confluent/discovery/protocols/pxe.py

@@ -393,7 +393,7 @@ def snoop(handler, protocol=None, nodeguess=None):
                     if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
                         idx, recv = struct.unpack('II', cmsgarr[16:24])
                         recv = ipfromint(recv)
-                    rqv = memoryview(rawbuffer)
+                    rqv = memoryview(rawbuffer)[:i]
                     if rawbuffer[0] == 1:  # Boot request
                         process_dhcp4req(handler, nodeguess, cfg, net4, idx, recv, rqv)
                 elif netc == net6:

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -108,12 +108,15 @@ def _process_snoop(peer, rsp, mac, known_peers, newmacs, peerbymacaddress, byeha
                 if not value.endswith('/DeviceDescription.json'):
                     return
         if handler:
-            retdata = check_fish(('/DeviceDescription.json', peerdata))
-            if retdata:
-                known_peers.add(peer)
-                newmacs.add(mac)
-                peerbymacaddress[mac] = retdata
-                machandlers[mac] = handler
+            eventlet.spawn_n(check_fish_handler, handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer)
+
+def check_fish_handler(handler, peerdata, known_peers, newmacs, peerbymacaddress, machandlers, mac, peer):
+    retdata = check_fish(('/DeviceDescription.json', peerdata))
+    if retdata:
+        known_peers.add(peer)
+        newmacs.add(mac)
+        peerbymacaddress[mac] = retdata
+        machandlers[mac] = handler
 
 
 def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):

confluent_server/confluent/httpapi.py

@@ -21,6 +21,10 @@ try:
     import Cookie
 except ModuleNotFoundError:
     import http.cookies as Cookie
+try:
+    import confluent.webauthn as webauthn
+except ImportError:
+    webauthn = None
 import confluent.auth as auth
 import confluent.config.attributes as attribs
 import confluent.consoleserver as consoleserver
@@ -207,6 +211,8 @@ def _should_skip_authlog(env):
     if '/sessions/current/async' in env['PATH_INFO']:
         # this is effectively invisible
         return True
+    if '/sessions/current/webauthn/registered_credentials' in env['PATH_INFO']:
+        return True
     if (env['REQUEST_METHOD'] == 'GET' and
             ('/sensors/' in env['PATH_INFO'] or
              '/health/' in env['PATH_INFO'] or
@@ -263,18 +269,24 @@ def _csrf_valid(env, session):
             env['HTTP_CONFLUENTAUTHTOKEN'] == session['csrftoken'])
 
 
-def _authorize_request(env, operation):
+def _authorize_request(env, operation, reqbody):
     """Grant/Deny access based on data from wsgi env
 
     """
     authdata = None
     name = ''
     sessionid = None
+    sessid = None
     cookie = Cookie.SimpleCookie()
     element = env['PATH_INFO']
     if element.startswith('/sessions/current/'):
-        element = None
-    if 'HTTP_COOKIE' in env:
+        if (element.startswith('/sessions/current/webauthn/registered_credentials/')
+                or element.startswith('/sessions/current/webauthn/validate/')):
+            name = element.rsplit('/')[-1]
+            authdata = auth.authorize(name, element=element, operation=operation)
+        else:
+            element = None
+    if (not authdata) and 'HTTP_COOKIE' in env:
         cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
         if cidx >= 0:
             sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
@@ -322,18 +334,13 @@ def _authorize_request(env, operation):
             return {'code': 403}
         elif not authdata:
             return {'code': 401}
-        sessid = util.randomstring(32)
-        while sessid in httpsessions:
-            sessid = util.randomstring(32)
-        httpsessions[sessid] = {'name': name, 'expiry': time.time() + 90,
-                                'skipuserobject': authdata[4],
-                                'inflight': set([])}
-        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
-            httpsessions[sessid]['csrftoken'] = util.randomstring(32)
-        cookie['confluentsessionid'] = util.stringify(sessid)
-        cookie['confluentsessionid']['secure'] = 1
-        cookie['confluentsessionid']['httponly'] = 1
-        cookie['confluentsessionid']['path'] = '/'
+        sessid = _establish_http_session(env, authdata, name, cookie)
+    if authdata and element and element.startswith('/sessions/current/webauthn/validate/'):
+        if webauthn:
+            for rsp in webauthn.handle_api_request(element, env, None, authdata[2], authdata[1], None, reqbody, None):
+                if rsp['verified']:
+                    sessid = _establish_http_session(env, authdata, name, cookie)
+                    break
     skiplog = _should_skip_authlog(env)
     if authdata:
         auditmsg = {
@@ -352,17 +359,32 @@ def _authorize_request(env, operation):
         auditmsg['user'] = util.stringify(authdata[2])
         if sessid is not None:
             authinfo['sessionid'] = sessid
+            if 'csrftoken' in httpsessions[sessid]:
+                authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
+            httpsessions[sessid]['cfgmgr'] = authdata[1]
         if not skiplog:
             auditlog.log(auditmsg)
-        if 'csrftoken' in httpsessions[sessid]:
-            authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
-        httpsessions[sessid]['cfgmgr'] = authdata[1]
         return authinfo
     elif authdata is None:
         return {'code': 401}
     else:
         return {'code': 403}
 
+def _establish_http_session(env, authdata, name, cookie):
+    sessid = util.randomstring(32)
+    while sessid in httpsessions:
+        sessid = util.randomstring(32)
+    httpsessions[sessid] = {'name': name, 'expiry': time.time() + 90,
+                                'skipuserobject': authdata[4],
+                                'inflight': set([])}
+    if 'HTTP_CONFLUENTAUTHTOKEN' in env:
+        httpsessions[sessid]['csrftoken'] = util.randomstring(32)
+    cookie['confluentsessionid'] = util.stringify(sessid)
+    cookie['confluentsessionid']['secure'] = 1
+    cookie['confluentsessionid']['httponly'] = 1
+    cookie['confluentsessionid']['path'] = '/'
+    return sessid
+
 
 def _pick_mimetype(env):
     """Detect the http indicated mime to send back.
@@ -391,10 +413,31 @@ def _assign_consessionid(consolesession):
                                'expiry': time.time() + 60}
     return sessid
 
+
+def websockify_data(data):
+    if isinstance(data, dict):
+        data = json.dumps(data)
+        data = u'!' + data
+    else:
+        try:
+            data = data.decode('utf8')
+        except UnicodeDecodeError:
+            data = data.decode('cp437')
+        data = u' ' + data
+    return data
+
+def datacallback_bound(clientsessid, ws):
+    def datacallback(data):
+        data = websockify_data(data)
+        ws.send('${0}$'.format(clientsessid) + data)
+    return datacallback
+
 @eventlet.websocket.WebSocketWSGI.configured(
-    supported_protocols=['confluent.console'])
+    supported_protocols=['confluent.console', 'confluent.asyncweb'])
 def wsock_handler(ws):
     sessid = ws.wait()
+    if not sessid:
+        return
     sessid = sessid.replace('ConfluentSessionId:', '')
     sessid = sessid[:-1]
     currsess = httpsessions.get(sessid, None)
@@ -411,7 +454,89 @@ def wsock_handler(ws):
     authdata = auth.authorize(name, ws.path)
     if not authdata:
         return
+    cfgmgr = httpsessions[sessid]['cfgmgr']
+    username = httpsessions[sessid]['name']
+    if ws.path == '/sessions/current/async':
+        myconsoles = {}
+        def asyncwscallback(rsp):
+            rsp = json.dumps(rsp.raw())
+            ws.send(u'!' + rsp)
+        mythreadid = greenlet.getcurrent()
+        currsess['inflight'].add(mythreadid)
+        asess = None
+        try:
+            for asess in confluent.asynchttp.handle_async(
+                    {}, {}, currsess['inflight'], asyncwscallback):
+                ws.send(u' ASYNCID: {0}'.format(asess.asyncid))
+                clientmsg = True
+                while clientmsg:
+                    clientmsg = ws.wait()
+                    if clientmsg:
+                        if clientmsg[0] == '?':
+                            ws.send('?')
+                        elif clientmsg[0] == '$':
+                            targid, data = clientmsg[1:].split('$', 1)
+                            if data[0] == ' ':
+                                myconsoles[targid].write(data[1:])
+                        elif clientmsg[0] == '!':
+                            msg = json.loads(clientmsg[1:])
+                            action = msg.get('operation', None)
+                            if action == 'start':
+                                targ = msg['target']
+                                if '/console/session' in targ or '/shell/sessions' in targ:
+                                    width = msg['width']
+                                    height = msg['height']
+                                    clientsessid = '{0}'.format(msg['sessid'])
+                                    skipreplay = msg.get('skipreplay', False)
+                                    delimit = None
+                                    if '/console/session' in targ:
+                                        delimit = '/console/session'
+                                        shellsession = False
+                                    else:
+                                        delimit = '/shell/sessions'
+                                        shellsession = True
+                                    node = targ.split(delimit, 1)[0]
+                                    node = node.rsplit('/', 1)[-1]
+                                    auditmsg = {'operation': 'start', 'target': targ,
+                                                'user': util.stringify(username)}
+                                    auditlog.log(auditmsg)
+                                    datacallback = datacallback_bound(clientsessid, ws)
+                                    if shellsession:
+                                        consession = shellserver.ShellSession(
+                                            node=node, configmanager=cfgmgr,
+                                            username=username, skipreplay=skipreplay,
+                                            datacallback=datacallback,
+                                            width=width, height=height)
+                                    else:
+                                        consession = consoleserver.ConsoleSession(
+                                            node=node, configmanager=cfgmgr,
+                                            username=username, skipreplay=skipreplay,
+                                            datacallback=datacallback,
+                                            width=width, height=height)
+                                    myconsoles[clientsessid] = consession
+                            elif action == 'stop':
+                                sessid = '{0}'.format(msg.get('sessid', None))
+                                if sessid in myconsoles:
+                                    myconsoles[sessid].destroy()
+                                    del myconsoles[sessid]
+                        else:
+                            print(repr(clientmsg))
+        finally:
+            for cons in myconsoles:
+                myconsoles[cons].destroy()
+            if asess:
+                asess.destroy()
+            currsess['inflight'].discard(mythreadid)
+        return
     if '/console/session' in ws.path or '/shell/sessions/' in ws.path:
+        def datacallback(data):
+            ws.send(websockify_data(data))
+        geom = ws.wait()
+        geom = geom[1:]
+        geom = json.loads(geom)
+        width = geom['width']
+        height = geom['height']
+        skipreplay = geom.get('skipreplay', False)
         #hard bake JSON into this path, do not support other incarnations
         if '/console/session' in ws.path:
             prefix, _, _ = ws.path.partition('/console/session')
@@ -420,24 +545,7 @@ def wsock_handler(ws):
             prefix, _, _ = ws.path.partition('/shell/sessions')
             shellsession = True
         _, _, nodename = prefix.rpartition('/')
-        geom = ws.wait()
-        geom = geom[1:]
-        geom = json.loads(geom)
-        width = geom['width']
-        height = geom['height']
-        skipreplay = geom.get('skipreplay', False)
-        cfgmgr = httpsessions[sessid]['cfgmgr']
-        username = httpsessions[sessid]['name']
-        def datacallback(data):
-            if isinstance(data, dict):
-                data = json.dumps(data)
-                ws.send(u'!' + data)
-            else:
-                try:
-                    data = data.decode('utf8')
-                except UnicodeDecodeError:
-                    data = data.decode('cp437')
-                ws.send(u' ' + data)
+
         try:
             if shellsession:
                 consession = shellserver.ShellSession(
@@ -453,6 +561,8 @@ def wsock_handler(ws):
                 )
         except exc.NotFoundException:
             return
+        mythreadid = greenlet.getcurrent()
+        currsess['inflight'].add(mythreadid)
         clientmsg = ws.wait()
         try:
             while clientmsg is not None:
@@ -470,6 +580,7 @@ def wsock_handler(ws):
                     ws.send(u'?')
                 clientmsg = ws.wait()
         finally:
+            currsess['inflight'].discard(mythreadid)
             consession.destroy()
 
 
@@ -514,7 +625,7 @@ def resourcehandler_backend(env, start_response):
     if operation != 'retrieve' and 'restexplorerop' in querydict:
         operation = querydict['restexplorerop']
         del querydict['restexplorerop']
-    authorized = _authorize_request(env, operation)
+    authorized = _authorize_request(env, operation, reqbody)
     if 'logout' in authorized:
         start_response('200 Successful logout', headers)
         yield('{"result": "200 - Successful logout"}')
@@ -543,7 +654,7 @@ def resourcehandler_backend(env, start_response):
         raise Exception("Unrecognized code from auth engine")
     headers.extend(
         ("Set-Cookie", m.OutputString())
-        for m in authorized['cookie'].values())
+        for m in authorized.get('cookie', {}).values())
     cfgmgr = authorized['cfgmgr']
     if (operation == 'create') and env['PATH_INFO'] == '/sessions/current/async':
         pagecontent = ""
@@ -741,6 +852,14 @@ def resourcehandler_backend(env, start_response):
             tlvdata.unicode_dictvalues(sessinfo)
             yield json.dumps(sessinfo)
             return
+        elif url.startswith('/sessions/current/webauthn/'):
+            if not webauthn:
+                start_response('501 Not Implemented', headers)
+                yield ''
+                return
+            for rsp in webauthn.handle_api_request(url, env, start_response, authorized['username'], cfgmgr, headers, reqbody, authorized):
+                yield rsp
+            return
         resource = '.' + url[url.rindex('/'):]
         lquerydict = copy.deepcopy(querydict)
         try:

confluent_server/confluent/messages.py

@@ -515,6 +515,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputVolumes(path, nodes, inputdata)
     elif 'inventory/firmware/updates/active' in '/'.join(path) and inputdata:
         return InputFirmwareUpdate(path, nodes, inputdata, configmanager)
+    elif ('/'.join(path).startswith('power/inlets/') or '/'.join(path).startswith('power/outlets/')) and inputdata:
+        return InputPowerMessage(path, nodes, inputdata)
     elif '/'.join(path).startswith('media/detach'):
         return DetachMedia(path, nodes, inputdata)
     elif '/'.join(path).startswith('media/') and inputdata:
@@ -526,6 +528,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
     elif '/'.join(path).startswith(
             'configuration/management_controller/licenses') and inputdata:
         return InputLicense(path, nodes, inputdata, configmanager)
+    elif path == ['deployment', 'ident_image']:
+        return InputIdentImage(path, nodes, inputdata)
     elif inputdata:
         raise exc.InvalidArgumentException(
             'No known input handler for request')
@@ -887,6 +891,12 @@ class ConfluentInputMessage(ConfluentMessage):
     def is_valid_key(self, key):
         return key in self.valid_values
 
+
+class InputIdentImage(ConfluentInputMessage):
+    keyname = 'ident_image'
+    valid_values = ['create']
+
+
 class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([
         'on',

confluent_server/confluent/netutil.py

@@ -274,6 +274,19 @@ class NetManager(object):
             myattribs['current_nic'] = False
 
 
+def get_flat_net_config(configmanager, node):
+    fnc = get_full_net_config(configmanager, node)
+    dft = fnc.get('default', {})
+    if dft:
+        ret = [dft]
+    else:
+        ret = []
+    for nc in fnc.get('extranets', {}):
+        nc = fnc['extranets'][nc]
+        if nc:
+            ret.append(nc)
+    return ret
+
 def get_full_net_config(configmanager, node, serverip=None):
     cfd = configmanager.get_node_attributes(node, ['net.*'])
     cfd = cfd.get(node, {})
@@ -294,7 +307,7 @@ def get_full_net_config(configmanager, node, serverip=None):
             if iface not in attribs:
                 attribs[iface] = {}
             attribs[iface][attrib] = val
-    myaddrs = None
+    myaddrs = []
     if serverip:
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
@@ -311,6 +324,11 @@ def get_full_net_config(configmanager, node, serverip=None):
     return retattrs
 
 
+def noneify(cfgdata):
+    for key in cfgdata:
+        if cfgdata[key] == '':
+            cfgdata[key] = None
+    return cfgdata
 
 # TODO(jjohnson2): have a method to arbitrate setting methods, to aid
 # in correct matching of net.* based on parameters, mainly for pxe
@@ -450,15 +468,15 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
                     if ipn_on_same_subnet(fam, svrip, candgwn, prefix):
                         candgws.append((fam, candgwn, prefix))
         if foundaddr:
-            return cfgdata
+            return noneify(cfgdata)
         if dhcprequested:
             if not cfgdata.get('ipv4_method', None):
                 cfgdata['ipv4_method'] = 'dhcp'
             if not cfgdata.get('ipv6_method', None):
                 cfgdata['ipv6_method'] = 'dhcp'
-            return cfgdata
+            return noneify(cfgdata)
         if ipbynodename == None and ip6bynodename == None:
-            return cfgdata
+            return noneify(cfgdata)
         for myaddr in myaddrs:
             fam, svrip, prefix = myaddr[:3]
             if fam == socket.AF_INET:
@@ -505,7 +523,7 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
                 bynodenamn = socket.inet_pton(fam, bynodename)
                 if ipn_on_same_subnet(fam, candgwn, bynodenamn, prefix):
                     cfgdata['ipv{}_gateway'.format(nver)] = socket.inet_ntop(fam, candgwn)
-        return cfgdata
+        return noneify(cfgdata)
     if ip is not None:
         for prefixinfo in get_prefix_len_for_ip(ip):
             fam, prefix = prefixinfo
@@ -526,7 +544,7 @@ def get_nic_config(configmanager, node, ip=None, mac=None, ifidx=None,
                 if ipn_on_same_subnet(fam, ipn, gwn, prefix):
                     cfgdata['ipv{}_gateway'.format(nver)] = gw
                     break
-    return cfgdata
+    return noneify(cfgdata)
 
 def get_addresses_by_serverip(serverip):
     if '.' in serverip:

confluent_server/confluent/osimage.py

@@ -197,7 +197,7 @@ def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist
         for entry in entries:
             if str(entry).endswith('TRANS.TBL'):
                 continue
-            if extractlist and str(entry) not in extractlist:
+            if extractlist and str(entry).lower() not in extractlist:
                 continue
             write_header(write_p, entry._entry_p)
             read_p = entry._archive_p
@@ -250,6 +250,11 @@ def check_rocky(isoinfo):
             arch = entry.split('.')[-2]
             cat = 'el8'
             break
+        if 'rocky-release-9' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            cat = 'el9'
+            break
     else:
         return None
     if arch == 'noarch' and '.discinfo' in isoinfo[1]:
@@ -270,6 +275,11 @@ def check_alma(isoinfo):
             arch = entry.split('.')[-2]
             cat = 'el8'
             break
+        elif 'almalinux-release-9' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            cat = 'el9'
+            break
     else:
         return None
     if arch == 'noarch' and '.discinfo' in isoinfo[1]:
@@ -369,7 +379,7 @@ def check_ubuntu(isoinfo):
         return {'name': 'ubuntu-{0}-{1}'.format(ver, arch),
                 'method': EXTRACT|COPY,
                 'extractlist': ['casper/vmlinuz', 'casper/initrd',
-                'EFI/BOOT/BOOTx64.EFI', 'EFI/BOOT/grubx64.efi'
+                'efi/boot/bootx64.efi', 'efi/boot/grubx64.efi'
                 ],
                 'copyto': 'install.iso',
                 'category': 'ubuntu{0}'.format(major)}

confluent_server/confluent/plugins/deployment/identimage.py

@@ -0,0 +1,76 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2022 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+import confluent.messages as msg
+import confluent.netutil as netutil
+import eventlet.green.subprocess as subprocess
+import os
+import shutil
+import tempfile
+import socket
+import yaml
+import json
+
+def mkdirp(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+
+def create_apikey():
+    alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./'
+    newpass = ''.join([alpha[x >> 2] for x in bytearray(os.urandom(32))])
+    return newpass
+
+
+def create_ident_image(node, configmanager):
+    tmpd = tempfile.mkdtemp()
+    ident = { 'nodename': node }
+    apikey = create_apikey()
+    configmanager.set_node_attributes({node: {'secret.selfapiarmtoken': apikey}})
+    ident['apitoken'] = apikey
+    # This particular mechanism does not (yet) do anything smart with collective
+    # It would be a reasonable enhancement to list all collective server addresses
+    # restricted by 'managercandidates'
+    ident['deploy_servers'] = []
+    for myaddr in netutil.get_my_addresses():
+        myaddr = socket.inet_ntop(myaddr[0], myaddr[1])
+        ident['deploy_servers'].append(myaddr)
+    ident['net_cfgs'] = netutil.get_flat_net_config(configmanager, node)
+    with open(os.path.join(tmpd, 'cnflnt.yml'), 'w') as yamlout:
+        yaml.safe_dump(ident, yamlout, default_flow_style=False)
+    with open(os.path.join(tmpd, 'cnflnt.jsn'), 'w') as jsonout:
+        json.dump(ident, jsonout)
+    mkdirp('/var/lib/confluent/private/identity_images/')
+    imgname = '/var/lib/confluent/private/identity_images/{0}.img'.format(node)
+    if os.path.exists(imgname):
+        os.remove(imgname)
+    subprocess.check_call(['/opt/confluent/bin/dir2img', tmpd, imgname, 'cnflnt_idnt'])
+    shutil.rmtree(tmpd)
+
+
+def update(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        create_ident_image(node, configmanager)
+        yield msg.CreatedResource(
+                'nodes/{0}/deployment/ident_image'.format(node))
+
+

confluent_server/confluent/plugins/hardwaremanagement/geist.py

@@ -0,0 +1,123 @@
+# Copyright 2022 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pyghmi.util.webclient as wc
+import confluent.util as util
+import confluent.messages as msg
+import confluent.exceptions as exc
+
+
+class GeistClient(object):
+    def __init__(self, pdu, configmanager):
+        self.node = pdu
+        self.configmanager = configmanager
+        self._token = None
+        self._wc = None
+        self.username = None
+
+    @property
+    def token(self):
+        if not self._token:
+            self._token = self.login(self.configmanager)
+        return self._token
+
+    @property
+    def wc(self):
+        if self._wc:
+            return self._wc
+        targcfg = self.configmanager.get_node_attributes(self.node,
+                                            ['hardwaremanagement.manager'],
+                                            decrypt=True)
+        targcfg = targcfg.get(self.node, {})
+        target = targcfg.get(
+            'hardwaremanagement.manager', {}).get('value', None)
+        if not target:
+            target = self.node
+        cv = util.TLSCertVerifier(
+            self.configmanager, self.node,
+            'pubkeys.tls_hardwaremanager').verify_cert
+        self._wc = wc.SecureHTTPConnection(target, verifycallback=cv)
+        return self._wc
+
+    def login(self, configmanager):
+        credcfg = configmanager.get_node_attributes(self.node,
+                                            ['secret.hardwaremanagementuser',
+                                             'secret.hardwaremanagementpassword'],
+                                            decrypt=True)
+        credcfg = credcfg.get(self.node, {})
+        username = credcfg.get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        passwd = credcfg.get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
+        if not isinstance(username, str):
+            username = username.decode('utf8')
+        if not isinstance(passwd, str):
+            passwd = passwd.decode('utf8')
+        if not username or not passwd:
+            raise Exception('Missing username or password')
+        self.username = username
+        rsp = self.wc.grab_json_response(
+            '/api/auth/{0}'.format(username),
+            {'cmd': 'login', 'data': {'password': passwd}})
+        token = rsp['data']['token']
+        return token
+
+    def logout(self):
+        if self._token:
+            self.wc.grab_json_response('/api/auth/{0}'.format(self.username),
+                                       {'cmd': 'logout', 'token': self.token})
+            self._token = None
+
+    def get_outlet(self, outlet):
+        rsp = self.wc.grab_json_response('/api/dev')
+        rsp = rsp['data']
+        if len(rsp) != 1:
+            raise Exception('Multiple PDUs not supported per pdu')
+        pduname = list(rsp)[0]
+        outlet = rsp[pduname]['outlet'][str(int(outlet) - 1)]
+        state = outlet['state'].split('2')[-1]
+        return state
+
+    def set_outlet(self, outlet, state):
+        rsp = self.wc.grab_json_response('/api/dev')
+        if len(rsp['data']) != 1:
+            self.logout()
+            raise Exception('Multiple PDUs per endpoint not supported')
+        pdu = list(rsp['data'])[0]
+        outlet = int(outlet) - 1
+        rsp = self.wc.grab_json_response(
+            '/api/dev/{0}/outlet/{1}'.format(pdu, outlet),
+            {'cmd': 'control', 'token': self.token,
+            'data': {'action': state, 'delay': False}})
+
+
+def retrieve(nodes, element, configmanager, inputdata):
+    if 'outlets' not in element:
+        for node in nodes:
+            yield  msg.ConfluentResourceUnavailable(node, 'Not implemented')
+        return
+    for node in nodes:
+        gc = GeistClient(node, configmanager)
+        state = gc.get_outlet(element[-1])
+        yield msg.PowerState(node=node, state=state)
+
+def update(nodes, element, configmanager, inputdata):
+    if 'outlets' not in element:
+        yield msg.ConfluentResourceUnavailable(node, 'Not implemented')
+        return
+    for node in nodes:
+        gc = GeistClient(node, configmanager)
+        newstate = inputdata.powerstate(node)
+        gc.set_outlet(element[-1], newstate)
+    return retrieve(nodes, element, configmanager, inputdata)

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -110,7 +110,7 @@ def exithandler():
 
 atexit.register(exithandler)
 
-_ipmiworkers = greenpool.GreenPool(128)
+_ipmiworkers = greenpool.GreenPool(512)
 
 _ipmithread = None
 _ipmiwaiters = []
@@ -482,7 +482,7 @@ class IpmiHandler(object):
         self.tenant = cfg.tenant
         tenant = cfg.tenant
         while ((node, tenant) not in persistent_ipmicmds or
-                not persistent_ipmicmds[(node, tenant)].ipmi_session.logged or
+                not (persistent_ipmicmds[(node, tenant)].ipmi_session.logged or persistent_ipmicmds[(node, tenant)].ipmi_session.logging) or
                 persistent_ipmicmds[(node, tenant)].ipmi_session.broken):
             try:
                 persistent_ipmicmds[(node, tenant)].close_confluent()
@@ -514,6 +514,8 @@ class IpmiHandler(object):
                     raise exc.TargetEndpointUnreachable(ge.strerror)
                 raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
+        while not self.ipmicmd.ipmi_session.broken and not self.ipmicmd.ipmi_session.logged:
+            self.ipmicmd.ipmi_session.wait_for_rsp(3)
 
     bootdevices = {
         'optical': 'cd'

confluent_server/confluent/plugins/hardwaremanagement/pdu.py

@@ -0,0 +1,79 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import confluent.core as core
+import confluent.messages as msg
+import pyghmi.exceptions as pygexc
+import confluent.exceptions as exc
+
+def retrieve(nodes, element, configmanager, inputdata):
+    emebs = configmanager.get_node_attributes(
+        nodes, (u'power.*pdu', u'power.*outlet'))
+    if element == ['power', 'inlets']:
+        outletnames = set([])
+        for node in nodes:
+            for attrib in emebs[node]:
+                attrib = attrib.replace('power.', '').rsplit('.', 1)
+                if len(attrib) > 1:
+                    outletnames.add('inlet_' + attrib[0])
+                else:
+                    outletnames.add('default')
+        if outletnames:
+            outletnames.add('all')
+        for inlet in outletnames:
+            yield msg.ChildCollection(inlet)
+    elif len(element) == 3:
+        inletname = element[-1]
+        outlets = get_outlets(nodes, emebs, inletname)
+        for node in outlets:
+            for pgroup in outlets[node]:
+                pdu = outlets[node][pgroup]['pdu']
+                outlet = outlets[node][pgroup]['outlet']
+                for rsp in core.handle_path(
+                        '/nodes/{0}/power/outlets/{1}'.format(pdu, outlet),
+                        'retrieve', configmanager):
+                    yield msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node)
+
+def get_outlets(nodes, emebs, inletname):
+    outlets = {}
+    for node in nodes:
+        if node not in outlets:
+            outlets[node] = {}
+        for attrib in emebs[node]:
+            v = emebs[node][attrib].get('value', None)
+            if not v:
+                continue
+            attrib = attrib.replace('power.', '').rsplit('.', 1)
+            if len(attrib) > 1:
+                pgroup = 'inlet_' + attrib[0]
+            else:
+                pgroup = 'default'
+            if inletname == 'all' or pgroup == inletname:
+                if pgroup not in outlets[node]:
+                    outlets[node][pgroup] = {}
+                outlets[node][pgroup][attrib[-1]] = v
+    return outlets
+
+
+def update(nodes, element, configmanager, inputdata):
+    emebs = configmanager.get_node_attributes(
+        nodes, (u'power.*pdu', u'power.*outlet'))
+    inletname = element[-1]
+    outlets = get_outlets(nodes, emebs, inletname)
+    for node in outlets:
+        for pgroup in outlets[node]:
+            pdu = outlets[node][pgroup]['pdu']
+            outlet = outlets[node][pgroup]['outlet']
+            for rsp in core.handle_path('/nodes/{0}/power/outlets/{1}'.format(pdu, outlet),
+                                        'update', configmanager, inputdata={'state': inputdata.powerstate(node)}):
+                yield msg.KeyValueData({pgroup: rsp.kvpairs['state']['value']}, node)

confluent_server/confluent/selfservice.py

@@ -10,6 +10,9 @@ import eventlet.green.socket as socket
 import eventlet.green.subprocess as subprocess
 import confluent.discovery.handlers.xcc as xcc
 import confluent.discovery.handlers.tsm as tsm
+import base64
+import hmac
+import hashlib
 import crypt
 import json
 import os
@@ -60,13 +63,56 @@ def handle_request(env, start_response):
     global currlocale
     global currtzvintage
     configmanager.check_quorum()
+    cfg = configmanager.ConfigManager(None)
     nodename = env.get('HTTP_CONFLUENT_NODENAME', None)
+    if env['PATH_INFO'] == '/self/registerapikey':
+        crypthmac = env.get('HTTP_CONFLUENT_CRYPTHMAC', None)
+        if int(env.get('CONTENT_LENGTH', 65)) > 64:
+            start_response('400 Bad Request', [])
+            yield 'Bad Request'
+            return
+        cryptkey = env['wsgi.input'].read(int(env['CONTENT_LENGTH']))
+        if not (crypthmac and cryptkey):
+            start_response('401 Unauthorized', [])
+            yield 'Unauthorized'
+            return
+        hmackey = cfg.get_node_attributes(nodename, ['secret.selfapiarmtoken'], decrypt=True)
+        hmackey = hmackey.get(nodename, {}).get('secret.selfapiarmtoken', {}).get('value', None)
+        if not hmackey:
+            start_response('401 Unauthorized', [])
+            yield 'Unauthorized'
+            return
+        if not isinstance(hmackey, bytes):
+            hmackey = hmackey.encode('utf8')
+        if not isinstance(cryptkey, bytes):
+            cryptkey = cryptkey.encode('utf8')
+        try:
+            crypthmac = base64.b64decode(crypthmac)
+        except Exception:
+            start_response('400 Bad Request', [])
+            yield 'Bad Request'
+            return
+        righthmac = hmac.new(hmackey, cryptkey, hashlib.sha256).digest()
+        if righthmac == crypthmac:
+            cfgupdate = {nodename: {'crypted.selfapikey': {'hashvalue': cryptkey}}}
+            cfg.set_node_attributes(cfgupdate)
+            cfg.clear_node_attributes([nodename], ['secret.selfapiarmtoken'])
+            start_response('200 OK', [])
+            yield 'Accepted'
+            return
+        start_response('401 Unauthorized', [])
+        yield 'Unauthorized'
+        return
     apikey = env.get('HTTP_CONFLUENT_APIKEY', None)
     if not (nodename and apikey):
         start_response('401 Unauthorized', [])
         yield 'Unauthorized'
         return
-    cfg = configmanager.ConfigManager(None)
+    if len(apikey) > 48:
+        start_response('401', [])
+        yield 'Unauthorized'
+        return
+    
     ea = cfg.get_node_attributes(nodename, ['crypted.selfapikey', 'deployment.apiarmed'])
     eak = ea.get(
         nodename, {}).get('crypted.selfapikey', {}).get('hashvalue', None)

confluent_server/confluent/sockapi.py

@@ -502,8 +502,17 @@ class SockApi(object):
     def watch_resolv(self):
         while True:
             watcher = libc.inotify_init1(os.O_NONBLOCK)
-            if libc.inotify_add_watch(watcher, b'/etc/resolv.conf', 0xcda) <= -1:
-                break
+            resolvpath = '/etc/resolv.conf'
+            while True:
+                try:
+                    resolvpath = os.readlink(resolvpath)
+                except Exception:
+                    break
+            if not isinstance(resolvpath, bytes):
+                resolvpath = resolvpath.encode('utf8')
+            if libc.inotify_add_watch(watcher, resolvpath, 0xcc2) <= -1:
+                eventlet.sleep(15)
+                continue
             select.select((watcher,), (), (), 86400)
             try:
                 os.read(watcher, 1024)

confluent_server/confluent/util.py

@@ -29,6 +29,14 @@ import struct
 import eventlet.green.subprocess as subprocess
 
 
+def mkdirp(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+
+
 def run(cmd):
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = process.communicate()

confluent_server/confluent/webauthn.py

@@ -0,0 +1,113 @@
+import base64
+import confluent.tlvdata as tlvdata
+import confluent.util as util
+import json
+import pywarp
+import pywarp.backends
+import pywarp.credentials
+
+creds = {}
+challenges = {}
+
+class TestBackend(pywarp.backends.CredentialStorageBackend):
+    def __init__(self):
+        global creds
+        try:
+            with open('/tmp/mycreds.json', 'r') as ji:
+                creds = json.load(ji)
+        except Exception:
+            pass
+
+    def get_credential_by_email(self, email):
+        if not isinstance(email, str):
+            email = email.decode('utf8')
+        cred = creds[email]
+        cid = base64.b64decode(cred['cid'])
+        cpk = base64.b64decode(cred['cpk'])
+        return pywarp.credentials.Credential(credential_id=cid, credential_public_key=cpk)
+
+    def save_credential_for_user(self, email, credential):
+        if not isinstance(email, str):
+            email = email.decode('utf8')
+        credential = {'cid': base64.b64encode(credential.id).decode('utf8'), 'cpk': base64.b64encode(bytes(credential.public_key)).decode('utf8')}
+        creds[email] = credential
+        with open('/tmp/mycreds.json', 'w') as jo:
+            json.dump(creds, jo)
+
+    def save_challenge_for_user(self, email, challenge, type):
+        if not isinstance(email, str):
+            email = email.decode('utf8')
+        challenges[email] = challenge
+
+    def get_challenge_for_user(self, email, type):
+        if not isinstance(email, str):
+            email = email.decode('utf8')
+        return challenges[email]
+
+
+def handle_api_request(url, env, start_response, username, cfm, headers, reqbody, authorized):
+    if env['REQUEST_METHOD'] != 'POST':
+        raise Exception('Only POST supported for webauthn operations')
+    url = url.replace('/sessions/current/webauthn', '')
+    if url == '/registration_options':
+        rp = pywarp.RelyingPartyManager('Confluent Web UI', credential_storage_backend=TestBackend(), require_attestation=False)
+        userinfo = cfm.get_user(username)
+        if not userinfo:
+            cfm.create_user(username, role='Stub')
+            userinfo = cfm.get_user(username)
+        authid = userinfo.get('authid', None)
+        if not authid:
+            authid = util.randomstring(64)
+            cfm.set_user(username, {'authid': authid})
+        opts = rp.get_registration_options(username)
+        # pywarp generates an id derived
+        # from username, which is a 'must not' in the spec
+        # we replace that with a complying approach
+        opts['user']['id'] = authid
+        if 'icon' in opts['user']:
+            del opts['user']['icon']
+        if 'id' in opts['rp']:
+            del opts['rp']['id']
+        start_response('200 OK', headers)
+        yield json.dumps(opts)
+    elif url.startswith('/registered_credentials/'):
+        username = url.rsplit('/', 1)[-1]
+        rp = pywarp.RelyingPartyManager('Confluent Web UI', credential_storage_backend=TestBackend())
+        if not isinstance(username, bytes):
+            username = username.encode('utf8')
+        opts = rp.get_authentication_options(username)
+        opts['challenge'] = base64.b64encode(opts['challenge']).decode('utf8')
+        start_response('200 OK', headers)
+        yield json.dumps(opts)
+    elif url.startswith('/validate/'):
+        username = url.rsplit('/', 1)[-1]
+        if not isinstance(username, bytes):
+            username = username.encode('utf8')
+        rp = pywarp.RelyingPartyManager('Confluent Web UI', credential_storage_backend=TestBackend())
+        req = json.loads(reqbody)
+        for x in req:
+            req[x] = base64.b64decode(req[x].replace('-', '+').replace('_', '/'))
+        req['email'] = username
+        rsp = rp.verify(**req)
+        if start_response:
+            start_response('200 OK', headers)
+            sessinfo = {'username': username}
+            if 'authtoken' in authorized:
+                sessinfo['authtoken'] = authorized['authtoken']
+            if 'sessionid' in authorized:
+                sessinfo['sessionid'] = authorized['sessionid']
+            tlvdata.unicode_dictvalues(sessinfo)
+            yield json.dumps(sessinfo)
+        else:
+            yield rsp
+    elif url == '/register_credential':
+        rp = pywarp.RelyingPartyManager('Confluent Web UI', credential_storage_backend=TestBackend(), require_attestation=False)
+        req = json.loads(reqbody)
+        for x in req:
+            req[x] = base64.b64decode(req[x].replace('-', '+').replace('_', '/'))
+        if not isinstance(username, bytes):
+            username = username.encode('utf8')
+        req['email'] = username
+        rsp = rp.register(**req)
+        start_response('200 OK', headers)
+        yield json.dumps(rsp)

confluent_server/confluent_server.spec.tmpl

@@ -19,7 +19,7 @@ Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-
 %if "%{dist}" == ".el9"
 Requires: python3-pyghmi >= 1.0.34, python3-eventlet, python3-greenlet, python3-pycryptodomex >= 3.4.7, confluent_client == %{version}, python3-pyparsing, python3-paramiko, python3-dns, python3-netifaces, python3-pyasn1 >= 0.2.3, python3-pysnmp >= 4.3.4, python3-lxml, python3-eficompressor, python3-setuptools, python3-dateutil, python3-enum34, python3-asn1crypto, python3-cffi, python3-pyOpenSSL, python3-monotonic, python3-websocket-client python3-msgpack python3-libarchive-c python3-yaml openssl iproute
 %else
-Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python2-websocket-client python2-msgpack python-libarchive-c python-yaml
+Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client == %{version}, python-pyparsing, python-paramiko, python-dnspython, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-lxml, python-eficompressor, python-setuptools, python-dateutil, python2-websocket-client python2-msgpack python-libarchive-c python-yaml python-monotonic
 %endif
 %endif
 Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
@@ -71,7 +71,7 @@ true
 getent group confluent > /dev/null || /usr/sbin/groupadd -r confluent
 getent passwd confluent > /dev/null || /usr/sbin/useradd -r -g confluent -d /var/lib/confluent -s /sbin/nologin confluent
 mkdir -p /etc/confluent /var/lib/confluent /var/log/confluent /var/cache/confluent
-chown confluent:confluent /etc/confluent /var/log/confluent /var/cache/confluent
+chown confluent:confluent /etc/confluent /var/lib/confluent /var/log/confluent /var/cache/confluent
 
 %post
 sysctl -p /usr/lib/sysctl.d/confluent.conf >& /dev/null
@@ -99,4 +99,5 @@ true
 rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
+%license /opt/confluent/share/licenses/confluent_server/LICENSE
 %defattr(-,root,root)

confluent_server/makesetup

@@ -9,3 +9,4 @@ sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py
 if [ -f confluent/client.py ]; then 
     echo '__version__ = "'$VERSION'"' > confluent/__init__.py
 fi
+cp ../LICENSE .

confluent_server/setup.py.tmpl

@@ -14,6 +14,7 @@ setup(
               'confluent/discovery/handlers/',
               'confluent/networking/',
               'confluent/plugins/hardwaremanagement/',
+              'confluent/plugins/deployment/',
               'confluent/plugins/console/',
               'confluent/plugins/shell/',
               'confluent/collective/',
@@ -24,6 +25,7 @@ setup(
     scripts=['bin/confluent', 'bin/confluentdbutil', 'bin/collective', 'bin/osdeploy'],
     data_files=[('/etc/init.d', ['sysvinit/confluent']),
                 ('/usr/lib/sysctl.d', ['sysctl/confluent.conf']),
+                ('/opt/confluent/share/licenses/confluent_server', ['LICENSE']),
                 ('/usr/lib/systemd/system', ['systemd/confluent.service']),
 		('/opt/confluent/lib/python/confluent/plugins/console/', [])],
 

confluent_vtbufferd/COPYING.tmt

@@ -0,0 +1,13 @@
+Copyright (c) 2017 Rob King
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+  * Neither the name of the copyright holder nor the
+    names of contributors may be used to endorse or promote products
+    derived from this software without specific prior written permission.

confluent_vtbufferd/buildrpm

@@ -4,7 +4,7 @@ if [ "$NUMCOMMITS" != "$VERSION"  ]; then
     VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
 fi
 mkdir -p dist/confluent_vtbufferd-$VERSION
-cp *.c *.h Makefile dist/confluent_vtbufferd-$VERSION
+cp ../LICENSE COPYING.tmt *.c *.h Makefile dist/confluent_vtbufferd-$VERSION
 cd dist
 tar czf confluent_vtbufferd-$VERSION.tar.gz confluent_vtbufferd-$VERSION
 cd -

confluent_vtbufferd/confluent_vtbufferd.spec.tmpl

@@ -17,21 +17,6 @@ Url: https://github.com/lenovo/confluent/
 %description
 Service for managing in-memory VT emulation for confluent.
 
-Contains third party open source code:
-
-Copyright (c) 2017 Rob King
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-  * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-  * Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-  * Neither the name of the copyright holder nor the
-    names of contributors may be used to endorse or promote products
-    derived from this software without specific prior written permission.
 %prep
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
@@ -40,8 +25,12 @@ modification, are permitted provided that the following conditions are met:
 make
 
 %install
-mkdir -p $RPM_BUILD_ROOT/opt/confluent/bin
+mkdir -p $RPM_BUILD_ROOT/opt/confluent/bin $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd
 cp vtbufferd $RPM_BUILD_ROOT/opt/confluent/bin/
+cp COPYING.tmt $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd
+cp LICENSE $RPM_BUILD_ROOT/opt/confluent/share/licenses/vtbufferd
 
 %files
 /opt/confluent/bin/vtbufferd
+%license /opt/confluent/share/licenses/vtbufferd/COPYING.tmt
+%license /opt/confluent/share/licenses/vtbufferd/LICENSE

genesis/97genesis/install-base

@@ -10,7 +10,7 @@ dracut_install awk egrep dirname bc expr sort
 dracut_install ssh sshd vi reboot lspci parted tmux mkfs mkfs.ext4 mkfs.xfs xfs_db mkswap
 dracut_install efibootmgr
 dracut_install du df ssh-keygen scp clear dhclient lldpd lldpcli tee
-dracut_install /lib64/libnss_dns-2.28.so /lib64/libnss_dns.so.2
+dracut_install /lib64/libnss_dns-2.28.so /lib64/libnss_dns.so.2 /lib64/libnss_myhostname.so.2
 dracut_install ldd uptime /usr/lib64/libnl-3.so.200
 dracut_install poweroff date /etc/nsswitch.conf /etc/services /etc/protocols
 dracut_install  /usr/share/terminfo/x/xterm /usr/share/terminfo/l/linux /usr/share/terminfo/v/vt100 /usr/share/terminfo/x/xterm-color /usr/share/terminfo/s/screen /usr/share/terminfo/x/xterm-256color /usr/share/terminfo/p/putty-256color /usr/share/terminfo/p/putty /usr/share/terminfo/d/dumb

genesis/buildgenesis.sh

@@ -1,18 +1,40 @@
-cd $(dirname $0)
+pushd $(dirname $0)
+rm -rf licenses
 cp -a 97genesis /usr/lib/dracut/modules.d/
 cat /usr/lib/dracut/modules.d/97genesis/install-* > /usr/lib/dracut/modules.d/97genesis/install
 chmod +x /usr/lib/dracut/modules.d/97genesis/install /usr/lib/dracut/modules.d/97genesis/installkernel
 mkdir -p boot/initramfs
 mkdir -p boot/efi/boot
 dracut --no-early-microcode --xz -N -m "genesis base" -f boot/initramfs/distribution $(uname -r)
+tdir=$(mktemp -d)
+tfile=$(mktemp)
+cp boot/initramfs/distribution $tdir
+pushd $tdir
+xzcat distribution|cpio -dumi
+rm distribution
+find . -type f -exec rpm -qf /{} \; 2> /dev/null | grep -v 'not owned' | sort -u > $tfile
+popd
+rm -rf $tdir
+cp $tfile rpmlist
+cp confluent-genesis.spec confluent-genesis-out.spec
+for r in $(cat rpmlist); do
+	#rpm -qi $r | grep ^License|sed -e 's/^.*:/${r}:/' >> licenselist
+	for l in $(rpm -qL $r); do
+		lo=${l#/usr/share/}
+		lo=${lo#licenses/}
+		mkdir -p licenses/$(dirname $lo)
+		cp $l licenses/$lo
+		echo %license /opt/confluent/genesis/%{arch}/licenses/$lo >> confluent-genesis-out.spec
+	done
+done
 cp -f /boot/vmlinuz-$(uname -r) boot/kernel
 cp /boot/efi/EFI/BOOT/BOOTX64.EFI boot/efi/boot
 cp /boot/efi/EFI/centos/grubx64.efi boot/efi/boot/grubx64.efi
 mkdir -p ~/rpmbuild/SOURCES/
-tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot
-rpmbuild -bb confluent-genesis.spec
+tar cf ~/rpmbuild/SOURCES/confluent-genesis.tar boot rpmlist licenses
+rpmbuild -bb confluent-genesis-out.spec
 rm -rf /usr/lib/dracut/modules.d/97genesis
-cd -
+popd
 # getting src rpms would be nice, but centos isn't consistent..
 # /usr/lib/dracut/skipcpio /opt/confluent/genesis/x86_64/boot/initramfs/distribution | xzcat | cpio -dumiv
 # rpm -qf $(find . -type f | sed -e 's/^.//') |sort -u|grep -v 'not owned' > ../rpmlist

genesis/confluent-genesis.spec

@@ -28,6 +28,7 @@ find . -type f -exec chmod o+r {} +
 find . -type f -exec chmod -x {} +
 
 %files
+/opt/confluent/genesis/%{arch}/rpmlist
 /opt/confluent/genesis/%{arch}/boot/efi/boot/BOOTX64.EFI
 /opt/confluent/genesis/%{arch}/boot/efi/boot/grubx64.efi
 /opt/confluent/genesis/%{arch}/boot/initramfs/distribution

imgutil/buildrpm

@@ -5,6 +5,7 @@ if [ "$NUMCOMMITS" != "$VERSION"  ]; then
     VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_imgutil.spec.tmpl > confluent_imgutil.spec
+cp ../LICENSE .
 cd ..
 tar Jcvf confluent_imgutil.tar.xz imgutil
 mv confluent_imgutil.tar.xz ~/rpmbuild/SOURCES/

imgutil/confluent_imgutil.spec.tmpl

@@ -7,6 +7,20 @@ URL: https://hpc.lenovo.com/
 Source: confluent_imgutil.tar.xz
 BuildArch: noarch
 BuildRoot: /tmp/
+%if "%{dist}" == ".el8"
+Requires: squashfs-tools
+%else
+%if "%{dist}" == ".el9"
+Requires: squashfs-tools
+%else
+%if "%{dist}" == ".el7"
+Requires: squashfs-tools
+%else
+Requires: squashfs
+%endif
+%endif
+%endif
+
 
 %description
 Utility for generating, modifying, and packing confluent deployment
@@ -21,6 +35,8 @@ mkdir -p opt/confluent/bin
 mv imgutil opt/confluent/bin/
 chmod a+x opt/confluent/bin/imgutil
 mv ubuntu suse15 el7 el8 opt/confluent/lib/imgutil/
+mkdir -p opt/confluent/share/licenses/confluent_imgutil
+cp LICENSE opt/confluent/share/licenses/confluent_imgutil
 
 %install
 cp -a opt %{buildroot}/
@@ -28,3 +44,4 @@ cp -a opt %{buildroot}/
 %files
 /opt/confluent/bin/imgutil
 /opt/confluent/lib/imgutil
+%license /opt/confluent/share/licenses/confluent_imgutil/LICENSE