Enable more plugins for collective routing

Clarify when things go wrong due to certificate.

LICENSE

@@ -175,28 +175,3 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

confluent_client/bin/confetty

@@ -972,7 +972,7 @@ def consume_termdata(fh, bufferonly=False):
         data = None
     if type(data) == dict:
         updatestatus(data)
-        return
+        return ''
     if data is not None:
         data = client.stringify(data)
         if clearpowermessage:
@@ -1021,6 +1021,7 @@ def consume_termdata(fh, bufferonly=False):
             doexit = True
             inconsole = False
             sys.stdout.write("\r\n[remote disconnected]\r\n")
+    return ''
 
 if __name__ == '__main__':
     errcode = 0

confluent_client/bin/confluent2hosts

@@ -0,0 +1,212 @@
+#!/usr/bin/python2
+import argparse
+import os
+import re
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+def partitionhostsline(line):
+    comment = ''
+    if '#' in line:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    if not line.strip():
+        return '', [], comment
+    ipaddr, names = line.split(None, 1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.byname6 = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            byname = self.byname
+            if ':' in currip:
+                byname = self.byname6
+            for name in names:
+                byname[name] = x
+
+    def add_entry(self, ip, names):
+        targ = self.byname
+        if ':' in ip:
+            targ = self.byname6
+        line = '{:<39} {}'.format(ip, names)
+        x = len(self.sourcelines)
+        self.sourcelines.append(line)
+        for name in names.split():
+            if not name:
+                continue
+            targ[name] = x
+        self.byip[ip] = x
+
+
+    def read_target(self, targetfile):
+        if not os.path.exists(targetfile):
+            return
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            while lines and not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                byname = self.byname
+                if ':' in currip:
+                    byname = self.byname6
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this entry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in byname:
+                        x = byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while self.targlines and not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while self.sourcelines and not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+def main():
+    ap = argparse.ArgumentParser(description="Create/amend /etc/hosts file for given noderange")
+    ap.add_argument('noderange', help='Noderange to generate/update /etc/hosts for')
+    ap.add_argument('-a', '--attrib', help='Pull ip addresses and hostnames from attribute database', action='store_true')
+    ap.add_argument('-i', '--ip', help='Expression to generate addresses (e.g. 172.16.1.{n1} or fd2b:246f:8a50::{n1:x})')
+    ap.add_argument('-n', '--name', help='Expression for name to add ({node}-compute, etc). If unspecified, "{node} {node}.{dns.domain}" will be used', action='append')
+    args = ap.parse_args()
+    c = client.Command()
+    if args.name:
+        names = ' '.join(args.name)
+    else:
+        names = '{node} {node}.{dns.domain}'
+    merger = HostMerger()
+    if not args.ip and not args.attrib:
+        sys.stderr.write('-a or -i is currently required\n')
+        sys.exit(1)
+    if args.attrib:
+        ip4bynode = {}
+        ip6bynode = {}
+        namesbynode = {}
+        domainsbynode = {}
+        for ent in c.read('/noderange/{0}/attributes/current'.format(args.noderange)):
+            ent = ent.get('databynode', {})
+            for node in ent:
+                for attrib in ent[node]:
+                    val = ent[node][attrib]
+                    if not isinstance(val, dict):
+                        continue
+                    val = val.get('value', None)
+                    if attrib == 'dns.domain':
+                        domainsbynode[node] = val
+                    if attrib.startswith('net.'):
+                        nameparts = attrib.split('.')
+                        if len(nameparts) == 2:
+                            currnet = None
+                        else:
+                            currnet = '.'.join(nameparts[1:-1])
+                        for currdict in (ip4bynode, ip6bynode, namesbynode):
+                            if node not in currdict:
+                                currdict[node] = {}
+                        if attrib.endswith('.ipv4_address') and val:
+                            ip4bynode[node][currnet] = val.split('/', 1)[0]
+                        elif attrib.endswith('.ipv6_address') and val:
+                            ip6bynode[node][currnet] = val.split('/', 1)[0]
+                        elif attrib.endswith('.hostname'):
+                            namesbynode[node][currnet] = re.split('\s+|,', val)
+        for node in ip4bynode:
+            mydomain = domainsbynode.get(node, None)
+            for ipdb in (ip4bynode, ip6bynode):
+                for currnet in ipdb[node]:
+                    if args.name:
+                        sys.stderr.write('Custom name and attribute addresses not currently supported together\n')
+                        sys.exit(1)
+                    else:
+                        names = list(namesbynode.get(node, {}).get(currnet, [node]))
+                        if mydomain:
+                            for name in names:
+                                if mydomain in names:
+                                    break
+                            else:
+                                for name in list(names):
+                                    names.append('{0}.{1}'.format(name, mydomain))
+                    names = ' '.join(names)
+                    merger.add_entry(ipdb[node][currnet], names)
+        merger.write_out('/etc/whatnowhosts')
+    else:
+        namesbynode = {}
+        ipbynode = {}
+        expurl = '/noderange/{0}/attributes/expression'.format(args.noderange)
+        expression = names
+        exitcode = 0
+        exitcode |= expand_expression(c, namesbynode, expurl, names)
+        exitcode |= expand_expression(c, ipbynode, expurl, args.ip)
+        if exitcode:
+            sys.exit(exitcode)
+        for node in ipbynode:
+            merger.add_entry(ipbynode[node], namesbynode[node])
+    if os.path.exists('/etc/hosts'):
+        merger.read_target('/etc/hosts')
+        os.rename('/etc/hosts', '/etc/hosts.confluentbkup')
+    merger.write_out('/etc/hosts')
+
+
+
+def expand_expression(c, namesbynode, expurl, expression):
+    exitcode = 0
+    for exp in c.create(expurl, {'expression': expression}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            namesbynode[node] = ex[node]['value']
+    return exitcode
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/dir2img

@@ -21,17 +21,17 @@ def create_image(directory, image, label=None):
             currsz = (currsz // 512 +1) * 512
             datasz += currsz
     datasz += ents * 32768
-    datasz = datasz // 512 + 1
+    datasz = datasz // 4096 + 1
     with open(image, 'wb') as imgfile:
-        imgfile.seek(datasz * 512 - 1)
+        imgfile.seek(datasz * 4096 - 1)
         imgfile.write(b'\x00')
     if label:
         subprocess.check_call(['mformat', '-i', image, '-v', label,
                                '-r', '16', '-d', '1', '-t', str(datasz),
-                               '-s', '1','-h', '1', '::'])
+                               '-s', '4','-h', '2', '::'])
     else:
         subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
-                                str(datasz), '-s', '1','-h', '1', '::'])
+                                str(datasz), '-s', '4','-h', '2', '::'])
     # Some clustered filesystems will have the lock from mformat
     # linger after close (mformat doesn't unlock)
     # do a blocking wait for shared lock and then explicitly
@@ -58,4 +58,4 @@ if __name__ == '__main__':
     label = None
     if len(sys.argv) > 3:
         label = sys.argv[3]
-    create_image(sys.argv[1], sys.argv[2], label)
+    create_image(sys.argv[1], sys.argv[2], label)

confluent_client/bin/nodeattrib

@@ -105,7 +105,10 @@ if len(args) > 1:
             requestargs=args[1:]
     except:
         pass
-
+elif options.clear or options.environment or options.prompt:
+    sys.stderr.write('Attribute names required with specified options\n')
+    argparser.print_help()
+    exitcode = 400
 if exitcode != 0:
     sys.exit(exitcode)
 

confluent_client/bin/nodebmcreset

@@ -47,14 +47,14 @@ exitcode = 0
 errorNodes = set([])
 session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'configuration/management_controller/reset', 'reset', key='state', errnodes=errorNodes)   # = 0 if successful
-if success != 0:
-    sys.exit(success)
 
 # Determine which nodes were successful and print them
 
 allNodes = set([])
 
 for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
+    if 'error' in node and success != 0:
+        sys.exit(success)
     allNodes.add(node['item']['href'].replace("/", ""))
 
 goodNodes = allNodes - errorNodes

confluent_client/bin/nodeconfig

@@ -43,7 +43,7 @@ def bailout(msg, code=1):
     sys.exit(code)
 
 
-argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
+argparser = optparse.OptionParser(usage="Usage: %prog [options] <noderange> [setting|setting=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '
@@ -289,13 +289,11 @@ else:
     for path in queryparms:
         if options.comparedefault:
             continue
-        rc = client.print_attrib_path(path, session, list(queryparms[path]),
+        rcode |= client.print_attrib_path(path, session, list(queryparms[path]),
                                       NullOpt(), queryparms[path])
-        if rc:
-            sys.exit(rc)
     if printsys == 'all' or printextbmc or printbmc or printallbmc:
         if printbmc or not printextbmc:
-         rcode = client.print_attrib_path(
+         rcode |= client.print_attrib_path(
                         '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
                         session, printbmc, options, attrprefix='bmc.')
         if options.extra:

confluent_client/bin/nodeconsole

@@ -25,6 +25,7 @@ if path.startswith('/opt'):
     sys.path.append(path)
 import confluent.client as client
 import confluent.sortutil as sortutil
+import confluent.logreader as logreader
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -35,10 +36,21 @@ argparser = optparse.OptionParser(
            "console")
 argparser.add_option('-t', '--tile', action='store_true', default=False,
                      help='Tile console windows in the terminal')
+argparser.add_option('-l', '--log', action='store_true', default=False,
+                     help='Enter log replay mode instead of showing a live console')
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
+if options.log:
+    logname = args[0]
+    if not os.path.exists(logname) and logname[0] != '/':
+        logname = os.path.join('/var/log/confluent/consoles', logname)
+    if not os.path.exists(logname):
+        sys.stderr.write('Unable to locate {0} on local system\n'.format(logname))
+        sys.exit(1)
+    logreader.replay_to_console(logname)
+    sys.exit(0)
 if options.tile:
     null = open('/dev/null', 'w')
     nodes = []

confluent_client/bin/nodedeploy

@@ -85,10 +85,39 @@ def main(args):
         sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n')
     c = client.Command()
     c.stop_if_noderange_over(args.noderange, args.maxnodes)
+    errnodes = set([])
+    if not args.clear and args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
     if args.clear:
         cleararm(args.noderange, c)
         clearpending(args.noderange, c)
     elif args.profile:
+        profnames = []
+        for prof in c.read('/deployment/profiles/'):
+            profname = prof.get('item', {}).get('href', None)
+            if profname:
+                profname = profname.replace('/', '')
+                profnames.append(profname)
+                if profname == args.profile:
+                    break
+        else:
+            sys.stderr.write('The specified profile "{}" is not an available profile\n'.format(args.profile))
+            if profnames:
+                sys.stderr.write('The following profiles are available:\n')
+                for profname in profnames:
+                    sys.stderr.write('    ' + profname + '\n')
+            else:
+                sys.stderr.write('No deployment profiles available, try osdeploy fiimport or imgutil capture\n')
+            sys.exit(1)
         armonce(args.noderange, c)
         setpending(args.noderange, args.profile, c)
     else:
@@ -122,18 +151,7 @@ def main(args):
                 armed = ''
             print('{0}: {1}{2}'.format(node, profile, armed))
         sys.exit(0)
-    errnodes = set([])
     if args.network and not args.prepareonly:
-        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
-                                        bootmode='uefi',
-                                        persistent=False,
-                                        errnodes=errnodes)
-        if errnodes:
-            sys.stderr.write(
-                'Unable to set boot device for following nodes: {0}\n'.format(
-                    ','.join(errnodes)))
-            return 1
-        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
         return rc
     return 0
 

confluent_client/bin/nodediscover

@@ -48,6 +48,7 @@ columnmapping = {
     'Advertised IP': 'otheripaddrs',
     'Other IP': 'otheripaddrs',
 }
+#TODO: add chassis uuid
 
 
 def print_disco(options, session, currmac, outhandler, columns):

confluent_client/bin/nodeeventlog

@@ -37,7 +37,7 @@ if sys.version_info[0] < 3:
     sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [clear]")
+    usage="Usage: %prog [options] <noderange> [clear]")
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to clear if clearing log, '

confluent_client/bin/nodefirmware

@@ -88,9 +88,9 @@ client.check_globbing(noderange)
 
 def get_update_progress(session, url):
     for res in session.read(url):
-        status = res['phase']
-        percent = res['progress']
-        detail = res['detail']
+        status = res.get('phase', 'error')
+        percent = res.get('progress', None)
+        detail = res.get('detail', repr(res)),
         if status == 'error':
             text = 'error!'
         else:
@@ -147,10 +147,12 @@ def update_firmware(session, filename):
 def show_firmware(session):
     global exitcode
     firmware_shown = False
+    nodes_matched = False
     for component in components:
         for res in session.read(
                 '/noderange/{0}/inventory/firmware/all/{1}'.format(
                     noderange, component)):
+            nodes_matched = True
             exitcode |= client.printerror(res)
             if 'databynode' not in res:
                 continue
@@ -161,7 +163,9 @@ def show_firmware(session):
                     for prefix in inv:
                         firmware_shown = True
                         printfirm(node, prefix, inv[prefix])
-    if not firmware_shown and not exitcode:
+    if not nodes_matched:
+        sys.stderr.write('No matching nodes for noderange "{0}"\n'.format(noderange))
+    elif not firmware_shown and not exitcode:
         argparser.print_help()
 
 

confluent_client/bin/nodegroupdefine

@@ -45,14 +45,15 @@ except IndexError:
 client.check_globbing(noderange)
 session = client.Command()
 exitcode = 0
-attribs = {'name': noderange}
-for arg in args[1:]:
-    key, val = arg.split('=', 1)
-    attribs[key] = val
-for r in session.create('/nodegroups/', attribs):
-    if 'error' in r:
-        sys.stderr.write(r['error'] + '\n')
-        exitcode |= 1
-    if 'created' in r:
-        print('{0}: created'.format(r['created']))
+for group in noderange.split(','):
+    attribs = {'name': group}
+    for arg in args[1:]:
+        key, val = arg.split('=', 1)
+        attribs[key] = val
+    for r in session.create('/nodegroups/', attribs):
+        if 'error' in r:
+            sys.stderr.write(r['error'] + '\n')
+            exitcode |= 1
+        if 'created' in r:
+            print('{0}: created'.format(r['created']))
 sys.exit(exitcode)

confluent_client/bin/nodegroupremove

@@ -43,10 +43,11 @@ noderange = args[0]
 client.check_globbing(noderange)
 session = client.Command()
 exitcode = 0
-for r in session.delete('/nodegroups/{0}'.format(noderange)):
-    if 'error' in r:
-        sys.stderr.write(r['error'] + '\n')
-        exitcode |= 1
-    if 'deleted' in r:
-        print('{0}: deleted'.format(r['deleted']))
+for group in noderange.split(','):
+    for r in session.delete('/nodegroups/{0}'.format(group)):
+        if 'error' in r:
+            sys.stderr.write(r['error'] + '\n')
+            exitcode |= 1
+        if 'deleted' in r:
+            print('{0}: deleted'.format(r['deleted']))
 sys.exit(exitcode)

confluent_client/bin/nodeinventory

@@ -53,6 +53,9 @@ def print_mem_info(node, prefix, meminfo):
         memdescfmt += '3-{1} '
     elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
+    elif 'DCPMM' in meminfo['memory_type']:
+        memdescfmt = '{0}GB {1} '
+        meminfo['module_type'] = 'DCPMM'
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return

confluent_client/bin/nodelicense

@@ -36,7 +36,7 @@ exitcode = 0
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
+          "%prog <noderange> [list][install <license_filename>|save <directory>|delete <license_feature_name>]")
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to delete licenses from, '

confluent_client/bin/nodeping

@@ -0,0 +1,140 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def run():
+
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog [options] noderange")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for ping, with {}, it is the entire name evaluated as an expression, otherwise it is used as a suffix')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    (options, args) = argparser.parse_args()
+    if len(args) < 1:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    nodemap = {}
+    if options.substitutename:
+        subname = options.substitutename
+        if '{' not in subname:
+            subname = '{node}' + subname
+        for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                        {'expression': subname}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                nodemap[node] = ex[node]['value']
+    for rsp in c.read('/noderange/{0}/nodes/'.format(args[0])):
+        if 'error' in rsp:
+            sys.stderr.write(rsp['error'] + '\n')
+            exitcode |= rsp.get('errorcode', 1)
+        if rsp.get('item', {}).get('href', None):
+            node = rsp['item']['href'][:-1]
+            pingnode = nodemap.get(node, node)
+            cmdv = ['ping', '-c', '1', '-W', '1', pingnode]
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(pingnode, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((pingnode, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.readline()
+                if not data:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout':
+                            if ret:
+                                print('{0}: no_ping'.format(node))
+                            else:
+                                print('{0}: ping'.format(node))
+                            if pendingexecs:
+                                node, cmdv = pendingexecs.popleft()
+                                run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodepower

@@ -32,8 +32,8 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange "
-          "([status|on|off|shutdown|boot|reset])")
+    usage="Usage: %prog [options] <noderange> "
+          "([status|on|off|shutdown|boot|reset|pdu_status|pdu_off|pdu_on])")
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
                      help='Show previous power state')
@@ -51,22 +51,27 @@ except IndexError:
 client.check_globbing(noderange)
 setstate = None
 if len(args) > 1:
+    setstate = args[1]
     if setstate == 'softoff':
         setstate = 'shutdown'
-    elif not args[1] in ('stat', 'state', 'status'):
-        setstate = args[1]
 
-if setstate not in (None, 'on', 'off', 'shutdown', 'boot', 'reset'):
+if setstate not in (None, 'on', 'off', 'shutdown', 'boot', 'reset', 'pdu_status', 'pdu_stat', 'pdu_on', 'pdu_off', 'status', 'stat', 'state'):
     argparser.print_help()
     sys.exit(1)
 session = client.Command()
 exitcode = 0
 session.add_precede_key('oldstate')
+powurl = 'state'
+if setstate and setstate.startswith('pdu_'):
+    setstate = setstate.replace('pdu_', '')
+    powurl = 'inlets/all'
+if setstate in ('status', 'state', 'stat'):
+    setstate = None
 
 if options.previous:
     # get previous states
     prev = {}
-    for rsp in session.read("/noderange/{0}/power/state".format(noderange)):
+    for rsp in session.read("/noderange/{0}/power/{1}".format(noderange, powurl)):
         # gets previous (current) states
 
         databynode = rsp["databynode"]
@@ -77,4 +82,7 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))
+def outhandler(node, res):
+    for k in res[node]:
+        client.cprint('{0}: {1}: {2}'.format(node, k.replace('inlet_', ''), res[node][k]))
+sys.exit(session.simple_noderange_command(noderange, '/power/{0}'.format(powurl), setstate, promptover=options.maxnodes, key='state', outhandler=outhandler))

confluent_client/bin/noderemove

@@ -33,7 +33,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='''\n       %prog noderange
+    usage='''\n       %prog <noderange>
              \n ''')
 argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '

confluent_client/bin/nodersync

@@ -40,7 +40,7 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog location noderange:location",
+        usage="Usage: %prog <file/directorylist> <noderange>:<destination>",
     )
     argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '

confluent_client/bin/noderun

@@ -35,11 +35,14 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 import confluent.sortutil as sortutil
+devnull = None
 
 
 def run():
+    global devnull
+    devnull = open(os.devnull, 'rb')
     argparser = optparse.OptionParser(
-        usage="Usage: %prog [options] noderange commandexpression",
+        usage="Usage: %prog [options] <noderange> <command expression>",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
@@ -131,8 +134,14 @@ def run():
 
 
 def run_cmdv(node, cmdv, all, pipedesc):
-    nopen = subprocess.Popen(
-        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    try:
+        nopen = subprocess.Popen(
+            cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as e:
+        if e.errno == 2:
+            sys.stderr.write('{0}: Unable to find local executable file "{1}"'.format(node, cmdv[0]))
+            return
+        raise
     pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
                               'type': 'stdout'}
     pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,

confluent_client/bin/nodesetboot

@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|usb|floppy]')
+    usage='Usage: %prog [options] <noderange> [default|cd|network|setup|hd|usb|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')

confluent_client/bin/nodeshell

@@ -34,22 +34,27 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 import confluent.sortutil as sortutil
-
+devnull = None
 
 def run():
-
+    global devnull
+    devnull = open(os.devnull, 'rb')
     argparser = optparse.OptionParser(
         usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-l', '--loginname', type='str',
+                         help='Username to use when connecting, defaults to current user.')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
     argparser.add_option('-p', '--port', type='int', default=0,
                          help='Specify a custom port for ssh')
     argparser.add_option('-s', '--substitutename',
                          help='Use a different name other than the nodename for ssh')
+    argparser.add_option('-x', '--noexpression', action='store_true',
+                         help='Suppress expression expansion of command')
     argparser.add_option('-m', '--maxnodes', type='int',
                      help='Specify a maximum number of '
                           'nodes to run remote ssh command to, '
@@ -86,26 +91,39 @@ def run():
             ex = exp.get('databynode', ())
             for node in ex:
                 nodemap[node] = ex[node]['value']
-    for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
-                        {'expression': cmdstr}):
-        if 'error' in exp:
-            sys.stderr.write(exp['error'] + '\n')
-            exitcode |= exp.get('errorcode', 1)
-        ex = exp.get('databynode', ())
-        for node in ex:
-            cmd = ex[node]['value']
-            sshnode = nodemap.get(node, node)
-            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
-                cmd = cmd.encode('utf-8')
-            if options.port:
-                cmdv = ['ssh', '-p', '{0}'.format(options.port), sshnode, cmd]
-            else:
-                cmdv = ['ssh', sshnode, cmd]
-            if currprocs < concurrentprocs:
-                currprocs += 1
-                run_cmdv(node, cmdv, all, pipedesc)
-            else:
-                pendingexecs.append((node, cmdv))
+    cmdparms = []
+    if options.noexpression:
+        for res in c.read('/noderange/{0}/nodes/'.format(args[0])):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode |= res.get('errorcode', 1)
+                break
+            node = res['item']['href'][:-1]
+            cmdparms.append((node, cmdstr))
+    else:
+        for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                            {'expression': cmdstr}):
+            if 'error' in exp:
+                sys.stderr.write(exp['error'] + '\n')
+                exitcode |= exp.get('errorcode', 1)
+            ex = exp.get('databynode', ())
+            for node in ex:
+                cmdparms.append((node, ex[node]['value']))
+    for node, cmd in cmdparms:
+        sshnode = nodemap.get(node, node)
+        if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+            cmd = cmd.encode('utf-8')
+        cmdv = ['ssh']
+        if options.port:
+            cmdv += ['-p', '{0}'.format(options.port)]
+        if options.loginname:
+            cmdv += ['-l', options.loginname]
+        cmdv += [sshnode, cmd]
+        if currprocs < concurrentprocs:
+            currprocs += 1
+            run_cmdv(node, cmdv, all, pipedesc)
+        else:
+            pendingexecs.append((node, cmdv))
     if not all or exitcode:
         sys.exit(exitcode)
     rdy, _, _ = select.select(all, [], [], 10)
@@ -145,6 +163,7 @@ def run():
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:
+                    line = line.lstrip('\x08')
                     sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
@@ -154,7 +173,7 @@ def run():
 
 def run_cmdv(node, cmdv, all, pipedesc):
     nopen = subprocess.Popen(
-        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        cmdv, stdin=devnull, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
                               'type': 'stdout'}
     pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,

confluent_client/bin/nodestorage

@@ -20,6 +20,10 @@ import os
 import signal
 import sys
 
+try:
+    input = raw_input
+except NameError:
+    pass
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 except AttributeError:
@@ -195,6 +199,8 @@ def main():
                          help='Comma separated list of disks to use, or the '
                               'word "rest" to indicate use of all available '
                               'disks')
+    argparser.add_option('-i', '--interactive', default=False, action='store_true',
+                         help='Confirm any potentially destructive changes interactively')
     argparser.add_option('-s', '--size', type='str',
                          help='Comma separated list of sizes to use when '
                               'creating volumes.  The sizes may be absolute '
@@ -230,6 +236,13 @@ def main():
     except KeyError:
         argparser.print_help()
         sys.exit(1)
+    if options.interactive and operation != 'show':
+        showstorage(noderange, None, None)
+        sys.stdout.write('The above configuration will be changed by this action, do you wish to continue (y/N): ')
+        confirm = input()
+        if confirm and confirm.lower()[0] != 'y':
+            sys.stdout.write('Aborting\n')
+            sys.exit(1)
     handler(noderange, options, args[2:])
 
 

confluent_client/confluent/client.py

@@ -42,9 +42,9 @@ _attraliases = {
 }
 
 try:
-    input = raw_input
+    getinput = raw_input
 except NameError:
-    pass
+    getinput = input
 
 
 class NestedDict(dict):
@@ -208,7 +208,7 @@ class Command(object):
     def add_precede_dict(self, dict):
         self._prevdict = dict
 
-    def handle_results(self, ikey, rc, res, errnodes=None):
+    def handle_results(self, ikey, rc, res, errnodes=None, outhandler=None):
         if 'error' in res:
             if errnodes is not None:
                 errnodes.add(self._currnoderange)
@@ -245,10 +245,12 @@ class Command(object):
                         node, val, self._prevdict[node]))
                 else:
                     cprint('{0}: {1}'.format(node, val))
+            elif outhandler:
+                outhandler(node, res)
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None, errnodes=None, promptover=None, **kwargs):
+                                 key=None, errnodes=None, promptover=None, outhandler=None, **kwargs):
         try:
             self._currnoderange = noderange
             rc = 0
@@ -262,13 +264,13 @@ class Command(object):
             if input is None:
                 for res in self.read('/noderange/{0}/{1}'.format(
                         noderange, resource)):
-                    rc = self.handle_results(ikey, rc, res, errnodes)
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
             else:
                 self.stop_if_noderange_over(noderange, promptover)
                 kwargs[ikey] = input
                 for res in self.update('/noderange/{0}/{1}'.format(
                         noderange, resource), kwargs):
-                    rc = self.handle_results(ikey, rc, res, errnodes)
+                    rc = self.handle_results(ikey, rc, res, errnodes, outhandler)
             self._currnoderange = None
             return rc
         except KeyboardInterrupt:
@@ -284,9 +286,9 @@ class Command(object):
                 nodename = list(self.read(
                     '/noderange/{0}/nodes/'.format(noderange)))[0].get('item', {}).get('href', None)
                 nodename = nodename[:-1]
-                p = input('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+                p = getinput('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
             else:
-                p = input('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+                p = getinput('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
             if p.lower() != 'y':
                 sys.stderr.write('Aborting at user request\n')
                 sys.exit(1)
@@ -401,7 +403,7 @@ class Command(object):
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    replace = input(
+                    replace = getinput(
                         "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
@@ -507,7 +509,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                                                                   printattr)
                         else:
                             attrout = '{0}: {1}:'.format(node, printattr)
-                    elif 'broken' in currattr:
+                    elif isinstance(currattr, dict) and 'broken' in currattr:
                         attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                                   '{2}'.format(node, printattr,
                                                currattr['broken'])
@@ -524,7 +526,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                         blame = options.blame
                     except AttributeError:
                         blame = False
-                    if blame or 'broken' in currattr:
+                    if blame or (isinstance(currattr, dict) and 'broken' in currattr):
                         blamedata = []
                         if 'inheritedfrom' in currattr:
                             blamedata.append('inherited from group {0}'.format(
@@ -633,7 +635,7 @@ def printgroupattributes(session, requestargs, showtype, nodetype, noderange, op
                         attrout = '{0}: {1}: ********'.format(noderange, attr)
                     else:
                         attrout = '{0}: {1}:'.format(noderange, attr)
-                elif 'broken' in currattr:
+                elif isinstance(currattr, dict) and 'broken' in currattr:
                     attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
                               '{2}'.format(noderange, attr,
                                            currattr['broken'])

confluent_client/confluent/logreader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 import collections
 import os
 import struct
@@ -10,10 +10,13 @@ import termios
 import tty
 
 def writeout(data):
-    done = False
+    if isinstance(data, str) and not isinstance(data, bytes):
+        data = data.encode('utf8')
     try:
-        sys.stdout.write(data)
-        done = True
+        if hasattr(sys.stdout, 'buffer'):
+            sys.stdout.buffer.write(data)
+        else:
+            sys.stdout.write(data)
     except IOError:
         time.sleep(0.1)
         pass
@@ -21,14 +24,15 @@ def writeout(data):
 
 class LogReplay(object):
     def __init__(self, logfile, cblfile):
-        self.bin = open(cblfile, 'r')
-        self.txt = open(logfile, 'r')
+        self.bin = open(cblfile, 'rb')
+        self.txt = open(logfile, 'rb')
         self.cleardata = []
         self.clearidx = 0
         self.pendingdata = collections.deque([])
         self.priordata = collections.deque([])
         self.laststamp = None
         self.needclear = False
+        self.lasttxt = ''
 
     def _rewind(self, datasize=None):
         curroffset = self.bin.tell() - 16
@@ -58,14 +62,14 @@ class LogReplay(object):
 
     def get_output(self, reverse=False):
         endoffset = None
-        output = ''
+        output = b''
         if reverse:  # Forget the uncommited future, if present
-            output += '\x1b[2J\x1b[H'
+            output += b'\x1b[2J\x1b[H'
             endoffset, priordata = self._rewind(4096)
             if priordata is not None:
                 return priordata, 1
         elif self.needclear:
-            output += '\x1b[2J\x1b[H'
+            output += b'\x1b[2J\x1b[H'
             self.needclear = False
         if self.cleardata and self.clearidx < len(self.cleardata):
             datachunk = self.cleardata[self.clearidx]
@@ -76,7 +80,7 @@ class LogReplay(object):
         while (not reverse) or (self.bin.tell() < endoffset):
             record = self.bin.read(16)
             if not record:
-                return '', 0
+                return b'', 0
             record = struct.unpack('!BBIHIBBH', record)
             if record[0] > 16:
                 # Unsupported record, skip
@@ -97,10 +101,8 @@ class LogReplay(object):
                 if reverse and self.bin.tell() < endoffset:
                     output += txtout
                     continue
-                if '\x1b[2J' in txtout:
-                    self.cleardata = txtout.split('\x1b[2J')
-                    for idx in range(1, len(self.cleardata)):
-                        self.cleardata[idx] = '\x1b[2J' + self.cleardata[idx]
+                self.paginate(txtout)
+                if self.cleardata:
                     self.clearidx = 0
                     if not self.cleardata[0]:
                         self.cleardata = self.cleardata[1:]
@@ -116,8 +118,26 @@ class LogReplay(object):
                 break
         if endoffset is not None and endoffset >= 0:
             self.bin.seek(endoffset)
+        self.lasttxt = output
         return output, 1
 
+    def paginate(self, txtout):
+        cleardata = [txtout]
+        nextcleardata = []
+        for sep in (b'\x1b[2J', b'\x1b[H\x1b[J'):
+            replacementcleardata = []
+            for txtout in cleardata:
+                nextcleardata = txtout.split(sep)
+                if len(nextcleardata) > 1:
+                    for idx in range(1, len(nextcleardata)):
+                        nextcleardata[idx] = sep + nextcleardata[idx]
+                replacementcleardata.extend(nextcleardata)
+                nextcleardata = []
+            cleardata = replacementcleardata
+        if len(cleardata) > 1:
+            self.cleardata = cleardata
+        return
+
     def begin(self):
         self.needclear = True
         self.bin.seek(0)
@@ -125,8 +145,36 @@ class LogReplay(object):
     def end(self):
         self.bin.seek(0, 2)
 
+    def search(self, searchstr, skipfirst=False):
+        overlap = len(searchstr)
+        firstoffset = self.bin.tell()
+        lastoffset = firstoffset
+        output = self.lasttxt
+        while searchstr not in output:
+            output, delay = self.get_output()
+            if not self.cleardata and lastoffset == self.bin.tell():
+                self.bin.seek(firstoffset)
+                return b'', 0
+            lastoffset = self.bin.tell()
+        if skipfirst:
+            output, delay = self.get_output()
+            while searchstr not in output:
+                output, delay = self.get_output()
+                if not self.cleardata and lastoffset == self.bin.tell():
+                    self.bin.seek(firstoffset)
+                    return b'', 0
+                lastoffset = self.bin.tell()
+        clear = b'\x1b[2J\x1b[H'
+        txtchunks = self.lasttxt.split(searchstr)
+        output = clear + txtchunks[0]
+        txtchunks = txtchunks[1:]
+        while txtchunks:
+            output += b'\x1b[7m' + searchstr + b'\x1b[27m' + txtchunks[0]
+            txtchunks = txtchunks[1:]
+        return output, 0
 
-def main(txtfile, binfile):
+
+def _replay_to_console(txtfile, binfile):
     replay = LogReplay(txtfile, binfile)
     oldtcattr = termios.tcgetattr(sys.stdin.fileno())
     tty.setraw(sys.stdin.fileno())
@@ -135,7 +183,9 @@ def main(txtfile, binfile):
     reverse = False
     skipnext = False
     quitit = False
+    searchstr = None
     writeout('\x1b[2J\x1b[;H')
+    prepend = ''
     try:
         while not quitit:
             if not skipnext:
@@ -143,10 +193,20 @@ def main(txtfile, binfile):
             skipnext = False
             reverse = False
             if newdata:
+                if prepend:
+                    newdata = prepend + newdata
+                    prepend = b''
                 writeout(newdata)
+                newdata = ''
                 writeout('\x1b]0;[Time: {0}]\x07'.format(
                     time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))))
-                sys.stdout.flush()
+                try:
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                except IOError:
+                    pass
             while True:
                 select.select((sys.stdin,), (), (), 86400)
                 myinput = sys.stdin.read()
@@ -168,32 +228,68 @@ def main(txtfile, binfile):
                     break
                 elif myinput.lower() == 'd':
                     writeout('\x1b];{0}\x07'.format(replay.debuginfo()))
+                    if hasattr(sys.stdout, 'buffer'):
+                        sys.stdout.buffer.flush()
+                    else:
+                        sys.stdout.flush()
+                elif myinput.lower() == '/':
+                    sys.stdout.write('\x1b7\x1b[99999;0H\x1b[2K')
+                    searchstr = ''
+                    nxtchr = '/'
+                    while '\r' not in searchstr:
+                        sys.stdout.write(nxtchr)
+                        sys.stdout.flush()
+                        select.select((sys.stdin,), (), (), 86400)
+                        nxtchr = sys.stdin.read(1)
+                        if nxtchr in ('\x08', '\x7f'):
+                            searchstr = searchstr[:-1]
+                            nxtchr = '\x08 \x08'
+                        else:
+                            searchstr += nxtchr
+                    if not isinstance(searchstr, bytes):
+                        searchstr = searchstr.encode('utf8')
+                    searchstr = searchstr[:-1]
+                    newdata, delay = replay.search(searchstr)
+                    if not newdata:
+                        sys.stdout.write('\x1b[1K\rNo match found')
+                        prepend = b'\x1b[1K\x1b8'
+                    else:
+                        sys.stdout.write('\x1b[1K\x1b8')
                     sys.stdout.flush()
+                    skipnext = True
+                    break
+                elif myinput.lower() == 'n' and searchstr:
+                    newdata, delay = replay.search(searchstr, True)
+                    if not newdata:
+                        sys.stdout.write('\x1b7\x1b[99999;0H\x1b[2KNo more matches found')
+                        sys.stdout.flush()
+                        prepend = b'\x1b[1K\x1b8'
+                    skipnext = True
+                    break
                 else:
                     pass # print(repr(myinput))
     except Exception:
         currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
         fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
         termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
-        writeout('\x1b[m')
+        writeout('\x1b[m\x1b[?25h\n')
         raise
     currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
     fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
     termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
-    writeout('\x1b[m')
+    writeout('\x1b[m\x1b[?25h\n')
+
+def replay_to_console(txtfile):
+    if os.path.exists(txtfile + '.cbl'):
+        binfile = txtfile + '.cbl'
+    else:
+        fileparts = txtfile.split('.')
+        prefix = '.'.join(fileparts[:-1])
+        binfile = prefix + '.cbl.' + fileparts[-1]
+        if not os.path.exists(binfile):
+            sys.stderr.write('Unable to locate cbl file\n')
+            sys.exit(1)
+    _replay_to_console(txtfile, binfile)
 
 if __name__ == '__main__':
-    txtfile = sys.argv[1]
-    if len(sys.argv) > 2:
-        binfile = sys.argv[2]
-    else:
-        if os.path.exists(txtfile + '.cbl'):
-            binfile = txtfile + '.cbl'
-        else:
-            fileparts = txtfile.split('.')
-            prefix = '.'.join(fileparts[:-1])
-            binfile = prefix + '.cbl.' + fileparts[-1]
-            if not os.path.exists(binfile):
-                sys.stderr.write('Unable to locate cbl file\n')
-                sys.exit(1)
-    main(txtfile, binfile)
+    replay_to_console(sys.argv[1])

confluent_client/confluent/screensqueeze.py

@@ -17,9 +17,9 @@ import sys
 import struct
 import termios
 
-def get_screenwidth():
+def get_screengeom():
     return struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
-                                           b'....'))[1]
+                                           b'....'))
 class ScreenPrinter(object):
 
     def __init__(self, noderange, client, textlen=4):
@@ -44,7 +44,7 @@ class ScreenPrinter(object):
         self.fieldwidth = maxlen + textlen + 1  # 1 for column
 
     def set_output(self, node, text):
-        if self.nodeoutput[node] == text:
+        if self.nodeoutput.get(node, None) == text:
             return
         self.nodeoutput[node] = text
         if len(text) >= self.textlen:
@@ -54,23 +54,31 @@ class ScreenPrinter(object):
 
     def drawscreen(self):
         if self.squeeze:
-            currwidth = get_screenwidth()
+            currheight, currwidth = get_screengeom()
+            currheight -= 1
             numfields = currwidth // self.fieldwidth
             fieldformat = '{{0:>{0}}}:{{1:{1}}}'.format(self.nodenamelen,
                                                         self.textlen)
             sys.stdout.write('\x1b[2J\x1b[;H')  # clear screen
+            if len(self.nodelist) < (numfields * currheight):
+                numfields = len(self.nodelist) // currheight + 1
         else:
             numfields = 1
             fieldformat = '{0}: {1}'
-        currfields = 0
-        for node in self.nodelist:
-            if currfields >= numfields:
+        if self.squeeze:
+            columns = [self.nodelist[x:x+currheight] for x in range(0, len(self.nodelist), currheight)]
+            for currow in range(0, len(columns[0])):
+                for col in columns:
+                    try:
+                        node = col[currow]
+                        sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
+                    except IndexError:
+                        break
+                sys.stdout.write('\n')
+        else:
+            for node in self.nodelist:
+                sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
                 sys.stdout.write('\n')
-                currfields = 1
-            else:
-                currfields += 1
-            sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
-        sys.stdout.write('\n')
         sys.stdout.flush()
 
 
@@ -80,8 +88,8 @@ class ScreenPrinter(object):
 if __name__ == '__main__':
     import confluent.client as client
     c = client.Command()
-    p = ScreenPrinter('n1-n13', c)
-    p.set_output('n3', 'Upload: 67%')
+    p = ScreenPrinter('d1-d12', c)
+    p.set_output('d3', 'Upload: 67%')
 
 
 

confluent_client/confluent/termhandler.py

@@ -53,13 +53,16 @@ class TermHandler(object):
             connection = None
             try:
                 connection, address = self.socket.accept()
-                connection.sendall("confetty control v1--\n")
+                connection.sendall(b"confetty control v1--\n")
                 cmd = connection.recv(8)
-                if 'GETWINID' == cmd:
-                    connection.sendall(os.environ['WINDOWID'])
+                if b'GETWINID' == cmd:
+                    winid = os.environ['WINDOWID']
+                    if not isinstance(winid, bytes):
+                        winid = winid.encode('utf8')
+                    connection.sendall(winid)
                 connection.close()
             except BaseException:
                 pass
             finally:
                 if connection is not None:
-                    connection.close()
+                    connection.close()

confluent_client/confluent/tlvdata.py

@@ -19,8 +19,12 @@ import array
 import ctypes
 import ctypes.util
 import confluent.tlv as tlv
-import eventlet.green.socket as socket
-import eventlet.green.select as select
+try:
+    import eventlet.green.socket as socket
+    import eventlet.green.select as select
+except ImportError:
+    import socket
+    import select
 from datetime import datetime
 import json
 import os
@@ -240,8 +244,12 @@ def recv(handle):
         cdata = cmsgarr[CMSG_LEN(0).value:]
         data = rawbuffer[:i]
         if cmsg.cmsg_level == socket.SOL_SOCKET and cmsg.cmsg_type == SCM_RIGHTS:
-            filehandles.fromstring(bytes(
-                cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
+            try:
+                filehandles.fromstring(bytes(
+                    cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
+            except AttributeError:
+                filehandles.frombytes(bytes(
+                    cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
         data = json.loads(bytes(data))
         return ClientFile(data['filename'], data['mode'], filehandles[0])
     else:

confluent_client/confluent_client.spec.tmpl

@@ -27,19 +27,28 @@ a confluent server.
 %if "%{dist}" == ".el8"
 python3 setup.py build
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py build
+%else
 python2 setup.py build
 %endif
+%endif
 
 %install
 %if "%{dist}" == ".el8"
 python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
 %else
+%if "%{dist}" == ".el9"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
 python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
 %endif
+%endif
 
 
 %clean
 rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
+%license /opt/confluent/share/licenses/confluent_client/LICENSE
 %defattr(-,root,root)

confluent_client/confluent_env.sh

@@ -37,7 +37,7 @@ alias nodereseat='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export C
 alias noderun='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; noderun'
 alias nodesensors='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesensors'
 alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesetboot'
-alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
+alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage -i'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
 alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
 # Do not continue for non-bash shells, the rest of this sets up bash completion functions
@@ -156,6 +156,40 @@ _confluent_osimage_completion()
     fi
 }
 
+_confluent_imgutil_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 2 ]; then
+        COMPREPLY=($(compgen -W "build exec unpack pack capture" -- ${COMP_WORDS[COMP_CWORD]}))
+        return
+    elif [ ${CMPARGS[1]} == 'build' ]; then
+        COMPREPLY=($(compgen -W "-s $(ls /var/lib/confluent/distributions)" -- ${COMP_WORDS[COMP_CWORD]}))
+	    return
+    elif [ ${CMPARGS[1]} == 'unpack' ]; then
+        if [ $NUMARGS == 3 ]; then
+            COMPREPLY=($(compgen -W "-s $(ls /var/lib/confluent/public/os)" -- ${COMP_WORDS[COMP_CWORD]}))
+            return
+        fi
+        compopt -o dirnames
+	    return
+    elif [ ${CMPARGS[1]} == 'pack' ]; then
+        if [ $NUMARGS == 3 ]; then
+            compopt -o dirnames
+        fi
+        COMPREPLY=()
+        return
+    elif [ ${CMPARGS[1]} == 'capture' ]; then
+        if [ $NUMARGS == 3 ]; then
+            _confluent_nn_completion
+            return
+        fi
+        return
+    elif [ ${CMPARGS[1]} == 'exec' ]; then
+        compopt -o dirnames
+	    COMPREPLY=($(compgen -W "-v" -- ${COMP_WORDS[COMP_CWORD]}))
+        return
+    fi
+}
 _confluent_nodedeploy_completion()
 {
     _confluent_get_args
@@ -238,9 +272,6 @@ _confluent_nodeattrib_completion()
 _confluent_nn_completion()
 {
     _confluent_get_args
-    if [ $NUMARGS -gt 2 ]; then
-        return;
-    fi
     INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
@@ -302,6 +333,7 @@ complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
 complete -F _confluent_nodedeploy_completion nodedeploy
 complete -F _confluent_osimage_completion osdeploy
+complete -F _confluent_imgutil_completion imgutil
 complete -F _confluent_ng_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth

confluent_client/doc/man/collective.ronn

@@ -6,8 +6,8 @@ collective(1) -- Check and manage a confluent collective
 `collective invite <server>`  
 `collective join <server> [-i TOKEN]`  
 `collective show`  
-`collective gencert`
-`collective delete`
+`collective gencert`  
+`collective delete`  
 
 
 ## DESCRIPTION

confluent_client/doc/man/confluentdbutil.ronn

@@ -15,7 +15,7 @@ the json files (password protected, removed from the files, or unprotected).
 
 ## OPTIONS
 
-* `-p`, `--password`:
+* `-p PASSWORD`, `--password=PASSWORD`:
   If specified, information such as usernames and passwords will be encrypted
   using the given password.
   

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -34,7 +34,13 @@ For the `groups` attribute, it is possible to add a group by doing
 `groups,=<newgroup>` and to remove by doing `groups^=<oldgroup>`
 
 Note that `nodeattrib <group>` will likely not provide the expected behavior.
-See nodegroupattrib(8) command on how to manage attributes on a group level.
+See nodegroupattrib(8) command on how to manage attributes on a group level.  Running
+nodeattrib on a group will simply set node-specific attributes on each individual
+member of the group.
+
+Additionally, if wanting to change from a node level attribute value to inherit from a
+member group, it is required to use '-c' to clear the attribute.  Conversely, assigning
+to a blank value will allow masking a group defined attribute with an empty value.
 
 ## OPTIONS
 

confluent_client/doc/man/nodeconfig.ronn

@@ -27,7 +27,7 @@ actually be in effect until a reboot.
   If combined with `-x`, will show all differing values except those indicated
   by `-x`
   
-* `-b`, `--batch`:
+* `-b settings.batch`, `--batch=settings.batch`:
    Provide arguments as lines of a file, rather than the command line.  
    
 * `-d`, `--detail`:
@@ -47,7 +47,7 @@ actually be in effect until a reboot.
   Include advanced settings, which are normally not intended to be used
   without direction from the relevant server vendor.
 
-* `-r`, `--restoredefault`:
+* `-r COMPONENT`, `--restoredefault=COMPONENT`:
   Request that the specified component of the targeted nodes will have its
   configuration reset to default.  Currently the only component implemented
   is uefi.

confluent_client/doc/man/nodeconsole.ronn

@@ -22,6 +22,11 @@ a new session.
   Use tmux to arrange consoles of the given noderange into a tiled layout on
   the terminal screen
 
+* `-l`, `--log`:
+  Perform a log reply on the current, local log in /var/log/confluent/consoles.
+  If in collective mode, this only makes sense to use on the current collective
+  manager at this time.
+
 ## ESCAPE SEQUENCE COMMANDS
 
 While connected to a console, a number of commands may be performed through escape

confluent_client/doc/man/nodelicense.ronn

@@ -3,7 +3,7 @@ nodelicense(8) -- Manage license keys on BMC
 
 ## SYNOPSIS
 
-`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
+`nodelicense <noderange> [list][install <license_filename>|save <directory>|delete <license_feature_name>]`
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodeping.ronn

@@ -0,0 +1,47 @@
+nodeping(8) -- Pings a node or a noderange.
+==============================
+## SYNOPSIS
+`nodeping [options] noderange`  
+
+## DESCRIPTION  
+**nodeping** is a command that pings the default NIC on a node.
+It can also be used with the `-s` flag to change the ping location to something that is 'non primary'
+
+
+## OPTIONS
+* ` -f` COUNT, `-c` COUNT, --count=COUNT  
+   Number of commands to run at a time  
+* `-h`, `--help`:  
+  Show help message and exit      
+* `-s` SUBSTITUTENAME, --substitutename=SUBSTITUTENAME  
+  Use a different name other than the nodename for ping. This may be a 
+  expression, such as {bmc} or, if no { character is present, it is treated as a suffix.  -s -eth1 would make n1 become n1-eth1, for example. 
+
+   
+## EXAMPLES
+ * Pinging a node :  
+  `# nodeping <node>`  
+  `node : ping`    
+    
+* Pinging a group:  
+  `# nodeping <groupname>`    
+  `Node1 : ping  
+    Node2 : ping  
+    Node3 : ping`  
+
+* Pinging BMC on a node:  
+  `# nodeping -s {bmc} <noderange>`  
+  ` Node-bmc : ping`  
+
+* Pinging by specifying a suffix:
+  `# nodeping d1-d4 -s -eth1`  
+  `d2-eth1: no_ping`  
+  `d1-eth1: no_ping`  
+  `d3-eth1: no_ping`  
+  `d4-eth1: no_ping`  
+
+* Fail to ping node:  
+  `# nodeping <node>`  
+  `node : no_ping`  
+ 
+

confluent_client/doc/man/nodepower.ronn

@@ -3,8 +3,7 @@ nodepower(8) -- Check or change power state of confluent nodes
 
 ## SYNOPSIS
 
-`nodepower <noderange>`  
-`nodepower <noderange> [-p] [on|off|boot|shutdown|reset|status]`
+`nodepower [options] <noderange> ([status|on|off|shutdown|boot|reset])`
 
 ## DESCRIPTION
 
@@ -25,10 +24,13 @@ respond.
 * `reset`:  Request immediate reset of nodes of the noderange.  Nodes that are
 off will not react to this request.
 * `status`:  Behave identically to having no argument passed at all.
+* `pdu_status`: Query state of associated PDU outlets, if configured.
+* `pdu_on`: Energize all PDU outlets associated with the noderange.
+* `pdu_off`: De-energize all PDU outlets associated with the noderange.
 
 ## OPTIONS
 
-* `-p`, '--showprevious':
+* `-p`, `--showprevious`:
    Show previous power state for all directives that may change power state.
    
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:

confluent_client/doc/man/nodersync.ronn

@@ -12,7 +12,7 @@ noderange. This will present progress as percentage for all nodes.
 
 ## OPTIONS
 
-* `-c`, `-f`, `--count`:
+* `-f COUNT`, `-c COUNT`, `--count=COUNT`:
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 

confluent_client/doc/man/noderun.ronn

@@ -3,7 +3,7 @@ noderun(8) -- Run arbitrary commands per node in a noderange
 
 ## SYNOPSIS
 
-`noderun <noderange> <command string>`
+`noderun [options] <noderange> <command expression>`
 
 ## DESCRIPTION
 
@@ -15,7 +15,7 @@ themselves, see nodeshell(8).
 
 ## OPTIONS
 
-* `-f`, `-c`, `--count`:
+* `-f COUNT`, `-c COUNT`, `--count=COUNT`:
   Number of commands to run at a time
   
 * `-n`, `--nonodeprefix`:

confluent_client/doc/man/nodeshell.ronn

@@ -17,6 +17,9 @@ as stderr, unlike psh which combines all stdout and stderr into stdout.
 * `-c COUNT`, `-f COUNT`, `--count=COUNT`
   Specify the maximum number of instances to run concurrently
 
+* `-l LOGINNAME`, `--loginname=LOGINNAME`
+  Username to use when connecting, defaults to current user.
+
 * `-m MAXNODES`, `--maxnodes=MAXNODES`
   Specify a maximum number of nodes to run remote ssh command to, prompting
   if over the threshold

confluent_client/setup.py.tmpl

@@ -1,7 +1,9 @@
 from setuptools import setup
 import os
 
-data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh'])]
+data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh']),
+              ('/opt/confluent/share/licenses/confluent_client/', ['LICENSE'])
+              ]
 try:
     scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')]
     data_files.append(('/opt/confluent/share/man/man1', ['man/man1/' + x for x in os.listdir('man/man1')]))

confluent_osdeploy/buildrpm

@@ -5,7 +5,19 @@ if [ "$NUMCOMMITS" != "$VERSION"  ]; then
 fi
 sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec
 cd ..
+cp ../LICENSE .
 tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy
 mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
 cd -
+mkdir -p el9bin/opt/confluent/bin
+mkdir -p el9bin/stateless-bin
+podman run --privileged --rm -v $(pwd)/utils:/buildutils -i -t fedorabuilder make -C /buildutils
+cd utils
+mv confluent_imginfo copernicus clortho autocons ../el9bin/opt/confluent/bin
+mv start_root urlmount ../el9bin/stateless-bin/
+make clean
+cd ..
+tar Jcvf confluent_el9bin.tar.xz el9bin/
+mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
+rm -rf el9bin
 rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/common/initramfs/opt/confluent/bin/apiclient

@@ -0,0 +1,427 @@
+#!/usr/bin/python
+try:
+    import http.client as client
+except ImportError:
+    import httplib as client
+import ctypes
+import ctypes.util
+import glob
+import os
+import select
+import socket
+import subprocess
+import ssl
+import sys
+import struct
+import time
+
+class InvalidApiKey(Exception):
+    pass
+
+def msg_align(len):
+    return (len + 3) & ~3
+
+cryptname = ctypes.util.find_library('crypt')
+if not cryptname:
+    if os.path.exists('/lib64/libcrypt.so.2') or os.path.exists('/usr/lib64/libcrypt.so.2'):
+        cryptname = 'libcrypt.so.2'
+    else:
+        cryptname = 'libcrypt.so.1'
+c_libcrypt = ctypes.CDLL(cryptname)
+c_crypt = c_libcrypt.crypt
+c_crypt.argtypes = (ctypes.c_char_p, ctypes.c_char_p)
+c_crypt.restype = ctypes.c_char_p
+
+
+def get_my_addresses():
+    nlhdrsz = struct.calcsize('IHHII')
+    ifaddrsz = struct.calcsize('BBBBI')
+    # RTM_GETADDR = 22
+    # nlmsghdr struct: u32 len, u16 type, u16 flags, u32 seq, u32 pid
+    nlhdr = struct.pack('IHHII', nlhdrsz + ifaddrsz, 22, 0x301, 0, 0)
+    # ifaddrmsg struct: u8 family, u8 prefixlen, u8 flags, u8 scope, u32 index
+    ifaddrmsg = struct.pack('BBBBI', 0, 0, 0, 0, 0)
+    s = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
+    s.bind((0, 0))
+    s.sendall(nlhdr + ifaddrmsg)
+    addrs = []
+    while True:
+        pdata = s.recv(65536)
+        v = memoryview(pdata)
+        if struct.unpack('H', v[4:6])[0] == 3:  # netlink done message
+            break
+        while len(v):
+            length, typ = struct.unpack('IH', v[:6])
+            if typ == 20:
+                fam, plen, _, scope, ridx = struct.unpack('BBBBI', v[nlhdrsz:nlhdrsz+ifaddrsz])
+                if scope in (253, 0):
+                    rta = v[nlhdrsz+ifaddrsz:length]
+                    while len(rta):
+                        rtalen, rtatyp = struct.unpack('HH', rta[:4])
+                        if rtalen < 4:
+                            break
+                        if rtatyp == 1:
+                            addrs.append((fam, rta[4:rtalen], plen, ridx))
+                        rta = rta[msg_align(rtalen):]
+            v = v[msg_align(length):]
+    return addrs
+
+
+def scan_confluents():
+    srvs = {}
+    s6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    s6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    s6.bind(('::', 1900))
+    s4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    s4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+    s4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    s4.bind(('0.0.0.0', 1900))
+    doneidxs = set([])
+    msg = 'M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:'
+    with open('/etc/confluent/confluent.deploycfg') as dcfg:
+        for line in dcfg.read().split('\n'):
+            if line.startswith('confluent_uuid:'):
+                confluentuuid = line.split(': ')[1]
+                msg += '/confluentuuid=' + confluentuuid
+                break
+    with open('/sys/devices/virtual/dmi/id/product_uuid') as uuidin:
+        msg += '/uuid=' + uuidin.read().strip()
+    for addrf in glob.glob('/sys/class/net/*/address'):
+        with open(addrf) as addrin:
+            hwaddr = addrin.read().strip()
+            msg += '/mac=' + hwaddr
+    msg = msg.encode('utf8')
+    for addr in get_my_addresses():
+        if addr[0] == socket.AF_INET6:
+            if addr[-1] in doneidxs:
+                continue
+            try:
+                s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addr[-1])
+            except TypeError:
+                s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, addr[-1].tobytes())
+            try:
+                s6.sendto(msg, ('ff02::c', 1900))
+            except OSError:
+                pass
+            doneidxs.add(addr[-1])
+        elif addr[0] == socket.AF_INET:
+            try:
+                s4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, addr[1])
+            except TypeError:
+                s4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF, addr[1].tobytes())
+            try:
+                s4.sendto(msg, ('239.255.255.250', 1900))
+            except OSError:
+                pass
+    r = select.select((s4, s6), (), (), 4)
+    srvlist = []
+    if r:
+        r = r[0]
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            rsp = rsp.split(b'\r\n')
+            current = None
+            for line in rsp:
+                if line.startswith(b'NODENAME: '):
+                    current = {}
+                elif line.startswith(b'DEFAULTNET: 1'):
+                    current['isdefault'] = True
+                elif line.startswith(b'MGTIFACE: '):
+                    current['mgtiface'] = line.replace(b'MGTIFACE: ', b'').strip().decode('utf8')
+            if len(peer) > 2:
+                current['myidx'] = peer[-1]
+            currip = peer[0]
+            if currip.startswith('fe80::') and '%' not in currip:
+                currip = '{0}%{1}'.format(currip, peer[-1])
+            srvs[currip] = current
+            srvlist.append(currip)
+        r = select.select((s4, s6), (), (), 2)
+        if r:
+            r = r[0]
+    return srvlist, srvs
+
+
+def get_net_apikey(nodename, mgr):
+    alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789./'
+    newpass = ''.join([alpha[x >> 2] for x in bytearray(os.urandom(32))])
+    salt = '$5$' + ''.join([alpha[x >> 2] for x in bytearray(os.urandom(8))])
+    newpass = newpass.encode('utf8')
+    salt = salt.encode('utf8')
+    crypted = c_crypt(newpass, salt)
+    for addrinfo in socket.getaddrinfo(mgr, 13001, 0, socket.SOCK_STREAM):
+        try:
+            clisock = socket.socket(addrinfo[0], addrinfo[1])
+            clisock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            if addrinfo[0] == socket.AF_INET:
+                cliaddr = ('0.0.0.0', 302)
+            else:
+                cliaddr = ('::', 302)
+            clisock.bind(cliaddr)
+            clisock.connect(addrinfo[-1])
+            rsp = clisock.recv(8)
+            if rsp != b'\xc2\xd1-\xa8\x80\xd8j\xba':
+                raise Exception('Unrecognized credential banner')
+            hellostr = bytearray([1, len(nodename)]) + bytearray(nodename.encode('utf8')) + bytearray(b'\x00\x00')
+            clisock.send(hellostr)
+            rsp = bytearray(clisock.recv(2))
+            if not rsp:
+                continue
+            if rsp[0] == 128:
+                continue
+            if rsp[0] == 2:
+                echotoken = clisock.recv(rsp[1])
+                clisock.recv(2)  # drain \x00\x00
+                clisock.send(bytes(bytearray([3, rsp[1]])))
+                clisock.send(echotoken)
+                clisock.send(bytes(bytearray([4, len(crypted)])))
+                clisock.send(crypted)
+                clisock.send(b'\x00\x00')
+            rsp = bytearray(clisock.recv(2))
+            if rsp[0] == 5:
+                return newpass.decode('utf8')
+        finally:
+            clisock.close()
+    return ''
+
+
+def get_apikey(nodename, hosts, errout=None):
+    apikey = ""
+    if os.path.exists('/etc/confluent/confluent.apikey'):
+        apikey = open('/etc/confluent/confluent.apikey').read().strip()
+        if apikey:
+            return apikey
+    while not apikey:
+        for host in hosts:
+            try:
+                apikey = get_net_apikey(nodename, host)
+            except OSError:
+                apikey = None
+            if apikey:
+                break
+        else:
+            srvlist, _ = scan_confluents()
+            for host in srvlist:
+                try:
+                    apikey = get_net_apikey(nodename, host)
+                except OSError:
+                    apikey = None
+                if apikey:
+                    break
+        if not apikey:
+            errmsg = "Failed getting API token, check deployment.apiarmed attribute on {}\n".format(nodename)
+            sys.stderr.write(errmsg)
+            if errout:
+                errout.write(errmsg)
+            time.sleep(10)
+    with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
+        apiout.write(apikey)
+    apikey = apikey.strip()
+    os.chmod('/etc/confluent/confluent.apikey', 0o600)
+    return apikey
+
+class HTTPSClient(client.HTTPConnection, object):
+    def __init__(self, usejson=False, port=443, host=None, errout=None):
+        self.errout = None
+        if errout:
+            self.errout = open(errout, 'w')
+            self.errout.flush()
+        self.stdheaders = {}
+        mgtiface = None
+        if usejson:
+            self.stdheaders['ACCEPT'] = 'application/json'
+        if host:
+            self.hosts = [host]
+            with open('/etc/confluent/confluent.info') as cinfo:
+                info = cinfo.read().split('\n')
+                for line in info:
+                    if line.startswith('NODENAME:'):
+                        node = line.split(' ')[1]
+                        self.stdheaders['CONFLUENT_NODENAME'] = node
+        else:
+            self.hosts = []
+            info = open('/etc/confluent/confluent.info').read().split('\n')
+            havedefault = '0'
+            for line in info:
+                if line.startswith('NODENAME:'):
+                    node = line.split(' ')[1]
+                    self.stdheaders['CONFLUENT_NODENAME'] = node
+                if line.startswith('MANAGER:') and not host:
+                    host = line.split(' ')[1]
+                if line.startswith('EXTMGRINFO:'):
+                    extinfo = line.split(' ')[1]
+                    extinfo = extinfo.split('|')
+                    if not mgtiface:
+                        host, mgtiface, havedefault = extinfo[:3]
+                    if havedefault == '0' and extinfo[2] == '1':
+                        host, mgtiface, havedefault = extinfo[:3]
+            if '%' in host:
+                ifidx = host.split('%', 1)[1]
+                with open('/tmp/confluent.ifidx', 'w+') as ifout:
+                    ifout.write(ifidx)
+            self.hosts.append(host)
+            try:
+                info = open('/etc/confluent/confluent.deploycfg').read().split('\n')
+            except Exception:
+                info = None
+            if info:
+                for line in info:
+                    if line.startswith('deploy_server: ') or line.startswith('deploy_server_v6: '):
+                        self.hosts.append(line.split(': ', 1)[1])
+        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, self.hosts, errout=self.errout)
+        if mgtiface:
+            self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
+        self.port = port
+        self.host = None
+        self.node = node
+        host = self.check_connections()
+        client.HTTPConnection.__init__(self, host, port)
+        self.connect()
+
+
+    def set_header(self, key, val):
+        self.stdheaders[key] = val
+
+    def check_connections(self):
+        foundsrv = None
+        hosts = self.hosts
+        for timeo in (0.1, 5):
+            for host in hosts:
+                try:
+                    addrinf = socket.getaddrinfo(host, self.port)[0]
+                    psock = socket.socket(addrinf[0])
+                    psock.settimeout(timeo)
+                    psock.connect(addrinf[4])
+                    foundsrv = host
+                    psock.close()
+                    break
+                except OSError:
+                    continue
+            else:
+                continue
+            break
+        if not foundsrv:
+            srvlist, srvs = scan_confluents()
+            hosts = []
+            for srv in srvlist:
+                if srvs[srv].get('isdefault', False):
+                    hosts = [srv] + hosts
+                else:
+                    hosts = hosts + [srv]
+            for host in hosts:
+                try:
+                    addrinf = socket.getaddrinfo(host, self.port)[0]
+                    psock = socket.socket(addrinf[0])
+                    psock.settimeout(timeo)
+                    psock.connect(addrinf[4])
+                    foundsrv = host
+                    psock.close()
+                    break
+                except OSError:
+                    continue
+            else:
+                raise Exception('Unable to reach any hosts')
+        return foundsrv
+
+    def connect(self):
+        addrinf = socket.getaddrinfo(self.host, self.port)[0]
+        psock = socket.socket(addrinf[0])
+        psock.connect(addrinf[4])
+        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx.load_verify_locations('/etc/confluent/ca.pem')
+        host = self.host.split('%', 1)[0]
+        if '[' not in host and ':' in host:
+            self.stdheaders['Host'] = '[{0}]'.format(host)
+        else:
+            self.stdheaders['Host'] = '{0}'.format(host)
+        ctx.verify_mode = ssl.CERT_REQUIRED
+        ctx.check_hostname = True
+        try:
+            self.sock = ctx.wrap_socket(psock, server_hostname=host)
+        except ssl.SSLError:
+            errmsg = 'Error validating certificate on deployer (try `osdeploy initialize -t` on the deployment server {0})\n'.format(host)
+            sys.stderr.write(errmsg)
+            if self.errout:
+                self.errout.write(errmsg)
+                self.errout.flush()
+            sys.exit(1)
+
+    def grab_url(self, url, data=None, returnrsp=False):
+        return self.grab_url_with_status(url, data, returnrsp)[1]
+
+    def grab_url_with_status(self, url, data=None, returnrsp=False):
+        if data:
+            method = 'POST'
+        else:
+            method = 'GET'
+        authed = False
+        while not authed:
+            authed = True
+            self.request(method, url, data, headers=self.stdheaders)
+            rsp = self.getresponse()
+            if rsp.status >= 200 and rsp.status < 300:
+                if returnrsp:
+                    return rsp.status, rsp
+                else:
+                    return rsp.status, rsp.read()
+            if rsp.status == 401:
+                authed = False
+                rsp.read()
+                with open('/etc/confluent/confluent.apikey', 'w+') as akfile:
+                    akfile.write('')
+                self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(
+                    self.node, [self.host], errout=self.errout)
+        raise Exception(rsp.read())
+
+if __name__ == '__main__':
+    data = None
+    usejson = False
+    if '-j' in sys.argv:
+        usejson = True
+    if len(sys.argv) == 1:
+        HTTPSClient()
+        sys.exit(0)
+    try:
+        outbin = sys.argv.index('-o')
+        sys.argv.pop(outbin)
+        outbin = sys.argv.pop(outbin)
+    except ValueError:
+        outbin = None
+    try:
+        waitfor = sys.argv.index('-w')
+        sys.argv.pop(waitfor)
+        waitfor = int(sys.argv.pop(waitfor))
+    except ValueError:
+        waitfor = None
+    try:
+        data = sys.argv.index('-d')
+        sys.argv.pop(data)
+        data = sys.argv.pop(data)
+    except ValueError:
+        data = None
+    try:
+        errout = sys.argv.index('-e')
+        sys.argv.pop(errout)
+        errout = sys.argv.pop(errout)
+    except ValueError:
+        errout = None
+    if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
+        data = open(sys.argv[-1]).read()
+    if outbin:
+        with open(outbin, 'ab+') as outf:
+            reader = HTTPSClient(usejson=usejson, errout=errout).grab_url(
+                sys.argv[1], data, returnrsp=True)
+            chunk = reader.read(16384)
+            while chunk:
+                outf.write(chunk)
+                chunk = reader.read(16384)
+        sys.exit(0)
+    if waitfor:
+        client = HTTPSClient(usejson, errout=errout)
+        status = 201
+        while status != waitfor:
+            status, rsp = client.grab_url_with_status(sys.argv[1], data)
+            sys.stdout.write(rsp.decode())
+    else:
+        sys.stdout.write(HTTPSClient(usejson, errout=errout).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -1,163 +0,0 @@
-#!/usr/bin/python
-try:
-    import http.client as client
-except ImportError:
-    import httplib as client
-import os
-import socket
-import subprocess
-import ssl
-import sys
-
-def get_apikey(nodename, mgr):
-    sealnew = True
-    if os.path.exists('/etc/confluent/confluent.apikey'):
-        return open('/etc/confluent/confluent.apikey').read().strip()
-    apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
-    if not isinstance(apikey, str):
-        apikey = apikey.decode('utf8')
-    if apikey.startswith('SEALED:'):
-        sealnew = False
-        with open('/etc/confluent/confluent.sealedapikey', 'w+') as apiout:
-            apiout.write(apikey[7:])
-        with open('/etc/confluent/confluent.sealedapikey') as inp:
-            sp = subprocess.Popen(['/usr/bin/clevis-decrypt-tpm2'],
-                stdin=inp, stdout=subprocess.PIPE)
-            apikey = sp.communicate()[0]
-            if not isinstance(apikey, str):
-                apikey = apikey.decode('utf8')
-    with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
-        apiout.write(apikey)
-    if sealnew and os.path.exists('/usr/bin/clevis-encrypt-tpm2'):
-        try:
-            with open('/etc/confluent/confluent.apikey') as apin:
-                sealed = subprocess.check_output(
-                    ['/usr/bin/clevis-encrypt-tpm2', '{}'], stdin=apin)
-            print(HTTPSClient().grab_url('/confluent-api/self/saveapikey', sealed).decode())
-        except Exception:
-            sys.stderr.write('Unable to persist API key through TPM2 sealing\n')
-    apikey = apikey.strip()
-    os.chmod('/etc/confluent/confluent.apikey', 0o600)
-    return apikey
-
-class HTTPSClient(client.HTTPConnection, object):
-    def __init__(self, json=False, port=443):
-        self.stdheaders = {}
-        info = open('/etc/confluent/confluent.info').read().split('\n')
-        host = None
-        mgtiface = None
-        havedefault = '0'
-        for line in info:
-            if line.startswith('NODENAME:'):
-                node = line.split(' ')[1]
-                self.stdheaders['CONFLUENT_NODENAME'] = node
-            if line.startswith('MANAGER:') and not host:
-                host = line.split(' ')[1]
-            if line.startswith('EXTMGRINFO:'):
-                extinfo = line.split(' ')[1]
-                extinfo = extinfo.split('|')
-                if not mgtiface:
-                    host, mgtiface, havedefault = extinfo[:3]
-                if havedefault == '0' and extinfo[2] == '1':
-                    host, mgtiface, havedefault = extinfo[:3]
-        if '%' in host:
-            ifidx = host.split('%', 1)[1]
-            with open('/tmp/confluent.ifidx', 'w+') as ifout:
-                ifout.write(ifidx)
-        if json:
-            self.stdheaders['ACCEPT'] = 'application/json'
-        try:
-            info = open('/etc/confluent/confluent.deploycfg').read().split('\n')
-        except Exception:
-            info = None
-        if info:
-            for line in info:
-                if line.startswith('deploy_server: '):
-                    host = line.split(': ', 1)[1]
-                    break
-        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, host)
-        if mgtiface:
-            self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
-        client.HTTPConnection.__init__(self, host, port)
-        self.connect()
-
-    def set_header(self, key, val):
-        self.stdheaders[key] = val
-
-    def connect(self):
-        addrinf = socket.getaddrinfo(self.host, self.port)[0]
-        psock = socket.socket(addrinf[0])
-        psock.connect(addrinf[4])
-        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-        ctx.load_verify_locations('/etc/confluent/ca.pem')
-        host = self.host.split('%', 1)[0]
-        if '[' not in host and ':' in host:
-            self.stdheaders['Host'] = '[{0}]'.format(host)
-        else:
-            self.stdheaders['Host'] = '{0}'.format(host)
-        ctx.verify_mode = ssl.CERT_REQUIRED
-        ctx.check_hostname = True
-        self.sock = ctx.wrap_socket(psock, server_hostname=host)
-
-    def grab_url(self, url, data=None, returnrsp=False):
-        return self.grab_url_with_status(url, data, returnrsp)[1]
-
-    def grab_url_with_status(self, url, data=None, returnrsp=False):
-        if data:
-            method = 'POST'
-        else:
-            method = 'GET'
-        self.request(method, url, data, headers=self.stdheaders)
-        rsp = self.getresponse()
-        if rsp.status >= 200 and rsp.status < 300:
-            if returnrsp:
-                return rsp.status, rsp
-            else:
-                return rsp.status, rsp.read()
-        raise Exception(rsp.read())
-
-if __name__ == '__main__':
-    data = None
-    json = False
-    if '-j' in sys.argv:
-        json = True
-    if len(sys.argv) == 1:
-        HTTPSClient()
-        sys.exit(0)
-    try:
-        outbin = sys.argv.index('-o')
-        sys.argv.pop(outbin)
-        outbin = sys.argv.pop(outbin)
-    except ValueError:
-        outbin = None
-    try:
-        waitfor = sys.argv.index('-w')
-        sys.argv.pop(waitfor)
-        waitfor = int(sys.argv.pop(waitfor))
-    except ValueError:
-        waitfor = None
-    try:
-        data = sys.argv.index('-d')
-        sys.argv.pop(data)
-        data = sys.argv.pop(data)
-    except ValueError:
-        data = None
-    if outbin:
-        with open(outbin, 'ab+') as outf:
-            reader = HTTPSClient(json=json).grab_url(
-                sys.argv[1], data, returnrsp=True)
-            chunk = reader.read(16384)
-            while chunk:
-                outf.write(chunk)
-                chunk = reader.read(16384)
-        sys.exit(0)
-    if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
-        data = open(sys.argv[-1]).read()
-    if waitfor:
-        client = HTTPSClient(json=json)
-        status = 201
-        while status != waitfor:
-            status, rsp = client.grab_url_with_status(sys.argv[1], data)
-            sys.stdout.write(rsp.decode())
-    else:
-        sys.stdout.write(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/common/profile/scripts/confignet

@@ -0,0 +1,355 @@
+#!/usr/bin/python
+
+import glob
+import json
+import os
+import socket
+import sys
+import time
+import shlex
+import subprocess
+try:
+    from importlib.machinery import SourceFileLoader
+    def load_source(mod, path):
+        return SourceFileLoader(mod, path).load_module()
+except ImportError:
+    from imp import load_source
+
+try:
+    apiclient = load_source('apiclient', '/opt/confluent/bin/apiclient')
+except IOError:
+    apiclient = load_source('apiclient', '/etc/confluent/apiclient')
+
+def add_lla(iface, mac):
+    pieces = mac.split(':')
+    initbyte = int(pieces[0], 16) ^ 2
+    lla = 'fe80::{0:x}{1}:{2}ff:fe{3}:{4}{5}/64'.format(initbyte, pieces[1], pieces[2], pieces[3], pieces[4], pieces[5])
+    try:
+        subprocess.check_call(['ip', 'addr', 'add', 'dev', iface, lla, 'scope', 'link'])
+    except Exception:
+        return None
+    return lla
+
+#cli = apiclient.HTTPSClient(json=True)
+#c = cli.grab_url_with_status('/confluent-api/self/netcfg')
+def add_missing_llas():
+    #NetworkManager goes out of its way to suppress ipv6 lla, so will just add some
+    added = {}
+    linkinfo = subprocess.check_output(['ip', '-br', 'l']).decode('utf8')
+    ifaces = {}
+    for line in linkinfo.split('\n'):
+        line = line.strip().split()
+        if not line or 'LOOPBACK' in line[-1] or 'NO-CARRIER' in line[-1]:
+            continue
+        if 'UP' not in line[-1]:
+            subprocess.call(['ip', 'link', 'set', line[0], 'up'])
+        ifaces[line[0]] = line[2]
+    ips = {}
+    ipinfo = subprocess.check_output(['ip', '-br', '-6', 'a']).decode('utf8')
+    for line in ipinfo.split('\n'):
+        line = line.strip().split(None, 2)
+        if not line:
+            continue
+        ips[line[0]] = line[2]
+    for iface in ifaces:
+        for addr in ips.get(iface, '').split():
+            if addr.startswith('fe80::'):
+                break
+        else:
+            newlla = add_lla(iface, ifaces[iface])
+            if newlla:
+                added[iface] = newlla
+    return added
+
+def rm_tmp_llas(tmpllas):
+    for iface in tmpllas:
+        subprocess.check_call(['ip', 'addr', 'del', 'dev', iface, tmpllas[iface]])
+
+def await_tentative():
+    maxwait = 10
+    while b'tentative' in subprocess.check_output(['ip', 'a']):
+        if maxwait == 0:
+            break
+        maxwait -= 1
+        time.sleep(1)
+
+def map_idx_to_name():
+    map = {}
+    devtype = {}
+    prevdev = None
+    for line in subprocess.check_output(['ip', 'l']).decode('utf8').splitlines():
+        if line.startswith(' ') and 'link/' in line:
+            typ = line.split()[0].split('/')[1]
+            devtype[prevdev] = typ if type != 'ether' else 'ethernet'
+        if line.startswith(' '):
+            continue
+        idx, iface, rst = line.split(':', 2)
+        prevdev = iface.strip()
+        rst = rst.split()
+        try:
+            midx = rst.index('master')
+            continue
+        except ValueError:
+            pass
+        idx = int(idx)
+        iface = iface.strip()
+        map[idx] = iface
+    return map, devtype
+
+
+def get_interface_name(iname, settings):
+    explicitname = settings.get('interface_names', None)
+    if explicitname:
+        return explicitname
+    if settings.get('current_nic', False):
+        return iname
+    return None
+
+class WickedManager(object):
+    def __init__(self):
+        self.teamidx = 0
+        self.read_connections()
+
+    def read_connections(self):
+        self.cfgbydev = {}
+        for ifcfg in glob.glob('/etc/sysconfig/network/ifcfg-*'):
+            devname = ifcfg.replace('/etc/sysconfig/network/ifcfg-', '')
+            if devname == 'lo':
+                continue
+            currcfg = {}
+            self.cfgbydev[devname] = currcfg
+            for cfg in open(ifcfg).read().splitlines():
+                cfg = cfg.split('#', 1)[0]
+                kv = ' '.join(shlex.split(cfg)).split('=', 1)
+                if len(kv) != 2:
+                    continue
+                k, v = kv
+                k = k.strip()
+                v = v.strip()
+                currcfg[k] = v
+
+    def apply_configuration(self, cfg):
+        stgs = cfg['settings']
+        ipcfg = 'STARTMODE=auto\n'
+        routecfg = ''
+        bootproto4 = stgs.get('ipv4_method', 'none')
+        bootproto6 = stgs.get('ipv6_method', 'none')
+        if bootproto4 == 'dhcp' and bootproto6 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp\n'
+        elif bootproto4 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp4\n'
+        elif bootproto6 == 'dhcp':
+            ipcfg += 'BOOTPROTO=dhcp6\n'
+        else:
+            ipcfg += 'BOOTPROTO=static\n'
+        if stgs.get('ipv4_address', None):
+            ipcfg += 'IPADDR=' + stgs['ipv4_address'] + '\n'
+        v4gw = stgs.get('ipv4_gateway', None)
+        if stgs.get('ipv6_address', None):
+            ipcfg += 'IPADDR_V6=' + stgs['ipv6_address'] + '\n'
+        v6gw = stgs.get('ipv6_gateway', None)
+        cname = None
+        if len(cfg['interfaces']) > 1:  # creating new team
+            if not stgs.get('team_mode', None):
+                sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces'])))
+                return
+            if not stgs.get('connection_name', None):
+                stgs['connection_name'] = 'bond{0}'.format(self.teamidx)
+                self.teamidx += 1
+            cname = stgs['connection_name']
+            with open('/etc/sysconfig/network/ifcfg-{0}'.format(cname), 'w') as teamout:
+                teamout.write(ipcfg)
+                if stgs['team_mode'] == 'lacp':
+                    stgs['team_mode'] = '802.3ad'
+                teamout.write("BONDING_MODULE_OPTS='mode={0} miimon=100'\nBONDING_MASTER=yes\n".format(stgs['team_mode']))
+                idx = 1
+                for iface in cfg['interfaces']:
+                    subprocess.call(['wicked', 'ifdown', iface])
+                    try:
+                        os.remove('/etc/sysconfig/network/ifcfg-{0}'.format(iface))
+                        os.remove('/etc/sysconfig/network/ifroute-{0}'.format(iface))
+                    except OSError:
+                        pass
+                    teamout.write('BONDING_SLAVE{0}={1}\n'.format(idx, iface))
+                    idx += 1
+        else:
+            cname = list(cfg['interfaces'])[0]
+            priorcfg = self.cfgbydev.get(cname, {})
+            for cf in priorcfg:
+                if cf.startswith('TEAM_'):
+                    ipcfg += '{0}={1}\n'.format(cf, priorcfg[cf])
+            with open('/etc/sysconfig/network/ifcfg-{0}'.format(cname), 'w') as iout:
+                iout.write(ipcfg)
+        if v4gw:
+            routecfg += 'default {0} - {1}\n'.format(v4gw, cname)
+        if v6gw:
+            routecfg += 'default {0} - {1}\n'.format(v6gw, cname)
+        if routecfg:
+            with open('/etc/sysconfig/network/ifroute-{0}'.format(cname), 'w') as routeout:
+                routeout.write(routecfg)
+        subprocess.call(['wicked', 'ifup', cname])
+
+
+class NetworkManager(object):
+    def __init__(self, devtypes):
+        self.connections = {}
+        self.uuidbyname = {}
+        self.uuidbydev = {}
+        self.connectiondetail = {}
+        self.read_connections()
+        self.teamidx = 0
+        self.devtypes = devtypes
+
+    def read_connections(self):
+        self.connections = {}
+        self.uuidbyname = {}
+        self.uuidbydev = {}
+        self.connectiondetail = {}
+        ci = subprocess.check_output(['nmcli', '-t', 'c']).decode('utf8')
+        for inf in ci.splitlines():
+            n, u, t, dev = inf.split(':')
+            if n == 'NAME':
+                continue
+            if dev == '--':
+                dev = None
+            self.uuidbyname[n] = u
+            if dev:
+                self.uuidbydev[dev] = u
+            self.connections[u] = {'name': n, 'uuid': u, 'type': t, 'dev': dev}
+            deats = {}
+            for deat in subprocess.check_output(['nmcli', 'c', 's', u]).decode('utf8').splitlines():
+                k, v = deat.split(':', 1)
+                v = v.strip()
+                if v == '--':
+                    continue
+                if '(default)' in v:
+                    continue
+                deats[k] = v
+            self.connectiondetail[u] = deats
+
+
+    def add_team_member(self, team, member):
+        bondcfg = {}
+        if member in self.uuidbydev:
+            myuuid = self.uuidbydev[member]
+            deats = self.connectiondetail[myuuid]
+            currteam = deats.get('connection.master', None)
+            if currteam == team:
+                return
+            for stg in ('ipv4.dhcp-hostname', 'ipv4.dns', 'ipv6.dns', 'ipv6.dhcp-hostname'):
+                if deats.get(stg, None):
+                    bondcfg[stg] = deats[stg]
+        if member in self.uuidbyname:
+            subprocess.check_call(['nmcli', 'c', 'del', self.uuidbyname[member]])
+        subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team-slave', 'master', team, 'con-name', member, 'connection.interface-name', member])
+        if bondcfg:
+            args = []
+            for parm in bondcfg:
+                args.append(parm)
+                args.append(bondcfg[parm])
+            subprocess.check_call(['nmcli', 'c', 'm', team] + args)
+
+    def apply_configuration(self, cfg):
+        cmdargs = {}
+        stgs = cfg['settings']
+        cmdargs['ipv6.method'] = stgs.get('ipv6_method', 'link-local')
+        if stgs.get('ipv6_address', None):
+            cmdargs['ipv6.addresses'] = stgs['ipv6_address']
+        cmdargs['ipv4.method'] = stgs.get('ipv4_method', 'disabled')
+        if stgs.get('ipv4_address', None):
+            cmdargs['ipv4.addresses'] = stgs['ipv4_address']
+        if stgs.get('ipv4_gateway', None):
+            cmdargs['ipv4.gateway'] = stgs['ipv4_gateway']
+        if stgs.get('ipv6_gateway', None):
+            cmdargs['ipv6.gateway'] = stgs['ipv6_gateway']
+        if len(cfg['interfaces']) > 1:  # team time.. should be..
+            if not cfg['settings'].get('team_mode', None):
+                sys.stderr.write("Warning, multiple interfaces ({0}) without a team_mode, skipping setup\n".format(','.join(cfg['interfaces'])))
+                return
+            if not cfg['settings'].get('connection_name', None):
+                cfg['settings']['connection_name'] = 'team{0}'.format(self.teamidx)
+                self.teamidx += 1
+            cname = cfg['settings']['connection_name']
+            cargs = []
+            for arg in cmdargs:
+                cargs.append(arg)
+                cargs.append(cmdargs[arg])
+            subprocess.check_call(['nmcli', 'c', 'add', 'type', 'team', 'con-name', cname, 'connection.interface-name', cname, 'team.runner', stgs['team_mode']] + cargs)
+            for iface in cfg['interfaces']:
+                self.add_team_member(cname, iface)
+            subprocess.check_call(['nmcli', 'c', 'u', cname])
+        else:
+            cname = stgs.get('connection_name', None)
+            iname = list(cfg['interfaces'])[0]
+            if not cname:
+                cname = iname
+            u = self.uuidbyname.get(cname, None)
+            cargs = []
+            for arg in cmdargs:
+                cargs.append(arg)
+                cargs.append(cmdargs[arg])
+            if u:
+                cmdargs['connection.interface-name'] = iname
+                subprocess.check_call(['nmcli', 'c', 'm', u] + cargs)
+                subprocess.check_call(['nmcli', 'c', 'u', u])
+            else:
+                subprocess.check_call(['nmcli', 'c', 'add', 'type', self.devtypes[iname], 'con-name', cname, 'connection.interface-name', iname] + cargs)
+
+
+
+if __name__ == '__main__':
+    havefirewall = subprocess.call(['systemctl', 'status', 'firewalld'])
+    havefirewall = havefirewall == 0
+    if havefirewall:
+        subprocess.check_call(['systemctl', 'stop', 'firewalld'])
+    tmpllas = add_missing_llas()
+    await_tentative()
+    idxmap, devtypes = map_idx_to_name()
+    netname_to_interfaces = {}
+    myaddrs = apiclient.get_my_addresses()
+    srvs, _ = apiclient.scan_confluents()
+    doneidxs = set([])
+    for srv in srvs:
+        s = socket.create_connection((srv, 443))
+        myname = s.getsockname()
+        s.close()
+        if len(myname) == 4:
+            curridx = myname[-1]
+        else:
+            myname = myname[0]
+            myname = socket.inet_pton(socket.AF_INET, myname)
+            for addr in myaddrs:
+                if myname == addr[1].tobytes():
+                    curridx = addr[-1]
+        if curridx in doneidxs:
+            continue
+        status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+        nc = json.loads(nc)
+        iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
+        if iname:
+            for iname in iname.split(','):
+                if 'default' in netname_to_interfaces:
+                    netname_to_interfaces['default']['interfaces'].add(iname)
+                else:
+                    netname_to_interfaces['default'] = {'interfaces': set([iname]), 'settings': nc['default']}
+        for netname in nc.get('extranets', {}):
+            uname = '_' + netname
+            iname = get_interface_name(idxmap[curridx], nc['extranets'][netname])
+            if iname:
+                for iname in iname.split(','):
+                    if uname in netname_to_interfaces:
+                        netname_to_interfaces[uname]['interfaces'].add(iname)
+                    else:
+                        netname_to_interfaces[uname] = {'interfaces': set([iname]), 'settings': nc['extranets'][netname]}
+        doneidxs.add(curridx)
+    rm_tmp_llas(tmpllas)
+    if os.path.exists('/usr/bin/nmcli'):
+        nm = NetworkManager(devtypes)
+    elif os.path.exists('/usr/sbin/wicked'):
+        nm = WickedManager()
+    for netn in netname_to_interfaces:
+        nm.apply_configuration(netname_to_interfaces[netn])
+    if havefirewall:
+        subprocess.check_call(['systemctl', 'start', 'firewalld'])
+    await_tentative()
+

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -4,8 +4,9 @@ Release: 1
 Summary: OS Deployment support for confluent
 
 License: Apache2
-URL: http://hpc.lenovo.com/
+URL: https://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
+Source1: confluent_el9bin.tar.xz
 BuildArch: noarch
 Requires: confluent_ipxe mtools tar
 BuildRoot: /tmp
@@ -17,20 +18,46 @@ This contains support utilities for enabling deployment of x86_64 architecture s
 %define debug_package %{nil}
 
 %prep
-%setup -n confluent_osdeploy
+%setup -n confluent_osdeploy -a 1
 
 %build
 mkdir -p opt/confluent/bin
+mkdir -p stateless-bin
 cd utils
 make all
-cp copernicus clortho autocons ../opt/confluent/bin
+cp confluent_imginfo copernicus clortho autocons ../opt/confluent/bin
+cp start_root urlmount ../stateless-bin/
 cd ..
-for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do
+ln -s el8 el9
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04 ubuntu22.04 coreos el9; do
     mkdir ${os}out
     cd ${os}out
-    cp -a ../opt .
+    if [ -d ../${os}bin ]; then 
+        cp -a ../${os}bin/opt .
+    else
+        cp -a ../opt .
+    fi
     cp -a ../${os}/initramfs/* .
-    cp -a ../common/* .
+    cp -a ../common/initramfs/* .
+    find . | cpio -H newc -o > ../addons.cpio
+    mv ../addons.cpio .
+    cd ..
+done
+for os in el7 el8 suse15 el9 ubuntu20.04; do
+    mkdir ${os}disklessout
+    cd ${os}disklessout
+    if [ -d ../${os}bin ]; then
+        cp -a ../${os}bin/opt .
+    else
+        cp -a ../opt .
+    fi
+    cp -a ../${os}-diskless/initramfs/* .
+    cp -a ../common/initramfs/* .
+    if [ -d ../${os}bin ]; then 
+        cp -a ../${os}bin/stateless-bin/* opt/confluent/bin
+    else
+        cp -a ../stateless-bin/* opt/confluent/bin
+    fi
     find . | cpio -H newc -o > ../addons.cpio
     mv ../addons.cpio .
     cd ..
@@ -39,7 +66,7 @@ mkdir esxi7out
 cd esxi7out
 cp -a ../opt .
 cp -a ../esxi7/initramfs/* .
-cp -a ../common/* .
+cp -a ../common/initramfs/* .
 chmod +x bin/* opt/confluent/bin/*
 tar zcvf ../addons.tgz *
 mv ../addons.tgz .
@@ -48,13 +75,28 @@ cp -a esxi7out esxi6out
 cp -a esxi7 esxi6
 
 %install
-for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
+mkdir -p %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+cp LICENSE %{buildroot}/opt/confluent/share/licenses/confluent_osdeploy/
+for os in rhvh4 el7 el8 el9 genesis suse15 ubuntu20.04 ubuntu22.04 esxi6 esxi7 coreos; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
-    find %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles -name .gitignore -exec rm -f {} +
+    for targ in %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles/*; do
+        cp -a common/profile/* $targ
+    done
+    if [ -d ${os}disklessout ]; then
+        mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs
+        mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/profiles
+        cp ${os}disklessout/addons.* %{buildroot}/opt/confluent/lib/osdeploy/${os}-diskless/initramfs
+        cp -a ${os}-diskless/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os-diskless/profiles
+        for targ in %{buildroot}/opt/confluent/lib/osdeploy/$os-diskless/profiles/*; do
+            cp -a common/profile/* $targ
+        done
+    fi
 done
+find %{buildroot}/opt/confluent/lib/osdeploy/ -name .gitignore -exec rm -f {} +
 
 %files
 /opt/confluent/lib/osdeploy
+%license /opt/confluent/share/licenses/confluent_osdeploy/LICENSE

confluent_osdeploy/coreos/initramfs/etc/systemd/system/initrd-root-fs.target.requires/confluent-rootfs.service

@@ -0,0 +1 @@
+/usr/lib/systemd/system/confluent-rootfs.service

confluent_osdeploy/coreos/initramfs/opt/confluent/bin/initconfluent.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+TRIES=0
+oum=$(umask)
+umask 0077
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
+umask $oum
+if [ -e /dev/disk/by-label/CNFLNT_IDNT ]; then
+    tmnt=$(mktemp -d)
+    tcfg=$(mktemp)
+    mount /dev/disk/by-label/CNFLNT_IDNT $tmnt
+    cd $tmnt
+    deploysrvs=$(sed -n '/^deploy_servers:/, /^[^-]/p' cnflnt.yml |grep ^-|sed -e 's/^- //'|grep -v :)
+    nodename=$(grep ^nodename: cnflnt.yml|awk '{print $2}')
+    sed -n '/^net_cfgs:/, /^[^- ]/{/^[^- ]/!p}' cnflnt.yml |sed -n '/^-/, /^-/{/^-/!p}'| sed -e 's/^[- ]*//'> $tcfg
+    autoconfigmethod=$(grep ^ipv4_method: $tcfg)
+    autoconfigmethod=${autoconfigmethod#ipv4_method: }
+    if [ "$autoconfigmethod" = "dhcp" ]; then
+        /usr/libexec/nm-initrd-generator ip=:dhcp
+    else
+        v4addr=$(grep ^ipv4_address: $tcfg)
+        v4addr=${v4addr#ipv4_address: }
+        v4gw=$(grep ^ipv4_gateway: $tcfg)
+        v4gw=${v4gw#ipv4_gateway: }
+        if [ "$v4gw" = "null" ]; then
+            v4gw=""
+        fi
+        v4nm=$(grep ipv4_netmask: $tcfg)
+        v4nm=${v4nm#ipv4_netmask: }
+        /usr/libexec/nm-initrd-generator ip=$v4addr::$v4gw:$v4nm:$hostname::none
+    fi
+    NetworkManager --configure-and-quit=initrd --no-daemon
+    hmackeyfile=$(mktemp)
+    echo -n $(grep ^apitoken: cnflnt.yml|awk '{print $2}') > $hmackeyfile
+    cd -
+    umount $tmnt
+    passfile=$(mktemp)
+    passcrypt=$(mktemp)
+    hmacfile=$(mktemp)
+    ln -s /opt/confluent/bin/clortho /opt/confluent/bin/genpasshmac
+    /opt/confluent/bin/genpasshmac $passfile $passcrypt $hmacfile $hmackeyfile
+    for deployer in $deploysrvs; do
+        if curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_CRYPTHMAC: $(cat $hmacfile)" -d@$passcrypt -k https://$deployer/confluent-api/self/registerapikey; then
+            cp $passfile /etc/confluent/confluent.apikey
+            confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+            curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$deployer/confluent-api/self/profileprivate/pending/config.ign > /config.ign
+            [ -s /config.ign ] || rm /config.ign
+            confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+            confluent_profile=${confluent_profile#profile: }
+            curl -sf https://$deployer/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C /
+            exit 0
+        fi
+    done
+fi
+cd /sys/class/net
+echo "Searching for confluent deployment server...." > /dev/console
+while ! grep ^NODE /etc/confluent/confluent.info; do
+    while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+        TRIES=$((TRIES + 1))
+        for currif in *; do
+            ip link set $currif up
+        done
+        /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+    done
+    if ! grep ^NODE /etc/confluent/confluent.info; then
+        echo 'Current net config:' > /dev/console
+        ip -br a > /dev/console
+        exit 1
+    fi
+done
+echo "Found confluent deployment services on local network" > /dev/console
+cd /
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+cat /tls/*.pem > /etc/confluent/ca.pem
+confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
+echo "Connecting to confluent server: $confluent_mgr" > /dev/console
+if [[ $confluent_mgr == *%* ]]; then
+    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+needseal=1
+oldumask=$(umask)
+umask 0077
+while [ -z "$confluent_apikey" ]; do
+    /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
+    if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
+        needseal=0
+        sed -e s/^SEALED:// /etc/confluent/confluent.apikey | clevis-decrypt-tpm2 > /etc/confluent/confluent.apikey.decrypt
+        mv /etc/confluent/confluent.apikey.decrypt /etc/confluent/confluent.apikey
+    fi
+    confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+    if [ -z "$confluent_apikey" ]; then
+        echo "Unable to acquire node api key, no TPM2 sealed nor fresh token available, retrying..." > /dev/console
+        sleep 10
+    fi
+done
+if [[ $confluent_mgr == *:* ]]; then
+    confluent_mgr="[$confluent_mgr]"
+fi
+if [ $needseal == 1 ]; then
+    sealed=$(echo $confluent_apikey | clevis-encrypt-tpm2 {})
+    if [ ! -z "$sealed" ]; then
+        curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d $sealed https://$confluent_mgr/confluent-api/self/saveapikey
+    fi
+fi
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/profileprivate/pending/config.ign > /config.ign
+[ -s /config.ign ] || rm /config.ign
+
+umask $oldumask
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+confluent_profile=${confluent_profile#profile: }
+
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+
+if [ -e /lib/nm-lib.sh ]; then
+    . /lib/nm-lib.sh
+    nm_generate_connections
+    sed -i 's/method=disabled/method=link-local/' /run/NetworkManager/system-connections/*.nmconnection
+   if [ -f /run/NetworkManager/system-connections/$ifname.nmconnection ]; then
+        rm /run/NetworkManager/system-connections/default_connection.nmconnection
+    fi
+    if [[ "$ifname" == ib* ]]; then
+        sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
+        if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
+            echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
+        fi
+    fi
+fi
+#cat /proc/cmdline /etc/cmdline.d/01-confluent.conf | tr '\n' ' ' > /run/fakecmdline
+#chcon system_u:object_r:proc_t:s0 /run/fakecmdline
+#mount -o bind /run/fakecmdline /proc/cmdline
+
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/rootfs.img | rdcore stream-hash /etc/coreos-live-want-rootfs | bsdtar -xf - -C /

confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,10 @@
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+    fi
+fi

confluent_osdeploy/coreos/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,93 @@
+#!/bin/sh
+
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/)
+if [[ $confluent_mgr = *:* ]]; then
+    confluent_mgr=[$confluent_mgr]
+fi
+
+mkdir -p /sysroot/var/home/core/.ssh/
+chmod 700 /sysroot/var/home/core/.ssh
+cat /ssh/*.rootpubkey >> /sysroot/var/home/core/.ssh/authorized_keys
+chmod 600 /sysroot/var/home/core/.ssh/authorized_keys
+cp -a /etc/confluent /sysroot/etc/confluent
+mkdir -p /sysroot/opt/confluent/bin/
+cat > /sysroot/opt/confluent/bin/setssh.sh << 'EOF'
+#!/bin/sh
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/)
+if [[ $confluent_mgr = *:* ]]; then
+    confluent_mgr=[$confluent_mgr]
+fi
+
+for i in  /etc/ssh/ssh_host*key.pub; do
+    certname=${i/.pub/-cert.pub}
+    curl --cacert /etc/confluent/ca.pem -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$i https://$confluent_mgr/confluent-api/self/sshcert > $certname
+    if ! grep HostKey ${i%.pub} /etc/ssh/sshd_config > /dev/null; then
+        echo HostKey ${i%.pub} >> /etc/ssh/sshd_config
+    fi
+    if ! grep HostCertificate $certname /etc/ssh/sshd_config > /dev/null; then
+        echo HostCertificate $certname >> /etc/ssh/sshd_config
+    fi
+done
+EOF
+
+chmod 755 /sysroot/opt/confluent/bin/setssh.sh
+chcon system_u:object_r:bin_t:s0 /sysroot/opt/confluent/bin/setssh.sh
+
+
+cat > /sysroot/etc/systemd/system/confluent-ssh.service << EOF
+[Service]
+Type=oneshot
+ExecStart=/opt/confluent/bin/setssh.sh
+
+[Unit]
+PartOf=sshd.service
+Before=sshd.service
+After=sshd-keygen.target
+Wants=sshd-keygen.target
+Requires=network-online.target
+After=network-online.target
+
+
+[Install]
+WantedBy=sshd.service
+EOF
+
+mkdir -p /sysroot/etc/systemd/system/sshd.service.wants
+ln -s /etc/systemd/system/confluent-ssh.service /sysroot/etc/systemd/system/sshd.service.wants/
+ln -s /etc/systemd/system/confluent-onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/
+
+
+cat > /sysroot/opt/confluent/bin/onboot.sh << 'EOF'
+#!/bin/sh
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info| head -n 1| awk '{print $2}' | sed -e s/%/%25/)
+if [[ $confluent_mgr = *:* ]]; then
+    confluent_mgr=[$confluent_mgr]
+fi
+
+curl -sf https://$confluent_mgr/confluent-public/os/${confluent_profile}/scripts/onboot.sh > /tmp/onboot.sh
+[ -s /tmp/onboot.sh ] && . /tmp/onboot.sh
+exit 0
+EOF
+chmod 755 /sysroot/opt/confluent/bin/onboot.sh
+chcon system_u:object_r:bin_t:s0 /sysroot/opt/confluent/bin/onboot.sh
+
+
+cat > /sysroot/etc/systemd/system/confluent-onboot.service << EOF
+[Service]
+Type=oneshot
+ExecStart=/opt/confluent/bin/onboot.sh
+
+[Unit]
+Requires=sshd.service
+After=sshd.service
+Requires=network-online.target
+After=network-online.target
+EOF
+chcon -h system_u:object_r:systemd_unit_file_t:s0 /sysroot/etc/systemd/system/confluent-ssh.service /sysroot/etc/systemd/system/sshd.service.wants/confluent-ssh.service /sysroot/etc/systemd/system/confluent-onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/confluent-onboot.service
+
+cp -a /opt/confluent/bin/* /sysroot/opt/confluent/bin/
+chcon system_u:object_r:bin_t:s0 /sysroot/opt/confluent/bin/*

confluent_osdeploy/coreos/initramfs/usr/lib/systemd/system/confluent-rootfs.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Confluent initialization
+DefaultDependencies=false
+
+After=basic.target
+# Network is enabled here
+After=dracut-initqueue.service
+Before=ignition-setup-user.service
+Before=coreos-ignition-setup-user.service
+Before=coreos-livepxe-rootfs.service
+
+
+
+# If we fail, the boot will fail.  Be explicit about it.
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/opt/confluent/bin/initconfluent.sh
+

confluent_osdeploy/coreos/profiles/default/initprofile.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+profname=$(basename $2)
+if file -L $1/images/ignition.img | grep gzip > /dev/null; then
+    mkdir -p /var/lib/confluent/private/os/${profname}/pending/
+    cd /var/lib/confluent/private/os/${profname}/pending/
+    zcat $1/images/ignition.img | cpio -dumiv
+fi
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution && \
+mkdir -p $2/boot/efi/boot/ && \
+ln -s $1/images/pxeboot/rootfs.img $2/ && \
+(mcopy -i $1/images/efiboot.img ::efi/redhat/grubx64.efi $2/boot/efi/boot/ || \
+mcopy -i $1/images/efiboot.img ::efi/boot/grubx64.efi $2/boot/efi/boot/) && \
+mcopy -i $1/images/efiboot.img ::efi/boot/bootx64.efi $2/boot/efi/boot/

confluent_osdeploy/coreos/profiles/default/profile.yaml

@@ -0,0 +1,5 @@
+label: RedHat CoreOS %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal
+#the above boots and will have ignition agent phone home and await instructions
+#kernelargs: quiet random.trust_cpu=on ignition.firstboot ignition.platform.id=metal  coreos.inst=yes coreos.inst.install_dev=sda 
+#the above runs the installer to write to disk

confluent_osdeploy/el7-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -0,0 +1,245 @@
+get_remote_apikey() {
+    while [ -z "$confluent_apikey" ]; do
+        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
+        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
+            # we don't support remote sealed api keys anymore
+            echo > /etc/confluent/confluent.apikey
+        fi
+        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+        if [ -z "$confluent_apikey" ]; then
+            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
+            sleep 10
+        elif [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_startauthsession ]; then
+            tmpdir=$(mktemp -d)
+            cd $tmpdir
+            tpm2_startauthsession --session=session.ctx
+            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
+            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
+            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
+            tpm2_evictcontrol -Q -c data.ctx
+            tpm2_flushcontext session.ctx
+            cd - > /dev/null
+            rm -rf $tmpdir
+        fi
+    done
+}
+root=1
+rootok=1
+netroot=confluent
+clear
+mkdir -p /etc/ssh
+mkdir -p /var/tmp/
+mkdir -p /var/empty/sshd
+mkdir -p /etc/confluent
+sed -i '/^root:x/d' /etc/passwd
+echo root:x:0:0::/:/bin/bash >> /etc/passwd
+echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
+
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    autocons=${autocons##*/}
+    echo "Automatic console configured for $autocons"
+fi
+echo "Initializing confluent diskless environment"
+echo -n "udevd: "
+/usr/lib/systemd/systemd-udevd --daemon
+echo -n "Loading drivers..."
+udevadm trigger
+udevadm trigger --type=devices --action=add
+udevadm settle
+modprobe ib_ipoib
+modprobe ib_umad
+modprobe hfi1
+modprobe mlx5_ib
+echo "done"
+cat > /etc/ssh/sshd_config << EOF
+Port 2222
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
+PermitRootLogin yes
+AuthorizedKeysFile      .ssh/authorized_keys
+EOF
+mkdir /root/.ssh
+mkdir /.ssh
+cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
+cp /root/.ssh/authorized_keys /.ssh/
+cat /tls/*.pem > /etc/confluent/ca.pem
+mkdir -p /etc/pki/tls/certs
+cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
+TRIES=0
+oldumask=$(umask)
+umask 0077
+tpmdir=$(mktemp -d)
+cd $tpmdir
+lasthdl=""
+if [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_getcap ]; then
+    for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
+        tpm2_startauthsession --policy-session --session=session.ctx
+        tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+        unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
+        tpm2_flushcontext session.ctx
+        if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
+            confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
+            echo $confluent_apikey > /etc/confluent/confluent.apikey
+            if [ -n "$lasthdl" ]; then
+                tpm2_evictcontrol -c $lasthdl
+            fi
+            lasthdl=$hdl
+        fi
+    done
+fi
+cd - > /dev/null
+rm -rf $tpmdir
+touch /etc/confluent/confluent.info
+cd /sys/class/net
+echo -n "Scanning for network configuration..."
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    TRIES=$((TRIES + 1))
+    for i in *; do
+        ip link set $i up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+hostname $nodename
+confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
+fi
+if [[ $confluent_mgr == *%* ]]; then
+    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+
+ready=0
+while [ $ready = "0" ]; do
+    get_remote_apikey
+    if [[ $confluent_mgr == *:* ]]; then
+        confluent_mgr="[${confluent_mgr/\%/%25}]"
+    fi
+    tmperr=$(mktemp)
+    curl -gsSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
+    if grep 401 $tmperr > /dev/null; then
+        confluent_apikey=""
+        if [ -n "$lasthdl" ]; then
+            tpm2_evictcontrol -c $lasthdl
+        fi
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    else
+        ready=1
+    fi
+    rm $tmperr
+done
+if [ ! -z "$autocons" ] && grep textconsole: true /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+if [ -c /dev/tpmrm0 -a -x /usr/bin/tpm2_pcrextend ]; then
+    tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
+fi
+umask $oldumask
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo -n "Attempting to use dhcp to bring up $ifname..."
+    dhclient $ifname
+    echo "Complete:"
+    ip addr show dev $ifname
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#prefix: }
+    echo "Setting up $ifname as static at $v4addr/$v4nm"
+    ip addr add dev $ifname $v4addr/$v4nm
+    if [ ! -z "$v4gw" ]; then
+        ip route add default via $v4gw
+    fi
+    mkdir -p /run/NetworkManager/system-connections
+    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+type=ethernet
+autoconnect-retries=1
+EOC
+    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+[ethernet]
+mac-address-blacklist=
+
+[ipv4]
+EOC
+    echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v4gw" ]; then
+        echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+may-fail=false
+method=manual
+
+[ipv6]
+addr-gen-mode=eui64
+method=auto
+
+[proxy]
+EOC
+fi
+chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+echo -n "Initializing ssh..."
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    curl -gsf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_urls=""
+for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
+    if [[ $addr == *:* ]]; then
+        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
+    else
+        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    fi
+done
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+mkdir -p /etc/confluent
+curl -gsf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+source_remote imageboot.sh

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
+logshowpid=$!
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    touch /etc/confluent/firstboot.ran
+
+    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+
+    run_remote firstboot.custom
+    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+    run_remote_parts firstboot.d
+
+    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+    run_remote_config firstboot.d
+fi
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service
+rm /etc/confluent/firstboot.ran
+kill $logshowpid

confluent_osdeploy/el7-diskless/profiles/default/scripts/functions

@@ -0,0 +1,196 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/el7-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,93 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1,376 @@
+#!/usr/bin/python3
+import glob
+import json
+import os
+import re
+import time
+import shutil
+import socket
+import stat
+import struct
+import sys
+import subprocess
+
+def get_next_part_meta(img, imgsize):
+    if img.tell() == imgsize:
+        return None
+    pathlen = struct.unpack('!H', img.read(2))[0]
+    mountpoint = img.read(pathlen).decode('utf8')
+    jsonlen = struct.unpack('!I', img.read(4))[0]
+    metadata = json.loads(img.read(jsonlen).decode('utf8'))
+    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip filesystem type
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip orig devname (redundant with json)
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip padding
+    nextlen = struct.unpack('!Q', img.read(8))[0]
+    img.seek(nextlen, 1)  # go to next section
+    return metadata
+
+def get_multipart_image_meta(img):
+    img.seek(0, 2)
+    imgsize = img.tell()
+    img.seek(16)
+    seekamt = img.read(1)
+    img.seek(struct.unpack('B', seekamt)[0], 1)
+    partinfo = get_next_part_meta(img, imgsize)
+    while partinfo:
+        yield partinfo
+        partinfo = get_next_part_meta(img, imgsize)
+
+def get_image_metadata(imgpath):
+    with open(imgpath, 'rb') as img:
+        header = img.read(16)
+        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
+            for md in get_multipart_image_meta(img):
+                yield md
+        else:
+            raise Exception('Installation from single part image not supported')
+
+class PartedRunner():
+    def __init__(self, disk):
+        self.disk = disk
+
+    def run(self, command):
+        command = command.split()
+        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
+        return subprocess.check_output(command).decode('utf8')
+
+def fixup(rootdir, vols):
+    devbymount = {}
+    for vol in vols:
+        devbymount[vol['mount']] = vol['targetdisk']
+    fstabfile = os.path.join(rootdir, 'etc/fstab')
+    with open(fstabfile) as tfile:
+        fstab = tfile.read().split('\n')
+    while not fstab[0]:
+        fstab = fstab[1:]
+    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
+        os.unlink(os.path.join(rootdir, '.autorelabel'))
+    with open(fstabfile, 'w') as tfile:
+        for tab in fstab:
+            entry = tab.split()
+            if tab.startswith('#ORIGFSTAB#'):
+                if entry[1] in devbymount:
+                    targetdev = devbymount[entry[1]]
+                    if targetdev.startswith('/dev/localstorage/'):
+                        entry[0] = targetdev
+                    else:
+                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
+                        uuid = uuid.strip()
+                        entry[0] = 'UUID={}'.format(uuid)
+                elif entry[2] == 'swap':
+                    entry[0] = '/dev/mapper/localstorage-swap'
+                entry[0] = entry[0].ljust(42)
+                entry[1] = entry[1].ljust(16)
+                entry[3] = entry[3].ljust(28)
+                tab = '\t'.join(entry)
+            tfile.write(tab + '\n')
+    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
+        nameout.write(socket.gethostname() + '\n')
+    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
+    policy = None
+    if os.path.exists(selinuxconfig):
+        with open(selinuxconfig) as cfgin:
+            sec = cfgin.read().split('\n')
+            for l in sec:
+                l = l.split('#', 1)[0]
+                if l.startswith('SELINUXTYPE='):
+                    _, policy = l.split('=')
+    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
+        os.unlink(sshkey)
+    for sshkey in glob.glob('/etc/ssh/*_key*'):
+        newkey = os.path.join(rootdir, sshkey[1:])
+        shutil.copy2(sshkey, newkey)
+        finfo = os.stat(sshkey)
+        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/sysconfig/network-scripts/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/NetworkManager/system-connections/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob('/run/NetworkManager/system-connections/*'):
+        newcfg = ifcfg.split('/')[-1]
+        newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
+        shutil.copy2(ifcfg, newcfg)
+    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
+    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    if policy:
+        sys.stdout.write('Applying SELinux labeling...')
+        sys.stdout.flush()
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
+        sys.stdout.write('Done\n')
+        sys.stdout.flush()
+    for metafs in ('proc', 'sys', 'dev'):
+        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+        defgrub = defgrubin.read().split('\n')
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+        for gline in defgrub:
+            gline = gline.split()
+            newline = []
+            for ent in gline:
+                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    continue
+                newline.append(ent)
+            defgrubout.write(' '.join(newline) + '\n')
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
+    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    newroot = None
+    with open('/etc/shadow') as shadowin:
+        shents = shadowin.read().split('\n')
+        for shent in shents:
+            shent = shent.split(':')
+            if not shent:
+                continue
+            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
+                newroot = shent[1]
+    if newroot:
+        shlines = None
+        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
+            shlines = oshadow.read().split('\n')
+        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
+            for line in shlines:
+                if line.startswith('root:'):
+                    line = line.split(':')
+                    line[1] = newroot
+                    line = ':'.join(line)
+                oshadow.write(line + '\n')
+    partnum = None
+    targblock = None
+    for vol in vols:
+        if vol['mount'] == '/boot/efi':
+            targdev = vol['targetdisk']
+            partnum = re.search('(\d+)$', targdev).group(1)
+            targblock = re.search('(.*)\d+$', targdev).group(1)
+    if targblock:
+        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
+        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
+        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    #other network interfaces
+
+
+def had_swap():
+    with open('/etc/fstab') as tabfile:
+        tabs = tabfile.read().split('\n')
+        for tab in tabs:
+            tab = tab.split()
+            if len(tab) < 3:
+                continue
+            if tab[2] == 'swap':
+                return True
+    return False
+
+def install_to_disk(imgpath):
+    lvmvols = {}
+    deftotsize = 0
+    mintotsize = 0
+    deflvmsize = 0
+    minlvmsize = 0
+    biggestsize = 0
+    biggestfs = None
+    plainvols = {}
+    allvols = []
+    swapsize = 0
+    if had_swap():
+        with open('/proc/meminfo') as meminfo:
+            swapsize = meminfo.read().split('\n')[0]
+        swapsize = int(swapsize.split()[1])
+        if swapsize < 2097152:
+            swapsize = swapsize * 2
+        elif swapsize > 8388608 and swapsize < 67108864:
+            swapsize = swapsize * 0.5
+        elif swapsize >= 67108864:
+            swapsize = 33554432
+        swapsize = int(swapsize * 1024)
+    deftotsize = swapsize
+    mintotsize = swapsize
+    for fs in get_image_metadata(imgpath):
+        allvols.append(fs)
+        deftotsize += fs['initsize']
+        mintotsize += fs['minsize']
+        if fs['initsize'] > biggestsize:
+            biggestfs = fs
+            biggestsize = fs['initsize']
+        if fs['device'].startswith('/dev/mapper'):
+            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
+            deflvmsize += fs['initsize']
+            minlvmsize += fs['minsize']
+        else:
+            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+    with open('/tmp/installdisk') as diskin:
+        instdisk = diskin.read()
+    instdisk = '/dev/' + instdisk
+    parted = PartedRunner(instdisk)
+    dinfo = parted.run('unit s print')
+    dinfo = dinfo.split('\n')
+    sectors = 0
+    sectorsize = 0
+    for inf in dinfo:
+        if inf.startswith('Disk {0}:'.format(instdisk)):
+            _, sectors = inf.split(': ')
+            sectors = int(sectors.replace('s', ''))
+        if inf.startswith('Sector size (logical/physical):'):
+            _, sectorsize = inf.split(':')
+            sectorsize = sectorsize.split('/')[0]
+            sectorsize = sectorsize.replace('B', '')
+            sectorsize = int(sectorsize)
+    # for now, only support resizing/growing the largest partition
+    minexcsize = deftotsize - biggestfs['initsize']
+    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
+    minsectors = mintotsize // sectorsize
+    if sectors < (minsectors + 65536):
+        raise Exception('Disk too small to fit image')
+    biggestsectors = sectors - (minexcsize // sectorsize)
+    biggestsize = sectorsize * biggestsectors
+    parted.run('mklabel gpt')
+    curroffset = 2048
+    for volidx in sorted(plainvols):
+        vol = plainvols[volidx]
+        if vol is not biggestfs:
+            size = vol['initsize'] // sectorsize
+        else:
+            size = biggestsize // sectorsize
+        size += 2047 - (size % 2048)
+        end = curroffset + size
+        if end > sectors:
+            end = sectors
+        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
+        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        curroffset += size + 1
+    if not lvmvols:
+        if swapsize:
+            swapsize = swapsize // sectorsize
+            swapsize += 2047 - (size % 2048)
+            end = curroffset + swapsize
+            if end > sectors:
+                end = sectors
+            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
+            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+    else:
+        parted.run('mkpart lvm {}s 100%'.format(curroffset))
+        lvmpart = instdisk + '{}'.format(volidx + 1)
+        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
+        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
+        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
+        vginfo = vginfo.split('\n')
+        pesize = 0
+        pes = 0
+        for infline in vginfo:
+            infline = infline.split()
+            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
+                pesize = int(infline[2])
+            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
+                pes = int(infline[4])
+        takeaway = swapsize // pesize
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                continue
+            takeaway += vol['initsize'] // pesize
+            takeaway += 1
+        biggestextents = pes - takeaway
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                extents = biggestextents
+            else:
+                extents = vol['initsize'] // pesize
+                extents += 1
+            if vol['mount'] == '/':
+                lvname = 'root'
+            else:
+                lvname = vol['mount'].replace('/', '_')
+            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
+            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
+        if swapsize:
+            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
+            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
+        os.makedirs('/run/imginst/targ')
+        for vol in allvols:
+            with open(vol['targetdisk'], 'wb') as partition:
+                partition.write(b'\x00' * 1 * 1024 * 1024)
+            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
+            source = vol['mount'].replace('/', '_')
+            source = '/run/imginst/sources/' + source
+            blankfsstat = os.statvfs('/run/imginst/targ')
+            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
+            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
+            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
+                stillrunning = copier.poll()
+                lastprogress = 0.0
+                while stillrunning is None:
+                    currfsstat = os.statvfs('/run/imginst/targ')
+                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
+                    currused -= blankused
+                    with open('/proc/meminfo') as meminf:
+                        for line in meminf.read().split('\n'):
+                            if line.startswith('Dirty:'):
+                                _, dirty, _ = line.split()
+                                dirty = int(dirty) * 1024
+                    progress = (currused - dirty) / vol['minsize']
+                    if progress < lastprogress:
+                        progress = lastprogress
+                    if progress > 0.99:
+                        progress = 0.99
+                    lastprogress = progress
+                    progress = progress * 100
+                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                    sys.stdout.flush()
+                    time.sleep(0.5)
+                    stillrunning = copier.poll()
+                if stillrunning != 0:
+                    raise Exception("Error copying volume")
+                with subprocess.Popen(['sync']) as syncrun:
+                    stillrunning = syncrun.poll()
+                    while stillrunning is None:
+                        with open('/proc/meminfo') as meminf:
+                            for line in meminf.read().split('\n'):
+                                if line.startswith('Dirty:'):
+                                    _, dirty, _ = line.split()
+                                    dirty = int(dirty) * 1024
+                        progress = (vol['minsize'] - dirty) / vol['minsize']
+                        if progress < lastprogress:
+                            progress = lastprogress
+                        if progress > 0.99:
+                            progress = 0.99
+                        lastprogress = progress
+                        progress = progress * 100
+                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                        sys.stdout.flush()
+                        time.sleep(0.5)
+                        stillrunning = syncrun.poll()
+                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+            subprocess.check_call(['umount', '/run/imginst/targ'])
+        for vol in allvols:
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
+        fixup('/run/imginst/targ', allvols)
+
+
+if __name__ == '__main__':
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el7-diskless/profiles/default/scripts/imageboot.sh

@@ -0,0 +1,128 @@
+. /lib/dracut-lib.sh
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+    mount -t tmpfs untethered /mnt/remoteimg
+    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+else
+    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    modprobe fuse
+    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
+fi
+/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
+modprobe loop
+loopdev=$(losetup -f)
+export mountsrc=$loopdev
+losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
+if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+        sleep 1
+    done
+    cipher=$(head -n 1 /tmp/rootimg.key)
+    key=$(tail -n 1 /tmp/rootimg.key)
+    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
+    len=$(((len-4096)/512))
+    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
+    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
+    mountsrc=/dev/mapper/cryptimg
+fi
+
+if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
+    mount -o ro $mountsrc /mnt/remote
+elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
+    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
+    . /tmp/setupmount.sh
+    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
+    . /tmp/mountparts.sh
+fi
+
+
+#mount -t tmpfs overlay /mnt/overlay
+modprobe zram
+memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+echo $memtot > /sys/block/zram0/disksize
+mkfs.xfs /dev/zram0 > /dev/null
+mount -o discard /dev/zram0 /mnt/overlay
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /mnt/overlay/upper /mnt/overlay/work
+    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+    done
+fi
+mkdir -p /sysroot/etc/ssh
+mkdir -p /sysroot/etc/confluent
+mkdir -p /sysroot/root/.ssh
+cp /root/.ssh/* /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cp /etc/confluent/* /sysroot/etc/confluent/
+cp /etc/ssh/*key* /sysroot/etc/ssh/
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
+done
+
+mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
+if [ ! -z "$autocons" ]; then
+    autocons=${autocons%,*}
+    mkdir -p /run/systemd/generator/getty.target.wants
+    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
+fi
+while [ ! -e /sysroot/sbin/init ]; do
+    echo "Failed to access root filesystem or it is missing /sbin/init"
+    echo "System should be accessible through ssh at port 2222 with the appropriate key"
+    while [ ! -e /sysroot/sbin/init ]; do
+        sleep 1
+    done
+done
+rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
+rootpassword=${rootpassword#rootpassword: }
+if [ "$rootpassword" = "null" ]; then
+    rootpassword=""
+fi
+
+if [ ! -z "$rootpassword" ]; then
+    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
+fi
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
+sshconf=/sysroot/etc/ssh/ssh_config
+if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
+chmod 640 /sysroot/etc/ssh/*_key
+chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/ update-ca-trust
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+mkdir -p /sysroot/opt/confluent/bin
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+chmod +x /sysroot/opt/confluent/bin/onboot.sh
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
+cp /etc/confluent/functions /sysroot/etc/confluent/functions
+if grep installtodisk /proc/cmdline > /dev/null; then
+    . /etc/confluent/functions
+    run_remote installimage
+    exec reboot -f
+fi
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+exec /opt/confluent/bin/start_root

confluent_osdeploy/el7-diskless/profiles/default/scripts/installimage

@@ -0,0 +1,43 @@
+#!/bin/bash
+. /etc/confluent/functions
+# the image will be used to deploy itself
+# provide both access to image (for parsing metadata)
+# and existing mounts of image (to take advantage of caching)
+mount -o bind /sys /sysroot/sys
+mount -o bind /dev /sysroot/dev
+mount -o bind /proc /sysroot/proc
+mount -o bind /run /sysroot/run
+
+
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /sysroot/run/imginst/sources/_
+    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
+        srcname=${srcmount#/dev/mapper/mproot}
+        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
+        mkdir -p /sysroot/run/imginst/sources/$srcname
+        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
+    done
+fi
+cd /sysroot/run
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+if [ ! -f /sysroot/tmp/installdisk ]; then
+    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
+    while [ ! -f /sysroot/tmp/installdisk ]; do
+        sleep 1
+    done
+fi
+lvm vgchange -a n
+udevadm control -e
+chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
+chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
+echo "Port 22" >> /etc/ssh/sshd_config
+echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
+echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+
+chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
+chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Confluent onboot hook
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/onboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/onboot.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script is executed on each boot as it is
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-onboot.log
+exec 2>> /var/log/confluent/confluent-onboot.log
+chmod 600 /var/log/confluent/confluent-onboot.log
+tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
+logshowpid=$!
+
+run_remote_python syncfileclient
+run_remote_python confignet
+
+run_remote onboot.custom
+# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
+run_remote_parts onboot.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
+run_remote_config onboot.d
+
+#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+kill $logshowpid

confluent_osdeploy/el7-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# This script is executed 'chrooted' into a cloned disk target before rebooting
+#
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+tail -f /var/log/confluent/confluent-post.log > /dev/console &
+logshowpid=$!
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+mkdir -p /opt/confluent/bin
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+systemctl enable firstboot
+selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}')
+if [ ! -z "$selinuxpolicy" ]; then
+    setfiles /etc/selinux/${selinuxpolicy}/contexts/files/file_contexts /etc/
+fi
+run_remote_python syncfileclient
+run_remote post.custom
+# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+
+kill $logshowpid
+

confluent_osdeploy/el7-diskless/profiles/default/scripts/syncfileclient

@@ -0,0 +1,279 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+import pwd
+import grp
+try:
+    from importlib.machinery import SourceFileLoader
+    def load_source(mod, path):
+        return SourceFileLoader(mod, path).load_module()
+except ImportError:
+    from imp import load_source
+
+try:
+    apiclient = load_source('apiclient', '/opt/confluent/bin/apiclient')
+except IOError:
+    apiclient = load_source('apiclient', '/etc/confluent/apiclient')
+
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger(object):
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid < minid or uid > maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            lastrsp = ''
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
+    finally:
+        shutil.rmtree(tmpdir)
+        shutil.rmtree(appendoncedir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/el7-diskless/profiles/default/syncfiles

@@ -0,0 +1,29 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
+# This file lists files to synchronize or merge to the deployed systems from the deployment server
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
+# /some/path/hosts -> /etc/hosts
+
+# If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
+# /etc/hosts
+
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
+# It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
+MERGE:
+# /etc/passwd
+# /etc/group

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -18,4 +18,6 @@ mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
 done
-cp /opt/confluent/bin/apiclient /sysroot/etc/confluent
+mkdir -p /sysroot/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent/

confluent_osdeploy/el7/profiles/default/kickstart

@@ -35,10 +35,11 @@ reboot
 %packages
 #-kernel-uek # This can opt out of the UEK for the relevant distribution
 chrony
-rsync
-python
-tar
 pciutils
+python
+rsync
+tar
+-iwl*-firmware
 %include /tmp/addonpackages
 %end
 
@@ -51,7 +52,7 @@ pciutils
 
 %pre
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pre.sh > /tmp/preinst.sh
 . /tmp/preinst.sh
 %end
@@ -59,7 +60,7 @@ curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/pr
 %post --nochroot
 mkdir -p /mnt/sysimage/etc/confluent
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/prechroot.sh > /tmp/postinst.sh
 . /tmp/postinst.sh
 
@@ -76,7 +77,7 @@ systemctl enable firstboot
 chgrp ssh_keys /etc/ssh/ssh*key
 restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /opt/confluent/bin/firstboot.sh
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/post.sh > /tmp/postinst.sh
 . /tmp/postinst.sh
 %end

confluent_osdeploy/el7/profiles/default/scripts/configbmc

@@ -267,7 +267,7 @@ def get_lan_channel(s):
         medtype = int(rsp[1]) & 0b1111111
         if medtype not in (4, 6):
             continue
-        rsp = s.raw_command(0xc, 2, bytearray([2, chan, 5, 0, 0]))
+        rsp = s.raw_command(0xc, 2, bytearray([chan, 5, 0, 0]))
         if rsp.get('code', 1) == 0:
             return chan
     return 1

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -7,12 +7,16 @@
 
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 confluent_apikey=$(cat /etc/confluent/confluent.apikey)
-confluent_mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
 export nodename confluent_mgr confluent_profile
 . /etc/confluent/functions
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
 exec >> /var/log/confluent/confluent-firstboot.log
 exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
 tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
 logshowpid=$!
 

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -1,6 +1,50 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
 function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
     if [ -z "$confluent_mgr" ]; then
         confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
     fi
     if [ -z "$confluent_profile" ]; then
         confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
@@ -19,30 +63,42 @@ fetch_remote() {
 }
 
 source_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    scriptlist=$(/usr/libexec/platform-python /etc/confluent/apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
     for script in $scriptlist; do
         source_remote $1/$script
     done
+    rm -rf $confluentscripttmpdir
     unset confluentscripttmpdir
 }
 
 run_remote_parts() {
-    confluentscripttmpdir=$(mktemp -d)
-    scriptlist=$(/usr/libexec/platform-python /etc/confluent/apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
     for script in $scriptlist; do
         run_remote $1/$script
     done
+    rm -rf $confluentscripttmpdir
     unset confluentscripttmpdir
 }
 
 source_remote() {
     set_confluent_vars
+    unsettmpdir=0
     echo
     echo '---------------------------------------------------------------------------'
     echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
     fi
     echo Sourcing from $confluentscripttmpdir
     cd $confluentscripttmpdir
@@ -51,19 +107,27 @@ source_remote() {
     chmod +x $1
     cmd=$1
     shift
-    source $cmd
+    source ./$cmd
     cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
     return $retcode
 }
 
 run_remote() {
     requestedcmd="'$*'"
+    unsettmpdir=0
     set_confluent_vars
     echo
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
     if [ -z "$confluentscripttmpdir" ]; then
-        confluentscripttmpdir=$(mktemp -d)
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
     fi
     echo Executing in $confluentscripttmpdir
     cd $confluentscripttmpdir
@@ -81,6 +145,11 @@ run_remote() {
         echo "$requestedcmd exited with code $retcode"
     fi
     cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
     return $retcode
 }
 
@@ -92,20 +161,18 @@ run_remote_python() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
-    tmpdir=$(mktemp -d)
-    echo Executing in $tmpdir
-    cd $tmpdir
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
     mkdir -p $(dirname $1)
     curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $*
-    else
-        /usr/bin/python $*
-    fi
+    confluentpython $*
     retcode=$?
     echo "'$*' exited with code $retcode"
     cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
     return $retcode
 }
 
@@ -118,16 +185,12 @@ run_remote_config() {
     fi
     echo '---------------------------------------------------------------------------'
     echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
-    if [ -x /usr/libexec/platform-python ]; then
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    else
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
-        /usr/bin/python $apiclient /confluent-api/self/remoteconfig/status -w 204
-    fi
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
     echo
     echo 'Completed remote configuration'
     echo '---------------------------------------------------------------------------'
     return
 }
-
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/el7/profiles/default/scripts/post.sh

@@ -1,7 +1,9 @@
 #!/bin/sh
 mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
 exec >> /var/log/confluent/confluent-post.log
 exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
 tail -f /var/log/confluent/confluent-post.log > /dev/tty &
 logshowpid=$!
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')

confluent_osdeploy/el7/profiles/default/scripts/pre.sh

@@ -13,6 +13,7 @@ if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
 fi
 exec >> /tmp/confluent-pre.log
 exec 2>> /tmp/confluent-pre.log
+chmod 600 /tmp/confluent-pre.log
 tail -f /tmp/confluent-pre.log > /dev/tty &
 logshowpid=$!
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
@@ -78,5 +79,9 @@ if [ -e /tmp/installdisk -a ! -e /tmp/partitioning ]; then
     echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
     echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
 fi
-python /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
+if [ -f /opt/confluent/bin/apiclient ]; then
+    python /opt/confluent/bin/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
+else
+    python /etc/confluent/apiclient /confluent-public/os/$confluent_profile/kickstart.custom -o /tmp/kickstart.custom
+fi
 kill $logshowpid

confluent_osdeploy/el7/profiles/default/scripts/prechroot.sh

@@ -10,8 +10,12 @@
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 export confluent_mgr confluent_profile nodename
 cp -a /etc/confluent /mnt/sysimage/etc
+mkdir -p /mnt/sysimage/opt/confluent/bin
+cp /opt/confluent/bin/apiclient /mnt/sysimage/opt/confluent/bin/
 chmod -R og-rwx /mnt/sysimage/etc/confluent
 cp /tmp/functions /mnt/sysimage/etc/confluent/
+hostnamectl set-hostname $nodename
+cp /etc/hostname /mnt/sysimage/etc/hostname
 . /tmp/functions
 if [ -f /tmp/cryptboot ]; then
     cp /tmp/cryptboot /mnt/sysimage/tmp/

confluent_osdeploy/el7/profiles/default/scripts/syncfileclient

@@ -3,6 +3,8 @@ import importlib
 import tempfile
 import json
 import os
+import pwd
+import grp
 import shutil
 try:
     from importlib.machinery import SourceFileLoader
@@ -31,7 +33,7 @@ def partitionhostsline(line):
     names = names.split()
     return ipaddr, names, comment
 
-class HostMerger:
+class HostMerger(object):
     def __init__(self):
         self.byip = {}
         self.byname = {}
@@ -81,8 +83,12 @@ class HostMerger:
     def write_out(self, targetfile):
         while not self.targlines[-1]:
             self.targlines = self.targlines[:-1]
+            if not self.targlines:
+                break
         while not self.sourcelines[-1]:
             self.sourcelines = self.sourcelines[:-1]
+            if not self.sourcelines:
+                break
         with open(targetfile, 'w') as hosts:
             for line in self.targlines:
                 hosts.write(line + '\n')
@@ -135,12 +141,14 @@ class CredMerger:
                 except ValueError:
                     continue
                 if targfile:
-                    if uid <= minid or uid >= maxid:
+                    if uid < minid or uid > maxid:
                         self.targdata.append(line)
                     else:
                         self.discardnames[name] = 1
                 else:
-                    if uid >= minid and uid <= maxid:
+                    if name[0] in ('+', '#', '@'):
+                        self.sourcedata.append(line)
+                    elif uid >= minid and uid <= maxid:
                         self.sourcedata.append(line)
 
     def read_shadow(self, source):
@@ -173,6 +181,8 @@ class CredMerger:
                     shadout.write(line + '\n')
                 for line in self.sourcedata:
                     name, _ = line.split(':', 1)
+                    if name[0] in ('+', '#', '@'):
+                        continue
                     if name in self.shadowednames:
                         continue
                     shadout.write(name + ':!:::::::\n')
@@ -188,17 +198,35 @@ class CredMerger:
                         continue
                     shadout.write(name + ':!::\n')
 
+def appendonce(basepath, filename):
+    with open(filename, 'rb') as filehdl:
+        thedata = filehdl.read()
+    targname = filename.replace(basepath, '')
+    try:
+        with open(targname, 'rb') as filehdl:
+            targdata = filehdl.read()
+    except IOError:
+        targdata = b''
+    if thedata in targdata:
+        return
+    with open(targname, 'ab') as targhdl:
+        targhdl.write(thedata)
+
 def synchronize():
     tmpdir = tempfile.mkdtemp()
+    appendoncedir = tempfile.mkdtemp()
     try:
         ac = apiclient.HTTPSClient()
-        data = json.dumps({'merge': tmpdir})
+        data = json.dumps({'merge': tmpdir, 'appendonce': appendoncedir})
         status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
         if status == 202:
+            lastrsp = ''
             while status != 204:
                 status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
                 if not isinstance(rsp, str):
                     rsp = rsp.decode('utf8')
+                if status == 200:
+                    lastrsp = rsp
             pendpasswd = os.path.join(tmpdir, 'etc/passwd')
             if os.path.exists(pendpasswd):
                 cm = CredMerger()
@@ -217,6 +245,30 @@ def synchronize():
                 cm.read_source(pendhosts)
                 cm.read_target('/etc/hosts')
                 cm.write_out('/etc/hosts')
+            for dirn in os.walk(appendoncedir):
+                for filen in dirn[2]:
+                    appendonce(appendoncedir, os.path.join(dirn[0], filen))
+            if lastrsp:
+                lastrsp = json.loads(lastrsp)
+                opts = lastrsp.get('options', {})
+                for fname in opts:
+                    uid = -1
+                    gid = -1
+                    for opt in opts[fname]:
+                        if opt == 'owner':
+                            try:
+                                uid = pwd.getpwnam(opts[fname][opt]['name']).pw_uid
+                            except KeyError:
+                                uid = opts[fname][opt]['id']
+                        elif opt == 'group':
+                            try:
+                                gid = grp.getgrnam(opts[fname][opt]['name']).gr_gid
+                            except KeyError:
+                                gid = opts[fname][opt]['id']
+                        elif opt == 'permissions':
+                            os.chmod(fname, int(opts[fname][opt], 8))
+                    if uid != -1 or gid != -1:
+                        os.chown(fname, uid, gid)
     finally:
         shutil.rmtree(tmpdir)
 

confluent_osdeploy/el7/profiles/default/syncfiles

@@ -1,11 +1,27 @@
+# It is advised to avoid /var/lib/confluent/public as a source for syncing.  /var/lib/confluent/public
+# is served without authentication and thus any sensitive content would be a risk. If wanting to host
+# syncfiles on a common share, it is suggested to have /var/lib/confluent be the share and use some other
+# subdirectory other than public.
+#
+# Syncing is performed as the 'confluent' user, so all source files must be accessible by the confluent user.
+#
 # This file lists files to synchronize or merge to the deployed systems from the deployment server
-# To specify taking /some/path/hosts on the deploymennt server and duplicating it to /etc/hosts:
+# To specify taking /some/path/hosts on the deployment server and duplicating it to /etc/hosts:
+# Note particularly the use of '->' to delineate source from target.
 # /some/path/hosts -> /etc/hosts
 
 # If wanting to simply use the same path for source and destinaiton, the -> may be skipped:
 # /etc/hosts
 
-# Entries under MERGE will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
+# More function is available, for example to limit the entry to run only on n1 through n8, and to set
+# owner, group, and permissions in octal notation:
+# /example/source -> n1-n8:/etc/target (owner=root,group=root,permissions=600)
+
+# Entries under APPENDONCE: will be added to specified target, only if the target does not already
+# contain the data in the source already in its entirety. This allows append in a fashion that
+# is friendly to being run repeatedly
+
+# Entries under MERGE: will attempt to be intelligently merged. This supports /etc/group and /etc/passwd
 # Any supporting entries in /etc/shadow or /etc/gshadow are added automatically, with password disabled
 # It also will not inject 'system' ids (under 1,000 usually) as those tend to be local and rpm managed.
 MERGE:

confluent_osdeploy/el8-diskless/initramfs/usr/lib/dracut/hooks/cmdline/10-confluentdiskless.sh

@@ -0,0 +1,253 @@
+get_remote_apikey() {
+    while [ -z "$confluent_apikey" ]; do
+        /opt/confluent/bin/clortho $nodename $confluent_mgr > /etc/confluent/confluent.apikey
+        if grep ^SEALED: /etc/confluent/confluent.apikey > /dev/null; then
+            # we don't support remote sealed api keys anymore
+            echo > /etc/confluent/confluent.apikey
+        fi
+        confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+        if [ -z "$confluent_apikey" ]; then
+            echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..."
+            if [ ! -z "$autoconsdev" ]; then echo "Unable to acquire node api key, set deployment.apiarmed=once on node '$nodename', retrying..." > $autoconsdev; fi
+            sleep 10
+        elif [ -c /dev/tpmrm0 ]; then
+            tmpdir=$(mktemp -d)
+            cd $tmpdir
+            tpm2_startauthsession --session=session.ctx
+            tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+            tpm2_createprimary -G ecc -Q --key-context=prim.ctx
+            (echo -n "CONFLUENT_APIKEY:";cat /etc/confluent/confluent.apikey) | tpm2_create -Q --policy=pcr15.sha256.policy --public=data.pub --private=data.priv -i - -C prim.ctx
+            tpm2_load -Q --parent-context=prim.ctx --public=data.pub --private=data.priv --name=confluent.apikey --key-context=data.ctx
+            tpm2_evictcontrol -Q -c data.ctx
+            tpm2_flushcontext session.ctx
+            cd - > /dev/null
+            rm -rf $tmpdir
+        fi
+    done
+}
+root=1
+rootok=1
+netroot=confluent
+clear
+mkdir -p /etc/ssh
+mkdir -p /var/tmp/
+mkdir -p /var/empty/sshd
+mkdir -p /etc/confluent
+sed -i '/^root:x/d' /etc/passwd
+echo root:x:0:0::/:/bin/bash >> /etc/passwd
+echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
+
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    autoconsdev=${autocons%,*}
+    autocons=${autocons##*/}
+    echo "Automatic console configured for $autocons"
+fi
+echo "Initializing confluent diskless environment"
+echo -n "udevd: "
+/usr/lib/systemd/systemd-udevd --daemon
+echo -n "Loading drivers..."
+udevadm trigger
+udevadm trigger --type=devices --action=add
+udevadm settle
+modprobe ib_ipoib
+modprobe ib_umad
+modprobe hfi1
+modprobe mlx5_ib
+echo "done"
+cat > /etc/ssh/sshd_config << EOF
+Port 2222
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
+PermitRootLogin yes
+AuthorizedKeysFile      .ssh/authorized_keys
+EOF
+mkdir /root/.ssh
+mkdir /.ssh
+cat /ssh/*pubkey > /root/.ssh/authorized_keys 2>/dev/null
+cp /root/.ssh/authorized_keys /.ssh/
+cat /tls/*.pem > /etc/confluent/ca.pem
+mkdir -p /etc/pki/tls/certs
+cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
+TRIES=0
+oldumask=$(umask)
+umask 0077
+tpmdir=$(mktemp -d)
+cd $tpmdir
+lasthdl=""
+if [ -c /dev/tpmrm0 ]; then
+    for hdl in $(tpm2_getcap handles-persistent|awk '{print $2}'); do
+        tpm2_startauthsession --policy-session --session=session.ctx
+        tpm2_policypcr -Q --session=session.ctx --pcr-list="sha256:15" --policy=pcr15.sha256.policy
+        unsealeddata=$(tpm2_unseal --auth=session:session.ctx -Q -c $hdl 2>/dev/null)
+        tpm2_flushcontext session.ctx
+        if [[ $unsealeddata == "CONFLUENT_APIKEY:"* ]]; then
+            confluent_apikey=${unsealeddata#CONFLUENT_APIKEY:}
+            echo $confluent_apikey > /etc/confluent/confluent.apikey
+            if [ -n "$lasthdl" ]; then
+                tpm2_evictcontrol -c $lasthdl
+            fi
+            lasthdl=$hdl
+        fi
+    done
+fi
+cd - > /dev/null
+rm -rf $tpmdir
+touch /etc/confluent/confluent.info
+cd /sys/class/net
+echo -n "Scanning for network configuration..."
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    TRIES=$((TRIES + 1))
+    for i in *; do
+        ip link set $i up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+hostname $nodename
+confluent_mgr=$(grep '^EXTMGRINFO:.*1$' /etc/confluent/confluent.info | head -n 1 | awk -F': ' '{print $2}' | awk -F'|' '{print $1}')
+if [ -z "$confluent_mgr" ]; then
+    confluent_mgr=$(grep ^MANAGER: /etc/confluent/confluent.info|head -n 1 | awk '{print $2}')
+fi
+if [[ $confluent_mgr == *%* ]]; then
+    echo $confluent_mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    ifidx=$(cat /tmp/confluent.ifidx)
+    ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+    ifname=${ifname%:}
+fi
+
+ready=0
+while [ $ready = "0" ]; do
+    get_remote_apikey
+    if [[ $confluent_mgr == *:* ]]; then
+        confluent_mgr="[$confluent_mgr]"
+    fi
+    tmperr=$(mktemp)
+    curl -sSf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg 2> $tmperr
+    if grep 401 $tmperr > /dev/null; then
+        confluent_apikey=""
+        if [ -n "$lasthdl" ]; then
+            tpm2_evictcontrol -c $lasthdl
+        fi
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    elif grep 'SSL' $tmperr > /dev/null; then
+        confluent_mgr=${confluent_mgr#[}
+        confluent_mgr=${confluent_mgr%]}
+    	echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)'
+	if [ ! -z "$autoconsdev" ]; then echo 'Failure establishing TLS conneection to '$confluent_mgr' (try `osdeploy initialize -t` on the deployment server)' > $autoconsdev; fi
+    	sleep 10
+    else
+        ready=1
+    fi
+    rm $tmperr
+done
+if [ ! -z "$autocons" ] && grep "textconsole: true" /etc/confluent/confluent.deploycfg > /dev/null; then /opt/confluent/bin/autocons -c > /dev/null; fi
+if [ -c /dev/tpmrm0 ]; then
+    tpm2_pcrextend 15:sha256=2fbe96c50dde38ce9cd2764ddb79c216cfbcd3499568b1125450e60c45dd19f2
+fi
+umask $oldumask
+autoconfigmethod=$(grep ^ipv4_method: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo -n "Attempting to use dhcp to bring up $ifname..."
+    dhclient $ifname
+    echo "Complete:"
+    ip addr show dev $ifname
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#prefix: }
+    echo "Setting up $ifname as static at $v4addr/$v4nm"
+    ip addr add dev $ifname $v4addr/$v4nm
+    if [ ! -z "$v4gw" ]; then
+        ip route add default via $v4gw
+    fi
+    mkdir -p /run/NetworkManager/system-connections
+    cat > /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+[connection]
+EOC
+    echo id=${ifname} >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    echo uuid=$(uuidgen) >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+type=ethernet
+autoconnect-retries=1
+EOC
+    echo interface-name=$ifname >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+multi-connect=1
+permissions=
+wait-device-timeout=60000
+
+[ethernet]
+mac-address-blacklist=
+
+[ipv4]
+EOC
+    echo address1=$v4addr/$v4nm >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    if [ ! -z "$v4gw" ]; then
+        echo gateway=$v4gw >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    fi
+    nameserversec=0
+    nameservers=""
+    while read -r entry; do
+        if [ $nameserversec = 1 ]; then
+            if [[ $entry == "-"* ]]; then
+                nameservers="$nameservers"${entry#- }";"
+                continue
+            fi
+        fi
+        nameserversec=0
+        if [ "${entry%:*}" = "nameservers" ]; then
+            nameserversec=1
+            continue
+        fi
+    done < /etc/confluent/confluent.deploycfg
+    echo dns=$nameservers >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+    dnsdomain=${dnsdomain#dnsdomain: }
+    echo dns-search=$dnsdomain >> /run/NetworkManager/system-connections/$ifname.nmconnection
+    cat >> /run/NetworkManager/system-connections/$ifname.nmconnection << EOC
+may-fail=false
+method=manual
+
+[ipv6]
+addr-gen-mode=eui64
+method=auto
+
+[proxy]
+EOC
+fi
+chmod 600 /run/NetworkManager/system-connections/*.nmconnection
+echo -n "Initializing ssh..."
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    curl -sf -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" -d @$pubkey  https://$confluent_mgr/confluent-api/self/sshcert > $certfile
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+confluent_urls=""
+for addr in $(grep ^MANAGER: /etc/confluent/confluent.info|awk '{print $2}'|sed -e s/%/%25/); do
+    if [[ $addr == *:* ]]; then
+        confluent_urls="$confluent_urls $confluent_proto://[$addr]/confluent-public/os/$confluent_profile/rootimg.sfs"
+    else
+        confluent_urls="$confluent_urls $confluent_proto://$addr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    fi
+done
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg| awk '{print $2}')
+mkdir -p /etc/confluent
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
+. /etc/confluent/functions
+source_remote imageboot.sh

confluent_osdeploy/el8-diskless/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/opt/confluent/bin/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el8-diskless/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+exec >> /var/log/confluent/confluent-firstboot.log
+exec 2>> /var/log/confluent/confluent-firstboot.log
+chmod 600 /var/log/confluent/confluent-firstboot.log
+tail -f /var/log/confluent/confluent-firstboot.log > /dev/console &
+logshowpid=$!
+while ! ping -c 1 $confluent_mgr >& /dev/null; do
+	sleep 1
+done
+
+if [ ! -f /etc/confluent/firstboot.ran ]; then
+    touch /etc/confluent/firstboot.ran
+
+    cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+
+    run_remote firstboot.custom
+    # Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+    run_remote_parts firstboot.d
+
+    # Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+    run_remote_config firstboot.d
+fi
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service
+rm /etc/confluent/firstboot.ran
+kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/functions

@@ -0,0 +1,196 @@
+#!/bin/bash
+function test_mgr() {
+    if curl -s https://${1}/confluent-api/ > /dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+function confluentpython() {
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    elif [ -x /usr/bin/python3 ]; then
+        /usr/bin/python3 $*
+    elif [ -x /usr/bin/python ]; then
+        /usr/bin/python $*
+    elif [ -x /usr/bin/python2 ]; then
+        /usr/bin/python2 $*
+    fi
+}
+
+function set_confluent_vars() {
+    if [ -z "$nodename" ]; then
+        nodename=$(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
+    fi
+    if [[ "$confluent_mgr" == *"%"* ]]; then
+        confluent_mgr=""
+    fi
+    if [ -z "$confluent_mgr" ]; then
+        confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+        if ! test_mgr $confluent_mgr; then
+            confluent_mgr=$(grep ^deploy_server_v6: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+            if [[ "$confluent_mgr" = *":"* ]]; then
+                confluent_mgr="[$confluent_mgr]"
+            fi
+        fi
+        if ! test_mgr $confluent_mgr; then
+            BESTMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|1$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            OKMGRS=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info | grep '|0$' | sed -e 's/EXTMGRINFO: //' -e 's/|.*//')
+            for confluent_mgr in $BESTMGRS $OKMGRS; do
+                if [[ $confluent_mgr == *":"* ]]; then
+                    confluent_mgr="[$confluent_mgr]"
+                fi
+                if test_mgr $confluent_mgr; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if [ -z "$confluent_profile" ]; then
+        confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+source_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        source_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+run_remote_parts() {
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    scriptlist=$(confluentpython $apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1/$script
+    done
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+}
+
+source_remote() {
+    set_confluent_vars
+    unsettmpdir=0
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Sourcing $1 from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Sourcing from $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    shift
+    source ./$cmd
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    rm -rf $confluentscripttmpdir
+    return $retcode
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    unsettmpdir=0
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    if [ -z "$confluentscripttmpdir" ]; then
+        confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+        unsettmpdir=1
+    fi
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    fetch_remote $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    if [ $retcode -ne 0 ]; then
+        echo "$requestedcmd exited with code $retcode"
+    fi
+    cd - > /dev/null
+    if [ "$unsettmpdir" = 1 ]; then
+        rm -rf $confluentscripttmpdir
+        unset confluentscripttmpdir
+        unsettmpdir=0
+    fi
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/
+    confluentscripttmpdir=$(mktemp -d /tmp/confluentscripts.XXXXXXXXX)
+    echo Executing in $confluentscripttmpdir
+    cd $confluentscripttmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    confluentpython $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    rm -rf $confluentscripttmpdir
+    unset confluentscripttmpdir
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $confluent_mgr under profile $confluent_profile
+    confluentpython $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    confluentpython $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+#If invoked as a command, use the arguments to actually run a function
+(return 0 2>/dev/null) || $1 "${@:2}"

confluent_osdeploy/el8-diskless/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,93 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+        if os.path.exists('/sys/block/{0}/size'.format(self.name)):
+            with open('/sys/block/{0}/size'.format(self.name), 'r') as sizesrc:
+                self.size = int(sizesrc.read()) * 512
+        if int(self.size) < 536870912:
+            raise Exception("Device too small for install")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el8-diskless/profiles/default/scripts/image2disk.py

@@ -0,0 +1,376 @@
+#!/usr/bin/python3
+import glob
+import json
+import os
+import re
+import time
+import shutil
+import socket
+import stat
+import struct
+import sys
+import subprocess
+
+def get_next_part_meta(img, imgsize):
+    if img.tell() == imgsize:
+        return None
+    pathlen = struct.unpack('!H', img.read(2))[0]
+    mountpoint = img.read(pathlen).decode('utf8')
+    jsonlen = struct.unpack('!I', img.read(4))[0]
+    metadata = json.loads(img.read(jsonlen).decode('utf8'))
+    img.seek(16, 1) # skip the two 64-bit values we don't use, they are in json
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip filesystem type
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip orig devname (redundant with json)
+    nextlen = struct.unpack('!H', img.read(2))[0]
+    img.seek(nextlen, 1) # skip padding
+    nextlen = struct.unpack('!Q', img.read(8))[0]
+    img.seek(nextlen, 1)  # go to next section
+    return metadata
+
+def get_multipart_image_meta(img):
+    img.seek(0, 2)
+    imgsize = img.tell()
+    img.seek(16)
+    seekamt = img.read(1)
+    img.seek(struct.unpack('B', seekamt)[0], 1)
+    partinfo = get_next_part_meta(img, imgsize)
+    while partinfo:
+        yield partinfo
+        partinfo = get_next_part_meta(img, imgsize)
+
+def get_image_metadata(imgpath):
+    with open(imgpath, 'rb') as img:
+        header = img.read(16)
+        if header == b'\x63\x7b\x9d\x26\xb7\xfd\x48\x30\x89\xf9\x11\xcf\x18\xfd\xff\xa1':
+            for md in get_multipart_image_meta(img):
+                yield md
+        else:
+            raise Exception('Installation from single part image not supported')
+
+class PartedRunner():
+    def __init__(self, disk):
+        self.disk = disk
+
+    def run(self, command):
+        command = command.split()
+        command = ['parted', '-a', 'optimal', '-s', self.disk] + command
+        return subprocess.check_output(command).decode('utf8')
+
+def fixup(rootdir, vols):
+    devbymount = {}
+    for vol in vols:
+        devbymount[vol['mount']] = vol['targetdisk']
+    fstabfile = os.path.join(rootdir, 'etc/fstab')
+    with open(fstabfile) as tfile:
+        fstab = tfile.read().split('\n')
+    while not fstab[0]:
+        fstab = fstab[1:]
+    if os.path.exists(os.path.join(rootdir, '.autorelabel')):
+        os.unlink(os.path.join(rootdir, '.autorelabel'))
+    with open(fstabfile, 'w') as tfile:
+        for tab in fstab:
+            entry = tab.split()
+            if tab.startswith('#ORIGFSTAB#'):
+                if entry[1] in devbymount:
+                    targetdev = devbymount[entry[1]]
+                    if targetdev.startswith('/dev/localstorage/'):
+                        entry[0] = targetdev
+                    else:
+                        uuid = subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', targetdev]).decode('utf8')
+                        uuid = uuid.strip()
+                        entry[0] = 'UUID={}'.format(uuid)
+                elif entry[2] == 'swap':
+                    entry[0] = '/dev/mapper/localstorage-swap'
+                entry[0] = entry[0].ljust(42)
+                entry[1] = entry[1].ljust(16)
+                entry[3] = entry[3].ljust(28)
+                tab = '\t'.join(entry)
+            tfile.write(tab + '\n')
+    with open(os.path.join(rootdir, 'etc/hostname'), 'w') as nameout:
+        nameout.write(socket.gethostname() + '\n')
+    selinuxconfig = os.path.join(rootdir, 'etc/selinux/config')
+    policy = None
+    if os.path.exists(selinuxconfig):
+        with open(selinuxconfig) as cfgin:
+            sec = cfgin.read().split('\n')
+            for l in sec:
+                l = l.split('#', 1)[0]
+                if l.startswith('SELINUXTYPE='):
+                    _, policy = l.split('=')
+    for sshkey in glob.glob(os.path.join(rootdir, 'etc/ssh/*_key*')):
+        os.unlink(sshkey)
+    for sshkey in glob.glob('/etc/ssh/*_key*'):
+        newkey = os.path.join(rootdir, sshkey[1:])
+        shutil.copy2(sshkey, newkey)
+        finfo = os.stat(sshkey)
+        os.chown(newkey, finfo[stat.ST_UID], finfo[stat.ST_GID])
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/sysconfig/network-scripts/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob(os.path.join(rootdir, 'etc/NetworkManager/system-connections/*')):
+        os.unlink(ifcfg)
+    for ifcfg in glob.glob('/run/NetworkManager/system-connections/*'):
+        newcfg = ifcfg.split('/')[-1]
+        newcfg = os.path.join(rootdir, 'etc/NetworkManager/system-connections/{0}'.format(newcfg))
+        shutil.copy2(ifcfg, newcfg)
+    shutil.rmtree(os.path.join(rootdir, 'etc/confluent/'))
+    shutil.copytree('/etc/confluent', os.path.join(rootdir, 'etc/confluent'))
+    if policy:
+        sys.stdout.write('Applying SELinux labeling...')
+        sys.stdout.flush()
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'etc')])
+        subprocess.check_call(['setfiles', '-r', rootdir, os.path.join(rootdir, 'etc/selinux/{}/contexts/files/file_contexts'.format(policy)), os.path.join(rootdir, 'opt')])
+        sys.stdout.write('Done\n')
+        sys.stdout.flush()
+    for metafs in ('proc', 'sys', 'dev'):
+        subprocess.check_call(['mount', '-o', 'bind', '/{}'.format(metafs), os.path.join(rootdir, metafs)])
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub')) as defgrubin:
+        defgrub = defgrubin.read().split('\n')
+    with open(os.path.join(rootdir, 'etc/sysconfig/grub'), 'w') as defgrubout:
+        for gline in defgrub:
+            gline = gline.split()
+            newline = []
+            for ent in gline:
+                if ent.startswith('resume=') or ent.startswith('rd.lvm.lv'):
+                    continue
+                newline.append(ent)
+            defgrubout.write(' '.join(newline) + '\n')
+    grubcfg = subprocess.check_output(['find', os.path.join(rootdir, 'boot'), '-name', 'grub.cfg']).decode('utf8').strip().replace(rootdir, '/')
+    subprocess.check_call(['chroot', rootdir, 'grub2-mkconfig', '-o', grubcfg])
+    newroot = None
+    with open('/etc/shadow') as shadowin:
+        shents = shadowin.read().split('\n')
+        for shent in shents:
+            shent = shent.split(':')
+            if not shent:
+                continue
+            if shent[0] == 'root' and shent[1] not in ('*', '!!', ''):
+                newroot = shent[1]
+    if newroot:
+        shlines = None
+        with open(os.path.join(rootdir, 'etc/shadow')) as oshadow:
+            shlines = oshadow.read().split('\n')
+        with open(os.path.join(rootdir, 'etc/shadow'), 'w') as oshadow:
+            for line in shlines:
+                if line.startswith('root:'):
+                    line = line.split(':')
+                    line[1] = newroot
+                    line = ':'.join(line)
+                oshadow.write(line + '\n')
+    partnum = None
+    targblock = None
+    for vol in vols:
+        if vol['mount'] == '/boot/efi':
+            targdev = vol['targetdisk']
+            partnum = re.search('(\d+)$', targdev).group(1)
+            targblock = re.search('(.*)\d+$', targdev).group(1)
+    if targblock:
+        shimpath = subprocess.check_output(['find', os.path.join(rootdir, 'boot/efi'), '-name', 'shimx64.efi']).decode('utf8').strip()
+        shimpath = shimpath.replace(rootdir, '/').replace('/boot/efi', '').replace('//', '/').replace('/', '\\')
+        subprocess.check_call(['efibootmgr', '-c', '-d', targblock, '-l', shimpath, '--part', partnum])
+    #other network interfaces
+
+
+def had_swap():
+    with open('/etc/fstab') as tabfile:
+        tabs = tabfile.read().split('\n')
+        for tab in tabs:
+            tab = tab.split()
+            if len(tab) < 3:
+                continue
+            if tab[2] == 'swap':
+                return True
+    return False
+
+def install_to_disk(imgpath):
+    lvmvols = {}
+    deftotsize = 0
+    mintotsize = 0
+    deflvmsize = 0
+    minlvmsize = 0
+    biggestsize = 0
+    biggestfs = None
+    plainvols = {}
+    allvols = []
+    swapsize = 0
+    if had_swap():
+        with open('/proc/meminfo') as meminfo:
+            swapsize = meminfo.read().split('\n')[0]
+        swapsize = int(swapsize.split()[1])
+        if swapsize < 2097152:
+            swapsize = swapsize * 2
+        elif swapsize > 8388608 and swapsize < 67108864:
+            swapsize = swapsize * 0.5
+        elif swapsize >= 67108864:
+            swapsize = 33554432
+        swapsize = int(swapsize * 1024)
+    deftotsize = swapsize
+    mintotsize = swapsize
+    for fs in get_image_metadata(imgpath):
+        allvols.append(fs)
+        deftotsize += fs['initsize']
+        mintotsize += fs['minsize']
+        if fs['initsize'] > biggestsize:
+            biggestfs = fs
+            biggestsize = fs['initsize']
+        if fs['device'].startswith('/dev/mapper'):
+            lvmvols[fs['device'].replace('/dev/mapper/', '')] = fs
+            deflvmsize += fs['initsize']
+            minlvmsize += fs['minsize']
+        else:
+            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+    with open('/tmp/installdisk') as diskin:
+        instdisk = diskin.read()
+    instdisk = '/dev/' + instdisk
+    parted = PartedRunner(instdisk)
+    dinfo = parted.run('unit s print')
+    dinfo = dinfo.split('\n')
+    sectors = 0
+    sectorsize = 0
+    for inf in dinfo:
+        if inf.startswith('Disk {0}:'.format(instdisk)):
+            _, sectors = inf.split(': ')
+            sectors = int(sectors.replace('s', ''))
+        if inf.startswith('Sector size (logical/physical):'):
+            _, sectorsize = inf.split(':')
+            sectorsize = sectorsize.split('/')[0]
+            sectorsize = sectorsize.replace('B', '')
+            sectorsize = int(sectorsize)
+    # for now, only support resizing/growing the largest partition
+    minexcsize = deftotsize - biggestfs['initsize']
+    mintotsize = deftotsize - biggestfs['initsize'] + biggestfs['minsize']
+    minsectors = mintotsize // sectorsize
+    if sectors < (minsectors + 65536):
+        raise Exception('Disk too small to fit image')
+    biggestsectors = sectors - (minexcsize // sectorsize)
+    biggestsize = sectorsize * biggestsectors
+    parted.run('mklabel gpt')
+    curroffset = 2048
+    for volidx in sorted(plainvols):
+        vol = plainvols[volidx]
+        if vol is not biggestfs:
+            size = vol['initsize'] // sectorsize
+        else:
+            size = biggestsize // sectorsize
+        size += 2047 - (size % 2048)
+        end = curroffset + size
+        if end > sectors:
+            end = sectors
+        parted.run('mkpart primary {}s {}s'.format(curroffset, end))
+        vol['targetdisk'] = instdisk + '{0}'.format(volidx)
+        curroffset += size + 1
+    if not lvmvols:
+        if swapsize:
+            swapsize = swapsize // sectorsize
+            swapsize += 2047 - (size % 2048)
+            end = curroffset + swapsize
+            if end > sectors:
+                end = sectors
+            parted.run('mkpart swap {}s {}s'.format(curroffset, end))
+            subprocess.check_call(['mkswap', instdisk + '{}'.format(volidx + 1)])
+    else:
+        parted.run('mkpart lvm {}s 100%'.format(curroffset))
+        lvmpart = instdisk + '{}'.format(volidx + 1)
+        subprocess.check_call(['pvcreate', '-ff', '-y', lvmpart])
+        subprocess.check_call(['vgcreate', 'localstorage', lvmpart])
+        vginfo = subprocess.check_output(['vgdisplay', 'localstorage', '--units', 'b']).decode('utf8')
+        vginfo = vginfo.split('\n')
+        pesize = 0
+        pes = 0
+        for infline in vginfo:
+            infline = infline.split()
+            if len(infline) >= 3 and infline[:2] == ['PE', 'Size']:
+                pesize = int(infline[2])
+            if len(infline) >= 5 and infline[:2] == ['Free', 'PE']:
+                pes = int(infline[4])
+        takeaway = swapsize // pesize
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                continue
+            takeaway += vol['initsize'] // pesize
+            takeaway += 1
+        biggestextents = pes - takeaway
+        for volidx in lvmvols:
+            vol = lvmvols[volidx]
+            if vol is biggestfs:
+                extents = biggestextents
+            else:
+                extents = vol['initsize'] // pesize
+                extents += 1
+            if vol['mount'] == '/':
+                lvname = 'root'
+            else:
+                lvname = vol['mount'].replace('/', '_')
+            subprocess.check_call(['lvcreate', '-l', '{}'.format(extents), '-y', '-n', lvname, 'localstorage'])
+            vol['targetdisk'] = '/dev/localstorage/{}'.format(lvname)
+        if swapsize:
+            subprocess.check_call(['lvcreate', '-y', '-l', '{}'.format(swapsize // pesize), '-n', 'swap', 'localstorage'])
+            subprocess.check_call(['mkswap', '/dev/localstorage/swap'])
+        os.makedirs('/run/imginst/targ')
+        for vol in allvols:
+            with open(vol['targetdisk'], 'wb') as partition:
+                partition.write(b'\x00' * 1 * 1024 * 1024)
+            subprocess.check_call(['mkfs.{}'.format(vol['filesystem']), vol['targetdisk']])
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ'])
+            source = vol['mount'].replace('/', '_')
+            source = '/run/imginst/sources/' + source
+            blankfsstat = os.statvfs('/run/imginst/targ')
+            blankused = (blankfsstat.f_blocks - blankfsstat.f_bfree) * blankfsstat.f_bsize
+            sys.stdout.write('\nWriting {0}: '.format(vol['mount']))
+            with subprocess.Popen(['cp', '-ax', source + '/.', '/run/imginst/targ']) as copier:
+                stillrunning = copier.poll()
+                lastprogress = 0.0
+                while stillrunning is None:
+                    currfsstat = os.statvfs('/run/imginst/targ')
+                    currused = (currfsstat.f_blocks - currfsstat.f_bfree) * currfsstat.f_bsize
+                    currused -= blankused
+                    with open('/proc/meminfo') as meminf:
+                        for line in meminf.read().split('\n'):
+                            if line.startswith('Dirty:'):
+                                _, dirty, _ = line.split()
+                                dirty = int(dirty) * 1024
+                    progress = (currused - dirty) / vol['minsize']
+                    if progress < lastprogress:
+                        progress = lastprogress
+                    if progress > 0.99:
+                        progress = 0.99
+                    lastprogress = progress
+                    progress = progress * 100
+                    sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                    sys.stdout.flush()
+                    time.sleep(0.5)
+                    stillrunning = copier.poll()
+                if stillrunning != 0:
+                    raise Exception("Error copying volume")
+                with subprocess.Popen(['sync']) as syncrun:
+                    stillrunning = syncrun.poll()
+                    while stillrunning is None:
+                        with open('/proc/meminfo') as meminf:
+                            for line in meminf.read().split('\n'):
+                                if line.startswith('Dirty:'):
+                                    _, dirty, _ = line.split()
+                                    dirty = int(dirty) * 1024
+                        progress = (vol['minsize'] - dirty) / vol['minsize']
+                        if progress < lastprogress:
+                            progress = lastprogress
+                        if progress > 0.99:
+                            progress = 0.99
+                        lastprogress = progress
+                        progress = progress * 100
+                        sys.stdout.write('\x1b[1K\rWriting {0}: {1:3.2f}%'.format(vol['mount'], progress))
+                        sys.stdout.flush()
+                        time.sleep(0.5)
+                        stillrunning = syncrun.poll()
+                sys.stdout.write('\x1b[1K\rDone writing {0}'.format(vol['mount']))
+                sys.stdout.write('\n')
+                sys.stdout.flush()
+            subprocess.check_call(['umount', '/run/imginst/targ'])
+        for vol in allvols:
+            subprocess.check_call(['mount', vol['targetdisk'], '/run/imginst/targ/' + vol['mount']])
+        fixup('/run/imginst/targ', allvols)
+
+
+if __name__ == '__main__':
+    install_to_disk(os.environ['mountsrc'])

confluent_osdeploy/el8-diskless/profiles/default/scripts/imageboot.sh

@@ -0,0 +1,127 @@
+. /lib/dracut-lib.sh
+mkdir -p /mnt/remoteimg /mnt/remote /mnt/overlay
+if [ "untethered" = "$(getarg confluent_imagemethod)" ]; then
+    mount -t tmpfs untethered /mnt/remoteimg
+    curl https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs -o /mnt/remoteimg/rootimg.sfs
+else
+    confluent_urls="$confluent_urls https://$confluent_mgr/confluent-public/os/$confluent_profile/rootimg.sfs"
+    /opt/confluent/bin/urlmount $confluent_urls /mnt/remoteimg
+fi
+/opt/confluent/bin/confluent_imginfo /mnt/remoteimg/rootimg.sfs > /tmp/rootimg.info
+loopdev=$(losetup -f)
+export mountsrc=$loopdev
+losetup -r $loopdev /mnt/remoteimg/rootimg.sfs
+if grep '^Format: confluent_crypted' /tmp/rootimg.info > /dev/null; then
+    while ! curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/profileprivate/pending/rootimg.key > /tmp/rootimg.key; do
+        echo "Unable to retrieve private key from $confluent_mgr (verify that confluent can access /var/lib/confluent/private/$confluent_profile/pending/rootimg.key)"
+        sleep 1
+    done
+    cipher=$(head -n 1 /tmp/rootimg.key)
+    key=$(tail -n 1 /tmp/rootimg.key)
+    len=$(wc -c /mnt/remoteimg/rootimg.sfs | awk '{print $1}')
+    len=$(((len-4096)/512))
+    dmsetup create cryptimg --table "0 $len crypt $cipher $key 0 $loopdev 8"
+    /opt/confluent/bin/confluent_imginfo /dev/mapper/cryptimg > /tmp/rootimg.info
+    mountsrc=/dev/mapper/cryptimg
+fi
+
+if grep '^Format: squashfs' /tmp/rootimg.info > /dev/null; then
+    mount -o ro $mountsrc /mnt/remote
+elif grep  '^Format: confluent_multisquash' /tmp/rootimg.info; then
+    tail -n +3 /tmp/rootimg.info  | awk '{gsub("/", "_"); print "echo 0 " $4 " linear '$mountsrc' " $3 " | dmsetup create mproot" $7}' > /tmp/setupmount.sh
+    . /tmp/setupmount.sh
+    cat /tmp/setupmount.sh |awk '{printf "mount /dev/mapper/"$NF" "; sub("mproot", ""); gsub("_", "/"); print "/mnt/remote"$NF}' > /tmp/mountparts.sh
+    . /tmp/mountparts.sh
+fi
+
+
+#mount -t tmpfs overlay /mnt/overlay
+modprobe zram
+memtot=$(grep ^MemTotal: /proc/meminfo|awk '{print $2}')
+memtot=$((memtot/2))$(grep ^MemTotal: /proc/meminfo | awk '{print $3'})
+echo $memtot > /sys/block/zram0/disksize
+mkfs.xfs /dev/zram0 > /dev/null
+mount -o discard /dev/zram0 /mnt/overlay
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /mnt/overlay/upper /mnt/overlay/work
+    mount -t overlay -o upperdir=/mnt/overlay/upper,workdir=/mnt/overlay/work,lowerdir=/mnt/remote disklessroot /sysroot
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $3}'); do
+        mkdir -p /mnt/overlay${srcmount}/upper /mnt/overlay${srcmount}/work
+        mount -t overlay -o upperdir=/mnt/overlay${srcmount}/upper,workdir=/mnt/overlay${srcmount}/work,lowerdir=${srcmount} disklesspart /sysroot${srcmount#/mnt/remote}
+    done
+fi
+mkdir -p /sysroot/etc/ssh
+mkdir -p /sysroot/etc/confluent
+mkdir -p /sysroot/root/.ssh
+cp /root/.ssh/* /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cp /etc/confluent/* /sysroot/etc/confluent/
+cp /etc/ssh/*key* /sysroot/etc/ssh/
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /sysroot/etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /sysroot/etc/ssh/sshd_config
+done
+
+mkdir -p /sysroot/dev /sysroot/sys /sysroot/proc /sysroot/run
+if [ ! -z "$autocons" ]; then
+    autocons=${autocons%,*}
+    mkdir -p /run/systemd/generator/getty.target.wants
+    ln -s /usr/lib/systemd/system/serial-getty@.service /run/systemd/generator/getty.target.wants/serial-getty@${autocons}.service
+fi
+while [ ! -e /sysroot/sbin/init ]; do
+    echo "Failed to access root filesystem or it is missing /sbin/init"
+    echo "System should be accessible through ssh at port 2222 with the appropriate key"
+    while [ ! -e /sysroot/sbin/init ]; do
+        sleep 1
+    done
+done
+rootpassword=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg)
+rootpassword=${rootpassword#rootpassword: }
+if [ "$rootpassword" = "null" ]; then
+    rootpassword=""
+fi
+
+if [ ! -z "$rootpassword" ]; then
+    sed -i "s@root:[^:]*:@root:$rootpassword:@" /sysroot/etc/shadow
+fi
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+echo HostbasedAuthentication yes >> /sysroot/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /sysroot/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /sysroot/etc/ssh/sshd_config
+sshconf=/sysroot/etc/ssh/ssh_config
+if [ -d /sysroot/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/sysroot/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+curl -sf -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$confluent_mgr/confluent-api/self/nodelist > /sysroot/etc/ssh/shosts.equiv
+cp /sysroot/etc/ssh/shosts.equiv /sysroot/root/.shosts
+chmod 640 /sysroot/etc/ssh/*_key
+chroot /sysroot chgrp ssh_keys /etc/ssh/*_key
+cp /tls/*.pem /sysroot/etc/pki/ca-trust/source/anchors/
+chroot /sysroot/ update-ca-trust
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.service > /sysroot/etc/systemd/system/onboot.service
+mkdir -p /sysroot/opt/confluent/bin
+curl -sf https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/onboot.sh > /sysroot/opt/confluent/bin/onboot.sh
+chmod +x /sysroot/opt/confluent/bin/onboot.sh
+cp /opt/confluent/bin/apiclient /sysroot/opt/confluent/bin
+ln -s /etc/systemd/system/onboot.service /sysroot/etc/systemd/system/multi-user.target.wants/onboot.service
+cp /etc/confluent/functions /sysroot/etc/confluent/functions
+if grep installtodisk /proc/cmdline > /dev/null; then
+    . /etc/confluent/functions
+    run_remote installimage
+    exec reboot -f
+fi
+mv /lib/modules/$(uname -r) /lib/modules/$(uname -r)-ramfs
+ln -s /sysroot/lib/modules/$(uname -r) /lib/modules/
+kill $(grep -l ^/usr/lib/systemd/systemd-udevd  /proc/*/cmdline|cut -d/ -f 3)
+exec /opt/confluent/bin/start_root

confluent_osdeploy/el8-diskless/profiles/default/scripts/installimage

@@ -0,0 +1,43 @@
+#!/bin/bash
+. /etc/confluent/functions
+# the image will be used to deploy itself
+# provide both access to image (for parsing metadata)
+# and existing mounts of image (to take advantage of caching)
+mount -o bind /sys /sysroot/sys
+mount -o bind /dev /sysroot/dev
+mount -o bind /proc /sysroot/proc
+mount -o bind /run /sysroot/run
+
+
+if [ ! -f /tmp/mountparts.sh ]; then
+    mkdir -p /sysroot/run/imginst/sources/_
+    mount -o bind /mnt/remote /sysroot/run/imginst/sources/_
+else
+    for srcmount in $(cat /tmp/mountparts.sh | awk '{print $2}'); do
+        srcname=${srcmount#/dev/mapper/mproot}
+        srcdir=$(echo $srcmount | sed -e 's!/dev/mapper/mproot!/mnt/remote!' -e 's!_!/!g')
+        mkdir -p /sysroot/run/imginst/sources/$srcname
+        mount -o bind $srcdir /sysroot/run/imginst/sources/$srcname
+    done
+fi
+cd /sysroot/run
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_python getinstalldisk"
+chroot /sysroot/ bash -c "source /etc/confluent/functions; run_remote_parts pre.d"
+if [ ! -f /sysroot/tmp/installdisk ]; then
+    echo 'Unable to find a suitable installation target device, ssh to port 2222 to investigate'
+    while [ ! -f /sysroot/tmp/installdisk ]; do
+        sleep 1
+    done
+fi
+lvm vgchange -a n
+udevadm control -e
+chroot /sysroot /usr/lib/systemd/systemd-udevd --daemon
+chroot /sysroot bash -c "source /etc/confluent/functions; run_remote_python image2disk.py"
+echo "Port 22" >> /etc/ssh/sshd_config
+echo 'Match LocalPort 22' >> /etc/ssh/sshd_config
+echo '    ChrootDirectory /sysroot/run/imginst/targ' >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+
+chroot /sysroot/run/imginst/targ bash -c "source /etc/confluent/functions; run_remote post.sh"
+chroot /sysroot bash -c "umount \$(tac /proc/mounts|awk '{print \$2}'|grep ^/run/imginst/targ)"
+

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Confluent onboot hook
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/opt/confluent/bin/onboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el8-diskless/profiles/default/scripts/onboot.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# This script is executed on each boot as it is
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-onboot.log
+exec 2>> /var/log/confluent/confluent-onboot.log
+chmod 600 /var/log/confluent/confluent-onboot.log
+tail -f /var/log/confluent/confluent-onboot.log > /dev/console &
+logshowpid=$!
+
+rpm --import /etc/pki/rpm-gpg/*
+
+run_remote_python add_local_repositories
+run_remote_python syncfileclient
+run_remote_python confignet
+
+run_remote onboot.custom
+# onboot scripts may be placed into onboot.d, e.g. onboot.d/01-firstaction.sh, onboot.d/02-secondaction.sh
+run_remote_parts onboot.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
+run_remote_config onboot.d
+
+#curl -X POST -d 'status: booted' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+kill $logshowpid

confluent_osdeploy/el8-diskless/profiles/default/scripts/post.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# This script is executed 'chrooted' into a cloned disk target before rebooting
+#
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+confluent_apikey=$(cat /etc/confluent/confluent.apikey)
+confluent_mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+confluent_profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+export nodename confluent_mgr confluent_profile
+. /etc/confluent/functions
+mkdir -p /var/log/confluent
+chmod 700 /var/log/confluent
+exec >> /var/log/confluent/confluent-post.log
+exec 2>> /var/log/confluent/confluent-post.log
+chmod 600 /var/log/confluent/confluent-post.log
+tail -f /var/log/confluent/confluent-post.log > /dev/console &
+logshowpid=$!
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.service > /etc/systemd/system/firstboot.service
+mkdir -p /opt/confluent/bin
+curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/firstboot.sh > /opt/confluent/bin/firstboot.sh
+chmod +x /opt/confluent/bin/firstboot.sh
+systemctl enable firstboot
+selinuxpolicy=$(grep ^SELINUXTYPE /etc/selinux/config |awk -F= '{print $2}')
+if [ ! -z "$selinuxpolicy" ]; then
+    setfiles /etc/selinux/${selinuxpolicy}/contexts/files/file_contexts /etc/
+fi
+run_remote_python syncfileclient
+run_remote post.custom
+# post scripts may be placed into post.d, e.g. post.d/01-firstaction.sh, post.d/02-secondaction.sh
+run_remote_parts post.d
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post.d
+
+curl -sf -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $confluent_apikey" https://$confluent_mgr/confluent-api/self/updatestatus
+
+kill $logshowpid
+