Have tty2 also run a shell

Some may find switching VTs to be more intuitive
than tmux session management. Provide one extra
on tty2 for such a scenario.

.gitignore

@@ -1,4 +1,7 @@
 *.pyc
 .*.
 confluent_client/man/man*
+.vscode
 .*.sw*
+.sw*
+.idea/*

confluent_client/bin/collate

@@ -88,15 +88,18 @@ while fullline:
     for line in fullline.split('\n'):
         if not line:
             continue
-        if ': ' not in line:
+        if ':' not in line:
             line = 'UNKNOWN: ' + line
         if options.log:
-            node, output = line.split(': ', 1)
+            node, output = line.split(':', 1)
+            output = output.lstrip()
             currlog = options.log.format(node=node, nodename=node)
             with open(currlog, mode='a')  as log:
                 log.write(output + '\n')
             continue
-        grouped.add_line(*line.split(': ', 1))
+        node, output = line.split(':', 1)
+        output = output.lstrip()
+        grouped.add_line(node, output)
         if options.watch:
             if not holdoff:
                 holdoff = os.times()[4] + 0.250

confluent_client/bin/confetty

@@ -292,6 +292,8 @@ def rcompleter(text, state):
 
 
 def parse_command(command):
+    if isinstance(command, list):
+        return command
     try:
         args = shlex.split(command, posix=True)
     except ValueError as ve:
@@ -371,7 +373,7 @@ def do_command(command, server):
     if argv[0] == 'exit':
         if os.environ.get('TERM', '') not in ('linux'):
             sys.stdout.write('\x1b]0;\x07')
-        raise Bailout()
+        raise BailOut()
     elif argv[0] in ('help', '?'):
         return print_help()
     elif argv[0] == 'cd':
@@ -912,8 +914,7 @@ def main():
         do_command("start /nodes/%s/console/session" % session_node, netserver)
         doexit = True
     elif shellargs:
-        command = " ".join(shellargs)
-        do_command(command, netserver)
+        do_command(shellargs, netserver)
         quitconfetty(fullexit=True, fixterm=False)
 
     powerstate = None
@@ -947,6 +948,8 @@ def main():
                             clearpowermessage = False
                         try:
                             sys.stdout.write(data)
+                        except UnicodeEncodeError:
+                            sys.stdout.buffer.write(data.encode('utf8'))
                         except IOError:  # Some times circumstances are bad
                             # resort to byte at a time...
                             for d in data:
@@ -957,7 +960,13 @@ def main():
                             # don't bother churning if minute does not change
                             laststate['showtime'] = now
                             updatestatus()
-                        sys.stdout.flush()
+                        try:
+                            sys.stdout.flush()
+                        except Exception:
+                            # EWOULDBLOCK causes this to raise, ignore
+                            # this scenario comfortable that it
+                            # will come out soon enough
+                            pass
                     else:
                         deadline = 5
                         connected = False
@@ -1013,6 +1022,13 @@ if __name__ == '__main__':
         main()
     except BailOut as e:
         errcode = e.errorcode
+    except Exception as e:
+        import traceback
+        try:
+            quitconfetty()
+        except Exception:
+            pass
+        traceback.print_exc()
     finally:
         if deadline and os.times()[4] < deadline:
             sys.stderr.write('[Exited early, hit enter to continue]')

confluent_client/bin/dir2img

@@ -2,12 +2,13 @@
 # This will take a given directory and make a 'big floppy image'
 # out of it, suitable for nodemedia upload.
 
+import fcntl
 import glob
 import os
 import subprocess
 import sys
 
-def create_image(directory, image):
+def create_image(directory, image, label=None):
     ents = 0
     datasz = 512
     for dir in os.walk(sys.argv[1]):
@@ -24,12 +25,29 @@ def create_image(directory, image):
     with open(image, 'wb') as imgfile:
         imgfile.seek(datasz * 512 - 1)
         imgfile.write(b'\x00')
-    subprocess.check_call(['mformat', '-i', image, '-d', '1', '-t',
-                            str(datasz), '-s', '1','-h', '1', '::'])
+    if label:
+        subprocess.check_call(['mformat', '-i', image, '-v', label,
+                               '-r', '16', '-d', '1', '-t', str(datasz),
+                               '-s', '1','-h', '1', '::'])
+    else:
+        subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
+                                str(datasz), '-s', '1','-h', '1', '::'])
+    # Some clustered filesystems will have the lock from mformat
+    # linger after close (mformat doesn't unlock)
+    # do a blocking wait for shared lock and then explicitly
+    # unlock between calls to mtools
+    with open(image, 'rb') as imgfile:
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_SH)
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_UN)
     cpycmd = ['mcopy', '-i', image, '-s']
     cpycmd.extend(glob.glob('{0}/*'.format(directory)))
     cpycmd.append('::')
     subprocess.check_call(cpycmd)
+    # though not necessary for us, make sure dir2img doesn't have a lingering
+    # flock from mcopy for any subsequent commands
+    with open(image, 'rb') as imgfile:
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_SH)
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_UN)
 
 
 if __name__ == '__main__':
@@ -37,4 +55,7 @@ if __name__ == '__main__':
         sys.stderr.write("Usage: {0} <directory> <imagefile>".format(
             sys.argv[0]))
         sys.exit(1)
-    create_image(sys.argv[1], sys.argv[2])
+    label = None
+    if len(sys.argv) > 3:
+        label = sys.argv[3]
+    create_image(sys.argv[1], sys.argv[2], label)

confluent_client/bin/nodeconfig

@@ -134,9 +134,10 @@ def _assign_value():
     assignment[key] = value
 
 
-def parse_config_line(arguments):
+def parse_config_line(arguments, single=False):
     global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
-    for param in arguments:
+    for pidx in range(0, len(arguments)):
+        param = arguments[pidx]
         if param == 'show':
             continue  # forgive muscle memory of pasu users
         if param == 'set':
@@ -146,7 +147,12 @@ def parse_config_line(arguments):
         if needval:
             key = needval
             needval = None
-            value = param
+            if single:
+                value = ' '.join(arguments[pidx:])
+                _assign_value()
+                break
+            else:
+                value = param
             _assign_value()
             continue
         if '=' in param or param[-1] == ':' or forceset:
@@ -215,7 +221,7 @@ if options.batch:
             pass
         argset = argset.strip()
         if argset:
-            parse_config_line(shlex.split(argset))
+            parse_config_line(shlex.split(argset), single=True)
         argset = argfile.readline()
 else:
     parse_config_line(args[1:])
@@ -272,7 +278,6 @@ if setmode:
             rcode |= client.printerror(fr)
             for node in fr.get('databynode', []):
                 r = fr['databynode'][node]
-                rcode |= client.printerror(r, node)
                 if 'value' not in r:
                     continue
                 keyval = r['value']

confluent_client/bin/nodedeploy

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+
 import argparse
 import os
 import sys
@@ -9,6 +10,25 @@ if path.startswith('/opt'):
     sys.path.append(path)
 
 import confluent.client as client
+import confluent.sortutil as sortutil
+
+def cleararm(nr, cli):
+    nodes = set([])
+    for rsp in cli.read('/noderange/{0}/attributes/current'.format(nr)):
+        for node in rsp.get('databynode', {}):
+            nodeinfo = rsp['databynode'][node]
+            for attr in nodeinfo:
+                if attr == 'deployment.apiarmed':
+                    curr = nodeinfo[attr].get('value', '')
+                    if curr == 'continuous':
+                        nodes.add(node)
+    noderange = nr
+    if nodes:
+        noderange += ',-({0})'.format(','.join(nodes))
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(noderange),
+                          {'deployment.apiarmed': ''}):
+        pass
+
 
 def armonce(nr, cli):
     nodes = set([])
@@ -27,40 +47,94 @@ def armonce(nr, cli):
                           {'deployment.apiarmed': 'once'}):
         pass
 
+
 def setpending(nr, profile, cli):
+    args = {'deployment.pendingprofile': profile}
+    if not profile.startswith('genesis-'):
+        args['deployment.stagedprofile'] = ''
+        args['deployment.profile'] = ''
     for rsp in cli.update('/noderange/{0}/attributes/current'.format(nr),
-                          {'deployment.pendingprofile': profile}):
+                          args):
         pass
 
 
+def clearpending(nr, cli):
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(nr),
+                          {'deployment.pendingprofile': ''}):
+        pass
+
 def main(args):
     ap = argparse.ArgumentParser(description='Deploy OS to nodes')
-    ap.add_argument('-n', '--network', help='Initiate deployment over PXE', action='store_true')
+    ap.add_argument('-c', '--clear', help='Clear any pending deployment action', action='store_true')
+    ap.add_argument('-n', '--network', help='Initiate deployment over PXE/HTTP', action='store_true')
+    ap.add_argument('-p', '--prepareonly', help='Prepare only, skip any interaction with a BMC associated with this deployment action', action='store_true')
     ap.add_argument('-m', '--maxnodes', help='Specifiy a maximum nodes to be deployed')
     ap.add_argument('noderange', help='Set of nodes to deploy')
-    ap.add_argument('profile', help='Profile name to deploy')
-    args = ap.parse_args(args)
-    if not args.network:
-        sys.stderr.write('Currently only network (-n) deployment is supported\n')
+    ap.add_argument('profile', nargs='?', help='Profile name to deploy')
+    args, extra = ap.parse_known_args(args)
+    if args.profile is None and len(extra) == 1:
+        args.profile = extra[0]
+        extra = extra[1:]
+    if args.profile and not args.network:
+        sys.stderr.write('-n is a required argument currently to perform an install, optionally with -p\n')
         return 1
+    if not args.profile and args.network:
+        sys.stderr.write('Both noderange and a profile name are required arguments to request a network deployment\n')
+        return 1
+    if extra:
+        sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n')
     c = client.Command()
     c.stop_if_noderange_over(args.noderange, args.maxnodes)
-    armonce(args.noderange, c)
-    setpending(args.noderange, args.profile, c)
+    if args.clear:
+        cleararm(args.noderange, c)
+        clearpending(args.noderange, c)
+    elif args.profile:
+        armonce(args.noderange, c)
+        setpending(args.noderange, args.profile, c)
+    else:
+        databynode = {}
+        for r in c.read('/noderange/{0}/attributes/current'.format(args.noderange)):
+            dbn = r.get('databynode', {})
+            for node in dbn:
+                if node not in databynode:
+                    databynode[node] = {}
+                for attr in dbn[node]:
+                    if attr in ('deployment.pendingprofile', 'deployment.apiarmed', 'deployment.stagedprofile', 'deployment.profile'):
+                        databynode[node][attr] = dbn[node][attr].get('value', '')
+        for node in sortutil.natural_sort(databynode):
+            profile = databynode[node].get('deployment.pendingprofile', '')
+            if profile:
+                profile = 'pending: {}'.format(profile)
+            else:
+                profile = databynode[node].get('deployment.stagedprofile', '')
+                if profile:
+                    profile = 'staged: {}'.format(profile)
+                else:
+                    profile = databynode[node].get('deployment.profile', '')
+                    if profile:
+                        profile = 'completed: {}'.format(profile)
+                    else:
+                        profile= 'No profile pending or applied'
+            armed = databynode[node].get('deployment.apiarmed', '')
+            if armed in ('once', 'continuous'):
+                armed = ' (node authentication armed)'
+            else:
+                armed = ''
+            print('{0}: {1}{2}'.format(node, profile, armed))
+        sys.exit(0)
     errnodes = set([])
-    rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
-                                     bootmode='uefi',
-                                     persistent=False,
-                                     errnodes=errnodes)
-    if errnodes:
-        sys.stderr.write(
-            'Unable to set boot device for following nodes: {0}\n'.format(
-                ','.join(errnodes)))
-        return 1
-    rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
-    return rc
-
-
+    if args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
+        return rc
     return 0
 
 if __name__ == '__main__':

confluent_client/bin/nodefirmware

@@ -149,7 +149,6 @@ def show_firmware(session):
             if 'databynode' not in res:
                 continue
             for node in res['databynode']:
-                exitcode |= client.printerror(res['databynode'][node], node)
                 if 'firmware' not in res['databynode'][node]:
                     continue
                 for inv in res['databynode'][node]['firmware']:

confluent_client/bin/nodehealth

@@ -50,6 +50,7 @@ codemappings = {
     'warning': 1,
     'critical': 2,
     'failed': 2,
+    'unknown': 2,
 }
 
 def main():

confluent_client/bin/nodemedia

@@ -187,5 +187,7 @@ def main():
         argparser.print_help()
         sys.exit(1)
     handler(noderange, media)
+
 if __name__ == '__main__':
     main()
+    sys.exit(exitcode)

confluent_client/bin/nodesetboot

@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|usb|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')

confluent_client/bin/stats

@@ -22,13 +22,17 @@ import io
 import numpy as np
 
 import os
-import sixel
 import subprocess
 import sys
 
-class DumbWriter(sixel.SixelWriter):
-    def restore_position(self, output):
-        return
+try:
+    import sixel
+
+    class DumbWriter(sixel.SixelWriter):
+        def restore_position(self, output):
+            return
+except ImportError:
+    pass
 
 
 def plot(gui, output, plotdata, bins):

confluent_client/confluent/client.py

@@ -36,7 +36,9 @@ SO_PASSCRED = 16
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
     'bmcuser': 'secret.hardwaremanagementuser',
+    'switchuser': 'secret.hardwaremanagementuser',
     'bmcpass': 'secret.hardwaremanagementpassword',
+    'switchpass': 'secret.hardwaremanagementpassword',
 }
 
 try:
@@ -109,6 +111,12 @@ def printerror(res, node=None):
     exitcode = 0
     if 'errorcode' in res:
         exitcode = res['errorcode']
+    for node in res.get('databynode', {}):
+        exitcode = res['databynode'][node].get('errorcode', exitcode)
+        if 'error' in res['databynode'][node]:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['databynode'][node]['error']))
+            if exitcode == 0:
+                exitcode = 1
     if 'error' in res:
         if node:
             sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
@@ -465,7 +473,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                             if isinstance(val, list):
                                 val = ','.join(val)
                             attrout = '{0}: {1}: {2}'.format(
-                                node, printattr, val)
+                                node, printattr, val).strip()
                         else:
                             attrout = '{0}: {1}:'.format(node, printattr)
                     elif 'isset' in currattr:

confluent_client/confluent_client.spec.tmpl

@@ -7,7 +7,7 @@ Name: %{name}
 Version: %{version}
 Release: %{release}
 Source0: %{name}-%{version}.tar.gz
-License: UNKNOWN
+License: Apache2
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}

confluent_client/confluent_env.sh

@@ -91,7 +91,7 @@ _confluent_nodeidentify_completion()
 
 _confluent_nodesetboot_completion()
 {
-    COMP_CANDIDATES=("default,cd,network,setup,hd -h -b -p")
+    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy,usb -h -b -p")
     _confluent_generic_completion
 }
 
@@ -142,7 +142,7 @@ _confluent_osimage_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 2 ]; then
-        COMPREPLY=($(compgen -W "initialize import" -- ${COMP_WORDS[COMP_CWORD]}))
+        COMPREPLY=($(compgen -W "initialize import updateboot" -- ${COMP_WORDS[COMP_CWORD]}))
         return
     elif [ ${CMPARGS[1]} == 'initialize' ]; then
         COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]}))
@@ -150,6 +150,9 @@ _confluent_osimage_completion()
         compopt -o default
         COMPREPLY=()
         return
+    elif [ ${CMPARGS[1]} == 'updateboot' ]; then
+        COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return
     fi
 }
 
@@ -255,6 +258,8 @@ _confluent_nr_completion()
     CMPARGS=($COMP_LINE)
     _confluent_get_args
     if [ $NUMARGS -gt 2 ]; then
+        compopt -o default
+        COMPREPLY=()
         return;
     fi
     INPUT=${COMP_WORDS[COMP_CWORD]}
@@ -296,7 +301,7 @@ complete -F _confluent_nn_completion nodeconsole
 complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
 complete -F _confluent_nodedeploy_completion nodedeploy
-complete -F _confluent_osimage_completion osimage
+complete -F _confluent_osimage_completion osdeploy
 complete -F _confluent_ng_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth

confluent_client/doc/man/nodedeploy.ronn

@@ -0,0 +1,42 @@
+nodedeploy(8) -- Request preparation and/or initiating a node deployment
+=========================================================================
+
+## SYNOPSIS
+
+`nodedeploy <noderange>`  
+`nodedeploy <noderange> -c`  
+`nodedeploy <noderange> [-n] [-p] <name of profile>`  
+
+## DESCRIPTION
+
+**nodedeploy** configures attributes and interacts with BMC devices as appropriate
+to change the operating system on the noderange to that indicated by `profile`. Tab completion
+is supported under bash to help list and select a profile name. `-n` indicates that it
+will be a network based deployment (either HTTP or PXE) and `-p` suppresses any activity
+that requires the BMC (e.g. setting the next boot device and rebooting the system). Currently
+**nodedeploy** only supports `-n` style deployment flows. Without options it shows the current
+deployment status.
+
+## OPTIONS
+
+* '-c', '--clear`:
+  Remove any pending deployment action
+
+* `-n`, `--network`:
+  Prepare for either an HTTP or PXE based deployment, setting boot device to network and rebooting unless `-p` is specified.
+
+* `-p`, `--prepareonly`:
+  Prepare the network services for deployment, but do not interact with BMCs. This is intended for scenarios where
+  the boot device control and server restart will be handled outside of confluent.
+
+## EXAMPLES
+* Begin the instalalation of a profile of CentOS 8.2:  
+   `# nodedeploy d4 -n centos-8.2-x86_64-default`  
+   `d4: network`  
+   `d4: reset`  
+
+* Check current deployment state of nodes:
+  `# nodedeploy d4`  
+  `d4: pending: centos-8.2-x86_64-default (node authentication armed)`  
+
+

confluent_client/doc/man/nodesensors.ronn

@@ -29,6 +29,10 @@ interval of 1 second is used.
   Perform the specified number of readings, waiting `-i` indicated interval or 1 second if not
   otherwise indicated.
 
+* `-s`, `--skipnumberless`:
+  Return only sensors that provide numeric data (e.g. temperature, fanspeed),
+  ignoring sensors that do not provide numeric data.
+
 ## EXAMPLES
 * Retrieving all temperature related sensors from one system  
   `# nodesensors n1 temperature`  

confluent_client/doc/man/nodesetboot.ronn

@@ -4,7 +4,7 @@ nodesetboot(8) -- Check or set next boot device for noderange
 ## SYNOPSIS
 
 `nodesetboot <noderange>`  
-`nodesetboot [options] <noderange> [default|cd|network|setup|hd]`
+`nodesetboot [options] <noderange> [default|cd|network|setup|hd|usb|floppy]`
 
 ## DESCRIPTION
 
@@ -42,6 +42,14 @@ control.
   remote media mounted as CD/DVD, and detachable hard disks drives such as usb
   key devices.
   
+* `floppy`:
+  Request boot from floppy.  Generally speaking firmware uses this to mean a USB
+  flash drive or similar (whether virtual or physical).
+
+* `usb`:
+  Request boot from usb.  Generally speaking firmware uses this to mean a USB
+  flash drive or similar (whether virtual or physical).
+
 * `network`:
   Request boot to network
   

confluent_client/doc/man/osdeploy.ronn

@@ -0,0 +1,17 @@
+osdeploy(8) --- Configure general OS deployment facilities of confluent
+==========================================================================
+
+## SYNOPSIS
+
+`osdeploy import <iso>`  
+`osdeploy updateboot <profile>`  
+`osdeploy initialize [-h] [-g] [-u] [-s] [-k] [-t] [-p] [-i] [-l]`  
+
+## DESCRIPTION
+
+**osdeploy** manages the facilities and os deployment content of a confluent server. The
+`import` subcommand will generate profiles from an iso image. `updateboot` will take any
+changes in the specified update that need to be pushed into boot configuration and/or images
+and ensure those needed changes are performed. `initialize` provides assistance in setting
+up the most commonly required facilities. Run `osdeploy initialize -h` for more detail
+on the options offered by `osdeploy initialize`

confluent_client/doc/man/stats.ronn

@@ -0,0 +1,49 @@
+stats(8) -- Common basic statistics on typical numeric data in output
+==============================
+
+## SYNOPSIS
+
+`<other command> | stats [-c N] [-d D] [-x|-g|-t|-o image.png] [-s N] [-v] [-b N]
+
+## DESCRIPTION
+
+The **stats** command helps analyze common numerical data such as performance numbers
+or temperatures or any other numerical value.
+
+By default it looks for the last numerical output on the first line to identify the numerical column
+and analyze that number. This can be overriden by **-c COLUMN** to indicate a column. By default,
+whitespace and commas are treated to delimit columns, and **-d DELIMITER** can be used to override.
+
+By default it outputs basic statistics, but a histogram is available either text or through X11 output
+or sixel or output to an image file depending on whether **-x**, **-g**, **-t*, or **-o image.png** is
+used.
+
+## OPTIONS
+
+ * `-c N`:
+   Select column number. Defaults to last column that appears numeric
+
+ * `-d D`:
+   Specify a custom column delimiter
+
+ * `-x`:
+     Output in Sixel format (supported by mlterm and PuTTY, among others)
+
+ * `-g`:
+   Try to open histogram as an X window
+
+ * `-t`:
+   Output histogram as bars rendered by =
+
+ * `-o image.png`:
+   Write graphical histogram to image.png.
+
+ * `-s N`:
+   Ignore specified number of lines as header content before processing numbers
+
+ * `-v`:
+   Treat value before : on each line as a label, and show which labels belong to which histogram buckets.
+
+ * `-b N`:
+   Specify a custom number of buckets for histogram. The default is 10.
+   

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -0,0 +1,135 @@
+#!/usr/bin/python
+try:
+    import http.client as client
+except ImportError:
+    import httplib as client
+import os
+import socket
+import subprocess
+import ssl
+import sys
+
+def get_apikey(nodename, mgr):
+    sealnew = True
+    if os.path.exists('/etc/confluent/confluent.apikey'):
+        return open('/etc/confluent/confluent.apikey').read().strip()
+    apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
+    if not isinstance(apikey, str):
+        apikey = apikey.decode('utf8')
+    if apikey.startswith('SEALED:'):
+        sealnew = False
+        with open('/etc/confluent/confluent.sealedapikey', 'w+') as apiout:
+            apiout.write(apikey[7:])
+        with open('/etc/confluent/confluent.sealedapikey') as inp:
+            sp = subprocess.Popen(['/usr/bin/clevis-decrypt-tpm2'],
+                stdin=inp, stdout=subprocess.PIPE)
+            apikey = sp.communicate()[0]
+            if not isinstance(apikey, str):
+                apikey = apikey.decode('utf8')
+    with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
+        apiout.write(apikey)
+    if sealnew and os.path.exists('/usr/bin/clevis-encrypt-tpm2'):
+        try:
+            with open('/etc/confluent/confluent.apikey') as apin:
+                sealed = subprocess.check_output(
+                    ['/usr/bin/clevis-encrypt-tpm2', '{}'], stdin=apin)
+            print(HTTPSClient().grab_url('/confluent-api/self/saveapikey', sealed).decode())
+        except Exception:
+            sys.stderr.write('Unable to persist API key through TPM2 sealing\n')
+    apikey = apikey.strip()
+    os.chmod('/etc/confluent/confluent.apikey', 0o600)
+    return apikey
+
+class HTTPSClient(client.HTTPConnection, object):
+    def __init__(self, json=False, port=443):
+        self.stdheaders = {}
+        info = open('/etc/confluent/confluent.info').read().split('\n')
+        host = None
+        mgtiface = None
+        havedefault = '0'
+        for line in info:
+            if line.startswith('NODENAME:'):
+                node = line.split(' ')[1]
+                self.stdheaders['CONFLUENT_NODENAME'] = node
+            if line.startswith('MANAGER:') and not host:
+                host = line.split(' ')[1]
+            if line.startswith('EXTMGRINFO:'):
+                extinfo = line.split(' ')[1]
+                extinfo = extinfo.split('|')
+                if not mgtiface:
+                    host, mgtiface, havedefault = extinfo[:3]
+                if havedefault == '0' and extinfo[2] == '1':
+                    host, mgtiface, havedefault = extinfo[:3]
+        if '%' in host:
+            ifidx = host.split('%', 1)[1]
+            with open('/tmp/confluent.ifidx', 'w+') as ifout:
+                ifout.write(ifidx)
+        if json:
+            self.stdheaders['ACCEPT'] = 'application/json'
+        try:
+            info = open('/etc/confluent/confluent.deploycfg').read().split('\n')
+        except Exception:
+            info = None
+        if info:
+            for line in info:
+                if line.startswith('deploy_server: '):
+                    host = line.split(': ', 1)[1]
+                    break
+        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, host)
+        if mgtiface:
+            self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
+        client.HTTPConnection.__init__(self, host, port)
+        self.connect()
+
+    def set_header(self, key, val):
+        self.stdheaders[key] = val
+
+    def connect(self):
+        addrinf = socket.getaddrinfo(self.host, self.port)[0]
+        psock = socket.socket(addrinf[0])
+        psock.connect(addrinf[4])
+        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx.load_verify_locations('/etc/confluent/ca.pem')
+        host = self.host.split('%', 1)[0]
+        if '[' not in host and ':' in host:
+            self.stdheaders['Host'] = '[{0}]'.format(host)
+        else:
+            self.stdheaders['Host'] = '{0}'.format(host)
+        ctx.verify_mode = ssl.CERT_REQUIRED
+        ctx.check_hostname = True
+        self.sock = ctx.wrap_socket(psock, server_hostname=host)
+
+    def grab_url(self, url, data=None, returnrsp=False):
+        if data:
+            method = 'POST'
+        else:
+            method = 'GET'
+        self.request(method, url, data, headers=self.stdheaders)
+        rsp = self.getresponse()
+        if rsp.status >= 200 and rsp.status < 300:
+            if returnrsp:
+                return rsp
+            else:
+                return rsp.read()
+        raise Exception(rsp.read())
+
+if __name__ == '__main__':
+    data = None
+    json = False
+    if '-j' in sys.argv:
+        json = True
+    if len(sys.argv) == 1:
+        HTTPSClient()
+        sys.exit(0)
+    if sys.argv[-2] == '-o':
+        with open(sys.argv[3], 'wb') as outf:
+            reader = HTTPSClient(json=json).grab_url(
+                sys.argv[1], data, returnrsp=True)
+            chunk = reader.read(16384)
+            while chunk:
+                outf.write(chunk)
+                chunk = reader.read(16384)
+        sys.exit(0)
+    if os.path.exists(sys.argv[-1]):
+        data = open(sys.argv[-1]).read()
+    sys.stdout.write(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -7,7 +7,7 @@ License: Apache2
 URL: http://hpc.lenovo.com/
 Source0: confluent_osdeploy.tar.xz
 BuildArch: noarch
-Requires: confluent_ipxe mtools
+Requires: confluent_ipxe mtools tar
 BuildRoot: /tmp
 
 %description
@@ -25,22 +25,35 @@ cd utils
 make all
 cp copernicus clortho autocons ../opt/confluent/bin
 cd ..
-for os in el8 suse15 ubuntu20.04; do
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do
     mkdir ${os}out
     cd ${os}out
     cp -a ../opt .
     cp -a ../${os}/initramfs/* .
+    cp -a ../common/* .
     find . | cpio -H newc -o > ../addons.cpio
     mv ../addons.cpio .
     cd ..
 done
+mkdir esxi7out
+cd esxi7out
+cp -a ../opt .
+cp -a ../esxi7/initramfs/* .
+cp -a ../common/* .
+chmod +x bin/* opt/confluent/bin/*
+tar zcvf ../addons.tgz *
+mv ../addons.tgz .
+cd ..
+cp -a esxi7out esxi6out
+cp -a esxi7 esxi6
 
 %install
-for os in el8 suse15 ubuntu20.04; do
+for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
-    cp ${os}out/addons.cpio %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
+    cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+    find %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles -name .gitignore -exec rm -f {} +
 done
 
 %files

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+echo -n "" >> /tmp/net.ifaces
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+        echo "Initializing auto detected console when installer starts" > $autocons
+    fi
+fi
+if grep console=ttyS /proc/cmdline >& /dev/null; then
+    echo "Serial console has been requested in the kernel arguments, the local video may not show progress" > /dev/tty1
+fi
+. /lib/anaconda-lib.sh
+wait_for_kickstart

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+[ -e /tmp/confluent.initq ] && return 0
+if [ -f /tmp/dd_disk ]; then
+    for dd in $(cat /tmp/dd_disk); do
+        if [ -e $dd ]; then
+            driver-updates --disk $dd $dd
+        fi
+    done
+fi
+mkdir -p /etc/confluent
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=0
+echo -n > /etc/confluent/confluent.info
+cd /sys/class/net
+while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+    TRIES=$((TRIES + 1))
+    for currif in *; do
+        ip link set $currif up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+oum=$(umask)
+python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+echo $ifname > /tmp/net.ifaces
+
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+hostname=$nodename
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+    hostname=$hostname.$dnsdomain
+fi
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
+proto=${proto#protocol: }
+textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+	autocons=$(cat /tmp/01-autocons.devnode)
+	if [ ! -z "$autocons" ]; then
+	    echo Auto-configuring installed system to use text console
+	    echo Auto-configuring installed system to use text console > $autocons
+	    cp /tmp/01-autocons.conf /etc/cmdline.d/
+	else
+	    echo "Unable to automatically detect requested text console"
+	fi
+fi
+
+echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf
+echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf
+kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
+root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
+export kickstart
+export root
+autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/finished/confluent.sh

@@ -0,0 +1 @@
+[ -e /tmp/confluent.initq ]

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+while [ -h $BUNDLENAME ]; do
+    BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
+done
+
+cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
+mkdir -p /sysroot/etc/confluent/
+cp -a /tls /sysroot/etc/confluent
+cp -a /etc/confluent/* /sysroot/etc/confluent/
+sed -i 's/install::/install:*:/' /sysroot/etc/shadow
+sed -i 's/root::/root:*:/' /sysroot/etc/shadow
+mkdir -p /sysroot/root/.ssh
+#chmod 700 /sysroot/root/.ssh
+cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+#chmod 600 /sysroot/root/.ssh/authorized_keys
+mkdir -p /sysroot/etc/ssh/
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent

confluent_osdeploy/el7/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+

confluent_osdeploy/el7/profiles/default/kickstart

@@ -0,0 +1,80 @@
+# In this OS profile, data is largely filled in during the %pre
+# phase, rather than the kickstart actually having the content.
+# None of the files shall be replaced during an upgrade in
+# /var/lib/confluent/public/os/<profile>, so customization should
+# be done by modifying files in /var/lib/confluent/public/os/<profile>
+
+# /tmp/rootpw will provide a 'rootpw' line, either locking password if not configured
+# or the crypted form.
+%include /tmp/rootpw
+
+# timezone is fetched from confluent server, which provides the
+# timezone that the management server itself is in by default.
+%include /tmp/timezone
+
+# similar to timezone, confluent is asked to provide the
+# deployment servers language info and replicate that
+# to the deployment target.
+%include /tmp/langinfo
+
+# bootloader may be specified if crypted.grubpassword is set on
+# a node. This will become a bootloader line if a password
+# was specified
+%include /tmp/grubpw
+
+# The default partition scheme is applied to a single drive, using
+# the getinstalldisk script to make a best guess as to the most
+# appropriate device. See pre.sh and getinstalldisk to customize
+# the automatic behavior, or comment out/delete the 
+# following line and provide your own manual partition plan
+# instead
+%include /tmp/partitioning
+
+reboot
+
+%packages
+chrony
+rsync
+python
+pciutils
+%include /tmp/addonpackages
+%end
+
+
+# A kickstart.custom file is provided to enable easily adding
+# kickstart content without modifying the stock file.
+# While the stock file may be safely modified, using the .custom
+# files may allow for more easily rebasing custom profiles to newer
+# stock profiles
+%include /tmp/kickstart.custom
+
+%pre
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
+. /tmp/preinst.sh
+%end
+
+%post --nochroot
+mkdir -p /mnt/sysimage/etc/confluent
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+
+# Hook firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
+chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+%end
+
+%post
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+systemctl enable firstboot
+chgrp ssh_keys /etc/ssh/ssh*key
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+%end

confluent_osdeploy/el7/profiles/default/kickstart.custom

@@ -0,0 +1,7 @@
+# Any custom kickstart additions may go here.  It may be required to
+# modify base kickstart file to change certain portions, but adding
+# custom content to this file allows for most convenient way to 
+# refresh to a newer base profile if desired.
+# Two common example customizations are provided:
+#firewall --disabled
+#selinux --disabled

confluent_osdeploy/el7/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/el7/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/etc/confluent/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el7/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/etc/confluent/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+export nodename mgr profile
+. /etc/confluent/functions
+
+
+run_remote firstboot.custom
+
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -0,0 +1,72 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    else
+        /usr/bin/python $*
+    fi
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el7/profiles/default/scripts/infiniband/mofed.post

@@ -0,0 +1,20 @@
+# To use this script, rename or copy the mofed image to either ofed.tgz or ofed.iso
+# and modify the script below if wanting to use the iso instead of tgz
+
+# It checks for mellanox devices and opts not to install, so this script could be added
+# to a general profile without causing mofed to install on non-mellanox systems
+. /etc/confluent/functions
+if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
+    # Uncomment the following three lines and comment out the next
+    # two lines to use the .iso instead of the tgz packaging
+    #fetch_remote infiniband/mofed.iso
+    #mkdir MLNX_OFED
+    #mount -o loop ofed.iso MLNX_OFED
+    fetch_remote infiniband/mofed.tgz
+    tar xf infiniband/mofed.tgz
+    # The rest is common between tar and iso
+    cd MLNX_OFED*
+    mount -o loop ofed
+    ./mlnxofedinstall --force
+fi
+

confluent_osdeploy/el7/profiles/default/scripts/infiniband/mofed.pre

@@ -0,0 +1,10 @@
+# Add needed base packages to the install
+cat << EOF >> /tmp/addonpackages
+perl
+pkgconf-pkg-config
+tcsh
+lsof
+tk
+gcc-gfortran
+tcl
+EOF

confluent_osdeploy/el7/profiles/default/scripts/post.custom

@@ -0,0 +1,7 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.
+# An example for installing OFED for infiniband follows (see the file for more detail):
+#run_remote infiniband/mofed.post
+

confluent_osdeploy/el7/profiles/default/scripts/post.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# need to copy over ssh key info
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+
+chmod 700 /etc/confluent
+chmod og-rwx /etc/confluent/*
+
+export mgr profile nodename
+. /etc/confluent/functions
+
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+
+
+if [ -f /tmp/cryptboot ]; then
+    run_remote tpm_luks.sh
+fi
+# By default, the install repository is ignored, change
+# this by manually adding local repositories
+
+rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-*
+run_remote_python add_local_repositories
+# This script will execute in the installed system, but using the installer kernel prior to reboot.
+# This is an appropriate place to run post install activities that do not require the actual installed
+# kernel to run. For example adding drivers that would be needed for first boot to run cleanly.
+# If, for example, there is a post script that has a dependency on a driver or filesystem that
+# cannot work until booting into the installer, use firstboot.sh instead
+
+# run_remote will download and execute from /var/lib/confluent/public/os/<profile>/scripts/ directory
+# run_remote_python will use the appropriate python interpreter path to run the specified script
+# A post.custom is provided to more conveniently hold customizations, see the post.custom file.
+
+# run_remote example.sh
+# run_remote_python example.py
+run_remote post.custom

confluent_osdeploy/el7/profiles/default/scripts/pre.custom

@@ -0,0 +1,14 @@
+. /tmp/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.
+#
+#Here is an example to locally configure the platform BMC according
+#to confluent configuration so that the BMC would be on the correct
+#network:
+#run_remote_python configbmc -c
+
+#Some addons improve efficiency by adding dependencies during install
+#here is an example for adding OFED install prereqs to the install
+#run_remote infiniband/mofed.pre
+

confluent_osdeploy/el7/profiles/default/scripts/pre.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# This runs prior to the installer beginning. This is used to rewrite the 
+# scripted install file, merging data from confluent and identifying
+# the most appropriate install source.
+
+# If you want to use a more custom partition plan, the easiest
+# method is to edit the kicktstart file and comment out or
+# delete %include /tmp/partitioning
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+locale=$(grep ^locale: /etc/confluent/confluent.deploycfg)
+locale=${locale#locale: }
+keymap=$(grep ^keymap: /etc/confluent/confluent.deploycfg)
+keymap=${keymap#keymap: }
+echo lang $locale > /tmp/langinfo
+echo keyboard --vckeymap=$keymap >> /tmp/langinfo
+tz=$(grep ^timezone: /etc/confluent/confluent.deploycfg)
+tz=${tz#timezone: }
+ntpsrvs=""
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntpsrvs="--ntpservers="$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd,)
+fi
+echo timezone $ntpsrvs $tz --utc > /tmp/timezone
+rootpw=$(grep ^rootpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$rootpw" = null ]; then
+    echo "rootpw --lock" > /tmp/rootpw
+else
+    echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
+fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
+grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$grubpw" = "null" ]; then
+    touch /tmp/grubpw
+else
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
+fi
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host_*_key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.anaconda
+done
+/usr/sbin/sshd -f /etc/ssh/sshd_config.anaconda
+if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
+    consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
+    TMUX= tmux a <> $consoledev >&0 2>&1 &
+fi
+touch /tmp/addonpackages
+cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+LUKSPARTY=''
+if [ "$cryptboot" == "tpm2" ]; then
+	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
+	echo $cryptboot >> /tmp/cryptboot
+fi
+
+
+export mgr profile nodename
+curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /tmp/functions
+. /tmp/functions
+run_remote_python getinstalldisk
+if [ -e /tmp/installdisk ]; then
+    echo clearpart --all --initlabel >> /tmp/partitioning
+    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
+    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+fi
+python /etc/confluent/apiclient /confluent-public/os/$profile/kickstart.custom -o /tmp/kickstart.custom
+run_remote pre.custom

confluent_osdeploy/el7/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# This script runs after install is complete, but inside the installer 
+# environment. This is useful for carrying work done in pre/during the
+# installer into the installed environment.
+
+# It is almost certainly more useful to use post.sh or firstboot.sh
+# for customization, which will run in a more normal mechanism
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+export mgr profile nodename
+cp -a /etc/confluent /mnt/sysimage/etc
+cp /tmp/functions /mnt/sysimage/etc/confluent/
+. /tmp/functions
+cp /tmp/cryptboot /mnt/sysimage/tmp/
+
+# Preserve the ssh setup work done for the installer
+# by copying into the target system and setting up
+# host based authentication
+run_remote setupssh.sh

confluent_osdeploy/el7/profiles/default/scripts/setupssh.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+grep HostCert /etc/ssh/sshd_config.anaconda >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedAuthentication yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /mnt/sysimage/etc/ssh/sshd_config
+sshconf=/mnt/sysimage/etc/ssh/ssh_config
+if [ -d /mnt/sysimage/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/mnt/sysimage/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+
+cp /etc/ssh/ssh_host_* /mnt/sysimage/etc/ssh/
+mkdir /mnt/sysimage/root/.ssh/
+chmod 700 /mnt/sysimage/root/.ssh/
+cp /root/.ssh/authorized_keys /mnt/sysimage/root/.ssh/
+chmod 600 /mnt/sysimage/root/.ssh/authorized_keys
+cp /etc/ssh/ssh_known_hosts /mnt/sysimage/etc/ssh/
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+cp /tmp/allnodes /mnt/sysimage/etc/ssh/shosts.equiv
+cp /tmp/allnodes /mnt/sysimage/root/.shosts

confluent_osdeploy/el7/profiles/default/scripts/tpm_luks.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cryptdisk=$(blkid -t TYPE="crypto_LUKS"|sed -e s/:.*//)
+clevis luks bind -f -d $cryptdisk -k - tpm2 '{}' < /etc/confluent/confluent.apikey
+cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -11,5 +11,8 @@ if ! grep console= /proc/cmdline >& /dev/null; then
         echo "Initializing auto detected console when installer starts" > $autocons
     fi
 fi
+if grep console=ttyS /proc/cmdline >& /dev/null; then
+    echo "Serial console has been requested in the kernel arguments, the local video may not show progress" > /dev/tty1
+fi
 . /lib/anaconda-lib.sh
 wait_for_kickstart

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -1,52 +1,65 @@
 #!/bin/sh
 [ -e /tmp/confluent.initq ] && return 0
-echo -n "" > /tmp/confluent.initq
-cd /sys/class/net
-for currif in *; do
-    ip link set $currif up
-done
-cd -
-while ! grep MANAGER /tmp/confluent.info >& /dev/null; do
-    /opt/confluent/bin/copernicus -t > /tmp/confluent.info
-done
-read ifidx <<EOF
-$(grep ^MANAGER /tmp/confluent.info|grep fe80|sed -e s/.*%//)
-EOF
-read mgr << EOF
-$(grep ^MANAGER /tmp/confluent.info|grep fe80|awk '{print $2}')
-EOF
-mgridx=${mgr#*%}
-ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
-ifname=${ifname%:}
-echo $ifname > /tmp/net.ifaces
-nodename=$(grep ^NODENAME /tmp/confluent.info|awk '{print $2}')
-#TODO: blkid --label <whatever> to find mounted api
-
-if [ -z "$apikey" ]; then
-    apikey=$(/opt/confluent/bin/clortho $nodename $mgr)
+if [ -f /tmp/dd_disk ]; then
+    for dd in $(cat /tmp/dd_disk); do
+        if [ -e $dd ]; then
+            driver-updates --disk $dd $dd
+        fi
+    done
 fi
+vlaninfo=$(getarg vlan)
+if [ ! -z "$vlaninfo" ]; then
+        vldev=${vlaninfo#*:}
+        vlid=${vlaninfo#*.}
+        vlid=${vlid%:*}
+        ip link add link $vldev name $vldev.$vlid type vlan id $vlid
+fi
+TRIES=0
 oum=$(umask)
 umask 0077
-echo $apikey > /etc/confluent.apikey
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
 umask $oum
-mgr="[$mgr]"
-curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/deploycfg > /tmp/confluent.deploycfg
+cd /sys/class/net
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+    TRIES=$((TRIES + 1))
+    for currif in *; do
+        ip link set $currif up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
 
-dnsdomain=$(grep ^dnsdomain: /tmp/confluent.deploycfg)
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+cat /tls/*.pem > /etc/confluent/ca.pem
+/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+ifname=${ifname%@*}
+echo $ifname > /tmp/net.ifaces
+
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 hostname=$nodename
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
     hostname=$hostname.$dnsdomain
 fi
-mgr=$(grep ^deploy_server: /tmp/confluent.deploycfg)
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
 mgr=${mgr#deploy_server: }
-profilename=$(grep ^profile: /tmp/confluent.deploycfg)
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
 profilename=${profilename#profile: }
-proto=$(grep ^protocol: /tmp/confluent.deploycfg)
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
 proto=${proto#protocol: }
-textconsole=$(grep ^textconsole: /tmp/confluent.deploycfg)
+textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
 textconsole=${textconsole#textconsole: }
-if [ $textconsole = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
 	autocons=$(cat /tmp/01-autocons.devnode)
 	if [ ! -z "$autocons" ]; then
 	    echo Auto-configuring installed system to use text console
@@ -63,19 +76,19 @@ kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
 root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
 export kickstart
 export root
-autoconfigmethod=$(grep ipv4_method /tmp/confluent.deploycfg)
+autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
 autoconfigmethod=${autoconfigmethod#ipv4_method: }
 if [ "$autoconfigmethod" = "dhcp" ]; then
     echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
 else
-    v4addr=$(grep ^ipv4_address: /tmp/confluent.deploycfg)
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
     v4addr=${v4addr#ipv4_address: }
-    v4gw=$(grep ^ipv4_gateway: /tmp/confluent.deploycfg)
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
     v4gw=${v4gw#ipv4_gateway: }
     if [ "$v4gw" = "null" ]; then
         v4gw=""
     fi
-    v4nm=$(grep ipv4_netmask: /tmp/confluent.deploycfg)
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
     v4nm=${v4nm#ipv4_netmask: }
     echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
 fi
@@ -92,5 +105,17 @@ while read -r entry; do
         nameserversec=1
         continue
     fi
-done < /tmp/confluent.deploycfg
+done < /etc/confluent/confluent.deploycfg
+if [ -e /lib/nm-lib.sh ]; then
+    . /lib/nm-lib.sh
+    nm_generate_connections
+    if [[ "$ifname" == ib* ]]; then
+        sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
+        if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
+            echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
+        fi
+    fi
+fi
 

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -6,7 +6,9 @@ done
 
 cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
 mkdir -p /sysroot/etc/confluent/
+chmod 700 /sysroot/etc/confluent
 cp -a /tls /sysroot/etc/confluent
+cp /etc/confluent/* /sysroot/etc/confluent
 sed -i 's/install::/install:*:/' /sysroot/etc/shadow
 sed -i 's/root::/root:*:/' /sysroot/etc/shadow
 mkdir -p /sysroot/root/.ssh
@@ -17,7 +19,4 @@ mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do
     echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
 done
-cp /etc/confluent.apikey /sysroot/etc/
-cp /etc/confluent.apikey /sysroot/etc/confluent/
-cp /tmp/confluent.deploycfg /tmp/confluent.info /sysroot/etc/
-cp /tmp/confluent.deploycfg /tmp/confluent.info /sysroot/etc/confluent
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/' $2/profile.yaml
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot && \

confluent_osdeploy/el8/profiles/default/kickstart

@@ -17,6 +17,11 @@
 # to the deployment target.
 %include /tmp/langinfo
 
+# bootloader may be specified if crypted.grubpassword is set on
+# a node. This will become a bootloader line if a password
+# was specified
+%include /tmp/grubpw
+
 # The default partition scheme is applied to a single drive, using
 # the getinstalldisk script to make a best guess as to the most
 # appropriate device. See pre.sh and getinstalldisk to customize
@@ -29,35 +34,49 @@ reboot
 
 %packages
 @^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
 chrony
 rsync
 python3
+tar
+pciutils
+%include /tmp/addonpackages
+%include /tmp/cryptpkglist
 %end
 
+# A kickstart.custom file is provided to enable easily adding
+# kickstart content without modifying the stock file.
+# While the stock file may be safely modified, using the .custom
+# files may allow for more easily rebasing custom profiles to newer
+# stock profiles
+%include /tmp/kickstart.custom
+
 %pre
-profile=$(grep ^profile: /etc/confluent.deploycfg |awk '{print $2}')
-mgr=$(grep deploy_server /etc/confluent.deploycfg |awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
 . /tmp/preinst.sh
 %end
 
 %post --nochroot
 mkdir -p /mnt/sysimage/etc/confluent
-profile=$(grep ^profile: /etc/confluent.deploycfg |awk '{print $2}')
-mgr=$(grep deploy_server /etc/confluent.deploycfg |awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
 . /tmp/postinst.sh
 
 # Hook firstboot.sh
 curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
-curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
-chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+mkdir -p /mnt/sysimage/opt/confluent/bin
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/opt/confluent/bin/firstboot.sh
+chmod +x /mnt/sysimage/opt/confluent/bin/firstboot.sh
 %end
 
 %post
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
 systemctl enable firstboot
 chgrp ssh_keys /etc/ssh/ssh*key
-restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /opt/confluent/bin/firstboot.sh
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh

confluent_osdeploy/el8/profiles/default/kickstart.custom

@@ -0,0 +1,7 @@
+# Any custom kickstart additions may go here.  It may be required to
+# modify base kickstart file to change certain portions, but adding
+# custom content to this file allows for most convenient way to 
+# refresh to a newer base profile if desired.
+# Two common example customizations are provided:
+#firewall --disabled
+#selinux --disabled

confluent_osdeploy/el8/profiles/default/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
-kernelargs: quiet
+kernelargs: quiet  # These arguments are passed to the installer
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/etc/confluent/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el8/profiles/default/scripts/configbmc

@@ -0,0 +1,414 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import ctypes
+import fcntl
+import json
+from select import select
+import socket
+import struct
+import os
+import subprocess
+import sys
+import time
+
+if os.path.exists('/opt/confluent/bin/apiclient'):
+    apiclient = '/opt/confluent/bin/apiclient'
+elif os.path.exists('/etc/confluent/apiclient'):
+    apiclient = '/etc/confluent/apiclient'
+
+class IpmiMsg(ctypes.Structure):
+    _fields_ = [('netfn', ctypes.c_ubyte),
+                ('cmd', ctypes.c_ubyte),
+                ('data_len', ctypes.c_short),
+                ('data', ctypes.POINTER(ctypes.c_ubyte))]
+
+
+class IpmiSystemInterfaceAddr(ctypes.Structure):
+    _fields_ = [('addr_type', ctypes.c_int),
+                ('channel', ctypes.c_short),
+                ('lun', ctypes.c_ubyte)]
+
+
+class IpmiRecv(ctypes.Structure):
+    _fields_ = [('recv_type', ctypes.c_int),
+                ('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+class IpmiReq(ctypes.Structure):
+    _fields_ = [('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+_IONONE = 0
+_IOWRITE = 1
+_IOREAD = 2
+IPMICTL_SET_MY_ADDRESS_CMD = (
+    _IOREAD << 30 | ctypes.sizeof(ctypes.c_uint) << 16
+    | ord('i') << 8 | 17)  # from ipmi.h
+IPMICTL_SEND_COMMAND = (
+    _IOREAD << 30 | ctypes.sizeof(IpmiReq) << 16
+    | ord('i') << 8 | 13)  # from ipmi.h
+# next is really IPMICTL_RECEIVE_MSG_TRUNC, but will only use that
+IPMICTL_RECV = (
+    (_IOWRITE | _IOREAD) << 30 | ctypes.sizeof(IpmiRecv) << 16
+    | ord('i') << 8 | 11)  # from ipmi.h
+BMC_SLAVE_ADDR = ctypes.c_uint(0x20)
+CURRCHAN = 0xf
+ADDRTYPE = 0xc
+
+
+class Session(object):
+    def __init__(self, devnode='/dev/ipmi0'):
+        """Create a local session inband
+
+        :param: devnode: The path to the ipmi device
+        """
+        self.ipmidev = open(devnode, 'r+')
+        fcntl.ioctl(self.ipmidev, IPMICTL_SET_MY_ADDRESS_CMD, BMC_SLAVE_ADDR)
+        # the interface is initted, create some reusable memory for our session
+        self.databuffer = ctypes.create_string_buffer(4096)
+        self.req = IpmiReq()
+        self.rsp = IpmiRecv()
+        self.addr = IpmiSystemInterfaceAddr()
+        self.req.msg.data = ctypes.cast(
+            ctypes.addressof(self.databuffer),
+            ctypes.POINTER(ctypes.c_ubyte))
+        self.rsp.msg.data = self.req.msg.data
+        self.userid = None
+        self.password = None
+
+    def await_reply(self):
+        rd, _, _ = select((self.ipmidev,), (), (), 1)
+        while not rd:
+            rd, _, _ = select((self.ipmidev,), (), (), 1)
+
+    def pause(self, seconds):
+        time.sleep(seconds)
+
+    @property
+    def parsed_rsp(self):
+        response = {'netfn': self.rsp.msg.netfn, 'command': self.rsp.msg.cmd,
+                    'code': bytearray(self.databuffer.raw)[0],
+                    'data': bytearray(
+                        self.databuffer.raw[1:self.rsp.msg.data_len])}
+        return response
+
+    def await_config(s, bmccfg, channel):
+        vlan = bmccfg.get('bmcvlan', None)
+        ipv4 = bmccfg.get('bmcipv4', None)
+        prefix = bmccfg.get('prefixv4', None)
+        gw = bmccfg.get('bmcgw', None)
+
+
+
+    def raw_command(self,
+                    netfn,
+                    command,
+                    data=(),
+                    bridge_request=None,
+                    retry=True,
+                    delay_xmit=None,
+                    timeout=None,
+                    waitall=False, rslun=0):
+        self.addr.channel = CURRCHAN
+        self.addr.addr_type = ADDRTYPE
+        self.addr.lun = rslun
+        self.req.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        self.req.addr = ctypes.pointer(self.addr)
+        self.req.msg.netfn = netfn
+        self.req.msg.cmd = command
+        if data:
+            data = memoryview(bytearray(data))
+            try:
+                self.databuffer[:len(data)] = data[:len(data)]
+            except ValueError:
+                self.databuffer[:len(data)] = data[:len(data)].tobytes()
+        self.req.msg.data_len = len(data)
+        fcntl.ioctl(self.ipmidev, IPMICTL_SEND_COMMAND, self.req)
+        self.await_reply()
+        self.rsp.msg.data_len = 4096
+        self.rsp.addr = ctypes.pointer(self.addr)
+        self.rsp.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        fcntl.ioctl(self.ipmidev, IPMICTL_RECV, self.rsp)
+        return self.parsed_rsp
+
+
+def _is_tsm(model):
+    return model in ('7y00', '7z01', '7y98', '7y99')
+
+def set_port(s, port, vendor, model):
+    oport = port
+    if vendor not in ('IBM', 'Lenovo'):
+        raise Exception('{0} not implemented'.format(vendor))
+    if _is_tsm(model):
+        return set_port_tsm(s, port, model)
+    else:
+        set_port_xcc(s, port, model)
+        return 1
+
+
+def set_port_tsm(s, port, model):
+    oport = port
+    sys.stdout.write('Setting TSM port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    if port == 'ocp':
+        s.raw_command(0x32, 0x71, b'\x00\x01\x00')
+    elif port == 'dedicated':
+        s.raw_command(0x32, 0x71, b'\x00\x00\x00')
+    else:
+        raise Exception("Unsupported port for TSM")
+    timer = 15
+    while timer:
+        timer = timer - 1
+        time.sleep(1.0)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    if port == 'ocp':
+        iface = 0
+        s.raw_command(0x32, 0x71, b'\x00\x00\x03')
+    elif port == 'dedicated':
+        iface = 1
+        s.raw_command(0x32, 0x71, b'\x00\x01\x03')
+    rsp = s.raw_command(0x32, 0x72, bytearray([4, iface, 0]))
+    print('Complete')
+    return int(rsp['data'][0])
+
+
+def set_port_xcc(s, port, model):
+    oport = port
+    if port.lower() == 'dedicated':
+        port = b'\x01'
+    elif port.lower() in ('ml2', 'ocp'):
+        port = b'\x02\x00'
+    elif port.lower() == 'lom':
+        if model == '7x58':
+            port = b'\x00\x02'
+        else:
+            port = b'\x00\x00'
+    else:
+        port = port.split(' ')
+        port = bytes(bytearray([int(x) for x in port]))
+    currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if port == currport:
+        sys.stdout.write('XCC port already set to "{}"\n'.format(oport))
+        return
+    sys.stdout.write('Setting XCC port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    s.raw_command(0xc, 1, b'\x01\xc0' + port)
+    tries = 60
+    while currport != port and tries:
+        tries -= 1
+        time.sleep(0.5)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+        currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if not tries:
+        raise Exception('Timeout attempting to set port')
+    sys.stdout.write('Complete\n')
+
+
+def check_vlan(s, vlan, channel):
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    currvlan = bytes(s.raw_command(0xc, 2, bytearray([channel, 0x14 ,0, 0]))['data'][1:])
+    if bytearray(currvlan)[1] & 0b10000000 == 0:
+        currvlan = b'\x00\x00'
+    return currvlan == vlan
+
+
+def set_vlan(s, vlan, channel):
+    ovlan = vlan
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    if check_vlan(s, ovlan, channel):
+        sys.stdout.write('VLAN already configured to "{0}"\n'.format(ovlan))
+        return False
+    rsp = s.raw_command(0xc, 1, bytearray([channel, 0x14]) + vlan)
+    if rsp.get('code', 1) == 0:
+        print('VLAN configured to "{}"'.format(ovlan))
+    else:
+        print('Error setting vlan: ' + repr(rsp))
+    return True
+
+
+def get_lan_channel(s):
+    for chan in range(1, 16):
+        rsp = s.raw_command(6, 0x42, bytearray([chan]))['data']
+        if not rsp:
+            continue
+        medtype = int(rsp[1]) & 0b1111111
+        if medtype not in (4, 6):
+            continue
+        rsp = s.raw_command(0xc, 2, bytearray([2, chan, 5, 0, 0]))
+        if rsp.get('code', 1) == 0:
+            return chan
+    return 1
+
+
+def check_ipv4(s, ipaddr, channel):
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
+    return rsp == ipaddr
+
+def set_ipv4(s, ipaddr, channel):
+    oipaddr = ipaddr
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    if check_ipv4(s, oipaddr, channel):
+        print('IP Address already set to {}'.format(oipaddr))
+        return False
+    rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+    if rsp != 1:
+        sys.stdout.write("Changing configuration to static...")
+        sys.stdout.flush()
+        resp = s.raw_command(0xc, 1, bytearray([channel, 4, 1]))
+        tries = 0
+        while rsp != 1 and tries < 30:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            tries += 1
+            time.sleep(0.5)
+            rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+        sys.stdout.write('Complete\n')
+        sys.stdout.flush()
+    print('Setting IP to {}'.format(oipaddr))
+    s.raw_command(0xc, 1, bytearray([channel, 3]) + ipaddr)
+    return True
+
+
+def check_subnet(s, prefix, channel):
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
+    return rsp == mask
+
+def set_subnet(s, prefix, channel):
+    oprefix = prefix
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    if check_subnet(s, prefix, channel):
+        print('Subnet Mask already set to /{}'.format(oprefix))
+        return False
+    print('Setting subnet mask to /{}'.format(oprefix))
+    s.raw_command(0xc, 1, bytearray([channel, 6]) + mask)
+    return True
+
+
+def check_gateway(s, gw, channel):
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    return rsp == gw
+
+def set_gateway(s, gw, channel):
+    ogw = gw
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    if check_gateway(s, ogw, channel):
+        print('Gateway already set to {}'.format(ogw))
+        return False
+    print('Setting gateway to {}'.format(ogw))
+    s.raw_command(0xc, 1, bytearray([channel, 12]) + gw)
+    return True
+
+def dotwait():
+    sys.stdout.write('.')
+    sys.stdout.flush()
+    time.sleep(0.5)
+
+def main():
+    a = argparse.ArgumentParser(description='Locally configure a BMC device')
+    a.add_argument('-v', help='vlan id or off to disable vlan tagging')
+    a.add_argument('-p', help='Which port to use (dedicated, lom, ocp, ml2)')
+    a.add_argument('-i', help='JSON configuration file to read for '
+                   'configuration')
+    a.add_argument('-c', help='Use Confluent API to direct BMC configuration',
+                   action='store_true')
+    args = a.parse_args()
+    if args.i:
+        bmccfg = json.load(open(args.i))
+    elif args.c:
+        bmccfgsrc = subprocess.check_output(
+            [sys.executable, apiclient, '/confluent-api/self/bmcconfig', '-j'])
+        bmccfg = json.loads(bmccfgsrc)
+    else:
+        bmccfg = {}
+    if args.p is not None:
+        bmccfg['bmcport'] = args.p
+    if args.v is not None:
+        bmccfg['bmcvlan'] = args.v
+    vendor = open('/sys/devices/virtual/dmi/id/sys_vendor').read()
+    vendor = vendor.strip()
+    try:
+        model = open('/sys/devices/virtual/dmi/id/product_sku').read().strip()
+        if model == 'none':
+            raise Exception('No SKU')
+    except Exception:
+        model = open('/sys/devices/virtual/dmi/id/product_name').read()
+    if vendor in ('Lenovo', 'IBM'):
+        if '[' in model and ']' in model:
+            model = model.split('[')[1].split(']')[0]
+        model = model[:4].lower()
+    s = Session('/dev/ipmi0')
+    if not bmccfg:
+        print("No BMC configuration specified, exiting.")
+        return
+    if bmccfg.get('bmcport', None):
+        channel = set_port(s, bmccfg['bmcport'], vendor, model)
+    else:
+        channel = get_lan_channel(s)
+    awaitvlan = False
+    awaitip = False
+    awaitprefix = False
+    awaitgw = False
+    if bmccfg.get('bmcvlan', None):
+        awaitvlan = set_vlan(s, bmccfg['bmcvlan'], channel)
+    if bmccfg.get('bmcipv4', None):
+        awaitip = set_ipv4(s, bmccfg['bmcipv4'], channel)
+    if bmccfg.get('prefixv4', None):
+        awaitprefix = set_subnet(s, bmccfg['prefixv4'], channel)
+    if bmccfg.get('bmcgw', None):
+        awaitgw = set_gateway(s, bmccfg['bmcgw'], channel)
+    sys.stdout.write('Waiting for changes to take effect...')
+    sys.stdout.flush()
+    while awaitvlan and not check_vlan(s, bmccfg['bmcvlan'], channel):
+        dotwait()
+    while awaitip and not check_ipv4(s, bmccfg['bmcipv4'], channel):
+        dotwait()
+    while awaitprefix and not check_subnet(s, bmccfg['prefixv4'], channel):
+        dotwait()
+    while awaitprefix and not check_gateway(s, bmccfg['bmcgw'], channel):
+        dotwait()
+    sys.stdout.write('done\n')
+    sys.stdout.flush()
+    #await_config(s, bmccfg, channel)
+
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el8/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el8/profiles/default/scripts/firstboot.service

@@ -4,7 +4,7 @@ Requires=network-online.target
 After=network-online.target
 
 [Service]
-ExecStart=/etc/confluent/firstboot.sh
+ExecStart=/opt/confluent/bin/firstboot.sh
 
 [Install]
 WantedBy=multi-user.target

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -1,7 +1,24 @@
 #!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 apikey=$(cat /etc/confluent/confluent.apikey)
 mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
-curl --capath /etc/confluent/tls -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+export nodename mgr profile
+. /etc/confluent/functions
+
+
+run_remote firstboot.custom
+# Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+run_remote_parts firstboot
+
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -1,14 +1,75 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+run_remote_parts() {
+    scriptlist=$(/usr/libexec/platform-python /etc/confluent/apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1.d/$script
+    done
+}
+
 run_remote() {
-    cd $(mktemp -d)
-    curl -f https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
     chmod +x $1
-    ./$1
-    cd -
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
 }
 
 run_remote_python() {
-    cd $(mktemp -d)
-    curl -f https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
-    /usr/libexec/platform-python $1
-    cd -
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    /usr/libexec/platform-python $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
 }

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.post

@@ -0,0 +1,20 @@
+# To use this script, rename or copy the mofed image to either ofed.tgz or ofed.iso
+# and modify the script below if wanting to use the iso instead of tgz
+
+# It checks for mellanox devices and opts not to install, so this script could be added
+# to a general profile without causing mofed to install on non-mellanox systems
+. /etc/confluent/functions
+if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
+    # Uncomment the following three lines and comment out the next
+    # two lines to use the .iso instead of the tgz packaging
+    #fetch_remote infiniband/mofed.iso
+    #mkdir MLNX_OFED
+    #mount -o loop ofed.iso MLNX_OFED
+    fetch_remote infiniband/mofed.tgz
+    tar xf infiniband/mofed.tgz
+    # The rest is common between tar and iso
+    cd MLNX_OFED*
+    mount -o loop ofed
+    ./mlnxofedinstall --force
+fi
+

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.pre

@@ -0,0 +1,10 @@
+# Add needed base packages to the install
+cat << EOF >> /tmp/addonpackages
+perl
+pkgconf-pkg-config
+tcsh
+lsof
+tk
+gcc-gfortran
+tcl
+EOF

confluent_osdeploy/el8/profiles/default/scripts/post.custom

@@ -0,0 +1,7 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.
+
+# An example for installing OFED for infiniband follows (see the file for more detail):
+#run_remote infiniband/mofed.post

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -1,19 +1,38 @@
 #!/bin/sh
 # need to copy over ssh key info
 nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+
+chmod 700 /etc/confluent
+chmod og-rwx /etc/confluent/*
 export mgr profile nodename
 . /etc/confluent/functions
 
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+
+
+if [ -f /tmp/cryptboot ]; then
+    run_remote tpm_luks.sh
+fi
+
+# By default, the install repository is ignored, change
+# this by manually adding local repositories
+
+rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-*
+run_remote_python add_local_repositories
 # This script will execute in the installed system, but using the installer kernel prior to reboot.
 # This is an appropriate place to run post install activities that do not require the actual installed
 # kernel to run. For example adding drivers that would be needed for first boot to run cleanly.
 # If, for example, there is a post script that has a dependency on a driver or filesystem that
 # cannot work until booting into the installer, use firstboot.sh instead
 
-# run_remote will download and execute from /var/lib/confluent/public/<os>/scripts/ directory
+# run_remote will download and execute from /var/lib/confluent/public/os/<profile>/scripts/ directory
 # run_remote_python will use the appropriate python interpreter path to run the specified script
-
-# Add content as below:
+# A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
 # run_remote example.sh
 # run_remote_python example.py
+run_remote post.custom
+
+# Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh
+run_remote_parts post

confluent_osdeploy/el8/profiles/default/scripts/pre.custom

@@ -0,0 +1,13 @@
+. /tmp/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.
+#
+#Here is an example to locally configure the platform BMC according
+#to confluent configuration so that the BMC would be on the correct
+#network:
+#run_remote_python configbmc -c
+
+#Some addons improve efficiency by adding dependencies during install
+#here is an example for adding OFED install prereqs to the install
+#run_remote infiniband/mofed.pre

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -1,32 +1,56 @@
 #!/bin/sh
 
 # This runs prior to the installer beginning. This is used to rewrite the 
-# scripted install file, merging data from confluennt and identifying
+# scripted install file, merging data from confluent and identifying
 # the most appropriate install source.
 
 # If you want to use a more custom partition plan, the easiest
 # method is to edit the kicktstart file and comment out or
 # delete %include /tmp/partitioning
 
-nodename=$(grep ^NODENAME /etc/confluent.info|awk '{print $2}')
-locale=$(grep ^locale: /etc/confluent.deploycfg)
+/usr/libexec/platform-python /etc/confluent/apiclient >& /dev/null
+nicname=$(ip link|grep ^$(cat /tmp/confluent.ifidx): | awk '{print $2}' | awk -F: '{print $1}')
+nmcli c u $nicname
+while ip -6 addr | grep tentative > /dev/null; do
+   sleep 0.5
+done
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+locale=$(grep ^locale: /etc/confluent/confluent.deploycfg)
 locale=${locale#locale: }
-keymap=$(grep ^keymap: /etc/confluent.deploycfg)
+keymap=$(grep ^keymap: /etc/confluent/confluent.deploycfg)
 keymap=${keymap#keymap: }
 echo lang $locale > /tmp/langinfo
 echo keyboard --vckeymap=$keymap >> /tmp/langinfo
-tz=$(grep ^timezone: /etc/confluent.deploycfg)
+tz=$(grep ^timezone: /etc/confluent/confluent.deploycfg)
 tz=${tz#timezone: }
-echo timezone $tz --utc > /tmp/timezone
-rootpw=$(grep ^rootpassword /etc/confluent.deploycfg | awk '{print $2}')
+ntpsrvs=""
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntpsrvs="--ntpservers="$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd,)
+fi
+echo timezone $ntpsrvs $tz --utc > /tmp/timezone
+rootpw=$(grep ^rootpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
 if [ "$rootpw" = null ]; then
     echo "rootpw --lock" > /tmp/rootpw
 else
     echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
 fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
+grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$grubpw" = "null" ]; then
+    touch /tmp/grubpw
+else
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
+fi
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
-    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
     echo HostCertificate $certfile >> /etc/ssh/sshd_config.anaconda
 done
 /usr/sbin/sshd -f /etc/ssh/sshd_config.anaconda
@@ -34,6 +58,17 @@ if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
     consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
     tmux a <> $consoledev >&0 2>&1 &
 fi
+cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+LUKSPARTY=''
+touch /tmp/cryptpkglist
+touch /tmp/addonpackages
+if [ "$cryptboot" == "tpm2" ]; then
+	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
+	echo $cryptboot >> /tmp/cryptboot
+	echo clevis-dracut >> /tmp/cryptpkglist
+fi
+
+
 export mgr profile nodename
 curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /tmp/functions
 . /tmp/functions
@@ -41,5 +76,7 @@ run_remote_python getinstalldisk
 if [ -e /tmp/installdisk ]; then
     echo clearpart --all --initlabel >> /tmp/partitioning
     echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
-    echo autopart --nohome >> /tmp/partitioning
+    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
 fi
+curl -f https://$mgr/confluent-public/os/$profile/kickstart.custom > /tmp/kickstart.custom
+run_remote pre.custom

confluent_osdeploy/el8/profiles/default/scripts/prechroot.sh

@@ -7,11 +7,13 @@
 # It is almost certainly more useful to use post.sh or firstboot.sh
 # for customization, which will run in a more normal mechanism
 
-nodename=$(grep ^NODENAME /etc/confluent.info|awk '{print $2}')
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
 export mgr profile nodename
 cp -a /etc/confluent /mnt/sysimage/etc
+chmod -R og-rwx /mnt/sysimage/etc/confluent
 cp /tmp/functions /mnt/sysimage/etc/confluent/
 . /tmp/functions
+cp /tmp/cryptboot /mnt/sysimage/tmp/
 
 # Preserve the ssh setup work done for the installer
 # by copying into the target system and setting up

confluent_osdeploy/el8/profiles/default/scripts/setupssh.sh

@@ -3,7 +3,7 @@ grep HostCert /etc/ssh/sshd_config.anaconda >> /mnt/sysimage/etc/ssh/sshd_config
 echo HostbasedAuthentication yes >> /mnt/sysimage/etc/ssh/sshd_config
 echo HostbasedUsesNameFromPacketOnly yes >> /mnt/sysimage/etc/ssh/sshd_config
 echo IgnoreRhosts no >> /mnt/sysimage/etc/ssh/sshd_config
-sshconf=/etc/ssh/ssh_config
+sshconf=/mnt/sysimage/etc/ssh/ssh_config
 if [ -d /mnt/sysimage/etc/ssh/ssh_config.d/ ]; then
     sshconf=/mnt/sysimage/etc/ssh/ssh_config.d/01-confluent.conf
 fi
@@ -18,6 +18,6 @@ chmod 700 /mnt/sysimage/root/.ssh/
 cp /root/.ssh/authorized_keys /mnt/sysimage/root/.ssh/
 chmod 600 /mnt/sysimage/root/.ssh/authorized_keys
 cp /etc/ssh/ssh_known_hosts /mnt/sysimage/etc/ssh/
-curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
 cp /tmp/allnodes /mnt/sysimage/etc/ssh/shosts.equiv
 cp /tmp/allnodes /mnt/sysimage/root/.shosts

confluent_osdeploy/el8/profiles/default/scripts/tpm_luks.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cryptdisk=$(blkid -t TYPE="crypto_LUKS"|sed -e s/:.*//)
+clevis luks bind -f -d $cryptdisk -k - tpm2 '{}' < /etc/confluent/confluent.apikey
+cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -0,0 +1,25 @@
+#!/bin/sh
+# copernicus is hard coded, easier to script a fake sysfs
+uuid=$(vsish -e get /hardware/bios/dmiInfo|grep -A15 UUID|sed -e 's/.*://'|sed -e ':a;N;$!ba;s/\n//g' | sed -e 's/ *0x//g')
+uuid=${uuid:0:8}-${uuid:8:4}-${uuid:12:4}-${uuid:16:4}-${uuid:20:12}
+mkdir -p /sys/devices/virtual/dmi/id/
+echo $uuid > /sys/devices/virtual/dmi/id/product_uuid
+mkdir -p /etc/confluent
+localcli network firewall unload
+touch /etc/confluent/confluent.info
+while ! grep NODENAME /etc/confluent/confluent.info; do
+    /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+done
+node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+mgr=$(grep MANAGER: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
+cp /opt/confluent/bin/clortho /clortho
+/clortho $node $mgr > /etc/confluent/confluent.apikey
+cat /tls/*.pem > /etc/confluent/ca.pem
+/opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
+/opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg
+/opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/modinstall > /tmp/modinstall
+mv /etc/confluent/confluent.deploycfg.new /etc/confluent/confluent.deploycfg
+export node mgr profile
+. /tmp/modinstall
+exec /bin/install

confluent_osdeploy/esxi7/profiles/hypervisor/initprofile.sh

@@ -0,0 +1 @@
+#!/bin/sh

confluent_osdeploy/esxi7/profiles/hypervisor/kickstart

@@ -0,0 +1,11 @@
+accepteula
+clearpart --firstdisk --overwritevmfs
+install --firstdisk --overwritevmfs
+%include /tmp/ksnet
+%include /tmp/rootpw
+reboot
+%post
+localcli network firewall unload
+STATUP=$(mktemp)
+echo '{"status": "complete"}' > $STATUP
+/opt/confluent/bin/apiclient /confluent-api/self/updatestatus $STATUP

confluent_osdeploy/esxi7/profiles/hypervisor/profile.yaml

@@ -0,0 +1,3 @@
+label: Confluent installation of VMware ESXi %%VERSION%% Hypervisor
+ostype: esxi
+kernelargs: runweasel

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -0,0 +1,36 @@
+#!/usr/bin/python
+
+nodename = None
+for inf in open('/etc/confluent/confluent.info', 'r').read().split('\n'):
+    if inf.startswith('NODENAME: '):
+        nodename = inf.replace('NODENAME: ', '')
+        break
+
+deploycfg = open('/etc/confluent/confluent.deploycfg', 'r').read().split('\n')
+cfg = {}
+nslist = False
+nameservers = []
+for line in deploycfg:
+    kv = line.split(': ')
+    if not kv[0]:
+        continue
+    if len(kv) == 2:
+        cfg[kv[0]] = kv[1]
+    if kv[0] == 'nameservers:':
+        nslist = True
+        continue
+    if nslist and kv[0].startswith('- '):
+        nameservers.append(kv[0].split(' ', 1)[1])
+    else:
+        nslist=False
+cfg['nameservers'] = ','.join(nameservers)
+
+netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if cfg['ipv4_method'] == 'static':
+    netline += ' --ip={0} --netmask={1}'.format(cfg['ipv4_address'], cfg['ipv4_netmask'])
+    if cfg.get('ipv4_gateway', 'null') not in (None, '', 'null'):
+        netline += ' --gateway={0}'.format(cfg['ipv4_gateway'])
+    if cfg['nameservers']:
+        netline += ' --nameserver={0}'.format(cfg['nameservers'])
+print(netline)
+    

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/modinstall

@@ -0,0 +1,9 @@
+#!/bin/sh
+mv /etc/confluent/confluent.deploycfg /etc/confluent/confluent.newdeploycfg
+/opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/makeksnet >> /tmp/makeksnet
+mv /etc/confluent/confluent.newdeploycfg /etc/confluent/confluent.deploycfg
+chmod +x /tmp/makeksnet
+/tmp/makeksnet > /tmp/ksnet
+rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
+echo rootpw --iscrypted $rootpw > /tmp/rootpw
+export BOOT_CMDLINE=ks=/etc/confluent/ks.cfg

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -0,0 +1,103 @@
+#!/bin/bash
+export LANG=en_US.utf8
+mkdir -p /etc/confluent
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+    autocons=${autocons%,*}
+    echo $autocons > /tmp/01-autocons.devnode
+    if [ ! -z "$autocons" ]; then
+        echo "Using $(cat /tmp/01-autocons.conf)"
+        (while :; do tmux a <> $autocons >&0 2>&1; done) &
+        (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
+    else
+        (while :; do tmux a <> /dev/console >&0 2>&1; done) &
+    fi
+else
+    (while :; do tmux a <> /dev/console >&0 2>&1; done) &
+    (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
+fi
+(while :; do TERM=linux tmux <> /dev/tty2 >&0 2>&1; done) &
+echo -n "udevd: "
+/usr/lib/systemd/systemd-udevd --daemon
+echo -n "Loading drivers..."
+udevadm trigger
+udevadm trigger --type=devices --action=add
+udevadm settle
+modprobe ib_ipoib
+modprobe ib_umad
+modprobe hfi1
+modprobe mlx5_ib
+echo "done"
+cat > /etc/ssh/sshd_config << EOF
+Port 22
+Port 3389
+PermitRootLogin yes
+AuthorizedKeysFile      .ssh/authorized_keys
+EOF
+mkdir ~/.ssh
+cat /ssh/*.rootpubkey > ~/.ssh/authorized_keys
+cat /tls/*.pem > /etc/confluent/ca.pem
+mkdir -p /etc/pki/tls/certs
+cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt
+TRIES=0
+touch /etc/confluent/confluent.info
+cd /sys/class/net
+echo -n "Scanning for network configuration..."
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 30 ]; do
+    TRIES=$((TRIES + 1))
+    for i in *; do
+        ip link set $i up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+hostname $nodename
+/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo -n "Attempting to use dhcp to bring up $ifname..."
+    dhclient $ifname
+    echo "Complete:"
+    ip addr show dev $ifname
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ^prefix: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#prefix: }
+    echo "Setting up $ifname as static at $v4addr/$v4nm"
+    ip addr add dev $ifname $v4addr/$v4nm
+    if [ ! -z "$v4gw" ]; then
+        ip route add default via $v4gw
+    fi
+fi
+echo -n "Initialiing ssh..."
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host*key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    privfile=${pubkey%.pub}
+    /usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/sshcert $pubkey > $certfile
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
+    echo HostKey $privfile >> /etc/ssh/sshd_config
+done
+/usr/sbin/sshd
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+export profile mgr
+echo "Running https://$mgr/confluent-public/os/$profile/scripts/onboot.sh"
+/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/functions > /tmp/functions
+. /tmp/functions
+run_remote onboot.sh
+while :; do
+    bash
+done

confluent_osdeploy/genesis/initramfs/usr/lib/dracut/hooks/cmdline/10-genesis.sh

@@ -0,0 +1,16 @@
+root=1
+rootok=1
+netroot=genesis
+clear
+echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bashrc
+echo PS1="'"'[genesis running on \H \w]$ '"'" >> ~/.bash_profile
+mkdir -p /etc/ssh
+mkdir -p /var/tmp/
+mkdir -p /var/empty/sshd
+sed -i '/^root:x/d' /etc/passwd
+echo root:x:0:0::/:/bin/bash >> /etc/passwd
+echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
+tmux new-session -d sh /opt/confluent/bin/rungenesis
+while :; do
+	sleep 86400
+done

confluent_osdeploy/genesis/profiles/default/profile.yaml

@@ -0,0 +1,2 @@
+label: Genesis
+kernelargs: quiet

confluent_osdeploy/genesis/profiles/default/scripts/configbmc

@@ -0,0 +1,431 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import ctypes
+import fcntl
+import json
+from select import select
+import socket
+import struct
+import os
+import subprocess
+import sys
+import time
+
+if os.path.exists('/opt/confluent/bin/apiclient'):
+    apiclient = '/opt/confluent/bin/apiclient'
+elif os.path.exists('/etc/confluent/apiclient'):
+    apiclient = '/etc/confluent/apiclient'
+
+class IpmiMsg(ctypes.Structure):
+    _fields_ = [('netfn', ctypes.c_ubyte),
+                ('cmd', ctypes.c_ubyte),
+                ('data_len', ctypes.c_short),
+                ('data', ctypes.POINTER(ctypes.c_ubyte))]
+
+
+class IpmiSystemInterfaceAddr(ctypes.Structure):
+    _fields_ = [('addr_type', ctypes.c_int),
+                ('channel', ctypes.c_short),
+                ('lun', ctypes.c_ubyte)]
+
+
+class IpmiRecv(ctypes.Structure):
+    _fields_ = [('recv_type', ctypes.c_int),
+                ('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+class IpmiReq(ctypes.Structure):
+    _fields_ = [('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+_IONONE = 0
+_IOWRITE = 1
+_IOREAD = 2
+IPMICTL_SET_MY_ADDRESS_CMD = (
+    _IOREAD << 30 | ctypes.sizeof(ctypes.c_uint) << 16
+    | ord('i') << 8 | 17)  # from ipmi.h
+IPMICTL_SEND_COMMAND = (
+    _IOREAD << 30 | ctypes.sizeof(IpmiReq) << 16
+    | ord('i') << 8 | 13)  # from ipmi.h
+# next is really IPMICTL_RECEIVE_MSG_TRUNC, but will only use that
+IPMICTL_RECV = (
+    (_IOWRITE | _IOREAD) << 30 | ctypes.sizeof(IpmiRecv) << 16
+    | ord('i') << 8 | 11)  # from ipmi.h
+BMC_SLAVE_ADDR = ctypes.c_uint(0x20)
+CURRCHAN = 0xf
+ADDRTYPE = 0xc
+
+
+class Session(object):
+    def __init__(self, devnode='/dev/ipmi0'):
+        """Create a local session inband
+
+        :param: devnode: The path to the ipmi device
+        """
+        self.ipmidev = open(devnode, 'r+')
+        fcntl.ioctl(self.ipmidev, IPMICTL_SET_MY_ADDRESS_CMD, BMC_SLAVE_ADDR)
+        # the interface is initted, create some reusable memory for our session
+        self.databuffer = ctypes.create_string_buffer(4096)
+        self.req = IpmiReq()
+        self.rsp = IpmiRecv()
+        self.addr = IpmiSystemInterfaceAddr()
+        self.req.msg.data = ctypes.cast(
+            ctypes.addressof(self.databuffer),
+            ctypes.POINTER(ctypes.c_ubyte))
+        self.rsp.msg.data = self.req.msg.data
+        self.userid = None
+        self.password = None
+
+    def await_reply(self):
+        rd, _, _ = select((self.ipmidev,), (), (), 1)
+        while not rd:
+            rd, _, _ = select((self.ipmidev,), (), (), 1)
+
+    def pause(self, seconds):
+        time.sleep(seconds)
+
+    @property
+    def parsed_rsp(self):
+        response = {'netfn': self.rsp.msg.netfn, 'command': self.rsp.msg.cmd,
+                    'code': bytearray(self.databuffer.raw)[0],
+                    'data': bytearray(
+                        self.databuffer.raw[1:self.rsp.msg.data_len])}
+        return response
+
+    def await_config(s, bmccfg, channel):
+        vlan = bmccfg.get('bmcvlan', None)
+        ipv4 = bmccfg.get('bmcipv4', None)
+        prefix = bmccfg.get('prefixv4', None)
+        gw = bmccfg.get('bmcgw', None)
+
+
+
+    def raw_command(self,
+                    netfn,
+                    command,
+                    data=(),
+                    bridge_request=None,
+                    retry=True,
+                    delay_xmit=None,
+                    timeout=None,
+                    waitall=False, rslun=0):
+        self.addr.channel = CURRCHAN
+        self.addr.addr_type = ADDRTYPE
+        self.addr.lun = rslun
+        self.req.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        self.req.addr = ctypes.pointer(self.addr)
+        self.req.msg.netfn = netfn
+        self.req.msg.cmd = command
+        if data:
+            data = memoryview(bytearray(data))
+            try:
+                self.databuffer[:len(data)] = data[:len(data)]
+            except ValueError:
+                self.databuffer[:len(data)] = data[:len(data)].tobytes()
+        self.req.msg.data_len = len(data)
+        fcntl.ioctl(self.ipmidev, IPMICTL_SEND_COMMAND, self.req)
+        self.await_reply()
+        self.rsp.msg.data_len = 4096
+        self.rsp.addr = ctypes.pointer(self.addr)
+        self.rsp.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        fcntl.ioctl(self.ipmidev, IPMICTL_RECV, self.rsp)
+        return self.parsed_rsp
+
+
+def _is_tsm(model):
+    return model in ('7y00', '7z01', '7y98', '7y99')
+
+def set_port(s, port, vendor, model):
+    oport = port
+    if vendor not in ('IBM', 'Lenovo'):
+        raise Exception('{0} not implemented'.format(vendor))
+    if _is_tsm(model):
+        return set_port_tsm(s, port, model)
+    else:
+        set_port_xcc(s, port, model)
+        return 1
+
+
+def get_remote_config_mod(vendor, model):
+    if vendor in ('IBM', 'Lenovo'):
+        if _is_tsm(model):
+            return 'tsm'
+        else:
+            return 'xcc'
+    return None
+
+def set_port_tsm(s, port, model):
+    oport = port
+    sys.stdout.write('Setting TSM port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    if port == 'ocp':
+        s.raw_command(0x32, 0x71, b'\x00\x01\x00')
+    elif port == 'dedicated':
+        s.raw_command(0x32, 0x71, b'\x00\x00\x00')
+    else:
+        raise Exception("Unsupported port for TSM")
+    timer = 15
+    while timer:
+        timer = timer - 1
+        time.sleep(1.0)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    if port == 'ocp':
+        iface = 0
+        s.raw_command(0x32, 0x71, b'\x00\x00\x03')
+    elif port == 'dedicated':
+        iface = 1
+        s.raw_command(0x32, 0x71, b'\x00\x01\x03')
+    rsp = s.raw_command(0x32, 0x72, bytearray([4, iface, 0]))
+    print('Complete')
+    return int(rsp['data'][0])
+
+
+def set_port_xcc(s, port, model):
+    oport = port
+    if port.lower() == 'dedicated':
+        port = b'\x01'
+    elif port.lower() in ('ml2', 'ocp'):
+        port = b'\x02\x00'
+    elif port.lower() == 'lom':
+        if model == '7x58':
+            port = b'\x00\x02'
+        else:
+            port = b'\x00\x00'
+    else:
+        port = port.split(' ')
+        port = bytes(bytearray([int(x) for x in port]))
+    currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if port == currport:
+        sys.stdout.write('XCC port already set to "{}"\n'.format(oport))
+        return
+    sys.stdout.write('Setting XCC port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    s.raw_command(0xc, 1, b'\x01\xc0' + port)
+    tries = 60
+    while currport != port and tries:
+        tries -= 1
+        time.sleep(0.5)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+        currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if not tries:
+        raise Exception('Timeout attempting to set port')
+    sys.stdout.write('Complete\n')
+
+
+def check_vlan(s, vlan, channel):
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    currvlan = bytes(s.raw_command(0xc, 2, bytearray([channel, 0x14 ,0, 0]))['data'][1:])
+    if bytearray(currvlan)[1] & 0b10000000 == 0:
+        currvlan = b'\x00\x00'
+    return currvlan == vlan
+
+
+def set_vlan(s, vlan, channel):
+    ovlan = vlan
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    if check_vlan(s, ovlan, channel):
+        sys.stdout.write('VLAN already configured to "{0}"\n'.format(ovlan))
+        return False
+    rsp = s.raw_command(0xc, 1, bytearray([channel, 0x14]) + vlan)
+    if rsp.get('code', 1) == 0:
+        print('VLAN configured to "{}"'.format(ovlan))
+    else:
+        print('Error setting vlan: ' + repr(rsp))
+    return True
+
+
+def get_lan_channel(s):
+    for chan in range(1, 16):
+        rsp = s.raw_command(6, 0x42, bytearray([chan]))['data']
+        if not rsp:
+            continue
+        medtype = int(rsp[1]) & 0b1111111
+        if medtype not in (4, 6):
+            continue
+        rsp = s.raw_command(0xc, 2, bytearray([2, chan, 5, 0, 0]))
+        if rsp.get('code', 1) == 0:
+            return chan
+    return 1
+
+
+def check_ipv4(s, ipaddr, channel):
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
+    return rsp == ipaddr
+
+def set_ipv4(s, ipaddr, channel):
+    oipaddr = ipaddr
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    if check_ipv4(s, oipaddr, channel):
+        print('IP Address already set to {}'.format(oipaddr))
+        return False
+    rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+    if rsp != 1:
+        sys.stdout.write("Changing configuration to static...")
+        sys.stdout.flush()
+        resp = s.raw_command(0xc, 1, bytearray([channel, 4, 1]))
+        tries = 0
+        while rsp != 1 and tries < 30:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            tries += 1
+            time.sleep(0.5)
+            rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+        sys.stdout.write('Complete\n')
+        sys.stdout.flush()
+    print('Setting IP to {}'.format(oipaddr))
+    s.raw_command(0xc, 1, bytearray([channel, 3]) + ipaddr)
+    return True
+
+
+def check_subnet(s, prefix, channel):
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
+    return rsp == mask
+
+def set_subnet(s, prefix, channel):
+    oprefix = prefix
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    if check_subnet(s, prefix, channel):
+        print('Subnet Mask already set to /{}'.format(oprefix))
+        return False
+    print('Setting subnet mask to /{}'.format(oprefix))
+    s.raw_command(0xc, 1, bytearray([channel, 6]) + mask)
+    return True
+
+
+def check_gateway(s, gw, channel):
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    return rsp == gw
+
+def set_gateway(s, gw, channel):
+    ogw = gw
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    if check_gateway(s, ogw, channel):
+        print('Gateway already set to {}'.format(ogw))
+        return False
+    print('Setting gateway to {}'.format(ogw))
+    s.raw_command(0xc, 1, bytearray([channel, 12]) + gw)
+    return True
+
+def dotwait():
+    sys.stdout.write('.')
+    sys.stdout.flush()
+    time.sleep(0.5)
+
+def main():
+    a = argparse.ArgumentParser(description='Locally configure a BMC device')
+    a.add_argument('-v', help='vlan id or off to disable vlan tagging')
+    a.add_argument('-p', help='Which port to use (dedicated, lom, ocp, ml2)')
+    a.add_argument('-i', help='JSON configuration file to read for '
+                   'configuration')
+    a.add_argument('-c', help='Use Confluent API to direct BMC configuration',
+                   action='store_true')
+    args = a.parse_args()
+    if args.i:
+        bmccfg = json.load(open(args.i))
+    elif args.c:
+        bmccfgsrc = subprocess.check_output(
+            [sys.executable, apiclient, '/confluent-api/self/bmcconfig', '-j'])
+        bmccfg = json.loads(bmccfgsrc)
+    else:
+        bmccfg = {}
+    if args.p is not None:
+        bmccfg['bmcport'] = args.p
+    if args.v is not None:
+        bmccfg['bmcvlan'] = args.v
+    vendor = open('/sys/devices/virtual/dmi/id/sys_vendor').read()
+    vendor = vendor.strip()
+    try:
+        model = open('/sys/devices/virtual/dmi/id/product_sku').read().strip()
+        if model == 'none':
+            raise Exception('No SKU')
+    except Exception:
+        model = open('/sys/devices/virtual/dmi/id/product_name').read()
+    if vendor in ('Lenovo', 'IBM'):
+        if '[' in model and ']' in model:
+            model = model.split('[')[1].split(']')[0]
+        model = model[:4].lower()
+    s = Session('/dev/ipmi0')
+    if not bmccfg:
+        print("No BMC configuration specified, exiting.")
+        return
+    if bmccfg.get('bmcport', None):
+        channel = set_port(s, bmccfg['bmcport'], vendor, model)
+    else:
+        channel = get_lan_channel(s)
+    awaitvlan = False
+    awaitip = False
+    awaitprefix = False
+    awaitgw = False
+    if bmccfg.get('bmcvlan', None):
+        awaitvlan = set_vlan(s, bmccfg['bmcvlan'], channel)
+    if bmccfg.get('bmcipv4', None):
+        awaitip = set_ipv4(s, bmccfg['bmcipv4'], channel)
+    if bmccfg.get('prefixv4', None):
+        awaitprefix = set_subnet(s, bmccfg['prefixv4'], channel)
+    if bmccfg.get('bmcgw', None):
+        awaitgw = set_gateway(s, bmccfg['bmcgw'], channel)
+    sys.stdout.write('Waiting for changes to take effect...')
+    sys.stdout.flush()
+    while awaitvlan and not check_vlan(s, bmccfg['bmcvlan'], channel):
+        dotwait()
+    while awaitip and not check_ipv4(s, bmccfg['bmcipv4'], channel):
+        dotwait()
+    while awaitprefix and not check_subnet(s, bmccfg['prefixv4'], channel):
+        dotwait()
+    while awaitgw and not check_gateway(s, bmccfg['bmcgw'], channel):
+        dotwait()
+    sys.stdout.write('done\n')
+    sys.stdout.flush()
+    cfgmod = get_remote_config_mod(vendor, model)
+    if cfgmod:
+        with open('configbmc.configmod', 'w+') as cm:
+            cm.write('configmod: {0}\n'.format(cfgmod))
+        sys.stdout.write('Requesting remote configuration of authentication...')
+        sys.stdout.flush()
+        bmccfgsrc = subprocess.check_output(
+            [sys.executable, apiclient, '/confluent-api/self/remoteconfigbmc', 'configbmc.configmod'])
+        sys.stdout.write('done\n')
+        sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -0,0 +1,65 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    /usr/libexec/platform-python $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}

confluent_osdeploy/genesis/profiles/default/scripts/onboot.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+. /tmp/functions
+# This runs whenever this genesis profile boots for customization
+# purposes
+
+# run_remote and run_remote_python are available to download scripts and
+# execute them.
+
+# This is an example to request the BMC be configured on the network
+# according to how confluent has things configured:
+# run_remote_python configbmc -c

confluent_osdeploy/rhvh4/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+echo -n "" >> /tmp/net.ifaces
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+        echo "Initializing auto detected console when installer starts" > $autocons
+    fi
+fi
+. /lib/anaconda-lib.sh
+wait_for_kickstart

confluent_osdeploy/rhvh4/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -0,0 +1,120 @@
+#!/bin/sh
+[ -e /tmp/confluent.initq ] && return 0
+if [ -f /tmp/dd_disk ]; then
+    for dd in $(cat /tmp/dd_disk); do
+        if [ -e $dd ]; then
+            driver-updates --disk $dd $dd
+        fi
+    done
+fi
+mkdir -p /etc/confluent
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=0
+echo -n > /etc/confluent/confluent.info
+cd /sys/class/net
+while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+    TRIES=$((TRIES + 1))
+    for currif in *; do
+        ip link set $currif up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+oum=$(umask)
+# 4.3 is rhel7 with useless curl but has python, 4.4 has usable curl but no python..
+if [ -x /usr/bin/python ]; then
+    python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+else
+    mgr=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info| sed -e 's/^EXTMGRINFO: //' | awk -F'|' '{print $1 " " $2 " " $3}' |grep 1$ | awk 'NR < 2')
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info| sed -e 's/^EXTMGRINFO: //' | awk -F'|' '{print $1 " " $2 " " $3}' | awk 'NR < 2')
+    fi
+    mgtiface=$(echo $mgr | awk '{print $2}')
+    mgr=$(echo $mgr | awk '{print $1}')
+    if [ ! -f /etc/confluent/confluent.apikey ]; then
+        /opt/confluent/bin/clortho $nodename $mgr > /etc/confluent/confluent.apikey
+    fi
+    if echo $mgr | grep '%' > /dev/null; then
+        echo $mgr | awk -F% '{print $2}' > /tmp/confluent.ifidx
+    fi
+    apikey=$(cat /etc/confluent/confluent.apikey)
+    if echo $mgr | grep ':' > /dev/null; then
+        mgr="[$mgr]"
+    fi
+    curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -H "CONFLUENT_MGTIFACE: $mgtiface" https://$mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+fi
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+echo $ifname > /tmp/net.ifaces
+
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+hostname=$nodename
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+    hostname=$hostname.$dnsdomain
+fi
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
+proto=${proto#protocol: }
+textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+	autocons=$(cat /tmp/01-autocons.devnode)
+	if [ ! -z "$autocons" ]; then
+	    echo Auto-configuring installed system to use text console
+	    echo Auto-configuring installed system to use text console > $autocons
+	    cp /tmp/01-autocons.conf /etc/cmdline.d/
+	else
+	    echo "Unable to automatically detect requested text console"
+	fi
+fi
+
+# RHV must not have a repo
+#echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf
+echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf
+kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
+root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
+export kickstart
+export root
+autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+

confluent_osdeploy/rhvh4/initramfs/usr/lib/dracut/hooks/initqueue/finished/confluent.sh

@@ -0,0 +1 @@
+[ -e /tmp/confluent.initq ]

confluent_osdeploy/rhvh4/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+while [ -h $BUNDLENAME ]; do
+    BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
+done
+
+cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
+mkdir -p /sysroot/etc/confluent/
+cp -a /tls /sysroot/etc/confluent
+cp -a /etc/confluent/* /sysroot/etc/confluent/
+sed -i 's/install::/install:*:/' /sysroot/etc/shadow
+sed -i 's/root::/root:*:/' /sysroot/etc/shadow
+mkdir -p /sysroot/root/.ssh
+#chmod 700 /sysroot/root/.ssh
+cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+#chmod 600 /sysroot/root/.ssh/authorized_keys
+mkdir -p /sysroot/etc/ssh/
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent

confluent_osdeploy/rhvh4/profiles/default/initprofile.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+ln -s $(find $1 -name '*host-image*rpm') $2/image.rpm
+

confluent_osdeploy/rhvh4/profiles/default/kickstart

@@ -0,0 +1,74 @@
+# In this OS profile, data is largely filled in during the %pre
+# phase, rather than the kickstart actually having the content.
+# None of the files shall be replaced during an upgrade in
+# /var/lib/confluent/public/os/<profile>, so customization should
+# be done by modifying files in /var/lib/confluent/public/os/<profile>
+
+# /tmp/rootpw will provide a 'rootpw' line, either locking password if not configured
+# or the crypted form.
+%include /tmp/rootpw
+
+# timezone is fetched from confluent server, which provides the
+# timezone that the management server itself is in by default.
+%include /tmp/timezone
+
+# similar to timezone, confluent is asked to provide the
+# deployment servers language info and replicate that
+# to the deployment target.
+%include /tmp/langinfo
+
+# bootloader may be specified if crypted.grubpassword is set on
+# a node. This will become a bootloader line if a password
+# was specified
+%include /tmp/grubpw
+
+# The default partition scheme is applied to a single drive, using
+# the getinstalldisk script to make a best guess as to the most
+# appropriate device. See pre.sh and getinstalldisk to customize
+# the automatic behavior, or comment out/delete the 
+# following line and provide your own manual partition plan
+# instead
+%include /tmp/partitioning
+
+liveimg --url=file:///tmp/install.img
+
+reboot
+
+# A kickstart.custom file is provided to enable easily adding
+# kickstart content without modifying the stock file.
+# While the stock file may be safely modified, using the .custom
+# files may allow for more easily rebasing custom profiles to newer
+# stock profiles
+%include /tmp/kickstart.custom
+
+%pre
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
+. /tmp/preinst.sh
+%end
+
+%post --nochroot
+mkdir -p /mnt/sysimage/etc/confluent
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+
+# Hook firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
+chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+%end
+
+%post
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+nodectl init
+systemctl enable firstboot
+chgrp ssh_keys /etc/ssh/ssh*key
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+%end

confluent_osdeploy/rhvh4/profiles/default/kickstart.custom

@@ -0,0 +1,4 @@
+# Any custom kickstart additions may go here.  It may be required to
+# modify base kickstart file to change certain portions, but adding
+# custom content to this file allows for most convenient way to 
+# refresh to a newer base profile if desired.

confluent_osdeploy/rhvh4/profiles/default/profile.yaml

@@ -0,0 +1,3 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/rhvh4/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/rhvh4/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/etc/confluent/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/rhvh4/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+export nodename mgr profile
+. /etc/confluent/functions
+
+
+run_remote firstboot.custom
+
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service

confluent_osdeploy/rhvh4/profiles/default/scripts/functions

@@ -0,0 +1,38 @@
+run_remote() {
+    requestedcmd="'$*'"
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    /usr/libexec/platform-python $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/rhvh4/profiles/default/scripts/post.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/rhvh4/profiles/default/scripts/post.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# need to copy over ssh key info
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+
+chmod 700 /etc/confluent
+chmod og-rwx /etc/confluent/*
+
+export mgr profile nodename
+. /etc/confluent/functions
+
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+
+
+if [ -f /tmp/cryptboot ]; then
+    run_remote tpm_luks.sh
+fi
+# This script will execute in the installed system, but using the installer kernel prior to reboot.
+# This is an appropriate place to run post install activities that do not require the actual installed
+# kernel to run. For example adding drivers that would be needed for first boot to run cleanly.
+# If, for example, there is a post script that has a dependency on a driver or filesystem that
+# cannot work until booting into the installer, use firstboot.sh instead
+
+# run_remote will download and execute from /var/lib/confluent/public/os/<profile>/scripts/ directory
+# run_remote_python will use the appropriate python interpreter path to run the specified script
+# A post.custom is provided to more conveniently hold customizations, see the post.custom file.
+
+# run_remote example.sh
+# run_remote_python example.py
+run_remote post.custom

confluent_osdeploy/rhvh4/profiles/default/scripts/pre.custom

@@ -0,0 +1,4 @@
+. /tmp/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/rhvh4/profiles/default/scripts/pre.sh

@@ -0,0 +1,100 @@
+#!/bin/sh
+
+# This runs prior to the installer beginning. This is used to rewrite the 
+# scripted install file, merging data from confluent and identifying
+# the most appropriate install source.
+
+# If you want to use a more custom partition plan, the easiest
+# method is to edit the kicktstart file and comment out or
+# delete %include /tmp/partitioning
+
+mgraw=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info| sed -e 's/^EXTMGRINFO: //' | awk -F'|' '{print $1 " " $2 " " $3}' |grep 1$ | awk 'NR < 2')
+if [ -z "$mgraw" ]; then
+    mgraw=$(grep ^EXTMGRINFO: /etc/confluent/confluent.info| sed -e 's/^EXTMGRINFO: //' | awk -F'|' '{print $1 " " $2 " " $3}' | awk 'NR < 2')
+fi
+mgraw=$(echo $mgraw | awk '{print $1}')
+if echo $mgraw | grep '%' > /dev/null; then
+    echo $mgraw | awk -F% '{print $2}' > /tmp/confluent.ifidx
+fi
+
+iface=$(grep -H $(cat /tmp/confluent.ifidx) /sys/class/net/*/ifindex | awk -F/ '{print $5}')
+nmcli c u $iface
+while ip -6 addr | grep tentative > /dev/null; do
+   sleep 0.5
+done
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+locale=$(grep ^locale: /etc/confluent/confluent.deploycfg)
+locale=${locale#locale: }
+keymap=$(grep ^keymap: /etc/confluent/confluent.deploycfg)
+keymap=${keymap#keymap: }
+echo lang $locale > /tmp/langinfo
+echo keyboard --vckeymap=$keymap >> /tmp/langinfo
+tz=$(grep ^timezone: /etc/confluent/confluent.deploycfg)
+tz=${tz#timezone: }
+ntpsrvs=""
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntpsrvs="--ntpservers="$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd,)
+fi
+echo timezone $ntpsrvs $tz --utc > /tmp/timezone
+rootpw=$(grep ^rootpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$rootpw" = null ]; then
+    echo "rootpw --lock" > /tmp/rootpw
+else
+    echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
+fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
+grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$grubpw" = "null" ]; then
+    touch /tmp/grubpw
+else
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
+fi
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host_*_key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.anaconda
+done
+cp /etc/ssh/sshd_config.anaconda /etc/ssh/sshd_config
+/usr/sbin/sshd -f /etc/ssh/sshd_config.anaconda
+systemctl start sshd
+if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
+    consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
+    TMUX= tmux a <> $consoledev >&0 2>&1 &
+fi
+cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+LUKSPARTY=''
+if [ "$cryptboot" == "tpm2" ]; then
+	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
+	echo $cryptboot >> /tmp/cryptboot
+fi
+
+
+export mgr profile nodename
+curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /tmp/functions
+. /tmp/functions
+run_remote_python getinstalldisk
+if [ -e /tmp/installdisk ]; then
+    echo clearpart --all --initlabel >> /tmp/partitioning
+    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
+    echo autopart --type=thinp --nohome $LUKSPARTY >> /tmp/partitioning
+fi
+cd $(mktemp -d)
+if [ -x /usr/bin/python ]; then
+	python=/usr/bin/python
+elif [ -x /usr/libexec/platform-python ]; then
+	python=/usr/libexec/platform-python
+fi
+$python /etc/confluent/apiclient /confluent-public/os/$profile/image.rpm -o image.rpm
+rpm2cpio image.rpm | cpio -dumi
+ln -s $(find $(pwd) -name *img) /tmp/install.img
+cd -
+$python /etc/confluent/apiclient /confluent-public/os/$profile/kickstart.custom -o /tmp/kickstart.custom
+run_remote pre.custom