Handle unresolvable switches gracecfully

Rather than a trace log,
keep it to only the event log.

.gitignore

@@ -1 +1,4 @@
-.idea
+*.pyc
+.*.
+confluent_client/man/man*
+.*.sw*

confluent_client/addattribs.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+import os
+import sys
+path = os.path.dirname(os.path.realpath(__file__))
+try:
+    sys.path.remove(path)
+except Exception:
+    pass
+path = os.path.realpath(os.path.join(path, '..', 'confluent_server'))
+sys.path.append(path)
+
+import confluent.config.attributes as attr
+import shutil
+
+shutil.copyfile('doc/man/nodeattrib.ronn.tmpl', 'doc/man/nodeattrib.ronn')
+shutil.copyfile('doc/man/nodegroupattrib.ronn.tmpl', 'doc/man/nodegroupattrib.ronn')
+with open('doc/man/nodeattrib.ronn', 'a') as outf:
+    for field in sorted(attr.node):
+        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+with open('doc/man/nodegroupattrib.ronn', 'a') as outf:
+    for field in sorted(attr.node):
+        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+
+

confluent_client/bin/collate

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -43,6 +43,8 @@ argparser.add_option('-d', '--diff', action='store_true',
                           'output group and others')
 argparser.add_option('-w', '--watch', action='store_true',
                      help='Show intermediate results while running')
+argparser.add_option('-g', '--groupcount', action='store_true',
+                     help='Show count of output groups rather than the actual output')                     
 argparser.add_option('-s', '--skipcommon', action='store_true',
                      help='Do not print most common result, only non modal '
                           'groups, useful when combined with -d')
@@ -69,6 +71,9 @@ def print_current():
     if options.diff:
         grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
                           reverse=options.reverse, basenode=options.base)
+    elif options.groupcount:
+        grouped.generate_byoutput()
+        print(len(grouped.byoutput))
     else:
         grouped.print_all(skipmodal=options.skipcommon,
                                count=options.count,
@@ -83,15 +88,18 @@ while fullline:
     for line in fullline.split('\n'):
         if not line:
             continue
-        if ': ' not in line:
+        if ':' not in line:
             line = 'UNKNOWN: ' + line
         if options.log:
-            node, output = line.split(': ', 1)
+            node, output = line.split(':', 1)
+            output = output.lstrip()
             currlog = options.log.format(node=node, nodename=node)
             with open(currlog, mode='a')  as log:
                 log.write(output + '\n')
             continue
-        grouped.add_line(*line.split(': ', 1))
+        node, output = line.split(':', 1)
+        output = output.lstrip()
+        grouped.add_line(node, output)
         if options.watch:
             if not holdoff:
                 holdoff = os.times()[4] + 0.250

confluent_client/bin/confetty

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -76,6 +76,11 @@ import confluent.termhandler as termhandler
 import confluent.tlvdata as tlvdata
 import confluent.client as client
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 conserversequence = '\x05c'  # ctrl-e, c
 clearpowermessage = False
 
@@ -89,6 +94,10 @@ except NameError:
 netserver = None
 laststate = {}
 
+try:
+    input = raw_input
+except NameError:
+    pass
 
 class BailOut(Exception):
     def __init__(self, errorcode=0):
@@ -195,9 +204,9 @@ def prompt():
     if os.environ.get('TERM', '') not in ('linux'):
         sys.stdout.write('\x1b]0;confetty: %s\x07' % target)
     try:
-        return raw_input(target + ' -> ')
+        return input(target + ' -> ')
     except KeyboardInterrupt:
-        print ""
+        print("")
         return ""
     except EOFError:  # ctrl-d
         print("exit")
@@ -283,6 +292,8 @@ def rcompleter(text, state):
 
 
 def parse_command(command):
+    if isinstance(command, list):
+        return command
     try:
         args = shlex.split(command, posix=True)
     except ValueError as ve:
@@ -295,8 +306,11 @@ currchildren = None
 
 
 def print_result(res):
+    global exitcode
     if 'errorcode' in res or 'error' in res:
-        print res['error']
+        print(res['error'])
+        if 'errorcode' in res:
+            exitcode |= res['errorcode']
         return
     if 'databynode' in res:
         print_result(res['databynode'])
@@ -309,9 +323,9 @@ def print_result(res):
             attrstr = '%s=%s' % (key, recurse_format(res[key]))
         elif not isinstance(res[key], dict):
             try:
-                print '{0}: {1}'.format(key, res[key])
+                print('{0}: {1}'.format(key, res[key]))
             except UnicodeEncodeError:
-                print '{0}: {1}'.format(key, repr(res[key]))
+                print('{0}: {1}'.format(key, repr(res[key])))
             continue
         elif 'value' in res[key] and res[key]['value'] is not None:
             attrstr = '%s="%s"' % (key, res[key]['value'])
@@ -324,7 +338,7 @@ def print_result(res):
         else:
             sys.stdout.write('{0}: '.format(key))
             if isinstance(res[key], str) or isinstance(res[key], unicode):
-                print res[key]
+                print(res[key])
             else:
                 print_result(res[key])
             continue
@@ -423,10 +437,10 @@ def do_command(command, server):
         for res in session.read(targpath):
             if 'item' in res:  # a link relation
                 if type(res['item']) == dict:
-                    print res['item']["href"]
+                    print(res['item']["href"])
                 else:
                     for item in res['item']:
-                        print item["href"]
+                        print(item["href"])
             else:  # generic attributes to list
                 if 'error' in res:
                     sys.stderr.write(res['error'] + '\n')
@@ -504,6 +518,13 @@ def makecall(callout, args):
             if 'errorcode' in response:
                 exitcode = response['errorcode']
             sys.stderr.write('Error: ' + response['error'] + '\n')
+        if 'databynode' in response:
+            lresponse = response['databynode']
+            for node in lresponse:
+                if 'errorcode' in lresponse[node]:
+                    exitcode = lresponse[node]['errorcode']
+                if 'error' in lresponse[node]:
+                    sys.stderr.write('{0}: Error - {1}\n'.format(node, lresponse[node]['error']))
 
 
 def clearvalues(resource, attribs):
@@ -844,7 +865,7 @@ def server_connect():
         passphrase = os.environ['CONFLUENT_PASSPHRASE']
         session.authenticate(username, passphrase)
     while not session.authenticated:
-        username = raw_input("Name: ")
+        username = input("Name: ")
         passphrase = getpass.getpass("Passphrase: ")
         session.authenticate(username, passphrase)
 
@@ -864,7 +885,7 @@ def main():
     global inconsole
     try:
         server_connect()
-    except EOFError, KeyboardInterrupt:
+    except (EOFError, KeyboardInterrupt) as _:
         raise BailOut(0)
     except socket.gaierror:
         sys.stderr.write('Could not connect to confluent\n')
@@ -893,8 +914,7 @@ def main():
         do_command("start /nodes/%s/console/session" % session_node, netserver)
         doexit = True
     elif shellargs:
-        command = " ".join(shellargs)
-        do_command(command, netserver)
+        do_command(shellargs, netserver)
         quitconfetty(fullexit=True, fixterm=False)
 
     powerstate = None
@@ -922,6 +942,7 @@ def main():
                         updatestatus(data)
                         continue
                     if data is not None:
+                        data = client.stringify(data)
                         if clearpowermessage:
                             sys.stdout.write("\x1b[2J\x1b[;H")
                             clearpowermessage = False
@@ -937,7 +958,13 @@ def main():
                             # don't bother churning if minute does not change
                             laststate['showtime'] = now
                             updatestatus()
-                        sys.stdout.flush()
+                        try:
+                            sys.stdout.flush()
+                        except Exception:
+                            # EWOULDBLOCK causes this to raise, ignore
+                            # this scenario comfortable that it
+                            # will come out soon enough
+                            pass
                     else:
                         deadline = 5
                         connected = False
@@ -945,7 +972,7 @@ def main():
                             try:
                                 server_connect()
                                 connected = True
-                            except (socket.gaierror, socket.error):
+                            except Exception:
                                 pass
                             if not connected:
                                 time.sleep(1)
@@ -997,4 +1024,6 @@ if __name__ == '__main__':
         if deadline and os.times()[4] < deadline:
             sys.stderr.write('[Exited early, hit enter to continue]')
             sys.stdin.readline()
+        if errcode == 0:
+            errcode = exitcode
         sys.exit(errcode)

confluent_client/bin/confluent2ansible

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import optparse
 import signal
 import sys

confluent_client/bin/confluent2lxca

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal

confluent_client/bin/confluent2xcat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal
@@ -26,7 +26,7 @@ def lookupdata(data, key):
 
 def main():
     argparser = optparse.OptionParser(
-    usage='''\n       %prog noderange -o ansible.hosts
+    usage='''\n       %prog noderange -o xcatnodes.def
              \n ''')
     argparser.add_option('-o', '--output',
                          help='xCAT stanza file')

confluent_client/bin/dir2img

@@ -1,19 +1,53 @@
-#!/bin/sh
+#!/usr/bin/python
 # This will take a given directory and make a 'big floppy image'
 # out of it, suitable for nodemedia upload.
 
-if [ -z "$1" -o -z "$2" ]; then
-        echo "Usage: $0 <directory> <imagefile>"
-        exit 1
-fi
-# Get the needed payload side
-SIZE=$(du -sB 512 $1|awk '{print $1}')
-ENTRIES=$(find $1 |wc -l)
-# Also, each file and directory has overhead, pad size by 32kb per file,
-# which should be overkill but other values proved to be inadequate
-# A deeper understanding of VFAT would probably allow for more precise value
-SIZE=$((SIZE + ENTRIES * 64))
-dd if=/dev/zero of=$2 bs=512 count=$SIZE
-# Make a big single sided floppy with many many tracks, saves on complex math
-mformat -i $2 -d 1 -t $SIZE -s 1 -h 1 ::
-mcopy -i $2 -s $1/* ::
+import fcntl
+import glob
+import os
+import subprocess
+import sys
+
+def create_image(directory, image):
+    ents = 0
+    datasz = 512
+    for dir in os.walk(sys.argv[1]):
+        ents += 1
+        for filen in dir[2]:
+            ents += 1
+            filename = os.path.join(dir[0], filen)
+            currsz = os.path.getsize(filename)
+            # assuming up to 65k cluster
+            currsz = (currsz // 512 +1) * 512
+            datasz += currsz
+    datasz += ents * 32768
+    datasz = datasz // 512 + 1
+    with open(image, 'wb') as imgfile:
+        imgfile.seek(datasz * 512 - 1)
+        imgfile.write(b'\x00')
+    subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
+                            str(datasz), '-s', '1','-h', '1', '::'])
+    # Some clustered filesystems will have the lock from mformat
+    # linger after close (mformat doesn't unlock)
+    # do a blocking wait for shared lock and then explicitly
+    # unlock between calls to mtools
+    with open(image, 'rb') as imgfile:
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_SH)
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_UN)
+    cpycmd = ['mcopy', '-i', image, '-s']
+    cpycmd.extend(glob.glob('{0}/*'.format(directory)))
+    cpycmd.append('::')
+    subprocess.check_call(cpycmd)
+    # though not necessary for us, make sure dir2img doesn't have a lingering
+    # flock from mcopy for any subsequent commands
+    with open(image, 'rb') as imgfile:
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_SH)
+        fcntl.flock(imgfile.fileno(), fcntl.LOCK_UN)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        sys.stderr.write("Usage: {0} <directory> <imagefile>".format(
+            sys.argv[0]))
+        sys.exit(1)
+    create_image(sys.argv[1], sys.argv[2])

confluent_client/bin/nodeattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -50,6 +50,9 @@ argparser.add_option('-c', '--clear', action='store_true',
                      help='Clear attributes')
 argparser.add_option('-p', '--prompt', action='store_true',
                      help='Prompt for attribute values interactively')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Prompt if trying to set attributes on more '
+                          'than specified number of nodes')
 (options, args) = argparser.parse_args()
 
 
@@ -87,6 +90,7 @@ if len(args) > 1:
                     if oneval != twoval:
                         print('Values did not match.')
                 argassign[arg] = twoval
+        session.stop_if_noderange_over(noderange, options.maxnodes)
         exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
     try:
         # setting user output to what the user inputs
@@ -126,6 +130,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodebmcreset

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,8 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Number of nodes to affect before prompting for confirmation')
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,8 +45,10 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'configuration/management_controller/reset', 'reset', key='state', errnodes=errorNodes)   # = 0 if successful
+if success != 0:
+    sys.exit(success)
 
 # Determine which nodes were successful and print them
 
@@ -56,7 +60,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": BMC Reset Successful"
+    print(node + ": BMC Reset Successful")
 
 
 sys.exit(success)

confluent_client/bin/nodeboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -42,6 +42,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                      default=False,
                      help='Request the boot device be persistent rather than '
                           'one time')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to boot, '
+                          'prompting if over the threshold')                        
 
 (options, args) = argparser.parse_args()
 
@@ -54,8 +58,8 @@ except IndexError:
     sys.exit(1)
 client.check_globbing(noderange)
 bootdev = None
-if len(sys.argv) > 2:
-    bootdev = sys.argv[2]
+if len(args) > 1:
+    bootdev = args[1]
     if bootdev in ('net', 'pxe'):
         bootdev = 'network'
 session = client.Command()
@@ -66,6 +70,7 @@ else:
     bootmode = 'uefi'
 
 errnodes = set([])
+session.stop_if_noderange_over(noderange, options.maxnodes)
 rc = session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist,

confluent_client/bin/nodeconfig

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -54,6 +54,12 @@ argparser.add_option('-d', '--detail', dest='detail',
                      action='store_true', default=False,
                      help='Provide verbose information as available, such as '
                           'help text and possible valid values')
+argparser.add_option('-e', '--extra', dest='extra',
+                     action='store_true', default=False,
+                     help='Access extra configuration.  Extra configuration is generally '
+                           'reserved for unpopular or redundant options that may be slow to '
+                           'read.  Notably the IMM category on Lenovo settings is considered '
+                           'to be extra configuration')
 argparser.add_option('-x', '--exclude', dest='exclude',
                      action='store_true', default=False,
                      help='Treat positional arguments as items to not '
@@ -68,6 +74,10 @@ argparser.add_option('-r', '--restoredefault', default=False,
                      help='Restore the configuration of the node '
                           'to factory default for given component. '
                           'Currently only uefi is supported')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure, '
+                          'prompting if over the threshold')
 (options, args) = argparser.parse_args()
 
 cfgpaths = {
@@ -98,6 +108,9 @@ setmode = None
 assignment = {}
 queryparms = {}
 printsys = []
+printbmc = []
+printextbmc = []
+printallbmc = False
 setsys = {}
 forceset = False
 needval = None
@@ -122,7 +135,7 @@ def _assign_value():
 
 
 def parse_config_line(arguments):
-    global setmode, forceset, key, value, needval, candidate, path, attrib
+    global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
     for param in arguments:
         if param == 'show':
             continue  # forgive muscle memory of pasu users
@@ -154,6 +167,8 @@ def parse_config_line(arguments):
             if setmode != False:
                 bailout('Cannot do set and query in same command')
             if '.' not in param:
+                if param == 'bmc':
+                    printallbmc = True
                 matchedparms = False
                 for candidate in cfgpaths:
                     if candidate.startswith('{0}.'.format(param)):
@@ -169,10 +184,19 @@ def parse_config_line(arguments):
                                 del queryparms[path]
                             except KeyError:
                                 pass
-                if not matchedparms:
+                if param.lower() == 'imm':
+                    printextbmc.append(param)
+                    options.extra = True
+                elif not matchedparms:
                     printsys.append(param)
             elif param not in cfgpaths:
-                printsys.append(param)
+                if param.startswith('bmc.'):
+                    printbmc.append(param.replace('bmc.', ''))
+                elif param.lower().startswith('imm'):
+                    options.extra = True
+                    printextbmc.append(param)
+                else:
+                    printsys.append(param)
             else:
                 path, attrib = cfgpaths[param]
                 path = '/noderange/{0}/{1}'.format(noderange, path)
@@ -197,19 +221,29 @@ else:
     parse_config_line(args[1:])
 session = client.Command()
 rcode = 0
-if options.restoredefault and options.restoredefault.lower() in (
+if options.restoredefault:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    if options.restoredefault.lower() in (
         'sys', 'system', 'uefi', 'bios'):
-    for fr in session.update(
-            '/noderange/{0}/configuration/system/clear'.format(noderange),
-            {'clear': True}):
-        rcode |= client.printerror(fr)
-    sys.exit(rcode)
-elif options.restoredefault:
-    sys.stderr.write(
-        'Unrecognized component to restore defaults: {0}\n'.format(
-            options.restoredefault))
-    sys.exit(1)
+        for fr in session.update(
+                '/noderange/{0}/configuration/system/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    elif options.restoredefault.lower() in (
+        'bmc', 'imm', 'xcc'):
+        for fr in session.update(
+                '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    else:
+        sys.stderr.write(
+            'Unrecognized component to restore defaults: {0}\n'.format(
+                options.restoredefault))
+        sys.exit(1)
 if setmode:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     if options.exclude:
         sys.stderr.write('Cannot use exclude and assign at the same time\n')
         sys.exit(1)
@@ -217,8 +251,12 @@ if setmode:
     attrnamebypath = {}
     for key in assignment:
         if key not in cfgpaths:
-            path = 'configuration/system/all'
-            attrib = key
+            if key.startswith('bmc.'):
+                path = 'configuration/management_controller/extended/all'
+                attrib = key.replace('bmc.', '')
+            else:
+                path = 'configuration/system/all'
+                attrib = key
         else:
             path, attrib = cfgpaths[key]
         if path not in updatebypath:
@@ -250,6 +288,15 @@ else:
                                       NullOpt(), queryparms[path])
         if rc:
             sys.exit(rc)
+    if printsys == 'all' or printextbmc or printbmc or printallbmc:
+        if printbmc or not printextbmc:
+         rcode = client.print_attrib_path(
+                        '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
+                        session, printbmc, options, attrprefix='bmc.')
+        if options.extra:
+            rcode |= client.print_attrib_path(
+                    '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange),
+                    session, printextbmc, options)
     if printsys or options.exclude:
         if printsys == 'all':
             printsys = []

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -51,27 +51,28 @@ if options.tile:
         nodes.append(node)
     initial = True
     pane = 0
+    sessname = 'nodeconsole_{0}'.format(os.getpid())
     for node in sortutil.natural_sort(nodes):
+        panename = '{0}:{1}'.format(sessname, pane)
         if initial:
             initial = False
             subprocess.call(
                 ['tmux', 'new-session', '-d', '-s',
-                 'nodeconsole_{0}'.format(os.getpid()), '-x', '800', '-y',
+                 sessname, '-x', '800', '-y',
                  '800', '{0} -m 5 start /nodes/{1}/console/session'.format(
                     confettypath, node)])
         else:
-            subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
-            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
-            pane += 1
+            subprocess.call(['tmux', 'select-pane', '-t', sessname])
+            subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
             subprocess.call(
-                ['tmux', 'split', '-h',
+                    ['tmux', 'split', '-h', '-t', sessname,
                  '{0} -m 5 start /nodes/{1}/console/session'.format(
                      confettypath, node)])
-            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
-    subprocess.call(['tmux', 'select-pane', '-t', '0'])
-    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
-        os.getpid()))
+            subprocess.call(['tmux', 'select-layout', '-t', sessname, 'tiled'], stdout=null)
+            pane += 1
+    subprocess.call(['tmux', 'select-pane', '-t', sessname])
+    subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
+    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
              '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodedefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/noderange/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodedeploy

@@ -0,0 +1,141 @@
+#!/usr/bin/python3
+
+
+import argparse
+import os
+import sys
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+def cleararm(nr, cli):
+    nodes = set([])
+    for rsp in cli.read('/noderange/{0}/attributes/current'.format(nr)):
+        for node in rsp.get('databynode', {}):
+            nodeinfo = rsp['databynode'][node]
+            for attr in nodeinfo:
+                if attr == 'deployment.apiarmed':
+                    curr = nodeinfo[attr].get('value', '')
+                    if curr == 'continuous':
+                        nodes.add(node)
+    noderange = nr
+    if nodes:
+        noderange += ',-({0})'.format(','.join(nodes))
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(noderange),
+                          {'deployment.apiarmed': ''}):
+        pass
+
+
+def armonce(nr, cli):
+    nodes = set([])
+    for rsp in cli.read('/noderange/{0}/attributes/current'.format(nr)):
+        for node in rsp.get('databynode', {}):
+            nodeinfo = rsp['databynode'][node]
+            for attr in nodeinfo:
+                if attr == 'deployment.apiarmed':
+                    curr = nodeinfo[attr].get('value', '')
+                    if curr == 'continuous':
+                        nodes.add(node)
+    noderange = nr
+    if nodes:
+        noderange += ',-({0})'.format(','.join(nodes))
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(noderange),
+                          {'deployment.apiarmed': 'once'}):
+        pass
+
+
+def setpending(nr, profile, cli):
+    args = {'deployment.pendingprofile': profile}
+    if not profile.startswith('genesis-'):
+        args['deployment.stagedprofile'] = ''
+        args['deployment.profile'] = ''
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(nr),
+                          args):
+        pass
+
+
+def clearpending(nr, cli):
+    for rsp in cli.update('/noderange/{0}/attributes/current'.format(nr),
+                          {'deployment.pendingprofile': ''}):
+        pass
+
+def main(args):
+    ap = argparse.ArgumentParser(description='Deploy OS to nodes')
+    ap.add_argument('-c', '--clear', help='Clear any pending deployment action', action='store_true')
+    ap.add_argument('-n', '--network', help='Initiate deployment over PXE/HTTP', action='store_true')
+    ap.add_argument('-p', '--prepareonly', help='Prepare only, skip any interaction with a BMC associated with this deployment action', action='store_true')
+    ap.add_argument('-m', '--maxnodes', help='Specifiy a maximum nodes to be deployed')
+    ap.add_argument('noderange', help='Set of nodes to deploy')
+    ap.add_argument('profile', nargs='?', help='Profile name to deploy')
+    args, extra = ap.parse_known_args(args)
+    if args.profile is None and len(extra) == 1:
+        args.profile = extra[0]
+        extra = extra[1:]
+    if args.profile and not args.network:
+        sys.stderr.write('-n is a required argument currently to perform an install, optionally with -p\n')
+        return 1
+    if not args.profile and args.network:
+        sys.stderr.write('profile is a required argument to request a network deployment\n')
+        return 1
+    if extra:
+        sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n')
+    c = client.Command()
+    c.stop_if_noderange_over(args.noderange, args.maxnodes)
+    if args.clear:
+        cleararm(args.noderange, c)
+        clearpending(args.noderange, c)
+    elif args.profile:
+        armonce(args.noderange, c)
+        setpending(args.noderange, args.profile, c)
+    else:
+        databynode = {}
+        for r in c.read('/noderange/{0}/attributes/current'.format(args.noderange)):
+            dbn = r.get('databynode', {})
+            for node in dbn:
+                if node not in databynode:
+                    databynode[node] = {}
+                for attr in dbn[node]:
+                    if attr in ('deployment.pendingprofile', 'deployment.apiarmed', 'deployment.stagedprofile', 'deployment.profile'):
+                        databynode[node][attr] = dbn[node][attr].get('value', '')
+        for node in sortutil.natural_sort(databynode):
+            profile = databynode[node].get('deployment.pendingprofile', '')
+            if profile:
+                profile = 'pending: {}'.format(profile)
+            else:
+                profile = databynode[node].get('deployment.stagedprofile', '')
+                if profile:
+                    profile = 'staged: {}'.format(profile)
+                else:
+                    profile = databynode[node].get('deployment.profile', '')
+                    if profile:
+                        profile = 'completed: {}'.format(profile)
+                    else:
+                        profile= 'No profile pending or applied'
+            armed = databynode[node].get('deployment.apiarmed', '')
+            if armed in ('once', 'continuous'):
+                armed = ' (node authentication armed)'
+            else:
+                armed = ''
+            print('{0}: {1}{2}'.format(node, profile, armed))
+        sys.exit(0)
+    errnodes = set([])
+    if args.network and not args.prepareonly:
+        rc = c.simple_noderange_command(args.noderange, '/boot/nextdevice', 'network',
+                                        bootmode='uefi',
+                                        persistent=False,
+                                        errnodes=errnodes)
+        if errnodes:
+            sys.stderr.write(
+                'Unable to set boot device for following nodes: {0}\n'.format(
+                    ','.join(errnodes)))
+            return 1
+        rc |= c.simple_noderange_command(args.noderange, '/power/state', 'boot')
+        return rc
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

confluent_client/bin/nodediscover

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -92,6 +92,8 @@ def process_header(header):
             fields.append('hardwaremanagement.manager')
         elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
             fields.append('net.bmc.ipv4_gateway')
+        elif datum in ('bmc_gateway', 'xcc_gateway', 'imm_gateway'):
+            fields.append('net.bmc.ipv4_gateway')
         elif datum in ('bmcuser', 'username', 'user'):
             fields.append('secret.hardwaremanagementuser')
         elif datum in ('bmcpass', 'password', 'pass'):
@@ -152,6 +154,7 @@ def import_csv(options, session):
         for field in fields:
             if field in unique_fields:
                 unique_data[field] = set([])
+        broken = False
         for record in records:
             currfields = list(fields)
             nodedatum = {}
@@ -168,14 +171,16 @@ def import_csv(options, session):
                 nodedatum[currfield] = datum
             if not datum_complete(nodedatum):
                 sys.exit(1)
-            if not search_record(nodedatum, options, session):
+            if not search_record(nodedatum, options, session) and not broken:
                 blocking_scan(session)
             if not search_record(nodedatum, options, session):
                 sys.stderr.write(
                     "Could not match the following data: " +
                     repr(nodedatum) + '\n')
-                sys.exit(1)
+                broken = True
             nodedata.append(nodedatum)
+    if broken:
+        sys.exit(1)
     for datum in nodedata:
         maclist = search_record(datum, options, session)
         datum = datum_to_attrib(datum)
@@ -232,8 +237,12 @@ def clear_discovery(options, session):
             else:
                 print(repr(res))
 
-def list_matching_macs(options, session):
+def list_matching_macs(options, session, node=None, checknode=True):
     path = '/discovery/'
+    if node:
+        path += 'by-node/{0}/'.format(node)
+    elif checknode and options.node:
+        path += 'by-node/{0}/'.format(options.node)
     if options.model:
         path += 'by-model/{0}/'.format(options.model)
     if options.serial:
@@ -256,25 +265,25 @@ def list_matching_macs(options, session):
         path += 'by-mac/'
         return [x['item']['href'] for x in session.read(path)]
 
-def assign_discovery(options, session):
+def assign_discovery(options, session, needid=True):
     abort = False
     if options.importfile:
         return import_csv(options, session)
-    if not (options.serial or options.uuid or options.mac):
+    if not options.node:
+        sys.stderr.write("Node (-n) must be specified for assignment\n")
+        abort = True
+    if needid and not (options.serial or options.uuid or options.mac):
         sys.stderr.write(
             "UUID (-u), serial (-s), or ether address (-e) required for "
             "assignment\n")
         abort = True
-    if not options.node:
-        sys.stderr.write("Node (-n) must be specified for assignment\n")
-        abort = True
     if abort:
         sys.exit(1)
-    matches = list_matching_macs(options, session)
+    matches = list_matching_macs(options, session, None if needid else options.node, False)
     if not matches:
         # Do a rescan to catch missing requested data
         blocking_scan(session)
-        matches = list_matching_macs(options, session)
+        matches = list_matching_macs(options, session, None if needid else options.node, False)
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)
@@ -332,7 +341,7 @@ def main():
     parser.add_option('-o', '--order', dest='order',
                       help='Order output by given field', metavar='ORDER')
     (options, args) = parser.parse_args()
-    if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear'):
         parser.print_help()
         sys.exit(1)
     session = client.Command()
@@ -342,6 +351,8 @@ def main():
         clear_discovery(options, session)
     if args[0] == 'assign':
         assign_discovery(options, session)
+    if args[0] == 'reassign':
+        assign_discovery(options, session, False)
     if args[0] == 'rescan':
         blocking_scan(session)
         print("Rescan complete")

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,10 +33,15 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog [options] noderange [clear]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to clear if clearing log, '
+                          'prompting if over the threshold')            
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -45,11 +50,11 @@ except IndexError:
     sys.exit(1)
 client.check_globbing(noderange)
 deletemode = False
-if len(sys.argv) > 3:
+if len(args) > 2:
     argparser.print_help()
     sys.exit(1)
-if len(sys.argv) == 3:
-    if sys.argv[2] == 'clear':
+if len(args) == 2:
+    if args[1] == 'clear':
         deletemode = True
     else:
         argparser.print_help()
@@ -64,6 +69,8 @@ def format_event(evt):
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
+    if evt.get('log_id', None):
+        retparts.append(evt['log_id'] + ':')
     if 'component_type' in evt and evt['component_type'] is not None:
         dscparts.append(evt['component_type'])
     if 'component' in evt and evt['component'] is not None:
@@ -77,11 +84,15 @@ def format_event(evt):
             pass
         dscparts.append(evttext)
     retparts.append(' - '.join(dscparts))
-    return ' '.join(retparts)
+    msg = evt.get('message')
+    if not msg:
+        msg = ''
+    return ' '.join(retparts) + msg
 
 
 if deletemode:
     func = session.delete
+    session.stop_if_noderange_over(noderange, options.maxnodes)
 else:
     func = session.read
 for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
@@ -98,4 +109,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
             if 'events' in thisdata:
                evtdata = thisdata['events']
                for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print('{0}: {1}'.format(node, format_event(evt)))

confluent_client/bin/nodefirmware

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -59,6 +59,10 @@ argparser = optparse.OptionParser(
           "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='When updating, prompt if more than the specified '
+                          'number of servers will be affected')
+             
 (options, args) = argparser.parse_args()
 upfile = None
 try:
@@ -95,6 +99,7 @@ def get_update_progress(session, url):
 
 def update_firmware(session, filename):
     global exitcode
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     output = sq.ScreenPrinter(noderange, session)
     nodeurls = {}
     filename = os.path.abspath(filename)
@@ -151,7 +156,7 @@ def show_firmware(session):
                     for prefix in inv:
                         firmware_shown = True
                         printfirm(node, prefix, inv[prefix])
-    if not firmware_shown:
+    if not firmware_shown and not exitcode:
         argparser.print_help()
 
 
@@ -163,4 +168,4 @@ try:
         update_firmware(session, upfile)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -97,7 +97,7 @@ if len(args) > 1:
 
         requestargs=args[1:]
     except Exception as e:
-        print str(e)
+        print(str(e))
 
 if exitcode != 0:
     sys.exit(exitcode)
@@ -123,6 +123,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupdefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/nodegroups/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouplist

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo

confluent_client/bin/nodegroupremove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -49,4 +49,4 @@ for r in session.delete('/nodegroups/{0}'.format(noderange)):
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouprename

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo

confluent_client/bin/nodehealth

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,7 +32,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
 (options, args) = argparser.parse_args()
@@ -49,6 +50,7 @@ codemappings = {
     'warning': 1,
     'critical': 2,
     'failed': 2,
+    'unknown': 2,
 }
 
 def main():

confluent_client/bin/nodeidentify

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -31,7 +31,7 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off|blink]")
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -42,9 +42,6 @@ client.check_globbing(noderange)
 identifystate = None
 if len(sys.argv) > 2:
     identifystate = sys.argv[2]
-else:
-    argparser.print_help()
-    sys.exit(1)
 session = client.Command()
 exitcode = 0
 sys.exit(

confluent_client/bin/nodeinventory

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -34,7 +34,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 filters = []
 
@@ -50,15 +51,17 @@ def print_mem_info(node, prefix, meminfo):
     memdescfmt = '{0}GB PC'
     if meminfo['memory_type'] == 'DDR3 SDRAM':
         memdescfmt += '3-{1} '
-    elif meminfo['memory_type'] == 'DDR4 SDRAM':
+    elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
-    if meminfo['ecc']:
+    if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
     capacity = meminfo['capacity_mb'] / 1024
-    memdescfmt += meminfo['module_type']
+    modtype = meminfo.get('module_type', None)
+    if modtype:
+        memdescfmt += modtype
     memdesc = memdescfmt.format(capacity, meminfo['speed'])
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
@@ -66,10 +69,11 @@ def print_mem_info(node, prefix, meminfo):
     print('{0}: {1} model: {2}'.format(node, prefix, meminfo['model']))
     print('{0}: {1} serial number: {2}'.format(node, prefix,
                                                meminfo['serial']))
-    print('{0}: {1} manufacture date: {2}'.format(node, prefix,
-                                                  meminfo['manufacture_date']))
-    print('{0}: {1} manufacture location: {2}'.format(
-        node, prefix, meminfo['manufacture_location']))
+    if 'manufacture_date' in meminfo:
+        print('{0}: {1} manufacture date: {2}'.format(node, prefix,
+                                                      meminfo['manufacture_date']))
+        print('{0}: {1} manufacture location: {2}'.format(
+            node, prefix, meminfo['manufacture_location']))
 
 exitcode = 0
 
@@ -142,7 +146,7 @@ try:
                                 databynode[node] = {}
                             databynode[node][prefix] = inv
                         else:
-                            print '{0}: {1}: Not Present'.format(node, prefix)
+                            print('{0}: {1}: Not Present'.format(node, prefix))
                     continue
                 info = inv['information']
                 info.pop('board_extra', None)

confluent_client/bin/nodelicense

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo
@@ -36,21 +36,33 @@ exitcode = 0
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][install <file>]")
+          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete licenses from, '
+                          'prompting if over the threshold')                  
 (options, args) = argparser.parse_args()
 upfile = None
+downdir = None
+delete = False
 try:
     noderange = args[0]
     if len(args) > 1:
         if args[1] == 'install':
             upfile = args[2]
-        else:
-            components = ['all']
+        elif args[1] == 'save':
+            downdir = args[2]
+        elif args[1] == 'delete':
+            delete = args[2]
+        elif args[1] != 'list':
+            argparser.print_help()
+            sys.exit(1)        
 except IndexError:
     argparser.print_help()
     sys.exit(1)
 client.check_globbing(noderange)
 
+
 def install_license(session, filename):
     global exitcode
     resource = '/noderange/{0}/configuration/' \
@@ -58,27 +70,76 @@ def install_license(session, filename):
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
     for res in session.create(resource, instargs):
-        pass # print(repr(res))
-    show_licenses()
+        for node in res.get('databynode', []):
+            if 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(
+                    node, res['databynode'][node]['error']))
+                sys.exit(res['databynode'][node].get('errorcode', 1))        
+    show_licenses(session)
+
+
+def save_licenses(session, dirname):
+    global exitcode
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/save_licenses'.format(noderange)
+    filename = os.path.abspath(dirname)
+    instargs = {'dirname': filename}
+    for res in session.create(resource, instargs):
+        for node in res.get('databynode', {}):
+            fname = res['databynode'][node].get('filename', None)
+            if fname:
+                print('{0}: Saved license to {1}'.format(node, fname))
+            elif 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(node, res['databynode'][node]['error']))
+            else:
+                sys.stderr.write('{0}: {1}\n'.format(node, repr(res['databynode'][node])))
+
 
 def show_licenses(session):
     global exitcode
-    firmware_shown = False
     for res in session.read(
             '/noderange/{0}/configuration/management_controller/licenses/'
             'all'.format(noderange)):
         for node in res.get('databynode', {}):
             for license in res['databynode'][node].get('License', []):
-                print('{0}: {1}'.format(node, license.get('feature',
-                                                          'Unknown')))
+                msg = '{0}: {1}'.format(node, license.get('feature',
+                                                          'Unknown'))
+                if license.get('state', 'Active') != 'Active':
+                    msg += ' ({0})'.format(license['state'])
+                print(msg)
 
 
+def delete_license(session, licname):
+    global exitcode
+    licstodel = []
+    for res in list(session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange))):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                if license.get('feature', None) == licname:
+                    prefix = '/nodes/{0}/configuration/management_controller/licenses/'.format(node)
+                    for currlic in list(session.read(prefix)):
+                        currlic = currlic.get('item', {}).get('href', 'all')
+                        if currlic == 'all':
+                            continue
+                        currname = list(session.read(prefix + currlic))[0]
+                        currname = currname.get('License', [{}])[0].get('feature', None)
+                        if currname == licname:
+                            list(session.delete(prefix + currlic))
+    show_licenses(session)
+
 try:
     session = client.Command()
-    if upfile is None:
-        show_licenses(session)
-    else:
+    if upfile:
         install_license(session, upfile)
+    elif downdir:
+        save_licenses(session, downdir)
+    elif delete:
+        session.stop_if_noderange_over(noderange, options.maxnodes)
+        delete_license(session, delete)
+    else:
+        show_licenses(session)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -61,7 +61,7 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print res['item']['href'].replace('/', '')
+                print(res['item']['href'].replace('/', ''))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodemedia

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -83,7 +83,8 @@ def list_media(noderange, media):
         for node in res.get('databynode', []):
             url = res['databynode'][node].get('url', None)
             name =  res['databynode'][node].get('name', None)
-            if url and not res['databynode'][node].get('secure', False):
+            if (url and not url.startswith('file:') and
+                    not res['databynode'][node].get('secure', False)):
                 name += ' (insecure)'
             if not name:
                 continue
@@ -187,4 +188,4 @@ def main():
         sys.exit(1)
     handler(noderange, media)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodepower

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -37,6 +37,11 @@ argparser = optparse.OptionParser(
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
                      help='Show previous power state')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to change power state, '
+                          'prompting if over the threshold')
+
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -72,4 +77,4 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))

confluent_client/bin/noderemove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -35,6 +35,10 @@ import confluent.client as client
 argparser = optparse.OptionParser(
     usage='''\n       %prog noderange
              \n ''')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete, '
+                          'prompting if over the threshold')      
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
@@ -43,10 +47,11 @@ noderange = args[0]
 client.check_globbing(noderange)
 session = client.Command()
 exitcode = 0
+session.stop_if_noderange_over(noderange, options.maxnodes)
 for r in session.delete('/noderange/{0}'.format(noderange)):
     if 'error' in r:
         sys.stderr.write(r['error'] + '\n')
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/noderename

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo

confluent_client/bin/nodereseat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,10 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to reseat, '
+                          'prompting if over the threshold')   
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,7 +47,7 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'power/reseat', 'reseat', key='reseat', errnodes=errorNodes)   # = 0 if successful
 
 # Determine which nodes were successful and print them
@@ -56,7 +60,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": Reseat successful"
+    print(node + ": Reseat successful")
 
 
 sys.exit(success)

confluent_client/bin/nodersync

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -42,6 +42,10 @@ def run():
     argparser = optparse.OptionParser(
         usage="Usage: %prog location noderange:location",
     )
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run rsync to, '
+                          'prompting if over the threshold')   
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -55,7 +59,7 @@ def run():
     noderange, targpath = args[-1].split(':', 1)
     client.check_globbing(noderange)
     c = client.Command()
-    cmdstr = " ".join(args[:-1])
+    cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
     cmdstr += ' {node}:' + targpath
 
@@ -64,15 +68,17 @@ def run():
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
-
+    c.stop_if_noderange_over(noderange, options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
-                        {'expression': cmdstr}):
+            {'expression': cmdstr}):
         if 'error' in exp:
             sys.stderr.write(exp['error'] + '\n')
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -98,7 +104,7 @@ def run():
                     if desc['type'] == 'stdout':
                         if node not in pernodeout:
                             pernodeout[node] = ''
-                        pernodeout[node] += data
+                        pernodeout[node] += client.stringify(data)
                         if '\n' in pernodeout[node]:
                             currout, pernodeout[node] = pernodeout[node].split('\n', 1)
                             if currout:
@@ -117,7 +123,7 @@ def run():
                         output.set_output(node, 'error!')
                         if node not in nodeerrs:
                             nodeerrs[node] = ''
-                        nodeerrs[node] += data
+                        nodeerrs[node] += client.stringify(data)
                 else:
                     pop = desc['popen']
                     ret = pop.poll()

confluent_client/bin/noderun

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -46,6 +46,10 @@ def run():
                          help='Number of commands to run at a time')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run the command with, '
+                          'prompting if over the threshold')                           
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -63,7 +67,7 @@ def run():
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -71,7 +75,9 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -95,6 +101,7 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
+                        data = client.stringify(data)
                         if options.nonodeprefix:
                             sys.stderr.write(data)
                         else:
@@ -112,6 +119,7 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
+                line = client.stringify(line)
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:

confluent_client/bin/nodesensors

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -114,7 +114,8 @@ def sensorpass(showout=True, appendtime=False):
                         continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
-                            sensedata['states'].remove(redundant_state)
+                            if sensedata.get('states', False):
+                                sensedata['states'].remove(redundant_state)
                         except ValueError:
                             pass
                     resultdata[node][sensedata['name']] = sensedata
@@ -132,11 +133,12 @@ def sensorpass(showout=True, appendtime=False):
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):
                             showval += sensedata['units']
-                        if sensedata['health'] != 'ok':
+                        if sensedata.get('health', 'ok') != 'ok':
                             datadescription = [sensedata['health']]
                         else:
                             datadescription = []
-                        datadescription.extend(sensedata['states'])
+                        if sensedata.get('states', False):
+                            datadescription.extend(sensedata['states'])
                         if datadescription:
                             if showval == '':
                                 showval += u' {0}'.format(
@@ -147,8 +149,11 @@ def sensorpass(showout=True, appendtime=False):
                         if appendtime:
                             showval += ' @' + time.strftime(
                                 '%Y-%m-%dT%H:%M:%S')
-                        print(u'{0}: {1}:{2}'.format(
-                            node, sensedata['name'], showval).encode('utf-8'))
+                        printval = u'{0}: {1}:{2}'.format(
+                            node, sensedata['name'], showval)
+                        if not isinstance(printval, str):
+                            printval = printval.encode('utf-8')
+                        print(printval)
                         sys.stdout.flush()
     return resultdata
 
@@ -156,7 +161,7 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            if showtime.is_integer():
+            if isinstance(showtime, int):
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
             else:
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
@@ -183,6 +188,7 @@ def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
             except KeyError:
                 rowdata.append('N/A')
         csvwriter.writerow(rowdata)
+        sys.stdout.flush()
 
 
 def main():
@@ -196,7 +202,11 @@ def main():
             orderedsensors.append(name)
         orderedsensors.sort()
         for name in orderedsensors:
-            headernames.append(sensorheaders[name].encode('utf-8'))
+            headername = sensorheaders[name]
+            if (not isinstance(headername, str) and
+                    not isinstance(headername, bytes)):
+                headername = headername.encode('utf-8')
+            headernames.append(headername)
     if options.csv:
         linebyline = False
         csvwriter = csv.writer(sys.stdout)
@@ -220,7 +230,7 @@ def main():
                     sys.exit(exitcode)
             sleeptime = nextstart - os.times()[4]
             if sleeptime > 0:
-                time.sleep(nextstart - os.times()[4])
+                time.sleep(sleeptime)
     else:
         if options.csv:
             format_csv(csvwriter, orderedsensors, resdata, showtime=False)

confluent_client/bin/nodesetboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')
@@ -43,7 +43,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 argparser.add_option('-u', '--uefi', dest='uefi', action='store_true',
                      default=True,
                      help='Request UEFI style boot (rather than BIOS)')
-
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to modify next boot device, '
+                          'prompting if over the threshold')                        
 (options, args) = argparser.parse_args()
 
 try:
@@ -63,6 +66,7 @@ if options.biosmode:
     bootmode = 'bios'
 else:
     bootmode = 'uefi'
+session.stop_if_noderange_over(noderange, options.maxnodes)
 sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist))

confluent_client/bin/nodeshell

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -46,6 +46,12 @@ def run():
                          help='Number of commands to run at a time')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
+    argparser.add_option('-p', '--port', type='int', default=0,
+                         help='Specify a custom port for ssh')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run remote ssh command to, '
+                          'prompting if over the threshold')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -64,7 +70,7 @@ def run():
     pendingexecs = deque()
     exitcode = 0
 
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -72,8 +78,13 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
-            cmdv = ['ssh', node, cmd]
+            cmd = ex[node]['value']
+            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+                cmd = cmd.encode('utf-8')
+            if options.port:
+                cmdv = ['ssh', '-p', '{0}'.format(options.port), node, cmd]
+            else:
+                cmdv = ['ssh', node, cmd]
             if currprocs < concurrentprocs:
                 currprocs += 1
                 run_cmdv(node, cmdv, all, pipedesc)
@@ -96,6 +107,7 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
+                        data = client.stringify(data)
                         if options.nonodeprefix:
                             sys.stderr.write(data)
                         else:
@@ -113,6 +125,7 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
+                line = client.stringify(line)
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:

confluent_client/bin/nodestorage

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -63,10 +63,16 @@ def _print_cfg(scfg):
             sys.stderr.write(e['error'] + '\n')
             exitcode = e.get('errorcode', 1)
         for node in e.get('databynode', {}):
+            curr = e['databynode'][node]
+            if 'error' in curr:
+                if 'no available drives' in curr['error']:
+                    curr['error'] += ' (drives must be in unconfigured state to be available, they must not be in jbod or online state)'
+                sys.stderr.write('{0}: {1}\n'.format(node, curr['error']))
+                exitcode = curr.get('errorcode', 1)
+                continue
             if node not in storagebynode:
                 storagebynode[node] = {'disks': [], 'arrays': [],
                                        'volumes': []}
-            curr = e['databynode'][node]
             storagebynode[node][curr['type'] + 's'].append(curr)
     for node in storagebynode:
         for disk in sorted(storagebynode[node]['disks'],
@@ -108,6 +114,7 @@ def createstorage(noderange, options, args):
         sys.stderr.write('-r and -d are required arguments to create array\n')
         sys.exit(1)
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     names = options.name
     if names is None:
         names = ''.join(args)
@@ -115,6 +122,8 @@ def createstorage(noderange, options, args):
              'name': names}
     if options.size:
         parms['size'] = options.size
+    if options.stripsizes:
+        parms['stripsizes'] = options.stripsizes
     _print_cfg(session.create(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names), parms))
@@ -130,6 +139,7 @@ def deletestorage(noderange, options, args):
     else:
         names = options.name
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for rsp in session.delete(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names)):
@@ -147,13 +157,27 @@ def deletestorage(noderange, options, args):
             print(repr(rsp))
 
 
-# def setstorage(noderange, options, args):
-#     pass
+def setdisk(noderange, options, args):
+    if options.disks is None:
+        if len(args):
+            names = args.pop(0)
+        else:
+            sys.stderr.write('-d is required to indicate disk to modify\n')
+            sys.exit(1)
+    else:
+        names = options.disks
+    if not len(args) or args[0] not in ('hotspare', 'jbod', 'unconfigured'):
+        sys.stderr.write('diskset requires valid state as argument (hotspare, jbod, unconfigured)\n')
+        sys.exit(1)
+    session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    scfg = session.update('/noderange/{0}/configuration/storage/disks/{1}'.format(noderange, names), {'state': args[0]})
+    _print_cfg(scfg)
 
 funmap = {
     'create': createstorage,
     'show': showstorage,
-#     'set': setstorage,
+    'diskset': setdisk,
     'delete': deletestorage,
     'rm': deletestorage,
 }
@@ -162,7 +186,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|delete]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',
@@ -183,6 +207,14 @@ def main():
                               'naming volumes, or selecting a volume for '
                               'delete.  Default behavior is to use '
                               'implementation provided default names.')
+    argparser.add_option('-z', '--stripsizes', type='str',
+                         help='Comma separated list of stripsizes to use when creating volumes. '
+                            'This value is in kilobytes.  The default behavior is to allow the '
+                            'storage controller to decide.')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure storage on, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
     if len(args) == 1:
         args.append('show')

confluent_client/bin/nodesupport

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -64,7 +64,7 @@ def printerror(res, node=None):
 
 
 
-def download_servicedata(noderange, media):
+def download_servicedata(noderange, media, options):
     global exitcode
     session = client.Command()
     output = sq.ScreenPrinter(noderange, session)
@@ -73,6 +73,7 @@ def download_servicedata(noderange, media):
     upargs = {'filename': filename}
     noderrs = {}
     nodeurls = {}
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -121,6 +122,10 @@ def main():
                'management server (the confluent server if running remote, '
                'and the collective.manager if in collective)\n'
                '\n\nSee `man %prog` for more info.\n')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to download diagnostic data from, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
     media = None
     try:
@@ -142,6 +147,6 @@ def main():
     except KeyError:
         argparser.print_help()
         sys.exit(1)
-    handler(noderange, media)
+    handler(noderange, media, options)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/stats

@@ -0,0 +1,167 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import fcntl
+import io
+import numpy as np
+
+import os
+import sixel
+import subprocess
+import sys
+
+class DumbWriter(sixel.SixelWriter):
+    def restore_position(self, output):
+        return
+
+
+def plot(gui, output, plotdata, bins):
+    import matplotlib as mpl
+    if not gui:
+        mpl.use('Agg')
+    import matplotlib.pyplot as plt
+    n, bins, patches = plt.hist(plotdata, bins)
+    plt.show()
+    if not gui:
+        if output:
+            tdata = output
+        else:
+            tdata = io.BytesIO()
+        plt.savefig(tdata)
+    if not gui and not output:
+        writer = DumbWriter()
+        writer.draw(tdata)
+    return n, bins
+
+def textplot(plotdata, bins):
+    n, bins = np.histogram(plotdata, bins)
+    labels = []
+    for bin in bins:
+        labels.append('{0:0.1f}'.format(bin))
+    width = 80
+    # Since this will be primarily piped into, hard to get 
+    # terminal width
+    labelwidth = 0
+    for lab in labels:
+        if len(lab) > labelwidth:
+            labelwidth = len(lab)
+    width -= (labelwidth) + 1
+    labelfmt = '{{0:>{0}s}}|'.format(labelwidth)
+    maxn = 0.0
+    for lgth in n:
+        if lgth > maxn:
+            maxn = float(lgth)
+    for i in range(len(n)):
+        print(labelfmt.format(labels[i]) + '=' * int(np.round((n[i]/maxn) * width)))
+    return n, bins
+
+histogram = False
+aparser = argparse.ArgumentParser(description='Quick access to common statistics')
+aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
+aparser.add_argument('-d', default=None, help='Value used to separate columns')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
+aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
+aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
+aparser.add_argument('-t', default=False, action='store_true', help='Output a histogram in text format')
+aparser.add_argument('-v', default=False, action='store_true', help='Attempt to list nodes relevant to each histogram bar (requires -s, -o, or -t)')
+aparser.add_argument('-b', type=int, default=10, help='Number of bins to use in histogram (default is 10)')
+args = aparser.parse_args(sys.argv[1:])
+plotdata = []
+headlines = int(args.s)
+while headlines >= 0:
+    data = sys.stdin.readline()
+    headlines -= 1
+if args.d:
+    delimiter = args.d
+else:
+    if '\t' in data:
+        delimiter = '\t'
+    elif ' ' in data:
+        delimiter = ' '
+    elif ',' in data:
+        delimiter = ','
+    else:
+        delimiter = ' '  # handle single column
+data = list(csv.reader([data], delimiter=delimiter))[0]
+nodebydatum = {}
+idx = args.c - 1
+autoidx = False
+while data:
+    node = None
+    if ':' in data[0]:
+        node, data[0] = data[0].split(':', 1)
+    else:
+        node = data[0]
+    if idx == -1 and not autoidx:
+        while not autoidx:
+            try:
+                datum = float(data[idx])
+            except ValueError:
+                idx -= 1
+                continue
+            except IndexError:
+                sys.stderr.write('Unable to identify a numerical column\n')
+                sys.exit(1)
+            autoidx = True
+    else:
+        datum = float(data[idx])
+    if node:
+        if datum in nodebydatum:
+            nodebydatum[datum].add(node)
+        else:
+            nodebydatum[datum] = set([node])
+    plotdata.append(datum)
+    data = sys.stdin.readline()
+    data = list(csv.reader([data], delimiter=delimiter))[0]
+n = None
+if args.g or args.o or args.x:
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+if args.t:
+    n, bins = textplot(plotdata, bins=args.b)
+print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))
+if args.v and n is not None and nodebydatum:
+    print('')
+    currbin = bins[0]
+    bins = bins[1:]
+    currbinmembers = []
+    for datum in sorted(nodebydatum):
+        if datum > bins[0]:
+           nextbin = None
+           endbin = bins[0]
+           while len(bins) and bins[0] < datum:
+                nextbin = bins[0]
+                bins = bins[1:]
+           if not nextbin:
+               nextbin = np.max(plotdata)
+           print('Entries between {0} and {1}'.format(currbin, endbin))
+           currbin = nextbin
+           print('-' * 80)
+           print(','.join(sorted(currbinmembers)))
+           print('')
+           print('')
+           currbinmembers = []
+        for node in nodebydatum[datum]:
+            currbinmembers.append(node)
+    if currbinmembers:
+       print('Entries between {0} and {1}'.format(currbin, np.max(plotdata)))
+       print('-' * 80)
+       print(','.join(sorted(currbinmembers)))
+       print('')
+       print('')

confluent_client/confluent/client.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import anydbm as dbm
+try:
+    import anydbm as dbm
+except ImportError:
+    import dbm
 import csv
 import errno
 import fnmatch
@@ -33,9 +36,24 @@ SO_PASSCRED = 16
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
     'bmcuser': 'secret.hardwaremanagementuser',
+    'switchuser': 'secret.hardwaremanagementuser',
     'bmcpass': 'secret.hardwaremanagementpassword',
+    'switchpass': 'secret.hardwaremanagementpassword',
 }
 
+try:
+    input = raw_input
+except NameError:
+    pass
+
+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
 
 class Tabulator(object):
     def __init__(self, headers):
@@ -104,7 +122,10 @@ def printerror(res, node=None):
 
 
 def cprint(txt):
-    print(txt)
+    try:
+        print(txt)
+    except UnicodeEncodeError:
+        print(txt.encode('utf8'))
     sys.stdout.flush()
 
 def _parseserver(string):
@@ -204,7 +225,7 @@ class Command(object):
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None, errnodes=None, **kwargs):
+                                 key=None, errnodes=None, promptover=None, **kwargs):
         try:
             self._currnoderange = noderange
             rc = 0
@@ -220,6 +241,7 @@ class Command(object):
                         noderange, resource)):
                     rc = self.handle_results(ikey, rc, res, errnodes)
             else:
+                self.stop_if_noderange_over(noderange, promptover)
                 kwargs[ikey] = input
                 for res in self.update('/noderange/{0}/{1}'.format(
                         noderange, resource), kwargs):
@@ -229,6 +251,33 @@ class Command(object):
         except KeyboardInterrupt:
             cprint('')
             return 0
+    
+    def stop_if_noderange_over(self, noderange, maxnodes):
+        if maxnodes is None:
+            return
+        nsize = self.get_noderange_size(noderange)
+        if nsize > maxnodes:
+            if nsize == 1:
+                nodename = list(self.read(
+                    '/noderange/{0}/nodes/'.format(noderange)))[0].get('item', {}).get('href', None)
+                nodename = nodename[:-1]
+                p = input('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+            else:
+                p = input('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+            if p.lower() != 'y':
+                sys.stderr.write('Aborting at user request\n')
+                sys.exit(1)
+                raise Exception("Aborting at user request")
+        
+
+    def get_noderange_size(self, noderange):
+        numnodes = 0
+        for node in self.read('/noderange/{0}/nodes/'.format(noderange)):
+            if node.get('item', {}).get('href', None):
+                numnodes += 1
+            else:
+                raise Exception("Error trying to size noderange {0}".format(noderange))
+        return numnodes
 
     def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs):
         try:
@@ -322,13 +371,14 @@ class Command(object):
         if knownhosts:
             certdata = self.connection.getpeercert(binary_form=True)
             fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
+            fingerprint = fingerprint.encode('utf-8')
             hostid = '@'.join((port, server))
             khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
             if hostid in khf:
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    replace = raw_input(
+                    replace = input(
                         "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
@@ -381,8 +431,12 @@ def printattributes(session, requestargs, showtype, nodetype, noderange, options
     path = '/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)
     return print_attrib_path(path, session, requestargs, options)
 
+def _sort_attrib(k):
+    if isinstance(k[1], dict) and k[1].get('sortid', None) is not None:
+        return k[1]['sortid']
+    return k[0]
 
-def print_attrib_path(path, session, requestargs, options, rename=None):
+def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
     seenattributes = set([])
     for res in session.read(path):
@@ -391,9 +445,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
-            for attr, val in sorted(
-                    res['databynode'][node].items(),
-                    key=lambda (k, v): v.get('sortid', k) if isinstance(v, dict) else k):
+            for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
                     continue
@@ -405,6 +457,8 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                     printattr = rename.get(attr, attr)
                 else:
                     printattr = attr
+                if attrprefix:
+                    printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
                 if show_attr(attr, requestargs, seenattributes, options):
                     if 'value' in currattr:
@@ -413,7 +467,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                             if isinstance(val, list):
                                 val = ','.join(val)
                             attrout = '{0}: {1}: {2}'.format(
-                                node, printattr, val)
+                                node, printattr, val).strip()
                         else:
                             attrout = '{0}: {1}:'.format(node, printattr)
                     elif 'isset' in currattr:
@@ -478,11 +532,14 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                             details = False
                         if details:
                             if currattr.get('help', None):
-                                attrout += ' (Help: {0})'.format(
-                                    currattr['help'].encode('utf-8'))
+                                attrout += u' (Help: {0})'.format(
+                                    currattr['help'])
                             if currattr.get('possible', None):
-                                attrout += ' (Choices: {0})'.format(
-                                    ','.join(currattr['possible']))
+                                try:
+                                    attrout += u' (Choices: {0})'.format(
+                                        ','.join(currattr['possible']))
+                                except TypeError:
+                                    pass
                         cprint(attrout)
     if not exitcode:
         if requestargs:
@@ -652,4 +709,4 @@ def check_globbing(noderange):
                 'bash or change directories such that there is no filename '
                 'that would conflict.'
                 '\n'.format(noderange))
-            sys.exit(1)
+            sys.exit(1)

confluent_client/confluent/tlvdata.py

@@ -25,6 +25,11 @@ try:
 except NameError:
     unicode = str
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 def decodestr(value):
     ret = None
     try:
@@ -34,11 +39,13 @@ def decodestr(value):
             ret = value.decode('cp437')
         except UnicodeDecodeError:
             ret = value
+    except AttributeError:
+        return value
     return ret
 
 def unicode_dictvalues(dictdata):
     for key in dictdata:
-        if isinstance(dictdata[key], str):
+        if isinstance(dictdata[key], bytes):
             dictdata[key] = decodestr(dictdata[key])
         elif isinstance(dictdata[key], datetime):
             dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -49,7 +56,7 @@ def unicode_dictvalues(dictdata):
 
 
 def _unicode_list(currlist):
-    for i in xrange(len(currlist)):
+    for i in range(len(currlist)):
         if isinstance(currlist[i], str):
             currlist[i] = decodestr(currlist[i])
         elif isinstance(currlist[i], dict):
@@ -64,7 +71,7 @@ def send(handle, data):
             data = data.encode('utf-8')
         except AttributeError:
             pass
-    if isinstance(data, str) or isinstance(data, unicode):
+    if isinstance(data, bytes) or isinstance(data, unicode):
         # plain text, e.g. console data
         tl = len(data)
         if tl == 0:

confluent_client/confluent_client.spec.tmpl

@@ -7,7 +7,7 @@ Name: %{name}
 Version: %{version}
 Release: %{release}
 Source0: %{name}-%{version}.tar.gz
-License: UNKNOWN
+License: Apache2
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
@@ -24,10 +24,18 @@ a confluent server.
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+%if "%{dist}" == ".el8"
+python3 setup.py build
+%else
+python2 setup.py build
+%endif
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%if "%{dist}" == ".el8"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%endif
 
 
 %clean

confluent_client/confluent_env.sh

@@ -19,6 +19,7 @@ alias nodebmcreset='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 alias nodeboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeboot'
 alias nodeconfig='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconfig'
 alias nodeconsole='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconsole'
+alias nodedeploy='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodedeploy'
 alias nodedefine='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodedefine'
 alias nodeeventlog='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeeventlog'
 alias nodefirmware='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodefirmware'
@@ -39,14 +40,15 @@ alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
 alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
+# Do not continue for non-bash shells, the rest of this sets up bash completion functions
+[ -z "$BASH_VERSION" -o -z "$PS1" ] && return
 
 
 _confluent_get_args()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
+    NUMARGS=$((COMP_CWORD+1))
+    if [ "${COMP_WORDS[COMP_CWORD]}" == '' ]; then
         CMPARGS+=("")
     fi
     GENNED=""
@@ -73,7 +75,7 @@ function _confluent_generic_completion()
 {
     _confluent_get_args
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
     fi
     if [ $NUMARGS -lt 3 ]; then
         _confluent_nr_completion
@@ -82,14 +84,14 @@ function _confluent_generic_completion()
 }
 _confluent_nodeidentify_completion()
 {
-    COMP_CANDIDATES=("on,off -h")
+    COMP_CANDIDATES=("on,off,blink -h")
     _confluent_generic_completion
 }
 
 
 _confluent_nodesetboot_completion()
 {
-    COMP_CANDIDATES=("default,cd,network,setup,hd -h -b -p")
+    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy -h -b -p")
     _confluent_generic_completion
 }
 
@@ -109,7 +111,7 @@ _confluent_nodemedia_completion()
         return
     fi
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -lt 3 ]; then
@@ -122,7 +124,7 @@ _confluent_nodefirmware_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[2]} == 'update' ]; then
@@ -136,11 +138,42 @@ _confluent_nodefirmware_completion()
     fi
 }
 
+_confluent_osimage_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 2 ]; then
+        COMPREPLY=($(compgen -W "initialize import updateboot" -- ${COMP_WORDS[COMP_CWORD]}))
+        return
+    elif [ ${CMPARGS[1]} == 'initialize' ]; then
+        COMPREPLY=($(compgen -W "-h -u -s -t -i" -- ${COMP_WORDS[COMP_CWORD]}))
+    elif [ ${CMPARGS[1]} == 'import' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    elif [ ${CMPARGS[1]} == 'updateboot' ]; then
+        COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return
+    fi
+}
+
+_confluent_nodedeploy_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        COMPREPLY=($(compgen -W "-n $(confetty show /deployment/profiles|sed -e 's/\///')" -- "${COMP_WORDS[COMP_CWORD]}"))
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
 _confluent_nodeshell_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -c -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -c -- ${COMP_WORDS[COMP_CWORD]}))
         return
     fi
     if [ $NUMARGS -gt 3 ]; then
@@ -158,7 +191,7 @@ _confluent_nodelicense_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "install list" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
@@ -166,6 +199,11 @@ _confluent_nodelicense_completion()
         COMPREPLY=()
         return
     fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'save' ]; then
+        compopt -o dirnames
+        COMPREPLY=()
+        return
+    fi
     if [ $NUMARGS -lt 3 ]; then
          _confluent_nr_completion
          return
@@ -176,7 +214,7 @@ _confluent_nodesupport_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'servicedata' ]; then
@@ -203,41 +241,38 @@ _confluent_nn_completion()
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_nr_completion()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
-    fi
     _confluent_get_args
     if [ $NUMARGS -gt 2 ]; then
+        compopt -o default
+        COMPREPLY=()
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    #COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_ng_completion()
 {
@@ -245,17 +280,17 @@ _confluent_ng_completion()
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 complete -F _confluent_nodeattrib_completion nodeattrib
 complete -F _confluent_nodeattrib_completion nodegroupattrib
@@ -265,12 +300,14 @@ complete -F _confluent_nr_completion nodeconfig
 complete -F _confluent_nn_completion nodeconsole
 complete -F _confluent_nr_completion nodeeventlog
 complete -F _confluent_nodefirmware_completion nodefirmware
+complete -F _confluent_nodedeploy_completion nodedeploy
+complete -F _confluent_osimage_completion osdeploy
 complete -F _confluent_ng_completion nodegroupattrib
 complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth
 complete -F _confluent_nodeidentify_completion nodeidentify
 complete -F _confluent_nr_completion nodeinventory
-complete -F _confluent_nr_completion nodelist
+complete -F _confluent_nodeattrib_completion nodelist
 complete -F _confluent_nodemedia_completion nodemedia
 complete -F _confluent_nodepower_completion nodepower
 complete -F _confluent_nr_completion noderemove

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -100,3 +100,5 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)
+
+## ATTRIBUTES

confluent_client/doc/man/nodeconfig.ronn

@@ -22,6 +22,11 @@ given as a node expression, as documented in the man page for nodeattribexpressi
   If combined with `-x`, will show all differing values except those indicated
   by `-x`
 
+* `-e`, `--extra`:
+   Read settings that are generally not needed, but may be slow to retrieve.
+   Notably this includes the IMM category of Lenovo systems.  The most popular
+   IMM settings are available through faster 'bmc' attributes.
+
 * `-x`, `--exclude`:
   Rather than listing only the specified configuration parameters, list all
   attributes except for the specified ones

confluent_client/doc/man/nodedeploy.ronn

@@ -0,0 +1,42 @@
+nodedeploy(8) -- Request preparation and/or initiating a node deployment
+=========================================================================
+
+## SYNOPSIS
+
+`nodedeploy <noderange>`  
+`nodedeploy <noderange> -c`  
+`nodedeploy <noderange> [-n] [-p] <name of profile>`  
+
+## DESCRIPTION
+
+**nodedeploy** configures attributes and interacts with BMC devices as appropriate
+to change the operating system on the noderange to that indicated by `profile`. Tab completion
+is supported under bash to help list and select a profile name. `-n` indicates that it
+will be a network based deployment (either HTTP or PXE) and `-p` suppresses any activity
+that requires the BMC (e.g. setting the next boot device and rebooting the system). Currently
+**nodedeploy** only supports `-n` style deployment flows. Without options it shows the current
+deployment status.
+
+## OPTIONS
+
+* '-c', '--clear`:
+  Remove any pending deployment action
+
+* `-n`, `--network`:
+  Prepare for either an HTTP or PXE based deployment, setting boot device to network and rebooting unless `-p` is specified.
+
+* `-p`, `--prepareonly`:
+  Prepare the network services for deployment, but do not interact with BMCs. This is intended for scenarios where
+  the boot device control and server restart will be handled outside of confluent.
+
+## EXAMPLES
+* Begin the instalalation of a profile of CentOS 8.2:  
+   `# nodedeploy d4 -n centos-8.2-x86_64-default`  
+   `d4: network`  
+   `d4: reset`  
+
+* Check current deployment state of nodes:
+  `# nodedeploy d4`  
+  `d4: pending: centos-8.2-x86_64-default (node authentication armed)`  
+
+

confluent_client/doc/man/nodediscover.ronn

@@ -23,6 +23,9 @@ data may be filtered by various parameters, as denoted in the options below.
 **nodediscover assign** performs manual discovery, assigning an entry to a node
 identity or, using `-i`, using a csv file to assign nodes all at once.  For
 example, a spreadsheet of serial numbers to desired node names could be used.
+Note that if you see that the host is unreachable, it may be due to the IP
+address on the endpoint having changed since last detected. In such a case, it
+may help to **clear** and try **assign** again.
 
 **nodediscover rescan** requests the server to do an active sweep for new
 devices.  Generally every effort is made to passively detect devices as they

confluent_client/doc/man/nodefirmware.ronn

@@ -13,7 +13,11 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 will retrieve all firmware, but can be directed to fetch specific firmware by
 calling out the name of the firmware (e.g. uefi or xcc) or request reading only
 core firmware firmware by using the word 'core', which is generally a quicker
-operation.
+operation.  Different hardwaremanagement.method indicated plugins may have
+different capabilities available.  For example, the 'core' distinction may
+not be relevant to redfish.  Additionally, the Lenovo XCC makes certain
+information available over IPMI that is not otherwise available (for example
+the FPGA version where applicable).
 
 In the update form, it accepts a single file and attempts to update it using
 the out of band facilities.  Firmware updates can end in one of three states:

confluent_client/doc/man/nodegroupattrib.ronn.tmpl

@@ -41,3 +41,5 @@ the attributes are set on the node versus a group to which a node belongs.
 ## SEE ALSO
 
 nodeattrib(8), nodeattribexpressions(5)
+
+## ATTRIBUTES

confluent_client/doc/man/nodeidentify.ronn

@@ -3,7 +3,7 @@ nodeidentify(8) -- Control the identify LED of confluent nodes
 
 ## SYNOPSIS
 
-`nodidentify <noderange> [on|off]`  
+`nodidentify <noderange> [on|off|blink]`  
 
 ## DESCRIPTION
 
@@ -13,6 +13,7 @@ options are supported:
 
 * `on`: Turn on the identify LED
 * `off`: Turn off the identify LED
+* `blink`: Set the identify LED to blink (when supported by the system)
 
 ## EXAMPLES:
 

confluent_client/doc/man/nodelicense.ronn

@@ -3,9 +3,15 @@ nodelicense(8) -- Manage license keys on BMC
 
 ## SYNOPSIS
 
-`nodelicense <noderange> [list|install <filename>]`
+`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
 
 ## DESCRIPTION
 
-`nodelicense` shows and installs license keys on supported BMCs
+`nodelicense` manages license keys on supported BMCs. Without an argument, the command
+lists currently installed license.  Using `delete` will remove the specified license name
+from th eBMC.  The `save` subcommand will take the passed directory (which may be in the form
+of /path/to/{node}/ to have the node name substituted for each node) and back up installed licenses
+to that directory. The `install` command will take the specified filename and install.  The filename
+argument may be of the form  xcc_fod_0034_7X21{id.serial}.key to have the serial number substituted
+to allow unique licenses to be specified in a single command.
 

confluent_client/doc/man/noderange.ronn

@@ -42,7 +42,7 @@ The attribute name may use a wildcard:
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
 
-Exclusions can be done by prepending a ‘-‘ before a portion of a noderange:
+Exclusions can be done by prepending a '-' before a portion of a noderange:
 `rack1,-n2`
 `compute,-rack1`
 `compute,-location.row=12`
@@ -54,7 +54,7 @@ To indicate nodes that match multiple selections at once (set intersection), the
 For complex expressions, () may be used to indicate order of expanding the noderange to be explicit
 `rack1,-(console.logging=full@compute)`
 
-Noderange syntax can also indicate ‘pagination’, or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
+Noderange syntax can also indicate 'pagination', or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
 `rack1>3<6`
 
 The above would show the seventh through ninth nodes of the rack1 group. Like all other noderange operations, this may be combined with any of the above, but must appear as the very last operation. Ordering is done with a natural sort.

confluent_client/doc/man/nodersync.ronn

@@ -0,0 +1,21 @@
+nodersync(8) -- Run rsync in parallel against a noderange
+=========================================================================
+
+## SYNOPSIS
+
+`nodersync <file/directorylist> <noderange>:<destination>`  
+
+## DESCRIPTION
+
+Supervises execution of rsync to push files or a directory tree to the specified
+noderange. This will present progress as percentage for all nodes.
+
+## OPTIONS
+
+* `-m`:
+  Specify maximum number of nodes for noderange max.
+
+* `-c`:
+  Specify how many rsync executions to do concurrently.  If noderange
+  exceeds the count, then excess nodes will wait until one of the
+  active count completes.

confluent_client/doc/man/nodesensors.ronn

@@ -29,6 +29,10 @@ interval of 1 second is used.
   Perform the specified number of readings, waiting `-i` indicated interval or 1 second if not
   otherwise indicated.
 
+* `-s`, `--skipnumberless`:
+  Return only sensors that provide numeric data (e.g. temperature, fanspeed),
+  ignoring sensors that do not provide numeric data.
+
 ## EXAMPLES
 * Retrieving all temperature related sensors from one system  
   `# nodesensors n1 temperature`  

confluent_client/doc/man/nodesetboot.ronn

@@ -4,7 +4,7 @@ nodesetboot(8) -- Check or set next boot device for noderange
 ## SYNOPSIS
 
 `nodesetboot <noderange>`  
-`nodesetboot [options] <noderange> [default|cd|network|setup|hd]`
+`nodesetboot [options] <noderange> [default|cd|network|setup|hd|floppy]`
 
 ## DESCRIPTION
 
@@ -42,6 +42,10 @@ control.
   remote media mounted as CD/DVD, and detachable hard disks drives such as usb
   key devices.
   
+* `floppy`:
+  Request boot from floppy.  Generally speaking firmware uses this to mean a USB
+  flash drive or similar (whether virtual or physical).
+
 * `network`:
   Request boot to network
   

confluent_client/doc/man/nodestorage.ronn

@@ -35,8 +35,8 @@ the noderange.
     `$ nodestorage d5 delete somedata`  
     `Deleted: somedata`  
 
-* Creating a raid5 of 4 disks and a volume named `somedata`:  
-    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata
+* Creating a raid5 of 4 disks and a volume named `somedata`:
+    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata`  
     `d5: Volume somedata: Size: 1.905 TB`  
     `d5: Volume somedata: State: Optimal`  
     `d5: Volume somedata: Array 1-2`  

confluent_client/doc/man/nodesupport.ronn

@@ -7,7 +7,7 @@ nodesupport(8) -- Utilities for interacting with vendor support
 
 ## DESCRIPTION
 
-`nodesupport` provides capabilities associated with interactiong with support.
+`nodesupport` provides capabilities associated with interacting with support.
 Currently it only has the `servicedata` subcommand.  `servicedata` takes
 an argument that is either a directory name (that can be used for a single node
 or multiple nodes) or a file name (only to be used with single node noderange).
@@ -16,6 +16,9 @@ connects to the managed system, so it will download to the remote system if runn
 remotely and will download to the collective.manager indicated system if
 running in collective mode.
 
+Note that due to vendor filename requirements, any filename may have vendor
+specific suffixes added to any file produced.
+
 ## EXAMPLES
 
 * Download support data from a single node to a specific filename  

confluent_client/doc/man/osdeploy.ronn

@@ -0,0 +1,17 @@
+osdeploy(8) --- Configure general OS deployment facilities of confluent
+==========================================================================
+
+## SYNOPSIS
+
+`osdeploy import <iso>`  
+`osdeploy updateboot <profile>`  
+`osdeploy initialize [-h] [-g] [-u] [-s] [-k] [-t] [-p] [-i] [-l]`  
+
+## DESCRIPTION
+
+**osdeploy** manages the facilities and os deployment content of a confluent server. The
+`import` subcommand will generate profiles from an iso image. `updateboot` will take any
+changes in the specified update that need to be pushed into boot configuration and/or images
+and ensure those needed changes are performed. `initialize` provides assistance in setting
+up the most commonly required facilities. Run `osdeploy initialize -h` for more detail
+on the options offered by `osdeploy initialize`

confluent_client/makeman

@@ -1,4 +1,6 @@
 #!/bin/sh
+cd `dirname $0`
+python3 addattribs.py || python2 addattribs.py
 cd `dirname $0`/doc/man
 mkdir -p ../../man/man1
 mkdir -p ../../man/man5

confluent_client/samples/nodeattrib_from_switch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 
 # This is a sample python script for going through all observed mac addresses
 # and assuming they are BMC related and printing nodeattrib commands

confluent_common/confluent_common.spec.tmpl

@@ -22,10 +22,10 @@ This provides the modules common for both client and server
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+python2 setup.py build
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
 
 %clean
 rm -rf $RPM_BUILD_ROOT

confluent_osdeploy/buildrpm

@@ -0,0 +1,11 @@
+VERSION=`git describe|cut -d- -f 1`
+NUMCOMMITS=`git describe|cut -d- -f 2`
+if [ "$NUMCOMMITS" != "$VERSION"  ]; then
+    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
+fi
+sed -e "s/#VERSION#/$VERSION/" confluent_osdeploy.spec.tmpl > confluent_osdeploy.spec
+cd ..
+tar Jcvf confluent_osdeploy.tar.xz confluent_osdeploy
+mv confluent_osdeploy.tar.xz ~/rpmbuild/SOURCES/
+cd -
+rpmbuild -ba confluent_osdeploy.spec

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+try:
+    import http.client as client
+except ImportError:
+    import httplib as client
+import os
+import socket
+import subprocess
+import ssl
+import sys
+
+def get_apikey(nodename, mgr):
+    if os.path.exists('/etc/confluent/confluent.apikey'):
+        return open('/etc/confluent/confluent.apikey').read().strip()
+    apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
+    if not isinstance(apikey, str):
+        apikey = apikey.decode('utf8')
+    with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
+        apiout.write(apikey)
+    apikey = apikey.strip()
+    os.chmod('/etc/confluent/confluent.apikey', 0o600)
+    return apikey
+
+class HTTPSClient(client.HTTPConnection, object):
+    def __init__(self, json=False, port=443):
+        self.stdheaders = {}
+        info = open('/etc/confluent/confluent.info').read().split('\n')
+        host = None
+        mgtiface = None
+        havedefault = '0'
+        for line in info:
+            if line.startswith('NODENAME:'):
+                node = line.split(' ')[1]
+                self.stdheaders['CONFLUENT_NODENAME'] = node
+            if line.startswith('MANAGER:') and not host:
+                host = line.split(' ')[1]
+            if line.startswith('EXTMGRINFO:'):
+                extinfo = line.split(' ')[1]
+                extinfo = extinfo.split('|')
+                if not mgtiface:
+                    host, mgtiface, havedefault = extinfo[:3]
+                if havedefault == '0' and extinfo[2] == '1':
+                    host, mgtiface, havedefault = extinfo[:3]
+        if '%' in host:
+            ifidx = host.split('%', 1)[1]
+            with open('/tmp/confluent.ifidx', 'w+') as ifout:
+                ifout.write(ifidx)
+        if json:
+            self.stdheaders['ACCEPT'] = 'application/json'
+        self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, host)
+        if mgtiface:
+            self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
+        client.HTTPConnection.__init__(self, host, port)
+        self.connect()
+
+    def set_header(self, key, val):
+        self.stdheaders[key] = val
+
+    def connect(self):
+        addrinf = socket.getaddrinfo(self.host, self.port)[0]
+        psock = socket.socket(addrinf[0])
+        psock.connect(addrinf[4])
+        ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        ctx.load_verify_locations('/etc/confluent/ca.pem')
+        host = self.host.split('%', 1)[0]
+        if '[' not in host and ':' in host:
+            self.stdheaders['Host'] = '[{0}]'.format(host)
+        else:
+            self.stdheaders['Host'] = '{0}'.format(host)
+        ctx.verify_mode = ssl.CERT_REQUIRED
+        ctx.check_hostname = True
+        self.sock = ctx.wrap_socket(psock, server_hostname=host)
+
+    def grab_url(self, url, data=None, returnrsp=False):
+        if data:
+            method = 'POST'
+        else:
+            method = 'GET'
+        self.request(method, url, data, headers=self.stdheaders)
+        rsp = self.getresponse()
+        if rsp.status >= 200 and rsp.status < 300:
+            if returnrsp:
+                return rsp
+            else:
+                return rsp.read()
+        raise Exception(rsp.read())
+
+if __name__ == '__main__':
+    data = None
+    json = False
+    if '-j' in sys.argv:
+        json = True
+    if len(sys.argv) == 1:
+        HTTPSClient()
+        sys.exit(0)
+    if sys.argv[-2] == '-o':
+        with open(sys.argv[3], 'wb') as outf:
+            reader = HTTPSClient(json=json).grab_url(
+                sys.argv[1], data, returnrsp=True)
+            chunk = reader.read(16384)
+            while chunk:
+                outf.write(chunk)
+                chunk = reader.read(16384)
+        sys.exit(0)
+    if os.path.exists(sys.argv[-1]):
+        data = open(sys.argv[-1]).read()
+    print(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -0,0 +1,59 @@
+Name: confluent_osdeploy-x86_64
+Version: #VERSION#
+Release: 1
+Summary: OS Deployment support for confluent
+
+License: Apache2
+URL: http://hpc.lenovo.com/
+Source0: confluent_osdeploy.tar.xz
+BuildArch: noarch
+Requires: confluent_ipxe mtools tar
+BuildRoot: /tmp
+
+%description
+This contains support utilities for enabling deployment of x86_64 architecture systems
+
+
+%define debug_package %{nil}
+
+%prep
+%setup -n confluent_osdeploy
+
+%build
+mkdir -p opt/confluent/bin
+cd utils
+make all
+cp copernicus clortho autocons ../opt/confluent/bin
+cd ..
+for os in rhvh4 el7 genesis el8 suse15 ubuntu20.04; do
+    mkdir ${os}out
+    cd ${os}out
+    cp -a ../opt .
+    cp -a ../${os}/initramfs/* .
+    cp -a ../common/* .
+    find . | cpio -H newc -o > ../addons.cpio
+    mv ../addons.cpio .
+    cd ..
+done
+mkdir esxi7out
+cd esxi7out
+cp -a ../opt .
+cp -a ../esxi7/initramfs/* .
+cp -a ../common/* .
+chmod +x bin/* opt/confluent/bin/*
+tar zcvf ../addons.tgz *
+mv ../addons.tgz .
+cd ..
+cp -a esxi7out esxi6out
+cp -a esxi7 esxi6
+
+%install
+for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
+    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
+    mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+    cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
+    cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+done
+
+%files
+/opt/confluent/lib/osdeploy

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+echo -n "" >> /tmp/net.ifaces
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+        echo "Initializing auto detected console when installer starts" > $autocons
+    fi
+fi
+if grep console=ttyS /proc/cmdline >& /dev/null; then
+    echo "Serial console has been requested in the kernel arguments, the local video may not show progress" > /dev/tty1
+fi
+. /lib/anaconda-lib.sh
+wait_for_kickstart

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+[ -e /tmp/confluent.initq ] && return 0
+if [ -f /tmp/dd_disk ]; then
+    for dd in $(cat /tmp/dd_disk); do
+        if [ -e $dd ]; then
+            driver-updates --disk $dd $dd
+        fi
+    done
+fi
+mkdir -p /etc/confluent
+cat /tls/*.pem > /etc/confluent/ca.pem
+TRIES=0
+echo -n > /etc/confluent/confluent.info
+cd /sys/class/net
+while ! awk -F'|' '{print $3}' /etc/confluent/confluent.info |grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+    TRIES=$((TRIES + 1))
+    for currif in *; do
+        ip link set $currif up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+oum=$(umask)
+python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+echo $ifname > /tmp/net.ifaces
+
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+hostname=$nodename
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+    hostname=$hostname.$dnsdomain
+fi
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
+proto=${proto#protocol: }
+textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+	autocons=$(cat /tmp/01-autocons.devnode)
+	if [ ! -z "$autocons" ]; then
+	    echo Auto-configuring installed system to use text console
+	    echo Auto-configuring installed system to use text console > $autocons
+	    cp /tmp/01-autocons.conf /etc/cmdline.d/
+	else
+	    echo "Unable to automatically detect requested text console"
+	fi
+fi
+
+echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf
+echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf
+kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
+root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
+export kickstart
+export root
+autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/initqueue/finished/confluent.sh

@@ -0,0 +1 @@
+[ -e /tmp/confluent.initq ]

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+while [ -h $BUNDLENAME ]; do
+    BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
+done
+
+cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
+mkdir -p /sysroot/etc/confluent/
+cp -a /tls /sysroot/etc/confluent
+cp -a /etc/confluent/* /sysroot/etc/confluent/
+sed -i 's/install::/install:*:/' /sysroot/etc/shadow
+sed -i 's/root::/root:*:/' /sysroot/etc/shadow
+mkdir -p /sysroot/root/.ssh
+#chmod 700 /sysroot/root/.ssh
+cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+#chmod 600 /sysroot/root/.ssh/authorized_keys
+mkdir -p /sysroot/etc/ssh/
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent

confluent_osdeploy/el7/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+

confluent_osdeploy/el7/profiles/default/kickstart

@@ -0,0 +1,78 @@
+# In this OS profile, data is largely filled in during the %pre
+# phase, rather than the kickstart actually having the content.
+# None of the files shall be replaced during an upgrade in
+# /var/lib/confluent/public/os/<profile>, so customization should
+# be done by modifying files in /var/lib/confluent/public/os/<profile>
+
+# /tmp/rootpw will provide a 'rootpw' line, either locking password if not configured
+# or the crypted form.
+%include /tmp/rootpw
+
+# timezone is fetched from confluent server, which provides the
+# timezone that the management server itself is in by default.
+%include /tmp/timezone
+
+# similar to timezone, confluent is asked to provide the
+# deployment servers language info and replicate that
+# to the deployment target.
+%include /tmp/langinfo
+
+# bootloader may be specified if crypted.grubpassword is set on
+# a node. This will become a bootloader line if a password
+# was specified
+%include /tmp/grubpw
+
+# The default partition scheme is applied to a single drive, using
+# the getinstalldisk script to make a best guess as to the most
+# appropriate device. See pre.sh and getinstalldisk to customize
+# the automatic behavior, or comment out/delete the 
+# following line and provide your own manual partition plan
+# instead
+%include /tmp/partitioning
+
+reboot
+
+%packages
+chrony
+rsync
+python
+%end
+
+
+# A kickstart.custom file is provided to enable easily adding
+# kickstart content without modifying the stock file.
+# While the stock file may be safely modified, using the .custom
+# files may allow for more easily rebasing custom profiles to newer
+# stock profiles
+%include /tmp/kickstart.custom
+
+%pre
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
+. /tmp/preinst.sh
+%end
+
+%post --nochroot
+mkdir -p /mnt/sysimage/etc/confluent
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+
+# Hook firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
+chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+%end
+
+%post
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+systemctl enable firstboot
+chgrp ssh_keys /etc/ssh/ssh*key
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+%end

confluent_osdeploy/el7/profiles/default/kickstart.custom

@@ -0,0 +1,7 @@
+# Any custom kickstart additions may go here.  It may be required to
+# modify base kickstart file to change certain portions, but adding
+# custom content to this file allows for most convenient way to 
+# refresh to a newer base profile if desired.
+# Two common example customizations are provided:
+#firewall --disabled
+#selinux --disabled

confluent_osdeploy/el7/profiles/default/profile.yaml

@@ -0,0 +1,2 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet

confluent_osdeploy/el7/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/etc/confluent/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el7/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/etc/confluent/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el7/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+export nodename mgr profile
+. /etc/confluent/functions
+
+
+run_remote firstboot.custom
+
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -0,0 +1,38 @@
+run_remote() {
+    requestedcmd="'$*'"
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    if [ -x /usr/bin/chcon ]; then
+        chcon system_u:object_r:bin_t:s0 $cmd
+    fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    /usr/libexec/platform-python $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -0,0 +1,88 @@
+import subprocess
+import os
+
+class DiskInfo(object):
+    def __init__(self, devname):
+        self.name = devname
+        self.wwn = None
+        self.path = None
+        self.model = ''
+        self.size = 0
+        self.driver = None
+        self.mdcontainer = ''
+        devnode = '/dev/{0}'.format(devname)
+        qprop = subprocess.check_output(
+            ['udevadm', 'info', '--query=property', devnode])
+        if not isinstance(qprop, str):
+            qprop = qprop.decode('utf8')
+        for prop in qprop.split('\n'):
+            if '=' not in prop:
+                continue
+            k, v = prop.split('=', 1)
+            if k == 'DEVTYPE' and v != 'disk':
+                raise Exception('Not a disk')
+            elif k == 'DM_NAME':
+                raise Exception('Device Mapper')
+            elif k == 'ID_MODEL':
+                self.model = v
+            elif k == 'DEVPATH':
+                self.path = v
+            elif k == 'ID_WWN':
+                self.wwn = v
+            elif k == 'MD_CONTAINER':
+                self.mdcontainer = v
+        attrs = subprocess.check_output(['udevadm', 'info', '-a', devnode])
+        if not isinstance(attrs, str):
+            attrs = attrs.decode('utf8')
+        for attr in attrs.split('\n'):
+            if '==' not in attr:
+                continue
+            k, v = attr.split('==', 1)
+            k = k.strip()
+            if k == 'ATTRS{size}':
+                self.size = v.replace('"', '')
+            elif (k == 'DRIVERS' and not self.driver
+                    and v not in ('"sd"', '""')):
+                self.driver = v.replace('"', '')
+        if not self.driver and 'imsm' not in self.mdcontainer:
+            raise Exception("No driver detected")
+
+    @property
+    def priority(self):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+            return 0
+        if 'imsm' in self.mdcontainer:
+            return 1
+        if self.driver == 'ahci':
+            return 2
+        if self.driver.startswith('megaraid'):
+            return 3
+        if self.driver.startswith('mpt'):
+            return 4
+        return 99
+
+    def __repr__(self):
+        return repr({
+            'name': self.name,
+            'path': self.path,
+            'wwn': self.wwn,
+            'driver': self.driver,
+            'size': self.size,
+            'model': self.model,
+        })
+
+
+def main():
+    disks = []
+    for disk in sorted(os.listdir('/sys/class/block')):
+        try:
+            disk = DiskInfo(disk)
+            disks.append(disk)
+        except Exception as e:
+            print("Skipping {0}: {1}".format(disk, str(e)))
+    nd = [x.name for x in sorted(disks, key=lambda x: x.priority)]
+    if nd:
+        open('/tmp/installdisk', 'w').write(nd[0])
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el7/profiles/default/scripts/post.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7/profiles/default/scripts/post.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# need to copy over ssh key info
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+
+chmod 700 /etc/confluent
+chmod og-rwx /etc/confluent/*
+
+export mgr profile nodename
+. /etc/confluent/functions
+
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+
+
+if [ -f /tmp/cryptboot ]; then
+    run_remote tpm_luks.sh
+fi
+# By default, the install repository is ignored, change
+# this by manually adding local repositories
+
+rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-*
+run_remote_python add_local_repositories
+# This script will execute in the installed system, but using the installer kernel prior to reboot.
+# This is an appropriate place to run post install activities that do not require the actual installed
+# kernel to run. For example adding drivers that would be needed for first boot to run cleanly.
+# If, for example, there is a post script that has a dependency on a driver or filesystem that
+# cannot work until booting into the installer, use firstboot.sh instead
+
+# run_remote will download and execute from /var/lib/confluent/public/os/<profile>/scripts/ directory
+# run_remote_python will use the appropriate python interpreter path to run the specified script
+# A post.custom is provided to more conveniently hold customizations, see the post.custom file.
+
+# run_remote example.sh
+# run_remote_python example.py
+run_remote post.custom

confluent_osdeploy/el7/profiles/default/scripts/pre.custom

@@ -0,0 +1,4 @@
+. /tmp/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el7/profiles/default/scripts/pre.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+
+# This runs prior to the installer beginning. This is used to rewrite the 
+# scripted install file, merging data from confluent and identifying
+# the most appropriate install source.
+
+# If you want to use a more custom partition plan, the easiest
+# method is to edit the kicktstart file and comment out or
+# delete %include /tmp/partitioning
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+locale=$(grep ^locale: /etc/confluent/confluent.deploycfg)
+locale=${locale#locale: }
+keymap=$(grep ^keymap: /etc/confluent/confluent.deploycfg)
+keymap=${keymap#keymap: }
+echo lang $locale > /tmp/langinfo
+echo keyboard --vckeymap=$keymap >> /tmp/langinfo
+tz=$(grep ^timezone: /etc/confluent/confluent.deploycfg)
+tz=${tz#timezone: }
+ntpsrvs=""
+if grep ^ntpservers: /etc/confluent/confluent.deploycfg > /dev/null; then
+    ntpsrvs="--ntpservers="$(sed -n '/^ntpservers:/,/^[^-]/p' /etc/confluent/confluent.deploycfg|sed 1d|sed '$d' | sed -e 's/^- //' | paste -sd,)
+fi
+echo timezone $ntpsrvs $tz --utc > /tmp/timezone
+rootpw=$(grep ^rootpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$rootpw" = null ]; then
+    echo "rootpw --lock" > /tmp/rootpw
+else
+    echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
+fi
+grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
+if [ "$grubpw" = "null" ]; then
+    touch /tmp/grubpw
+else
+    echo "bootloader --iscrypted --password=$grubpw" > /tmp/grubpw
+fi
+ssh-keygen -A
+for pubkey in /etc/ssh/ssh_host_*_key.pub; do
+    certfile=${pubkey/.pub/-cert.pub}
+    curl -f -X POST -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" -d @$pubkey https://$mgr/confluent-api/self/sshcert > $certfile
+    echo HostCertificate $certfile >> /etc/ssh/sshd_config.anaconda
+done
+/usr/sbin/sshd -f /etc/ssh/sshd_config.anaconda
+if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
+    consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
+    TMUX= tmux a <> $consoledev >&0 2>&1 &
+fi
+cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
+LUKSPARTY=''
+if [ "$cryptboot" == "tpm2" ]; then
+	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
+	echo $cryptboot >> /tmp/cryptboot
+fi
+
+
+export mgr profile nodename
+curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /tmp/functions
+. /tmp/functions
+run_remote_python getinstalldisk
+if [ -e /tmp/installdisk ]; then
+    echo clearpart --all --initlabel >> /tmp/partitioning
+    echo ignoredisk --only-use $(cat /tmp/installdisk) >> /tmp/partitioning
+    echo autopart --nohome $LUKSPARTY >> /tmp/partitioning
+fi
+python /etc/confluent/apiclient /confluent-public/os/$profile/kickstart.custom -o /tmp/kickstart.custom
+run_remote pre.custom

confluent_osdeploy/el7/profiles/default/scripts/prechroot.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# This script runs after install is complete, but inside the installer 
+# environment. This is useful for carrying work done in pre/during the
+# installer into the installed environment.
+
+# It is almost certainly more useful to use post.sh or firstboot.sh
+# for customization, which will run in a more normal mechanism
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+export mgr profile nodename
+cp -a /etc/confluent /mnt/sysimage/etc
+cp /tmp/functions /mnt/sysimage/etc/confluent/
+. /tmp/functions
+cp /tmp/cryptboot /mnt/sysimage/tmp/
+
+# Preserve the ssh setup work done for the installer
+# by copying into the target system and setting up
+# host based authentication
+run_remote setupssh.sh

confluent_osdeploy/el7/profiles/default/scripts/setupssh.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+grep HostCert /etc/ssh/sshd_config.anaconda >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedAuthentication yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo HostbasedUsesNameFromPacketOnly yes >> /mnt/sysimage/etc/ssh/sshd_config
+echo IgnoreRhosts no >> /mnt/sysimage/etc/ssh/sshd_config
+sshconf=/mnt/sysimage/etc/ssh/ssh_config
+if [ -d /mnt/sysimage/etc/ssh/ssh_config.d/ ]; then
+    sshconf=/mnt/sysimage/etc/ssh/ssh_config.d/01-confluent.conf
+fi
+echo 'Host *' >> $sshconf
+echo '    HostbasedAuthentication yes' >> $sshconf
+echo '    EnableSSHKeysign yes' >> $sshconf
+echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
+
+cp /etc/ssh/ssh_host_* /mnt/sysimage/etc/ssh/
+mkdir /mnt/sysimage/root/.ssh/
+chmod 700 /mnt/sysimage/root/.ssh/
+cp /root/.ssh/authorized_keys /mnt/sysimage/root/.ssh/
+chmod 600 /mnt/sysimage/root/.ssh/authorized_keys
+cp /etc/ssh/ssh_known_hosts /mnt/sysimage/etc/ssh/
+curl -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$mgr/confluent-api/self/nodelist > /tmp/allnodes
+cp /tmp/allnodes /mnt/sysimage/etc/ssh/shosts.equiv
+cp /tmp/allnodes /mnt/sysimage/root/.shosts

confluent_osdeploy/el7/profiles/default/scripts/tpm_luks.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+cryptdisk=$(blkid -t TYPE="crypto_LUKS"|sed -e s/:.*//)
+clevis luks bind -f -d $cryptdisk -k - tpm2 '{}' < /etc/confluent/confluent.apikey
+cryptsetup luksRemoveKey $cryptdisk < /etc/confluent/confluent.apikey

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/cmdline/01-confluent.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+echo -n "" >> /tmp/net.ifaces
+cat /tls/*.0 >> /etc/pki/tls/certs/ca-bundle.crt
+if ! grep console= /proc/cmdline >& /dev/null; then
+    autocons=$(/opt/confluent/bin/autocons)
+    if [ -n "$autocons" ]; then
+        echo console=$autocons |sed -e 's!/dev/!!' >> /tmp/01-autocons.conf
+        autocons=${autocons%,*}
+	echo $autocons > /tmp/01-autocons.devnode
+	echo "Detected firmware specified console at $(cat /tmp/01-autocons.conf)" > $autocons
+        echo "Initializing auto detected console when installer starts" > $autocons
+    fi
+fi
+if grep console=ttyS /proc/cmdline >& /dev/null; then
+    echo "Serial console has been requested in the kernel arguments, the local video may not show progress" > /dev/tty1
+fi
+. /lib/anaconda-lib.sh
+wait_for_kickstart

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+[ -e /tmp/confluent.initq ] && return 0
+if [ -f /tmp/dd_disk ]; then
+    for dd in $(cat /tmp/dd_disk); do
+        if [ -e $dd ]; then
+            driver-updates --disk $dd $dd
+        fi
+    done
+fi
+TRIES=0
+oum=$(umask)
+umask 0077
+mkdir -p /etc/confluent
+echo -n > /etc/confluent/confluent.info
+umask $oum
+cd /sys/class/net
+while ! grep ^EXTMGRINFO: /etc/confluent/confluent.info | awk -F'|' '{print $3}' | grep 1 >& /dev/null && [ "$TRIES" -lt 60 ]; do
+    TRIES=$((TRIES + 1))
+    for currif in *; do
+        ip link set $currif up
+    done
+    /opt/confluent/bin/copernicus -t > /etc/confluent/confluent.info
+done
+cd /
+grep ^EXTMGRINFO: /etc/confluent/confluent.info || return 0  # Do absolutely nothing if no data at all yet
+echo -n "" > /tmp/confluent.initq
+# restart cmdline
+echo -n "" > /etc/cmdline.d/01-confluent.conf
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+#TODO: blkid --label <whatever> to find mounted api
+
+cat /tls/*.pem > /etc/confluent/ca.pem
+/usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
+ifidx=$(cat /tmp/confluent.ifidx)
+ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
+ifname=${ifname%:}
+echo $ifname > /tmp/net.ifaces
+
+dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
+dnsdomain=${dnsdomain#dnsdomain: }
+hostname=$nodename
+if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
+    hostname=$hostname.$dnsdomain
+fi
+mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
+mgr=${mgr#deploy_server: }
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+profilename=${profilename#profile: }
+proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
+proto=${proto#protocol: }
+textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
+textconsole=${textconsole#textconsole: }
+if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
+	autocons=$(cat /tmp/01-autocons.devnode)
+	if [ ! -z "$autocons" ]; then
+	    echo Auto-configuring installed system to use text console
+	    echo Auto-configuring installed system to use text console > $autocons
+	    cp /tmp/01-autocons.conf /etc/cmdline.d/
+	else
+	    echo "Unable to automatically detect requested text console"
+	fi
+fi
+
+echo inst.repo=$proto://$mgr/confluent-public/os/$profilename/distribution >> /etc/cmdline.d/01-confluent.conf
+echo inst.ks=$proto://$mgr/confluent-public/os/$profilename/kickstart >> /etc/cmdline.d/01-confluent.conf
+kickstart=$proto://$mgr/confluent-public/os/$profilename/kickstart
+root=anaconda-net:$proto://$mgr/confluent-public/os/$profilename/distribution
+export kickstart
+export root
+autoconfigmethod=$(grep ipv4_method /etc/confluent/confluent.deploycfg)
+autoconfigmethod=${autoconfigmethod#ipv4_method: }
+if [ "$autoconfigmethod" = "dhcp" ]; then
+    echo ip=$ifname:dhcp >>  /etc/cmdline.d/01-confluent.conf
+else
+    v4addr=$(grep ^ipv4_address: /etc/confluent/confluent.deploycfg)
+    v4addr=${v4addr#ipv4_address: }
+    v4gw=$(grep ^ipv4_gateway: /etc/confluent/confluent.deploycfg)
+    v4gw=${v4gw#ipv4_gateway: }
+    if [ "$v4gw" = "null" ]; then
+        v4gw=""
+    fi
+    v4nm=$(grep ipv4_netmask: /etc/confluent/confluent.deploycfg)
+    v4nm=${v4nm#ipv4_netmask: }
+    echo ip=$v4addr::$v4gw:$v4nm:$hostname:$ifname:none >> /etc/cmdline.d/01-confluent.conf
+fi
+nameserversec=0
+while read -r entry; do
+    if [ $nameserversec = 1 ]; then
+        if [[ $entry == "-"* ]] && [[ $entry != "- ''" ]]; then
+            echo nameserver=${entry#- } >> /etc/cmdline.d/01-confluent.conf
+            continue
+        fi
+    fi
+    nameserversec=0
+    if [ "${entry%:*}" = "nameservers" ]; then
+        nameserversec=1
+        continue
+    fi
+done < /etc/confluent/confluent.deploycfg
+

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+BUNDLENAME=/sysroot/etc/pki/tls/certs/ca-bundle.crt
+while [ -h $BUNDLENAME ]; do
+    BUNDLENAME=/sysroot/$(readlink $BUNDLENAME)
+done
+
+cat /etc/pki/tls/certs/ca-bundle.crt > $BUNDLENAME
+mkdir -p /sysroot/etc/confluent/
+chmod 700 /sysroot/etc/confluent
+cp -a /tls /sysroot/etc/confluent
+cp /etc/confluent/* /sysroot/etc/confluent
+sed -i 's/install::/install:*:/' /sysroot/etc/shadow
+sed -i 's/root::/root:*:/' /sysroot/etc/shadow
+mkdir -p /sysroot/root/.ssh
+chmod 700 /sysroot/root/.ssh
+cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+chmod 600 /sysroot/root/.ssh/authorized_keys
+mkdir -p /sysroot/etc/ssh/
+for i in /ssh/*.ca; do
+    echo '@cert-authority *' $(cat $i) >> /sysroot/etc/ssh/ssh_known_hosts
+done
+cp /opt/confluent/bin/apiclient /sysroot/etc/confluent

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
+ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
+mkdir -p $2/boot/efi/boot && \
+ln -s $1/EFI/BOOT/BOOTX64.EFI $1/EFI/BOOT/grubx64.efi $2/boot/efi/boot/
+

confluent_osdeploy/el8/profiles/default/kickstart

@@ -0,0 +1,79 @@
+# In this OS profile, data is largely filled in during the %pre
+# phase, rather than the kickstart actually having the content.
+# None of the files shall be replaced during an upgrade in
+# /var/lib/confluent/public/os/<profile>, so customization should
+# be done by modifying files in /var/lib/confluent/public/os/<profile>
+
+# /tmp/rootpw will provide a 'rootpw' line, either locking password if not configured
+# or the crypted form.
+%include /tmp/rootpw
+
+# timezone is fetched from confluent server, which provides the
+# timezone that the management server itself is in by default.
+%include /tmp/timezone
+
+# similar to timezone, confluent is asked to provide the
+# deployment servers language info and replicate that
+# to the deployment target.
+%include /tmp/langinfo
+
+# bootloader may be specified if crypted.grubpassword is set on
+# a node. This will become a bootloader line if a password
+# was specified
+%include /tmp/grubpw
+
+# The default partition scheme is applied to a single drive, using
+# the getinstalldisk script to make a best guess as to the most
+# appropriate device. See pre.sh and getinstalldisk to customize
+# the automatic behavior, or comment out/delete the 
+# following line and provide your own manual partition plan
+# instead
+%include /tmp/partitioning
+
+reboot
+
+%packages
+@^minimal-environment
+chrony
+rsync
+python3
+%include /tmp/cryptpkglist
+%end
+
+# A kickstart.custom file is provided to enable easily adding
+# kickstart content without modifying the stock file.
+# While the stock file may be safely modified, using the .custom
+# files may allow for more easily rebasing custom profiles to newer
+# stock profiles
+%include /tmp/kickstart.custom
+
+%pre
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/pre.sh > /tmp/preinst.sh
+. /tmp/preinst.sh
+%end
+
+%post --nochroot
+mkdir -p /mnt/sysimage/etc/confluent
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+
+# Hook firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
+chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+%end
+
+%post
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+systemctl enable firstboot
+chgrp ssh_keys /etc/ssh/ssh*key
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
+curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh
+. /tmp/postinst.sh
+%end

confluent_osdeploy/el8/profiles/default/kickstart.custom

@@ -0,0 +1,7 @@
+# Any custom kickstart additions may go here.  It may be required to
+# modify base kickstart file to change certain portions, but adding
+# custom content to this file allows for most convenient way to 
+# refresh to a newer base profile if desired.
+# Two common example customizations are provided:
+#firewall --disabled
+#selinux --disabled

confluent_osdeploy/el8/profiles/default/profile.yaml

@@ -0,0 +1,2 @@
+label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
+kernelargs: quiet

confluent_osdeploy/el8/profiles/default/scripts/add_local_repositories

@@ -0,0 +1,43 @@
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+    import cStringIO
+import imp
+import sys
+apiclient = imp.load_source('apiclient', '/etc/confluent/apiclient')
+repo = None
+server = None
+profile = None
+with open('/etc/confluent/confluent.deploycfg') as dplcfgfile:
+    lines = dplcfgfile.read().split('\n')
+    for line in lines:
+        if line.startswith('deploy_server:'):
+           _, server = line.split(' ', 1)
+        if line.startswith('profile: '):
+           _, profile = line.split(' ', 1)
+
+path = '/confluent-public/os/{0}/distribution/'.format(profile)
+clnt = apiclient.HTTPSClient()
+cfgdata = clnt.grab_url(path + '.treeinfo').decode()
+c = configparser.ConfigParser()
+try:
+    c.read_string(cfgdata)
+except AttributeError:
+    f = cStringIO.StringIO(cfgdata)
+    c.readfp(f)
+for sec in c.sections():
+    if sec.startswith('variant-'):
+        try:
+            repopath = c.get(sec, 'repository')
+        except Exception:
+            continue
+        _, varname = sec.split('-', 1)
+        reponame = '/etc/yum.repos.d/local-{0}.repo'.format(varname.lower())
+        with open(reponame, 'w') as repout:
+            repout.write('[local-{0}]\n'.format(varname.lower()))
+            repout.write('name=Local install repository for {0}\n'.format(varname))
+            if repopath[0] == '.':
+                repopath = repopath[1:]
+            repout.write('baseurl=https://{}/confluent-public/os/{}/distribution/{}\n'.format(server, profile, repopath))
+            repout.write('enabled=1\n')

confluent_osdeploy/el8/profiles/default/scripts/configbmc

@@ -0,0 +1,351 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import ctypes
+import fcntl
+import json
+from select import select
+import socket
+import struct
+import os
+import subprocess
+import sys
+import time
+
+if os.path.exists('/opt/confluent/bin/apiclient'):
+    apiclient = '/opt/confluent/bin/apiclient'
+elif os.path.exists('/etc/confluent/apiclient'):
+    apiclient = '/etc/confluent/apiclient'
+
+class IpmiMsg(ctypes.Structure):
+    _fields_ = [('netfn', ctypes.c_ubyte),
+                ('cmd', ctypes.c_ubyte),
+                ('data_len', ctypes.c_short),
+                ('data', ctypes.POINTER(ctypes.c_ubyte))]
+
+
+class IpmiSystemInterfaceAddr(ctypes.Structure):
+    _fields_ = [('addr_type', ctypes.c_int),
+                ('channel', ctypes.c_short),
+                ('lun', ctypes.c_ubyte)]
+
+
+class IpmiRecv(ctypes.Structure):
+    _fields_ = [('recv_type', ctypes.c_int),
+                ('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+class IpmiReq(ctypes.Structure):
+    _fields_ = [('addr', ctypes.POINTER(IpmiSystemInterfaceAddr)),
+                ('addr_len', ctypes.c_uint),
+                ('msgid', ctypes.c_long),
+                ('msg', IpmiMsg)]
+
+
+_IONONE = 0
+_IOWRITE = 1
+_IOREAD = 2
+IPMICTL_SET_MY_ADDRESS_CMD = (
+    _IOREAD << 30 | ctypes.sizeof(ctypes.c_uint) << 16
+    | ord('i') << 8 | 17)  # from ipmi.h
+IPMICTL_SEND_COMMAND = (
+    _IOREAD << 30 | ctypes.sizeof(IpmiReq) << 16
+    | ord('i') << 8 | 13)  # from ipmi.h
+# next is really IPMICTL_RECEIVE_MSG_TRUNC, but will only use that
+IPMICTL_RECV = (
+    (_IOWRITE | _IOREAD) << 30 | ctypes.sizeof(IpmiRecv) << 16
+    | ord('i') << 8 | 11)  # from ipmi.h
+BMC_SLAVE_ADDR = ctypes.c_uint(0x20)
+CURRCHAN = 0xf
+ADDRTYPE = 0xc
+
+
+class Session(object):
+    def __init__(self, devnode='/dev/ipmi0'):
+        """Create a local session inband
+
+        :param: devnode: The path to the ipmi device
+        """
+        self.ipmidev = open(devnode, 'r+')
+        fcntl.ioctl(self.ipmidev, IPMICTL_SET_MY_ADDRESS_CMD, BMC_SLAVE_ADDR)
+        # the interface is initted, create some reusable memory for our session
+        self.databuffer = ctypes.create_string_buffer(4096)
+        self.req = IpmiReq()
+        self.rsp = IpmiRecv()
+        self.addr = IpmiSystemInterfaceAddr()
+        self.req.msg.data = ctypes.cast(
+            ctypes.addressof(self.databuffer),
+            ctypes.POINTER(ctypes.c_ubyte))
+        self.rsp.msg.data = self.req.msg.data
+        self.userid = None
+        self.password = None
+
+    def await_reply(self):
+        rd, _, _ = select((self.ipmidev,), (), (), 1)
+        while not rd:
+            rd, _, _ = select((self.ipmidev,), (), (), 1)
+
+    def pause(self, seconds):
+        time.sleep(seconds)
+
+    @property
+    def parsed_rsp(self):
+        response = {'netfn': self.rsp.msg.netfn, 'command': self.rsp.msg.cmd,
+                    'code': bytearray(self.databuffer.raw)[0],
+                    'data': bytearray(
+                        self.databuffer.raw[1:self.rsp.msg.data_len])}
+        return response
+
+    def raw_command(self,
+                    netfn,
+                    command,
+                    data=(),
+                    bridge_request=None,
+                    retry=True,
+                    delay_xmit=None,
+                    timeout=None,
+                    waitall=False, rslun=0):
+        self.addr.channel = CURRCHAN
+        self.addr.addr_type = ADDRTYPE
+        self.addr.lun = rslun
+        self.req.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        self.req.addr = ctypes.pointer(self.addr)
+        self.req.msg.netfn = netfn
+        self.req.msg.cmd = command
+        if data:
+            data = memoryview(bytearray(data))
+            try:
+                self.databuffer[:len(data)] = data[:len(data)]
+            except ValueError:
+                self.databuffer[:len(data)] = data[:len(data)].tobytes()
+        self.req.msg.data_len = len(data)
+        fcntl.ioctl(self.ipmidev, IPMICTL_SEND_COMMAND, self.req)
+        self.await_reply()
+        self.rsp.msg.data_len = 4096
+        self.rsp.addr = ctypes.pointer(self.addr)
+        self.rsp.addr_len = ctypes.sizeof(IpmiSystemInterfaceAddr)
+        fcntl.ioctl(self.ipmidev, IPMICTL_RECV, self.rsp)
+        return self.parsed_rsp
+
+
+def _is_tsm(model):
+    return model in ('7y00', '7z01', '7y98', '7y99')
+
+def set_port(s, port, vendor, model):
+    oport = port
+    if vendor not in ('IBM', 'Lenovo'):
+        raise Exception('{0} not implemented'.format(vendor))
+    if _is_tsm(model):
+        return set_port_tsm(s, port, model)
+    else:
+        set_port_xcc(s, port, model)
+        return 1
+
+
+def set_port_tsm(s, port, model):
+    oport = port
+    sys.stdout.write('Setting TSM port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    if port == 'ocp':
+        s.raw_command(0x32, 0x71, b'\x00\x01\x00')
+    elif port == 'dedicated':
+        s.raw_command(0x32, 0x71, b'\x00\x00\x00')
+    else:
+        raise Exception("Unsupported port for TSM")
+    timer = 15
+    while timer:
+        time.sleep(1.0)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+    if port == 'ocp':
+        iface = 0
+        s.raw_command(0x32, 0x71, b'\x00\x00\x03')
+    elif port == 'dedicated':
+        iface = 1
+        s.raw_command(0x32, 0x71, b'\x00\x01\x03')
+    rsp = s.raw_command(0x32, 0x72, bytearray(4, iface, 0))
+    print('Complete')
+    return int(rsp['data'][0])
+
+
+def set_port_xcc(s, port, model):
+    oport = port
+    if port.lower() == 'dedicated':
+        port = b'\x01'
+    elif port.lower() in ('ml2', 'ocp'):
+        port = b'\x02\x00'
+    elif port.lower() == 'lom':
+        if model == '7x58':
+            port = b'\x00\x02'
+        else:
+            port = b'\x00\x00'
+    else:
+        port = port.split(' ')
+        port = bytes(bytearray([int(x) for x in port]))
+    currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if port == currport:
+        sys.stdout.write('XCC port already set to "{}"\n'.format(oport))
+        return
+    sys.stdout.write('Setting XCC port to "{}"...'.format(oport))
+    sys.stdout.flush()
+    s.raw_command(0xc, 1, b'\x01\xc0' + port)
+    tries = 60
+    while currport != port and tries:
+        tries -= 1
+        time.sleep(0.5)
+        sys.stdout.write('.')
+        sys.stdout.flush()
+        currport = bytes(s.raw_command(0xc, 2, b'\x01\xc0\x00\x00')['data'][1:])
+    if not tries:
+        raise Exception('Timeout attempting to set port')
+    sys.stdout.write('Complete\n')
+
+
+def set_vlan(s, vlan, channel):
+    ovlan = vlan
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    currvlan = bytes(s.raw_command(0xc, 2, bytearray([channel, 0x14 ,0, 0]))['data'][1:])
+    if bytearray(currvlan)[1] & 0b10000000 == 0:
+        currvlan = b'\x00\x00'
+    if currvlan == vlan:
+        sys.stdout.write('VLAN already configured to "{0}"\n'.format(ovlan))
+        return False
+    rsp = s.raw_command(0xc, 1, bytearray([channel, 0x14]) + vlan)
+    if rsp.get('code', 1) == 0:
+        print('VLAN configured to "{}"'.format(ovlan))
+    else:
+        print('Error setting vlan: ' + repr(rsp))
+    return
+
+
+def get_lan_channel(s):
+    for chan in range(1, 16):
+        rsp = s.raw_command(6, 0x42, bytearray([chan]))['data']
+        if not rsp:
+            continue
+        medtype = int(rsp[1]) & 0b1111111
+        if medtype not in (4, 6):
+            continue
+        rsp = s.raw_command(0xc, 2, bytearray([2, chan, 5, 0, 0]))
+        if rsp.get('code', 1) == 0:
+            return chan
+    return 1
+
+def set_ipv4(s, ipaddr, channel):
+    oipaddr = ipaddr
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
+    if rsp == ipaddr:
+        print('IP Address already set to {}'.format(oipaddr))
+        return
+    rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+    if rsp != 1:
+        sys.stdout.print("Changing configuration to static...")
+        sys.stdout.flush()
+        resp = s.raw_command(0xc, 1, bytearray([channel, 4, 1]))
+        tries = 0
+        while rsp != 1 and tries < 30:
+            sys.stdout.write('.')
+            tries += 1
+            time.sleep(0.5)
+            rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
+        sys.stdout.write('Complete')
+    print('Setting IP to {}'.format(oipaddr))
+    s.raw_command(0xc, 1, bytearray([channel, 3]) + ipaddr)
+
+
+def set_subnet(s, prefix, channel):
+    oprefix = prefix
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
+    if rsp == mask:
+        print('Subnet Mask already set to /{}'.format(oprefix))
+        return
+    print('Setting subnet mask to /{}'.format(oprefix))
+    s.raw_command(0xc, 1, bytearray([channel, 6]) + mask)
+
+
+def set_gateway(s, gw, channel):
+    ogw = gw
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    if rsp == gw:
+        print('Gateway already set to {}'.format(ogw))
+        return
+    print('Setting gateway to {}'.format(ogw))
+    s.raw_command(0xc, 1, bytearray([channel, 12]) + gw)
+
+def main():
+    a = argparse.ArgumentParser(description='Locally configure a BMC device')
+    a.add_argument('-v', help='vlan id or off to disable vlan tagging')
+    a.add_argument('-p', help='Which port to use (dedicated, lom, ocp, ml2)')
+    a.add_argument('-i', help='JSON configuration file to read for '
+                   'configuration')
+    a.add_argument('-c', help='Use Confluent API to direct BMC configuration',
+                   action='store_true')
+    args = a.parse_args()
+    if args.i:
+        bmccfg = json.load(open(args.i))
+    elif args.c:
+        bmccfgsrc = subprocess.check_output(
+            [sys.executable, apiclient, '/confluent-api/self/bmcconfig', '-j'])
+        bmccfg = json.loads(bmccfgsrc)
+    else:
+        bmccfg = {}
+    if args.p is not None:
+        bmccfg['bmcport'] = args.p
+    if args.v is not None:
+        bmccfg['bmcvlan'] = args.v
+    vendor = open('/sys/devices/virtual/dmi/id/sys_vendor').read()
+    vendor = vendor.strip()
+    try:
+        model = open('/sys/devices/virtual/dmi/id/product_sku').read()
+    except Exception:
+        model = open('/sys/devices/virtual/dmi/id/product_name').read()
+    if vendor in ('Lenovo', 'IBM'):
+        if '[' in model and ']' in model:
+            model = model.split('[')[1].split(']')[0]
+        model = model[:4].lower()
+    s = Session('/dev/ipmi0')
+    if not bmccfg:
+        print("No BMC configuration specified, exiting.")
+        return
+    if bmccfg.get('bmcport', None):
+        channel = set_port(s, bmccfg['bmcport'], vendor, model)
+    else:
+        channel = get_lan_channel(s)
+    if bmccfg.get('bmcvlan', None):
+        set_vlan(s, bmccfg['bmcvlan'], channel)
+    if bmccfg.get('bmcipv4', None):
+        set_ipv4(s, bmccfg['bmcipv4'], channel)
+    if bmccfg.get('prefixv4', None):
+        set_subnet(s, bmccfg['prefixv4'], channel)
+    if bmccfg.get('bmcgw', None):
+        set_gateway(s, bmccfg['bmcgw'], channel)
+    #await_config(s, bmccfg, channel)
+
+
+if __name__ == '__main__':
+    main()

confluent_osdeploy/el8/profiles/default/scripts/firstboot.custom

@@ -0,0 +1,4 @@
+. /etc/confluent/functions
+# This is a convenient place to keep customizations separate from modifying the stock scripts
+# While modification of the stock scripts is fine, it may be easier to rebase to a newer
+# stock profile if the '.custom' files are used.

confluent_osdeploy/el8/profiles/default/scripts/firstboot.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=First Boot Process
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/etc/confluent/firstboot.sh
+
+[Install]
+WantedBy=multi-user.target
+

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# This script is executed on the first boot after install has
+# completed. It is best to edit the middle of the file as
+# noted below so custom commands are executed before
+# the script notifies confluent that install is fully complete.
+
+nodename=$(grep ^NODENAME /etc/confluent/confluent.info|awk '{print $2}')
+apikey=$(cat /etc/confluent/confluent.apikey)
+mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|awk '{print $2}')
+cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
+export nodename mgr profile
+. /etc/confluent/functions
+
+
+run_remote firstboot.custom
+
+
+curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
+systemctl disable firstboot
+rm /etc/systemd/system/firstboot.service