Fix non-client fallback

If client doesn't provide file,
then we need to open by name.

.gitignore

@@ -1,4 +1,7 @@
 *.pyc
 .*.
 confluent_client/man/man*
+.vscode
 .*.sw*
+.sw*
+.idea/*

confluent_client/bin/confetty

@@ -373,7 +373,7 @@ def do_command(command, server):
     if argv[0] == 'exit':
         if os.environ.get('TERM', '') not in ('linux'):
             sys.stdout.write('\x1b]0;\x07')
-        raise Bailout()
+        raise BailOut()
     elif argv[0] in ('help', '?'):
         return print_help()
     elif argv[0] == 'cd':
@@ -948,6 +948,8 @@ def main():
                             clearpowermessage = False
                         try:
                             sys.stdout.write(data)
+                        except UnicodeEncodeError:
+                            sys.stdout.buffer.write(data.encode('utf8'))
                         except IOError:  # Some times circumstances are bad
                             # resort to byte at a time...
                             for d in data:
@@ -1020,6 +1022,13 @@ if __name__ == '__main__':
         main()
     except BailOut as e:
         errcode = e.errorcode
+    except Exception as e:
+        import traceback
+        try:
+            quitconfetty()
+        except Exception:
+            pass
+        traceback.print_exc()
     finally:
         if deadline and os.times()[4] < deadline:
             sys.stderr.write('[Exited early, hit enter to continue]')

confluent_client/bin/dir2img

@@ -8,7 +8,7 @@ import os
 import subprocess
 import sys
 
-def create_image(directory, image):
+def create_image(directory, image, label=None):
     ents = 0
     datasz = 512
     for dir in os.walk(sys.argv[1]):
@@ -25,8 +25,13 @@ def create_image(directory, image):
     with open(image, 'wb') as imgfile:
         imgfile.seek(datasz * 512 - 1)
         imgfile.write(b'\x00')
-    subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
-                            str(datasz), '-s', '1','-h', '1', '::'])
+    if label:
+        subprocess.check_call(['mformat', '-i', image, '-v', label,
+                               '-r', '16', '-d', '1', '-t', str(datasz),
+                               '-s', '1','-h', '1', '::'])
+    else:
+        subprocess.check_call(['mformat', '-i', image, '-r', '16', '-d', '1', '-t',
+                                str(datasz), '-s', '1','-h', '1', '::'])
     # Some clustered filesystems will have the lock from mformat
     # linger after close (mformat doesn't unlock)
     # do a blocking wait for shared lock and then explicitly
@@ -50,4 +55,7 @@ if __name__ == '__main__':
         sys.stderr.write("Usage: {0} <directory> <imagefile>".format(
             sys.argv[0]))
         sys.exit(1)
-    create_image(sys.argv[1], sys.argv[2])
+    label = None
+    if len(sys.argv) > 3:
+        label = sys.argv[3]
+    create_image(sys.argv[1], sys.argv[2], label)

confluent_client/bin/nodeconfig

@@ -134,9 +134,10 @@ def _assign_value():
     assignment[key] = value
 
 
-def parse_config_line(arguments):
+def parse_config_line(arguments, single=False):
     global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
-    for param in arguments:
+    for pidx in range(0, len(arguments)):
+        param = arguments[pidx]
         if param == 'show':
             continue  # forgive muscle memory of pasu users
         if param == 'set':
@@ -146,7 +147,12 @@ def parse_config_line(arguments):
         if needval:
             key = needval
             needval = None
-            value = param
+            if single:
+                value = ' '.join(arguments[pidx:])
+                _assign_value()
+                break
+            else:
+                value = param
             _assign_value()
             continue
         if '=' in param or param[-1] == ':' or forceset:
@@ -215,7 +221,7 @@ if options.batch:
             pass
         argset = argset.strip()
         if argset:
-            parse_config_line(shlex.split(argset))
+            parse_config_line(shlex.split(argset), single=True)
         argset = argfile.readline()
 else:
     parse_config_line(args[1:])
@@ -272,7 +278,6 @@ if setmode:
             rcode |= client.printerror(fr)
             for node in fr.get('databynode', []):
                 r = fr['databynode'][node]
-                rcode |= client.printerror(r, node)
                 if 'value' not in r:
                     continue
                 keyval = r['value']

confluent_client/bin/nodedeploy

@@ -79,7 +79,7 @@ def main(args):
         sys.stderr.write('-n is a required argument currently to perform an install, optionally with -p\n')
         return 1
     if not args.profile and args.network:
-        sys.stderr.write('profile is a required argument to request a network deployment\n')
+        sys.stderr.write('Both noderange and a profile name are required arguments to request a network deployment\n')
         return 1
     if extra:
         sys.stderr.write('Unrecognized arguments: ' + repr(extra) + '\n')

confluent_client/bin/nodediscover

@@ -263,7 +263,11 @@ def list_matching_macs(options, session, node=None, checknode=True):
         return [options.mac.replace(':', '-')]
     else:
         path += 'by-mac/'
-        return [x['item']['href'] for x in session.read(path)]
+        ret = []
+        for x in session.read(path):
+            if 'item' in x and 'href' in x['item']:
+                ret.append(x['item']['href'])
+        return ret
 
 def assign_discovery(options, session, needid=True):
     abort = False

confluent_client/bin/nodefirmware

@@ -109,6 +109,12 @@ def update_firmware(session, filename):
     if options.backup:
         upargs['bank'] = 'backup'
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -149,7 +155,6 @@ def show_firmware(session):
             if 'databynode' not in res:
                 continue
             for node in res['databynode']:
-                exitcode |= client.printerror(res['databynode'][node], node)
                 if 'firmware' not in res['databynode'][node]:
                     continue
                 for inv in res['databynode'][node]['firmware']:

confluent_client/bin/nodelicense

@@ -69,6 +69,12 @@ def install_license(session, filename):
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, instargs):
         for node in res.get('databynode', []):
             if 'error' in res['databynode'][node]:

confluent_client/bin/nodemedia

@@ -111,6 +111,12 @@ def upload_media(noderange, media):
     resource = '/noderange/{0}/media/uploads/'.format(noderange)
     upargs = {'filename': filename}
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     nodeurls = {}
     for res in session.create(resource, upargs):
         if 'created' not in res:

confluent_client/bin/nodesetboot

@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|floppy]')
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|usb|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')

confluent_client/bin/nodestorage

@@ -186,7 +186,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',

confluent_client/bin/stats

@@ -22,13 +22,17 @@ import io
 import numpy as np
 
 import os
-import sixel
 import subprocess
 import sys
 
-class DumbWriter(sixel.SixelWriter):
-    def restore_position(self, output):
-        return
+try:
+    import sixel
+
+    class DumbWriter(sixel.SixelWriter):
+        def restore_position(self, output):
+            return
+except ImportError:
+    pass
 
 
 def plot(gui, output, plotdata, bins):

confluent_client/confluent/client.py

@@ -46,6 +46,13 @@ try:
 except NameError:
     pass
 
+
+class NestedDict(dict):
+    def __missing__(self, key):
+        value = self[key] = type(self)()
+        return value
+
+
 def stringify(instr):
     # Normalize unicode and bytes to 'str', correcting for
     # current python version
@@ -111,6 +118,12 @@ def printerror(res, node=None):
     exitcode = 0
     if 'errorcode' in res:
         exitcode = res['errorcode']
+    for node in res.get('databynode', {}):
+        exitcode = res['databynode'][node].get('errorcode', exitcode)
+        if 'error' in res['databynode'][node]:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['databynode'][node]['error']))
+            if exitcode == 0:
+                exitcode = 1
     if 'error' in res:
         if node:
             sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
@@ -148,6 +161,7 @@ class Command(object):
         self._prevkeyname = None
         self.connection = None
         self._currnoderange = None
+        self.unixdomain = False
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
                 self.serverloc = os.environ['CONFLUENT_HOST']
@@ -157,11 +171,13 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+            self.unixdomain = True
         elif self.serverloc == '/var/run/confluent/api.sock':
             raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
-        tlvdata.recv(self.connection)
+        self.protversion = int(tlvdata.recv(self.connection).split(
+            b'--')[1].strip()[1:])
         authdata = tlvdata.recv(self.connection)
         if authdata['authpassed'] == 1:
             self.authenticated = True
@@ -172,6 +188,13 @@ class Command(object):
             passphrase = os.environ['CONFLUENT_PASSPHRASE']
             self.authenticate(username, passphrase)
 
+    def add_file(self, name, handle, mode):
+        if self.protversion < 3:
+            raise Exception('Not supported with connected confluent server')
+        if not self.unixdomain:
+            raise Exception('Can only add a file to a unix domain connection')
+        tlvdata.send(self.connection, {'filename': name, 'mode': mode}, handle)
+
     def authenticate(self, username, password):
         tlvdata.send(self.connection,
                      {'username': username, 'password': password})
@@ -411,7 +434,7 @@ def send_request(operation, path, server, parameters=None):
         result = tlvdata.recv(server)
 
 
-def attrrequested(attr, attrlist, seenattributes):
+def attrrequested(attr, attrlist, seenattributes, node=None):
     for candidate in attrlist:
         truename = candidate
         if candidate.startswith('hm'):
@@ -419,10 +442,16 @@ def attrrequested(attr, attrlist, seenattributes):
         if candidate in _attraliases:
             candidate = _attraliases[candidate]
         if fnmatch.fnmatch(attr.lower(), candidate.lower()):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = True
             return True
         elif attr.lower().startswith(candidate.lower() + '.'):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = 1
             return True
     return False
 
@@ -438,13 +467,15 @@ def _sort_attrib(k):
 
 def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
-    seenattributes = set([])
+    seenattributes = NestedDict()
+    allnodes = set([])
     for res in session.read(path):
         if 'error' in res:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
+            allnodes.add(node)
             for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
@@ -452,7 +483,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attr == 'errorcode':
                     exitcode |= val
                     continue
-                seenattributes.add(attr)
+                seenattributes[node][attr] = True
                 if rename:
                     printattr = rename.get(attr, attr)
                 else:
@@ -460,7 +491,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attrprefix:
                     printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
-                if show_attr(attr, requestargs, seenattributes, options):
+                if show_attr(attr, requestargs, seenattributes, options, node):
                     if 'value' in currattr:
                         if currattr['value'] is not None:
                             val = currattr['value']
@@ -541,23 +572,39 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                                 except TypeError:
                                     pass
                         cprint(attrout)
+    somematched = set([])
+    printmissing = set([])
+    badnodes = NestedDict()
     if not exitcode:
         if requestargs:
             for attr in requestargs:
-                if attr not in seenattributes:
-                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
-                    exitcode = 1
+                for node in allnodes:
+                    if attr in seenattributes[node]:
+                        somematched.add(attr)
+                    else:
+                        badnodes[node][attr] = True
+                        exitcode = 1
+        for node in sortutil.natural_sort(badnodes):
+            for attr in badnodes[node]:
+                if attr in somematched:
+                    sys.stderr.write(
+                        'Error: {0} matches no valid value for {1}\n'.format(
+                            attr, node))
+                else:
+                    printmissing.add(attr)
+        for missing in printmissing:
+            sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
     return exitcode
 
 
-def show_attr(attr, requestargs, seenattributes, options):
+def show_attr(attr, requestargs, seenattributes, options, node):
     try:
         reverse = options.exclude
     except AttributeError:
         reverse = False
     if requestargs is None or requestargs == []:
         return True
-    processattr = attrrequested(attr, requestargs, seenattributes)
+    processattr = attrrequested(attr, requestargs, seenattributes, node)
     if reverse:
         processattr = not processattr
     return processattr

confluent_client/confluent/tlv.py

@@ -17,4 +17,4 @@
 
 
 class Types(object):
-    text, json = range(2)
+    text, json, filehandle = range(3)

confluent_client/confluent/tlvdata.py

@@ -15,9 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import array
+import ctypes
+import ctypes.util
 import confluent.tlv as tlv
+import eventlet.green.socket as socket
+import eventlet.green.select as select
 from datetime import datetime
 import json
+import os
 import struct
 
 try:
@@ -30,6 +36,72 @@ try:
 except NameError:
     pass
 
+class iovec(ctypes.Structure):   # from uio.h
+    _fields_ = [('iov_base', ctypes.c_void_p),
+                ('iov_len', ctypes.c_size_t)]
+
+
+iovec_ptr = ctypes.POINTER(iovec)
+
+
+class cmsghdr(ctypes.Structure):  # also from bits/socket.h
+    _fields_ = [('cmsg_len', ctypes.c_size_t),
+                ('cmsg_level', ctypes.c_int),
+                ('cmsg_type', ctypes.c_int)]
+
+    @classmethod
+    def init_data(cls, cmsg_len, cmsg_level, cmsg_type, cmsg_data):
+        Data = ctypes.c_ubyte * ctypes.sizeof(cmsg_data)
+        class _flexhdr(ctypes.Structure):
+            _fields_ = cls._fields_ + [('cmsg_data', Data)]
+
+        datab = Data(*bytearray(cmsg_data))
+        return _flexhdr(cmsg_len=cmsg_len, cmsg_level=cmsg_level,
+                        cmsg_type=cmsg_type, cmsg_data=datab)
+
+
+def CMSG_LEN(length):
+    sizeof_cmshdr = ctypes.sizeof(cmsghdr)
+    return ctypes.c_size_t(CMSG_ALIGN(sizeof_cmshdr).value + length)
+
+
+SCM_RIGHTS = 1
+
+
+class msghdr(ctypes.Structure):  # from bits/socket.h
+    _fields_ = [('msg_name', ctypes.c_void_p),
+                ('msg_namelen', ctypes.c_uint),
+                ('msg_iov', ctypes.POINTER(iovec)),
+                ('msg_iovlen', ctypes.c_size_t),
+                ('msg_control', ctypes.c_void_p),
+                ('msg_controllen', ctypes.c_size_t),
+                ('msg_flags', ctypes.c_int)]
+
+
+def CMSG_ALIGN(length):  # bits/socket.h
+    ret = (length + ctypes.sizeof(ctypes.c_size_t) - 1
+           & ~(ctypes.sizeof(ctypes.c_size_t) - 1))
+    return ctypes.c_size_t(ret)
+
+
+def CMSG_SPACE(length):  # bits/socket.h
+    ret = CMSG_ALIGN(length).value + CMSG_ALIGN(ctypes.sizeof(cmsghdr)).value
+    return ctypes.c_size_t(ret)
+
+
+class ClientFile(object):
+    def __init__(self, name, mode, fd):
+        self.fileobject = os.fdopen(fd, mode)
+        self.filename = name
+
+libc = ctypes.CDLL(ctypes.util.find_library('c'))
+recvmsg = libc.recvmsg
+recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+recvmsg.restype = ctypes.c_int
+sendmsg = libc.sendmsg
+sendmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+sendmsg.restype = ctypes.c_size_t
+
 def decodestr(value):
     ret = None
     try:
@@ -65,7 +137,7 @@ def _unicode_list(currlist):
             _unicode_list(currlist[i])
 
 
-def send(handle, data):
+def send(handle, data, filehandle=None):
     if isinstance(data, unicode):
         try:
             data = data.encode('utf-8')
@@ -93,9 +165,27 @@ def send(handle, data):
         if tl > 16777215:
             raise Exception("JSON data exceeds protocol limits")
         # xor in the type (0b1 << 24)
-        tl |= 16777216
-        handle.sendall(struct.pack("!I", tl))
-        handle.sendall(sdata)
+        if filehandle is None:
+            tl |= 16777216
+            handle.sendall(struct.pack("!I", tl))
+            handle.sendall(sdata)
+        else:
+            tl |= (2 << 24)
+            handle.sendall(struct.pack("!I", tl))
+            cdtype = ctypes.c_ubyte * len(sdata)
+            cdata = cdtype.from_buffer(bytearray(sdata))
+            ciov = iovec(iov_base=ctypes.addressof(cdata),
+                         iov_len=ctypes.c_size_t(ctypes.sizeof(cdata)))
+            fd = ctypes.c_int(filehandle)
+            cmh = cmsghdr.init_data(
+                cmsg_len=CMSG_LEN(
+                    ctypes.sizeof(fd)), cmsg_level=socket.SOL_SOCKET,
+                    cmsg_type=SCM_RIGHTS, cmsg_data=fd)
+            mh = msghdr(msg_name=None, msg_len=0, msg_iov=iovec_ptr(ciov),
+                        msg_iovlen=1, msg_control=ctypes.addressof(cmh),
+                        msg_controllen=ctypes.c_size_t(ctypes.sizeof(cmh)))
+            sendmsg(handle.fileno(), mh, 0)
+
 
 def recvall(handle, size):
     rd = handle.recv(size)
@@ -125,12 +215,42 @@ def recv(handle):
     datatype = (tl & 2130706432) >> 24  # grab 7 bits from near beginning
     if dlen == 0:
         return None
-    data = handle.recv(dlen)
-    while len(data) < dlen:
-        ndata = handle.recv(dlen - len(data))
-        if not ndata:
-            raise Exception("Error reading data")
-        data += ndata
+    if datatype == tlv.Types.filehandle:
+        filehandles = array.array('i')
+        rawbuffer = bytearray(2048)
+        pkttype = ctypes.c_ubyte * 2048
+        data = pkttype.from_buffer(rawbuffer)
+        cmsgsize = CMSG_SPACE(ctypes.sizeof(ctypes.c_int)).value
+        cmsgarr = bytearray(cmsgsize)
+        cmtype = ctypes.c_ubyte * cmsgsize
+        cmsg = cmtype.from_buffer(cmsgarr)
+        cmsg.cmsg_level = socket.SOL_SOCKET
+        cmsg.cmsg_type = SCM_RIGHTS
+        cmsg.cmsg_len = CMSG_LEN(ctypes.sizeof(ctypes.c_int))
+        iov = iovec()
+        iov.iov_base = ctypes.addressof(data)
+        iov.iov_len = 2048
+        msg = msghdr()
+        msg.msg_iov = ctypes.pointer(iov)
+        msg.msg_iovlen = 1
+        msg.msg_control = ctypes.addressof(cmsg)
+        msg.msg_controllen = ctypes.sizeof(cmsg)
+        select.select([handle], [], [])
+        i = recvmsg(handle.fileno(), ctypes.pointer(msg), 0)
+        cdata = cmsgarr[CMSG_LEN(0).value:]
+        data = rawbuffer[:i]
+        if cmsg.cmsg_level == socket.SOL_SOCKET and cmsg.cmsg_type == SCM_RIGHTS:
+            filehandles.fromstring(bytes(
+                cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
+        data = json.loads(bytes(data))
+        return ClientFile(data['filename'], data['mode'], filehandles[0])
+    else:
+        data = handle.recv(dlen)
+        while len(data) < dlen:
+            ndata = handle.recv(dlen - len(data))
+            if not ndata:
+                raise Exception("Error reading data")
+            data += ndata
     if datatype == tlv.Types.text:
         return data
     elif datatype == tlv.Types.json:

confluent_client/confluent_env.sh

@@ -91,7 +91,7 @@ _confluent_nodeidentify_completion()
 
 _confluent_nodesetboot_completion()
 {
-    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy -h -b -p")
+    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy,usb -h -b -p")
     _confluent_generic_completion
 }
 

confluent_client/doc/man/nodeconfig.ronn

@@ -14,6 +14,12 @@ running configuration on the node firmware.  Calling without '=' will show the
 current value, and '=' will change the value.  Network information can be
 given as a node expression, as documented in the man page for nodeattribexpressions(5).
 
+Note that when using nodeconfig to submit changes, it will exit when the change
+is accepted, but the endpoint may not have fully processed it. Doing a show
+immediately after doing a set may reflect older information. Also, if changing
+BIOS/UEFI settings, the change may appear in output, but generally won't
+actually be in effect until a reboot.
+
 ## OPTIONS
 
 * `-c`, `--comparedefault`:

confluent_client/doc/man/nodesetboot.ronn

@@ -4,7 +4,7 @@ nodesetboot(8) -- Check or set next boot device for noderange
 ## SYNOPSIS
 
 `nodesetboot <noderange>`  
-`nodesetboot [options] <noderange> [default|cd|network|setup|hd|floppy]`
+`nodesetboot [options] <noderange> [default|cd|network|setup|hd|usb|floppy]`
 
 ## DESCRIPTION
 
@@ -46,6 +46,10 @@ control.
   Request boot from floppy.  Generally speaking firmware uses this to mean a USB
   flash drive or similar (whether virtual or physical).
 
+* `usb`:
+  Request boot from usb.  Generally speaking firmware uses this to mean a USB
+  flash drive or similar (whether virtual or physical).
+
 * `network`:
   Request boot to network
   

confluent_client/doc/man/nodestorage.ronn

@@ -3,12 +3,15 @@ nodestorage(8) -- Examine/Modify storage configuration of a node
 
 ## SYNOPSIS
 
-`nodestorage <noderange> [show|create|delete] [options]`  
+`nodestorage <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]`  
 
 ## DESCRIPTION
 
 `nodestorage` provides access to the remote storage configuration of
-the noderange.
+the noderange. The `show` subcommand will show current storage configuration,
+`create` can be used to create new arrays or volumes, `delete` can be used to
+remove volumes and arrays, and `diskset` can modify the usage of disks
+indicated by `-d` to either be `unconfigured`, `jbod`, or `hotspare`.
 
 ## OPTIONS
 * `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
@@ -29,6 +32,15 @@ the noderange.
   volumes, or selecting a volume for delete.  Default
   behavior is to use implementation provided default
 
+* `-z` **STRIPSIZES**, `--stripsizes`=**STRIPSIZES**:
+  Comma separated list of stripsizes to use when creating volumes.
+  This value is in kilobytes.  The default behavior is to allow the
+  storage controller to decide
+
+* `-m` **MAXNODES**, `--maxnodes`=**MAXNODES**:
+  Specify a maximum number of nodes to configure storage on, prompting
+  if over the threshold
+
 ## EXAMPLES
 
 * Deleting the volume `somedata`:

confluent_client/doc/man/stats.ronn

@@ -0,0 +1,49 @@
+stats(8) -- Common basic statistics on typical numeric data in output
+==============================
+
+## SYNOPSIS
+
+`<other command> | stats [-c N] [-d D] [-x|-g|-t|-o image.png] [-s N] [-v] [-b N]
+
+## DESCRIPTION
+
+The **stats** command helps analyze common numerical data such as performance numbers
+or temperatures or any other numerical value.
+
+By default it looks for the last numerical output on the first line to identify the numerical column
+and analyze that number. This can be overriden by **-c COLUMN** to indicate a column. By default,
+whitespace and commas are treated to delimit columns, and **-d DELIMITER** can be used to override.
+
+By default it outputs basic statistics, but a histogram is available either text or through X11 output
+or sixel or output to an image file depending on whether **-x**, **-g**, **-t*, or **-o image.png** is
+used.
+
+## OPTIONS
+
+ * `-c N`:
+   Select column number. Defaults to last column that appears numeric
+
+ * `-d D`:
+   Specify a custom column delimiter
+
+ * `-x`:
+     Output in Sixel format (supported by mlterm and PuTTY, among others)
+
+ * `-g`:
+   Try to open histogram as an X window
+
+ * `-t`:
+   Output histogram as bars rendered by =
+
+ * `-o image.png`:
+   Write graphical histogram to image.png.
+
+ * `-s N`:
+   Ignore specified number of lines as header content before processing numbers
+
+ * `-v`:
+   Treat value before : on each line as a label, and show which labels belong to which histogram buckets.
+
+ * `-b N`:
+   Specify a custom number of buckets for histogram. The default is 10.
+   

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -10,13 +10,32 @@ import ssl
 import sys
 
 def get_apikey(nodename, mgr):
+    sealnew = True
     if os.path.exists('/etc/confluent/confluent.apikey'):
         return open('/etc/confluent/confluent.apikey').read().strip()
     apikey = subprocess.check_output(['/opt/confluent/bin/clortho', nodename, mgr])
     if not isinstance(apikey, str):
         apikey = apikey.decode('utf8')
+    if apikey.startswith('SEALED:'):
+        sealnew = False
+        with open('/etc/confluent/confluent.sealedapikey', 'w+') as apiout:
+            apiout.write(apikey[7:])
+        with open('/etc/confluent/confluent.sealedapikey') as inp:
+            sp = subprocess.Popen(['/usr/bin/clevis-decrypt-tpm2'],
+                stdin=inp, stdout=subprocess.PIPE)
+            apikey = sp.communicate()[0]
+            if not isinstance(apikey, str):
+                apikey = apikey.decode('utf8')
     with open('/etc/confluent/confluent.apikey', 'w+') as apiout:
         apiout.write(apikey)
+    if sealnew and os.path.exists('/usr/bin/clevis-encrypt-tpm2'):
+        try:
+            with open('/etc/confluent/confluent.apikey') as apin:
+                sealed = subprocess.check_output(
+                    ['/usr/bin/clevis-encrypt-tpm2', '{}'], stdin=apin)
+            print(HTTPSClient().grab_url('/confluent-api/self/saveapikey', sealed).decode())
+        except Exception:
+            sys.stderr.write('Unable to persist API key through TPM2 sealing\n')
     apikey = apikey.strip()
     os.chmod('/etc/confluent/confluent.apikey', 0o600)
     return apikey
@@ -47,6 +66,15 @@ class HTTPSClient(client.HTTPConnection, object):
                 ifout.write(ifidx)
         if json:
             self.stdheaders['ACCEPT'] = 'application/json'
+        try:
+            info = open('/etc/confluent/confluent.deploycfg').read().split('\n')
+        except Exception:
+            info = None
+        if info:
+            for line in info:
+                if line.startswith('deploy_server: '):
+                    host = line.split(': ', 1)[1]
+                    break
         self.stdheaders['CONFLUENT_APIKEY'] = get_apikey(node, host)
         if mgtiface:
             self.stdheaders['CONFLUENT_MGTIFACE'] = mgtiface
@@ -104,4 +132,4 @@ if __name__ == '__main__':
         sys.exit(0)
     if os.path.exists(sys.argv[-1]):
         data = open(sys.argv[-1]).read()
-    print(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())
+    sys.stdout.write(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/confluent_osdeploy.spec.tmpl

@@ -53,6 +53,7 @@ for os in rhvh4 el7 el8 genesis suse15 ubuntu20.04 esxi6 esxi7; do
     mkdir -p %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
     cp ${os}out/addons.* %{buildroot}/opt/confluent/lib/osdeploy/$os/initramfs
     cp -a $os/profiles/* %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles
+    find %{buildroot}/opt/confluent/lib/osdeploy/$os/profiles -name .gitignore -exec rm -f {} +
 done
 
 %files

confluent_osdeploy/el7/profiles/default/kickstart

@@ -36,6 +36,8 @@ reboot
 chrony
 rsync
 python
+pciutils
+%include /tmp/addonpackages
 %end
 
 

confluent_osdeploy/el7/profiles/default/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
 kernelargs: quiet
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/el7/profiles/default/scripts/functions

@@ -1,12 +1,37 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
 run_remote() {
     requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
     echo
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
     chmod +x $1
     cmd=$1
@@ -23,14 +48,23 @@ run_remote() {
 
 run_remote_python() {
     echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
-    /usr/libexec/platform-python $*
+    if [ -x /usr/libexec/platform-python ]; then
+        /usr/libexec/platform-python $*
+    else
+        /usr/bin/python $*
+    fi
     retcode=$?
     echo "'$*' exited with code $retcode"
     cd - > /dev/null

confluent_osdeploy/el7/profiles/default/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el7/profiles/default/scripts/infiniband/mofed.post

@@ -0,0 +1,20 @@
+# To use this script, rename or copy the mofed image to either ofed.tgz or ofed.iso
+# and modify the script below if wanting to use the iso instead of tgz
+
+# It checks for mellanox devices and opts not to install, so this script could be added
+# to a general profile without causing mofed to install on non-mellanox systems
+. /etc/confluent/functions
+if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
+    # Uncomment the following three lines and comment out the next
+    # two lines to use the .iso instead of the tgz packaging
+    #fetch_remote infiniband/mofed.iso
+    #mkdir MLNX_OFED
+    #mount -o loop ofed.iso MLNX_OFED
+    fetch_remote infiniband/mofed.tgz
+    tar xf infiniband/mofed.tgz
+    # The rest is common between tar and iso
+    cd MLNX_OFED*
+    mount -o loop ofed
+    ./mlnxofedinstall --force
+fi
+

confluent_osdeploy/el7/profiles/default/scripts/infiniband/mofed.pre

@@ -0,0 +1,10 @@
+# Add needed base packages to the install
+cat << EOF >> /tmp/addonpackages
+perl
+pkgconf-pkg-config
+tcsh
+lsof
+tk
+gcc-gfortran
+tcl
+EOF

confluent_osdeploy/el7/profiles/default/scripts/post.custom

@@ -2,3 +2,6 @@
 # This is a convenient place to keep customizations separate from modifying the stock scripts
 # While modification of the stock scripts is fine, it may be easier to rebase to a newer
 # stock profile if the '.custom' files are used.
+# An example for installing OFED for infiniband follows (see the file for more detail):
+#run_remote infiniband/mofed.post
+

confluent_osdeploy/el7/profiles/default/scripts/pre.custom

@@ -2,3 +2,13 @@
 # This is a convenient place to keep customizations separate from modifying the stock scripts
 # While modification of the stock scripts is fine, it may be easier to rebase to a newer
 # stock profile if the '.custom' files are used.
+#
+#Here is an example to locally configure the platform BMC according
+#to confluent configuration so that the BMC would be on the correct
+#network:
+#run_remote_python configbmc -c
+
+#Some addons improve efficiency by adding dependencies during install
+#here is an example for adding OFED install prereqs to the install
+#run_remote infiniband/mofed.pre
+

confluent_osdeploy/el7/profiles/default/scripts/pre.sh

@@ -28,11 +28,19 @@ if [ "$rootpw" = null ]; then
 else
     echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
 fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
 grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
 if [ "$grubpw" = "null" ]; then
     touch /tmp/grubpw
 else
-    echo "bootloader --iscrypted --password=$grubpw" > /tmp/grubpw
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
 fi
 ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host_*_key.pub; do
@@ -45,6 +53,7 @@ if [ -f "/run/install/cmdline.d/01-autocons.conf" ]; then
     consoledev=$(cat /run/install/cmdline.d/01-autocons.conf | sed -e 's!console=!/dev/!' -e 's/,.*//')
     TMUX= tmux a <> $consoledev >&0 2>&1 &
 fi
+touch /tmp/addonpackages
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 if [ "$cryptboot" == "tpm2" ]; then

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -7,6 +7,13 @@ if [ -f /tmp/dd_disk ]; then
         fi
     done
 fi
+vlaninfo=$(getarg vlan)
+if [ ! -z "$vlaninfo" ]; then
+        vldev=${vlaninfo#*:}
+        vlid=${vlaninfo#*.}
+        vlid=${vlid%:*}
+        ip link add link $vldev name $vldev.$vlid type vlan id $vlid
+fi
 TRIES=0
 oum=$(umask)
 umask 0077
@@ -35,6 +42,7 @@ cat /tls/*.pem > /etc/confluent/ca.pem
 ifidx=$(cat /tmp/confluent.ifidx)
 ifname=$(ip link |grep ^$ifidx:|awk '{print $2}')
 ifname=${ifname%:}
+ifname=${ifname%@*}
 echo $ifname > /tmp/net.ifaces
 
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
@@ -98,4 +106,16 @@ while read -r entry; do
         continue
     fi
 done < /etc/confluent/confluent.deploycfg
+if [ -e /lib/nm-lib.sh ]; then
+    . /lib/nm-lib.sh
+    nm_generate_connections
+    if [[ "$ifname" == ib* ]]; then
+        sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
+        if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
+            echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
+        fi
+    fi
+fi
 

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot && \

confluent_osdeploy/el8/profiles/default/kickstart

@@ -34,9 +34,13 @@ reboot
 
 %packages
 @^minimal-environment
+#-kernel-uek # This can opt out of the UEK for the relevant distribution
 chrony
 rsync
 python3
+tar
+pciutils
+%include /tmp/addonpackages
 %include /tmp/cryptpkglist
 %end
 
@@ -63,15 +67,16 @@ curl -f https://$mgr/confluent-public/os/$profile/scripts/prechroot.sh > /tmp/po
 
 # Hook firstboot.sh
 curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.service > /mnt/sysimage/etc/systemd/system/firstboot.service
-curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/etc/confluent/firstboot.sh
-chmod +x /mnt/sysimage/etc/confluent/firstboot.sh
+mkdir -p /mnt/sysimage/opt/confluent/bin
+curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/sysimage/opt/confluent/bin/firstboot.sh
+chmod +x /mnt/sysimage/opt/confluent/bin/firstboot.sh
 %end
 
 %post
 cat /etc/confluent/tls/*.pem >> /etc/pki/tls/certs/ca-bundle.crt
 systemctl enable firstboot
 chgrp ssh_keys /etc/ssh/ssh*key
-restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /etc/confluent/firstboot.sh
+restorecon /etc/ssh/ssh*key /root/.shosts /etc/ssh/shosts.equiv /etc/ssh/ssh_config.d/* /opt/confluent/bin/firstboot.sh
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 mgr=$(grep deploy_server /etc/confluent/confluent.deploycfg |awk '{print $2}')
 curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/postinst.sh

confluent_osdeploy/el8/profiles/default/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
-kernelargs: quiet
+kernelargs: quiet  # These arguments are passed to the installer
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/el8/profiles/default/scripts/configbmc

@@ -111,6 +111,14 @@ class Session(object):
                         self.databuffer.raw[1:self.rsp.msg.data_len])}
         return response
 
+    def await_config(s, bmccfg, channel):
+        vlan = bmccfg.get('bmcvlan', None)
+        ipv4 = bmccfg.get('bmcipv4', None)
+        prefix = bmccfg.get('prefixv4', None)
+        gw = bmccfg.get('bmcgw', None)
+
+
+
     def raw_command(self,
                     netfn,
                     command,
@@ -217,8 +225,7 @@ def set_port_xcc(s, port, model):
     sys.stdout.write('Complete\n')
 
 
-def set_vlan(s, vlan, channel):
-    ovlan = vlan
+def check_vlan(s, vlan, channel):
     if vlan == 'off':
         vlan = b'\x00\x00'
     else:
@@ -229,7 +236,19 @@ def set_vlan(s, vlan, channel):
     currvlan = bytes(s.raw_command(0xc, 2, bytearray([channel, 0x14 ,0, 0]))['data'][1:])
     if bytearray(currvlan)[1] & 0b10000000 == 0:
         currvlan = b'\x00\x00'
-    if currvlan == vlan:
+    return currvlan == vlan
+
+
+def set_vlan(s, vlan, channel):
+    ovlan = vlan
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    if check_vlan(s, ovlan, channel):
         sys.stdout.write('VLAN already configured to "{0}"\n'.format(ovlan))
         return False
     rsp = s.raw_command(0xc, 1, bytearray([channel, 0x14]) + vlan)
@@ -237,7 +256,7 @@ def set_vlan(s, vlan, channel):
         print('VLAN configured to "{}"'.format(ovlan))
     else:
         print('Error setting vlan: ' + repr(rsp))
-    return
+    return True
 
 
 def get_lan_channel(s):
@@ -253,13 +272,18 @@ def get_lan_channel(s):
             return chan
     return 1
 
+
+def check_ipv4(s, ipaddr, channel):
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
+    return rsp == ipaddr
+
 def set_ipv4(s, ipaddr, channel):
     oipaddr = ipaddr
     ipaddr = bytearray(socket.inet_aton(ipaddr))
-    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
-    if rsp == ipaddr:
+    if check_ipv4(s, oipaddr, channel):
         print('IP Address already set to {}'.format(oipaddr))
-        return
+        return False
     rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
     if rsp != 1:
         sys.stdout.write("Changing configuration to static...")
@@ -276,29 +300,47 @@ def set_ipv4(s, ipaddr, channel):
         sys.stdout.flush()
     print('Setting IP to {}'.format(oipaddr))
     s.raw_command(0xc, 1, bytearray([channel, 3]) + ipaddr)
+    return True
 
 
+def check_subnet(s, prefix, channel):
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
+    return rsp == mask
+
 def set_subnet(s, prefix, channel):
     oprefix = prefix
     prefix = int(prefix)
     mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
-    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
-    if rsp == mask:
+    if check_subnet(s, prefix, channel):
         print('Subnet Mask already set to /{}'.format(oprefix))
-        return
+        return False
     print('Setting subnet mask to /{}'.format(oprefix))
     s.raw_command(0xc, 1, bytearray([channel, 6]) + mask)
+    return True
 
 
+def check_gateway(s, gw, channel):
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    return rsp == gw
+
 def set_gateway(s, gw, channel):
     ogw = gw
     gw = bytearray(socket.inet_aton(gw))
     rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
-    if rsp == gw:
+    if check_gateway(s, ogw, channel):
         print('Gateway already set to {}'.format(ogw))
-        return
+        return False
     print('Setting gateway to {}'.format(ogw))
     s.raw_command(0xc, 1, bytearray([channel, 12]) + gw)
+    return True
+
+def dotwait():
+    sys.stdout.write('.')
+    sys.stdout.flush()
+    time.sleep(0.5)
 
 def main():
     a = argparse.ArgumentParser(description='Locally configure a BMC device')
@@ -341,14 +383,30 @@ def main():
         channel = set_port(s, bmccfg['bmcport'], vendor, model)
     else:
         channel = get_lan_channel(s)
+    awaitvlan = False
+    awaitip = False
+    awaitprefix = False
+    awaitgw = False
     if bmccfg.get('bmcvlan', None):
-        set_vlan(s, bmccfg['bmcvlan'], channel)
+        awaitvlan = set_vlan(s, bmccfg['bmcvlan'], channel)
     if bmccfg.get('bmcipv4', None):
-        set_ipv4(s, bmccfg['bmcipv4'], channel)
+        awaitip = set_ipv4(s, bmccfg['bmcipv4'], channel)
     if bmccfg.get('prefixv4', None):
-        set_subnet(s, bmccfg['prefixv4'], channel)
+        awaitprefix = set_subnet(s, bmccfg['prefixv4'], channel)
     if bmccfg.get('bmcgw', None):
-        set_gateway(s, bmccfg['bmcgw'], channel)
+        awaitgw = set_gateway(s, bmccfg['bmcgw'], channel)
+    sys.stdout.write('Waiting for changes to take effect...')
+    sys.stdout.flush()
+    while awaitvlan and not check_vlan(s, bmccfg['bmcvlan'], channel):
+        dotwait()
+    while awaitip and not check_ipv4(s, bmccfg['bmcipv4'], channel):
+        dotwait()
+    while awaitprefix and not check_subnet(s, bmccfg['prefixv4'], channel):
+        dotwait()
+    while awaitprefix and not check_gateway(s, bmccfg['bmcgw'], channel):
+        dotwait()
+    sys.stdout.write('done\n')
+    sys.stdout.flush()
     #await_config(s, bmccfg, channel)
 
 

confluent_osdeploy/el8/profiles/default/scripts/firstboot.service

@@ -4,7 +4,7 @@ Requires=network-online.target
 After=network-online.target
 
 [Service]
-ExecStart=/etc/confluent/firstboot.sh
+ExecStart=/opt/confluent/bin/firstboot.sh
 
 [Install]
 WantedBy=multi-user.target

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -15,6 +15,8 @@ export nodename mgr profile
 
 
 run_remote firstboot.custom
+# Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+run_remote_parts firstboot
 
 
 curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -1,12 +1,44 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+run_remote_parts() {
+    scriptlist=$(/usr/libexec/platform-python /etc/confluent/apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1.d/$script
+    done
+}
+
 run_remote() {
     requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
     echo
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
     chmod +x $1
     cmd=$1
@@ -23,12 +55,17 @@ run_remote() {
 
 run_remote_python() {
     echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     /usr/libexec/platform-python $*
     retcode=$?

confluent_osdeploy/el8/profiles/default/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.post

@@ -0,0 +1,20 @@
+# To use this script, rename or copy the mofed image to either ofed.tgz or ofed.iso
+# and modify the script below if wanting to use the iso instead of tgz
+
+# It checks for mellanox devices and opts not to install, so this script could be added
+# to a general profile without causing mofed to install on non-mellanox systems
+. /etc/confluent/functions
+if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
+    # Uncomment the following three lines and comment out the next
+    # two lines to use the .iso instead of the tgz packaging
+    #fetch_remote infiniband/mofed.iso
+    #mkdir MLNX_OFED
+    #mount -o loop ofed.iso MLNX_OFED
+    fetch_remote infiniband/mofed.tgz
+    tar xf infiniband/mofed.tgz
+    # The rest is common between tar and iso
+    cd MLNX_OFED*
+    mount -o loop ofed
+    ./mlnxofedinstall --force
+fi
+

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.pre

@@ -0,0 +1,10 @@
+# Add needed base packages to the install
+cat << EOF >> /tmp/addonpackages
+perl
+pkgconf-pkg-config
+tcsh
+lsof
+tk
+gcc-gfortran
+tcl
+EOF

confluent_osdeploy/el8/profiles/default/scripts/post.custom

@@ -2,3 +2,6 @@
 # This is a convenient place to keep customizations separate from modifying the stock scripts
 # While modification of the stock scripts is fine, it may be easier to rebase to a newer
 # stock profile if the '.custom' files are used.
+
+# An example for installing OFED for infiniband follows (see the file for more detail):
+#run_remote infiniband/mofed.post

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -33,3 +33,6 @@ run_remote_python add_local_repositories
 # run_remote example.sh
 # run_remote_python example.py
 run_remote post.custom
+
+# Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh
+run_remote_parts post

confluent_osdeploy/el8/profiles/default/scripts/pre.custom

@@ -7,3 +7,7 @@
 #to confluent configuration so that the BMC would be on the correct
 #network:
 #run_remote_python configbmc -c
+
+#Some addons improve efficiency by adding dependencies during install
+#here is an example for adding OFED install prereqs to the install
+#run_remote infiniband/mofed.pre

confluent_osdeploy/el8/profiles/default/scripts/pre.sh

@@ -34,11 +34,19 @@ if [ "$rootpw" = null ]; then
 else
     echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
 fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
 grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
 if [ "$grubpw" = "null" ]; then
     touch /tmp/grubpw
 else
-    echo "bootloader --iscrypted --password=$grubpw" > /tmp/grubpw
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
 fi
 for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
@@ -53,6 +61,7 @@ fi
 cryptboot=$(grep ^encryptboot: /etc/confluent/confluent.deploycfg | awk '{print $2}')
 LUKSPARTY=''
 touch /tmp/cryptpkglist
+touch /tmp/addonpackages
 if [ "$cryptboot" == "tpm2" ]; then
 	LUKSPARTY="--encrypted --passphrase=$(cat /etc/confluent/confluent.apikey)"
 	echo $cryptboot >> /tmp/cryptboot

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -7,18 +7,33 @@ echo $uuid > /sys/devices/virtual/dmi/id/product_uuid
 mkdir -p /etc/confluent
 localcli network firewall unload
 touch /etc/confluent/confluent.info
+begin=$(date +%s)
 while ! grep NODENAME /etc/confluent/confluent.info; do
+    echo "Searching for deployment service on local network..."
     /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+    esxcfg-nics -l
+    if [ $(date +%s) -gt $((begin + 90)) ]; then 
+        if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
+            chvt 1
+            sleep 1
+            chvt 2
+            /etc/init.d/ESXShell start
+        fi
+        echo "Unable to locate a deployment system on the local network, verify network connectivity"
+        echo "A debug session has been made available on Alt-F1"
+        sleep 30
+    fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
 mgr=$(grep MANAGER: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
 cp /opt/confluent/bin/clortho /clortho
 /clortho $node $mgr > /etc/confluent/confluent.apikey
 cat /tls/*.pem > /etc/confluent/ca.pem
-/opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
-profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/^profile: //')
+/opt/confluent/bin/apiclient /confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg.new
+profile=$(grep ^profile: /etc/confluent/confluent.deploycfg.new | sed -e 's/^profile: //')
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/kickstart > /etc/confluent/ks.cfg
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/modinstall > /tmp/modinstall
+mv /etc/confluent/confluent.deploycfg.new /etc/confluent/confluent.deploycfg
 export node mgr profile
 . /tmp/modinstall
 exec /bin/install

confluent_osdeploy/esxi7/profiles/hypervisor/kickstart

@@ -4,3 +4,8 @@ install --firstdisk --overwritevmfs
 %include /tmp/ksnet
 %include /tmp/rootpw
 reboot
+%post --interpreter=busybox
+localcli network firewall unload
+STATUP=$(mktemp)
+echo '{"status": "complete"}' > $STATUP
+/opt/confluent/bin/apiclient /confluent-api/self/updatestatus $STATUP

confluent_osdeploy/esxi7/profiles/hypervisor/profile.yaml

@@ -1,3 +1,3 @@
-label: VMware ESXi %%VERSION%% Hypervisor
+label: Confluent installation of VMware ESXi %%VERSION%% Hypervisor
 ostype: esxi
 kernelargs: runweasel

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -1,5 +1,8 @@
 #!/usr/bin/python
+import re
+import subprocess
 
+uplinkmatch = re.compile('^\s*Uplinks:\s*(.*)')
 nodename = None
 for inf in open('/etc/confluent/confluent.info', 'r').read().split('\n'):
     if inf.startswith('NODENAME: '):
@@ -24,8 +27,15 @@ for line in deploycfg:
     else:
         nslist=False
 cfg['nameservers'] = ','.join(nameservers)
-
+vswinfo = subprocess.check_output(['localcli', 'network', 'vswitch', 'standard', 'list']).decode('utf8')
+vmnic = None
+for info in vswinfo.split('\n'):
+    upinfo = uplinkmatch.match(info)
+    if upinfo:
+        vmnic = upinfo.group(1)
 netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if vmnic:
+    netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':
     netline += ' --ip={0} --netmask={1}'.format(cfg['ipv4_address'], cfg['ipv4_netmask'])
     if cfg.get('ipv4_gateway', 'null') not in (None, '', 'null'):

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/modinstall

@@ -1,5 +1,7 @@
 #!/bin/sh
+mv /etc/confluent/confluent.deploycfg /etc/confluent/confluent.newdeploycfg
 /opt/confluent/bin/apiclient /confluent-public/os/$profile/scripts/makeksnet >> /tmp/makeksnet
+mv /etc/confluent/confluent.newdeploycfg /etc/confluent/confluent.deploycfg
 chmod +x /tmp/makeksnet
 /tmp/makeksnet > /tmp/ksnet
 rootpw=$(grep ^rootpassword: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -9,8 +9,11 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     if [ ! -z "$autocons" ]; then
         echo "Using $(cat /tmp/01-autocons.conf)"
         tmux a <> $autocons >&0 2>&1 &
+    else
+        tmux a <> /dev/console >&0 2>&1 &
     fi
 else
+    tmux a <> /dev/console >&0 2>&1 &
     tmux a <> /dev/tty1 >&0 2>&1 &
 fi
 echo -n "udevd: "
@@ -25,6 +28,9 @@ modprobe hfi1
 modprobe mlx5_ib
 echo "done"
 cat > /etc/ssh/sshd_config << EOF
+Port 22
+Port 3389
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
 PermitRootLogin yes
 AuthorizedKeysFile      .ssh/authorized_keys
 EOF
@@ -78,7 +84,9 @@ for pubkey in /etc/ssh/ssh_host*key.pub; do
     certfile=${pubkey/.pub/-cert.pub}
     privfile=${pubkey%.pub}
     /usr/libexec/platform-python /opt/confluent/bin/apiclient /confluent-api/self/sshcert $pubkey > $certfile
-    echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    if [ -s $certfile ]; then
+        echo HostCertificate $certfile >> /etc/ssh/sshd_config
+    fi
     echo HostKey $privfile >> /etc/ssh/sshd_config
 done
 /usr/sbin/sshd
@@ -92,11 +100,3 @@ run_remote onboot.sh
 while :; do
     bash
 done
-
-    
-
-
-
-
-cd -
-

confluent_osdeploy/genesis/initramfs/usr/lib/dracut/hooks/cmdline/10-genesis.sh

@@ -10,4 +10,7 @@ mkdir -p /var/empty/sshd
 sed -i '/^root:x/d' /etc/passwd
 echo root:x:0:0::/:/bin/bash >> /etc/passwd
 echo sshd:x:30:30:SSH User:/var/empty/sshd:/sbin/nologin >> /etc/passwd
-tmux new-session sh /opt/confluent/bin/rungenesis
+tmux new-session -d sh /opt/confluent/bin/rungenesis
+while :; do
+	sleep 86400
+done

confluent_osdeploy/genesis/profiles/default/scripts/configbmc

@@ -111,6 +111,14 @@ class Session(object):
                         self.databuffer.raw[1:self.rsp.msg.data_len])}
         return response
 
+    def await_config(s, bmccfg, channel):
+        vlan = bmccfg.get('bmcvlan', None)
+        ipv4 = bmccfg.get('bmcipv4', None)
+        prefix = bmccfg.get('prefixv4', None)
+        gw = bmccfg.get('bmcgw', None)
+
+
+
     def raw_command(self,
                     netfn,
                     command,
@@ -157,6 +165,14 @@ def set_port(s, port, vendor, model):
         return 1
 
 
+def get_remote_config_mod(vendor, model):
+    if vendor in ('IBM', 'Lenovo'):
+        if _is_tsm(model):
+            return 'tsm'
+        else:
+            return 'xcc'
+    return None
+
 def set_port_tsm(s, port, model):
     oport = port
     sys.stdout.write('Setting TSM port to "{}"...'.format(oport))
@@ -217,8 +233,7 @@ def set_port_xcc(s, port, model):
     sys.stdout.write('Complete\n')
 
 
-def set_vlan(s, vlan, channel):
-    ovlan = vlan
+def check_vlan(s, vlan, channel):
     if vlan == 'off':
         vlan = b'\x00\x00'
     else:
@@ -229,7 +244,19 @@ def set_vlan(s, vlan, channel):
     currvlan = bytes(s.raw_command(0xc, 2, bytearray([channel, 0x14 ,0, 0]))['data'][1:])
     if bytearray(currvlan)[1] & 0b10000000 == 0:
         currvlan = b'\x00\x00'
-    if currvlan == vlan:
+    return currvlan == vlan
+
+
+def set_vlan(s, vlan, channel):
+    ovlan = vlan
+    if vlan == 'off':
+        vlan = b'\x00\x00'
+    else:
+        vlan = int(vlan)
+        if vlan:
+            vlan = vlan | 32768
+        vlan = struct.pack('<H', vlan)
+    if check_vlan(s, ovlan, channel):
         sys.stdout.write('VLAN already configured to "{0}"\n'.format(ovlan))
         return False
     rsp = s.raw_command(0xc, 1, bytearray([channel, 0x14]) + vlan)
@@ -237,7 +264,7 @@ def set_vlan(s, vlan, channel):
         print('VLAN configured to "{}"'.format(ovlan))
     else:
         print('Error setting vlan: ' + repr(rsp))
-    return
+    return True
 
 
 def get_lan_channel(s):
@@ -253,13 +280,18 @@ def get_lan_channel(s):
             return chan
     return 1
 
+
+def check_ipv4(s, ipaddr, channel):
+    ipaddr = bytearray(socket.inet_aton(ipaddr))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
+    return rsp == ipaddr
+
 def set_ipv4(s, ipaddr, channel):
     oipaddr = ipaddr
     ipaddr = bytearray(socket.inet_aton(ipaddr))
-    rsp = s.raw_command(0xc, 2, bytearray([channel, 3, 0, 0]))['data'][-4:]
-    if rsp == ipaddr:
+    if check_ipv4(s, oipaddr, channel):
         print('IP Address already set to {}'.format(oipaddr))
-        return
+        return False
     rsp = int(s.raw_command(0xc, 2, bytearray([channel, 4, 0, 0]))['data'][1]) & 0b1111
     if rsp != 1:
         sys.stdout.write("Changing configuration to static...")
@@ -276,29 +308,47 @@ def set_ipv4(s, ipaddr, channel):
         sys.stdout.flush()
     print('Setting IP to {}'.format(oipaddr))
     s.raw_command(0xc, 1, bytearray([channel, 3]) + ipaddr)
+    return True
 
 
+def check_subnet(s, prefix, channel):
+    prefix = int(prefix)
+    mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
+    return rsp == mask
+
 def set_subnet(s, prefix, channel):
     oprefix = prefix
     prefix = int(prefix)
     mask = bytearray(struct.pack('!I', (2**32 - 1) ^ (2**(32 - prefix) - 1)))
-    rsp = s.raw_command(0xc, 2, bytearray([channel, 6, 0, 0]))['data'][-4:]
-    if rsp == mask:
+    if check_subnet(s, prefix, channel):
         print('Subnet Mask already set to /{}'.format(oprefix))
-        return
+        return False
     print('Setting subnet mask to /{}'.format(oprefix))
     s.raw_command(0xc, 1, bytearray([channel, 6]) + mask)
+    return True
 
 
+def check_gateway(s, gw, channel):
+    gw = bytearray(socket.inet_aton(gw))
+    rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
+    return rsp == gw
+
 def set_gateway(s, gw, channel):
     ogw = gw
     gw = bytearray(socket.inet_aton(gw))
     rsp = s.raw_command(0xc, 2, bytearray([channel, 12, 0, 0]))['data'][-4:]
-    if rsp == gw:
+    if check_gateway(s, ogw, channel):
         print('Gateway already set to {}'.format(ogw))
-        return
+        return False
     print('Setting gateway to {}'.format(ogw))
     s.raw_command(0xc, 1, bytearray([channel, 12]) + gw)
+    return True
+
+def dotwait():
+    sys.stdout.write('.')
+    sys.stdout.flush()
+    time.sleep(0.5)
 
 def main():
     a = argparse.ArgumentParser(description='Locally configure a BMC device')
@@ -341,15 +391,40 @@ def main():
         channel = set_port(s, bmccfg['bmcport'], vendor, model)
     else:
         channel = get_lan_channel(s)
+    awaitvlan = False
+    awaitip = False
+    awaitprefix = False
+    awaitgw = False
     if bmccfg.get('bmcvlan', None):
-        set_vlan(s, bmccfg['bmcvlan'], channel)
+        awaitvlan = set_vlan(s, bmccfg['bmcvlan'], channel)
     if bmccfg.get('bmcipv4', None):
-        set_ipv4(s, bmccfg['bmcipv4'], channel)
+        awaitip = set_ipv4(s, bmccfg['bmcipv4'], channel)
     if bmccfg.get('prefixv4', None):
-        set_subnet(s, bmccfg['prefixv4'], channel)
+        awaitprefix = set_subnet(s, bmccfg['prefixv4'], channel)
     if bmccfg.get('bmcgw', None):
-        set_gateway(s, bmccfg['bmcgw'], channel)
-    #await_config(s, bmccfg, channel)
+        awaitgw = set_gateway(s, bmccfg['bmcgw'], channel)
+    sys.stdout.write('Waiting for changes to take effect...')
+    sys.stdout.flush()
+    while awaitvlan and not check_vlan(s, bmccfg['bmcvlan'], channel):
+        dotwait()
+    while awaitip and not check_ipv4(s, bmccfg['bmcipv4'], channel):
+        dotwait()
+    while awaitprefix and not check_subnet(s, bmccfg['prefixv4'], channel):
+        dotwait()
+    while awaitprefix and not check_gateway(s, bmccfg['bmcgw'], channel):
+        dotwait()
+    sys.stdout.write('done\n')
+    sys.stdout.flush()
+    cfgmod = get_remote_config_mod(vendor, model)
+    if cfgmod:
+        with open('configbmc.configmod', 'w+') as cm:
+            cm.write('configmod: {0}\n'.format(cfgmod))
+        sys.stdout.write('Requesting remote configuration of authentication...')
+        sys.stdout.flush()
+        bmccfgsrc = subprocess.check_output(
+            [sys.executable, apiclient, '/confluent-api/self/remoteconfigbmc', 'configbmc.configmod'])
+        sys.stdout.write('done\n')
+        sys.stdout.flush()
 
 
 if __name__ == '__main__':

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -1,12 +1,35 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
 run_remote() {
     requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
     echo
     echo '---------------------------------------------------------------------------'
     echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
     chmod +x $1
     cmd=$1
@@ -23,12 +46,16 @@ run_remote() {
 
 run_remote_python() {
     echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
     echo '---------------------------------------------------------------------------'
     echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
     tmpdir=$(mktemp -d)
     echo Executing in $tmpdir
     cd $tmpdir
-    curl -f -sS https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
     if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
     /usr/libexec/platform-python $*
     retcode=$?

confluent_osdeploy/rhvh4/profiles/default/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
 kernelargs: quiet
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/rhvh4/profiles/default/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/rhvh4/profiles/default/scripts/pre.sh

@@ -42,11 +42,19 @@ if [ "$rootpw" = null ]; then
 else
     echo "rootpw --iscrypted $rootpw" > /tmp/rootpw
 fi
+curl -f https://$mgr/confluent-public/os/$profile/profile.yaml > /tmp/instprofile.yaml
+blargs=$(grep ^installedargs: /tmp/instprofile.yaml | sed -e 's/#.*//' -e 's/^installedargs: //')
+if [ ! -z "$blargs" ]; then
+	blargs=' --append="'$blargs'"'
+fi
 grubpw=$(grep ^grubpassword /etc/confluent/confluent.deploycfg | awk '{print $2}')
 if [ "$grubpw" = "null" ]; then
     touch /tmp/grubpw
 else
-    echo "bootloader --iscrypted --password=$grubpw" > /tmp/grubpw
+    blargs=" --iscrypted --password=$grubpw $blargs"
+fi
+if [ ! -z "$blargs" ]; then
+    echo "bootloader $blargs" > /tmp/grubpw
 fi
 ssh-keygen -A
 for pubkey in /etc/ssh/ssh_host_*_key.pub; do

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -12,7 +12,7 @@ autocons=""
 if ! grep console /proc/cmdline > /dev/null; then
 	autocons=$(/opt/confluent/bin/autocons)
 	if [ ! -z "$autocons" ]; then
-		echo "Serial console detected from firmmware: $autocons" > ${autocons%,*}
+		echo "Serial console detected from firmware: $autocons" > ${autocons%,*}
 	fi
 fi
 mkdir -p /etc/confluent
@@ -52,6 +52,9 @@ if [ -z "$mgtiface" ]; then
 else
     curl -H "CONFLUENT_MGTIFACE: $mgtiface" -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $(cat /etc/confluent/confluent.apikey)" https://$mgr/confluent-api/self/deploycfg > /etc/confluent/confluent.deploycfg
 fi
+profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
+profilename=${profilename#profile: }
+curl https://$mgr/confluent-public/os/$profilename/profile.yaml > /tmp/profile.yaml
 dnsdomain=$(grep ^dnsdomain: /etc/confluent/confluent.deploycfg)
 dnsdomain=${dnsdomain#dnsdomain: }
 if [ ! -z "$dnsdomain" ] && [ "$dnsdomain" != "null" ]; then
@@ -60,11 +63,15 @@ fi
 textconsole=$(grep ^textconsole: /etc/confluent/confluent.deploycfg)
 textconsole=${textconsole#textconsole: }
 if [ "$textconsole" = "true" ] && ! grep console= /proc/cmdline > /dev/null && [ ! -z "$autocons" ]; then
-	echo "Serial console autodetected and enabled, will be available upon install completion" > ${autocons%,*}
-	echo "The installer will run in text mode on the graphics console. When ssh is available," > ${autocons%,*}
-	echo "install progress may be checked by using ssh to access and run the screendump command" > ${autocons%,*}
-	echo "Install time serial console would require the profile to have console=${autocons#/dev/}" > ${autocons%,*}
-	echo "specified in the kernel command line arguments" > ${autocons%,*}
+	echo "Serial console has been autodected and enabled read-only for install" > ${autocons%,*}
+	echo "It will be read-write after install" > ${autocons%,*}
+	echo "If a fully functional console is desired over serial, add console=${autocons#/dev/} " > ${autocons%,*}
+	echo "to kerneralgs in the profile.yaml file of the profile and run 'osdeploy updateboot <profile>" > ${autocons%,*}
+	#echo "Serial console autodetected and enabled, will be available upon install completion" > ${autocons%,*}
+	#echo "The installer will run in text mode on the graphics console. When ssh is available," > ${autocons%,*}
+	#echo "install progress may be checked by using ssh to access and run the screendump command" > ${autocons%,*}
+	#echo "Install time serial console would require the profile to have console=${autocons#/dev/}" > ${autocons%,*}
+	#echo "specified in the kernel command line arguments" > ${autocons%,*}
 	echo ${autocons%,*} > /tmp/autoconsdev
 	sed -e s'/$/ 'console=${autocons#*/dev/}/ /proc/cmdline > /etc/fakecmdline
 	mount -o bind /etc/fakecmdline /proc/cmdline
@@ -98,7 +105,7 @@ else
 			fi
 		fi
 		nameserversec=0
-		if [ ${entry%:*} = "nameservers" ]; then
+		if [ "${entry%:*}" = "nameservers" ]; then
 			nameserversec=1
 			continue
 		fi
@@ -107,11 +114,16 @@ fi
 echo done
 mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg)
 mgr=${mgr#deploy_server: }
-profilename=$(grep ^profile: /etc/confluent/confluent.deploycfg)
-profilename=${profilename#profile: }
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg)
 proto=${proto#protocol: }
 
+append=$(grep ^installedargs: /tmp/profile.yaml | sed -e 's/^installedargs: //' -e 's/#.*//')
+if [ -z "$append" ]; then
+    echo "<bootloader/>" > /tmp/bootloader.xml
+else
+    echo "<bootloader><global><append>$append</append></global></bootloader>" > /tmp/bootloader.xml
+fi
+
 echo "<media_url>${proto}://${mgr}/confluent-public/os/${profilename}/product</media_url>" > /tmp/pkgurl
 
 echo "AutoYaST: $proto://$mgr/confluent-public/os/$profilename/autoyast" >> /etc/linuxrc.d/01-confluent

confluent_osdeploy/suse15/profiles/hpc/autoyast

@@ -28,6 +28,7 @@ dynamic behavior and replace with static configuration.
         </add_on_products>
 </add-on>
     %%ENDIFSLE%%
+    <xi:include href="file:///tmp/bootloader.xml"/>
     <software>
        %%IFSLE%%
        <products config:type="list">
@@ -126,6 +127,7 @@ curl -f $proto://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /mnt/e
 curl -f $proto://$mgr/confluent-public/os/$profile/scripts/post.sh > /mnt/etc/confluent/post.sh
 chmod +x /mnt/etc/confluent/firstboot.sh
 chmod +x /mnt/etc/confluent/post.sh
+cp /mnt/etc/confluent/post.sh /mnt/var/adm/autoinstall/scripts/
 ]]>
           </source>
         </script>

confluent_osdeploy/suse15/profiles/hpc/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
-kernelargs: quiet
+kernelargs: quiet # These arguments are passed to the installer
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.custom

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# This script runs at the end of the final boot
+
+. /etc/confluent/functions
+
+# Custom scripts may go here
+# run_remote example.sh
+# run_remote_python example.py

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -7,9 +7,7 @@ mgr=$(grep ^deploy_server /etc/confluent/confluent.deploycfg|awk '{print $2}')
 profile=$(grep ^profile: /etc/confluent/confluent.deploycfg|sed -e 's/^rootpassword: //')
 proto=$(grep ^protocol: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 apikey=$(cat /etc/confluent/confluent.apikey)
-curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -f -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus
 . /etc/confluent/functions
 
-# Custom scripts may go here
-# run_remote example.sh
-# run_remote_python example.py
+run_remote firstboot.custom
+curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -f -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/suse15/profiles/hpc/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/suse15/profiles/hpc/scripts/post.custom

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# This script runs at the end of install in the installed system
+# but still under the installer kernel.
+
+# This is a good place to run most customizations that do not have any
+# dependency upon the install target kernel being active.
+
+# If there are dependencies on the kernel (drivers or special filesystems)
+# then firstboot.sh would be the script to customize.
+
+. /etc/confluent/functions
+
+# Examples:
+# run_remote script.sh
+# run_remote_python script.py

confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh

@@ -21,10 +21,7 @@ chmod og-rwx /etc/confluent/*
 export mgr profile nodename
 . /etc/confluent/functions
 
+run_remote post.custom
+
 curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
 
-# Customizations may go here
-
-# Examples:
-# run_remote script.sh
-# run_remote_python script.py

confluent_osdeploy/ubuntu20.04/initramfs/custom-installation/post.sh

@@ -2,5 +2,6 @@
 deploycfg=/custom-installation/confluent/confluent.deploycfg
 mgr=$(grep ^deploy_server $deploycfg|awk '{print $2}')
 profile=$(grep ^profile: $deploycfg|awk '{print $2}')
+export deploycfg mgr profile
 curl -f https://$mgr/confluent-public/os/$profile/scripts/post.sh > /tmp/post.sh
 . /tmp/post.sh

confluent_osdeploy/ubuntu20.04/profiles/default/profile.yaml

@@ -1,2 +1,3 @@
 label: %%DISTRO%% %%VERSION%% %%ARCH%% (Default Profile)
 kernelargs: quiet osprofile=%%PROFILE%%
+#installedargs: example # These arguments would be added to the installed system

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/getinstalldisk

@@ -49,7 +49,7 @@ class DiskInfo(object):
 
     @property
     def priority(self):
-        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2'):
+        if self.model.lower() in ('thinksystem_m.2_vd', 'thinksystem m.2', 'thinksystem_m.2'):
             return 0
         if 'imsm' in self.mdcontainer:
             return 1

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -29,6 +29,7 @@ if grep ^ntpservers: /target/etc/confluent/confluent.deploycfg > /dev/null; then
     sed -i "s/#NTP=/NTP=$ntps/" /target/etc/systemd/timesyncd.conf
 fi
 textcons=$(grep ^textconsole: /target/etc/confluent/confluent.deploycfg |awk '{print $2}')
+updategrub=0
 if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
     cons=""
     if [ -f /custom-installation/autocons.info ]; then
@@ -36,11 +37,18 @@ if [ "$textcons" = "true" ] && ! grep console= /proc/cmdline > /dev/null; then
     fi
     if [ ! -z "$cons" ]; then
         sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 console='${cons#/dev/}'"/' /target/etc/default/grub
-	mount -o bind /dev /target/dev
-	mount -o bind /proc /target/proc
-	mount -o bind /sys /target/sys
-	chroot /target update-grub
-	umount /target/sys /target/dev /target/proc
+        updategrub=1
     fi
 fi
+kargs=$(curl https://$mgr/confluent-public/os/$profile/profile.yaml | grep ^installedargs: | sed -e 's/#.*//')
+if [ ! -z "$kargs" ]; then
+    sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub
+fi
+if [ 1 = $updategrub ]; then
+    mount -o bind /dev /target/dev
+    mount -o bind /proc /target/proc
+    mount -o bind /sys /target/sys
+    chroot /target update-grub
+    umount /target/sys /target/dev /target/proc
+fi
 

confluent_osdeploy/utils/autocons.c

@@ -51,6 +51,7 @@ int main(int argc, char* argv[]) {
     } else {
         exit(0);
     }
+    ttyf = open(buff, O_RDWR | O_NOCTTY);
     if (currspeed == SPEED9600) {
         cspeed = B9600;
         strcpy(offset, ",9600");
@@ -66,8 +67,6 @@ int main(int argc, char* argv[]) {
     } else {
         exit(0);
     }
-    printf("%s\n", buff);
-    ttyf = open(buff, O_RDWR | O_NOCTTY);
     tcgetattr(ttyf, &tty);
     if (cspeed) {
         cfsetospeed(&tty, B115200);
@@ -75,6 +74,7 @@ int main(int argc, char* argv[]) {
     }
     tcsetattr(ttyf, TCSANOW, &tty);
     ioctl(ttyf, TIOCCONS, 0);
+    printf("%s\n", buff);
 
 }
 

confluent_osdeploy/utils/clortho.c

@@ -36,7 +36,8 @@ unsigned char* genpasswd(int len) {
 int main(int argc, char* argv[]) {
         int sock, ret;
         char slen;
-        unsigned char currlen, currtype;
+        unsigned char currtype;
+        size_t currlen;
         unsigned char* passwd;
         unsigned char* cryptedpass;
         unsigned char* macaddr;
@@ -107,10 +108,21 @@ int main(int argc, char* argv[]) {
         ret = read(sock, buffer, 2);
         while (buffer[0] != 255) {
             currtype = buffer[0];
-            currlen = buffer[1];
+	    if (currtype & 0b10000000) {
+                currlen = buffer[1] << 8;
+                read(sock, buffer, 1);
+                currlen |= buffer[0];
+            } else {
+                currlen = buffer[1];
+	    }
             memset(buffer, 0, MAXPACKET);
+	    if (currlen > 1000) {
+                fprintf(stderr, "Received oversized message\n");
+		exit(1);
+            }
             if (currlen) {
-                ret = read(sock, buffer, currlen);  // Max is 255, well under MAX_PACKET
+                ret = read(sock, buffer, currlen);  // Max is 1000, well under MAX_PACKET
+                buffer[currlen] = 0;
             }
             if (currtype == 2) {
                 dprintf(sock, "\x03%c", currlen);
@@ -118,6 +130,10 @@ int main(int argc, char* argv[]) {
                 slen = strlen(cryptedpass) & 0xff;
                 dprintf(sock, "\x04%c%s", slen, cryptedpass);
                 ret = write(sock, "\x00\x00", 2);
+            } else if (currtype == 128) {
+		printf("SEALED:%s", buffer);
+		printf("\n");
+		exit(0);
             } else if (currtype == 5) {
                 printf("%s", passwd);
                 printf("\n");

confluent_server/bin/confluentdbutil

@@ -74,7 +74,7 @@ if args[0] == 'restore':
             for targdir in os.walk('/etc/confluent'):
                 os.chown(targdir[0], owner, group)
                 for f in targdir[2]:
-                    os.chown(os.patht.join(targdir[0], f), owner, group)
+                    os.chown(os.path.join(targdir[0], f), owner, group)
     except Exception as e:
         print(str(e))
         sys.exit(1)

confluent_server/bin/osdeploy

@@ -1,5 +1,7 @@
 #!/usr/bin/python2
 
+__author__ = 'jjohnson2,bfinley'
+
 import argparse
 import glob
 import os
@@ -50,7 +52,11 @@ def main(args):
         'updateboot',
         help='Push profile.yaml of the named profile data into boot assets as appropriate')
     upb.add_argument('profile', help='Profile to update boot assets')
+    osls = sp.add_parser('list', help='List OS images available for deployment')
     cmdset = ap.parse_args()
+
+    if cmdset.command == 'list':
+        return oslist()
     if cmdset.command == 'import':
         return osimport(cmdset.imagefile)
     if cmdset.command == 'initialize':
@@ -308,9 +314,36 @@ def updateboot(profilename):
             print(repr(rsp))
 
 
+def oslist():
+    c = client.Command()
+    print("Distributions:")
+    for rsp in c.read('/deployment/distributions'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+    print("Profiles:")
+    for rsp in c.read('/deployment/profiles'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+
 def osimport(imagefile):
     c = client.Command()
     imagefile = os.path.abspath(imagefile)
+    if c.unixdomain:
+        ofile = open(imagefile, 'rb')
+        try:
+            c.add_file(imagefile, ofile.fileno(), 'rb')
+        except Exception:
+            pass
     importing = False
     shortname = None
     for rsp in c.create('/deployment/importing/', {'filename': imagefile}):

confluent_server/confluent/collective/manager.py

@@ -19,6 +19,7 @@ import confluent.collective.invites as invites
 import confluent.config.configmanager as cfm
 import confluent.exceptions as exc
 import confluent.log as log
+import confluent.noderange as noderange
 import confluent.tlvdata as tlvdata
 import confluent.util as util
 import eventlet
@@ -27,6 +28,7 @@ import eventlet.green.ssl as ssl
 import eventlet.green.threading as threading
 import greenlet
 import random
+import time
 import sys
 try:
     import OpenSSL.crypto as crypto
@@ -38,6 +40,7 @@ except ImportError:
 currentleader = None
 follower = None
 retrythread = None
+failovercheck = None
 
 class ContextBool(object):
     def __init__(self):
@@ -73,8 +76,8 @@ def connect_to_leader(cert=None, name=None, leader=None):
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
             vers = banner.split()[2]
-            if vers != b'v2':
-                raise Exception('This instance only supports protocol 2, synchronize versions between collective members')
+            if vers not in (b'v2', b'v3'):
+                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -193,15 +196,21 @@ def connect_to_collective(cert, member):
         raise Exception("Certificate mismatch in the collective")
     return remote
 
-
+mycachedname = [None, 0]
 def get_myname():
+    if mycachedname[1] > time.time() - 15:
+        return mycachedname[0]
     try:
         with open('/etc/confluent/cfg/myname', 'r') as f:
-            return f.read().strip()
+            mycachedname[0] = f.read().strip()
+            mycachedname[1] = time.time()
+            return mycachedname[0]
     except IOError:
         myname = socket.gethostname()
         with open('/etc/confluent/cfg/myname', 'w') as f:
             f.write(myname)
+        mycachedname[0] = myname
+        mycachedname[1] = time.time()
         return myname
 
 def handle_connection(connection, cert, request, local=False):
@@ -567,6 +576,7 @@ def become_leader(connection):
         if dronecandidate in skipem or member == myname:
             continue
         eventlet.spawn_n(try_assimilate, dronecandidate)
+    schedule_rebalance()
 
 
 def startup():
@@ -576,7 +586,53 @@ def startup():
         return
     eventlet.spawn_n(start_collective)
 
+def check_managers():
+    global failovercheck
+    if not follower:
+        c = cfm.ConfigManager(None)
+        collinfo = {}
+        populate_collinfo(collinfo)
+        availmanagers = {}
+        offlinemgrs = set(collinfo['offline'])
+        offlinemgrs.add('')
+        for offline in collinfo['offline']:
+            nodes = noderange.NodeRange(
+                'collective.manager=={}'.format(offline), c).nodes
+            managercandidates = c.get_node_attributes(
+                nodes, 'collective.managercandidates')
+            expandednoderanges = {}
+            for node in nodes:
+                if node not in managercandidates:
+                    continue
+                targets = managercandidates[node].get('collective.managercandidates', {}).get('value', None)
+                if not targets:
+                    continue
+                if not availmanagers:
+                    for active in collinfo['active']:
+                        availmanagers[active] = len(
+                            noderange.NodeRange(
+                                'collective.manager=={}'.format(active), c).nodes)
+                    availmanagers[collinfo['leader']] = len(
+                            noderange.NodeRange(
+                                'collective.manager=={}'.format(
+                                    collinfo['leader']), c).nodes)
+                if targets not in expandednoderanges:
+                    expandednoderanges[targets] = set(
+                        noderange.NodeRange(targets, c).nodes) - offlinemgrs
+                targets = sorted(expandednoderanges[targets], key=availmanagers.get)
+                if not targets:
+                    continue
+                c.set_node_attributes({node: {'collective.manager': {'value': targets[0]}}})
+                availmanagers[targets[0]] += 1
+    failovercheck = None
+
+def schedule_rebalance():
+    global failovercheck
+    if not failovercheck:
+        failovercheck = eventlet.spawn_after(10, check_managers)
+
 def start_collective():
+    cfm.membership_callback = schedule_rebalance
     global follower
     global retrythread
     if follower:

confluent_server/confluent/config/attributes.py

@@ -129,6 +129,14 @@ node = {
                         'Generally this is not directly modified, but is modified '
                         'by the "nodedeploy" command'),
     },
+    'deployment.sealedapikey': {
+        'description': 'This attribute is used by some images to save a sealed '
+                       'version of a node apikey, so that a subsequent run with '
+                       'same TPM2 will use the TPM2 to protect the API key rather '
+                       'than local network verification. If this is set, then '
+                       'an api key request will receive this if the api key grant '
+                       'is not armed',
+    },
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
@@ -181,6 +189,10 @@ node = {
 #                        'autonode.servername, so that would not need to be '
 #                        'copied ')
 #    },
+#    'collective.allowedmanagers': {
+#        'description': ('Restricted set of deployment and managers in automatic selectien
+#    },
+#    ssh.equivnodes - control the list of nodes that go into equiv...
     'collective.manager': {
         'description': ('When in collective mode, the member of the '
                         'collective currently considered to be responsible '
@@ -189,6 +201,14 @@ node = {
                         'indicates candidate managers, either for '
                         'high availability or load balancing purposes.')
     },
+    'collective.managercandidates': {
+        'description': ('A noderange of nodes permitted to be a manager for '
+                        'the node. This controls failover and deployment.  If '
+                        'not defined, all managers may deploy and no '
+                        'automatic failover will be performed. '
+                        'Using this requires that collective members be '
+                        'defined as nodes for noderange expansion')
+    },
     'deployment.pendingprofile': {
         'description': ('An OS profile that is pending deployment.  This indicates to '
                         'the network boot subsystem what should be offered when a potential '
@@ -328,7 +348,7 @@ node = {
         'description': ('Indicate logging level to apply to console.  Valid '
                         'values are currently "full", "interactive", and '
                         '"none". Defaults to "full".'),
-        'validvalues': ('full', 'interactive', 'none'),
+        'validvalues': ('full', 'memory', 'interactive', 'none'),
     },
     'console.method': {
         'description': ('Indicate the method used to access the console of '
@@ -522,6 +542,13 @@ node = {
         'description': ('Password to use when connecting to the hardware '
                         'manager'),
     },
+    'ssh.trustnodes': {
+        'description': ('Nodes that are allowed to ssh into the node, '
+                        'expressed in noderange syntax.  This is used during '
+                        'deployment if the confluent SSH certificate '
+                        'authority is configured.  Default behavior is for '
+                        'all nodes to trust each other.'),
+    },
     'pubkeys.addpolicy': {
         'description': ('Policy to use when encountering unknown public '
                         'keys.  Choices are "automatic" to accept and '

confluent_server/confluent/config/configmanager.py

@@ -115,6 +115,7 @@ _attraliases = {
 }
 _validroles = ('Administrator', 'Operator', 'Monitor')
 
+membership_callback = None
 
 def attrib_supports_expression(attrib):
     if not isinstance(attrib, str):
@@ -409,6 +410,8 @@ def _push_rpc(stream, payload):
         except Exception:
             logException()
             del cfgstreams[stream]
+            if membership_callback:
+                membership_callback()
             stream.close()
 
 
@@ -615,6 +618,8 @@ def relay_slaved_requests(name, listener):
                 except Exception:
                     pass
                 del cfgstreams[name]
+                if membership_callback:
+                    membership_callback()
             cfgstreams[name] = listener
             lh = StreamHandler(listener)
             _hasquorum = len(cfgstreams) >= (
@@ -682,6 +687,8 @@ def relay_slaved_requests(name, listener):
                         _push_rpc,
                         [(cfgstreams[s], payload) for s in cfgstreams]):
                     pass
+            if membership_callback:
+                membership_callback()
             if not cfgstreams and not cfgleader:  # last one out, set cfgleader to boolean to mark dead collective
                 stop_following(True)
                 return False
@@ -739,6 +746,8 @@ def stop_leading():
             del cfgstreams[stream]
         except KeyError:
             pass  # may have already been deleted..
+        if membership_callback:
+            membership_callback()
 
 
 _oldcfgstore = None
@@ -776,7 +785,7 @@ def commit_clear():
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
     # per collective member
-    for globvar in _oldcfgstore['globals']:
+    for globvar in _oldcfgstore.get('globals', ()):
         if globvar.endswith('_key'):
             continue
         _cfgstore['globals'][globvar] = _oldcfgstore['globals'][globvar]
@@ -1129,6 +1138,7 @@ class ConfigManager(object):
         return _cfgstore['tenant'][self.tenant]
 
     def __init__(self, tenant, decrypt=False, username=None):
+        self.clientfiles = {}
         global _cfgstore
         with _initlock:
             if _cfgstore is None:
@@ -1163,6 +1173,13 @@ class ConfigManager(object):
             self._bg_sync_to_file()
             self.wait_for_sync()
 
+    def add_client_file(self, clientfile):
+        self.clientfiles[clientfile.filename] = clientfile.fileobject
+
+    def close_client_files(self):
+        for f in self.clientfiles:
+            self.clientfiles[f].close()
+
     def get_collective_member(self, name):
         return get_collective_member(name)
 

confluent_server/confluent/consoleserver.py

@@ -234,7 +234,7 @@ class ConsoleHandler(object):
             self._isondemand = False
         else:
             if (attrvalue[self.node]['console.logging']['value'] not in (
-                'full', '', 'buffer')):
+                'full', '', 'memory')):
                 self._isondemand = True
             if (attrvalue[self.node]['console.logging']['value']) in ('none', 'memory'):
                 self._dologging = False
@@ -609,10 +609,10 @@ class ConsoleHandler(object):
         if self.shiftin is not None:  # detected that terminal requested a
             # shiftin character set, relay that to the terminal that cannected
             retdata += b'\x1b)' + self.shiftin
-        if self.appmodedetected:
-            retdata += b'\x1b[?1h'
-        else:
-            retdata += b'\x1b[?1l'
+        #if self.appmodedetected:
+        #    retdata += b'\x1b[?1h'
+        #else:
+        #    retdata += b'\x1b[?1l'
         return retdata, connstate
 
     def write(self, data):

confluent_server/confluent/core.py

@@ -180,7 +180,8 @@ def handle_deployment(configmanager, inputdata, pathcomponents,
                     yield imp
                 return
             elif operation == 'create':
-                importer = osimage.MediaImporter(inputdata['filename'])
+                importer = osimage.MediaImporter(inputdata['filename'],
+                                                 configmanager)
                 yield msg.KeyValueData({'target': importer.targpath,
                                         'name': importer.importkey})
                 return

confluent_server/confluent/credserver.py

@@ -22,6 +22,16 @@ import eventlet
 import eventlet.green.socket as socket
 import eventlet.greenpool
 import os
+import struct
+
+# cred grant tlvs:
+# 0, 0 - null
+# 1, len, <nodename>
+# 2, len, token - echo request
+# 3, len, token - echo reply
+# 4, len, crypted - crypted apikey
+# 5, 0, accept key
+# 128, len, len, key - sealed key
 
 class CredServer(object):
     def __init__(self):
@@ -38,11 +48,20 @@ class CredServer(object):
                 client.close()
                 return
             nodename = util.stringify(client.recv(tlv[1]))
-            tlv = bytearray(client.recv(2))
-            apiarmed = self.cfm.get_node_attributes(nodename, 'deployment.apiarmed')
-            apiarmed = apiarmed.get(nodename, {}).get('deployment.apiarmed', {}).get(
+            tlv = bytearray(client.recv(2))  # should always be null
+            apimats = self.cfm.get_node_attributes(nodename,
+                ['deployment.apiarmed', 'deployment.sealedapikey'])
+            apiarmed = apimats.get(nodename, {}).get('deployment.apiarmed', {}).get(
                 'value', None)
             if not apiarmed:
+                if apimats.get(nodename, {}).get(
+                    'deployment.sealedapikey', {}).get('value', None):
+                    sealed = apimats[nodename]['deployment.sealedapikey'][
+                        'value']
+                    if not isinstance(sealed, bytes):
+                        sealed = sealed.encode('utf8')
+                    reply = b'\x80' + struct.pack('>H', len(sealed) + 1) + sealed + b'\x00'
+                    client.send(reply)
                 client.close()
                 return
             if apiarmed not in ('once', 'continuous'):

confluent_server/confluent/discovery/core.py

@@ -230,6 +230,8 @@ def send_discovery_datum(info):
     yield msg.KeyValueData({'serialnumber': sn})
     yield msg.KeyValueData({'modelnumber': mn})
     yield msg.KeyValueData({'uuid': uuid})
+    if 'enclosure.uuid' in info:
+        yield msg.KeyValueData({'enclosure_uuid': info['enclosure.uuid']})
     if 'enclosure.bay' in info:
         yield msg.KeyValueData({'bay': int(info['enclosure.bay'])})
     yield msg.KeyValueData({'macs': [info.get('hwaddr', '')]})
@@ -240,6 +242,16 @@ def send_discovery_datum(info):
     yield msg.KeyValueData({'types': types})
     if 'otheraddresses' in info:
         yield msg.KeyValueData({'otheripaddrs': list(info['otheraddresses'])})
+    if 'location' in info:
+        yield msg.KeyValueData({'location': info['location']})
+    if 'room' in info:
+        yield msg.KeyValueData({'room': info['room']})
+    if 'rack' in info:
+        yield msg.KeyValueData({'rack': info['rack']})
+    if 'u' in info:
+        yield msg.KeyValueData({'lowest_u': info['u']})
+    if 'hostname' in info:
+        yield msg.KeyValueData({'hostname': info['hostname']})
 
 
 def _info_matches(info, criteria):
@@ -772,12 +784,14 @@ def get_chained_smm_name(nodename, cfg, handler, nl=None, checkswitch=True):
                                                'extend a single enclosure')
         cd = cfg.get_node_attributes(nodename, ['hardwaremanagement.manager',
                                                 'pubkeys.tls_hardwaremanager'])
-        smmaddr = cd[nodename]['hardwaremanagement.manager']['value']
         pkey = cd[nodename].get('pubkeys.tls_hardwaremanager', {}).get(
             'value', None)
         if not pkey:
             # We cannot continue through a break in the chain
             return None, False
+        smmaddr = cd.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', None)
+        if not smmaddr:
+            return None, False
         if pkey:
             cv = util.TLSCertVerifier(
                 cfg, nodename, 'pubkeys.tls_hardwaremanager').verify_cert
@@ -800,10 +814,10 @@ def get_smm_neighbor_fingerprints(smmaddr, cv):
     neighs = wc.grab_json_response('/scripts/neighdata.json')
     if not neighs:
         return
-    for idx in (4, 5):
-        if 'sha256' not in neighs[idx]:
+    for neigh in neighs:
+        if 'sha256' not in neigh:
             continue
-        yield 'sha256$' + b64tohex(neighs[idx]['sha256'])
+        yield 'sha256$' + b64tohex(neigh['sha256'])
 
 
 def get_nodename(cfg, handler, info):
@@ -855,6 +869,14 @@ def get_nodename(cfg, handler, info):
                         # while this started by switch, it was disambiguated
                         info['verified'] = v
                         return newnodename, None
+                    else:
+                        errorstr = ('Attempt to discover SMM in chain but '
+                                   'unable to follow chain to the specific '
+                                   'SMM, it may be waiting on an upstream '
+                                   'SMM, chain starts with {0}'.format(
+                                       nodename))
+                        log.log({'error': errorstr})
+                        return None, None
         if (nodename and
                 not handler.discoverable_by_switch(macinfo['maccount'])):
             if handler.devname == 'SMM':
@@ -1040,6 +1062,20 @@ def eval_node(cfg, handler, info, nodename, manual=False):
 
 
 def discover_node(cfg, handler, info, nodename, manual):
+    if manual:
+        if not cfg.is_node(nodename):
+            raise exc.InvalidArgumentException(
+                '{0} is not a defined node, must be defined before an '
+                'endpoint may be assigned to it'.format(nodename))
+        if handler.https_supported:
+            currcert = handler.https_cert
+            if currcert:
+                currprint = util.get_fingerprint(currcert, 'sha256')
+                prevnode = nodes_by_fprint.get(currprint, None)
+                if prevnode and prevnode != nodename:
+                    raise exc.InvalidArgumentException(
+                    'Attempt to assign {0} conflicts with existing node {1} '
+                    'based on TLS certificate.'.format(nodename, prevnode))
     known_nodes[nodename][info['hwaddr']] = info
     if info['hwaddr'] in unknown_info:
         del unknown_info[info['hwaddr']]
@@ -1129,11 +1165,13 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
     # use uuid based scheme in lieu of tls cert, ideally only
     # for stateless 'discovery' targets like pxe, where data does not
     # change
-    uuidinfo = cfg.get_node_attributes(nodename, ['id.uuid', 'id.serial', 'id.model', 'net*.bootable'])
+    uuidinfo = cfg.get_node_attributes(nodename, ['id.uuid', 'id.serial', 'id.model', 'net*.hwaddr', 'net*.bootable'])
     if manual or policies & set(('open', 'pxe')):
         enrich_pxe_info(info)
         attribs = {}
         olduuid = uuidinfo.get(nodename, {}).get('id.uuid', None)
+        if isinstance(olduuid, dict):
+            olduuid = olduuid.get('value', None)
         uuid = info.get('uuid', None)
         if uuid and uuid != olduuid:
             attribs['id.uuid'] = info['uuid']
@@ -1146,7 +1184,9 @@ def do_pxe_discovery(cfg, handler, info, manual, nodename, policies):
         for attrname in uuidinfo.get(nodename, {}):
             if attrname.endswith('.bootable') and uuidinfo[nodename][attrname].get('value', None):
                 newattrname = attrname[:-8] + 'hwaddr'
-                attribs[newattrname] = info['hwaddr']
+                oldhwaddr = uuidinfo.get(nodename, {}).get(newattrname, {}).get('value', None)
+                if info['hwaddr'] != oldhwaddr:
+                    attribs[newattrname] = info['hwaddr']
         if attribs:
             cfg.set_node_attributes({nodename: attribs})
     if info['uuid'] in known_pxe_uuids:
@@ -1278,11 +1318,11 @@ known_pxe_uuids = {}
 def _map_unique_ids(nodes=None):
     global nodes_by_uuid
     global nodes_by_fprint
-    nodes_by_uuid = {}
-    nodes_by_fprint = {}
     # Map current known ids based on uuid and fingperprints for fast lookup
     cfg = cfm.ConfigManager(None)
     if nodes is None:
+        nodes_by_uuid = {}
+        nodes_by_fprint = {}
         nodes = cfg.list_nodes()
     bigmap = cfg.get_node_attributes(nodes,
                                      ('id.uuid',
@@ -1304,7 +1344,7 @@ def _map_unique_ids(nodes=None):
             del nodes_by_uuid[uuid_by_nodes[node]]
         if node in fprint_by_nodes:
             del nodes_by_fprint[fprint_by_nodes[node]]
-        uuid = bigmap[node].get('id.uuid', {}).get('value', None)
+        uuid = bigmap[node].get('id.uuid', {}).get('value', '').lower()
         if uuid_is_valid(uuid):
             nodes_by_uuid[uuid] = node
         fprint = bigmap[node].get(

confluent_server/confluent/discovery/handlers/imm.py

@@ -47,10 +47,28 @@ class NodeHandler(bmchandler.NodeHandler):
             self.info['uuid'] = uuidprefix + '-' + '-'.join(
                 wronguuid.split('-')[3:])
             self.info['uuid'] = self.info['uuid'].lower()
+        room = slpattrs.get('room-id', [None])[0]
+        if room:
+            self.info['room'] = room
+        rack = slpattrs.get('rack-id', [None])[0]
+        if rack:
+            self.info['rack'] = rack
+        name = slpattrs.get('name', [None])[0]
+        if name:
+            self.info['hostname'] = name
+        unumber = slpattrs.get('lowest-u', [None])[0]
+        if unumber:
+            self.info['u'] = unumber
+        location = slpattrs.get('location', [None])[0]
+        if location:
+            self.info['location'] = location
         if ff not in ('dense-computing', 'BC2'):
             # do not probe unless it's a dense platform
             return
         self.isdense = True
+        encuuid = slpattrs.get('chassis-uuid', [None])[0]
+        if encuuid:
+            self.info['enclosure.uuid'] = encuuid
         slot = int(slpattrs.get('slot', ['0'])[0])
         if slot != 0:
             self.info['enclosure.bay'] = slot

confluent_server/confluent/discovery/handlers/smm.py

@@ -27,7 +27,16 @@ import confluent.netutil as netutil
 import confluent.util as util
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
-from xml.etree.ElementTree import fromstring
+from xml.etree.ElementTree import fromstring as rfromstring
+
+def fromstring(inputdata):
+    if isinstance(inputdata, bytes):
+        cmpstr = b'!entity'
+    else:
+        cmpstr = '!entity'
+    if cmpstr in inputdata.lower():
+        raise Exception('!ENTITY not supported in this interface')
+    return rfromstring(inputdata)
 
 def fixuuid(baduuid):
     # SMM dumps it out in hex
@@ -41,7 +50,7 @@ def fixuuid(baduuid):
 class NodeHandler(bmchandler.NodeHandler):
     is_enclosure = True
     devname = 'SMM'
-    maxmacs = 6  # support an enclosure, but try to avoid catching daisy chain
+    maxmacs = 14  # support an enclosure, but try to avoid catching daisy chain
 
     def scan(self):
         # the UUID is in a weird order, fix it up to match
@@ -82,6 +91,14 @@ class NodeHandler(bmchandler.NodeHandler):
 
     def _webconfignet(self, wc, nodename):
         cfg = self.configmanager
+        if 'service:lenovo-smm2' in self.info.get('services', []):
+            # need to enable ipmi for now..
+            wc.request('POST', '/data', 'set=DoCmd(0x06,0x40,0x01,0x82,0x84)')
+            rsp = wc.getresponse()
+            rsp.read()
+            wc.request('POST', '/data', 'set=DoCmd(0x06,0x40,0x01,0x42,0x44)')
+            rsp = wc.getresponse()
+            rsp.read()
         cd = cfg.get_node_attributes(
             nodename, ['hardwaremanagement.manager'])
         smmip = cd.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', None)
@@ -196,7 +213,7 @@ class NodeHandler(bmchandler.NodeHandler):
             raise Exception('Cannot support default password and setting password rules at same time')
         if passwd == 'PASSW0RD':
             # We must avoid hitting the web interface due to forced password change, best effert
-            self._bmcconfig(nodename)
+            raise Exception('Using the default password is no longer supported')
         else:
             # Switch to full web based configuration, to mitigate risks with the SMM
             wc = self._webconfigcreds(username, passwd)

confluent_server/confluent/discovery/handlers/tsm.py

@@ -229,6 +229,19 @@ class NodeHandler(generic.NodeHandler):
         rsp, status = wc.grab_json_response_with_status('/api/session', method='DELETE')
 
 
+def remote_nodecfg(nodename, cfm):
+    cfg = cfm.get_node_attributes(
+            nodename, 'hardwaremanagement.manager')
+    ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get(
+        'value', None)
+    ipaddr = getaddrinfo(ipaddr, 0)[0][-1]
+    if not ipaddr:
+        raise Excecption('Cannot remote configure a system without known '
+                         'address')
+    info = {'addresses': [ipaddr]}
+    nh = NodeHandler(info, cfm)
+    nh.config(nodename)
+
 if __name__ == '__main__':
     import confluent.config.configmanager as cfm
     c = cfm.ConfigManager(None)

confluent_server/confluent/discovery/handlers/xcc.py

@@ -39,6 +39,9 @@ def fixup_uuid(uuidprop):
     return '-'.join(uuid).upper()
 
 
+class LockedUserException(Exception):
+    pass
+
 
 
 class NodeHandler(immhandler.NodeHandler):
@@ -66,11 +69,13 @@ class NodeHandler(immhandler.NodeHandler):
         if ff not in ('dense-computing', [u'dense-computing']):
             # skip preconfig for non-SD530 servers
             return
+        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
+        if not currfirm.startswith('TEI'):
+            return
         self.trieddefault = None  # Reset state on a preconfig attempt
         # attempt to enable SMM
         #it's normal to get a 'not supported' (193) for systems without an SMM
         # need to branch on 3.00+ firmware
-        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
         currfirm = currfirm.split(':')
         if len(currfirm) > 1:
             currfirm = float(currfirm[1])
@@ -136,8 +141,14 @@ class NodeHandler(immhandler.NodeHandler):
                    'Content-Type': 'application/json'}
         wc.request('POST', '/api/login', adata, headers)
         rsp = wc.getresponse()
+        try:
+            rspdata = json.loads(rsp.read())
+        except Exception:
+            rspdata = {}
         if rsp.status != 200 and password == 'PASSW0RD':
-            rsp.read()
+            if rspdata.get('locktime', 0) > 0:
+                raise LockedUserException(
+                    'The user "{0}" has been locked out for too many incorrect password attempts'.format(username))
             adata = json.dumps({
                 'username': username,
                 'password': newpassword,
@@ -146,16 +157,21 @@ class NodeHandler(immhandler.NodeHandler):
                        'Content-Type': 'application/json'}
             wc.request('POST', '/api/login', adata, headers)
             rsp = wc.getresponse()
+            try:
+                rspdata = json.loads(rsp.read())
+            except Exception:
+                rspdata = {}
             if rsp.status == 200:
                 pwdchanged = True
                 password = newpassword
             else:
-                rsp.read()
-                return (None, None)
+                if rspdata.get('locktime', 0) > 0:
+                    raise LockedUserException(
+                        'The user "{0}" has been locked out for too many incorrect password attempts'.format(username))
+                return (None, rspdata)
         if rsp.status == 200:
             self._currcreds = (username, password)
             wc.set_basic_credentials(username, password)
-            rspdata = json.loads(rsp.read())
             wc.set_header('Content-Type', 'application/json')
             wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
             if '_csrf_token' in wc.cookies:
@@ -178,12 +194,16 @@ class NodeHandler(immhandler.NodeHandler):
                 wc = self.wc
                 self.set_password_policy('', wc)
             return (wc, pwdchanged)
-        return (None, None)
+        elif rspdata.get('locktime', 0) > 0:
+            raise LockedUserException(
+                'The user "{0}" has been locked out by too many incorrect password attempts'.format(username))
+        return (None, rspdata)
 
     @property
     def wc(self):
         passwd = None
         isdefault = True
+        errinfo = {}
         if self._wc is None:
             self._wc = webclient.SecureHTTPConnection(
                 self.ipaddr, 443, verifycallback=self.validate_cert)
@@ -205,6 +225,9 @@ class NodeHandler(immhandler.NodeHandler):
                 'secret.hardwaremanagementpassword'], decrypt=True)
             user, passwd, isdefault = self.get_node_credentials(
                 nodename, creds, 'USERID', 'PASSW0RD')
+        if not inpreconfig and isdefault:
+            raise Exception('Default user/password is not supported. Please set "secret.hardwaremanagementuser" and "secret.hardwaremanagementpassword" for {} to a non-default value. If the XCC is currently at defaults, it will automatically change to the specified values'.format(nodename))
+        savedexc = None
         if not self.trieddefault:
             if not passwd:
                 # So in preconfig context, we don't have admin permission to
@@ -215,7 +238,12 @@ class NodeHandler(immhandler.NodeHandler):
                 # This is replacing one well known password (PASSW0RD) with another
                 # (TempW0rd42)
                 passwd = 'TempW0rd42'
-            wc, pwdchanged = self.get_webclient('USERID', 'PASSW0RD', passwd)
+            try:
+                wc, pwdchanged = self.get_webclient('USERID', 'PASSW0RD', passwd)
+            except LockedUserException as lue:
+                wc = None
+                pwdchanged = 'The user "USERID" has been locked out by too many incorrect password attempts'
+                savedexc = lue
             if wc:
                 if pwdchanged:
                     if inpreconfig:
@@ -223,16 +251,26 @@ class NodeHandler(immhandler.NodeHandler):
                     else:
                         self._needpasswordchange = False
                 return wc
+            else:
+                errinfo = pwdchanged
         self.trieddefault = True
         if isdefault:
             return
         self._atdefaultcreds = False
         if self.tmppasswd:
-            wc, _ = self.get_webclient('USERID', self.tmppasswd, passwd)
+            if savedexc:
+                raise savedexc
+            wc, errinfo = self.get_webclient('USERID', self.tmppasswd, passwd)
         else:
-            wc, _ = self.get_webclient(user, passwd, None)
+            if user == 'USERID' and savedexc:
+                raise savedexc
+            wc, errinfo = self.get_webclient(user, passwd, None)
         if wc:
             return wc
+        else:
+            if errinfo.get('description', '') == 'Invalid credentials':
+                raise Exception('The stored confluent password for user "{}" was not accepted by the XCC'.format(user))
+            raise Exception('Error connecting to webservice: ' + repr(errinfo))
 
     def set_password_policy(self, strruleset, wc):
         ruleset = {'USER_GlobalMinPassChgInt': '0'}
@@ -286,9 +324,13 @@ class NodeHandler(immhandler.NodeHandler):
             wc.grab_json_response('/api/function',
                                 {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
         if username != 'USERID':
-            wc.grab_json_response(
+            rsp, status = wc.grab_json_response_with_status(
                 '/api/function',
                 {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,'.format(uid, username)})
+            if status == 200 and rsp.get('return', 0) == 762:
+                rsp, status = wc.grab_json_response_with_status(
+                    '/api/function',
+                    {'USER_UserModify': '{0},{1},,1,Administrator,0,0,0,0,,8,'.format(uid, username)})
             self.tmppasswd = None
         self._currcreds = (username, passwd)
 
@@ -338,6 +380,10 @@ class NodeHandler(immhandler.NodeHandler):
                     nwc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
                     nwc.grab_json_response('/api/providers/logout')
                     nwc, pwdchanged = self.get_webclient(user, tpass, passwd)
+                    if not nwc:
+                        if not pwdchanged:
+                            pwdchanged = 'Unknown'
+                        raise Exception('Error converting from sha356account: ' + repr(pwdchanged))
                     if not pwdchanged:
                         nwc.grab_json_response(
                             '/api/function',
@@ -431,3 +477,16 @@ class NodeHandler(immhandler.NodeHandler):
             if em:
                 self.configmanager.set_node_attributes(
                     {em: {'id.uuid': enclosureuuid}})
+
+def remote_nodecfg(nodename, cfm):
+    cfg = cfm.get_node_attributes(
+            nodename, 'hardwaremanagement.manager')
+    ipaddr = cfg.get(nodename, {}).get('hardwaremanagement.manager', {}).get(
+        'value', None)
+    ipaddr = getaddrinfo(ipaddr, 0)[0][-1]
+    if not ipaddr:
+        raise Excecption('Cannot remote configure a system without known '
+                         'address')
+    info = {'addresses': [ipaddr]}
+    nh = NodeHandler(info, cfm)
+    nh.config(nodename)

confluent_server/confluent/discovery/protocols/pxe.py

@@ -23,6 +23,8 @@
 # option 97 = UUID (wireformat)
 
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
+import confluent.noderange as noderange
 import confluent.log as log
 import confluent.netutil as netutil
 import ctypes
@@ -264,9 +266,7 @@ def proxydhcp():
         if not myipn:
             continue
         if opts.get(77, None) == b'iPXE':
-            cfd = cfg.get_node_attributes(node, ('deployment.*'))
-            profile = cfd.get(node, {}).get(
-                'deployment.pendingprofile', {}).get('value', None)
+            profile = get_deployment_profile(node, cfg)
             if not profile:
                 continue
             myip = socket.inet_ntoa(myipn)
@@ -423,22 +423,36 @@ def remap_nodes(nodeattribs, configmanager):
     for node in updates:
         for attrib in updates[node]:
             if attrib == 'id.uuid':
-                uuidmap[updates[node][attrib]['value']] = node
+                uuidmap[updates[node][attrib]['value'].lower()] = node
             elif 'hwaddr' in attrib:
-                macmap[updates[node][attrib]['value']] = node
+                macmap[updates[node][attrib]['value'].lower()] = node
 
 
+def get_deployment_profile(node, cfg, cfd=None):
+    if not cfd:
+        cfd = cfg.get_node_attributes(node, ('deployment.*'))
+    profile = cfd.get(node, {}).get('deployment.pendingprofile', {}).get('value', None)
+    if not profile:
+        return None
+    candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None)
+    if candmgrs:
+        candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+        if collective.get_myname() not in candmgrs:
+            return None
+    return profile
+
 staticassigns = {}
 myipbypeer = {}
 def check_reply(node, info, packet, sock, cfg, reqview):
     httpboot = info['architecture'] == 'uefi-httpboot'
     replen = 275  # default is going to be 286
     cfd = cfg.get_node_attributes(node, ('deployment.*'))
-    profile = cfd.get(node, {}).get('deployment.pendingprofile', {}).get('value', None)
-    myipn = info['netinfo']['recvip']
-    myipn = socket.inet_aton(myipn)
+    profile = get_deployment_profile(node, cfg, cfd)
     if not profile:
         return
+    myipn = info['netinfo']['recvip']
+    myipn = socket.inet_aton(myipn)
+
     rqtype = packet[53][0]
     insecuremode = cfd.get(node, {}).get('deployment.useinsecureprotocols',
         {}).get('value', 'never')

confluent_server/confluent/discovery/protocols/slp.py

@@ -19,6 +19,7 @@ import confluent.util as util
 import confluent.log as log
 import os
 import random
+import eventlet.greenpool
 import eventlet.green.select as select
 import eventlet.green.socket as socket
 import struct
@@ -198,7 +199,14 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
     :param addresses:  Pass through of addresses argument from find_targets
     :return:
     """
-    if addresses is None:
+    if addresses is not None:
+        for addr in addresses:
+            for saddr in socket.getaddrinfo(addr, 427):
+                if saddr[0] == socket.AF_INET:
+                    net4.sendto(data, saddr[4])
+                elif saddr[0] == socket.AF_INET6:
+                    net.sendto(data, saddr[4])
+    else:
         data = _generate_request_payload(srvtype, True, xid)
         net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
         v6addrs = []
@@ -360,6 +368,9 @@ def _add_attributes(parsed):
         return
 
 
+def unicast_scan(address):
+    pass
+
 def query_srvtypes(target):
     """Query the srvtypes advertised by the target
 
@@ -465,15 +476,16 @@ def snoop(handler, protocol=None):
             # will now yield dupe info over time
             known_peers = set([])
             peerbymacaddress = {}
-            neighutil.update_neigh()
             while r:
                 for s in r:
                     (rsp, peer) = s.recvfrom(9000)
                     ip = peer[0].partition('%')[0]
-                    if ip not in neighutil.neightable:
-                        continue
                     if peer in known_peers:
                         continue
+                    if ip not in neighutil.neightable:
+                        neighutil.update_neigh()
+                    if ip not in neighutil.neightable:
+                        continue
                     known_peers.add(peer)
                     mac = neighutil.neightable[ip]
                     if mac in peerbymacaddress:
@@ -579,6 +591,8 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
         # reduced chance of many responses overwhelming receive buffer.
     _grab_rsps((net, net4), rsps, 1, xidmap)
     # now to analyze and flesh out the responses
+    handleids = set([])
+    gp = eventlet.greenpool.GreenPool(128)
     for id in rsps:
         for srvurl in rsps[id].get('urls', ()):
             if len(srvurl) > 4:
@@ -593,14 +607,22 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
                     break
             else:
                 continue
-        _add_attributes(rsps[id])
+        gp.spawn_n(_add_attributes, rsps[id])
+        handleids.add(id)
+    gp.waitall()
+    for id in handleids:
         if 'service:lighttpd' in rsps[id]['services']:
             currinf = rsps[id]
             curratt = currinf.get('attributes', {})
             if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
                currinf['services'] = ['service:lenovo-tsm']
-               curratt['enclosure-serial-number'] = curratt['Product-Serial']
-               curratt['enclosure-machinetype-model'] = curratt['Machine-Type']
+               serialnumber = curratt.get('Product-Serial', curratt.get('SerialNumber', None))
+               if serialnumber:
+                   curratt['enclosure-serial-number'] = serialnumber
+               mtm = curratt.get('Machine-Type', curratt.get('Product-Name', None))
+               if mtm:
+                   mtm[0] = mtm[0].rstrip()
+                   curratt['enclosure-machinetype-model'] = mtm
             else:
                 continue
         del rsps[id]['payload']

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -29,12 +29,15 @@
 
 
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
 import confluent.neighutil as neighutil
+import confluent.noderange as noderange
 import confluent.util as util
 import confluent.log as log
 import confluent.netutil as netutil
 import eventlet.green.select as select
 import eventlet.green.socket as socket
+import eventlet.greenpool as gp
 import time
 try:
     from eventlet.green.urllib.request import urlopen
@@ -118,7 +121,6 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
             newmacs = set([])
             machandlers = {}
             r, _, _ = select.select((net4, net6), (), (), 60)
-            neighutil.update_neigh()
             while r:
                 for s in r:
                     (rsp, peer) = s.recvfrom(9000)
@@ -128,10 +130,12 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                     method, _, _ = rsp[0].split(b' ', 2)
                     if method == b'NOTIFY':
                         ip = peer[0].partition('%')[0]
-                        if ip not in neighutil.neightable:
-                            continue
                         if peer in known_peers:
                             continue
+                        if ip not in neighutil.neightable:
+                            neighutil.update_neigh()
+                        if ip not in neighutil.neightable:
+                            continue
                         mac = neighutil.neightable[ip]
                         known_peers.add(peer)
                         newmacs.add(mac)
@@ -184,10 +188,15 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                         # planned for
                                         cfg = cfm.ConfigManager(None)
                                         cfd = cfg.get_node_attributes(
-                                            node, 'deployment.pendingprofile')
+                                            node, ['deployment.pendingprofile', 'collective.managercandidates'])
                                         if not cfd.get(node, {}).get(
                                                 'deployment.pendingprofile', {}).get('value', None):
                                             break
+                                        candmgrs = cfd.get(node, {}).get('collective.managercandidates', {}).get('value', None)
+                                        if candmgrs:
+                                            candmgrs = noderange.NodeRange(candmgrs, cfg).nodes
+                                            if collective.get_myname() not in candmgrs:
+                                                break
                                         currtime = time.time()
                                         seconds = int(currtime)
                                         msecs = int(currtime * 1000 % 1000)
@@ -200,6 +209,8 @@ def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
                                                 cfg, node, ifidx=iface)
                                             if ncfg.get('matchesnodename', None):
                                                 reply += 'DEFAULTNET: 1\r\n'
+                                        elif not netutil.address_is_local(peer[0]):
+                                            continue
                                         if not isinstance(reply, bytes):
                                             reply = reply.encode('utf8')
                                         s.sendto(reply, peer)
@@ -274,13 +285,26 @@ def _find_service(service, target):
         if timeout < 0:
             timeout = 0
         r, _, _ = select.select((net4, net6), (), (), timeout)
+    querypool = gp.GreenPool()
+    pooltargs = []
     for nid in peerdata:
         for url in peerdata[nid].get('urls', ()):
             if url.endswith('/desc.tmpl'):
-                info = urlopen(url).read()
-                if b'<friendlyName>Athena</friendlyName>' in info:
-                    peerdata[nid]['services'] = ['service:thinkagile-storage']
-                    yield peerdata[nid]
+                pooltargs.append((url, peerdata[nid]))
+    for pi in querypool.imap(check_cpstorage, pooltargs):
+        if pi is not None:
+            yield pi
+
+def check_cpstorage(urldata):
+    url, data = urldata
+    try:
+        info = urlopen(url, timeout=1).read()
+        if b'<friendlyName>Athena</friendlyName>' in info:
+            data['services'] = ['service:thinkagile-storage']
+            return data
+    except Exception:
+        pass
+    return None
 
 
 def _parse_ssdp(peer, rsp, peerdata):
@@ -327,10 +351,6 @@ def _parse_ssdp(peer, rsp, peerdata):
 
 if __name__ == '__main__':
 
-    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1']):
+    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1'], '10.240.52.189'):
         print(repr(rsp))
-    def fun(a):
-        print(repr(a))
-    def byefun(a):
-        print('bye' + repr(a))
-    snoop(fun, byefun)
+    

confluent_server/confluent/firmwaremanager.py

@@ -32,11 +32,12 @@ uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
 _tracelog = None
+filecontentbyname = {}
 
 
-def execupdate(handler, filename, updateobj, type, owner, node):
+def execupdate(handler, filename, updateobj, type, owner, node, datfile):
     global _tracelog
-    if type != 'ffdc':
+    if type != 'ffdc' and not datfile:
         errstr = False
         if not os.path.exists(filename):
             errstr =  '{0} does not appear to exist on {1}, or is in a directory with permissions forbidding confluent user/group access'.format(
@@ -64,9 +65,10 @@ def execupdate(handler, filename, updateobj, type, owner, node):
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
-                                 bank=updateobj.bank)
+                                 data=datfile, bank=updateobj.bank)
         else:
-            completion = handler(filename, progress=updateobj.handle_progress)
+            completion = handler(filename, progress=updateobj.handle_progress,
+                                 data=datfile)
         if type == 'ffdc' and completion:
             filename = completion
             completion = None
@@ -90,14 +92,20 @@ def execupdate(handler, filename, updateobj, type, owner, node):
 
 class Updater(object):
     def __init__(self, node, handler, filename, tenant=None, name=None,
-                 bank=None, type='firmware', owner=None):
+                 bank=None, type='firmware', owner=None, configmanager=None):
         self.bank = bank
         self.node = node
         self.phase = 'initializing'
         self.detail = ''
         self.percent = 0.0
+        if configmanager and filename in configmanager.clientfiles:
+            cf = configmanager.clientfiles[filename]
+            datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+        else:
+            datfile = None
+        self.datfile = datfile
         self.updateproc = updatepool.spawn(execupdate, handler, filename,
-                                           self, type, owner, node)
+                                           self, type, owner, node, datfile)
         if type == 'firmware':
             myparty = updatesbytarget
         elif type == 'mediaupload':
@@ -120,6 +128,8 @@ class Updater(object):
 
     def cancel(self):
         self.updateproc.kill()
+        if self.datfile:
+            self.datfile.close()
 
     @property
     def progress(self):

confluent_server/confluent/forwarder.py

@@ -34,6 +34,8 @@ def handle_connection(incoming, outgoing):
         for mysock in r:
             data = mysock.recv(32768)
             if not data:
+                incoming.close()
+                outgoing.close()
                 return
             if mysock == incoming:
                 outgoing.sendall(data)
@@ -72,6 +74,7 @@ def forward_video():
             vidclient.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
         except Exception:
             conn.close()
+            vidclient.close()
             continue
         eventlet.spawn_n(handle_connection, conn, vidclient)
 

confluent_server/confluent/httpapi.py

@@ -65,16 +65,6 @@ opmap = {
 }
 
 
-class RobustCookie(Cookie.SimpleCookie):
-    # this is very bad form, but BaseCookie has a terrible flaw
-    def _BaseCookie__set(self, K, rval, cval):
-        try:
-            super(RobustCookie, self)._BaseCookie__set(K, rval, cval)
-        except Cookie.CookieError:
-            # empty value if SimpleCookie rejects
-            dict.__setitem__(self, K, Cookie.Morsel())
-
-
 def group_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the group").html() + '<br>'
@@ -284,11 +274,10 @@ def _authorize_request(env, operation):
     if element.startswith('/sessions/current/'):
         element = None
     if 'HTTP_COOKIE' in env:
-        #attempt to use the cookie.  If it matches
-        cc = RobustCookie()
-        cc.load(env['HTTP_COOKIE'])
-        if 'confluentsessionid' in cc:
-            sessionid = cc['confluentsessionid'].value
+        cidx = (env['HTTP_COOKIE']).find('confluentsessionid=')
+        if cidx >= 0:
+            sessionid = env['HTTP_COOKIE'][cidx+19:cidx+51]
+            sessid = sessionid
             sessid = sessionid
             if sessionid in httpsessions:
                 if _csrf_valid(env, httpsessions[sessionid]):

confluent_server/confluent/messages.py

@@ -1147,6 +1147,7 @@ class BootDevice(ConfluentChoiceMessage):
         'default',
         'cd',
         'floppy',
+        'usb',
     ])
 
     valid_bootmodes = set([

confluent_server/confluent/networking/lldp.py

@@ -148,11 +148,10 @@ def get_fingerprint(switch, port, configmanager, portmatch):
 def _extract_extended_desc(info, source, integritychecked):
     source = str(source)
     info['verified']  = bool(integritychecked)
-    if source.startswith('Lenovo SMM;'):
-        info['peerdescription'] = 'Lenovo SMM'
-        if ';S2=' in source:
-            info['peersha256fingerprint'] = source.replace('Lenovo SMM;S2=',
-                                                           '')
+    if source.startswith('Lenovo ') and ';S2=' in source:
+        desc, fprint = source.split(';S2=', 1)
+        info['peerdescription'] = desc
+        info['peersha256fingerprint'] = fprint
     else:
         info['peerdescription'] = source
 

confluent_server/confluent/networking/macmap.py

@@ -138,7 +138,9 @@ def _affluent_map_switch(args):
     wc =  webclient.SecureHTTPConnection(
                 switch, 443, verifycallback=kv, timeout=5)
     wc.set_basic_credentials(user, password)
-    macs = wc.grab_json_response('/affluent/macs/by-port')
+    macs, retcode = wc.grab_json_response_with_status('/affluent/macs/by-port')
+    if retcode != 200:
+        raise Exception("No affluent detected")
     _macsbyswitch[switch] = macs
 
     for iface in macs:
@@ -193,6 +195,7 @@ def _map_switch_backend(args):
     haveqbridge = False
     mactobridge = {}
     conn = snmp.Session(switch, password, user)
+    ifnamemap = get_portnamemap(conn)
     for vb in conn.walk('1.3.6.1.2.1.17.7.1.2.2.1.2'):
         haveqbridge = True
         oid, bridgeport = vb
@@ -214,16 +217,32 @@ def _map_switch_backend(args):
                 *([int(x) for x in oid[-6:]])
             )
             mactobridge[macaddr] = int(bridgeport)
+    #ciscoiftovlanmap = {}
+    vlanstocheck = set([])
+    for vb in conn.walk('.1.3.6.1.4.1.9.9.68.1.2.2.1.2'):
+        vlanstocheck.add(vb[1])
+    #ciscotrunktovlanmap = {}
+    for vb in conn.walk('.1.3.6.1.4.1.9.9.46.1.6.1.1.5'):
+        vlanstocheck.add(vb[1])
+    if not vlanstocheck:
+        vlanstocheck.add(None)
     bridgetoifmap = {}
-    for vb in conn.walk('1.3.6.1.2.1.17.1.4.1.2'):
-        bridgeport, ifidx = vb
-        bridgeport = int(str(bridgeport).rsplit('.', 1)[1])
-        try:
-            bridgetoifmap[bridgeport] = int(ifidx)
-        except ValueError:
-            # ifidx might be '', skip in such a case
-            continue
-    ifnamemap = get_portnamemap(conn)
+    for vlan in vlanstocheck:
+        if vlan:
+            if user:
+                conn = snmp.Session(switch, password, user, 'vlan-{}'.format(vlan))
+            else:
+                if not isinstance(password, str):
+                    password = password.decode('utf8')
+                conn = snmp.Session(switch, '{}@{}'.format(password, vlan))
+        for vb in conn.walk('1.3.6.1.2.1.17.1.4.1.2'):
+            bridgeport, ifidx = vb
+            bridgeport = int(str(bridgeport).rsplit('.', 1)[1])
+            try:
+                bridgetoifmap[bridgeport] = int(ifidx)
+            except ValueError:
+                # ifidx might be '', skip in such a case
+                continue
     maccounts = {}
     bridgetoifvalid = False
     for mac in mactobridge:
@@ -375,12 +394,19 @@ def _full_updatemacmap(configmanager):
                         continue
                     if curswitch not in _switchportmap:
                         _switchportmap[curswitch] = {}
-                    if portname in _switchportmap[curswitch]:
-                        log.log({'error': 'Duplicate switch topology config '
-                                          'for {0} and {1}'.format(
-                                            node,
+                    if (portname in _switchportmap[curswitch] and
+                            _switchportmap[curswitch][portname] != node):
+                        if _switchportmap[curswitch][portname] is None:
+                            errstr = ('Duplicate switch attributes for {0} and '
+                                      'a previously logged duplicate'.format(
+                                         node))
+                        else:
+                            errstr = ('Duplicate switch topology config '
+                                      'for {0} and {1}'.format(
+                                                node,
                                             _switchportmap[curswitch][
-                                                portname])})
+                                                portname]))
+                        log.log({'error': errstr})
                         _switchportmap[curswitch][portname] = None
                     else:
                         _switchportmap[curswitch][portname] = node

confluent_server/confluent/osimage.py

@@ -40,6 +40,22 @@ from libarchive.ffi import (
 def relax_umask():
     os.umask(0o22)
 
+
+def makedirs(path, mode):
+    try:
+        os.makedirs(path, 0o755)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+
+def symlink(src, targ):
+    try:
+        os.symlink(src, targ)
+    except OSError as e:
+        if e.errno != 17:
+            raise
+
+
 def update_boot(profilename):
     if profilename.startswith('/var/lib/confluent/public'):
         profiledir = profilename
@@ -59,6 +75,7 @@ def update_boot(profilename):
         update_boot_esxi(profiledir, profile, label)
 
 def update_boot_esxi(profiledir, profile, label):
+    profname = os.path.basename(profiledir)
     kernelargs = profile.get('kernelargs', '')
     oum = os.umask(0o22)
     bootcfg = open('{0}/distribution/BOOT.CFG'.format(profiledir), 'r').read()
@@ -89,7 +106,7 @@ def update_boot_esxi(profiledir, profile, label):
         else:
             newbootcfg += cfgline + '\n'
             efibootcfg += cfgline + '\n'
-    os.makedirs('{0}/boot/efi/boot/'.format(profiledir), 0o755)
+    makedirs('{0}/boot/efi/boot/'.format(profiledir), 0o755)
     bcfgout = os.open('{0}/boot/efi/boot/boot.cfg'.format(profiledir), os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o644)
     bcfg = os.fdopen(bcfgout, 'w')
     try:
@@ -102,7 +119,7 @@ def update_boot_esxi(profiledir, profile, label):
         bcfg.write(newbootcfg)
     finally:
         bcfg.close()
-    os.symlink('/var/lib/confluent/public/site/initramfs.tgz',
+    symlink('/var/lib/confluent/public/site/initramfs.tgz',
                '{0}/boot/site.tgz'.format(profiledir))
     for fn in filesneeded:
         if fn.startswith('/'):
@@ -110,8 +127,10 @@ def update_boot_esxi(profiledir, profile, label):
         sourcefile = '{0}/distribution/{1}'.format(profiledir, fn)
         if not os.path.exists(sourcefile):
             sourcefile = '{0}/distribution/{1}'.format(profiledir, fn.upper())
-        os.symlink(sourcefile, '{0}/boot/{1}'.format(profiledir, fn))
-    os.symlink('{0}/distribution/EFI/BOOT/BOOTX64.EFI'.format(profiledir), '{0}/boot/efi/boot/bootx64.efi'.format(profiledir))
+        symlink(sourcefile, '{0}/boot/{1}'.format(profiledir, fn))
+    symlink('{0}/distribution/EFI/BOOT/BOOTX64.EFI'.format(profiledir), '{0}/boot/efi/boot/bootx64.efi'.format(profiledir))
+    if os.path.exists('{0}/distribution/EFI/BOOT/CRYPTO64.EFI'.format(profiledir)):
+        symlink('{0}/distribution/EFI/BOOT/CRYPTO64.EFI'.format(profiledir), '{0}/boot/efi/boot/crypto64.efi'.format(profiledir))
     ipout = os.open(profiledir + '/boot.ipxe', os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0o644)
     ipxeout = os.fdopen(ipout, 'w')
     try:
@@ -124,10 +143,11 @@ def update_boot_esxi(profiledir, profile, label):
         ipxeout.close()
     subprocess.check_call(
         ['/opt/confluent/bin/dir2img', '{0}/boot'.format(profiledir),
-         '{0}/boot.img'.format(profiledir)], preexec_fn=relax_umask)
+         '{0}/boot.img'.format(profiledir), profname], preexec_fn=relax_umask)
 
 
 def update_boot_linux(profiledir, profile, label):
+    profname = os.path.basename(profiledir)
     kernelargs = profile.get('kernelargs', '')
     grubcfg = "set timeout=5\nmenuentry '"
     grubcfg += label
@@ -162,7 +182,7 @@ def update_boot_linux(profiledir, profile, label):
         ipxeout.close()
     subprocess.check_call(
         ['/opt/confluent/bin/dir2img', '{0}/boot'.format(profiledir),
-         '{0}/boot.img'.format(profiledir)], preexec_fn=relax_umask)
+         '{0}/boot.img'.format(profiledir), profname], preexec_fn=relax_umask)
 
 
 def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist=None):
@@ -198,21 +218,43 @@ def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist
         callback({'progress': float(sizedone) / float(totalsize)})
 
 
-def extract_file(filepath, flags=0, callback=lambda x: None, imginfo=(), extractlist=None):
+def extract_file(archfile, flags=0, callback=lambda x: None, imginfo=(), extractlist=None):
     """Extracts an archive from a file into the current directory."""
     totalsize = 0
     for img in imginfo:
         if not imginfo[img]:
             continue
         totalsize += imginfo[img]
-    with libarchive.file_reader(filepath) as archive:
+    archfile.seek(0)
+    with libarchive.fd_reader(archfile.fileno()) as archive:
         extract_entries(archive, flags, callback, totalsize, extractlist)
 
 
+def check_alma(isoinfo):
+    ver = None
+    arch = None
+    cat = None
+    for entry in isoinfo[0]:
+        if 'almalinux-release-8' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            cat = 'el8'
+            break
+    else:
+        return None
+    if arch == 'noarch' and '.discinfo' in isoinfo[1]:
+        prodinfo = isoinfo[1]['.discinfo']
+        arch = prodinfo.split(b'\n')[2]
+        if not isinstance(arch, str):
+            arch = arch.decode('utf-8')
+    return {'name': 'alma-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': cat}
+
+
 def check_centos(isoinfo):
     ver = None
     arch = None
     cat = None
+    isstream = ''
     for entry in isoinfo[0]:
         if 'centos-release-7' in entry:
             dotsplit = entry.split('.')
@@ -225,9 +267,25 @@ def check_centos(isoinfo):
             arch = entry.split('.')[-2]
             cat = 'el8'
             break
+        elif 'centos-stream-release-8' in entry:
+            ver = entry.split('-')[3]
+            arch = entry.split('.')[-2]
+            cat = 'el8'
+            isstream = '_stream'
+            break
+        elif 'centos-linux-release-8' in entry:
+            ver = entry.split('-')[3]
+            arch = entry.split('.')[-2]
+            cat = 'el8'
+            break
     else:
         return None
-    return {'name': 'centos-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': cat}
+    if arch == 'noarch' and '.discinfo' in isoinfo[1]:
+        prodinfo = isoinfo[1]['.discinfo']
+        arch = prodinfo.split(b'\n')[2]
+        if not isinstance(arch, str):
+            arch = arch.decode('utf-8')
+    return {'name': 'centos{2}-{0}-{1}'.format(ver, arch, isstream), 'method': EXTRACT, 'category': cat}
 
 def check_esxi(isoinfo):
     if '.DISCINFO' not in isoinfo[1]:
@@ -324,9 +382,26 @@ def check_sles(isoinfo):
     return None
 
 
+def _priv_check_oraclelinux(isoinfo):
+    ver = None
+    arch = None
+    for entry in isoinfo[0]:
+        if 'oraclelinux-release-' in entry and 'release-el7' not in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            break
+    else:
+        return None
+    major = ver.split('.', 1)[0]
+    return {'name': 'oraclelinux-{0}-{1}'.format(ver, arch), 'method': EXTRACT,
+            'category': 'el{0}'.format(major)}
+
 def check_rhel(isoinfo):
     ver = None
     arch = None
+    isoracle = _priv_check_oraclelinux(isoinfo)
+    if isoracle:
+        return isoracle
     for entry in isoinfo[0]:
         if 'redhat-release-7' in entry:
             dotsplit = entry.split('.')
@@ -366,10 +441,10 @@ def check_rhel(isoinfo):
     return {'name': 'rhel-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': 'el{0}'.format(major)}
 
 
-def scan_iso(filename):
+def scan_iso(archive):
     filesizes = {}
     filecontents = {}
-    with libarchive.file_reader(filename) as reader:
+    with libarchive.fd_reader(archive.fileno()) as reader:
         for ent in reader:
             if str(ent).endswith('TRANS.TBL'):
                 continue
@@ -382,35 +457,41 @@ def scan_iso(filename):
     return filesizes, filecontents
 
 
-def fingerprint(filename):
-    with open(filename, 'rb') as archive:
-        header = archive.read(32768)
-        archive.seek(32769)
-        if archive.read(6) == b'CD001\x01':
-            # ISO image
-            isoinfo = scan_iso(filename)
-            name = None
-            for fun in globals():
-                if fun.startswith('check_'):
-                    name = globals()[fun](isoinfo)
-                    if name:
-                        return name, isoinfo[0]
-            return None
-        else:
-            sum = hashlib.sha256(header)
-            if sum.digest() in HEADERSUMS:
-                archive.seek(32768)
+def fingerprint(archive):
+    header = archive.read(32768)
+    archive.seek(32769)
+    if archive.read(6) == b'CD001\x01':
+        # ISO image
+        archive.seek(0)
+        isoinfo = scan_iso(archive)
+        archive.seek(0)
+        name = None
+        for fun in globals():
+            if fun.startswith('check_'):
+                name = globals()[fun](isoinfo)
+                if name:
+                    return name, isoinfo[0]
+        return None
+    else:
+        sum = hashlib.sha256(header)
+        if sum.digest() in HEADERSUMS:
+            archive.seek(32768)
+            chunk = archive.read(32768)
+            while chunk:
+                sum.update(chunk)
                 chunk = archive.read(32768)
-                while chunk:
-                    sum.update(chunk)
-                    chunk = archive.read(32768)
-                imginfo = HASHPRINTS.get(sum.hexdigest(), None)
-                if imginfo:
-                    return imginfo, None
+            imginfo = HASHPRINTS.get(sum.hexdigest(), None)
+            if imginfo:
+                return imginfo, None
 
 
-def import_image(filename, callback, backend=False):
-    identity = fingerprint(filename)
+def import_image(filename, callback, backend=False, mfd=None):
+    if mfd:
+        archive = os.fdopen(int(mfd), 'rb')
+    else:
+        archive = open(filename, 'rb')
+    archive.seek(0)
+    identity = fingerprint(archive)
     if not identity:
         return -1
     identity, imginfo = identity
@@ -426,11 +507,13 @@ def import_image(filename, callback, backend=False):
         print('Importing OS to ' + targpath + ':')
     printit({'progress': 0.0})
     if EXTRACT & identity['method']:
-        extract_file(filename, callback=callback, imginfo=imginfo, extractlist=identity.get('extractlist', None))
+        extract_file(archive, callback=callback, imginfo=imginfo, extractlist=identity.get('extractlist', None))
     if COPY & identity['method']:
         basename = identity.get('copyto', os.path.basename(filename))
         targpath = os.path.join(targpath, basename)
-        shutil.copyfile(filename, targpath)
+        archive.seek(0)
+        with open(targpath, 'wb') as targ:
+            shutil.copyfileobj(archive, targ)
     with open(targpath + '/distinfo.yaml', 'w') as distinfo:
         distinfo.write(yaml.dump(identity, default_flow_style=False))
     if 'subname' in identity:
@@ -507,10 +590,15 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname,
 
 class MediaImporter(object):
 
-    def __init__(self, media):
+    def __init__(self, media, cfm=None):
         self.worker = None
         self.profiles = []
-        identity = fingerprint(media)
+        medfile = None
+        if cfm and media in cfm.clientfiles:
+            medfile = cfm.clientfiles[media]
+        else:
+            medfile = open(media, 'rb')
+        identity = fingerprint(medfile)
         if not identity:
             raise exc.InvalidArgumentException('Unsupported Media')
         self.percent = 0.0
@@ -537,6 +625,7 @@ class MediaImporter(object):
         if os.path.exists(self.targpath):
             raise Exception('{0} already exists'.format(self.targpath))
         self.filename = os.path.abspath(media)
+        self.medfile = medfile
         self.importer = eventlet.spawn(self.importmedia)
 
     def stop(self):
@@ -549,10 +638,11 @@ class MediaImporter(object):
 
     def importmedia(self):
         os.environ['PYTHONPATH'] = ':'.join(sys.path)
+        os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
         with open(os.devnull, 'w') as devnull:
             self.worker = subprocess.Popen(
                 [sys.executable, __file__, self.filename, '-b'],
-                stdin=devnull, stdout=subprocess.PIPE)
+                stdin=devnull, stdout=subprocess.PIPE, close_fds=False)
         wkr = self.worker
         currline = b''
         while wkr.poll() is None:
@@ -597,6 +687,7 @@ def get_importing_status(importkey):
 if __name__ == '__main__':
     os.umask(0o022)
     if len(sys.argv) > 2:
-        sys.exit(import_image(sys.argv[1], callback=printit, backend=True))
+        mfd = os.environ.get('CONFLUENT_MEDIAFD', None)
+        sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd))
     else:
         sys.exit(import_image(sys.argv[1], callback=printit))

confluent_server/confluent/plugins/configuration/attributes.py

@@ -87,7 +87,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                     desc = ''
             if 'value' in currattr or 'expression' in currattr:
                 yield msg.Attributes(kv={attribute: currattr}, desc=desc)
-            elif 'cryptvalue' in currattr:
+            elif 'cryptvalue' in currattr or 'hashvalue' in currattr:
                 yield msg.CryptedAttributes(
                     kv={attribute: currattr},
                     desc=desc)

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -110,7 +110,7 @@ def exithandler():
 
 atexit.register(exithandler)
 
-_ipmiworkers = greenpool.GreenPool()
+_ipmiworkers = greenpool.GreenPool(128)
 
 _ipmithread = None
 _ipmiwaiters = []
@@ -462,6 +462,7 @@ persistent_ipmicmds = {}
 class IpmiHandler(object):
     def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
                  realop):
+        self.cfm = cfg
         self.invmap = {}
         self.output = output
         self.sensorcategory = None
@@ -589,7 +590,7 @@ class IpmiHandler(object):
     def handle_update(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                     self.inputdata.nodefile(self.node), self.tenant,
-                                    bank=self.inputdata.bank)
+                                    bank=self.inputdata.bank, configmanager=self.cfm)
         self.output.put(
             msg.CreatedResource(
                 'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
@@ -598,7 +599,7 @@ class IpmiHandler(object):
     def handle_media_upload(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                      self.inputdata.nodefile(self.node), self.tenant,
-                                     type='mediaupload')
+                                     type='mediaupload', configmanager=self.cfm)
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
@@ -1524,14 +1525,21 @@ class IpmiHandler(object):
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
             filename = self.inputdata.nodefile(self.node)
-            if not os.access(filename, os.R_OK):
+            datfile = None
+            if filename in self.cfm.clientfiles:
+                cf = self.cfm.clientfiles[filename]
+                datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+            if datfile is None and not os.access(filename, os.R_OK):
                 errstr =  ('{0} is not readable by confluent on {1} '
                            '(ensure confluent user or group can access file '
                            'and parent directories)').format(
                                filename, socket.gethostname())
                 self.output.put(msg.ConfluentNodeError(self.node, errstr))
                 return
-            self.ipmicmd.apply_license(filename)
+            try:
+                self.ipmicmd.apply_license(filename, data=datfile)
+            finally:
+                datfile.close()
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
             i = 1

confluent_server/confluent/plugins/hardwaremanagement/redfish.py

@@ -348,6 +348,7 @@ persistent_ipmicmds = {}
 class IpmiHandler(object):
     def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
                  realop):
+        self.cfm = cfg
         self.sensormap = {}
         self.invmap = {}
         self.output = output
@@ -453,7 +454,8 @@ class IpmiHandler(object):
     def handle_update(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                     self.inputdata.nodefile(self.node), self.tenant,
-                                    bank=self.inputdata.bank)
+                                    bank=self.inputdata.bank,
+                                    configmanager=self.cfm)
         self.output.put(
             msg.CreatedResource(
                 'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
@@ -462,7 +464,7 @@ class IpmiHandler(object):
     def handle_media_upload(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                      self.inputdata.nodefile(self.node), self.tenant,
-                                     type='mediaupload')
+                                     type='mediaupload', configmanager=self.cfm)
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
@@ -1368,14 +1370,21 @@ class IpmiHandler(object):
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
             filename = self.inputdata.nodefile(self.node)
-            if not os.access(filename, os.R_OK):
+            datfile = None
+            if filename in self.cfm.clientfiles:
+                cf = self.cfm.clientfiles[filename]
+                datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+            if datfile is None and not os.access(filename, os.R_OK):
                 errstr =  ('{0} is not readable by confluent on {1} '
                            '(ensure confluent user or group can access file '
                            'and parent directories)').format(
                                filename, socket.gethostname())
                 self.output.put(msg.ConfluentNodeError(self.node, errstr))
                 return
-            self.ipmicmd.apply_license(self.inputdata.nodefile(self.node))
+            try:
+                self.ipmicmd.apply_license(filename, data=datfile)
+            finally:
+                datfile.close()
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
             i = 1