Fix up build process for vtbufferd

The tmt library had a bug with ECH with cursor position.

Flesh out the dump_vt to translate the attributes to VT codes
for the buffer to look the same as the attributes indicate.

.gitignore

@@ -1,7 +1,6 @@
 *.pyc
 .*.
 confluent_client/man/man*
-confluent_client/doc/man
 .vscode
 .*.sw*
 .sw*

confluent_client/bin/confetty

@@ -948,6 +948,8 @@ def main():
                             clearpowermessage = False
                         try:
                             sys.stdout.write(data)
+                        except UnicodeEncodeError:
+                            sys.stdout.buffer.write(data.encode('utf8'))
                         except IOError:  # Some times circumstances are bad
                             # resort to byte at a time...
                             for d in data:

confluent_client/bin/nodeconfig

@@ -278,7 +278,6 @@ if setmode:
             rcode |= client.printerror(fr)
             for node in fr.get('databynode', []):
                 r = fr['databynode'][node]
-                rcode |= client.printerror(r, node)
                 if 'value' not in r:
                     continue
                 keyval = r['value']

confluent_client/bin/nodediscover

@@ -263,7 +263,11 @@ def list_matching_macs(options, session, node=None, checknode=True):
         return [options.mac.replace(':', '-')]
     else:
         path += 'by-mac/'
-        return [x['item']['href'] for x in session.read(path)]
+        ret = []
+        for x in session.read(path):
+            if 'item' in x and 'href' in x['item']:
+                ret.append(x['item']['href'])
+        return ret
 
 def assign_discovery(options, session, needid=True):
     abort = False

confluent_client/bin/nodefirmware

@@ -109,6 +109,12 @@ def update_firmware(session, filename):
     if options.backup:
         upargs['bank'] = 'backup'
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -149,7 +155,6 @@ def show_firmware(session):
             if 'databynode' not in res:
                 continue
             for node in res['databynode']:
-                exitcode |= client.printerror(res['databynode'][node], node)
                 if 'firmware' not in res['databynode'][node]:
                     continue
                 for inv in res['databynode'][node]['firmware']:

confluent_client/bin/nodelicense

@@ -69,6 +69,12 @@ def install_license(session, filename):
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, instargs):
         for node in res.get('databynode', []):
             if 'error' in res['databynode'][node]:

confluent_client/bin/nodemedia

@@ -111,6 +111,12 @@ def upload_media(noderange, media):
     resource = '/noderange/{0}/media/uploads/'.format(noderange)
     upargs = {'filename': filename}
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     nodeurls = {}
     for res in session.create(resource, upargs):
         if 'created' not in res:
@@ -187,5 +193,7 @@ def main():
         argparser.print_help()
         sys.exit(1)
     handler(noderange, media)
+
 if __name__ == '__main__':
     main()
+    sys.exit(exitcode)

confluent_client/bin/nodesetboot

@@ -32,7 +32,7 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|floppy]')
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd|usb|floppy]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')

confluent_client/bin/nodestorage

@@ -186,7 +186,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',

confluent_client/bin/nodesupport

@@ -116,7 +116,7 @@ class OptParser(optparse.OptionParser):
 def main():
     argparser = OptParser(
         usage="Usage: %prog <noderange> servicedata "
-              "<filename>",
+              "<directory or filename>",
         epilog='\nservicedata will save service data to the given '
                'directory.  It is saved to the location on the relevant '
                'management server (the confluent server if running remote, '

confluent_client/bin/stats

@@ -37,6 +37,8 @@ except ImportError:
 
 def plot(gui, output, plotdata, bins):
     import matplotlib as mpl
+    if gui and mpl.get_backend() == 'agg':
+        sys.stderr.write('Error: No GUI backend available and -g specified!\n')
     if not gui:
         mpl.use('Agg')
     import matplotlib.pyplot as plt

confluent_client/confluent/client.py

@@ -46,6 +46,13 @@ try:
 except NameError:
     pass
 
+
+class NestedDict(dict):
+    def __missing__(self, key):
+        value = self[key] = type(self)()
+        return value
+
+
 def stringify(instr):
     # Normalize unicode and bytes to 'str', correcting for
     # current python version
@@ -154,6 +161,7 @@ class Command(object):
         self._prevkeyname = None
         self.connection = None
         self._currnoderange = None
+        self.unixdomain = False
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
                 self.serverloc = os.environ['CONFLUENT_HOST']
@@ -163,11 +171,13 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+            self.unixdomain = True
         elif self.serverloc == '/var/run/confluent/api.sock':
             raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
-        tlvdata.recv(self.connection)
+        self.protversion = int(tlvdata.recv(self.connection).split(
+            b'--')[1].strip()[1:])
         authdata = tlvdata.recv(self.connection)
         if authdata['authpassed'] == 1:
             self.authenticated = True
@@ -178,6 +188,13 @@ class Command(object):
             passphrase = os.environ['CONFLUENT_PASSPHRASE']
             self.authenticate(username, passphrase)
 
+    def add_file(self, name, handle, mode):
+        if self.protversion < 3:
+            raise Exception('Not supported with connected confluent server')
+        if not self.unixdomain:
+            raise Exception('Can only add a file to a unix domain connection')
+        tlvdata.send(self.connection, {'filename': name, 'mode': mode}, handle)
+
     def authenticate(self, username, password):
         tlvdata.send(self.connection,
                      {'username': username, 'password': password})
@@ -417,7 +434,7 @@ def send_request(operation, path, server, parameters=None):
         result = tlvdata.recv(server)
 
 
-def attrrequested(attr, attrlist, seenattributes):
+def attrrequested(attr, attrlist, seenattributes, node=None):
     for candidate in attrlist:
         truename = candidate
         if candidate.startswith('hm'):
@@ -425,10 +442,16 @@ def attrrequested(attr, attrlist, seenattributes):
         if candidate in _attraliases:
             candidate = _attraliases[candidate]
         if fnmatch.fnmatch(attr.lower(), candidate.lower()):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = True
             return True
         elif attr.lower().startswith(candidate.lower() + '.'):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = 1
             return True
     return False
 
@@ -444,13 +467,15 @@ def _sort_attrib(k):
 
 def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
-    seenattributes = set([])
+    seenattributes = NestedDict()
+    allnodes = set([])
     for res in session.read(path):
         if 'error' in res:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
+            allnodes.add(node)
             for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
@@ -458,7 +483,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attr == 'errorcode':
                     exitcode |= val
                     continue
-                seenattributes.add(attr)
+                seenattributes[node][attr] = True
                 if rename:
                     printattr = rename.get(attr, attr)
                 else:
@@ -466,7 +491,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attrprefix:
                     printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
-                if show_attr(attr, requestargs, seenattributes, options):
+                if show_attr(attr, requestargs, seenattributes, options, node):
                     if 'value' in currattr:
                         if currattr['value'] is not None:
                             val = currattr['value']
@@ -547,23 +572,39 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                                 except TypeError:
                                     pass
                         cprint(attrout)
+    somematched = set([])
+    printmissing = set([])
+    badnodes = NestedDict()
     if not exitcode:
         if requestargs:
             for attr in requestargs:
-                if attr not in seenattributes:
-                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
-                    exitcode = 1
+                for node in allnodes:
+                    if attr in seenattributes[node]:
+                        somematched.add(attr)
+                    else:
+                        badnodes[node][attr] = True
+                        exitcode = 1
+        for node in sortutil.natural_sort(badnodes):
+            for attr in badnodes[node]:
+                if attr in somematched:
+                    sys.stderr.write(
+                        'Error: {0} matches no valid value for {1}\n'.format(
+                            attr, node))
+                else:
+                    printmissing.add(attr)
+        for missing in printmissing:
+            sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
     return exitcode
 
 
-def show_attr(attr, requestargs, seenattributes, options):
+def show_attr(attr, requestargs, seenattributes, options, node):
     try:
         reverse = options.exclude
     except AttributeError:
         reverse = False
     if requestargs is None or requestargs == []:
         return True
-    processattr = attrrequested(attr, requestargs, seenattributes)
+    processattr = attrrequested(attr, requestargs, seenattributes, node)
     if reverse:
         processattr = not processattr
     return processattr

confluent_client/confluent/tlv.py

@@ -17,4 +17,4 @@
 
 
 class Types(object):
-    text, json = range(2)
+    text, json, filehandle = range(3)

confluent_client/confluent/tlvdata.py

@@ -15,9 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import array
+import ctypes
+import ctypes.util
 import confluent.tlv as tlv
+import eventlet.green.socket as socket
+import eventlet.green.select as select
 from datetime import datetime
 import json
+import os
 import struct
 
 try:
@@ -30,6 +36,72 @@ try:
 except NameError:
     pass
 
+class iovec(ctypes.Structure):   # from uio.h
+    _fields_ = [('iov_base', ctypes.c_void_p),
+                ('iov_len', ctypes.c_size_t)]
+
+
+iovec_ptr = ctypes.POINTER(iovec)
+
+
+class cmsghdr(ctypes.Structure):  # also from bits/socket.h
+    _fields_ = [('cmsg_len', ctypes.c_size_t),
+                ('cmsg_level', ctypes.c_int),
+                ('cmsg_type', ctypes.c_int)]
+
+    @classmethod
+    def init_data(cls, cmsg_len, cmsg_level, cmsg_type, cmsg_data):
+        Data = ctypes.c_ubyte * ctypes.sizeof(cmsg_data)
+        class _flexhdr(ctypes.Structure):
+            _fields_ = cls._fields_ + [('cmsg_data', Data)]
+
+        datab = Data(*bytearray(cmsg_data))
+        return _flexhdr(cmsg_len=cmsg_len, cmsg_level=cmsg_level,
+                        cmsg_type=cmsg_type, cmsg_data=datab)
+
+
+def CMSG_LEN(length):
+    sizeof_cmshdr = ctypes.sizeof(cmsghdr)
+    return ctypes.c_size_t(CMSG_ALIGN(sizeof_cmshdr).value + length)
+
+
+SCM_RIGHTS = 1
+
+
+class msghdr(ctypes.Structure):  # from bits/socket.h
+    _fields_ = [('msg_name', ctypes.c_void_p),
+                ('msg_namelen', ctypes.c_uint),
+                ('msg_iov', ctypes.POINTER(iovec)),
+                ('msg_iovlen', ctypes.c_size_t),
+                ('msg_control', ctypes.c_void_p),
+                ('msg_controllen', ctypes.c_size_t),
+                ('msg_flags', ctypes.c_int)]
+
+
+def CMSG_ALIGN(length):  # bits/socket.h
+    ret = (length + ctypes.sizeof(ctypes.c_size_t) - 1
+           & ~(ctypes.sizeof(ctypes.c_size_t) - 1))
+    return ctypes.c_size_t(ret)
+
+
+def CMSG_SPACE(length):  # bits/socket.h
+    ret = CMSG_ALIGN(length).value + CMSG_ALIGN(ctypes.sizeof(cmsghdr)).value
+    return ctypes.c_size_t(ret)
+
+
+class ClientFile(object):
+    def __init__(self, name, mode, fd):
+        self.fileobject = os.fdopen(fd, mode)
+        self.filename = name
+
+libc = ctypes.CDLL(ctypes.util.find_library('c'))
+recvmsg = libc.recvmsg
+recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+recvmsg.restype = ctypes.c_int
+sendmsg = libc.sendmsg
+sendmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+sendmsg.restype = ctypes.c_size_t
+
 def decodestr(value):
     ret = None
     try:
@@ -65,7 +137,7 @@ def _unicode_list(currlist):
             _unicode_list(currlist[i])
 
 
-def send(handle, data):
+def send(handle, data, filehandle=None):
     if isinstance(data, unicode):
         try:
             data = data.encode('utf-8')
@@ -93,9 +165,27 @@ def send(handle, data):
         if tl > 16777215:
             raise Exception("JSON data exceeds protocol limits")
         # xor in the type (0b1 << 24)
-        tl |= 16777216
-        handle.sendall(struct.pack("!I", tl))
-        handle.sendall(sdata)
+        if filehandle is None:
+            tl |= 16777216
+            handle.sendall(struct.pack("!I", tl))
+            handle.sendall(sdata)
+        else:
+            tl |= (2 << 24)
+            handle.sendall(struct.pack("!I", tl))
+            cdtype = ctypes.c_ubyte * len(sdata)
+            cdata = cdtype.from_buffer(bytearray(sdata))
+            ciov = iovec(iov_base=ctypes.addressof(cdata),
+                         iov_len=ctypes.c_size_t(ctypes.sizeof(cdata)))
+            fd = ctypes.c_int(filehandle)
+            cmh = cmsghdr.init_data(
+                cmsg_len=CMSG_LEN(
+                    ctypes.sizeof(fd)), cmsg_level=socket.SOL_SOCKET,
+                    cmsg_type=SCM_RIGHTS, cmsg_data=fd)
+            mh = msghdr(msg_name=None, msg_len=0, msg_iov=iovec_ptr(ciov),
+                        msg_iovlen=1, msg_control=ctypes.addressof(cmh),
+                        msg_controllen=ctypes.c_size_t(ctypes.sizeof(cmh)))
+            sendmsg(handle.fileno(), mh, 0)
+
 
 def recvall(handle, size):
     rd = handle.recv(size)
@@ -125,12 +215,42 @@ def recv(handle):
     datatype = (tl & 2130706432) >> 24  # grab 7 bits from near beginning
     if dlen == 0:
         return None
-    data = handle.recv(dlen)
-    while len(data) < dlen:
-        ndata = handle.recv(dlen - len(data))
-        if not ndata:
-            raise Exception("Error reading data")
-        data += ndata
+    if datatype == tlv.Types.filehandle:
+        filehandles = array.array('i')
+        rawbuffer = bytearray(2048)
+        pkttype = ctypes.c_ubyte * 2048
+        data = pkttype.from_buffer(rawbuffer)
+        cmsgsize = CMSG_SPACE(ctypes.sizeof(ctypes.c_int)).value
+        cmsgarr = bytearray(cmsgsize)
+        cmtype = ctypes.c_ubyte * cmsgsize
+        cmsg = cmtype.from_buffer(cmsgarr)
+        cmsg.cmsg_level = socket.SOL_SOCKET
+        cmsg.cmsg_type = SCM_RIGHTS
+        cmsg.cmsg_len = CMSG_LEN(ctypes.sizeof(ctypes.c_int))
+        iov = iovec()
+        iov.iov_base = ctypes.addressof(data)
+        iov.iov_len = 2048
+        msg = msghdr()
+        msg.msg_iov = ctypes.pointer(iov)
+        msg.msg_iovlen = 1
+        msg.msg_control = ctypes.addressof(cmsg)
+        msg.msg_controllen = ctypes.sizeof(cmsg)
+        select.select([handle], [], [])
+        i = recvmsg(handle.fileno(), ctypes.pointer(msg), 0)
+        cdata = cmsgarr[CMSG_LEN(0).value:]
+        data = rawbuffer[:i]
+        if cmsg.cmsg_level == socket.SOL_SOCKET and cmsg.cmsg_type == SCM_RIGHTS:
+            filehandles.fromstring(bytes(
+                cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
+        data = json.loads(bytes(data))
+        return ClientFile(data['filename'], data['mode'], filehandles[0])
+    else:
+        data = handle.recv(dlen)
+        while len(data) < dlen:
+            ndata = handle.recv(dlen - len(data))
+            if not ndata:
+                raise Exception("Error reading data")
+            data += ndata
     if datatype == tlv.Types.text:
         return data
     elif datatype == tlv.Types.json:

confluent_client/confluent_env.sh

@@ -91,7 +91,7 @@ _confluent_nodeidentify_completion()
 
 _confluent_nodesetboot_completion()
 {
-    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy -h -b -p")
+    COMP_CANDIDATES=("default,cd,network,setup,hd,floppy,usb -h -b -p")
     _confluent_generic_completion
 }
 

confluent_client/doc/man/collate.ronn

@@ -3,7 +3,7 @@ collate(1) -- Organize text input by node
 
 ## SYNOPSIS
 
-`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r] [-l lognametemplate]`
+`<other command> | collate [-a] [-b] [-d] [-w] [-g] [-s] [-c] [-r] [-l lognametemplate]`
 
 ## DESCRIPTION
 
@@ -21,19 +21,23 @@ node and group names sorted alphanumerically.
    when output differs along a telling group boundary.  For example, if
    output suggests a large number of nodes are unreachable, abbreviate
    showing 'rack1' as being unreachable may make more obvious a possible cause.
+   
+* `-b`, `--base`:
+   Use given node as reference for comparison when using
+   -d, instead of using the most common result      
 
 * `-d`, `--diff`:
    Express all but the most common result group in terms of diff from
    the most common result group
-
-* `-l`, `--log`:
-   Save output per node to individual log files, replacing {node} in the name
-   with the nodename of each
-
+   
 * `-w`, `--watch`:
    Update results dynamically as data becomes available, rather than
    waiting    for the command to fully complete.
-
+   
+* `-g`, `--groupcount`:
+   Show count of output groups rather than the actual
+   output
+   
 * `-s`, `--skipcommon`:
    Suppress printing of the most common result text group.  This is used to
    focus on stray output against a well known and expected result.
@@ -45,6 +49,10 @@ node and group names sorted alphanumerically.
  * `-r`, `--reverse`:
     Rather than starting with most common to least common, start with
     the least common and print the most common last.
+    
+* `-l`, `--log`:
+   Save output per node to individual log files, replacing {node} in the name
+   with the nodename of each
    
 ## EXAMPLES
  * Organizing power state of multiple nodes:  

confluent_client/doc/man/collective.ronn

@@ -6,7 +6,8 @@ collective(1) -- Check and manage a confluent collective
 `collective invite <server>`  
 `collective join <server> [-i TOKEN]`  
 `collective show`  
-`collective gencert`  
+`collective gencert`
+`collective delete`
 
 
 ## DESCRIPTION

confluent_client/doc/man/confetty.ronn

@@ -15,6 +15,18 @@ be mostly an aid for developing client software, with
 day to day administration generally being easier with
 the various function specific commands.
 
+## OPTIONS
+
+* `-s`, `--server`:
+  Confluent instance to connect to
+
+* `-c`, `--control`:
+  Path to offer terminal control
+
+* `-m`, `--mintime`:
+  Minimum time to run or else pause for input (used to
+  keep a terminal from closing quickly on error)
+
 ## COMMANDS
 
 The CLI may be navigated by shell commands and some other

confluent_client/doc/man/confluentdbutil.ronn

@@ -3,8 +3,7 @@ confluentdbutil(8) -- Backup or restore confluent database
 
 ## SYNOPSIS
 
-
-`confluentdbutil [options] <dump|restore> <path>`
+`confluentdbutil [options] [dump|restore] <path>`
 
 ## DESCRIPTION
 
@@ -19,8 +18,25 @@ the json files (password protected, removed from the files, or unprotected).
 * `-p`, `--password`:
   If specified, information such as usernames and passwords will be encrypted
   using the given password.
+  
+* `-i`, `--interactivepassword`:
+  Prompt for password.  
+  
 * `-r`, `--redact`:
   Indicates to replace usernames and passwords with a dummy string rather
   than included.
+  
 * `-u`, `--unprotected`:
-  The keys.json file will include the encryption keys without any protection.
+  The keys.json file will include the encryption keys without any protection.
+  
+* `-s`, `--skipkeys`:
+  This specifies to dump the encrypted data without
+  dumping the keys needed to decrypt it.  This is
+  suitable for an automated incremental backup, where an
+  earlier password protected dump has a protected
+  keys.json file, and only the protected data is needed.
+  keys do not change and as such they do not require
+  incremental backup.
+  
+  * `-h`, `--help`:
+  Show help message and exit

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -40,12 +40,25 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
 
 * `-b`, `--blame`:
   Annotate inherited and expression based attributes to show their base value.
+  
 * `-c`, `--clear`:
-  Clear specified nodeattributes
+  Clear specified nodeattributes.
+  
 * `-e`, `--environment`:
-  Set specified attributes based on exported environment variable of matching name.  Environment variable names may be lower case or all upper case.  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  Set specified attributes based on exported environment variable of matching name. 
+  Environment variable names may be lower case or all upper case. 
+  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  
 * `-p`, `--prompt`:
-  Request interactive prompting to provide values rather than the command line or environment variables.
+  Request interactive prompting to provide values rather than the command line
+  or environment variables.
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Prompt if trying to set attributes on more than
+  specified number of nodes.
+
+* `-h`, `--help`:
+  Show help message and exit
 
 ## EXAMPLES
 * Listing matching nodes of a simple noderange:

confluent_client/doc/man/nodebmcreset.ronn

@@ -9,6 +9,15 @@ nodebmcreset(8) -- Reset management controller
 
 `nodebmcreset` allows you to reset the management controller of the specified noderange
 
+## OPTIONS
+
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Number of nodes to affect before prompting for
+  confirmation
+  
+* `-h`, `--help`:
+  Show help message and exit  
+
 ## EXAMPLES:
 
 * Reset the management controller for nodes n1 through n4:

confluent_client/doc/man/nodeboot.ronn

@@ -3,8 +3,7 @@ nodeboot(8) -- Reboot a confluent node to a specific device
 
 ## SYNOPSIS
 
-`nodeboot <noderange>`  
-`nodeboot [options] <noderange>` [default|cd|network|setup|hd]  
+`nodeboot [options] <noderange> [default|cd|network|setup|hd]`  
 
 ## DESCRIPTION
 
@@ -25,6 +24,13 @@ device without inducing a reboot, see the `nodesetboot` command.
 * `-p`, `--persist`:
   For a system that supports it, mark the boot override to persist rather than
   be a one time change.  Many systems do not support this functionality.
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Specify a maximum number of nodes to boot, prompting
+  if over the threshold
+  
+* `-h`, `--help`:
+  Show help message and exit  
 
 * `default`:
   Request a normal default boot with no particular device override

confluent_client/doc/man/nodeconfig.ronn

@@ -3,8 +3,7 @@ nodeconfig(8) -- Show or change node configuration
 
 ## SYNOPSIS
 
-`nodeconfig <noderange> [options] [<configuration>..]`  
-`nodeconfig <noderange> [options] [<configuration=value>..]`  
+`nodeconfig [options] <noderange> [setting|setting=value]`
 
 ## DESCRIPTION
 
@@ -14,6 +13,12 @@ running configuration on the node firmware.  Calling without '=' will show the
 current value, and '=' will change the value.  Network information can be
 given as a node expression, as documented in the man page for nodeattribexpressions(5).
 
+Note that when using nodeconfig to submit changes, it will exit when the change
+is accepted, but the endpoint may not have fully processed it. Doing a show
+immediately after doing a set may reflect older information. Also, if changing
+BIOS/UEFI settings, the change may appear in output, but generally won't
+actually be in effect until a reboot.
+
 ## OPTIONS
 
 * `-c`, `--comparedefault`:
@@ -21,6 +26,13 @@ given as a node expression, as documented in the man page for nodeattribexpressi
   no configuration values are specified, it will show only those that differ.
   If combined with `-x`, will show all differing values except those indicated
   by `-x`
+  
+* `-b`, `--batch`:
+   Provide arguments as lines of a file, rather than the command line.  
+   
+* `-d`, `--detail`:
+  Provide detailed data as available.  This can include help text and valid
+  values for a setting.   
 
 * `-e`, `--extra`:
    Read settings that are generally not needed, but may be slow to retrieve.
@@ -31,18 +43,21 @@ given as a node expression, as documented in the man page for nodeattribexpressi
   Rather than listing only the specified configuration parameters, list all
   attributes except for the specified ones
 
-* `-d`, `--detail`:
-  Provide detailed data as available.  This can include help text and valid
-  values for a setting.
+* `-a`, `--advanced`:
+  Include advanced settings, which are normally not intended to be used
+  without direction from the relevant server vendor.
 
 * `-r`, `--restoredefault`:
   Request that the specified component of the targeted nodes will have its
   configuration reset to default.  Currently the only component implemented
   is uefi.
 
-* `-b`, '--batch':
-   Provide arguments as lines of a file, rather than the command line.
-
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Specify a maximum number of nodes to configure, prompting if over 
+  the threshold  
+  
+* `-h`, `--help`:
+  Show help message and exit  
 
 ## EXAMPLES
 * Showing the current IP configuration of noderange BMC/IMM/XCC:

confluent_client/doc/man/nodedeploy.ronn

@@ -3,9 +3,7 @@ nodedeploy(8) -- Request preparation and/or initiating a node deployment
 
 ## SYNOPSIS
 
-`nodedeploy <noderange>`  
-`nodedeploy <noderange> -c`  
-`nodedeploy <noderange> [-n] [-p] <name of profile>`  
+`nodedeploy [-h] [-c] [-n] [-p] [-m MAXNODES] <noderange> [profile]`  
 
 ## DESCRIPTION
 
@@ -19,7 +17,7 @@ deployment status.
 
 ## OPTIONS
 
-* '-c', '--clear`:
+* `-c`, `--clear`:
   Remove any pending deployment action
 
 * `-n`, `--network`:
@@ -28,6 +26,12 @@ deployment status.
 * `-p`, `--prepareonly`:
   Prepare the network services for deployment, but do not interact with BMCs. This is intended for scenarios where
   the boot device control and server restart will be handled outside of confluent.
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`: 
+  Specifiy a maximum nodes to be deployed.
+  
+* `-h`, `--help`:
+  Show help message and exit  
 
 ## EXAMPLES
 * Begin the instalalation of a profile of CentOS 8.2:  

confluent_client/doc/man/nodeeventlog.ronn

@@ -3,14 +3,22 @@ nodeeventlog(8) -- Pull eventlog from confluent nodes
 
 ## SYNOPSIS
 
-`nodeeventlog <noderange>`  
-`nodeeventlog <noderange> [clear]`
+`nodeeventlog [options] <noderange> [clear]`  
 
 ## DESCRIPTION
 
 `nodeeventlog` pulls and optionally clears the event log from the requested
 noderange.
 
+## OPTIONS
+
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Specify a maximum number of nodes to clear if clearing log, prompting if 
+  over the threshold 
+  
+* `-h`, `--help`:
+  Show help message and exit  
+
 ## EXAMPLES
 * Pull the event log from n2 and n3:
   `# nodeeventlog n2,n3`  

confluent_client/doc/man/nodefirmware.ronn

@@ -3,9 +3,7 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 
 ## SYNOPSIS
 
-`nodefirmware <noderange>`  
-`nodefirmware <noderange> list|<components>|core`  
-`nodefirmware <noderange> update [--backup] <filename>`  
+`nodefirmware <noderange> [list][update [--backup <file>]]|[<components>]`
 
 ## DESCRIPTION
 
@@ -26,6 +24,18 @@ the out of band facilities.  Firmware updates can end in one of three states:
 * `pending`:  The firmware update process has completed, but the firmware will not be active until the relevant component next resets.  Generally speaking, for UEFI update the system will need a reboot, and for BMC updates, the `nodebmcreset` command will begin the process to activate the firmware.
 * `complete`:  The firmware update process has completed and activation has proceeded.  Note that while the activation process has commenced, the component may still be in the process of rebooting when nodefirmware exits.
 
+## OPTIONS
+
+* `-b`, `--backup`:
+  Target a backup bank rather than primary
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  When updating, prompt if more than the specified number of servers will 
+  be affected  
+  
+* `-h`, `--help`:
+  Show help message and exit    
+
 ## EXAMPLES
 
 * Pull firmware from a node:

confluent_client/doc/man/nodegroupattrib.ronn.tmpl

@@ -25,8 +25,17 @@ the attributes are set on the node versus a group to which a node belongs.
 
 ## OPTIONS
 
+* `-b`, `--blame`:
+  Show information about how attributes inherited
+
+* `-e`, `--environment`:
+  Set attributes, but from environment variable of same name
+
 * `-c`, `--clear`:
   Clear specified nodeattributes.
+  
+* `-p`, `--prompt`:
+  Prompt for attribute values interactively
 
 ## EXAMPLES
 

confluent_client/doc/man/nodelicense.ronn

@@ -15,3 +15,7 @@ to that directory. The `install` command will take the specified filename and in
 argument may be of the form  xcc_fod_0034_7X21{id.serial}.key to have the serial number substituted
 to allow unique licenses to be specified in a single command.
 
+## OPTIONS
+
+* `-m`, `--maxnodes`:
+  Specify a maximum number of nodes to delete licenses from, prompting if over the threshold

confluent_client/doc/man/nodemedia.ronn

@@ -34,8 +34,8 @@ the host platform attempts.
 
 ## OPTIONS
 
-* `-c`:
-  Specify the maximum number of instances to run concurrently.
+* `-h`, `--help`:
+  Show help message and exit
 
 ## EXAMPLES
 * Listing currently mounted media:

confluent_client/doc/man/nodepower.ronn

@@ -30,6 +30,9 @@ off will not react to this request.
 
 * `-p`, '--showprevious':
    Show previous power state for all directives that may change power state.
+   
+* `-h`, `--help`:
+  Show help message and exit   
 
 ## EXAMPLES
 * Get power state of nodes n1 through n4:

confluent_client/doc/man/noderemove.ronn

@@ -9,6 +9,11 @@ noderemove(8) -- Remove nodes from the confluent management service
 
 `noderemove` simply removes the given noderange from the confluent database.
 
+## OPTIONS
+
+* `-m`, `--maxnodes`:
+    Specify a maximum number of nodes to delete, prompting if over the
+    threshold
 
 ## EXAMPLES
 

confluent_client/doc/man/nodereseat.ronn

@@ -11,9 +11,13 @@ nodereseat(8) -- Request a reseat of a node
 node's slot.  This should be equivalent to removing the system entirely from
 the chassis and putting it back in, but without actually having to do so.
 
+## OPTIONS
+
+* `-m`, `--maxnodes`:
+    Specify a maximum number of nodes to reseat, prompting if over the threshold
+
 ## EXAMPLES
 
 * Reseating the node `s1`:
     `# nodereseat s1`  
     `s1: Reseat successful`  
-

confluent_client/doc/man/nodersync.ronn

@@ -12,10 +12,12 @@ noderange. This will present progress as percentage for all nodes.
 
 ## OPTIONS
 
-* `-m`:
-  Specify maximum number of nodes for noderange max.
-
-* `-c`:
+* `-c`, `-f`, `--count`:
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
-  active count completes.
+  active count completes. 
+  
+* `-m`, `--maxnodes`:
+  Specify a maximum number of nodes to run rsync to, prompting if over the
+  threshold
+  

confluent_client/doc/man/noderun.ronn

@@ -13,6 +13,21 @@ nodeattribexpressions(5) are expanded prior to execution of the command.  For
 noderun, the commands are locally executed.  To execute commands on the nodes
 themselves, see nodeshell(8).
 
+## OPTIONS
+
+* `-f`, `-c`, `--count`:
+  Number of commands to run at a time
+  
+* `-n`, `--nonodeprefix`:
+  Do not prefix output with node names
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Specify a maximum number of nodes to run the command with, prompting if over
+  the threshold
+  
+* `-h`, `--help`:
+  Show help message and exit
+
 ## EXAMPLES
 
 * Run ping against nodes n1 through n4:

confluent_client/doc/man/nodesetboot.ronn

@@ -2,9 +2,8 @@ nodesetboot(8) -- Check or set next boot device for noderange
 ====================================================
 
 ## SYNOPSIS
-
-`nodesetboot <noderange>`  
-`nodesetboot [options] <noderange> [default|cd|network|setup|hd|floppy]`
+  
+`nodesetboot [options] <noderange> [default|cd|network|setup|hd|usb|floppy]`
 
 ## DESCRIPTION
 
@@ -21,6 +20,8 @@ to use `nodesetboot <noderange> setup` and then initiate a reboot from within
 the operating system with ssh or similar rather than using the remote hardware
 control.
 
+Running the command with no target queries the current setting.
+
 ## OPTIONS
 
 * `-b`, `--bios`:
@@ -33,6 +34,13 @@ control.
 
 * `-u`, `--uefi`:
   This flag does nothing, it is for command compatibility with xCAT's rsetboot
+  
+* `-m MAXNODES`, `--maxnodes=MAXNODES`:
+  Specify a maximum number of nodes to modify next boot device, prompting if
+  over the threshold
+  
+* `-h`, `--help`:
+  Show help message and exit  
 
 * `default`:
   Request a normal default boot with no particular device override
@@ -46,6 +54,10 @@ control.
   Request boot from floppy.  Generally speaking firmware uses this to mean a USB
   flash drive or similar (whether virtual or physical).
 
+* `usb`:
+  Request boot from usb.  Generally speaking firmware uses this to mean a USB
+  flash drive or similar (whether virtual or physical).
+
 * `network`:
   Request boot to network
   

confluent_client/doc/man/nodeshell.ronn

@@ -3,7 +3,7 @@ nodeshell(8) -- Execute command on many nodes in a noderange through ssh
 
 ## SYNOPSIS
 
-`nodeshell <noderange> [options] <command to execute on each node>`
+`nodeshell [options] <noderange> <command to execute on each node>`
 
 ## DESCRIPTION
 
@@ -13,10 +13,20 @@ nodeattribexpressions(5).  `nodeshell` provides stdout as stdout and stderr
 as stderr, unlike psh which combines all stdout and stderr into stdout.
 
 ## OPTIONS
+  
+* `-c COUNT`, `-f COUNT`, `--count=COUNT`
+  Specify the maximum number of instances to run concurrently
 
-* `-c`:
-  Specify the maximum number of instances to run concurrently.
+* `-m MAXNODES`, `--maxnodes=MAXNODES`
+  Specify a maximum number of nodes to run remote ssh command to, prompting
+  if over the threshold
+ 
+* `-n`, `--nonodeprefix`
+  Do not prefix output with node names  
 
+* `-p PORT`, `--port=PORT`
+  Specify a custom port for ssh
+       
 ## EXAMPLES
 
 * Running `echo hi` on for nodes:

confluent_client/doc/man/nodestorage.ronn

@@ -3,12 +3,15 @@ nodestorage(8) -- Examine/Modify storage configuration of a node
 
 ## SYNOPSIS
 
-`nodestorage <noderange> [show|create|delete] [options]`  
+`nodestorage <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]`  
 
 ## DESCRIPTION
 
 `nodestorage` provides access to the remote storage configuration of
-the noderange.
+the noderange. The `show` subcommand will show current storage configuration,
+`create` can be used to create new arrays or volumes, `delete` can be used to
+remove volumes and arrays, and `diskset` can modify the usage of disks
+indicated by `-d` to either be `unconfigured`, `jbod`, or `hotspare`.
 
 ## OPTIONS
 * `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
@@ -29,6 +32,15 @@ the noderange.
   volumes, or selecting a volume for delete.  Default
   behavior is to use implementation provided default
 
+* `-z` **STRIPSIZES**, `--stripsizes`=**STRIPSIZES**:
+  Comma separated list of stripsizes to use when creating volumes.
+  This value is in kilobytes.  The default behavior is to allow the
+  storage controller to decide
+
+* `-m` **MAXNODES**, `--maxnodes`=**MAXNODES**:
+  Specify a maximum number of nodes to configure storage on, prompting
+  if over the threshold
+
 ## EXAMPLES
 
 * Deleting the volume `somedata`:

confluent_client/doc/man/nodesupport.ronn

@@ -19,6 +19,12 @@ running in collective mode.
 Note that due to vendor filename requirements, any filename may have vendor
 specific suffixes added to any file produced.
 
+## OPTIONS
+
+* `-m`, `--maxnodes`:
+  Specify a maximum number of nodes to download diagnostic data from, prompting
+    if over the threshold
+
 ## EXAMPLES
 
 * Download support data from a single node to a specific filename  

confluent_client/doc/man/osdeploy.ronn

@@ -5,7 +5,7 @@ osdeploy(8) --- Configure general OS deployment facilities of confluent
 
 `osdeploy import <iso>`  
 `osdeploy updateboot <profile>`  
-`osdeploy initialize [-h] [-g] [-u] [-s] [-k] [-t] [-p] [-i] [-l]`  
+`osdeploy initialize [-h] [-g] [-u] [-s] [-k] [-t] [-p] [-i] [-l] [-a]`  
 
 ## DESCRIPTION
 

confluent_client/doc/man/stats.ronn

@@ -0,0 +1,49 @@
+stats(8) -- Common basic statistics on typical numeric data in output
+==============================
+
+## SYNOPSIS
+
+`<other command> | stats [-c N] [-d D] [-x|-g|-t|-o image.png] [-s N] [-v] [-b N]
+
+## DESCRIPTION
+
+The **stats** command helps analyze common numerical data such as performance numbers
+or temperatures or any other numerical value.
+
+By default it looks for the last numerical output on the first line to identify the numerical column
+and analyze that number. This can be overriden by **-c COLUMN** to indicate a column. By default,
+whitespace and commas are treated to delimit columns, and **-d DELIMITER** can be used to override.
+
+By default it outputs basic statistics, but a histogram is available either text or through X11 output
+or sixel or output to an image file depending on whether **-x**, **-g**, **-t*, or **-o image.png** is
+used.
+
+## OPTIONS
+
+ * `-c N`:
+   Select column number. Defaults to last column that appears numeric
+
+ * `-d D`:
+   Specify a custom column delimiter
+
+ * `-x`:
+     Output in Sixel format (supported by mlterm and PuTTY, among others)
+
+ * `-g`:
+   Try to open histogram as an X window
+
+ * `-t`:
+   Output histogram as bars rendered by =
+
+ * `-o image.png`:
+   Write graphical histogram to image.png.
+
+ * `-s N`:
+   Ignore specified number of lines as header content before processing numbers
+
+ * `-v`:
+   Treat value before : on each line as a label, and show which labels belong to which histogram buckets.
+
+ * `-b N`:
+   Specify a custom number of buckets for histogram. The default is 10.
+   

confluent_osdeploy/common/opt/confluent/bin/apiclient

@@ -100,6 +100,9 @@ class HTTPSClient(client.HTTPConnection, object):
         self.sock = ctx.wrap_socket(psock, server_hostname=host)
 
     def grab_url(self, url, data=None, returnrsp=False):
+        return self.grab_url_with_status(url, data, returnrsp)[1]
+
+    def grab_url_with_status(self, url, data=None, returnrsp=False):
         if data:
             method = 'POST'
         else:
@@ -108,9 +111,9 @@ class HTTPSClient(client.HTTPConnection, object):
         rsp = self.getresponse()
         if rsp.status >= 200 and rsp.status < 300:
             if returnrsp:
-                return rsp
+                return rsp.status, rsp
             else:
-                return rsp.read()
+                return rsp.status, rsp.read()
         raise Exception(rsp.read())
 
 if __name__ == '__main__':
@@ -121,8 +124,26 @@ if __name__ == '__main__':
     if len(sys.argv) == 1:
         HTTPSClient()
         sys.exit(0)
-    if sys.argv[-2] == '-o':
-        with open(sys.argv[3], 'wb') as outf:
+    try:
+        outbin = sys.argv.index('-o')
+        sys.argv.pop(outbin)
+        outbin = sys.argv.pop(outbin)
+    except ValueError:
+        outbin = None
+    try:
+        waitfor = sys.argv.index('-w')
+        sys.argv.pop(waitfor)
+        waitfor = int(sys.argv.pop(waitfor))
+    except ValueError:
+        waitfor = None
+    try:
+        data = sys.argv.index('-d')
+        sys.argv.pop(data)
+        data = sys.argv.pop(data)
+    except ValueError:
+        data = None
+    if outbin:
+        with open(outbin, 'wb') as outf:
             reader = HTTPSClient(json=json).grab_url(
                 sys.argv[1], data, returnrsp=True)
             chunk = reader.read(16384)
@@ -130,6 +151,13 @@ if __name__ == '__main__':
                 outf.write(chunk)
                 chunk = reader.read(16384)
         sys.exit(0)
-    if os.path.exists(sys.argv[-1]):
+    if len(sys.argv) > 2 and os.path.exists(sys.argv[-1]):
         data = open(sys.argv[-1]).read()
-    sys.stdout.write(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())
+    if waitfor:
+        client = HTTPSClient(json=json)
+        status = 201
+        while status != waitfor:
+            status, rsp = client.grab_url_with_status(sys.argv[1], data)
+            sys.stdout.write(rsp.decode())
+    else:
+        sys.stdout.write(HTTPSClient(json=json).grab_url(sys.argv[1], data).decode())

confluent_osdeploy/el7/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -12,7 +12,7 @@ sed -i 's/install::/install:*:/' /sysroot/etc/shadow
 sed -i 's/root::/root:*:/' /sysroot/etc/shadow
 mkdir -p /sysroot/root/.ssh
 #chmod 700 /sysroot/root/.ssh
-cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+cat /ssh/*pubkey > /sysroot/root/.ssh/authorized_keys
 #chmod 600 /sysroot/root/.ssh/authorized_keys
 mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do

confluent_osdeploy/el7/profiles/default/scripts/infiniband/mofed.post

@@ -9,12 +9,11 @@ if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
     # two lines to use the .iso instead of the tgz packaging
     #fetch_remote infiniband/mofed.iso
     #mkdir MLNX_OFED
-    #mount -o loop ofed.iso MLNX_OFED
+    #mount -o loop infiniband/mofed.iso MLNX_OFED
     fetch_remote infiniband/mofed.tgz
-    tar xf mofed.tgz
+    tar xf infiniband/mofed.tgz
     # The rest is common between tar and iso
     cd MLNX_OFED*
-    mount -o loop ofed
     ./mlnxofedinstall --force
 fi
 

confluent_osdeploy/el7/profiles/default/scripts/post.sh

@@ -30,6 +30,10 @@ run_remote_python add_local_repositories
 # run_remote_python will use the appropriate python interpreter path to run the specified script
 # A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
+# This will induce server side processing of the syncfile contents if
+# present
+run_remote_python syncfileclient
+
 # run_remote example.sh
 # run_remote_python example.py
 run_remote post.custom

confluent_osdeploy/el7/profiles/default/scripts/syncfileclient

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger:
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid <= minid or uid >= maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/initqueue/01-confluent.sh

@@ -109,5 +109,13 @@ done < /etc/confluent/confluent.deploycfg
 if [ -e /lib/nm-lib.sh ]; then
     . /lib/nm-lib.sh
     nm_generate_connections
+    if [[ "$ifname" == ib* ]]; then
+        sed -i s/type=ethernet/type=infiniband/ /run/NetworkManager/system-connections/$ifname.nmconnection
+        if ! grep '\[infiniband\]' /run/NetworkManager/system-connections/$ifname.nmconnection > /dev/null; then
+            echo >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo '[infiniband]' >> /run/NetworkManager/system-connections/$ifname.nmconnection
+            echo transport-mode=datagram >> /run/NetworkManager/system-connections/$ifname.nmconnection
+        fi
+    fi
 fi
 

confluent_osdeploy/el8/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -13,7 +13,7 @@ sed -i 's/install::/install:*:/' /sysroot/etc/shadow
 sed -i 's/root::/root:*:/' /sysroot/etc/shadow
 mkdir -p /sysroot/root/.ssh
 chmod 700 /sysroot/root/.ssh
-cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+cat /ssh/*pubkey > /sysroot/root/.ssh/authorized_keys
 chmod 600 /sysroot/root/.ssh/authorized_keys
 mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do

confluent_osdeploy/el8/profiles/default/ansible/firstboot.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/el8/profiles/default/ansible/post.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot && \

confluent_osdeploy/el8/profiles/default/scripts/firstboot.sh

@@ -19,6 +19,9 @@ run_remote firstboot.custom
 run_remote_parts firstboot
 
 
+# Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+run_remote_config firstboot
+
 curl -X POST -d 'status: complete' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
 systemctl disable firstboot
 rm /etc/systemd/system/firstboot.service

confluent_osdeploy/el8/profiles/default/scripts/functions

@@ -73,3 +73,21 @@ run_remote_python() {
     cd - > /dev/null
     return $retcode
 }
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $mgr under profile $profile
+    /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+

confluent_osdeploy/el8/profiles/default/scripts/infiniband/mofed.post

@@ -11,7 +11,7 @@ if lspci -d 15b3:: -n |grep 15b3 > /dev/null; then
     #mkdir MLNX_OFED
     #mount -o loop ofed.iso MLNX_OFED
     fetch_remote infiniband/mofed.tgz
-    tar xf mofed.tgz
+    tar xf infiniband/mofed.tgz
     # The rest is common between tar and iso
     cd MLNX_OFED*
     mount -o loop ofed

confluent_osdeploy/el8/profiles/default/scripts/post.sh

@@ -8,8 +8,6 @@ chmod og-rwx /etc/confluent/*
 export mgr profile nodename
 . /etc/confluent/functions
 
-curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
-
 
 if [ -f /tmp/cryptboot ]; then
     run_remote tpm_luks.sh
@@ -30,9 +28,17 @@ run_remote_python add_local_repositories
 # run_remote_python will use the appropriate python interpreter path to run the specified script
 # A post.custom is provided to more conveniently hold customizations, see the post.custom file.
 
+# This will induce server side processing of the syncfile contents if
+# present
+run_remote_python syncfileclient
+
 # run_remote example.sh
 # run_remote_python example.py
 run_remote post.custom
 
 # Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh
 run_remote_parts post
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post
+curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/el8/profiles/default/scripts/prechroot.sh

@@ -14,6 +14,10 @@ chmod -R og-rwx /mnt/sysimage/etc/confluent
 cp /tmp/functions /mnt/sysimage/etc/confluent/
 . /tmp/functions
 cp /tmp/cryptboot /mnt/sysimage/tmp/
+echo Port 2222 >> /etc/ssh/sshd_config.anaconda
+echo Match LocalPort 22 >> /etc/ssh/sshd_config.anaconda
+echo "    ChrootDirectory /mnt/sysimage" >> /etc/ssh/sshd_config.anaconda
+kill -HUP $(cat /run/sshd.pid)
 
 # Preserve the ssh setup work done for the installer
 # by copying into the target system and setting up

confluent_osdeploy/el8/profiles/default/scripts/syncfileclient

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger:
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid <= minid or uid >= maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -7,8 +7,22 @@ echo $uuid > /sys/devices/virtual/dmi/id/product_uuid
 mkdir -p /etc/confluent
 localcli network firewall unload
 touch /etc/confluent/confluent.info
+begin=$(date +%s)
 while ! grep NODENAME /etc/confluent/confluent.info; do
+    echo "Searching for deployment service on local network..."
     /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+    esxcfg-nics -l
+    if [ $(date +%s) -gt $((begin + 90)) ]; then 
+        if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
+            chvt 1
+            sleep 1
+            chvt 2
+            /etc/init.d/ESXShell start
+        fi
+        echo "Unable to locate a deployment system on the local network, verify network connectivity"
+        echo "A debug session has been made available on Alt-F1"
+        sleep 30
+    fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
 mgr=$(grep MANAGER: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')

confluent_osdeploy/esxi7/profiles/hypervisor/kickstart

@@ -4,3 +4,8 @@ install --firstdisk --overwritevmfs
 %include /tmp/ksnet
 %include /tmp/rootpw
 reboot
+%post --interpreter=busybox
+localcli network firewall unload
+STATUP=$(mktemp)
+echo '{"status": "complete"}' > $STATUP
+/opt/confluent/bin/apiclient /confluent-api/self/updatestatus $STATUP

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -1,5 +1,8 @@
 #!/usr/bin/python
+import re
+import subprocess
 
+uplinkmatch = re.compile('^\s*Uplinks:\s*(.*)')
 nodename = None
 for inf in open('/etc/confluent/confluent.info', 'r').read().split('\n'):
     if inf.startswith('NODENAME: '):
@@ -24,8 +27,15 @@ for line in deploycfg:
     else:
         nslist=False
 cfg['nameservers'] = ','.join(nameservers)
-
+vswinfo = subprocess.check_output(['localcli', 'network', 'vswitch', 'standard', 'list']).decode('utf8')
+vmnic = None
+for info in vswinfo.split('\n'):
+    upinfo = uplinkmatch.match(info)
+    if upinfo:
+        vmnic = upinfo.group(1)
 netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if vmnic:
+    netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':
     netline += ' --ip={0} --netmask={1}'.format(cfg['ipv4_address'], cfg['ipv4_netmask'])
     if cfg.get('ipv4_gateway', 'null') not in (None, '', 'null'):

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -8,14 +8,16 @@ if ! grep console= /proc/cmdline >& /dev/null; then
     echo $autocons > /tmp/01-autocons.devnode
     if [ ! -z "$autocons" ]; then
         echo "Using $(cat /tmp/01-autocons.conf)"
-        tmux a <> $autocons >&0 2>&1 &
+        (while :; do tmux a <> $autocons >&0 2>&1; done) &
+        (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
     else
-        tmux a <> /dev/console >&0 2>&1 &
+        (while :; do tmux a <> /dev/console >&0 2>&1; done) &
     fi
 else
-    tmux a <> /dev/console >&0 2>&1 &
-    tmux a <> /dev/tty1 >&0 2>&1 &
+    (while :; do tmux a <> /dev/console >&0 2>&1; done) &
+    (while :; do TERM=linux tmux a <> /dev/tty1 >&0 2>&1; done) &
 fi
+(while :; do TERM=linux tmux <> /dev/tty2 >&0 2>&1; done) &
 echo -n "udevd: "
 /usr/lib/systemd/systemd-udevd --daemon
 echo -n "Loading drivers..."
@@ -30,11 +32,12 @@ echo "done"
 cat > /etc/ssh/sshd_config << EOF
 Port 22
 Port 3389
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
 PermitRootLogin yes
 AuthorizedKeysFile      .ssh/authorized_keys
 EOF
 mkdir ~/.ssh
-cat /ssh/*.rootpubkey > ~/.ssh/authorized_keys
+cat /ssh/*pubkey > ~/.ssh/authorized_keys 2>/dev/null
 cat /tls/*.pem > /etc/confluent/ca.pem
 mkdir -p /etc/pki/tls/certs
 cat /tls/*.pem > /etc/pki/tls/certs/ca-bundle.crt

confluent_osdeploy/genesis/profiles/default/ansible/onboot.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/genesis/profiles/default/profile.yaml

@@ -1,2 +1,2 @@
 label: Genesis
-kernelarags: quiet
+kernelargs: quiet

confluent_osdeploy/genesis/profiles/default/scripts/configbmc

@@ -411,7 +411,7 @@ def main():
         dotwait()
     while awaitprefix and not check_subnet(s, bmccfg['prefixv4'], channel):
         dotwait()
-    while awaitprefix and not check_gateway(s, bmccfg['bmcgw'], channel):
+    while awaitgw and not check_gateway(s, bmccfg['bmcgw'], channel):
         dotwait()
     sys.stdout.write('done\n')
     sys.stdout.flush()

confluent_osdeploy/genesis/profiles/default/scripts/functions

@@ -63,3 +63,20 @@ run_remote_python() {
     cd - > /dev/null
     return $retcode
 }
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $mgr under profile $profile
+    /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    /usr/libexec/platform-python $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}

confluent_osdeploy/genesis/profiles/default/scripts/onboot.sh

@@ -6,6 +6,13 @@
 # run_remote and run_remote_python are available to download scripts and
 # execute them.
 
+# This will induce server side processing of the syncfile contents if
+# present
+run_remote_python syncfileclient
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/onboot.d/
+run_remote_config onboot
+
 # This is an example to request the BMC be configured on the network
 # according to how confluent has things configured:
 # run_remote_python configbmc -c

confluent_osdeploy/genesis/profiles/default/scripts/syncfileclient

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger:
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid <= minid or uid >= maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/rhvh4/initramfs/usr/lib/dracut/hooks/pre-pivot/01-confluent.sh

@@ -12,7 +12,7 @@ sed -i 's/install::/install:*:/' /sysroot/etc/shadow
 sed -i 's/root::/root:*:/' /sysroot/etc/shadow
 mkdir -p /sysroot/root/.ssh
 #chmod 700 /sysroot/root/.ssh
-cat /ssh/*.rootpubkey > /sysroot/root/.ssh/authorized_keys
+cat /ssh/*pubkey > /sysroot/root/.ssh/authorized_keys
 #chmod 600 /sysroot/root/.ssh/authorized_keys
 mkdir -p /sysroot/etc/ssh/
 for i in /ssh/*.ca; do

confluent_osdeploy/suse15/initramfs/opt/confluent/bin/suseagent

@@ -1,7 +1,7 @@
 #!/bin/bash
 echo "Installing certificates"
 echo '<authorized_keys xmlns="http://www.suse.com/1.0/yast2ns" xmlns:config="http://www.suse.com/1.0/configns" config:type="list">' > /tmp/rootkeys.xml
-for pub in /ssh/*.rootpubkey; do
+for pub in /ssh/*pubkey; do
     echo '<listentry>'$(cat $pub)'</listentry>' >> /tmp/rootkeys.xml
 done
 echo '</authorized_keys>' >> /tmp/rootkeys.xml

confluent_osdeploy/suse15/profiles/hpc/ansible/firstboot.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/suse15/profiles/hpc/ansible/post.d/README.txt

@@ -0,0 +1,25 @@
+Ansible playbooks ending in .yml or .yaml that are placed into this directory will be executed at the
+appropriate phase of the install process.
+
+The 'hosts' may be omitted, and if included will be ignored, replaced with the host that is specifically
+requesting the playbooks be executed.
+
+Also, the playbooks will be executed on the deployment server. Hence it may be slower in aggregate than
+running content under scripts/ which ask much less of the deployment server
+
+Here is an example of what a playbook would look like broadly:
+
+- name: Example
+  gather_facts: no
+  tasks:
+       - name: Example1
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.4 test1
+           create: yes
+       - name: Example2
+         lineinfile:
+           path: /etc/hosts
+           line: 1.2.3.5 test2
+           create: yes
+

confluent_osdeploy/suse15/profiles/hpc/autoyast

@@ -39,6 +39,7 @@ dynamic behavior and replace with static configuration.
         <pattern>base</pattern>
       </patterns>
       <packages config:type="list">
+        <package>python3</package>
         <package>openssl</package>
         <package>chrony</package>
         <package>rsync</package>

confluent_osdeploy/suse15/profiles/hpc/scripts/firstboot.sh

@@ -10,4 +10,11 @@ apikey=$(cat /etc/confluent/confluent.apikey)
 . /etc/confluent/functions
 
 run_remote firstboot.custom
+
+# Firstboot scripts may be placed into firstboot.d, e.g. firstboot.d/01-firstaction.sh, firstboot.d/02-secondaction.sh
+run_remote_parts firstboot
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/firstboot.d/
+run_remote_config firstboot
+
 curl --capath /etc/confluent/tls -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -f -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/suse15/profiles/hpc/scripts/functions

@@ -33,3 +33,20 @@ run_remote_python() {
     cd - > /dev/null
     return $retcode
 }
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $mgr under profile $profile
+    python3 $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    python3 $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}

confluent_osdeploy/suse15/profiles/hpc/scripts/post.sh

@@ -21,7 +21,17 @@ chmod og-rwx /etc/confluent/*
 export mgr profile nodename
 . /etc/confluent/functions
 
+# This will induce server side processing of the syncfile contents if
+# present
+run_remote_python syncfileclient
+
 run_remote post.custom
 
+# Also, scripts may be placed into 'post.d', e.g. post.d/01-runfirst.sh, post.d/02-runsecond.sh
+run_remote_parts post
+
+# Induce execution of remote configuration, e.g. ansible plays in ansible/post.d/
+run_remote_config post
+
 curl -X POST -d 'status: staged' -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" https://$mgr/confluent-api/self/updatestatus
 

confluent_osdeploy/suse15/profiles/hpc/scripts/pre.sh

@@ -16,7 +16,8 @@ if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "n
 fi
 
 mkdir ~/.ssh
-cat /ssh/*.rootpubkey > ~/.ssh/authorized_keys
+cat /ssh/*pubkey > ~/.ssh/authorized_keys 2>/dev/null
+
 ssh-keygen -A
 for i in  /etc/ssh/ssh_host*key.pub; do
     certname=${i/.pub/-cert.pub}

confluent_osdeploy/suse15/profiles/hpc/scripts/prechroot.sh

@@ -17,5 +17,14 @@ cp -a /tls /mnt/etc/confluent/
 cp -a /tls/* /mnt/var/lib/ca-certificates/openssl
 cp -a /tls/* /mnt/var/lib/ca-certificates/pem
 cp -a /tls/*.pem /mnt/etc/pki/trust/anchors
+cat /tls/*.pem > /mnt/etc/confluent/ca.pem
+cp /opt/confluent/bin/apiclient /mnt/etc/confluent
 
 run_remote setupssh.sh
+
+echo Port 22 >> /etc/ssh/sshd_config
+echo Port 2222 >> /etc/ssh/sshd_config
+echo Match LocalPort 22 >> /etc/ssh/sshd_config
+echo "    ChrootDirectory /mnt" >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+

confluent_osdeploy/suse15/profiles/hpc/scripts/syncfileclient

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger:
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid <= minid or uid >= maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/firstboot.sh

@@ -11,4 +11,8 @@ apikey=$(cat /etc/confluent/confluent.apikey)
 mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg |awk '{print $2}')
 hostnamectl set-hostname $(grep ^NODENAME: /etc/confluent/confluent.info | awk '{print $2}')
 touch /etc/cloud/cloud-init.disabled
+source /etc/confluent/functions
+
+run_remote_parts firstboot
+run_remote_config firstboot
 curl --capath /etc/confluent/tls -f -H "CONFLUENT_NODENAME: $nodename" -H "CONFLUENT_APIKEY: $apikey" -X POST -d "status: complete" https://$mgr/confluent-api/self/updatestatus

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/functions

@@ -0,0 +1,93 @@
+function set_confluent_vars() {
+    if [ -z "$mgr" ]; then
+        mgr=$(grep ^deploy_server: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+    if [ -z "$profile" ]; then
+        profile=$(grep ^profile: /etc/confluent/confluent.deploycfg | sed -e 's/[^ ]*: //')
+    fi
+}
+
+fetch_remote() {
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $1 failed to download; return 1; fi
+}
+
+run_remote_parts() {
+    scriptlist=$(python3 /etc/confluent/apiclient /confluent-api/self/scriptlist/$1|sed -e 's/^- //')
+    for script in $scriptlist; do
+        run_remote $1.d/$script
+    done
+}
+
+run_remote() {
+    requestedcmd="'$*'"
+    curlargs=""
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    set_confluent_vars
+    echo
+    echo '---------------------------------------------------------------------------'
+    echo Running $requestedcmd from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo $requestedcmd failed to download; return 1; fi
+    chmod +x $1
+    cmd=$1
+    #if [ -x /usr/bin/chcon ]; then
+    #    chcon system_u:object_r:bin_t:s0 $cmd
+    #fi
+    shift
+    ./$cmd $*
+    retcode=$?
+    echo "$requestedcmd exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_python() {
+    echo
+    set_confluent_vars
+    if [ -f /etc/confluent/ca.pem ]; then
+        curlargs=" --cacert /etc/confluent/ca.pem"
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Running python script "'$*'" from https://$mgr/confluent-public/os/$profile/scripts/
+    tmpdir=$(mktemp -d)
+    echo Executing in $tmpdir
+    cd $tmpdir
+    mkdir -p $(dirname $1)
+    curl -f -sS $curlargs https://$mgr/confluent-public/os/$profile/scripts/$1 > $1
+    if [ $? != 0 ]; then echo "'$*'" failed to download; return 1; fi
+    python3 $*
+    retcode=$?
+    echo "'$*' exited with code $retcode"
+    cd - > /dev/null
+    return $retcode
+}
+
+run_remote_config() {
+    echo
+    set_confluent_vars
+    apiclient=/opt/confluent/bin/apiclient
+    if [ -f /etc/confluent/apiclient ]; then
+        apiclient=/etc/confluent/apiclient
+    fi
+    echo '---------------------------------------------------------------------------'
+    echo Requesting to run remote configuration for "'$*'" from $mgr under profile $profile
+    python3 $apiclient /confluent-api/self/remoteconfig/"$*" -d {}
+    python3 $apiclient /confluent-api/self/remoteconfig/status -w 204
+    echo
+    echo 'Completed remote configuration'
+    echo '---------------------------------------------------------------------------'
+    return
+}
+

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/post.sh

@@ -21,6 +21,8 @@ echo '    EnableSSHKeysign yes' >> $sshconf
 echo '    HostbasedKeyTypes *ed25519*' >> $sshconf
 
 curl -f https://$mgr/confluent-public/os/$profile/scripts/firstboot.sh > /target/etc/confluent/firstboot.sh
+curl -f https://$mgr/confluent-public/os/$profile/scripts/functions > /target/etc/confluent/functions
+source /target/etc/confluent/functions
 chmod +x /target/etc/confluent/firstboot.sh
 cp /tmp/allnodes /target/root/.shosts
 cp /tmp/allnodes /target/etc/ssh/shosts.equiv
@@ -44,11 +46,30 @@ kargs=$(curl https://$mgr/confluent-public/os/$profile/profile.yaml | grep ^inst
 if [ ! -z "$kargs" ]; then
     sed -i 's/GRUB_CMDLINE_LINUX="\([^"]*\)"/GRUB_CMDLINE_LINUX="\1 '"${kargs}"'"/' /target/etc/default/grub
 fi
-if [ 1 = $updategrub ]; then
-    mount -o bind /dev /target/dev
-    mount -o bind /proc /target/proc
-    mount -o bind /sys /target/sys
-    chroot /target update-grub
-    umount /target/sys /target/dev /target/proc
-fi
+mkdir -p /opt/confluent/bin
+mkdir -p /etc/confluent
+cp -a /target/etc/confluent/* /etc/confluent
+cp /custom-installation/confluent/bin/apiclient /opt/confluent/bin/
+cp /custom-installation/confluent/bin/apiclient /target/etc/confluent/
+
+mount -o bind /dev /target/dev
+mount -o bind /proc /target/proc
+mount -o bind /sys /target/sys
+if [ 1 = $updategrub ]; then
+    chroot /target update-grub
+fi
+echo "Port 22" >> /etc/ssh/sshd_config
+echo "Port 2222" >> /etc/ssh/sshd_config
+echo "Match LocalPort 22" >> /etc/ssh/sshd_config
+echo "    ChrootDirectory /target" >> /etc/ssh/sshd_config
+kill -HUP $(cat /run/sshd.pid)
+cat /target/etc/confluent/tls/*.pem > /target/etc/confluent/ca.pem
+cat /target/etc/confluent/tls/*.pem > /etc/confluent/ca.pem
+chroot /target bash -c "source /etc/confluent/functions; run_remote_python syncfileclient"
+chroot /target bash -c "source /etc/confluent/functions; run_remote_parts post"
+source /target/etc/confluent/functions
+
+run_remote_config post
+
+umount /target/sys /target/dev /target/proc
 

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/pre.sh

@@ -9,7 +9,7 @@ if [ "$cryptboot" != "" ]  && [ "$cryptboot" != "none" ] && [ "$cryptboot" != "n
 fi
 
 
-cat /custom-installation/ssh/*.rootpubkey > /root/.ssh/authorized_keys
+cat /custom-installation/ssh/*pubkey > /root/.ssh/authorized_keys
 nodename=$(grep ^NODENAME: /custom-installation/confluent/confluent.info|awk '{print $2}')
 apikey=$(cat /custom-installation/confluent/confluent.apikey)
 for pubkey in /etc/ssh/ssh_host*key.pub; do

confluent_osdeploy/ubuntu20.04/profiles/default/scripts/syncfileclient

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+import importlib
+import tempfile
+import json
+import os
+import shutil
+from importlib.machinery import SourceFileLoader
+try:
+    apiclient = SourceFileLoader('apiclient', '/opt/confluent/bin/apiclient').load_module()
+except FileNotFoundError:
+    apiclient = SourceFileLoader('apiclient', '/etc/confluent/apiclient').load_module()
+
+
+def partitionhostsline(line):
+    comment = ''
+    try:
+        cmdidx = line.index('#')
+        comment = line[cmdidx:]
+        line = line[:cmdidx].strip()
+    except ValueError:
+        pass
+    if not line:
+        return '', [], comment
+    ipaddr, names = line.split(maxsplit=1)
+    names = names.split()
+    return ipaddr, names, comment
+
+class HostMerger:
+    def __init__(self):
+        self.byip = {}
+        self.byname = {}
+        self.sourcelines = []
+        self.targlines = []
+
+    def read_source(self, sourcefile):
+        with open(sourcefile, 'r') as hfile:
+            self.sourcelines = hfile.read().split('\n')
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        for x in range(len(self.sourcelines)):
+            line = self.sourcelines[x]
+            currip, names, comment = partitionhostsline(line)
+            if currip:
+                self.byip[currip] = x
+            for name in names:
+                self.byname[name] = x
+
+    def read_target(self, targetfile):
+        with open(targetfile, 'r') as hfile:
+            lines = hfile.read().split('\n')
+            if not lines[-1]:
+                lines = lines[:-1]
+            for y in range(len(lines)):
+                line = lines[y]
+                currip, names, comment = partitionhostsline(line)
+                if currip in self.byip:
+                    x = self.byip[currip]
+                    if self.sourcelines[x] is None:
+                        # have already consumed this enntry
+                        continue
+                    self.targlines.append(self.sourcelines[x])
+                    self.sourcelines[x] = None
+                    continue
+                for name in names:
+                    if name in self.byname:
+                        x = self.byname[name]
+                        if self.sourcelines[x] is None:
+                            break
+                        self.targlines.append(self.sourcelines[x])
+                        self.sourcelines[x] = None
+                        break
+                else:
+                    self.targlines.append(line)
+
+    def write_out(self, targetfile):
+        while not self.targlines[-1]:
+            self.targlines = self.targlines[:-1]
+        while not self.sourcelines[-1]:
+            self.sourcelines = self.sourcelines[:-1]
+        with open(targetfile, 'w') as hosts:
+            for line in self.targlines:
+                hosts.write(line + '\n')
+            for line in self.sourcelines:
+                if line is not None:
+                    hosts.write(line + '\n')
+
+
+class CredMerger:
+    def __init__(self):
+        try:
+            with open('/etc/login.defs', 'r') as ldefs:
+                defs = ldefs.read().split('\n')
+        except FileNotFoundError:
+            defs = []
+        lkup = {}
+        self.discardnames = {}
+        self.shadowednames = {}
+        for line in defs:
+            try:
+                line = line[:line.index('#')]
+            except ValueError:
+                pass
+            keyval = line.split()
+            if len(keyval) < 2:
+                continue
+            lkup[keyval[0]] = keyval[1]
+        self.uidmin = int(lkup.get('UID_MIN', 1000))
+        self.uidmax = int(lkup.get('UID_MAX', 60000))
+        self.gidmin = int(lkup.get('GID_MIN', 1000))
+        self.gidmax = int(lkup.get('GID_MAX', 60000))
+        self.shadowlines = None
+
+    def read_passwd(self, source, targfile=False):
+        self.read_generic(source, self.uidmin, self.uidmax, targfile)
+
+    def read_group(self, source, targfile=False):
+        self.read_generic(source, self.gidmin, self.gidmax, targfile)
+
+    def read_generic(self, source, minid, maxid, targfile):
+        if targfile:
+            self.targdata = []
+        else:
+            self.sourcedata = []
+        with open(source, 'r') as inputfile:
+            for line in inputfile.read().split('\n'):
+                try:
+                    name, _, uid, _ = line.split(':', 3)
+                    uid = int(uid)
+                except ValueError:
+                    continue
+                if targfile:
+                    if uid <= minid or uid >= maxid:
+                        self.targdata.append(line)
+                    else:
+                        self.discardnames[name] = 1
+                else:
+                    if uid >= minid and uid <= maxid:
+                        self.sourcedata.append(line)
+
+    def read_shadow(self, source):
+        self.shadowlines = []
+        try:
+            with open(source, 'r') as inshadow:
+                for line in inshadow.read().split('\n'):
+                    try:
+                        name, _ = line.split(':' , 1)
+                    except ValueError:
+                        continue
+                    if name in self.discardnames:
+                        continue
+                    self.shadowednames[name] = 1
+                    self.shadowlines.append(line)
+        except FileNotFoundError:
+            return
+
+    def write_out(self, outfile):
+        with open(outfile, 'w') as targ:
+            for line in self.targdata:
+                targ.write(line + '\n')
+            for line in self.sourcedata:
+                targ.write(line + '\n')
+        if outfile == '/etc/passwd':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/shadow')
+            with open('/etc/shadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':', 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!:::::::\n')
+        if outfile == '/etc/group':
+            if self.shadowlines is None:
+                self.read_shadow('/etc/gshadow')
+            with open('/etc/gshadow', 'w') as shadout:
+                for line in self.shadowlines:
+                    shadout.write(line + '\n')
+                for line in self.sourcedata:
+                    name, _ = line.split(':' , 1)
+                    if name in self.shadowednames:
+                        continue
+                    shadout.write(name + ':!::\n')
+
+def synchronize():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        ac = apiclient.HTTPSClient()
+        data = json.dumps({'merge': tmpdir})
+        status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles', data)
+        if status == 202:
+            while status != 204:
+                status, rsp = ac.grab_url_with_status('/confluent-api/self/remotesyncfiles')
+                if not isinstance(rsp, str):
+                    rsp = rsp.decode('utf8')
+            pendpasswd = os.path.join(tmpdir, 'etc/passwd')
+            if os.path.exists(pendpasswd):
+                cm = CredMerger()
+                cm.read_passwd(pendpasswd, targfile=False)
+                cm.read_passwd('/etc/passwd', targfile=True)
+                cm.write_out('/etc/passwd')
+            pendgroup = os.path.join(tmpdir, 'etc/group')
+            if os.path.exists(pendgroup):
+                cm = CredMerger()
+                cm.read_group(pendgroup, targfile=False)
+                cm.read_group('/etc/group', targfile=True)
+                cm.write_out('/etc/group')
+            pendhosts = os.path.join(tmpdir, 'etc/hosts')
+            if os.path.exists(pendhosts):
+                cm = HostMerger()
+                cm.read_source(pendhosts)
+                cm.read_target('/etc/hosts')
+                cm.write_out('/etc/hosts')
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+    synchronize()

confluent_osdeploy/utils/autocons.c

@@ -1,3 +1,4 @@
+#include <errno.h>
 #include <termios.h>
 #include <sys/ioctl.h>
 #include <fcntl.h>
@@ -20,20 +21,29 @@
 
 int main(int argc, char* argv[]) {
     struct termios tty;
+    struct termios tty2;
+    struct winsize ws;
+    unsigned width, height;
     int ttyf;
-    int spcr;
+    int tmpi;
     int currspeed;
+    int flags;
     speed_t cspeed;
     char buff[128];
+    int bufflen;
+    fd_set set;
+    struct timeval timeout;
     char* offset;
     uint64_t address;
-    spcr = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
-    if (spcr < 0) {
+    bufflen = 0;
+    tmpi = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
+    if (tmpi < 0) {
         exit(0);
     }
-    if (read(spcr, buff, 80) < 80) {
+    if (read(tmpi, buff, 80) < 80) {
         exit(0);
     }
+    close(tmpi);
     if (buff[8] != 2) exit(0); //revision 2
     if (buff[36] != 0) exit(0); //16550 only
     if (buff[40] != 1) exit(0); //IO only
@@ -69,12 +79,51 @@ int main(int argc, char* argv[]) {
     }
     tcgetattr(ttyf, &tty);
     if (cspeed) {
-        cfsetospeed(&tty, B115200);
-        cfsetispeed(&tty, B115200);
+        cfsetospeed(&tty, cspeed);
+        cfsetispeed(&tty, cspeed);
     }
+    printf("%s\n", buff);
+    tcgetattr(ttyf, &tty2);
+    cfmakeraw(&tty2);
+    tcsetattr(ttyf, TCSANOW, &tty2);
+    flags = fcntl(ttyf, F_GETFL, 0);
+    fcntl(ttyf, F_SETFL, flags | O_NONBLOCK);
+    while (read(ttyf, buff, 64) > 0) {
+        // Drain any pending reads
+    }
+    timeout.tv_sec = 0;
+    timeout.tv_usec = 500000;
+    FD_ZERO(&set);
+    FD_SET(ttyf, &set);
+    write(ttyf, "\0337\033[999;999H\033[6n\0338", 18);
+    while (select(ttyf + 1, &set, NULL, NULL, &timeout) > 0) {
+        if ((tmpi = read(ttyf, buff + bufflen, 127 - bufflen)) < 0) {
+            if (errno == EAGAIN || errno == EWOULDBLOCK) {
+                continue;
+            } else {
+                break;
+            }
+        }
+        bufflen += tmpi;
+        buff[bufflen] = 0;
+        if (strchr(buff, 'R')) {
+            break;
+        }
+    }
+    fcntl(ttyf, F_SETFL, flags);
+    ws.ws_xpixel = 0;
+    ws.ws_ypixel = 0;
+    if (sscanf(buff, "\033[%u;%uR", &height, &width) == 2) {
+        ws.ws_col = width;
+        ws.ws_row = height;
+    } else {
+        ws.ws_col = 100;
+        ws.ws_row = 31;
+    }
+    if (ws.ws_col < 80) { ws.ws_col = 80; }
+    if (ws.ws_row < 24) { ws.ws_col = 24; }
+    ioctl(ttyf, TIOCSWINSZ, &ws);
     tcsetattr(ttyf, TCSANOW, &tty);
     ioctl(ttyf, TIOCCONS, 0);
-    printf("%s\n", buff);
-
 }
 

confluent_server/bin/confluentdbutil

@@ -74,7 +74,7 @@ if args[0] == 'restore':
             for targdir in os.walk('/etc/confluent'):
                 os.chown(targdir[0], owner, group)
                 for f in targdir[2]:
-                    os.chown(os.patht.join(targdir[0], f), owner, group)
+                    os.chown(os.path.join(targdir[0], f), owner, group)
     except Exception as e:
         print(str(e))
         sys.exit(1)

confluent_server/bin/osdeploy

@@ -1,5 +1,7 @@
 #!/usr/bin/python2
 
+__author__ = 'jjohnson2,bfinley'
+
 import argparse
 import glob
 import os
@@ -36,6 +38,7 @@ def main(args):
     ap = argparse.ArgumentParser(description='Manage OS deployment resources')
     sp = ap.add_subparsers(dest='command')
     wiz = sp.add_parser('initialize', help='Do OS deployment preparation')
+    wiz.add_argument('-a', help='Initialize SSH access by confluent to nodes for automation such as ansible playbook execution or syncfiles', action='store_true')
     wiz.add_argument('-g', help='Initialize a Genesis profile to boot systems into a rescue or staging environment', action='store_true')
     wiz.add_argument('-u', help='Pull in root user key for node deployment', action='store_true')
     wiz.add_argument('-s', help='Set up SSH CA for managing node to node ssh and known hosts', action='store_true')
@@ -50,7 +53,11 @@ def main(args):
         'updateboot',
         help='Push profile.yaml of the named profile data into boot assets as appropriate')
     upb.add_argument('profile', help='Profile to update boot assets')
+    osls = sp.add_parser('list', help='List OS images available for deployment')
     cmdset = ap.parse_args()
+
+    if cmdset.command == 'list':
+        return oslist()
     if cmdset.command == 'import':
         return osimport(cmdset.imagefile)
     if cmdset.command == 'initialize':
@@ -94,6 +101,8 @@ def initialize_genesis():
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/initramfs/addons.cpio')
         os.symlink('/opt/confluent/genesis/x86_64/boot/kernel',
                 '/var/lib/confluent/public/os/genesis-x86_64/boot/kernel')
+        shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/ansible/',
+                        '/var/lib/confluent/public/os/genesis-x86_64/ansible/')
         shutil.copytree('/opt/confluent/lib/osdeploy/genesis/profiles/default/scripts/',
                         '/var/lib/confluent/public/os/genesis-x86_64/scripts/')
         shutil.copyfile('/opt/confluent/lib/osdeploy/genesis/profiles/default/profile.yaml',
@@ -200,6 +209,8 @@ def initialize(cmdset):
         cmdset.l = input().strip().lower().startswith('y')
         sys.stdout.write('Update tftp directory with binaries to support PXE (-p) (y/n): ')
         cmdset.p = input().strip().lower().startswith('y')
+        sys.stdout.write('Initialize confluent ssh user key so confluent can execute remote automation (e.g. Ansible plays) (-a) (y/n): ')
+        cmdset.a = input().strip().lower().startswith('y')
         sys.stdout.write('Generate new TLS certificates for HTTP, replacing any existing certificate (-t)? (y/n): ')
         cmdset.t = input().strip().lower().startswith('y')
         if not cmdset.t:
@@ -235,6 +246,9 @@ def initialize(cmdset):
     if cmdset.s:
         didsomething = True
         sshutil.initialize_ca()
+    if cmdset.a:
+        didsomething = True
+        sshutil.initialize_root_key(True, True)
     if cmdset.p:
         install_tftp_content()
     if cmdset.l:
@@ -308,9 +322,36 @@ def updateboot(profilename):
             print(repr(rsp))
 
 
+def oslist():
+    c = client.Command()
+    print("Distributions:")
+    for rsp in c.read('/deployment/distributions'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+    print("Profiles:")
+    for rsp in c.read('/deployment/profiles'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+
 def osimport(imagefile):
     c = client.Command()
     imagefile = os.path.abspath(imagefile)
+    if c.unixdomain:
+        ofile = open(imagefile, 'rb')
+        try:
+            c.add_file(imagefile, ofile.fileno(), 'rb')
+        except Exception:
+            pass
     importing = False
     shortname = None
     for rsp in c.create('/deployment/importing/', {'filename': imagefile}):

confluent_server/confluent/certutil.py

@@ -72,7 +72,7 @@ def create_certificate(keyout=None, certout=None):
     if not keyout:
         raise Exception('Unable to locate TLS certificate path automatically')
     shortname = socket.gethostname().split('.')[0]
-    longname = socket.getfqdn()
+    longname = shortname # socket.getfqdn()
     subprocess.check_call(
         ['openssl', 'ecparam', '-name', 'secp384r1', '-genkey', '-out',
          keyout])
@@ -81,7 +81,7 @@ def create_certificate(keyout=None, certout=None):
     # there exists non-compliant clients that fail with them as IP
     san.extend(['DNS:{0}'.format(x) for x in get_ip_addresses()])
     san.append('DNS:{0}'.format(shortname))
-    san.append('DNS:{0}'.format(longname))
+    #san.append('DNS:{0}'.format(longname))
     san = ','.join(san)
     sslcfg = get_openssl_conf_location()
     tmpconfig = tempfile.mktemp()

confluent_server/confluent/collective/manager.py

@@ -76,8 +76,8 @@ def connect_to_leader(cert=None, name=None, leader=None):
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
             vers = banner.split()[2]
-            if vers != b'v2':
-                raise Exception('This instance only supports protocol 2, synchronize versions between collective members')
+            if vers not in (b'v2', b'v3'):
+                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()

confluent_server/confluent/config/configmanager.py

@@ -785,7 +785,7 @@ def commit_clear():
     # currently defined as local to each collective member
     # currently just 'autosense' which is intended to be active
     # per collective member
-    for globvar in _oldcfgstore['globals']:
+    for globvar in _oldcfgstore.get('globals', ()):
         if globvar.endswith('_key'):
             continue
         _cfgstore['globals'][globvar] = _oldcfgstore['globals'][globvar]
@@ -1138,6 +1138,7 @@ class ConfigManager(object):
         return _cfgstore['tenant'][self.tenant]
 
     def __init__(self, tenant, decrypt=False, username=None):
+        self.clientfiles = {}
         global _cfgstore
         with _initlock:
             if _cfgstore is None:
@@ -1172,6 +1173,13 @@ class ConfigManager(object):
             self._bg_sync_to_file()
             self.wait_for_sync()
 
+    def add_client_file(self, clientfile):
+        self.clientfiles[clientfile.filename] = clientfile.fileobject
+
+    def close_client_files(self):
+        for f in self.clientfiles:
+            self.clientfiles[f].close()
+
     def get_collective_member(self, name):
         return get_collective_member(name)
 

confluent_server/confluent/consoleserver.py

@@ -33,42 +33,67 @@ import confluent.tlvdata as tlvdata
 import confluent.util as util
 import eventlet
 import eventlet.event
+import eventlet.green.os as os
+import eventlet.green.select as select
 import eventlet.green.socket as socket
+import eventlet.green.subprocess as subprocess
 import eventlet.green.ssl as ssl
-import pyte
+import eventlet.semaphore as semaphore
+import fcntl
 import random
+import struct
 import time
 import traceback
 
 _handled_consoles = {}
 
 _tracelog = None
+_bufferdaemon = None
+_bufferlock = None
 
 try:
     range = xrange
 except NameError:
     pass
 
-pytecolors2ansi = {
-    'black': 0,
-    'red': 1,
-    'green': 2,
-    'brown': 3,
-    'blue': 4,
-    'magenta': 5,
-    'cyan': 6,
-    'white': 7,
-    'default': 9,
-}
-# might be able to use IBMPC map from pyte charsets,
-# in that case, would have to mask out certain things (like ESC)
-# in the same way that Screen's draw method would do
-# for now at least get some of the arrows in there (note ESC is one
-# of those arrows... so skip it...
-ansichars = dict(zip((0x18, 0x19), u'\u2191\u2193'))
+
+def chunk_output(output, n):
+    for i in range(0, len(output), n):
+        yield output[i:i + n]
+
+def get_buffer_output(nodename):
+    out = _bufferdaemon.stdin
+    instream = _bufferdaemon.stdout
+    if not isinstance(nodename, bytes):
+        nodename = nodename.encode('utf8')
+    outdata = b''
+    with _bufferlock:
+        out.write(struct.pack('I', len(nodename)))
+        out.write(nodename)
+        out.flush()
+        select.select((instream,), (), (), 30)
+        while not outdata or outdata[-1]:
+            chunk = instream.read(128)
+            if chunk:
+                outdata += chunk
+            else:
+                select.select((instream,), (), (), 0)
+        return outdata[:-1]
 
 
-def _utf8_normalize(data, shiftin, decoder):
+def send_output(nodename, output):
+    if not isinstance(nodename, bytes):
+        nodename = nodename.encode('utf8')
+    with _bufferlock:
+        _bufferdaemon.stdin.write(struct.pack('I', len(nodename) | (1 << 29)))
+        _bufferdaemon.stdin.write(nodename)
+        _bufferdaemon.stdin.flush()
+        for chunk in chunk_output(output, 8192):
+            _bufferdaemon.stdin.write(struct.pack('I', len(chunk) | (2 << 29)))
+            _bufferdaemon.stdin.write(chunk)
+            _bufferdaemon.stdin.flush()
+
+def _utf8_normalize(data, decoder):
     # first we give the stateful decoder a crack at the byte stream,
     # we may come up empty in the event of a partial multibyte
     try:
@@ -88,60 +113,9 @@ def _utf8_normalize(data, shiftin, decoder):
     # Finally, the low part of ascii is valid utf-8, but we are going to be
     # more interested in the cp437 versions (since this is console *output*
     # not input
-    if shiftin is None:
-        data = data.translate(ansichars)
     return data.encode('utf-8')
 
 
-def pytechars2line(chars, maxlen=None):
-    line = b'\x1b[m'  # start at default params
-    lb = False  # last bold
-    li = False  # last italic
-    lu = False  # last underline
-    ls = False  # last strikethrough
-    lr = False  # last reverse
-    lfg = 'default'  # last fg color
-    lbg = 'default'   # last bg color
-    hasdata = False
-    len = 1
-    for charidx in range(maxlen):
-        char = chars[charidx]
-        csi = bytearray([])
-        if char.fg != lfg:
-            csi.append(30 + pytecolors2ansi.get(char.fg, 9))
-            lfg = char.fg
-        if char.bg != lbg:
-            csi.append(40 + pytecolors2ansi.get(char.bg, 9))
-            lbg = char.bg
-        if char.bold != lb:
-            lb = char.bold
-            csi.append(1 if lb else 22)
-        if char.italics != li:
-            li = char.italics
-            csi.append(3 if li else 23)
-        if char.underscore != lu:
-            lu = char.underscore
-            csi.append(4 if lu else 24)
-        if char.strikethrough != ls:
-            ls = char.strikethrough
-            csi.append(9 if ls else 29)
-        if char.reverse != lr:
-            lr = char.reverse
-            csi.append(7 if lr else 27)
-        if csi:
-            line += b'\x1b[' + b';'.join(['{0}'.format(x).encode('utf-8') for x in csi]) + b'm'
-        if not hasdata and char.data.rstrip():
-            hasdata = True
-        chardata = char.data
-        if not isinstance(chardata, bytes):
-            chardata = chardata.encode('utf-8')
-        line += chardata
-        if maxlen and len >= maxlen:
-            break
-        len += 1
-    return line, hasdata
-
-
 class ConsoleHandler(object):
     _plugin_path = '/nodes/{0}/_console/session'
     _logtobuffer = True
@@ -161,15 +135,15 @@ class ConsoleHandler(object):
         self.node = node
         self.connectstate = 'unconnected'
         self._isalive = True
-        self.buffer = pyte.Screen(100, 31)
-        self.termstream = pyte.ByteStream()
-        self.termstream.attach(self.buffer)
+        #self.buffer = pyte.Screen(100, 31)
+        #self.termstream = pyte.ByteStream()
+        #self.termstream.attach(self.buffer)
         self.livesessions = set([])
         self.utf8decoder = codecs.getincrementaldecoder('utf-8')()
         if self._logtobuffer:
             self.logger = log.Logger(node, console=True,
                                      tenant=configmanager.tenant)
-        (text, termstate, timestamp) = (b'', 0, False)
+        timestamp = False
         # when reading from log file, we will use wall clock
         # it should usually match walltime.
         self.lasttime = 0
@@ -182,13 +156,7 @@ class ConsoleHandler(object):
                 # guess
                 self.lasttime = util.monotonic_time()
         self.clearbuffer()
-        self.appmodedetected = False
-        self.shiftin = None
         self.reconnect = None
-        if termstate & 1:
-            self.appmodedetected = True
-        if termstate & 2:
-            self.shiftin = b'0'
         self.users = {}
         self._attribwatcher = None
         self._console = None
@@ -216,10 +184,7 @@ class ConsoleHandler(object):
         if not isinstance(data, bytes):
             data = data.encode('utf-8')
         try:
-            self.termstream.feed(data)
-        except StopIteration:  # corrupt parser state, start over
-            self.termstream = pyte.ByteStream()
-            self.termstream.attach(self.buffer)
+            send_output(self.node, data)
         except Exception:
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
@@ -536,19 +501,7 @@ class ConsoleHandler(object):
             return
         if not isinstance(data, bytes):
             data = data.encode('utf-8')
-        if b'\x1b[?1l' in data:  # request for ansi mode cursor keys
-            self.appmodedetected = False
-        if b'\x1b[?1h' in data:  # remember the session wants the client to use
-            # 'application mode'  Thus far only observed on esxi
-            self.appmodedetected = True
-        if b'\x1b)0' in data:
-            # console indicates it wants access to special drawing characters
-            self.shiftin = b'0'
         eventdata = 0
-        if self.appmodedetected:
-            eventdata |= 1
-        if self.shiftin is not None:
-            eventdata |= 2
         # TODO: analyze buffer for registered events, examples:
         #   panics
         #   certificate signing request
@@ -557,7 +510,7 @@ class ConsoleHandler(object):
             self.clearerror = False
             self.feedbuffer(b'\x1bc\x1b[2J\x1b[1;1H')
             self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
-        self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
+        self._send_rcpts(_utf8_normalize(data, self.utf8decoder))
         self.log(data, eventdata=eventdata)
         self.lasttime = util.monotonic_time()
         self.feedbuffer(data)
@@ -583,36 +536,7 @@ class ConsoleHandler(object):
             'connectstate': self.connectstate,
             'clientcount': len(self.livesessions),
         }
-        retdata = b'\x1b[H\x1b[J'  # clear screen
-        pendingbl = b''  # pending blank lines
-        maxlen = 0
-        for line in self.buffer.display:
-            line = line.rstrip()
-            if len(line) > maxlen:
-                maxlen = len(line)
-        for line in range(self.buffer.lines):
-            nline, notblank = pytechars2line(self.buffer.buffer[line], maxlen)
-            if notblank:
-                if pendingbl:
-                    retdata += pendingbl
-                    pendingbl = b''
-                retdata += nline + b'\r\n'
-            else:
-                pendingbl += nline + b'\r\n'
-        if len(retdata) >  6:
-            retdata = retdata[:-2]  # remove the last \r\n
-        cursordata = '\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
-                                            self.buffer.cursor.x + 1)
-        if not isinstance(cursordata, bytes):
-            cursordata = cursordata.encode('utf-8')
-        retdata += cursordata
-        if self.shiftin is not None:  # detected that terminal requested a
-            # shiftin character set, relay that to the terminal that cannected
-            retdata += b'\x1b)' + self.shiftin
-        #if self.appmodedetected:
-        #    retdata += b'\x1b[?1h'
-        #else:
-        #    retdata += b'\x1b[?1l'
+        retdata = get_buffer_output(self.node)
         return retdata, connstate
 
     def write(self, data):
@@ -661,7 +585,16 @@ def _start_tenant_sessions(cfm):
 
 def start_console_sessions():
     global _tracelog
+    global _bufferdaemon
+    global _bufferlock
+    _bufferlock = semaphore.Semaphore()
     _tracelog = log.Logger('trace')
+    _bufferdaemon = subprocess.Popen(
+        ['/opt/confluent/bin/vtbufferd'], stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE)
+    fl = fcntl.fcntl(_bufferdaemon.stdout.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(_bufferdaemon.stdout.fileno(),
+                fcntl.F_SETFL, fl | os.O_NONBLOCK)
     configmodule.hook_new_configmanagers(_start_tenant_sessions)
 
 

confluent_server/confluent/core.py

@@ -180,7 +180,8 @@ def handle_deployment(configmanager, inputdata, pathcomponents,
                     yield imp
                 return
             elif operation == 'create':
-                importer = osimage.MediaImporter(inputdata['filename'])
+                importer = osimage.MediaImporter(inputdata['filename'],
+                                                 configmanager)
                 yield msg.KeyValueData({'target': importer.targpath,
                                         'name': importer.importkey})
                 return

confluent_server/confluent/discovery/core.py

@@ -811,13 +811,17 @@ def get_smm_neighbor_fingerprints(smmaddr, cv):
     if ':' in smmaddr:
         smmaddr = '[{0}]'.format(smmaddr)
     wc = webclient.SecureHTTPConnection(smmaddr, verifycallback=cv)
-    neighs = wc.grab_json_response('/scripts/neighdata.json')
+    try:
+        neighs = wc.grab_json_response('/scripts/neighdata.json')
+    except Exception:
+        log.log({'error': 'Failure getting LLDP information from {}'.format(smmaddr)})
+        return
     if not neighs:
         return
-    for idx in (4, 5):
-        if 'sha256' not in neighs[idx]:
+    for neigh in neighs:
+        if 'sha256' not in neigh:
             continue
-        yield 'sha256$' + b64tohex(neighs[idx]['sha256'])
+        yield 'sha256$' + b64tohex(neigh['sha256'])
 
 
 def get_nodename(cfg, handler, info):

confluent_server/confluent/discovery/handlers/smm.py

@@ -27,7 +27,16 @@ import confluent.netutil as netutil
 import confluent.util as util
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
-from xml.etree.ElementTree import fromstring
+from xml.etree.ElementTree import fromstring as rfromstring
+
+def fromstring(inputdata):
+    if isinstance(inputdata, bytes):
+        cmpstr = b'!entity'
+    else:
+        cmpstr = '!entity'
+    if cmpstr in inputdata.lower():
+        raise Exception('!ENTITY not supported in this interface')
+    return rfromstring(inputdata)
 
 def fixuuid(baduuid):
     # SMM dumps it out in hex

confluent_server/confluent/discovery/protocols/pxe.py

@@ -494,6 +494,7 @@ def check_reply(node, info, packet, sock, cfg, reqview):
         log.log({'error': nicerr})
     if niccfg.get('ipv4_broken', False):
         # Received a request over a nic with no ipv4 configured, ignore it
+        log.log({'error': 'Skipping boot reply to {0} due to no viable IPv4 configuration on deployment system'.format(node)})
         return
     clipn = None
     if niccfg['ipv4_address'] and niccfg['ipv4_method'] != 'firmwaredhcp':
@@ -557,6 +558,16 @@ def check_reply(node, info, packet, sock, cfg, reqview):
         staticassigns[fulladdr] = (clipn, repview[:replen + 28].tobytes())
     elif fulladdr in staticassigns:
         del staticassigns[fulladdr]
+    if httpboot:
+        boottype = 'HTTP'
+    else:
+        boottype = 'PXE'
+    if clipn:
+        ipinfo = 'with static address {0}'.format(niccfg['ipv4_address'])
+    else:
+        ipinfo = 'without address'
+    log.log({
+        'info': 'Offering {0} boot {1} to {2}'.format(boottype, ipinfo, node)})
     send_raw_packet(repview, replen + 28, reqview, info)
 
 def send_raw_packet(repview, replen, reqview, info):

confluent_server/confluent/discovery/protocols/slp.py

@@ -19,6 +19,7 @@ import confluent.util as util
 import confluent.log as log
 import os
 import random
+import eventlet.greenpool
 import eventlet.green.select as select
 import eventlet.green.socket as socket
 import struct
@@ -106,6 +107,8 @@ def _parse_slp_packet(packet, peer, rsps, xidmap):
     if '%' in addr:
         addr = addr[:addr.index('%')]
     mac = None
+    if addr not in neighutil.neightable:
+        neighutil.update_neigh()
     if addr in neighutil.neightable:
         identifier = neighutil.neightable[addr]
         mac = identifier
@@ -198,7 +201,14 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
     :param addresses:  Pass through of addresses argument from find_targets
     :return:
     """
-    if addresses is None:
+    if addresses is not None:
+        for addr in addresses:
+            for saddr in socket.getaddrinfo(addr, 427):
+                if saddr[0] == socket.AF_INET:
+                    net4.sendto(data, saddr[4])
+                elif saddr[0] == socket.AF_INET6:
+                    net.sendto(data, saddr[4])
+    else:
         data = _generate_request_payload(srvtype, True, xid)
         net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
         v6addrs = []
@@ -532,6 +542,8 @@ def active_scan(handler, protocol=None):
     for scanned in scan():
         for addr in scanned['addresses']:
             ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                neighutil.update_neigh()
             if ip not in neighutil.neightable:
                 continue
             if addr in known_peers:
@@ -583,6 +595,8 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
         # reduced chance of many responses overwhelming receive buffer.
     _grab_rsps((net, net4), rsps, 1, xidmap)
     # now to analyze and flesh out the responses
+    handleids = set([])
+    gp = eventlet.greenpool.GreenPool(128)
     for id in rsps:
         for srvurl in rsps[id].get('urls', ()):
             if len(srvurl) > 4:
@@ -597,14 +611,22 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
                     break
             else:
                 continue
-        _add_attributes(rsps[id])
+        gp.spawn_n(_add_attributes, rsps[id])
+        handleids.add(id)
+    gp.waitall()
+    for id in handleids:
         if 'service:lighttpd' in rsps[id]['services']:
             currinf = rsps[id]
             curratt = currinf.get('attributes', {})
             if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
                currinf['services'] = ['service:lenovo-tsm']
-               curratt['enclosure-serial-number'] = curratt['Product-Serial']
-               curratt['enclosure-machinetype-model'] = curratt['Machine-Type']
+               serialnumber = curratt.get('Product-Serial', curratt.get('SerialNumber', None))
+               if serialnumber:
+                   curratt['enclosure-serial-number'] = serialnumber
+               mtm = curratt.get('Machine-Type', curratt.get('Product-Name', None))
+               if mtm:
+                   mtm[0] = mtm[0].rstrip()
+                   curratt['enclosure-machinetype-model'] = mtm
             else:
                 continue
         del rsps[id]['payload']

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -62,6 +62,8 @@ def active_scan(handler, protocol=None):
     for scanned in scan(['urn:dmtf-org:service:redfish-rest:1']):
         for addr in scanned['addresses']:
             ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                neighutil.update_neigh()
             if ip not in neighutil.neightable:
                 continue
             if addr in known_peers:
@@ -311,6 +313,8 @@ def _parse_ssdp(peer, rsp, peerdata):
     ip = peer[0].partition('%')[0]
     nid = ip
     mac = None
+    if ip not in neighutil.neightable:
+        neighutil.update_neigh()
     if ip in neighutil.neightable:
         nid = neighutil.neightable[ip]
         mac = nid
@@ -351,10 +355,6 @@ def _parse_ssdp(peer, rsp, peerdata):
 
 if __name__ == '__main__':
 
-    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1']):
+    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1'], '10.240.52.189'):
         print(repr(rsp))
-    def fun(a):
-        print(repr(a))
-    def byefun(a):
-        print('bye' + repr(a))
-    snoop(fun, byefun)
+    

confluent_server/confluent/firmwaremanager.py

@@ -32,11 +32,12 @@ uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
 _tracelog = None
+filecontentbyname = {}
 
 
-def execupdate(handler, filename, updateobj, type, owner, node):
+def execupdate(handler, filename, updateobj, type, owner, node, datfile):
     global _tracelog
-    if type != 'ffdc':
+    if type != 'ffdc' and not datfile:
         errstr = False
         if not os.path.exists(filename):
             errstr =  '{0} does not appear to exist on {1}, or is in a directory with permissions forbidding confluent user/group access'.format(
@@ -64,9 +65,10 @@ def execupdate(handler, filename, updateobj, type, owner, node):
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
-                                 bank=updateobj.bank)
+                                 data=datfile, bank=updateobj.bank)
         else:
-            completion = handler(filename, progress=updateobj.handle_progress)
+            completion = handler(filename, progress=updateobj.handle_progress,
+                                 data=datfile)
         if type == 'ffdc' and completion:
             filename = completion
             completion = None
@@ -90,14 +92,20 @@ def execupdate(handler, filename, updateobj, type, owner, node):
 
 class Updater(object):
     def __init__(self, node, handler, filename, tenant=None, name=None,
-                 bank=None, type='firmware', owner=None):
+                 bank=None, type='firmware', owner=None, configmanager=None):
         self.bank = bank
         self.node = node
         self.phase = 'initializing'
         self.detail = ''
         self.percent = 0.0
+        if configmanager and filename in configmanager.clientfiles:
+            cf = configmanager.clientfiles[filename]
+            datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+        else:
+            datfile = None
+        self.datfile = datfile
         self.updateproc = updatepool.spawn(execupdate, handler, filename,
-                                           self, type, owner, node)
+                                           self, type, owner, node, datfile)
         if type == 'firmware':
             myparty = updatesbytarget
         elif type == 'mediaupload':
@@ -120,6 +128,8 @@ class Updater(object):
 
     def cancel(self):
         self.updateproc.kill()
+        if self.datfile:
+            self.datfile.close()
 
     @property
     def progress(self):

confluent_server/confluent/messages.py

@@ -1054,7 +1054,7 @@ class InputNetworkConfiguration(ConfluentInputMessage):
         if 'ipv4_gateway' not in inputdata:
             inputdata['ipv4_gateway'] = None
 
-        if 'ipv4_configuration' in inputdata:
+        if 'ipv4_configuration' in inputdata and inputdata['ipv4_configuration']:
             if inputdata['ipv4_configuration'].lower() not in ['dhcp','static']:
                 raise exc.InvalidArgumentException(
                                             'Unrecognized ipv4_configuration')
@@ -1147,6 +1147,7 @@ class BootDevice(ConfluentChoiceMessage):
         'default',
         'cd',
         'floppy',
+        'usb',
     ])
 
     valid_bootmodes = set([

confluent_server/confluent/netutil.py

@@ -82,8 +82,11 @@ _idxtoifnamemap = {}
 def _rebuildidxmap():
     _idxtoifnamemap.clear()
     for iname in os.listdir('/sys/class/net'):
-        ci = int(open('/sys/class/net/{0}/ifindex'.format(iname)).read())
-        _idxtoifnamemap[ci] = iname
+        try:
+            ci = int(open('/sys/class/net/{0}/ifindex'.format(iname)).read())
+            _idxtoifnamemap[ci] = iname
+        except Exception:  # there may be non interface in /sys/class/net
+            pass
 
 
 def myiptonets(svrip):

confluent_server/confluent/networking/lldp.py

@@ -139,7 +139,10 @@ def get_fingerprint(switch, port, configmanager, portmatch):
             continue
         if info.get('switch', None) != switch:
             continue
-        if portmatch(info.get('port'), port):
+        if portmatch(info.get('portid', None), port):
+            return ('sha256$' + b64tohex(info['peersha256fingerprint']),
+                    info.get('verified', False))
+        elif portmatch(info.get('port', None), port):
             return ('sha256$' + b64tohex(info['peersha256fingerprint']),
                     info.get('verified', False))
     return None, False
@@ -235,19 +238,23 @@ def _extract_neighbor_data_b(args):
         idx = oidindex[0][-1]
         idxtoifname[idx] = _lldpdesc_to_ifname(sid, idx, str(oidindex[1]))
     for remotedesc in conn.walk('1.0.8802.1.1.2.1.4.1.1.10'):
-        iname = idxtoifname[remotedesc[0][-2]]
+        iname = idxtoifname.get(remotedesc[0][-2],
+                                idxtoportid[remotedesc[0][-2]])
         _init_lldp(lldpdata, iname, remotedesc[0][-2], idxtoportid, switch)
         _extract_extended_desc(lldpdata[iname], remotedesc[1], user)
     for remotename in conn.walk('1.0.8802.1.1.2.1.4.1.1.9'):
-        iname = idxtoifname[remotename[0][-2]]
+        iname = idxtoifname.get(remotename[0][-2],
+                                idxtoportid[remotename[0][-2]])
         _init_lldp(lldpdata, iname, remotename[0][-2], idxtoportid, switch)
         lldpdata[iname]['peername'] = str(remotename[1])
     for remotename in conn.walk('1.0.8802.1.1.2.1.4.1.1.7'):
-        iname = idxtoifname[remotename[0][-2]]
+        iname = idxtoifname.get(remotename[0][-2],
+                                idxtoportid[remotename[0][-2]])
         _init_lldp(lldpdata, iname, remotename[0][-2], idxtoportid, switch)
         lldpdata[iname]['peerportid'] = sanitize(remotename[1])
     for remoteid in conn.walk('1.0.8802.1.1.2.1.4.1.1.5'):
-        iname = idxtoifname[remoteid[0][-2]]
+        iname = idxtoifname.get(remoteid[0][-2],
+                                idxtoportid[remoteid[0][-2]])
         _init_lldp(lldpdata, iname, remoteid[0][-2], idxtoportid, switch)
         lldpdata[iname]['peerchassisid'] = sanitize(remoteid[1])
     for entry in lldpdata:

confluent_server/confluent/networking/macmap.py

@@ -82,6 +82,8 @@ _blacklistnames = (
 
 
 def _namesmatch(switchdesc, userdesc):
+    if switchdesc is None:
+        return False
     if switchdesc == userdesc:
         return True
     try:
@@ -217,13 +219,18 @@ def _map_switch_backend(args):
                 *([int(x) for x in oid[-6:]])
             )
             mactobridge[macaddr] = int(bridgeport)
-    #ciscoiftovlanmap = {}
     vlanstocheck = set([])
-    for vb in conn.walk('.1.3.6.1.4.1.9.9.68.1.2.2.1.2'):
-        vlanstocheck.add(vb[1])
-    #ciscotrunktovlanmap = {}
-    for vb in conn.walk('.1.3.6.1.4.1.9.9.46.1.6.1.1.5'):
-        vlanstocheck.add(vb[1])
+    try:
+        #ciscoiftovlanmap = {}
+        for vb in conn.walk('.1.3.6.1.4.1.9.9.68.1.2.2.1.2'):
+            vlanstocheck.add(vb[1])
+        #ciscotrunktovlanmap = {}
+        for vb in conn.walk('.1.3.6.1.4.1.9.9.46.1.6.1.1.5'):
+            vlanstocheck.add(vb[1])
+    except Exception:
+        # We might have crashed snmp on a non-cisco switch
+        # in such a case, delay 8 seconds to allow recovery to complete
+        eventlet.sleep(8)
     if not vlanstocheck:
         vlanstocheck.add(None)
     bridgetoifmap = {}