Fix non-client fallback

If client doesn't provide file,
then we need to open by name.

confluent_client/bin/nodediscover

@@ -263,7 +263,11 @@ def list_matching_macs(options, session, node=None, checknode=True):
         return [options.mac.replace(':', '-')]
     else:
         path += 'by-mac/'
-        return [x['item']['href'] for x in session.read(path)]
+        ret = []
+        for x in session.read(path):
+            if 'item' in x and 'href' in x['item']:
+                ret.append(x['item']['href'])
+        return ret
 
 def assign_discovery(options, session, needid=True):
     abort = False

confluent_client/bin/nodefirmware

@@ -109,6 +109,12 @@ def update_firmware(session, filename):
     if options.backup:
         upargs['bank'] = 'backup'
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):

confluent_client/bin/nodelicense

@@ -69,6 +69,12 @@ def install_license(session, filename):
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     for res in session.create(resource, instargs):
         for node in res.get('databynode', []):
             if 'error' in res['databynode'][node]:

confluent_client/bin/nodemedia

@@ -111,6 +111,12 @@ def upload_media(noderange, media):
     resource = '/noderange/{0}/media/uploads/'.format(noderange)
     upargs = {'filename': filename}
     noderrs = {}
+    if session.unixdomain:
+        of = open(filename, 'rb')
+        try:
+            session.add_file(filename, of.fileno(), 'rb')
+        except Exception:
+            pass
     nodeurls = {}
     for res in session.create(resource, upargs):
         if 'created' not in res:

confluent_client/bin/nodestorage

@@ -186,7 +186,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',

confluent_client/confluent/client.py

@@ -46,6 +46,13 @@ try:
 except NameError:
     pass
 
+
+class NestedDict(dict):
+    def __missing__(self, key):
+        value = self[key] = type(self)()
+        return value
+
+
 def stringify(instr):
     # Normalize unicode and bytes to 'str', correcting for
     # current python version
@@ -154,6 +161,7 @@ class Command(object):
         self._prevkeyname = None
         self.connection = None
         self._currnoderange = None
+        self.unixdomain = False
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
                 self.serverloc = os.environ['CONFLUENT_HOST']
@@ -163,11 +171,13 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+            self.unixdomain = True
         elif self.serverloc == '/var/run/confluent/api.sock':
             raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
-        tlvdata.recv(self.connection)
+        self.protversion = int(tlvdata.recv(self.connection).split(
+            b'--')[1].strip()[1:])
         authdata = tlvdata.recv(self.connection)
         if authdata['authpassed'] == 1:
             self.authenticated = True
@@ -178,6 +188,13 @@ class Command(object):
             passphrase = os.environ['CONFLUENT_PASSPHRASE']
             self.authenticate(username, passphrase)
 
+    def add_file(self, name, handle, mode):
+        if self.protversion < 3:
+            raise Exception('Not supported with connected confluent server')
+        if not self.unixdomain:
+            raise Exception('Can only add a file to a unix domain connection')
+        tlvdata.send(self.connection, {'filename': name, 'mode': mode}, handle)
+
     def authenticate(self, username, password):
         tlvdata.send(self.connection,
                      {'username': username, 'password': password})
@@ -417,7 +434,7 @@ def send_request(operation, path, server, parameters=None):
         result = tlvdata.recv(server)
 
 
-def attrrequested(attr, attrlist, seenattributes):
+def attrrequested(attr, attrlist, seenattributes, node=None):
     for candidate in attrlist:
         truename = candidate
         if candidate.startswith('hm'):
@@ -425,10 +442,16 @@ def attrrequested(attr, attrlist, seenattributes):
         if candidate in _attraliases:
             candidate = _attraliases[candidate]
         if fnmatch.fnmatch(attr.lower(), candidate.lower()):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = True
             return True
         elif attr.lower().startswith(candidate.lower() + '.'):
-            seenattributes.add(truename)
+            if node is None:
+                seenattributes.add(truename)
+            else:
+                seenattributes[node][truename] = 1
             return True
     return False
 
@@ -444,13 +467,15 @@ def _sort_attrib(k):
 
 def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
-    seenattributes = set([])
+    seenattributes = NestedDict()
+    allnodes = set([])
     for res in session.read(path):
         if 'error' in res:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
+            allnodes.add(node)
             for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
@@ -458,7 +483,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attr == 'errorcode':
                     exitcode |= val
                     continue
-                seenattributes.add(attr)
+                seenattributes[node][attr] = True
                 if rename:
                     printattr = rename.get(attr, attr)
                 else:
@@ -466,7 +491,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                 if attrprefix:
                     printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
-                if show_attr(attr, requestargs, seenattributes, options):
+                if show_attr(attr, requestargs, seenattributes, options, node):
                     if 'value' in currattr:
                         if currattr['value'] is not None:
                             val = currattr['value']
@@ -547,23 +572,39 @@ def print_attrib_path(path, session, requestargs, options, rename=None, attrpref
                                 except TypeError:
                                     pass
                         cprint(attrout)
+    somematched = set([])
+    printmissing = set([])
+    badnodes = NestedDict()
     if not exitcode:
         if requestargs:
             for attr in requestargs:
-                if attr not in seenattributes:
-                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
-                    exitcode = 1
+                for node in allnodes:
+                    if attr in seenattributes[node]:
+                        somematched.add(attr)
+                    else:
+                        badnodes[node][attr] = True
+                        exitcode = 1
+        for node in sortutil.natural_sort(badnodes):
+            for attr in badnodes[node]:
+                if attr in somematched:
+                    sys.stderr.write(
+                        'Error: {0} matches no valid value for {1}\n'.format(
+                            attr, node))
+                else:
+                    printmissing.add(attr)
+        for missing in printmissing:
+            sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
     return exitcode
 
 
-def show_attr(attr, requestargs, seenattributes, options):
+def show_attr(attr, requestargs, seenattributes, options, node):
     try:
         reverse = options.exclude
     except AttributeError:
         reverse = False
     if requestargs is None or requestargs == []:
         return True
-    processattr = attrrequested(attr, requestargs, seenattributes)
+    processattr = attrrequested(attr, requestargs, seenattributes, node)
     if reverse:
         processattr = not processattr
     return processattr

confluent_client/confluent/tlv.py

@@ -17,4 +17,4 @@
 
 
 class Types(object):
-    text, json = range(2)
+    text, json, filehandle = range(3)

confluent_client/confluent/tlvdata.py

@@ -15,9 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import array
+import ctypes
+import ctypes.util
 import confluent.tlv as tlv
+import eventlet.green.socket as socket
+import eventlet.green.select as select
 from datetime import datetime
 import json
+import os
 import struct
 
 try:
@@ -30,6 +36,72 @@ try:
 except NameError:
     pass
 
+class iovec(ctypes.Structure):   # from uio.h
+    _fields_ = [('iov_base', ctypes.c_void_p),
+                ('iov_len', ctypes.c_size_t)]
+
+
+iovec_ptr = ctypes.POINTER(iovec)
+
+
+class cmsghdr(ctypes.Structure):  # also from bits/socket.h
+    _fields_ = [('cmsg_len', ctypes.c_size_t),
+                ('cmsg_level', ctypes.c_int),
+                ('cmsg_type', ctypes.c_int)]
+
+    @classmethod
+    def init_data(cls, cmsg_len, cmsg_level, cmsg_type, cmsg_data):
+        Data = ctypes.c_ubyte * ctypes.sizeof(cmsg_data)
+        class _flexhdr(ctypes.Structure):
+            _fields_ = cls._fields_ + [('cmsg_data', Data)]
+
+        datab = Data(*bytearray(cmsg_data))
+        return _flexhdr(cmsg_len=cmsg_len, cmsg_level=cmsg_level,
+                        cmsg_type=cmsg_type, cmsg_data=datab)
+
+
+def CMSG_LEN(length):
+    sizeof_cmshdr = ctypes.sizeof(cmsghdr)
+    return ctypes.c_size_t(CMSG_ALIGN(sizeof_cmshdr).value + length)
+
+
+SCM_RIGHTS = 1
+
+
+class msghdr(ctypes.Structure):  # from bits/socket.h
+    _fields_ = [('msg_name', ctypes.c_void_p),
+                ('msg_namelen', ctypes.c_uint),
+                ('msg_iov', ctypes.POINTER(iovec)),
+                ('msg_iovlen', ctypes.c_size_t),
+                ('msg_control', ctypes.c_void_p),
+                ('msg_controllen', ctypes.c_size_t),
+                ('msg_flags', ctypes.c_int)]
+
+
+def CMSG_ALIGN(length):  # bits/socket.h
+    ret = (length + ctypes.sizeof(ctypes.c_size_t) - 1
+           & ~(ctypes.sizeof(ctypes.c_size_t) - 1))
+    return ctypes.c_size_t(ret)
+
+
+def CMSG_SPACE(length):  # bits/socket.h
+    ret = CMSG_ALIGN(length).value + CMSG_ALIGN(ctypes.sizeof(cmsghdr)).value
+    return ctypes.c_size_t(ret)
+
+
+class ClientFile(object):
+    def __init__(self, name, mode, fd):
+        self.fileobject = os.fdopen(fd, mode)
+        self.filename = name
+
+libc = ctypes.CDLL(ctypes.util.find_library('c'))
+recvmsg = libc.recvmsg
+recvmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+recvmsg.restype = ctypes.c_int
+sendmsg = libc.sendmsg
+sendmsg.argtypes = [ctypes.c_int, ctypes.POINTER(msghdr), ctypes.c_int]
+sendmsg.restype = ctypes.c_size_t
+
 def decodestr(value):
     ret = None
     try:
@@ -65,7 +137,7 @@ def _unicode_list(currlist):
             _unicode_list(currlist[i])
 
 
-def send(handle, data):
+def send(handle, data, filehandle=None):
     if isinstance(data, unicode):
         try:
             data = data.encode('utf-8')
@@ -93,9 +165,27 @@ def send(handle, data):
         if tl > 16777215:
             raise Exception("JSON data exceeds protocol limits")
         # xor in the type (0b1 << 24)
-        tl |= 16777216
-        handle.sendall(struct.pack("!I", tl))
-        handle.sendall(sdata)
+        if filehandle is None:
+            tl |= 16777216
+            handle.sendall(struct.pack("!I", tl))
+            handle.sendall(sdata)
+        else:
+            tl |= (2 << 24)
+            handle.sendall(struct.pack("!I", tl))
+            cdtype = ctypes.c_ubyte * len(sdata)
+            cdata = cdtype.from_buffer(bytearray(sdata))
+            ciov = iovec(iov_base=ctypes.addressof(cdata),
+                         iov_len=ctypes.c_size_t(ctypes.sizeof(cdata)))
+            fd = ctypes.c_int(filehandle)
+            cmh = cmsghdr.init_data(
+                cmsg_len=CMSG_LEN(
+                    ctypes.sizeof(fd)), cmsg_level=socket.SOL_SOCKET,
+                    cmsg_type=SCM_RIGHTS, cmsg_data=fd)
+            mh = msghdr(msg_name=None, msg_len=0, msg_iov=iovec_ptr(ciov),
+                        msg_iovlen=1, msg_control=ctypes.addressof(cmh),
+                        msg_controllen=ctypes.c_size_t(ctypes.sizeof(cmh)))
+            sendmsg(handle.fileno(), mh, 0)
+
 
 def recvall(handle, size):
     rd = handle.recv(size)
@@ -125,12 +215,42 @@ def recv(handle):
     datatype = (tl & 2130706432) >> 24  # grab 7 bits from near beginning
     if dlen == 0:
         return None
-    data = handle.recv(dlen)
-    while len(data) < dlen:
-        ndata = handle.recv(dlen - len(data))
-        if not ndata:
-            raise Exception("Error reading data")
-        data += ndata
+    if datatype == tlv.Types.filehandle:
+        filehandles = array.array('i')
+        rawbuffer = bytearray(2048)
+        pkttype = ctypes.c_ubyte * 2048
+        data = pkttype.from_buffer(rawbuffer)
+        cmsgsize = CMSG_SPACE(ctypes.sizeof(ctypes.c_int)).value
+        cmsgarr = bytearray(cmsgsize)
+        cmtype = ctypes.c_ubyte * cmsgsize
+        cmsg = cmtype.from_buffer(cmsgarr)
+        cmsg.cmsg_level = socket.SOL_SOCKET
+        cmsg.cmsg_type = SCM_RIGHTS
+        cmsg.cmsg_len = CMSG_LEN(ctypes.sizeof(ctypes.c_int))
+        iov = iovec()
+        iov.iov_base = ctypes.addressof(data)
+        iov.iov_len = 2048
+        msg = msghdr()
+        msg.msg_iov = ctypes.pointer(iov)
+        msg.msg_iovlen = 1
+        msg.msg_control = ctypes.addressof(cmsg)
+        msg.msg_controllen = ctypes.sizeof(cmsg)
+        select.select([handle], [], [])
+        i = recvmsg(handle.fileno(), ctypes.pointer(msg), 0)
+        cdata = cmsgarr[CMSG_LEN(0).value:]
+        data = rawbuffer[:i]
+        if cmsg.cmsg_level == socket.SOL_SOCKET and cmsg.cmsg_type == SCM_RIGHTS:
+            filehandles.fromstring(bytes(
+                cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
+        data = json.loads(bytes(data))
+        return ClientFile(data['filename'], data['mode'], filehandles[0])
+    else:
+        data = handle.recv(dlen)
+        while len(data) < dlen:
+            ndata = handle.recv(dlen - len(data))
+            if not ndata:
+                raise Exception("Error reading data")
+            data += ndata
     if datatype == tlv.Types.text:
         return data
     elif datatype == tlv.Types.json:

confluent_client/doc/man/nodeconfig.ronn

@@ -14,6 +14,12 @@ running configuration on the node firmware.  Calling without '=' will show the
 current value, and '=' will change the value.  Network information can be
 given as a node expression, as documented in the man page for nodeattribexpressions(5).
 
+Note that when using nodeconfig to submit changes, it will exit when the change
+is accepted, but the endpoint may not have fully processed it. Doing a show
+immediately after doing a set may reflect older information. Also, if changing
+BIOS/UEFI settings, the change may appear in output, but generally won't
+actually be in effect until a reboot.
+
 ## OPTIONS
 
 * `-c`, `--comparedefault`:

confluent_client/doc/man/nodestorage.ronn

@@ -3,12 +3,15 @@ nodestorage(8) -- Examine/Modify storage configuration of a node
 
 ## SYNOPSIS
 
-`nodestorage <noderange> [show|create|delete] [options]`  
+`nodestorage <noderange> [show|create|delete|diskset] [hotspare|jbod|unconfigured] [options]`  
 
 ## DESCRIPTION
 
 `nodestorage` provides access to the remote storage configuration of
-the noderange.
+the noderange. The `show` subcommand will show current storage configuration,
+`create` can be used to create new arrays or volumes, `delete` can be used to
+remove volumes and arrays, and `diskset` can modify the usage of disks
+indicated by `-d` to either be `unconfigured`, `jbod`, or `hotspare`.
 
 ## OPTIONS
 * `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
@@ -29,6 +32,15 @@ the noderange.
   volumes, or selecting a volume for delete.  Default
   behavior is to use implementation provided default
 
+* `-z` **STRIPSIZES**, `--stripsizes`=**STRIPSIZES**:
+  Comma separated list of stripsizes to use when creating volumes.
+  This value is in kilobytes.  The default behavior is to allow the
+  storage controller to decide
+
+* `-m` **MAXNODES**, `--maxnodes`=**MAXNODES**:
+  Specify a maximum number of nodes to configure storage on, prompting
+  if over the threshold
+
 ## EXAMPLES
 
 * Deleting the volume `somedata`:

confluent_osdeploy/el8/profiles/default/initprofile.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/' $2/profile.yaml
+sed -i 's/centos/CentOS/; s/rhel/Red Hat Enterprise Linux/; s/oraclelinux/Oracle Linux/; s/alma/AlmaLinux/' $2/profile.yaml
 ln -s $1/images/pxeboot/vmlinuz $2/boot/kernel && \
 ln -s $1/images/pxeboot/initrd.img $2/boot/initramfs/distribution
 mkdir -p $2/boot/efi/boot && \

confluent_osdeploy/esxi7/initramfs/bin/dcuiweasel

@@ -7,8 +7,22 @@ echo $uuid > /sys/devices/virtual/dmi/id/product_uuid
 mkdir -p /etc/confluent
 localcli network firewall unload
 touch /etc/confluent/confluent.info
+begin=$(date +%s)
 while ! grep NODENAME /etc/confluent/confluent.info; do
+    echo "Searching for deployment service on local network..."
     /opt/confluent/bin/copernicus > /etc/confluent/confluent.info
+    esxcfg-nics -l
+    if [ $(date +%s) -gt $((begin + 90)) ]; then 
+        if [ ! -f /var/run/vmware/show-esx-shell-login ]; then
+            chvt 1
+            sleep 1
+            chvt 2
+            /etc/init.d/ESXShell start
+        fi
+        echo "Unable to locate a deployment system on the local network, verify network connectivity"
+        echo "A debug session has been made available on Alt-F1"
+        sleep 30
+    fi
 done
 node=$(grep NODENAME: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')
 mgr=$(grep MANAGER: /etc/confluent/confluent.info|head -n 1|awk '{print $2}')

confluent_osdeploy/esxi7/profiles/hypervisor/kickstart

@@ -4,7 +4,7 @@ install --firstdisk --overwritevmfs
 %include /tmp/ksnet
 %include /tmp/rootpw
 reboot
-%post
+%post --interpreter=busybox
 localcli network firewall unload
 STATUP=$(mktemp)
 echo '{"status": "complete"}' > $STATUP

confluent_osdeploy/esxi7/profiles/hypervisor/scripts/makeksnet

@@ -1,5 +1,8 @@
 #!/usr/bin/python
+import re
+import subprocess
 
+uplinkmatch = re.compile('^\s*Uplinks:\s*(.*)')
 nodename = None
 for inf in open('/etc/confluent/confluent.info', 'r').read().split('\n'):
     if inf.startswith('NODENAME: '):
@@ -24,8 +27,15 @@ for line in deploycfg:
     else:
         nslist=False
 cfg['nameservers'] = ','.join(nameservers)
-
+vswinfo = subprocess.check_output(['localcli', 'network', 'vswitch', 'standard', 'list']).decode('utf8')
+vmnic = None
+for info in vswinfo.split('\n'):
+    upinfo = uplinkmatch.match(info)
+    if upinfo:
+        vmnic = upinfo.group(1)
 netline = 'network --hostname={0} --bootproto={1}'.format(nodename, cfg['ipv4_method'])
+if vmnic:
+    netline += ' --device={0}'.format(vmnic)
 if cfg['ipv4_method'] == 'static':
     netline += ' --ip={0} --netmask={1}'.format(cfg['ipv4_address'], cfg['ipv4_netmask'])
     if cfg.get('ipv4_gateway', 'null') not in (None, '', 'null'):

confluent_osdeploy/genesis/initramfs/opt/confluent/bin/rungenesis

@@ -30,6 +30,7 @@ echo "done"
 cat > /etc/ssh/sshd_config << EOF
 Port 22
 Port 3389
+Subsystem       sftp    /usr/libexec/openssh/sftp-server
 PermitRootLogin yes
 AuthorizedKeysFile      .ssh/authorized_keys
 EOF

confluent_server/bin/osdeploy

@@ -1,5 +1,7 @@
 #!/usr/bin/python2
 
+__author__ = 'jjohnson2,bfinley'
+
 import argparse
 import glob
 import os
@@ -50,7 +52,11 @@ def main(args):
         'updateboot',
         help='Push profile.yaml of the named profile data into boot assets as appropriate')
     upb.add_argument('profile', help='Profile to update boot assets')
+    osls = sp.add_parser('list', help='List OS images available for deployment')
     cmdset = ap.parse_args()
+
+    if cmdset.command == 'list':
+        return oslist()
     if cmdset.command == 'import':
         return osimport(cmdset.imagefile)
     if cmdset.command == 'initialize':
@@ -308,9 +314,36 @@ def updateboot(profilename):
             print(repr(rsp))
 
 
+def oslist():
+    c = client.Command()
+    print("Distributions:")
+    for rsp in c.read('/deployment/distributions'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+    print("Profiles:")
+    for rsp in c.read('/deployment/profiles'):
+        if 'error' in rsp:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print("  " + rsp['item']['href'].replace('/', ''))
+    print("")
+
+
 def osimport(imagefile):
     c = client.Command()
     imagefile = os.path.abspath(imagefile)
+    if c.unixdomain:
+        ofile = open(imagefile, 'rb')
+        try:
+            c.add_file(imagefile, ofile.fileno(), 'rb')
+        except Exception:
+            pass
     importing = False
     shortname = None
     for rsp in c.create('/deployment/importing/', {'filename': imagefile}):

confluent_server/confluent/collective/manager.py

@@ -76,8 +76,8 @@ def connect_to_leader(cert=None, name=None, leader=None):
         with cfm._initlock:
             banner = tlvdata.recv(remote)  # the banner
             vers = banner.split()[2]
-            if vers != b'v2':
-                raise Exception('This instance only supports protocol 2, synchronize versions between collective members')
+            if vers not in (b'v2', b'v3'):
+                raise Exception('This instance only supports protocol 2 or 3, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()

confluent_server/confluent/config/configmanager.py

@@ -1138,6 +1138,7 @@ class ConfigManager(object):
         return _cfgstore['tenant'][self.tenant]
 
     def __init__(self, tenant, decrypt=False, username=None):
+        self.clientfiles = {}
         global _cfgstore
         with _initlock:
             if _cfgstore is None:
@@ -1172,6 +1173,13 @@ class ConfigManager(object):
             self._bg_sync_to_file()
             self.wait_for_sync()
 
+    def add_client_file(self, clientfile):
+        self.clientfiles[clientfile.filename] = clientfile.fileobject
+
+    def close_client_files(self):
+        for f in self.clientfiles:
+            self.clientfiles[f].close()
+
     def get_collective_member(self, name):
         return get_collective_member(name)
 

confluent_server/confluent/core.py

@@ -180,7 +180,8 @@ def handle_deployment(configmanager, inputdata, pathcomponents,
                     yield imp
                 return
             elif operation == 'create':
-                importer = osimage.MediaImporter(inputdata['filename'])
+                importer = osimage.MediaImporter(inputdata['filename'],
+                                                 configmanager)
                 yield msg.KeyValueData({'target': importer.targpath,
                                         'name': importer.importkey})
                 return

confluent_server/confluent/discovery/protocols/slp.py

@@ -199,7 +199,14 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
     :param addresses:  Pass through of addresses argument from find_targets
     :return:
     """
-    if addresses is None:
+    if addresses is not None:
+        for addr in addresses:
+            for saddr in socket.getaddrinfo(addr, 427):
+                if saddr[0] == socket.AF_INET:
+                    net4.sendto(data, saddr[4])
+                elif saddr[0] == socket.AF_INET6:
+                    net.sendto(data, saddr[4])
+    else:
         data = _generate_request_payload(srvtype, True, xid)
         net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
         v6addrs = []

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -351,10 +351,6 @@ def _parse_ssdp(peer, rsp, peerdata):
 
 if __name__ == '__main__':
 
-    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1']):
+    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1'], '10.240.52.189'):
         print(repr(rsp))
-    def fun(a):
-        print(repr(a))
-    def byefun(a):
-        print('bye' + repr(a))
-    snoop(fun, byefun)
+    

confluent_server/confluent/firmwaremanager.py

@@ -32,11 +32,12 @@ uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
 _tracelog = None
+filecontentbyname = {}
 
 
-def execupdate(handler, filename, updateobj, type, owner, node):
+def execupdate(handler, filename, updateobj, type, owner, node, datfile):
     global _tracelog
-    if type != 'ffdc':
+    if type != 'ffdc' and not datfile:
         errstr = False
         if not os.path.exists(filename):
             errstr =  '{0} does not appear to exist on {1}, or is in a directory with permissions forbidding confluent user/group access'.format(
@@ -64,9 +65,10 @@ def execupdate(handler, filename, updateobj, type, owner, node):
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
-                                 bank=updateobj.bank)
+                                 data=datfile, bank=updateobj.bank)
         else:
-            completion = handler(filename, progress=updateobj.handle_progress)
+            completion = handler(filename, progress=updateobj.handle_progress,
+                                 data=datfile)
         if type == 'ffdc' and completion:
             filename = completion
             completion = None
@@ -90,14 +92,20 @@ def execupdate(handler, filename, updateobj, type, owner, node):
 
 class Updater(object):
     def __init__(self, node, handler, filename, tenant=None, name=None,
-                 bank=None, type='firmware', owner=None):
+                 bank=None, type='firmware', owner=None, configmanager=None):
         self.bank = bank
         self.node = node
         self.phase = 'initializing'
         self.detail = ''
         self.percent = 0.0
+        if configmanager and filename in configmanager.clientfiles:
+            cf = configmanager.clientfiles[filename]
+            datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+        else:
+            datfile = None
+        self.datfile = datfile
         self.updateproc = updatepool.spawn(execupdate, handler, filename,
-                                           self, type, owner, node)
+                                           self, type, owner, node, datfile)
         if type == 'firmware':
             myparty = updatesbytarget
         elif type == 'mediaupload':
@@ -120,6 +128,8 @@ class Updater(object):
 
     def cancel(self):
         self.updateproc.kill()
+        if self.datfile:
+            self.datfile.close()
 
     @property
     def progress(self):

confluent_server/confluent/osimage.py

@@ -218,17 +218,38 @@ def extract_entries(entries, flags=0, callback=None, totalsize=None, extractlist
         callback({'progress': float(sizedone) / float(totalsize)})
 
 
-def extract_file(filepath, flags=0, callback=lambda x: None, imginfo=(), extractlist=None):
+def extract_file(archfile, flags=0, callback=lambda x: None, imginfo=(), extractlist=None):
     """Extracts an archive from a file into the current directory."""
     totalsize = 0
     for img in imginfo:
         if not imginfo[img]:
             continue
         totalsize += imginfo[img]
-    with libarchive.file_reader(filepath) as archive:
+    archfile.seek(0)
+    with libarchive.fd_reader(archfile.fileno()) as archive:
         extract_entries(archive, flags, callback, totalsize, extractlist)
 
 
+def check_alma(isoinfo):
+    ver = None
+    arch = None
+    cat = None
+    for entry in isoinfo[0]:
+        if 'almalinux-release-8' in entry:
+            ver = entry.split('-')[2]
+            arch = entry.split('.')[-2]
+            cat = 'el8'
+            break
+    else:
+        return None
+    if arch == 'noarch' and '.discinfo' in isoinfo[1]:
+        prodinfo = isoinfo[1]['.discinfo']
+        arch = prodinfo.split(b'\n')[2]
+        if not isinstance(arch, str):
+            arch = arch.decode('utf-8')
+    return {'name': 'alma-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': cat}
+
+
 def check_centos(isoinfo):
     ver = None
     arch = None
@@ -420,10 +441,10 @@ def check_rhel(isoinfo):
     return {'name': 'rhel-{0}-{1}'.format(ver, arch), 'method': EXTRACT, 'category': 'el{0}'.format(major)}
 
 
-def scan_iso(filename):
+def scan_iso(archive):
     filesizes = {}
     filecontents = {}
-    with libarchive.file_reader(filename) as reader:
+    with libarchive.fd_reader(archive.fileno()) as reader:
         for ent in reader:
             if str(ent).endswith('TRANS.TBL'):
                 continue
@@ -436,35 +457,41 @@ def scan_iso(filename):
     return filesizes, filecontents
 
 
-def fingerprint(filename):
-    with open(filename, 'rb') as archive:
-        header = archive.read(32768)
-        archive.seek(32769)
-        if archive.read(6) == b'CD001\x01':
-            # ISO image
-            isoinfo = scan_iso(filename)
-            name = None
-            for fun in globals():
-                if fun.startswith('check_'):
-                    name = globals()[fun](isoinfo)
-                    if name:
-                        return name, isoinfo[0]
-            return None
-        else:
-            sum = hashlib.sha256(header)
-            if sum.digest() in HEADERSUMS:
-                archive.seek(32768)
+def fingerprint(archive):
+    header = archive.read(32768)
+    archive.seek(32769)
+    if archive.read(6) == b'CD001\x01':
+        # ISO image
+        archive.seek(0)
+        isoinfo = scan_iso(archive)
+        archive.seek(0)
+        name = None
+        for fun in globals():
+            if fun.startswith('check_'):
+                name = globals()[fun](isoinfo)
+                if name:
+                    return name, isoinfo[0]
+        return None
+    else:
+        sum = hashlib.sha256(header)
+        if sum.digest() in HEADERSUMS:
+            archive.seek(32768)
+            chunk = archive.read(32768)
+            while chunk:
+                sum.update(chunk)
                 chunk = archive.read(32768)
-                while chunk:
-                    sum.update(chunk)
-                    chunk = archive.read(32768)
-                imginfo = HASHPRINTS.get(sum.hexdigest(), None)
-                if imginfo:
-                    return imginfo, None
+            imginfo = HASHPRINTS.get(sum.hexdigest(), None)
+            if imginfo:
+                return imginfo, None
 
 
-def import_image(filename, callback, backend=False):
-    identity = fingerprint(filename)
+def import_image(filename, callback, backend=False, mfd=None):
+    if mfd:
+        archive = os.fdopen(int(mfd), 'rb')
+    else:
+        archive = open(filename, 'rb')
+    archive.seek(0)
+    identity = fingerprint(archive)
     if not identity:
         return -1
     identity, imginfo = identity
@@ -480,11 +507,13 @@ def import_image(filename, callback, backend=False):
         print('Importing OS to ' + targpath + ':')
     printit({'progress': 0.0})
     if EXTRACT & identity['method']:
-        extract_file(filename, callback=callback, imginfo=imginfo, extractlist=identity.get('extractlist', None))
+        extract_file(archive, callback=callback, imginfo=imginfo, extractlist=identity.get('extractlist', None))
     if COPY & identity['method']:
         basename = identity.get('copyto', os.path.basename(filename))
         targpath = os.path.join(targpath, basename)
-        shutil.copyfile(filename, targpath)
+        archive.seek(0)
+        with open(targpath, 'wb') as targ:
+            shutil.copyfileobj(archive, targ)
     with open(targpath + '/distinfo.yaml', 'w') as distinfo:
         distinfo.write(yaml.dump(identity, default_flow_style=False))
     if 'subname' in identity:
@@ -561,10 +590,15 @@ def generate_stock_profiles(defprofile, distpath, targpath, osname,
 
 class MediaImporter(object):
 
-    def __init__(self, media):
+    def __init__(self, media, cfm=None):
         self.worker = None
         self.profiles = []
-        identity = fingerprint(media)
+        medfile = None
+        if cfm and media in cfm.clientfiles:
+            medfile = cfm.clientfiles[media]
+        else:
+            medfile = open(media, 'rb')
+        identity = fingerprint(medfile)
         if not identity:
             raise exc.InvalidArgumentException('Unsupported Media')
         self.percent = 0.0
@@ -591,6 +625,7 @@ class MediaImporter(object):
         if os.path.exists(self.targpath):
             raise Exception('{0} already exists'.format(self.targpath))
         self.filename = os.path.abspath(media)
+        self.medfile = medfile
         self.importer = eventlet.spawn(self.importmedia)
 
     def stop(self):
@@ -603,10 +638,11 @@ class MediaImporter(object):
 
     def importmedia(self):
         os.environ['PYTHONPATH'] = ':'.join(sys.path)
+        os.environ['CONFLUENT_MEDIAFD'] = '{0}'.format(self.medfile.fileno())
         with open(os.devnull, 'w') as devnull:
             self.worker = subprocess.Popen(
                 [sys.executable, __file__, self.filename, '-b'],
-                stdin=devnull, stdout=subprocess.PIPE)
+                stdin=devnull, stdout=subprocess.PIPE, close_fds=False)
         wkr = self.worker
         currline = b''
         while wkr.poll() is None:
@@ -651,6 +687,7 @@ def get_importing_status(importkey):
 if __name__ == '__main__':
     os.umask(0o022)
     if len(sys.argv) > 2:
-        sys.exit(import_image(sys.argv[1], callback=printit, backend=True))
+        mfd = os.environ.get('CONFLUENT_MEDIAFD', None)
+        sys.exit(import_image(sys.argv[1], callback=printit, backend=True, mfd=mfd))
     else:
         sys.exit(import_image(sys.argv[1], callback=printit))

confluent_server/confluent/plugins/configuration/attributes.py

@@ -87,7 +87,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                     desc = ''
             if 'value' in currattr or 'expression' in currattr:
                 yield msg.Attributes(kv={attribute: currattr}, desc=desc)
-            elif 'cryptvalue' in currattr:
+            elif 'cryptvalue' in currattr or 'hashvalue' in currattr:
                 yield msg.CryptedAttributes(
                     kv={attribute: currattr},
                     desc=desc)

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -462,6 +462,7 @@ persistent_ipmicmds = {}
 class IpmiHandler(object):
     def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
                  realop):
+        self.cfm = cfg
         self.invmap = {}
         self.output = output
         self.sensorcategory = None
@@ -589,7 +590,7 @@ class IpmiHandler(object):
     def handle_update(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                     self.inputdata.nodefile(self.node), self.tenant,
-                                    bank=self.inputdata.bank)
+                                    bank=self.inputdata.bank, configmanager=self.cfm)
         self.output.put(
             msg.CreatedResource(
                 'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
@@ -598,7 +599,7 @@ class IpmiHandler(object):
     def handle_media_upload(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                      self.inputdata.nodefile(self.node), self.tenant,
-                                     type='mediaupload')
+                                     type='mediaupload', configmanager=self.cfm)
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
@@ -1524,14 +1525,21 @@ class IpmiHandler(object):
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
             filename = self.inputdata.nodefile(self.node)
-            if not os.access(filename, os.R_OK):
+            datfile = None
+            if filename in self.cfm.clientfiles:
+                cf = self.cfm.clientfiles[filename]
+                datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+            if datfile is None and not os.access(filename, os.R_OK):
                 errstr =  ('{0} is not readable by confluent on {1} '
                            '(ensure confluent user or group can access file '
                            'and parent directories)').format(
                                filename, socket.gethostname())
                 self.output.put(msg.ConfluentNodeError(self.node, errstr))
                 return
-            self.ipmicmd.apply_license(filename)
+            try:
+                self.ipmicmd.apply_license(filename, data=datfile)
+            finally:
+                datfile.close()
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
             i = 1

confluent_server/confluent/plugins/hardwaremanagement/redfish.py

@@ -348,6 +348,7 @@ persistent_ipmicmds = {}
 class IpmiHandler(object):
     def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
                  realop):
+        self.cfm = cfg
         self.sensormap = {}
         self.invmap = {}
         self.output = output
@@ -453,7 +454,8 @@ class IpmiHandler(object):
     def handle_update(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                     self.inputdata.nodefile(self.node), self.tenant,
-                                    bank=self.inputdata.bank)
+                                    bank=self.inputdata.bank,
+                                    configmanager=self.cfm)
         self.output.put(
             msg.CreatedResource(
                 'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
@@ -462,7 +464,7 @@ class IpmiHandler(object):
     def handle_media_upload(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                      self.inputdata.nodefile(self.node), self.tenant,
-                                     type='mediaupload')
+                                     type='mediaupload', configmanager=self.cfm)
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
@@ -1368,14 +1370,21 @@ class IpmiHandler(object):
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
             filename = self.inputdata.nodefile(self.node)
-            if not os.access(filename, os.R_OK):
+            datfile = None
+            if filename in self.cfm.clientfiles:
+                cf = self.cfm.clientfiles[filename]
+                datfile = os.fdopen(os.dup(cf.fileno()), cf.mode)
+            if datfile is None and not os.access(filename, os.R_OK):
                 errstr =  ('{0} is not readable by confluent on {1} '
                            '(ensure confluent user or group can access file '
                            'and parent directories)').format(
                                filename, socket.gethostname())
                 self.output.put(msg.ConfluentNodeError(self.node, errstr))
                 return
-            self.ipmicmd.apply_license(self.inputdata.nodefile(self.node))
+            try:
+                self.ipmicmd.apply_license(filename, data=datfile)
+            finally:
+                datfile.close()
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
             i = 1

confluent_server/confluent/sockapi.py

@@ -77,6 +77,8 @@ except ImportError:
 
 plainsocket = None
 
+libc = ctypes.CDLL(ctypes.util.find_library('c'))
+
 def _should_authlog(path, operation):
     if (operation == 'retrieve' and
             ('/sensors/' in path or '/health/' in path or
@@ -126,7 +128,8 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             cfm = authdata[1]
             authenticated = True
     # version 0 == original, version 1 == pickle3 allowed, 2 = pickle forbidden, msgpack allowed
-    send_data(connection, "Confluent -- v2 --")
+    # v3 - filehandle allowed
+    send_data(connection, "Confluent -- v3 --")
     while not authenticated:  # prompt for name and passphrase
         send_data(connection, {'authpassed': 0})
         response = tlvdata.recv(connection)
@@ -156,7 +159,7 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             cfm = authdata[1]
     send_data(connection, {'authpassed': 1})
     request = tlvdata.recv(connection)
-    if request and 'collective' in request:
+    if request and isinstance(request, dict) and 'collective' in request:
         if skipauth:
             if not libssl:
                 tlvdata.send(
@@ -197,7 +200,7 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
                                       'error': 'Unexpected error - ' + str(e)})
             send_data(connection, {'_requestdone': 1})
         request = tlvdata.recv(connection)
-
+    cfm.close_client_files()
 
 def send_response(responses, connection):
     if responses is None:
@@ -208,6 +211,9 @@ def send_response(responses, connection):
 
 
 def process_request(connection, request, cfm, authdata, authname, skipauth):
+    if isinstance(request, tlvdata.ClientFile):
+        cfm.add_client_file(request)
+        return
     if not isinstance(request, dict):
         raise exc.InvalidArgumentException
     operation = request['operation']
@@ -475,7 +481,6 @@ class SockApi(object):
         self.unixdomainserver = eventlet.spawn(_unixdomainhandler)
 
     def watch_for_cert(self):
-        libc = ctypes.CDLL(ctypes.util.find_library('c'))
         watcher = libc.inotify_init1(os.O_NONBLOCK)
         if libc.inotify_add_watch(watcher, b'/etc/confluent/', 0x100) > -1:
             while True:

genesis/97genesis/install-base

@@ -1,7 +1,7 @@
 #!/bin/sh
 dracut_install /usr/bin/clevis-encrypt-tpm2 /usr/bin/clevis-decrypt-tpm2 /usr/bin/jose /usr/bin/mktemp
 dracut_install /lib64/libtss2-tcti-device.so.0
-dracut_install /usr/bin/tpm2_create /usr/bin/tpm2_pcrlist /usr/bin/tpm2_createpolicy /usr/bin/tpm2_createprimary /usr/bin/tpm2_load /usr/bin/tpm2_unseal
+dracut_install /usr/bin/tpm2_create /usr/bin/tpm2_pcrread /usr/bin/tpm2_createpolicy /usr/bin/tpm2_createprimary /usr/bin/tpm2_load /usr/bin/tpm2_unseal
 dracut_install curl openssl tar ipmitool cpio gzip lsmod ethtool
 dracut_install modprobe touch echo cut wc bash netstat uniq grep ip hostname
 dracut_install awk egrep dirname bc expr sort

genesis/97genesis/install-python

@@ -8,39 +8,10 @@ dracut_install /usr/lib64/python3.6/os.py
 dracut_install /usr/lib64/gconv/gconv-modules.cache
 dracut_install /usr/lib64/python3.6
 dracut_install /usr/lib64/python3.6/collections
-dracut_install /usr/lib64/python3.6/collections/__pycache__/abc.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/collections/__pycache__/__init__.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/ctypes
-dracut_install /usr/lib64/python3.6/ctypes/__pycache__/_endian.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/ctypes/__pycache__/__init__.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/email
-dracut_install /usr/lib64/python3.6/email/__pycache__/base64mime.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/charset.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/_encoded_words.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/encoders.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/errors.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/feedparser.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/header.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/__init__.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/iterators.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/message.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/_parseaddr.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/parser.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/_policybase.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/quoprimime.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/email/__pycache__/utils.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/encodings
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/aliases.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/__init__.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/latin_1.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/utf_8.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/ascii.cpython-36.opt-1.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/ascii.cpython-36.opt-2.pyc
-dracut_install /usr/lib64/python3.6/encodings/__pycache__/ascii.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/http
-dracut_install /usr/lib64/python3.6/http/__pycache__/client.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/http/__pycache__/cookies.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/http/__pycache__/__init__.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/lib-dynload
 dracut_install /usr/lib64/python3.6/lib-dynload/fcntl.cpython-36m-x86_64-linux-gnu.so
 dracut_install /usr/lib64/python3.6/lib-dynload/binascii.cpython-36m-x86_64-linux-gnu.so
@@ -59,66 +30,8 @@ dracut_install /usr/lib64/python3.6/lib-dynload/_socket.cpython-36m-x86_64-linux
 dracut_install /usr/lib64/python3.6/lib-dynload/_ssl.cpython-36m-x86_64-linux-gnu.so
 dracut_install /usr/lib64/python3.6/lib-dynload/_struct.cpython-36m-x86_64-linux-gnu.so
 dracut_install /usr/lib64/python3.6/lib-dynload/unicodedata.cpython-36m-x86_64-linux-gnu.so
-dracut_install /usr/lib64/python3.6/__pycache__/abc.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/argparse.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/base64.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/bisect.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/_bootlocale.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/calendar.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/codecs.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/_collections_abc.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/configparser.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/copy.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/copyreg.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/datetime.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/enum.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/functools.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/genericpath.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/gettext.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/hashlib.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/heapq.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/io.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/ipaddress.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/keyword.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/linecache.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/locale.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/operator.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/os.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/posixpath.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/quopri.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/random.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/re.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/reprlib.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/selectors.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/shlex.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/signal.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/_sitebuiltins.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/site.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/socket.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/sre_compile.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/sre_constants.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/sre_parse.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/ssl.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/stat.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/string.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/struct.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/subprocess.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/sysconfig.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/_sysconfigdata_m_linux_x86_64-linux-gnu.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/textwrap.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/threading.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/token.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/tokenize.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/traceback.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/types.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/uu.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/warnings.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/weakref.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/__pycache__/_weakrefset.cpython-36.pyc
 dracut_install /usr/lib64/python3.6/site-packages
 dracut_install /usr/lib64/python3.6/urllib
-dracut_install /usr/lib64/python3.6/urllib/__pycache__/__init__.cpython-36.pyc
-dracut_install /usr/lib64/python3.6/urllib/__pycache__/parse.cpython-36.pyc
 dracut_install /usr/lib/locale/en_US.utf8/LC_ADDRESS
 dracut_install /usr/lib/locale/en_US.utf8/LC_COLLATE
 dracut_install /usr/lib/locale/en_US.utf8/LC_CTYPE
@@ -235,3 +148,77 @@ dracut_install /usr/lib/locale/en_US.utf8/LC_NAME
 dracut_install /usr/lib/locale/en_US.utf8/LC_NUMERIC
 dracut_install /usr/lib/locale/en_US.utf8/LC_PAPER
 dracut_install /usr/lib64/python3.6/json/__init__.py /usr/lib64/python3.6/json/decoder.py /usr/lib64/python3.6/json/encoder.py /usr/lib64/python3.6/json/scanner.py /usr/lib64/python3.6/json/tool.py /usr/lib64/python3.6/lib-dynload/_json.cpython-36m-x86_64-linux-gnu.so
+
+# ansible dependencies
+dracut_install /usr/lib64/python3.6/runpy.py
+dracut_install /usr/lib64/python3.6/importlib/__init__.py
+dracut_install /usr/lib64/python3.6/importlib/_bootstrap.py
+dracut_install /usr/lib64/python3.6/importlib/_bootstrap_external.py
+dracut_install /usr/lib64/python3.6/importlib/abc.py
+dracut_install /usr/lib64/python3.6/importlib/machinery.py
+dracut_install /usr/lib64/python3.6/importlib/util.py
+dracut_install /usr/lib64/python3.6/contextlib.py
+dracut_install /usr/lib64/python3.6/pkgutil.py
+dracut_install /usr/lib64/python3.6/shutil.py
+dracut_install /usr/lib64/python3.6/fnmatch.py
+dracut_install /usr/lib64/python3.6/tempfile.py
+dracut_install /usr/lib64/python3.6/zipfile.py
+dracut_install /usr/lib64/python3.6/encodings/cp437.py
+dracut_install /usr/lib64/python3.6/lib-dynload/zlib.cpython-36m-x86_64-linux-gnu.so
+dracut_install /usr/lib64/python3.6/lib-dynload/grp.cpython-36m-x86_64-linux-gnu.so
+dracut_install /usr/lib64/python3.6/lib-dynload/array.cpython-36m-x86_64-linux-gnu.so
+dracut_install /usr/lib64/python3.6/__future__.py
+dracut_install /usr/lib64/python3.6/platform.py
+dracut_install /usr/lib64/python3.6/logging/__init__.py
+dracut_install /usr/lib64/python3.6/logging/config.py
+dracut_install /usr/lib64/python3.6/logging/handlers.py
+dracut_install /usr/lib64/python3.6/optparse.py
+dracut_install /usr/lib64/python3.6/ast.py
+dracut_install /usr/lib64/python3.6/multiprocessing/__init__.py
+dracut_install /usr/lib64/python3.6/multiprocessing/connection.py
+dracut_install /usr/lib64/python3.6/multiprocessing/context.py
+dracut_install /usr/lib64/python3.6/multiprocessing/dummy/__init__.py
+dracut_install /usr/lib64/python3.6/multiprocessing/dummy/connection.py
+dracut_install /usr/lib64/python3.6/multiprocessing/forkserver.py
+dracut_install /usr/lib64/python3.6/multiprocessing/heap.py
+dracut_install /usr/lib64/python3.6/multiprocessing/managers.py
+dracut_install /usr/lib64/python3.6/multiprocessing/pool.py
+dracut_install /usr/lib64/python3.6/multiprocessing/popen_fork.py
+dracut_install /usr/lib64/python3.6/multiprocessing/popen_forkserver.py
+dracut_install /usr/lib64/python3.6/multiprocessing/popen_spawn_posix.py
+dracut_install /usr/lib64/python3.6/multiprocessing/popen_spawn_win32.py
+dracut_install /usr/lib64/python3.6/multiprocessing/process.py
+dracut_install /usr/lib64/python3.6/multiprocessing/queues.py
+dracut_install /usr/lib64/python3.6/multiprocessing/reduction.py
+dracut_install /usr/lib64/python3.6/multiprocessing/resource_sharer.py
+dracut_install /usr/lib64/python3.6/multiprocessing/semaphore_tracker.py
+dracut_install /usr/lib64/python3.6/multiprocessing/sharedctypes.py
+dracut_install /usr/lib64/python3.6/multiprocessing/spawn.py
+dracut_install /usr/lib64/python3.6/multiprocessing/synchronize.py
+dracut_install /usr/lib64/python3.6/multiprocessing/util.py
+dracut_install /usr/lib64/python3.6/pickle.py
+dracut_install /usr/lib64/python3.6/_compat_pickle.py
+dracut_install /usr/lib64/python3.6/queue.py
+dracut_install /usr/lib64/python3.6/glob.py
+dracut_install /usr/lib64/python3.6/distutils/__init__.py
+dracut_install /usr/lib64/python3.6/distutils/archive_util.py
+dracut_install /usr/lib64/python3.6/distutils/cmd.py
+dracut_install /usr/lib64/python3.6/distutils/config.py
+dracut_install /usr/lib64/python3.6/distutils/core.py
+dracut_install /usr/lib64/python3.6/distutils/debug.py
+dracut_install /usr/lib64/python3.6/distutils/dep_util.py
+dracut_install /usr/lib64/python3.6/distutils/dir_util.py
+dracut_install /usr/lib64/python3.6/distutils/errors.py
+dracut_install /usr/lib64/python3.6/distutils/extension.py
+dracut_install /usr/lib64/python3.6/distutils/fancy_getopt.py
+dracut_install /usr/lib64/python3.6/distutils/file_util.py
+dracut_install /usr/lib64/python3.6/distutils/filelist.py
+dracut_install /usr/lib64/python3.6/distutils/log.py
+dracut_install /usr/lib64/python3.6/distutils/spawn.py
+dracut_install /usr/lib64/python3.6/distutils/sysconfig.py
+dracut_install /usr/lib64/python3.6/distutils/text_file.py
+dracut_install /usr/lib64/python3.6/distutils/util.py
+dracut_install /usr/lib64/python3.6/distutils/version.py
+dracut_install /usr/lib64/python3.6/distutils/versionpredicate.py
+dracut_install /usr/lib64/python3.6/getpass.py
+dracut_install /usr/libexec/openssh/sftp-server

genesis/confluent-genesis.spec

@@ -1,5 +1,5 @@
 %define arch x86_64
-Version: 3.1.0
+Version: 3.2.0
 Release:  1
 Name: confluent-genesis-%{arch}
 BuildArch: noarch

misc/slpscan.py

@@ -0,0 +1,857 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is extracting the confluent slp support into
+# a standalone script, for use in environments that
+# can't run full confluent and/or would prefer
+# a utility style approach to SLP
+
+
+import os
+import random
+import select
+import socket
+import struct
+
+_slp_services = set([
+    'service:management-hardware.IBM:integrated-management-module2',
+    'service:lenovo-smm',
+    'service:lenovo-smm2',
+    'service:ipmi',
+    'service:lighttpd',
+    'service:management-hardware.Lenovo:lenovo-xclarity-controller',
+    'service:management-hardware.IBM:chassis-management-module',
+    'service:management-hardware.Lenovo:chassis-management-module',
+    'service:io-device.Lenovo:management-module',
+])
+
+# SLP has a lot of ambition that was unfulfilled in practice.
+# So we have a static footer here to always use 'DEFAULT' scope, no LDAP
+# predicates, and no authentication for service requests
+srvreqfooter = b'\x00\x07DEFAULT\x00\x00\x00\x00'
+# An empty instance of the attribute list extension
+# which is defined in RFC 3059, used to indicate support for that capability
+attrlistext = b'\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00'
+
+try:
+    IPPROTO_IPV6 = socket.IPPROTO_IPV6
+except AttributeError:
+    IPPROTO_IPV6 = 41  # Assume Windows value if socket is missing it
+
+
+def _parse_slp_header(packet):
+    packet = bytearray(packet)
+    if len(packet) < 16 or packet[0] != 2:
+        # discard packets that are obviously useless
+        return None
+    parsed = {
+        'function': packet[1],
+    }
+    (offset, parsed['xid'], langlen) = struct.unpack('!IHH',
+                                           bytes(b'\x00' + packet[7:14]))
+    parsed['lang'] = packet[14:14 + langlen].decode('utf-8')
+    parsed['payload'] = packet[14 + langlen:]
+    if offset:
+        parsed['offset'] = 14 + langlen
+        parsed['extoffset'] = offset
+    return parsed
+
+
+def _pop_url(payload):
+    urllen = struct.unpack('!H', bytes(payload[3:5]))[0]
+    url = bytes(payload[5:5+urllen]).decode('utf-8')
+    if payload[5+urllen] != 0:
+        raise Exception('Auth blocks unsupported')
+    payload = payload[5+urllen+1:]
+    return url, payload
+
+
+def _parse_SrvRply(parsed):
+    """ Modify passed dictionary to have parsed data
+
+
+    :param parsed:
+    :return:
+    """
+    payload = parsed['payload']
+    if len(payload) < 4:
+        return
+    ecode, ucount = struct.unpack('!HH', bytes(payload[0:4]))
+    if ecode:
+        parsed['errorcode'] = ecode
+    payload = payload[4:]
+    parsed['urls'] = []
+    while ucount:
+        ucount -= 1
+        url, payload = _pop_url(payload)
+        parsed['urls'].append(url)
+
+
+def _parse_slp_packet(packet, peer, rsps, xidmap):
+    parsed = _parse_slp_header(packet)
+    if not parsed:
+        return
+    addr = peer[0]
+    if '%' in addr:
+        addr = addr[:addr.index('%')]
+    mac = None
+    if addr in neightable:
+        identifier = neightable[addr]
+        mac = identifier
+    else:
+        identifier = addr
+    if (identifier, parsed['xid']) in rsps:
+        # avoid obviously duplicate entries
+        parsed = rsps[(identifier, parsed['xid'])]
+    else:
+        rsps[(identifier, parsed['xid'])] = parsed
+    if mac and 'hwaddr' not in parsed:
+        parsed['hwaddr'] = mac
+    if parsed['xid'] in xidmap:
+        parsed['services'] = [xidmap[parsed['xid']]]
+    if 'addresses' in parsed:
+        if peer not in parsed['addresses']:
+            parsed['addresses'].append(peer)
+    else:
+        parsed['addresses'] = [peer]
+    if parsed['function'] == 2:  # A service reply
+        _parse_SrvRply(parsed)
+
+
+def _v6mcasthash(srvtype):
+    # The hash algorithm described by RFC 3111
+    nums = bytearray(srvtype.encode('utf-8'))
+    hashval = 0
+    for i in nums:
+        hashval *= 33
+        hashval += i
+        hashval &= 0xffff  # only need to track the lowest 16 bits
+    hashval &= 0x3ff
+    hashval |= 0x1000
+    return '{0:x}'.format(hashval)
+
+
+def _generate_slp_header(payload, multicast, functionid, xid, extoffset=0):
+    if multicast:
+        flags = 0x2000
+    else:
+        flags = 0
+    packetlen = len(payload) + 16  # we have a fixed 16 byte header supported
+    if extoffset:  # if we have an offset, add 16 to account for this function
+        # generating a 16 byte header
+        extoffset += 16
+    if packetlen > 1400:
+        # For now, we aren't intending to support large SLP transmits
+        # raise an exception to help identify if such a requirement emerges
+        raise Exception("TODO: Transmit overflow packets")
+    # We always do SLP v2, and only v2
+    header = bytearray([2, functionid])
+    # SLP uses 24 bit packed integers, so in such places we pack 32 then
+    # discard the high byte
+    header.extend(struct.pack('!IH', packetlen, flags)[1:])
+    # '2' below refers to the length of the language tag
+    header.extend(struct.pack('!IHH', extoffset, xid, 2)[1:])
+    # we only do english (in SLP world, it's not like non-english appears...)
+    header.extend(b'en')
+    return header
+
+def _generate_attr_request(service, xid):
+    service = service.encode('utf-8')
+    payload = bytearray(struct.pack('!HH', 0, len(service)) + service)
+    payload.extend(srvreqfooter)
+    header = _generate_slp_header(payload, False, functionid=6, xid=xid)
+    return header + payload
+
+
+
+def _generate_request_payload(srvtype, multicast, xid, prlist=''):
+    prlist = prlist.encode('utf-8')
+    payload = bytearray(struct.pack('!H', len(prlist)) + prlist)
+    srvtype = srvtype.encode('utf-8')
+    payload.extend(struct.pack('!H', len(srvtype)) + srvtype)
+    payload.extend(srvreqfooter)
+    extoffset = len(payload)
+    payload.extend(attrlistext)
+    header = _generate_slp_header(payload, multicast, functionid=1, xid=xid,
+                                  extoffset=extoffset)
+    return header + payload
+
+
+def _find_srvtype(net, net4, srvtype, addresses, xid):
+    """Internal function to find a single service type
+
+    Helper to do singleton requests to srvtype
+
+    :param net: Socket active
+    :param srvtype: Service type to do now
+    :param addresses:  Pass through of addresses argument from find_targets
+    :return:
+    """
+    data = _generate_request_payload(srvtype, True, xid)
+    if addresses is not None:
+        for addr in addresses:
+            for saddr in socket.getaddrinfo(addr, 427):
+                if saddr[0] == socket.AF_INET:
+                    net4.sendto(data, saddr[4])
+                elif saddr[0] == socket.AF_INET6:
+                    net.sendto(data, saddr[4])
+    else:
+        net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+        v6addrs = []
+        v6hash = _v6mcasthash(srvtype)
+        # do 'interface local' and 'link local'
+        # it shouldn't make sense, but some configurations work with interface
+        # local that do not work with link local
+        v6addrs.append(('ff01::1:' + v6hash, 427, 0, 0))
+        v6addrs.append(('ff02::1:' + v6hash, 427, 0, 0))
+        for idx in list_interface_indexes():
+            # IPv6 multicast is by index, so lead with that
+            net.setsockopt(IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, idx)
+            for sa in v6addrs:
+                try:
+                    net.sendto(data, sa)
+                except socket.error:
+                    # if we hit an interface without ipv6 multicast,
+                    # this can cause an error, skip such an interface
+                    # case in point, 'lo'
+                    pass
+        for i4 in list_ips():
+            if 'broadcast' not in i4:
+                continue
+            addr = i4['addr']
+            bcast = i4['broadcast']
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                           socket.inet_aton(addr))
+            try:
+                net4.sendto(data, ('239.255.255.253', 427))
+            except socket.error as se:
+                # On occasion, multicasting may be disabled
+                # tolerate this scenario and move on
+                if se.errno != 101:
+                    raise
+            net4.sendto(data, (bcast, 427))
+
+
+def _grab_rsps(socks, rsps, interval, xidmap):
+    r = None
+    res = select.select(socks, (), (), interval)
+    if res:
+        r = res[0]
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            refresh_neigh()
+            _parse_slp_packet(rsp, peer, rsps, xidmap)
+            res = select.select(socks, (), (), interval)
+            if not res:
+                r = None
+            else:
+                r = res[0]
+
+
+
+def _parse_attrlist(attrstr):
+    attribs = {}
+    previousattrlen = None
+    attrstr = stringify(attrstr)
+    while attrstr:
+        if len(attrstr) == previousattrlen:
+            raise Exception('Looping in attrstr parsing')
+        previousattrlen = len(attrstr)
+        if attrstr[0] == '(':
+            if ')' not in attrstr:
+                attribs['INCOMPLETE'] = True
+                return attribs
+            currattr = attrstr[1:attrstr.index(')')]
+            if '=' not in currattr:  # Not allegedly kosher, but still..
+                attribs[currattr] = None
+            else:
+                attrname, attrval = currattr.split('=', 1)
+                attribs[attrname] = []
+                for val in attrval.split(','):
+                    if val[:3] == '\\FF':  # we should make this bytes
+                        finalval = bytearray([])
+                        for bnum in attrval[3:].split('\\'):
+                            if bnum == '':
+                                continue
+                            finalval.append(int(bnum, 16))
+                        val = finalval
+                        if 'uuid' in attrname and len(val) == 16:
+                            lebytes = struct.unpack_from(
+                                '<IHH', memoryview(val[:8]))
+                            bebytes = struct.unpack_from(
+                                '>HHI', memoryview(val[8:]))
+                            val = '{0:08X}-{1:04X}-{2:04X}-{3:04X}-' \
+                                  '{4:04X}{5:08X}'.format(
+                                lebytes[0], lebytes[1], lebytes[2], bebytes[0],
+                                bebytes[1], bebytes[2]
+                            ).lower()
+                    attribs[attrname].append(val)
+            attrstr = attrstr[attrstr.index(')'):]
+        elif attrstr[0] == ','[0]:
+            attrstr = attrstr[1:]
+        elif ',' in attrstr:
+            currattr = attrstr[:attrstr.index(',')]
+            attribs[currattr] = None
+            attrstr = attrstr[attrstr.index(','):]
+        else:
+            currattr = attrstr
+            attribs[currattr] = None
+            attrstr = None
+    return attribs
+
+
+def _parse_attrs(data, parsed, xid=None):
+    headinfo = _parse_slp_header(data)
+    if xid is None:
+        xid = parsed['xid']
+    if headinfo['function'] != 7 or headinfo['xid'] != xid:
+        return
+    payload = headinfo['payload']
+    if struct.unpack('!H', bytes(payload[:2]))[0] != 0:
+        return
+    length = struct.unpack('!H', bytes(payload[2:4]))[0]
+    attrstr = bytes(payload[4:4+length])
+    parsed['attributes'] = _parse_attrlist(attrstr)
+
+
+def fix_info(info, handler):
+    if '_attempts' not in info:
+        info['_attempts'] = 10
+    if info['_attempts'] == 0:
+        return
+    info['_attempts'] -= 1
+    _add_attributes(info)
+    handler(info)
+
+def _add_attributes(parsed):
+    xid = parsed.get('xid', 42)
+    attrq = _generate_attr_request(parsed['services'][0], xid)
+    target = None
+    # prefer reaching out to an fe80 if present, to be highly robust
+    # in face of network changes
+    for addr in parsed['addresses']:
+        if addr[0].startswith('fe80'):
+            target = addr
+    # however if no fe80 seen, roll with the first available address
+    if not target:
+        target = parsed['addresses'][0]
+    if len(target) == 4:
+        net = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        net.settimeout(2.0)
+        net.connect(target)
+    except socket.error:
+        return
+    try:
+        net.sendall(attrq)
+        rsp = net.recv(8192)
+        net.close()
+        _parse_attrs(rsp, parsed, xid)
+    except Exception as e:
+        # this can be a messy area, just degrade the quality of rsp
+        # in a bad situation
+        return
+
+
+def unicast_scan(address):
+    pass
+
+def query_srvtypes(target):
+    """Query the srvtypes advertised by the target
+
+    :param target: A sockaddr tuple (if you get the peer info)
+    """
+    payload = b'\x00\x00\xff\xff\x00\x07DEFAULT'
+    header = _generate_slp_header(payload, False, functionid=9, xid=1)
+    packet = header + payload
+    if len(target) == 2:
+        net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    elif len(target) == 4:
+        net = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        raise Exception('Unrecognized target {0}'.format(repr(target)))
+    tries = 3
+    connected = False
+    while tries and not connected:
+        tries -= 1
+        try:
+            net.settimeout(1.0)
+            net.connect(target)
+            connected = True
+        except socket.error:
+            pass
+    if not connected:
+        return [u'']
+    net.sendall(packet)
+    rs = net.recv(8192)
+    net.close()
+    parsed = _parse_slp_header(rs)
+    if parsed:
+        payload = parsed['payload']
+        if payload[:2] != '\x00\x00':
+            return
+        stypelen = struct.unpack('!H', bytes(payload[2:4]))[0]
+        stypes = payload[4:4+stypelen].decode('utf-8')
+        return stypes.split(',')
+
+def rescan(handler):
+    known_peers = set([])
+    for scanned in scan():
+        for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neightable:
+                continue
+            if addr in known_peers:
+                break
+            known_peers.add(addr)
+        else:
+            handler(scanned)
+
+
+def snoop(handler, protocol=None):
+    """Watch for SLP activity
+
+    handler will be called with a dictionary of relevant attributes
+
+    :param handler:
+    :return:
+    """
+    try:
+        active_scan(handler, protocol)
+    except Exception as e:
+        raise
+    net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    slpg = socket.inet_pton(socket.AF_INET6, 'ff01::123')
+    slpg2 = socket.inet_pton(socket.AF_INET6, 'ff02::123')
+    for i6idx in list_interface_indexes():
+        mreq = slpg + struct.pack('=I', i6idx)
+        net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
+        mreq = slpg2 + struct.pack('=I', i6idx)
+        net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    for i4 in list_ips():
+        if 'broadcast' not in i4:
+            continue
+        slpmcast = socket.inet_aton('239.255.255.253') + \
+            socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            slpmcast)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            # socket in use can occur when aliased ipv4 are encountered
+    net.bind(('', 427))
+    net4.bind(('', 427))
+
+    while True:
+        try:
+            newmacs = set([])
+            r, _, _ = select.select((net, net4), (), (), 60)
+            # clear known_peers and peerbymacaddress
+            # to avoid stale info getting in...
+            # rely upon the select(0.2) to catch rapid fire and aggregate ip
+            # addresses that come close together
+            # calling code needs to understand deeper context, as snoop
+            # will now yield dupe info over time
+            known_peers = set([])
+            peerbymacaddress = {}
+            while r:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    ip = peer[0].partition('%')[0]
+                    if peer in known_peers:
+                        continue
+                    if ip not in neightable:
+                        update_neigh()
+                    if ip not in neightable:
+                        continue
+                    known_peers.add(peer)
+                    mac = neightable[ip]
+                    if mac in peerbymacaddress:
+                        peerbymacaddress[mac]['addresses'].append(peer)
+                    else:
+                        q = query_srvtypes(peer)
+                        if not q or not q[0]:
+                            # SLP might have started and not ready yet
+                            # ignore for now
+                            known_peers.discard(peer)
+                            continue
+                        # we want to prioritize the very well known services
+                        svcs = []
+                        for svc in q:
+                            if svc in _slp_services:
+                                svcs.insert(0, svc)
+                            else:
+                                svcs.append(svc)
+                        peerbymacaddress[mac] = {
+                            'services': svcs,
+                            'addresses': [peer],
+                        }
+                    newmacs.add(mac)
+                r, _, _ = select.select((net, net4), (), (), 0.2)
+            for mac in newmacs:
+                peerbymacaddress[mac]['xid'] = 1
+                _add_attributes(peerbymacaddress[mac])
+                peerbymacaddress[mac]['hwaddr'] = mac
+                peerbymacaddress[mac]['protocol'] = protocol
+                for srvurl in peerbymacaddress[mac].get('urls', ()):
+                    if len(srvurl) > 4:
+                        srvurl = srvurl[:-3]
+                    if srvurl.endswith('://Athena:'):
+                        continue
+                if 'service:ipmi' in peerbymacaddress[mac]['services']:
+                    continue
+                if 'service:lightttpd' in peerbymacaddress[mac]['services']:
+                    currinf = peerbymacaddress[mac]
+                    curratt = currinf.get('attributes', {})
+                    if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+                        peerbymacaddress[mac]['services'] = ['service:lenovo-tsm']
+                    else:
+                        continue
+                handler(peerbymacaddress[mac])
+        except Exception as e:
+            raise
+
+
+def active_scan(handler, protocol=None):
+    known_peers = set([])
+    for scanned in scan():
+        for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neightable:
+                continue
+            if addr in known_peers:
+                break
+            known_peers.add(addr)
+        else:
+            scanned['protocol'] = protocol
+            handler(scanned)
+
+
+def scan(srvtypes=_slp_services, addresses=None, localonly=False):
+    """Find targets providing matching requested srvtypes
+
+    This is a generator that will iterate over respondants to the SrvType
+    requested.
+
+    :param srvtypes: An iterable list of the service types to find
+    :param addresses: An iterable of addresses/ranges.  Default is to scan
+                      local network segment using multicast and broadcast.
+                      Each address can be a single address, hyphen-delimited
+                      range, or an IP/CIDR indication of a network.
+    :return: Iterable set of results
+    """
+    net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    # increase RCVBUF to max, mitigate chance of
+    # failure due to full buffer.
+    net.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16777216)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16777216)
+    # SLP is very poor at scanning large counts and managing it, so we
+    # must make the best of it
+    # Some platforms/config default to IPV6ONLY, we are doing IPv4
+    # too, so force it
+    #net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+    # we are going to do broadcast, so allow that...
+    initxid = random.randint(0, 32768)
+    xididx = 0
+    xidmap = {}
+    # First we give fast repsonders of each srvtype individual chances to be
+    # processed, mitigating volume of response traffic
+    rsps = {}
+    for srvtype in srvtypes:
+        xididx += 1
+        _find_srvtype(net, net4, srvtype, addresses, initxid + xididx)
+        xidmap[initxid + xididx] = srvtype
+        _grab_rsps((net, net4), rsps, 0.1, xidmap)
+        # now do a more slow check to work to get stragglers,
+        # but fortunately the above should have taken the brunt of volume, so
+        # reduced chance of many responses overwhelming receive buffer.
+    _grab_rsps((net, net4), rsps, 1, xidmap)
+    # now to analyze and flesh out the responses
+    for id in rsps:
+        for srvurl in rsps[id].get('urls', ()):
+            if len(srvurl) > 4:
+                srvurl = srvurl[:-3]
+            if srvurl.endswith('://Athena:'):
+                continue
+        if 'service:ipmi' in rsps[id]['services']:
+            continue
+        if localonly:
+            for addr in rsps[id]['addresses']:
+                if 'fe80' in addr[0]:
+                    break
+            else:
+                continue
+        _add_attributes(rsps[id])
+        if 'service:lighttpd' in rsps[id]['services']:
+            currinf = rsps[id]
+            curratt = currinf.get('attributes', {})
+            if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+               currinf['services'] = ['service:lenovo-tsm']
+               serialnumber = curratt.get('Product-Serial', curratt.get('SerialNumber', None))
+               if serialnumber:
+                   curratt['enclosure-serial-number'] = serialnumber
+               mtm = curratt.get('Machine-Type', curratt.get('Product-Name', None))
+               if mtm:
+                   mtm[0] = mtm[0].rstrip()
+                   curratt['enclosure-machinetype-model'] = mtm
+            else:
+                continue
+        del rsps[id]['payload']
+        del rsps[id]['function']
+        del rsps[id]['xid']
+        yield rsps[id]
+
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A consolidated manage of neighbor table information management.
+# Ultimately, this should use AF_NETLINK, but in the interest of time,
+# use ip neigh for the moment
+
+import subprocess
+import os
+
+neightable = {}
+neightime = 0
+
+import re
+
+_validmac = re.compile('..:..:..:..:..:..')
+
+
+def update_neigh():
+    global neightable
+    global neightime
+    neightable = {}
+    if os.name == 'nt':
+        return
+    ipn = subprocess.Popen(['ip', 'neigh'], stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+    (neighdata, err) = ipn.communicate()
+    neighdata = stringify(neighdata)
+    for entry in neighdata.split('\n'):
+        entry = entry.split(' ')
+        if len(entry) < 5 or not entry[4]:
+            continue
+        if entry[0] in ('192.168.0.100', '192.168.70.100', '192.168.70.125'):
+            # Note that these addresses are common static ip addresses
+            # that are hopelessly ambiguous if there are many
+            # so ignore such entries and move on
+            # ideally the system network steers clear of this landmine of
+            # a subnet, but just in case
+            continue
+        if not _validmac.match(entry[4]):
+            continue
+        neightable[entry[0]] = entry[4]
+    neightime = os.times()[4]
+
+
+def refresh_neigh():
+    global neightime
+    if os.name == 'nt':
+        return
+    if os.times()[4] > (neightime + 30):
+        update_neigh()
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2014 IBM Corporation
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Various utility functions that do not neatly fit into one category or another
+import base64
+import hashlib
+import netifaces
+import os
+import re
+import socket
+import ssl
+import struct
+
+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8', errors='replace')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
+
+def list_interface_indexes():
+    # Getting the interface indexes in a portable manner
+    # would be better, but there's difficulty from a python perspective.
+    # For now be linux specific
+    try:
+        for iface in os.listdir('/sys/class/net/'):
+            if not os.path.exists('/sys/class/net/{0}/ifindex'.format(iface)):
+                continue
+            ifile = open('/sys/class/net/{0}/ifindex'.format(iface), 'r')
+            intidx = int(ifile.read())
+            ifile.close()
+            yield intidx
+    except (IOError, OSError):
+        # Probably situation is non-Linux, just do limited support for
+        # such platforms until other people come along
+        for iface in netifaces.interfaces():
+            addrinfo = netifaces.ifaddresses(iface).get(socket.AF_INET6, [])
+            for addr in addrinfo:
+                v6addr = addr.get('addr', '').partition('%')[2]
+                if v6addr:
+                    yield(int(v6addr))
+                    break
+        return
+
+
+def list_ips():
+    # Used for getting addresses to indicate the multicast address
+    # as well as getting all the broadcast addresses
+    for iface in netifaces.interfaces():
+        addrs = netifaces.ifaddresses(iface)
+        if netifaces.AF_INET in addrs:
+            for addr in addrs[netifaces.AF_INET]:
+                yield addr
+
+def randomstring(length=20):
+    """Generate a random string of requested length
+
+    :param length: The number of characters to produce, defaults to 20
+    """
+    chunksize = length // 4
+    if length % 4 > 0:
+        chunksize += 1
+    strval = base64.urlsafe_b64encode(os.urandom(chunksize * 3))
+    return stringify(strval[0:length])
+
+
+def securerandomnumber(low=0, high=4294967295):
+    """Return a random number within requested range
+
+    Note that this function will not return smaller than 0 nor larger
+    than 2^32-1 no matter what is requested.
+    The python random number facility does not provide characteristics
+    appropriate for secure rng, go to os.urandom
+
+    :param low: Smallest number to return (defaults to 0)
+    :param high: largest number to return (defaults to 2^32-1)
+    """
+    number = -1
+    while number < low or number > high:
+        number = struct.unpack("I", os.urandom(4))[0]
+    return number
+
+
+def monotonic_time():
+    """Return a monotoc time value
+
+    In scenarios like timeouts and such, monotonic timing is preferred.
+    """
+    # for now, just support POSIX systems
+    return os.times()[4]
+
+
+def get_certificate_from_file(certfile):
+    cert = open(certfile, 'r').read()
+    inpemcert = False
+    prunedcert = ''
+    for line in cert.split('\n'):
+        if '-----BEGIN CERTIFICATE-----' in line:
+            inpemcert = True
+        if inpemcert:
+            prunedcert += line
+        if '-----END CERTIFICATE-----' in line:
+            break
+    return ssl.PEM_cert_to_DER_cert(prunedcert)
+
+
+def get_fingerprint(certificate, algo='sha512'):
+    if algo == 'sha256':
+        return 'sha256$' + hashlib.sha256(certificate).hexdigest()
+    elif algo == 'sha512':
+        return 'sha512$' + hashlib.sha512(certificate).hexdigest()
+    raise Exception('Unsupported fingerprint algorithm ' + algo)
+
+
+def cert_matches(fingerprint, certificate):
+    if not fingerprint or not certificate:
+        return False
+    algo, _, fp = fingerprint.partition('$')
+    newfp = None
+    if algo in ('sha512', 'sha256'):
+        newfp = get_fingerprint(certificate, algo)
+    return newfp and fingerprint == newfp
+
+
+numregex = re.compile('([0-9]+)')
+
+def naturalize_string(key):
+    """Analyzes string in a human way to enable natural sort
+
+    :param nodename: The node name to analyze
+    :returns: A structure that can be consumed by 'sorted'
+    """
+    return [int(text) if text.isdigit() else text.lower()
+            for text in re.split(numregex, key)]
+
+def natural_sort(iterable):
+    """Return a sort using natural sort if possible
+
+    :param iterable:
+    :return:
+    """
+    try:
+        return sorted(iterable, key=naturalize_string)
+    except TypeError:
+        # The natural sort attempt failed, fallback to ascii sort
+        return sorted(iterable)
+
+
+if __name__ == '__main__':
+    print(repr(list(scan(addresses=['10.240.52.189']))))