Merge branch 'master' into nodesearch

Fix HTML view with unicode
Fix handling autosense with python3
2026-09-29 08:41:00 +00:00 · 2019-10-17 13:46:23 -04:00 · 2019-10-17 11:25:03 -04:00 · 2019-10-17 09:47:46 -04:00 · 2019-10-17 09:31:01 -04:00 · 2019-10-16 13:50:23 -04:00
140 changed files with 9103 additions and 7758 deletions
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -35,6 +35,9 @@ import confluent.client
 argparser = optparse.OptionParser(usage="Usage: <other command> | %prog [options]")
 argparser.add_option('-a', '--abbreviate', action='store_true',
                     help='Attempt to use confluent server to shorten noderanges')
+argparser.add_option('-b', '--base',
+                     help='Use given node as reference for comparison when '
+                          'using -d, instead of using the most common result')
 argparser.add_option('-d', '--diff', action='store_true',
                     help='Show what differs between most common '
                          'output group and others')
@@ -48,6 +51,8 @@ argparser.add_option('-c', '--count', action='store_true',
 argparser.add_option('-r', '--reverse', action='store_true',
                     help='Reverse sort order to show biggest output group '
                          'last')
+argparser.add_option('-l', '--log', action='store', type='string', dest='log',
+                     help='Log each output to file, using {node} as a placeholder for node.')
 (options, args) = argparser.parse_args()
 if sys.stdin.isatty():
    argparser.print_help()
@@ -63,7 +68,7 @@ else:
 def print_current():
    if options.diff:
        grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
-                          reverse=options.reverse)
+                          reverse=options.reverse, basenode=options.base)
    else:
        grouped.print_all(skipmodal=options.skipcommon,
                               count=options.count,
@@ -80,6 +85,12 @@ while fullline:
            continue
        if ': ' not in line:
            line = 'UNKNOWN: ' + line
+        if options.log:
+            node, output = line.split(': ', 1)
+            currlog = options.log.format(node=node, nodename=node)
+            with open(currlog, mode='a')  as log:
+                log.write(output + '\n')
+            continue
        grouped.add_line(*line.split(': ', 1))
        if options.watch:
            if not holdoff:
@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,6 +49,7 @@ import select
 import shlex
 import signal
 import socket
+import struct
 import sys
 import time
 try:
@@ -88,6 +89,15 @@ except NameError:
 netserver = None
 laststate = {}

+try:
+    input = raw_input
+except NameError:
+    pass
+
+class BailOut(Exception):
+    def __init__(self, errorcode=0):
+        self.errorcode = errorcode
+

 def print_help():
    print("confetty provides a filesystem like interface to confluent.  "
@@ -189,9 +199,9 @@ def prompt():
    if os.environ.get('TERM', '') not in ('linux'):
        sys.stdout.write('\x1b]0;confetty: %s\x07' % target)
    try:
-        return raw_input(target + ' -> ')
+        return input(target + ' -> ')
    except KeyboardInterrupt:
-        print ""
+        print("")
        return ""
    except EOFError:  # ctrl-d
        print("exit")
@@ -290,7 +300,7 @@ currchildren = None

 def print_result(res):
    if 'errorcode' in res or 'error' in res:
-        print res['error']
+        print(res['error'])
        return
    if 'databynode' in res:
        print_result(res['databynode'])
@@ -303,9 +313,9 @@ def print_result(res):
            attrstr = '%s=%s' % (key, recurse_format(res[key]))
        elif not isinstance(res[key], dict):
            try:
-                print '{0}: {1}'.format(key, res[key])
+                print('{0}: {1}'.format(key, res[key]))
            except UnicodeEncodeError:
-                print '{0}: {1}'.format(key, repr(res[key]))
+                print('{0}: {1}'.format(key, repr(res[key])))
            continue
        elif 'value' in res[key] and res[key]['value'] is not None:
            attrstr = '%s="%s"' % (key, res[key]['value'])
@@ -318,7 +328,7 @@ def print_result(res):
        else:
            sys.stdout.write('{0}: '.format(key))
            if isinstance(res[key], str) or isinstance(res[key], unicode):
-                print res[key]
+                print(res[key])
            else:
                print_result(res[key])
            continue
@@ -353,7 +363,7 @@ def do_command(command, server):
    if argv[0] == 'exit':
        if os.environ.get('TERM', '') not in ('linux'):
            sys.stdout.write('\x1b]0;\x07')
-        sys.exit(0)
+        raise Bailout()
    elif argv[0] in ('help', '?'):
        return print_help()
    elif argv[0] == 'cd':
@@ -417,10 +427,10 @@ def do_command(command, server):
        for res in session.read(targpath):
            if 'item' in res:  # a link relation
                if type(res['item']) == dict:
-                    print res['item']["href"]
+                    print(res['item']["href"])
                else:
                    for item in res['item']:
-                        print item["href"]
+                        print(item["href"])
            else:  # generic attributes to list
                if 'error' in res:
                    sys.stderr.write(res['error'] + '\n')
@@ -434,6 +444,10 @@ def do_command(command, server):
        currconsole = targpath
        startrequest = {'operation': 'start', 'path': targpath,
                        'parameters': {}}
+        height, width = struct.unpack(
+            'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
+        startrequest['parameters']['width'] = width
+        startrequest['parameters']['height'] = height
        for param in argv[2:]:
            (parmkey, parmval) = param.split("=")
            startrequest['parameters'][parmkey] = parmval
@@ -478,7 +492,8 @@ def createresource(args):
        collection = targpath
    else:
        collection, _, resname = targpath.rpartition('/')
-        keydata['name'] = resname
+        if 'name' not in keydata:
+            keydata['name'] = resname
    makecall(session.create, (collection, keydata))


@@ -493,6 +508,13 @@ def makecall(callout, args):
            if 'errorcode' in response:
                exitcode = response['errorcode']
            sys.stderr.write('Error: ' + response['error'] + '\n')
+        if 'databynode' in response:
+            lresponse = response['databynode']
+            for node in lresponse:
+                if 'errorcode' in lresponse[node]:
+                    exitcode = lresponse[node]['errorcode']
+                if 'error' in lresponse[node]:
+                    sys.stderr.write('{0}: Error - {1}\n'.format(node, lresponse[node]['error']))


 def clearvalues(resource, attribs):
@@ -575,11 +597,20 @@ def fullpath_target(currpath, forcepath=False):
        ntarget += '/'
    return ntarget

+def do_resize(a, b):
+    if not inconsole:
+        return
+    height, width = struct.unpack(
+        'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
+    tlvdata.send(session.connection, {'operation': 'resize', 'width': width,
+                                      'height': height})
+

 def startconsole(nodename):
    global inconsole
    global consolename
    global didconsole
+    signal.signal(signal.SIGWINCH, do_resize)
    didconsole = True
    consolename = nodename
    tty.setraw(sys.stdin.fileno())
@@ -598,11 +629,12 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
        if oldtcattr is not None:
            termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
    # Request default color scheme, to undo potential weirdness of terminal
-    sys.stdout.write('\x1b[m')
+    if sys.stdout.isatty():
+        sys.stdout.write('\x1b[m')
    if fullexit:
-        if os.environ.get('TERM', '') not in ('linux'):
+        if sys.stdout.isatty() and os.environ.get('TERM', '') not in ('linux'):
            sys.stdout.write('\x1b]0;\x07')
-        sys.exit(code)
+        raise BailOut(code)
    else:
        tlvdata.send(session.connection, {'operation': 'stop',
                                          'path': currconsole})
@@ -795,6 +827,10 @@ parser.add_option("-s", "--server", dest="netserver",
 parser.add_option("-c", "--control", dest="controlpath",
                  help="Path to offer terminal control",
                  metavar="PATH")
+parser.add_option(
+    '-m', '--mintime', default=0,
+    help='Minimum time to run or else pause for input (used to keep a '
+         'terminal from closing quickly on error)')
 opts, shellargs = parser.parse_args()

 username = None
@@ -819,49 +855,10 @@ def server_connect():
        passphrase = os.environ['CONFLUENT_PASSPHRASE']
        session.authenticate(username, passphrase)
    while not session.authenticated:
-        username = raw_input("Name: ")
+        username = input("Name: ")
        passphrase = getpass.getpass("Passphrase: ")
        session.authenticate(username, passphrase)

-
-try:
-    server_connect()
-except EOFError, KeyboardInterrupt:
-    sys.exit(0)
-except socket.gaierror:
-    sys.stderr.write('Could not connect to confluent\n')
-    sys.exit(1)
-# clear on start can help with readable of TUI, but it
-# can be annoying, so for now don't do it.
-# sys.stdout.write('\x1b[H\x1b[J')
-# sys.stdout.flush()
-
-if sys.stdout.isatty():
-    import readline
-
-    readline.parse_and_bind("tab: complete")
-    readline.parse_and_bind("set bell-style none")
-    dl = readline.get_completer_delims().replace('-', '')
-    readline.set_completer_delims(dl)
-    readline.set_completer(completer)
-
-doexit = False
-inconsole = False
-pendingcommand = ""
-session_node = get_session_node(shellargs)
-if session_node is not None:
-    consoleonly = True
-    do_command("start /nodes/%s/console/session" % session_node, netserver)
-    doexit = True
-elif shellargs:
-    command = " ".join(shellargs)
-    do_command(command, netserver)
-    quitconfetty(fullexit=True, fixterm=False)
-
-powerstate = None
-powertime = None
-
-
 def check_power_state():
    tlvdata.send(
        session.connection,
@@ -870,86 +867,148 @@ def check_power_state():
    return


-while inconsole or not doexit:
-    if inconsole:
-        try:
-            rdylist, _, _ = select.select(
-                (sys.stdin, session.connection), (), (), 10)
-        except select.error:
-            rdylist = ()
-        for fh in rdylist:
-            if fh == session.connection:
-                # this only should get called in the
-                # case of a console session
-                # each command should slurp up all relevant
-                # recv potential
-                #fh.read()
-                try:
-                    data = tlvdata.recv(fh)
-                except Exception:
-                    data = None
-                if type(data) == dict:
-                    updatestatus(data)
-                    continue
-                if data is not None:
-                    if clearpowermessage:
-                        sys.stdout.write("\x1b[2J\x1b[;H")
-                        clearpowermessage = False
-                    try:
-                        sys.stdout.write(data)
-                    except IOError:  # Some times circumstances are bad
-                        # resort to byte at a time...
-                        for d in data:
-                            sys.stdout.write(d)
-                    now = time.time()
-                    if ('showtime' not in laststate or
-                            (now // 60) != laststate['showtime'] // 60):
-                        # don't bother churning if minute does not change
-                        laststate['showtime'] = now
-                        updatestatus()
-                    sys.stdout.flush()
-                else:
-                    deadline = 5
-                    connected = False
-                    while not connected and deadline > 0:
-                        try:
-                            server_connect()
-                            connected = True
-                        except (socket.gaierror, socket.error):
-                            pass
-                        if not connected:
-                            time.sleep(1)
-                            deadline -=1
-                    if connected:
-                        do_command(
-                            "start /nodes/%s/console/session skipreplay=True" % consolename,
-                            netserver)
-                    else:
-                        doexit = True
-                        inconsole = False
-                        sys.stdout.write("\r\n[remote disconnected]\r\n")
-                    break
-            else:
-                try:
-                    myinput = fh.read()
-                    myinput = check_escape_seq(myinput, fh)
-                    if myinput:
-                        tlvdata.send(session.connection, myinput)
-                except IOError:
-                    pass
-        if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
-            powertime = time.time()
-            powerstate = True
-            check_power_state()
-    else:
-        currcommand = prompt()
-        try:
-            do_command(currcommand, netserver)
-        except socket.error:
+if sys.stdout.isatty():
+    import readline
+
+
+def main():
+    global inconsole
+    try:
+        server_connect()
+    except (EOFError, KeyboardInterrupt) as _:
+        raise BailOut(0)
+    except socket.gaierror:
+        sys.stderr.write('Could not connect to confluent\n')
+        raise BailOut(1)
+    # clear on start can help with readable of TUI, but it
+    # can be annoying, so for now don't do it.
+    # sys.stdout.write('\x1b[H\x1b[J')
+    # sys.stdout.flush()
+    global powerstate, powertime, clearpowermessage
+
+
+    if sys.stdout.isatty():
+
+        readline.parse_and_bind("tab: complete")
+        readline.parse_and_bind("set bell-style none")
+        dl = readline.get_completer_delims().replace('-', '')
+        readline.set_completer_delims(dl)
+        readline.set_completer(completer)
+
+    doexit = False
+    inconsole = False
+    pendingcommand = ""
+    session_node = get_session_node(shellargs)
+    if session_node is not None:
+        consoleonly = True
+        do_command("start /nodes/%s/console/session" % session_node, netserver)
+        doexit = True
+    elif shellargs:
+        command = " ".join(shellargs)
+        do_command(command, netserver)
+        quitconfetty(fullexit=True, fixterm=False)
+
+    powerstate = None
+    powertime = None
+
+    while inconsole or not doexit:
+        if inconsole:
+            try:
+                rdylist, _, _ = select.select(
+                    (sys.stdin, session.connection), (), (), 10)
+            except select.error:
+                rdylist = ()
+            for fh in rdylist:
+                if fh == session.connection:
+                    # this only should get called in the
+                    # case of a console session
+                    # each command should slurp up all relevant
+                    # recv potential
+                    #fh.read()
+                    try:
+                        data = tlvdata.recv(fh)
+                    except Exception:
+                        data = None
+                    if type(data) == dict:
+                        updatestatus(data)
+                        continue
+                    if data is not None:
+                        data = client.stringify(data)
+                        if clearpowermessage:
+                            sys.stdout.write("\x1b[2J\x1b[;H")
+                            clearpowermessage = False
+                        try:
+                            sys.stdout.write(data)
+                        except IOError:  # Some times circumstances are bad
+                            # resort to byte at a time...
+                            for d in data:
+                                sys.stdout.write(d)
+                        now = time.time()
+                        if ('showtime' not in laststate or
+                                (now // 60) != laststate['showtime'] // 60):
+                            # don't bother churning if minute does not change
+                            laststate['showtime'] = now
+                            updatestatus()
+                        sys.stdout.flush()
+                    else:
+                        deadline = 5
+                        connected = False
+                        while not connected and deadline > 0:
+                            try:
+                                server_connect()
+                                connected = True
+                            except (socket.gaierror, socket.error):
+                                pass
+                            if not connected:
+                                time.sleep(1)
+                                deadline -=1
+                        if connected:
+                            do_command(
+                                "start /nodes/%s/console/session skipreplay=True" % consolename,
+                                netserver)
+                        else:
+                            doexit = True
+                            inconsole = False
+                            sys.stdout.write("\r\n[remote disconnected]\r\n")
+                        break
+                else:
+                    try:
+                        myinput = fh.read()
+                        myinput = check_escape_seq(myinput, fh)
+                        if myinput:
+                            tlvdata.send(session.connection, myinput)
+                    except IOError:
+                        pass
+            if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
+                powertime = time.time()
+                powerstate = True
+                check_power_state()
+        else:
+            currcommand = prompt()
            try:
-                server_connect()
                do_command(currcommand, netserver)
            except socket.error:
-                doexit = True
-                sys.stdout.write('Lost connection to server')
-quitconfetty(fullexit=True)
+                try:
+                    server_connect()
+                    do_command(currcommand, netserver)
+                except socket.error:
+                    doexit = True
+                    sys.stdout.write('Lost connection to server')
+    quitconfetty(fullexit=True)
+
+if __name__ == '__main__':
+    errcode = 0
+    deadline = 0
+    if opts.mintime:
+        deadline = os.times()[4] + float(opts.mintime)
+    try:
+        main()
+    except BailOut as e:
+        errcode = e.errorcode
+    finally:
+        if deadline and os.times()[4] < deadline:
+            sys.stderr.write('[Exited early, hit enter to continue]')
+            sys.stdin.readline()
+        if errcode == 0:
+            errcode = exitcode
+        sys.exit(errcode)
@@ -0,0 +1,71 @@
+#!/usr/bin/python2
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o ansible.hosts
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='Ansible hosts file')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not options.output:
+        sys.stderr.write('Output file must be specified by -o\n')
+        sys.exit(1)
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    nodesbygroup = {}
+    with open(options.output, 'w') as importfile:
+        needempty = False
+        for node in sortutil.natural_sort(databynode):
+            data = databynode[node]
+            if not data.get('groups', []):
+                importfile.write(node + '\n')
+                needempty = True
+            for g in data.get('groups', []):
+                if g not in nodesbygroup:
+                    nodesbygroup[g] = set([node])
+                else:
+                    nodesbygroup[g].add(node)
+        if needempty:
+            importfile.write('\n')
+        for group in sortutil.natural_sort(nodesbygroup):
+            importfile.write('[{0}]\n'.format(group))
+            for node in sortutil.natural_sort(nodesbygroup[group]):
+                importfile.write('{0}\n'.format(node))
+            importfile.write('\n')
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,76 @@
+#!/usr/bin/python2
+import csv
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+headers = '# Type,Serial Number,Current IP,Current username,Current password,New password,Recovery password,Switch enable password,New IPv4,IPv4 Subnet mask,IPv4 Default gateway,IPv4 DNS1,IPv4 DNS2,New IPv6,IPv6 Prefix,IPv6 Default gateway,IPv6 DNS1,IPv6 DNS2,Domain,Host name,Display name,Rack,Lowest Rack Unit,Height,Force,Stored credentials ID,Managed authentication,Group Name'.split(',')
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o bulkimport.csv
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='File to write for bulk import into xClarity Administrator')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not options.output:
+        sys.stderr.write('Output file must be specified by -o\n')
+        sys.exit(1)
+    if 'XCCUSER' not in os.environ or 'XCCPASS' not in os.environ:
+        sys.stderr.write('Must specify XCCUSER and XCCPASS in environment variables\n')
+        sys.exit(1)
+    xccuser = os.environ['XCCUSER']
+    xccpass = os.environ['XCCPASS']
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    for res in sess.read(
+            '/noderange/{0}/configuration/management_controller/net_interfaces/management'.format(
+                    noderange)):
+        for node in res.get('databynode', {}):
+            currip = res['databynode'][node].get('ipv4_address', {}).get(
+                'value', '')
+            if currip:
+                databynode[node]['hardwaremanagement.manager'] = {
+                    'value': currip.split('/', 1)[0]}
+    with open(options.output, 'w') as importfile:
+        bulkimport = csv.writer(importfile)
+        bulkimport.writerow(headers)
+        for node in sortutil.natural_sort(databynode):
+            data = databynode[node]
+            row = ['server', lookupdata(data, 'id.serial'), lookupdata(data, 'hardwaremanagement.manager'), xccuser, xccpass, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', node, lookupdata(data, 'location.rack'), lookupdata(data, 'location.u'), '', '', '','', '']
+            bulkimport.writerow(row)
+
+
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,103 @@
+#!/usr/bin/python2
+import csv
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o xcatnodes.def
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='xCAT stanza file')
+    argparser.add_option('-m', '--macs',
+                         help='xCAT macs.csv to write')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not (options.output or options.macs):
+        sys.stderr.write('Output file must be specified by -o or -m\n')
+        sys.exit(1)
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    if options.output:
+        with open(options.output, 'w') as importfile:
+            for node in sortutil.natural_sort(databynode):
+                xcatattrs = collect_attribute_data(databynode, node)
+                importfile.write('{0}:\n'.format(node))
+                importfile.write('   objtype=node\n')
+                importfile.write('   arch=x86_64\n')
+                importfile.write('   netboot=xnba\n')
+                importfile.write('   mgt=ipmi\n')
+                for attr in xcatattrs:
+                    if xcatattrs[attr] is None:
+                        continue
+                    importfile.write('   {0}={1}\n'.format(attr,
+                                                           xcatattrs[attr]))
+                importfile.write('\n')
+    if options.macs:
+        with open(options.macs, 'w') as importfile:
+            macsv = csv.writer(importfile)
+            macsv.writerow(['#node', 'mac'])
+            for node in sortutil.natural_sort(databynode):
+                xcatattrs = collect_attribute_data(databynode, node)
+                macsv.writerow([node, xcatattrs['mac']])
+
+
+
+def collect_attribute_data(databynode, node):
+    data = databynode[node]
+    xcatattrs = {}
+    xcatattrs['groups'] = ','.join(data.get('groups', []))
+    xcatattrs['bmc'] = data.get('hardwaremanagement.manager', {}).get(
+        'value', None)
+    xcatattrs['mpa'] = data.get('enclosure.manager', {}).get(
+        'value', None)
+    xcatattrs['slotid'] = data.get('enclosure.bay', {}).get(
+        'value', None)
+    gotmac = False
+    for key in data:
+        if key.startswith('net.') and 'hwaddr' in key:
+            currmac = data[key].get('value', None)
+            if currmac:
+                if gotmac:
+                    sys.stderr.write(
+                        'Ignoring {0} and using only {1} for mac, '
+                        'multiple macs not supported by '
+                        'confluent2xcat\n'.format(key, gotmac))
+                    continue
+                gotmac = key
+                xcatattrs['mac'] = currmac
+    return xcatattrs
+
+
+if __name__ == '__main__':
+    main()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -17,6 +17,7 @@

 __author__ = 'alin37'

+from getpass import getpass
 import optparse
 import os
 import signal
@@ -47,6 +48,8 @@ argparser.add_option('-e', '--environment', action='store_true',
                          'same name')
 argparser.add_option('-c', '--clear', action='store_true',
                     help='Clear attributes')
+argparser.add_option('-p', '--prompt', action='store_true',
+                     help='Prompt for attribute values interactively')
 (options, args) = argparser.parse_args()


@@ -67,12 +70,24 @@ exitcode = 0
 nodetype="noderange"

 if len(args) > 1:
-    if "=" in args[1] or options.clear or options.environment:
+    if "=" in args[1] or options.clear or options.environment or options.prompt:
        if "=" in args[1] and options.clear:
            print("Can not clear and set at the same time!")
            argparser.print_help()
            sys.exit(1)
-        exitcode=client.updateattrib(session,args,nodetype, noderange, options)
+        argassign = None
+        if options.prompt:
+            argassign = {}
+            for arg in args[1:]:
+                oneval = 1
+                twoval = 2
+                while oneval != twoval:
+                    oneval = getpass('Enter value for {0}: '.format(arg))
+                    twoval = getpass('Confirm value for {0}: '.format(arg))
+                    if oneval != twoval:
+                        print('Values did not match.')
+                argassign[arg] = twoval
+        exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
    try:
        # setting user output to what the user inputs
        if args[1] == 'all':
@@ -111,6 +126,6 @@ else:
            sys.stderr.write(res['error'] + '\n')
            exitcode = 1
        else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))

-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -56,7 +56,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes

 for node in goodNodes:
-    print node + ": BMC Reset Successful"
+    print(node + ": BMC Reset Successful")


 sys.exit(success)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015 Lenovo
@@ -35,6 +35,9 @@ argparser = optparse.OptionParser()
 argparser.add_option('-b', '--bios', dest='biosmode',
                     action='store_true', default=False,
                     help='Request BIOS style boot (rather than UEFI)')
+argparser.add_option('-u', '--uefi', dest='uefimode',
+                     action='store_true', default=False,
+                     help='Request UEFI style boot (rather than BIOS)')
 argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                     default=False,
                     help='Request the boot device be persistent rather than '
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -43,7 +43,7 @@ def bailout(msg, code=1):
    sys.exit(code)


-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                     action='store_true', default=False,
                     help='Compare given settings to default or list settings '
@@ -58,6 +58,16 @@ argparser.add_option('-x', '--exclude', dest='exclude',
                     action='store_true', default=False,
                     help='Treat positional arguments as items to not '
                          'examine, compare, or restore default')
+argparser.add_option('-a', '--advanced', dest='advanced',
+                     action='store_true', default=False,
+                     help='Include advanced settings, which are normally not '
+                          'intended to be used without direction from the '
+                          'relevant server vendor.')
+argparser.add_option('-r', '--restoredefault', default=False,
+                     dest='restoredefault', metavar="COMPONENT",
+                     help='Restore the configuration of the node '
+                          'to factory default for given component. '
+                          'Currently only uefi is supported')
 (options, args) = argparser.parse_args()

 cfgpaths = {
@@ -88,6 +98,8 @@ setmode = None
 assignment = {}
 queryparms = {}
 printsys = []
+printbmc = []
+printallbmc = False
 setsys = {}
 forceset = False
 needval = None
@@ -112,7 +124,7 @@ def _assign_value():


 def parse_config_line(arguments):
-    global setmode, forceset, key, value, needval, candidate, path, attrib
+    global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
    for param in arguments:
        if param == 'show':
            continue  # forgive muscle memory of pasu users
@@ -122,6 +134,7 @@ def parse_config_line(arguments):
            continue
        if needval:
            key = needval
+            needval = None
            value = param
            _assign_value()
            continue
@@ -143,6 +156,8 @@ def parse_config_line(arguments):
            if setmode != False:
                bailout('Cannot do set and query in same command')
            if '.' not in param:
+                if param == 'bmc':
+                    printallbmc = True
                matchedparms = False
                for candidate in cfgpaths:
                    if candidate.startswith('{0}.'.format(param)):
@@ -161,7 +176,10 @@ def parse_config_line(arguments):
                if not matchedparms:
                    printsys.append(param)
            elif param not in cfgpaths:
-                printsys.append(param)
+                if param.startswith('bmc.'):
+                    printbmc.append(param.replace('bmc.', ''))
+                else:
+                    printsys.append(param)
            else:
                path, attrib = cfgpaths[param]
                path = '/noderange/{0}/{1}'.format(noderange, path)
@@ -174,12 +192,38 @@ if options.batch:
    argfile = open(options.batch, 'r')
    argset = argfile.readline()
    while argset:
-        parse_config_line(shlex.split(argset))
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            parse_config_line(shlex.split(argset))
        argset = argfile.readline()
 else:
    parse_config_line(args[1:])
 session = client.Command()
 rcode = 0
+if options.restoredefault:
+    if options.restoredefault.lower() in (
+        'sys', 'system', 'uefi', 'bios'):
+        for fr in session.update(
+                '/noderange/{0}/configuration/system/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    elif options.restoredefault.lower() in (
+        'bmc', 'imm', 'xcc'):
+        for fr in session.update(
+                '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    else:
+        sys.stderr.write(
+            'Unrecognized component to restore defaults: {0}\n'.format(
+                options.restoredefault))
+        sys.exit(1)
 if setmode:
    if options.exclude:
        sys.stderr.write('Cannot use exclude and assign at the same time\n')
@@ -188,8 +232,12 @@ if setmode:
    attrnamebypath = {}
    for key in assignment:
        if key not in cfgpaths:
-            path = 'configuration/system/all'
-            attrib = key
+            if key.startswith('bmc.'):
+                path = 'configuration/management_controller/extended/all'
+                attrib = key.replace('bmc.', '')
+            else:
+                path = 'configuration/system/all'
+                attrib = key
        else:
            path, attrib = cfgpaths[key]
        if path not in updatebypath:
@@ -221,10 +269,19 @@ else:
                                      NullOpt(), queryparms[path])
        if rc:
            sys.exit(rc)
+    if printbmc or printallbmc:
+        rcode = client.print_attrib_path(
+                    '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
+                    session, printbmc, options, attrprefix='bmc.')
    if printsys or options.exclude:
        if printsys == 'all':
            printsys = []
-        path = '/noderange/{0}/configuration/system/all'.format(noderange)
+        if (options.comparedefault or printsys == []) and not options.advanced:
+            path = '/noderange/{0}/configuration/system/all'.format(noderange)
+        else:
+            path = '/noderange/{0}/configuration/system/advanced'.format(
+                noderange)
+        
        rcode = client.print_attrib_path(path, session, printsys,
                                         options)
-sys.exit(rcode)
+sys.exit(rcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015 Lenovo
@@ -17,7 +17,14 @@

 import optparse
 import os
+import subprocess
 import sys
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+import confluent.client as client
+import confluent.sortutil as sortutil

 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -26,9 +33,45 @@ argparser = optparse.OptionParser(
           "ctrl-'e', then release ctrl, then 'c', then '?' for a full list. "
           "For example, ctrl-'e', then 'c', then '.' will exit the current "
           "console")
+argparser.add_option('-t', '--tile', action='store_true', default=False,
+                     help='Tile console windows in the terminal')
 (options, args) = argparser.parse_args()
 if len(args) != 1:
    argparser.print_help()
    sys.exit(1)
-os.execl(confettypath, confettypath, 'start',
-         '/nodes/{0}/console/session'.format(args[0]))
+if options.tile:
+    null = open('/dev/null', 'w')
+    nodes = []
+    sess = client.Command()
+    for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
+        node = res.get('item', {}).get('href', '/').replace('/', '')
+        if not node:
+            sys.stderr.write(res.get('error', repr(res)) + '\n')
+            sys.exit(1)
+        nodes.append(node)
+    initial = True
+    pane = 0
+    for node in sortutil.natural_sort(nodes):
+        if initial:
+            initial = False
+            subprocess.call(
+                ['tmux', 'new-session', '-d', '-s',
+                 'nodeconsole_{0}'.format(os.getpid()), '-x', '800', '-y',
+                 '800', '{0} -m 5 start /nodes/{1}/console/session'.format(
+                    confettypath, node)])
+        else:
+            subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
+            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
+            pane += 1
+            subprocess.call(
+                ['tmux', 'split', '-h',
+                 '{0} -m 5 start /nodes/{1}/console/session'.format(
+                     confettypath, node)])
+            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
+    subprocess.call(['tmux', 'select-pane', '-t', '0'])
+    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
+    os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
+        os.getpid()))
+else:
+    os.execl(confettypath, confettypath, 'start',
+             '/nodes/{0}/console/session'.format(args[0]))
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/noderange/', attribs):
        exitcode |= 1
    if 'created' in r:
        print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -19,6 +19,7 @@ import csv
 import optparse
 import os
 import sys
+import time

 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
@@ -26,31 +27,49 @@ if path.startswith('/opt'):
    sys.path.append(path)

 import confluent.client as client
+import confluent.sortutil as sortutil

-tabformat = '{0:>15}|{1:>15}|{2:>15}|{3:>36}|{4:>17}|{5:>12}|{6:>48}'
-columns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
-           'Current IP Addresses']
-delimit = ['-' * 15, '-' * 15, '-' * 15, '-' * 36, '-' * 17, '-' * 12,
-           '-' * 48]
+defcolumns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
+              'Current IP Addresses']
+columnmapping = {
+    'Node': 'nodename',
+    'Model': 'modelnumber',
+    'Serial': 'serialnumber',
+    'UUID': 'uuid',
+    'Type': 'types',
+    'Current IP Addresses': 'ipaddrs',
+    'IP': 'ipaddrs',
+    'Bay': 'bay',
+    'Mac Address': 'macs',
+    'Mac': 'macs',
+    'Switch': 'switch',
+    'Port': 'port',
+    'Advertised IP': 'otheripaddrs',
+    'Other IP': 'otheripaddrs',
+}


-def dumpmacs(procinfo):
-    return ','.join(procinfo['macs']) # + procinfo.get('relatedmacs', []))
-
-
-def print_disco(options, session, currmac):
+def print_disco(options, session, currmac, outhandler, columns):
    procinfo = {}
    for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):

        procinfo.update(tmpinfo)
-    record = [procinfo['nodename'], procinfo['modelnumber'],
-              procinfo['serialnumber'], procinfo['uuid'], dumpmacs(procinfo),
-              ','.join(procinfo['types']),
-              ','.join(sorted(procinfo['ipaddrs']))]
-    if options.csv:
-        csv.writer(sys.stdout).writerow(record)
-    else:
-        print(tabformat.format(*record))
+    if 'Switch' in columns or 'Port' in columns:
+        for tmpinfo in session.read(
+                '/networking/macs/by-mac/{0}'.format(currmac)):
+            if 'ports' in tmpinfo:
+                # The api sorts so that the most specific available value
+                # is last
+                procinfo.update(tmpinfo['ports'][-1])
+    record = []
+    for col in columns:
+        rawcol = columnmapping[col]
+        rawval = procinfo.get(rawcol, '')
+        if isinstance(rawval, list):
+            record.append(','.join(rawval))
+        else:
+            record.append(str(rawval))
+    outhandler.add_row(record)


 def process_header(header):
@@ -72,7 +91,9 @@ def process_header(header):
        elif datum in ('bmc', 'imm', 'xcc'):
            fields.append('hardwaremanagement.manager')
        elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
-            fields.append('net.bmc.gateway')
+            fields.append('net.bmc.ipv4_gateway')
+        elif datum in ('bmc_gateway', 'xcc_gateway', 'imm_gateway'):
+            fields.append('net.bmc.ipv4_gateway')
        elif datum in ('bmcuser', 'username', 'user'):
            fields.append('secret.hardwaremanagementuser')
        elif datum in ('bmcpass', 'password', 'pass'):
@@ -122,24 +143,44 @@ def datum_to_attrib(datum):
    del datum['node']
    return datum

+unique_fields = frozenset(['serial', 'mac', 'uuid'])
+
 def import_csv(options, session):
    nodedata = []
+    unique_data = {}
    with open(options.importfile, 'r') as datasrc:
        records = csv.reader(datasrc)
        fields = process_header(next(records))
+        for field in fields:
+            if field in unique_fields:
+                unique_data[field] = set([])
+        broken = False
        for record in records:
            currfields = list(fields)
            nodedatum = {}
            for datum in record:
-                nodedatum[currfields.pop(0)] = datum
+                currfield = currfields.pop(0)
+                if currfield in unique_fields:
+                    if datum in unique_data[currfield]:
+                        sys.stderr.write(
+                            "Import contains duplicate values "
+                            "({0} with value {1}\n".format(currfield, datum)
+                        )
+                        sys.exit(1)
+                    unique_data[currfield].add(datum)
+                nodedatum[currfield] = datum
            if not datum_complete(nodedatum):
                sys.exit(1)
+            if not search_record(nodedatum, options, session) and not broken:
+                blocking_scan(session)
            if not search_record(nodedatum, options, session):
                sys.stderr.write(
                    "Could not match the following data: " +
                    repr(nodedatum) + '\n')
-                sys.exit(1)
+                broken = True
            nodedata.append(nodedatum)
+    if broken:
+        sys.exit(1)
    for datum in nodedata:
        maclist = search_record(datum, options, session)
        datum = datum_to_attrib(datum)
@@ -165,13 +206,28 @@ def import_csv(options, session):


 def list_discovery(options, session):
-    if options.csv:
-        csv.writer(sys.stdout).writerow(columns)
+    orderby = None
+    if options.fields:
+        columns = []
+        for field in options.fields.split(','):
+            for cdt in columnmapping:
+                if cdt.lower().replace(
+                        ' ', '') == field.lower().replace(' ', ''):
+                    columns.append(cdt)
    else:
-        print(tabformat.format(*columns))
-        print(tabformat.format(*delimit))
+        columns = defcolumns
+    if options.order:
+        for field in columns:
+            if options.order.lower() == field.lower():
+                orderby = field
+    outhandler = client.Tabulator(columns)
    for mac in list_matching_macs(options, session):
-        print_disco(options, session, mac)
+        print_disco(options, session, mac, outhandler, columns)
+    if options.csv:
+        outhandler.write_csv(sys.stdout, orderby)
+    else:
+        for row in outhandler.get_table(orderby):
+            print(row)

 def clear_discovery(options, session):
    for mac in list_matching_macs(options, session):
@@ -181,8 +237,10 @@ def clear_discovery(options, session):
            else:
                print(repr(res))

-def list_matching_macs(options, session):
+def list_matching_macs(options, session, node=None):
    path = '/discovery/'
+    if node:
+        path += 'by-node/{0}/'.format(node)
    if options.model:
        path += 'by-model/{0}/'.format(options.model)
    if options.serial:
@@ -205,21 +263,25 @@ def list_matching_macs(options, session):
        path += 'by-mac/'
        return [x['item']['href'] for x in session.read(path)]

-def assign_discovery(options, session):
+def assign_discovery(options, session, needid=True):
    abort = False
    if options.importfile:
        return import_csv(options, session)
-    if not (options.serial or options.uuid or options.mac):
+    if not options.node:
+        sys.stderr.write("Node (-n) must be specified for assignment\n")
+        abort = True
+    if needid and not (options.serial or options.uuid or options.mac):
        sys.stderr.write(
            "UUID (-u), serial (-s), or ether address (-e) required for "
            "assignment\n")
        abort = True
-    if not options.node:
-        sys.stderr.write("Node (-n) must be specified for assignment\n")
-        abort = True
    if abort:
        sys.exit(1)
-    matches = list_matching_macs(options, session)
+    matches = list_matching_macs(options, session, None if needid else options.node)
+    if not matches:
+        # Do a rescan to catch missing requested data
+        blocking_scan(session)
+        matches = list_matching_macs(options, session, None if needid else options.node)
    if not matches:
        sys.stderr.write("No matching discovery candidates found\n")
        sys.exit(1)
@@ -230,6 +292,11 @@ def assign_discovery(options, session):
        else:
            print(repr(res))

+def blocking_scan(session):
+    list(session.update('/discovery/rescan', {'rescan': 'start'}))
+    while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
+        time.sleep(0.5)
+    list(session.update('/networking/macs/rescan', {'rescan': 'start'}))


 def main():
@@ -266,8 +333,13 @@ def main():
    parser.add_option('-i', '--import', dest='importfile',
                      help='Import bulk assignment data from given CSV file',
                      metavar='IMPORT.CSV')
+    parser.add_option('-f', '--fields', dest='fields',
+                      help='Select fields for output',
+                      metavar='FIELDS')
+    parser.add_option('-o', '--order', dest='order',
+                      help='Order output by given field', metavar='ORDER')
    (options, args) = parser.parse_args()
-    if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear'):
        parser.print_help()
        sys.exit(1)
    session = client.Command()
@@ -277,10 +349,12 @@ def main():
        clear_discovery(options, session)
    if args[0] == 'assign':
        assign_discovery(options, session)
+    if args[0] == 'reassign':
+        assign_discovery(options, session, False)
    if args[0] == 'rescan':
-        list(session.update('/discovery/rescan', {'rescan': 'start'}))
-        print("Rescan initiated")
+        blocking_scan(session)
+        print("Rescan complete")


 if __name__ == '__main__':
-    main()
+    main()
@@ -1,7 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,7 +33,8 @@ if path.startswith('/opt'):

 import confluent.client as client

-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)

 argparser = optparse.OptionParser(
    usage="Usage: %prog [options] noderange [clear]")
@@ -64,6 +65,8 @@ def format_event(evt):
        display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
        retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
    dscparts = []
+    if evt.get('log_id', None):
+        retparts.append(evt['log_id'] + ':')
    if 'component_type' in evt and evt['component_type'] is not None:
        dscparts.append(evt['component_type'])
    if 'component' in evt and evt['component'] is not None:
@@ -77,7 +80,10 @@ def format_event(evt):
            pass
        dscparts.append(evttext)
    retparts.append(' - '.join(dscparts))
-    return ' '.join(retparts)
+    msg = evt.get('message')
+    if not msg:
+        msg = ''
+    return ' '.join(retparts) + msg


 if deletemode:
@@ -98,4 +104,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
            if 'events' in thisdata:
               evtdata = thisdata['events']
               for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print('{0}: {1}'.format(node, format_event(evt)))
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2016-2017 Lenovo
@@ -37,14 +37,14 @@ exitcode = 0


 def printfirm(node, prefix, data):
-    if 'model' in data:
+    if 'model' in data and data['model']:
        prefix += ' ' + data['model']
    builddesc = []
-    if 'build' in data:
+    if 'build' in data and data['build']:
        builddesc.append(data['build'])
-    if 'date' in data:
+    if 'date' in data and data['date']:
        builddesc.append(data['date'])
-    if 'version' in data:
+    if 'version' in data and data['version']:
        version = data['version']
        if builddesc:
            version += ' ({0})'.format(' '.join(builddesc))
@@ -55,7 +55,8 @@ def printfirm(node, prefix, data):
 components = ['all']

 argparser = optparse.OptionParser(
-    usage="Usage: %prog <noderange> [update [--backup <file>]]|[<components>]")
+    usage="Usage: "
+          "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                     help='Target a backup bank rather than primary')
 (options, args) = argparser.parse_args()
@@ -66,9 +67,15 @@ try:
        if args[1] == 'update':
            upfile = args[2]
        else:
+            if args[1] == 'list':
+                comps = args[2:]
+            else:
+                comps = args[1:]
            components = []
-            for arg in args[1:]:
+            for arg in comps:
                components += arg.split(',')
+            if not components:
+                components = ['all']

 except IndexError:
    argparser.print_help()
@@ -144,7 +151,7 @@ def show_firmware(session):
                    for prefix in inv:
                        firmware_shown = True
                        printfirm(node, prefix, inv[prefix])
-    if not firmware_shown:
+    if not firmware_shown and not exitcode:
        argparser.print_help()


@@ -156,4 +163,4 @@ try:
        update_firmware(session, upfile)
 except KeyboardInterrupt:
    print('')
-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -17,6 +17,7 @@

 __author__ = 'alin37'

+from getpass import getpass
 import optparse
 import os
 import signal
@@ -35,9 +36,9 @@ import confluent.client as client

 argparser = optparse.OptionParser(
    usage='''\n       %prog [options]   \
-             \n       %prog [options] nodegroup [list of attributes]  \
+             \n       %prog [options] nodegroup [list of attributes|all]  \
             \n       %prog [options] nodegroup nodes=value1,value2   \
-             \n       %prog -e nodegroup <attribute names to set>  \             
+             \n       %prog -e nodegroup <attribute names to set>  \
             \n       %prog [options] nodegroup nodes=value1,value2
             \n ''')
 argparser.add_option('-b', '--blame', action='store_true',
@@ -47,6 +48,8 @@ argparser.add_option('-e', '--environment', action='store_true',
                          'same name')
 argparser.add_option('-c', '--clear', action='store_true',
                     help='Clear variables')
+argparser.add_option('-p', '--prompt', action='store_true',
+                     help='Prompt for attribute values interactively')
 (options, args) = argparser.parse_args()


@@ -72,7 +75,19 @@ if len(args) > 1:
        print("Can not clear and set at the same time!")
        argparser.print_help()
        sys.exit(1)
-    exitcode=client.updateattrib(session,args,nodetype, nodegroups, options)
+    argassign = None
+    if options.prompt:
+        argassign = {}
+        for arg in args[1:]:
+            oneval = 1
+            twoval = 2
+            while oneval != twoval:
+                oneval = getpass('Enter value for {0}: '.format(arg))
+                twoval = getpass('Confirm value for {0}: '.format(arg))
+                if oneval != twoval:
+                    print('Values did not match.')
+            argassign[arg] = twoval
+    exitcode=client.updateattrib(session,args,nodetype, nodegroups, options, argassign)
    try:
        # setting user output to what the user inputs
        if args[1] == 'all':
@@ -82,7 +97,7 @@ if len(args) > 1:

        requestargs=args[1:]
    except Exception as e:
-        print str(e)
+        print(str(e))

 if exitcode != 0:
    sys.exit(exitcode)
@@ -108,6 +123,6 @@ else:
            sys.stderr.write(res['error'] + '\n')
            exitcode = 1
        else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))

-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/nodegroups/', attribs):
        exitcode |= 1
    if 'created' in r:
        print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -0,0 +1,62 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    showtype='all'
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) > 0:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print(res['item']['href'].replace('/', ''))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -49,4 +49,4 @@ for r in session.delete('/nodegroups/{0}'.format(noderange)):
        exitcode |= 1
    if 'deleted' in r:
        print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -0,0 +1,63 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog <group> <new group name>\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) != 2:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.update(
+            '/nodegroups/{0}/attributes/rename'.format(args[0]),
+            {'rename': args[1]}):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print('{0}: {1}'.format(res['oldname'], res['newname']))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -32,7 +32,8 @@ if path.startswith('/opt'):

 import confluent.client as client

-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)

 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
 (options, args) = argparser.parse_args()
@@ -44,6 +45,13 @@ except IndexError:
 client.check_globbing(noderange)


+codemappings = {
+    'ok': 0,
+    'warning': 1,
+    'critical': 2,
+    'failed': 2,
+}
+
 def main():
    global session, exitcode, healthbynode, healthexplanations, health, node, sensor, explanation
    session = client.Command()
@@ -66,9 +74,11 @@ def main():
            if 'error' in health[node]:
                sys.stderr.write('{0}: Error: {1}\n'.format(
                    node, health[node]['error']))
-                exitcode |= 1
+                exitcode = 3
            if 'health' in health[node]:
                healthbynode[node] = health[node]['health']['value']
+                if codemappings[healthbynode[node]] > exitcode:
+                    exitcode = codemappings[healthbynode[node]]
            if 'sensors' in health[node]:
                healthexplanations[node] = []
                for sensor in health[node]['sensors']:
@@ -94,5 +104,5 @@ try:
    main()
 except KeyboardInterrupt:
    print('')
-    sys.exit(0)
+sys.exit(exitcode)

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -31,7 +31,7 @@ if path.startswith('/opt'):

 import confluent.client as client

-argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off|blink]")
 (options, args) = argparser.parse_args()
 try:
    noderange = args[0]
@@ -42,9 +42,6 @@ client.check_globbing(noderange)
 identifystate = None
 if len(sys.argv) > 2:
    identifystate = sys.argv[2]
-else:
-    argparser.print_help()
-    sys.exit(1)
 session = client.Command()
 exitcode = 0
 sys.exit(
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2016-2017 Lenovo
@@ -34,7 +34,8 @@ if path.startswith('/opt'):

 import confluent.client as client

-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)

 filters = []

@@ -50,15 +51,17 @@ def print_mem_info(node, prefix, meminfo):
    memdescfmt = '{0}GB PC'
    if meminfo['memory_type'] == 'DDR3 SDRAM':
        memdescfmt += '3-{1} '
-    elif meminfo['memory_type'] == 'DDR4 SDRAM':
+    elif 'DDR4' in meminfo['memory_type']:
        memdescfmt += '4-{1} '
    else:
        print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
        return
-    if meminfo['ecc']:
+    if meminfo.get('ecc', False):
        memdescfmt += 'ECC '
    capacity = meminfo['capacity_mb'] / 1024
-    memdescfmt += meminfo['module_type']
+    modtype = meminfo.get('module_type', None)
+    if modtype:
+        memdescfmt += modtype
    memdesc = memdescfmt.format(capacity, meminfo['speed'])
    print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
    print('{0}: {1} manufacturer: {2}'.format(
@@ -66,10 +69,11 @@ def print_mem_info(node, prefix, meminfo):
    print('{0}: {1} model: {2}'.format(node, prefix, meminfo['model']))
    print('{0}: {1} serial number: {2}'.format(node, prefix,
                                               meminfo['serial']))
-    print('{0}: {1} manufacture date: {2}'.format(node, prefix,
-                                                  meminfo['manufacture_date']))
-    print('{0}: {1} manufacture location: {2}'.format(
-        node, prefix, meminfo['manufacture_location']))
+    if 'manufacture_date' in meminfo:
+        print('{0}: {1} manufacture date: {2}'.format(node, prefix,
+                                                      meminfo['manufacture_date']))
+        print('{0}: {1} manufacture location: {2}'.format(
+            node, prefix, meminfo['manufacture_location']))

 exitcode = 0

@@ -87,6 +91,7 @@ def printerror(res, node=None):


 url = '/noderange/{0}/inventory/hardware/all/all'
+usedprefixes = set([])

 argparser = optparse.OptionParser(
    usage="Usage: %prog <noderange> [serial|model|uuid|mac]")
@@ -129,6 +134,11 @@ try:
                continue
            for inv in res['databynode'][node]['inventory']:
                prefix = inv['name']
+                idx = 2
+                while (node, prefix) in usedprefixes:
+                    prefix = '{0} {1}'.format(inv['name'], idx)
+                    idx += 1
+                usedprefixes.add((node, prefix))
                if not inv['present']:
                    if not filters:
                        if options.json:
@@ -136,7 +146,7 @@ try:
                                databynode[node] = {}
                            databynode[node][prefix] = inv
                        else:
-                            print '{0}: {1}: Not Present'.format(node, prefix)
+                            print('{0}: {1}: Not Present'.format(node, prefix))
                    continue
                info = inv['information']
                info.pop('board_extra', None)
@@ -0,0 +1,146 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+exitcode = 0
+
+argparser = optparse.OptionParser(
+    usage="Usage: "
+          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
+(options, args) = argparser.parse_args()
+upfile = None
+downdir = None
+delete = False
+try:
+    noderange = args[0]
+    if len(args) > 1:
+        if args[1] == 'install':
+            upfile = args[2]
+        elif args[1] == 'save':
+            downdir = args[2]
+        elif args[1] == 'delete':
+            delete = args[2]
+        elif args[1] != 'list':
+            argparser.print_help()
+            sys.exit(1)
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+
+
+def install_license(session, filename):
+    global exitcode
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate requested file {0}\n'.format(
+            filename))
+        sys.exit(404)
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/licenses/'.format(noderange)
+    filename = os.path.abspath(filename)
+    instargs = {'filename': filename}
+    for res in session.create(resource, instargs):
+        for node in res.get('databynode', []):
+            if 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(
+                    node, res['databynode'][node]['error']))
+                sys.exit(res['databynode'][node].get('errorcode', 1))        
+    show_licenses(session)
+
+
+def save_licenses(session, dirname):
+    global exitcode
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/save_licenses'.format(noderange)
+    filename = os.path.abspath(dirname)
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate specified directory {0}\n'.format(
+            filename))
+        sys.exit(404)
+    instargs = {'dirname': filename}
+    for res in session.create(resource, instargs):
+        for node in res.get('databynode', {}):
+            fname = res['databynode'][node].get('filename', None)
+            if fname:
+                print('{0}: Saved license to {1}'.format(node, fname))
+            else:
+                sys.stderr.write('{0}: {1}', node, repr(res['databynode'][node]))
+
+
+def show_licenses(session):
+    global exitcode
+    for res in session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                msg = '{0}: {1}'.format(node, license.get('feature',
+                                                          'Unknown'))
+                if license.get('state', 'Active') != 'Active':
+                    msg += ' ({0})'.format(license['state'])
+                print(msg)
+
+
+def delete_license(session, licname):
+    global exitcode
+    licstodel = []
+    for res in list(session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange))):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                if license.get('feature', None) == licname:
+                    prefix = '/nodes/{0}/configuration/management_controller/licenses/'.format(node)
+                    for currlic in list(session.read(prefix)):
+                        currlic = currlic.get('item', {}).get('href', 'all')
+                        if currlic == 'all':
+                            continue
+                        currname = list(session.read(prefix + currlic))[0]
+                        currname = currname.get('License', [{}])[0].get('feature', None)
+                        if currname == licname:
+                            list(session.delete(prefix + currlic))
+    show_licenses(session)
+
+try:
+    session = client.Command()
+    if upfile:
+        install_license(session, upfile)
+    elif downdir:
+        save_licenses(session, downdir)
+    elif delete:
+        delete_license(session, delete)
+    else:
+        show_licenses(session)
+except KeyboardInterrupt:
+    print('')
+sys.exit(exitcode)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -61,7 +61,7 @@ def main():
                sys.stderr.write(res['error'] + '\n')
                exitcode = 1
            else:
-                print res['item']['href'].replace('/', '')
+                print(res['item']['href'].replace('/', ''))

    sys.exit(exitcode)

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2018 Lenovo
@@ -83,7 +83,8 @@ def list_media(noderange, media):
        for node in res.get('databynode', []):
            url = res['databynode'][node].get('url', None)
            name =  res['databynode'][node].get('name', None)
-            if url and not res['databynode'][node].get('secure', False):
+            if (url and not url.startswith('file:') and
+                    not res['databynode'][node].get('secure', False)):
                name += ' (insecure)'
            if not name:
                continue
@@ -187,4 +188,4 @@ def main():
        sys.exit(1)
    handler(noderange, media)
 if __name__ == '__main__':
-    main()
+    main()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -72,4 +72,4 @@ if options.previous:
    # add dictionary to session
    session.add_precede_dict(prev)

-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -49,4 +49,4 @@ for r in session.delete('/noderange/{0}'.format(noderange)):
        exitcode |= 1
    if 'deleted' in r:
        print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)
@@ -0,0 +1,52 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> <newname>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+identifystate = None
+if len(sys.argv) > 2:
+    newname = sys.argv[2]
+else:
+    argparser.print_help()
+    sys.exit(1)
+session = client.Command()
+exitcode = 0
+sys.exit(
+    session.simple_noderange_command(noderange, 'attributes/rename', newname))
+
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -56,7 +56,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes

 for node in goodNodes:
-    print node + ": Reseat successful"
+    print(node + ": Reseat successful")


 sys.exit(success)
@@ -0,0 +1,155 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+import confluent.sortutil as sortutil
+
+
+def run():
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog location noderange:location",
+    )
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of nodes to concurrently rsync')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2 or ':' not in args[-1]:
+        argparser.print_help()
+        sys.exit(1)
+    concurrentprocs = options.count
+    noderange, targpath = args[-1].split(':', 1)
+    client.check_globbing(noderange)
+    c = client.Command()
+    cmdstr = ' '.join(args[:-1])
+    cmdstr = 'rsync -av --info=progress2 ' + cmdstr
+    cmdstr += ' {node}:' + targpath
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
+            {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
+            cmdv = shlex.split(cmd)
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    nodeerrs = {}
+    pernodeout = {}
+    pernodefile = {}
+    output = sq.ScreenPrinter(noderange, c)
+
+    while all:
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.read(1)
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = ''
+                        pernodeout[node] += stringify(data)
+                        if '\n' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\n', 1)
+                            if currout:
+                                pernodefile[node] = os.path.basename(currout)
+                        if '\r' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\r', 1)
+                            if currout:
+                                currout = currout.split()
+                                try:
+                                    currout = currout[1]
+                                    output.set_output(node, '{0}:{1}'.format(pernodefile[node], currout))
+                                except IndexError:
+                                    pernodefile = currout[0]
+                                    pass
+                    else:
+                        output.set_output(node, 'error!')
+                        if node not in nodeerrs:
+                            nodeerrs[node] = ''
+                        nodeerrs[node] += client.stringify(data)
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if node not in nodeerrs:
+                            output.set_output(node, 'complete')
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    for node in nodeerrs:
+        for line in nodeerrs[node].split('\n'):
+            sys.stderr.write('{0}: {1}\n'.format(node, line))
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2016-2017 Lenovo
@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil

 def run():
    argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
        epilog="Expressions are the same as in attributes, e.g. "
               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
    argparser.add_option('-f', '-c', '--count', type='int', default=168,
                         help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
    # among other things, FD_SETSIZE limits.  Besides, spawning too many
    # processes can be unkind for the unaware on memory pressure and such...
    argparser.disable_interspersed_args()
@@ -69,7 +71,9 @@ def run():
            exitcode |= exp.get('errorcode', 1)
        ex = exp.get('databynode', ())
        for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
            cmdv = shlex.split(cmd)
            if currprocs < concurrentprocs:
                currprocs += 1
@@ -85,7 +89,7 @@ def run():
            desc = pipedesc[r]
            node = desc['node']
            data = True
-            while data and select.select([r], [], [], 0):
+            while data and select.select([r], [], [], 0)[0]:
                data = r.readline()
                if data:
                    if desc['type'] == 'stdout':
@@ -93,7 +97,11 @@ def run():
                            pernodeout[node] = []
                        pernodeout[node].append(data)
                    else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        data = client.stringify(data)
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                        sys.stderr.flush()
                else:
                    pop = desc['popen']
@@ -107,7 +115,11 @@ def run():
                            run_cmdv(node, cmdv, all, pipedesc)
        for node in sortutil.natural_sort(pernodeout):
            for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
        sys.stdout.flush()
        if all:
            rdy, _, _ = select.select(all, [], [], 10)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015-2017 Lenovo
@@ -114,7 +114,8 @@ def sensorpass(showout=True, appendtime=False):
                        continue
                    for redundant_state in ('Non-Critical', 'Critical'):
                        try:
-                            sensedata['states'].remove(redundant_state)
+                            if sensedata.get('states', False):
+                                sensedata['states'].remove(redundant_state)
                        except ValueError:
                            pass
                    resultdata[node][sensedata['name']] = sensedata
@@ -132,11 +133,12 @@ def sensorpass(showout=True, appendtime=False):
                            showval = u' {0} '.format(sensedata['value'])
                        if sensedata['units'] not in (None, u''):
                            showval += sensedata['units']
-                        if sensedata['health'] != 'ok':
+                        if sensedata.get('health', 'ok') != 'ok':
                            datadescription = [sensedata['health']]
                        else:
                            datadescription = []
-                        datadescription.extend(sensedata['states'])
+                        if sensedata.get('states', False):
+                            datadescription.extend(sensedata['states'])
                        if datadescription:
                            if showval == '':
                                showval += u' {0}'.format(
@@ -147,8 +149,11 @@ def sensorpass(showout=True, appendtime=False):
                        if appendtime:
                            showval += ' @' + time.strftime(
                                '%Y-%m-%dT%H:%M:%S')
-                        print(u'{0}: {1}:{2}'.format(
-                            node, sensedata['name'], showval).encode('utf-8'))
+                        printval = u'{0}: {1}:{2}'.format(
+                            node, sensedata['name'], showval)
+                        if not isinstance(printval, str):
+                            printval = printval.encode('utf-8')
+                        print(printval)
                        sys.stdout.flush()
    return resultdata

@@ -156,7 +161,7 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
    for nodekey in resdata:
        if showtime:
-            if showtime.is_integer():
+            if isinstance(showtime, int):
                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
            else:
                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
@@ -183,6 +188,7 @@ def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
            except KeyError:
                rowdata.append('N/A')
        csvwriter.writerow(rowdata)
+        sys.stdout.flush()


 def main():
@@ -196,7 +202,11 @@ def main():
            orderedsensors.append(name)
        orderedsensors.sort()
        for name in orderedsensors:
-            headernames.append(sensorheaders[name].encode('utf-8'))
+            headername = sensorheaders[name]
+            if (not isinstance(headername, str) and
+                    not isinstance(headername, bytes)):
+                headername = headername.encode('utf-8')
+            headernames.append(headername)
    if options.csv:
        linebyline = False
        csvwriter = csv.writer(sys.stdout)
@@ -220,7 +230,7 @@ def main():
                    sys.exit(exitcode)
            sleeptime = nextstart - os.times()[4]
            if sleeptime > 0:
-                time.sleep(nextstart - os.times()[4])
+                time.sleep(sleeptime)
    else:
        if options.csv:
            format_csv(csvwriter, orderedsensors, resdata, showtime=False)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2015 Lenovo
@@ -53,8 +53,8 @@ except IndexError:
    argparser.print_help()
    sys.exit(1)
 bootdev = None
-if len(sys.argv) > 2:
-    bootdev = sys.argv[2]
+if len(args) > 1:
+    bootdev = args[1]
    if bootdev in ('net', 'pxe'):
        bootdev = 'network'
 session = client.Command()
@@ -65,4 +65,4 @@ else:
    bootmode = 'uefi'
 sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                     bootmode=bootmode,
-                                     persistent=options.persist))
+                                     persistent=options.persist))
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2016-2017 Lenovo
@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil
 def run():

    argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
        epilog="Expressions are the same as in attributes, e.g. "
               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
    argparser.add_option('-f', '-c', '--count', type='int', default=168,
                         help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
    # among other things, FD_SETSIZE limits.  Besides, spawning too many
    # processes can be unkind for the unaware on memory pressure and such...
    argparser.disable_interspersed_args()
@@ -70,7 +72,9 @@ def run():
            exitcode |= exp.get('errorcode', 1)
        ex = exp.get('databynode', ())
        for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+                cmd = cmd.encode('utf-8')
            cmdv = ['ssh', node, cmd]
            if currprocs < concurrentprocs:
                currprocs += 1
@@ -94,7 +98,11 @@ def run():
                            pernodeout[node] = []
                        pernodeout[node].append(data)
                    else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        data = client.stringify(data)
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                        sys.stderr.flush()
                else:
                    pop = desc['popen']
@@ -108,7 +116,11 @@ def run():
                            run_cmdv(node, cmdv, all, pipedesc)
        for node in sortutil.natural_sort(pernodeout):
            for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
        sys.stdout.flush()
        if all:
            rdy, _, _ = select.select(all, [], [], 10)
@@ -0,0 +1,224 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2018 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+exitcode = 0
+
+class OptParser(optparse.OptionParser):
+
+    def format_epilog(self, formatter):
+        return self.expand_prog_name(self.epilog)
+
+def mbtohuman(mb):
+    if mb > 1000000:
+        return '{0:.3f} TB'.format(mb/1000000.0)
+    if mb > 1000:
+        return '{0:.3f} GB'.format(mb/1000.0)
+    return '{0:.3f} MB'.format(mb)
+
+def showstorage(noderange, options, args):
+    global exitcode
+    session = client.Command()
+    disks = {}
+    arrays = {}
+    volumes = {}
+    scfg = session.read('/noderange/{0}/configuration/storage/all'.format(
+        noderange))
+    _print_cfg(scfg)
+
+
+def _print_cfg(scfg):
+    global exitcode
+    storagebynode = {}
+    for e in scfg:
+        if 'error' in e:
+            sys.stderr.write(e['error'] + '\n')
+            exitcode = e.get('errorcode', 1)
+        for node in e.get('databynode', {}):
+            if node not in storagebynode:
+                storagebynode[node] = {'disks': [], 'arrays': [],
+                                       'volumes': []}
+            curr = e['databynode'][node]
+            storagebynode[node][curr['type'] + 's'].append(curr)
+    for node in storagebynode:
+        for disk in sorted(storagebynode[node]['disks'],
+                           key=lambda x: x['name']):
+            print('{0}: Disk {1} Description: {2}'.format(
+                node, disk['name'], disk['description']))
+            print('{0}: Disk {1} State: {2}'.format(node, disk['name'],
+                                                    disk['state']))
+            print('{0}: Disk {1} FRU: {2}'.format(node, disk['name'],
+                                                  disk['fru']))
+            print('{0}: Disk {1} Serial Number: {2}'.format(node, disk['name'],
+                                                            disk['serial']))
+            if disk['array']:
+                print('{0}: Disk {1} Array: {2}'.format(node, disk['name'],
+                                                        disk['array']))
+        for arr in storagebynode[node]['arrays']:
+            print('{0}: Array {1} Available Capacity: {2}'.format(
+                node, arr['id'], mbtohuman(arr['available'])))
+            print('{0}: Array {1} Total Capacity: {2}'.format(
+                node, arr['id'], mbtohuman(arr['capacity'])))
+            print('{0}: Array {1} RAID: {2}'.format(node, arr['id'],
+                                                    arr['raid']))
+            print('{0}: Array {1} Disks: {2}'.format(node, arr['id'], ','.join(
+                arr['disks'])))
+            print(
+                '{0}: Array {1} Volumes: {2}'.format(node, arr['id'], ','.join(
+                    arr['volumes'])))
+        for vol in storagebynode[node]['volumes']:
+            print('{0}: Volume {1}: Size: {2}'.format(node, vol['name'],
+                                                      mbtohuman(vol['size'])))
+            print('{0}: Volume {1}: State: {2}'.format(node, vol['name'],
+                                                       vol['state']))
+            print('{0}: Volume {1}: Array {2}'.format(node, vol['name'],
+                                                      vol['array']))
+
+
+def createstorage(noderange, options, args):
+    if options.raidlevel is None or options.disks is None:
+        sys.stderr.write('-r and -d are required arguments to create array\n')
+        sys.exit(1)
+    session = client.Command()
+    names = options.name
+    if names is None:
+        names = ''.join(args)
+    parms = {'disks': options.disks, 'raidlevel': options.raidlevel,
+             'name': names}
+    if options.size:
+        parms['size'] = options.size
+    if options.stripsizes:
+        parms['stripsizes'] = options.stripsizes
+    _print_cfg(session.create(
+            '/noderange/{0}/configuration/storage/volumes/{1}'.format(
+                noderange, names), parms))
+
+
+def deletestorage(noderange, options, args):
+    if options.name is None:
+        if len(args) == 1:
+            names = args[0]
+        else:
+            sys.stderr.write('-n is required to indicate volume(s) to delete\n')
+            sys.exit(1)
+    else:
+        names = options.name
+    session = client.Command()
+    for rsp in session.delete(
+            '/noderange/{0}/configuration/storage/volumes/{1}'.format(
+                noderange, names)):
+        if 'deleted' in rsp:
+            print('Deleted: {0}'.format(rsp['deleted']))
+        elif 'databynode' in rsp:
+            for node in rsp['databynode']:
+                if 'error' in rsp['databynode'][node]:
+                    sys.stderr.write('{0}: {1}\n'.format(
+                        node, rsp['databynode'][node]['error']))
+                else:
+                    sys.stderr.write('{0}: {1}\n'.format(
+                        node, repr(rsp['databynode'][node])))
+        else:
+            print(repr(rsp))
+
+
+def setdisk(noderange, options, args):
+    if options.disks is None:
+        if len(args):
+            names = args.pop(0)
+        else:
+            sys.stderr.write('-d is required to indicate disk to modify\n')
+            sys.exit(1)
+    else:
+        names = options.disks
+    if not len(args) or args[0] not in ('hotspare', 'jbod', 'unconfigured'):
+        sys.stderr.write('diskset requires valid state as argument (hotspare, jbod, unconfigured)\n')
+        sys.exit(1)
+    session = client.Command()
+    scfg = session.update('/noderange/{0}/configuration/storage/disks/{1}'.format(noderange, names), {'state': args[0]})
+    _print_cfg(scfg)
+
+funmap = {
+    'create': createstorage,
+    'show': showstorage,
+    'diskset': setdisk,
+    'delete': deletestorage,
+    'rm': deletestorage,
+}
+
+
+
+def main():
+    argparser = OptParser(
+        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
+        epilog='',
+    )
+    argparser.add_option('-r', '--raidlevel', type='int',
+                         help='RAID level to use when creating an array')
+    argparser.add_option('-d', '--disks', type='str',
+                         help='Comma separated list of disks to use, or the '
+                              'word "rest" to indicate use of all available '
+                              'disks')
+    argparser.add_option('-s', '--size', type='str',
+                         help='Comma separated list of sizes to use when '
+                              'creating volumes.  The sizes may be absolute '
+                              'size (e.g. 16gb), percentage (10%) or the word '
+                              '"rest" to use remaining capacity, default '
+                              'behavior is to use all capacity to make a '
+                              'volume')
+    argparser.add_option('-n', '--name', type='str',
+                         help='Comma separated list of names to use when '
+                              'naming volumes, or selecting a volume for '
+                              'delete.  Default behavior is to use '
+                              'implementation provided default names.')
+    argparser.add_option('-z', '--stripsizes', type='str',
+                         help='Comma separated list of stripsizes to use when creating volumes. '
+                            'This value is in kilobytes.  The default behavior is to allow the '
+                            'storage controller to decide.')
+    (options, args) = argparser.parse_args()
+    if len(args) == 1:
+        args.append('show')
+    try:
+        noderange = args[0]
+        operation = args[1]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    try:
+        handler = funmap[operation]
+    except KeyError:
+        argparser.print_help()
+        sys.exit(1)
+    handler(noderange, options, args[2:])
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,147 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2018 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+
+exitcode = 0
+
+
+def get_update_progress(session, url):
+    for res in session.read(url):
+        status = res['phase']
+        percent = res['progress']
+        detail = res['detail']
+        if status == 'error':
+            text = 'error!'
+        else:
+            text = '{0}: {1:3.0f}%'.format(status, percent)
+    return text, status, detail
+
+
+def printerror(res, node=None):
+    global exitcode
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+    for node in res.get('databynode', ()):
+        printerror(res['databynode'][node], node)
+
+
+
+def download_servicedata(noderange, media):
+    global exitcode
+    session = client.Command()
+    output = sq.ScreenPrinter(noderange, session)
+    filename = os.path.abspath(media)
+    resource = '/noderange/{0}/support/servicedata/'.format(noderange)
+    upargs = {'filename': filename}
+    noderrs = {}
+    nodeurls = {}
+    for res in session.create(resource, upargs):
+        if 'created' not in res:
+            for nodename in res.get('databynode', ()):
+                output.set_output(nodename, 'error!')
+                noderrs[nodename] = res['databynode'][nodename].get(
+                    'error', 'Unknown Error')
+            continue
+        watchurl = res['created']
+        currnode = watchurl.split('/')[1]
+        nodeurls[currnode] = '/' + watchurl
+    while nodeurls:
+        for node in list(nodeurls):
+            progress, status, err = get_update_progress(
+                session, nodeurls[node])
+            if status == 'error':
+                exitcode = 1
+                noderrs[node] = err
+            if status in ('error', 'complete', 'pending'):
+                list(session.delete(nodeurls[node]))
+                del nodeurls[node]
+            output.set_output(node, progress)
+        time.sleep(2)
+    allerrnodes = ','.join(noderrs)
+    if noderrs:
+        sys.stderr.write(
+            'Nodes had errors retrieving service data ({0})!\n'.format(allerrnodes))
+    for node in noderrs:
+        sys.stderr.write('{0}: {1}\n'.format(node, noderrs[node]))
+
+funmap = {
+    'servicedata': download_servicedata,
+}
+
+
+class OptParser(optparse.OptionParser):
+
+    def format_epilog(self, formatter):
+        return self.expand_prog_name(self.epilog)
+
+def main():
+    argparser = OptParser(
+        usage="Usage: %prog <noderange> servicedata "
+              "<filename>",
+        epilog='\nservicedata will save service data to the given '
+               'directory.  It is saved to the location on the relevant '
+               'management server (the confluent server if running remote, '
+               'and the collective.manager if in collective)\n'
+               '\n\nSee `man %prog` for more info.\n')
+    (options, args) = argparser.parse_args()
+    media = None
+    try:
+        noderange = args[0]
+        operation = args[1]
+        arglength = 2
+        if operation == 'servicedata':
+            media = args[2]
+            arglength = 3
+        if len(args) > arglength:
+            argparser.print_help()
+            sys.exit(1)
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    try:
+        handler = funmap[operation]
+    except KeyError:
+        argparser.print_help()
+        sys.exit(1)
+    handler(noderange, media)
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,167 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import fcntl
+import io
+import numpy as np
+
+import os
+import sixel
+import subprocess
+import sys
+
+class DumbWriter(sixel.SixelWriter):
+    def restore_position(self, output):
+        return
+
+
+def plot(gui, output, plotdata, bins):
+    import matplotlib as mpl
+    if not gui:
+        mpl.use('Agg')
+    import matplotlib.pyplot as plt
+    n, bins, patches = plt.hist(plotdata, bins)
+    plt.show()
+    if not gui:
+        if output:
+            tdata = output
+        else:
+            tdata = io.BytesIO()
+        plt.savefig(tdata)
+    if not gui and not output:
+        writer = DumbWriter()
+        writer.draw(tdata)
+    return n, bins
+
+def textplot(plotdata, bins):
+    n, bins = np.histogram(plotdata, bins)
+    labels = []
+    for bin in bins:
+        labels.append('{0:0.1f}'.format(bin))
+    width = 80
+    # Since this will be primarily piped into, hard to get 
+    # terminal width
+    labelwidth = 0
+    for lab in labels:
+        if len(lab) > labelwidth:
+            labelwidth = len(lab)
+    width -= (labelwidth) + 1
+    labelfmt = '{{0:>{0}s}}|'.format(labelwidth)
+    maxn = 0.0
+    for lgth in n:
+        if lgth > maxn:
+            maxn = float(lgth)
+    for i in range(len(n)):
+        print(labelfmt.format(labels[i]) + '=' * int(np.round((n[i]/maxn) * width)))
+    return n, bins
+
+histogram = False
+aparser = argparse.ArgumentParser(description='Quick access to common statistics')
+aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
+aparser.add_argument('-d', default=None, help='Value used to separate columns')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
+aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
+aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
+aparser.add_argument('-t', default=False, action='store_true', help='Output a histogram in text format')
+aparser.add_argument('-v', default=False, action='store_true', help='Attempt to list nodes relevant to each histogram bar (requires -s, -o, or -t)')
+aparser.add_argument('-b', type=int, default=10, help='Number of bins to use in histogram (default is 10)')
+args = aparser.parse_args(sys.argv[1:])
+plotdata = []
+headlines = int(args.s)
+while headlines >= 0:
+    data = sys.stdin.readline()
+    headlines -= 1
+if args.d:
+    delimiter = args.d
+else:
+    if '\t' in data:
+        delimiter = '\t'
+    elif ' ' in data:
+        delimiter = ' '
+    elif ',' in data:
+        delimiter = ','
+    else:
+        delimiter = ' '  # handle single column
+data = list(csv.reader([data], delimiter=delimiter))[0]
+nodebydatum = {}
+idx = args.c - 1
+autoidx = False
+while data:
+    node = None
+    if ':' in data[0]:
+        node, data[0] = data[0].split(':', 1)
+    else:
+        node = data[0]
+    if idx == -1 and not autoidx:
+        while not autoidx:
+            try:
+                datum = float(data[idx])
+            except ValueError:
+                idx -= 1
+                continue
+            except IndexError:
+                sys.stderr.write('Unable to identify a numerical column\n')
+                sys.exit(1)
+            autoidx = True
+    else:
+        datum = float(data[idx])
+    if node:
+        if datum in nodebydatum:
+            nodebydatum[datum].add(node)
+        else:
+            nodebydatum[datum] = set([node])
+    plotdata.append(datum)
+    data = sys.stdin.readline()
+    data = list(csv.reader([data], delimiter=delimiter))[0]
+n = None
+if args.g or args.o or args.x:
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+if args.t:
+    n, bins = textplot(plotdata, bins=args.b)
+print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))
+if args.v and n is not None and nodebydatum:
+    print('')
+    currbin = bins[0]
+    bins = bins[1:]
+    currbinmembers = []
+    for datum in sorted(nodebydatum):
+        if datum > bins[0]:
+           nextbin = None
+           endbin = bins[0]
+           while len(bins) and bins[0] < datum:
+                nextbin = bins[0]
+                bins = bins[1:]
+           if not nextbin:
+               nextbin = np.max(plotdata)
+           print('Entries between {0} and {1}'.format(currbin, endbin))
+           currbin = nextbin
+           print('-' * 80)
+           print(','.join(sorted(currbinmembers)))
+           print('')
+           print('')
+           currbinmembers = []
+        for node in nodebydatum[datum]:
+            currbinmembers.append(node)
+    if currbinmembers:
+       print('Entries between {0} and {1}'.format(currbin, np.max(plotdata)))
+       print('-' * 80)
+       print(','.join(sorted(currbinmembers)))
+       print('')
+       print('')
@@ -0,0 +1 @@
+../confluent_server/builddeb
@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-import anydbm as dbm
+try:
+    import anydbm as dbm
+except ImportError:
+    import dbm
+import csv
 import errno
+import fnmatch
 import hashlib
 import os
 import shlex
@@ -24,6 +29,7 @@ import socket
 import ssl
 import sys
 import confluent.tlvdata as tlvdata
+import confluent.sortutil as sortutil

 SO_PASSCRED = 16

@@ -34,6 +40,67 @@ _attraliases = {
 }


+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
+
+class Tabulator(object):
+    def __init__(self, headers):
+        self.headers = headers
+        self.rows = []
+
+    def add_row(self, row):
+        self.rows.append(row)
+
+    def get_table(self, order=None):
+        i = 0
+        fmtstr = ''
+        separator = []
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            neededlen = len(head)
+            for row in self.rows:
+                if len(row[i]) > neededlen:
+                    neededlen = len(row[i])
+            separator.append('-' * (neededlen + 1))
+            fmtstr += '{{{0}:>{1}}}|'.format(i, neededlen + 1)
+            i = i + 1
+        fmtstr = fmtstr[:-1]
+        yield fmtstr.format(*self.headers)
+        yield fmtstr.format(*separator)
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                yield fmtstr.format(*row)
+        else:
+            for row in self.rows:
+                yield fmtstr.format(*row)
+
+    def write_csv(self, output, order=None):
+        output = csv.writer(output)
+        output.writerow(self.headers)
+        i = 0
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            i = i + 1
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                output.writerow(row)
+        else:
+            for row in self.rows:
+                output.writerow(row)
+
+
 def printerror(res, node=None):
    exitcode = 0
    if 'errorcode' in res:
@@ -49,7 +116,10 @@ def printerror(res, node=None):


 def cprint(txt):
-    print(txt)
+    try:
+        print(txt)
+    except UnicodeEncodeError:
+        print(txt.encode('utf8'))
    sys.stdout.flush()

 def _parseserver(string):
@@ -81,6 +151,8 @@ class Command(object):
            self.serverloc = server
        if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
            self._connect_unix()
+        elif self.serverloc == '/var/run/confluent/api.sock':
+            raise Exception('Confluent service is not available')
        else:
            self._connect_tls()
        tlvdata.recv(self.connection)
@@ -265,14 +337,19 @@ class Command(object):
        if knownhosts:
            certdata = self.connection.getpeercert(binary_form=True)
            fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
+            fingerprint = fingerprint.encode('utf-8')
            hostid = '@'.join((port, server))
            khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
            if hostid in khf:
                if fingerprint == khf[hostid]:
                    return
                else:
-                    replace = raw_input(
-                        "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    try:
+                        replace = raw_input(
+                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    except NameError:
+                        replace = input(
+                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                    if replace not in ('y', 'Y'):
                        raise Exception("BAD CERTIFICATE")
            cprint('Adding new key for %s:%s' % (server, port))
@@ -311,7 +388,7 @@ def attrrequested(attr, attrlist, seenattributes):
            candidate = candidate.replace('hm', 'hardwaremanagement', 1)
        if candidate in _attraliases:
            candidate = _attraliases[candidate]
-        if candidate.lower() == attr.lower():
+        if fnmatch.fnmatch(attr.lower(), candidate.lower()):
            seenattributes.add(truename)
            return True
        elif attr.lower().startswith(candidate.lower() + '.'):
@@ -325,7 +402,7 @@ def printattributes(session, requestargs, showtype, nodetype, noderange, options
    return print_attrib_path(path, session, requestargs, options)


-def print_attrib_path(path, session, requestargs, options, rename=None):
+def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
    exitcode = 0
    seenattributes = set([])
    for res in session.read(path):
@@ -336,7 +413,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
        for node in sorted(res['databynode']):
            for attr, val in sorted(
                    res['databynode'][node].items(),
-                    key=lambda (k, v): v.get('sortid', k) if isinstance(v, dict) else k):
+                    key=lambda k: k[1].get('sortid', k[0]) if isinstance(k[1], dict) else k[0]):
                if attr == 'error':
                    sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
                    continue
@@ -348,6 +425,8 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                    printattr = rename.get(attr, attr)
                else:
                    printattr = attr
+                if attrprefix:
+                    printattr = attrprefix + printattr
                currattr = res['databynode'][node][attr]
                if show_attr(attr, requestargs, seenattributes, options):
                    if 'value' in currattr:
@@ -407,9 +486,12 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                                (currattr.get('default', None) is not None and
                                currattr.get('value', None) is not None and
                                currattr['value'] != currattr['default'])):
+                            cval = ','.join(currattr['value']) if isinstance(
+                                currattr['value'], list) else currattr['value']
+                            dval = ','.join(currattr['default']) if isinstance(
+                                currattr['default'], list) else currattr['default']
                            cprint('{0}: {1}: {2} (Default: {3})'.format(
-                                node, printattr, currattr['value'],
-                                currattr['default']))
+                                node, printattr, cval, dval))
                    else:

                        try:
@@ -418,11 +500,14 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                            details = False
                        if details:
                            if currattr.get('help', None):
-                                attrout += ' (Help: {0})'.format(
-                                    currattr['help'].encode('utf-8'))
+                                attrout += u' (Help: {0})'.format(
+                                    currattr['help'])
                            if currattr.get('possible', None):
-                                attrout += ' (Choices: {0})'.format(
-                                    ','.join(currattr['possible']))
+                                try:
+                                    attrout += u' (Choices: {0})'.format(
+                                        ','.join(currattr['possible']))
+                                except TypeError:
+                                    pass
                        cprint(attrout)
    if not exitcode:
        if requestargs:
@@ -493,7 +578,7 @@ def printgroupattributes(session, requestargs, showtype, nodetype, noderange, op
                    exitcode = 1
    return exitcode

-def updateattrib(session, updateargs, nodetype, noderange, options):
+def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=None):
    # update attribute
    exitcode = 0
    if options.clear:
@@ -526,6 +611,14 @@ def updateattrib(session, updateargs, nodetype, noderange, options):
                                                            'attributes/all',
                                                            value, key)
        sys.exit(exitcode)
+    elif dictassign:
+        for key in dictassign:
+            if nodetype == 'nodegroups':
+                exitcode = session.simple_nodegroups_command(
+                    noderange, 'attributes/all', dictassign[key], key)
+            else:
+                exitcode = session.simple_noderange_command(
+                    noderange, 'attributes/all', dictassign[key], key)
    else:
        if "=" in updateargs[1]:
            try:
@@ -584,4 +677,4 @@ def check_globbing(noderange):
                'bash or change directories such that there is no filename '
                'that would conflict.'
                '\n'.format(noderange))
-            sys.exit(1)
+            sys.exit(1)
@@ -44,6 +44,8 @@ class ScreenPrinter(object):
        self.fieldwidth = maxlen + textlen + 1  # 1 for column

    def set_output(self, node, text):
+        if self.nodeoutput[node] == text:
+            return
        self.nodeoutput[node] = text
        if len(text) >= self.textlen:
            self.textlen = len(text) + 1
@@ -169,13 +169,18 @@ class GroupedData(object):
        output.flush()

    def print_deviants(self, output=sys.stdout, skipmodal=False, reverse=False,
-                       count=False):
+                       count=False, basenode=None):
        self.generate_byoutput()
        modaloutput = None
        ismodal = True
        revoutput = []
+        if basenode:
+            for checkout in self.byoutput:
+                if basenode in self.byoutput[checkout]:
+                    modaloutput = checkout
        for outdata in sorted(
-                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                self.byoutput, key=lambda x: [0 if modaloutput == x else 1,
+                                              0 - len(self.byoutput[x]),
                                              humanify_nodename(
                                                  self.get_group_text(
                                                      self.byoutput[x]
@@ -25,6 +25,11 @@ try:
 except NameError:
    unicode = str

+try:
+    range = xrange
+except NameError:
+    pass
+
 def decodestr(value):
    ret = None
    try:
@@ -34,11 +39,13 @@ def decodestr(value):
            ret = value.decode('cp437')
        except UnicodeDecodeError:
            ret = value
+    except AttributeError:
+        return value
    return ret

 def unicode_dictvalues(dictdata):
    for key in dictdata:
-        if isinstance(dictdata[key], str):
+        if isinstance(dictdata[key], bytes):
            dictdata[key] = decodestr(dictdata[key])
        elif isinstance(dictdata[key], datetime):
            dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -49,7 +56,7 @@ def unicode_dictvalues(dictdata):


 def _unicode_list(currlist):
-    for i in xrange(len(currlist)):
+    for i in range(len(currlist)):
        if isinstance(currlist[i], str):
            currlist[i] = decodestr(currlist[i])
        elif isinstance(currlist[i], dict):
@@ -64,7 +71,7 @@ def send(handle, data):
            data = data.encode('utf-8')
        except AttributeError:
            pass
-    if isinstance(data, str) or isinstance(data, unicode):
+    if isinstance(data, bytes) or isinstance(data, unicode):
        # plain text, e.g. console data
        tl = len(data)
        if tl == 0:
@@ -24,10 +24,18 @@ a confluent server.
 %setup -n %{name}-%{version} -n %{name}-%{version}

 %build
-python setup.py build
+%if "%{dist}" == ".el8"
+python3 setup.py build
+%else
+python2 setup.py build
+%endif

 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%if "%{dist}" == ".el8"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%endif


 %clean
@@ -36,4 +36,256 @@ alias nodereseat='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export C
 alias noderun='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; noderun'
 alias nodesensors='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesensors'
 alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesetboot'
+alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
+alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
+# Do not continue for non-bash shells, the rest of this sets up bash completion functions
+[ -z "$BASH_VERSION" -o -z "$PS1" ] && return
+
+
+_confluent_get_args()
+{
+    CMPARGS=($COMP_LINE)
+    NUMARGS=${#CMPARGS[@]}
+    if [ "${COMP_WORDS[-1]}" == '' ]; then
+        NUMARGS=$((NUMARGS+1))
+        CMPARGS+=("")
+    fi
+    GENNED=""
+    for CAND in ${COMP_CANDIDATES[@]}; do
+        candarray=(${CAND//,/ })
+        matched=0
+        for c in "${candarray[@]}"; do
+            for arg in "${CMPARGS[@]}"; do
+                if [ "$arg" = "$c" ]; then
+                    matched=1
+                    break
+                fi
+            done
+        done
+        if [ 0 = $matched ]; then
+            for c in "${candarray[@]}"; do
+                GENNED+=" $c"
+            done
+        fi
+    done
+}
+
+function _confluent_generic_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+_confluent_nodeidentify_completion()
+{
+    COMP_CANDIDATES=("on,off,blink -h")
+    _confluent_generic_completion
+}
+
+
+_confluent_nodesetboot_completion()
+{
+    COMP_CANDIDATES=("default,cd,network,setup,hd -h -b -p")
+    _confluent_generic_completion
+}
+
+_confluent_nodepower_completion()
+{
+   COMP_CANDIDATES=("boot,off,on,status -h -p")
+   _confluent_generic_completion
+}
+
+_confluent_nodemedia_completion()
+{
+    COMP_CANDIDATES=("list,upload,attach,detachall -h")
+    _confluent_get_args
+    if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[-2]} == 'upload' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodefirmware_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[2]} == 'update' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodeshell_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -c -- ${COMP_WORDS[-1]}))
+        return
+    fi
+    if [ $NUMARGS -gt 3 ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodelicense_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'save' ]; then
+        compopt -o dirnames
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+         _confluent_nr_completion
+         return
+    fi
+}
+
+_confluent_nodesupport_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'servicedata' ]; then
+        compopt -o  dirnames
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+         _confluent_nr_completion
+         return
+    fi
+}
+
+_confluent_nodeattrib_completion()
+{
+    COMP_CANDIDATES=$(nodeattrib '~.>1' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_generic_completion
+}
+
+
+_confluent_nn_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+_confluent_nr_completion()
+{
+    CMPARGS=($COMP_LINE)
+    NUMARGS=${#CMPARGS[@]}
+    if [ "${COMP_WORDS[-1]}" == '' ]; then
+        NUMARGS=$((NUMARGS+1))
+    fi
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    #COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+_confluent_ng_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+complete -F _confluent_nodeattrib_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodegroupattrib
+complete -F _confluent_nr_completion nodebmcreset
+complete -F _confluent_nodesetboot_completion nodeboot
+complete -F _confluent_nr_completion nodeconfig
+complete -F _confluent_nn_completion nodeconsole
+complete -F _confluent_nr_completion nodeeventlog
+complete -F _confluent_nodefirmware_completion nodefirmware
+complete -F _confluent_ng_completion nodegroupattrib
+complete -F _confluent_ng_completion nodegroupremove
+complete -F _confluent_nr_completion nodehealth
+complete -F _confluent_nodeidentify_completion nodeidentify
+complete -F _confluent_nr_completion nodeinventory
+complete -F _confluent_nodeattrib_completion nodelist
+complete -F _confluent_nodemedia_completion nodemedia
+complete -F _confluent_nodepower_completion nodepower
+complete -F _confluent_nr_completion noderemove
+complete -F _confluent_nr_completion nodereseat
+complete -F _confluent_nodeshell_completion noderun
+complete -F _confluent_nr_completion nodesensors
+complete -F _confluent_nodesetboot_completion nodesetboot
+complete -F _confluent_nodeshell_completion nodeshell
+complete -F _confluent_nodesupport_completion nodesupport
+complete -F _confluent_nodelicense_completion nodelicense
+
@@ -3,7 +3,7 @@ collate(1) -- Organize text input by node

 ## SYNOPSIS

-`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r]`
+`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r] [-l lognametemplate]`

 ## DESCRIPTION

@@ -26,6 +26,10 @@ node and group names sorted alphanumerically.
   Express all but the most common result group in terms of diff from
   the most common result group

+* `-l`, `--log`:
+   Save output per node to individual log files, replacing {node} in the name
+   with the nodename of each
+
 * `-w`, `--watch`:
   Update results dynamically as data becomes available, rather than
   waiting    for the command to fully complete.
@@ -85,4 +89,4 @@ node and group names sorted alphanumerically.
  `  Processors.ExecuteDisableBit=Enable`  

   
-   
+   
@@ -0,0 +1,55 @@
+collective(1) -- Check and manage a confluent collective
+==============================
+
+## SYNOPSIS
+
+`collective invite <server>`  
+`collective join <server> [-i TOKEN]`  
+`collective show`  
+`collective gencert`  
+
+
+## DESCRIPTION
+
+**collective** helps manage the collective mode of confluent, where multiple
+confluent servers are linked together to act as one.  For example, the procedure to set up
+a collective to run on three servers called mgt1, mgt2, and mgt3, first install and start
+confluent as usual on the three servers.  On mgt1, run `collective invite mgt2` and an
+invitation token will be output.  On mgt2, either run `collective join mgt1` to paste
+the token interactively, or `collective join mgt1 -i <token>`.  At this point, either
+mgt1 or mgt2 can bring in mgt3.  For example on mgt2 run `collective invite mgt3` and
+on mgt3 run `collective join mgt2 -i <token>`
+
+This can be linked together in the following manner with ssh:
+on mgt1:
+  `# ssh mgt2 collective join mgt1 -i $(collective invite mgt2)`
+
+Note that a collective is only redundant with 3 or more members.  The collective
+will function so long as more than half of the members are online.  A collective
+of two members is supported, but without redundancy.
+
+Also note that the collective leader role is dynamic, but has no impact on interacting
+with confluent.  It is merely an internal role that can dynamically change depending
+on circumstances.
+
+## OPTIONS
+
+ * `-i`:
+    Provide the token as an argument rather than interactively.
+   
+## EXAMPLES
+ * Inviting a server called mgt2:
+  `# collective invite mgt2`  
+  `bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+
+* On mgt2, joining mgt1:
+  `# collective join mgt1 -i bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+  `Success`  
+
+* Showing the collective state:
+  `# collective show`  
+  `Quorum: True`  
+  `Leader: mgt1`  
+  `Active collective members:`  
+  `  mgt2`  
+
@@ -7,6 +7,7 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  

 ## DESCRIPTION

@@ -22,6 +23,9 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.

+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 If the word all is specified, then all available attributes are given.
 Omitting any attribute name or the word 'all' will display only attributes
 that are currently set.
@@ -40,6 +44,8 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
  Clear specified nodeattributes
 * `-e`, `--environment`:
  Set specified attributes based on exported environment variable of matching name.  Environment variable names may be lower case or all upper case.  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+* `-p`, `--prompt`:
+  Request interactive prompting to provide values rather than the command line or environment variables.

 ## EXAMPLES
 * Listing matching nodes of a simple noderange:
@@ -85,6 +91,12 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
    `n1: console.method: `  
    `n2: console.method: `  

+* List all switches that a node is described as connected to:
+    `# nodeattrib d1 net.*switch`  
+    `d1: net.mgt.switch: mgtswitch1`  
+    `d1: net.pxe.switch: pxeswitch1`  
+    `d1: net.switch:`  
+
 ## SEE ALSO

 nodegroupattrib(8), nodeattribexpressions(5)
@@ -9,6 +9,18 @@ expression to generate the value.
 An expression will contain some directives wrapped in `{}` characters. Within
 `{}` are a number of potential substitute values and operations.

+Note that syntax of expressions can have overlap with the shell syntax.
+For example:
+
+`$ echo (n2)`  
+`-bash: syntax error near unexpected token `n2'`  
+
+In such a case, it helps to quote the expression to allow it to be passed:
+
+`$ echo '(n2)'`  
+`(n2)`  
+
+
 The most common operation is to extract a number from the nodename. These 
 values are available as n1, n2, etc. So for example attributes for a node named 
 b1o2r3u4 would have {n1} as 1, {n2} as 2, {n3} as 3, and {n4} as 4. 
@@ -4,12 +4,44 @@ nodeboot(8) -- Reboot a confluent node to a specific device
 ## SYNOPSIS

 `nodeboot <noderange>`  
-`nodeboot <noderange>` [net|setup]  
+`nodeboot [options] <noderange>` [default|cd|network|setup|hd]  

 ## DESCRIPTION

 **nodeboot** reboots nodes in a noderange.  If an additional argument is given,
-it sets the node to specifically boot to that as the next boot.
+it sets the node to specifically boot to that as the next boot.  This 
+performs an immediate reboot without waiting for the OS.  To set the boot
+device without inducing a reboot, see the `nodesetboot` command.
+
+## OPTIONS
+
+* `-b`, `--bios`:
+  For a system that supports both BIOS and UEFI style boot, request BIOS style
+  boot if supported (some platforms will UEFI boot with this flag anyway).
+
+* `-u`, `--uefi`:
+  This flag does nothing, it is for command compatibility with xCAT's rsetboot
+
+* `-p`, `--persist`:
+  For a system that supports it, mark the boot override to persist rather than
+  be a one time change.  Many systems do not support this functionality.
+
+* `default`:
+  Request a normal default boot with no particular device override
+
+* `cd`:
+  Request boot from media.  Note that this can include physical CD,
+  remote media mounted as CD/DVD, and detachable hard disks drives such as usb
+  key devices.
+  
+* `network`:
+  Request boot to network
+  
+* `setup`:
+  Request to enter the firmware configuration menu (e.g. F1 setup) on next boot.
+
+* `hd`:
+  Boot straight to hard disk drive

 ## EXAMPLES
 * Booting n3 and n4 to the default boot behavior:
@@ -3,8 +3,8 @@ nodeconfig(8) -- Show or change node configuration

 ## SYNOPSIS

-`nodeconfig <noderange> [options] [<configuration>..]`
-`nodeconfig <noderange> [options] [<configuration=value>..]`
+`nodeconfig <noderange> [options] [<configuration>..]`  
+`nodeconfig <noderange> [options] [<configuration=value>..]`  

 ## DESCRIPTION

@@ -30,6 +30,11 @@ given as a node expression, as documented in the man page for nodeattribexpressi
  Provide detailed data as available.  This can include help text and valid
  values for a setting.

+* `-r`, `--restoredefault`:
+  Request that the specified component of the targeted nodes will have its
+  configuration reset to default.  Currently the only component implemented
+  is uefi.
+
 * `-b`, '--batch':
   Provide arguments as lines of a file, rather than the command line.

@@ -2,7 +2,7 @@ nodeconsole(8) -- Open a console to a confluent node
 =====================================================

 ## SYNOPSIS
-`nodeconsole <node>`
+`nodeconsole [options] <noderange>`

 ## DESCRIPTION

@@ -16,6 +16,12 @@ will initiate an automatic retry interval that is randomized between 2 and 4 min
 The reopen escape sequence below requests an immediate retry, as does connecting
 a new session.

+## OPTIONS
+
+* `-t`, `--tile`:
+  Use tmux to arrange consoles of the given noderange into a tiled layout on
+  the terminal screen
+
 ## ESCAPE SEQUENCE COMMANDS

 While connected to a console, a number of commands may be performed through escape
@@ -1,4 +1,5 @@
 nodedefine(8) -- Define new confluent nodes
+===================================================================

 ## SYNOPSIS

@@ -75,6 +75,20 @@ the nodes.
 * `-e MAC`, `--ethaddr=MAC`:
  Operate against the system with the specified MAC
  address
+* `-f FIELDS`, `--fields=FIELDS`:
+  Request a custom set of fields.  The available fields are:  
+  Node: The node name if a correlation has been identified  
+  Model: The model number  
+  Serial: The serial number  
+  UUID: The UUID as it should appear in DMI  
+  Type: Device type (e.g. lenovo-xcc, pxe-client, etc)  
+  IP: The confirmed working IP addresses associated with the record  
+  Mac: Mac address of the relevant network interface  
+  Switch: The nearest detected switch to the entry  
+  Port: Port of the switch that most closely connects to the network interface  
+  Advertised IP: IP addresses that may not have been confirmed, but are advertised  
+* `-o ORDER`, `--order=ORDER`:
+  Order output by given field.  Field names are the same as documented in the -f argument.
 * `-t TYPE`, `--type=TYPE`:
  Operate against the system of the specified type
 * `-c`, `--csv`:
@@ -4,7 +4,7 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 ## SYNOPSIS

 `nodefirmware <noderange>`  
-`nodefirmware <noderange> <components>|core`  
+`nodefirmware <noderange> list|<components>|core`  
 `nodefirmware <noderange> update [--backup] <filename>`  

 ## DESCRIPTION
@@ -7,6 +7,7 @@ nodegroupattrib(8) -- List or change confluent nodegroup attributes
 `nodegroupattrib <group> [<nodeattribute>...]`  
 `nodegroupattrib <group> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodegroupattrib <group> [-c] [<nodeattribute1> <nodeattribute2=value2> ...]`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  

 ## DESCRIPTION

@@ -1,4 +1,5 @@
 nodegroupdefine(8) -- Define new confluent node group
+===================================================================

 ## SYNOPSIS

@@ -6,7 +7,7 @@ nodegroupdefine(8) -- Define new confluent node group

 ## DESCRIPTION

-`nodegroupdefine` allows the definition of a new node for the confluent management
+`nodegroupdefine` allows the definition of a new nodegroup for the confluent management
 service.  It may only define a single group name at a time.
 It has the same syntax as `nodegroupattrib(8)`, and the commands differ in
 that `nodegroupattrib(8)` will error if a node group does not exist.
@@ -0,0 +1,14 @@
+nodegrouplist(8) -- List the defined confluent nodegroups
+===================================================================
+
+## SYNOPSIS
+
+`nodegrouplist`  
+
+## DESCRIPTION
+
+`nodegrouplist` lists the currently defined groups in confluent.
+
+## SEE ALSO
+
+nodeattrib(8), nodeattribexpressions(5), nodegroupattrib(8)
@@ -3,7 +3,7 @@ nodeidentify(8) -- Control the identify LED of confluent nodes

 ## SYNOPSIS

-`nodidentify <noderange> [on|off]`  
+`nodidentify <noderange> [on|off|blink]`  

 ## DESCRIPTION

@@ -13,6 +13,7 @@ options are supported:

 * `on`: Turn on the identify LED
 * `off`: Turn off the identify LED
+* `blink`: Set the identify LED to blink (when supported by the system)

 ## EXAMPLES:

@@ -0,0 +1,17 @@
+nodelicense(8) -- Manage license keys on BMC
+=================================================================
+
+## SYNOPSIS
+
+`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
+
+## DESCRIPTION
+
+`nodelicense` manages license keys on supported BMCs. Without an argument, the command
+lists currently installed license.  Using `delete` will remove the specified license name
+from th eBMC.  The `save` subcommand will take the passed directory (which may be in the form
+of /path/to/{node}/ to have the node name substituted for each node) and back up installed licenses
+to that directory. The `install` command will take the specified filename and install.  The filename
+argument may be of the form  xcc_fod_0034_7X21{id.serial}.key to have the serial number substituted
+to allow unique licenses to be specified in a single command.
+
@@ -17,6 +17,9 @@ displayed.  If `-b` is specified, it will also display information on
 how inherited and expression based attributes are defined.  There is more
 information on node attributes in nodeattributes(5) man page.

+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 ## OPTIONS

 * `-b`, `--blame`:
@@ -36,10 +36,13 @@ Also, regular expressions may be used to indicate nodes with names matching cert
 The other major noderange primitive is indicating nodes by some attribute value:
 `location.rack=7`

+The attribute name may use a wildcard:
+`net.*switch=switch1`
+
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`

-Exclusions can be done by prepending a ‘-‘ before a portion of a noderange:
+Exclusions can be done by prepending a '-' before a portion of a noderange:
 `rack1,-n2`
 `compute,-rack1`
 `compute,-location.row=12`
@@ -51,7 +54,7 @@ To indicate nodes that match multiple selections at once (set intersection), the
 For complex expressions, () may be used to indicate order of expanding the noderange to be explicit
 `rack1,-(console.logging=full@compute)`

-Noderange syntax can also indicate ‘pagination’, or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
+Noderange syntax can also indicate 'pagination', or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
 `rack1>3<6`

 The above would show the seventh through ninth nodes of the rack1 group. Like all other noderange operations, this may be combined with any of the above, but must appear as the very last operation. Ordering is done with a natural sort.
@@ -30,7 +30,10 @@ control.
 * `-p`, `--persist`:
  For a system that supports it, mark the boot override to persist rather than
  be a one time change.  Many systems do not support this functionality.
-  
+
+* `-u`, `--uefi`:
+  This flag does nothing, it is for command compatibility with xCAT's rsetboot
+
 * `default`:
  Request a normal default boot with no particular device override

@@ -0,0 +1,65 @@
+nodestorage(8) -- Examine/Modify storage configuration of a node
+============================================
+
+## SYNOPSIS
+
+`nodestorage <noderange> [show|create|delete] [options]`  
+
+## DESCRIPTION
+
+`nodestorage` provides access to the remote storage configuration of
+the noderange.
+
+## OPTIONS
+* `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
+  RAID level to use when creating an array
+
+* `-d` **DISKS**, `--disks`=**DISKS**:
+   Comma separated list of disks to use, or the word "rest" to
+   indicate use of all available disks
+
+* `-s` **SIZE**, `--size`=**SIZE**:
+  Comma separated list of sizes to use when creating
+  volumes.  The sizes may be absolute size (e.g. 16gb),
+  percentage (10%) or the word "rest" to use remaining
+  capacity, default behavior is to use all capacity to
+  make a volume
+* `-n` **NAME**, `--name`=**NAME**:
+  Comma separated list of names to use when naming
+  volumes, or selecting a volume for delete.  Default
+  behavior is to use implementation provided default
+
+## EXAMPLES
+
+* Deleting the volume `somedata`:
+    `$ nodestorage d5 delete somedata`  
+    `Deleted: somedata`  
+
+* Creating a raid5 of 4 disks and a volume named `somedata`:
+    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata`  
+    `d5: Volume somedata: Size: 1.905 TB`  
+    `d5: Volume somedata: State: Optimal`  
+    `d5: Volume somedata: Array 1-2`  
+
+* Showing current storage configuration of `d3`:
+    `$ nodestorage d3`  
+    `d3: Disk m.2-0 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-0 State: online`  
+    `d3: Disk m.2-0 FRU: 00LF428`  
+    `d3: Disk m.2-0 Serial Number: H6B80054`  
+    `d3: Disk m.2-0 Array: 0-0`  
+    `d3: Disk m.2-1 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-1 State: online`  
+    `d3: Disk m.2-1 FRU: 00LF428`  
+    `d3: Disk m.2-1 Serial Number: H6B80059`  
+    `d3: Disk m.2-1 Array: 0-0`  
+    `d3: Array 0-0 Available Capacity: 0.000 MB`  
+    `d3: Array 0-0 Total Capacity: 131.072 GB`  
+    `d3: Array 0-0 RAID: RAID 1`  
+    `d3: Array 0-0 Disks: m.2-0,m.2-1`  
+    `d3: Array 0-0 Volumes: new_vd`  
+    `d3: Volume new_vd: Size: 122.040 GB`  
+    `d3: Volume new_vd: State: Optimal`  
+    `d3: Volume new_vd: Array 0-0`  
+
+
@@ -0,0 +1,29 @@
+nodesupport(8) -- Utilities for interacting with vendor support
+=================================================================
+
+## SYNOPSIS
+
+`nodesupport <noderange> servicedata <directory or filename>`  
+
+## DESCRIPTION
+
+`nodesupport` provides capabilities associated with interactiong with support.
+Currently it only has the `servicedata` subcommand.  `servicedata` takes
+an argument that is either a directory name (that can be used for a single node
+or multiple nodes) or a file name (only to be used with single node noderange).
+Note that the file will be downloaded to the confluent server that actually
+connects to the managed system, so it will download to the remote system if running
+remotely and will download to the collective.manager indicated system if
+running in collective mode.
+
+## EXAMPLES
+
+* Download support data from a single node to a specific filename  
+`# nodesupport d1 servicedata svcdata.out`  
+`d1:initializing:  15%`  
+
+* Download support data from multiple nodes to a directory  
+`# nodesupport d1-d4 servicedata service/`    
+`d1:initializing:   0% d2:initializing:   0% d3:initializing:   0% d4:initializing:   0%`  
+`# ls service/`  
+`d1.svcdata d2.svcdata  d3.svcdata  d4.svcdata`  
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2

 # This is a sample python script for going through all observed mac addresses
 # and assuming they are BMC related and printing nodeattrib commands
@@ -22,10 +22,10 @@ This provides the modules common for both client and server
 %setup -n %{name}-%{version} -n %{name}-%{version}

 %build
-python setup.py build
+python2 setup.py build

 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin

 %clean
 rm -rf $RPM_BUILD_ROOT
@@ -1,17 +0,0 @@
-[metadata]
-name = confluent_common
-summary = Confluent Common Libraries
-description-file =
-    README.txt
-author = Jarrod Johnson
-author-email = jjohnson2@lenovo.com
-home-page = http://xcat.sf.net/
-classifier =
-    Intended Audience :: Information Technology
-    Intended Audience :: System Administrators
-    License :: OSI Approved :: Apache Software License
-    Operating System :: POSIX :: Linux
-    Programming Language :: Python :: 2.6
-    Programming Language :: Python :: 2.7
-
-[files]
@@ -1,3 +1,4 @@
+include pam/*
 include sysvinit/*
 include systemd/*
 include sysctl/*
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2

 import argparse
 import errno
@@ -22,7 +22,7 @@ except NameError:
    pass

 def make_certificate():
-    umask = os.umask(0077)
+    umask = os.umask(0o77)
    try:
        os.makedirs('/etc/confluent/cfg')
    except OSError as e:
@@ -61,13 +61,27 @@ def join_collective(server, invitation):
        make_certificate()
    s = client.Command().connection
    while not invitation:
-        invitation = raw_input('Paste the invitation here: ')
+        invitation = input('Paste the invitation here: ')
    tlvdata.send(s, {'collective': {'operation': 'join',
                                    'invitation': invitation,
                                    'server': server}})
    res = tlvdata.recv(s)
-    print(res.get('collective',
-                  {'status': 'Unknown response: ' + repr(res)})['status'])
+    res = res.get('collective',
+                  {'status': 'Unknown response: ' + repr(res)})
+    print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)
+
+def delete_member(name):
+    s = client.Command().connection
+    tlvdata.send(s, {'collective': {'operation': 'delete',
+                                    'member': name}})
+    res = tlvdata.recv(s)
+    res = res.get('collective',
+                  {'status': 'Unknown response: ' + repr(res)})
+    print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)


 def show_collective():
@@ -100,11 +114,13 @@ def main():
    sl = sp.add_parser('show', help='Show information about the collective')
    ic = sp.add_parser('invite', help='Generate a invitation to allow a new '
                                    'confluent instance to join as a '
-                                    'collective member')
+                                    'collective member.  Run collective invite -h for more information')
    ic.add_argument('name', help='Name of server to invite to join the '
                                 'collective')
-    jc = sp.add_parser('join', help='Join a collective')
-    jc.add_argument('server', help='A server currently in the collective')
+    dc = sp.add_parser('delete', help='Delete a member of a collective')
+    dc.add_argument('name', help='Name of server to delete from collective')
+    jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
+    jc.add_argument('server', help='Existing collective member that ran invite and generated a token')
    jc.add_argument('-i', help='Invitation provided by runniing invite on an '
                               'existing collective member')
    cmdset = a.parse_args()
@@ -116,6 +132,8 @@ def main():
        join_collective(cmdset.server, cmdset.i)
    elif cmdset.command == 'show':
        show_collective()
+    elif cmdset.command == 'delete':
+        delete_member(cmdset.name)

 if __name__ == '__main__':
    main()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
@@ -0,0 +1,55 @@
+from os.path import exists
+import shutil
+import socket
+import subprocess
+import tempfile
+
+def get_openssl_conf_location():
+    if exists('/etc/pki/tls/openssl.cnf'):
+        return '/etc/pki/tls/openssl.cnf'
+    elif exists('/etc/ssl/openssl.cnf');
+        return '/etc/ssl/openssl.cnf'
+    else:
+        raise Exception("Cannot find openssl config file")
+
+def get_ip_addresses():
+    lines = subprocess.check_output('ip addr'.split(' '))
+    for line in lines.split('\n'):
+        if line.startswith('    inet6 '):
+            line = line.replace('    inet6 ', '').split('/')[0]
+            if line.startswith('fe80::'):
+                continue
+            if line == '::1':
+                continue
+        elif line.startswith('    inet '):
+            line = line.replace('    inet ', '').split('/')[0]
+            if line == '127.0.0.1':
+                continue
+            if line.startswith('169.254.'):
+                continue
+        else:
+            continue
+        yield line
+
+def create_certificate():
+    shortname = socket.gethostname().split('.')[0]
+    longname = socket.getfqdn()
+    subprocess.check_call(
+        'openssl ecparam -name secp384r1 -genkey -out privkey.pem'.split(' '))
+    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
+    san.append('DNS:{0}'.format(shortname))
+    san.append('DNS:{0}'.format(longname))
+    san = ','.join(san)
+    sslcfg = get_openssl_conf_location()
+    tmpconfig = tempfile.mktemp()
+    shutil.copy2(sslcfg, tmpconfig)
+    with open(tmpconfig, 'a') as cfgfile:
+        cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
+    subprocess.check_call(
+        'openssl req -new -x509 -key privkey.pem -days 7300 -out cert.pem '
+        '-subj /CN={0} -extensions SAN '
+        '-config {1}'.format(longname, tmpconfig).split(' ')
+    )
+
+if __name__ == '__main__':
+    create_certificate()
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2017 Lenovo
@@ -82,7 +82,7 @@ elif args[0] == 'dump':
              "or -s to do encrypted backup that requires keys.json from "
              "another backup to restore.")
        sys.exit(1)
-    os.umask(077)
+    os.umask(0o77)
    main._initsecurity(conf.get_config())
    if not os.path.exists(dumpdir):
        os.makedirs(dumpdir)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
@@ -0,0 +1,64 @@
+#!/bin/bash
+cd `dirname $0`
+PKGNAME=$(basename $(pwd))
+DPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+OPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+if grep wheezy /etc/os-release; then
+    DPKGNAME=python-$DPKGNAME
+fi
+cd ..
+mkdir -p /tmp/confluent # $DPKGNAME
+cp -a * .git /tmp/confluent # $DPKGNAME
+cd /tmp/confluent/$PKGNAME
+if [ -x ./makeman ]; then
+    ./makeman
+fi
+./makesetup
+VERSION=`cat VERSION`
+cat > setup.cfg << EOF
+[install]
+install-purelib=/opt/confluent/lib/python
+install-scripts=/opt/confluent/bin
+
+[sdist_dsc]
+package=$DPKGNAME
+EOF
+
+python setup.py sdist > /dev/null 2>&1
+py2dsc dist/*.tar.gz
+shopt -s extglob
+cd deb_dist/!(*.orig)/
+if [ "$OPKGNAME" = "confluent-server" ]; then
+    if grep wheezy /etc/os-release; then
+        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl/' debian/control
+    else
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python-lxml, python-eficompressor, python-pycryptodome, python-dateutil/' debian/control
+    fi
+    if grep wheezy /etc/os-release; then
+        echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
+    else
+        echo 'confluent_client confluent-client' >> debian/pydist-overrides
+    fi
+fi
+head -n -1 debian/control > debian/control1
+mv debian/control1 debian/control
+echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
+#echo 'Provides: python-'$DPKGNAME >> debian/control
+#echo 'Conflicts: python-'$DPKGNAME >> debian/control
+#echo 'Replaces: python-'$DPKGNAME' (<<2)' >> debian/control
+#echo 'Breaks: python-'$DPKGNAME' (<<2)' >> debian/control
+
+dpkg-buildpackage -rfakeroot -uc  -us -i
+if [ $? -ne 0 ]; then
+    echo "[ERROR] rpmbuild returned non-zero, run: rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec"
+    exit 1
+else
+    cd -
+    # Clean up the generated files in this directory
+    rm -rf $PKGNAME.egg-info dist setup.py
+    rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d)
+    if [ ! -z "$1" ]; then
+        mv deb_dist/* $1/
+    fi
+fi
+exit 0
@@ -6,7 +6,7 @@ fi
 ./makesetup
 VERSION=`cat VERSION`
 PKGNAME=$(basename $(pwd))
-python setup.py sdist > /dev/null 2>&1
+python3 setup.py sdist > /dev/null 2>&1
 cp dist/*.tar.gz ~/rpmbuild/SOURCES
 sed -e 's/#VERSION#/'$VERSION/ $PKGNAME.spec.tmpl > ~/rpmbuild/SPECS/$PKGNAME.spec
 rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec 2> /dev/null |grep ^Wrote:
@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

-# Copyright 2016 Lenovo
+# Copyright 2016-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,6 +83,8 @@ class AsyncSession(object):
        self.reaper = eventlet.spawn_after(15, self.destroy)

    def add(self, requestid, rsp):
+        if self.responses is None:
+            return
        self.responses.append((requestid, rsp))
        if self._evt:
            self._evt.send()
@@ -107,6 +109,7 @@ class AsyncSession(object):
        for console in self.consoles:
            _consolesessions[console]['session'].destroy()
        self.consoles = None
+        self.responses = None
        del _asyncsessions[self.asyncid]

    def run_handler(self, handler, requestid):
@@ -166,6 +169,9 @@ def handle_async(env, querydict, threadset):
        currsess = AsyncSession()
        yield messages.AsyncSession(currsess.asyncid)
        return
+    if  querydict['asyncid'] not in _asyncsessions:
+        raise exc.InvalidArgumentException(
+                'Invalid or expired async id')
    mythreadid = greenlet.getcurrent()
    threadset.add(mythreadid)
    loggedout = None
@@ -23,11 +23,15 @@ import confluent.config.configmanager as configmanager
 import eventlet
 import eventlet.tpool
 import Cryptodome.Protocol.KDF as KDF
+from fnmatch import fnmatch
 import hashlib
 import hmac
 import multiprocessing
+import confluent.userutil as userutil
+import confluent.util as util
+pam = None
 try:
-    import PAM
+    import confluent.pam as pam
 except ImportError:
    pass
 import time
@@ -39,24 +43,59 @@ _passchecking = {}
 authworkers = None
 authcleaner = None

+_allowedbyrole = {
+    'Operator': {
+        'retrieve': ['*'],
+        'create': [
+            '/noderange/',
+            '/nodes/',
+            '/node*/media/uploads/',
+            '/node*/inventory/firmware/updates/*',
+            '/node*/suppport/servicedata*',
+            '/node*/attributes/expression',
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+            '/node*/configuration/*',
+        ],
+        'update': [
+            '/discovery/*', 
+            '/networking/macs/rescan',
+            '/node*/power/state', 
+            '/node*/power/reseat',
+            '/node*/attributes/*',
+            '/node*/media/*tach',
+            '/node*/boot/nextdevice',
+            '/node*/identify',
+            '/node*/configuration/*',
+        ],
+        'start': [
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+        ],
+        'delete': [
+            '/discovery/*',
+            '/node*',
+        ],
+    },
+    'Monitor': {
+        'retrieve': [
+            '/node*/health/hardware',
+            '/node*/power/state',
+            '/node*/sensors/*',
+            '/nodes/',
+            '/',
+        ],
+    }
+}

-class Credentials(object):
-    def __init__(self, username, passphrase):
-        self.username = username
-        self.passphrase = passphrase
-        self.haspam = False
-
-    def pam_conv(self, auth, query_list):
-        # use stored credentials in a pam conversation
-        self.haspam = True
-        resp = []
-        for query_entry in query_list:
-            query, pamtype = query_entry
-            if query.startswith('Password'):
-                resp.append((self.passphrase, 0))
-            else:
-                return None
-        return resp
+_deniedbyrole = {
+    # This supersedes the above and is only consulted after the allowed has happened
+    'Operator': {
+        'update': [
+            '/node*/configuration/management_controller/users/*',
+        ]
+    }
+}


 def _prune_passcache():
@@ -65,9 +104,9 @@ def _prune_passcache():
    while True:
        curtime = time.time()
        for passent in _passcache.iterkeys():
-            if passent[2] < curtime - 10:
+            if passent[2] < curtime - 90:
                del _passcache[passent]
-        eventlet.sleep(10)
+        eventlet.sleep(90)


 def _get_usertenant(name, tenant=False):
@@ -81,11 +120,13 @@ def _get_usertenant(name, tenant=False):
    administrator account a tenant gets.
    Otherwise, just assume a user in the default tenant
    """
+    if not isinstance(name, bytes):
+        name = name.encode('utf-8')
    if not isinstance(tenant, bool):
        # if not boolean, it must be explicit tenant
        user = name
-    elif '/' in name:  # tenant scoped name
-        tenant, user = name.split('/', 1)
+    elif b'/' in name:  # tenant scoped name
+        tenant, user = name.split(b'/', 1)
    elif configmanager.is_tenant(name):
        # the account is the implicit tenant owner account
        user = name
@@ -93,6 +134,9 @@ def _get_usertenant(name, tenant=False):
    else:  # assume it is a non-tenant user account
        user = name
        tenant = None
+    user = util.stringify(user)
+    if tenant:
+        tenant = util.stringify(tenant)
    yield user
    yield tenant

@@ -112,21 +156,37 @@ def authorize(name, element, tenant=False, operation='create',
            and the relevant ConfigManager object for the context of the
            request.
    """
-    if operation not in ('create', 'start', 'update', 'retrieve', 'delete'):
-        return None
+    # skipuserobj is a leftover from the now abandoned plan to use pam session
+    # to do authorization and authentication.  Now confluent always does authorization
+    # even if pam does authentication.
+    if operation not in ('create', 'start', 'update', 'retrieve', 'delete', None):
+        return False
    user, tenant = _get_usertenant(name, tenant)
    if tenant is not None and not configmanager.is_tenant(tenant):
-        return None
+        return False
    manager = configmanager.ConfigManager(tenant, username=user)
-    if skipuserobj:
-        return None, manager, user, tenant, skipuserobj
    userobj = manager.get_user(user)
+    if not userobj:
+        for group in userutil.grouplist(user):
+            userobj = manager.get_usergroup(group)
+            if userobj:
+                break
    if userobj:  # returning
+        role = userobj.get('role', 'Administrator')
+        if element and role != 'Administrator':
+            for rule in _allowedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    break
+            else:
+                return False
+            for rule in _deniedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    return False
        return userobj, manager, user, tenant, skipuserobj
-    return None
+    return False


-def check_user_passphrase(name, passphrase, element=None, tenant=False):
+def check_user_passphrase(name, passphrase, operation=None, element=None, tenant=False):
    """Check a a login name and passphrase for authenticity and authorization

    The function combines authentication and authorization into one function.
@@ -157,63 +217,66 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
        # would normally make an event and wait
        # but here there's no need for that
        eventlet.sleep(0.5)
-    credobj = Credentials(user, passphrase)
-    try:
-        pammy = PAM.pam()
-        pammy.start(_pamservice, user, credobj.pam_conv)
-        pammy.authenticate()
-        pammy.acct_mgmt()
-        del pammy
-        return authorize(user, element, tenant, skipuserobj=False)
-    except NameError:
-        pass
-    except PAM.error:
-        if credobj.haspam:
-            return None
+    cfm = configmanager.ConfigManager(tenant, username=user)
+    ucfg = cfm.get_user(user)
+    if ucfg is None:
+        try:
+            for group in userutil.grouplist(user):
+                ucfg = cfm.get_usergroup(group)
+                if ucfg:
+                    break
+        except KeyError:
+            pass
+    if ucfg is None:
+        eventlet.sleep(0.05)
+        return None
    if (user, tenant) in _passcache:
-        if passphrase == _passcache[(user, tenant)]:
-            return authorize(user, element, tenant)
+        if hashlib.sha256(passphrase).digest() == _passcache[(user, tenant)]:
+            return authorize(user, element, tenant, operation=operation)
        else:
            # In case of someone trying to guess,
            # while someone is legitimately logged in
            # invalidate cache and force the slower check
            del _passcache[(user, tenant)]
-            return None
-    cfm = configmanager.ConfigManager(tenant, username=user)
-    ucfg = cfm.get_user(user)
-    if ucfg is None or 'cryptpass' not in ucfg:
-        eventlet.sleep(0.05)  # stall even on test for existence of a username
-        return None
-    _passchecking[(user, tenant)] = True
-    # TODO(jbjohnso): WORKERPOOL
-    # PBKDF2 is, by design, cpu intensive
-    # throw it at the worker pool when implemented
-    # maybe a distinct worker pool, wondering about starving out non-auth stuff
-    salt, crypt = ucfg['cryptpass']
-    # execute inside tpool to get greenthreads to give it a special thread
-    # world
-    #TODO(jbjohnso): util function to generically offload a call
-    #such a beast could be passed into pyghmi as a way for pyghmi to
-    #magically get offload of the crypto functions without having
-    #to explicitly get into the eventlet tpool game
-    global authworkers
-    global authcleaner
-    if authworkers is None:
-        authworkers = multiprocessing.Pool(processes=1)
-    else:
-        authcleaner.cancel()
-    authcleaner = eventlet.spawn_after(30, _clean_authworkers)
-    crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
-    del _passchecking[(user, tenant)]
-    eventlet.sleep(0.05)  # either way, we want to stall so that client can't
+    if 'cryptpass' in ucfg:
+        _passchecking[(user, tenant)] = True
+        # TODO(jbjohnso): WORKERPOOL
+        # PBKDF2 is, by design, cpu intensive
+        # throw it at the worker pool when implemented
+        # maybe a distinct worker pool, wondering about starving out non-auth stuff
+        salt, crypt = ucfg['cryptpass']
+        # execute inside tpool to get greenthreads to give it a special thread
+        # world
+        # TODO(jbjohnso): util function to generically offload a call
+        # such a beast could be passed into pyghmi as a way for pyghmi to
+        # magically get offload of the crypto functions without having
+        # to explicitly get into the eventlet tpool game
+        global authworkers
+        global authcleaner
+        if authworkers is None:
+            authworkers = multiprocessing.Pool(processes=1)
+        else:
+            authcleaner.cancel()
+        authcleaner = eventlet.spawn_after(30, _clean_authworkers)
+        crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
+        del _passchecking[(user, tenant)]
+        eventlet.sleep(
+            0.05)  # either way, we want to stall so that client can't
        # determine failure because there is a delay, valid response will
        # delay as well
-    if crypt == crypted:
-        _passcache[(user, tenant)] = passphrase
-        return authorize(user, element, tenant)
+        if crypt == crypted:
+            _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+            return authorize(user, element, tenant, operation)
+    if pam:
+        pammy = pam.pam()
+        usergood = pammy.authenticate(user, passphrase)
+        del pammy
+        if usergood:
+            _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+            return authorize(user, element, tenant, operation, skipuserobj=False)    
+    eventlet.sleep(0.05)  # stall even on test for existence of a username
    return None

-
 def _apply_pbkdf(passphrase, salt):
    return KDF.PBKDF2(passphrase, salt, 32, 10000,
                      lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
@@ -231,4 +294,4 @@ def _do_pbkdf(passphrase, salt):
    # compute.  However, we do want to wait for result, so we have
    # one of the exceedingly rare sort of circumstances where 'apply'
    # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
+    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
@@ -18,13 +18,16 @@ import base64
 import confluent.collective.invites as invites
 import confluent.config.configmanager as cfm
 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.util as util
 import eventlet
 import eventlet.green.socket as socket
 import eventlet.green.ssl as ssl
 import eventlet.green.threading as threading
+import greenlet
 import random
+import sys
 try:
    import OpenSSL.crypto as crypto
 except ImportError:
@@ -33,43 +36,57 @@ except ImportError:
    crypto = None

 currentleader = None
-cfginitlock = None
 follower = None
 retrythread = None

 class ContextBool(object):
    def __init__(self):
        self.active = False
+        self.mylock = threading.RLock()

    def __enter__(self):
        self.active = True
+        self.mylock.__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.active = False
+        self.mylock.__exit__(exc_type, exc_val, exc_tb)

 connecting = ContextBool()
 leader_init = ContextBool()

-
 def connect_to_leader(cert=None, name=None, leader=None):
    global currentleader
-    global cfginitlock
    global follower
-    if cfginitlock is None:
-        cfginitlock = threading.RLock()
    if leader is None:
        leader = currentleader
+    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+             'subsystem': 'collective'})
    try:
        remote = connect_to_collective(cert, leader)
-    except socket.error:
+    except socket.error as e:
+        log.log({'error': 'Collective connection attempt to {0} failed: {1}'
+                          ''.format(leader, str(e)),
+                 'subsystem': 'collective'})
        return False
    with connecting:
-        with cfginitlock:
-            tlvdata.recv(remote)  # the banner
+        with cfm._initlock:
+            banner = tlvdata.recv(remote)  # the banner
+            vers = banner.split()[2]
+            pvers = 0
+            reqver = 4
+            if vers == b'v0':
+                pvers = 2
+            elif vers == b'v1':
+                pvers = 4
+            if sys.version_info[0] < 3:
+                pvers = 2
+                reqver = 2
            tlvdata.recv(remote)  # authpassed... 0..
            if name is None:
                name = get_myname()
            tlvdata.send(remote, {'collective': {'operation': 'connect',
+                                                 'protover': reqver,
                                                 'name': name,
                                                 'txcount': cfm._txcount}})
            keydata = tlvdata.recv(remote)
@@ -77,10 +94,16 @@ def connect_to_leader(cert=None, name=None, leader=None):
                return False
            if 'error' in keydata:
                if 'backoff' in keydata:
-                    eventlet.spawn_after(random.random(), connect_to_leader,
-                                         cert, name, leader)
-                    return True
+                    log.log({
+                        'info': 'Collective initialization in progress on '
+                                '{0}'.format(leader),
+                        'subsystem': 'collective'})
+                    return False
                if 'leader' in keydata:
+                    log.log(
+                        {'info': 'Prospective leader {0} has redirected this '
+                                 'member to {1}'.format(leader, keydata['leader']),
+                         'subsystem': 'collective'})
                    ldrc = cfm.get_collective_member_by_address(
                        keydata['leader'])
                    if ldrc and ldrc['name'] == name:
@@ -88,18 +111,27 @@ def connect_to_leader(cert=None, name=None, leader=None):
                    return connect_to_leader(name=name,
                                             leader=keydata['leader'])
                if 'txcount' in keydata:
+                    log.log({'info':
+                                 'Prospective leader {0} has inferior '
+                                 'transaction count, becoming leader'
+                                 ''.format(leader), 'subsystem': 'collective',
+                             'subsystem': 'collective'})
                    return become_leader(remote)
-                print(keydata['error'])
                return False
-            if follower is not None:
                follower.kill()
                cfm.stop_following()
                follower = None
+            if follower:
+                follower.kill()
+                cfm.stop_following()
+                follower = None
+            log.log({'info': 'Following leader {0}'.format(leader),
+                     'subsystem': 'collective'})
            colldata = tlvdata.recv(remote)
            globaldata = tlvdata.recv(remote)
            dbi = tlvdata.recv(remote)
            dbsize = dbi['dbsize']
-            dbjson = ''
+            dbjson = b''
            while (len(dbjson) < dbsize):
                ndata = remote.recv(dbsize - len(dbjson))
                if not ndata:
@@ -128,17 +160,28 @@ def connect_to_leader(cert=None, name=None, leader=None):
                raise
            currentleader = leader
        #spawn this as a thread...
-        follower = eventlet.spawn(follow_leader, remote)
+        follower = eventlet.spawn(follow_leader, remote, pvers)
    return True


-def follow_leader(remote):
+def follow_leader(remote, proto):
    global currentleader
-    cfm.follow_channel(remote)
-    # The leader has folded, time to startup again...
-    cfm.stop_following()
-    currentleader = None
-    eventlet.spawn_n(start_collective)
+    cleanexit = False
+    try:
+        cfm.follow_channel(remote, proto)
+    except greenlet.GreenletExit:
+        cleanexit = True
+    finally:
+        if cleanexit:
+            log.log({'info': 'Previous following cleanly closed',
+                     'subsystem': 'collective'})
+            return
+        log.log({'info': 'Current leader has disappeared, restarting '
+                         'collective membership', 'subsystem': 'collective'})
+        # The leader has folded, time to startup again...
+        cfm.stop_following()
+        currentleader = None
+        eventlet.spawn_n(start_collective)


 def connect_to_collective(cert, member):
@@ -177,8 +220,7 @@ def handle_connection(connection, cert, request, local=False):
    else:
        if not local:
            return
-
-        if 'show' == operation:
+        if operation in ('show', 'delete'):
            if not list(cfm.list_collective()):
                tlvdata.send(connection,
                             {'collective': {'error': 'Collective mode not '
@@ -215,7 +257,23 @@ def handle_connection(connection, cert, request, local=False):
                collinfo['quorum'] = True
            except exc.DegradedCollective:
                collinfo['quorum'] = False
-            tlvdata.send(connection, {'collective':  collinfo})
+            if operation == 'show':
+                tlvdata.send(connection, {'collective':  collinfo})
+            elif operation == 'delete':
+                todelete = request['member']
+                if (todelete == collinfo['leader'] or 
+                       todelete in collinfo['active']):
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is still active, stop the confluent service to remove it'.format(todelete)}})
+                    return
+                if todelete not in collinfo['offline']:
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is not a recognized collective member'.format(todelete)}})
+                    return
+                cfm.del_collective_member(todelete)
+                tlvdata.send(connection,
+                    {'collective': {'status': 'Successfully deleted {0}'.format(todelete)}})
+                connection.close()
            return
        if 'invite' == operation:
            try:
@@ -236,7 +294,8 @@ def handle_connection(connection, cert, request, local=False):
            invitation = request['invitation']
            try:
                invitation = base64.b64decode(invitation)
-                name, invitation = invitation.split('@', 1)
+                name, invitation = invitation.split(b'@', 1)
+                name = util.stringify(name)
            except Exception:
                tlvdata.send(
                    connection,
@@ -291,6 +350,8 @@ def handle_connection(connection, cert, request, local=False):
            f = open('/etc/confluent/cfg/myname', 'w')
            f.write(name)
            f.close()
+            log.log({'info': 'Connecting to collective due to join',
+                     'subsystem': 'collective'})
            eventlet.spawn_n(connect_to_leader, rsp['collective'][
                'fingerprint'], name)
    if 'enroll' == operation:
@@ -320,6 +381,11 @@ def handle_connection(connection, cert, request, local=False):
    if 'assimilate' == operation:
        drone = request['name']
        droneinfo = cfm.get_collective_member(drone)
+        if not droneinfo:
+            tlvdata.send(connection,
+                         {'error': 'Unrecognized leader, '
+                                   'redo invitation process'})
+            return
        if not util.cert_matches(droneinfo['fingerprint'], cert):
            tlvdata.send(connection,
                         {'error': 'Invalid certificate, '
@@ -329,8 +395,22 @@ def handle_connection(connection, cert, request, local=False):
            tlvdata.send(connection,
                         {'error': 'Refusing to be assimilated by inferior'
                                   'transaction count',
-                          'txcount': cfm._txcount})
+                          'txcount': cfm._txcount,})
            return
+        if connecting.active:
+            # don't try to connect while actively already trying to connect
+            tlvdata.send(connection, {'status': 0})
+            connection.close()
+            return
+        if (currentleader == connection.getpeername()[0] and
+                follower and not follower.dead):
+            # if we are happily following this leader already, don't stir
+            # the pot
+            tlvdata.send(connection, {'status': 0})
+            connection.close()
+            return
+        log.log({'info': 'Connecting in response to assimilation',
+                 'subsystem': 'collective'})
        eventlet.spawn_n(connect_to_leader, None, None,
                         leader=connection.getpeername()[0])
        tlvdata.send(connection, {'status': 0})
@@ -350,6 +430,7 @@ def handle_connection(connection, cert, request, local=False):
        tlvdata.send(connection, collinfo)
    if 'connect' == operation:
        drone = request['name']
+        folver = request.get('protover', 2)
        droneinfo = cfm.get_collective_member(drone)
        if not (droneinfo and util.cert_matches(droneinfo['fingerprint'],
                                                cert)):
@@ -359,6 +440,11 @@ def handle_connection(connection, cert, request, local=False):
            connection.close()
            return
        myself = connection.getsockname()[0]
+        if connecting.active:
+            tlvdata.send(connection, {'error': 'Connecting right now',
+                                      'backoff': True})
+            connection.close()
+            return
        if myself != get_leader(connection):
            tlvdata.send(
                connection,
@@ -366,17 +452,14 @@ def handle_connection(connection, cert, request, local=False):
                          'in another castle', 'leader': currentleader})
            connection.close()
            return
-        if connecting.active:
-            tlvdata.send(connection, {'error': 'Connecting right now',
-                                      'backoff': True})
-            connection.close()
-            return
        if request['txcount'] > cfm._txcount:
            retire_as_leader()
            tlvdata.send(connection,
                         {'error': 'Client has higher tranasaction count, '
                                   'should assimilate me, connecting..',
                          'txcount': cfm._txcount})
+            log.log({'info': 'Connecting to leader due to superior '
+                             'transaction count', 'subsystem': collective})
            eventlet.spawn_n(connect_to_leader, None, None,
                             connection.getpeername()[0])
            connection.close()
@@ -396,7 +479,7 @@ def handle_connection(connection, cert, request, local=False):
            connection.sendall(cfgdata)
        #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
        # so far unused anyway
-        if not cfm.relay_slaved_requests(drone, connection):
+        if not cfm.relay_slaved_requests(drone, connection, folver):
            if not retrythread:  # start a recovery if everyone else seems
                # to have disappeared
                retrythread = eventlet.spawn_after(30 + random.random(),
@@ -430,11 +513,36 @@ def try_assimilate(drone):
    tlvdata.recv(remote)  # the banner
    tlvdata.recv(remote)  # authpassed... 0..
    answer = tlvdata.recv(remote)
-    if answer and 'error' in answer:
+    if not answer:
+        log.log(
+            {'error':
+                 'No answer from {0} while trying to assimilate'.format(
+                     drone),
+            'subsystem': 'collective'})
+        return
+    if 'txcount' in answer:
+        log.log({'info': 'Deferring to {0} due to transaction count'.format(
+            drone), 'subsystem': 'collective'})
        connect_to_leader(None, None, leader=remote.getpeername()[0])
+        return
+    if 'error' in answer:
+        log.log({
+            'error': 'Error encountered while attempting to '
+                     'assimilate {0}: {1}'.format(drone, answer['error']),
+            'subsystem': 'collective'})
+        return
+    log.log({'info': 'Assimilated {0} into collective'.format(drone),
+             'subsystem': 'collective'})
+

 def get_leader(connection):
    if currentleader is None or connection.getpeername()[0] == currentleader:
+        if currentleader is None:
+            msg = 'Becoming leader as no leader known'
+        else:
+            msg = 'Becoming leader because {0} attempted to connect and it ' \
+                  'is current leader'.format(currentleader)
+        log.log({'info': msg, 'subsystem': 'collective'})
        become_leader(connection)
    return currentleader

@@ -446,27 +554,34 @@ def retire_as_leader():
 def become_leader(connection):
    global currentleader
    global follower
+    global retrythread
+    log.log({'info': 'Becoming leader of collective',
+             'subsystem': 'collective'})
    if follower:
        follower.kill()
+        cfm.stop_following()
        follower = None
+    if retrythread:
+        retrythread.cancel()
+        retrythread = None
    currentleader = connection.getsockname()[0]
    skipaddr = connection.getpeername()[0]
    myname = get_myname()
+    skipem = set(cfm.cfgstreams)
+    skipem.add(currentleader)
+    skipem.add(skipaddr)
    for member in cfm.list_collective():
        dronecandidate = cfm.get_collective_member(member)['address']
-        if dronecandidate in (currentleader, skipaddr) or member == myname:
+        if dronecandidate in skipem or member == myname:
            continue
        eventlet.spawn_n(try_assimilate, dronecandidate)


 def startup():
-    global cfginitlock
    members = list(cfm.list_collective())
    if len(members) < 2:
        # Not in collective mode, return
        return
-    if cfginitlock is None:
-        cfginitlock = threading.RLock()
    eventlet.spawn_n(start_collective)

 def start_collective():
@@ -474,7 +589,15 @@ def start_collective():
    global retrythread
    if follower:
        follower.kill()
+        cfm.stop_following()
        follower = None
+    try:
+        if cfm.cfgstreams:
+            cfm.check_quorum()
+            # Do not start if we have quorum and are leader
+            return
+    except exc.DegradedCollective:
+        pass
    if leader_init.active:  # do not start trying to connect if we are
        # xmitting data to a follower
        return
@@ -485,6 +608,8 @@ def start_collective():
        if cfm.cfgleader is None:
            cfm.stop_following(True)
        ldrcandidate = cfm.get_collective_member(member)['address']
+        log.log({'info': 'Performing startup attempt to {0}'.format(
+            ldrcandidate), 'subsystem': 'collective'})
        if connect_to_leader(name=myname, leader=ldrcandidate):
            break
    else:
@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -93,9 +93,19 @@ node = {
        'description': ('List of static groups for which this node is '
                        'considered a member'),
    },
-    #'type': {
-    #    'description': ('Classification of node as system, vm, etc')
-    #},
+    'type': {
+        'description': ('Classification of node as server or switch'),
+        'validvalues': ('switch', 'server'),
+    },
+    'api.key': {
+        'description': ('Crypt of api key for self api requests by node'),
+    },
+    'api.armed': {
+        'description': ('Indicates whether an insecure api key request is allowed. '
+                        'The format is an expiration time in ISO8601 format. When '
+                        'the indicated time passes or the first time a node claims '
+                        'the key, key grants will not be allowed.'),
+    },
    #'id': {
    #    'description': ('Numeric identifier for node')
    #},
@@ -156,6 +166,14 @@ node = {
                        'indicates candidate managers, either for '
                        'high availability or load balancing purposes.')
    },
+    'discovery.passwordrules': {
+        'description':  'Any specified rules shall be configured on the BMC '
+                        'upon discovery.  "expiration=no,loginfailures=no,complexity=no,reuse=no" '
+                        'would disable password expiration, login failures '
+                        'triggering a lockout, password complexity requirements,'
+                        'and any restrictions around reusing an old password.',
+        'validlistkeys': ('expiration', 'loginfailures', 'complexity', 'reuse'),
+    },
    'discovery.policy': {
        'description':  'Policy to use for auto-configuration of discovered '
                        'and identified nodes. Valid values are "manual", '
@@ -165,6 +183,7 @@ node = {
                        'so long as the node has no existing public key. '
                        '"open" allows discovery even if a known public key '
                        'is already stored',
+        'validlist': ('manual', 'permissive', 'pxe', 'open'),
    },
    'info.note': {
        'description':  'A field used for administrators to make arbitrary '
@@ -173,6 +192,9 @@ node = {
                        'freeform text data without concern for issues in how '
                        'the server will process it.',
    },
+    'location.height': {
+        'description': 'Height in RU of the system (defaults to query the systems)',
+    },
    'location.room': {
        'description': 'Room description for the node',
    },
@@ -252,11 +274,13 @@ node = {
    'console.logging': {
        'description': ('Indicate logging level to apply to console.  Valid '
                        'values are currently "full", "interactive", and '
-                        '"none". Defaults to "full".')
+                        '"none". Defaults to "full".'),
+        'validvalues': ('full', 'interactive', 'none'),
    },
    'console.method': {
        'description': ('Indicate the method used to access the console of '
-                        'the managed node.')
+                        'the managed node.'),
+        'validvalues': ('ssh', 'ipmi'),
    },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),
@@ -16,7 +16,10 @@


 # This defines config variable to store the global configuration for confluent
-import ConfigParser
+try:
+    import ConfigParser
+except ModuleNotFoundError:
+    import configparser as ConfigParser
 import os

 _config = None
@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2017 Lenovo
+# Copyright 2017-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ def _utf8_normalize(data, shiftin, decoder):


 def pytechars2line(chars, maxlen=None):
-    line = '\x1b[m'  # start at default params
+    line = b'\x1b[m'  # start at default params
    lb = False  # last bold
    li = False  # last italic
    lu = False  # last underline
@@ -106,7 +106,7 @@ def pytechars2line(chars, maxlen=None):
    len = 1
    for charidx in range(maxlen):
        char = chars[charidx]
-        csi = []
+        csi = bytearray([])
        if char.fg != lfg:
            csi.append(30 + pytecolors2ansi[char.fg])
            lfg = char.fg
@@ -129,10 +129,13 @@ def pytechars2line(chars, maxlen=None):
            lr = char.reverse
            csi.append(7 if lr else 27)
        if csi:
-            line += b'\x1b[' + b';'.join(['{0}'.format(x) for x in csi]) + b'm'
-        if not hasdata and char.data.encode('utf-8').rstrip():
+            line += b'\x1b[' + b';'.join(['{0}'.format(x).encode('utf-8') for x in csi]) + b'm'
+        if not hasdata and char.data.rstrip():
            hasdata = True
-        line += char.data.encode('utf-8')
+        chardata = char.data
+        if not isinstance(chardata, bytes):
+            chardata = chardata.encode('utf-8')
+        line += chardata
        if maxlen and len >= maxlen:
            break
        len += 1
@@ -145,8 +148,10 @@ class ConsoleHandler(object):
    _genwatchattribs = frozenset(('console.method', 'console.logging',
                                  'collective.manager'))

-    def __init__(self, node, configmanager):
+    def __init__(self, node, configmanager, width=80, height=24):
        self.clearpending = False
+        self.clearerror = False
+        self.initsize = (width, height)
        self._dologging = True
        self._is_local = True
        self._isondemand = False
@@ -183,7 +188,7 @@ class ConsoleHandler(object):
        if termstate & 1:
            self.appmodedetected = True
        if termstate & 2:
-            self.shiftin = '0'
+            self.shiftin = b'0'
        self.users = {}
        self._attribwatcher = None
        self._console = None
@@ -208,6 +213,8 @@ class ConsoleHandler(object):
        return retrytime + (retrytime * random.random())

    def feedbuffer(self, data):
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
        try:
            self.termstream.feed(data)
        except StopIteration:  # corrupt parser state, start over
@@ -307,8 +314,11 @@ class ConsoleHandler(object):

    def clearbuffer(self):
        self.feedbuffer(
-            '\x1bc[no replay buffer due to console.logging attribute set to '
-            'none or interactive,\r\nconnection loss, or service restart]')
+            '\x1bc[No data has been received from the remote console since ' \
+            'connecting.  This could\r\nbe due to having the console.logging ' \
+            'attribute set to none or interactive,\r\nserial console not '  \
+            'being enabled or incorrectly configured in the OS or\r\nfirmware, ' \
+            'or the console simply not having any output since last connection]')
        self.clearpending = True

    def _detach(self):
@@ -359,16 +369,33 @@ class ConsoleHandler(object):
        except (exc.NotImplementedException, exc.NotFoundException):
            self._console = None
        except:
-            _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
-                          event=log.Events.stacktrace)
+            if _tracelog:
+                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                              event=log.Events.stacktrace)
+            else:
+                print(traceback.format_exc())
        if not isinstance(self._console, conapi.Console):
            self.clearbuffer()
            self.connectstate = 'unconnected'
            self.error = 'misconfigured'
            self._send_rcpts({'connectstate': self.connectstate,
                              'error': self.error})
+            self.feedbuffer(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self._send_rcpts(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self.clearerror = True
            return
+        if self.clearerror:
+            self.clearerror = False
+            self.clearbuffer()
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
        self.send_break = self._console.send_break
+        self.resize = self._console.resize
        if self._attribwatcher:
            self.cfgmgr.remove_watcher(self._attribwatcher)
            self._attribwatcher = None
@@ -380,6 +407,7 @@ class ConsoleHandler(object):
            self._attribwatcher = self.cfgmgr.watch_attributes(
                (self.node,), attribstowatch, self._attribschanged)
        try:
+            self.resize(width=self.initsize[0], height=self.initsize[1])
            self._console.connect(self.get_console_output)
        except exc.TargetEndpointBadCredentials:
            self.clearbuffer()
@@ -425,6 +453,7 @@ class ConsoleHandler(object):

    def _got_disconnected(self):
        if self.connectstate != 'unconnected':
+            self._console.close()
            self.connectstate = 'unconnected'
            self.log(
                logdata='console disconnected', ltype=log.DataTypes.event,
@@ -502,17 +531,19 @@ class ConsoleHandler(object):
            if data == conapi.ConsoleEvent.Disconnect:
                self._got_disconnected()
            return
-        elif data == '':
+        elif data in (b'', u''):
            # ignore empty strings from a cconsole provider
            return
-        if '\x1b[?1l' in data:  # request for ansi mode cursor keys
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+        if b'\x1b[?1l' in data:  # request for ansi mode cursor keys
            self.appmodedetected = False
-        if '\x1b[?1h' in data:  # remember the session wants the client to use
+        if b'\x1b[?1h' in data:  # remember the session wants the client to use
            # 'application mode'  Thus far only observed on esxi
            self.appmodedetected = True
-        if '\x1b)0' in data:
+        if b'\x1b)0' in data:
            # console indicates it wants access to special drawing characters
-            self.shiftin = '0'
+            self.shiftin = b'0'
        eventdata = 0
        if self.appmodedetected:
            eventdata |= 1
@@ -521,10 +552,11 @@ class ConsoleHandler(object):
        # TODO: analyze buffer for registered events, examples:
        #   panics
        #   certificate signing request
-        if self.clearpending:
+        if self.clearpending or self.clearerror:
            self.clearpending = False
-            self.feedbuffer(b'\x1bc')
-            self._send_rcpts(b'\x1bc')
+            self.clearerror = False
+            self.feedbuffer(b'\x1bc\x1b[2J\x1b[1;1H')
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
        self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
        self.log(data, eventdata=eventdata)
        self.lasttime = util.monotonic_time()
@@ -564,25 +596,30 @@ class ConsoleHandler(object):
                if pendingbl:
                    retdata += pendingbl
                    pendingbl = b''
-                retdata += nline + '\r\n'
+                retdata += nline + b'\r\n'
            else:
-                pendingbl += nline + '\r\n'
+                pendingbl += nline + b'\r\n'
        if len(retdata) >  6:
            retdata = retdata[:-2]  # remove the last \r\n
-        retdata += b'\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
-                                           self.buffer.cursor.x + 1)
+        cursordata = '\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
+                                            self.buffer.cursor.x + 1)
+        if not isinstance(cursordata, bytes):
+            cursordata = cursordata.encode('utf-8')
+        retdata += cursordata
        if self.shiftin is not None:  # detected that terminal requested a
            # shiftin character set, relay that to the terminal that cannected
-            retdata += '\x1b)' + self.shiftin
+            retdata += b'\x1b)' + self.shiftin
        if self.appmodedetected:
-            retdata += '\x1b[?1h'
+            retdata += b'\x1b[?1h'
        else:
-            retdata += '\x1b[?1l'
+            retdata += b'\x1b[?1l'
        return retdata, connstate

    def write(self, data):
        if self.connectstate == 'connected':
            try:
+                if isinstance(data, str) and not isinstance(data, bytes):
+                    data = data.encode('utf-8')
                self._console.write(data)
            except Exception:
                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
@@ -597,11 +634,14 @@ def disconnect_node(node, configmanager):
        del _handled_consoles[consk]


-def _nodechange(added, deleting, configmanager):
-    for node in added:
-        connect_node(node, configmanager)
+def _nodechange(added, deleting, renamed, configmanager):
    for node in deleting:
        disconnect_node(node, configmanager)
+    for node in renamed:
+        disconnect_node(node, configmanager)
+        connect_node(renamed[node], configmanager)
+    for node in added:
+        connect_node(node, configmanager)


 def _start_tenant_sessions(cfm):
@@ -625,17 +665,20 @@ def start_console_sessions():
    configmodule.hook_new_configmanagers(_start_tenant_sessions)


-def connect_node(node, configmanager, username=None):
+def connect_node(node, configmanager, username=None, direct=True, width=80,
+                 height=24):
    attrval = configmanager.get_node_attributes(node, 'collective.manager')
    myc = attrval.get(node, {}).get('collective.manager', {}).get(
        'value', None)
    myname = collective.get_myname()
-    if myc and myc != collective.get_myname():
+    if myc and myc != collective.get_myname() and direct:
        minfo = configmodule.get_collective_member(myc)
-        return ProxyConsole(node, minfo, myname, configmanager, username)
+        return ProxyConsole(node, minfo, myname, configmanager, username,
+                            width, height)
    consk = (node, configmanager.tenant)
    if consk not in _handled_consoles:
-        _handled_consoles[consk] = ConsoleHandler(node, configmanager)
+        _handled_consoles[consk] = ConsoleHandler(node, configmanager, width,
+                                                  height)
    return _handled_consoles[consk]

 # A stub console handler that just passes through to a remote confluent
@@ -644,8 +687,10 @@ def connect_node(node, configmanager, username=None):
 class ProxyConsole(object):
    _genwatchattribs = frozenset(('collective.manager',))

-    def __init__(self, node, managerinfo, myname, configmanager, user):
+    def __init__(self, node, managerinfo, myname, configmanager, user, width,
+                 height):
        self.skipreplay = True
+        self.initsize = (width, height)
        self.managerinfo = managerinfo
        self.myname = myname
        self.cfm = configmanager
@@ -656,6 +701,7 @@ class ProxyConsole(object):
        self._attribwatcher = configmanager.watch_attributes(
            (self.node,), self._genwatchattribs, self._attribschanged)

+
    def _attribschanged(self, nodeattribs, configmanager, **kwargs):
        if self.clisession:
            self.clisession.detach()
@@ -697,6 +743,8 @@ class ProxyConsole(object):
                'tenant': self.cfm.tenant,
                'node': self.node,
                'skipreplay': self.skipreplay,
+                'width': self.initsize[0],
+                'height': self.initsize[1],
                #TODO(jjohnson2): declare myself as a proxy,
                #facilitate redirect rather than relay on manager change
            },
@@ -736,6 +784,10 @@ class ProxyConsole(object):
    def reopen(self):
        tlvdata.send(self.remote, {'operation': 'reopen'})

+    def resize(self, width, height):
+        tlvdata.send(self.remote, {'operation': 'resize', 'width': width,
+                                   'height': height})
+

 # this represents some api view of a console handler.  This handles things like
 # holding the caller specific queue data, for example, when http api should be
@@ -758,10 +810,9 @@ class ConsoleSession(object):
                         'get_next_output' non-functional
    :param skipreplay: If true, will skip the attempt to redraw the screen
    """
-    connector = connect_node

    def __init__(self, node, configmanager, username, datacallback=None,
-                 skipreplay=False):
+                 skipreplay=False, direct=True, width=80, height=24):
        self.registered = False
        self.tenant = configmanager.tenant
        if not configmanager.is_node(node):
@@ -769,6 +820,10 @@ class ConsoleSession(object):
        self.username = username
        self.node = node
        self.configmanager = configmanager
+        self.direct = direct  # true if client is directly connected versus
+                              # relay
+        self.width = width
+        self.height = height
        self.connect_session()
        self.registered = True
        self._evt = None
@@ -797,12 +852,16 @@ class ConsoleSession(object):
        between console and shell.
        """
        self.conshdl = connect_node(self.node, self.configmanager,
-                                    self.username)
+                                    self.username, self.direct, self.width,
+                                    self.height)
    def send_break(self):
        """Send break to remote system
        """
        self.conshdl.send_break()

+    def resize(self, width, height):
+        self.conshdl.resize(width, height)
+
    def get_buffer_age(self):
        """Get the age in seconds of the buffered data

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@

 import confluent
 import confluent.alerts as alerts
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.config.attributes as attrscheme
 import confluent.config.configmanager as cfm
@@ -62,8 +63,10 @@ import itertools
 import os
 try:
    import cPickle as pickle
+    pargs = {}
 except ImportError:
    import pickle
+    pargs = {'encoding': 'utf-8'}
 import socket
 import struct
 import sys
@@ -85,7 +88,10 @@ def seek_element(currplace, currkey):

 def nested_lookup(nestdict, key):
    try:
-        return reduce(seek_element, key, nestdict)
+        currloc = nestdict
+        for currk in key:
+            currloc = seek_element(currloc, currk)
+        return currloc
    except TypeError:
        raise exc.NotFoundException("Invalid element requested")

@@ -105,6 +111,8 @@ def load_plugins():
        for plugin in os.listdir(plugindir):
            if plugin.startswith('.'):
                continue
+            if '__pycache__' in plugin:
+                continue
            (plugin, plugtype) = os.path.splitext(plugin)
            if plugtype == '.sh':
                pluginmap[plugin] = shellmodule.Plugin(
@@ -123,7 +131,7 @@ def load_plugins():


 rootcollections = ['discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'version']
+                   'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' , 'users/', 'version']


 class PluginRoute(object):
@@ -143,6 +151,7 @@ def _init_core():
    # be enumerated in any collection
    noderesources = {
        'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
            'all': PluginRoute({'handler': 'attributes'}),
            'current': PluginRoute({'handler': 'attributes'}),
            'expression': PluginRoute({'handler': 'attributes'}),
@@ -161,10 +170,22 @@ def _init_core():
                        'default': 'ipmi',
                    }),
                },
+                'clear': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                }),
                'users': PluginCollection({
                    'pluginattrs': ['hardwaremanagement.method'],
                    'default': 'ipmi',
                }),
+                'licenses': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'save_licenses': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                'net_interfaces': PluginCollection({
                    'pluginattrs': ['hardwaremanagement.method'],
                    'default': 'ipmi',
@@ -195,12 +216,48 @@ def _init_core():
                        'default': 'ipmi',
                    }),
                },
+                'extended': {
+                    'all': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'advanced': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),                    
+                },
+            },
+            'storage': {
+                'all': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'arrays': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'disks': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'volumes': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                })
            },
            'system': {
                'all': PluginRoute({
                    'pluginattrs': ['hardwaremanagement.method'],
                    'default': 'ipmi',
                }),
+                'advanced': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'clear': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                })
            },
        },
        '_console': {
@@ -233,6 +290,10 @@ def _init_core():
                'default': 'ipmi',
            }),
        },
+        'description': PluginRoute({
+            'pluginattrs': ['hardwaremanagement.method'],
+            'default': 'ipmi',
+        }),
        'events': {
            'hardware': {
                'log': PluginRoute({
@@ -334,10 +395,18 @@ def _init_core():
            },

        },
+        'support': {
+            'servicedata': PluginCollection({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+        },
    }

    nodegroupresources = {
        'attributes': {
+            'check': PluginRoute({'handler': 'attributes'}),
+            'rename': PluginRoute({'handler': 'attributes'}),
            'all': PluginRoute({'handler': 'attributes'}),
            'current': PluginRoute({'handler': 'attributes'}),
        },
@@ -353,17 +422,37 @@ def create_user(inputdata, configmanager):
    configmanager.create_user(username, attributemap=inputdata)


+def create_usergroup(inputdata, configmanager):
+    try:
+        groupname = inputdata['name']
+        del inputdata['name']
+    except (KeyError, ValueError):
+        raise exc.InvalidArgumentException()
+    configmanager.create_usergroup(groupname)
+
+
+def update_usergroup(groupname, attribmap, configmanager):
+    try:
+        configmanager.set_usergroup(groupname, attribmap)
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
+
 def update_user(name, attribmap, configmanager):
    try:
        configmanager.set_user(name, attribmap)
-    except ValueError:
-        raise exc.InvalidArgumentException()
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))


+def show_usergroup(groupname, configmanager):
+    groupinfo = configmanager.get_usergroup(groupname)
+    for attr in groupinfo:
+        yield msg.Attributes(kv={attr: groupinfo[attr]})
+
 def show_user(name, configmanager):
    userobj = configmanager.get_user(name)
    rv = {}
-    for attr in attrscheme.user.iterkeys():
+    for attr in attrscheme.user:
        rv[attr] = None
        if attr == 'password':
            if 'cryptpass' in userobj:
@@ -376,6 +465,10 @@ def show_user(name, configmanager):
                rv[attr] = userobj[attr]
            yield msg.Attributes(kv={attr: rv[attr]},
                                 desc=attrscheme.user[attr]['description'])
+    if 'role' in userobj:
+        yield msg.Attributes(kv={'role': userobj['role']})
+
+


 def stripnode(iterablersp, node):
@@ -408,6 +501,10 @@ def delete_user(user, configmanager):
    configmanager.del_user(user)
    yield msg.DeletedResource(user)

+def delete_usergroup(usergroup, configmanager):
+    configmanager.del_usergroup(usergroup)
+    yield msg.DeletedResource(usergroup)
+

 def delete_nodegroup_collection(collectionpath, configmanager):
    if len(collectionpath) == 2:  # just the nodegroup
@@ -567,6 +664,15 @@ class BadPlugin(object):
            self.node, self.plugin + ' is not a supported plugin')


+class BadCollective(object):
+    def __init__(self, node):
+        self.node = node
+
+    def error(self, *args, **kwargs):
+        yield msg.ConfluentNodeError(
+            self.node, 'collective mode is active, but collective.manager '
+                       'is not set for this node')
+
 def abbreviate_noderange(configmanager, inputdata, operation):
    if operation != 'create':
        raise exc.InvalidArgumentException('Must be a create with nodes in list')
@@ -583,7 +689,10 @@ def handle_dispatch(connection, cert, dispatch, peername):
            cfm.get_collective_member(peername)['fingerprint'], cert):
        connection.close()
        return
-    dispatch = pickle.loads(dispatch)
+    pversion = 0
+    if bytearray(dispatch)[0] == 0x80:
+        pversion = bytearray(dispatch)[1]
+    dispatch = pickle.loads(dispatch, **pargs)
    configmanager = cfm.ConfigManager(dispatch['tenant'])
    nodes = dispatch['nodes']
    inputdata = dispatch['inputdata']
@@ -602,7 +711,7 @@ def handle_dispatch(connection, cert, dispatch, peername):
                plugpath = nodeattr[node][attrname]['value']
            elif 'default' in plugroute:
                plugpath = plugroute['default']
-        if plugpath is not None:
+        if plugpath:
            try:
                hfunc = getattr(pluginmap[plugpath], operation)
            except KeyError:
@@ -619,14 +728,18 @@ def handle_dispatch(connection, cert, dispatch, peername):
                configmanager=configmanager,
                inputdata=inputdata))
        for res in itertools.chain(*passvalues):
-            _forward_rsp(connection, res)
+            _forward_rsp(connection, res, pversion)
    except Exception as res:
-        _forward_rsp(connection, res)
+        _forward_rsp(connection, res, pversion)
    connection.sendall('\x00\x00\x00\x00\x00\x00\x00\x00')


-def _forward_rsp(connection, res):
-    r = pickle.dumps(res)
+def _forward_rsp(connection, res, pversion):
+    try:
+        r = pickle.dumps(res, protocol=pversion)
+    except TypeError:
+        r = pickle.dumps(Exception(
+            'Cannot serialize error, check collective.manager error logs for details' + str(res)), protocol=pversion)
    rlen = len(r)
    if not rlen:
        return
@@ -636,6 +749,8 @@ def _forward_rsp(connection, res):

 def handle_node_request(configmanager, inputdata, operation,
                        pathcomponents, autostrip=True):
+    if log.logfull:
+        raise exc.TargetResourceUnavailable('Filesystem full, free up space and restart confluent service')
    iscollection = False
    routespec = None
    if pathcomponents[0] == 'noderange':
@@ -759,7 +874,8 @@ def handle_node_request(configmanager, inputdata, operation,
                        continue
                elif list(cfm.list_collective()):
                    badcollnodes.append(node)
-            if plugpath is not None:
+                    continue
+            if plugpath:
                try:
                    hfunc = getattr(pluginmap[plugpath], operation)
                except KeyError:
@@ -769,11 +885,8 @@ def handle_node_request(configmanager, inputdata, operation,
                    nodesbyhandler[hfunc].append(node)
                else:
                    nodesbyhandler[hfunc] = [node]
-        if badcollnodes:
-            raise exc.ConfluentException(
-                'collective management active, '
-                'collective.manager must be set for {0}'.format(
-                    ','.join(badcollnodes)))
+        for bn in badcollnodes:
+            nodesbyhandler[BadCollective(bn).error] = [bn]
        workers = greenpool.GreenPool()
        numworkers = 0
        for hfunc in nodesbyhandler:
@@ -809,6 +922,8 @@ def iterate_queue(numworkers, passvalues, strip=False):
        if nv == 'theend':
            completions += 1
        else:
+            if isinstance(nv, Exception):
+                raise nv
            if strip and not isinstance(nv, console.Console):
                nv.strip_node(strip)
            yield nv
@@ -822,6 +937,8 @@ def addtoqueue(theq, fun, kwargs):
        else:
            for pv in result:
                theq.put(pv)
+    except Exception as e:
+        theq.put(e)
    finally:
        theq.put('theend')

@@ -831,24 +948,40 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
    a = configmanager.get_collective_member(manager)
    try:
        remote = socket.create_connection((a['address'], 13001))
-        remote.settimeout(90)
+        remote.settimeout(180)
        remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                 keyfile='/etc/confluent/privkey.pem',
                                 certfile='/etc/confluent/srvcert.pem')
    except Exception:
        for node in nodes:
-            yield msg.ConfluentResourceUnavailable(
-                node, 'Collective member {0} is unreachable'.format(a['name']))
+            if a:
+                yield msg.ConfluentResourceUnavailable(
+                    node, 'Collective member {0} is unreachable'.format(
+                        a['name']))
+            else:
+                yield msg.ConfluentResourceUnavailable(
+                    node,
+                    '"{0}" is not recognized as a collective member'.format(
+                        manager))
+
        return
    if not util.cert_matches(a['fingerprint'], remote.getpeercert(
            binary_form=True)):
        raise Exception("Invalid certificate on peer")
-    tlvdata.recv(remote)
+    banner = tlvdata.recv(remote)
+    vers = banner.split()[2]
+    if vers == b'v0':
+        pvers = 2
+    elif vers == b'v1':
+        pvers = 4
+    if sys.version_info[0] < 3:
+        pvers = 2
    tlvdata.recv(remote)
    myname = collective.get_myname()
    dreq = pickle.dumps({'name': myname, 'nodes': list(nodes),
                         'path': element,'tenant': configmanager.tenant,
-                         'operation': operation, 'inputdata': inputdata})
+                         'operation': operation, 'inputdata': inputdata},
+                         protocol=pvers)
    tlvdata.send(remote, {'dispatch': {'name': myname, 'length': len(dreq)}})
    remote.sendall(dreq)
    while True:
@@ -895,7 +1028,10 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
                            a['name']))
                return
            rsp += nrsp
-        rsp = pickle.loads(rsp)
+        try:
+            rsp = pickle.loads(rsp, **pargs)
+        except UnicodeDecodeError:
+            rsp = pickle.loads(rsp, encoding='latin1')
        if isinstance(rsp, Exception):
            raise rsp
        yield rsp
@@ -941,6 +1077,31 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
            configmanager, inputdata, operation, pathcomponents)
    elif pathcomponents[0] == 'version':
        return (msg.Attributes(kv={'version': confluent.__version__}),)
+    elif pathcomponents[0] == 'usergroups':
+        # TODO: when non-administrator accounts exist,
+        # they must only be allowed to see their own user
+        try:
+            usergroup = pathcomponents[1]
+        except IndexError:  # it's just users/
+            if operation == 'create':
+                inputdata = msg.get_input_message(
+                    pathcomponents, operation, inputdata,
+                    configmanager=configmanager)
+                create_usergroup(inputdata.attribs, configmanager)
+            return iterate_collections(configmanager.list_usergroups(),
+                                       forcecollection=False)
+        if usergroup not in configmanager.list_usergroups():
+            raise exc.NotFoundException("Invalid usergroup %s" % usergroup)
+        if operation == 'retrieve':
+            return show_usergroup(usergroup, configmanager)
+        elif operation == 'delete':
+            return delete_usergroup(usergroup, configmanager)
+        elif operation == 'update':
+            inputdata = msg.get_input_message(
+                pathcomponents, operation, inputdata,
+                configmanager=configmanager)
+            update_usergroup(usergroup, inputdata.attribs, configmanager)
+            return show_usergroup(usergroup, configmanager)
    elif pathcomponents[0] == 'users':
        # TODO: when non-administrator accounts exist,
        # they must only be allowed to see their own user
@@ -0,0 +1,87 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.config.configmanager as cfm
+import datetime
+import eventlet
+import eventlet.green.socket as socket
+import eventlet.greenpool
+import os
+
+class CredServer(object):
+    def __init__(self, bindhost='::', bindport=301, ttl=1):
+        self.srv = socket.socket(socket.AF_INET6)
+        self.srv.setsockopt(socket.SOL_IP, socket.IP_TTL, ttl)
+        self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.srv.bind((bindhost, bindport))
+        self.srv.listen(32)
+        self.gpool = eventlet.greenpool.GreenPool(256)
+        self.cfm = cfm.ConfigManager(None)
+        self.runtime = eventlet.spawn(self.listen)
+
+    def listen(self):
+        while True:
+            client, info = self.srv.accept()
+            if info[1] > 1023:
+                client.close()
+                continue
+            self.gpool.spawn_n(self.handle_client, client)
+    
+    def handle_client(self, client):
+            client.send('\xc2\xd1-\xa8\x80\xd8j\xba')
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 1:
+                client.close()
+                return
+            nodename = client.recv(tlv[1])
+            tlv = bytearray(client.recv(2))
+            apiarmed = self.cfm.get_node_attributes(nodename, 'api.armed')
+            apiarmed = apiarmed.get(nodename, {}).get('api.armed', {}).get('value', None)
+            if not apiarmed:
+                client.close()
+                return
+            now = datetime.datetime.utcnow()
+            expiry = datetime.datetime.strptime(apiarmed, "%Y-%m-%dT%H:%M:%SZ")
+            if now > expiry:
+                self.cfm.set_node_attributes({nodename: {'api.armed': ''}})
+                client.close()
+                return
+            client.send(b'\x02\x20')
+            rttoken = os.urandom(32)
+            client.send(rttoken)
+            client.send('\x00\x00')
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 3:
+                client.close()
+                return
+            echotoken = client.recv(tlv[1])
+            if echotoken != rttoken:
+                client.close()
+                return
+            tlv = bytearray(client.recv(2))
+            if tlv[0] != 4:
+                client.close()
+                return
+            echotoken = client.recv(tlv[1])
+            self.cfm.set_node_attributes({nodename: {'api.key': echotoken, 'api.armed': ''}})
+            client.recv(2)  # drain end of message
+            client.send('\x05\x00') # report success     
+            client.close()
+
+if __name__ == '__main__':
+    a = CredServer()
+    while True:
+        eventlet.sleep(86400)
@@ -63,10 +63,13 @@

 import base64
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
-#import confluent.discovery.protocols.ssdp as ssdp
+import confluent.discovery.protocols.ssdp as ssdp
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
+import confluent.discovery.handlers.cpstorage as cpstorage
+import confluent.discovery.handlers.tsm as tsm
 import confluent.discovery.handlers.pxe as pxeh
 import confluent.discovery.handlers.smm as smm
 import confluent.discovery.handlers.xcc as xcc
@@ -86,6 +89,13 @@ import eventlet.greenpool
 import eventlet.semaphore

 autosensors = set()
+scanner = None
+
+try:
+    unicode
+except NameError:
+    unicode = str
+
 class nesteddict(dict):

    def __missing__(self, key):
@@ -98,6 +108,8 @@ nodehandlers = {
    'service:management-hardware.IBM:integrated-management-module2': imm,
    'pxe-client': pxeh,
    'service:io-device.Lenovo:management-module': None,
+    'service:thinkagile-storage': cpstorage,
+    'service:lenovo-tsm': tsm,
 }

 servicenames = {
@@ -106,6 +118,8 @@ servicenames = {
    'service:management-hardware.Lenovo:lenovo-xclarity-controller': 'lenovo-xcc',
    'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
    'service:io-device.Lenovo:management-module': 'lenovo-switch',
+    'service:thinkagile-storage': 'thinkagile-storagebmc',
+    'service:lenovo-tsm': 'lenovo-tsm',
 }

 servicebyname = {
@@ -114,6 +128,8 @@ servicebyname = {
    'lenovo-xcc': 'service:management-hardware.Lenovo:lenovo-xclarity-controller',
    'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2',
    'lenovo-switch': 'service:io-device.Lenovo:management-module',
+    'thinkagile-storage': 'service:thinkagile-storagebmc',
+    'lenovo-tsm': 'service:lenovo-tsm',
 }

 discopool = eventlet.greenpool.GreenPool(500)
@@ -209,6 +225,8 @@ def send_discovery_datum(info):
        if infotype in servicenames:
            types.append(servicenames[infotype])
    yield msg.KeyValueData({'types': types})
+    if 'otheraddresses' in info:
+        yield msg.KeyValueData({'otheripaddrs': list(info['otheraddresses'])})


 def _info_matches(info, criteria):
@@ -358,7 +376,7 @@ def handle_autosense_config(operation, inputdata):
        yield msg.KeyValueData({'enabled': autosense})
    elif operation == 'update':
        enabled = inputdata['enabled']
-        if type(enabled) in (unicode, str):
+        if type(enabled) in (unicode, bytes):
            enabled = enabled.lower() in ('true', '1', 'y', 'yes', 'enable',
                                          'enabled')
        if autosense == enabled:
@@ -381,6 +399,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
            raise exc.InvalidArgumentException()
        rescan()
        return (msg.KeyValueData({'rescan': 'started'}),)
+
    elif operation in ('update', 'create'):
        if 'node' not in inputdata:
            raise exc.InvalidArgumentException('Missing node name in input')
@@ -391,8 +410,15 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
                'Unable to {0} to {1}'.format(operation,
                                              '/'.join(pathcomponents)))
        handler = info['handler'].NodeHandler(info, configmanager)
-        eval_node(configmanager, handler, info, inputdata['node'],
-                  manual=True)
+        try:
+            eval_node(configmanager, handler, info, inputdata['node'],
+                      manual=True)
+        except Exception as e:
+            # or... incorrect passworod provided..
+            if 'Incorrect password' in str(e) or 'Unauthorized name' in str(e):
+                return [msg.ConfluentTargetInvalidCredentials(
+                    inputdata['node'])]
+            raise
        return [msg.AssignedResource(inputdata['node'])]
    elif operation == 'delete':
        mac = _get_mac_from_query(pathcomponents)
@@ -416,6 +442,8 @@ def handle_read_api_request(pathcomponents):
    # TODO(jjohnson2): This should be more generalized...
    #  odd indexes into components are 'by-'*, even indexes
    # starting at 2 are parameters to previous index
+    if pathcomponents == ['discovery', 'rescan']:
+        return (msg.KeyValueData({'scanning': bool(scanner)}),)
    subcats, queryparms, indexof, coll = _parameterize_path(pathcomponents[1:])
    if len(pathcomponents) == 1:
        dirlist = [msg.ChildCollection(x + '/') for x in sorted(list(subcats))]
@@ -458,6 +486,9 @@ def _recheck_nodes_backend(nodeattribs, configmanager):
    _map_unique_ids(nodeattribs)
    # for the nodes whose attributes have changed, consider them as potential
    # strangers
+    if nodeattribs:
+        macmap.vintage = 0  # expire current mac map data, in case
+        # the attributes changed impacted the result
    for node in nodeattribs:
        if node in known_nodes:
            for somemac in known_nodes[node]:
@@ -571,6 +602,11 @@ def detected(info):
            break
    else:  # no nodehandler, ignore for now
        return
+    if (handler and not handler.NodeHandler.adequate(info) and
+            info.get('protocol', None)):
+        eventlet.spawn_after(10, info['protocol'].fix_info, info,
+                             safe_detected)
+        return
    try:
        snum = info['attributes']['enclosure-serial-number'][0].strip()
        if snum:
@@ -617,6 +653,9 @@ def detected(info):
    uuid = info.get('uuid', None)
    if uuid_is_valid(uuid):
        known_uuids[uuid][info['hwaddr']] = info
+    info['otheraddresses'] = set([])
+    for i4addr in info.get('attributes', {}).get('ipv4-address', []):
+        info['otheraddresses'].add(i4addr)
    if handler and handler.https_supported and not handler.https_cert:
        if handler.cert_fail_reason == 'unreachable':
            log.log(
@@ -671,7 +710,8 @@ def detected(info):

 def b64tohex(b64str):
    bd = base64.b64decode(b64str)
-    return ''.join(['{0:02x}'.format(ord(x)) for x in bd])
+    bd = bytearray(bd)
+    return ''.join(['{0:02x}'.format(x) for x in bd])


 def get_enclosure_chain_head(nodename, cfg):
@@ -686,6 +726,10 @@ def get_enclosure_chain_head(nodename, cfg):
        if ne in members:
            raise exc.InvalidArgumentException(
                'Circular chain that includes ' + nodename)
+        if not cfg.is_node(ne):
+            raise exc.InvalidArgumentException(
+                '{0} is chained to nonexistent node {1} '.format(
+                    nodename, ne))
        nodename = ne
        members.append(nodename)
    return nodename
@@ -827,6 +871,8 @@ def get_nodename_from_chained_smms(cfg, handler, info):
                nodename = newnodename
    return nodename

+def get_node_by_uuid(uuid):
+    return nodes_by_uuid.get(uuid, None)

 def get_nodename_from_enclosures(cfg, info):
    nodename = None
@@ -851,7 +897,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
        handler.probe()  # unicast interrogation as possible to get more data
        # switch concurrently
        # do some preconfig, for example, to bring a SMM online if applicable
-        handler.preconfig()
+        handler.preconfig(nodename)
    except Exception as e:
        unknown_info[info['hwaddr']] = info
        info['discostatus'] = 'unidentified'
@@ -1026,6 +1072,14 @@ def discover_node(cfg, handler, info, nodename, manual):
                traceback.print_exc()
                return False
            newnodeattribs = {}
+            if cfm.list_collective():
+                # We are in a collective, check collective.manager
+                cmc = cfg.get_node_attributes(nodename, 'collective.manager')
+                cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
+                if not cm:
+                    # Node is being discovered in collective, but no collective.manager, default
+                    # to the collective member actually able to execute the discovery
+                    newnodeattribs['collective.manager'] = collective.get_myname()
            if 'uuid' in info:
                newnodeattribs['id.uuid'] = info['uuid']
            if 'serialnumber' in info:
@@ -1091,6 +1145,7 @@ needaddhandled = False
 def _handle_nodelist_change(configmanager):
    global needaddhandled
    global nodeaddhandler
+    macmap.vintage = 0  # the current mac map is probably inaccurate
    _recheck_nodes((), configmanager)
    if needaddhandled:
        needaddhandled = False
@@ -1099,10 +1154,18 @@ def _handle_nodelist_change(configmanager):
        nodeaddhandler = None


-def newnodes(added, deleting, configmanager):
+def newnodes(added, deleting, renamed, configmanager):
    global attribwatcher
    global needaddhandled
    global nodeaddhandler
+    alldeleting = set(deleting) | set(renamed)
+    for node in alldeleting:
+        if node not in known_nodes:
+            continue
+        for mac in known_nodes[node]:
+            if mac in known_info:
+                del known_info[mac]
+        del known_nodes[node]
    _map_unique_ids()
    configmanager.remove_watcher(attribwatcher)
    allnodes = configmanager.list_nodes()
@@ -1142,12 +1205,17 @@ def _periodic_recheck(configmanager):

 def rescan():
    _map_unique_ids()
-    eventlet.spawn_n(slp.active_scan, safe_detected)
+    global scanner
+    if scanner:
+        return
+    else:
+        scanner = eventlet.spawn(slp.active_scan, safe_detected, slp)


 def start_detection():
    global attribwatcher
    global rechecker
+    global rechecktime
    _map_unique_ids()
    cfg = cfm.ConfigManager(None)
    allnodes = cfg.list_nodes()
@@ -1162,8 +1230,7 @@ def start_detection():
    if rechecker is None:
        rechecktime = util.monotonic_time() + 900
        rechecker = eventlet.spawn_after(900, _periodic_recheck, cfg)
-
-    # eventlet.spawn_n(ssdp.snoop, safe_detected)
+    eventlet.spawn_n(ssdp.snoop, None, None, ssdp, get_node_by_uuid)

 def stop_autosense():
    for watcher in list(autosensors):
@@ -1171,8 +1238,8 @@ def stop_autosense():
        autosensors.discard(watcher)

 def start_autosense():
-    autosensors.add(eventlet.spawn(slp.snoop, safe_detected))
-    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected))
+    autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe))


 nodes_by_fprint = {}
@@ -1194,7 +1261,7 @@ def _map_unique_ids(nodes=None):
    uuid_by_nodes = {}
    fprint_by_nodes = {}
    for uuid in nodes_by_uuid:
-        if not uuid_is_valid():
+        if not uuid_is_valid(uuid):
            continue
        node = nodes_by_uuid[uuid]
        if node in bigmap:
@@ -26,16 +26,27 @@ ipmicommand.session.threading = eventlet.green.threading
 ipmicommand.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo
 getaddrinfo = eventlet.support.greendns.getaddrinfo

-DEFAULT_USER = 'USERID'
-DEFAULT_PASS = 'PASSW0RD'


 class NodeHandler(generic.NodeHandler):
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'

-    def _get_ipmicmd(self, user=DEFAULT_USER, password=DEFAULT_PASS):
-        return ipmicommand.Command(self.ipaddr, user, password)
+    def _get_ipmicmd(self, user=None, password=None):
+        priv = None
+        if user is None or password is None:
+            if self.trieddefault:
+                raise pygexc.IpmiException()
+            priv = 4  # manually indicate priv to avoid double-attempt
+        if user is None:
+            user = self.DEFAULT_USER
+        if password is None:
+            password = self.DEFAULT_PASS
+        return ipmicommand.Command(self.ipaddr, user, password,
+                                   privlevel=priv, keepalive=False)

    def __init__(self, info, configmanager):
+        self.trieddefault = None
        super(NodeHandler, self).__init__(info, configmanager)

    def probe(self):
@@ -45,35 +56,37 @@ class NodeHandler(generic.NodeHandler):
    def config(self, nodename, reset=False):
        self._bmcconfig(nodename, reset)

-    def _bmcconfig(self, nodename, reset=False):
+    def _bmcconfig(self, nodename, reset=False, customconfig=None, vc=None):
        # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
        # In general, try to use https automation, to make it consistent
        # between hypothetical secure path and today.
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
        try:
            ic = self._get_ipmicmd()
-            passwd = DEFAULT_PASS
+            passwd = self.DEFAULT_PASS
        except pygexc.IpmiException as pi:
-            creds = self.configmanager.get_node_attributes(
-                nodename,
-                ['secret.hardwaremanagementuser',
-                 'secret.hardwaremanagementpassword'], decrypt=True)
-            user = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementuser', {}).get('value', None)
            havecustomcreds = False
-            if user is not None and user != DEFAULT_USER:
+            if user is not None and user != self.DEFAULT_USER:
                havecustomcreds = True
            else:
-                user = DEFAULT_USER
-            passwd = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementpassword', {}).get('value', None)
-            if passwd is not None and passwd != DEFAULT_PASS:
+                user = self.DEFAULT_USER
+            if passwd is not None and passwd != self.DEFAULT_PASS:
                havecustomcreds = True
            else:
-                passwd = DEFAULT_PASS
+                passwd = self.DEFAULT_PASS
            if havecustomcreds:
                ic = self._get_ipmicmd(user, passwd)
            else:
                raise
+        if vc:
+            ic.register_key_handler(vc)
        currusers = ic.get_users()
        lanchan = ic.get_network_channel()
        userdata = ic.xraw_command(netfn=6, command=0x44, data=(lanchan,
@@ -93,45 +106,33 @@ class NodeHandler(generic.NodeHandler):
            raise exc.TargetEndpointBadCredentials(
                'secret.hardwaremanagementuser and/or '
                'secret.hardwaremanagementpassword was not configured')
-        if ('hardwaremanagement.manager' in cd and
-                cd['hardwaremanagement.manager']['value'] and
-                not cd['hardwaremanagement.manager']['value'].startswith(
-                    'fe80::')):
-            newip = cd['hardwaremanagement.manager']['value']
-            newipinfo = getaddrinfo(newip, 0)[0]
-            # This getaddrinfo is repeated in get_nic_config, could be
-            # optimized, albeit with a more convoluted api..
-            newip = newipinfo[-1][0]
-            if ':' in newip:
-                raise exc.NotImplementedException('IPv6 remote config TODO')
-            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
-            plen = netconfig['prefix']
-            newip = '{0}/{1}'.format(newip, plen)
-            ic.set_net_configuration(ipv4_address=newip,
-                                     ipv4_configuration='static',
-                                     ipv4_gateway=netconfig['ipv4_gateway'])
-        elif self.ipaddr.startswith('fe80::'):
-            cfg.set_node_attributes(
-                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
-        else:
-            raise exc.TargetEndpointUnreachable(
-                'hardwaremanagement.manager must be set to desired address')
        newuser = cd['secret.hardwaremanagementuser']['value']
        newpass = cd['secret.hardwaremanagementpassword']['value']
        for uid in currusers:
            if currusers[uid]['name'] == newuser:
                # Use existing account that has been created
                newuserslot = uid
+                if newpass != passwd:  # don't mess with existing if no change
+                    ic.set_user_password(newuserslot, password=newpass)
+                    ic = self._get_ipmicmd(user, passwd)
+                    if vc:
+                        ic.register_key_handler(vc)
                break
        else:
            newuserslot = lockedusers + 1
            if newuserslot < 2:
                newuserslot = 2
+            if newpass != passwd:  # don't mess with existing if no change
+                ic.set_user_password(newuserslot, password=newpass)
            ic.set_user_name(newuserslot, newuser)
-            ic.set_user_access(newuserslot, lanchan,
-                               privilege_level='administrator')
-        if newpass != passwd:  # don't mess with existing if no change
-            ic.set_user_password(newuserslot, password=newpass)
+            if havecustomcreds:
+                ic = self._get_ipmicmd(user, passwd)
+                if vc:
+                    ic.register_key_handler(vc)
+            #We are remote operating on the account we are
+            #using, no need to try to set user access
+            #ic.set_user_access(newuserslot, lanchan,
+            #                   privilege_level='administrator')
        # Now to zap others
        for uid in currusers:
            if uid != newuserslot:
@@ -152,6 +153,36 @@ class NodeHandler(generic.NodeHandler):
                            # name...
                            # the user will remain, but that is life
                            raise
+        if customconfig:
+            customconfig(ic)
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            # This getaddrinfo is repeated in get_nic_config, could be
+            # optimized, albeit with a more convoluted api..
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
+            plen = netconfig['prefix']
+            newip = '{0}/{1}'.format(newip, plen)
+            currcfg = ic.get_net_configuration()
+            if currcfg['ipv4_address'] != newip:
+                # do not change the ipv4_config if the current config looks
+                # like it is already accurate
+                ic.set_net_configuration(ipv4_address=newip,
+                                         ipv4_configuration='static',
+                                         ipv4_gateway=netconfig[
+                                             'ipv4_gateway'])
+        elif self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address')
        if reset:
            ic.reset_bmc()
        return ic
@@ -0,0 +1,21 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.bmc as bmchandler
+
+class NodeHandler(bmchandler.NodeHandler):
+    DEFAULT_USER = 'admin'
+    DEFAULT_PASS = 'admin'
+    devname = 'BMC'
+    maxmacs = 2
@@ -40,6 +40,12 @@ class NodeHandler(object):
                targsa = info['addresses'][0]
        self.ipaddr = targsa[0]

+    @classmethod
+    def adequate(cls, info):
+        # Check if the referenced info is really enough, if false, a rescan
+        # may occur against the target in a short while
+        return True
+
    def scan(self):
        # Do completely passive things to enhance data.
        # Probe is permitted to for example attempt a login
@@ -51,7 +57,7 @@ class NodeHandler(object):
        # serial number and uuid to flesh out data as needed
        return

-    def preconfig(self):
+    def preconfig(self, possiblenode):
        return

    def discoverable_by_switch(self, macs):
@@ -62,6 +68,23 @@ class NodeHandler(object):
    def _savecert(self, certificate):
        self._fp = certificate
        return True
+    
+    def get_node_credentials(self, nodename, creds, defuser, defpass):
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        havecustomcreds = False
+        if user is not None and user != defuser:
+            havecustomcreds = True
+        else:
+            user = defuser
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
+        if passwd is not None and passwd != defpass:
+            havecustomcreds = True
+        else:
+            passwd = defpass
+        return user, passwd, not havecustomcreds
+

    @property
    def cert_fail_reason(self):
@@ -12,15 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
 import pyghmi.exceptions as pygexc
 import pyghmi.ipmi.private.util as pygutil
-import string
+import confluent.util as util
 import struct

 class NodeHandler(bmchandler.NodeHandler):
    devname = 'IMM'

+    @classmethod
+    def adequate(cls, info):
+        # We can sometimes receive a partially initialized SLP packet
+        # This is not adequate for being satisfied
+        return bool(info.get('attributes', {}))
+
    def scan(self):
        slpattrs = self.info.get('attributes', {})
        self.isdense = False
@@ -32,13 +39,14 @@ class NodeHandler(bmchandler.NodeHandler):
        if wronguuid:
            # we need to fix the first three portions of the uuid
            uuidprefix = wronguuid.split('-')[:3]
-            uuidprefix = struct.pack(
-                '<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+            uuidprefix = codecs.encode(struct.pack(
+                '<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+            uuidprefix = util.stringify(uuidprefix)
            uuidprefix = uuidprefix[:8] + '-' + uuidprefix[8:12] + '-' + \
                         uuidprefix[12:16]
            self.info['uuid'] = uuidprefix + '-' + '-'.join(
                wronguuid.split('-')[3:])
-            self.info['uuid'] = string.lower(self.info['uuid'])
+            self.info['uuid'] = self.info['uuid'].lower()
        if ff not in ('dense-computing', 'BC2'):
            # do not probe unless it's a dense platform
            return
@@ -12,21 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
+import confluent.exceptions as exc
+import eventlet
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 import struct
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+import eventlet.support.greendns
+import confluent.netutil as netutil
+import confluent.util as util
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+from xml.etree.ElementTree import fromstring

 def fixuuid(baduuid):
    # SMM dumps it out in hex
    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
-    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode(
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]),
        'hex')
+    a = util.stringify(a)
    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
    return '-'.join(uuid).lower()

 class NodeHandler(bmchandler.NodeHandler):
    is_enclosure = True
    devname = 'SMM'
-    maxmacs = 5  # support an enclosure, but try to avoid catching daisy chain
+    maxmacs = 6  # support an enclosure, but try to avoid catching daisy chain

    def scan(self):
        # the UUID is in a weird order, fix it up to match
@@ -36,9 +51,147 @@ class NodeHandler(bmchandler.NodeHandler):
            uuid = fixuuid(uuid[0])
            self.info['uuid'] = uuid

+    def _validate_cert(self, certificate):
+        # Assumption is by the time we call config, that discovery core has
+        # vetted self._fp.  Our job here then is just to make sure that
+        # the currect connection matches the previously saved cert
+        if not self._fp:  # circumstances are that we haven't validated yet
+            self._fp = certificate
+        return certificate == self._fp
+
+    def _webconfigrules(self, wc):
+        rules = []
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                rules.append('passwordDurationDays:' + value)
+                warndays = '5' if int(value) > 5 else value
+                rules.append('passwordExpireWarningDays:' + warndays)
+            if name.lower() in ('lockout', 'loginfailures'):
+                rules.append('passwordFailAllowdNum:' + value)
+            if name.lower() == 'reuse':
+                rules.append('passwordReuseCheckNum:' + value)
+        if rules:
+            apirequest = 'set={0}'.format(','.join(rules))
+            wc.request('POST', '/data', apirequest)
+            wc.getresponse().read()
+
+    def _webconfignet(self, wc, nodename):
+        cfg = self.configmanager
+        cd = cfg.get_node_attributes(
+            nodename, ['hardwaremanagement.manager'])
+        smmip = cd.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', None)
+        if smmip and ':' not in smmip:
+            smmip = getaddrinfo(smmip, 0)[0]
+            smmip = smmip[-1][0]
+            if smmip and ':' in smmip:
+                raise exc.NotImplementedException('IPv6 not supported')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=smmip)
+            netmask = netutil.cidr_to_mask(netconfig['prefix'])
+            setdata = 'set=ifIndex:0,v4DHCPEnabled:0,v4IPAddr:{0},v4NetMask:{1}'.format(smmip, netmask)
+            gateway = netconfig.get('ipv4_gateway', None)
+            if gateway:
+                setdata += ',v4Gateway:{0}'.format(gateway)
+            wc.request('POST', '/data', setdata)
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            if '<statusCode>0' not in rspdata:
+                raise Exception("Error configuring SMM Network")
+            return
+        if smmip and ':' in smmip and not smmip.startswith('fe80::'):
+            raise exc.NotImplementedException('IPv6 configuration TODO')
+        if self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                    {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+
+    def _webconfigcreds(self, username, password):
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self._validate_cert)
+        wc.connect()
+        authdata = {  # start by trying factory defaults
+            'user': 'USERID',
+            'password': 'PASSW0RD',
+        }
+        headers = {'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded'}
+        wc.request('POST', '/data/login', urlencode(authdata), headers)
+        rsp = wc.getresponse()
+        rspdata = util.stringify(rsp.read())
+        if 'authResult>0' not in rspdata:
+            # default credentials are refused, try with the actual
+            authdata['user'] = username
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = util.stringify(rsp.read())
+            if 'renew_account' in rspdata:
+                raise Exception('Configured password has expired')
+            if 'authResult>0' not in rspdata:
+                raise Exception('Unknown username/password on SMM')
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+        if 'renew_account' in rspdata:
+            passwdchange = {'oripwd': 'PASSW0RD', 'newpwd': password}
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            wc.request('POST', '/data/changepwd', urlencode(passwdchange))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+        if 'authResult>0' in rspdata:
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            if username == 'USERID':
+                return wc
+            wc.request('POST', '/data', 'set=user(2,1,{0},511,,4,15,0)'.format(username))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            wc.request('POST', '/data/logout')
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['user'] = username
+            wc.request('POST', '/data/login', urlencode(authdata, headers))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+
    def config(self, nodename):
        # SMM for now has to reset to assure configuration applies
-        super(NodeHandler, self).config(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        username = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', 'USERID')
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', 'PASSW0RD')
+        if passwd == 'PASSW0RD' and self.ruleset:
+            raise Exception('Cannot support default password and setting password rules at same time')
+        if passwd == 'PASSW0RD':
+            # We must avoid hitting the web interface due to forced password change, best effert
+            self._bmcconfig(nodename)
+        else:
+            # Switch to full web based configuration, to mitigate risks with the SMM
+            wc = self._webconfigcreds(username, passwd)
+            self._webconfigrules(wc)
+            self._webconfignet(wc, nodename)
+

 # notes for smm:
 # POST to:
@@ -53,4 +206,4 @@ class NodeHandler(bmchandler.NodeHandler):
 # with body user=USERID&password=Passw0rd!4321
 # yields:
 # <?xml version="1.0" encoding="UTF-8"?><root> <status>ok</status> <authResult>0</authResult> <forwardUrl>index.html</forwardUrl> </root>
-# note forwardUrl, if password change needed, will indicate something else
+# note forwardUrl, if password change needed, will indicate something else
@@ -0,0 +1,191 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.generic as generic
+import confluent.exceptions as exc
+import confluent.netutil as netutil
+import confluent.util as util
+import eventlet.support.greendns
+import json
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+class NodeHandler(generic.NodeHandler):
+    devname = 'TSM'
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'
+
+    def __init__(self, info, configmanager):
+        self.trieddefault = None
+        self.targuser = None
+        self.curruser = None
+        self.currpass = None
+        self.targpass = None
+        self.nodename = None
+        self.csrftok = None
+        self.channel = None
+        self.atdefault = True
+        super(NodeHandler, self).__init__(info, configmanager)
+
+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def _get_wc(self):
+        authdata = {  # start by trying factory defaults
+            'username': self.DEFAULT_USER,
+            'password': self.DEFAULT_PASS,
+        }
+        if not self.trieddefault:
+            wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert)
+            rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            if status > 400:
+                rsp = util.stringify(rsp)
+                self.trieddefault = True
+                if '555' in rsp:
+                    passchange = {
+                        'Password': self.targpass, 
+                        'RetypePassword': self.targpass,
+                        'param': 4,
+                        'default_password': self.DEFAULT_PASS,
+                        'username': self.DEFAULT_USER
+                        }
+                    rsp, status = wc.grab_json_response_with_status('/api/reset-pass', urlencode(passchange))
+                    authdata['password'] = self.targpass
+                    rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+                    self.csrftok = rsp['CSRFToken']
+                    self.channel = rsp['channel']
+                    self.curruser = self.DEFAULT_USER
+                    self.currpass = self.targpass
+                    return wc
+            else:
+                self.curruser = self.DEFAULT_USER
+                self.currpass = self.DEFAULT_PASS
+                self.csrftok = rsp['CSRFToken']
+                self.channel = rsp['channel']
+                return wc
+        if self.curruser:
+            authdata['username'] = self.curruser
+            authdata['password'] = self.currpass
+            rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            if rsp.status != 200:
+                return None
+            self.csrftok = rsp['CSRFToken']
+            self.channel = rsp['channel']
+            return wc
+        authdata['username'] = self.targuser
+        authdata['password'] = self.targpass
+        rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+        if status != 200:
+            return None
+        self.curruser = self.targuser
+        self.currpass = self.targpass
+        self.csrftok = rsp['CSRFToken']
+        self.channel = rsp['channel']
+        return wc
+
+    def config(self, nodename):
+        self.nodename = nodename
+        creds = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
+        cd = creds.get(nodename, {})
+        user, passwd, _ = self.get_node_credentials(
+                nodename, creds, self.DEFAULT_USER, self.DEFAULT_PASS)
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
+        self.targuser = user
+        self.targpass = passwd
+        wc = self._get_wc()
+        wc.set_header('X-CSRFTOKEN', self.csrftok)
+        curruserinfo = {}
+        authupdate = False
+        wc.set_header('Content-Type', 'application/json')
+        if user != self.curruser:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['name'] = user
+        if passwd != self.currpass:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['changepassword'] = 0
+            authchg['password_size'] = 'bytes_20'
+            authchg['password'] = passwd
+            authchg['confirm_password'] = passwd
+        if authupdate:
+            rsp, status = wc.grab_json_response_with_status('/api/settings/users/2', authchg, method='PUT')
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            # IPMI must be enabled per user config
+            wc.grab_json_response('/api/settings/ipmilanconfig', {
+                'ipv4_enable': 1, 'ipv6_enable': 1,
+                'uncheckedipv4lanEnable': 0, 'uncheckedipv6lanEnable': 0,
+                'checkedipv4lanEnable': 1, 'checkedipv6lanEnable': 1})
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            currnet = wc.grab_json_response('/api/settings/network')
+            for net in currnet:
+                if net['channel_number'] == self.channel and net['lan_enable'] == 0:
+                    # ignore false indication and switch to 8 (dedicated)
+                    self.channel = 8
+                if net['channel_number'] == self.channel:
+                    # we have found the interface to potentially manipulate
+                    if net['ipv4_address'] != newip:
+                        netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+                        newmask = netutil.cidr_to_mask(netconfig['prefix'])
+                        net['ipv4_address'] = newip
+                        net['ipv4_subnet'] = newmask
+                        if netconfig['ipv4_gateway']:
+                            net['ipv4_gateway'] = netconfig['ipv4_gateway']
+                        net['ipv4_dhcp_enable'] = 0
+                        rsp, status = wc.grab_json_response_with_status(
+                            '/api/settings/network/{0}'.format(net['id']), net, method='PUT')
+                    break
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        rsp, status = wc.grab_json_response_with_status('/api/session', method='DELETE')
+
+
+if __name__ == '__main__':
+    import confluent.config.configmanager as cfm
+    c = cfm.ConfigManager(None)
+    import sys
+    info = {'addresses': [[sys.argv[1]]] }
+    print(repr(info))
+    testr = NodeHandler(info, c)
+    testr.config(sys.argv[2])
@@ -1,4 +1,4 @@
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,51 +12,408 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import base64
+import codecs
 import confluent.discovery.handlers.imm as immhandler
+import confluent.netutil as netutil
+import confluent.util as util
+import errno
+import eventlet
+import eventlet.support.greendns
+import json
+import os
 import pyghmi.exceptions as pygexc
-import pyghmi.ipmi.oem.lenovo.imm as imm
+import eventlet.green.socket as socket
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import struct
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+
+def fixup_uuid(uuidprop):
+    baduuid = ''.join(uuidprop.split())
+    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+    a = util.stringify(a)
+    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
+    return '-'.join(uuid).upper()
+



 class NodeHandler(immhandler.NodeHandler):
    devname = 'XCC'

-    def preconfig(self):
+    def __init__(self, info, configmanager):
+        self._wc = None
+        self.nodename = None
+        self.tmpnodename = None
+        self.tmppasswd = None
+        self._atdefaultcreds = True
+        self._needpasswordchange = True
+        self._currcreds = (None, None)
+        super(NodeHandler, self).__init__(info, configmanager)
+
+    @classmethod
+    def adequate(cls, info):
+        # We can sometimes receive a partially initialized SLP packet
+        # This is not adequate for being satisfied
+        return bool(info.get('attributes', {}))
+
+    def preconfig(self, possiblenode):
+        self.tmpnodename = possiblenode
        ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
        if ff not in ('dense-computing', [u'dense-computing']):
+            # skip preconfig for non-SD530 servers
            return
+        self.trieddefault = None  # Reset state on a preconfig attempt
        # attempt to enable SMM
        #it's normal to get a 'not supported' (193) for systems without an SMM
+        # need to branch on 3.00+ firmware
+        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
+        currfirm = currfirm.split(':')
+        if len(currfirm) > 1:
+            currfirm = float(currfirm[1])
+        disableipmi = False
+        if currfirm >= 3:
+            # IPMI is disabled and we need it, also we need to go to *some* password
+            wc = self.wc
+            if not wc:
+                # We cannot try to enable SMM here without risking real credentials
+                # on the wire to untrusted parties
+                return
+            wc.grab_json_response('/api/providers/logout')
+            wc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = wc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                disableipmi = True
+                _, _ = wc.grab_json_response_with_status(
+                    '/redfish/v1/Managers/1/NetworkProtocol',
+                    {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
        ipmicmd = None
        try:
-            ipmicmd = self._get_ipmicmd()
+            ipmicmd = self._get_ipmicmd(self._currcreds[0], self._currcreds[1])
            ipmicmd.xraw_command(netfn=0x3a, command=0xf1, data=(1,))
        except pygexc.IpmiException as e:
            if (e.ipmicode != 193 and 'Unauthorized name' not in str(e) and
                    'Incorrect password' not in str(e)):
                # raise an issue if anything other than to be expected
+                if disableipmi:
+                    _, _ = wc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
                raise
+            self.trieddefault = True
+        if disableipmi:
+            _, _ = wc.grab_json_response_with_status(
+                '/redfish/v1/Managers/1/NetworkProtocol',
+                {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
        #TODO: decide how to clean out if important
        #as it stands, this can step on itself
        #if ipmicmd:
        #    ipmicmd.ipmi_session.logout()

+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def get_webclient(self, username, password, newpassword):
+        wc = self._wc.dupe()
+        try:
+            wc.connect()
+        except socket.error as se:
+            if se.errno != errno.ECONNREFUSED:
+                raise
+            return (None, None)
+        pwdchanged = False
+        adata = json.dumps({'username': util.stringify(username),
+                            'password': util.stringify(password)
+                            })
+        headers = {'Connection': 'keep-alive',
+                   'Content-Type': 'application/json'}
+        wc.request('POST', '/api/login', adata, headers)
+        rsp = wc.getresponse()
+        if rsp.status != 200 and password == 'PASSW0RD':
+            rsp.read()
+            adata = json.dumps({
+                'username': username,
+                'password': newpassword,
+                })
+            headers = {'Connection': 'keep-alive',
+                       'Content-Type': 'application/json'}
+            wc.request('POST', '/api/login', adata, headers)
+            rsp = wc.getresponse()
+            if rsp.status == 200:
+                pwdchanged = True
+                password = newpassword
+            else:
+                rsp.read()
+                return (None, None)
+        if rsp.status == 200:
+            self._currcreds = (username, password)
+            wc.set_basic_credentials(username, password)
+            rspdata = json.loads(rsp.read())
+            wc.set_header('Content-Type', 'application/json')
+            wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            if rspdata.get('pwchg_required', None) == 'true':
+                wc.request('POST', '/api/function', json.dumps(
+                    {'USER_UserPassChange': '1,{0}'.format(newpassword)}))
+                rsp = wc.getresponse()
+                rsp.read()
+                if rsp.status != 200:
+                    return (None, None)
+                self._currcreds = (username, newpassword)
+                wc.set_basic_credentials(username, newpassword)
+                pwdchanged = True
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            return (wc, pwdchanged)
+
+    @property
+    def wc(self):
+        passwd = None
+        isdefault = True
+        if self._wc is None:
+            self._wc = webclient.SecureHTTPConnection(
+                self.ipaddr, 443, verifycallback=self.validate_cert)
+            self._wc.connect()
+        nodename = None
+        if self.nodename:
+            nodename = self.nodename
+            inpreconfig = False
+        elif self.tmpnodename:
+            nodename = None
+            inpreconfig = True
+        if self._currcreds[0] is not None:
+            wc, pwdchanged = self.get_webclient(self._currcreds[0], self._currcreds[1], None)
+            if wc:
+                return wc
+        if nodename:
+            creds = self.configmanager.get_node_attributes(
+                nodename, ['secret.hardwaremanagementuser',
+                'secret.hardwaremanagementpassword'], decrypt=True)
+            user, passwd, isdefault = self.get_node_credentials(
+                nodename, creds, 'USERID', 'PASSW0RD')
+        if not self.trieddefault:
+            if not passwd:
+                # So in preconfig context, we don't have admin permission to
+                # actually divulge anything to the target
+                # however the target *will* demand a new password... if it's currently
+                # PASSW0RD
+                # use TempW0rd42 to avoid divulging a real password on the line
+                # This is replacing one well known password (PASSW0RD) with another 
+                # (TempW0rd42)
+                passwd = 'TempW0rd42'
+            wc, pwdchanged = self.get_webclient('USERID', 'PASSW0RD', passwd)
+            if wc:
+                if pwdchanged:
+                    if inpreconfig:
+                        self.tmppasswd = passwd
+                    else:
+                        self._needpasswordchange = False
+                return wc
+        self.trieddefault = True
+        if isdefault:
+            return
+        self._atdefaultcreds = False
+        if self.tmppasswd:
+            wc, _ = self.get_webclient('USERID', self.tmppasswd, passwd)
+        else:
+            wc, _ = self.get_webclient(user, passwd, None)
+        if wc:
+            return wc
+
+    def set_password_policy(self):
+        ruleset = {'USER_GlobalMinPassChgInt': '0'}
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                ruleset['USER_GlobalPassExpPeriod'] = value
+                if int(value) < 5:
+                    ruleset['USER_GlobalPassExpWarningPeriod'] = value
+            if name.lower() in ('lockout', 'loginfailures'):
+                if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                    value = '0'
+                ruleset['USER_GlobalMaxLoginFailures'] = value
+            if name.lower() == 'complexity':
+                ruleset['USER_GlobalPassComplexRequired'] = value
+            if name.lower() == 'reuse':
+                ruleset['USER_GlobalMinPassReuseCycle'] = value
+        try:
+            wc = self.wc
+            wc.grab_json_response('/api/dataset', ruleset)
+            wc.grab_json_response('/api/providers/logout')
+        except Exception as e:
+            print(repr(e))
+            pass
+
+    def _get_next_userid(self, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        userinfo = userinfo['items'][0]['users']
+        for user in userinfo:
+            if user['users_user_name'] == '':
+                return user['users_user_id']
+
+    def _setup_xcc_account(self, username, passwd, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        uid = None
+        for user in userinfo['items'][0]['users']:
+            if user['users_user_name'] == username:
+                uid = user['users_user_id']
+                break
+        else:
+            for user in userinfo['items'][0]['users']:
+                if user['users_user_name'] == 'USERID':
+                    uid = user['users_user_id']
+                    break
+        if not uid:
+            raise Exception("XCC has neither the default user nor configured user")
+        # The following will work if the password is force change or normal..
+        if self._needpasswordchange and self.tmppasswd != passwd:
+            wc.grab_json_response('/api/function',
+                                {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
+        if username != 'USERID':
+            wc.grab_json_response(
+                '/api/function',
+                {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,'.format(uid, username)})
+            self.tmppasswd = None
+        self._currcreds = (username, passwd)
+
+    def _convert_sha256account(self, user, passwd, wc):
+        # First check if the specified user is sha256...
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        curruser = None
+        uid = None
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
+        for userent in userinfo['items'][0]['users']:
+            if userent['users_user_name'] == user:
+                curruser = userent
+                break
+        if curruser.get('users_pass_is_sha256', 0):
+            self._wc = None
+            wc = self.wc
+            nwc = wc.dupe()
+            # Have to convert it for being useful with most Lenovo automation tools
+            # This requires deleting the account entirely and trying again
+            tmpuid = self._get_next_userid(wc)
+            try:
+                tpass = base64.b64encode(os.urandom(9)) + 'Iw47$'
+                userparams = "{0},6pmu0ezczzcp,{1},1,4,0,0,0,0,,8,".format(tmpuid, tpass)
+                result = wc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                wc.grab_json_response('/api/providers/logout')
+                adata = json.dumps({
+                    'username': '6pmu0ezczzcp',
+                    'password': tpass,
+                })
+                headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+                nwc.request('POST', '/api/login', adata, headers)
+                rsp = nwc.getresponse()
+                if rsp.status == 200:
+                    rspdata = json.loads(rsp.read())
+                    nwc.set_header('Content-Type', 'application/json')
+                    nwc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+                    if '_csrf_token' in wc.cookies:
+                        nwc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+                    if rspdata.get('reason', False):
+                        newpass = base64.b64encode(os.urandom(9)) + 'q4J$'
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(tmpuid, newpass)})
+                    nwc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(curruser['users_user_id'], user)})
+                    userparams = "{0},{1},{2},1,4,0,0,0,0,,8,".format(curruser['users_user_id'], user, tpass)
+                    nwc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                    nwc.grab_json_response('/api/providers/logout')
+                    nwc, pwdchanged = self.get_webclient(user, tpass, passwd)
+                    if not pwdchanged:
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(curruser['users_user_id'], passwd)})
+                    nwc.grab_json_response('/api/providers/logout')
+            finally:
+                self._wc = None
+                wc = self.wc
+                wc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(tmpuid, '6pmu0ezczzcp')})
+                wc.grab_json_response('/api/providers/logout')
+
    def config(self, nodename, reset=False):
+        self.nodename = nodename
        # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
        # In general, try to use https automation, to make it consistent
        # between hypothetical secure path and today.
-        ic = self._bmcconfig(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        wc = self.wc
+        creds = self.configmanager.get_node_attributes(
+            self.nodename, ['secret.hardwaremanagementuser',
+            'secret.hardwaremanagementpassword'], decrypt=True)
+        user, passwd, isdefault = self.get_node_credentials(nodename, creds, 'USERID', 'PASSW0RD')
+        self.set_password_policy()
+        if self._atdefaultcreds:
+            if not isdefault:
+                self._setup_xcc_account(user, passwd, wc)
+        self._convert_sha256account(user, passwd, wc)
+        cd = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
+        cd = cd.get(nodename, {})
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            nwc = wc.dupe()
+            nwc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = nwc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                # User has indicated IPMI support, but XCC is currently disabled
+                # change XCC to be consistent
+                _, _ = nwc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+            newmask = netutil.cidr_to_mask(netconfig['prefix'])
+            currinfo = wc.grab_json_response('/api/providers/logoninfo')
+            currip = currinfo.get('items', [{}])[0].get('ipv4_address', '')
+            # do not change the ipv4_config if the current config looks right already
+            if currip != newip:
+                statargs = {
+                    'ENET_IPv4Ena': '1', 'ENET_IPv4AddrSource': '0',
+                    'ENET_IPv4StaticIPAddr': newip, 'ENET_IPv4StaticIPNetMask': newmask
+                    }
+                if netconfig['ipv4_gateway']:
+                    statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
+                wc.grab_json_response('/api/dataset', statargs)
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        wc.grab_json_response('/api/providers/logout')
        ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
        if ff not in ('dense-computing', [u'dense-computing']):
            return
-        # Ok, we can get the enclosure uuid now..
-        ic.oem_init()
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
+        enclosureuuid = self.info.get('attributes', {}).get('chassis-uuid', [None])[0]
        if enclosureuuid:
-            enclosureuuid = imm.fixup_uuid(enclosureuuid).lower()
+            enclosureuuid = enclosureuuid.lower()
            em = self.configmanager.get_node_attributes(nodename,
                                                        'enclosure.manager')
            em = em.get(nodename, {}).get('enclosure.manager', {}).get(
@@ -65,8 +422,3 @@ class NodeHandler(immhandler.NodeHandler):
            if em:
                self.configmanager.set_node_attributes(
                    {em: {'id.uuid': enclosureuuid}})
-
-# TODO(jjohnson2): web based init config for future prevalidated cert scheme
-#    def config(self, nodename):
-#        return
-
@@ -34,8 +34,8 @@ pxearchs = {


 def decode_uuid(rawguid):
-    lebytes = struct.unpack_from('<IHH', buffer(rawguid[:8]))
-    bebytes = struct.unpack_from('>HHI', buffer(rawguid[8:]))
+    lebytes = struct.unpack_from('<IHH', rawguid[:8])
+    bebytes = struct.unpack_from('>HHI', rawguid[8:])
    return '{0:08X}-{1:04X}-{2:04X}-{3:04X}-{4:04X}{5:08X}'.format(
        lebytes[0], lebytes[1], lebytes[2], bebytes[0], bebytes[1], bebytes[2]).lower()

@@ -72,7 +72,7 @@ def find_info_in_options(rq, optidx):
        return uuid, arch
    return uuid, arch

-def snoop(handler):
+def snoop(handler, protocol=None):
    #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
    #prominent
    #TODO(jjohnson2): IP_PKTINFO, recvmsg to get the destination ip, per
@@ -98,7 +98,7 @@ def snoop(handler):
            netaddr = ':'.join(['{0:02x}'.format(x) for x in netaddr])
            optidx = 0
            try:
-                optidx = rq.index('\x63\x82\x53\x63') + 4
+                optidx = rq.index(b'\x63\x82\x53\x63') + 4
            except ValueError:
                continue
            uuid, arch = find_info_in_options(rq, optidx)
@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ import traceback
 _slp_services = set([
    'service:management-hardware.IBM:integrated-management-module2',
    'service:lenovo-smm',
+    'service:ipmi',
+    'service:lighttpd',
    'service:management-hardware.Lenovo:lenovo-xclarity-controller',
    'service:management-hardware.IBM:chassis-management-module',
    'service:management-hardware.Lenovo:chassis-management-module',
@@ -47,7 +49,6 @@ except AttributeError:
    IPPROTO_IPV6 = 41  # Assume Windows value if socket is missing it


-
 def _parse_slp_header(packet):
    packet = bytearray(packet)
    if len(packet) < 16 or packet[0] != 2:
@@ -245,24 +246,23 @@ def _grab_rsps(socks, rsps, interval, xidmap):

 def _parse_attrlist(attrstr):
    attribs = {}
+    previousattrlen = None
+    attrstr = util.stringify(attrstr)
    while attrstr:
+        if len(attrstr) == previousattrlen:
+            raise Exception('Looping in attrstr parsing')
+        previousattrlen = len(attrstr)
        if attrstr[0] == '(':
            if ')' not in attrstr:
                attribs['INCOMPLETE'] = True
                return attribs
            currattr = attrstr[1:attrstr.index(')')]
            if '=' not in currattr:  # Not allegedly kosher, but still..
-                currattr = currattr.decode('utf-8')
                attribs[currattr] = None
            else:
-                attrname, attrval = currattr.split('=')
-                attrname = attrname.decode('utf-8')
+                attrname, attrval = currattr.split('=', 1)
                attribs[attrname] = []
                for val in attrval.split(','):
-                    try:
-                        val = val.decode('utf-8')
-                    except UnicodeDecodeError:
-                        val = '*DECODEERROR*'
                    if val[:3] == '\\FF':  # we should make this bytes
                        finalval = bytearray([])
                        for bnum in attrval[3:].split('\\'):
@@ -272,9 +272,9 @@ def _parse_attrlist(attrstr):
                        val = finalval
                        if 'uuid' in attrname and len(val) == 16:
                            lebytes = struct.unpack_from(
-                                '<IHH', buffer(val[:8]))
+                                '<IHH', memoryview(val[:8]))
                            bebytes = struct.unpack_from(
-                                '>HHI', buffer(val[8:]))
+                                '>HHI', memoryview(val[8:]))
                            val = '{0:08X}-{1:04X}-{2:04X}-{3:04X}-' \
                                  '{4:04X}{5:08X}'.format(
                                lebytes[0], lebytes[1], lebytes[2], bebytes[0],
@@ -282,7 +282,7 @@ def _parse_attrlist(attrstr):
                            ).lower()
                    attribs[attrname].append(val)
            attrstr = attrstr[attrstr.index(')'):]
-        elif attrstr[0] == ',':
+        elif attrstr[0] == ','[0]:
            attrstr = attrstr[1:]
        elif ',' in attrstr:
            currattr = attrstr[:attrstr.index(',')]
@@ -295,9 +295,11 @@ def _parse_attrlist(attrstr):
    return attribs


-def _parse_attrs(data, parsed):
+def _parse_attrs(data, parsed, xid=None):
    headinfo = _parse_slp_header(data)
-    if headinfo['function'] != 7 or headinfo['xid'] != parsed['xid']:
+    if xid is None:
+        xid = parsed['xid']
+    if headinfo['function'] != 7 or headinfo['xid'] != xid:
        return
    payload = headinfo['payload']
    if struct.unpack('!H', bytes(payload[:2]))[0] != 0:
@@ -307,8 +309,18 @@ def _parse_attrs(data, parsed):
    parsed['attributes'] = _parse_attrlist(attrstr)


+def fix_info(info, handler):
+    if '_attempts' not in info:
+        info['_attempts'] = 10
+    if info['_attempts'] == 0:
+        return
+    info['_attempts'] -= 1
+    _add_attributes(info)
+    handler(info)
+
 def _add_attributes(parsed):
-    attrq = _generate_attr_request(parsed['services'][0], parsed['xid'])
+    xid = parsed.get('xid', 42)
+    attrq = _generate_attr_request(parsed['services'][0], xid)
    target = None
    # prefer reaching out to an fe80 if present, to be highly robust
    # in face of network changes
@@ -323,13 +335,14 @@ def _add_attributes(parsed):
    else:
        net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
+        net.settimeout(1.0)
        net.connect(target)
    except socket.error:
        return
    net.sendall(attrq)
    rsp = net.recv(8192)
    net.close()
-    _parse_attrs(rsp, parsed)
+    _parse_attrs(rsp, parsed, xid)


 def query_srvtypes(target):
@@ -351,6 +364,7 @@ def query_srvtypes(target):
    while tries and not connected:
        tries -= 1
        try:
+            net.settimeout(1.0)
            net.connect(target)
            connected = True
        except socket.error:
@@ -383,7 +397,7 @@ def rescan(handler):
            handler(scanned)


-def snoop(handler):
+def snoop(handler, protocol=None):
    """Watch for SLP activity

    handler will be called with a dictionary of relevant attributes
@@ -392,7 +406,7 @@ def snoop(handler):
    :return:
    """
    tracelog = log.Logger('trace')
-    active_scan(handler)
+    active_scan(handler, protocol)
    net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
    net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
    slpg = socket.inet_pton(socket.AF_INET6, 'ff01::123')
@@ -469,13 +483,26 @@ def snoop(handler):
                peerbymacaddress[mac]['xid'] = 1
                _add_attributes(peerbymacaddress[mac])
                peerbymacaddress[mac]['hwaddr'] = mac
+                peerbymacaddress[mac]['protocol'] = protocol
+                if 'service:ipmi' in peerbymacaddress[mac]['services']:
+                    if 'service:ipmi//Athena:623' in peerbymacaddress[mac].get('urls', ()):
+                        peerbymacaddress[mac]['services'] = ['service:thinkagile-storage']
+                    else:
+                        continue
+                if 'service:lightttpd' in peerbymacaddress[mac]['services']:
+                    currinf = peerbymacaddress[mac]
+                    curratt = currinf.get('attributes', {})
+                    if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+                        peerbymacaddress[mac]['services'] = ['service:lenovo-tsm']
+                    else:
+                        continue
                handler(peerbymacaddress[mac])
        except Exception as e:
            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)


-def active_scan(handler):
+def active_scan(handler, protocol=None):
    known_peers = set([])
    for scanned in scan():
        for addr in scanned['addresses']:
@@ -486,6 +513,7 @@ def active_scan(handler):
                break
            known_peers.add(addr)
        else:
+            scanned['protocol'] = protocol
            handler(scanned)


@@ -531,6 +559,11 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
    _grab_rsps((net, net4), rsps, 1, xidmap)
    # now to analyze and flesh out the responses
    for id in rsps:
+        if 'service:ipmi' in rsps[id]['services']:
+            if 'service:ipmi://Athena:623' in rsps[id]['urls']:
+                rsps[id]['services'] = ['service:thinkagile-storage']
+            else:
+                continue
        if localonly:
            for addr in rsps[id]['addresses']:
                if 'fe80' in addr[0]:
@@ -538,6 +571,15 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
            else:
                continue
        _add_attributes(rsps[id])
+        if 'service:lighttpd' in rsps[id]['services']:
+            currinf = rsps[id]
+            curratt = currinf.get('attributes', {})
+            if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+               currinf['services'] = ['service:lenovo-tsm']
+               curratt['enclosure-serial-number'] = curratt['Product-Serial']
+               curratt['enclosure-machinetype-model'] = curratt['Machine-Type']
+            else:
+                continue
        del rsps[id]['payload']
        del rsps[id]['function']
        del rsps[id]['xid']
@@ -30,9 +30,11 @@

 import confluent.neighutil as neighutil
 import confluent.util as util
+import confluent.log as log
 import eventlet.green.select as select
 import eventlet.green.socket as socket
 import struct
+import traceback

 mcastv4addr = '239.255.255.250'
 mcastv6addr = 'ff02::c'
@@ -51,7 +53,7 @@ def scan(services, target=None):
            yield rply


-def snoop(handler, byehandler=None):
+def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
    """Watch for SSDP notify messages

    The handler shall be called on any service coming online.
@@ -67,6 +69,7 @@ def snoop(handler, byehandler=None):
    # Normally, I like using v6/v4 agnostic socket. However, since we are
    # dabbling in multicast wizardry here, such sockets can cause big problems,
    # so we will have two distinct sockets
+    tracelog = log.Logger('trace')
    known_peers = set([])
    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
@@ -78,57 +81,85 @@ def snoop(handler, byehandler=None):
    for i4 in util.list_ips():
        ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
                     socket.inet_aton(i4['addr'])
-        net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
-                        ssdp4mcast)
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            ssdp4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    net4.bind(('', 1900))
    net6.bind(('', 1900))
    peerbymacaddress = {}
    while True:
-        newmacs = set([])
-        machandlers = {}
-        r, _, _ = select.select((net4, net6), (), (), 60)
-        neighutil.update_neigh()
-        while r:
-            for s in r:
-                (rsp, peer) = s.recvfrom(9000)
-                rsp = rsp.split('\r\n')
-                method, _, _ = rsp[0].split(' ', 2)
-                if method == 'NOTIFY':
-                    ip = peer[0].partition('%')[0]
-                    if ip not in neighutil.neightable:
-                        continue
-                    if peer in known_peers:
-                        continue
-                    mac = neighutil.neightable[ip]
-                    known_peers.add(peer)
-                    newmacs.add(mac)
-                    if mac in peerbymacaddress:
-                        peerbymacaddress[mac]['peers'].append(peer)
-                    else:
-                        peerbymacaddress[mac] = {
-                            'hwaddr': mac,
-                            'peers': [peer],
-                        }
-                        peerdata = peerbymacaddress[mac]
+        try:
+            newmacs = set([])
+            machandlers = {}
+            r, _, _ = select.select((net4, net6), (), (), 60)
+            neighutil.update_neigh()
+            while r:
+                for s in r:
+                    (rsp, peer) = s.recvfrom(9000)
+                    rsp = rsp.split('\r\n')
+                    method, _, _ = rsp[0].split(' ', 2)
+                    if method == 'NOTIFY':
+                        ip = peer[0].partition('%')[0]
+                        if ip not in neighutil.neightable:
+                            continue
+                        if peer in known_peers:
+                            continue
+                        mac = neighutil.neightable[ip]
+                        known_peers.add(peer)
+                        newmacs.add(mac)
+                        if mac in peerbymacaddress:
+                            peerbymacaddress[mac]['peers'].append(peer)
+                        else:
+                            peerbymacaddress[mac] = {
+                                'hwaddr': mac,
+                                'peers': [peer],
+                            }
+                            peerdata = peerbymacaddress[mac]
+                            for headline in rsp[1:]:
+                                if not headline:
+                                    continue
+                                header, _, value = headline.partition(':')
+                                header = header.strip()
+                                value = value.strip()
+                                if header == 'NT':
+                                    peerdata['service'] = value
+                                elif header == 'NTS':
+                                    if value == 'ssdp:byebye':
+                                        machandlers[mac] = byehandler
+                                    elif value == 'ssdp:alive':
+                                        machandlers[mac] = None # handler
+                    elif method == 'M-SEARCH':
+                        if not uuidlookup:
+                            continue
+                        #ip = peer[0].partition('%')[0]
                        for headline in rsp[1:]:
                            if not headline:
                                continue
-                            header, _, value = headline.partition(':')
-                            header = header.strip()
-                            value = value.strip()
-                            if header == 'NT':
-                                peerdata['service'] = value
-                            elif header == 'NTS':
-                                if value == 'ssdp:byebye':
-                                    machandlers[mac] = byehandler
-                                elif value == 'ssdp:alive':
-                                    machandlers[mac] = handler
-            r, _, _ = select.select((net4, net6), (), (), 0.1)
-        for mac in newmacs:
-            thehandler = machandlers.get(mac, None)
-            if thehandler:
-                thehandler(peerbymacaddress[mac])
+                            headline = headline.partition(':')
+                            if len(headline) < 3:
+                                continue
+                            if  headline[0] == 'ST' and headline[-1].startswith(' urn:xcat.org:service:confluent:'):
+                                for query in headline[-1].split('/'):
+                                    if query.startswith('uuid='):
+                                        curruuid = query.split('=', 1)[1].lower()
+                                        node = uuidlookup(curruuid)
+                                        if not node:
+                                            break
+                                        reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}'.format(node)
+                                        s.sendto(reply, peer)
+                r, _, _ = select.select((net4, net6), (), (), 0.2)
+            for mac in newmacs:
+                thehandler = machandlers.get(mac, None)
+                if thehandler:
+                    thehandler(peerbymacaddress[mac])
+        except Exception:
+                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                             event=log.Events.stacktrace)


 def _find_service(service, target):
@@ -167,6 +198,7 @@ def _find_service(service, target):
            net4.sendto(smsg.format(bcast, service), (bcast, 1900))
    # SSDP by spec encourages responses to spread out over a 3 second interval
    # hence we must be a bit more patient
+    deadline = util.monotonic_time() + 4
    r, _, _ = select.select((net4, net6), (), (), 4)
    peerdata = {}
    while r:
@@ -174,7 +206,10 @@ def _find_service(service, target):
            (rsp, peer) = s.recvfrom(9000)
            neighutil.refresh_neigh()
            _parse_ssdp(peer, rsp, peerdata)
-        r, _, _ = select.select((net4, net6), (), (), 4)
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
    for nid in peerdata:
        yield peerdata[nid]

@@ -194,6 +229,8 @@ def _parse_ssdp(peer, rsp, peerdata):
    if code == '200':
        if nid in peerdata:
            peerdatum = peerdata[nid]
+            if peer not in peerdatum['peers']:
+                peerdatum['peers'].append(peer)
        else:
            peerdatum = {
                'peers': [peer],
@@ -106,6 +106,7 @@ class PubkeyInvalid(ConfluentException):
        super(PubkeyInvalid, self).__init__(self, text)
        self.fingerprint = fingerprint
        self.attrname = attribname
+        self.message = text
        bodydata = {'message': text,
                    'event': event,
                    'fingerprint': fingerprint,
@@ -19,22 +19,31 @@
 # the time comes

 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.messages as msg
 import eventlet
 import os
+import pwd
 import socket
+import traceback

 updatesbytarget = {}
 uploadsbytarget = {}
+downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
+_tracelog = None


-def execupdate(handler, filename, updateobj, type):
-    if not os.path.exists(filename):
+def execupdate(handler, filename, updateobj, type, owner, node):
+    global _tracelog
+    if type != 'ffdc' and not os.path.exists(filename):
        errstr =  '{0} does not appear to exist on {1}'.format(
            filename, socket.gethostname())
        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                   'detail': errstr})
+        return
+    if type == 'ffdc' and os.path.isdir(filename):
+        filename += '/' + node + '.svcdata'
    try:
        if type == 'firmware':
            completion = handler(filename, progress=updateobj.handle_progress,
@@ -43,6 +52,9 @@ def execupdate(handler, filename, updateobj, type):
            completion = handler(filename, progress=updateobj.handle_progress)
        if completion is None:
            completion = 'complete'
+        if owner:
+            pwent = pwd.getpwnam(owner)
+            os.chown(filename, pwent.pw_uid, pwent.pw_gid)
        updateobj.handle_progress({'phase': completion, 'progress': 100.0})
    except exc.PubkeyInvalid as pi:
        errstr = 'Certificate mismatch detected, does not match value in ' \
@@ -50,23 +62,28 @@ def execupdate(handler, filename, updateobj, type):
        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                   'detail': errstr})
    except Exception as e:
+        if _tracelog is None:
+            _tracelog = log.Logger('trace')
+        _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                   'detail': str(e)})

 class Updater(object):
    def __init__(self, node, handler, filename, tenant=None, name=None,
-                 bank=None, type='firmware'):
+                 bank=None, type='firmware', owner=None):
        self.bank = bank
        self.node = node
        self.phase = 'initializing'
        self.detail = ''
        self.percent = 0.0
        self.updateproc = updatepool.spawn(execupdate, handler, filename,
-                                           self, type)
+                                           self, type, owner, node)
        if type == 'firmware':
            myparty = updatesbytarget
        elif type == 'mediaupload':
            myparty = uploadsbytarget
+        elif type == 'ffdc':
+            myparty = downloadsbytarget
        if (node, tenant) not in myparty:
            myparty[(node, tenant)] = {}
        if name is None:
@@ -96,6 +113,8 @@ def remove_updates(nodes, tenant, element, type='firmware'):
    upid = element[-1]
    if type == 'firmware':
        myparty = updatesbytarget
+    elif type == 'ffdc':
+        myparty = downloadsbytarget
    else:
        myparty = uploadsbytarget
    for node in nodes:
@@ -115,6 +134,9 @@ def list_updates(nodes, tenant, element, type='firmware'):
    if type == 'mediaupload':
        myparty = uploadsbytarget
        verb = 'upload'
+    elif type == 'ffdc':
+        verb = 'download'
+        myparty = downloadsbytarget
    else:
        myparty = updatesbytarget
        verb = 'update'
@@ -49,6 +49,7 @@ def forward_port(sock, target, clientip, sessionid):
            continue
        try:
            client = socket.create_connection((target, 443))
+            client.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
        except Exception:
            conn.close()
            continue
@@ -68,6 +69,7 @@ def forward_video():
        try:
            vidclient = socket.create_connection((vidtargetbypeer[cli[0]],
                                                  3900))
+            vidclient.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
        except Exception:
            conn.close()
            continue
@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4

 # Copyright 2014 IBM Corporation
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,10 @@
 # This SCGI server provides a http wrap to confluent api
 # It additionally manages httprequest console sessions
 import base64
-import Cookie
+try:
+    import Cookie
+except ModuleNotFoundError:
+    import http.cookies as Cookie
 import confluent.auth as auth
 import confluent.config.attributes as attribs
 import confluent.consoleserver as consoleserver
@@ -39,7 +42,10 @@ import socket
 import sys
 import traceback
 import time
-import urlparse
+try:
+    import urlparse
+except ModuleNotFoundError:
+    import urllib.parse as urlparse
 import eventlet.wsgi
 #scgi = eventlet.import_patched('flup.server.scgi')
 tlvdata = confluent.tlvdata
@@ -74,7 +80,7 @@ def group_creation_resources():
    yield confluent.messages.ListAttributes(kv={'nodes': []},
                                            desc='Nodes to add to the group'
                                            ).html() + '<br>\n'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
        if attr == 'groups':
            continue
        if attr.startswith("secret."):
@@ -95,7 +101,7 @@ def group_creation_resources():
 def node_creation_resources():
    yield confluent.messages.Attributes(
        kv={'name': None}, desc="Name of the node").html() + '<br>'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
        if attr.startswith("secret."):
            yield confluent.messages.CryptedAttributes(
                kv={attr: None},
@@ -126,7 +132,7 @@ def user_creation_resources():
            'description': (''),
        },
    }
-    for attr in sorted(credential.iterkeys()):
+    for attr in sorted(credential):
        if attr == "password":
            yield confluent.messages.CryptedAttributes(
                kv={attr: None},
@@ -176,7 +182,7 @@ def _get_query_dict(env, reqbody, reqtype):
    if reqbody is not None:
        if "application/x-www-form-urlencoded" in reqtype:
            pbody = urlparse.parse_qs(reqbody, True)
-            for ky in pbody.iterkeys():
+            for ky in pbody:
                if len(pbody[ky]) > 1:  # e.g. REST explorer
                    na = [i for i in pbody[ky] if i != '']
                    qdict[ky] = na
@@ -184,7 +190,7 @@ def _get_query_dict(env, reqbody, reqtype):
                    qdict[ky] = pbody[ky][0]
        elif 'application/json' in reqtype:
            pbody = json.loads(reqbody)
-            for key in pbody.iterkeys():
+            for key in pbody:
                qdict[key] = pbody[key]
    if 'restexplorerhonorkey' in qdict:
        nqdict = {}
@@ -269,6 +275,9 @@ def _authorize_request(env, operation):
    name = ''
    sessionid = None
    cookie = Cookie.SimpleCookie()
+    element = env['PATH_INFO']
+    if element.startswith('/sessions/current/'):
+        element = None
    if 'HTTP_COOKIE' in env:
        #attempt to use the cookie.  If it matches
        cc = RobustCookie()
@@ -290,7 +299,7 @@ def _authorize_request(env, operation):
                    httpsessions[sessionid]['expiry'] = time.time() + 90
                    name = httpsessions[sessionid]['name']
                    authdata = auth.authorize(
-                        name, element=None,
+                        name, element=element, operation=operation,
                        skipuserobj=httpsessions[sessionid]['skipuserobject'])
    if (not authdata) and 'HTTP_AUTHORIZATION' in env:
        if env['PATH_INFO'] == '/sessions/current/logout':
@@ -302,9 +311,11 @@ def _authorize_request(env, operation):
                return {'code': 401}
            return ('logout',)
        name, passphrase = base64.b64decode(
-            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
-        authdata = auth.check_user_passphrase(name, passphrase, element=None)
-        if not authdata:
+            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(b':', 1)
+        authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
+        if authdata is False:
+            return {'code': 403}
+        elif not authdata:
            return {'code': 401}
        sessid = util.randomstring(32)
        while sessid in httpsessions:
@@ -314,14 +325,14 @@ def _authorize_request(env, operation):
                                'inflight': set([])}
        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
            httpsessions[sessid]['csrftoken'] = util.randomstring(32)
-        cookie['confluentsessionid'] = sessid
+        cookie['confluentsessionid'] = util.stringify(sessid)
        cookie['confluentsessionid']['secure'] = 1
        cookie['confluentsessionid']['httponly'] = 1
        cookie['confluentsessionid']['path'] = '/'
    skiplog = _should_skip_authlog(env)
    if authdata:
        auditmsg = {
-            'user': name,
+            'user': util.stringify(name),
            'operation': operation,
            'target': env['PATH_INFO'],
        }
@@ -333,7 +344,7 @@ def _authorize_request(env, operation):
        if authdata[3] is not None:
            auditmsg['tenant'] = authdata[3]
            authinfo['tenant'] = authdata[3]
-        auditmsg['user'] = authdata[2]
+        auditmsg['user'] = util.stringify(authdata[2])
        if sessid is not None:
            authinfo['sessionid'] = sessid
        if not skiplog:
@@ -341,15 +352,10 @@ def _authorize_request(env, operation):
        if 'csrftoken' in httpsessions[sessid]:
            authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
        return authinfo
-    else:
+    elif authdata is None:
        return {'code': 401}
-        # TODO(jbjohnso): actually evaluate the request for authorization
-        # In theory, the x509 or http auth stuff will get translated and then
-        # passed on to the core authorization function in an appropriate form
-        # expresses return in the form of http code
-        # 401 if there is no known identity
-        # 403 if valid identity, but no access
-        # going to run 200 just to get going for now
+    else:
+        return {'code': 403}


 def _pick_mimetype(env):
@@ -384,11 +390,11 @@ def resourcehandler(env, start_response):
    try:
        for rsp in resourcehandler_backend(env, start_response):
            yield rsp
-    except:
+    except Exception as e:
        tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                     event=log.Events.stacktrace)
-        start_response('500 - Internal Server Error', [])
-        yield '500 - Internal Server Error'
+        start_response('500 - ' + str(e), [])
+        yield '500 - ' + str(e)
        return


@@ -429,7 +435,7 @@ def resourcehandler_backend(env, start_response):
        return
    if authorized['code'] == 403:
        start_response('403 Forbidden', badauth)
-        yield 'authorization failed'
+        yield 'Forbidden'
        return
    if authorized['code'] != 200:
        raise Exception("Unrecognized code from auth engine")
@@ -446,6 +452,8 @@ def resourcehandler_backend(env, start_response):
                            httpsessions[authorized['sessionid']]['inflight'])):
                pagecontent += rsp
            start_response("200 OK", headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
            yield pagecontent
            return
        except exc.ConfluentException as e:
@@ -469,6 +477,10 @@ def resourcehandler_backend(env, start_response):
        funport = forwarder.get_port(targip, env['HTTP_X_FORWARDED_FOR'],
                                     authorized['sessionid'])
        host = env['HTTP_X_FORWARDED_HOST']
+        if ']' in host:
+            host = host.split(']')[0] + ']'
+        elif ':' in host:
+            host = host.rsplit(':', 1)[0]
        url = 'https://{0}:{1}/'.format(host, funport)
        start_response('302', [('Location', url)])
        yield 'Our princess is in another castle!'
@@ -487,7 +499,7 @@ def resourcehandler_backend(env, start_response):
            auditmsg = {
                'operation': 'start',
                'target': env['PATH_INFO'],
-                'user': authorized['username'],
+                'user': util.stringify(authorized['username']),
            }
            if 'tenant' in authorized:
                auditmsg['tenant'] = authorized['tenant']
@@ -496,6 +508,8 @@ def resourcehandler_backend(env, start_response):
            skipreplay = False
            if 'skipreplay' in querydict and querydict['skipreplay']:
                skipreplay = True
+            width = querydict.get('width', 80)
+            height = querydict.get('height', 24)
            datacallback = None
            async = None
            if 'HTTP_CONFLUENTASYNCID' in env:
@@ -507,13 +521,13 @@ def resourcehandler_backend(env, start_response):
                    consession = shellserver.ShellSession(
                        node=nodename, configmanager=cfgmgr,
                        username=authorized['username'], skipreplay=skipreplay,
-                        datacallback=datacallback
+                        datacallback=datacallback, width=width, height=height
                    )
                else:
                    consession = consoleserver.ConsoleSession(
                        node=nodename, configmanager=cfgmgr,
                        username=authorized['username'], skipreplay=skipreplay,
-                        datacallback=datacallback
+                        datacallback=datacallback, width=width, height=height
                    )
            except exc.NotFoundException:
                start_response("404 Not found", headers)
@@ -548,6 +562,9 @@ def resourcehandler_backend(env, start_response):
        elif 'action' in querydict:
            if querydict['action'] == 'break':
                consolesessions[querydict['session']]['session'].send_break()
+            elif querydict['action'] == 'resize':
+                consolesessions[querydict['session']]['session'].resize(
+                    width=querydict['width'], height=querydict['height'])
            elif querydict['action'] == 'reopen':
                consolesessions[querydict['session']]['session'].reopen()
            else:
@@ -617,6 +634,7 @@ def resourcehandler_backend(env, start_response):
            sessinfo = {'username': authorized['username']}
            if 'authtoken' in authorized:
                sessinfo['authtoken'] = authorized['authtoken']
+            tlvdata.unicode_dictvalues(sessinfo)
            yield json.dumps(sessinfo)
            return
        resource = '.' + url[url.rindex('/'):]
@@ -638,6 +656,8 @@ def resourcehandler_backend(env, start_response):
                for datum in _assemble_json(hdlr, resource, url, extension):
                    pagecontent += datum
            start_response('200 OK', headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
            yield pagecontent
        except exc.ConfluentException as e:
            if ((not isinstance(e, exc.LockedCredentials)) and
@@ -722,7 +742,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
    for rsp in responses:
        if isinstance(rsp, confluent.messages.LinkRelation):
            haldata = rsp.raw()
-            for hk in haldata.iterkeys():
+            for hk in haldata:
                if 'href' in haldata[hk]:
                    if isinstance(haldata[hk]['href'], int):
                        haldata[hk]['href'] = str(haldata[hk]['href'])
@@ -738,7 +758,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                    links[hk] = haldata[hk]
        else:
            rsp = rsp.raw()
-            for dk in rsp.iterkeys():
+            for dk in rsp:
                if dk in rspdata:
                    if isinstance(rspdata[dk], list):
                        if isinstance(rsp[dk], list):
@@ -757,8 +777,8 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                        rspdata[dk] = rsp[dk]
    rspdata["_links"] = links
    tlvdata.unicode_dictvalues(rspdata)
-    yield json.dumps(
-        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
+    yield util.stringify(json.dumps(
+        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8'))


 def serve(bind_host, bind_port):
@@ -784,8 +804,17 @@ def serve(bind_host, bind_port):
                'Failed to open HTTP due to busy port, trying again in'
                ' a second\n')
            eventlet.sleep(1)
-    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                         debug=False, socket_timeout=60)
+    # TCP_FASTOPEN
+    try:
+        sock.setsockopt(socket.SOL_TCP, 23, 5)
+    except Exception:
+        pass  # we gave it our best shot there
+    try:
+        eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                             debug=False, socket_timeout=60)
+    except TypeError:
+        # Older eventlet in place, skip arguments it does not understand
+        eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)


 class HttpApi(object):
@@ -41,6 +41,11 @@ class Console(object):
        the remote console"""
        return

+    def resize(self, width, height):
+        """This function is responsible for relaying resize request from
+        client terminal to remote console"""
+        return
+
    def ping(self):
        """This function is a hint to the console plugin that now would be a
        nice time to assess health of console connection.  Plugins that see
--- a/Show More
+++ b/Show More