Fix XCC discovery issues

A couple of issues could occur during discover that should be
stepped over.

confluent_client/bin/collate

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -35,6 +35,9 @@ import confluent.client
 argparser = optparse.OptionParser(usage="Usage: <other command> | %prog [options]")
 argparser.add_option('-a', '--abbreviate', action='store_true',
                      help='Attempt to use confluent server to shorten noderanges')
+argparser.add_option('-b', '--base',
+                     help='Use given node as reference for comparison when '
+                          'using -d, instead of using the most common result')
 argparser.add_option('-d', '--diff', action='store_true',
                      help='Show what differs between most common '
                           'output group and others')
@@ -65,7 +68,7 @@ else:
 def print_current():
     if options.diff:
         grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
-                          reverse=options.reverse)
+                          reverse=options.reverse, basenode=options.base)
     else:
         grouped.print_all(skipmodal=options.skipcommon,
                                count=options.count,

confluent_client/bin/confetty

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -89,6 +89,10 @@ except NameError:
 netserver = None
 laststate = {}
 
+try:
+    input = raw_input
+except NameError:
+    pass
 
 class BailOut(Exception):
     def __init__(self, errorcode=0):
@@ -195,9 +199,9 @@ def prompt():
     if os.environ.get('TERM', '') not in ('linux'):
         sys.stdout.write('\x1b]0;confetty: %s\x07' % target)
     try:
-        return raw_input(target + ' -> ')
+        return input(target + ' -> ')
     except KeyboardInterrupt:
-        print ""
+        print("")
         return ""
     except EOFError:  # ctrl-d
         print("exit")
@@ -296,7 +300,7 @@ currchildren = None
 
 def print_result(res):
     if 'errorcode' in res or 'error' in res:
-        print res['error']
+        print(res['error'])
         return
     if 'databynode' in res:
         print_result(res['databynode'])
@@ -309,9 +313,9 @@ def print_result(res):
             attrstr = '%s=%s' % (key, recurse_format(res[key]))
         elif not isinstance(res[key], dict):
             try:
-                print '{0}: {1}'.format(key, res[key])
+                print('{0}: {1}'.format(key, res[key]))
             except UnicodeEncodeError:
-                print '{0}: {1}'.format(key, repr(res[key]))
+                print('{0}: {1}'.format(key, repr(res[key])))
             continue
         elif 'value' in res[key] and res[key]['value'] is not None:
             attrstr = '%s="%s"' % (key, res[key]['value'])
@@ -324,7 +328,7 @@ def print_result(res):
         else:
             sys.stdout.write('{0}: '.format(key))
             if isinstance(res[key], str) or isinstance(res[key], unicode):
-                print res[key]
+                print(res[key])
             else:
                 print_result(res[key])
             continue
@@ -423,10 +427,10 @@ def do_command(command, server):
         for res in session.read(targpath):
             if 'item' in res:  # a link relation
                 if type(res['item']) == dict:
-                    print res['item']["href"]
+                    print(res['item']["href"])
                 else:
                     for item in res['item']:
-                        print item["href"]
+                        print(item["href"])
             else:  # generic attributes to list
                 if 'error' in res:
                     sys.stderr.write(res['error'] + '\n')
@@ -504,6 +508,13 @@ def makecall(callout, args):
             if 'errorcode' in response:
                 exitcode = response['errorcode']
             sys.stderr.write('Error: ' + response['error'] + '\n')
+        if 'databynode' in response:
+            lresponse = response['databynode']
+            for node in lresponse:
+                if 'errorcode' in lresponse[node]:
+                    exitcode = lresponse[node]['errorcode']
+                if 'error' in lresponse[node]:
+                    sys.stderr.write('{0}: Error - {1}\n'.format(node, lresponse[node]['error']))
 
 
 def clearvalues(resource, attribs):
@@ -844,7 +855,7 @@ def server_connect():
         passphrase = os.environ['CONFLUENT_PASSPHRASE']
         session.authenticate(username, passphrase)
     while not session.authenticated:
-        username = raw_input("Name: ")
+        username = input("Name: ")
         passphrase = getpass.getpass("Passphrase: ")
         session.authenticate(username, passphrase)
 
@@ -864,7 +875,7 @@ def main():
     global inconsole
     try:
         server_connect()
-    except EOFError, KeyboardInterrupt:
+    except (EOFError, KeyboardInterrupt) as _:
         raise BailOut(0)
     except socket.gaierror:
         sys.stderr.write('Could not connect to confluent\n')
@@ -922,6 +933,7 @@ def main():
                         updatestatus(data)
                         continue
                     if data is not None:
+                        data = client.stringify(data)
                         if clearpowermessage:
                             sys.stdout.write("\x1b[2J\x1b[;H")
                             clearpowermessage = False
@@ -997,4 +1009,6 @@ if __name__ == '__main__':
         if deadline and os.times()[4] < deadline:
             sys.stderr.write('[Exited early, hit enter to continue]')
             sys.stdin.readline()
+        if errcode == 0:
+            errcode = exitcode
         sys.exit(errcode)

confluent_client/bin/confluent2ansible

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import optparse
 import signal
 import sys

confluent_client/bin/confluent2lxca

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal

confluent_client/bin/confluent2xcat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal
@@ -26,7 +26,7 @@ def lookupdata(data, key):
 
 def main():
     argparser = optparse.OptionParser(
-    usage='''\n       %prog noderange -o ansible.hosts
+    usage='''\n       %prog noderange -o xcatnodes.def
              \n ''')
     argparser.add_option('-o', '--output',
                          help='xCAT stanza file')

confluent_client/bin/nodeattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -50,6 +50,9 @@ argparser.add_option('-c', '--clear', action='store_true',
                      help='Clear attributes')
 argparser.add_option('-p', '--prompt', action='store_true',
                      help='Prompt for attribute values interactively')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Prompt if trying to set attributes on more '
+                          'than specified number of nodes')
 (options, args) = argparser.parse_args()
 
 
@@ -87,6 +90,7 @@ if len(args) > 1:
                     if oneval != twoval:
                         print('Values did not match.')
                 argassign[arg] = twoval
+        session.stop_if_noderange_over(noderange, options.maxnodes)
         exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
     try:
         # setting user output to what the user inputs
@@ -126,6 +130,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodebmcreset

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,8 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Number of nodes to affect before prompting for confirmation')
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,7 +45,7 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'configuration/management_controller/reset', 'reset', key='state', errnodes=errorNodes)   # = 0 if successful
 
 # Determine which nodes were successful and print them
@@ -56,7 +58,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": BMC Reset Successful"
+    print(node + ": BMC Reset Successful")
 
 
 sys.exit(success)

confluent_client/bin/nodeboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -42,6 +42,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                      default=False,
                      help='Request the boot device be persistent rather than '
                           'one time')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to boot, '
+                          'prompting if over the threshold')                        
 
 (options, args) = argparser.parse_args()
 
@@ -66,6 +70,7 @@ else:
     bootmode = 'uefi'
 
 errnodes = set([])
+session.stop_if_noderange_over(noderange, options.maxnodes)
 rc = session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist,

confluent_client/bin/nodeconfig

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -43,7 +43,7 @@ def bailout(msg, code=1):
     sys.exit(code)
 
 
-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '
@@ -68,6 +68,10 @@ argparser.add_option('-r', '--restoredefault', default=False,
                      help='Restore the configuration of the node '
                           'to factory default for given component. '
                           'Currently only uefi is supported')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure, '
+                          'prompting if over the threshold')                            
 (options, args) = argparser.parse_args()
 
 cfgpaths = {
@@ -98,6 +102,8 @@ setmode = None
 assignment = {}
 queryparms = {}
 printsys = []
+printbmc = []
+printallbmc = False
 setsys = {}
 forceset = False
 needval = None
@@ -122,7 +128,7 @@ def _assign_value():
 
 
 def parse_config_line(arguments):
-    global setmode, forceset, key, value, needval, candidate, path, attrib
+    global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
     for param in arguments:
         if param == 'show':
             continue  # forgive muscle memory of pasu users
@@ -154,6 +160,8 @@ def parse_config_line(arguments):
             if setmode != False:
                 bailout('Cannot do set and query in same command')
             if '.' not in param:
+                if param == 'bmc':
+                    printallbmc = True
                 matchedparms = False
                 for candidate in cfgpaths:
                     if candidate.startswith('{0}.'.format(param)):
@@ -172,7 +180,10 @@ def parse_config_line(arguments):
                 if not matchedparms:
                     printsys.append(param)
             elif param not in cfgpaths:
-                printsys.append(param)
+                if param.startswith('bmc.'):
+                    printbmc.append(param.replace('bmc.', ''))
+                else:
+                    printsys.append(param)
             else:
                 path, attrib = cfgpaths[param]
                 path = '/noderange/{0}/{1}'.format(noderange, path)
@@ -185,25 +196,41 @@ if options.batch:
     argfile = open(options.batch, 'r')
     argset = argfile.readline()
     while argset:
-        parse_config_line(shlex.split(argset))
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            parse_config_line(shlex.split(argset))
         argset = argfile.readline()
 else:
     parse_config_line(args[1:])
 session = client.Command()
 rcode = 0
-if options.restoredefault and options.restoredefault.lower() in (
+if options.restoredefault:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    if options.restoredefault.lower() in (
         'sys', 'system', 'uefi', 'bios'):
-    for fr in session.update(
-            '/noderange/{0}/configuration/system/clear'.format(noderange),
-            {'clear': True}):
-        rcode |= client.printerror(fr)
-    sys.exit(rcode)
-elif options.restoredefault:
-    sys.stderr.write(
-        'Unrecognized component to restore defaults: {0}\n'.format(
-            options.restoredefault))
-    sys.exit(1)
+        for fr in session.update(
+                '/noderange/{0}/configuration/system/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    elif options.restoredefault.lower() in (
+        'bmc', 'imm', 'xcc'):
+        for fr in session.update(
+                '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    else:
+        sys.stderr.write(
+            'Unrecognized component to restore defaults: {0}\n'.format(
+                options.restoredefault))
+        sys.exit(1)
 if setmode:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     if options.exclude:
         sys.stderr.write('Cannot use exclude and assign at the same time\n')
         sys.exit(1)
@@ -211,8 +238,12 @@ if setmode:
     attrnamebypath = {}
     for key in assignment:
         if key not in cfgpaths:
-            path = 'configuration/system/all'
-            attrib = key
+            if key.startswith('bmc.'):
+                path = 'configuration/management_controller/extended/all'
+                attrib = key.replace('bmc.', '')
+            else:
+                path = 'configuration/system/all'
+                attrib = key
         else:
             path, attrib = cfgpaths[key]
         if path not in updatebypath:
@@ -244,6 +275,10 @@ else:
                                       NullOpt(), queryparms[path])
         if rc:
             sys.exit(rc)
+    if printbmc or printallbmc:
+        rcode = client.print_attrib_path(
+                    '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
+                    session, printbmc, options, attrprefix='bmc.')
     if printsys or options.exclude:
         if printsys == 'all':
             printsys = []
@@ -252,6 +287,7 @@ else:
         else:
             path = '/noderange/{0}/configuration/system/advanced'.format(
                 noderange)
+        
         rcode = client.print_attrib_path(path, session, printsys,
                                          options)
 sys.exit(rcode)

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -40,6 +40,7 @@ if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
 if options.tile:
+    null = open('/dev/null', 'w')
     nodes = []
     sess = client.Command()
     for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
@@ -60,13 +61,15 @@ if options.tile:
                     confettypath, node)])
         else:
             subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
+            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
             pane += 1
             subprocess.call(
                 ['tmux', 'split', '-h',
                  '{0} -m 5 start /nodes/{1}/console/session'.format(
                      confettypath, node)])
-            subprocess.call(['tmux', 'select-layout', 'tiled'])
+            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
     subprocess.call(['tmux', 'select-pane', '-t', '0'])
+    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
     os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
         os.getpid()))
 else:

confluent_client/bin/nodedefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/noderange/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodediscover

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -69,10 +69,7 @@ def print_disco(options, session, currmac, outhandler, columns):
             record.append(','.join(rawval))
         else:
             record.append(str(rawval))
-    if options.csv:
-        csv.writer(sys.stdout).writerow(record)
-    else:
-        outhandler.add_row(record)
+    outhandler.add_row(record)
 
 
 def process_header(header):
@@ -95,6 +92,8 @@ def process_header(header):
             fields.append('hardwaremanagement.manager')
         elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
             fields.append('net.bmc.ipv4_gateway')
+        elif datum in ('bmc_gateway', 'xcc_gateway', 'imm_gateway'):
+            fields.append('net.bmc.ipv4_gateway')
         elif datum in ('bmcuser', 'username', 'user'):
             fields.append('secret.hardwaremanagementuser')
         elif datum in ('bmcpass', 'password', 'pass'):
@@ -155,6 +154,7 @@ def import_csv(options, session):
         for field in fields:
             if field in unique_fields:
                 unique_data[field] = set([])
+        broken = False
         for record in records:
             currfields = list(fields)
             nodedatum = {}
@@ -171,14 +171,16 @@ def import_csv(options, session):
                 nodedatum[currfield] = datum
             if not datum_complete(nodedatum):
                 sys.exit(1)
-            if not search_record(nodedatum, options, session):
+            if not search_record(nodedatum, options, session) and not broken:
                 blocking_scan(session)
             if not search_record(nodedatum, options, session):
                 sys.stderr.write(
                     "Could not match the following data: " +
                     repr(nodedatum) + '\n')
-                sys.exit(1)
+                broken = True
             nodedata.append(nodedatum)
+    if broken:
+        sys.exit(1)
     for datum in nodedata:
         maclist = search_record(datum, options, session)
         datum = datum_to_attrib(datum)
@@ -204,7 +206,6 @@ def import_csv(options, session):
 
 
 def list_discovery(options, session):
-    outhandler = None
     orderby = None
     if options.fields:
         columns = []
@@ -219,13 +220,12 @@ def list_discovery(options, session):
         for field in columns:
             if options.order.lower() == field.lower():
                 orderby = field
-    if options.csv:
-        csv.writer(sys.stdout).writerow(columns)
-    else:
-        outhandler = client.Tabulator(columns)
+    outhandler = client.Tabulator(columns)
     for mac in list_matching_macs(options, session):
         print_disco(options, session, mac, outhandler, columns)
-    if outhandler:
+    if options.csv:
+        outhandler.write_csv(sys.stdout, orderby)
+    else:
         for row in outhandler.get_table(orderby):
             print(row)
 
@@ -237,8 +237,10 @@ def clear_discovery(options, session):
             else:
                 print(repr(res))
 
-def list_matching_macs(options, session):
+def list_matching_macs(options, session, node=None):
     path = '/discovery/'
+    if node:
+        path += 'by-node/{0}/'.format(node)
     if options.model:
         path += 'by-model/{0}/'.format(options.model)
     if options.serial:
@@ -261,25 +263,25 @@ def list_matching_macs(options, session):
         path += 'by-mac/'
         return [x['item']['href'] for x in session.read(path)]
 
-def assign_discovery(options, session):
+def assign_discovery(options, session, needid=True):
     abort = False
     if options.importfile:
         return import_csv(options, session)
-    if not (options.serial or options.uuid or options.mac):
+    if not options.node:
+        sys.stderr.write("Node (-n) must be specified for assignment\n")
+        abort = True
+    if needid and not (options.serial or options.uuid or options.mac):
         sys.stderr.write(
             "UUID (-u), serial (-s), or ether address (-e) required for "
             "assignment\n")
         abort = True
-    if not options.node:
-        sys.stderr.write("Node (-n) must be specified for assignment\n")
-        abort = True
     if abort:
         sys.exit(1)
-    matches = list_matching_macs(options, session)
+    matches = list_matching_macs(options, session, None if needid else options.node)
     if not matches:
         # Do a rescan to catch missing requested data
         blocking_scan(session)
-        matches = list_matching_macs(options, session)
+        matches = list_matching_macs(options, session, None if needid else options.node)
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)
@@ -294,6 +296,7 @@ def blocking_scan(session):
     list(session.update('/discovery/rescan', {'rescan': 'start'}))
     while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
         time.sleep(0.5)
+    list(session.update('/networking/macs/rescan', {'rescan': 'start'}))
 
 
 def main():
@@ -336,7 +339,7 @@ def main():
     parser.add_option('-o', '--order', dest='order',
                       help='Order output by given field', metavar='ORDER')
     (options, args) = parser.parse_args()
-    if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear'):
         parser.print_help()
         sys.exit(1)
     session = client.Command()
@@ -346,10 +349,12 @@ def main():
         clear_discovery(options, session)
     if args[0] == 'assign':
         assign_discovery(options, session)
+    if args[0] == 'reassign':
+        assign_discovery(options, session, False)
     if args[0] == 'rescan':
         blocking_scan(session)
         print("Rescan complete")
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,10 +33,15 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog [options] noderange [clear]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to clear if clearing log, '
+                          'prompting if over the threshold')            
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -45,11 +50,11 @@ except IndexError:
     sys.exit(1)
 client.check_globbing(noderange)
 deletemode = False
-if len(sys.argv) > 3:
+if len(args) > 2:
     argparser.print_help()
     sys.exit(1)
-if len(sys.argv) == 3:
-    if sys.argv[2] == 'clear':
+if len(args) == 2:
+    if args[1] == 'clear':
         deletemode = True
     else:
         argparser.print_help()
@@ -64,6 +69,8 @@ def format_event(evt):
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
+    if evt.get('log_id', None):
+        retparts.append(evt['log_id'] + ':')
     if 'component_type' in evt and evt['component_type'] is not None:
         dscparts.append(evt['component_type'])
     if 'component' in evt and evt['component'] is not None:
@@ -77,11 +84,15 @@ def format_event(evt):
             pass
         dscparts.append(evttext)
     retparts.append(' - '.join(dscparts))
-    return ' '.join(retparts)
+    msg = evt.get('message')
+    if not msg:
+        msg = ''
+    return ' '.join(retparts) + msg
 
 
 if deletemode:
     func = session.delete
+    session.stop_if_noderange_over(noderange, options.maxnodes)
 else:
     func = session.read
 for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
@@ -98,4 +109,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
             if 'events' in thisdata:
                evtdata = thisdata['events']
                for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print('{0}: {1}'.format(node, format_event(evt)))

confluent_client/bin/nodefirmware

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -59,6 +59,10 @@ argparser = optparse.OptionParser(
           "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='When updating, prompt if more than the specified '
+                          'number of servers will be affected')
+             
 (options, args) = argparser.parse_args()
 upfile = None
 try:
@@ -95,6 +99,7 @@ def get_update_progress(session, url):
 
 def update_firmware(session, filename):
     global exitcode
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     output = sq.ScreenPrinter(noderange, session)
     nodeurls = {}
     filename = os.path.abspath(filename)
@@ -151,7 +156,7 @@ def show_firmware(session):
                     for prefix in inv:
                         firmware_shown = True
                         printfirm(node, prefix, inv[prefix])
-    if not firmware_shown:
+    if not firmware_shown and not exitcode:
         argparser.print_help()
 
 
@@ -163,4 +168,4 @@ try:
         update_firmware(session, upfile)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -36,9 +36,9 @@ import confluent.client as client
 
 argparser = optparse.OptionParser(
     usage='''\n       %prog [options]   \
-             \n       %prog [options] nodegroup [list of attributes]  \
+             \n       %prog [options] nodegroup [list of attributes|all]  \
              \n       %prog [options] nodegroup nodes=value1,value2   \
-             \n       %prog -e nodegroup <attribute names to set>  \             
+             \n       %prog -e nodegroup <attribute names to set>  \
              \n       %prog [options] nodegroup nodes=value1,value2
              \n ''')
 argparser.add_option('-b', '--blame', action='store_true',
@@ -97,7 +97,7 @@ if len(args) > 1:
 
         requestargs=args[1:]
     except Exception as e:
-        print str(e)
+        print(str(e))
 
 if exitcode != 0:
     sys.exit(exitcode)
@@ -123,6 +123,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupdefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/nodegroups/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouplist

@@ -0,0 +1,62 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    showtype='all'
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) > 0:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print(res['item']['href'].replace('/', ''))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodegroupremove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -49,4 +49,4 @@ for r in session.delete('/nodegroups/{0}'.format(noderange)):
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouprename

@@ -0,0 +1,63 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog <group> <new group name>\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) != 2:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.update(
+            '/nodegroups/{0}/attributes/rename'.format(args[0]),
+            {'rename': args[1]}):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print('{0}: {1}'.format(res['oldname'], res['newname']))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodehealth

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,7 +32,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
 (options, args) = argparser.parse_args()

confluent_client/bin/nodeidentify

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -31,7 +31,7 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off|blink]")
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -42,9 +42,6 @@ client.check_globbing(noderange)
 identifystate = None
 if len(sys.argv) > 2:
     identifystate = sys.argv[2]
-else:
-    argparser.print_help()
-    sys.exit(1)
 session = client.Command()
 exitcode = 0
 sys.exit(

confluent_client/bin/nodeinventory

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -34,7 +34,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 filters = []
 
@@ -50,15 +51,17 @@ def print_mem_info(node, prefix, meminfo):
     memdescfmt = '{0}GB PC'
     if meminfo['memory_type'] == 'DDR3 SDRAM':
         memdescfmt += '3-{1} '
-    elif meminfo['memory_type'] == 'DDR4 SDRAM':
+    elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
-    if meminfo['ecc']:
+    if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
     capacity = meminfo['capacity_mb'] / 1024
-    memdescfmt += meminfo['module_type']
+    modtype = meminfo.get('module_type', None)
+    if modtype:
+        memdescfmt += modtype
     memdesc = memdescfmt.format(capacity, meminfo['speed'])
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
@@ -66,10 +69,11 @@ def print_mem_info(node, prefix, meminfo):
     print('{0}: {1} model: {2}'.format(node, prefix, meminfo['model']))
     print('{0}: {1} serial number: {2}'.format(node, prefix,
                                                meminfo['serial']))
-    print('{0}: {1} manufacture date: {2}'.format(node, prefix,
-                                                  meminfo['manufacture_date']))
-    print('{0}: {1} manufacture location: {2}'.format(
-        node, prefix, meminfo['manufacture_location']))
+    if 'manufacture_date' in meminfo:
+        print('{0}: {1} manufacture date: {2}'.format(node, prefix,
+                                                      meminfo['manufacture_date']))
+        print('{0}: {1} manufacture location: {2}'.format(
+            node, prefix, meminfo['manufacture_location']))
 
 exitcode = 0
 
@@ -142,7 +146,7 @@ try:
                                 databynode[node] = {}
                             databynode[node][prefix] = inv
                         else:
-                            print '{0}: {1}: Not Present'.format(node, prefix)
+                            print('{0}: {1}: Not Present'.format(node, prefix))
                     continue
                 info = inv['information']
                 info.pop('board_extra', None)

confluent_client/bin/nodelicense

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo
@@ -36,49 +36,116 @@ exitcode = 0
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][install <file>]")
+          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete licenses from, '
+                          'prompting if over the threshold')                  
 (options, args) = argparser.parse_args()
 upfile = None
+downdir = None
+delete = False
 try:
     noderange = args[0]
     if len(args) > 1:
         if args[1] == 'install':
             upfile = args[2]
-        else:
-            components = ['all']
+        elif args[1] == 'save':
+            downdir = args[2]
+        elif args[1] == 'delete':
+            delete = args[2]
+        elif args[1] != 'list':
+            argparser.print_help()
+            sys.exit(1)        
 except IndexError:
     argparser.print_help()
     sys.exit(1)
 client.check_globbing(noderange)
 
+
 def install_license(session, filename):
     global exitcode
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate requested file {0}\n'.format(
+            filename))
+        sys.exit(404)
     resource = '/noderange/{0}/configuration/' \
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
     for res in session.create(resource, instargs):
-        pass # print(repr(res))
-    show_licenses()
+        for node in res.get('databynode', []):
+            if 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(
+                    node, res['databynode'][node]['error']))
+                sys.exit(res['databynode'][node].get('errorcode', 1))        
+    show_licenses(session)
+
+
+def save_licenses(session, dirname):
+    global exitcode
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/save_licenses'.format(noderange)
+    filename = os.path.abspath(dirname)
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate specified directory {0}\n'.format(
+            filename))
+        sys.exit(404)
+    instargs = {'dirname': filename}
+    for res in session.create(resource, instargs):
+        for node in res.get('databynode', {}):
+            fname = res['databynode'][node].get('filename', None)
+            if fname:
+                print('{0}: Saved license to {1}'.format(node, fname))
+            else:
+                sys.stderr.write('{0}: {1}', node, repr(res['databynode'][node]))
+
 
 def show_licenses(session):
     global exitcode
-    firmware_shown = False
     for res in session.read(
             '/noderange/{0}/configuration/management_controller/licenses/'
             'all'.format(noderange)):
         for node in res.get('databynode', {}):
             for license in res['databynode'][node].get('License', []):
-                print('{0}: {1}'.format(node, license.get('feature',
-                                                          'Unknown')))
+                msg = '{0}: {1}'.format(node, license.get('feature',
+                                                          'Unknown'))
+                if license.get('state', 'Active') != 'Active':
+                    msg += ' ({0})'.format(license['state'])
+                print(msg)
 
 
+def delete_license(session, licname):
+    global exitcode
+    licstodel = []
+    for res in list(session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange))):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                if license.get('feature', None) == licname:
+                    prefix = '/nodes/{0}/configuration/management_controller/licenses/'.format(node)
+                    for currlic in list(session.read(prefix)):
+                        currlic = currlic.get('item', {}).get('href', 'all')
+                        if currlic == 'all':
+                            continue
+                        currname = list(session.read(prefix + currlic))[0]
+                        currname = currname.get('License', [{}])[0].get('feature', None)
+                        if currname == licname:
+                            list(session.delete(prefix + currlic))
+    show_licenses(session)
+
 try:
     session = client.Command()
-    if upfile is None:
-        show_licenses(session)
-    else:
+    if upfile:
         install_license(session, upfile)
+    elif downdir:
+        save_licenses(session, downdir)
+    elif delete:
+        session.stop_if_noderange_over(noderange, options.maxnodes)
+        delete_license(session, delete)
+    else:
+        show_licenses(session)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -61,7 +61,7 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print res['item']['href'].replace('/', '')
+                print(res['item']['href'].replace('/', ''))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodemedia

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -83,7 +83,8 @@ def list_media(noderange, media):
         for node in res.get('databynode', []):
             url = res['databynode'][node].get('url', None)
             name =  res['databynode'][node].get('name', None)
-            if url and not res['databynode'][node].get('secure', False):
+            if (url and not url.startswith('file:') and
+                    not res['databynode'][node].get('secure', False)):
                 name += ' (insecure)'
             if not name:
                 continue
@@ -187,4 +188,4 @@ def main():
         sys.exit(1)
     handler(noderange, media)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodepower

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -37,6 +37,11 @@ argparser = optparse.OptionParser(
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
                      help='Show previous power state')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to change power state, '
+                          'prompting if over the threshold')
+
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -72,4 +77,4 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))

confluent_client/bin/noderemove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -35,6 +35,10 @@ import confluent.client as client
 argparser = optparse.OptionParser(
     usage='''\n       %prog noderange
              \n ''')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete, '
+                          'prompting if over the threshold')      
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
@@ -43,10 +47,11 @@ noderange = args[0]
 client.check_globbing(noderange)
 session = client.Command()
 exitcode = 0
+session.stop_if_noderange_over(noderange, options.maxnodes)
 for r in session.delete('/noderange/{0}'.format(noderange)):
     if 'error' in r:
         sys.stderr.write(r['error'] + '\n')
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/noderename

@@ -0,0 +1,52 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> <newname>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+identifystate = None
+if len(sys.argv) > 2:
+    newname = sys.argv[2]
+else:
+    argparser.print_help()
+    sys.exit(1)
+session = client.Command()
+exitcode = 0
+sys.exit(
+    session.simple_noderange_command(noderange, 'attributes/rename', newname))
+

confluent_client/bin/nodereseat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,10 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to reseat, '
+                          'prompting if over the threshold')   
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,7 +47,7 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'power/reseat', 'reseat', key='reseat', errnodes=errorNodes)   # = 0 if successful
 
 # Determine which nodes were successful and print them
@@ -56,7 +60,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": Reseat successful"
+    print(node + ": Reseat successful")
 
 
 sys.exit(success)

confluent_client/bin/nodersync

@@ -0,0 +1,159 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+import confluent.sortutil as sortutil
+
+
+def run():
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog location noderange:location",
+    )
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run rsync to, '
+                          'prompting if over the threshold')   
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of nodes to concurrently rsync')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2 or ':' not in args[-1]:
+        argparser.print_help()
+        sys.exit(1)
+    concurrentprocs = options.count
+    noderange, targpath = args[-1].split(':', 1)
+    client.check_globbing(noderange)
+    c = client.Command()
+    cmdstr = ' '.join(args[:-1])
+    cmdstr = 'rsync -av --info=progress2 ' + cmdstr
+    cmdstr += ' {node}:' + targpath
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+    c.stop_if_noderange_over(noderange, options.maxnodes)
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
+            {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
+            cmdv = shlex.split(cmd)
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    nodeerrs = {}
+    pernodeout = {}
+    pernodefile = {}
+    output = sq.ScreenPrinter(noderange, c)
+
+    while all:
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.read(1)
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = ''
+                        pernodeout[node] += client.stringify(data)
+                        if '\n' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\n', 1)
+                            if currout:
+                                pernodefile[node] = os.path.basename(currout)
+                        if '\r' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\r', 1)
+                            if currout:
+                                currout = currout.split()
+                                try:
+                                    currout = currout[1]
+                                    output.set_output(node, '{0}:{1}'.format(pernodefile[node], currout))
+                                except IndexError:
+                                    pernodefile = currout[0]
+                                    pass
+                    else:
+                        output.set_output(node, 'error!')
+                        if node not in nodeerrs:
+                            nodeerrs[node] = ''
+                        nodeerrs[node] += client.stringify(data)
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if node not in nodeerrs:
+                            output.set_output(node, 'complete')
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    for node in nodeerrs:
+        for line in nodeerrs[node].split('\n'):
+            sys.stderr.write('{0}: {1}\n'.format(node, line))
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/noderun

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -39,11 +39,17 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run the command with, '
+                          'prompting if over the threshold')                           
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -61,7 +67,7 @@ def run():
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -69,7 +75,9 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -93,7 +101,11 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        data = client.stringify(data)
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -107,7 +119,11 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodesensors

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -114,7 +114,8 @@ def sensorpass(showout=True, appendtime=False):
                         continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
-                            sensedata['states'].remove(redundant_state)
+                            if sensedata.get('states', False):
+                                sensedata['states'].remove(redundant_state)
                         except ValueError:
                             pass
                     resultdata[node][sensedata['name']] = sensedata
@@ -132,11 +133,12 @@ def sensorpass(showout=True, appendtime=False):
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):
                             showval += sensedata['units']
-                        if sensedata['health'] != 'ok':
+                        if sensedata.get('health', 'ok') != 'ok':
                             datadescription = [sensedata['health']]
                         else:
                             datadescription = []
-                        datadescription.extend(sensedata['states'])
+                        if sensedata.get('states', False):
+                            datadescription.extend(sensedata['states'])
                         if datadescription:
                             if showval == '':
                                 showval += u' {0}'.format(
@@ -147,8 +149,11 @@ def sensorpass(showout=True, appendtime=False):
                         if appendtime:
                             showval += ' @' + time.strftime(
                                 '%Y-%m-%dT%H:%M:%S')
-                        print(u'{0}: {1}:{2}'.format(
-                            node, sensedata['name'], showval).encode('utf-8'))
+                        printval = u'{0}: {1}:{2}'.format(
+                            node, sensedata['name'], showval)
+                        if not isinstance(printval, str):
+                            printval = printval.encode('utf-8')
+                        print(printval)
                         sys.stdout.flush()
     return resultdata
 
@@ -156,7 +161,7 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            if showtime.is_integer():
+            if isinstance(showtime, int):
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
             else:
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
@@ -183,6 +188,7 @@ def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
             except KeyError:
                 rowdata.append('N/A')
         csvwriter.writerow(rowdata)
+        sys.stdout.flush()
 
 
 def main():
@@ -196,7 +202,11 @@ def main():
             orderedsensors.append(name)
         orderedsensors.sort()
         for name in orderedsensors:
-            headernames.append(sensorheaders[name].encode('utf-8'))
+            headername = sensorheaders[name]
+            if (not isinstance(headername, str) and
+                    not isinstance(headername, bytes)):
+                headername = headername.encode('utf-8')
+            headernames.append(headername)
     if options.csv:
         linebyline = False
         csvwriter = csv.writer(sys.stdout)
@@ -220,7 +230,7 @@ def main():
                     sys.exit(exitcode)
             sleeptime = nextstart - os.times()[4]
             if sleeptime > 0:
-                time.sleep(nextstart - os.times()[4])
+                time.sleep(sleeptime)
     else:
         if options.csv:
             format_csv(csvwriter, orderedsensors, resdata, showtime=False)

confluent_client/bin/nodesetboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -43,7 +43,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 argparser.add_option('-u', '--uefi', dest='uefi', action='store_true',
                      default=True,
                      help='Request UEFI style boot (rather than BIOS)')
-
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to modify next boot device, '
+                          'prompting if over the threshold')                        
 (options, args) = argparser.parse_args()
 
 try:
@@ -63,6 +66,7 @@ if options.biosmode:
     bootmode = 'bios'
 else:
     bootmode = 'uefi'
+session.stop_if_noderange_over(noderange, options.maxnodes)
 sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist))

confluent_client/bin/nodeshell

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -39,11 +39,17 @@ import confluent.sortutil as sortutil
 def run():
 
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run remote ssh command to, '
+                          'prompting if over the threshold')   
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -53,7 +59,7 @@ def run():
         sys.exit(1)
     client.check_globbing(args[0])
     concurrentprocs = options.count
-    c = client.Command()
+    c = client.Command()    
     cmdstr = " ".join(args[1:])
 
     currprocs = 0
@@ -62,7 +68,7 @@ def run():
     pendingexecs = deque()
     exitcode = 0
 
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -70,7 +76,9 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+                cmd = cmd.encode('utf-8')
             cmdv = ['ssh', node, cmd]
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -94,7 +102,11 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        data = client.stringify(data)
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -108,7 +120,11 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                line = client.stringify(line)
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodestorage

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -63,10 +63,16 @@ def _print_cfg(scfg):
             sys.stderr.write(e['error'] + '\n')
             exitcode = e.get('errorcode', 1)
         for node in e.get('databynode', {}):
+            curr = e['databynode'][node]
+            if 'error' in curr:
+                if 'no available drives' in curr['error']:
+                    curr['error'] += ' (drives must be in unconfigured state to be available, they must not be in jbod or online state)'
+                sys.stderr.write('{0}: {1}\n'.format(node, curr['error']))
+                exitcode = curr.get('errorcode', 1)
+                continue
             if node not in storagebynode:
                 storagebynode[node] = {'disks': [], 'arrays': [],
                                        'volumes': []}
-            curr = e['databynode'][node]
             storagebynode[node][curr['type'] + 's'].append(curr)
     for node in storagebynode:
         for disk in sorted(storagebynode[node]['disks'],
@@ -108,6 +114,7 @@ def createstorage(noderange, options, args):
         sys.stderr.write('-r and -d are required arguments to create array\n')
         sys.exit(1)
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     names = options.name
     if names is None:
         names = ''.join(args)
@@ -115,6 +122,8 @@ def createstorage(noderange, options, args):
              'name': names}
     if options.size:
         parms['size'] = options.size
+    if options.stripsizes:
+        parms['stripsizes'] = options.stripsizes
     _print_cfg(session.create(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names), parms))
@@ -130,6 +139,7 @@ def deletestorage(noderange, options, args):
     else:
         names = options.name
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for rsp in session.delete(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names)):
@@ -147,13 +157,27 @@ def deletestorage(noderange, options, args):
             print(repr(rsp))
 
 
-def setstorage(noderange, options, args):
-    pass
+def setdisk(noderange, options, args):
+    if options.disks is None:
+        if len(args):
+            names = args.pop(0)
+        else:
+            sys.stderr.write('-d is required to indicate disk to modify\n')
+            sys.exit(1)
+    else:
+        names = options.disks
+    if not len(args) or args[0] not in ('hotspare', 'jbod', 'unconfigured'):
+        sys.stderr.write('diskset requires valid state as argument (hotspare, jbod, unconfigured)\n')
+        sys.exit(1)
+    session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    scfg = session.update('/noderange/{0}/configuration/storage/disks/{1}'.format(noderange, names), {'state': args[0]})
+    _print_cfg(scfg)
 
 funmap = {
     'create': createstorage,
     'show': showstorage,
-    'set': setstorage,
+    'diskset': setdisk,
     'delete': deletestorage,
     'rm': deletestorage,
 }
@@ -162,7 +186,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|set|delete]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',
@@ -183,7 +207,17 @@ def main():
                               'naming volumes, or selecting a volume for '
                               'delete.  Default behavior is to use '
                               'implementation provided default names.')
+    argparser.add_option('-z', '--stripsizes', type='str',
+                         help='Comma separated list of stripsizes to use when creating volumes. '
+                            'This value is in kilobytes.  The default behavior is to allow the '
+                            'storage controller to decide.')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure storage on, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
+    if len(args) == 1:
+        args.append('show')
     try:
         noderange = args[0]
         operation = args[1]
@@ -200,4 +234,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodesupport

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -64,7 +64,7 @@ def printerror(res, node=None):
 
 
 
-def download_servicedata(noderange, media):
+def download_servicedata(noderange, media, options):
     global exitcode
     session = client.Command()
     output = sq.ScreenPrinter(noderange, session)
@@ -73,6 +73,7 @@ def download_servicedata(noderange, media):
     upargs = {'filename': filename}
     noderrs = {}
     nodeurls = {}
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -114,11 +115,17 @@ class OptParser(optparse.OptionParser):
 
 def main():
     argparser = OptParser(
-        usage="Usage: %prog <noderange> [servicedata] "
+        usage="Usage: %prog <noderange> servicedata "
               "<filename>",
         epilog='\nservicedata will save service data to the given '
-               'directory\n'
+               'directory.  It is saved to the location on the relevant '
+               'management server (the confluent server if running remote, '
+               'and the collective.manager if in collective)\n'
                '\n\nSee `man %prog` for more info.\n')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to download diagnostic data from, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
     media = None
     try:
@@ -140,6 +147,6 @@ def main():
     except KeyError:
         argparser.print_help()
         sys.exit(1)
-    handler(noderange, media)
+    handler(noderange, media, options)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/stats

@@ -0,0 +1,167 @@
+#!/usr/bin/python2
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import fcntl
+import io
+import numpy as np
+
+import os
+import sixel
+import subprocess
+import sys
+
+class DumbWriter(sixel.SixelWriter):
+    def restore_position(self, output):
+        return
+
+
+def plot(gui, output, plotdata, bins):
+    import matplotlib as mpl
+    if not gui:
+        mpl.use('Agg')
+    import matplotlib.pyplot as plt
+    n, bins, patches = plt.hist(plotdata, bins)
+    plt.show()
+    if not gui:
+        if output:
+            tdata = output
+        else:
+            tdata = io.BytesIO()
+        plt.savefig(tdata)
+    if not gui and not output:
+        writer = DumbWriter()
+        writer.draw(tdata)
+    return n, bins
+
+def textplot(plotdata, bins):
+    n, bins = np.histogram(plotdata, bins)
+    labels = []
+    for bin in bins:
+        labels.append('{0:0.1f}'.format(bin))
+    width = 80
+    # Since this will be primarily piped into, hard to get 
+    # terminal width
+    labelwidth = 0
+    for lab in labels:
+        if len(lab) > labelwidth:
+            labelwidth = len(lab)
+    width -= (labelwidth) + 1
+    labelfmt = '{{0:>{0}s}}|'.format(labelwidth)
+    maxn = 0.0
+    for lgth in n:
+        if lgth > maxn:
+            maxn = float(lgth)
+    for i in range(len(n)):
+        print(labelfmt.format(labels[i]) + '=' * int(np.round((n[i]/maxn) * width)))
+    return n, bins
+
+histogram = False
+aparser = argparse.ArgumentParser(description='Quick access to common statistics')
+aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
+aparser.add_argument('-d', default=None, help='Value used to separate columns')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
+aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
+aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
+aparser.add_argument('-t', default=False, action='store_true', help='Output a histogram in text format')
+aparser.add_argument('-v', default=False, action='store_true', help='Attempt to list nodes relevant to each histogram bar (requires -s, -o, or -t)')
+aparser.add_argument('-b', type=int, default=10, help='Number of bins to use in histogram (default is 10)')
+args = aparser.parse_args(sys.argv[1:])
+plotdata = []
+headlines = int(args.s)
+while headlines >= 0:
+    data = sys.stdin.readline()
+    headlines -= 1
+if args.d:
+    delimiter = args.d
+else:
+    if '\t' in data:
+        delimiter = '\t'
+    elif ' ' in data:
+        delimiter = ' '
+    elif ',' in data:
+        delimiter = ','
+    else:
+        delimiter = ' '  # handle single column
+data = list(csv.reader([data], delimiter=delimiter))[0]
+nodebydatum = {}
+idx = args.c - 1
+autoidx = False
+while data:
+    node = None
+    if ':' in data[0]:
+        node, data[0] = data[0].split(':', 1)
+    else:
+        node = data[0]
+    if idx == -1 and not autoidx:
+        while not autoidx:
+            try:
+                datum = float(data[idx])
+            except ValueError:
+                idx -= 1
+                continue
+            except IndexError:
+                sys.stderr.write('Unable to identify a numerical column\n')
+                sys.exit(1)
+            autoidx = True
+    else:
+        datum = float(data[idx])
+    if node:
+        if datum in nodebydatum:
+            nodebydatum[datum].add(node)
+        else:
+            nodebydatum[datum] = set([node])
+    plotdata.append(datum)
+    data = sys.stdin.readline()
+    data = list(csv.reader([data], delimiter=delimiter))[0]
+n = None
+if args.g or args.o or args.x:
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+if args.t:
+    n, bins = textplot(plotdata, bins=args.b)
+print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))
+if args.v and n is not None and nodebydatum:
+    print('')
+    currbin = bins[0]
+    bins = bins[1:]
+    currbinmembers = []
+    for datum in sorted(nodebydatum):
+        if datum > bins[0]:
+           nextbin = None
+           endbin = bins[0]
+           while len(bins) and bins[0] < datum:
+                nextbin = bins[0]
+                bins = bins[1:]
+           if not nextbin:
+               nextbin = np.max(plotdata)
+           print('Entries between {0} and {1}'.format(currbin, endbin))
+           currbin = nextbin
+           print('-' * 80)
+           print(','.join(sorted(currbinmembers)))
+           print('')
+           print('')
+           currbinmembers = []
+        for node in nodebydatum[datum]:
+            currbinmembers.append(node)
+    if currbinmembers:
+       print('Entries between {0} and {1}'.format(currbin, np.max(plotdata)))
+       print('-' * 80)
+       print(','.join(sorted(currbinmembers)))
+       print('')
+       print('')

confluent_client/builddeb

@@ -0,0 +1 @@
+../confluent_server/builddeb

confluent_client/confluent/client.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import anydbm as dbm
+try:
+    import anydbm as dbm
+except ImportError:
+    import dbm
+import csv
 import errno
 import fnmatch
 import hashlib
@@ -35,6 +39,19 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
 }
 
+try:
+    input = raw_input
+except NameError:
+    pass
+
+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
 
 class Tabulator(object):
     def __init__(self, headers):
@@ -70,6 +87,23 @@ class Tabulator(object):
             for row in self.rows:
                 yield fmtstr.format(*row)
 
+    def write_csv(self, output, order=None):
+        output = csv.writer(output)
+        output.writerow(self.headers)
+        i = 0
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            i = i + 1
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                output.writerow(row)
+        else:
+            for row in self.rows:
+                output.writerow(row)
+
 
 def printerror(res, node=None):
     exitcode = 0
@@ -86,7 +120,10 @@ def printerror(res, node=None):
 
 
 def cprint(txt):
-    print(txt)
+    try:
+        print(txt)
+    except UnicodeEncodeError:
+        print(txt.encode('utf8'))
     sys.stdout.flush()
 
 def _parseserver(string):
@@ -118,6 +155,8 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+        elif self.serverloc == '/var/run/confluent/api.sock':
+            raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
         tlvdata.recv(self.connection)
@@ -184,7 +223,7 @@ class Command(object):
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None, errnodes=None, **kwargs):
+                                 key=None, errnodes=None, promptover=None, **kwargs):
         try:
             self._currnoderange = noderange
             rc = 0
@@ -200,6 +239,7 @@ class Command(object):
                         noderange, resource)):
                     rc = self.handle_results(ikey, rc, res, errnodes)
             else:
+                self.stop_if_noderange_over(noderange, promptover)
                 kwargs[ikey] = input
                 for res in self.update('/noderange/{0}/{1}'.format(
                         noderange, resource), kwargs):
@@ -209,6 +249,33 @@ class Command(object):
         except KeyboardInterrupt:
             cprint('')
             return 0
+    
+    def stop_if_noderange_over(self, noderange, maxnodes):
+        if maxnodes is None:
+            return
+        nsize = self.get_noderange_size(noderange)
+        if nsize > maxnodes:
+            if nsize == 1:
+                nodename = list(self.read(
+                    '/noderange/{0}/nodes/'.format(noderange)))[0].get('item', {}).get('href', None)
+                nodename = nodename[:-1]
+                p = input('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+            else:
+                p = input('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+            if p.lower() != 'y':
+                sys.stderr.write('Aborting at user request\n')
+                sys.exit(1)
+                raise Exception("Aborting at user request")
+        
+
+    def get_noderange_size(self, noderange):
+        numnodes = 0
+        for node in self.read('/noderange/{0}/nodes/'.format(noderange)):
+            if node.get('item', {}).get('href', None):
+                numnodes += 1
+            else:
+                raise Exception("Error trying to size noderange {0}".format(noderange))
+        return numnodes
 
     def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs):
         try:
@@ -302,13 +369,14 @@ class Command(object):
         if knownhosts:
             certdata = self.connection.getpeercert(binary_form=True)
             fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
+            fingerprint = fingerprint.encode('utf-8')
             hostid = '@'.join((port, server))
             khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
             if hostid in khf:
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    replace = raw_input(
+                    replace = input(
                         "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
@@ -361,8 +429,12 @@ def printattributes(session, requestargs, showtype, nodetype, noderange, options
     path = '/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)
     return print_attrib_path(path, session, requestargs, options)
 
+def _sort_attrib(k):
+    if isinstance(k[1], dict) and k[1].get('sortid', None) is not None:
+        return k[1]['sortid']
+    return k[0]
 
-def print_attrib_path(path, session, requestargs, options, rename=None):
+def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
     seenattributes = set([])
     for res in session.read(path):
@@ -371,9 +443,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
-            for attr, val in sorted(
-                    res['databynode'][node].items(),
-                    key=lambda (k, v): v.get('sortid', k) if isinstance(v, dict) else k):
+            for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
                     continue
@@ -385,6 +455,8 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                     printattr = rename.get(attr, attr)
                 else:
                     printattr = attr
+                if attrprefix:
+                    printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
                 if show_attr(attr, requestargs, seenattributes, options):
                     if 'value' in currattr:
@@ -444,9 +516,12 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                                 (currattr.get('default', None) is not None and
                                 currattr.get('value', None) is not None and
                                 currattr['value'] != currattr['default'])):
+                            cval = ','.join(currattr['value']) if isinstance(
+                                currattr['value'], list) else currattr['value']
+                            dval = ','.join(currattr['default']) if isinstance(
+                                currattr['default'], list) else currattr['default']
                             cprint('{0}: {1}: {2} (Default: {3})'.format(
-                                node, printattr, currattr['value'],
-                                currattr['default']))
+                                node, printattr, cval, dval))
                     else:
 
                         try:
@@ -455,11 +530,14 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                             details = False
                         if details:
                             if currattr.get('help', None):
-                                attrout += ' (Help: {0})'.format(
-                                    currattr['help'].encode('utf-8'))
+                                attrout += u' (Help: {0})'.format(
+                                    currattr['help'])
                             if currattr.get('possible', None):
-                                attrout += ' (Choices: {0})'.format(
-                                    ','.join(currattr['possible']))
+                                try:
+                                    attrout += u' (Choices: {0})'.format(
+                                        ','.join(currattr['possible']))
+                                except TypeError:
+                                    pass
                         cprint(attrout)
     if not exitcode:
         if requestargs:
@@ -629,4 +707,4 @@ def check_globbing(noderange):
                 'bash or change directories such that there is no filename '
                 'that would conflict.'
                 '\n'.format(noderange))
-            sys.exit(1)
+            sys.exit(1)

confluent_client/confluent/screensqueeze.py

@@ -44,6 +44,8 @@ class ScreenPrinter(object):
         self.fieldwidth = maxlen + textlen + 1  # 1 for column
 
     def set_output(self, node, text):
+        if self.nodeoutput[node] == text:
+            return
         self.nodeoutput[node] = text
         if len(text) >= self.textlen:
             self.textlen = len(text) + 1

confluent_client/confluent/textgroup.py

@@ -169,13 +169,18 @@ class GroupedData(object):
         output.flush()
 
     def print_deviants(self, output=sys.stdout, skipmodal=False, reverse=False,
-                       count=False):
+                       count=False, basenode=None):
         self.generate_byoutput()
         modaloutput = None
         ismodal = True
         revoutput = []
+        if basenode:
+            for checkout in self.byoutput:
+                if basenode in self.byoutput[checkout]:
+                    modaloutput = checkout
         for outdata in sorted(
-                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                self.byoutput, key=lambda x: [0 if modaloutput == x else 1,
+                                              0 - len(self.byoutput[x]),
                                               humanify_nodename(
                                                   self.get_group_text(
                                                       self.byoutput[x]

confluent_client/confluent/tlvdata.py

@@ -25,6 +25,11 @@ try:
 except NameError:
     unicode = str
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 def decodestr(value):
     ret = None
     try:
@@ -34,11 +39,13 @@ def decodestr(value):
             ret = value.decode('cp437')
         except UnicodeDecodeError:
             ret = value
+    except AttributeError:
+        return value
     return ret
 
 def unicode_dictvalues(dictdata):
     for key in dictdata:
-        if isinstance(dictdata[key], str):
+        if isinstance(dictdata[key], bytes):
             dictdata[key] = decodestr(dictdata[key])
         elif isinstance(dictdata[key], datetime):
             dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -49,7 +56,7 @@ def unicode_dictvalues(dictdata):
 
 
 def _unicode_list(currlist):
-    for i in xrange(len(currlist)):
+    for i in range(len(currlist)):
         if isinstance(currlist[i], str):
             currlist[i] = decodestr(currlist[i])
         elif isinstance(currlist[i], dict):
@@ -64,7 +71,7 @@ def send(handle, data):
             data = data.encode('utf-8')
         except AttributeError:
             pass
-    if isinstance(data, str) or isinstance(data, unicode):
+    if isinstance(data, bytes) or isinstance(data, unicode):
         # plain text, e.g. console data
         tl = len(data)
         if tl == 0:

confluent_client/confluent_client.spec.tmpl

@@ -24,10 +24,18 @@ a confluent server.
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+%if "%{dist}" == ".el8"
+python3 setup.py build
+%else
+python2 setup.py build
+%endif
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%if "%{dist}" == ".el8"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%endif
 
 
 %clean

confluent_client/confluent_env.sh

@@ -39,14 +39,15 @@ alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
 alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
+# Do not continue for non-bash shells, the rest of this sets up bash completion functions
+[ -z "$BASH_VERSION" -o -z "$PS1" ] && return
 
 
 _confluent_get_args()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
+    NUMARGS=$((COMP_CWORD+1))
+    if [ "${COMP_WORDS[COMP_CWORD]}" == '' ]; then
         CMPARGS+=("")
     fi
     GENNED=""
@@ -73,7 +74,7 @@ function _confluent_generic_completion()
 {
     _confluent_get_args
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
     fi
     if [ $NUMARGS -lt 3 ]; then
         _confluent_nr_completion
@@ -82,7 +83,7 @@ function _confluent_generic_completion()
 }
 _confluent_nodeidentify_completion()
 {
-    COMP_CANDIDATES=("on,off -h")
+    COMP_CANDIDATES=("on,off,blink -h")
     _confluent_generic_completion
 }
 
@@ -109,7 +110,7 @@ _confluent_nodemedia_completion()
         return
     fi
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -lt 3 ]; then
@@ -122,7 +123,7 @@ _confluent_nodefirmware_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[2]} == 'update' ]; then
@@ -140,7 +141,7 @@ _confluent_nodeshell_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -c -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -c -- ${COMP_WORDS[COMP_CWORD]}))
         return
     fi
     if [ $NUMARGS -gt 3 ]; then
@@ -158,7 +159,7 @@ _confluent_nodelicense_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "install list" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
@@ -166,6 +167,11 @@ _confluent_nodelicense_completion()
         COMPREPLY=()
         return
     fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'save' ]; then
+        compopt -o dirnames
+        COMPREPLY=()
+        return
+    fi
     if [ $NUMARGS -lt 3 ]; then
          _confluent_nr_completion
          return
@@ -176,7 +182,7 @@ _confluent_nodesupport_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'servicedata' ]; then
@@ -190,47 +196,49 @@ _confluent_nodesupport_completion()
     fi
 }
 
+_confluent_nodeattrib_completion()
+{
+    COMP_CANDIDATES=$(nodeattrib '~.>1' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_generic_completion
+}
+
+
 _confluent_nn_completion()
 {
     _confluent_get_args
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_nr_completion()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
-    fi
     _confluent_get_args
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    #COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_ng_completion()
 {
@@ -238,19 +246,20 @@ _confluent_ng_completion()
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
-complete -F _confluent_nr_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodegroupattrib
 complete -F _confluent_nr_completion nodebmcreset
 complete -F _confluent_nodesetboot_completion nodeboot
 complete -F _confluent_nr_completion nodeconfig
@@ -262,7 +271,7 @@ complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth
 complete -F _confluent_nodeidentify_completion nodeidentify
 complete -F _confluent_nr_completion nodeinventory
-complete -F _confluent_nr_completion nodelist
+complete -F _confluent_nodeattrib_completion nodelist
 complete -F _confluent_nodemedia_completion nodemedia
 complete -F _confluent_nodepower_completion nodepower
 complete -F _confluent_nr_completion noderemove

confluent_client/doc/man/collective.ronn

@@ -0,0 +1,55 @@
+collective(1) -- Check and manage a confluent collective
+==============================
+
+## SYNOPSIS
+
+`collective invite <server>`  
+`collective join <server> [-i TOKEN]`  
+`collective show`  
+`collective gencert`  
+
+
+## DESCRIPTION
+
+**collective** helps manage the collective mode of confluent, where multiple
+confluent servers are linked together to act as one.  For example, the procedure to set up
+a collective to run on three servers called mgt1, mgt2, and mgt3, first install and start
+confluent as usual on the three servers.  On mgt1, run `collective invite mgt2` and an
+invitation token will be output.  On mgt2, either run `collective join mgt1` to paste
+the token interactively, or `collective join mgt1 -i <token>`.  At this point, either
+mgt1 or mgt2 can bring in mgt3.  For example on mgt2 run `collective invite mgt3` and
+on mgt3 run `collective join mgt2 -i <token>`
+
+This can be linked together in the following manner with ssh:
+on mgt1:
+  `# ssh mgt2 collective join mgt1 -i $(collective invite mgt2)`
+
+Note that a collective is only redundant with 3 or more members.  The collective
+will function so long as more than half of the members are online.  A collective
+of two members is supported, but without redundancy.
+
+Also note that the collective leader role is dynamic, but has no impact on interacting
+with confluent.  It is merely an internal role that can dynamically change depending
+on circumstances.
+
+## OPTIONS
+
+ * `-i`:
+    Provide the token as an argument rather than interactively.
+   
+## EXAMPLES
+ * Inviting a server called mgt2:
+  `# collective invite mgt2`  
+  `bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+
+* On mgt2, joining mgt1:
+  `# collective join mgt1 -i bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+  `Success`  
+
+* Showing the collective state:
+  `# collective show`  
+  `Quorum: True`  
+  `Leader: mgt1`  
+  `Active collective members:`  
+  `  mgt2`  
+

confluent_client/doc/man/nodeattrib.ronn

@@ -23,6 +23,9 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 If the word all is specified, then all available attributes are given.
 Omitting any attribute name or the word 'all' will display only attributes
 that are currently set.
@@ -88,6 +91,12 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
     `n1: console.method: `  
     `n2: console.method: `  
 
+* List all switches that a node is described as connected to:
+    `# nodeattrib d1 net.*switch`  
+    `d1: net.mgt.switch: mgtswitch1`  
+    `d1: net.pxe.switch: pxeswitch1`  
+    `d1: net.switch:`  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeattribexpressions.ronn

@@ -9,6 +9,18 @@ expression to generate the value.
 An expression will contain some directives wrapped in `{}` characters. Within
 `{}` are a number of potential substitute values and operations.
 
+Note that syntax of expressions can have overlap with the shell syntax.
+For example:
+
+`$ echo (n2)`  
+`-bash: syntax error near unexpected token `n2'`  
+
+In such a case, it helps to quote the expression to allow it to be passed:
+
+`$ echo '(n2)'`  
+`(n2)`  
+
+
 The most common operation is to extract a number from the nodename. These 
 values are available as n1, n2, etc. So for example attributes for a node named 
 b1o2r3u4 would have {n1} as 1, {n2} as 2, {n3} as 3, and {n4} as 4. 

confluent_client/doc/man/nodeconfig.ronn

@@ -3,8 +3,8 @@ nodeconfig(8) -- Show or change node configuration
 
 ## SYNOPSIS
 
-`nodeconfig <noderange> [options] [<configuration>..]`
-`nodeconfig <noderange> [options] [<configuration=value>..]`
+`nodeconfig <noderange> [options] [<configuration>..]`  
+`nodeconfig <noderange> [options] [<configuration=value>..]`  
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodefirmware.ronn

@@ -13,7 +13,11 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 will retrieve all firmware, but can be directed to fetch specific firmware by
 calling out the name of the firmware (e.g. uefi or xcc) or request reading only
 core firmware firmware by using the word 'core', which is generally a quicker
-operation.
+operation.  Different hardwaremanagement.method indicated plugins may have
+different capabilities available.  For example, the 'core' distinction may
+not be relevant to redfish.  Additionally, the Lenovo XCC makes certain
+information available over IPMI that is not otherwise available (for example
+the FPGA version where applicable).
 
 In the update form, it accepts a single file and attempts to update it using
 the out of band facilities.  Firmware updates can end in one of three states:

confluent_client/doc/man/nodegrouplist.ronn

@@ -0,0 +1,14 @@
+nodegrouplist(8) -- List the defined confluent nodegroups
+===================================================================
+
+## SYNOPSIS
+
+`nodegrouplist`  
+
+## DESCRIPTION
+
+`nodegrouplist` lists the currently defined groups in confluent.
+
+## SEE ALSO
+
+nodeattrib(8), nodeattribexpressions(5), nodegroupattrib(8)

confluent_client/doc/man/nodeidentify.ronn

@@ -3,7 +3,7 @@ nodeidentify(8) -- Control the identify LED of confluent nodes
 
 ## SYNOPSIS
 
-`nodidentify <noderange> [on|off]`  
+`nodidentify <noderange> [on|off|blink]`  
 
 ## DESCRIPTION
 
@@ -13,6 +13,7 @@ options are supported:
 
 * `on`: Turn on the identify LED
 * `off`: Turn off the identify LED
+* `blink`: Set the identify LED to blink (when supported by the system)
 
 ## EXAMPLES:
 

confluent_client/doc/man/nodelicense.ronn

@@ -3,9 +3,15 @@ nodelicense(8) -- Manage license keys on BMC
 
 ## SYNOPSIS
 
-`nodelicense <noderange> [list|install <filename>]`
+`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
 
 ## DESCRIPTION
 
-`nodelicense` shows and installs license keys on supported BMCs
+`nodelicense` manages license keys on supported BMCs. Without an argument, the command
+lists currently installed license.  Using `delete` will remove the specified license name
+from th eBMC.  The `save` subcommand will take the passed directory (which may be in the form
+of /path/to/{node}/ to have the node name substituted for each node) and back up installed licenses
+to that directory. The `install` command will take the specified filename and install.  The filename
+argument may be of the form  xcc_fod_0034_7X21{id.serial}.key to have the serial number substituted
+to allow unique licenses to be specified in a single command.
 

confluent_client/doc/man/nodelist.ronn

@@ -17,6 +17,9 @@ displayed.  If `-b` is specified, it will also display information on
 how inherited and expression based attributes are defined.  There is more
 information on node attributes in nodeattributes(5) man page.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 ## OPTIONS
 
 * `-b`, `--blame`:

confluent_client/doc/man/noderange.ronn

@@ -36,10 +36,13 @@ Also, regular expressions may be used to indicate nodes with names matching cert
 The other major noderange primitive is indicating nodes by some attribute value:
 `location.rack=7`
 
+The attribute name may use a wildcard:
+`net.*switch=switch1`
+
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
 
-Exclusions can be done by prepending a ‘-‘ before a portion of a noderange:
+Exclusions can be done by prepending a '-' before a portion of a noderange:
 `rack1,-n2`
 `compute,-rack1`
 `compute,-location.row=12`
@@ -51,7 +54,7 @@ To indicate nodes that match multiple selections at once (set intersection), the
 For complex expressions, () may be used to indicate order of expanding the noderange to be explicit
 `rack1,-(console.logging=full@compute)`
 
-Noderange syntax can also indicate ‘pagination’, or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
+Noderange syntax can also indicate 'pagination', or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
 `rack1>3<6`
 
 The above would show the seventh through ninth nodes of the rack1 group. Like all other noderange operations, this may be combined with any of the above, but must appear as the very last operation. Ordering is done with a natural sort.

confluent_client/doc/man/nodestorage.ronn

@@ -0,0 +1,65 @@
+nodestorage(8) -- Examine/Modify storage configuration of a node
+============================================
+
+## SYNOPSIS
+
+`nodestorage <noderange> [show|create|delete] [options]`  
+
+## DESCRIPTION
+
+`nodestorage` provides access to the remote storage configuration of
+the noderange.
+
+## OPTIONS
+* `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
+  RAID level to use when creating an array
+
+* `-d` **DISKS**, `--disks`=**DISKS**:
+   Comma separated list of disks to use, or the word "rest" to
+   indicate use of all available disks
+
+* `-s` **SIZE**, `--size`=**SIZE**:
+  Comma separated list of sizes to use when creating
+  volumes.  The sizes may be absolute size (e.g. 16gb),
+  percentage (10%) or the word "rest" to use remaining
+  capacity, default behavior is to use all capacity to
+  make a volume
+* `-n` **NAME**, `--name`=**NAME**:
+  Comma separated list of names to use when naming
+  volumes, or selecting a volume for delete.  Default
+  behavior is to use implementation provided default
+
+## EXAMPLES
+
+* Deleting the volume `somedata`:
+    `$ nodestorage d5 delete somedata`  
+    `Deleted: somedata`  
+
+* Creating a raid5 of 4 disks and a volume named `somedata`:
+    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata`  
+    `d5: Volume somedata: Size: 1.905 TB`  
+    `d5: Volume somedata: State: Optimal`  
+    `d5: Volume somedata: Array 1-2`  
+
+* Showing current storage configuration of `d3`:
+    `$ nodestorage d3`  
+    `d3: Disk m.2-0 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-0 State: online`  
+    `d3: Disk m.2-0 FRU: 00LF428`  
+    `d3: Disk m.2-0 Serial Number: H6B80054`  
+    `d3: Disk m.2-0 Array: 0-0`  
+    `d3: Disk m.2-1 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-1 State: online`  
+    `d3: Disk m.2-1 FRU: 00LF428`  
+    `d3: Disk m.2-1 Serial Number: H6B80059`  
+    `d3: Disk m.2-1 Array: 0-0`  
+    `d3: Array 0-0 Available Capacity: 0.000 MB`  
+    `d3: Array 0-0 Total Capacity: 131.072 GB`  
+    `d3: Array 0-0 RAID: RAID 1`  
+    `d3: Array 0-0 Disks: m.2-0,m.2-1`  
+    `d3: Array 0-0 Volumes: new_vd`  
+    `d3: Volume new_vd: Size: 122.040 GB`  
+    `d3: Volume new_vd: State: Optimal`  
+    `d3: Volume new_vd: Array 0-0`  
+
+

confluent_client/doc/man/nodesupport.ronn

@@ -7,10 +7,17 @@ nodesupport(8) -- Utilities for interacting with vendor support
 
 ## DESCRIPTION
 
-`nodesupport` provides capabilities associated with interactiong with support.
+`nodesupport` provides capabilities associated with interacting with support.
 Currently it only has the `servicedata` subcommand.  `servicedata` takes
 an argument that is either a directory name (that can be used for a single node
-or multiple nodes) or a file name (only to be used with single node noderange)
+or multiple nodes) or a file name (only to be used with single node noderange).
+Note that the file will be downloaded to the confluent server that actually
+connects to the managed system, so it will download to the remote system if running
+remotely and will download to the collective.manager indicated system if
+running in collective mode.
+
+Note that due to vendor filename requirements, any filename may have vendor
+specific suffixes added to any file produced.
 
 ## EXAMPLES
 

confluent_client/samples/nodeattrib_from_switch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 
 # This is a sample python script for going through all observed mac addresses
 # and assuming they are BMC related and printing nodeattrib commands

confluent_common/confluent_common.spec.tmpl

@@ -22,10 +22,10 @@ This provides the modules common for both client and server
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+python2 setup.py build
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
 
 %clean
 rm -rf $RPM_BUILD_ROOT

confluent_server/bin/collective

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 
 import argparse
 import errno
@@ -22,7 +22,7 @@ except NameError:
     pass
 
 def make_certificate():
-    umask = os.umask(0077)
+    umask = os.umask(0o77)
     try:
         os.makedirs('/etc/confluent/cfg')
     except OSError as e:
@@ -61,7 +61,7 @@ def join_collective(server, invitation):
         make_certificate()
     s = client.Command().connection
     while not invitation:
-        invitation = raw_input('Paste the invitation here: ')
+        invitation = input('Paste the invitation here: ')
     tlvdata.send(s, {'collective': {'operation': 'join',
                                     'invitation': invitation,
                                     'server': server}})
@@ -69,6 +69,19 @@ def join_collective(server, invitation):
     res = res.get('collective',
                   {'status': 'Unknown response: ' + repr(res)})
     print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)
+
+def delete_member(name):
+    s = client.Command().connection
+    tlvdata.send(s, {'collective': {'operation': 'delete',
+                                    'member': name}})
+    res = tlvdata.recv(s)
+    res = res.get('collective',
+                  {'status': 'Unknown response: ' + repr(res)})
+    print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)
 
 
 def show_collective():
@@ -101,11 +114,13 @@ def main():
     sl = sp.add_parser('show', help='Show information about the collective')
     ic = sp.add_parser('invite', help='Generate a invitation to allow a new '
                                     'confluent instance to join as a '
-                                    'collective member')
+                                    'collective member.  Run collective invite -h for more information')
     ic.add_argument('name', help='Name of server to invite to join the '
                                  'collective')
-    jc = sp.add_parser('join', help='Join a collective')
-    jc.add_argument('server', help='A server currently in the collective')
+    dc = sp.add_parser('delete', help='Delete a member of a collective')
+    dc.add_argument('name', help='Name of server to delete from collective')
+    jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
+    jc.add_argument('server', help='Existing collective member that ran invite and generated a token')
     jc.add_argument('-i', help='Invitation provided by runniing invite on an '
                                'existing collective member')
     cmdset = a.parse_args()
@@ -117,6 +132,8 @@ def main():
         join_collective(cmdset.server, cmdset.i)
     elif cmdset.command == 'show':
         show_collective()
+    elif cmdset.command == 'delete':
+        delete_member(cmdset.name)
 
 if __name__ == '__main__':
     main()

confluent_server/bin/confluent

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation

confluent_server/bin/confluentcertutil.py

@@ -1,11 +1,16 @@
+from os.path import exists
 import shutil
 import socket
 import subprocess
 import tempfile
 
 def get_openssl_conf_location():
-    # CentOS/RHAT
-    return '/etc/pki/tls/openssl.cnf'
+    if exists('/etc/pki/tls/openssl.cnf'):
+        return '/etc/pki/tls/openssl.cnf'
+    elif exists('/etc/ssl/openssl.cnf');
+        return '/etc/ssl/openssl.cnf'
+    else:
+        raise Exception("Cannot find openssl config file")
 
 def get_ip_addresses():
     lines = subprocess.check_output('ip addr'.split(' '))
@@ -47,4 +52,4 @@ def create_certificate():
     )
 
 if __name__ == '__main__':
-    create_certificate()
+    create_certificate()

confluent_server/bin/confluentdbutil

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -82,7 +82,7 @@ elif args[0] == 'dump':
               "or -s to do encrypted backup that requires keys.json from "
               "another backup to restore.")
         sys.exit(1)
-    os.umask(077)
+    os.umask(0o77)
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)

confluent_server/bin/confluentsrv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation

confluent_server/builddeb

@@ -0,0 +1,68 @@
+#!/bin/bash
+cd `dirname $0`
+PKGNAME=$(basename $(pwd))
+DPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+OPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+PYEXEC=python3
+DSCARGS="--with-python3=True --with-python2=False"
+if grep wheezy /etc/os-release; then
+    DPKGNAME=python-$DPKGNAME
+    PYEXEC=python
+    DSCARGS="--with-python3=False --with-python2=True"
+fi
+cd ..
+mkdir -p /tmp/confluent # $DPKGNAME
+cp -a * .git /tmp/confluent # $DPKGNAME
+cd /tmp/confluent/$PKGNAME
+if [ -x ./makeman ]; then
+    ./makeman
+fi
+./makesetup
+VERSION=`cat VERSION`
+cat > setup.cfg << EOF
+[install]
+install-purelib=/opt/confluent/lib/python
+install-scripts=/opt/confluent/bin
+
+[sdist_dsc]
+package=$DPKGNAME
+EOF
+
+$PYEXEC setup.py sdist > /dev/null 2>&1
+py2dsc $DSCARGS dist/*.tar.gz
+shopt -s extglob
+cd deb_dist/!(*.orig)/
+if [ "$OPKGNAME" = "confluent-server" ]; then
+    if grep wheezy /etc/os-release; then
+        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl/' debian/control
+    else
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket/' debian/control
+    fi
+    if grep wheezy /etc/os-release; then
+        echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
+    else
+        echo 'confluent_client confluent-client' >> debian/pydist-overrides
+    fi
+fi
+head -n -1 debian/control > debian/control1
+mv debian/control1 debian/control
+echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
+#echo 'Provides: python-'$DPKGNAME >> debian/control
+#echo 'Conflicts: python-'$DPKGNAME >> debian/control
+#echo 'Replaces: python-'$DPKGNAME' (<<2)' >> debian/control
+#echo 'Breaks: python-'$DPKGNAME' (<<2)' >> debian/control
+
+dpkg-buildpackage -rfakeroot -uc  -us -i
+if [ $? -ne 0 ]; then
+    echo "[ERROR] rpmbuild returned non-zero, run: rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec"
+    exit 1
+else
+    cd -
+    # Clean up the generated files in this directory
+    rm -rf $PKGNAME.egg-info dist setup.py
+    rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d)
+    if [ ! -z "$1" ]; then
+        mv deb_dist/* $1/
+    fi
+fi
+exit 0

confluent_server/buildrpm

@@ -6,7 +6,7 @@ fi
 ./makesetup
 VERSION=`cat VERSION`
 PKGNAME=$(basename $(pwd))
-python setup.py sdist > /dev/null 2>&1
+python3 setup.py sdist > /dev/null 2>&1
 cp dist/*.tar.gz ~/rpmbuild/SOURCES
 sed -e 's/#VERSION#/'$VERSION/ $PKGNAME.spec.tmpl > ~/rpmbuild/SPECS/$PKGNAME.spec
 rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec 2> /dev/null |grep ^Wrote:

confluent_server/confluent/auth.py

@@ -23,11 +23,15 @@ import confluent.config.configmanager as configmanager
 import eventlet
 import eventlet.tpool
 import Cryptodome.Protocol.KDF as KDF
+from fnmatch import fnmatch
 import hashlib
 import hmac
 import multiprocessing
+import confluent.userutil as userutil
+import confluent.util as util
+pam = None
 try:
-    import PAM
+    import confluent.pam as pam
 except ImportError:
     pass
 import time
@@ -39,24 +43,59 @@ _passchecking = {}
 authworkers = None
 authcleaner = None
 
+_allowedbyrole = {
+    'Operator': {
+        'retrieve': ['*'],
+        'create': [
+            '/noderange/',
+            '/nodes/',
+            '/node*/media/uploads/',
+            '/node*/inventory/firmware/updates/*',
+            '/node*/suppport/servicedata*',
+            '/node*/attributes/expression',
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+            '/node*/configuration/*',
+        ],
+        'update': [
+            '/discovery/*',
+            '/networking/macs/rescan',
+            '/node*/power/state',
+            '/node*/power/reseat',
+            '/node*/attributes/*',
+            '/node*/media/*tach',
+            '/node*/boot/nextdevice',
+            '/node*/identify',
+            '/node*/configuration/*',
+        ],
+        'start': [
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+        ],
+        'delete': [
+            '/discovery/*',
+            '/node*',
+        ],
+    },
+    'Monitor': {
+        'retrieve': [
+            '/node*/health/hardware',
+            '/node*/power/state',
+            '/node*/sensors/*',
+            '/nodes/',
+            '/',
+        ],
+    }
+}
 
-class Credentials(object):
-    def __init__(self, username, passphrase):
-        self.username = username
-        self.passphrase = passphrase
-        self.haspam = False
-
-    def pam_conv(self, auth, query_list):
-        # use stored credentials in a pam conversation
-        self.haspam = True
-        resp = []
-        for query_entry in query_list:
-            query, pamtype = query_entry
-            if query.startswith('Password'):
-                resp.append((self.passphrase, 0))
-            else:
-                return None
-        return resp
+_deniedbyrole = {
+    # This supersedes the above and is only consulted after the allowed has happened
+    'Operator': {
+        'update': [
+            '/node*/configuration/management_controller/users/*',
+        ]
+    }
+}
 
 
 def _prune_passcache():
@@ -65,9 +104,9 @@ def _prune_passcache():
     while True:
         curtime = time.time()
         for passent in _passcache.iterkeys():
-            if passent[2] < curtime - 10:
+            if passent[2] < curtime - 90:
                 del _passcache[passent]
-        eventlet.sleep(10)
+        eventlet.sleep(90)
 
 
 def _get_usertenant(name, tenant=False):
@@ -81,11 +120,13 @@ def _get_usertenant(name, tenant=False):
     administrator account a tenant gets.
     Otherwise, just assume a user in the default tenant
     """
+    if not isinstance(name, bytes):
+        name = name.encode('utf-8')
     if not isinstance(tenant, bool):
         # if not boolean, it must be explicit tenant
         user = name
-    elif '/' in name:  # tenant scoped name
-        tenant, user = name.split('/', 1)
+    elif b'/' in name:  # tenant scoped name
+        tenant, user = name.split(b'/', 1)
     elif configmanager.is_tenant(name):
         # the account is the implicit tenant owner account
         user = name
@@ -93,6 +134,9 @@ def _get_usertenant(name, tenant=False):
     else:  # assume it is a non-tenant user account
         user = name
         tenant = None
+    user = util.stringify(user)
+    if tenant:
+        tenant = util.stringify(tenant)
     yield user
     yield tenant
 
@@ -112,21 +156,37 @@ def authorize(name, element, tenant=False, operation='create',
             and the relevant ConfigManager object for the context of the
             request.
     """
-    if operation not in ('create', 'start', 'update', 'retrieve', 'delete'):
-        return None
+    # skipuserobj is a leftover from the now abandoned plan to use pam session
+    # to do authorization and authentication.  Now confluent always does authorization
+    # even if pam does authentication.
+    if operation not in ('create', 'start', 'update', 'retrieve', 'delete', None):
+        return False
     user, tenant = _get_usertenant(name, tenant)
     if tenant is not None and not configmanager.is_tenant(tenant):
-        return None
+        return False
     manager = configmanager.ConfigManager(tenant, username=user)
-    if skipuserobj:
-        return None, manager, user, tenant, skipuserobj
     userobj = manager.get_user(user)
+    if not userobj:
+        for group in userutil.grouplist(user):
+            userobj = manager.get_usergroup(group)
+            if userobj:
+                break
     if userobj:  # returning
+        role = userobj.get('role', 'Administrator')
+        if element and role != 'Administrator':
+            for rule in _allowedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    break
+            else:
+                return False
+            for rule in _deniedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    return False
         return userobj, manager, user, tenant, skipuserobj
-    return None
+    return False
 
 
-def check_user_passphrase(name, passphrase, element=None, tenant=False):
+def check_user_passphrase(name, passphrase, operation=None, element=None, tenant=False):
     """Check a a login name and passphrase for authenticity and authorization
 
     The function combines authentication and authorization into one function.
@@ -157,21 +217,27 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         # would normally make an event and wait
         # but here there's no need for that
         eventlet.sleep(0.5)
-    credobj = Credentials(user, passphrase)
     cfm = configmanager.ConfigManager(tenant, username=user)
     ucfg = cfm.get_user(user)
+    if ucfg is None:
+        try:
+            for group in userutil.grouplist(user):
+                ucfg = cfm.get_usergroup(group)
+                if ucfg:
+                    break
+        except KeyError:
+            pass
     if ucfg is None:
         eventlet.sleep(0.05)
         return None
     if (user, tenant) in _passcache:
         if hashlib.sha256(passphrase).digest() == _passcache[(user, tenant)]:
-            return authorize(user, element, tenant)
+            return authorize(user, element, tenant, operation=operation)
         else:
             # In case of someone trying to guess,
             # while someone is legitimately logged in
             # invalidate cache and force the slower check
             del _passcache[(user, tenant)]
-            return None
     if 'cryptpass' in ucfg:
         _passchecking[(user, tenant)] = True
         # TODO(jbjohnso): WORKERPOOL
@@ -200,23 +266,17 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         # delay as well
         if crypt == crypted:
             _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
-            return authorize(user, element, tenant)
-    try:
-        pammy = PAM.pam()
-        pammy.start(_pamservice, user, credobj.pam_conv)
-        pammy.authenticate()
-        pammy.acct_mgmt()
+            return authorize(user, element, tenant, operation)
+    if pam:
+        pammy = pam.pam()
+        usergood = pammy.authenticate(user, passphrase)
         del pammy
-        _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
-        return authorize(user, element, tenant, skipuserobj=False)
-    except NameError:
-        pass
-    except PAM.error:
-        pass
+        if usergood:
+            _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+            return authorize(user, element, tenant, operation, skipuserobj=False)    
     eventlet.sleep(0.05)  # stall even on test for existence of a username
     return None
 
-
 def _apply_pbkdf(passphrase, salt):
     return KDF.PBKDF2(passphrase, salt, 32, 10000,
                       lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
@@ -234,4 +294,4 @@ def _do_pbkdf(passphrase, salt):
     # compute.  However, we do want to wait for result, so we have
     # one of the exceedingly rare sort of circumstances where 'apply'
     # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
+    return authworkers.apply(_apply_pbkdf, [passphrase, salt])

confluent_server/confluent/collective/manager.py

@@ -27,6 +27,7 @@ import eventlet.green.ssl as ssl
 import eventlet.green.threading as threading
 import greenlet
 import random
+import sys
 try:
     import OpenSSL.crypto as crypto
 except ImportError:
@@ -70,11 +71,22 @@ def connect_to_leader(cert=None, name=None, leader=None):
         return False
     with connecting:
         with cfm._initlock:
-            tlvdata.recv(remote)  # the banner
+            banner = tlvdata.recv(remote)  # the banner
+            vers = banner.split()[2]
+            pvers = 0
+            reqver = 4
+            if vers == b'v0':
+                pvers = 2
+            elif vers == b'v1':
+                pvers = 4
+            if sys.version_info[0] < 3:
+                pvers = 2
+                reqver = 2
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
             tlvdata.send(remote, {'collective': {'operation': 'connect',
+                                                 'protover': reqver,
                                                  'name': name,
                                                  'txcount': cfm._txcount}})
             keydata = tlvdata.recv(remote)
@@ -119,7 +131,7 @@ def connect_to_leader(cert=None, name=None, leader=None):
             globaldata = tlvdata.recv(remote)
             dbi = tlvdata.recv(remote)
             dbsize = dbi['dbsize']
-            dbjson = ''
+            dbjson = b''
             while (len(dbjson) < dbsize):
                 ndata = remote.recv(dbsize - len(dbjson))
                 if not ndata:
@@ -148,15 +160,15 @@ def connect_to_leader(cert=None, name=None, leader=None):
                 raise
             currentleader = leader
         #spawn this as a thread...
-        follower = eventlet.spawn(follow_leader, remote)
+        follower = eventlet.spawn(follow_leader, remote, pvers, leader)
     return True
 
 
-def follow_leader(remote):
+def follow_leader(remote, proto, leader):
     global currentleader
     cleanexit = False
     try:
-        cfm.follow_channel(remote)
+        cfm.follow_channel(remote, proto)
     except greenlet.GreenletExit:
         cleanexit = True
     finally:
@@ -164,8 +176,8 @@ def follow_leader(remote):
             log.log({'info': 'Previous following cleanly closed',
                      'subsystem': 'collective'})
             return
-        log.log({'info': 'Current leader has disappeared, restarting '
-                         'collective membership', 'subsystem': 'collective'})
+        log.log({'info': 'Current leader ({0}) has disappeared, restarting '
+                         'collective membership'.format(leader), 'subsystem': 'collective'})
         # The leader has folded, time to startup again...
         cfm.stop_following()
         currentleader = None
@@ -208,8 +220,7 @@ def handle_connection(connection, cert, request, local=False):
     else:
         if not local:
             return
-
-        if 'show' == operation:
+        if operation in ('show', 'delete'):
             if not list(cfm.list_collective()):
                 tlvdata.send(connection,
                              {'collective': {'error': 'Collective mode not '
@@ -246,7 +257,23 @@ def handle_connection(connection, cert, request, local=False):
                 collinfo['quorum'] = True
             except exc.DegradedCollective:
                 collinfo['quorum'] = False
-            tlvdata.send(connection, {'collective':  collinfo})
+            if operation == 'show':
+                tlvdata.send(connection, {'collective':  collinfo})
+            elif operation == 'delete':
+                todelete = request['member']
+                if (todelete == collinfo['leader'] or 
+                       todelete in collinfo['active']):
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is still active, stop the confluent service to remove it'.format(todelete)}})
+                    return
+                if todelete not in collinfo['offline']:
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is not a recognized collective member'.format(todelete)}})
+                    return
+                cfm.del_collective_member(todelete)
+                tlvdata.send(connection,
+                    {'collective': {'status': 'Successfully deleted {0}'.format(todelete)}})
+                connection.close()
             return
         if 'invite' == operation:
             try:
@@ -267,7 +294,8 @@ def handle_connection(connection, cert, request, local=False):
             invitation = request['invitation']
             try:
                 invitation = base64.b64decode(invitation)
-                name, invitation = invitation.split('@', 1)
+                name, invitation = invitation.split(b'@', 1)
+                name = util.stringify(name)
             except Exception:
                 tlvdata.send(
                     connection,
@@ -375,7 +403,7 @@ def handle_connection(connection, cert, request, local=False):
             connection.close()
             return
         if (currentleader == connection.getpeername()[0] and
-                follower and follower.isAlive()):
+                follower and not follower.dead):
             # if we are happily following this leader already, don't stir
             # the pot
             tlvdata.send(connection, {'status': 0})
@@ -402,6 +430,7 @@ def handle_connection(connection, cert, request, local=False):
         tlvdata.send(connection, collinfo)
     if 'connect' == operation:
         drone = request['name']
+        folver = request.get('protover', 2)
         droneinfo = cfm.get_collective_member(drone)
         if not (droneinfo and util.cert_matches(droneinfo['fingerprint'],
                                                 cert)):
@@ -450,7 +479,7 @@ def handle_connection(connection, cert, request, local=False):
             connection.sendall(cfgdata)
         #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
         # so far unused anyway
-        if not cfm.relay_slaved_requests(drone, connection):
+        if not cfm.relay_slaved_requests(drone, connection, folver):
             if not retrythread:  # start a recovery if everyone else seems
                 # to have disappeared
                 retrythread = eventlet.spawn_after(30 + random.random(),

confluent_server/confluent/config/attributes.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -93,9 +93,10 @@ node = {
         'description': ('List of static groups for which this node is '
                         'considered a member'),
     },
-    #'type': {
-    #    'description': ('Classification of node as system, vm, etc')
-    #},
+    'type': {
+        'description': ('Classification of node as server or switch'),
+        'validvalues': ('switch', 'server'),
+    },
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
@@ -158,9 +159,11 @@ node = {
     },
     'discovery.passwordrules': {
         'description':  'Any specified rules shall be configured on the BMC '
-                        'upon discovery.  "expiration=no,loginfailures=no" '
-                        'would disable password expiration and login failures '
-                        'triggering a lockout.'
+                        'upon discovery.  "expiration=no,loginfailures=no,complexity=no,reuse=no" '
+                        'would disable password expiration, login failures '
+                        'triggering a lockout, password complexity requirements,'
+                        'and any restrictions around reusing an old password.',
+        'validlistkeys': ('expiration', 'loginfailures', 'complexity', 'reuse'),
     },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
@@ -171,6 +174,7 @@ node = {
                         'so long as the node has no existing public key. '
                         '"open" allows discovery even if a known public key '
                         'is already stored',
+        'validlist': ('manual', 'permissive', 'pxe', 'open'),
     },
     'info.note': {
         'description':  'A field used for administrators to make arbitrary '
@@ -179,6 +183,9 @@ node = {
                         'freeform text data without concern for issues in how '
                         'the server will process it.',
     },
+    'location.height': {
+        'description': 'Height in RU of the system (defaults to query the systems)',
+    },
     'location.room': {
         'description': 'Room description for the node',
     },
@@ -258,11 +265,13 @@ node = {
     'console.logging': {
         'description': ('Indicate logging level to apply to console.  Valid '
                         'values are currently "full", "interactive", and '
-                        '"none". Defaults to "full".')
+                        '"none". Defaults to "full".'),
+        'validvalues': ('full', 'interactive', 'none'),
     },
     'console.method': {
         'description': ('Indicate the method used to access the console of '
-                        'the managed node.')
+                        'the managed node.'),
+        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/conf.py

@@ -16,7 +16,10 @@
 
 
 # This defines config variable to store the global configuration for confluent
-import ConfigParser
+try:
+    import ConfigParser
+except ModuleNotFoundError:
+    import configparser as ConfigParser
 import os
 
 _config = None

confluent_server/confluent/consoleserver.py

@@ -94,7 +94,7 @@ def _utf8_normalize(data, shiftin, decoder):
 
 
 def pytechars2line(chars, maxlen=None):
-    line = '\x1b[m'  # start at default params
+    line = b'\x1b[m'  # start at default params
     lb = False  # last bold
     li = False  # last italic
     lu = False  # last underline
@@ -106,7 +106,7 @@ def pytechars2line(chars, maxlen=None):
     len = 1
     for charidx in range(maxlen):
         char = chars[charidx]
-        csi = []
+        csi = bytearray([])
         if char.fg != lfg:
             csi.append(30 + pytecolors2ansi[char.fg])
             lfg = char.fg
@@ -129,10 +129,13 @@ def pytechars2line(chars, maxlen=None):
             lr = char.reverse
             csi.append(7 if lr else 27)
         if csi:
-            line += b'\x1b[' + b';'.join(['{0}'.format(x) for x in csi]) + b'm'
-        if not hasdata and char.data.encode('utf-8').rstrip():
+            line += b'\x1b[' + b';'.join(['{0}'.format(x).encode('utf-8') for x in csi]) + b'm'
+        if not hasdata and char.data.rstrip():
             hasdata = True
-        line += char.data.encode('utf-8')
+        chardata = char.data
+        if not isinstance(chardata, bytes):
+            chardata = chardata.encode('utf-8')
+        line += chardata
         if maxlen and len >= maxlen:
             break
         len += 1
@@ -147,6 +150,7 @@ class ConsoleHandler(object):
 
     def __init__(self, node, configmanager, width=80, height=24):
         self.clearpending = False
+        self.clearerror = False
         self.initsize = (width, height)
         self._dologging = True
         self._is_local = True
@@ -184,7 +188,7 @@ class ConsoleHandler(object):
         if termstate & 1:
             self.appmodedetected = True
         if termstate & 2:
-            self.shiftin = '0'
+            self.shiftin = b'0'
         self.users = {}
         self._attribwatcher = None
         self._console = None
@@ -209,6 +213,8 @@ class ConsoleHandler(object):
         return retrytime + (retrytime * random.random())
 
     def feedbuffer(self, data):
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
         try:
             self.termstream.feed(data)
         except StopIteration:  # corrupt parser state, start over
@@ -308,8 +314,11 @@ class ConsoleHandler(object):
 
     def clearbuffer(self):
         self.feedbuffer(
-            '\x1bc[no replay buffer due to console.logging attribute set to '
-            'none or interactive,\r\nconnection loss, or service restart]')
+            '\x1bc[No data has been received from the remote console since ' \
+            'connecting.  This could\r\nbe due to having the console.logging ' \
+            'attribute set to none or interactive,\r\nserial console not '  \
+            'being enabled or incorrectly configured in the OS or\r\nfirmware, ' \
+            'or the console simply not having any output since last connection]')
         self.clearpending = True
 
     def _detach(self):
@@ -353,13 +362,17 @@ class ConsoleHandler(object):
         if self.reconnect:
             self.reconnect.cancel()
             self.reconnect = None
+        strerror = ('The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function')
         try:
             self._console = list(plugin.handle_path(
                 self._plugin_path.format(self.node),
                 "create", self.cfgmgr))[0]
         except (exc.NotImplementedException, exc.NotFoundException):
             self._console = None
-        except:
+        except Exception as e:
+            strerror = str(e)
             if _tracelog:
                 _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                               event=log.Events.stacktrace)
@@ -371,7 +384,16 @@ class ConsoleHandler(object):
             self.error = 'misconfigured'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
+            self.feedbuffer(
+                '\x1bc\x1b[2J\x1b[1;1H[{0}]'.format(strerror))
+            self._send_rcpts(
+                '\x1bc\x1b[2J\x1b[1;1H[{0}]'.format(strerror))
+            self.clearerror = True
             return
+        if self.clearerror:
+            self.clearerror = False
+            self.clearbuffer()
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self.send_break = self._console.send_break
         self.resize = self._console.resize
         if self._attribwatcher:
@@ -431,6 +453,7 @@ class ConsoleHandler(object):
 
     def _got_disconnected(self):
         if self.connectstate != 'unconnected':
+            self._console.close()
             self.connectstate = 'unconnected'
             self.log(
                 logdata='console disconnected', ltype=log.DataTypes.event,
@@ -508,17 +531,19 @@ class ConsoleHandler(object):
             if data == conapi.ConsoleEvent.Disconnect:
                 self._got_disconnected()
             return
-        elif data == '':
+        elif data in (b'', u''):
             # ignore empty strings from a cconsole provider
             return
-        if '\x1b[?1l' in data:  # request for ansi mode cursor keys
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+        if b'\x1b[?1l' in data:  # request for ansi mode cursor keys
             self.appmodedetected = False
-        if '\x1b[?1h' in data:  # remember the session wants the client to use
+        if b'\x1b[?1h' in data:  # remember the session wants the client to use
             # 'application mode'  Thus far only observed on esxi
             self.appmodedetected = True
-        if '\x1b)0' in data:
+        if b'\x1b)0' in data:
             # console indicates it wants access to special drawing characters
-            self.shiftin = '0'
+            self.shiftin = b'0'
         eventdata = 0
         if self.appmodedetected:
             eventdata |= 1
@@ -527,10 +552,11 @@ class ConsoleHandler(object):
         # TODO: analyze buffer for registered events, examples:
         #   panics
         #   certificate signing request
-        if self.clearpending:
+        if self.clearpending or self.clearerror:
             self.clearpending = False
-            self.feedbuffer(b'\x1bc')
-            self._send_rcpts(b'\x1bc')
+            self.clearerror = False
+            self.feedbuffer(b'\x1bc\x1b[2J\x1b[1;1H')
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
         self.log(data, eventdata=eventdata)
         self.lasttime = util.monotonic_time()
@@ -570,25 +596,30 @@ class ConsoleHandler(object):
                 if pendingbl:
                     retdata += pendingbl
                     pendingbl = b''
-                retdata += nline + '\r\n'
+                retdata += nline + b'\r\n'
             else:
-                pendingbl += nline + '\r\n'
+                pendingbl += nline + b'\r\n'
         if len(retdata) >  6:
             retdata = retdata[:-2]  # remove the last \r\n
-        retdata += b'\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
-                                           self.buffer.cursor.x + 1)
+        cursordata = '\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
+                                            self.buffer.cursor.x + 1)
+        if not isinstance(cursordata, bytes):
+            cursordata = cursordata.encode('utf-8')
+        retdata += cursordata
         if self.shiftin is not None:  # detected that terminal requested a
             # shiftin character set, relay that to the terminal that cannected
-            retdata += '\x1b)' + self.shiftin
+            retdata += b'\x1b)' + self.shiftin
         if self.appmodedetected:
-            retdata += '\x1b[?1h'
+            retdata += b'\x1b[?1h'
         else:
-            retdata += '\x1b[?1l'
+            retdata += b'\x1b[?1l'
         return retdata, connstate
 
     def write(self, data):
         if self.connectstate == 'connected':
             try:
+                if isinstance(data, str) and not isinstance(data, bytes):
+                    data = data.encode('utf-8')
                 self._console.write(data)
             except Exception:
                 _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
@@ -603,11 +634,14 @@ def disconnect_node(node, configmanager):
         del _handled_consoles[consk]
 
 
-def _nodechange(added, deleting, configmanager):
-    for node in added:
-        connect_node(node, configmanager)
+def _nodechange(added, deleting, renamed, configmanager):
     for node in deleting:
         disconnect_node(node, configmanager)
+    for node in renamed:
+        disconnect_node(node, configmanager)
+        connect_node(renamed[node], configmanager)
+    for node in added:
+        connect_node(node, configmanager)
 
 
 def _start_tenant_sessions(cfm):

confluent_server/confluent/core.py

@@ -35,6 +35,7 @@
 
 import confluent
 import confluent.alerts as alerts
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.config.attributes as attrscheme
 import confluent.config.configmanager as cfm
@@ -62,14 +63,16 @@ import itertools
 import os
 try:
     import cPickle as pickle
+    pargs = {}
 except ImportError:
     import pickle
+    pargs = {'encoding': 'utf-8'}
 import socket
 import struct
 import sys
 
 pluginmap = {}
-dispatch_plugins = (b'ipmi', u'ipmi')
+dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol')
 
 
 def seek_element(currplace, currkey):
@@ -85,7 +88,10 @@ def seek_element(currplace, currkey):
 
 def nested_lookup(nestdict, key):
     try:
-        return reduce(seek_element, key, nestdict)
+        currloc = nestdict
+        for currk in key:
+            currloc = seek_element(currloc, currk)
+        return currloc
     except TypeError:
         raise exc.NotFoundException("Invalid element requested")
 
@@ -105,6 +111,8 @@ def load_plugins():
         for plugin in os.listdir(plugindir):
             if plugin.startswith('.'):
                 continue
+            if '__pycache__' in plugin:
+                continue
             (plugin, plugtype) = os.path.splitext(plugin)
             if plugtype == '.sh':
                 pluginmap[plugin] = shellmodule.Plugin(
@@ -123,7 +131,7 @@ def load_plugins():
 
 
 rootcollections = ['discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'version']
+                   'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' , 'users/', 'version']
 
 
 class PluginRoute(object):
@@ -143,6 +151,7 @@ def _init_core():
     # be enumerated in any collection
     noderesources = {
         'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
             'expression': PluginRoute({'handler': 'attributes'}),
@@ -161,6 +170,10 @@ def _init_core():
                         'default': 'ipmi',
                     }),
                 },
+                'clear': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                }),
                 'users': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -169,6 +182,10 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'save_licenses': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                 'net_interfaces': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -199,6 +216,16 @@ def _init_core():
                         'default': 'ipmi',
                     }),
                 },
+                'extended': {
+                    'all': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'advanced': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
             },
             'storage': {
                 'all': PluginRoute({
@@ -378,6 +405,8 @@ def _init_core():
 
     nodegroupresources = {
         'attributes': {
+            'check': PluginRoute({'handler': 'attributes'}),
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
         },
@@ -393,17 +422,37 @@ def create_user(inputdata, configmanager):
     configmanager.create_user(username, attributemap=inputdata)
 
 
+def create_usergroup(inputdata, configmanager):
+    try:
+        groupname = inputdata['name']
+        del inputdata['name']
+    except (KeyError, ValueError):
+        raise exc.InvalidArgumentException()
+    configmanager.create_usergroup(groupname)
+
+
+def update_usergroup(groupname, attribmap, configmanager):
+    try:
+        configmanager.set_usergroup(groupname, attribmap)
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
+
 def update_user(name, attribmap, configmanager):
     try:
         configmanager.set_user(name, attribmap)
-    except ValueError:
-        raise exc.InvalidArgumentException()
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
 
 
+def show_usergroup(groupname, configmanager):
+    groupinfo = configmanager.get_usergroup(groupname)
+    for attr in groupinfo:
+        yield msg.Attributes(kv={attr: groupinfo[attr]})
+
 def show_user(name, configmanager):
     userobj = configmanager.get_user(name)
     rv = {}
-    for attr in attrscheme.user.iterkeys():
+    for attr in attrscheme.user:
         rv[attr] = None
         if attr == 'password':
             if 'cryptpass' in userobj:
@@ -416,6 +465,10 @@ def show_user(name, configmanager):
                 rv[attr] = userobj[attr]
             yield msg.Attributes(kv={attr: rv[attr]},
                                  desc=attrscheme.user[attr]['description'])
+    if 'role' in userobj:
+        yield msg.Attributes(kv={'role': userobj['role']})
+
+
 
 
 def stripnode(iterablersp, node):
@@ -448,6 +501,10 @@ def delete_user(user, configmanager):
     configmanager.del_user(user)
     yield msg.DeletedResource(user)
 
+def delete_usergroup(usergroup, configmanager):
+    configmanager.del_usergroup(usergroup)
+    yield msg.DeletedResource(usergroup)
+
 
 def delete_nodegroup_collection(collectionpath, configmanager):
     if len(collectionpath) == 2:  # just the nodegroup
@@ -632,7 +689,10 @@ def handle_dispatch(connection, cert, dispatch, peername):
             cfm.get_collective_member(peername)['fingerprint'], cert):
         connection.close()
         return
-    dispatch = pickle.loads(dispatch)
+    pversion = 0
+    if bytearray(dispatch)[0] == 0x80:
+        pversion = bytearray(dispatch)[1]
+    dispatch = pickle.loads(dispatch, **pargs)
     configmanager = cfm.ConfigManager(dispatch['tenant'])
     nodes = dispatch['nodes']
     inputdata = dispatch['inputdata']
@@ -651,7 +711,7 @@ def handle_dispatch(connection, cert, dispatch, peername):
                 plugpath = nodeattr[node][attrname]['value']
             elif 'default' in plugroute:
                 plugpath = plugroute['default']
-        if plugpath is not None:
+        if plugpath:
             try:
                 hfunc = getattr(pluginmap[plugpath], operation)
             except KeyError:
@@ -668,14 +728,18 @@ def handle_dispatch(connection, cert, dispatch, peername):
                 configmanager=configmanager,
                 inputdata=inputdata))
         for res in itertools.chain(*passvalues):
-            _forward_rsp(connection, res)
+            _forward_rsp(connection, res, pversion)
     except Exception as res:
-        _forward_rsp(connection, res)
+        _forward_rsp(connection, res, pversion)
     connection.sendall('\x00\x00\x00\x00\x00\x00\x00\x00')
 
 
-def _forward_rsp(connection, res):
-    r = pickle.dumps(res)
+def _forward_rsp(connection, res, pversion):
+    try:
+        r = pickle.dumps(res, protocol=pversion)
+    except TypeError:
+        r = pickle.dumps(Exception(
+            'Cannot serialize error, check collective.manager error logs for details' + str(res)), protocol=pversion)
     rlen = len(r)
     if not rlen:
         return
@@ -685,6 +749,8 @@ def _forward_rsp(connection, res):
 
 def handle_node_request(configmanager, inputdata, operation,
                         pathcomponents, autostrip=True):
+    if log.logfull:
+        raise exc.TargetResourceUnavailable('Filesystem full, free up space and restart confluent service')
     iscollection = False
     routespec = None
     if pathcomponents[0] == 'noderange':
@@ -809,7 +875,7 @@ def handle_node_request(configmanager, inputdata, operation,
                 elif list(cfm.list_collective()):
                     badcollnodes.append(node)
                     continue
-            if plugpath is not None:
+            if plugpath:
                 try:
                     hfunc = getattr(pluginmap[plugpath], operation)
                 except KeyError:
@@ -882,7 +948,7 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
     a = configmanager.get_collective_member(manager)
     try:
         remote = socket.create_connection((a['address'], 13001))
-        remote.settimeout(90)
+        remote.settimeout(180)
         remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                  keyfile='/etc/confluent/privkey.pem',
                                  certfile='/etc/confluent/srvcert.pem')
@@ -902,12 +968,20 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
     if not util.cert_matches(a['fingerprint'], remote.getpeercert(
             binary_form=True)):
         raise Exception("Invalid certificate on peer")
-    tlvdata.recv(remote)
+    banner = tlvdata.recv(remote)
+    vers = banner.split()[2]
+    if vers == b'v0':
+        pvers = 2
+    elif vers == b'v1':
+        pvers = 4
+    if sys.version_info[0] < 3:
+        pvers = 2
     tlvdata.recv(remote)
     myname = collective.get_myname()
     dreq = pickle.dumps({'name': myname, 'nodes': list(nodes),
                          'path': element,'tenant': configmanager.tenant,
-                         'operation': operation, 'inputdata': inputdata})
+                         'operation': operation, 'inputdata': inputdata},
+                         protocol=pvers)
     tlvdata.send(remote, {'dispatch': {'name': myname, 'length': len(dreq)}})
     remote.sendall(dreq)
     while True:
@@ -954,7 +1028,10 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
                             a['name']))
                 return
             rsp += nrsp
-        rsp = pickle.loads(rsp)
+        try:
+            rsp = pickle.loads(rsp, **pargs)
+        except UnicodeDecodeError:
+            rsp = pickle.loads(rsp, encoding='latin1')
         if isinstance(rsp, Exception):
             raise rsp
         yield rsp
@@ -1000,6 +1077,31 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
             configmanager, inputdata, operation, pathcomponents)
     elif pathcomponents[0] == 'version':
         return (msg.Attributes(kv={'version': confluent.__version__}),)
+    elif pathcomponents[0] == 'usergroups':
+        # TODO: when non-administrator accounts exist,
+        # they must only be allowed to see their own user
+        try:
+            usergroup = pathcomponents[1]
+        except IndexError:  # it's just users/
+            if operation == 'create':
+                inputdata = msg.get_input_message(
+                    pathcomponents, operation, inputdata,
+                    configmanager=configmanager)
+                create_usergroup(inputdata.attribs, configmanager)
+            return iterate_collections(configmanager.list_usergroups(),
+                                       forcecollection=False)
+        if usergroup not in configmanager.list_usergroups():
+            raise exc.NotFoundException("Invalid usergroup %s" % usergroup)
+        if operation == 'retrieve':
+            return show_usergroup(usergroup, configmanager)
+        elif operation == 'delete':
+            return delete_usergroup(usergroup, configmanager)
+        elif operation == 'update':
+            inputdata = msg.get_input_message(
+                pathcomponents, operation, inputdata,
+                configmanager=configmanager)
+            update_usergroup(usergroup, inputdata.attribs, configmanager)
+            return show_usergroup(usergroup, configmanager)
     elif pathcomponents[0] == 'users':
         # TODO: when non-administrator accounts exist,
         # they must only be allowed to see their own user

confluent_server/confluent/discovery/core.py

@@ -63,10 +63,13 @@
 
 import base64
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 #import confluent.discovery.protocols.ssdp as ssdp
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
+import confluent.discovery.handlers.cpstorage as cpstorage
+import confluent.discovery.handlers.tsm as tsm
 import confluent.discovery.handlers.pxe as pxeh
 import confluent.discovery.handlers.smm as smm
 import confluent.discovery.handlers.xcc as xcc
@@ -88,6 +91,11 @@ import eventlet.semaphore
 autosensors = set()
 scanner = None
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 class nesteddict(dict):
 
     def __missing__(self, key):
@@ -100,6 +108,8 @@ nodehandlers = {
     'service:management-hardware.IBM:integrated-management-module2': imm,
     'pxe-client': pxeh,
     'service:io-device.Lenovo:management-module': None,
+    'service:thinkagile-storage': cpstorage,
+    'service:lenovo-tsm': tsm,
 }
 
 servicenames = {
@@ -108,6 +118,8 @@ servicenames = {
     'service:management-hardware.Lenovo:lenovo-xclarity-controller': 'lenovo-xcc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
+    'service:thinkagile-storage': 'thinkagile-storagebmc',
+    'service:lenovo-tsm': 'lenovo-tsm',
 }
 
 servicebyname = {
@@ -116,6 +128,8 @@ servicebyname = {
     'lenovo-xcc': 'service:management-hardware.Lenovo:lenovo-xclarity-controller',
     'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2',
     'lenovo-switch': 'service:io-device.Lenovo:management-module',
+    'thinkagile-storage': 'service:thinkagile-storagebmc',
+    'lenovo-tsm': 'service:lenovo-tsm',
 }
 
 discopool = eventlet.greenpool.GreenPool(500)
@@ -362,7 +376,7 @@ def handle_autosense_config(operation, inputdata):
         yield msg.KeyValueData({'enabled': autosense})
     elif operation == 'update':
         enabled = inputdata['enabled']
-        if type(enabled) in (unicode, str):
+        if type(enabled) in (unicode, bytes):
             enabled = enabled.lower() in ('true', '1', 'y', 'yes', 'enable',
                                           'enabled')
         if autosense == enabled:
@@ -696,7 +710,8 @@ def detected(info):
 
 def b64tohex(b64str):
     bd = base64.b64decode(b64str)
-    return ''.join(['{0:02x}'.format(ord(x)) for x in bd])
+    bd = bytearray(bd)
+    return ''.join(['{0:02x}'.format(x) for x in bd])
 
 
 def get_enclosure_chain_head(nodename, cfg):
@@ -880,7 +895,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
         handler.probe()  # unicast interrogation as possible to get more data
         # switch concurrently
         # do some preconfig, for example, to bring a SMM online if applicable
-        handler.preconfig()
+        handler.preconfig(nodename)
     except Exception as e:
         unknown_info[info['hwaddr']] = info
         info['discostatus'] = 'unidentified'
@@ -934,9 +949,9 @@ def eval_node(cfg, handler, info, nodename, manual=False):
                     #     raise exc.InvalidArgumentException(errorstr)
                     # log.log({'error': errorstr})
                     if encuuid in pending_by_uuid:
-                        pending_by_uuid[encuuid].add(info)
+                        pending_by_uuid[encuuid].append(info)
                     else:
-                        pending_by_uuid[encuuid] = set([info])
+                        pending_by_uuid[encuuid] = [info]
                     return
                 # We found the real smm, replace the list with the actual smm
                 # to continue
@@ -1055,6 +1070,14 @@ def discover_node(cfg, handler, info, nodename, manual):
                 traceback.print_exc()
                 return False
             newnodeattribs = {}
+            if cfm.list_collective():
+                # We are in a collective, check collective.manager
+                cmc = cfg.get_node_attributes(nodename, 'collective.manager')
+                cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
+                if not cm:
+                    # Node is being discovered in collective, but no collective.manager, default
+                    # to the collective member actually able to execute the discovery
+                    newnodeattribs['collective.manager'] = collective.get_myname()
             if 'uuid' in info:
                 newnodeattribs['id.uuid'] = info['uuid']
             if 'serialnumber' in info:
@@ -1071,6 +1094,10 @@ def discover_node(cfg, handler, info, nodename, manual):
         info['discostatus'] = 'discovered'
         for i in pending_by_uuid.get(curruuid, []):
             eventlet.spawn_n(_recheck_single_unknown_info, cfg, i)
+        try:
+            del pending_by_uuid[curruuid]
+        except KeyError:
+            pass
         return True
     log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
                      'value allowing discovery (open or permissive)'.format(
@@ -1129,10 +1156,18 @@ def _handle_nodelist_change(configmanager):
         nodeaddhandler = None
 
 
-def newnodes(added, deleting, configmanager):
+def newnodes(added, deleting, renamed, configmanager):
     global attribwatcher
     global needaddhandled
     global nodeaddhandler
+    alldeleting = set(deleting) | set(renamed)
+    for node in alldeleting:
+        if node not in known_nodes:
+            continue
+        for mac in known_nodes[node]:
+            if mac in known_info:
+                del known_info[mac]
+        del known_nodes[node]
     _map_unique_ids()
     configmanager.remove_watcher(attribwatcher)
     allnodes = configmanager.list_nodes()
@@ -1182,6 +1217,7 @@ def rescan():
 def start_detection():
     global attribwatcher
     global rechecker
+    global rechecktime
     _map_unique_ids()
     cfg = cfm.ConfigManager(None)
     allnodes = cfg.list_nodes()

confluent_server/confluent/discovery/handlers/bmc.py

@@ -26,16 +26,27 @@ ipmicommand.session.threading = eventlet.green.threading
 ipmicommand.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
-DEFAULT_USER = 'USERID'
-DEFAULT_PASS = 'PASSW0RD'
 
 
 class NodeHandler(generic.NodeHandler):
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'
 
-    def _get_ipmicmd(self, user=DEFAULT_USER, password=DEFAULT_PASS):
-        return ipmicommand.Command(self.ipaddr, user, password)
+    def _get_ipmicmd(self, user=None, password=None):
+        priv = None
+        if user is None or password is None:
+            if self.trieddefault:
+                raise pygexc.IpmiException()
+            priv = 4  # manually indicate priv to avoid double-attempt
+        if user is None:
+            user = self.DEFAULT_USER
+        if password is None:
+            password = self.DEFAULT_PASS
+        return ipmicommand.Command(self.ipaddr, user, password,
+                                   privlevel=priv, keepalive=False)
 
     def __init__(self, info, configmanager):
+        self.trieddefault = None
         super(NodeHandler, self).__init__(info, configmanager)
 
     def probe(self):
@@ -45,37 +56,37 @@ class NodeHandler(generic.NodeHandler):
     def config(self, nodename, reset=False):
         self._bmcconfig(nodename, reset)
 
-    def _bmcconfig(self, nodename, reset=False, customconfig=None):
+    def _bmcconfig(self, nodename, reset=False, customconfig=None, vc=None):
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
         try:
             ic = self._get_ipmicmd()
-            passwd = DEFAULT_PASS
+            passwd = self.DEFAULT_PASS
         except pygexc.IpmiException as pi:
-            creds = self.configmanager.get_node_attributes(
-                nodename,
-                ['secret.hardwaremanagementuser',
-                 'secret.hardwaremanagementpassword'], decrypt=True)
-            user = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementuser', {}).get('value', None)
             havecustomcreds = False
-            if user is not None and user != DEFAULT_USER:
+            if user is not None and user != self.DEFAULT_USER:
                 havecustomcreds = True
             else:
-                user = DEFAULT_USER
-            passwd = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementpassword', {}).get('value', None)
-            if passwd is not None and passwd != DEFAULT_PASS:
+                user = self.DEFAULT_USER
+            if passwd is not None and passwd != self.DEFAULT_PASS:
                 havecustomcreds = True
             else:
-                passwd = DEFAULT_PASS
+                passwd = self.DEFAULT_PASS
             if havecustomcreds:
                 ic = self._get_ipmicmd(user, passwd)
             else:
                 raise
-        if customconfig:
-            customconfig(ic)
+        if vc:
+            ic.register_key_handler(vc)
         currusers = ic.get_users()
         lanchan = ic.get_network_channel()
         userdata = ic.xraw_command(netfn=6, command=0x44, data=(lanchan,
@@ -95,29 +106,6 @@ class NodeHandler(generic.NodeHandler):
             raise exc.TargetEndpointBadCredentials(
                 'secret.hardwaremanagementuser and/or '
                 'secret.hardwaremanagementpassword was not configured')
-        if ('hardwaremanagement.manager' in cd and
-                cd['hardwaremanagement.manager']['value'] and
-                not cd['hardwaremanagement.manager']['value'].startswith(
-                    'fe80::')):
-            newip = cd['hardwaremanagement.manager']['value']
-            newipinfo = getaddrinfo(newip, 0)[0]
-            # This getaddrinfo is repeated in get_nic_config, could be
-            # optimized, albeit with a more convoluted api..
-            newip = newipinfo[-1][0]
-            if ':' in newip:
-                raise exc.NotImplementedException('IPv6 remote config TODO')
-            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
-            plen = netconfig['prefix']
-            newip = '{0}/{1}'.format(newip, plen)
-            ic.set_net_configuration(ipv4_address=newip,
-                                     ipv4_configuration='static',
-                                     ipv4_gateway=netconfig['ipv4_gateway'])
-        elif self.ipaddr.startswith('fe80::'):
-            cfg.set_node_attributes(
-                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
-        else:
-            raise exc.TargetEndpointUnreachable(
-                'hardwaremanagement.manager must be set to desired address')
         newuser = cd['secret.hardwaremanagementuser']['value']
         newpass = cd['secret.hardwaremanagementpassword']['value']
         for uid in currusers:
@@ -126,6 +114,9 @@ class NodeHandler(generic.NodeHandler):
                 newuserslot = uid
                 if newpass != passwd:  # don't mess with existing if no change
                     ic.set_user_password(newuserslot, password=newpass)
+                    ic = self._get_ipmicmd(user, passwd)
+                    if vc:
+                        ic.register_key_handler(vc)
                 break
         else:
             newuserslot = lockedusers + 1
@@ -134,8 +125,14 @@ class NodeHandler(generic.NodeHandler):
             if newpass != passwd:  # don't mess with existing if no change
                 ic.set_user_password(newuserslot, password=newpass)
             ic.set_user_name(newuserslot, newuser)
-            ic.set_user_access(newuserslot, lanchan,
-                               privilege_level='administrator')
+            if havecustomcreds:
+                ic = self._get_ipmicmd(user, passwd)
+                if vc:
+                    ic.register_key_handler(vc)
+            #We are remote operating on the account we are
+            #using, no need to try to set user access
+            #ic.set_user_access(newuserslot, lanchan,
+            #                   privilege_level='administrator')
         # Now to zap others
         for uid in currusers:
             if uid != newuserslot:
@@ -156,6 +153,36 @@ class NodeHandler(generic.NodeHandler):
                             # name...
                             # the user will remain, but that is life
                             raise
+        if customconfig:
+            customconfig(ic)
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            # This getaddrinfo is repeated in get_nic_config, could be
+            # optimized, albeit with a more convoluted api..
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
+            plen = netconfig['prefix']
+            newip = '{0}/{1}'.format(newip, plen)
+            currcfg = ic.get_net_configuration()
+            if currcfg['ipv4_address'] != newip:
+                # do not change the ipv4_config if the current config looks
+                # like it is already accurate
+                ic.set_net_configuration(ipv4_address=newip,
+                                         ipv4_configuration='static',
+                                         ipv4_gateway=netconfig[
+                                             'ipv4_gateway'])
+        elif self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address')
         if reset:
             ic.reset_bmc()
         return ic

confluent_server/confluent/discovery/handlers/cpstorage.py

@@ -0,0 +1,21 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.bmc as bmchandler
+
+class NodeHandler(bmchandler.NodeHandler):
+    DEFAULT_USER = 'admin'
+    DEFAULT_PASS = 'admin'
+    devname = 'BMC'
+    maxmacs = 2

confluent_server/confluent/discovery/handlers/generic.py

@@ -57,7 +57,7 @@ class NodeHandler(object):
         # serial number and uuid to flesh out data as needed
         return
 
-    def preconfig(self):
+    def preconfig(self, possiblenode):
         return
 
     def discoverable_by_switch(self, macs):
@@ -69,6 +69,27 @@ class NodeHandler(object):
         self._fp = certificate
         return True
 
+    def get_node_credentials(self, nodename, creds, defuser, defpass):
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        havecustomcreds = False
+        if user and not isinstance(user, str):
+            user = user.decode('utf8')
+        if user is not None and user != defuser:
+            havecustomcreds = True
+        else:
+            user = defuser
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
+        if passwd and not isinstance(passwd, str):
+            passwd = passwd.decode('utf8')
+        if passwd is not None and passwd != defpass:
+            havecustomcreds = True
+        else:
+            passwd = defpass
+        return user, passwd, not havecustomcreds
+
+
     @property
     def cert_fail_reason(self):
         if self._certfailreason == 1:

confluent_server/confluent/discovery/handlers/imm.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
 import pyghmi.exceptions as pygexc
 import pyghmi.ipmi.private.util as pygutil
-import string
+import confluent.util as util
 import struct
 
 class NodeHandler(bmchandler.NodeHandler):
@@ -25,7 +26,7 @@ class NodeHandler(bmchandler.NodeHandler):
     def adequate(cls, info):
         # We can sometimes receive a partially initialized SLP packet
         # This is not adequate for being satisfied
-        return bool(info['attributes'])
+        return bool(info.get('attributes', {}))
 
     def scan(self):
         slpattrs = self.info.get('attributes', {})
@@ -38,13 +39,14 @@ class NodeHandler(bmchandler.NodeHandler):
         if wronguuid:
             # we need to fix the first three portions of the uuid
             uuidprefix = wronguuid.split('-')[:3]
-            uuidprefix = struct.pack(
-                '<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+            uuidprefix = codecs.encode(struct.pack(
+                '<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+            uuidprefix = util.stringify(uuidprefix)
             uuidprefix = uuidprefix[:8] + '-' + uuidprefix[8:12] + '-' + \
                          uuidprefix[12:16]
             self.info['uuid'] = uuidprefix + '-' + '-'.join(
                 wronguuid.split('-')[3:])
-            self.info['uuid'] = string.lower(self.info['uuid'])
+            self.info['uuid'] = self.info['uuid'].lower()
         if ff not in ('dense-computing', 'BC2'):
             # do not probe unless it's a dense platform
             return

confluent_server/confluent/discovery/handlers/smm.py

@@ -12,21 +12,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
+import confluent.exceptions as exc
+import eventlet
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 import struct
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+import eventlet.support.greendns
+import confluent.netutil as netutil
+import confluent.util as util
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+from xml.etree.ElementTree import fromstring
 
 def fixuuid(baduuid):
     # SMM dumps it out in hex
     uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
-    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode(
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]),
         'hex')
+    a = util.stringify(a)
     uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
     return '-'.join(uuid).lower()
 
 class NodeHandler(bmchandler.NodeHandler):
     is_enclosure = True
     devname = 'SMM'
-    maxmacs = 5  # support an enclosure, but try to avoid catching daisy chain
+    maxmacs = 6  # support an enclosure, but try to avoid catching daisy chain
 
     def scan(self):
         # the UUID is in a weird order, fix it up to match
@@ -36,9 +51,151 @@ class NodeHandler(bmchandler.NodeHandler):
             uuid = fixuuid(uuid[0])
             self.info['uuid'] = uuid
 
+    def _validate_cert(self, certificate):
+        # Assumption is by the time we call config, that discovery core has
+        # vetted self._fp.  Our job here then is just to make sure that
+        # the currect connection matches the previously saved cert
+        if not self._fp:  # circumstances are that we haven't validated yet
+            self._fp = certificate
+        return certificate == self._fp
+
+    def _webconfigrules(self, wc):
+        rules = []
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                rules.append('passwordDurationDays:' + value)
+                warndays = '5' if int(value) > 5 else value
+                rules.append('passwordExpireWarningDays:' + warndays)
+            if name.lower() in ('lockout', 'loginfailures'):
+                rules.append('passwordFailAllowdNum:' + value)
+            if name.lower() == 'reuse':
+                rules.append('passwordReuseCheckNum:' + value)
+        if rules:
+            apirequest = 'set={0}'.format(','.join(rules))
+            wc.request('POST', '/data', apirequest)
+            wc.getresponse().read()
+
+    def _webconfignet(self, wc, nodename):
+        cfg = self.configmanager
+        cd = cfg.get_node_attributes(
+            nodename, ['hardwaremanagement.manager'])
+        smmip = cd.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', None)
+        if smmip and ':' not in smmip:
+            smmip = getaddrinfo(smmip, 0)[0]
+            smmip = smmip[-1][0]
+            if smmip and ':' in smmip:
+                raise exc.NotImplementedException('IPv6 not supported')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=smmip)
+            netmask = netutil.cidr_to_mask(netconfig['prefix'])
+            setdata = 'set=ifIndex:0,v4DHCPEnabled:0,v4IPAddr:{0},v4NetMask:{1}'.format(smmip, netmask)
+            gateway = netconfig.get('ipv4_gateway', None)
+            if gateway:
+                setdata += ',v4Gateway:{0}'.format(gateway)
+            wc.request('POST', '/data', setdata)
+            rsp = wc.getresponse()
+            rspdata = util.stringify(rsp.read())
+            if '<statusCode>0' not in rspdata:
+                raise Exception("Error configuring SMM Network")
+            return
+        if smmip and ':' in smmip and not smmip.startswith('fe80::'):
+            raise exc.NotImplementedException('IPv6 configuration TODO')
+        if self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                    {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+
+    def _webconfigcreds(self, username, password):
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self._validate_cert)
+        wc.connect()
+        authdata = {  # start by trying factory defaults
+            'user': 'USERID',
+            'password': 'PASSW0RD',
+        }
+        headers = {'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded'}
+        wc.request('POST', '/data/login', urlencode(authdata), headers)
+        rsp = wc.getresponse()
+        rspdata = util.stringify(rsp.read())
+        if 'authResult>0' not in rspdata:
+            # default credentials are refused, try with the actual
+            authdata['user'] = username
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = util.stringify(rsp.read())
+            if 'renew_account' in rspdata:
+                raise Exception('Configured password has expired')
+            if 'authResult>0' not in rspdata:
+                raise Exception('Unknown username/password on SMM')
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+        if 'renew_account' in rspdata:
+            passwdchange = {'oripwd': 'PASSW0RD', 'newpwd': password}
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            wc.request('POST', '/data/changepwd', urlencode(passwdchange))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = util.stringify(rsp.read())
+        if 'authResult>0' in rspdata:
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            if username == 'USERID':
+                return wc
+            wc.request('POST', '/data', 'set=user(2,1,{0},511,,4,15,0)'.format(username))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            wc.request('POST', '/data/logout')
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['user'] = username
+            wc.request('POST', '/data/login', urlencode(authdata, headers))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+
     def config(self, nodename):
         # SMM for now has to reset to assure configuration applies
-        super(NodeHandler, self).config(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        username = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', 'USERID')
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', 'PASSW0RD')
+        if not isinstance(username, str):
+            username = username.decode('utf8')
+        if not isinstance(passwd, str):
+            passwd = passwd.decode('utf8')
+        if passwd == 'PASSW0RD' and self.ruleset:
+            raise Exception('Cannot support default password and setting password rules at same time')
+        if passwd == 'PASSW0RD':
+            # We must avoid hitting the web interface due to forced password change, best effert
+            self._bmcconfig(nodename)
+        else:
+            # Switch to full web based configuration, to mitigate risks with the SMM
+            wc = self._webconfigcreds(username, passwd)
+            self._webconfigrules(wc)
+            self._webconfignet(wc, nodename)
+
 
 # notes for smm:
 # POST to:
@@ -53,4 +210,4 @@ class NodeHandler(bmchandler.NodeHandler):
 # with body user=USERID&password=Passw0rd!4321
 # yields:
 # <?xml version="1.0" encoding="UTF-8"?><root> <status>ok</status> <authResult>0</authResult> <forwardUrl>index.html</forwardUrl> </root>
-# note forwardUrl, if password change needed, will indicate something else
+# note forwardUrl, if password change needed, will indicate something else

confluent_server/confluent/discovery/handlers/tsm.py

@@ -0,0 +1,221 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.generic as generic
+import confluent.exceptions as exc
+import confluent.netutil as netutil
+import confluent.util as util
+import eventlet.support.greendns
+import json
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+class NodeHandler(generic.NodeHandler):
+    devname = 'TSM'
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'
+
+    def __init__(self, info, configmanager):
+        self.trieddefault = None
+        self.targuser = None
+        self.curruser = None
+        self.currpass = None
+        self.targpass = None
+        self.nodename = None
+        self.csrftok = None
+        self.channel = None
+        self.atdefault = True
+        super(NodeHandler, self).__init__(info, configmanager)
+
+    def scan(self):
+        c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert)
+        i = c.grab_json_response('/redfish/v1/')
+        uuid = i.get('UUID', None)
+        if uuid:
+            self.info['uuid'] = uuid
+
+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def _get_wc(self):
+        authdata = {  # start by trying factory defaults
+            'username': self.DEFAULT_USER,
+            'password': self.DEFAULT_PASS,
+        }
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert)
+        wc.set_header('Content-Type', 'application/json')
+        authmode = 0
+        if not self.trieddefault:
+            rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+            if status == 403:
+                wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+                authmode = 1
+                rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            else:
+                authmode = 2
+            if status > 400:
+                rsp = util.stringify(rsp)
+                self.trieddefault = True
+                if '555' in rsp:
+                    passchange = {
+                        'Password': self.targpass, 
+                        'RetypePassword': self.targpass,
+                        'param': 4,
+                        'default_password': self.DEFAULT_PASS,
+                        'username': self.DEFAULT_USER
+                        }
+                    if authmode == 2:
+                        rsp, status = wc.grab_json_response_with_status('/api/reset-pass', passchange)
+                    else:
+                        rsp, status = wc.grab_json_response_with_status('/api/reset-pass', urlencode(passchange))
+                    authdata['password'] = self.targpass
+                    if authmode == 2:
+                        rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+                    else:
+                        rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+                    self.csrftok = rsp['CSRFToken']
+                    self.channel = rsp['channel']
+                    self.curruser = self.DEFAULT_USER
+                    self.currpass = self.targpass
+                    return wc
+            else:
+                self.curruser = self.DEFAULT_USER
+                self.currpass = self.DEFAULT_PASS
+                self.csrftok = rsp['CSRFToken']
+                self.channel = rsp['channel']
+                return wc
+        if self.curruser:
+            authdata['username'] = self.curruser
+            authdata['password'] = self.currpass
+            if authmode != 1:
+                rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+            if authmode == 1 or status == 403:
+                wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+                rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            if status != 200:
+                return None
+            self.csrftok = rsp['CSRFToken']
+            self.channel = rsp['channel']
+            return wc
+        authdata['username'] = self.targuser
+        authdata['password'] = self.targpass
+        if authmode != 1:
+            rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+        if authmode == 1 or status == 403:
+            wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+            rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+        if status != 200:
+            return None
+        self.curruser = self.targuser
+        self.currpass = self.targpass
+        self.csrftok = rsp['CSRFToken']
+        self.channel = rsp['channel']
+        return wc
+
+    def config(self, nodename):
+        self.nodename = nodename
+        creds = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
+        cd = creds.get(nodename, {})
+        user, passwd, _ = self.get_node_credentials(
+                nodename, creds, self.DEFAULT_USER, self.DEFAULT_PASS)
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
+        self.targuser = user
+        self.targpass = passwd
+        wc = self._get_wc()
+        wc.set_header('X-CSRFTOKEN', self.csrftok)
+        curruserinfo = {}
+        authupdate = False
+        wc.set_header('Content-Type', 'application/json')
+        if user != self.curruser:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['name'] = user
+        if passwd != self.currpass:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['changepassword'] = 0
+            authchg['password_size'] = 'bytes_20'
+            authchg['password'] = passwd
+            authchg['confirm_password'] = passwd
+        if authupdate:
+            rsp, status = wc.grab_json_response_with_status('/api/settings/users/2', authchg, method='PUT')
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            # IPMI must be enabled per user config
+            wc.grab_json_response('/api/settings/ipmilanconfig', {
+                'ipv4_enable': 1, 'ipv6_enable': 1,
+                'uncheckedipv4lanEnable': 0, 'uncheckedipv6lanEnable': 0,
+                'checkedipv4lanEnable': 1, 'checkedipv6lanEnable': 1})
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            currnet = wc.grab_json_response('/api/settings/network')
+            for net in currnet:
+                if net['channel_number'] == self.channel and net['lan_enable'] == 0:
+                    # ignore false indication and switch to 8 (dedicated)
+                    self.channel = 8
+                if net['channel_number'] == self.channel:
+                    # we have found the interface to potentially manipulate
+                    if net['ipv4_address'] != newip:
+                        netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+                        newmask = netutil.cidr_to_mask(netconfig['prefix'])
+                        net['ipv4_address'] = newip
+                        net['ipv4_subnet'] = newmask
+                        if netconfig['ipv4_gateway']:
+                            net['ipv4_gateway'] = netconfig['ipv4_gateway']
+                        net['ipv4_dhcp_enable'] = 0
+                        rsp, status = wc.grab_json_response_with_status(
+                            '/api/settings/network/{0}'.format(net['id']), net, method='PUT')
+                    break
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        rsp, status = wc.grab_json_response_with_status('/api/session', method='DELETE')
+
+
+if __name__ == '__main__':
+    import confluent.config.configmanager as cfm
+    c = cfm.ConfigManager(None)
+    import sys
+    info = {'addresses': [[sys.argv[1]]] }
+    print(repr(info))
+    testr = NodeHandler(info, c)
+    testr.config(sys.argv[2])

confluent_server/confluent/discovery/handlers/xcc.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import codecs
 import confluent.discovery.handlers.imm as immhandler
+import confluent.exceptions as exc
+import confluent.netutil as netutil
 import confluent.util as util
+import errno
+import eventlet
+import eventlet.support.greendns
+import json
+import os
 import pyghmi.exceptions as pygexc
-import pyghmi.ipmi.oem.lenovo.imm as imm
+import eventlet.green.socket as socket
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import struct
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+
+def fixup_uuid(uuidprop):
+    baduuid = ''.join(uuidprop.split())
+    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+    a = util.stringify(a)
+    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
+    return '-'.join(uuid).upper()
 
 
 
@@ -23,27 +44,71 @@ import pyghmi.ipmi.oem.lenovo.imm as imm
 class NodeHandler(immhandler.NodeHandler):
     devname = 'XCC'
 
+    def __init__(self, info, configmanager):
+        self._wc = None
+        self.nodename = None
+        self.tmpnodename = None
+        self.tmppasswd = None
+        self._atdefaultcreds = True
+        self._needpasswordchange = True
+        self._currcreds = (None, None)
+        super(NodeHandler, self).__init__(info, configmanager)
+
     @classmethod
     def adequate(cls, info):
         # We can sometimes receive a partially initialized SLP packet
         # This is not adequate for being satisfied
         return bool(info.get('attributes', {}))
 
-    def preconfig(self):
+    def preconfig(self, possiblenode):
+        self.tmpnodename = possiblenode
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
+            # skip preconfig for non-SD530 servers
             return
+        self.trieddefault = None  # Reset state on a preconfig attempt
         # attempt to enable SMM
         #it's normal to get a 'not supported' (193) for systems without an SMM
+        # need to branch on 3.00+ firmware
+        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
+        currfirm = currfirm.split(':')
+        if len(currfirm) > 1:
+            currfirm = float(currfirm[1])
+        disableipmi = False
+        if currfirm >= 3:
+            # IPMI is disabled and we need it, also we need to go to *some* password
+            wc = self.wc
+            if not wc:
+                # We cannot try to enable SMM here without risking real credentials
+                # on the wire to untrusted parties
+                return
+            wc.grab_json_response('/api/providers/logout')
+            wc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = wc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                disableipmi = True
+                _, _ = wc.grab_json_response_with_status(
+                    '/redfish/v1/Managers/1/NetworkProtocol',
+                    {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
         ipmicmd = None
         try:
-            ipmicmd = self._get_ipmicmd()
+            ipmicmd = self._get_ipmicmd(self._currcreds[0], self._currcreds[1])
             ipmicmd.xraw_command(netfn=0x3a, command=0xf1, data=(1,))
         except pygexc.IpmiException as e:
             if (e.ipmicode != 193 and 'Unauthorized name' not in str(e) and
-                    'Incorrect password' not in str(e)):
+                    'Incorrect password' not in str(e) and 
+                    str(e) != 'Session no longer connected'):
                 # raise an issue if anything other than to be expected
+                if disableipmi:
+                    _, _ = wc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
                 raise
+            self.trieddefault = True
+        if disableipmi:
+            _, _ = wc.grab_json_response_with_status(
+                '/redfish/v1/Managers/1/NetworkProtocol',
+                {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
         #TODO: decide how to clean out if important
         #as it stands, this can step on itself
         #if ipmicmd:
@@ -55,9 +120,122 @@ class NodeHandler(immhandler.NodeHandler):
         fprint = util.get_fingerprint(self.https_cert)
         return util.cert_matches(fprint, certificate)
 
-    def set_password_policy(self, ic):
+    def get_webclient(self, username, password, newpassword):
+        wc = self._wc.dupe()
+        try:
+            wc.connect()
+        except socket.error as se:
+            if se.errno != errno.ECONNREFUSED:
+                raise
+            return (None, None)
+        pwdchanged = False
+        adata = json.dumps({'username': util.stringify(username),
+                            'password': util.stringify(password)
+                            })
+        headers = {'Connection': 'keep-alive',
+                   'Content-Type': 'application/json'}
+        wc.request('POST', '/api/login', adata, headers)
+        rsp = wc.getresponse()
+        if rsp.status != 200 and password == 'PASSW0RD':
+            rsp.read()
+            adata = json.dumps({
+                'username': username,
+                'password': newpassword,
+                })
+            headers = {'Connection': 'keep-alive',
+                       'Content-Type': 'application/json'}
+            wc.request('POST', '/api/login', adata, headers)
+            rsp = wc.getresponse()
+            if rsp.status == 200:
+                pwdchanged = True
+                password = newpassword
+            else:
+                rsp.read()
+                return (None, None)
+        if rsp.status == 200:
+            self._currcreds = (username, password)
+            wc.set_basic_credentials(username, password)
+            rspdata = json.loads(rsp.read())
+            wc.set_header('Content-Type', 'application/json')
+            wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            if rspdata.get('pwchg_required', None) == 'true':
+                wc.request('POST', '/api/function', json.dumps(
+                    {'USER_UserPassChange': '1,{0}'.format(newpassword)}))
+                rsp = wc.getresponse()
+                rsp.read()
+                if rsp.status != 200:
+                    return (None, None)
+                self._currcreds = (username, newpassword)
+                wc.set_basic_credentials(username, newpassword)
+                pwdchanged = True
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            if pwdchanged:
+                # Remove the minimum change interval, to allow sane 
+                # password changes after provisional changes
+                self.set_password_policy('')
+            return (wc, pwdchanged)
+        return (None, None)
+
+    @property
+    def wc(self):
+        passwd = None
+        isdefault = True
+        if self._wc is None:
+            self._wc = webclient.SecureHTTPConnection(
+                self.ipaddr, 443, verifycallback=self.validate_cert)
+            self._wc.connect()
+        nodename = None
+        if self.nodename:
+            nodename = self.nodename
+            inpreconfig = False
+        elif self.tmpnodename:
+            nodename = None
+            inpreconfig = True
+        if self._currcreds[0] is not None:
+            wc, pwdchanged = self.get_webclient(self._currcreds[0], self._currcreds[1], None)
+            if wc:
+                return wc
+        if nodename:
+            creds = self.configmanager.get_node_attributes(
+                nodename, ['secret.hardwaremanagementuser',
+                'secret.hardwaremanagementpassword'], decrypt=True)
+            user, passwd, isdefault = self.get_node_credentials(
+                nodename, creds, 'USERID', 'PASSW0RD')
+        if not self.trieddefault:
+            if not passwd:
+                # So in preconfig context, we don't have admin permission to
+                # actually divulge anything to the target
+                # however the target *will* demand a new password... if it's currently
+                # PASSW0RD
+                # use TempW0rd42 to avoid divulging a real password on the line
+                # This is replacing one well known password (PASSW0RD) with another
+                # (TempW0rd42)
+                passwd = 'TempW0rd42'
+            wc, pwdchanged = self.get_webclient('USERID', 'PASSW0RD', passwd)
+            if wc:
+                if pwdchanged:
+                    if inpreconfig:
+                        self.tmppasswd = passwd
+                    else:
+                        self._needpasswordchange = False
+                return wc
+        self.trieddefault = True
+        if isdefault:
+            return
+        self._atdefaultcreds = False
+        if self.tmppasswd:
+            wc, _ = self.get_webclient('USERID', self.tmppasswd, passwd)
+        else:
+            wc, _ = self.get_webclient(user, passwd, None)
+        if wc:
+            return wc
+
+    def set_password_policy(self, strruleset):
         ruleset = {'USER_GlobalMinPassChgInt': '0'}
-        for rule in self.ruleset.split(','):
+        for rule in strruleset.split(','):
             if '=' not in rule:
                 continue
             name, value = rule.split('=')
@@ -71,29 +249,181 @@ class NodeHandler(immhandler.NodeHandler):
                 if value.lower() in ('no', 'none', 'disable', 'disabled'):
                     value = '0'
                 ruleset['USER_GlobalMaxLoginFailures'] = value
-        ic.register_key_handler(self.validate_cert)
-        ic.oem_init()
-        ic._oem.immhandler.wc.grab_json_response('/api/dataset', ruleset)
+            if name.lower() == 'complexity':
+                ruleset['USER_GlobalPassComplexRequired'] = value
+            if name.lower() == 'reuse':
+                ruleset['USER_GlobalMinPassReuseCycle'] = value
+        try:
+            wc = self.wc
+            wc.grab_json_response('/api/dataset', ruleset)
+            wc.grab_json_response('/api/providers/logout')
+        except Exception as e:
+            print(repr(e))
+            pass
+
+    def _get_next_userid(self, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        userinfo = userinfo['items'][0]['users']
+        for user in userinfo:
+            if user['users_user_name'] == '':
+                return user['users_user_id']
+
+    def _setup_xcc_account(self, username, passwd, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        uid = None
+        for user in userinfo['items'][0]['users']:
+            if user['users_user_name'] == username:
+                uid = user['users_user_id']
+                break
+        else:
+            for user in userinfo['items'][0]['users']:
+                if user['users_user_name'] == 'USERID':
+                    uid = user['users_user_id']
+                    break
+        if not uid:
+            raise Exception("XCC has neither the default user nor configured user")
+        # The following will work if the password is force change or normal..
+        if self._needpasswordchange and self.tmppasswd != passwd:
+            wc.grab_json_response('/api/function',
+                                {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
+        if username != 'USERID':
+            wc.grab_json_response(
+                '/api/function',
+                {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,'.format(uid, username)})
+            self.tmppasswd = None
+        self._currcreds = (username, passwd)
+
+    def _convert_sha256account(self, user, passwd, wc):
+        # First check if the specified user is sha256...
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        curruser = None
+        uid = None
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
+        for userent in userinfo['items'][0]['users']:
+            if userent['users_user_name'] == user:
+                curruser = userent
+                break
+        if curruser.get('users_pass_is_sha256', 0):
+            self._wc = None
+            wc = self.wc
+            nwc = wc.dupe()
+            # Have to convert it for being useful with most Lenovo automation tools
+            # This requires deleting the account entirely and trying again
+            tmpuid = self._get_next_userid(wc)
+            try:
+                tpass = base64.b64encode(os.urandom(9)) + 'Iw47$'
+                userparams = "{0},6pmu0ezczzcp,{1},1,4,0,0,0,0,,8,".format(tmpuid, tpass)
+                result = wc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                wc.grab_json_response('/api/providers/logout')
+                adata = json.dumps({
+                    'username': '6pmu0ezczzcp',
+                    'password': tpass,
+                })
+                headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+                nwc.request('POST', '/api/login', adata, headers)
+                rsp = nwc.getresponse()
+                if rsp.status == 200:
+                    rspdata = json.loads(rsp.read())
+                    nwc.set_header('Content-Type', 'application/json')
+                    nwc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+                    if '_csrf_token' in wc.cookies:
+                        nwc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+                    if rspdata.get('reason', False):
+                        newpass = base64.b64encode(os.urandom(9)) + 'q4J$'
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(tmpuid, newpass)})
+                    nwc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(curruser['users_user_id'], user)})
+                    userparams = "{0},{1},{2},1,4,0,0,0,0,,8,".format(curruser['users_user_id'], user, tpass)
+                    nwc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                    nwc.grab_json_response('/api/providers/logout')
+                    nwc, pwdchanged = self.get_webclient(user, tpass, passwd)
+                    if not pwdchanged:
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(curruser['users_user_id'], passwd)})
+                    nwc.grab_json_response('/api/providers/logout')
+            finally:
+                self._wc = None
+                wc = self.wc
+                wc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(tmpuid, '6pmu0ezczzcp')})
+                wc.grab_json_response('/api/providers/logout')
 
     def config(self, nodename, reset=False):
+        self.nodename = nodename
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
         dpp = self.configmanager.get_node_attributes(
             nodename, 'discovery.passwordrules')
-        self.ruleset = dpp.get(nodename, {}).get(
+        strruleset = dpp.get(nodename, {}).get(
             'discovery.passwordrules', {}).get('value', '')
-        ic = self._bmcconfig(nodename, customconfig=self.set_password_policy)
+        wc = self.wc
+        creds = self.configmanager.get_node_attributes(
+            self.nodename, ['secret.hardwaremanagementuser',
+            'secret.hardwaremanagementpassword'], decrypt=True)
+        user, passwd, isdefault = self.get_node_credentials(nodename, creds, 'USERID', 'PASSW0RD')
+        self.set_password_policy(strruleset)
+        if self._atdefaultcreds:
+            if isdefault and self.tmppasswd:
+                raise Exception(
+                    'Request to use default credentials, but refused by target after it has been changed to {0}'.format(self.tmppasswd))
+            if not isdefault:
+                self._setup_xcc_account(user, passwd, wc)
+        self._convert_sha256account(user, passwd, wc)
+        cd = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
+        cd = cd.get(nodename, {})
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            nwc = wc.dupe()
+            nwc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = nwc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                # User has indicated IPMI support, but XCC is currently disabled
+                # change XCC to be consistent
+                _, _ = nwc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+            newmask = netutil.cidr_to_mask(netconfig['prefix'])
+            currinfo = wc.grab_json_response('/api/providers/logoninfo')
+            currip = currinfo.get('items', [{}])[0].get('ipv4_address', '')
+            # do not change the ipv4_config if the current config looks right already
+            if currip != newip:
+                statargs = {
+                    'ENET_IPv4Ena': '1', 'ENET_IPv4AddrSource': '0',
+                    'ENET_IPv4StaticIPAddr': newip, 'ENET_IPv4StaticIPNetMask': newmask
+                    }
+                if netconfig['ipv4_gateway']:
+                    statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
+                wc.grab_json_response('/api/dataset', statargs)
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        wc.grab_json_response('/api/providers/logout')
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return
-        # Ok, we can get the enclosure uuid now..
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
+        enclosureuuid = self.info.get('attributes', {}).get('chassis-uuid', [None])[0]
         if enclosureuuid:
-            enclosureuuid = imm.fixup_uuid(enclosureuuid).lower()
+            enclosureuuid = enclosureuuid.lower()
             em = self.configmanager.get_node_attributes(nodename,
                                                         'enclosure.manager')
             em = em.get(nodename, {}).get('enclosure.manager', {}).get(
@@ -102,8 +432,3 @@ class NodeHandler(immhandler.NodeHandler):
             if em:
                 self.configmanager.set_node_attributes(
                     {em: {'id.uuid': enclosureuuid}})
-
-# TODO(jjohnson2): web based init config for future prevalidated cert scheme
-#    def config(self, nodename):
-#        return
-

confluent_server/confluent/discovery/protocols/pxe.py

@@ -34,8 +34,8 @@ pxearchs = {
 
 
 def decode_uuid(rawguid):
-    lebytes = struct.unpack_from('<IHH', buffer(rawguid[:8]))
-    bebytes = struct.unpack_from('>HHI', buffer(rawguid[8:]))
+    lebytes = struct.unpack_from('<IHH', rawguid[:8])
+    bebytes = struct.unpack_from('>HHI', rawguid[8:])
     return '{0:08X}-{1:04X}-{2:04X}-{3:04X}-{4:04X}{5:08X}'.format(
         lebytes[0], lebytes[1], lebytes[2], bebytes[0], bebytes[1], bebytes[2]).lower()
 
@@ -98,7 +98,7 @@ def snoop(handler, protocol=None):
             netaddr = ':'.join(['{0:02x}'.format(x) for x in netaddr])
             optidx = 0
             try:
-                optidx = rq.index('\x63\x82\x53\x63') + 4
+                optidx = rq.index(b'\x63\x82\x53\x63') + 4
             except ValueError:
                 continue
             uuid, arch = find_info_in_options(rq, optidx)

confluent_server/confluent/discovery/protocols/slp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ import traceback
 _slp_services = set([
     'service:management-hardware.IBM:integrated-management-module2',
     'service:lenovo-smm',
+    'service:ipmi',
+    'service:lighttpd',
     'service:management-hardware.Lenovo:lenovo-xclarity-controller',
     'service:management-hardware.IBM:chassis-management-module',
     'service:management-hardware.Lenovo:chassis-management-module',
@@ -47,7 +49,6 @@ except AttributeError:
     IPPROTO_IPV6 = 41  # Assume Windows value if socket is missing it
 
 
-
 def _parse_slp_header(packet):
     packet = bytearray(packet)
     if len(packet) < 16 or packet[0] != 2:
@@ -83,6 +84,8 @@ def _parse_SrvRply(parsed):
     :return:
     """
     payload = parsed['payload']
+    if len(payload) < 4:
+        return
     ecode, ucount = struct.unpack('!HH', bytes(payload[0:4]))
     if ecode:
         parsed['errorcode'] = ecode
@@ -233,36 +236,42 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
 
 
 def _grab_rsps(socks, rsps, interval, xidmap):
-    r, _, _ = select.select(socks, (), (), interval)
+    r = None
+    res = select.select(socks, (), (), interval)
+    if res:
+        r = res[0]
     while r:
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
             neighutil.refresh_neigh()
             _parse_slp_packet(rsp, peer, rsps, xidmap)
-            r, _, _ = select.select(socks, (), (), interval)
+            res = select.select(socks, (), (), interval)
+            if not res:
+                r = None
+            else:
+                r = res[0]
 
 
 
 def _parse_attrlist(attrstr):
     attribs = {}
+    previousattrlen = None
+    attrstr = util.stringify(attrstr)
     while attrstr:
+        if len(attrstr) == previousattrlen:
+            raise Exception('Looping in attrstr parsing')
+        previousattrlen = len(attrstr)
         if attrstr[0] == '(':
             if ')' not in attrstr:
                 attribs['INCOMPLETE'] = True
                 return attribs
             currattr = attrstr[1:attrstr.index(')')]
             if '=' not in currattr:  # Not allegedly kosher, but still..
-                currattr = currattr.decode('utf-8')
                 attribs[currattr] = None
             else:
                 attrname, attrval = currattr.split('=', 1)
-                attrname = attrname.decode('utf-8')
                 attribs[attrname] = []
                 for val in attrval.split(','):
-                    try:
-                        val = val.decode('utf-8')
-                    except UnicodeDecodeError:
-                        val = '*DECODEERROR*'
                     if val[:3] == '\\FF':  # we should make this bytes
                         finalval = bytearray([])
                         for bnum in attrval[3:].split('\\'):
@@ -272,9 +281,9 @@ def _parse_attrlist(attrstr):
                         val = finalval
                         if 'uuid' in attrname and len(val) == 16:
                             lebytes = struct.unpack_from(
-                                '<IHH', buffer(val[:8]))
+                                '<IHH', memoryview(val[:8]))
                             bebytes = struct.unpack_from(
-                                '>HHI', buffer(val[8:]))
+                                '>HHI', memoryview(val[8:]))
                             val = '{0:08X}-{1:04X}-{2:04X}-{3:04X}-' \
                                   '{4:04X}{5:08X}'.format(
                                 lebytes[0], lebytes[1], lebytes[2], bebytes[0],
@@ -282,7 +291,7 @@ def _parse_attrlist(attrstr):
                             ).lower()
                     attribs[attrname].append(val)
             attrstr = attrstr[attrstr.index(')'):]
-        elif attrstr[0] == ',':
+        elif attrstr[0] == ','[0]:
             attrstr = attrstr[1:]
         elif ',' in attrstr:
             currattr = attrstr[:attrstr.index(',')]
@@ -335,6 +344,7 @@ def _add_attributes(parsed):
     else:
         net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
+        net.settimeout(1.0)
         net.connect(target)
     except socket.error:
         return
@@ -363,6 +373,7 @@ def query_srvtypes(target):
     while tries and not connected:
         tries -= 1
         try:
+            net.settimeout(1.0)
             net.connect(target)
             connected = True
         except socket.error:
@@ -482,6 +493,18 @@ def snoop(handler, protocol=None):
                 _add_attributes(peerbymacaddress[mac])
                 peerbymacaddress[mac]['hwaddr'] = mac
                 peerbymacaddress[mac]['protocol'] = protocol
+                if 'service:ipmi' in peerbymacaddress[mac]['services']:
+                    if 'service:ipmi//Athena:623' in peerbymacaddress[mac].get('urls', ()):
+                        peerbymacaddress[mac]['services'] = ['service:thinkagile-storage']
+                    else:
+                        continue
+                if 'service:lightttpd' in peerbymacaddress[mac]['services']:
+                    currinf = peerbymacaddress[mac]
+                    curratt = currinf.get('attributes', {})
+                    if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+                        peerbymacaddress[mac]['services'] = ['service:lenovo-tsm']
+                    else:
+                        continue
                 handler(peerbymacaddress[mac])
         except Exception as e:
             tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
@@ -545,6 +568,11 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
     _grab_rsps((net, net4), rsps, 1, xidmap)
     # now to analyze and flesh out the responses
     for id in rsps:
+        if 'service:ipmi' in rsps[id]['services']:
+            if 'service:ipmi://Athena:623' in rsps[id].get('urls', ''):
+                rsps[id]['services'] = ['service:thinkagile-storage']
+            else:
+                continue
         if localonly:
             for addr in rsps[id]['addresses']:
                 if 'fe80' in addr[0]:
@@ -552,6 +580,15 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
             else:
                 continue
         _add_attributes(rsps[id])
+        if 'service:lighttpd' in rsps[id]['services']:
+            currinf = rsps[id]
+            curratt = currinf.get('attributes', {})
+            if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+               currinf['services'] = ['service:lenovo-tsm']
+               curratt['enclosure-serial-number'] = curratt['Product-Serial']
+               curratt['enclosure-machinetype-model'] = curratt['Machine-Type']
+            else:
+                continue
         del rsps[id]['payload']
         del rsps[id]['function']
         del rsps[id]['xid']

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -78,8 +78,13 @@ def snoop(handler, byehandler=None):
     for i4 in util.list_ips():
         ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
                      socket.inet_aton(i4['addr'])
-        net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
-                        ssdp4mcast)
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            ssdp4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     net4.bind(('', 1900))
     net6.bind(('', 1900))
@@ -167,6 +172,7 @@ def _find_service(service, target):
             net4.sendto(smsg.format(bcast, service), (bcast, 1900))
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
+    deadline = util.monotonic_time() + 4
     r, _, _ = select.select((net4, net6), (), (), 4)
     peerdata = {}
     while r:
@@ -174,7 +180,10 @@ def _find_service(service, target):
             (rsp, peer) = s.recvfrom(9000)
             neighutil.refresh_neigh()
             _parse_ssdp(peer, rsp, peerdata)
-        r, _, _ = select.select((net4, net6), (), (), 4)
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
     for nid in peerdata:
         yield peerdata[nid]
 
@@ -194,6 +203,8 @@ def _parse_ssdp(peer, rsp, peerdata):
     if code == '200':
         if nid in peerdata:
             peerdatum = peerdata[nid]
+            if peer not in peerdatum['peers']:
+                peerdatum['peers'].append(peer)
         else:
             peerdatum = {
                 'peers': [peer],

confluent_server/confluent/exceptions.py

@@ -107,11 +107,14 @@ class PubkeyInvalid(ConfluentException):
         self.fingerprint = fingerprint
         self.attrname = attribname
         self.message = text
+        certtxt = base64.b64encode(certificate)
+        if not isinstance(certtxt, str):
+            certtxt = certtxt.decode('utf8')
         bodydata = {'message': text,
                     'event': event,
                     'fingerprint': fingerprint,
                     'fingerprintfield': attribname,
-                    'certificate': base64.b64encode(certificate)}
+                    'certificate': certtxt}
         self.errorbody = json.dumps(bodydata)
 
     def get_error_body(self):

confluent_server/confluent/firmwaremanager.py

@@ -19,19 +19,23 @@
 # the time comes
 
 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.messages as msg
 import eventlet
 import os
 import pwd
 import socket
+import traceback
 
 updatesbytarget = {}
 uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
+_tracelog = None
 
 
 def execupdate(handler, filename, updateobj, type, owner, node):
+    global _tracelog
     if type != 'ffdc' and not os.path.exists(filename):
         errstr =  '{0} does not appear to exist on {1}'.format(
             filename, socket.gethostname())
@@ -39,13 +43,16 @@ def execupdate(handler, filename, updateobj, type, owner, node):
                                    'detail': errstr})
         return
     if type == 'ffdc' and os.path.isdir(filename):
-        filename += '/' + node + '.svcdata'
+        filename += '/' + node
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
                                  bank=updateobj.bank)
         else:
             completion = handler(filename, progress=updateobj.handle_progress)
+        if type == 'ffdc' and completion:
+            filename = completion
+            completion = None
         if completion is None:
             completion = 'complete'
         if owner:
@@ -58,6 +65,9 @@ def execupdate(handler, filename, updateobj, type, owner, node):
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': errstr})
     except Exception as e:
+        if _tracelog is None:
+            _tracelog = log.Logger('trace')
+        _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': str(e)})
 

confluent_server/confluent/forwarder.py

@@ -49,6 +49,7 @@ def forward_port(sock, target, clientip, sessionid):
             continue
         try:
             client = socket.create_connection((target, 443))
+            client.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
         except Exception:
             conn.close()
             continue
@@ -68,6 +69,7 @@ def forward_video():
         try:
             vidclient = socket.create_connection((vidtargetbypeer[cli[0]],
                                                   3900))
+            vidclient.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
         except Exception:
             conn.close()
             continue

confluent_server/confluent/httpapi.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,10 @@
 # This SCGI server provides a http wrap to confluent api
 # It additionally manages httprequest console sessions
 import base64
-import Cookie
+try:
+    import Cookie
+except ModuleNotFoundError:
+    import http.cookies as Cookie
 import confluent.auth as auth
 import confluent.config.attributes as attribs
 import confluent.consoleserver as consoleserver
@@ -39,7 +42,10 @@ import socket
 import sys
 import traceback
 import time
-import urlparse
+try:
+    import urlparse
+except ModuleNotFoundError:
+    import urllib.parse as urlparse
 import eventlet.wsgi
 #scgi = eventlet.import_patched('flup.server.scgi')
 tlvdata = confluent.tlvdata
@@ -74,7 +80,7 @@ def group_creation_resources():
     yield confluent.messages.ListAttributes(kv={'nodes': []},
                                             desc='Nodes to add to the group'
                                             ).html() + '<br>\n'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
         if attr == 'groups':
             continue
         if attr.startswith("secret."):
@@ -95,7 +101,7 @@ def group_creation_resources():
 def node_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the node").html() + '<br>'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
         if attr.startswith("secret."):
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
@@ -126,7 +132,7 @@ def user_creation_resources():
             'description': (''),
         },
     }
-    for attr in sorted(credential.iterkeys()):
+    for attr in sorted(credential):
         if attr == "password":
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
@@ -176,7 +182,7 @@ def _get_query_dict(env, reqbody, reqtype):
     if reqbody is not None:
         if "application/x-www-form-urlencoded" in reqtype:
             pbody = urlparse.parse_qs(reqbody, True)
-            for ky in pbody.iterkeys():
+            for ky in pbody:
                 if len(pbody[ky]) > 1:  # e.g. REST explorer
                     na = [i for i in pbody[ky] if i != '']
                     qdict[ky] = na
@@ -184,7 +190,7 @@ def _get_query_dict(env, reqbody, reqtype):
                     qdict[ky] = pbody[ky][0]
         elif 'application/json' in reqtype:
             pbody = json.loads(reqbody)
-            for key in pbody.iterkeys():
+            for key in pbody:
                 qdict[key] = pbody[key]
     if 'restexplorerhonorkey' in qdict:
         nqdict = {}
@@ -269,6 +275,9 @@ def _authorize_request(env, operation):
     name = ''
     sessionid = None
     cookie = Cookie.SimpleCookie()
+    element = env['PATH_INFO']
+    if element.startswith('/sessions/current/'):
+        element = None
     if 'HTTP_COOKIE' in env:
         #attempt to use the cookie.  If it matches
         cc = RobustCookie()
@@ -290,7 +299,7 @@ def _authorize_request(env, operation):
                     httpsessions[sessionid]['expiry'] = time.time() + 90
                     name = httpsessions[sessionid]['name']
                     authdata = auth.authorize(
-                        name, element=None,
+                        name, element=element, operation=operation,
                         skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
         if env['PATH_INFO'] == '/sessions/current/logout':
@@ -302,9 +311,11 @@ def _authorize_request(env, operation):
                 return {'code': 401}
             return ('logout',)
         name, passphrase = base64.b64decode(
-            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
-        authdata = auth.check_user_passphrase(name, passphrase, element=None)
-        if not authdata:
+            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(b':', 1)
+        authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
+        if authdata is False:
+            return {'code': 403}
+        elif not authdata:
             return {'code': 401}
         sessid = util.randomstring(32)
         while sessid in httpsessions:
@@ -314,14 +325,14 @@ def _authorize_request(env, operation):
                                 'inflight': set([])}
         if 'HTTP_CONFLUENTAUTHTOKEN' in env:
             httpsessions[sessid]['csrftoken'] = util.randomstring(32)
-        cookie['confluentsessionid'] = sessid
+        cookie['confluentsessionid'] = util.stringify(sessid)
         cookie['confluentsessionid']['secure'] = 1
         cookie['confluentsessionid']['httponly'] = 1
         cookie['confluentsessionid']['path'] = '/'
     skiplog = _should_skip_authlog(env)
     if authdata:
         auditmsg = {
-            'user': name,
+            'user': util.stringify(name),
             'operation': operation,
             'target': env['PATH_INFO'],
         }
@@ -333,7 +344,7 @@ def _authorize_request(env, operation):
         if authdata[3] is not None:
             auditmsg['tenant'] = authdata[3]
             authinfo['tenant'] = authdata[3]
-        auditmsg['user'] = authdata[2]
+        auditmsg['user'] = util.stringify(authdata[2])
         if sessid is not None:
             authinfo['sessionid'] = sessid
         if not skiplog:
@@ -341,15 +352,10 @@ def _authorize_request(env, operation):
         if 'csrftoken' in httpsessions[sessid]:
             authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
         return authinfo
-    else:
+    elif authdata is None:
         return {'code': 401}
-        # TODO(jbjohnso): actually evaluate the request for authorization
-        # In theory, the x509 or http auth stuff will get translated and then
-        # passed on to the core authorization function in an appropriate form
-        # expresses return in the form of http code
-        # 401 if there is no known identity
-        # 403 if valid identity, but no access
-        # going to run 200 just to get going for now
+    else:
+        return {'code': 403}
 
 
 def _pick_mimetype(env):
@@ -384,11 +390,11 @@ def resourcehandler(env, start_response):
     try:
         for rsp in resourcehandler_backend(env, start_response):
             yield rsp
-    except:
+    except Exception as e:
         tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                      event=log.Events.stacktrace)
-        start_response('500 - Internal Server Error', [])
-        yield '500 - Internal Server Error'
+        start_response('500 - ' + str(e), [])
+        yield '500 - ' + str(e)
         return
 
 
@@ -429,7 +435,7 @@ def resourcehandler_backend(env, start_response):
         return
     if authorized['code'] == 403:
         start_response('403 Forbidden', badauth)
-        yield 'authorization failed'
+        yield 'Forbidden'
         return
     if authorized['code'] != 200:
         raise Exception("Unrecognized code from auth engine")
@@ -446,6 +452,8 @@ def resourcehandler_backend(env, start_response):
                             httpsessions[authorized['sessionid']]['inflight'])):
                 pagecontent += rsp
             start_response("200 OK", headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
             yield pagecontent
             return
         except exc.ConfluentException as e:
@@ -469,6 +477,10 @@ def resourcehandler_backend(env, start_response):
         funport = forwarder.get_port(targip, env['HTTP_X_FORWARDED_FOR'],
                                      authorized['sessionid'])
         host = env['HTTP_X_FORWARDED_HOST']
+        if ']' in host:
+            host = host.split(']')[0] + ']'
+        elif ':' in host:
+            host = host.rsplit(':', 1)[0]
         url = 'https://{0}:{1}/'.format(host, funport)
         start_response('302', [('Location', url)])
         yield 'Our princess is in another castle!'
@@ -487,7 +499,7 @@ def resourcehandler_backend(env, start_response):
             auditmsg = {
                 'operation': 'start',
                 'target': env['PATH_INFO'],
-                'user': authorized['username'],
+                'user': util.stringify(authorized['username']),
             }
             if 'tenant' in authorized:
                 auditmsg['tenant'] = authorized['tenant']
@@ -622,6 +634,7 @@ def resourcehandler_backend(env, start_response):
             sessinfo = {'username': authorized['username']}
             if 'authtoken' in authorized:
                 sessinfo['authtoken'] = authorized['authtoken']
+            tlvdata.unicode_dictvalues(sessinfo)
             yield json.dumps(sessinfo)
             return
         resource = '.' + url[url.rindex('/'):]
@@ -643,6 +656,8 @@ def resourcehandler_backend(env, start_response):
                 for datum in _assemble_json(hdlr, resource, url, extension):
                     pagecontent += datum
             start_response('200 OK', headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
             yield pagecontent
         except exc.ConfluentException as e:
             if ((not isinstance(e, exc.LockedCredentials)) and
@@ -727,7 +742,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
     for rsp in responses:
         if isinstance(rsp, confluent.messages.LinkRelation):
             haldata = rsp.raw()
-            for hk in haldata.iterkeys():
+            for hk in haldata:
                 if 'href' in haldata[hk]:
                     if isinstance(haldata[hk]['href'], int):
                         haldata[hk]['href'] = str(haldata[hk]['href'])
@@ -743,7 +758,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                     links[hk] = haldata[hk]
         else:
             rsp = rsp.raw()
-            for dk in rsp.iterkeys():
+            for dk in rsp:
                 if dk in rspdata:
                     if isinstance(rspdata[dk], list):
                         if isinstance(rsp[dk], list):
@@ -762,8 +777,8 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                         rspdata[dk] = rsp[dk]
     rspdata["_links"] = links
     tlvdata.unicode_dictvalues(rspdata)
-    yield json.dumps(
-        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
+    yield util.stringify(json.dumps(
+        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8'))
 
 
 def serve(bind_host, bind_port):
@@ -790,9 +805,16 @@ def serve(bind_host, bind_port):
                 ' a second\n')
             eventlet.sleep(1)
     # TCP_FASTOPEN
-    sock.setsockopt(socket.SOL_TCP, 23, 5)
-    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                         debug=False, socket_timeout=60)
+    try:
+        sock.setsockopt(socket.SOL_TCP, 23, 5)
+    except Exception:
+        pass  # we gave it our best shot there
+    try:
+        eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                             debug=False, socket_timeout=60)
+    except TypeError:
+        # Older eventlet in place, skip arguments it does not understand
+        eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -51,11 +51,15 @@
 #    - leading bit reserved, 0 for now
 #    - length of metadata record 7 bits
 #    - type of data referenced by this entry (one byte), currently:
-#       0=text event, 1=json, 2=console data
+#       0=text event, 1=json, 2=console data, 3=event
 #    - offset into the text log to begin (4 bytes)
 #    - length of data referenced by this entry (2 bytes)
-#    - UTC timestamp of this entry in seconds since epoch (unsigned 32 bit?)
-#    - CRC32 over the record
+#    - UTC timestamp of this entry in seconds since epoch (unsigned 32 bit)
+#    - Event type (per 'Events' class below)
+#    - Event data (per event, currently used by connect/disconnect to represent
+#      single or multiple connections by user and for 'appmode' and 'shiftin'
+#      status for console
+#    - 2 reserved bytes
 #    (a future extended version might include suport for Forward Secure Sealing
 #    or other fields)
 
@@ -72,7 +76,13 @@ import stat
 import struct
 import time
 import traceback
+try:
+    unicode
+except NameError:
+    unicode = str
 
+daemonized = False
+logfull = False
 try:
     from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_SH
 except ImportError:
@@ -150,21 +160,36 @@ class BaseRotatingHandler(object):
         Output the record to the file, catering for rollover as described
         in doRollover().
         """
-        rolling_type = self.shouldRollover(binrecord, textrecord)
-        if rolling_type:
-            flock(self.textfile, LOCK_UN)
-            return self.doRollover(rolling_type)
-        return None
+        global logfull
+        try:
+            rolling_type = self.shouldRollover(binrecord, textrecord)
+            if rolling_type:
+                flock(self.textfile, LOCK_UN)
+                return self.doRollover(rolling_type)
+            return None
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
+
 
     def emit(self, binrecord, textrecord):
-        if self.textfile is None:
-            self.textfile = open(self.textpath, mode='ab')
-        if self.binfile is None:
-            self.binfile = open(self.binpath, mode='ab')
-        self.textfile.write(textrecord)
-        self.binfile.write(binrecord)
-        self.textfile.flush()
-        self.binfile.flush()
+        global logfull
+        try:
+            if self.textfile is None:
+                self.textfile = open(self.textpath, mode='ab')
+            if self.binfile is None:
+                self.binfile = open(self.binpath, mode='ab')
+            if not isinstance(textrecord, bytes):
+                textrecord = textrecord.encode('utf-8')
+            self.textfile.write(textrecord)
+            self.binfile.write(binrecord)
+            self.textfile.flush()
+            self.binfile.flush()
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
 
     def get_textfile_offset(self, data_len):
         if self.textfile is None:
@@ -561,28 +586,39 @@ class Logger(object):
                 textdate = time.strftime(
                     '%b %d %H:%M:%S ', time.localtime(tstamp))
             flock(textfile, LOCK_EX)
-            offset = textfile.tell() + len(textdate)
-            datalen = len(data)
-            eventaux = entry[4]
-            if eventaux is None:
-                eventaux = 0
-            # metadata length is always 16 for this code at the moment
-            binrecord = struct.pack(
-                ">BBIHIBBH", 16, ltype, offset, datalen, tstamp, evtdata,
-                eventaux, 0)
-            if self.isconsole:
-                if ltype == 2:
-                    textrecord = data
+            try:
+                offset = textfile.tell() + len(textdate)
+                datalen = len(data)
+                eventaux = entry[4]
+                if eventaux is None:
+                    eventaux = 0
+                # metadata length is always 16 for this code at the moment
+                binrecord = struct.pack(
+                    ">BBIHIBBH", 16, ltype, offset, datalen, tstamp, evtdata,
+                    eventaux, 0)
+                if self.isconsole:
+                    if ltype == 2:
+                        textrecord = data
+                    else:
+                        if not isinstance(textdate, bytes):
+                            textdate = textdate.encode('utf-8')
+                        if not isinstance(data, bytes):
+                            data = data.encode('utf-8')
+                        textrecord = textdate + data + b']'
                 else:
-                    textrecord = textdate + data + ']'
-            else:
-                textrecord = textdate + data
-                if not textrecord.endswith('\n'):
-                    textrecord += '\n'
-            files = self.handler.try_emit(binrecord, textrecord)
+                    textrecord = textdate + data
+                    if not textrecord.endswith('\n'):
+                        textrecord += '\n'
+                files = self.handler.try_emit(binrecord, textrecord)
+            except struct.error:
+                files = self.handler.doRollover(RollingTypes.size_rolling)
+            finally:
+                try:
+                    flock(textfile, LOCK_UN)
+                except Exception:
+                    pass
             if not files:
                 self.handler.emit(binrecord, textrecord)
-                flock(textfile, LOCK_UN)
             else:
                 # Log the rolling event at first, then log the last data
                 # which cause the rolling event.
@@ -716,7 +752,7 @@ class Logger(object):
         pass
 
     def log(self, logdata=None, ltype=None, event=0, eventdata=None):
-        if type(logdata) not in (str, unicode, dict):
+        if type(logdata) not in (bytes, unicode, dict):
             raise Exception("Unsupported logdata")
         if ltype is None:
             if type(logdata) == dict:
@@ -753,15 +789,17 @@ globaleventlog = None
 tracelog = None
 
 
-def log(logdata=None, ltype=None, event=0, eventdata=None):
+def log(logdata=None, ltype=None, event=0, eventdata=None, flush=False):
     global globaleventlog
     if globaleventlog is None:
         globaleventlog = Logger('events')
     globaleventlog.log(logdata, ltype, event, eventdata)
+    if flush:
+        globaleventlog.writedata()
 
 def logtrace():
     global tracelog
     if tracelog is None:
         tracelog = Logger('trace', buffered=False)
     tracelog.log(traceback.format_exc(), ltype=DataTypes.event,
-                 event=Events.stacktrace)
+                 event=Events.stacktrace)

confluent_server/confluent/main.py

@@ -43,9 +43,11 @@ except ImportError:
 import confluent.discovery.core as disco
 import eventlet
 dbgif = False
-if  map(int, (eventlet.__version__.split('.'))) > [0, 18]:
+try:
     import eventlet.backdoor as backdoor
     dbgif = True
+except Exception:
+    pass
 havefcntl = True
 try:
     import fcntl
@@ -72,7 +74,7 @@ def _daemonize():
     os.setsid()
     thispid = os.fork()
     if thispid > 0:
-        print 'confluent server starting as pid %d' % thispid
+        print('confluent server starting as pid {0}'.format(thispid))
         os._exit(0)
     os.closerange(0, 2)
     os.umask(63)
@@ -81,6 +83,7 @@ def _daemonize():
     os.dup2(0, 2)
     sys.stdout = log.Logger('stdout', buffered=False)
     sys.stderr = log.Logger('stderr', buffered=False)
+    log.daemonized = True
 
 
 def _updatepidfile():
@@ -224,6 +227,11 @@ def run():
     except:
         doexit()
         raise
+    try:
+        log.log({'info': 'Confluent management service starting'}, flush=True)
+    except (OSError, IOError) as e:
+        print(repr(e))
+        sys.exit(1)
     _daemonize()
     if havefcntl:
         _updatepidfile()
@@ -231,7 +239,7 @@ def run():
     signal.signal(signal.SIGTERM, terminate)
     collective.startup()
     if dbgif:
-        oumask = os.umask(0077)
+        oumask = os.umask(0o077)
         try:
             os.remove('/var/run/confluent/dbg.sock')
         except OSError:

confluent_server/confluent/messages.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,10 +20,17 @@
 # format.  This is also how different data formats are supported
 import confluent.exceptions as exc
 import confluent.config.configmanager as cfm
+import confluent.config.conf as cfgfile
 from copy import deepcopy
 from datetime import datetime
+import confluent.util as util
 import json
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 valid_health_values = set([
     'ok',
     'warning',
@@ -32,6 +39,17 @@ valid_health_values = set([
     'unknown',
 ])
 
+passcomplexity = cfgfile.get_option('policy', 'passwordcomplexity')
+passminlength = cfgfile.get_option('policy', 'passwordminlength')
+if passminlength:
+    passminlength = int(passminlength)
+else:
+    passminlength = 0
+if passcomplexity:
+    passcomplexity = int(passcomplexity)
+else:
+    passcomplexity = 0
+
 def simplify_name(name):
     return name.lower().replace(' ', '_').replace('/', '-').replace(
         '_-_', '-')
@@ -42,7 +60,9 @@ def _htmlify_structure(indict):
     if isinstance(indict, dict):
         for key in sorted(indict):
             ret += "<li>{0}: ".format(key)
-            if type(indict[key]) in (str, unicode, float, int):
+            if type(indict[key]) in (bytes, unicode):
+                ret += util.stringify(indict[key])
+            if type(indict[key]) in (float, int):
                 ret += str(indict[key])
             elif isinstance(indict[key], datetime):
                 ret += indict[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -50,13 +70,13 @@ def _htmlify_structure(indict):
                 ret += _htmlify_structure(indict[key])
     elif isinstance(indict, list):
         if len(indict) > 0:
-            if type(indict[0]) in (str, unicode, None):
+            if type(indict[0]) in (bytes, unicode, None):
                 nd = []
                 for datum in indict:
                     if datum is None:
                         nd.append('')
                     else:
-                        nd.append(datum)
+                        nd.append(util.stringify(datum))
                 ret += ",".join(nd)
             else:
                 for v in indict:
@@ -109,15 +129,16 @@ class ConfluentMessage(object):
             return self._generic_html_value(self.kvpairs)
         if not self.stripped:
             htmlout = ''
-            for node in self.kvpairs.iterkeys():
+            for node in self.kvpairs:
                 htmlout += '{0}:{1}\n'.format(
                     node, self._generic_html_value(self.kvpairs[node]))
             return htmlout
 
     def _generic_html_value(self, pairs):
         snippet = ""
-        for key in pairs.iterkeys():
+        for key in pairs:
             val = pairs[key]
+            key = util.stringify(key)
             value = self.defaultvalue
             if isinstance(val, dict) and 'type' in val:
                 valtype = val['type']
@@ -144,7 +165,7 @@ class ConfluentMessage(object):
                         '<input type="checkbox" name="restexplorerhonorkey" '
                         'value="{1}">\r').format(valtype, key, self.desc)
                 return snippet
-            if (isinstance(val, bool) or isinstance(val, str) or
+            if (isinstance(val, bool) or isinstance(val, bytes) or
                     isinstance(val, unicode)):
                 value = str(val)
             elif val is not None and 'value' in val:
@@ -267,6 +288,24 @@ class CreatedResource(ConfluentMessage):
         pass
 
 
+class RenamedResource(ConfluentMessage):
+    notnode = True
+    readonly = True
+
+    def __init__(self, oldname, newname):
+        self.kvpairs = {'oldname': oldname, 'newname': newname}
+
+    def strip_node(self, node):
+        pass
+
+
+class RenamedNode(ConfluentMessage):
+    def __init__(self, name, rename):
+        self.desc = 'New Name'
+        kv = {'rename': {'value': rename}}
+        self.kvpairs = {name: kv}
+
+
 class AssignedResource(ConfluentMessage):
     notnode = True
     readonly = True
@@ -291,15 +330,16 @@ class ConfluentChoiceMessage(ConfluentMessage):
             return self._create_option(self.kvpairs)
         else:
             htmlout = ''
-            for node in self.kvpairs.iterkeys():
+            for node in self.kvpairs:
                 htmlout += '{0}:{1}\n'.format(
                     node, self._create_option(self.kvpairs[node]))
             return htmlout
 
     def _create_option(self, pairdata):
         snippet = ''
-        for key in pairdata.iterkeys():
+        for key in pairdata:
             val = pairdata[key]
+            key = util.stringify(key)
             snippet += key + ':<select name="%s">' % key
             valid_values = self.valid_values
             if key in self.valid_paramset:
@@ -381,7 +421,9 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputReseatMessage(path, nodes, inputdata)
     elif path == ['attributes', 'expression']:
         return InputExpression(path, inputdata, nodes)
-    elif path[0] in ('attributes', 'users') and operation != 'retrieve':
+    elif path == ['attributes', 'rename']:
+        return InputConfigChangeSet(path, inputdata, nodes, configmanager)
+    elif path[0] in ('attributes', 'users', 'usergroups') and operation != 'retrieve':
         return InputAttributes(path, inputdata, nodes)
     elif path == ['boot', 'nextdevice'] and operation != 'retrieve':
         return InputBootDevice(path, nodes, inputdata)
@@ -418,10 +460,11 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
     elif (path[:4] == ['configuration', 'management_controller', 'ntp',
             'servers'] and operation != 'retrieve' and len(path) == 5):
         return InputNTPServer(path, nodes, inputdata)
-    elif (path[:3] == ['configuration', 'system', 'all'] and
+    elif (path[:3] in (['configuration', 'system', 'all'],
+            ['configuration', 'management_controller', 'extended']) and
             operation != 'retrieve'):
         return InputConfigChangeSet(path, inputdata, nodes, configmanager)
-    elif (path[:3] == ['configuration', 'system', 'clear'] and
+    elif (path[0] == 'configuration' and path[2] == 'clear'  and
             operation != 'retrieve'):
         return InputConfigClear(path, inputdata, nodes, configmanager)
     elif (path[:3] == ['configuration', 'storage', 'disks'] and
@@ -438,6 +481,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputMedia(path, nodes, inputdata, configmanager)
     elif '/'.join(path).startswith('support/servicedata') and inputdata:
         return InputMedia(path, nodes, inputdata, configmanager)
+    elif '/'.join(path).startswith('configuration/management_controller/save_licenses') and inputdata:
+        return InputMedia(path, nodes, inputdata, configmanager)
     elif '/'.join(path).startswith(
             'configuration/management_controller/licenses') and inputdata:
         return InputLicense(path, nodes, inputdata, configmanager)
@@ -448,7 +493,7 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
 class InputFirmwareUpdate(ConfluentMessage):
 
     def __init__(self, path, nodes, inputdata, configmanager):
-        self._filename = inputdata.get('filename', inputdata.get('url', None))
+        self._filename = inputdata.get('filename', inputdata.get('url', inputdata.get('dirname', None)))
         self.bank = inputdata.get('bank', None)
         self.nodes = nodes
         self.filebynode = {}
@@ -490,6 +535,10 @@ class Media(ConfluentMessage):
     def __init__(self, node, media):
         self.kvpairs = {node: {'name': media.name, 'url': media.url}}
 
+class SavedFile(ConfluentMessage):
+    def __init__(self, node, file):
+        self.kvpairs = {node: {'filename': file}}
+
 class InputAlertData(ConfluentMessage):
 
     def __init__(self, path, inputdata, nodes=None):
@@ -539,8 +588,6 @@ class InputConfigClear(ConfluentMessage):
             raise exc.InvalidArgumentException('Input must be {"clear":true}')
 
 class InputConfigChangeSet(InputExpression):
-    # For now, this is identical to InputExpression, later it may
-    # internalize formula expansion, but not now..
     def __init__(self, path, inputdata, nodes=None, configmanager=None):
         self.cfm = configmanager
         super(InputConfigChangeSet, self).__init__(path, inputdata, nodes)
@@ -550,7 +597,7 @@ class InputConfigChangeSet(InputExpression):
         endattrs = {}
         for attr in attrs:
             origval = attrs[attr]
-            if isinstance(origval, str) or isinstance(origval, unicode):
+            if isinstance(origval, bytes) or isinstance(origval, unicode):
                 origval = {'expression': origval}
             if 'expression' not in origval:
                 endattrs[attr] = attrs[attr]
@@ -577,7 +624,7 @@ class InputAttributes(ConfluentMessage):
         if nodes is None:
             self.attribs = inputdata
             for attrib in self.attribs:
-                if type(self.attribs[attrib]) in (str, unicode):
+                if type(self.attribs[attrib]) in (bytes, unicode):
                     try:
                         # ok, try to use format against the string
                         # store back result to the attribute to
@@ -598,12 +645,12 @@ class InputAttributes(ConfluentMessage):
         for node in nodes:
             self.nodeattribs[node] = inputdata
 
-    def get_attributes(self, node):
+    def get_attributes(self, node, validattrs=None):
         if node not in self.nodeattribs:
             return {}
         nodeattr = deepcopy(self.nodeattribs[node])
         for attr in nodeattr:
-            if type(nodeattr[attr]) in (str, unicode):
+            if type(nodeattr[attr]) in (bytes, unicode):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -613,15 +660,82 @@ class InputAttributes(ConfluentMessage):
                     # an expression string will error if format() done
                     # use that as cue to put it into config as an expr
                     nodeattr[attr] = {'expression': nodeattr[attr]}
+            if validattrs and 'validvalues' in validattrs.get(attr, []):
+                if (nodeattr[attr] and
+                        nodeattr[attr] not in validattrs[attr]['validvalues']):
+                    raise exc.InvalidArgumentException(
+                        'Attribute {0} does not accept value {1} (valid values would be {2})'.format(
+                            attr, nodeattr[attr], ','.join(validattrs[attr]['validvalues'])))
+            elif validattrs and 'validlist' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if v and v not in validattrs[attr]['validlist']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept list member '
+                            '{1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlist'])))
+            elif validattrs and 'validlistkeys' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if '=' not in v:
+                        raise exc.InvalidArgumentException(
+                            'Passed key {0} requires a parameter'.format(v))
+                    v = v.split('=', 1)[0]
+                    if v and v not in validattrs[attr]['validlistkeys']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept key {1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlistkeys'])
+                            )
+                        )
         return nodeattr
 
+def checkPassword(password, username):
+    lowercase = set('abcdefghijklmnopqrstuvwxyz')
+    uppercase = set('abcdefghijklmnopqrstuvwxyz'.upper())
+    numbers = set('0123456789')
+    special = set('`~!@#$%^&*()-_=+[{]};:"/?.>,<' + "'")
+    if len(password) < passminlength:
+        raise exc.InvalidArgumentException('Password must be at least {0} characters long'.format(passminlength))
+    if not isinstance(passcomplexity, int) or passcomplexity < 1:
+        return
+    if not bool(set(password.lower()) & lowercase):  # rule 1
+        raise exc.InvalidArgumentException('Password must contain at least one letter')
+    if passcomplexity < 2:
+        return
+    thepass = set(password)
+    if not bool(thepass & numbers):  # rule 2
+        raise exc.InvalidArgumentException('Password must contain at least one number')
+    if passcomplexity < 3:
+        return
+    classes = 0
+    for charclass in (lowercase, uppercase, special):
+        if bool(thepass & charclass):
+            classes += 1
+    if classes < 2:
+        raise exc.InvalidArgumentException('Password must contain at least two of upper case letter, lower case letter, and/or special character')
+    if passcomplexity < 4:
+        return
+    if username and password in (username, username[::-1]): # rule 4
+        raise exc.InvalidArgumentException('Password must not be similar to username')
+    if passcomplexity < 5:
+        return
+    for char in thepass:
+        if char * 3 in password:
+            raise exc.InvalidArgumentException('Password must not contain any of the same character repeated 3 times')
+
+
 
 class InputCredential(ConfluentMessage):
     valid_privilege_levels = set([
         'callback',
         'user',
+        'ReadOnly',
         'operator',
+        'Operator',
         'administrator',
+        'Administrator',
         'proprietary',
         'no_access',
     ])
@@ -639,33 +753,21 @@ class InputCredential(ConfluentMessage):
         if len(path) == 4:
             inputdata['uid'] = path[-1]
         # if the operation is 'create' check if all fields are present
-        missingattrs = []
-        for attrname in ('uid', 'privilege_level', 'username', 'password'):
-            if attrname not in inputdata:
-                missingattrs.append(attrname)
-        if missingattrs:
-            raise exc.InvalidArgumentException(
-                'Required fields missing: {0}'.format(','.join(missingattrs)))
-        if (isinstance(inputdata['uid'], str) and
+        if (type(inputdata['uid']) in (bytes, unicode) and
                 not inputdata['uid'].isdigit()):
-            raise exc.InvalidArgumentException('uid must be a number')
+            inputdata['uid'] = inputdata['uid']
         else:
             inputdata['uid'] = int(inputdata['uid'])
         if ('privilege_level' in inputdata and
               inputdata['privilege_level'] not in self.valid_privilege_levels):
             raise exc.InvalidArgumentException('privilege_level is not one of '
                                         + ','.join(self.valid_privilege_levels))
-        if 'username' in inputdata and len(inputdata['username']) > 16:
-            raise exc.InvalidArgumentException(
-                                        'name must be less than or = 16 chars')
-        if 'password' in inputdata and len(inputdata['password']) > 20:
-            raise exc.InvalidArgumentException('password has limit of 20 chars')
-
         if ('enabled' in inputdata and
             inputdata['enabled'] not in self.valid_enabled_values):
             raise exc.InvalidArgumentException('valid values for enabled are '
                                                             + 'yes and no')
-
+        if 'password' in inputdata and (passcomplexity or passminlength):
+            checkPassword(inputdata['password'], inputdata.get('username', None))
         if nodes is None:
             raise exc.InvalidArgumentException(
                 'This only supports per-node input')
@@ -677,7 +779,7 @@ class InputCredential(ConfluentMessage):
             return {}
         credential = deepcopy(self.credentials[node])
         for attr in credential:
-            if type(credential[attr]) in (str, unicode):
+            if type(credential[attr]) in (bytes, unicode):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -731,6 +833,7 @@ class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([
         'on',
         'off',
+        'blink',
     ])
 
     keyname = 'identify'
@@ -762,6 +865,9 @@ class InputVolumes(ConfluentInputMessage):
             sizes = inputdata.get('size', [None])
             if not isinstance(sizes, list):
                 sizes = sizes.split(',')
+            stripsizes = inputdata.get('stripsizes', [None])
+            if not isinstance(stripsizes, list):
+                stripsizes = stripsizes.split(',')
             disks = inputdata.get('disks', [])
             if not disks:
                 raise exc.InvalidArgumentException(
@@ -773,8 +879,15 @@ class InputVolumes(ConfluentInputMessage):
                     currname = volnames.pop(0)
                 else:
                     currname = None
+                if stripsizes:
+                    currstripsize = stripsizes.pop(0)
+                    if currstripsize:
+                        currstripsize = int(currstripsize)
+                else:
+                    currstripsize = None
                 inputdata.append(
                     {'name': currname, 'size': size,
+                     'stripsize': currstripsize,
                      'disks': disks,
                      'raidlevel': raidlvl})
         for node in nodes:
@@ -796,6 +909,7 @@ class InputVolumes(ConfluentInputMessage):
                 self.inputbynode[node].append({'name': volname,
                                                'size': volsize,
                                                'disks': disks,
+                                               'stripsize': input.get('stripsize', None),
                                                'raidlevel': raidlvl,
                                                })
 
@@ -1145,7 +1259,9 @@ class EventCollection(ConfluentMessage):
                 'event': event.get('event', None),
                 'severity': event['severity'],
                 'timestamp': event.get('timestamp', None),
+                'message': event.get('message', None),
                 'record_id': event.get('record_id', None),
+                'log_id': event.get('log_id', None),
             }
             if event['severity'] not in valid_health_values:
                 raise exc.NotImplementedException(
@@ -1199,14 +1315,15 @@ class AsyncSession(ConfluentMessage):
         self.kvpairs = {'asyncid': id}
 
 class User(ConfluentMessage):
-    def __init__(self, uid, username, privilege_level, name=None):
+    def __init__(self, uid, username, privilege_level, name=None, expiration=None):
         self.desc = 'foo'
         self.stripped = False
         self.notnode = name is None
         kvpairs = {'username': {'value': username},
                    'password': {'value': '', 'type': 'password'},
                    'privilege_level': {'value': privilege_level},
-                   'enabled': {'value': ''}
+                   'enabled': {'value': ''},
+                   'expiration': {'value': expiration},
                    }
         if self.notnode:
             self.kvpairs = kvpairs
@@ -1225,6 +1342,7 @@ class UserCollection(ConfluentMessage):
             entry = {
                 'uid': user['uid'],
                 'username': user['name'],
+                'expiration': user.get('expiration', None),
                 'privilege_level': user['access']['privilege_level']
             }
             userlist.append(entry)
@@ -1251,7 +1369,7 @@ class AlertDestination(ConfluentMessage):
 
 class InputAlertDestination(ConfluentMessage):
     valid_alert_params = {
-        'acknowledge': lambda x: False if type(x) in (unicode,str) and x.lower() == 'false' else bool(x),
+        'acknowledge': lambda x: False if type(x) in (unicode, bytes) and x.lower() == 'false' else bool(x),
         'acknowledge_timeout': lambda x: int(x) if x and x.isdigit() else None,
         'ip': lambda x: x,
         'retries': lambda x: int(x)
@@ -1261,6 +1379,8 @@ class InputAlertDestination(ConfluentMessage):
         self.alertcfg = {}
         if multinode:  # keys are node names
             for node in inputdata:
+                if not isinstance(inputdata[node], dict):
+                    break
                 self.alertcfg[node] = inputdata[node]
                 for key in inputdata[node]:
                     if key not in self.valid_alert_params:
@@ -1273,7 +1393,8 @@ class InputAlertDestination(ConfluentMessage):
                     else:
                         self.alertcfg[node][key] = \
                             self.valid_alert_params[key](inputdata[node][key])
-        else:
+            else:
+                return
             for key in inputdata:
                 if key not in self.valid_alert_params:
                     raise exc.InvalidArgumentException(
@@ -1356,12 +1477,13 @@ class Array(ConfluentMessage):
         }
 
 class Volume(ConfluentMessage):
-    def __init__(self, name, volname, size, state, array):
+    def __init__(self, name, volname, size, state, array, stripsize=None):
         self.kvpairs = {
             name: {
                 'type': 'volume',
                 'name': simplify_name(volname),
                 'label': volname,
+                'stripsize': stripsize,
                 'size': size,
                 'state': state,
                 'array': array,
@@ -1370,14 +1492,18 @@ class Volume(ConfluentMessage):
 
 class Disk(ConfluentMessage):
     valid_states = set([
+        'fault',
         'jbod',
         'unconfigured',
         'hotspare',
+        'rebuilding',
         'online',
     ])
     state_aliases = {
+        'unconfigured bad': 'fault',
         'unconfigured good': 'unconfigured',
         'global hot spare': 'hotspare',
+        'dedicated hot spare': 'hotspare',
     }
 
     def _normalize_state(self, instate):
@@ -1463,7 +1589,7 @@ class Attributes(ConfluentMessage):
         nkv = {}
         self.notnode = name is None
         for key in kv:
-            if type(kv[key]) in (str, unicode):
+            if type(kv[key]) in (bytes, unicode):
                 nkv[key] = {'value': kv[key]}
             else:
                 nkv[key] = kv[key]
@@ -1555,12 +1681,12 @@ class NTPServer(ConfluentMessage):
 class License(ConfluentMessage):
     readonly = True
 
-    def __init__(self, name=None, kvm=None, feature=None):
+    def __init__(self, name=None, kvm=None, feature=None, state=None):
         self.notnode = name is None
         self.desc = 'License'
 
         kv = []
-        kv.append({'kvm_availability': str(kvm), 'feature': feature})
+        kv.append({'kvm_availability': str(kvm), 'feature': feature, 'state': state})
         if self.notnode:
             self.kvpairs = {'License': kv}
         else:
@@ -1574,7 +1700,7 @@ class CryptedAttributes(Attributes):
         # for now, just keep the dictionary keys and discard crypt value
         self.desc = desc
         nkv = {}
-        for key in kv.iterkeys():
+        for key in kv:
             nkv[key] = {'isset': False}
             try:
                 if kv[key] is not None and kv[key]['cryptvalue'] != '':

confluent_server/confluent/neighutil.py

@@ -18,6 +18,7 @@
 # Ultimately, this should use AF_NETLINK, but in the interest of time,
 # use ip neigh for the moment
 
+import confluent.util as util
 import eventlet.green.subprocess as subprocess
 import os
 
@@ -39,6 +40,7 @@ def update_neigh():
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
     (neighdata, err) = ipn.communicate()
+    neighdata = util.stringify(neighdata)
     for entry in neighdata.split('\n'):
         entry = entry.split(' ')
         if len(entry) < 5 or not entry[4]:

confluent_server/confluent/netutil.py

@@ -24,6 +24,19 @@ import eventlet.support.greendns
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 
+def mask_to_cidr(mask):
+    maskn = socket.inet_pton(socket.AF_INET, mask)
+    maskn = struct.unpack('!I', maskn)[0]
+    cidr = 32
+    while maskn & 0b1 == 0 and cidr > 0:
+        cidr -= 1
+        maskn >>= 1
+    return cidr
+
+def cidr_to_mask(cidr):
+    return socket.inet_ntop(
+        socket.AF_INET, struct.pack('!I', (2**32 - 1) ^ (2**(32 - cidr) - 1)))
+
 def ip_on_same_subnet(first, second, prefix):
     addrinf = socket.getaddrinfo(first, None, 0, socket.SOCK_STREAM)[0]
     fam = addrinf[0]

confluent_server/confluent/networking/lldp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016, 2017 Lenovo
+# Copyright 2016-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -105,7 +105,11 @@ def close_enough(fuzz, literal):
     if fuzz == literal:
         return True
     fuzz = '^' + fuzz.replace('-', '[/: -]') + '$'
-    matcher = re.compile(fuzz)
+    try:
+        matcher = re.compile(fuzz)
+    except Exception:
+        raise exc.InvalidArgumentException(
+            'Invalid regular expression specified')
     return bool(matcher.match(literal))
 
 
@@ -121,7 +125,8 @@ def _dump_neighbordatum(info):
 
 def b64tohex(b64str):
     bd = base64.b64decode(b64str)
-    return ''.join(['{0:02x}'.format(ord(x)) for x in bd])
+    bd = bytearray(bd)
+    return ''.join(['{0:02x}'.format(x) for x in bd])
 
 def get_fingerprint(switch, port, configmanager, portmatch):
     update_switch_data(switch, configmanager)

confluent_server/confluent/networking/macmap.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016-2017 Lenovo
+# Copyright 2016-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -315,10 +315,12 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
         switches = set([])
         for node in nodelocations:
             cfg = nodelocations[node]
+            if cfg.get('type', {}).get('value', None) == 'switch':
+                switches.add(node)
             for attr in cfg:
                 if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                     continue
@@ -342,7 +344,7 @@ def _full_updatemacmap(configmanager):
                         _switchportmap[curswitch][portname] = None
                     else:
                         _switchportmap[curswitch][portname] = node
-        for switch in _macsbyswitch:
+        for switch in list(_macsbyswitch):
             if switch not in switches:
                 del _macsbyswitch[switch]
         switchauth = get_switchcreds(configmanager, switches)
@@ -378,7 +380,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
     if (operation in ('update', 'create') and
             pathcomponents == ['networking', 'macs', 'rescan']):
         if inputdata != {'rescan': 'start'}:
-            raise exc.InvalidArgumentException()
+            raise exc.InvalidArgumentException('Input must be rescan=start')
         eventlet.spawn_n(rescan, configmanager)
         return [msg.KeyValueData({'rescan': 'started'})]
     raise exc.NotImplementedException(
@@ -456,9 +458,21 @@ def handle_read_api_request(pathcomponents, configmanager):
                     portname = portname.replace('-', '/')
                 maclist = _macsbyswitch[switchname][portname]
             except KeyError:
-                raise exc.NotFoundException('No known macs for switch {0} '
-                                            'port {1}'.format(switchname,
-                                                              portname))
+                foundsomemacs = False
+                if switchname in _macsbyswitch:
+                    try:
+                        matcher = re.compile(portname)
+                    except Exception:
+                        raise exc.InvalidArgumentException('Invalid regular expression specified')
+                    maclist = []
+                    for actualport in _macsbyswitch[switchname]:
+                        if bool(matcher.match(actualport)):
+                            foundsomemacs = True
+                            maclist = maclist + _macsbyswitch[switchname][actualport]
+                if not foundsomemacs:
+                    raise exc.NotFoundException('No known macs for switch {0} '
+                                                'port {1}'.format(switchname,
+                                                                  portname))
             return [msg.ChildCollection(x.replace(':', '-'))
                     for x in sorted(maclist)]
         if len(pathcomponents) == 8:

confluent_server/confluent/networking/netutil.py

@@ -42,10 +42,12 @@ def get_switchcreds(configmanager, switches):
 
 def list_switches(configmanager):
     nodelocations = configmanager.get_node_attributes(
-        configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+        configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
     switches = set([])
     for node in nodelocations:
         cfg = nodelocations[node]
+        if cfg.get('type', {}).get('value', None) == 'switch':
+            switches.add(node)
         for attr in cfg:
             if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                 continue

confluent_server/confluent/noderange.py

@@ -25,6 +25,11 @@ import itertools
 import pyparsing as pp
 import re
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 # construct custom grammar with pyparsing
 _nodeword = pp.Word(pp.alphanums + '~^$/=-_:.*+!')
 _nodebracket = pp.QuotedString(quoteChar='[', endQuoteChar=']',
@@ -157,7 +162,7 @@ class NodeRange(object):
         pieces = seqrange.split(delimiter)
         if len(pieces) % 2 != 0:
             return self.failorreturn(seqrange)
-        halflen = len(pieces) / 2
+        halflen = len(pieces) // 2
         left = delimiter.join(pieces[:halflen])
         right = delimiter.join(pieces[halflen:])
         leftbits = _numextractor.parseString(left).asList()
@@ -166,7 +171,7 @@ class NodeRange(object):
             return self.failorreturn(seqrange)
         finalfmt = ''
         iterators = []
-        for idx in xrange(len(leftbits)):
+        for idx in range(len(leftbits)):
             if leftbits[idx] == rightbits[idx]:
                 finalfmt += leftbits[idx]
             elif leftbits[idx][0] in pp.alphas:
@@ -181,7 +186,7 @@ class NodeRange(object):
                 if leftnum > rightnum:
                     width = len(rightbits[idx])
                     minnum = rightnum
-                    maxnum = leftnum + 1  # xrange goes to n-1...
+                    maxnum = leftnum + 1  # range goes to n-1...
                 elif rightnum > leftnum:
                     width = len(leftbits[idx])
                     minnum = leftnum
@@ -189,7 +194,7 @@ class NodeRange(object):
                 else:  # differently padded, but same number...
                     return self.failorreturn(seqrange)
                 numformat = '{0:0%d}' % width
-                for num in xrange(minnum, maxnum):
+                for num in range(minnum, maxnum):
                     curseq.append(numformat.format(num))
         results = set([])
         for combo in itertools.product(*iterators):
@@ -222,7 +227,7 @@ class NodeRange(object):
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
             return set(self.cfm.filter_node_attributes(element, filternodes))
-        for idx in xrange(len(element)):
+        for idx in range(len(element)):
             if element[idx][0] == '[':
                 nodes = set([])
                 for numeric in NodeRange(element[idx][1:-1]).nodes:

confluent_server/confluent/pam.py

@@ -0,0 +1,235 @@
+# Pulled from:
+# https://raw.githubusercontent.com/FirefighterBlu3/python-pam/fe44b334970f421635d9e373b563c9e6566613bd/pam.py
+# and https://github.com/FirefighterBlu3/python-pam/pull/16/files
+# (c) 2007 Chris AtLee <chris@atlee.ca>
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license.php
+#
+# Original author: Chris AtLee
+#
+# Modified by David Ford, 2011-12-6
+# added py3 support and encoding
+# added pam_end
+# added pam_setcred to reset credentials after seeing Leon Walker's remarks
+# added byref as well
+# use readline to prestuff the getuser input
+
+'''
+PAM module for python
+
+Provides an authenticate function that will allow the caller to authenticate
+a user against the Pluggable Authentication Modules (PAM) on the system.
+
+Implemented using ctypes, so no compilation is necessary.
+'''
+
+__all__      = ['pam']
+__version__  = '1.8.4'
+__author__   = 'David Ford <david@blue-labs.org>'
+__released__ = '2018 June 15'
+
+import sys
+
+from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
+from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
+from ctypes import memmove
+from ctypes.util import find_library
+
+class PamHandle(Structure):
+    """wrapper class for pam_handle_t pointer"""
+    _fields_ = [ ("handle", c_void_p) ]
+
+    def __init__(self):
+        Structure.__init__(self)
+        self.handle = 0
+
+class PamMessage(Structure):
+    """wrapper class for pam_message structure"""
+    _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
+
+    def __repr__(self):
+        return "<PamMessage %i '%s'>" % (self.msg_style, self.msg)
+
+class PamResponse(Structure):
+    """wrapper class for pam_response structure"""
+    _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
+
+    def __repr__(self):
+        return "<PamResponse %i '%s'>" % (self.resp_retcode, self.resp)
+
+conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
+
+class PamConv(Structure):
+    """wrapper class for pam_conv structure"""
+    _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
+
+# Various constants
+PAM_PROMPT_ECHO_OFF       = 1
+PAM_PROMPT_ECHO_ON        = 2
+PAM_ERROR_MSG             = 3
+PAM_TEXT_INFO             = 4
+PAM_REINITIALIZE_CRED     = 8
+
+libc                      = CDLL(find_library("c"))
+libpam                    = CDLL(find_library("pam"))
+
+calloc                    = libc.calloc
+calloc.restype            = c_void_p
+calloc.argtypes           = [c_size_t, c_size_t]
+
+# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
+if hasattr(libpam, 'pam_end'):
+    pam_end                   = libpam.pam_end
+    pam_end.restype           = c_int
+    pam_end.argtypes          = [PamHandle, c_int]
+
+pam_start                 = libpam.pam_start
+pam_start.restype         = c_int
+pam_start.argtypes        = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
+
+pam_acct_mgmt             = libpam.pam_acct_mgmt
+pam_acct_mgmt.restype     = c_int
+pam_acct_mgmt.argtypes    = [PamHandle, c_int]
+
+pam_setcred               = libpam.pam_setcred
+pam_setcred.restype       = c_int
+pam_setcred.argtypes      = [PamHandle, c_int]
+
+pam_strerror              = libpam.pam_strerror
+pam_strerror.restype      = c_char_p
+pam_strerror.argtypes     = [PamHandle, c_int]
+
+pam_authenticate          = libpam.pam_authenticate
+pam_authenticate.restype  = c_int
+pam_authenticate.argtypes = [PamHandle, c_int]
+
+class pam():
+    code   = 0
+    reason = None
+
+    def __init__(self):
+        pass
+
+    def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+        """username and password authentication for the given service.
+
+           Returns True for success, or False for failure.
+
+           self.code (integer) and self.reason (string) are always stored and may
+           be referenced for the reason why authentication failed. 0/'Success' will
+           be stored for success.
+
+           Python3 expects bytes() for ctypes inputs.  This function will make
+           necessary conversions using the supplied encoding.
+
+        Inputs:
+          username: username to authenticate
+          password: password in plain text
+          service:  PAM service to authenticate against, defaults to 'login'
+
+        Returns:
+          success:  True
+          failure:  False
+        """
+
+        @conv_func
+        def my_conv(n_messages, messages, p_response, app_data):
+            """Simple conversation function that responds to any
+               prompt where the echo is off with the supplied password"""
+            # Create an array of n_messages response objects
+            addr = calloc(n_messages, sizeof(PamResponse))
+            response = cast(addr, POINTER(PamResponse))
+            p_response[0] = response
+            for i in range(n_messages):
+                if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
+                    dst = calloc(len(password)+1, sizeof(c_char))
+                    memmove(dst, cpassword, len(password))
+                    response[i].resp = dst
+                    response[i].resp_retcode = 0
+            return 0
+
+        # python3 ctypes prefers bytes
+        if sys.version_info >= (3,):
+            if isinstance(username, str): username = username.encode(encoding)
+            if isinstance(password, str): password = password.encode(encoding)
+            if isinstance(service, str):  service  = service.encode(encoding)
+        else:
+            if isinstance(username, unicode):
+                username = username.encode(encoding)
+            if isinstance(password, unicode):
+                password = password.encode(encoding)
+            if isinstance(service, unicode):
+                service  = service.encode(encoding)
+
+        if b'\x00' in username or b'\x00' in password or b'\x00' in service:
+            self.code = 4  # PAM_SYSTEM_ERR in Linux-PAM
+            self.reason = 'strings may not contain NUL'
+            return False
+
+        # do this up front so we can safely throw an exception if there's
+        # anything wrong with it
+        cpassword = c_char_p(password)
+
+        handle = PamHandle()
+        conv   = PamConv(my_conv, 0)
+        retval = pam_start(service, username, byref(conv), byref(handle))
+
+        if retval != 0:
+            # This is not an authentication error, something has gone wrong starting up PAM
+            self.code   = retval
+            self.reason = "pam_start() failed"
+            return False
+
+        retval = pam_authenticate(handle, 0)
+        auth_success = retval == 0
+
+        if auth_success:
+            retval = pam_acct_mgmt(handle, 0)
+            auth_success = retval == 0
+
+        if auth_success and resetcreds:
+            retval = pam_setcred(handle, PAM_REINITIALIZE_CRED)
+
+        # store information to inform the caller why we failed
+        self.code   = retval
+        self.reason = pam_strerror(handle, retval)
+        if sys.version_info >= (3,):
+            self.reason = self.reason.decode(encoding)
+
+        if hasattr(libpam, 'pam_end'):
+            pam_end(handle, retval)
+
+        return auth_success
+
+
+def authenticate(*vargs, **dargs):
+    """
+    Compatibility function for older versions of python-pam.
+    """
+    return pam().authenticate(*vargs, **dargs)
+
+
+if __name__ == "__main__":
+    import readline, getpass
+
+    def input_with_prefill(prompt, text):
+        def hook():
+            readline.insert_text(text)
+            readline.redisplay()
+        readline.set_pre_input_hook(hook)
+
+        if sys.version_info >= (3,):
+            result = input(prompt)
+        else:
+            result = raw_input(prompt)
+
+        readline.set_pre_input_hook()
+        return result
+
+    pam = pam()
+
+    username = input_with_prefill('Username: ', getpass.getuser())
+
+    # enter a valid username and an invalid/valid password, to verify both failure and success
+    pam.authenticate(username, getpass.getpass())
+    print('{} {}'.format(pam.code, pam.reason))

confluent_server/confluent/plugins/configuration/attributes.py

@@ -1,5 +1,5 @@
 # Copyright 2014 IBM Corporation
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ import confluent.exceptions as exc
 import confluent.messages as msg
 import confluent.config.attributes as allattributes
 import confluent.util as util
+from fnmatch import fnmatch
 
 
 def retrieve(nodes, element, configmanager, inputdata):
@@ -39,6 +40,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
     if element == 'all':
         theattrs = set(allattributes.node).union(set(grpcfg))
         theattrs.add('nodes')
+        theattrs.add('noderange')
         for attribute in sorted(theattrs):
             if attribute == 'groups':
                 continue
@@ -51,6 +53,10 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                 val = grpcfg[attribute]
             else:
                 val = {'value': None}
+            if attribute == 'noderange':
+                val['desc'] = 'The noderange this group is expanded ' \
+                    'to when used in noderange, exclusive with static ' \
+                    'nodes'
             if attribute.startswith('secret.'):
                 yield msg.CryptedAttributes(
                     kv={attribute: val},
@@ -69,6 +75,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
         for attribute in sorted(list(grpcfg)):
             currattr = grpcfg[attribute]
             if attribute == 'nodes':
+                if not currattr:
+                    continue
                 desc = 'The nodes belonging to this group'
             elif attribute == 'noderange':
                 desc = 'A dynamic noderange that this group refers to in noderange expansion'
@@ -92,8 +100,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                     kv={attribute: currattr},
                     desc=desc)
             else:
-                print attribute
-                print repr(currattr)
+                print(attribute)
+                print(repr(currattr))
                 raise Exception("BUGGY ATTRIBUTE FOR NODEGROUP")
 
 
@@ -126,7 +134,7 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                             attribute, {}).get('description', ''))
     elif element[-1] == 'current':
         for node in util.natural_sort(list(attributes)):
-            for attribute in sorted(attributes[node].iterkeys()):
+            for attribute in sorted(attributes[node]):
                 currattr = attributes[node][attribute]
                 try:
                     desc = allattributes.node[attribute]['description']
@@ -141,8 +149,8 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                     yield msg.ListAttributes(
                         node, {attribute: currattr}, desc)
                 else:
-                    print attribute
-                    print repr(currattr)
+                    print(attribute)
+                    print(repr(currattr))
                     raise Exception("BUGGY ATTRIBUTE FOR NODE")
 
 
@@ -156,9 +164,28 @@ def update(nodes, element, configmanager, inputdata):
 
 
 def update_nodegroup(group, element, configmanager, inputdata):
+    if element == 'check':
+        check = inputdata.attribs
+        decrypt = configmanager.decrypt
+        configmanager.decrypt = True
+        currinfo = configmanager.get_nodegroup_attributes(group, list(check))
+        configmanager.decrypt = decrypt
+        for inf in check:
+            checkvalue = check[inf]
+            if isinstance(checkvalue, dict):
+                checkvalue = checkvalue.get('value', None)
+            currvalue = currinfo.get(inf, {}).get('value')
+            if checkvalue == currvalue:
+                raise exc.InvalidArgumentException('Checked value matches existing value')
+        return retrieve_nodegroup(group, element, configmanager, inputdata)
+    if 'rename' in element:
+        namemap = {}
+        namemap[group] = inputdata.attribs['rename']
+        configmanager.rename_nodegroups(namemap)
+        return yield_rename_resources(namemap, isnode=False)
     try:
         clearattribs = []
-        for attrib in inputdata.attribs.iterkeys():
+        for attrib in inputdata.attribs:
             if inputdata.attribs[attrib] is None:
                 clearattribs.append(attrib)
         for attrib in clearattribs:
@@ -195,18 +222,56 @@ def create(nodes, element, configmanager, inputdata):
     if nodes is not None and element[-1] == 'expression':
         return _expand_expression(nodes, configmanager, inputdata)
 
+def yield_rename_resources(namemap, isnode):
+    for node in namemap:
+        if isnode:
+            yield msg.RenamedNode(node, namemap[node])
+        else:
+            yield msg.RenamedResource(node, namemap[node])
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
+    if not nodes:
+        raise exc.InvalidArgumentException(
+            'No action to take, noderange is empty (if trying to define '
+            'group attributes, use nodegroupattrib)')
+    if element[-1] == 'check':
+        for node in nodes:
+            check = inputdata.get_attributes(node, allattributes.node)
+            currinfo = configmanager.get_node_attributes(node, list(check), decrypt=True)
+            for inf in check:
+                checkvalue = check[inf]
+                if isinstance(checkvalue, dict):
+                    checkvalue = checkvalue.get('value', None)
+                currvalue = currinfo.get(node, {}).get(inf, {}).get('value')
+                if checkvalue == currvalue:
+                    raise exc.InvalidArgumentException('Checked value matches existing value')
+        return retrieve(nodes, element, configmanager, inputdata)
+    if 'rename' in element:
+        namemap = {}
+        for node in nodes:
+            rename = inputdata.get_attributes(node)
+            namemap[node] = rename['rename']
+        configmanager.rename_nodes(namemap)
+        return yield_rename_resources(namemap, isnode=True)
     for node in nodes:
-        updatenode = inputdata.get_attributes(node)
+        updatenode = inputdata.get_attributes(node, allattributes.node)
         clearattribs = []
         if updatenode:
-            for attrib in updatenode.iterkeys():
+            for attrib in list(updatenode):
                 if updatenode[attrib] is None:
-                    clearattribs.append(attrib)
-            if len(clearattribs) > 0:
-                for attrib in clearattribs:
                     del updatenode[attrib]
+                    if attrib in allattributes.node  or attrib.startswith('custom.') or attrib.startswith('net.'):
+                        clearattribs.append(attrib)
+                    else:
+                        foundattrib = False
+                        for candattrib in allattributes.node:
+                            if fnmatch(candattrib, attrib):
+                                clearattribs.append(candattrib)
+                                foundattrib = True
+                        if not foundattrib:
+                            raise exc.InvalidArgumentException("No attribute matches '" + attrib + "' (try wildcard if trying to clear a group)")
+            if len(clearattribs) > 0:
                 configmanager.clear_node_attributes([node], clearattribs)
             updatedict[node] = updatenode
     try:

confluent_server/confluent/plugins/console/tsmsol.py

@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+         
+
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        rc = rcmd.Command(self.origbmc, self.username,
+                          self.password,
+                          verifycallback=lambda x: True)
+        wc = rc.oem.wc
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/sol?CSRFTOKEN={1}'.format(self.bmc, rc.oem.csrftok), host=bmc, cookie='QSESSIONID={0}'.format(wc.cookies['QSESSIONID']))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)

confluent_server/confluent/plugins/hardwaremanagement/cnos.py

@@ -0,0 +1,197 @@
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#Noncritical:
+#  - One or more temperature sensors is in the warning range;
+#  - A panic dump exists in flash.
+#Critical:
+#  - One or more temperature sensors is in the failure range;
+#  - One or more fans are running < 100 RPM;
+#  - One power supply is off.
+
+
+import eventlet
+import eventlet.queue as queue
+import confluent.exceptions as exc
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import confluent.messages as msg
+import confluent.util as util
+
+class SwitchSensor(object):
+    def __init__(self, name, states, value=None, health=None):
+        self.name = name
+        self.value = value
+        self.states = states
+        self.health = health
+
+
+def cnos_login(node, configmanager, creds):
+    wc = webclient.SecureHTTPConnection(node, port=443, verifycallback=util.TLSCertVerifier(
+        configmanager, node, 'pubkeys.tls_hardwaremanager').verify_cert)
+    wc.set_basic_credentials(creds[node]['secret.hardwaremanagementuser']['value'], creds[node]['secret.hardwaremanagementpassword']['value'])
+    wc.request('GET', '/nos/api/login/')
+    rsp = wc.getresponse()
+    body = rsp.read()
+    if rsp.status == 401:  # CNOS gives 401 on first attempt...
+        wc.request('GET', '/nos/api/login/')
+        rsp = wc.getresponse()
+        body = rsp.read()
+    if rsp.status >= 200 and rsp.status < 300:
+        return wc
+    raise exc.TargetEndpointBadCredentials('Unable to authenticate')
+
+def update(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def delete(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def create(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def retrieve(nodes, element, configmanager, inputdata):
+    results = queue.LightQueue()
+    workers = set([])
+    if element == ['power', 'state']:
+        for node in nodes:
+            yield msg.PowerState(node=node, state='on')
+        return
+    elif element == ['health', 'hardware']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_health, configmanager, creds,
+                                       node, results))
+    elif element[:3] == ['inventory', 'hardware', 'all']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_inventory, configmanager,
+                                       creds, node, results, element))
+    elif element[:3] == ['inventory', 'firmware', 'all']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_firmware, configmanager,
+                                       creds, node, results, element))
+    else:
+        for node in nodes:
+            yield msg.ConfluentNodeError(node, 'Not Implemented')
+        return
+    currtimeout = 10
+    while workers:
+        try:
+            datum = results.get(10)
+            while datum:
+                if datum:
+                    yield datum
+                datum = results.get_nowait()
+        except queue.Empty:
+            pass
+        eventlet.sleep(0.001)
+        for t in list(workers):
+            if t.dead:
+                workers.discard(t)
+    try:
+        while True:
+            datum = results.get_nowait()
+            if datum:
+                yield datum
+    except queue.Empty:
+        pass
+
+
+def retrieve_inventory(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        results.put(msg.ChildCollection('system'))
+        return
+    wc = cnos_login(node, configmanager, creds)
+    sysinfo = wc.grab_json_response('/nos/api/sysinfo/inventory')
+    invinfo = {
+        'inventory': [{
+            'name': 'System',
+            'present': True,
+            'information': {
+                'Product name': sysinfo['Model'],
+                'Serial Number': sysinfo['Electronic Serial Number'],
+                'Board Serial Number': sysinfo['Serial Number'],
+                'Manufacturer': 'Lenovo',
+                'Model': sysinfo['Machine Type Model'],
+                'FRU Number': sysinfo['FRU'].strip(),
+            }
+        }]
+    }
+    results.put(msg.KeyValueData(invinfo, node))
+
+
+def retrieve_firmware(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        return
+    wc = cnos_login(node, configmanager, creds)
+    sysinfo = wc.grab_json_response('/nos/api/sysinfo/inventory')
+    items = [{
+        'Software': {'version': sysinfo['Software Revision']},
+        },
+        {
+        'BIOS': {'version': sysinfo['BIOS Revision']},
+        }]
+    results.put(msg.Firmware(items, node))
+
+def retrieve_health(configmanager, creds, node, results):
+    wc = cnos_login(node, configmanager, creds)
+    hinfo = wc.grab_json_response('/nos/api/sysinfo/globalhealthstatus')
+    summary = hinfo['status'].lower()
+    if summary == 'noncritical':
+        summary = 'warning'
+    results.put(msg.HealthSummary(summary, name=node))
+    state = None
+    badreadings = []
+    if summary != 'ok':  # temperature or dump or fans or psu
+        wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        if switchinfo:
+            badreadings.append(
+                SwitchSensor('Panicdump', ['Present'], health='warning'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/temperatures')
+        for temp in switchinfo:
+            if temp == 'Temperature threshold':
+                continue
+            if switchinfo[temp]['State'] != 'OK':
+                temphealth = switchinfo[temp]['State'].lower()
+                if temphealth == 'noncritical':
+                    temphealth = 'warning'
+                tempval = switchinfo[temp]['Temp']
+                badreadings.append(
+                    SwitchSensor(temp, [], value=tempval, health=temphealth))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/fans')
+        for fan in switchinfo:
+            if switchinfo[fan]['speed-rpm'] < 100:
+                badreadings.append(
+                    SwitchSensor(fan, [], value=switchinfo[fan]['speed-rpm'],
+                                 health='critical'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/power')
+        for psu in switchinfo:
+            if switchinfo[psu]['State'] != 'Normal ON':
+                psuname = switchinfo[psu]['Name']
+                badreadings.append(
+                    SwitchSensor(psuname, states=[switchinfo[psu]['State']],
+                                 health='critical'))
+    results.put(msg.SensorReadings(badreadings, name=node))