Add '..' to range operators

The perl-style range operator exists in some tools, we
might as well support that alongside the usual '-'
operator and ':' python style operator.

.gitignore

@@ -1 +1,4 @@
-.idea
+*.pyc
+.*.
+confluent_client/man/man*
+.*.sw*

confluent_client/addattribs.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+import os
+import sys
+path = os.path.dirname(os.path.realpath(__file__))
+try:
+    sys.path.remove(path)
+except Exception:
+    pass
+path = os.path.realpath(os.path.join(path, '..', 'confluent_server'))
+sys.path.append(path)
+
+import confluent.config.attributes as attr
+import shutil
+
+shutil.copyfile('doc/man/nodeattrib.ronn.tmpl', 'doc/man/nodeattrib.ronn')
+shutil.copyfile('doc/man/nodegroupattrib.ronn.tmpl', 'doc/man/nodegroupattrib.ronn')
+with open('doc/man/nodeattrib.ronn', 'a') as outf:
+    for field in sorted(attr.node):
+        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+with open('doc/man/nodegroupattrib.ronn', 'a') as outf:
+    for field in sorted(attr.node):
+        outf.write('\n* `{0}`:\n  {1}\n'.format(field, attr.node[field]['description']))
+
+

confluent_client/bin/collate

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -43,6 +43,8 @@ argparser.add_option('-d', '--diff', action='store_true',
                           'output group and others')
 argparser.add_option('-w', '--watch', action='store_true',
                      help='Show intermediate results while running')
+argparser.add_option('-g', '--groupcount', action='store_true',
+                     help='Show count of output groups rather than the actual output')                     
 argparser.add_option('-s', '--skipcommon', action='store_true',
                      help='Do not print most common result, only non modal '
                           'groups, useful when combined with -d')
@@ -69,6 +71,9 @@ def print_current():
     if options.diff:
         grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
                           reverse=options.reverse, basenode=options.base)
+    elif options.groupcount:
+        grouped.generate_byoutput()
+        print(len(grouped.byoutput))
     else:
         grouped.print_all(skipmodal=options.skipcommon,
                                count=options.count,

confluent_client/bin/confetty

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -76,6 +76,11 @@ import confluent.termhandler as termhandler
 import confluent.tlvdata as tlvdata
 import confluent.client as client
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 conserversequence = '\x05c'  # ctrl-e, c
 clearpowermessage = False
 
@@ -89,6 +94,10 @@ except NameError:
 netserver = None
 laststate = {}
 
+try:
+    input = raw_input
+except NameError:
+    pass
 
 class BailOut(Exception):
     def __init__(self, errorcode=0):
@@ -195,9 +204,9 @@ def prompt():
     if os.environ.get('TERM', '') not in ('linux'):
         sys.stdout.write('\x1b]0;confetty: %s\x07' % target)
     try:
-        return raw_input(target + ' -> ')
+        return input(target + ' -> ')
     except KeyboardInterrupt:
-        print ""
+        print("")
         return ""
     except EOFError:  # ctrl-d
         print("exit")
@@ -295,8 +304,11 @@ currchildren = None
 
 
 def print_result(res):
+    global exitcode
     if 'errorcode' in res or 'error' in res:
-        print res['error']
+        print(res['error'])
+        if 'errorcode' in res:
+            exitcode |= res['errorcode']
         return
     if 'databynode' in res:
         print_result(res['databynode'])
@@ -309,9 +321,9 @@ def print_result(res):
             attrstr = '%s=%s' % (key, recurse_format(res[key]))
         elif not isinstance(res[key], dict):
             try:
-                print '{0}: {1}'.format(key, res[key])
+                print('{0}: {1}'.format(key, res[key]))
             except UnicodeEncodeError:
-                print '{0}: {1}'.format(key, repr(res[key]))
+                print('{0}: {1}'.format(key, repr(res[key])))
             continue
         elif 'value' in res[key] and res[key]['value'] is not None:
             attrstr = '%s="%s"' % (key, res[key]['value'])
@@ -324,7 +336,7 @@ def print_result(res):
         else:
             sys.stdout.write('{0}: '.format(key))
             if isinstance(res[key], str) or isinstance(res[key], unicode):
-                print res[key]
+                print(res[key])
             else:
                 print_result(res[key])
             continue
@@ -423,10 +435,10 @@ def do_command(command, server):
         for res in session.read(targpath):
             if 'item' in res:  # a link relation
                 if type(res['item']) == dict:
-                    print res['item']["href"]
+                    print(res['item']["href"])
                 else:
                     for item in res['item']:
-                        print item["href"]
+                        print(item["href"])
             else:  # generic attributes to list
                 if 'error' in res:
                     sys.stderr.write(res['error'] + '\n')
@@ -851,7 +863,7 @@ def server_connect():
         passphrase = os.environ['CONFLUENT_PASSPHRASE']
         session.authenticate(username, passphrase)
     while not session.authenticated:
-        username = raw_input("Name: ")
+        username = input("Name: ")
         passphrase = getpass.getpass("Passphrase: ")
         session.authenticate(username, passphrase)
 
@@ -871,7 +883,7 @@ def main():
     global inconsole
     try:
         server_connect()
-    except EOFError, KeyboardInterrupt:
+    except (EOFError, KeyboardInterrupt) as _:
         raise BailOut(0)
     except socket.gaierror:
         sys.stderr.write('Could not connect to confluent\n')
@@ -929,6 +941,7 @@ def main():
                         updatestatus(data)
                         continue
                     if data is not None:
+                        data = client.stringify(data)
                         if clearpowermessage:
                             sys.stdout.write("\x1b[2J\x1b[;H")
                             clearpowermessage = False
@@ -952,7 +965,7 @@ def main():
                             try:
                                 server_connect()
                                 connected = True
-                            except (socket.gaierror, socket.error):
+                            except Exception:
                                 pass
                             if not connected:
                                 time.sleep(1)

confluent_client/bin/confluent2ansible

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import optparse
 import signal
 import sys

confluent_client/bin/confluent2lxca

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal

confluent_client/bin/confluent2xcat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 import csv
 import optparse
 import signal

confluent_client/bin/nodeattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -50,6 +50,9 @@ argparser.add_option('-c', '--clear', action='store_true',
                      help='Clear attributes')
 argparser.add_option('-p', '--prompt', action='store_true',
                      help='Prompt for attribute values interactively')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Prompt if trying to set attributes on more '
+                          'than specified number of nodes')
 (options, args) = argparser.parse_args()
 
 
@@ -87,6 +90,7 @@ if len(args) > 1:
                     if oneval != twoval:
                         print('Values did not match.')
                 argassign[arg] = twoval
+        session.stop_if_noderange_over(noderange, options.maxnodes)
         exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
     try:
         # setting user output to what the user inputs
@@ -128,4 +132,4 @@ else:
         else:
             print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodebmcreset

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,8 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Number of nodes to affect before prompting for confirmation')
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,8 +45,10 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'configuration/management_controller/reset', 'reset', key='state', errnodes=errorNodes)   # = 0 if successful
+if success != 0:
+    sys.exit(success)
 
 # Determine which nodes were successful and print them
 
@@ -56,7 +60,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": BMC Reset Successful"
+    print(node + ": BMC Reset Successful")
 
 
 sys.exit(success)

confluent_client/bin/nodeboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -42,6 +42,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                      default=False,
                      help='Request the boot device be persistent rather than '
                           'one time')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to boot, '
+                          'prompting if over the threshold')                        
 
 (options, args) = argparser.parse_args()
 
@@ -54,8 +58,8 @@ except IndexError:
     sys.exit(1)
 client.check_globbing(noderange)
 bootdev = None
-if len(sys.argv) > 2:
-    bootdev = sys.argv[2]
+if len(args) > 1:
+    bootdev = args[1]
     if bootdev in ('net', 'pxe'):
         bootdev = 'network'
 session = client.Command()
@@ -66,6 +70,7 @@ else:
     bootmode = 'uefi'
 
 errnodes = set([])
+session.stop_if_noderange_over(noderange, options.maxnodes)
 rc = session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist,

confluent_client/bin/nodeconfig

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -54,6 +54,12 @@ argparser.add_option('-d', '--detail', dest='detail',
                      action='store_true', default=False,
                      help='Provide verbose information as available, such as '
                           'help text and possible valid values')
+argparser.add_option('-e', '--extra', dest='extra',
+                     action='store_true', default=False,
+                     help='Access extra configuration.  Extra configuration is generally '
+                           'reserved for unpopular or redundant options that may be slow to '
+                           'read.  Notably the IMM category on Lenovo settings is considered '
+                           'to be extra configuration')
 argparser.add_option('-x', '--exclude', dest='exclude',
                      action='store_true', default=False,
                      help='Treat positional arguments as items to not '
@@ -68,6 +74,10 @@ argparser.add_option('-r', '--restoredefault', default=False,
                      help='Restore the configuration of the node '
                           'to factory default for given component. '
                           'Currently only uefi is supported')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure, '
+                          'prompting if over the threshold')
 (options, args) = argparser.parse_args()
 
 cfgpaths = {
@@ -98,6 +108,9 @@ setmode = None
 assignment = {}
 queryparms = {}
 printsys = []
+printbmc = []
+printextbmc = []
+printallbmc = False
 setsys = {}
 forceset = False
 needval = None
@@ -122,7 +135,7 @@ def _assign_value():
 
 
 def parse_config_line(arguments):
-    global setmode, forceset, key, value, needval, candidate, path, attrib
+    global setmode, printallbmc, forceset, key, value, needval, candidate, path, attrib
     for param in arguments:
         if param == 'show':
             continue  # forgive muscle memory of pasu users
@@ -154,6 +167,8 @@ def parse_config_line(arguments):
             if setmode != False:
                 bailout('Cannot do set and query in same command')
             if '.' not in param:
+                if param == 'bmc':
+                    printallbmc = True
                 matchedparms = False
                 for candidate in cfgpaths:
                     if candidate.startswith('{0}.'.format(param)):
@@ -169,10 +184,19 @@ def parse_config_line(arguments):
                                 del queryparms[path]
                             except KeyError:
                                 pass
-                if not matchedparms:
+                if param.lower() == 'imm':
+                    printextbmc.append(param)
+                    options.extra = True
+                elif not matchedparms:
                     printsys.append(param)
             elif param not in cfgpaths:
-                printsys.append(param)
+                if param.startswith('bmc.'):
+                    printbmc.append(param.replace('bmc.', ''))
+                elif param.lower().startswith('imm'):
+                    options.extra = True
+                    printextbmc.append(param)
+                else:
+                    printsys.append(param)
             else:
                 path, attrib = cfgpaths[param]
                 path = '/noderange/{0}/{1}'.format(noderange, path)
@@ -197,19 +221,29 @@ else:
     parse_config_line(args[1:])
 session = client.Command()
 rcode = 0
-if options.restoredefault and options.restoredefault.lower() in (
+if options.restoredefault:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
+    if options.restoredefault.lower() in (
         'sys', 'system', 'uefi', 'bios'):
-    for fr in session.update(
-            '/noderange/{0}/configuration/system/clear'.format(noderange),
-            {'clear': True}):
-        rcode |= client.printerror(fr)
-    sys.exit(rcode)
-elif options.restoredefault:
-    sys.stderr.write(
-        'Unrecognized component to restore defaults: {0}\n'.format(
-            options.restoredefault))
-    sys.exit(1)
+        for fr in session.update(
+                '/noderange/{0}/configuration/system/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    elif options.restoredefault.lower() in (
+        'bmc', 'imm', 'xcc'):
+        for fr in session.update(
+                '/noderange/{0}/configuration/management_controller/clear'.format(noderange),
+                {'clear': True}):
+            rcode |= client.printerror(fr)
+        sys.exit(rcode)
+    else:
+        sys.stderr.write(
+            'Unrecognized component to restore defaults: {0}\n'.format(
+                options.restoredefault))
+        sys.exit(1)
 if setmode:
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     if options.exclude:
         sys.stderr.write('Cannot use exclude and assign at the same time\n')
         sys.exit(1)
@@ -217,8 +251,12 @@ if setmode:
     attrnamebypath = {}
     for key in assignment:
         if key not in cfgpaths:
-            path = 'configuration/system/all'
-            attrib = key
+            if key.startswith('bmc.'):
+                path = 'configuration/management_controller/extended/all'
+                attrib = key.replace('bmc.', '')
+            else:
+                path = 'configuration/system/all'
+                attrib = key
         else:
             path, attrib = cfgpaths[key]
         if path not in updatebypath:
@@ -250,6 +288,15 @@ else:
                                       NullOpt(), queryparms[path])
         if rc:
             sys.exit(rc)
+    if printsys == 'all' or printextbmc or printbmc or printallbmc:
+        if printbmc or not printextbmc:
+         rcode = client.print_attrib_path(
+                        '/noderange/{0}/configuration/management_controller/extended/all'.format(noderange),
+                        session, printbmc, options, attrprefix='bmc.')
+        if options.extra:
+            rcode |= client.print_attrib_path(
+                    '/noderange/{0}/configuration/management_controller/extended/extra'.format(noderange),
+                    session, printextbmc, options)
     if printsys or options.exclude:
         if printsys == 'all':
             printsys = []

confluent_client/bin/nodeconsole

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -51,27 +51,28 @@ if options.tile:
         nodes.append(node)
     initial = True
     pane = 0
+    sessname = 'nodeconsole_{0}'.format(os.getpid())
     for node in sortutil.natural_sort(nodes):
+        panename = '{0}:{1}'.format(sessname, pane)
         if initial:
             initial = False
             subprocess.call(
                 ['tmux', 'new-session', '-d', '-s',
-                 'nodeconsole_{0}'.format(os.getpid()), '-x', '800', '-y',
+                 sessname, '-x', '800', '-y',
                  '800', '{0} -m 5 start /nodes/{1}/console/session'.format(
                     confettypath, node)])
         else:
-            subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
-            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
-            pane += 1
+            subprocess.call(['tmux', 'select-pane', '-t', sessname])
+            subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
             subprocess.call(
-                ['tmux', 'split', '-h',
+                    ['tmux', 'split', '-h', '-t', sessname,
                  '{0} -m 5 start /nodes/{1}/console/session'.format(
                      confettypath, node)])
-            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
-    subprocess.call(['tmux', 'select-pane', '-t', '0'])
-    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
-    os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
-        os.getpid()))
+            subprocess.call(['tmux', 'select-layout', '-t', sessname, 'tiled'], stdout=null)
+            pane += 1
+    subprocess.call(['tmux', 'select-pane', '-t', sessname])
+    subprocess.call(['tmux', 'set-option', '-t', panename, 'pane-border-status', 'top'], stderr=null)
+    os.execlp('tmux', 'tmux', 'attach', '-t', sessname)
 else:
     os.execl(confettypath, confettypath, 'start',
              '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodedefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/noderange/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodediscover

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -237,10 +237,12 @@ def clear_discovery(options, session):
             else:
                 print(repr(res))
 
-def list_matching_macs(options, session, node=None):
+def list_matching_macs(options, session, node=None, checknode=True):
     path = '/discovery/'
     if node:
         path += 'by-node/{0}/'.format(node)
+    elif checknode and options.node:
+        path += 'by-node/{0}/'.format(options.node)
     if options.model:
         path += 'by-model/{0}/'.format(options.model)
     if options.serial:
@@ -277,11 +279,11 @@ def assign_discovery(options, session, needid=True):
         abort = True
     if abort:
         sys.exit(1)
-    matches = list_matching_macs(options, session, None if needid else options.node)
+    matches = list_matching_macs(options, session, None if needid else options.node, False)
     if not matches:
         # Do a rescan to catch missing requested data
         blocking_scan(session)
-        matches = list_matching_macs(options, session, None if needid else options.node)
+        matches = list_matching_macs(options, session, None if needid else options.node, False)
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)

confluent_client/bin/nodeeventlog

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2019 Lenovo
@@ -38,6 +38,10 @@ if sys.version_info[0] < 3:
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog [options] noderange [clear]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to clear if clearing log, '
+                          'prompting if over the threshold')            
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -46,11 +50,11 @@ except IndexError:
     sys.exit(1)
 client.check_globbing(noderange)
 deletemode = False
-if len(sys.argv) > 3:
+if len(args) > 2:
     argparser.print_help()
     sys.exit(1)
-if len(sys.argv) == 3:
-    if sys.argv[2] == 'clear':
+if len(args) == 2:
+    if args[1] == 'clear':
         deletemode = True
     else:
         argparser.print_help()
@@ -88,6 +92,7 @@ def format_event(evt):
 
 if deletemode:
     func = session.delete
+    session.stop_if_noderange_over(noderange, options.maxnodes)
 else:
     func = session.read
 for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):

confluent_client/bin/nodefirmware

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -59,6 +59,10 @@ argparser = optparse.OptionParser(
           "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='When updating, prompt if more than the specified '
+                          'number of servers will be affected')
+             
 (options, args) = argparser.parse_args()
 upfile = None
 try:
@@ -95,6 +99,7 @@ def get_update_progress(session, url):
 
 def update_firmware(session, filename):
     global exitcode
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     output = sq.ScreenPrinter(noderange, session)
     nodeurls = {}
     filename = os.path.abspath(filename)
@@ -163,4 +168,4 @@ try:
         update_firmware(session, upfile)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupattrib

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -97,7 +97,7 @@ if len(args) > 1:
 
         requestargs=args[1:]
     except Exception as e:
-        print str(e)
+        print(str(e))
 
 if exitcode != 0:
     sys.exit(exitcode)
@@ -123,6 +123,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegroupdefine

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -55,4 +55,4 @@ for r in session.create('/nodegroups/', attribs):
         exitcode |= 1
     if 'created' in r:
         print('{0}: created'.format(r['created']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouplist

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo

confluent_client/bin/nodegroupremove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -49,4 +49,4 @@ for r in session.delete('/nodegroups/{0}'.format(noderange)):
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodegrouprename

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo

confluent_client/bin/nodehealth

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo

confluent_client/bin/nodeidentify

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -31,7 +31,7 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off|blink]")
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -42,9 +42,6 @@ client.check_globbing(noderange)
 identifystate = None
 if len(sys.argv) > 2:
     identifystate = sys.argv[2]
-else:
-    argparser.print_help()
-    sys.exit(1)
 session = client.Command()
 exitcode = 0
 sys.exit(

confluent_client/bin/nodeinventory

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo

confluent_client/bin/nodelicense

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo
@@ -37,6 +37,10 @@ exitcode = 0
 argparser = optparse.OptionParser(
     usage="Usage: "
           "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete licenses from, '
+                          'prompting if over the threshold')                  
 (options, args) = argparser.parse_args()
 upfile = None
 downdir = None
@@ -52,7 +56,7 @@ try:
             delete = args[2]
         elif args[1] != 'list':
             argparser.print_help()
-            sys.exit(1)
+            sys.exit(1)        
 except IndexError:
     argparser.print_help()
     sys.exit(1)
@@ -61,10 +65,6 @@ client.check_globbing(noderange)
 
 def install_license(session, filename):
     global exitcode
-    if not os.path.exists(filename):
-        sys.stderr.write('Unable to locate requested file {0}\n'.format(
-            filename))
-        sys.exit(404)
     resource = '/noderange/{0}/configuration/' \
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
@@ -83,18 +83,16 @@ def save_licenses(session, dirname):
     resource = '/noderange/{0}/configuration/' \
                'management_controller/save_licenses'.format(noderange)
     filename = os.path.abspath(dirname)
-    if not os.path.exists(filename):
-        sys.stderr.write('Unable to locate specified directory {0}\n'.format(
-            filename))
-        sys.exit(404)
     instargs = {'dirname': filename}
     for res in session.create(resource, instargs):
         for node in res.get('databynode', {}):
             fname = res['databynode'][node].get('filename', None)
             if fname:
                 print('{0}: Saved license to {1}'.format(node, fname))
+            elif 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(node, res['databynode'][node]['error']))
             else:
-                sys.stderr.write('{0}: {1}', node, repr(res['databynode'][node]))
+                sys.stderr.write('{0}: {1}\n'.format(node, repr(res['databynode'][node])))
 
 
 def show_licenses(session):
@@ -138,6 +136,7 @@ try:
     elif downdir:
         save_licenses(session, downdir)
     elif delete:
+        session.stop_if_noderange_over(noderange, options.maxnodes)
         delete_license(session, delete)
     else:
         show_licenses(session)

confluent_client/bin/nodelist

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo

confluent_client/bin/nodemedia

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -188,4 +188,4 @@ def main():
         sys.exit(1)
     handler(noderange, media)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodepower

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -37,6 +37,11 @@ argparser = optparse.OptionParser(
 argparser.add_option('-p', '--showprevious', dest='previous',
                      action='store_true', default=False,
                      help='Show previous power state')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to change power state, '
+                          'prompting if over the threshold')
+
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -72,4 +77,4 @@ if options.previous:
     # add dictionary to session
     session.add_precede_dict(prev)
 
-sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate, promptover=options.maxnodes))

confluent_client/bin/noderemove

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -35,6 +35,10 @@ import confluent.client as client
 argparser = optparse.OptionParser(
     usage='''\n       %prog noderange
              \n ''')
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to delete, '
+                          'prompting if over the threshold')      
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
@@ -43,10 +47,11 @@ noderange = args[0]
 client.check_globbing(noderange)
 session = client.Command()
 exitcode = 0
+session.stop_if_noderange_over(noderange, options.maxnodes)
 for r in session.delete('/noderange/{0}'.format(noderange)):
     if 'error' in r:
         sys.stderr.write(r['error'] + '\n')
         exitcode |= 1
     if 'deleted' in r:
         print('{0}: deleted'.format(r['deleted']))
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/noderename

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo

confluent_client/bin/nodereseat

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -32,6 +32,10 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to reseat, '
+                          'prompting if over the threshold')   
 (options, args) = argparser.parse_args()
 try:
     noderange = args[0]
@@ -43,7 +47,7 @@ session = client.Command()
 exitcode = 0
 
 errorNodes = set([])
-
+session.stop_if_noderange_over(noderange, options.maxnodes)
 success = session.simple_noderange_command(noderange, 'power/reseat', 'reseat', key='reseat', errnodes=errorNodes)   # = 0 if successful
 
 # Determine which nodes were successful and print them
@@ -56,7 +60,7 @@ for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
 goodNodes = allNodes - errorNodes
 
 for node in goodNodes:
-    print node + ": Reseat successful"
+    print(node + ": Reseat successful")
 
 
 sys.exit(success)

confluent_client/bin/nodersync

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -42,6 +42,10 @@ def run():
     argparser = optparse.OptionParser(
         usage="Usage: %prog location noderange:location",
     )
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run rsync to, '
+                          'prompting if over the threshold')   
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -55,7 +59,7 @@ def run():
     noderange, targpath = args[-1].split(':', 1)
     client.check_globbing(noderange)
     c = client.Command()
-    cmdstr = " ".join(args[:-1])
+    cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
     cmdstr += ' {node}:' + targpath
 
@@ -64,15 +68,17 @@ def run():
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
-
+    c.stop_if_noderange_over(noderange, options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
-                        {'expression': cmdstr}):
+            {'expression': cmdstr}):
         if 'error' in exp:
             sys.stderr.write(exp['error'] + '\n')
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -98,7 +104,7 @@ def run():
                     if desc['type'] == 'stdout':
                         if node not in pernodeout:
                             pernodeout[node] = ''
-                        pernodeout[node] += data
+                        pernodeout[node] += client.stringify(data)
                         if '\n' in pernodeout[node]:
                             currout, pernodeout[node] = pernodeout[node].split('\n', 1)
                             if currout:
@@ -117,7 +123,7 @@ def run():
                         output.set_output(node, 'error!')
                         if node not in nodeerrs:
                             nodeerrs[node] = ''
-                        nodeerrs[node] += data
+                        nodeerrs[node] += client.stringify(data)
                 else:
                     pop = desc['popen']
                     ret = pop.poll()

confluent_client/bin/noderun

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -46,6 +46,10 @@ def run():
                          help='Number of commands to run at a time')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run the command with, '
+                          'prompting if over the threshold')                           
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -63,7 +67,7 @@ def run():
     pipedesc = {}
     pendingexecs = deque()
     exitcode = 0
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -71,7 +75,9 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
+            cmd = ex[node]['value']
+            if not isinstance(cmd, bytes) and not isinstance(cmd, str):
+                cmd = cmd.encode('utf-8')
             cmdv = shlex.split(cmd)
             if currprocs < concurrentprocs:
                 currprocs += 1
@@ -95,6 +101,7 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
+                        data = client.stringify(data)
                         if options.nonodeprefix:
                             sys.stderr.write(data)
                         else:
@@ -112,6 +119,7 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
+                line = client.stringify(line)
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:

confluent_client/bin/nodesensors

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015-2017 Lenovo
@@ -149,8 +149,11 @@ def sensorpass(showout=True, appendtime=False):
                         if appendtime:
                             showval += ' @' + time.strftime(
                                 '%Y-%m-%dT%H:%M:%S')
-                        print(u'{0}: {1}:{2}'.format(
-                            node, sensedata['name'], showval).encode('utf8'))
+                        printval = u'{0}: {1}:{2}'.format(
+                            node, sensedata['name'], showval)
+                        if not isinstance(printval, str):
+                            printval = printval.encode('utf-8')
+                        print(printval)
                         sys.stdout.flush()
     return resultdata
 
@@ -199,7 +202,11 @@ def main():
             orderedsensors.append(name)
         orderedsensors.sort()
         for name in orderedsensors:
-            headernames.append(sensorheaders[name].encode('utf-8'))
+            headername = sensorheaders[name]
+            if (not isinstance(headername, str) and
+                    not isinstance(headername, bytes)):
+                headername = headername.encode('utf-8')
+            headernames.append(headername)
     if options.csv:
         linebyline = False
         csvwriter = csv.writer(sys.stdout)

confluent_client/bin/nodesetboot

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2015 Lenovo
@@ -43,7 +43,10 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 argparser.add_option('-u', '--uefi', dest='uefi', action='store_true',
                      default=True,
                      help='Request UEFI style boot (rather than BIOS)')
-
+argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to modify next boot device, '
+                          'prompting if over the threshold')                        
 (options, args) = argparser.parse_args()
 
 try:
@@ -63,6 +66,7 @@ if options.biosmode:
     bootmode = 'bios'
 else:
     bootmode = 'uefi'
+session.stop_if_noderange_over(noderange, options.maxnodes)
 sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist))

confluent_client/bin/nodeshell

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2016-2017 Lenovo
@@ -46,6 +46,12 @@ def run():
                          help='Number of commands to run at a time')
     argparser.add_option('-n', '--nonodeprefix', action='store_true',
                          help='Do not prefix output with node names')
+    argparser.add_option('-p', '--port', type='int', default=0,
+                         help='Specify a custom port for ssh')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to run remote ssh command to, '
+                          'prompting if over the threshold')   
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -64,7 +70,7 @@ def run():
     pendingexecs = deque()
     exitcode = 0
 
-
+    c.stop_if_noderange_over(args[0], options.maxnodes)
     for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
                         {'expression': cmdstr}):
         if 'error' in exp:
@@ -72,8 +78,13 @@ def run():
             exitcode |= exp.get('errorcode', 1)
         ex = exp.get('databynode', ())
         for node in ex:
-            cmd = ex[node]['value'].encode('utf-8')
-            cmdv = ['ssh', node, cmd]
+            cmd = ex[node]['value']
+            if not isinstance(cmd, str) and not isinstance(cmd, bytes):
+                cmd = cmd.encode('utf-8')
+            if options.port:
+                cmdv = ['ssh', '-p', '{0}'.format(options.port), node, cmd]
+            else:
+                cmdv = ['ssh', node, cmd]
             if currprocs < concurrentprocs:
                 currprocs += 1
                 run_cmdv(node, cmdv, all, pipedesc)
@@ -96,6 +107,7 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
+                        data = client.stringify(data)
                         if options.nonodeprefix:
                             sys.stderr.write(data)
                         else:
@@ -113,6 +125,7 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
+                line = client.stringify(line)
                 if options.nonodeprefix:
                     sys.stdout.write(line)
                 else:

confluent_client/bin/nodestorage

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -63,10 +63,16 @@ def _print_cfg(scfg):
             sys.stderr.write(e['error'] + '\n')
             exitcode = e.get('errorcode', 1)
         for node in e.get('databynode', {}):
+            curr = e['databynode'][node]
+            if 'error' in curr:
+                if 'no available drives' in curr['error']:
+                    curr['error'] += ' (drives must be in unconfigured state to be available, they must not be in jbod or online state)'
+                sys.stderr.write('{0}: {1}\n'.format(node, curr['error']))
+                exitcode = curr.get('errorcode', 1)
+                continue
             if node not in storagebynode:
                 storagebynode[node] = {'disks': [], 'arrays': [],
                                        'volumes': []}
-            curr = e['databynode'][node]
             storagebynode[node][curr['type'] + 's'].append(curr)
     for node in storagebynode:
         for disk in sorted(storagebynode[node]['disks'],
@@ -108,6 +114,7 @@ def createstorage(noderange, options, args):
         sys.stderr.write('-r and -d are required arguments to create array\n')
         sys.exit(1)
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     names = options.name
     if names is None:
         names = ''.join(args)
@@ -132,6 +139,7 @@ def deletestorage(noderange, options, args):
     else:
         names = options.name
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for rsp in session.delete(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names)):
@@ -162,6 +170,7 @@ def setdisk(noderange, options, args):
         sys.stderr.write('diskset requires valid state as argument (hotspare, jbod, unconfigured)\n')
         sys.exit(1)
     session = client.Command()
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     scfg = session.update('/noderange/{0}/configuration/storage/disks/{1}'.format(noderange, names), {'state': args[0]})
     _print_cfg(scfg)
 
@@ -202,6 +211,10 @@ def main():
                          help='Comma separated list of stripsizes to use when creating volumes. '
                             'This value is in kilobytes.  The default behavior is to allow the '
                             'storage controller to decide.')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to configure storage on, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
     if len(args) == 1:
         args.append('show')

confluent_client/bin/nodesupport

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2018 Lenovo
@@ -64,7 +64,7 @@ def printerror(res, node=None):
 
 
 
-def download_servicedata(noderange, media):
+def download_servicedata(noderange, media, options):
     global exitcode
     session = client.Command()
     output = sq.ScreenPrinter(noderange, session)
@@ -73,6 +73,7 @@ def download_servicedata(noderange, media):
     upargs = {'filename': filename}
     noderrs = {}
     nodeurls = {}
+    session.stop_if_noderange_over(noderange, options.maxnodes)
     for res in session.create(resource, upargs):
         if 'created' not in res:
             for nodename in res.get('databynode', ()):
@@ -121,6 +122,10 @@ def main():
                'management server (the confluent server if running remote, '
                'and the collective.manager if in collective)\n'
                '\n\nSee `man %prog` for more info.\n')
+    argparser.add_option('-m', '--maxnodes', type='int',
+                     help='Specify a maximum number of '
+                          'nodes to download diagnostic data from, '
+                          'prompting if over the threshold') 
     (options, args) = argparser.parse_args()
     media = None
     try:
@@ -142,6 +147,6 @@ def main():
     except KeyError:
         argparser.print_help()
         sys.exit(1)
-    handler(noderange, media)
+    handler(noderange, media, options)
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/stats

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2019 Lenovo

confluent_client/confluent/client.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,6 +39,19 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
 }
 
+try:
+    input = raw_input
+except NameError:
+    pass
+
+def stringify(instr):
+    # Normalize unicode and bytes to 'str', correcting for
+    # current python version
+    if isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.decode('utf-8')
+    elif not isinstance(instr, bytes) and not isinstance(instr, str):
+        return instr.encode('utf-8')
+    return instr
 
 class Tabulator(object):
     def __init__(self, headers):
@@ -107,7 +120,10 @@ def printerror(res, node=None):
 
 
 def cprint(txt):
-    print(txt)
+    try:
+        print(txt)
+    except UnicodeEncodeError:
+        print(txt.encode('utf8'))
     sys.stdout.flush()
 
 def _parseserver(string):
@@ -207,7 +223,7 @@ class Command(object):
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None, errnodes=None, **kwargs):
+                                 key=None, errnodes=None, promptover=None, **kwargs):
         try:
             self._currnoderange = noderange
             rc = 0
@@ -223,6 +239,7 @@ class Command(object):
                         noderange, resource)):
                     rc = self.handle_results(ikey, rc, res, errnodes)
             else:
+                self.stop_if_noderange_over(noderange, promptover)
                 kwargs[ikey] = input
                 for res in self.update('/noderange/{0}/{1}'.format(
                         noderange, resource), kwargs):
@@ -232,6 +249,33 @@ class Command(object):
         except KeyboardInterrupt:
             cprint('')
             return 0
+    
+    def stop_if_noderange_over(self, noderange, maxnodes):
+        if maxnodes is None:
+            return
+        nsize = self.get_noderange_size(noderange)
+        if nsize > maxnodes:
+            if nsize == 1:
+                nodename = list(self.read(
+                    '/noderange/{0}/nodes/'.format(noderange)))[0].get('item', {}).get('href', None)
+                nodename = nodename[:-1]
+                p = input('Command is about to affect node {0}, continue (y/n)? '.format(nodename))
+            else:
+                p = input('Command is about to affect {0} nodes, continue (y/n)? '.format(nsize))
+            if p.lower() != 'y':
+                sys.stderr.write('Aborting at user request\n')
+                sys.exit(1)
+                raise Exception("Aborting at user request")
+        
+
+    def get_noderange_size(self, noderange):
+        numnodes = 0
+        for node in self.read('/noderange/{0}/nodes/'.format(noderange)):
+            if node.get('item', {}).get('href', None):
+                numnodes += 1
+            else:
+                raise Exception("Error trying to size noderange {0}".format(noderange))
+        return numnodes
 
     def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs):
         try:
@@ -332,12 +376,8 @@ class Command(object):
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    try:
-                        replace = raw_input(
-                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
-                    except NameError:
-                        replace = input(
-                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    replace = input(
+                        "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
             cprint('Adding new key for %s:%s' % (server, port))
@@ -389,8 +429,12 @@ def printattributes(session, requestargs, showtype, nodetype, noderange, options
     path = '/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)
     return print_attrib_path(path, session, requestargs, options)
 
+def _sort_attrib(k):
+    if isinstance(k[1], dict) and k[1].get('sortid', None) is not None:
+        return k[1]['sortid']
+    return k[0]
 
-def print_attrib_path(path, session, requestargs, options, rename=None):
+def print_attrib_path(path, session, requestargs, options, rename=None, attrprefix=None):
     exitcode = 0
     seenattributes = set([])
     for res in session.read(path):
@@ -399,9 +443,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
             exitcode = 1
             continue
         for node in sorted(res['databynode']):
-            for attr, val in sorted(
-                    res['databynode'][node].items(),
-                    key=lambda k: k[1].get('sortid', k[0]) if isinstance(k[1], dict) else k[0]):
+            for attr, val in sorted(res['databynode'][node].items(), key=_sort_attrib):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
                     continue
@@ -413,6 +455,8 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                     printattr = rename.get(attr, attr)
                 else:
                     printattr = attr
+                if attrprefix:
+                    printattr = attrprefix + printattr
                 currattr = res['databynode'][node][attr]
                 if show_attr(attr, requestargs, seenattributes, options):
                     if 'value' in currattr:

confluent_client/confluent/tlvdata.py

@@ -25,6 +25,11 @@ try:
 except NameError:
     unicode = str
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 def decodestr(value):
     ret = None
     try:
@@ -40,7 +45,7 @@ def decodestr(value):
 
 def unicode_dictvalues(dictdata):
     for key in dictdata:
-        if isinstance(dictdata[key], str):
+        if isinstance(dictdata[key], bytes):
             dictdata[key] = decodestr(dictdata[key])
         elif isinstance(dictdata[key], datetime):
             dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -51,7 +56,7 @@ def unicode_dictvalues(dictdata):
 
 
 def _unicode_list(currlist):
-    for i in xrange(len(currlist)):
+    for i in range(len(currlist)):
         if isinstance(currlist[i], str):
             currlist[i] = decodestr(currlist[i])
         elif isinstance(currlist[i], dict):
@@ -66,7 +71,7 @@ def send(handle, data):
             data = data.encode('utf-8')
         except AttributeError:
             pass
-    if isinstance(data, str) or isinstance(data, unicode):
+    if isinstance(data, bytes) or isinstance(data, unicode):
         # plain text, e.g. console data
         tl = len(data)
         if tl == 0:

confluent_client/confluent_client.spec.tmpl

@@ -24,10 +24,18 @@ a confluent server.
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+%if "%{dist}" == ".el8"
+python3 setup.py build
+%else
+python2 setup.py build
+%endif
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%if "%{dist}" == ".el8"
+python3 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%else
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-scripts=/opt/confluent/bin --install-purelib=/opt/confluent/lib/python
+%endif
 
 
 %clean

confluent_client/confluent_env.sh

@@ -46,9 +46,8 @@ alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 _confluent_get_args()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
+    NUMARGS=$((COMP_CWORD+1))
+    if [ "${COMP_WORDS[COMP_CWORD]}" == '' ]; then
         CMPARGS+=("")
     fi
     GENNED=""
@@ -75,7 +74,7 @@ function _confluent_generic_completion()
 {
     _confluent_get_args
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
     fi
     if [ $NUMARGS -lt 3 ]; then
         _confluent_nr_completion
@@ -84,7 +83,7 @@ function _confluent_generic_completion()
 }
 _confluent_nodeidentify_completion()
 {
-    COMP_CANDIDATES=("on,off -h")
+    COMP_CANDIDATES=("on,off,blink -h")
     _confluent_generic_completion
 }
 
@@ -111,7 +110,7 @@ _confluent_nodemedia_completion()
         return
     fi
     if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
-        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -lt 3 ]; then
@@ -124,7 +123,7 @@ _confluent_nodefirmware_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[2]} == 'update' ]; then
@@ -142,7 +141,7 @@ _confluent_nodeshell_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -c -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -c -- ${COMP_WORDS[COMP_CWORD]}))
         return
     fi
     if [ $NUMARGS -gt 3 ]; then
@@ -160,7 +159,7 @@ _confluent_nodelicense_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
@@ -183,7 +182,7 @@ _confluent_nodesupport_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[COMP_CWORD]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'servicedata' ]; then
@@ -210,41 +209,36 @@ _confluent_nn_completion()
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_nr_completion()
 {
     CMPARGS=($COMP_LINE)
-    NUMARGS=${#CMPARGS[@]}
-    if [ "${COMP_WORDS[-1]}" == '' ]; then
-        NUMARGS=$((NUMARGS+1))
-    fi
     _confluent_get_args
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    #COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 _confluent_ng_completion()
 {
@@ -252,17 +246,17 @@ _confluent_ng_completion()
     if [ $NUMARGS -gt 2 ]; then
         return;
     fi
-    INPUT=${COMP_WORDS[-1]}
+    INPUT=${COMP_WORDS[COMP_CWORD]}
     INPUT=${INPUT##*,-}
     INPUT=${INPUT##*,}
     INPUT=${INPUT##*@}
     PREFIX=""
-    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
-        PREFIX=${COMP_WORDS[-1]}
+    if [ "$INPUT" != "${COMP_WORDS[COMP_CWORD]}" ]; then
+        PREFIX=${COMP_WORDS[COMP_CWORD]}
         PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
     fi
 
-    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[COMP_CWORD]}"))
 }
 complete -F _confluent_nodeattrib_completion nodeattrib
 complete -F _confluent_nodeattrib_completion nodegroupattrib

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -100,3 +100,5 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)
+
+## ATTRIBUTES

confluent_client/doc/man/nodeconfig.ronn

@@ -22,6 +22,11 @@ given as a node expression, as documented in the man page for nodeattribexpressi
   If combined with `-x`, will show all differing values except those indicated
   by `-x`
 
+* `-e`, `--extra`:
+   Read settings that are generally not needed, but may be slow to retrieve.
+   Notably this includes the IMM category of Lenovo systems.  The most popular
+   IMM settings are available through faster 'bmc' attributes.
+
 * `-x`, `--exclude`:
   Rather than listing only the specified configuration parameters, list all
   attributes except for the specified ones

confluent_client/doc/man/nodediscover.ronn

@@ -23,6 +23,9 @@ data may be filtered by various parameters, as denoted in the options below.
 **nodediscover assign** performs manual discovery, assigning an entry to a node
 identity or, using `-i`, using a csv file to assign nodes all at once.  For
 example, a spreadsheet of serial numbers to desired node names could be used.
+Note that if you see that the host is unreachable, it may be due to the IP
+address on the endpoint having changed since last detected. In such a case, it
+may help to **clear** and try **assign** again.
 
 **nodediscover rescan** requests the server to do an active sweep for new
 devices.  Generally every effort is made to passively detect devices as they

confluent_client/doc/man/nodefirmware.ronn

@@ -13,7 +13,11 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 will retrieve all firmware, but can be directed to fetch specific firmware by
 calling out the name of the firmware (e.g. uefi or xcc) or request reading only
 core firmware firmware by using the word 'core', which is generally a quicker
-operation.
+operation.  Different hardwaremanagement.method indicated plugins may have
+different capabilities available.  For example, the 'core' distinction may
+not be relevant to redfish.  Additionally, the Lenovo XCC makes certain
+information available over IPMI that is not otherwise available (for example
+the FPGA version where applicable).
 
 In the update form, it accepts a single file and attempts to update it using
 the out of band facilities.  Firmware updates can end in one of three states:

confluent_client/doc/man/nodegroupattrib.ronn.tmpl

@@ -41,3 +41,5 @@ the attributes are set on the node versus a group to which a node belongs.
 ## SEE ALSO
 
 nodeattrib(8), nodeattribexpressions(5)
+
+## ATTRIBUTES

confluent_client/doc/man/nodeidentify.ronn

@@ -3,7 +3,7 @@ nodeidentify(8) -- Control the identify LED of confluent nodes
 
 ## SYNOPSIS
 
-`nodidentify <noderange> [on|off]`  
+`nodidentify <noderange> [on|off|blink]`  
 
 ## DESCRIPTION
 
@@ -13,6 +13,7 @@ options are supported:
 
 * `on`: Turn on the identify LED
 * `off`: Turn off the identify LED
+* `blink`: Set the identify LED to blink (when supported by the system)
 
 ## EXAMPLES:
 

confluent_client/doc/man/noderange.ronn

@@ -42,7 +42,7 @@ The attribute name may use a wildcard:
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
 
-Exclusions can be done by prepending a ‘-‘ before a portion of a noderange:
+Exclusions can be done by prepending a '-' before a portion of a noderange:
 `rack1,-n2`
 `compute,-rack1`
 `compute,-location.row=12`
@@ -54,7 +54,7 @@ To indicate nodes that match multiple selections at once (set intersection), the
 For complex expressions, () may be used to indicate order of expanding the noderange to be explicit
 `rack1,-(console.logging=full@compute)`
 
-Noderange syntax can also indicate ‘pagination’, or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
+Noderange syntax can also indicate 'pagination', or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
 `rack1>3<6`
 
 The above would show the seventh through ninth nodes of the rack1 group. Like all other noderange operations, this may be combined with any of the above, but must appear as the very last operation. Ordering is done with a natural sort.

confluent_client/doc/man/nodersync.ronn

@@ -0,0 +1,21 @@
+nodersync(8) -- Run rsync in parallel against a noderange
+=========================================================================
+
+## SYNOPSIS
+
+`nodersync <file/directorylist> <noderange>:<destination>`  
+
+## DESCRIPTION
+
+Supervises execution of rsync to push files or a directory tree to the specified
+noderange. This will present progress as percentage for all nodes.
+
+## OPTIONS
+
+* `-m`:
+  Specify maximum number of nodes for noderange max.
+
+* `-c`:
+  Specify how many rsync executions to do concurrently.  If noderange
+  exceeds the count, then excess nodes will wait until one of the
+  active count completes.

confluent_client/doc/man/nodesupport.ronn

@@ -7,7 +7,7 @@ nodesupport(8) -- Utilities for interacting with vendor support
 
 ## DESCRIPTION
 
-`nodesupport` provides capabilities associated with interactiong with support.
+`nodesupport` provides capabilities associated with interacting with support.
 Currently it only has the `servicedata` subcommand.  `servicedata` takes
 an argument that is either a directory name (that can be used for a single node
 or multiple nodes) or a file name (only to be used with single node noderange).
@@ -16,6 +16,9 @@ connects to the managed system, so it will download to the remote system if runn
 remotely and will download to the collective.manager indicated system if
 running in collective mode.
 
+Note that due to vendor filename requirements, any filename may have vendor
+specific suffixes added to any file produced.
+
 ## EXAMPLES
 
 * Download support data from a single node to a specific filename  

confluent_client/makeman

@@ -1,4 +1,6 @@
 #!/bin/sh
+cd `dirname $0`
+python3 addattribs.py || python2 addattribs.py
 cd `dirname $0`/doc/man
 mkdir -p ../../man/man1
 mkdir -p ../../man/man5

confluent_client/samples/nodeattrib_from_switch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 
 # This is a sample python script for going through all observed mac addresses
 # and assuming they are BMC related and printing nodeattrib commands

confluent_common/confluent_common.spec.tmpl

@@ -22,10 +22,10 @@ This provides the modules common for both client and server
 %setup -n %{name}-%{version} -n %{name}-%{version}
 
 %build
-python setup.py build
+python2 setup.py build
 
 %install
-python setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
+python2 setup.py install --single-version-externally-managed -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES --install-purelib=/opt/confluent/lib/python --install-scripts=/opt/confluent/bin
 
 %clean
 rm -rf $RPM_BUILD_ROOT

confluent_server/bin/collective

@@ -1,8 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 
 import argparse
 import errno
 import os
+import pwd
 import socket
 import subprocess
 import sys
@@ -22,7 +23,7 @@ except NameError:
     pass
 
 def make_certificate():
-    umask = os.umask(0077)
+    umask = os.umask(0o77)
     try:
         os.makedirs('/etc/confluent/cfg')
     except OSError as e:
@@ -40,6 +41,12 @@ def make_certificate():
                               '/etc/confluent/srvcert.pem -subj /CN='
                               '{0}'.format(socket.gethostname()).split(' ')):
         raise Exception('Error generating certificate')
+    try:
+        uid = pwd.getpwnam('confluent').pw_uid
+        os.chown('/etc/confluent/privkey.pem', uid, -1)
+        os.chown('/etc/confluent/srvcert.pem', uid, -1)
+    except KeyError:
+        pass
     print('Certificate generated successfully')
     os.umask(umask)
 
@@ -61,7 +68,7 @@ def join_collective(server, invitation):
         make_certificate()
     s = client.Command().connection
     while not invitation:
-        invitation = raw_input('Paste the invitation here: ')
+        invitation = input('Paste the invitation here: ')
     tlvdata.send(s, {'collective': {'operation': 'join',
                                     'invitation': invitation,
                                     'server': server}})
@@ -69,12 +76,28 @@ def join_collective(server, invitation):
     res = res.get('collective',
                   {'status': 'Unknown response: ' + repr(res)})
     print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)
+
+def delete_member(name):
+    s = client.Command().connection
+    tlvdata.send(s, {'collective': {'operation': 'delete',
+                                    'member': name}})
+    res = tlvdata.recv(s)
+    res = res.get('collective',
+                  {'status': 'Unknown response: ' + repr(res)})
+    print(res.get('status', res.get('error', repr(res))))
+    if 'error' in res:
+        sys.exit(1)
 
 
 def show_collective():
     s = client.Command().connection
     tlvdata.send(s, {'collective': {'operation': 'show'}})
     res = tlvdata.recv(s)
+    if 'error' in res:
+        print(res['error'])
+        return
     if 'error' in res['collective']:
         print(res['collective']['error'])
         return
@@ -104,6 +127,8 @@ def main():
                                     'collective member.  Run collective invite -h for more information')
     ic.add_argument('name', help='Name of server to invite to join the '
                                  'collective')
+    dc = sp.add_parser('delete', help='Delete a member of a collective')
+    dc.add_argument('name', help='Name of server to delete from collective')
     jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
     jc.add_argument('server', help='Existing collective member that ran invite and generated a token')
     jc.add_argument('-i', help='Invitation provided by runniing invite on an '
@@ -117,6 +142,8 @@ def main():
         join_collective(cmdset.server, cmdset.i)
     elif cmdset.command == 'show':
         show_collective()
+    elif cmdset.command == 'delete':
+        delete_member(cmdset.name)
 
 if __name__ == '__main__':
     main()

confluent_server/bin/confluent

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
@@ -32,7 +32,7 @@ import confluent.main
 import multiprocessing
 if __name__ == '__main__':
     multiprocessing.freeze_support()
-    confluent.main.run()
+    confluent.main.run(sys.argv)
 #except:
 #   pass
 #p.disable()

confluent_server/bin/confluentcertutil.py

@@ -1,11 +1,16 @@
+from os.path import exists
 import shutil
 import socket
 import subprocess
 import tempfile
 
 def get_openssl_conf_location():
-    # CentOS/RHAT
-    return '/etc/pki/tls/openssl.cnf'
+    if exists('/etc/pki/tls/openssl.cnf'):
+        return '/etc/pki/tls/openssl.cnf'
+    elif exists('/etc/ssl/openssl.cnf'):
+        return '/etc/ssl/openssl.cnf'
+    else:
+        raise Exception("Cannot find openssl config file")
 
 def get_ip_addresses():
     lines = subprocess.check_output('ip addr'.split(' '))
@@ -47,4 +52,4 @@ def create_certificate():
     )
 
 if __name__ == '__main__':
-    create_certificate()
+    create_certificate()

confluent_server/bin/confluentdbutil

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2017 Lenovo
@@ -62,8 +62,11 @@ if args[0] == 'restore':
     if pid is not None:
         print("Confluent is running, must shut down to restore db")
         sys.exit(1)
+    password = options.password
+    if options.interactivepassword:
+        password = getpass.getpass('Enter password to restore backup: ')
     try:
-        cfm.restore_db_from_directory(dumpdir, options.password)
+        cfm.restore_db_from_directory(dumpdir, password)
     except Exception as e:
         print(str(e))
         sys.exit(1)
@@ -82,11 +85,11 @@ elif args[0] == 'dump':
               "or -s to do encrypted backup that requires keys.json from "
               "another backup to restore.")
         sys.exit(1)
-    os.umask(077)
+    os.umask(0o77)
     main._initsecurity(conf.get_config())
     if not os.path.exists(dumpdir):
         os.makedirs(dumpdir)
-    cfm.dump_db_to_directory(dumpdir, options.password, options.redact,
+    cfm.dump_db_to_directory(dumpdir, password, options.redact,
                              options.skipkeys)
 
 

confluent_server/bin/confluentsrv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python2
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation

confluent_server/builddeb

@@ -3,8 +3,12 @@ cd `dirname $0`
 PKGNAME=$(basename $(pwd))
 DPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
 OPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+PYEXEC=python3
+DSCARGS="--with-python3=True --with-python2=False"
 if grep wheezy /etc/os-release; then
     DPKGNAME=python-$DPKGNAME
+    PYEXEC=python
+    DSCARGS=""
 fi
 cd ..
 mkdir -p /tmp/confluent # $DPKGNAME
@@ -24,15 +28,15 @@ install-scripts=/opt/confluent/bin
 package=$DPKGNAME
 EOF
 
-python setup.py sdist > /dev/null 2>&1
-py2dsc dist/*.tar.gz
+$PYEXEC setup.py sdist > /dev/null 2>&1
+py2dsc $DSCARGS dist/*.tar.gz
 shopt -s extglob
 cd deb_dist/!(*.orig)/
 if [ "$OPKGNAME" = "confluent-server" ]; then
     if grep wheezy /etc/os-release; then
-        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex, python-dateutil, python-pyopenssl, python-msgpack/' debian/control
     else
-        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python-lxml, python-eficompressor, python-pycryptodome, python-dateutil/' debian/control
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python3-lxml, python3-eficompressor, python3-pycryptodome, python3-websocket, python3-msgpack, python3-eventlet, python3-pyparsing, python3-pyte, python3-pyghmi, python3-paramiko/' debian/control
     fi
     if grep wheezy /etc/os-release; then
         echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
@@ -40,6 +44,9 @@ if [ "$OPKGNAME" = "confluent-server" ]; then
         echo 'confluent_client confluent-client' >> debian/pydist-overrides
     fi
 fi
+if ! grep wheezy /etc/os-release; then
+    sed -i 's/^Package: python3-/Package: /' debian/control
+fi
 head -n -1 debian/control > debian/control1
 mv debian/control1 debian/control
 echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules

confluent_server/buildrpm

@@ -6,7 +6,7 @@ fi
 ./makesetup
 VERSION=`cat VERSION`
 PKGNAME=$(basename $(pwd))
-python setup.py sdist > /dev/null 2>&1
+python3 setup.py sdist > /dev/null 2>&1
 cp dist/*.tar.gz ~/rpmbuild/SOURCES
 sed -e 's/#VERSION#/'$VERSION/ $PKGNAME.spec.tmpl > ~/rpmbuild/SPECS/$PKGNAME.spec
 rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec 2> /dev/null |grep ^Wrote:

confluent_server/confluent/asynchttp.py

@@ -64,12 +64,12 @@ class AsyncTermRelation(object):
     # Need to keep an association of term object to async
     # This allows the async handler to know the context of
     # outgoing data to provide to calling code
-    def __init__(self, termid, async):
-        self.async = async
+    def __init__(self, termid, asynchdl):
+        self.asynchdl = asynchdl
         self.termid = termid
 
     def got_data(self, data):
-        self.async.add(self.termid, data)
+        self.asynchdl.add(self.termid, data)
 
 
 class AsyncSession(object):

confluent_server/confluent/auth.py

@@ -27,9 +27,13 @@ from fnmatch import fnmatch
 import hashlib
 import hmac
 import multiprocessing
+import os
+import pwd
 import confluent.userutil as userutil
+import confluent.util as util
+pam = None
 try:
-    import PAM
+    import confluent.pam as pam
 except ImportError:
     pass
 import time
@@ -56,9 +60,9 @@ _allowedbyrole = {
             '/node*/configuration/*',
         ],
         'update': [
-            '/discovery/*', 
+            '/discovery/*',
             '/networking/macs/rescan',
-            '/node*/power/state', 
+            '/node*/power/state',
             '/node*/power/reseat',
             '/node*/attributes/*',
             '/node*/media/*tach',
@@ -94,34 +98,6 @@ _deniedbyrole = {
         ]
     }
 }
-class Credentials(object):
-    def __init__(self, username, passphrase):
-        self.username = username
-        self.passphrase = passphrase
-        self.haspam = False
-
-    def pam_conv(self, auth, query_list):
-        # use stored credentials in a pam conversation
-        self.haspam = True
-        resp = []
-        for query_entry in query_list:
-            query, pamtype = query_entry
-            if query.startswith('Password'):
-                resp.append((self.passphrase, 0))
-            else:
-                return None
-        return resp
-
-
-def _prune_passcache():
-    # This function makes sure we don't remember a passphrase in memory more
-    # than 10 seconds
-    while True:
-        curtime = time.time()
-        for passent in _passcache.iterkeys():
-            if passent[2] < curtime - 10:
-                del _passcache[passent]
-        eventlet.sleep(10)
 
 
 def _get_usertenant(name, tenant=False):
@@ -135,11 +111,13 @@ def _get_usertenant(name, tenant=False):
     administrator account a tenant gets.
     Otherwise, just assume a user in the default tenant
     """
+    if not isinstance(name, bytes):
+        name = name.encode('utf-8')
     if not isinstance(tenant, bool):
         # if not boolean, it must be explicit tenant
         user = name
-    elif '/' in name:  # tenant scoped name
-        tenant, user = name.split('/', 1)
+    elif b'/' in name:  # tenant scoped name
+        tenant, user = name.split(b'/', 1)
     elif configmanager.is_tenant(name):
         # the account is the implicit tenant owner account
         user = name
@@ -147,6 +125,9 @@ def _get_usertenant(name, tenant=False):
     else:  # assume it is a non-tenant user account
         user = name
         tenant = None
+    user = util.stringify(user)
+    if tenant:
+        tenant = util.stringify(tenant)
     yield user
     yield tenant
 
@@ -227,7 +208,6 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
         # would normally make an event and wait
         # but here there's no need for that
         eventlet.sleep(0.5)
-    credobj = Credentials(user, passphrase)
     cfm = configmanager.ConfigManager(tenant, username=user)
     ucfg = cfm.get_user(user)
     if ucfg is None:
@@ -249,7 +229,6 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
             # while someone is legitimately logged in
             # invalidate cache and force the slower check
             del _passcache[(user, tenant)]
-            return None
     if 'cryptpass' in ucfg:
         _passchecking[(user, tenant)] = True
         # TODO(jbjohnso): WORKERPOOL
@@ -279,22 +258,40 @@ def check_user_passphrase(name, passphrase, operation=None, element=None, tenant
         if crypt == crypted:
             _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
             return authorize(user, element, tenant, operation)
-    try:
-        pammy = PAM.pam()
-        pammy.start(_pamservice, user, credobj.pam_conv)
-        pammy.authenticate()
-        pammy.acct_mgmt()
-        del pammy
-        _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
-        return authorize(user, element, tenant, operation, skipuserobj=False)
-    except NameError:
-        pass
-    except PAM.error:
-        pass
+    if pam:
+        pwe = None
+        try:
+            pwe = pwd.getpwnam(user)
+        except KeyError:
+            #pam won't work if the user doesn't exist, don't go further
+            eventlet.sleep(0.05)  # stall even on test for existence of a username
+            return None
+        if os.getuid() != 0:
+            # confluent is running with reduced privilege, however, pam_unix refuses
+            # to let a non-0 user check anothers password.
+            # We will fork and the child will assume elevated privilege to
+            # get unix_chkpwd helper to enable checking /etc/shadow
+            pid = os.fork()
+            if not pid:
+                usergood = False
+                try:
+                    # we change to the uid we are trying to authenticate as, because
+                    # pam_unix uses unix_chkpwd which reque
+                    os.setuid(pwe.pw_uid)
+                    usergood = pam.authenticate(user, passphrase, service=_pamservice)
+                finally:
+                    os._exit(0 if usergood else 1)
+            usergood = os.waitpid(pid, 0)[1] == 0
+        else:
+            # We are running as root, we don't need to fork in order to authenticate the
+            # user
+            usergood = pam.authenticate(user, passphrase, service=_pamservice)
+        if usergood:
+            _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+            return authorize(user, element, tenant, operation, skipuserobj=False)
     eventlet.sleep(0.05)  # stall even on test for existence of a username
     return None
 
-
 def _apply_pbkdf(passphrase, salt):
     return KDF.PBKDF2(passphrase, salt, 32, 10000,
                       lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
@@ -312,4 +309,4 @@ def _do_pbkdf(passphrase, salt):
     # compute.  However, we do want to wait for result, so we have
     # one of the exceedingly rare sort of circumstances where 'apply'
     # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
+    return authworkers.apply(_apply_pbkdf, [passphrase, salt])

confluent_server/confluent/collective/manager.py

@@ -27,6 +27,7 @@ import eventlet.green.ssl as ssl
 import eventlet.green.threading as threading
 import greenlet
 import random
+import sys
 try:
     import OpenSSL.crypto as crypto
 except ImportError:
@@ -70,7 +71,10 @@ def connect_to_leader(cert=None, name=None, leader=None):
         return False
     with connecting:
         with cfm._initlock:
-            tlvdata.recv(remote)  # the banner
+            banner = tlvdata.recv(remote)  # the banner
+            vers = banner.split()[2]
+            if vers != b'v2':
+                raise Exception('This instance only supports protocol 2, synchronize versions between collective members')
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
                 name = get_myname()
@@ -119,7 +123,7 @@ def connect_to_leader(cert=None, name=None, leader=None):
             globaldata = tlvdata.recv(remote)
             dbi = tlvdata.recv(remote)
             dbsize = dbi['dbsize']
-            dbjson = ''
+            dbjson = b''
             while (len(dbjson) < dbsize):
                 ndata = remote.recv(dbsize - len(dbjson))
                 if not ndata:
@@ -148,11 +152,11 @@ def connect_to_leader(cert=None, name=None, leader=None):
                 raise
             currentleader = leader
         #spawn this as a thread...
-        follower = eventlet.spawn(follow_leader, remote)
+        follower = eventlet.spawn(follow_leader, remote, leader)
     return True
 
 
-def follow_leader(remote):
+def follow_leader(remote, leader):
     global currentleader
     cleanexit = False
     try:
@@ -164,8 +168,8 @@ def follow_leader(remote):
             log.log({'info': 'Previous following cleanly closed',
                      'subsystem': 'collective'})
             return
-        log.log({'info': 'Current leader has disappeared, restarting '
-                         'collective membership', 'subsystem': 'collective'})
+        log.log({'info': 'Current leader ({0}) has disappeared, restarting '
+                         'collective membership'.format(leader), 'subsystem': 'collective'})
         # The leader has folded, time to startup again...
         cfm.stop_following()
         currentleader = None
@@ -208,8 +212,7 @@ def handle_connection(connection, cert, request, local=False):
     else:
         if not local:
             return
-
-        if 'show' == operation:
+        if operation in ('show', 'delete'):
             if not list(cfm.list_collective()):
                 tlvdata.send(connection,
                              {'collective': {'error': 'Collective mode not '
@@ -246,7 +249,23 @@ def handle_connection(connection, cert, request, local=False):
                 collinfo['quorum'] = True
             except exc.DegradedCollective:
                 collinfo['quorum'] = False
-            tlvdata.send(connection, {'collective':  collinfo})
+            if operation == 'show':
+                tlvdata.send(connection, {'collective':  collinfo})
+            elif operation == 'delete':
+                todelete = request['member']
+                if (todelete == collinfo['leader'] or 
+                       todelete in collinfo['active']):
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is still active, stop the confluent service to remove it'.format(todelete)}})
+                    return
+                if todelete not in collinfo['offline']:
+                    tlvdata.send(connection, {'collective':
+                            {'error': '{0} is not a recognized collective member'.format(todelete)}})
+                    return
+                cfm.del_collective_member(todelete)
+                tlvdata.send(connection,
+                    {'collective': {'status': 'Successfully deleted {0}'.format(todelete)}})
+                connection.close()
             return
         if 'invite' == operation:
             try:
@@ -267,7 +286,8 @@ def handle_connection(connection, cert, request, local=False):
             invitation = request['invitation']
             try:
                 invitation = base64.b64decode(invitation)
-                name, invitation = invitation.split('@', 1)
+                name, invitation = invitation.split(b'@', 1)
+                name = util.stringify(name)
             except Exception:
                 tlvdata.send(
                     connection,
@@ -375,7 +395,7 @@ def handle_connection(connection, cert, request, local=False):
             connection.close()
             return
         if (currentleader == connection.getpeername()[0] and
-                follower and follower.isAlive()):
+                follower and not follower.dead):
             # if we are happily following this leader already, don't stir
             # the pot
             tlvdata.send(connection, {'status': 0})

confluent_server/confluent/config/attributes.py

@@ -270,8 +270,9 @@ node = {
     },
     'console.method': {
         'description': ('Indicate the method used to access the console of '
-                        'the managed node.'),
-        'validvalues': ('ssh', 'ipmi'),
+                        'the managed node.  If not specified, then console '
+                        'is disabled'),
+        'validvalues': ('ssh', 'ipmi', 'tsmsol'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),
@@ -301,6 +302,7 @@ node = {
     'hardwaremanagement.method': {
         'description': 'The method used to perform operations such as power '
                        'control, get sensor data, get inventory, and so on. '
+                       'ipmi is used if not specified.'
     },
     'enclosure.bay': {
          'description': 'The bay in the enclosure, if any',

confluent_server/confluent/config/conf.py

@@ -16,7 +16,10 @@
 
 
 # This defines config variable to store the global configuration for confluent
-import ConfigParser
+try:
+    import ConfigParser
+except ModuleNotFoundError:
+    import configparser as ConfigParser
 import os
 
 _config = None

confluent_server/confluent/config/configmanager.py

@@ -46,7 +46,10 @@ import Cryptodome.Protocol.KDF as KDF
 from Cryptodome.Cipher import AES
 from Cryptodome.Hash import HMAC
 from Cryptodome.Hash import SHA256
-import anydbm as dbm
+try:
+    import anydbm as dbm
+except ModuleNotFoundError:
+    import dbm
 import ast
 import base64
 import confluent.config.attributes as allattributes
@@ -57,7 +60,10 @@ import confluent.util
 import confluent.netutil as netutil
 import confluent.exceptions as exc
 import copy
-import cPickle
+try:
+    import cPickle
+except ModuleNotFoundError:
+    import pickle as cPickle
 import errno
 import eventlet
 import eventlet.event as event
@@ -65,6 +71,7 @@ import eventlet.green.select as select
 import eventlet.green.threading as gthread
 import fnmatch
 import json
+import msgpack
 import operator
 import os
 import random
@@ -74,6 +81,10 @@ import struct
 import sys
 import threading
 import traceback
+try:
+    unicode
+except NameError:
+    unicode = str
 
 
 _masterkey = None
@@ -101,6 +112,14 @@ _attraliases = {
 }
 _validroles = ('Administrator', 'Operator', 'Monitor')
 
+
+def attrib_supports_expression(attrib):
+    attrib = _attraliases.get(attrib, attrib)
+    if attrib.startswith('secret.') or attrib.startswith('crypted.'):
+        return False
+    return True
+
+
 def _mkpath(pathname):
     try:
         os.makedirs(pathname)
@@ -299,12 +318,12 @@ def check_quorum():
 def exec_on_leader(function, *args):
     if isinstance(cfgleader, bool):
         raise exc.DegradedCollective()
-    xid = os.urandom(8)
+    xid = confluent.util.stringify(base64.b64encode(os.urandom(8)))
     while xid in _pendingchangesets:
-        xid = os.urandom(8)
+        xid = confluent.util.stringify(base64.b64encode(os.urandom(8)))
     _pendingchangesets[xid] = event.Event()
-    rpcpayload = cPickle.dumps({'function': function, 'args': args,
-                                'xid': xid})
+    rpcpayload = msgpack.packb({'function': function, 'args': args,
+                                'xid': xid}, use_bin_type=False)
     rpclen = len(rpcpayload)
     cfgleader.sendall(struct.pack('!Q', rpclen))
     cfgleader.sendall(rpcpayload)
@@ -314,15 +333,24 @@ def exec_on_leader(function, *args):
 
 
 def exec_on_followers(fnname, *args):
-    global _txcount
+    pushes = eventlet.GreenPool()
+    # Check health of collective prior to attempting
+    for _ in pushes.starmap(
+        _push_rpc, [(cfgstreams[s], b'') for s in cfgstreams]):
+        pass
     if len(cfgstreams) < (len(_cfgstore['collective']) // 2):
         # the leader counts in addition to registered streams
         raise exc.DegradedCollective()
+    exec_on_followers_unconditional(fnname, *args)
+
+
+def exec_on_followers_unconditional(fnname, *args):
+    global _txcount
     pushes = eventlet.GreenPool()
     _txcount += 1
-    payload = cPickle.dumps({'function': fnname, 'args': args,
-                             'txcount': _txcount})
-    for res in pushes.starmap(
+    payload = msgpack.packb({'function': fnname, 'args': args,
+                             'txcount': _txcount}, use_bin_type=False)
+    for _ in pushes.starmap(
             _push_rpc, [(cfgstreams[s], payload) for s in cfgstreams]):
         pass
 
@@ -368,9 +396,15 @@ def init_masterkey(password=None, autogen=True):
 
 def _push_rpc(stream, payload):
     with _rpclock:
-        stream.sendall(struct.pack('!Q', len(payload)))
-        if len(payload):
-            stream.sendall(payload)
+        try:
+            stream.sendall(struct.pack('!Q', len(payload)))
+            if len(payload):
+                stream.sendall(payload)
+            return True
+        except Exception:
+            logException()
+            del cfgstreams[stream]
+            stream.close()
 
 
 def decrypt_value(cryptvalue,
@@ -391,12 +425,12 @@ def decrypt_value(cryptvalue,
             raise Exception("bad HMAC value on crypted value")
         decrypter = AES.new(key, AES.MODE_CBC, iv)
         value = decrypter.decrypt(cipherdata)
-        padsize = ord(value[-1])
+        padsize = bytearray(value)[-1]
         pad = value[-padsize:]
         # Note that I cannot grasp what could be done with a subliminal
         # channel in padding in this case, but check the padding anyway
-        for padbyte in pad:
-            if ord(padbyte) != padsize:
+        for padbyte in bytearray(pad):
+            if padbyte != padsize:
                 raise Exception("bad padding in encrypted value")
         return value[0:-padsize]
     else:
@@ -462,11 +496,10 @@ def crypt_value(value,
         key = _masterkey
     iv = os.urandom(12)
     crypter = AES.new(key, AES.MODE_GCM, nonce=iv)
-    try:
-        cryptval, hmac = crypter.encrypt_and_digest(value)
-    except TypeError:
-        cryptval, hmac = crypter.encrypt_and_digest(value.encode('utf-8'))
-    return iv, cryptval, hmac, '\x02'
+    if not isinstance(value, bytes):
+        value = value.encode('utf-8')
+    cryptval, hmac = crypter.encrypt_and_digest(value)
+    return iv, cryptval, hmac, b'\x02'
 
 
 def _load_dict_from_dbm(dpath, tdb):
@@ -474,16 +507,19 @@ def _load_dict_from_dbm(dpath, tdb):
         dbe = dbm.open(tdb, 'r')
         currdict = _cfgstore
         for elem in dpath:
+            elem = confluent.util.stringify(elem)
             if elem not in currdict:
                 currdict[elem] = {}
             currdict = currdict[elem]
         try:
-            for tk in dbe:
-                currdict[tk] = cPickle.loads(dbe[tk])
+            for tk in dbe.keys():
+                tks = confluent.util.stringify(tk)
+                currdict[tks] = cPickle.loads(dbe[tk])
         except AttributeError:
             tk = dbe.firstkey()
             while tk != None:
-                currdict[tk] = cPickle.loads(dbe[tk])
+                tks = confluent.util.stringify(tk)
+                currdict[tks] = cPickle.loads(dbe[tk])
                 tk = dbe.nextkey(tk)
     except dbm.error:
         return
@@ -522,13 +558,7 @@ def set_global(globalname, value, sync=True):
     """
     if _cfgstore is None:
         init(not sync)
-    try:
-        globalname = globalname.encode('utf-8')
-    except AttributeError:
-        # We have to remove the unicode-ness of the string,
-        # but if it is already bytes in python 3, then we will
-        # get an attributeerror, so pass
-        pass
+    globalname = confluent.util.stringify(globalname)
     with _dirtylock:
         if 'dirtyglobals' not in _cfgstore:
             _cfgstore['dirtyglobals'] = set()
@@ -560,11 +590,18 @@ def relay_slaved_requests(name, listener):
             lh = StreamHandler(listener)
             _hasquorum = len(cfgstreams) >= (
                     len(_cfgstore['collective']) // 2)
-            payload = cPickle.dumps({'quorum': _hasquorum})
-            for _ in pushes.starmap(
-                    _push_rpc,
-                    [(cfgstreams[s], payload) for s in cfgstreams]):
-                pass
+            _newquorum = None
+            while _hasquorum != _newquorum:
+                if _newquorum is not None:
+                    _hasquorum = _newquorum
+                payload = msgpack.packb({'quorum': _hasquorum}, use_bin_type=False)
+                for _ in pushes.starmap(
+                        _push_rpc,
+                        [(cfgstreams[s], payload) for s in cfgstreams]):
+                    pass
+                _newquorum = len(cfgstreams) >= (
+                        len(_cfgstore['collective']) // 2)
+            _hasquorum = _newquorum
             if _hasquorum and _pending_collective_updates:
                 apply_pending_collective_updates()
             msg = lh.get_next_msg()
@@ -573,21 +610,27 @@ def relay_slaved_requests(name, listener):
                     raise Exception("Unexpected loss of node in followers: " + name)
                 sz = struct.unpack('!Q', msg)[0]
                 if sz != 0:
-                    rpc = ''
+                    rpc = b''
                     while len(rpc) < sz:
                         nrpc = listener.recv(sz - len(rpc))
                         if not nrpc:
                             raise Exception('Truncated client error')
                         rpc += nrpc
-                    rpc = cPickle.loads(rpc)
+                    rpc = msgpack.unpackb(rpc, raw=False)
                     exc = None
+                    if not (rpc['function'].startswith('_rpc_') or rpc['function'].endswith('_collective_member')):
+                        raise Exception('Unsupported function {0} called'.format(rpc['function']))
                     try:
                         globals()[rpc['function']](*rpc['args'])
+                    except ValueError as ve:
+                        exc = ['ValueError', str(ve)]
                     except Exception as e:
-                        exc = e
+                        exc = ['Exception', str(e)]
                     if 'xid' in rpc:
-                        _push_rpc(listener, cPickle.dumps({'xid': rpc['xid'],
-                                                           'exc': exc}))
+                        res = _push_rpc(listener, msgpack.packb({'xid': rpc['xid'],
+                                                           'exc': exc}, use_bin_type=False))
+                        if not res:
+                            break
                 try:
                     msg = lh.get_next_msg()
                 except Exception:
@@ -604,7 +647,7 @@ def relay_slaved_requests(name, listener):
             if cfgstreams:
                 _hasquorum = len(cfgstreams) >= (
                         len(_cfgstore['collective']) // 2)
-                payload = cPickle.dumps({'quorum': _hasquorum})
+                payload = msgpack.packb({'quorum': _hasquorum}, use_bin_type=False)
                 for _ in pushes.starmap(
                         _push_rpc,
                         [(cfgstreams[s], payload) for s in cfgstreams]):
@@ -632,7 +675,9 @@ class StreamHandler(object):
                 if confluent.util.monotonic_time() > self.expiry:
                     return None
                 if confluent.util.monotonic_time() > self.keepalive:
-                    _push_rpc(self.sock, b'')  # nulls are a keepalive
+                    res = _push_rpc(self.sock, b'')  # nulls are a keepalive
+                    if not res:
+                        return None
                     self.keepalive = confluent.util.monotonic_time() + 20
             self.expiry = confluent.util.monotonic_time() + 60
             msg = self.sock.recv(8)
@@ -723,28 +768,37 @@ def follow_channel(channel):
         while msg:
             sz = struct.unpack('!Q', msg)[0]
             if sz != 0:
-                rpc = ''
+                rpc = b''
                 while len(rpc) < sz:
                     nrpc = channel.recv(sz - len(rpc))
                     if not nrpc:
                         raise Exception('Truncated message error')
                     rpc += nrpc
-                rpc = cPickle.loads(rpc)
+                rpc = msgpack.unpackb(rpc, raw=False)
                 if 'txcount' in rpc:
                     _txcount = rpc['txcount']
                 if 'function' in rpc:
+                    if not (rpc['function'].startswith('_true') or rpc['function'].startswith('_rpc')):
+                        raise Exception("Received unsupported function call: {0}".format(rpc['function']))
                     try:
                         globals()[rpc['function']](*rpc['args'])
                     except Exception as e:
                         print(repr(e))
                 if 'xid' in rpc and rpc['xid']:
                     if rpc.get('exc', None):
-                        _pendingchangesets[rpc['xid']].send_exception(rpc['exc'])
+                        exctype, excstr = rpc['exc']
+                        if exctype == 'ValueError':
+                            exc = ValueError(excstr)
+                        else:
+                            exc = Exception(excstr)
+                        _pendingchangesets[rpc['xid']].send_exception(exc)
                     else:
                         _pendingchangesets[rpc['xid']].send()
                 if 'quorum' in rpc:
                     _hasquorum = rpc['quorum']
-                _push_rpc(channel, b'')  # use null as ACK
+                res = _push_rpc(channel, b'')  # use null as ACK
+                if not res:
+                    break
             msg = lh.get_next_msg()
     finally:
         # mark the connection as broken
@@ -761,6 +815,34 @@ def add_collective_member(name, address, fingerprint):
         exec_on_followers('_true_add_collective_member', name, address, fingerprint)
     _true_add_collective_member(name, address, fingerprint)
 
+def del_collective_member(name):
+    if cfgleader and not isinstance(cfgleader, bool):
+        return exec_on_leader('del_collective_member', name)
+    if cfgstreams:
+        exec_on_followers_unconditional('_true_del_collective_member', name)
+    _true_del_collective_member(name)
+
+def _true_del_collective_member(name, sync=True):
+    global cfgleader
+    name = confluent.util.stringify(name)
+    if _cfgstore is None:
+        return
+    if 'collective' not in _cfgstore:
+        return
+    if name not in _cfgstore['collective']:
+        return
+    del _cfgstore['collective'][name]
+    with _dirtylock:
+        if 'collectivedirty' not in _cfgstore:
+            _cfgstore['collectivedirty'] = set([])
+        _cfgstore['collectivedirty'].add(name)
+    if len(_cfgstore['collective']) < 2:
+        del _cfgstore['collective']
+        cfgleader = None
+    if sync:
+        ConfigManager._bg_sync_to_file()
+
+
 _pending_collective_updates = {}
 
 
@@ -785,10 +867,7 @@ def apply_pending_collective_updates():
 
 
 def _true_add_collective_member(name, address, fingerprint, sync=True):
-    try:
-        name = name.encode('utf-8')
-    except AttributeError:
-        pass
+    name = confluent.util.stringify(name)
     if _cfgstore is None:
         init(not sync)  # use not sync to avoid read from disk
     if 'collective' not in _cfgstore:
@@ -823,8 +902,7 @@ def get_collective_member_by_address(address):
 
 
 def _mark_dirtykey(category, key, tenant=None):
-    if type(key) in (str, unicode):
-        key = key.encode('utf-8')
+    key = confluent.util.stringify(key)
     with _dirtylock:
         if 'dirtykeys' not in _cfgstore:
             _cfgstore['dirtykeys'] = {}
@@ -1090,6 +1168,8 @@ class ConfigManager(object):
             attribute, match = expression.split('=')
         else:
             raise Exception('Invalid Expression')
+        if attribute.startswith('secret.'):
+            raise Exception('Filter by secret attributes is not supported')
         for node in nodes:
             try:
                 currvals = [self._cfgstore['nodes'][node][attribute]['value']]
@@ -1145,9 +1225,9 @@ class ConfigManager(object):
         Returns an identifier that can be used to unsubscribe from these
         notifications using remove_watcher
         """
-        notifierid = random.randint(0, sys.maxint)
+        notifierid = random.randint(0, sys.maxsize)
         while notifierid in self._notifierids:
-            notifierid = random.randint(0, sys.maxint)
+            notifierid = random.randint(0, sys.maxsize)
         self._notifierids[notifierid] = {'attriblist': []}
         if self.tenant not in self._attribwatchers:
             self._attribwatchers[self.tenant] = {}
@@ -1186,9 +1266,9 @@ class ConfigManager(object):
         # use in case of cancellation.
         # I anticipate no more than a handful of watchers of this sort, so
         # this loop should not have to iterate too many times
-        notifierid = random.randint(0, sys.maxint)
+        notifierid = random.randint(0, sys.maxsize)
         while notifierid in self._notifierids:
-            notifierid = random.randint(0, sys.maxint)
+            notifierid = random.randint(0, sys.maxsize)
         # going to track that this is a nodecollection type watcher,
         # but there is no additional data associated.
         self._notifierids[notifierid] = set(['nodecollection'])
@@ -1304,7 +1384,7 @@ class ConfigManager(object):
     def _true_create_usergroup(self, groupname, role="Administrator"):
         if 'usergroups' not in self._cfgstore:
             self._cfgstore['usergroups'] = {}
-        groupname = groupname.encode('utf-8')
+        groupname = confluent.util.stringify(groupname)
         if groupname in self._cfgstore['usergroups']:
             raise Exception("Duplicate groupname requested")
         self._cfgstore['usergroups'][groupname] = {'role': role}
@@ -1334,7 +1414,7 @@ class ConfigManager(object):
             return exec_on_leader('_rpc_master_set_user', self.tenant, name,
                                   attributemap)
         if cfgstreams:
-            exec_on_followers('_rpc_set_user', self.tenant, name)
+            exec_on_followers('_rpc_set_user', self.tenant, name, attributemap)
         self._true_set_user(name, attributemap)
 
     def _true_set_user(self, name, attributemap):
@@ -1406,7 +1486,7 @@ class ConfigManager(object):
                 raise Exception("Duplicate id requested")
         if 'users' not in self._cfgstore:
             self._cfgstore['users'] = {}
-        name = name.encode('utf-8')
+        name = confluent.util.stringify(name)
         if name in self._cfgstore['users']:
             raise Exception("Duplicate username requested")
         self._cfgstore['users'][name] = {'id': uid}
@@ -1414,9 +1494,10 @@ class ConfigManager(object):
             self._cfgstore['users'][name]['displayname'] = displayname
         _cfgstore['main']['idmap'][uid] = {
             'tenant': self.tenant,
-            'username': name
+            'username': name,
+            'role': role,
         }
-        if attributemap is not None:
+        if attributemap:
             self._true_set_user(name, attributemap)
         _mark_dirtykey('users', name, self.tenant)
         _mark_dirtykey('idmap', uid)
@@ -1579,6 +1660,7 @@ class ConfigManager(object):
                 if group in self._cfgstore['nodes'][node]['groups']:
                     self._cfgstore['nodes'][node]['groups'].remove(group)
                     self._node_removed_from_group(node, group, changeset)
+                    _mark_dirtykey('nodes', node, self.tenant)
         for node in nodes:
             if node not in self._cfgstore['nodes']:
                 self._cfgstore['nodes'][node] = {'groups': [group]}
@@ -1664,7 +1746,7 @@ class ConfigManager(object):
                                 "{0} node does not exist to add to {1}".format(
                                     node, group))
         for group in attribmap:
-            group = group.encode('utf-8')
+            group = confluent.util.stringify(group)
             if group not in self._cfgstore['nodegroups']:
                 self._cfgstore['nodegroups'][group] = {'nodes': set()}
             cfgobj = self._cfgstore['nodegroups'][group]
@@ -1722,8 +1804,8 @@ class ConfigManager(object):
         attributes = realattributes
         if type(groups) in (str, unicode):
             groups = (groups,)
-        for group in groups:
-                group = group.encode('utf-8')
+        for group in groups:                
+                group = confluent.util.stringify(group)
                 try:
                     groupentry = self._cfgstore['nodegroups'][group]
                 except KeyError:
@@ -1814,11 +1896,14 @@ class ConfigManager(object):
                             'nodeattrs': {node: [attrname]},
                             'callback': attribwatcher[watchkey][notifierid]
                         }
-        for watcher in notifdata.itervalues():
+        for watcher in notifdata:
+            watcher = notifdata[watcher]
             callback = watcher['callback']
             eventlet.spawn_n(_do_notifier, self, watcher, callback)
 
     def del_nodes(self, nodes):
+        if isinstance(nodes, set):
+            nodes = list(nodes)  # msgpack can't handle set
         if cfgleader:  # slaved to a collective
             return exec_on_leader('_rpc_master_del_nodes', self.tenant,
                                   nodes)
@@ -1828,14 +1913,15 @@ class ConfigManager(object):
 
     def _true_del_nodes(self, nodes):
         if self.tenant in self._nodecollwatchers:
-            for watcher in self._nodecollwatchers[self.tenant].itervalues():
+            for watcher in self._nodecollwatchers[self.tenant]:
+                watcher = self._nodecollwatchers[self.tenant][watcher]
                 watcher(added=(), deleting=nodes, renamed=(), configmanager=self)
         changeset = {}
         for node in nodes:
             # set a reserved attribute for the sake of the change notification
             # framework to trigger on
             changeset[node] = {'_nodedeleted': 1}
-            node = node.encode('utf-8')
+            node = confluent.util.stringify(node)
             if node in self._cfgstore['nodes']:
                 self._sync_groups_to_node(node=node, groups=[],
                                           changeset=changeset)
@@ -1883,7 +1969,7 @@ class ConfigManager(object):
                 realattributes.append(attrname)
         attributes = realattributes
         for node in nodes:
-            node = node.encode('utf-8')
+            node = confluent.util.stringify(node)
             try:
                 nodek = self._cfgstore['nodes'][node]
             except KeyError:
@@ -1954,7 +2040,8 @@ class ConfigManager(object):
             self._recalculate_expressions(cfgobj, formatter=exprmgr, node=renamemap[name], changeset=changeset)
         if self.tenant in self._nodecollwatchers:
             nodecollwatchers = self._nodecollwatchers[self.tenant]
-            for watcher in nodecollwatchers.itervalues():
+            for watcher in nodecollwatchers:
+                watcher = nodecollwatchers[watcher]
                 eventlet.spawn_n(_do_add_watcher, watcher, (), self, renamemap)
         self._bg_sync_to_file()
 
@@ -2011,7 +2098,7 @@ class ConfigManager(object):
         # first do a sanity check of the input upfront
         # this mitigates risk of arguments being partially applied
         for node in attribmap:
-            node = node.encode('utf-8')
+            node = confluent.util.stringify(node)
             if node == '':
                 raise ValueError('"{0}" is not a valid node name'.format(node))
             if autocreate:
@@ -2066,8 +2153,8 @@ class ConfigManager(object):
                             attrname, node)
                         raise ValueError(errstr)
                     attribmap[node][attrname] = attrval
-        for node in attribmap:
-            node = node.encode('utf-8')
+        for node in attribmap:            
+            node = confluent.util.stringify(node)
             exprmgr = None
             if node not in self._cfgstore['nodes']:
                 newnodes.append(node)
@@ -2110,7 +2197,8 @@ class ConfigManager(object):
         if newnodes:
             if self.tenant in self._nodecollwatchers:
                 nodecollwatchers = self._nodecollwatchers[self.tenant]
-                for watcher in nodecollwatchers.itervalues():
+                for watcher in nodecollwatchers:
+                    watcher = nodecollwatchers[watcher]
                     eventlet.spawn_n(_do_add_watcher, watcher, newnodes, self)
         self._bg_sync_to_file()
         #TODO: wait for synchronization to suceed/fail??)
@@ -2228,7 +2316,7 @@ class ConfigManager(object):
                                 target = dumpdata[confarea][element][attribute]['cryptvalue']
                             cryptval = []
                             for value in target:
-                                cryptval.append(base64.b64encode(value))
+                                cryptval.append(confluent.util.stringify(base64.b64encode(value)))
                             if attribute == 'cryptpass':
                                 dumpdata[confarea][element][attribute] = '!'.join(cryptval)
                             else:
@@ -2248,7 +2336,7 @@ class ConfigManager(object):
         _cfgstore = {}
         rootpath = cls._cfgdir
         try:
-            with open(os.path.join(rootpath, 'transactioncount'), 'r') as f:
+            with open(os.path.join(rootpath, 'transactioncount'), 'rb') as f:
                 txbytes = f.read()
                 if len(txbytes) == 8:
                     _txcount = struct.unpack('!Q', txbytes)[0]
@@ -2306,7 +2394,7 @@ class ConfigManager(object):
             if statelessmode:
                 return
             _mkpath(cls._cfgdir)
-            with open(os.path.join(cls._cfgdir, 'transactioncount'), 'w') as f:
+            with open(os.path.join(cls._cfgdir, 'transactioncount'), 'wb') as f:
                 f.write(struct.pack('!Q', _txcount))
             if (fullsync or 'dirtyglobals' in _cfgstore and
                     'globals' in _cfgstore):
@@ -2321,31 +2409,37 @@ class ConfigManager(object):
                     for globalkey in dirtyglobals:
                         if globalkey in _cfgstore['globals']:
                             globalf[globalkey] = \
-                                cPickle.dumps(_cfgstore['globals'][globalkey])
+                                cPickle.dumps(_cfgstore['globals'][globalkey], protocol=cPickle.HIGHEST_PROTOCOL)
                         else:
                             if globalkey in globalf:
                                 del globalf[globalkey]
                 finally:
                     globalf.close()
             if fullsync or 'collectivedirty' in _cfgstore:
-                collectivef = dbm.open(os.path.join(cls._cfgdir, "collective"),
-                                       'c', 384)
-                try:
-                    if fullsync:
-                        colls = _cfgstore['collective']
-                    else:
-                        with _dirtylock:
-                            colls = copy.deepcopy(_cfgstore['collectivedirty'])
-                            del _cfgstore['collectivedirty']
-                    for coll in colls:
-                        if coll in _cfgstore['collective']:
-                            collectivef[coll] = cPickle.dumps(
-                                _cfgstore['collective'][coll])
+                if len(_cfgstore.get('collective', ())) > 1:
+                    collectivef = dbm.open(os.path.join(cls._cfgdir, "collective"),
+                                        'c', 384)
+                    try:
+                        if fullsync:
+                            colls = _cfgstore['collective']
                         else:
-                            if coll in collectivef:
-                                del globalf[coll]
-                finally:
-                    collectivef.close()
+                            with _dirtylock:
+                                colls = copy.deepcopy(_cfgstore['collectivedirty'])
+                                del _cfgstore['collectivedirty']
+                        for coll in colls:
+                            if coll in _cfgstore['collective']:
+                                collectivef[coll] = cPickle.dumps(
+                                    _cfgstore['collective'][coll], protocol=cPickle.HIGHEST_PROTOCOL)
+                            else:
+                                if coll in collectivef:
+                                    del collectivef[coll]
+                    finally:
+                        collectivef.close()
+                else:
+                    try:
+                        os.remove(os.path.join(cls._cfgdir, "collective"))
+                    except OSError:
+                        pass
             if fullsync:
                 pathname = cls._cfgdir
                 currdict = _cfgstore['main']
@@ -2354,7 +2448,7 @@ class ConfigManager(object):
                     dbf = dbm.open(os.path.join(pathname, category), 'c', 384)  # 0600
                     try:
                         for ck in currdict[category]:
-                            dbf[ck] = cPickle.dumps(currdict[category][ck])
+                            dbf[ck] = cPickle.dumps(currdict[category][ck], protocol=cPickle.HIGHEST_PROTOCOL)
                     finally:
                         dbf.close()
             elif 'dirtykeys' in _cfgstore:
@@ -2378,7 +2472,7 @@ class ConfigManager(object):
                                     if ck in dbf:
                                         del dbf[ck]
                                 else:
-                                    dbf[ck] = cPickle.dumps(currdict[category][ck])
+                                    dbf[ck] = cPickle.dumps(currdict[category][ck], protocol=cPickle.HIGHEST_PROTOCOL)
                         finally:
                             dbf.close()
         willrun = False
@@ -2417,7 +2511,9 @@ def _restore_keys(jsond, password, newpassword=None, sync=True):
     else:
         keydata = json.loads(jsond)
     cryptkey = _parse_key(keydata['cryptkey'], password)
-    integritykey = _parse_key(keydata['integritykey'], password)
+    integritykey = None
+    if 'integritykey' in keydata:
+        integritykey = _parse_key(keydata['integritykey'], password)
     conf.init_config()
     cfg = conf.get_config()
     if cfg.has_option('security', 'externalcfgkey'):
@@ -2426,8 +2522,9 @@ def _restore_keys(jsond, password, newpassword=None, sync=True):
             newpassword = keyfile.read()
     set_global('master_privacy_key', _format_key(cryptkey,
                                                  password=newpassword), sync)
-    set_global('master_integrity_key', _format_key(integritykey,
-                                                   password=newpassword), sync)
+    if integritykey:    
+        set_global('master_integrity_key', _format_key(integritykey,
+                                                       password=newpassword), sync)
     _masterkey = cryptkey
     _masterintegritykey = integritykey
     if sync:
@@ -2439,20 +2536,21 @@ def _dump_keys(password, dojson=True):
         init_masterkey()
     cryptkey = _format_key(_masterkey, password=password)
     if 'passphraseprotected' in cryptkey:
-        cryptkey = '!'.join(map(base64.b64encode,
-                                cryptkey['passphraseprotected']))
+        cryptkey = '!'.join(
+            [confluent.util.stringify(base64.b64encode(x))
+             for x in cryptkey['passphraseprotected']])
     else:
-        cryptkey = '*unencrypted:{0}'.format(base64.b64encode(
-            cryptkey['unencryptedvalue']))
+        cryptkey = '*unencrypted:{0}'.format(confluent.util.stringify(base64.b64encode(
+            cryptkey['unencryptedvalue'])))
     keydata = {'cryptkey': cryptkey}
     if _masterintegritykey is not None:
         integritykey = _format_key(_masterintegritykey, password=password)
         if 'passphraseprotected' in integritykey:
-            integritykey = '!'.join(map(base64.b64encode,
-                                        integritykey['passphraseprotected']))
+            integritykey = '!'.join([confluent.util.stringify(base64.b64encode(x)) for x in
+                                        integritykey['passphraseprotected']])
         else:
-            integritykey = '*unencrypted:{0}'.format(base64.b64encode(
-                integritykey['unencryptedvalue']))
+            integritykey = '*unencrypted:{0}'.format(confluent.util.stringify(base64.b64encode(
+                integritykey['unencryptedvalue'])))
         keydata['integritykey'] = integritykey
     if dojson:
         return json.dumps(keydata, sort_keys=True, indent=4, separators=(',', ': '))

confluent_server/confluent/consoleserver.py

@@ -94,7 +94,7 @@ def _utf8_normalize(data, shiftin, decoder):
 
 
 def pytechars2line(chars, maxlen=None):
-    line = '\x1b[m'  # start at default params
+    line = b'\x1b[m'  # start at default params
     lb = False  # last bold
     li = False  # last italic
     lu = False  # last underline
@@ -106,12 +106,12 @@ def pytechars2line(chars, maxlen=None):
     len = 1
     for charidx in range(maxlen):
         char = chars[charidx]
-        csi = []
+        csi = bytearray([])
         if char.fg != lfg:
-            csi.append(30 + pytecolors2ansi[char.fg])
+            csi.append(30 + pytecolors2ansi.get(char.fg, 9))
             lfg = char.fg
         if char.bg != lbg:
-            csi.append(40 + pytecolors2ansi[char.bg])
+            csi.append(40 + pytecolors2ansi.get(char.bg, 9))
             lbg = char.bg
         if char.bold != lb:
             lb = char.bold
@@ -129,10 +129,13 @@ def pytechars2line(chars, maxlen=None):
             lr = char.reverse
             csi.append(7 if lr else 27)
         if csi:
-            line += b'\x1b[' + b';'.join(['{0}'.format(x) for x in csi]) + b'm'
-        if not hasdata and char.data.encode('utf-8').rstrip():
+            line += b'\x1b[' + b';'.join(['{0}'.format(x).encode('utf-8') for x in csi]) + b'm'
+        if not hasdata and char.data.rstrip():
             hasdata = True
-        line += char.data.encode('utf-8')
+        chardata = char.data
+        if not isinstance(chardata, bytes):
+            chardata = chardata.encode('utf-8')
+        line += chardata
         if maxlen and len >= maxlen:
             break
         len += 1
@@ -185,7 +188,7 @@ class ConsoleHandler(object):
         if termstate & 1:
             self.appmodedetected = True
         if termstate & 2:
-            self.shiftin = '0'
+            self.shiftin = b'0'
         self.users = {}
         self._attribwatcher = None
         self._console = None
@@ -210,6 +213,8 @@ class ConsoleHandler(object):
         return retrytime + (retrytime * random.random())
 
     def feedbuffer(self, data):
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
         try:
             self.termstream.feed(data)
         except StopIteration:  # corrupt parser state, start over
@@ -238,7 +243,7 @@ class ConsoleHandler(object):
     def check_collective(self, attrvalue):
         myc = attrvalue.get(self.node, {}).get('collective.manager', {}).get(
             'value', None)
-        if configmodule.list_collective() and not myc:
+        if list(configmodule.list_collective()) and not myc:
             self._is_local = False
             self._detach()
             self._disconnect()
@@ -357,13 +362,17 @@ class ConsoleHandler(object):
         if self.reconnect:
             self.reconnect.cancel()
             self.reconnect = None
+        strerror = ('The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function')
         try:
             self._console = list(plugin.handle_path(
                 self._plugin_path.format(self.node),
                 "create", self.cfgmgr))[0]
         except (exc.NotImplementedException, exc.NotFoundException):
             self._console = None
-        except:
+        except Exception as e:
+            strerror = str(e)
             if _tracelog:
                 _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                               event=log.Events.stacktrace)
@@ -376,13 +385,9 @@ class ConsoleHandler(object):
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
             self.feedbuffer(
-                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
-                'not configured,\r\nset it to a valid value for console '
-                'function]')
+                '\x1bc\x1b[2J\x1b[1;1H[{0}]'.format(strerror))
             self._send_rcpts(
-                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
-                'not configured,\r\nset it to a valid value for console '
-                'function]')
+                '\x1bc\x1b[2J\x1b[1;1H[{0}]'.format(strerror))
             self.clearerror = True
             return
         if self.clearerror:
@@ -448,6 +453,7 @@ class ConsoleHandler(object):
 
     def _got_disconnected(self):
         if self.connectstate != 'unconnected':
+            self._console.close()
             self.connectstate = 'unconnected'
             self.log(
                 logdata='console disconnected', ltype=log.DataTypes.event,
@@ -525,17 +531,19 @@ class ConsoleHandler(object):
             if data == conapi.ConsoleEvent.Disconnect:
                 self._got_disconnected()
             return
-        elif data == '':
+        elif data in (b'', u''):
             # ignore empty strings from a cconsole provider
             return
-        if '\x1b[?1l' in data:  # request for ansi mode cursor keys
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+        if b'\x1b[?1l' in data:  # request for ansi mode cursor keys
             self.appmodedetected = False
-        if '\x1b[?1h' in data:  # remember the session wants the client to use
+        if b'\x1b[?1h' in data:  # remember the session wants the client to use
             # 'application mode'  Thus far only observed on esxi
             self.appmodedetected = True
-        if '\x1b)0' in data:
+        if b'\x1b)0' in data:
             # console indicates it wants access to special drawing characters
-            self.shiftin = '0'
+            self.shiftin = b'0'
         eventdata = 0
         if self.appmodedetected:
             eventdata |= 1
@@ -588,25 +596,30 @@ class ConsoleHandler(object):
                 if pendingbl:
                     retdata += pendingbl
                     pendingbl = b''
-                retdata += nline + '\r\n'
+                retdata += nline + b'\r\n'
             else:
-                pendingbl += nline + '\r\n'
+                pendingbl += nline + b'\r\n'
         if len(retdata) >  6:
             retdata = retdata[:-2]  # remove the last \r\n
-        retdata += b'\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
-                                           self.buffer.cursor.x + 1)
+        cursordata = '\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
+                                            self.buffer.cursor.x + 1)
+        if not isinstance(cursordata, bytes):
+            cursordata = cursordata.encode('utf-8')
+        retdata += cursordata
         if self.shiftin is not None:  # detected that terminal requested a
             # shiftin character set, relay that to the terminal that cannected
-            retdata += '\x1b)' + self.shiftin
+            retdata += b'\x1b)' + self.shiftin
         if self.appmodedetected:
-            retdata += '\x1b[?1h'
+            retdata += b'\x1b[?1h'
         else:
-            retdata += '\x1b[?1l'
+            retdata += b'\x1b[?1l'
         return retdata, connstate
 
     def write(self, data):
         if self.connectstate == 'connected':
             try:
+                if isinstance(data, str) and not isinstance(data, bytes):
+                    data = data.encode('utf-8')
                 self._console.write(data)
             except Exception:
                 _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,

confluent_server/confluent/core.py

@@ -60,18 +60,19 @@ import eventlet.greenpool as greenpool
 import eventlet.green.ssl as ssl
 import eventlet.queue as queue
 import itertools
+import msgpack
 import os
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
 import socket
 import struct
 import sys
 
 pluginmap = {}
-dispatch_plugins = (b'ipmi', u'ipmi')
+dispatch_plugins = (b'ipmi', u'ipmi', b'redfish', u'redfish', b'tsmsol', u'tsmsol')
 
+try:
+    unicode
+except NameError:
+    unicode = str
 
 def seek_element(currplace, currkey):
     try:
@@ -86,7 +87,10 @@ def seek_element(currplace, currkey):
 
 def nested_lookup(nestdict, key):
     try:
-        return reduce(seek_element, key, nestdict)
+        currloc = nestdict
+        for currk in key:
+            currloc = seek_element(currloc, currk)
+        return currloc
     except TypeError:
         raise exc.NotFoundException("Invalid element requested")
 
@@ -106,6 +110,8 @@ def load_plugins():
         for plugin in os.listdir(plugindir):
             if plugin.startswith('.'):
                 continue
+            if '__pycache__' in plugin:
+                continue
             (plugin, plugtype) = os.path.splitext(plugin)
             if plugtype == '.sh':
                 pluginmap[plugin] = shellmodule.Plugin(
@@ -163,6 +169,10 @@ def _init_core():
                         'default': 'ipmi',
                     }),
                 },
+                'clear': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                }),
                 'users': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -205,6 +215,20 @@ def _init_core():
                         'default': 'ipmi',
                     }),
                 },
+                'extended': {
+                    'all': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'extra': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'advanced': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
             },
             'storage': {
                 'all': PluginRoute({
@@ -384,6 +408,7 @@ def _init_core():
 
     nodegroupresources = {
         'attributes': {
+            'check': PluginRoute({'handler': 'attributes'}),
             'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
@@ -395,18 +420,22 @@ def create_user(inputdata, configmanager):
     try:
         username = inputdata['name']
         del inputdata['name']
+        role = inputdata['role']
+        del inputdata['role']
     except (KeyError, ValueError):
-        raise exc.InvalidArgumentException()
-    configmanager.create_user(username, attributemap=inputdata)
+        raise exc.InvalidArgumentException('Missing user name or role')
+    configmanager.create_user(username, role, attributemap=inputdata)
 
 
 def create_usergroup(inputdata, configmanager):
     try:
         groupname = inputdata['name']
+        role = inputdata['role']
         del inputdata['name']
+        del inputdata['role']
     except (KeyError, ValueError):
-        raise exc.InvalidArgumentException()
-    configmanager.create_usergroup(groupname)
+        raise exc.InvalidArgumentException("Missing user name or role")
+    configmanager.create_usergroup(groupname, role)
 
 
 def update_usergroup(groupname, attribmap, configmanager):
@@ -430,7 +459,7 @@ def show_usergroup(groupname, configmanager):
 def show_user(name, configmanager):
     userobj = configmanager.get_user(name)
     rv = {}
-    for attr in attrscheme.user.iterkeys():
+    for attr in attrscheme.user:
         rv[attr] = None
         if attr == 'password':
             if 'cryptpass' in userobj:
@@ -667,13 +696,22 @@ def handle_dispatch(connection, cert, dispatch, peername):
             cfm.get_collective_member(peername)['fingerprint'], cert):
         connection.close()
         return
-    dispatch = pickle.loads(dispatch)
+    if dispatch[0:2] != b'\x01\x03':  # magic value to indicate msgpack
+        # We only support msgpack now
+        # The magic should preclude any pickle, as the first byte can never be
+        # under 0x20 or so.
+        connection.close()
+        return
+    dispatch = msgpack.unpackb(dispatch[2:], raw=False)
     configmanager = cfm.ConfigManager(dispatch['tenant'])
     nodes = dispatch['nodes']
     inputdata = dispatch['inputdata']
     operation = dispatch['operation']
     pathcomponents = dispatch['path']
     routespec = nested_lookup(noderesources, pathcomponents)
+    inputdata = msg.get_input_message(
+        pathcomponents, operation, inputdata, nodes, dispatch['isnoderange'],
+        configmanager)
     plugroute = routespec.routeinfo
     plugpath = None
     nodesbyhandler = {}
@@ -711,10 +749,18 @@ def handle_dispatch(connection, cert, dispatch, peername):
 
 def _forward_rsp(connection, res):
     try:
-        r = pickle.dumps(res)
-    except TypeError:
-        r = pickle.dumps(Exception(
-            'Cannot serialize error, check collective.manager error logs for details' + str(res)))
+       r = res.serialize()
+    except AttributeError:
+        if isinstance(res, Exception):
+            r = msgpack.packb(['Exception', str(res)], use_bin_type=False)
+        else:
+            r = msgpack.packb(
+                ['Exception', 'Unable to serialize response ' + repr(res)],
+                use_bin_type=False)
+    except Exception as e:
+        r = msgpack.packb(
+                ['Exception', 'Unable to serialize response ' + repr(res) + ' due to ' + str(e)],
+                use_bin_type=False)
     rlen = len(r)
     if not rlen:
         return
@@ -805,7 +851,7 @@ def handle_node_request(configmanager, inputdata, operation,
     del pathcomponents[0:2]
     passvalues = queue.Queue()
     plugroute = routespec.routeinfo
-    inputdata = msg.get_input_message(
+    msginputdata = msg.get_input_message(
         pathcomponents, operation, inputdata, nodes, isnoderange,
         configmanager)
     if 'handler' in plugroute:  # fixed handler definition, easy enough
@@ -816,7 +862,7 @@ def handle_node_request(configmanager, inputdata, operation,
         passvalue = hfunc(
             nodes=nodes, element=pathcomponents,
             configmanager=configmanager,
-            inputdata=inputdata)
+            inputdata=msginputdata)
         if isnoderange:
             return passvalue
         elif isinstance(passvalue, console.Console):
@@ -869,13 +915,13 @@ def handle_node_request(configmanager, inputdata, operation,
             workers.spawn(addtoqueue, passvalues, hfunc, {'nodes': nodesbyhandler[hfunc],
                                            'element': pathcomponents,
                 'configmanager': configmanager,
-                'inputdata': inputdata})
+                'inputdata': msginputdata})
         for manager in nodesbymanager:
             numworkers += 1
             workers.spawn(addtoqueue, passvalues, dispatch_request, {
                 'nodes': nodesbymanager[manager], 'manager': manager,
                 'element': pathcomponents, 'configmanager': configmanager,
-                'inputdata': inputdata, 'operation': operation})
+                'inputdata': inputdata, 'operation': operation, 'isnoderange': isnoderange})
         if isnoderange or not autostrip:
             return iterate_queue(numworkers, passvalues)
         else:
@@ -919,7 +965,7 @@ def addtoqueue(theq, fun, kwargs):
 
 
 def dispatch_request(nodes, manager, element, configmanager, inputdata,
-                     operation):
+                     operation, isnoderange):
     a = configmanager.get_collective_member(manager)
     try:
         remote = socket.create_connection((a['address'], 13001))
@@ -943,12 +989,20 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
     if not util.cert_matches(a['fingerprint'], remote.getpeercert(
             binary_form=True)):
         raise Exception("Invalid certificate on peer")
-    tlvdata.recv(remote)
+    banner = tlvdata.recv(remote)
+    vers = banner.split()[2]
+    if vers == b'v0':
+        pvers = 2
+    elif vers == b'v1':
+        pvers = 4
+    if sys.version_info[0] < 3:
+        pvers = 2
     tlvdata.recv(remote)
     myname = collective.get_myname()
-    dreq = pickle.dumps({'name': myname, 'nodes': list(nodes),
-                         'path': element,'tenant': configmanager.tenant,
-                         'operation': operation, 'inputdata': inputdata})
+    dreq =  b'\x01\x03' + msgpack.packb(
+        {'name': myname, 'nodes': list(nodes),
+        'path': element,'tenant': configmanager.tenant,
+        'operation': operation, 'inputdata': inputdata, 'isnoderange': isnoderange}, use_bin_type=False)
     tlvdata.send(remote, {'dispatch': {'name': myname, 'length': len(dreq)}})
     remote.sendall(dreq)
     while True:
@@ -995,9 +1049,14 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
                             a['name']))
                 return
             rsp += nrsp
-        rsp = pickle.loads(rsp)
+        try:
+            rsp = msg.msg_deserialize(rsp)
+        except Exception:
+            rsp = exc.deserialize_exc(rsp)
         if isinstance(rsp, Exception):
             raise rsp
+        if not rsp:
+            raise Exception('Error in cross-collective serialize/deserialze, see remote logs')
         yield rsp
 
 

confluent_server/confluent/discovery/core.py

@@ -65,9 +65,11 @@ import base64
 import confluent.config.configmanager as cfm
 import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
-#import confluent.discovery.protocols.ssdp as ssdp
+import confluent.discovery.protocols.ssdp as ssdp
 import confluent.discovery.protocols.slp as slp
 import confluent.discovery.handlers.imm as imm
+import confluent.discovery.handlers.cpstorage as cpstorage
+import confluent.discovery.handlers.tsm as tsm
 import confluent.discovery.handlers.pxe as pxeh
 import confluent.discovery.handlers.smm as smm
 import confluent.discovery.handlers.xcc as xcc
@@ -89,6 +91,11 @@ import eventlet.semaphore
 autosensors = set()
 scanner = None
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 class nesteddict(dict):
 
     def __missing__(self, key):
@@ -100,23 +107,35 @@ nodehandlers = {
     'service:management-hardware.Lenovo:lenovo-xclarity-controller': xcc,
     'service:management-hardware.IBM:integrated-management-module2': imm,
     'pxe-client': pxeh,
+    'onie-switch': None,
+    'cumulus-switch': None,
     'service:io-device.Lenovo:management-module': None,
+    'service:thinkagile-storage': cpstorage,
+    'service:lenovo-tsm': tsm,
 }
 
 servicenames = {
     'pxe-client': 'pxe-client',
+    'onie-switch': 'onie-switch',
+    'cumulus-switch': 'cumulus-switch',
     'service:lenovo-smm': 'lenovo-smm',
     'service:management-hardware.Lenovo:lenovo-xclarity-controller': 'lenovo-xcc',
     'service:management-hardware.IBM:integrated-management-module2': 'lenovo-imm2',
     'service:io-device.Lenovo:management-module': 'lenovo-switch',
+    'service:thinkagile-storage': 'thinkagile-storagebmc',
+    'service:lenovo-tsm': 'lenovo-tsm',
 }
 
 servicebyname = {
     'pxe-client': 'pxe-client',
+    'onie-switch': 'onie-switch',
+    'cumulus-switch': 'cumulus-switch',
     'lenovo-smm': 'service:lenovo-smm',
     'lenovo-xcc': 'service:management-hardware.Lenovo:lenovo-xclarity-controller',
     'lenovo-imm2': 'service:management-hardware.IBM:integrated-management-module2',
     'lenovo-switch': 'service:io-device.Lenovo:management-module',
+    'thinkagile-storage': 'service:thinkagile-storagebmc',
+    'lenovo-tsm': 'service:lenovo-tsm',
 }
 
 discopool = eventlet.greenpool.GreenPool(500)
@@ -363,7 +382,7 @@ def handle_autosense_config(operation, inputdata):
         yield msg.KeyValueData({'enabled': autosense})
     elif operation == 'update':
         enabled = inputdata['enabled']
-        if type(enabled) in (unicode, str):
+        if type(enabled) in (unicode, bytes):
             enabled = enabled.lower() in ('true', '1', 'y', 'yes', 'enable',
                                           'enabled')
         if autosense == enabled:
@@ -697,7 +716,8 @@ def detected(info):
 
 def b64tohex(b64str):
     bd = base64.b64decode(b64str)
-    return ''.join(['{0:02x}'.format(ord(x)) for x in bd])
+    bd = bytearray(bd)
+    return ''.join(['{0:02x}'.format(x) for x in bd])
 
 
 def get_enclosure_chain_head(nodename, cfg):
@@ -881,7 +901,7 @@ def eval_node(cfg, handler, info, nodename, manual=False):
         handler.probe()  # unicast interrogation as possible to get more data
         # switch concurrently
         # do some preconfig, for example, to bring a SMM online if applicable
-        handler.preconfig()
+        handler.preconfig(nodename)
     except Exception as e:
         unknown_info[info['hwaddr']] = info
         info['discostatus'] = 'unidentified'
@@ -935,9 +955,9 @@ def eval_node(cfg, handler, info, nodename, manual=False):
                     #     raise exc.InvalidArgumentException(errorstr)
                     # log.log({'error': errorstr})
                     if encuuid in pending_by_uuid:
-                        pending_by_uuid[encuuid].add(info)
+                        pending_by_uuid[encuuid].append(info)
                     else:
-                        pending_by_uuid[encuuid] = set([info])
+                        pending_by_uuid[encuuid] = [info]
                     return
                 # We found the real smm, replace the list with the actual smm
                 # to continue
@@ -1056,7 +1076,7 @@ def discover_node(cfg, handler, info, nodename, manual):
                 traceback.print_exc()
                 return False
             newnodeattribs = {}
-            if cfm.list_collective():
+            if list(cfm.list_collective()):
                 # We are in a collective, check collective.manager
                 cmc = cfg.get_node_attributes(nodename, 'collective.manager')
                 cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
@@ -1080,6 +1100,10 @@ def discover_node(cfg, handler, info, nodename, manual):
         info['discostatus'] = 'discovered'
         for i in pending_by_uuid.get(curruuid, []):
             eventlet.spawn_n(_recheck_single_unknown_info, cfg, i)
+        try:
+            del pending_by_uuid[curruuid]
+        except KeyError:
+            pass
         return True
     log.log({'info': 'Detected {0}, but discovery.policy is not set to a '
                      'value allowing discovery (open or permissive)'.format(
@@ -1193,12 +1217,21 @@ def rescan():
     if scanner:
         return
     else:
-        scanner = eventlet.spawn(slp.active_scan, safe_detected, slp)
+        scanner = eventlet.spawn(blocking_scan)
 
 
+def blocking_scan():
+    global scanner
+    slpscan = eventlet.spawn(slp.active_scan, safe_detected, slp)
+    ssdpscan = eventlet.spawn(ssdp.active_scan, safe_detected, ssdp)
+    slpscan.wait()
+    ssdpscan.wait()
+    scanner = None
+
 def start_detection():
     global attribwatcher
     global rechecker
+    global rechecktime
     _map_unique_ids()
     cfg = cfm.ConfigManager(None)
     allnodes = cfg.list_nodes()

confluent_server/confluent/discovery/handlers/bmc.py

@@ -26,11 +26,11 @@ ipmicommand.session.threading = eventlet.green.threading
 ipmicommand.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
-DEFAULT_USER = 'USERID'
-DEFAULT_PASS = 'PASSW0RD'
 
 
 class NodeHandler(generic.NodeHandler):
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'
 
     def _get_ipmicmd(self, user=None, password=None):
         priv = None
@@ -39,9 +39,9 @@ class NodeHandler(generic.NodeHandler):
                 raise pygexc.IpmiException()
             priv = 4  # manually indicate priv to avoid double-attempt
         if user is None:
-            user = DEFAULT_USER
+            user = self.DEFAULT_USER
         if password is None:
-            password = DEFAULT_PASS
+            password = self.DEFAULT_PASS
         return ipmicommand.Command(self.ipaddr, user, password,
                                    privlevel=priv, keepalive=False)
 
@@ -70,17 +70,17 @@ class NodeHandler(generic.NodeHandler):
             'secret.hardwaremanagementpassword', {}).get('value', None)
         try:
             ic = self._get_ipmicmd()
-            passwd = DEFAULT_PASS
+            passwd = self.DEFAULT_PASS
         except pygexc.IpmiException as pi:
             havecustomcreds = False
-            if user is not None and user != DEFAULT_USER:
+            if user is not None and user != self.DEFAULT_USER:
                 havecustomcreds = True
             else:
-                user = DEFAULT_USER
-            if passwd is not None and passwd != DEFAULT_PASS:
+                user = self.DEFAULT_USER
+            if passwd is not None and passwd != self.DEFAULT_PASS:
                 havecustomcreds = True
             else:
-                passwd = DEFAULT_PASS
+                passwd = self.DEFAULT_PASS
             if havecustomcreds:
                 ic = self._get_ipmicmd(user, passwd)
             else:

confluent_server/confluent/discovery/handlers/cpstorage.py

@@ -0,0 +1,70 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.bmc as bmchandler
+import eventlet
+import confluent.util as util
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+class NodeHandler(bmchandler.NodeHandler):
+    DEFAULT_USER = 'admin'
+    DEFAULT_PASS = 'admin'
+    devname = 'BMC'
+    maxmacs = 2
+
+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def get_webclient(self, user, passwd, newuser, newpass):
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443,
+                                            verifycallback=self.validate_cert)
+        wc.connect()
+        authdata = urlencode({'username': user, 'password': passwd,
+                              'weblogsign': 1})
+        res = wc.grab_json_response_with_status('/api/session', authdata)
+        if res[1] == 200:
+            if res[0].get('force_password', 1) == 0:
+                # Need to handle password change
+                passchange = {
+                    'Password': newpass,
+                    'RetypePassword': newpass,
+                    'param': 4,
+                    'username': 'admin',
+                    'privilege': 4,
+                }
+                passchange = urlencode(passchange)
+                rsp = wc.grab_json_response_with_status('/api/reset-pass',
+                                                        passchange)
+                rsp = wc.grab_json_response_with_status('/api/session',
+                                                        method='DELETE')
+
+    def config(self, nodename, reset=False):
+        self.nodename = nodename
+        creds = self.configmanager.get_node_attributes(
+            self.nodename, ['secret.hardwaremanagementuser',
+                            'secret.hardwaremanagementpassword'],
+            decrypt=True)
+        user, passwd, isdefault = self.get_node_credentials(
+                nodename, creds, 'admin', 'admin')
+        if not isdefault:
+            self.get_webclient(self.DEFAULT_USER, self.DEFAULT_PASS, user,
+                               passwd)
+        self._bmcconfig(nodename, False)

confluent_server/confluent/discovery/handlers/generic.py

@@ -57,7 +57,7 @@ class NodeHandler(object):
         # serial number and uuid to flesh out data as needed
         return
 
-    def preconfig(self):
+    def preconfig(self, possiblenode):
         return
 
     def discoverable_by_switch(self, macs):
@@ -68,17 +68,21 @@ class NodeHandler(object):
     def _savecert(self, certificate):
         self._fp = certificate
         return True
-    
+
     def get_node_credentials(self, nodename, creds, defuser, defpass):
         user = creds.get(nodename, {}).get(
             'secret.hardwaremanagementuser', {}).get('value', None)
         havecustomcreds = False
+        if user and not isinstance(user, str):
+            user = user.decode('utf8')
         if user is not None and user != defuser:
             havecustomcreds = True
         else:
             user = defuser
         passwd = creds.get(nodename, {}).get(
             'secret.hardwaremanagementpassword', {}).get('value', None)
+        if passwd and not isinstance(passwd, str):
+            passwd = passwd.decode('utf8')
         if passwd is not None and passwd != defpass:
             havecustomcreds = True
         else:

confluent_server/confluent/discovery/handlers/imm.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
 import pyghmi.exceptions as pygexc
 import pyghmi.ipmi.private.util as pygutil
+import confluent.util as util
 import struct
 
 class NodeHandler(bmchandler.NodeHandler):
@@ -37,8 +39,9 @@ class NodeHandler(bmchandler.NodeHandler):
         if wronguuid:
             # we need to fix the first three portions of the uuid
             uuidprefix = wronguuid.split('-')[:3]
-            uuidprefix = struct.pack(
-                '<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+            uuidprefix = codecs.encode(struct.pack(
+                '<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+            uuidprefix = util.stringify(uuidprefix)
             uuidprefix = uuidprefix[:8] + '-' + uuidprefix[8:12] + '-' + \
                          uuidprefix[12:16]
             self.info['uuid'] = uuidprefix + '-' + '-'.join(

confluent_server/confluent/discovery/handlers/smm.py

@@ -12,13 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import confluent.discovery.handlers.bmc as bmchandler
 import confluent.exceptions as exc
-import pyghmi.util.webclient as webclient
+import eventlet
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 import struct
-import urllib
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
 import eventlet.support.greendns
 import confluent.netutil as netutil
+import confluent.util as util
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 from xml.etree.ElementTree import fromstring
@@ -26,8 +32,9 @@ from xml.etree.ElementTree import fromstring
 def fixuuid(baduuid):
     # SMM dumps it out in hex
     uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
-    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode(
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]),
         'hex')
+    a = util.stringify(a)
     uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
     return '-'.join(uuid).lower()
 
@@ -83,6 +90,13 @@ class NodeHandler(bmchandler.NodeHandler):
             smmip = smmip[-1][0]
             if smmip and ':' in smmip:
                 raise exc.NotImplementedException('IPv6 not supported')
+            wc.request('POST', '/data', 'get=hostname')
+            rsp = wc.getresponse()
+            rspdata = fromstring(util.stringify(rsp.read()))
+            currip = rspdata.find('netConfig').find('ifConfigEntries').find(
+                'ifConfig').find('v4IPAddr').text
+            if currip == smmip:
+                return
             netconfig = netutil.get_nic_config(cfg, nodename, ip=smmip)
             netmask = netutil.cidr_to_mask(netconfig['prefix'])
             setdata = 'set=ifIndex:0,v4DHCPEnabled:0,v4IPAddr:{0},v4NetMask:{1}'.format(smmip, netmask)
@@ -91,7 +105,7 @@ class NodeHandler(bmchandler.NodeHandler):
                 setdata += ',v4Gateway:{0}'.format(gateway)
             wc.request('POST', '/data', setdata)
             rsp = wc.getresponse()
-            rspdata = rsp.read()
+            rspdata = util.stringify(rsp.read())
             if '<statusCode>0' not in rspdata:
                 raise Exception("Error configuring SMM Network")
             return
@@ -109,16 +123,16 @@ class NodeHandler(bmchandler.NodeHandler):
             'password': 'PASSW0RD',
         }
         headers = {'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded'}
-        wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+        wc.request('POST', '/data/login', urlencode(authdata), headers)
         rsp = wc.getresponse()
-        rspdata = rsp.read()
+        rspdata = util.stringify(rsp.read())
         if 'authResult>0' not in rspdata:
             # default credentials are refused, try with the actual
             authdata['user'] = username
             authdata['password'] = password
-            wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
             rsp = wc.getresponse()
-            rspdata = rsp.read()
+            rspdata = util.stringify(rsp.read())
             if 'renew_account' in rspdata:
                 raise Exception('Configured password has expired')
             if 'authResult>0' not in rspdata:
@@ -132,13 +146,13 @@ class NodeHandler(bmchandler.NodeHandler):
             tokens = fromstring(rspdata)
             st2 = tokens.findall('st2')[0].text
             wc.set_header('ST2', st2)
-            wc.request('POST', '/data/changepwd', urllib.urlencode(passwdchange))
+            wc.request('POST', '/data/changepwd', urlencode(passwdchange))
             rsp = wc.getresponse()
             rspdata = rsp.read()
             authdata['password'] = password
-            wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+            wc.request('POST', '/data/login', urlencode(authdata), headers)
             rsp = wc.getresponse()
-            rspdata = rsp.read()
+            rspdata = util.stringify(rsp.read())
         if 'authResult>0' in rspdata:
             tokens = fromstring(rspdata)
             st2 = tokens.findall('st2')[0].text
@@ -152,7 +166,7 @@ class NodeHandler(bmchandler.NodeHandler):
             rsp = wc.getresponse()
             rspdata = rsp.read()
             authdata['user'] = username
-            wc.request('POST', '/data/login', urllib.urlencode(authdata, headers))
+            wc.request('POST', '/data/login', urlencode(authdata, headers))
             rsp = wc.getresponse()
             rspdata = rsp.read()
             tokens = fromstring(rspdata)
@@ -174,6 +188,10 @@ class NodeHandler(bmchandler.NodeHandler):
             'secret.hardwaremanagementuser', {}).get('value', 'USERID')
         passwd = creds.get(nodename, {}).get(
             'secret.hardwaremanagementpassword', {}).get('value', 'PASSW0RD')
+        if not isinstance(username, str):
+            username = username.decode('utf8')
+        if not isinstance(passwd, str):
+            passwd = passwd.decode('utf8')
         if passwd == 'PASSW0RD' and self.ruleset:
             raise Exception('Cannot support default password and setting password rules at same time')
         if passwd == 'PASSW0RD':

confluent_server/confluent/discovery/handlers/tsm.py

@@ -0,0 +1,221 @@
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.generic as generic
+import confluent.exceptions as exc
+import confluent.netutil as netutil
+import confluent.util as util
+import eventlet.support.greendns
+import json
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
+
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+class NodeHandler(generic.NodeHandler):
+    devname = 'TSM'
+    DEFAULT_USER = 'USERID'
+    DEFAULT_PASS = 'PASSW0RD'
+
+    def __init__(self, info, configmanager):
+        self.trieddefault = None
+        self.targuser = None
+        self.curruser = None
+        self.currpass = None
+        self.targpass = None
+        self.nodename = None
+        self.csrftok = None
+        self.channel = None
+        self.atdefault = True
+        super(NodeHandler, self).__init__(info, configmanager)
+
+    def scan(self):
+        c = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert)
+        i = c.grab_json_response('/redfish/v1/')
+        uuid = i.get('UUID', None)
+        if uuid:
+            self.info['uuid'] = uuid
+
+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def _get_wc(self):
+        authdata = {  # start by trying factory defaults
+            'username': self.DEFAULT_USER,
+            'password': self.DEFAULT_PASS,
+        }
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self.validate_cert)
+        wc.set_header('Content-Type', 'application/json')
+        authmode = 0
+        if not self.trieddefault:
+            rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+            if status == 403:
+                wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+                authmode = 1
+                rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            else:
+                authmode = 2
+            if status > 400:
+                rsp = util.stringify(rsp)
+                self.trieddefault = True
+                if '555' in rsp:
+                    passchange = {
+                        'Password': self.targpass, 
+                        'RetypePassword': self.targpass,
+                        'param': 4,
+                        'default_password': self.DEFAULT_PASS,
+                        'username': self.DEFAULT_USER
+                        }
+                    if authmode == 2:
+                        rsp, status = wc.grab_json_response_with_status('/api/reset-pass', passchange)
+                    else:
+                        rsp, status = wc.grab_json_response_with_status('/api/reset-pass', urlencode(passchange))
+                    authdata['password'] = self.targpass
+                    if authmode == 2:
+                        rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+                    else:
+                        rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+                    self.csrftok = rsp['CSRFToken']
+                    self.channel = rsp['channel']
+                    self.curruser = self.DEFAULT_USER
+                    self.currpass = self.targpass
+                    return wc
+            else:
+                self.curruser = self.DEFAULT_USER
+                self.currpass = self.DEFAULT_PASS
+                self.csrftok = rsp['CSRFToken']
+                self.channel = rsp['channel']
+                return wc
+        if self.curruser:
+            authdata['username'] = self.curruser
+            authdata['password'] = self.currpass
+            if authmode != 1:
+                rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+            if authmode == 1 or status == 403:
+                wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+                rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+            if status != 200:
+                return None
+            self.csrftok = rsp['CSRFToken']
+            self.channel = rsp['channel']
+            return wc
+        authdata['username'] = self.targuser
+        authdata['password'] = self.targpass
+        if authmode != 1:
+            rsp, status = wc.grab_json_response_with_status('/api/session', authdata)
+        if authmode == 1 or status == 403:
+            wc.set_header('Content-Type', 'application/x-www-form-urlencoded')
+            rsp, status = wc.grab_json_response_with_status('/api/session', urlencode(authdata))
+        if status != 200:
+            return None
+        self.curruser = self.targuser
+        self.currpass = self.targpass
+        self.csrftok = rsp['CSRFToken']
+        self.channel = rsp['channel']
+        return wc
+
+    def config(self, nodename):
+        self.nodename = nodename
+        creds = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
+        cd = creds.get(nodename, {})
+        user, passwd, _ = self.get_node_credentials(
+                nodename, creds, self.DEFAULT_USER, self.DEFAULT_PASS)
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
+        self.targuser = user
+        self.targpass = passwd
+        wc = self._get_wc()
+        wc.set_header('X-CSRFTOKEN', self.csrftok)
+        curruserinfo = {}
+        authupdate = False
+        wc.set_header('Content-Type', 'application/json')
+        if user != self.curruser:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['name'] = user
+        if passwd != self.currpass:
+            authupdate = True
+            if not curruserinfo:
+                curruserinfo = wc.grab_json_response('/api/settings/users')
+                authchg = curruserinfo[1]
+            authchg['changepassword'] = 0
+            authchg['password_size'] = 'bytes_20'
+            authchg['password'] = passwd
+            authchg['confirm_password'] = passwd
+        if authupdate:
+            rsp, status = wc.grab_json_response_with_status('/api/settings/users/2', authchg, method='PUT')
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            # IPMI must be enabled per user config
+            wc.grab_json_response('/api/settings/ipmilanconfig', {
+                'ipv4_enable': 1, 'ipv6_enable': 1,
+                'uncheckedipv4lanEnable': 0, 'uncheckedipv6lanEnable': 0,
+                'checkedipv4lanEnable': 1, 'checkedipv6lanEnable': 1})
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            currnet = wc.grab_json_response('/api/settings/network')
+            for net in currnet:
+                if net['channel_number'] == self.channel and net['lan_enable'] == 0:
+                    # ignore false indication and switch to 8 (dedicated)
+                    self.channel = 8
+                if net['channel_number'] == self.channel:
+                    # we have found the interface to potentially manipulate
+                    if net['ipv4_address'] != newip:
+                        netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+                        newmask = netutil.cidr_to_mask(netconfig['prefix'])
+                        net['ipv4_address'] = newip
+                        net['ipv4_subnet'] = newmask
+                        if netconfig['ipv4_gateway']:
+                            net['ipv4_gateway'] = netconfig['ipv4_gateway']
+                        net['ipv4_dhcp_enable'] = 0
+                        rsp, status = wc.grab_json_response_with_status(
+                            '/api/settings/network/{0}'.format(net['id']), net, method='PUT')
+                    break
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        rsp, status = wc.grab_json_response_with_status('/api/session', method='DELETE')
+
+
+if __name__ == '__main__':
+    import confluent.config.configmanager as cfm
+    c = cfm.ConfigManager(None)
+    import sys
+    info = {'addresses': [[sys.argv[1]]] }
+    print(repr(info))
+    testr = NodeHandler(info, c)
+    testr.config(sys.argv[2])

confluent_server/confluent/discovery/handlers/xcc.py

@@ -13,16 +13,19 @@
 # limitations under the License.
 
 import base64
+import codecs
 import confluent.discovery.handlers.imm as immhandler
+import confluent.exceptions as exc
 import confluent.netutil as netutil
 import confluent.util as util
+import errno
 import eventlet
 import eventlet.support.greendns
 import json
 import os
 import pyghmi.exceptions as pygexc
-xcc = eventlet.import_patched('pyghmi.redfish.oem.lenovo.xcc')
-import pyghmi.util.webclient as webclient
+import eventlet.green.socket as socket
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 import struct
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
@@ -30,7 +33,8 @@ getaddrinfo = eventlet.support.greendns.getaddrinfo
 def fixup_uuid(uuidprop):
     baduuid = ''.join(uuidprop.split())
     uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
-    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+    a = codecs.encode(struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]), 'hex')
+    a = util.stringify(a)
     uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
     return '-'.join(uuid).upper()
 
@@ -41,10 +45,13 @@ class NodeHandler(immhandler.NodeHandler):
     devname = 'XCC'
 
     def __init__(self, info, configmanager):
-        self._xcchdlr = None
         self._wc = None
         self.nodename = None
+        self.tmpnodename = None
+        self.tmppasswd = None
         self._atdefaultcreds = True
+        self._needpasswordchange = True
+        self._currcreds = (None, None)
         super(NodeHandler, self).__init__(info, configmanager)
 
     @classmethod
@@ -53,23 +60,55 @@ class NodeHandler(immhandler.NodeHandler):
         # This is not adequate for being satisfied
         return bool(info.get('attributes', {}))
 
-    def preconfig(self):
+    def preconfig(self, possiblenode):
+        self.tmpnodename = possiblenode
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
+            # skip preconfig for non-SD530 servers
             return
         self.trieddefault = None  # Reset state on a preconfig attempt
         # attempt to enable SMM
         #it's normal to get a 'not supported' (193) for systems without an SMM
+        # need to branch on 3.00+ firmware
+        currfirm = self.info.get('attributes', {}).get('firmware-image-info', [''])[0]
+        currfirm = currfirm.split(':')
+        if len(currfirm) > 1:
+            currfirm = float(currfirm[1])
+        disableipmi = False
+        if currfirm >= 3:
+            # IPMI is disabled and we need it, also we need to go to *some* password
+            wc = self.wc
+            if not wc:
+                # We cannot try to enable SMM here without risking real credentials
+                # on the wire to untrusted parties
+                return
+            wc.grab_json_response('/api/providers/logout')
+            wc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = wc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                disableipmi = True
+                _, _ = wc.grab_json_response_with_status(
+                    '/redfish/v1/Managers/1/NetworkProtocol',
+                    {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
         ipmicmd = None
         try:
-            ipmicmd = self._get_ipmicmd()
+            ipmicmd = self._get_ipmicmd(self._currcreds[0], self._currcreds[1])
             ipmicmd.xraw_command(netfn=0x3a, command=0xf1, data=(1,))
         except pygexc.IpmiException as e:
             if (e.ipmicode != 193 and 'Unauthorized name' not in str(e) and
-                    'Incorrect password' not in str(e)):
+                    'Incorrect password' not in str(e) and 
+                    str(e) != 'Session no longer connected'):
                 # raise an issue if anything other than to be expected
+                if disableipmi:
+                    _, _ = wc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
                 raise
             self.trieddefault = True
+        if disableipmi:
+            _, _ = wc.grab_json_response_with_status(
+                '/redfish/v1/Managers/1/NetworkProtocol',
+                {'IPMI': {'ProtocolEnabled': False}}, method='PATCH')
         #TODO: decide how to clean out if important
         #as it stands, this can step on itself
         #if ipmicmd:
@@ -81,35 +120,123 @@ class NodeHandler(immhandler.NodeHandler):
         fprint = util.get_fingerprint(self.https_cert)
         return util.cert_matches(fprint, certificate)
 
+    def get_webclient(self, username, password, newpassword):
+        wc = self._wc.dupe()
+        try:
+            wc.connect()
+        except socket.error as se:
+            if se.errno != errno.ECONNREFUSED:
+                raise
+            return (None, None)
+        pwdchanged = False
+        adata = json.dumps({'username': util.stringify(username),
+                            'password': util.stringify(password)
+                            })
+        headers = {'Connection': 'keep-alive',
+                   'Content-Type': 'application/json'}
+        wc.request('POST', '/api/login', adata, headers)
+        rsp = wc.getresponse()
+        if rsp.status != 200 and password == 'PASSW0RD':
+            rsp.read()
+            adata = json.dumps({
+                'username': username,
+                'password': newpassword,
+                })
+            headers = {'Connection': 'keep-alive',
+                       'Content-Type': 'application/json'}
+            wc.request('POST', '/api/login', adata, headers)
+            rsp = wc.getresponse()
+            if rsp.status == 200:
+                pwdchanged = True
+                password = newpassword
+            else:
+                rsp.read()
+                return (None, None)
+        if rsp.status == 200:
+            self._currcreds = (username, password)
+            wc.set_basic_credentials(username, password)
+            rspdata = json.loads(rsp.read())
+            wc.set_header('Content-Type', 'application/json')
+            wc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            if rspdata.get('pwchg_required', None) == 'true':
+                wc.request('POST', '/api/function', json.dumps(
+                    {'USER_UserPassChange': '1,{0}'.format(newpassword)}))
+                rsp = wc.getresponse()
+                rsp.read()
+                if rsp.status != 200:
+                    return (None, None)
+                self._currcreds = (username, newpassword)
+                wc.set_basic_credentials(username, newpassword)
+                pwdchanged = True
+            if '_csrf_token' in wc.cookies:
+                wc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+            if pwdchanged:
+                # Remove the minimum change interval, to allow sane 
+                # password changes after provisional changes
+                wc = self.wc
+                self.set_password_policy('', wc)
+            return (wc, pwdchanged)
+        return (None, None)
+
     @property
     def wc(self):
+        passwd = None
+        isdefault = True
         if self._wc is None:
             self._wc = webclient.SecureHTTPConnection(
                 self.ipaddr, 443, verifycallback=self.validate_cert)
             self._wc.connect()
-            self._xcchdlr = xcc.OEMHandler(None, None, self._wc, False)
-        if not self.trieddefault:
-            self._xcchdlr.set_credentials('USERID', 'PASSW0RD')
-            wc = self._xcchdlr.get_webclient()
+        nodename = None
+        if self.nodename:
+            nodename = self.nodename
+            inpreconfig = False
+        elif self.tmpnodename:
+            nodename = None
+            inpreconfig = True
+        if self._currcreds[0] is not None:
+            wc, pwdchanged = self.get_webclient(self._currcreds[0], self._currcreds[1], None)
             if wc:
                 return wc
+        if nodename:
+            creds = self.configmanager.get_node_attributes(
+                nodename, ['secret.hardwaremanagementuser',
+                'secret.hardwaremanagementpassword'], decrypt=True)
+            user, passwd, isdefault = self.get_node_credentials(
+                nodename, creds, 'USERID', 'PASSW0RD')
+        if not self.trieddefault:
+            if not passwd:
+                # So in preconfig context, we don't have admin permission to
+                # actually divulge anything to the target
+                # however the target *will* demand a new password... if it's currently
+                # PASSW0RD
+                # use TempW0rd42 to avoid divulging a real password on the line
+                # This is replacing one well known password (PASSW0RD) with another
+                # (TempW0rd42)
+                passwd = 'TempW0rd42'
+            wc, pwdchanged = self.get_webclient('USERID', 'PASSW0RD', passwd)
+            if wc:
+                if pwdchanged:
+                    if inpreconfig:
+                        self.tmppasswd = passwd
+                    else:
+                        self._needpasswordchange = False
+                return wc
         self.trieddefault = True
-        creds = self.configmanager.get_node_attributes(
-            self.nodename, ['secret.hardwaremanagementuser',
-            'secret.hardwaremanagementpassword'], decrypt=True)
-        user, passwd, isdefault = self.get_node_credentials(
-            self.nodename, creds, 'USERID', 'PASSW0RD')
         if isdefault:
             return
         self._atdefaultcreds = False
-        self._xcchdlr.set_credentials(user, passwd)
-        wc = self._xcchdlr.get_webclient()
+        if self.tmppasswd:
+            wc, _ = self.get_webclient('USERID', self.tmppasswd, passwd)
+        else:
+            wc, _ = self.get_webclient(user, passwd, None)
         if wc:
             return wc
 
-    def set_password_policy(self):
+    def set_password_policy(self, strruleset, wc):
         ruleset = {'USER_GlobalMinPassChgInt': '0'}
-        for rule in self.ruleset.split(','):
+        for rule in strruleset.split(','):
             if '=' not in rule:
                 continue
             name, value = rule.split('=')
@@ -128,7 +255,7 @@ class NodeHandler(immhandler.NodeHandler):
             if name.lower() == 'reuse':
                 ruleset['USER_GlobalMinPassReuseCycle'] = value
         try:
-            self.wc.grab_json_response('/api/dataset', ruleset)
+            wc.grab_json_response('/api/dataset', ruleset)
         except Exception as e:
             print(repr(e))
             pass
@@ -155,18 +282,23 @@ class NodeHandler(immhandler.NodeHandler):
         if not uid:
             raise Exception("XCC has neither the default user nor configured user")
         # The following will work if the password is force change or normal..
-        wc.grab_json_response('/api/function',
-                             {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
+        if self._needpasswordchange and self.tmppasswd != passwd:
+            wc.grab_json_response('/api/function',
+                                {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
         if username != 'USERID':
             wc.grab_json_response(
                 '/api/function',
                 {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,'.format(uid, username)})
+            self.tmppasswd = None
+        self._currcreds = (username, passwd)
 
     def _convert_sha256account(self, user, passwd, wc):
         # First check if the specified user is sha256...
         userinfo = wc.grab_json_response('/api/dataset/imm_users')
         curruser = None
         uid = None
+        user = util.stringify(user)
+        passwd = util.stringify(passwd)
         for userent in userinfo['items'][0]['users']:
             if userent['users_user_name'] == user:
                 curruser = userent
@@ -182,6 +314,7 @@ class NodeHandler(immhandler.NodeHandler):
                 tpass = base64.b64encode(os.urandom(9)) + 'Iw47$'
                 userparams = "{0},6pmu0ezczzcp,{1},1,4,0,0,0,0,,8,".format(tmpuid, tpass)
                 result = wc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                wc.grab_json_response('/api/providers/logout')
                 adata = json.dumps({
                     'username': '6pmu0ezczzcp',
                     'password': tpass,
@@ -201,11 +334,20 @@ class NodeHandler(immhandler.NodeHandler):
                             '/api/function',
                             {'USER_UserPassChange': '{0},{1}'.format(tmpuid, newpass)})
                     nwc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(curruser['users_user_id'], user)})
-                    userparams = "{0},{1},{2},1,4,0,0,0,0,,8,".format(curruser['users_user_id'], user, passwd)
+                    userparams = "{0},{1},{2},1,4,0,0,0,0,,8,".format(curruser['users_user_id'], user, tpass)
                     nwc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                    nwc.grab_json_response('/api/providers/logout')
+                    nwc, pwdchanged = self.get_webclient(user, tpass, passwd)
+                    if not pwdchanged:
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(curruser['users_user_id'], passwd)})
+                    nwc.grab_json_response('/api/providers/logout')
             finally:
                 self._wc = None
-                self.wc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(tmpuid, '6pmu0ezczzcp')})
+                wc = self.wc
+                wc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(tmpuid, '6pmu0ezczzcp')})
+                wc.grab_json_response('/api/providers/logout')
 
     def config(self, nodename, reset=False):
         self.nodename = nodename
@@ -214,23 +356,38 @@ class NodeHandler(immhandler.NodeHandler):
         # between hypothetical secure path and today.
         dpp = self.configmanager.get_node_attributes(
             nodename, 'discovery.passwordrules')
-        self.ruleset = dpp.get(nodename, {}).get(
+        strruleset = dpp.get(nodename, {}).get(
             'discovery.passwordrules', {}).get('value', '')
         wc = self.wc
         creds = self.configmanager.get_node_attributes(
             self.nodename, ['secret.hardwaremanagementuser',
             'secret.hardwaremanagementpassword'], decrypt=True)
         user, passwd, isdefault = self.get_node_credentials(nodename, creds, 'USERID', 'PASSW0RD')
-        self.set_password_policy()
+        self.set_password_policy(strruleset, wc)
         if self._atdefaultcreds:
+            if isdefault and self.tmppasswd:
+                raise Exception(
+                    'Request to use default credentials, but refused by target after it has been changed to {0}'.format(self.tmppasswd))
             if not isdefault:
                 self._setup_xcc_account(user, passwd, wc)
         self._convert_sha256account(user, passwd, wc)
         cd = self.configmanager.get_node_attributes(
             nodename, ['secret.hardwaremanagementuser',
                        'secret.hardwaremanagementpassword',
-                       'hardwaremanagement.manager'], True)
+                       'hardwaremanagement.manager', 'hardwaremanagement.method', 'console.method'],
+                       True)
         cd = cd.get(nodename, {})
+        if (cd.get('hardwaremanagement.method', {}).get('value', 'ipmi') != 'redfish'
+                or cd.get('console.method', {}).get('value', None) == 'ipmi'):
+            nwc = wc.dupe()
+            nwc.set_basic_credentials(self._currcreds[0], self._currcreds[1])
+            rsp = nwc.grab_json_response('/redfish/v1/Managers/1/NetworkProtocol')
+            if not rsp.get('IPMI', {}).get('ProtocolEnabled', True):
+                # User has indicated IPMI support, but XCC is currently disabled
+                # change XCC to be consistent
+                _, _ = nwc.grab_json_response_with_status(
+                        '/redfish/v1/Managers/1/NetworkProtocol',
+                        {'IPMI': {'ProtocolEnabled': True}}, method='PATCH')
         if ('hardwaremanagement.manager' in cd and
                 cd['hardwaremanagement.manager']['value'] and
                 not cd['hardwaremanagement.manager']['value'].startswith(
@@ -242,17 +399,24 @@ class NodeHandler(immhandler.NodeHandler):
                 raise exc.NotImplementedException('IPv6 remote config TODO')
             netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
             newmask = netutil.cidr_to_mask(netconfig['prefix'])
-            # do not change the ipv4_config if the current config looks
-            statargs = {'ENET_IPv4Ena': '1', 'ENET_IPv4AddrSource': '0', 'ENET_IPv4StaticIPAddr': newip, 'ENET_IPv4StaticIPNetMask': newmask}
-            if netconfig['ipv4_gateway']:
-                statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
-            wc.grab_json_response('/api/dataset', statargs)
+            currinfo = wc.grab_json_response('/api/providers/logoninfo')
+            currip = currinfo.get('items', [{}])[0].get('ipv4_address', '')
+            # do not change the ipv4_config if the current config looks right already
+            if currip != newip:
+                statargs = {
+                    'ENET_IPv4Ena': '1', 'ENET_IPv4AddrSource': '0',
+                    'ENET_IPv4StaticIPAddr': newip, 'ENET_IPv4StaticIPNetMask': newmask
+                    }
+                if netconfig['ipv4_gateway']:
+                    statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
+                wc.grab_json_response('/api/dataset', statargs)
         elif self.ipaddr.startswith('fe80::'):
             self.configmanager.set_node_attributes(
                 {nodename: {'hardwaremanagement.manager': self.ipaddr}})
         else:
             raise exc.TargetEndpointUnreachable(
                 'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
+        wc.grab_json_response('/api/providers/logout')
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return

confluent_server/confluent/discovery/protocols/pxe.py

@@ -30,33 +30,67 @@ pxearchs = {
     '\x00\x07': 'uefi-x64',
     '\x00\x09': 'uefi-x64',
     '\x00\x0b': 'uefi-aarch64',
+    '\x00\x10': 'uefi-httpboot',
 }
 
 
+def stringify(value):
+    string = bytes(value)
+    if not isinstance(string, str):
+        string = string.decode('utf8')
+    return string
+
 def decode_uuid(rawguid):
-    lebytes = struct.unpack_from('<IHH', buffer(rawguid[:8]))
-    bebytes = struct.unpack_from('>HHI', buffer(rawguid[8:]))
+    lebytes = struct.unpack_from('<IHH', rawguid[:8])
+    bebytes = struct.unpack_from('>HHI', rawguid[8:])
     return '{0:08X}-{1:04X}-{2:04X}-{3:04X}-{4:04X}{5:08X}'.format(
         lebytes[0], lebytes[1], lebytes[2], bebytes[0], bebytes[1], bebytes[2]).lower()
 
 
+def _decode_ocp_vivso(rq, idx, size):
+    end = idx + size
+    vivso = {'service-type': 'onie-switch'}
+    while idx < end:
+        if rq[idx] == 3:
+            vivso['machine'] = stringify(rq[idx + 2:idx + 2 + rq[idx + 1]])
+        elif rq[idx] == 4:
+            vivso['arch'] = stringify(rq[idx + 2:idx + 2 + rq[idx + 1]])
+        elif rq[idx] == 5:
+            vivso['revision'] = stringify(rq[idx + 2:idx + 2 + rq[idx + 1]])
+        idx += rq[idx + 1] + 2
+    return '', None, vivso
+
+
 def find_info_in_options(rq, optidx):
     uuid = None
     arch = None
+    vivso = None
+    ztpurlrequested = False
+    iscumulus = False
     try:
         while uuid is None or arch is None:
             if rq[optidx] == 53:  # DHCP message type
                 # we want only length 1 and only discover (type 1)
                 if rq[optidx + 1] != 1 or rq[optidx + 2] != 1:
-                    return uuid, arch
+                    return uuid, arch, vivso
                 optidx += 3
+            elif rq[optidx] == 55:
+                if 239 in rq[optidx + 2:optidx + 2 + rq[optidx + 1]]:
+                    ztpurlrequested = True
+                optidx += rq[optidx + 1] + 2
+            elif rq[optidx] == 60:
+                vci = stringify(rq[optidx + 2:optidx + 2 + rq[optidx + 1]])
+                if vci.startswith('cumulus-linux'):
+                    iscumulus = True
+                    arch = vci.replace('cumulus-linux', '').strip()
+                optidx += rq[optidx + 1] + 2
             elif rq[optidx] == 97:
                 if rq[optidx + 1] != 17:
                     # 16 bytes of uuid and one reserved byte
-                    return uuid, arch
+                    return uuid, arch, vivso
                 if rq[optidx + 2] != 0:  # the reserved byte should be zero,
                     # anything else would be a new spec that we don't know yet
-                    return uuid, arch
+                    return uuid, arch, vivso
                 uuid = decode_uuid(rq[optidx + 3:optidx + 19])
                 optidx += 19
             elif rq[optidx] == 93:
@@ -66,11 +100,20 @@ def find_info_in_options(rq, optidx):
                 if archraw in pxearchs:
                     arch = pxearchs[archraw]
                 optidx += 4
+            elif rq[optidx] == 125:
+                #vivso = rq[optidx + 2:optidx + 2 + rq[optidx + 1]]
+                if rq[optidx + 2:optidx + 6] == b'\x00\x00\xa6\x7f':  # OCP
+                    return _decode_ocp_vivso(rq, optidx + 7, rq[optidx + 6])
+                optidx += rq[optidx + 1] + 2
             else:
                 optidx += rq[optidx + 1] + 2
     except IndexError:
-        return uuid, arch
-    return uuid, arch
+        pass
+    if not vivso and iscumulus and ztpurlrequested:
+        if not uuid:
+            uuid = ''
+        vivso = {'service-type': 'cumulus-switch', 'arch': arch}
+    return uuid, arch, vivso
 
 def snoop(handler, protocol=None):
     #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
@@ -98,10 +141,17 @@ def snoop(handler, protocol=None):
             netaddr = ':'.join(['{0:02x}'.format(x) for x in netaddr])
             optidx = 0
             try:
-                optidx = rq.index('\x63\x82\x53\x63') + 4
+                optidx = rq.index(b'\x63\x82\x53\x63') + 4
             except ValueError:
                 continue
-            uuid, arch = find_info_in_options(rq, optidx)
+            uuid, arch, vivso = find_info_in_options(rq, optidx)
+            if vivso:
+                # info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
+                handler({'hwaddr': netaddr, 'uuid': uuid,
+                         'architecture': vivso.get('arch', ''),
+                         'services': (vivso['service-type'],),
+                         'attributes': {'enclosure-machinetype-model': [vivso.get('machine', '')]}})
+                continue
             if uuid is None:
                 continue
             # We will fill out service to have something to byte into,

confluent_server/confluent/discovery/protocols/slp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@ import traceback
 _slp_services = set([
     'service:management-hardware.IBM:integrated-management-module2',
     'service:lenovo-smm',
+    'service:ipmi',
+    'service:lighttpd',
     'service:management-hardware.Lenovo:lenovo-xclarity-controller',
     'service:management-hardware.IBM:chassis-management-module',
     'service:management-hardware.Lenovo:chassis-management-module',
@@ -47,7 +49,6 @@ except AttributeError:
     IPPROTO_IPV6 = 41  # Assume Windows value if socket is missing it
 
 
-
 def _parse_slp_header(packet):
     packet = bytearray(packet)
     if len(packet) < 16 or packet[0] != 2:
@@ -83,6 +84,8 @@ def _parse_SrvRply(parsed):
     :return:
     """
     payload = parsed['payload']
+    if len(payload) < 4:
+        return
     ecode, ucount = struct.unpack('!HH', bytes(payload[0:4]))
     if ecode:
         parsed['errorcode'] = ecode
@@ -233,36 +236,42 @@ def _find_srvtype(net, net4, srvtype, addresses, xid):
 
 
 def _grab_rsps(socks, rsps, interval, xidmap):
-    r, _, _ = select.select(socks, (), (), interval)
+    r = None
+    res = select.select(socks, (), (), interval)
+    if res:
+        r = res[0]
     while r:
         for s in r:
             (rsp, peer) = s.recvfrom(9000)
             neighutil.refresh_neigh()
             _parse_slp_packet(rsp, peer, rsps, xidmap)
-            r, _, _ = select.select(socks, (), (), interval)
+            res = select.select(socks, (), (), interval)
+            if not res:
+                r = None
+            else:
+                r = res[0]
 
 
 
 def _parse_attrlist(attrstr):
     attribs = {}
+    previousattrlen = None
+    attrstr = util.stringify(attrstr)
     while attrstr:
+        if len(attrstr) == previousattrlen:
+            raise Exception('Looping in attrstr parsing')
+        previousattrlen = len(attrstr)
         if attrstr[0] == '(':
             if ')' not in attrstr:
                 attribs['INCOMPLETE'] = True
                 return attribs
             currattr = attrstr[1:attrstr.index(')')]
             if '=' not in currattr:  # Not allegedly kosher, but still..
-                currattr = currattr.decode('utf-8')
                 attribs[currattr] = None
             else:
                 attrname, attrval = currattr.split('=', 1)
-                attrname = attrname.decode('utf-8')
                 attribs[attrname] = []
                 for val in attrval.split(','):
-                    try:
-                        val = val.decode('utf-8')
-                    except UnicodeDecodeError:
-                        val = '*DECODEERROR*'
                     if val[:3] == '\\FF':  # we should make this bytes
                         finalval = bytearray([])
                         for bnum in attrval[3:].split('\\'):
@@ -272,9 +281,9 @@ def _parse_attrlist(attrstr):
                         val = finalval
                         if 'uuid' in attrname and len(val) == 16:
                             lebytes = struct.unpack_from(
-                                '<IHH', buffer(val[:8]))
+                                '<IHH', memoryview(val[:8]))
                             bebytes = struct.unpack_from(
-                                '>HHI', buffer(val[8:]))
+                                '>HHI', memoryview(val[8:]))
                             val = '{0:08X}-{1:04X}-{2:04X}-{3:04X}-' \
                                   '{4:04X}{5:08X}'.format(
                                 lebytes[0], lebytes[1], lebytes[2], bebytes[0],
@@ -282,7 +291,7 @@ def _parse_attrlist(attrstr):
                             ).lower()
                     attribs[attrname].append(val)
             attrstr = attrstr[attrstr.index(')'):]
-        elif attrstr[0] == ',':
+        elif attrstr[0] == ','[0]:
             attrstr = attrstr[1:]
         elif ',' in attrstr:
             currattr = attrstr[:attrstr.index(',')]
@@ -484,6 +493,20 @@ def snoop(handler, protocol=None):
                 _add_attributes(peerbymacaddress[mac])
                 peerbymacaddress[mac]['hwaddr'] = mac
                 peerbymacaddress[mac]['protocol'] = protocol
+                for srvurl in peerbymacaddress[mac].get('urls', ()):
+                    if len(srvurl) > 4:
+                        srvurl = srvurl[:-3]
+                    if srvurl.endswith('://Athena:'):
+                        continue
+                if 'service:ipmi' in peerbymacaddress[mac]['services']:
+                    continue
+                if 'service:lightttpd' in peerbymacaddress[mac]['services']:
+                    currinf = peerbymacaddress[mac]
+                    curratt = currinf.get('attributes', {})
+                    if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+                        peerbymacaddress[mac]['services'] = ['service:lenovo-tsm']
+                    else:
+                        continue
                 handler(peerbymacaddress[mac])
         except Exception as e:
             tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
@@ -547,6 +570,13 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
     _grab_rsps((net, net4), rsps, 1, xidmap)
     # now to analyze and flesh out the responses
     for id in rsps:
+        for srvurl in rsps[id].get('urls', ()):
+            if len(srvurl) > 4:
+                srvurl = srvurl[:-3]
+            if srvurl.endswith('://Athena:'):
+                continue
+        if 'service:ipmi' in rsps[id]['services']:
+            continue
         if localonly:
             for addr in rsps[id]['addresses']:
                 if 'fe80' in addr[0]:
@@ -554,6 +584,15 @@ def scan(srvtypes=_slp_services, addresses=None, localonly=False):
             else:
                 continue
         _add_attributes(rsps[id])
+        if 'service:lighttpd' in rsps[id]['services']:
+            currinf = rsps[id]
+            curratt = currinf.get('attributes', {})
+            if curratt.get('System-Manufacturing', [None])[0] == 'Lenovo' and curratt.get('type', [None])[0] == 'LenovoThinkServer':
+               currinf['services'] = ['service:lenovo-tsm']
+               curratt['enclosure-serial-number'] = curratt['Product-Serial']
+               curratt['enclosure-machinetype-model'] = curratt['Machine-Type']
+            else:
+                continue
         del rsps[id]['payload']
         del rsps[id]['function']
         del rsps[id]['xid']

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -32,6 +32,10 @@ import confluent.neighutil as neighutil
 import confluent.util as util
 import eventlet.green.select as select
 import eventlet.green.socket as socket
+try:
+    from eventlet.green.urllib.request import urlopen
+except (ImportError, AssertionError):
+    from eventlet.green.urllib2 import urlopen
 import struct
 
 mcastv4addr = '239.255.255.250'
@@ -45,6 +49,20 @@ smsg = ('M-SEARCH * HTTP/1.1\r\n'
         'MX: 3\r\n\r\n')
 
 
+def active_scan(handler, protocol=None):
+    known_peers = set([])
+    for scanned in scan(['urn:dmtf-org:service:redfish-rest:1']):
+        for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                continue
+            if addr in known_peers:
+                break
+            known_peers.add(addr)
+        else:
+            scanned['protocol'] = protocol
+            handler(scanned)
+
 def scan(services, target=None):
     for service in services:
         for rply in _find_service(service, target):
@@ -78,8 +96,13 @@ def snoop(handler, byehandler=None):
     for i4 in util.list_ips():
         ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
                      socket.inet_aton(i4['addr'])
-        net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
-                        ssdp4mcast)
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            ssdp4mcast)
+        except socket.error as e:
+            if e.errno != 98:
+                # errno 98 can happen if aliased, skip for now
+                raise
     net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     net4.bind(('', 1900))
     net6.bind(('', 1900))
@@ -104,11 +127,11 @@ def snoop(handler, byehandler=None):
                     known_peers.add(peer)
                     newmacs.add(mac)
                     if mac in peerbymacaddress:
-                        peerbymacaddress[mac]['peers'].append(peer)
+                        peerbymacaddress[mac]['addresses'].append(peer)
                     else:
                         peerbymacaddress[mac] = {
                             'hwaddr': mac,
-                            'peers': [peer],
+                            'addresses': [peer],
                         }
                         peerdata = peerbymacaddress[mac]
                         for headline in rsp[1:]:
@@ -180,7 +203,12 @@ def _find_service(service, target):
             timeout = 0
         r, _, _ = select.select((net4, net6), (), (), timeout)
     for nid in peerdata:
-        yield peerdata[nid]
+        for url in peerdata[nid].get('urls', ()):
+            if url.endswith('/desc.tmpl'):
+                info = urlopen(url).read()
+                if '<friendlyName>Athena</friendlyName>' in info:
+                    peerdata[nid]['services'] = ['service:thinkagile-storage']
+                    yield peerdata[nid]
 
 
 def _parse_ssdp(peer, rsp, peerdata):
@@ -198,11 +226,11 @@ def _parse_ssdp(peer, rsp, peerdata):
     if code == '200':
         if nid in peerdata:
             peerdatum = peerdata[nid]
-            if peer not in peerdatum['peers']:
-                peerdatum['peers'].append(peer)
+            if peer not in peerdatum['addresses']:
+                peerdatum['addresses'].append(peer)
         else:
             peerdatum = {
-                'peers': [peer],
+                'addresses': [peer],
                 'hwaddr': mac,
             }
             peerdata[nid] = peerdatum

confluent_server/confluent/exceptions.py

@@ -17,7 +17,17 @@
 
 import base64
 import json
+import msgpack
 
+def deserialize_exc(msg):
+    excd = msgpack.unpackb(msg, raw=False)
+    if excd[0] == 'Exception':
+        return Exception(excd[1])
+    if excd[0] not in globals():
+        return Exception('Cannot deserialize: {0}'.format(repr(excd)))
+    if not issubclass(excd[0], ConfluentException):
+        return Exception('Cannot deserialize: {0}'.format(repr(excd)))
+    return globals(excd[0])(*excd[1])
 
 class ConfluentException(Exception):
     apierrorcode = 500
@@ -27,6 +37,10 @@ class ConfluentException(Exception):
         errstr = ' - '.join((self._apierrorstr, str(self)))
         return json.dumps({'error': errstr })
 
+    def serialize(self):
+        return msgpack.packb([self.__class__.__name__, [str(self)]],
+                             use_bin_type=False)
+
     @property
     def apierrorstr(self):
         if str(self):
@@ -104,16 +118,24 @@ class PubkeyInvalid(ConfluentException):
 
     def __init__(self, text, certificate, fingerprint, attribname, event):
         super(PubkeyInvalid, self).__init__(self, text)
+        self.myargs = (text, certificate, fingerprint, attribname, event)
         self.fingerprint = fingerprint
         self.attrname = attribname
         self.message = text
+        certtxt = base64.b64encode(certificate)
+        if not isinstance(certtxt, str):
+            certtxt = certtxt.decode('utf8')
         bodydata = {'message': text,
                     'event': event,
                     'fingerprint': fingerprint,
                     'fingerprintfield': attribname,
-                    'certificate': base64.b64encode(certificate)}
+                    'certificate': certtxt}
         self.errorbody = json.dumps(bodydata)
 
+    def serialize(self):
+        return msgpack.packb([self.__class__.__name__, self.myargs],
+                             use_bin_type=False)
+
     def get_error_body(self):
         return self.errorbody
 

confluent_server/confluent/firmwaremanager.py

@@ -36,20 +36,40 @@ _tracelog = None
 
 def execupdate(handler, filename, updateobj, type, owner, node):
     global _tracelog
-    if type != 'ffdc' and not os.path.exists(filename):
-        errstr =  '{0} does not appear to exist on {1}'.format(
-            filename, socket.gethostname())
-        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
-                                   'detail': errstr})
-        return
+    if type != 'ffdc':
+        errstr = False
+        if not os.path.exists(filename):
+            errstr =  '{0} does not appear to exist on {1}'.format(
+                filename, socket.gethostname())
+        elif not os.access(filename, os.R_OK):
+            errstr =  '{0} is not readable by confluent on {1} (ensure confluent user or group can access file and parent directories)'.format(
+                filename, socket.gethostname())
+        if errstr:
+            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                    'detail': errstr})
+            return
     if type == 'ffdc' and os.path.isdir(filename):
-        filename += '/' + node + '.svcdata'
+        filename += '/' + node
+    if 'type' == 'ffdc':
+        errstr = False
+        if os.path.exists(filename):
+            errstr = '{0} already exists on {1}, cannot overwrite'.format(
+                filename, socket.gethostname())
+        elif not os.access(os.path.dirname(filename), os.W_OK):
+            errstr = '{0} directory not writable by confluent user/group on {1}, check the directory and parent directory ownership and permissions'.format(filename, socket.gethostname())
+        if errstr:
+            updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                    'detail': errstr})
+            return
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
                                  bank=updateobj.bank)
         else:
             completion = handler(filename, progress=updateobj.handle_progress)
+        if type == 'ffdc' and completion:
+            filename = completion
+            completion = None
         if completion is None:
             completion = 'complete'
         if owner:

confluent_server/confluent/forwarder.py

@@ -49,6 +49,7 @@ def forward_port(sock, target, clientip, sessionid):
             continue
         try:
             client = socket.create_connection((target, 443))
+            client.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
         except Exception:
             conn.close()
             continue
@@ -68,6 +69,7 @@ def forward_video():
         try:
             vidclient = socket.create_connection((vidtargetbypeer[cli[0]],
                                                   3900))
+            vidclient.setsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG, 1456)
         except Exception:
             conn.close()
             continue

confluent_server/confluent/httpapi.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,10 @@
 # This SCGI server provides a http wrap to confluent api
 # It additionally manages httprequest console sessions
 import base64
-import Cookie
+try:
+    import Cookie
+except ModuleNotFoundError:
+    import http.cookies as Cookie
 import confluent.auth as auth
 import confluent.config.attributes as attribs
 import confluent.consoleserver as consoleserver
@@ -39,7 +42,10 @@ import socket
 import sys
 import traceback
 import time
-import urlparse
+try:
+    import urlparse
+except ModuleNotFoundError:
+    import urllib.parse as urlparse
 import eventlet.wsgi
 #scgi = eventlet.import_patched('flup.server.scgi')
 tlvdata = confluent.tlvdata
@@ -74,7 +80,7 @@ def group_creation_resources():
     yield confluent.messages.ListAttributes(kv={'nodes': []},
                                             desc='Nodes to add to the group'
                                             ).html() + '<br>\n'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
         if attr == 'groups':
             continue
         if attr.startswith("secret."):
@@ -95,7 +101,7 @@ def group_creation_resources():
 def node_creation_resources():
     yield confluent.messages.Attributes(
         kv={'name': None}, desc="Name of the node").html() + '<br>'
-    for attr in sorted(attribs.node.iterkeys()):
+    for attr in sorted(attribs.node):
         if attr.startswith("secret."):
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
@@ -126,7 +132,7 @@ def user_creation_resources():
             'description': (''),
         },
     }
-    for attr in sorted(credential.iterkeys()):
+    for attr in sorted(credential):
         if attr == "password":
             yield confluent.messages.CryptedAttributes(
                 kv={attr: None},
@@ -176,7 +182,7 @@ def _get_query_dict(env, reqbody, reqtype):
     if reqbody is not None:
         if "application/x-www-form-urlencoded" in reqtype:
             pbody = urlparse.parse_qs(reqbody, True)
-            for ky in pbody.iterkeys():
+            for ky in pbody:
                 if len(pbody[ky]) > 1:  # e.g. REST explorer
                     na = [i for i in pbody[ky] if i != '']
                     qdict[ky] = na
@@ -184,7 +190,7 @@ def _get_query_dict(env, reqbody, reqtype):
                     qdict[ky] = pbody[ky][0]
         elif 'application/json' in reqtype:
             pbody = json.loads(reqbody)
-            for key in pbody.iterkeys():
+            for key in pbody:
                 qdict[key] = pbody[key]
     if 'restexplorerhonorkey' in qdict:
         nqdict = {}
@@ -305,7 +311,7 @@ def _authorize_request(env, operation):
                 return {'code': 401}
             return ('logout',)
         name, passphrase = base64.b64decode(
-            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
+            env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(b':', 1)
         authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
         if authdata is False:
             return {'code': 403}
@@ -319,14 +325,14 @@ def _authorize_request(env, operation):
                                 'inflight': set([])}
         if 'HTTP_CONFLUENTAUTHTOKEN' in env:
             httpsessions[sessid]['csrftoken'] = util.randomstring(32)
-        cookie['confluentsessionid'] = sessid
+        cookie['confluentsessionid'] = util.stringify(sessid)
         cookie['confluentsessionid']['secure'] = 1
         cookie['confluentsessionid']['httponly'] = 1
         cookie['confluentsessionid']['path'] = '/'
     skiplog = _should_skip_authlog(env)
     if authdata:
         auditmsg = {
-            'user': name,
+            'user': util.stringify(name),
             'operation': operation,
             'target': env['PATH_INFO'],
         }
@@ -338,7 +344,7 @@ def _authorize_request(env, operation):
         if authdata[3] is not None:
             auditmsg['tenant'] = authdata[3]
             authinfo['tenant'] = authdata[3]
-        auditmsg['user'] = authdata[2]
+        auditmsg['user'] = util.stringify(authdata[2])
         if sessid is not None:
             authinfo['sessionid'] = sessid
         if not skiplog:
@@ -410,7 +416,7 @@ def resourcehandler_backend(env, start_response):
         reqtype = env['CONTENT_TYPE']
     operation = opmap[env['REQUEST_METHOD']]
     querydict = _get_query_dict(env, reqbody, reqtype)
-    if 'restexplorerop' in querydict:
+    if operation != 'retrieve' and 'restexplorerop' in querydict:
         operation = querydict['restexplorerop']
         del querydict['restexplorerop']
     authorized = _authorize_request(env, operation)
@@ -446,6 +452,8 @@ def resourcehandler_backend(env, start_response):
                             httpsessions[authorized['sessionid']]['inflight'])):
                 pagecontent += rsp
             start_response("200 OK", headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
             yield pagecontent
             return
         except exc.ConfluentException as e:
@@ -491,7 +499,7 @@ def resourcehandler_backend(env, start_response):
             auditmsg = {
                 'operation': 'start',
                 'target': env['PATH_INFO'],
-                'user': authorized['username'],
+                'user': util.stringify(authorized['username']),
             }
             if 'tenant' in authorized:
                 auditmsg['tenant'] = authorized['tenant']
@@ -503,10 +511,10 @@ def resourcehandler_backend(env, start_response):
             width = querydict.get('width', 80)
             height = querydict.get('height', 24)
             datacallback = None
-            async = None
+            asynchdl = None
             if 'HTTP_CONFLUENTASYNCID' in env:
-                async = confluent.asynchttp.get_async(env, querydict)
-                termrel = async.set_term_relation(env)
+                asynchdl = confluent.asynchttp.get_async(env, querydict)
+                termrel = asynchdl.set_term_relation(env)
                 datacallback = termrel.got_data
             try:
                 if shellsession:
@@ -529,8 +537,8 @@ def resourcehandler_backend(env, start_response):
                 start_response("500 Internal Server Error", headers)
                 return
             sessid = _assign_consessionid(consession)
-            if async:
-                async.add_console_session(sessid)
+            if asynchdl:
+                asynchdl.add_console_session(sessid)
             start_response('200 OK', headers)
             yield '{"session":"%s","data":""}' % sessid
             return
@@ -626,6 +634,7 @@ def resourcehandler_backend(env, start_response):
             sessinfo = {'username': authorized['username']}
             if 'authtoken' in authorized:
                 sessinfo['authtoken'] = authorized['authtoken']
+            tlvdata.unicode_dictvalues(sessinfo)
             yield json.dumps(sessinfo)
             return
         resource = '.' + url[url.rindex('/'):]
@@ -647,6 +656,8 @@ def resourcehandler_backend(env, start_response):
                 for datum in _assemble_json(hdlr, resource, url, extension):
                     pagecontent += datum
             start_response('200 OK', headers)
+            if not isinstance(pagecontent, bytes):
+                pagecontent = pagecontent.encode('utf-8')
             yield pagecontent
         except exc.ConfluentException as e:
             if ((not isinstance(e, exc.LockedCredentials)) and
@@ -731,7 +742,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
     for rsp in responses:
         if isinstance(rsp, confluent.messages.LinkRelation):
             haldata = rsp.raw()
-            for hk in haldata.iterkeys():
+            for hk in haldata:
                 if 'href' in haldata[hk]:
                     if isinstance(haldata[hk]['href'], int):
                         haldata[hk]['href'] = str(haldata[hk]['href'])
@@ -747,7 +758,7 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                     links[hk] = haldata[hk]
         else:
             rsp = rsp.raw()
-            for dk in rsp.iterkeys():
+            for dk in rsp:
                 if dk in rspdata:
                     if isinstance(rspdata[dk], list):
                         if isinstance(rsp[dk], list):
@@ -766,8 +777,8 @@ def _assemble_json(responses, resource=None, url=None, extension=None):
                         rspdata[dk] = rsp[dk]
     rspdata["_links"] = links
     tlvdata.unicode_dictvalues(rspdata)
-    yield json.dumps(
-        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
+    yield util.stringify(json.dumps(
+        rspdata, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8'))
 
 
 def serve(bind_host, bind_port):

confluent_server/confluent/log.py

@@ -76,6 +76,10 @@ import stat
 import struct
 import time
 import traceback
+try:
+    unicode
+except NameError:
+    unicode = str
 
 daemonized = False
 logfull = False
@@ -176,6 +180,8 @@ class BaseRotatingHandler(object):
                 self.textfile = open(self.textpath, mode='ab')
             if self.binfile is None:
                 self.binfile = open(self.binpath, mode='ab')
+            if not isinstance(textrecord, bytes):
+                textrecord = textrecord.encode('utf-8')
             self.textfile.write(textrecord)
             self.binfile.write(binrecord)
             self.textfile.flush()
@@ -594,7 +600,11 @@ class Logger(object):
                     if ltype == 2:
                         textrecord = data
                     else:
-                        textrecord = textdate + data + ']'
+                        if not isinstance(textdate, bytes):
+                            textdate = textdate.encode('utf-8')
+                        if not isinstance(data, bytes):
+                            data = data.encode('utf-8')
+                        textrecord = textdate + data + b']'
                 else:
                     textrecord = textdate + data
                     if not textrecord.endswith('\n'):
@@ -742,7 +752,7 @@ class Logger(object):
         pass
 
     def log(self, logdata=None, ltype=None, event=0, eventdata=None):
-        if type(logdata) not in (str, unicode, dict):
+        if type(logdata) not in (bytes, unicode, dict):
             raise Exception("Unsupported logdata")
         if ltype is None:
             if type(logdata) == dict:
@@ -792,4 +802,4 @@ def logtrace():
     if tracelog is None:
         tracelog = Logger('trace', buffered=False)
     tracelog.log(traceback.format_exc(), ltype=DataTypes.event,
-                 event=Events.stacktrace)
+                 event=Events.stacktrace)

confluent_server/confluent/main.py

@@ -43,9 +43,11 @@ except ImportError:
 import confluent.discovery.core as disco
 import eventlet
 dbgif = False
-if  map(int, (eventlet.__version__.split('.'))) > [0, 18]:
+try:
     import eventlet.backdoor as backdoor
     dbgif = True
+except Exception:
+    pass
 havefcntl = True
 try:
     import fcntl
@@ -75,13 +77,16 @@ def _daemonize():
         print('confluent server starting as pid {0}'.format(thispid))
         os._exit(0)
     os.closerange(0, 2)
-    os.umask(63)
     os.open(os.devnull, os.O_RDWR)
     os.dup2(0, 1)
     os.dup2(0, 2)
+    log.daemonized = True
+
+
+def _redirectoutput():
+    os.umask(63)
     sys.stdout = log.Logger('stdout', buffered=False)
     sys.stderr = log.Logger('stderr', buffered=False)
-    log.daemonized = True
 
 
 def _updatepidfile():
@@ -204,7 +209,7 @@ def setlimits():
         pass
 
 
-def run():
+def run(args):
     setlimits()
     try:
         signal.signal(signal.SIGUSR1, dumptrace)
@@ -230,7 +235,10 @@ def run():
     except (OSError, IOError) as e:
         print(repr(e))
         sys.exit(1)
-    _daemonize()
+    if '-f' not in args:
+        _daemonize()
+    if '-o' not in args:
+        _redirectoutput()
     if havefcntl:
         _updatepidfile()
     signal.signal(signal.SIGINT, terminate)

confluent_server/confluent/messages.py

@@ -23,8 +23,15 @@ import confluent.config.configmanager as cfm
 import confluent.config.conf as cfgfile
 from copy import deepcopy
 from datetime import datetime
+import confluent.util as util
+import msgpack
 import json
 
+try:
+    unicode
+except NameError:
+    unicode = str
+
 valid_health_values = set([
     'ok',
     'warning',
@@ -54,7 +61,9 @@ def _htmlify_structure(indict):
     if isinstance(indict, dict):
         for key in sorted(indict):
             ret += "<li>{0}: ".format(key)
-            if type(indict[key]) in (str, unicode, float, int):
+            if type(indict[key]) in (bytes, unicode):
+                ret += util.stringify(indict[key])
+            if type(indict[key]) in (float, int):
                 ret += str(indict[key])
             elif isinstance(indict[key], datetime):
                 ret += indict[key].strftime('%Y-%m-%dT%H:%M:%S')
@@ -62,13 +71,13 @@ def _htmlify_structure(indict):
                 ret += _htmlify_structure(indict[key])
     elif isinstance(indict, list):
         if len(indict) > 0:
-            if type(indict[0]) in (str, unicode, None):
+            if type(indict[0]) in (bytes, unicode, None):
                 nd = []
                 for datum in indict:
                     if datum is None:
                         nd.append('')
                     else:
-                        nd.append(datum)
+                        nd.append(util.stringify(datum))
                 ret += ",".join(nd)
             else:
                 for v in indict:
@@ -76,6 +85,13 @@ def _htmlify_structure(indict):
     return ret + '</ul>'
 
 
+def msg_deserialize(packed):
+    m = msgpack.unpackb(packed, raw=False)
+    cls = globals()[m[0]]
+    if issubclass(cls, ConfluentMessage) or issubclass(cls, ConfluentNodeError):
+        return cls(*m[1:])
+    raise Exception("Unknown shenanigans")
+
 class ConfluentMessage(object):
     apicode = 200
     readonly = False
@@ -97,6 +113,15 @@ class ConfluentMessage(object):
         jsonsnippet = json.dumps(datasource, sort_keys=True, separators=(',', ':'))[1:-1]
         return jsonsnippet
 
+    def serialize(self):
+        msg = [self.__class__.__name__]
+        msg.extend(self.myargs)
+        return msgpack.packb(msg, use_bin_type=False)
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(*data)
+
     def raw(self):
         """Return pythonic representation of the response.
 
@@ -121,15 +146,16 @@ class ConfluentMessage(object):
             return self._generic_html_value(self.kvpairs)
         if not self.stripped:
             htmlout = ''
-            for node in self.kvpairs.iterkeys():
+            for node in self.kvpairs:
                 htmlout += '{0}:{1}\n'.format(
                     node, self._generic_html_value(self.kvpairs[node]))
             return htmlout
 
     def _generic_html_value(self, pairs):
         snippet = ""
-        for key in pairs.iterkeys():
+        for key in pairs:
             val = pairs[key]
+            key = util.stringify(key)
             value = self.defaultvalue
             if isinstance(val, dict) and 'type' in val:
                 valtype = val['type']
@@ -156,7 +182,7 @@ class ConfluentMessage(object):
                         '<input type="checkbox" name="restexplorerhonorkey" '
                         'value="{1}">\r').format(valtype, key, self.desc)
                 return snippet
-            if (isinstance(val, bool) or isinstance(val, str) or
+            if (isinstance(val, bool) or isinstance(val, bytes) or
                     isinstance(val, unicode)):
                 value = str(val)
             elif val is not None and 'value' in val:
@@ -202,6 +228,15 @@ class ConfluentNodeError(object):
         self.node = node
         self.error = errorstr
 
+    def serialize(self):
+        return msgpack.packb(
+            [self.__class__.__name__, self.node, self.error],
+            use_bin_type=False)
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(*data)
+
     def raw(self):
         return {'databynode': {self.node: {'errorcode': self.apicode,
                                            'error': self.error}}}
@@ -212,7 +247,7 @@ class ConfluentNodeError(object):
     def strip_node(self, node):
         # NOTE(jjohnson2): For single node errors, raise exception to
         # trigger what a developer of that medium would expect
-        raise Exception(self.error)
+        raise Exception('{0}: {1}'.format(self.node, self.error))
 
 
 class ConfluentResourceUnavailable(ConfluentNodeError):
@@ -251,9 +286,9 @@ class ConfluentTargetNotFound(ConfluentNodeError):
 
 class ConfluentTargetInvalidCredentials(ConfluentNodeError):
     apicode = 502
-    def __init__(self, node):
+    def __init__(self, node, errstr='bad credentials'):
         self.node = node
-        self.error = 'bad credentials'
+        self.error = errstr
 
     def strip_node(self, node):
         raise exc.TargetEndpointBadCredentials
@@ -262,6 +297,7 @@ class ConfluentTargetInvalidCredentials(ConfluentNodeError):
 class DeletedResource(ConfluentMessage):
     notnode = True
     def __init__(self, resource):
+        self.myargs = [resource]
         self.kvpairs = {'deleted': resource}
 
     def strip_node(self, node):
@@ -273,6 +309,7 @@ class CreatedResource(ConfluentMessage):
     readonly = True
 
     def __init__(self, resource):
+        self.myargs = [resource]
         self.kvpairs = {'created': resource}
 
     def strip_node(self, node):
@@ -284,6 +321,7 @@ class RenamedResource(ConfluentMessage):
     readonly = True
 
     def __init__(self, oldname, newname):
+        self.myargs = (oldname, newname)
         self.kvpairs = {'oldname': oldname, 'newname': newname}
 
     def strip_node(self, node):
@@ -292,6 +330,7 @@ class RenamedResource(ConfluentMessage):
 
 class RenamedNode(ConfluentMessage):
     def __init__(self, name, rename):
+        self.myargs = (name, rename)
         self.desc = 'New Name'
         kv = {'rename': {'value': rename}}
         self.kvpairs = {name: kv}
@@ -302,13 +341,16 @@ class AssignedResource(ConfluentMessage):
     readonly = True
 
     def __init__(self, resource):
+        self.myargs = [resource]
         self.kvpairs = {'assigned': resource}
 
+
 class ConfluentChoiceMessage(ConfluentMessage):
     valid_values = set()
     valid_paramset = {}
 
     def __init__(self, node, state):
+        self.myargs = (node, state)
         self.stripped = False
         self.kvpairs = {
             node: {
@@ -321,15 +363,16 @@ class ConfluentChoiceMessage(ConfluentMessage):
             return self._create_option(self.kvpairs)
         else:
             htmlout = ''
-            for node in self.kvpairs.iterkeys():
+            for node in self.kvpairs:
                 htmlout += '{0}:{1}\n'.format(
                     node, self._create_option(self.kvpairs[node]))
             return htmlout
 
     def _create_option(self, pairdata):
         snippet = ''
-        for key in pairdata.iterkeys():
+        for key in pairdata:
             val = pairdata[key]
+            key = util.stringify(key)
             snippet += key + ':<select name="%s">' % key
             valid_values = self.valid_values
             if key in self.valid_paramset:
@@ -381,6 +424,7 @@ class LinkRelation(ConfluentMessage):
 
 class ChildCollection(LinkRelation):
     def __init__(self, collname, candelete=False):
+        self.myargs = (collname, candelete)
         self.rel = 'item'
         self.href = collname
         self.candelete = candelete
@@ -450,10 +494,11 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
     elif (path[:4] == ['configuration', 'management_controller', 'ntp',
             'servers'] and operation != 'retrieve' and len(path) == 5):
         return InputNTPServer(path, nodes, inputdata)
-    elif (path[:3] == ['configuration', 'system', 'all'] and
+    elif (path[:3] in (['configuration', 'system', 'all'],
+            ['configuration', 'management_controller', 'extended']) and
             operation != 'retrieve'):
         return InputConfigChangeSet(path, inputdata, nodes, configmanager)
-    elif (path[:3] == ['configuration', 'system', 'clear'] and
+    elif (path[0] == 'configuration' and path[2] == 'clear'  and
             operation != 'retrieve'):
         return InputConfigClear(path, inputdata, nodes, configmanager)
     elif (path[:3] == ['configuration', 'storage', 'disks'] and
@@ -500,9 +545,21 @@ class InputFirmwareUpdate(ConfluentMessage):
             raise Exception('User requested substitutions, but code is '
                             'written against old api, code must be fixed or '
                             'skip {} expansion')
+        if self.filebynode[node].startswith('/etc/confluent'):
+            raise Exception(
+                'File transfer with /etc/confluent is not supported')
+        if self.filebynode[node].startswith('/var/log/confluent'):
+            raise Exception(
+                'File transfer with /var/log/confluent is not supported')
         return self._filename
 
     def nodefile(self, node):
+        if self.filebynode[node].startswith('/etc/confluent'):
+            raise Exception(
+                'File transfer with /etc/confluent is not supported')
+        if self.filebynode[node].startswith('/var/log/confluent'):
+            raise Exception(
+                'File transfer with /var/log/confluent is not supported')
         return self.filebynode[node]
 
 class InputMedia(InputFirmwareUpdate):
@@ -521,11 +578,15 @@ class DetachMedia(ConfluentMessage):
 
 
 class Media(ConfluentMessage):
-    def __init__(self, node, media):
-        self.kvpairs = {node: {'name': media.name, 'url': media.url}}
+    def __init__(self, node, media=None, rawmedia=None):
+        if media:
+            rawmedia = {'name': media.name, 'url': media.url}
+        self.myargs = (node, None, rawmedia)
+        self.kvpairs = {node: rawmedia}
 
 class SavedFile(ConfluentMessage):
     def __init__(self, node, file):
+        self.myargs = (node, file)
         self.kvpairs = {node: {'filename': file}}
 
 class InputAlertData(ConfluentMessage):
@@ -586,7 +647,7 @@ class InputConfigChangeSet(InputExpression):
         endattrs = {}
         for attr in attrs:
             origval = attrs[attr]
-            if isinstance(origval, str) or isinstance(origval, unicode):
+            if isinstance(origval, bytes) or isinstance(origval, unicode):
                 origval = {'expression': origval}
             if 'expression' not in origval:
                 endattrs[attr] = attrs[attr]
@@ -613,7 +674,9 @@ class InputAttributes(ConfluentMessage):
         if nodes is None:
             self.attribs = inputdata
             for attrib in self.attribs:
-                if type(self.attribs[attrib]) in (str, unicode):
+                if not cfm.attrib_supports_expression(attrib):
+                    continue
+                if type(self.attribs[attrib]) in (bytes, unicode):
                     try:
                         # ok, try to use format against the string
                         # store back result to the attribute to
@@ -639,7 +702,7 @@ class InputAttributes(ConfluentMessage):
             return {}
         nodeattr = deepcopy(self.nodeattribs[node])
         for attr in nodeattr:
-            if type(nodeattr[attr]) in (str, unicode):
+            if type(nodeattr[attr]) in (bytes, unicode) and cfm.attrib_supports_expression(attr):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -742,7 +805,7 @@ class InputCredential(ConfluentMessage):
         if len(path) == 4:
             inputdata['uid'] = path[-1]
         # if the operation is 'create' check if all fields are present
-        if (isinstance(inputdata['uid'], str) and
+        if (type(inputdata['uid']) in (bytes, unicode) and
                 not inputdata['uid'].isdigit()):
             inputdata['uid'] = inputdata['uid']
         else:
@@ -768,7 +831,7 @@ class InputCredential(ConfluentMessage):
             return {}
         credential = deepcopy(self.credentials[node])
         for attr in credential:
-            if type(credential[attr]) in (str, unicode):
+            if type(credential[attr]) in (bytes, unicode):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -822,6 +885,7 @@ class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([
         'on',
         'off',
+        'blink',
     ])
 
     keyname = 'identify'
@@ -1096,6 +1160,7 @@ class BootDevice(ConfluentChoiceMessage):
         }
 
     def __init__(self, node, device, bootmode='unspecified', persistent=False):
+        self.myargs = (node, device, bootmode, persistent)
         if device not in self.valid_values:
             raise Exception("Invalid boot device argument passed in:" +
                             repr(device))
@@ -1194,10 +1259,10 @@ class PowerState(ConfluentChoiceMessage):
 
     def __init__(self, node, state, oldstate=None):
         super(PowerState, self).__init__(node, state)
+        self.myargs = (node, state, oldstate)
         if oldstate is not None:
             self.kvpairs[node]['oldstate'] = {'value': oldstate}
 
-
 class BMCReset(ConfluentChoiceMessage):
     valid_values = set([
         'reset',
@@ -1213,13 +1278,13 @@ class NTPEnabled(ConfluentChoiceMessage):
 
     def __init__(self, node, enabled):
         self.stripped = False
+        self.myargs = (node, enabled)
         self.kvpairs = {
             node: {
                 'state': {'value': str(enabled)},
             }
         }
 
-
 class EventCollection(ConfluentMessage):
     """A collection of events
 
@@ -1239,6 +1304,8 @@ class EventCollection(ConfluentMessage):
     def __init__(self, events=(), name=None):
         eventdata = []
         self.notnode = name is None
+        self.myname = name
+        self.myargs = (eventdata, name)
         for event in events:
             entry = {
                 'id': event.get('id', None),
@@ -1266,6 +1333,10 @@ class AsyncCompletion(ConfluentMessage):
         self.stripped = True
         self.notnode = True
 
+    @classmethod
+    def deserialize(cls):
+        raise Exception("Not supported")
+
     def raw(self):
         return {'_requestdone': True}
 
@@ -1276,6 +1347,10 @@ class AsyncMessage(ConfluentMessage):
         self.notnode = True
         self.msgpair = pair
 
+    @classmethod
+    def deserialize(cls):
+        raise Exception("Not supported")
+
     def raw(self):
         rsp = self.msgpair[1]
         rspdict = None
@@ -1307,6 +1382,7 @@ class User(ConfluentMessage):
         self.desc = 'foo'
         self.stripped = False
         self.notnode = name is None
+        self.myargs = (uid, username, privilege_level, name, expiration)
         kvpairs = {'username': {'value': username},
                    'password': {'value': '', 'type': 'password'},
                    'privilege_level': {'value': privilege_level},
@@ -1326,7 +1402,11 @@ class UserCollection(ConfluentMessage):
         self.notnode = name is None
         self.desc = 'list of users'
         userlist = []
+        self.myargs = (userlist, name)
         for user in users:
+            if 'username' in user:  # processing an already translated dict
+                userlist.append(user)
+                continue
             entry = {
                 'uid': user['uid'],
                 'username': user['name'],
@@ -1340,8 +1420,10 @@ class UserCollection(ConfluentMessage):
             self.kvpairs = {name: {'users': userlist}}
 
 
+
 class AlertDestination(ConfluentMessage):
     def __init__(self, ip, acknowledge=False, acknowledge_timeout=None, retries=0, name=None):
+        self.myargs = (ip, acknowledge, acknowledge_timeout, retries, name)
         self.desc = 'foo'
         self.stripped = False
         self.notnode = name is None
@@ -1357,7 +1439,7 @@ class AlertDestination(ConfluentMessage):
 
 class InputAlertDestination(ConfluentMessage):
     valid_alert_params = {
-        'acknowledge': lambda x: False if type(x) in (unicode,str) and x.lower() == 'false' else bool(x),
+        'acknowledge': lambda x: False if type(x) in (unicode, bytes) and x.lower() == 'false' else bool(x),
         'acknowledge_timeout': lambda x: int(x) if x and x.isdigit() else None,
         'ip': lambda x: x,
         'retries': lambda x: int(x)
@@ -1367,6 +1449,8 @@ class InputAlertDestination(ConfluentMessage):
         self.alertcfg = {}
         if multinode:  # keys are node names
             for node in inputdata:
+                if not isinstance(inputdata[node], dict):
+                    break
                 self.alertcfg[node] = inputdata[node]
                 for key in inputdata[node]:
                     if key not in self.valid_alert_params:
@@ -1379,7 +1463,8 @@ class InputAlertDestination(ConfluentMessage):
                     else:
                         self.alertcfg[node][key] = \
                             self.valid_alert_params[key](inputdata[node][key])
-        else:
+            else:
+                return
             for key in inputdata:
                 if key not in self.valid_alert_params:
                     raise exc.InvalidArgumentException(
@@ -1403,7 +1488,11 @@ class SensorReadings(ConfluentMessage):
     def __init__(self, sensors=(), name=None):
         readings = []
         self.notnode = name is None
+        self.myargs = (readings, name)
         for sensor in sensors:
+            if isinstance(sensor, dict):
+                readings.append(sensor)
+                continue
             sensordict = {'name': sensor.name}
             if hasattr(sensor, 'value'):
                 sensordict['value'] = sensor.value
@@ -1428,6 +1517,13 @@ class Firmware(ConfluentMessage):
     readonly = True
 
     def __init__(self, data, name):
+        for datum in data:
+            for component in datum:
+                for field in datum[component]:
+                    tdatum = datum[component]
+                    if isinstance(tdatum[field], datetime):
+                        tdatum[field] = tdatum[field].strftime('%Y-%m-%dT%H:%M:%S')
+        self.myargs = (data, name)
         self.notnode = name is None
         self.desc = 'Firmware information'
         if self.notnode:
@@ -1440,6 +1536,7 @@ class KeyValueData(ConfluentMessage):
     readonly = True
 
     def __init__(self, kvdata, name=None):
+        self.myargs = (kvdata, name)
         self.notnode = name is None
         if self.notnode:
             self.kvpairs = kvdata
@@ -1449,6 +1546,7 @@ class KeyValueData(ConfluentMessage):
 class Array(ConfluentMessage):
     def __init__(self, name, disks=None, raid=None, volumes=None,
                  id=None, capacity=None, available=None):
+        self.myargs = (name, disks, raid, volumes, id, capacity, available)
         self.kvpairs = {
             name: {
                 'type': 'array',
@@ -1463,6 +1561,7 @@ class Array(ConfluentMessage):
 
 class Volume(ConfluentMessage):
     def __init__(self, name, volname, size, state, array, stripsize=None):
+        self.myargs = (name, volname, size, state, array, stripsize)
         self.kvpairs = {
             name: {
                 'type': 'volume',
@@ -1477,12 +1576,15 @@ class Volume(ConfluentMessage):
 
 class Disk(ConfluentMessage):
     valid_states = set([
+        'fault',
         'jbod',
         'unconfigured',
         'hotspare',
+        'rebuilding',
         'online',
     ])
     state_aliases = {
+        'unconfigured bad': 'fault',
         'unconfigured good': 'unconfigured',
         'global hot spare': 'hotspare',
         'dedicated hot spare': 'hotspare',
@@ -1500,6 +1602,8 @@ class Disk(ConfluentMessage):
     def __init__(self, name, label=None, description=None,
                  diskid=None, state=None, serial=None, fru=None,
                  array=None):
+        self.myargs = (name, label, description, diskid, state,
+                       serial, fru, array)
         state = self._normalize_state(state)
         self.kvpairs = {
             name: {
@@ -1521,6 +1625,7 @@ class LEDStatus(ConfluentMessage):
     readonly = True
 
     def __init__(self, data, name):
+        self.myargs = (data, name)
         self.notnode = name is None
         self.desc = 'led status'
 
@@ -1535,6 +1640,7 @@ class NetworkConfiguration(ConfluentMessage):
 
     def __init__(self, name=None, ipv4addr=None, ipv4gateway=None,
                  ipv4cfgmethod=None, hwaddr=None):
+        self.myargs = (name, ipv4addr, ipv4gateway, ipv4cfgmethod, hwaddr)
         self.notnode = name is None
         self.stripped = False
 
@@ -1555,6 +1661,7 @@ class HealthSummary(ConfluentMessage):
     valid_values = valid_health_values
 
     def __init__(self, health, name=None):
+        self.myargs = (health, name)
         self.stripped = False
         self.notnode = name is None
         if health not in self.valid_values:
@@ -1567,11 +1674,12 @@ class HealthSummary(ConfluentMessage):
 
 class Attributes(ConfluentMessage):
     def __init__(self, name=None, kv=None, desc=''):
+        self.myargs = (name, kv, desc)
         self.desc = desc
         nkv = {}
         self.notnode = name is None
         for key in kv:
-            if type(kv[key]) in (str, unicode):
+            if type(kv[key]) in (bytes, unicode):
                 nkv[key] = {'value': kv[key]}
             else:
                 nkv[key] = kv[key]
@@ -1587,6 +1695,7 @@ class ConfigSet(Attributes):
 
 class ListAttributes(ConfluentMessage):
     def __init__(self, name=None, kv=None, desc=''):
+        self.myargs = (name, kv, desc)
         self.desc = desc
         self.notnode = name is None
         if self.notnode:
@@ -1597,6 +1706,7 @@ class ListAttributes(ConfluentMessage):
 
 class MCI(ConfluentMessage):
     def __init__(self, name=None, mci=None):
+        self.myargs = (name, mci)
         self.notnode = name is None
         self.desc = 'BMC identifier'
 
@@ -1609,6 +1719,7 @@ class MCI(ConfluentMessage):
 
 class Hostname(ConfluentMessage):
     def __init__(self, name=None, hostname=None):
+        self.myargs = (name, hostname)
         self.notnode = name is None
         self.desc = 'BMC hostname'
 
@@ -1620,6 +1731,7 @@ class Hostname(ConfluentMessage):
 
 class DomainName(ConfluentMessage):
     def __init__(self, name=None, dn=None):
+        self.myargs = (name, dn)
         self.notnode = name is None
         self.desc = 'BMC domain name'
 
@@ -1634,6 +1746,7 @@ class NTPServers(ConfluentMessage):
     readonly = True
 
     def __init__(self, name=None, servers=None):
+        self.myargs = (name, servers)
         self.notnode = name is None
         self.desc = 'NTP Server'
 
@@ -1648,6 +1761,7 @@ class NTPServers(ConfluentMessage):
 
 class NTPServer(ConfluentMessage):
     def __init__(self, name=None, server=None):
+        self.myargs = (name, server)
         self.notnode = name is None
         self.desc = 'NTP Server'
 
@@ -1664,6 +1778,7 @@ class License(ConfluentMessage):
     readonly = True
 
     def __init__(self, name=None, kvm=None, feature=None, state=None):
+        self.myargs = (name, kvm, feature, state)
         self.notnode = name is None
         self.desc = 'License'
 
@@ -1679,10 +1794,11 @@ class CryptedAttributes(Attributes):
     defaulttype = 'password'
 
     def __init__(self, name=None, kv=None, desc=''):
+        self.myargs = (name, kv, desc)
         # for now, just keep the dictionary keys and discard crypt value
         self.desc = desc
         nkv = {}
-        for key in kv.iterkeys():
+        for key in kv:
             nkv[key] = {'isset': False}
             try:
                 if kv[key] is not None and kv[key]['cryptvalue'] != '':

confluent_server/confluent/neighutil.py

@@ -18,6 +18,7 @@
 # Ultimately, this should use AF_NETLINK, but in the interest of time,
 # use ip neigh for the moment
 
+import confluent.util as util
 import eventlet.green.subprocess as subprocess
 import os
 
@@ -39,6 +40,7 @@ def update_neigh():
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
     (neighdata, err) = ipn.communicate()
+    neighdata = util.stringify(neighdata)
     for entry in neighdata.split('\n'):
         entry = entry.split(' ')
         if len(entry) < 5 or not entry[4]:

confluent_server/confluent/networking/lldp.py

@@ -44,7 +44,7 @@ import eventlet
 from eventlet.greenpool import GreenPool
 import eventlet.semaphore
 import re
-
+webclient = eventlet.import_patched('pyghmi.util.webclient')
 # The interesting OIDs are:
 # lldpLocChassisId - to cross reference (1.0.8802.1.1.2.1.3.2.0)
 # lldpLocPortId - for cross referencing.. (1.0.8802.1.1.2.1.3.7.1.3)
@@ -85,6 +85,7 @@ _neighdata = {}
 _neighbypeerid = {}
 _updatelocks = {}
 _chassisidbyswitch = {}
+_noaffluent = set([])
 
 def lenovoname(idx, desc):
     if desc.isdigit():
@@ -125,7 +126,8 @@ def _dump_neighbordatum(info):
 
 def b64tohex(b64str):
     bd = base64.b64decode(b64str)
-    return ''.join(['{0:02x}'.format(ord(x)) for x in bd])
+    bd = bytearray(bd)
+    return ''.join(['{0:02x}'.format(x) for x in bd])
 
 def get_fingerprint(switch, port, configmanager, portmatch):
     update_switch_data(switch, configmanager)
@@ -170,21 +172,58 @@ def _init_lldp(data, iname, idx, idxtoportid, switch):
         data[iname] = {'port': iname, 'portid': str(idxtoportid[idx]),
                        'chassisid': _chassisidbyswitch[switch]}
 
+def _extract_neighbor_data_affluent(switch, user, password, cfm, lldpdata):
+    kv = util.TLSCertVerifier(cfm, switch,
+                                  'pubkeys.tls_hardwaremanager').verify_cert
+    wc =  webclient.SecureHTTPConnection(
+                switch, 443, verifycallback=kv, timeout=5)
+    wc.set_basic_credentials(user, password)
+    neighdata = wc.grab_json_response('/affluent/lldp/all')
+    chassisid = neighdata['chassis']['id']
+    _chassisidbyswitch[switch] = chassisid,
+    for record in neighdata['neighbors']:
+        localport = record['localport']
+        peerid = '{0}.{1}'.format(
+            record.get('peerchassisid', '').replace(':', '-').replace('/', '-'),
+            record.get('peerportid', '').replace(':', '-').replace('/', '-'),
+        )
+        portdata = {
+            'verified': True,  # It is over TLS after all
+            'peerdescription': record.get('peerdescription', None),
+            'peerchassisid': record['peerchassisid'],
+            'peername': record['peername'],
+            'switch': switch,
+            'chassisid': chassisid,
+            'portid': record['localport'],
+            'peerportid': record['peerportid'],
+            'port': record['localport'],
+            'peerid': peerid,
+        }
+        _neighbypeerid[peerid] = portdata
+        lldpdata[localport] = portdata
+    neighdata[switch] = lldpdata
+
+
 def _extract_neighbor_data_b(args):
     """Build LLDP data about elements connected to switch
 
     args are carried as a tuple, because of eventlet convenience
     """
-    switch, password, user, force = args[:4]
+    switch, password, user, cfm, force = args[:5]
     vintage = _neighdata.get(switch, {}).get('!!vintage', 0)
     now = util.monotonic_time()
     if vintage > (now - 60) and not force:
         return
+    lldpdata = {'!!vintage': now}
+    try:
+        return _extract_neighbor_data_affluent(switch, user, password, cfm, lldpdata)
+    except Exception:
+        pass
     conn = snmp.Session(switch, password, user)
     sid = None
-    lldpdata = {'!!vintage': now}
     for sysid in conn.walk('1.3.6.1.2.1.1.2'):
         sid = str(sysid[1][6:])
+    _noaffluent.add(switch)
     idxtoifname = {}
     idxtoportid = {}
     _chassisidbyswitch[switch] = sanitize(list(
@@ -267,8 +306,8 @@ def _extract_neighbor_data(args):
             return _extract_neighbor_data_b(args)
     except Exception as e:
         yieldexc = False
-        if len(args) >= 5:
-            yieldexc = args[4]
+        if len(args) >= 6:
+            yieldexc = args[5]
         if yieldexc:
             return e
         else:
@@ -357,10 +396,3 @@ def _handle_neighbor_query(pathcomponents, configmanager):
                 raise x
     return list_info(parms, listrequested)
 
-
-def _list_interfaces(switchname, configmanager):
-    switchcreds = get_switchcreds(configmanager, (switchname,))
-    switchcreds = switchcreds[0]
-    conn = snmp.Session(*switchcreds)
-    ifnames = netutil.get_portnamemap(conn)
-    return util.natural_sort(ifnames.values())

confluent_server/confluent/networking/macmap.py

@@ -45,13 +45,16 @@ from eventlet.greenpool import GreenPool
 import eventlet
 import eventlet.semaphore
 import re
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+
+noaffluent = set([])
 
 _macmap = {}
 _apimacmap = {}
 _macsbyswitch = {}
 _nodesbymac = {}
 _switchportmap = {}
-_neighdata = {}
 vintage = None
 
 
@@ -127,6 +130,36 @@ def _nodelookup(switch, ifname):
     return None
 
 
+def _affluent_map_switch(args):
+    switch, password, user, cfm = args
+    kv = util.TLSCertVerifier(cfm, switch,
+                                  'pubkeys.tls_hardwaremanager').verify_cert
+    wc =  webclient.SecureHTTPConnection(
+                switch, 443, verifycallback=kv, timeout=5)
+    wc.set_basic_credentials(user, password)
+    macs = wc.grab_json_response('/affluent/macs/by-port')
+    _macsbyswitch[switch] = macs
+
+    for iface in macs:
+        nummacs = len(macs[iface])
+        for mac in macs[iface]:
+            if mac in _macmap:
+                _macmap[mac].append((switch, iface, nummacs))
+            else:
+                _macmap[mac] = [(switch, iface, nummacs)]
+            nodename = _nodelookup(switch, iface)
+            if nodename is not None:
+                if mac in _nodesbymac and _nodesbymac[mac][0] != nodename:
+                    # For example, listed on both a real edge port
+                    # and by accident a trunk port
+                    log.log({'error': '{0} and {1} described by ambiguous'
+                                    ' switch topology values'.format(
+                                        nodename, _nodesbymac[mac][0])})
+                    _nodesbymac[mac] = (None, None)
+                else:
+                    _nodesbymac[mac] = (nodename, nummacs)
+
+
 def _map_switch_backend(args):
     """Manipulate portions of mac address map relevant to a given switch
     """
@@ -144,13 +177,18 @@ def _map_switch_backend(args):
     #   fallback if ifName is empty
     #
     global _macmap
-    if len(args) == 3:
-        switch, password, user = args
+    if len(args) == 4:
+        switch, password, user, cfm = args
         if not user:
             user = None
     else:
         switch, password = args
         user = None
+    if switch not in noaffluent:
+        try:
+            return _affluent_map_switch(args)
+        except Exception:
+            pass
     haveqbridge = False
     mactobridge = {}
     conn = snmp.Session(switch, password, user)
@@ -164,6 +202,7 @@ def _map_switch_backend(args):
             *([int(x) for x in oid[-6:]])
         )
         mactobridge[macaddr] = int(bridgeport)
+    noaffluent.add(switch)
     if not haveqbridge:
         for vb in conn.walk('1.3.6.1.2.1.17.4.3.1.2'):
             oid, bridgeport = vb

confluent_server/confluent/networking/netutil.py

@@ -36,7 +36,7 @@ def get_switchcreds(configmanager, switches):
                 'secret.hardwaremanagementuser', {}).get('value', None)
             if not user:
                 user = None
-        switchauth.append((switch, password, user))
+        switchauth.append((switch, password, user, configmanager))
     return switchauth
 
 

confluent_server/confluent/noderange.py

@@ -25,6 +25,11 @@ import itertools
 import pyparsing as pp
 import re
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 # construct custom grammar with pyparsing
 _nodeword = pp.Word(pp.alphanums + '~^$/=-_:.*+!')
 _nodebracket = pp.QuotedString(quoteChar='[', endQuoteChar=']',
@@ -157,7 +162,7 @@ class NodeRange(object):
         pieces = seqrange.split(delimiter)
         if len(pieces) % 2 != 0:
             return self.failorreturn(seqrange)
-        halflen = len(pieces) / 2
+        halflen = len(pieces) // 2
         left = delimiter.join(pieces[:halflen])
         right = delimiter.join(pieces[halflen:])
         leftbits = _numextractor.parseString(left).asList()
@@ -166,7 +171,7 @@ class NodeRange(object):
             return self.failorreturn(seqrange)
         finalfmt = ''
         iterators = []
-        for idx in xrange(len(leftbits)):
+        for idx in range(len(leftbits)):
             if leftbits[idx] == rightbits[idx]:
                 finalfmt += leftbits[idx]
             elif leftbits[idx][0] in pp.alphas:
@@ -181,7 +186,7 @@ class NodeRange(object):
                 if leftnum > rightnum:
                     width = len(rightbits[idx])
                     minnum = rightnum
-                    maxnum = leftnum + 1  # xrange goes to n-1...
+                    maxnum = leftnum + 1  # range goes to n-1...
                 elif rightnum > leftnum:
                     width = len(leftbits[idx])
                     minnum = leftnum
@@ -189,7 +194,7 @@ class NodeRange(object):
                 else:  # differently padded, but same number...
                     return self.failorreturn(seqrange)
                 numformat = '{0:0%d}' % width
-                for num in xrange(minnum, maxnum):
+                for num in range(minnum, maxnum):
                     curseq.append(numformat.format(num))
         results = set([])
         for combo in itertools.product(*iterators):
@@ -222,7 +227,7 @@ class NodeRange(object):
             if self.cfm is None:
                 raise Exception('Verification configmanager required')
             return set(self.cfm.filter_node_attributes(element, filternodes))
-        for idx in xrange(len(element)):
+        for idx in range(len(element)):
             if element[idx][0] == '[':
                 nodes = set([])
                 for numeric in NodeRange(element[idx][1:-1]).nodes:
@@ -245,6 +250,8 @@ class NodeRange(object):
                 return nodes
         if ':' in element:  # : range for less ambiguity
             return self.expandrange(element, ':')
+        elif '..' in element:
+            return self.expandrange(element, '..')
         elif '-' in element:
             return self.expandrange(element, '-')
         elif '+' in element:

confluent_server/confluent/pam.py

@@ -0,0 +1,235 @@
+# Pulled from:
+# https://raw.githubusercontent.com/FirefighterBlu3/python-pam/fe44b334970f421635d9e373b563c9e6566613bd/pam.py
+# and https://github.com/FirefighterBlu3/python-pam/pull/16/files
+# (c) 2007 Chris AtLee <chris@atlee.ca>
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license.php
+#
+# Original author: Chris AtLee
+#
+# Modified by David Ford, 2011-12-6
+# added py3 support and encoding
+# added pam_end
+# added pam_setcred to reset credentials after seeing Leon Walker's remarks
+# added byref as well
+# use readline to prestuff the getuser input
+
+'''
+PAM module for python
+
+Provides an authenticate function that will allow the caller to authenticate
+a user against the Pluggable Authentication Modules (PAM) on the system.
+
+Implemented using ctypes, so no compilation is necessary.
+'''
+
+__all__      = ['pam']
+__version__  = '1.8.4'
+__author__   = 'David Ford <david@blue-labs.org>'
+__released__ = '2018 June 15'
+
+import sys
+
+from ctypes import CDLL, POINTER, Structure, CFUNCTYPE, cast, byref, sizeof
+from ctypes import c_void_p, c_size_t, c_char_p, c_char, c_int
+from ctypes import memmove
+from ctypes.util import find_library
+
+class PamHandle(Structure):
+    """wrapper class for pam_handle_t pointer"""
+    _fields_ = [ ("handle", c_void_p) ]
+
+    def __init__(self):
+        Structure.__init__(self)
+        self.handle = 0
+
+class PamMessage(Structure):
+    """wrapper class for pam_message structure"""
+    _fields_ = [ ("msg_style", c_int), ("msg", c_char_p) ]
+
+    def __repr__(self):
+        return "<PamMessage %i '%s'>" % (self.msg_style, self.msg)
+
+class PamResponse(Structure):
+    """wrapper class for pam_response structure"""
+    _fields_ = [ ("resp", c_char_p), ("resp_retcode", c_int) ]
+
+    def __repr__(self):
+        return "<PamResponse %i '%s'>" % (self.resp_retcode, self.resp)
+
+conv_func = CFUNCTYPE(c_int, c_int, POINTER(POINTER(PamMessage)), POINTER(POINTER(PamResponse)), c_void_p)
+
+class PamConv(Structure):
+    """wrapper class for pam_conv structure"""
+    _fields_ = [ ("conv", conv_func), ("appdata_ptr", c_void_p) ]
+
+# Various constants
+PAM_PROMPT_ECHO_OFF       = 1
+PAM_PROMPT_ECHO_ON        = 2
+PAM_ERROR_MSG             = 3
+PAM_TEXT_INFO             = 4
+PAM_REINITIALIZE_CRED     = 8
+
+libc                      = CDLL(find_library("c"))
+libpam                    = CDLL(find_library("pam"))
+
+calloc                    = libc.calloc
+calloc.restype            = c_void_p
+calloc.argtypes           = [c_size_t, c_size_t]
+
+# bug #6 (@NIPE-SYSTEMS), some libpam versions don't include this function
+if hasattr(libpam, 'pam_end'):
+    pam_end                   = libpam.pam_end
+    pam_end.restype           = c_int
+    pam_end.argtypes          = [PamHandle, c_int]
+
+pam_start                 = libpam.pam_start
+pam_start.restype         = c_int
+pam_start.argtypes        = [c_char_p, c_char_p, POINTER(PamConv), POINTER(PamHandle)]
+
+pam_acct_mgmt             = libpam.pam_acct_mgmt
+pam_acct_mgmt.restype     = c_int
+pam_acct_mgmt.argtypes    = [PamHandle, c_int]
+
+pam_setcred               = libpam.pam_setcred
+pam_setcred.restype       = c_int
+pam_setcred.argtypes      = [PamHandle, c_int]
+
+pam_strerror              = libpam.pam_strerror
+pam_strerror.restype      = c_char_p
+pam_strerror.argtypes     = [PamHandle, c_int]
+
+pam_authenticate          = libpam.pam_authenticate
+pam_authenticate.restype  = c_int
+pam_authenticate.argtypes = [PamHandle, c_int]
+
+class pam():
+    code   = 0
+    reason = None
+
+    def __init__(self):
+        pass
+
+    def authenticate(self, username, password, service='login', encoding='utf-8', resetcreds=True):
+        """username and password authentication for the given service.
+
+           Returns True for success, or False for failure.
+
+           self.code (integer) and self.reason (string) are always stored and may
+           be referenced for the reason why authentication failed. 0/'Success' will
+           be stored for success.
+
+           Python3 expects bytes() for ctypes inputs.  This function will make
+           necessary conversions using the supplied encoding.
+
+        Inputs:
+          username: username to authenticate
+          password: password in plain text
+          service:  PAM service to authenticate against, defaults to 'login'
+
+        Returns:
+          success:  True
+          failure:  False
+        """
+
+        @conv_func
+        def my_conv(n_messages, messages, p_response, app_data):
+            """Simple conversation function that responds to any
+               prompt where the echo is off with the supplied password"""
+            # Create an array of n_messages response objects
+            addr = calloc(n_messages, sizeof(PamResponse))
+            response = cast(addr, POINTER(PamResponse))
+            p_response[0] = response
+            for i in range(n_messages):
+                if messages[i].contents.msg_style == PAM_PROMPT_ECHO_OFF:
+                    dst = calloc(len(password)+1, sizeof(c_char))
+                    memmove(dst, cpassword, len(password))
+                    response[i].resp = dst
+                    response[i].resp_retcode = 0
+            return 0
+
+        # python3 ctypes prefers bytes
+        if sys.version_info >= (3,):
+            if isinstance(username, str): username = username.encode(encoding)
+            if isinstance(password, str): password = password.encode(encoding)
+            if isinstance(service, str):  service  = service.encode(encoding)
+        else:
+            if isinstance(username, unicode):
+                username = username.encode(encoding)
+            if isinstance(password, unicode):
+                password = password.encode(encoding)
+            if isinstance(service, unicode):
+                service  = service.encode(encoding)
+
+        if b'\x00' in username or b'\x00' in password or b'\x00' in service:
+            self.code = 4  # PAM_SYSTEM_ERR in Linux-PAM
+            self.reason = 'strings may not contain NUL'
+            return False
+
+        # do this up front so we can safely throw an exception if there's
+        # anything wrong with it
+        cpassword = c_char_p(password)
+
+        handle = PamHandle()
+        conv   = PamConv(my_conv, 0)
+        retval = pam_start(service, username, byref(conv), byref(handle))
+
+        if retval != 0:
+            # This is not an authentication error, something has gone wrong starting up PAM
+            self.code   = retval
+            self.reason = "pam_start() failed"
+            return False
+
+        retval = pam_authenticate(handle, 0)
+        auth_success = retval == 0
+
+        if auth_success:
+            retval = pam_acct_mgmt(handle, 0)
+            auth_success = retval == 0
+
+        if auth_success and resetcreds:
+            retval = pam_setcred(handle, PAM_REINITIALIZE_CRED)
+
+        # store information to inform the caller why we failed
+        self.code   = retval
+        self.reason = pam_strerror(handle, retval)
+        if sys.version_info >= (3,):
+            self.reason = self.reason.decode(encoding)
+
+        if hasattr(libpam, 'pam_end'):
+            pam_end(handle, retval)
+
+        return auth_success
+
+
+def authenticate(*vargs, **dargs):
+    """
+    Compatibility function for older versions of python-pam.
+    """
+    return pam().authenticate(*vargs, **dargs)
+
+
+if __name__ == "__main__":
+    import readline, getpass
+
+    def input_with_prefill(prompt, text):
+        def hook():
+            readline.insert_text(text)
+            readline.redisplay()
+        readline.set_pre_input_hook(hook)
+
+        if sys.version_info >= (3,):
+            result = input(prompt)
+        else:
+            result = raw_input(prompt)
+
+        readline.set_pre_input_hook()
+        return result
+
+    pam = pam()
+
+    username = input_with_prefill('Username: ', getpass.getuser())
+
+    # enter a valid username and an invalid/valid password, to verify both failure and success
+    pam.authenticate(username, getpass.getpass())
+    print('{} {}'.format(pam.code, pam.reason))

confluent_server/confluent/plugins/configuration/attributes.py

@@ -134,7 +134,7 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                             attribute, {}).get('description', ''))
     elif element[-1] == 'current':
         for node in util.natural_sort(list(attributes)):
-            for attribute in sorted(attributes[node].iterkeys()):
+            for attribute in sorted(attributes[node]):
                 currattr = attributes[node][attribute]
                 try:
                     desc = allattributes.node[attribute]['description']
@@ -164,6 +164,20 @@ def update(nodes, element, configmanager, inputdata):
 
 
 def update_nodegroup(group, element, configmanager, inputdata):
+    if element == 'check':
+        check = inputdata.attribs
+        decrypt = configmanager.decrypt
+        configmanager.decrypt = True
+        currinfo = configmanager.get_nodegroup_attributes(group, list(check))
+        configmanager.decrypt = decrypt
+        for inf in check:
+            checkvalue = check[inf]
+            if isinstance(checkvalue, dict):
+                checkvalue = checkvalue.get('value', None)
+            currvalue = currinfo.get(inf, {}).get('value')
+            if checkvalue == currvalue:
+                raise exc.InvalidArgumentException('Checked value matches existing value')
+        return retrieve_nodegroup(group, element, configmanager, inputdata)
     if 'rename' in element:
         namemap = {}
         namemap[group] = inputdata.attribs['rename']
@@ -171,7 +185,7 @@ def update_nodegroup(group, element, configmanager, inputdata):
         return yield_rename_resources(namemap, isnode=False)
     try:
         clearattribs = []
-        for attrib in inputdata.attribs.iterkeys():
+        for attrib in inputdata.attribs:
             if inputdata.attribs[attrib] is None:
                 clearattribs.append(attrib)
         for attrib in clearattribs:
@@ -221,6 +235,18 @@ def update_nodes(nodes, element, configmanager, inputdata):
         raise exc.InvalidArgumentException(
             'No action to take, noderange is empty (if trying to define '
             'group attributes, use nodegroupattrib)')
+    if element[-1] == 'check':
+        for node in nodes:
+            check = inputdata.get_attributes(node, allattributes.node)
+            currinfo = configmanager.get_node_attributes(node, list(check), decrypt=True)
+            for inf in check:
+                checkvalue = check[inf]
+                if isinstance(checkvalue, dict):
+                    checkvalue = checkvalue.get('value', None)
+                currvalue = currinfo.get(node, {}).get(inf, {}).get('value')
+                if checkvalue == currvalue:
+                    raise exc.InvalidArgumentException('Checked value matches existing value')
+        return retrieve(nodes, element, configmanager, inputdata)
     if 'rename' in element:
         namemap = {}
         for node in nodes:

confluent_server/confluent/plugins/console/tsmsol.py

@@ -0,0 +1,160 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This plugin provides an ssh implementation comforming to the 'console'
+# specification.  consoleserver or shellserver would be equally likely
+# to use this.
+
+import confluent.exceptions as cexc
+import confluent.interface.console as conapi
+import confluent.log as log
+import confluent.util as util
+import pyghmi.exceptions as pygexc
+import pyghmi.redfish.command as rcmd
+import eventlet
+import eventlet.green.ssl as ssl
+try:
+    websocket = eventlet.import_patched('websocket')
+    wso = websocket.WebSocket
+except Exception:
+    wso = object
+
+def get_conn_params(node, configdata):
+    if 'secret.hardwaremanagementuser' in configdata:
+        username = configdata['secret.hardwaremanagementuser']['value']
+    else:
+        username = 'USERID'
+    if 'secret.hardwaremanagementpassword' in configdata:
+        passphrase = configdata['secret.hardwaremanagementpassword']['value']
+    else:
+        passphrase = 'PASSW0RD'  # for lack of a better guess
+    if 'hardwaremanagement.manager' in configdata:
+        bmc = configdata['hardwaremanagement.manager']['value']
+    else:
+        bmc = node
+    return {
+        'username': username,
+        'passphrase': passphrase,
+        'bmc': bmc,
+    }
+_configattributes = ('secret.hardwaremanagementuser',
+                     'secret.hardwaremanagementpassword',
+                     'hardwaremanagement.manager')
+
+class WrappedWebSocket(wso):
+
+    def set_verify_callback(self, callback):
+        self._certverify = callback
+
+    def connect(self, url, **options):
+        add_tls = url.startswith('wss://')
+        if add_tls:
+            hostname, port, resource, _ = websocket._url.parse_url(url)
+            if hostname[0] != '[' and ':' in hostname:
+                hostname = '[{0}]'.format(hostname)
+            if resource[0] != '/':
+                resource = '/{0}'.format(resource)
+            url = 'ws://{0}:443{1}'.format(hostname,resource)
+        else:
+            return super(WrappedWebSocket, self).connect(url, **options)
+        self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout)
+        self.sock, addrs = websocket._http.connect(url, self.sock_opt, websocket._http.proxy_info(**options),
+                                           options.pop('socket', None))
+        self.sock = ssl.wrap_socket(self.sock, cert_reqs=ssl.CERT_NONE)
+        # The above is supersedeed by the _certverify, which provides
+        # known-hosts style cert validaiton
+        bincert = self.sock.getpeercert(binary_form=True)
+        if not self._certverify(bincert):
+            raise pygexc.UnrecognizedCertificate('Unknown certificate', bincert)
+        try:
+            self.handshake_response = websocket._handshake.handshake(self.sock, *addrs, **options)
+            if self.handshake_response.status in websocket._handshake.SUPPORTED_REDIRECT_STATUSES:
+                options['redirect_limit'] = options.pop('redirect_limit', 3) - 1
+                if options['redirect_limit'] < 0:
+                     raise Exception('Redirect limit hit')
+                url = self.handshake_response.headers['location']
+                self.sock.close()
+                return self.connect(url, **options)
+            self.connected = True
+        except:
+            if self.sock:
+                self.sock.close()
+                self.sock = None
+            raise
+
+
+         
+
+
+
+class TsmConsole(conapi.Console):
+
+    def __init__(self, node, config):
+        self.node = node
+        self.ws = None
+        configdata = config.get_node_attributes([node], _configattributes, decrypt=True)
+        connparams = get_conn_params(node, configdata[node])
+        self.username = connparams['username']
+        self.password = connparams['passphrase']
+        self.bmc = connparams['bmc']
+        self.origbmc = connparams['bmc']
+        if ':' in self.bmc:
+            self.bmc = '[{0}]'.format(self.bmc)
+        self.datacallback = None
+        self.nodeconfig = config
+        self.connected = False
+
+
+    def recvdata(self):
+        while self.connected:
+            pendingdata = self.ws.recv()
+            if pendingdata == '':
+                self.datacallback(conapi.ConsoleEvent.Disconnect)
+                return
+            self.datacallback(pendingdata)
+
+    def connect(self, callback):
+        self.datacallback = callback
+        rc = rcmd.Command(self.origbmc, self.username,
+                          self.password,
+                          verifycallback=lambda x: True)
+        wc = rc.oem.wc
+        bmc = self.bmc
+        if '%' in self.bmc:
+            prefix = self.bmc.split('%')[0]
+            bmc = prefix + ']'
+        self.ws = WrappedWebSocket(host=bmc)
+        kv = util.TLSCertVerifier(
+            self.nodeconfig, self.node, 'pubkeys.tls_hardwaremanager').verify_cert
+        self.ws.set_verify_callback(kv)
+        self.ws.connect('wss://{0}/sol?CSRFTOKEN={1}'.format(self.bmc, rc.oem.csrftok), host=bmc, cookie='QSESSIONID={0}'.format(wc.cookies['QSESSIONID']))
+        self.connected = True
+        eventlet.spawn_n(self.recvdata)
+        return
+
+    def write(self, data):
+        self.ws.send(data)
+
+    def close(self):
+        if self.ws:
+            self.ws.close()
+        self.connected = False
+        self.datacallback = None
+
+def create(nodes, element, configmanager, inputdata):
+    if len(nodes) == 1:
+        return TsmConsole(nodes[0], configmanager)

confluent_server/confluent/plugins/hardwaremanagement/affluent.py

@@ -0,0 +1,153 @@
+
+# Copyright 2019-2020 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import eventlet
+import eventlet.queue as queue
+import confluent.exceptions as exc
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import confluent.messages as msg
+import confluent.util as util
+
+class SwitchSensor(object):
+    def __init__(self, name, states, value=None, health=None):
+        self.name = name
+        self.value = value
+        self.states = states
+        self.health = health
+
+class WebClient(object):
+    def __init__(self, node, configmanager, creds):
+        self.node = node
+        self.wc = webclient.SecureHTTPConnection(node, port=443, verifycallback=util.TLSCertVerifier(
+            configmanager, node, 'pubkeys.tls_hardwaremanager').verify_cert)
+        self.wc.set_basic_credentials(creds[node]['secret.hardwaremanagementuser']['value'], creds[node]['secret.hardwaremanagementpassword']['value'])
+    
+    def fetch(self, url, results):
+        rsp, status = self.wc.grab_json_response_with_status(url)
+        if status == 401:
+            results.put(msg.ConfluentTargetInvalidCredentials(self.node, 'Unable to authenticate'))
+            return {}
+        elif status != 200:
+            results.put(msg.ConfluentNodeError(self.node, 'Unknown error: ' + rsp + ' while retrieving ' + url))
+            return {}
+        return rsp
+    
+
+def update(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+
+def delete(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+
+def create(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+
+def _run_method(method, workers, results, configmanager, nodes, element):
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(method, configmanager, creds,
+                                       node, results, element))
+
+def retrieve(nodes, element, configmanager, inputdata):
+    results = queue.LightQueue()
+    workers = set([])
+    if element == ['power', 'state']:
+        for node in nodes:
+            yield msg.PowerState(node=node, state='on')
+        return
+    elif element == ['health', 'hardware']:
+        _run_method(retrieve_health, workers, results, configmanager, nodes, element)
+    elif element[:3] == ['inventory', 'hardware', 'all']:
+        _run_method(retrieve_inventory, workers, results, configmanager, nodes, element)
+    elif element[:3] == ['inventory', 'firmware', 'all']:
+        _run_method(retrieve_firmware, workers, results, configmanager, nodes, element)
+    elif element == ['sensors', 'hardware', 'all']:
+        _run_method(list_sensors, workers, results, configmanager, nodes, element)
+    elif element[:3] == ['sensors', 'hardware', 'all']:
+        _run_method(retrieve_sensors, workers, results, configmanager, nodes, element)
+    else:
+        for node in nodes:
+            yield msg.ConfluentNodeError(node, 'Not Implemented')
+        return
+    while workers:
+        try:
+            datum = results.get(10)
+            while datum:
+                if datum:
+                    yield datum
+                datum = results.get_nowait()
+        except queue.Empty:
+            pass
+        eventlet.sleep(0.001)
+        for t in list(workers):
+            if t.dead:
+                workers.discard(t)
+    try:
+        while True:
+            datum = results.get_nowait()
+            if datum:
+                yield datum
+    except queue.Empty:
+        pass
+
+
+def retrieve_inventory(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        results.put(msg.ChildCollection('system'))
+        return
+    wc = WebClient(node, configmanager, creds)
+    invinfo = wc.fetch('/affluent/inventory/hardware/all', results)
+    if invinfo:
+        results.put(msg.KeyValueData(invinfo, node))
+
+
+def retrieve_firmware(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        return
+    wc = WebClient(node, configmanager, creds)
+    fwinfo = wc.fetch('/affluent/inventory/firmware/all', results)
+    if fwinfo:
+        results.put(msg.Firmware(fwinfo, node))
+
+def list_sensors(configmanager, creds, node, results, element):
+    wc = WebClient(node, configmanager, creds)
+    sensors = wc.fetch('/affluent/sensors/hardware/all', results)
+    for sensor in sensors['item']:
+        results.put(msg.ChildCollection(sensor))
+
+def retrieve_sensors(configmanager, creds, node, results, element):
+    wc = WebClient(node, configmanager, creds)
+    sensors = wc.fetch('/affluent/sensors/hardware/all/{0}'.format(element[-1]), results)
+    if sensors:
+        results.put(msg.SensorReadings(sensors['sensors'], node))
+
+
+
+def retrieve_health(configmanager, creds, node, results, element):
+    wc = WebClient(node, configmanager, creds)
+    hinfo = wc.fetch('/affluent/health', results) 
+    if hinfo:
+        results.put(msg.HealthSummary(hinfo.get('health', 'unknown'), name=node))
+        results.put(msg.SensorReadings(hinfo.get('sensors', []), name=node))

confluent_server/confluent/plugins/hardwaremanagement/cnos.py

@@ -78,6 +78,18 @@ def retrieve(nodes, element, configmanager, inputdata):
         for node in nodes:
             workers.add(eventlet.spawn(retrieve_health, configmanager, creds,
                                        node, results))
+    elif element[:3] == ['inventory', 'hardware', 'all']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_inventory, configmanager,
+                                       creds, node, results, element))
+    elif element[:3] == ['inventory', 'firmware', 'all']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_firmware, configmanager,
+                                       creds, node, results, element))
     else:
         for node in nodes:
             yield msg.ConfluentNodeError(node, 'Not Implemented')
@@ -105,6 +117,44 @@ def retrieve(nodes, element, configmanager, inputdata):
         pass
 
 
+def retrieve_inventory(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        results.put(msg.ChildCollection('system'))
+        return
+    wc = cnos_login(node, configmanager, creds)
+    sysinfo = wc.grab_json_response('/nos/api/sysinfo/inventory')
+    invinfo = {
+        'inventory': [{
+            'name': 'System',
+            'present': True,
+            'information': {
+                'Product name': sysinfo['Model'],
+                'Serial Number': sysinfo['Electronic Serial Number'],
+                'Board Serial Number': sysinfo['Serial Number'],
+                'Manufacturer': 'Lenovo',
+                'Model': sysinfo['Machine Type Model'],
+                'FRU Number': sysinfo['FRU'].strip(),
+            }
+        }]
+    }
+    results.put(msg.KeyValueData(invinfo, node))
+
+
+def retrieve_firmware(configmanager, creds, node, results, element):
+    if len(element) == 3:
+        results.put(msg.ChildCollection('all'))
+        return
+    wc = cnos_login(node, configmanager, creds)
+    sysinfo = wc.grab_json_response('/nos/api/sysinfo/inventory')
+    items = [{
+        'Software': {'version': sysinfo['Software Revision']},
+        },
+        {
+        'BIOS': {'version': sysinfo['BIOS Revision']},
+        }]
+    results.put(msg.Firmware(items, node))
+
 def retrieve_health(configmanager, creds, node, results):
     wc = cnos_login(node, configmanager, creds)
     hinfo = wc.grab_json_response('/nos/api/sysinfo/globalhealthstatus')

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -29,6 +29,7 @@ import eventlet.queue as queue
 import eventlet.support.greendns
 from fnmatch import fnmatch
 import os
+import pwd
 import pyghmi.constants as pygconstants
 import pyghmi.exceptions as pygexc
 import pyghmi.storage as storage
@@ -42,6 +43,11 @@ import traceback
 if not hasattr(ssl, 'SSLEOFError'):
     ssl.SSLEOFError = None
 
+try:
+    range = xrange
+except NameError:
+    pass
+
 pci_cache = {}
 
 def get_dns_txt(qstring):
@@ -131,7 +137,7 @@ def hex2bin(hexstring):
     if len(hexvals) < 2:
         hexvals = hexstring.split(' ')
     if len(hexvals) < 2:
-        hexvals = [hexstring[i:i+2] for i in xrange(0, len(hexstring), 2)]
+        hexvals = [hexstring[i:i+2] for i in range(0, len(hexstring), 2)]
     bytedata = [int(i, 16) for i in hexvals]
     return bytearray(bytedata)
 
@@ -278,6 +284,7 @@ def _donothing(data):
 
 class IpmiConsole(conapi.Console):
     configattributes = frozenset(_configattributes)
+    bmctonodemapping = {}
 
     def __init__(self, node, config):
         self.error = None
@@ -295,10 +302,21 @@ class IpmiConsole(conapi.Console):
         self.bmc = connparams['bmc']
         self.port = connparams['port']
         self.connected = False
+        # ok, is self.bmc unique among nodes already
         # Cannot actually create console until 'connect', when we get callback
+        if (self.bmc in self.bmctonodemapping and
+                self.bmctonodemapping[self.bmc] != node):
+            raise Exception(
+                "Duplicate hardwaremanagement.manager attribute for {0} and {1}".format(
+                    node, self.bmctonodemapping[self.bmc]))
+        self.bmctonodemapping[self.bmc] = node
 
     def __del__(self):
         self.solconnection = None
+        try:
+            del self.bmctonodemapping[self.bmc]
+        except KeyError:
+            pass
 
     def handle_data(self, data):
         if type(data) == dict:
@@ -383,7 +401,7 @@ def perform_requests(operator, nodes, element, cfg, inputdata, realop):
                         raise datum
                     if (hasattr(datum, 'kvpairs') and datum.kvpairs and
                             len(datum.kvpairs) == 1):
-                        bundle.append((datum.kvpairs.keys()[0], datum))
+                        bundle.append((list(datum.kvpairs)[0], datum))
                         numnodes -= 1
                     else:
                         yield datum
@@ -491,8 +509,8 @@ class IpmiHandler(object):
                 #    raise exc.TargetEndpointUnreachable(
                 #        "Login process to " + connparams['bmc'] + " died")
             except socket.gaierror as ge:
-                if ge[0] == -2:
-                    raise exc.TargetEndpointUnreachable(ge[1])
+                if ge.errno == -2:
+                    raise exc.TargetEndpointUnreachable(ge.strerror)
                 raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
 
@@ -584,9 +602,13 @@ class IpmiHandler(object):
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
+    def get_diags(self, savefile, progress):
+        return self.ipmicmd.get_diagnostic_data(
+            savefile, progress=progress, autosuffix=True)
+
     def handle_servicedata_fetch(self):
         u = firmwaremanager.Updater(
-            self.node, self.ipmicmd.get_diagnostic_data,
+            self.node, self.get_diags,
             self.inputdata.nodefile(self.node), self.tenant, type='ffdc',
             owner=self.current_user)
         self.output.put(msg.CreatedResource(
@@ -623,12 +645,20 @@ class IpmiHandler(object):
             return self.handle_domain_name()
         elif self.element[1:3] == ['management_controller', 'ntp']:
             return self.handle_ntp()
+        elif self.element[1:4] == ['management_controller', 'extended', 'all']:
+            return self.handle_bmcconfig()
+        elif self.element[1:4] == ['management_controller', 'extended', 'advanced']:
+            return self.handle_bmcconfig(True)
+        elif self.element[1:4] == ['management_controller', 'extended', 'extra']:
+            return self.handle_bmcconfig(True, extended=True)
         elif self.element[1:3] == ['system', 'all']:
             return self.handle_sysconfig()
         elif self.element[1:3] == ['system', 'advanced']:
             return self.handle_sysconfig(True)
         elif self.element[1:3] == ['system', 'clear']:
             return self.handle_sysconfigclear()
+        elif self.element[1:3] == ['management_controller', 'clear']:
+            return self.handle_bmcconfigclear()
         elif self.element[1:3] == ['management_controller', 'licenses']:
             return self.handle_licenses()
         elif self.element[1:3] == ['management_controller', 'save_licenses']:
@@ -652,7 +682,7 @@ class IpmiHandler(object):
             if len(self.element) == 4:
                 # A list of destinations
                 maxdest = self.ipmicmd.get_alert_destination_count()
-                for alertidx in xrange(0, maxdest + 1):
+                for alertidx in range(0, maxdest + 1):
                     self.output.put(msg.ChildCollection(alertidx))
                 return
             elif len(self.element) == 5:
@@ -826,6 +856,10 @@ class IpmiHandler(object):
                     raise
                 if hasattr(reading, 'health'):
                     reading.health = _str_health(reading.health)
+                if hasattr(reading, 'unavailable') and reading.unavailable:
+                    self.output.put(msg.SensorReadings([EmptySensor(
+                        reading.name)], name=self.node))
+                    continue
                 readings.append(reading)
             self.output.put(msg.SensorReadings(readings, name=self.node))
         else:
@@ -841,9 +875,13 @@ class IpmiHandler(object):
                     self.ipmicmd.sensormap[sensorname])
                 if hasattr(reading, 'health'):
                     reading.health = _str_health(reading.health)
-                self.output.put(
-                    msg.SensorReadings([reading],
-                                       name=self.node))
+                if hasattr(reading, 'unavailable') and reading.unavailable:
+                    self.output.put(msg.SensorReadings([EmptySensor(
+                        reading.name)], name=self.node))
+                else:
+                    self.output.put(
+                        msg.SensorReadings([reading],
+                                           name=self.node))
             except pygexc.IpmiException as ie:
                 if ie.ipmicode == 203:
                     self.output.put(msg.ConfluentResourceUnavailable(
@@ -1296,6 +1334,9 @@ class IpmiHandler(object):
     def identify(self):
         if 'update' == self.op:
             identifystate = self.inputdata.inputbynode[self.node] == 'on'
+            if self.inputdata.inputbynode[self.node] == 'blink':
+                raise exc.InvalidArgumentException(
+                    '"blink" is not supported with ipmi')
             self.ipmicmd.set_identify(on=identifystate)
             self.output.put(msg.IdentifyState(
                 node=self.node, state=self.inputdata.inputbynode[self.node]))
@@ -1370,12 +1411,33 @@ class IpmiHandler(object):
             self.ipmicmd.set_domain_name(dn)
             return
 
+    def handle_bmcconfigclear(self):
+        if 'read' == self.op:
+            raise exc.InvalidArgumentException(
+                'Cannot read the "clear" resource')
+        self.ipmicmd.clear_bmc_configuration()
+
     def handle_sysconfigclear(self):
         if 'read' == self.op:
             raise exc.InvalidArgumentException(
                 'Cannot read the "clear" resource')
         self.ipmicmd.clear_system_configuration()
 
+    def handle_bmcconfig(self, advanced=False, extended=False):
+        if 'read' == self.op:
+            try:
+                if extended:
+                    bmccfg = self.ipmicmd.get_extended_bmc_configuration()
+                else:
+                    bmccfg = self.ipmicmd.get_bmc_configuration()
+                self.output.put(msg.ConfigSet(self.node, bmccfg))
+            except Exception as e:
+                self.output.put(
+                    msg.ConfluentNodeError(self.node, str(e)))
+        elif 'update' == self.op:
+            self.ipmicmd.set_bmc_configuration(
+                self.inputdata.get_attributes(self.node))
+
     def handle_sysconfig(self, advanced=False):
         if 'read' == self.op:
             try:
@@ -1441,14 +1503,33 @@ class IpmiHandler(object):
 
     def save_licenses(self):
         directory = self.inputdata.nodefile(self.node)
+        checkdir = directory
+        if not os.access(directory, os.W_OK):
+            raise exc.InvalidArgumentException(
+                'The confluent system user/group is unable to write to '
+                'directory {0}, check ownership and permissions'.format(
+                    checkdir))
         for saved in self.ipmicmd.save_licenses(directory):
+            try:
+                pwent = pwd.getpwnam(self.current_user)
+                os.chown(saved, pwent.pw_uid, pwent.pw_gid)
+            except KeyError:
+                pass
             self.output.put(msg.SavedFile(self.node, saved))
 
     def handle_licenses(self):
         if self.element[-1] == '':
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
-            self.ipmicmd.apply_license(self.inputdata.nodefile(self.node))
+            filename = self.inputdata.nodefile(self.node)
+            if not os.access(filename, os.R_OK):
+                errstr =  ('{0} is not readable by confluent on {1} '
+                           '(ensure confluent user or group can access file '
+                           'and parent directories)').format(
+                               filename, socket.gethostname())
+                self.output.put(msg.ConfluentNodeError(self.node, errstr))
+                return
+            self.ipmicmd.apply_license(filename)
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
             i = 1

confluent_server/confluent/plugins/hardwaremanagement/redfish.py

@@ -1,5 +1,5 @@
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import eventlet.queue as queue
 import eventlet.support.greendns
 from fnmatch import fnmatch
 import os
+import pwd
 import pyghmi.constants as pygconstants
 import pyghmi.exceptions as pygexc
 import pyghmi.storage as storage
@@ -169,12 +170,21 @@ class IpmiCommandWrapper(ipmicommand.Command):
         try:
             super(IpmiCommandWrapper, self).__init__(**kwargs)
         except socket.error as se:
-            if isinstance(se, socket.timeout) or (len(se) > 1 and se[1] == 'EHOSTUNREACH'):
+            if (hasattr(se, 'errno')
+                    and se.errno in (errno.ENETUNREACH, errno.EHOSTUNREACH, errno.EADDRNOTAVAIL)):
+                if hasattr(se, 'strerror'):
+                    raise exc.TargetEndpointUnreachable(se.strerror)
+                else:
+                    raise exc.TargetEndpointUnreachable(str(se))
+            if isinstance(se, socket.timeout):
                 raise exc.TargetEndpointUnreachable('timeout')
             raise
         except pygexc.PyghmiException as pe:
             if 'Access Denied' in str(pe):
                 raise exc.TargetEndpointBadCredentials()
+            if 'Redfish not ready' in str(pe):
+                raise exc.TargetEndpointUnreachable('Redfish not yet ready')
+            raise
 
     def close_confluent(self):
         if self._attribwatcher:
@@ -183,10 +193,8 @@ class IpmiCommandWrapper(ipmicommand.Command):
 
     def _attribschanged(self, nodeattribs, configmanager, **kwargs):
         try:
-            self.ipmi_session._mark_broken()
-        except AttributeError:
-            # if ipmi_session doesn't already exist,
-            # then do nothing
+            del persistent_ipmicmds[(self.node, configmanager.tenant)]
+        except KeyError:
             pass
 
     def get_health(self):
@@ -270,7 +278,7 @@ def perform_requests(operator, nodes, element, cfg, inputdata, realop):
                         raise datum
                     if (hasattr(datum, 'kvpairs') and datum.kvpairs and
                             len(datum.kvpairs) == 1):
-                        bundle.append((datum.kvpairs.keys()[0], datum))
+                        bundle.append((list(datum.kvpairs)[0], datum))
                         numnodes -= 1
                     else:
                         yield datum
@@ -300,6 +308,11 @@ def perform_request(operator, node, element,
         try:
             return IpmiHandler(operator, node, element, configdata, inputdata,
                                cfg, results, realop).handle_request()
+        except socket.error as se:
+            if hasattr(se, 'strerror'):
+                results.put(msg.ConfluentTargetTimeout(node, se.strerror))
+            else:
+                results.put(msg.ConfluentTargetTimeout(node, str(se)))
         except pygexc.IpmiException as ipmiexc:
             excmsg = str(ipmiexc)
             if excmsg in ('Session no longer connected', 'timeout'):
@@ -327,6 +340,8 @@ def perform_request(operator, node, element,
             traceback.print_exc()
         finally:
             results.put('Done')
+        if (node, cfg.tenant) in persistent_ipmicmds:
+            del persistent_ipmicmds[(node, cfg.tenant)]
 
 persistent_ipmicmds = {}
 
@@ -365,8 +380,8 @@ class IpmiHandler(object):
                 self.loggedin = True
                 self.ipmicmd = persistent_ipmicmds[(node, tenant)]
             except socket.gaierror as ge:
-                if ge[0] == -2:
-                    raise exc.TargetEndpointUnreachable(ge[1])
+                if ge.errno == -2:
+                    raise exc.TargetEndpointUnreachable(ge.strerror)
                 raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
 
@@ -451,9 +466,13 @@ class IpmiHandler(object):
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
+    def get_diags(self, savefile, progress):
+        return self.ipmicmd.get_diagnostic_data(
+            savefile, progress=progress, autosuffix=True)
+
     def handle_servicedata_fetch(self):
         u = firmwaremanager.Updater(
-            self.node, self.ipmicmd.get_diagnostic_data,
+            self.node, self.get_diags,
             self.inputdata.nodefile(self.node), self.tenant, type='ffdc',
             owner=self.current_user)
         self.output.put(msg.CreatedResource(
@@ -490,6 +509,8 @@ class IpmiHandler(object):
             return self.handle_domain_name()
         elif self.element[1:3] == ['management_controller', 'ntp']:
             return self.handle_ntp()
+        elif self.element[1:4] == ['management_controller', 'extended', 'all']:
+            return self.handle_bmcconfig()
         elif self.element[1:3] == ['system', 'all']:
             return self.handle_sysconfig()
         elif self.element[1:3] == ['system', 'advanced']:
@@ -682,6 +703,10 @@ class IpmiHandler(object):
                 try:
                     reading = self.ipmicmd.get_sensor_reading(
                         sensor['name'])
+                    if reading.unavailable:
+                        self.output.put(msg.SensorReadings([EmptySensor(
+                            sensor['name'])], name=self.node))
+                        continue
                 except pygexc.IpmiException as ie:
                     if ie.ipmicode == 203:
                         self.output.put(msg.SensorReadings([EmptySensor(
@@ -702,6 +727,11 @@ class IpmiHandler(object):
             try:
                 reading = self.ipmicmd.get_sensor_reading(
                     self.sensormap[sensorname])
+                if reading.unavailable:
+                    self.output.put(msg.ConfluentResourceUnavailable(
+                            self.node, 'Unavailable'
+                        ))
+                    return
                 if hasattr(reading, 'health'):
                     reading.health = _str_health(reading.health)
                 self.output.put(
@@ -1163,13 +1193,14 @@ class IpmiHandler(object):
     def identify(self):
         if 'update' == self.op:
             identifystate = self.inputdata.inputbynode[self.node] == 'on'
-            self.ipmicmd.set_identify(on=identifystate)
+            blinkstate = self.inputdata.inputbynode[self.node] == 'blink'
+            self.ipmicmd.set_identify(on=identifystate, blink=blinkstate)
             self.output.put(msg.IdentifyState(
                 node=self.node, state=self.inputdata.inputbynode[self.node]))
             return
         elif 'read' == self.op:
-            # ipmi has identify as read-only for now
-            self.output.put(msg.IdentifyState(node=self.node, state=''))
+            identify = self.ipmicmd.get_identify().get('identifystate', '')
+            self.output.put(msg.IdentifyState(node=self.node, state=identify))
             return
 
     def power(self):
@@ -1237,6 +1268,19 @@ class IpmiHandler(object):
             self.ipmicmd.set_domain_name(dn)
             return
 
+    def handle_bmcconfig(self, advanced=False):
+        if 'read' == self.op:
+            try:
+                self.output.put(msg.ConfigSet(
+                    self.node,
+                    self.ipmicmd.get_bmc_configuration()))
+            except Exception as e:
+                self.output.put(
+                    msg.ConfluentNodeError(self.node, str(e)))
+        elif 'update' == self.op:
+            self.ipmicmd.set_bmc_configuration(
+                self.inputdata.get_attributes(self.node))
+
     def handle_sysconfigclear(self):
         if 'read' == self.op:
             raise exc.InvalidArgumentException(
@@ -1304,13 +1348,31 @@ class IpmiHandler(object):
 
     def save_licenses(self):
         directory = self.inputdata.nodefile(self.node)
+        if not os.access(directory, os.W_OK):
+            raise exc.InvalidArgumentException(
+                'The confluent system user/group is unable to write to '
+                'directory {0}, check ownership and permissions'.format(
+                    directory))
         for saved in self.ipmicmd.save_licenses(directory):
+            try:
+                pwent = pwd.getpwnam(self.current_user)
+                os.chown(saved, pwent.pw_uid, pwent.pw_gid)
+            except KeyError:
+                pass
             self.output.put(msg.SavedFile(self.node, saved))
 
     def handle_licenses(self):
         if self.element[-1] == '':
             self.element = self.element[:-1]
         if self.op in ('create', 'update'):
+            filename = self.inputdata.nodefile(self.node)
+            if not os.access(filename, os.R_OK):
+                errstr =  ('{0} is not readable by confluent on {1} '
+                           '(ensure confluent user or group can access file '
+                           'and parent directories)').format(
+                               filename, socket.gethostname())
+                self.output.put(msg.ConfluentNodeError(self.node, errstr))
+                return
             self.ipmicmd.apply_license(self.inputdata.nodefile(self.node))
         if len(self.element) == 3:
             self.output.put(msg.ChildCollection('all'))
@@ -1389,5 +1451,3 @@ def delete(nodes, element, configmanager, inputdata):
                                               element, type='ffdc')
     return perform_requests(
         'delete', nodes, element, configmanager, inputdata, 'delete')
-
-