Have nodes/ipminode/power/state work for xcat node

If multiple colon designated settings are present, correctly
associate settings with appropriate key.

confluent_client/bin/collate

@@ -48,6 +48,8 @@ argparser.add_option('-c', '--count', action='store_true',
 argparser.add_option('-r', '--reverse', action='store_true',
                      help='Reverse sort order to show biggest output group '
                           'last')
+argparser.add_option('-l', '--log', action='store', type='string', dest='log',
+                     help='Log each output to file, using {node} as a placeholder for node.')
 (options, args) = argparser.parse_args()
 if sys.stdin.isatty():
     argparser.print_help()
@@ -80,6 +82,12 @@ while fullline:
             continue
         if ': ' not in line:
             line = 'UNKNOWN: ' + line
+        if options.log:
+            node, output = line.split(': ', 1)
+            currlog = options.log.format(node=node, nodename=node)
+            with open(currlog, mode='a')  as log:
+                log.write(output + '\n')
+            continue
         grouped.add_line(*line.split(': ', 1))
         if options.watch:
             if not holdoff:

confluent_client/bin/confetty

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,6 +49,7 @@ import select
 import shlex
 import signal
 import socket
+import struct
 import sys
 import time
 try:
@@ -89,6 +90,11 @@ netserver = None
 laststate = {}
 
 
+class BailOut(Exception):
+    def __init__(self, errorcode=0):
+        self.errorcode = errorcode
+
+
 def print_help():
     print("confetty provides a filesystem like interface to confluent.  "
           "Navigation is done using the same commands as would be used in a "
@@ -353,7 +359,7 @@ def do_command(command, server):
     if argv[0] == 'exit':
         if os.environ.get('TERM', '') not in ('linux'):
             sys.stdout.write('\x1b]0;\x07')
-        sys.exit(0)
+        raise Bailout()
     elif argv[0] in ('help', '?'):
         return print_help()
     elif argv[0] == 'cd':
@@ -434,6 +440,10 @@ def do_command(command, server):
         currconsole = targpath
         startrequest = {'operation': 'start', 'path': targpath,
                         'parameters': {}}
+        height, width = struct.unpack(
+            'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
+        startrequest['parameters']['width'] = width
+        startrequest['parameters']['height'] = height
         for param in argv[2:]:
             (parmkey, parmval) = param.split("=")
             startrequest['parameters'][parmkey] = parmval
@@ -478,7 +488,8 @@ def createresource(args):
         collection = targpath
     else:
         collection, _, resname = targpath.rpartition('/')
-        keydata['name'] = resname
+        if 'name' not in keydata:
+            keydata['name'] = resname
     makecall(session.create, (collection, keydata))
 
 
@@ -575,11 +586,20 @@ def fullpath_target(currpath, forcepath=False):
         ntarget += '/'
     return ntarget
 
+def do_resize(a, b):
+    if not inconsole:
+        return
+    height, width = struct.unpack(
+        'hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, b'....'))[:2]
+    tlvdata.send(session.connection, {'operation': 'resize', 'width': width,
+                                      'height': height})
+
 
 def startconsole(nodename):
     global inconsole
     global consolename
     global didconsole
+    signal.signal(signal.SIGWINCH, do_resize)
     didconsole = True
     consolename = nodename
     tty.setraw(sys.stdin.fileno())
@@ -598,11 +618,12 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
         if oldtcattr is not None:
             termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
     # Request default color scheme, to undo potential weirdness of terminal
-    sys.stdout.write('\x1b[m')
+    if sys.stdout.isatty():
+        sys.stdout.write('\x1b[m')
     if fullexit:
-        if os.environ.get('TERM', '') not in ('linux'):
+        if sys.stdout.isatty() and os.environ.get('TERM', '') not in ('linux'):
             sys.stdout.write('\x1b]0;\x07')
-        sys.exit(code)
+        raise BailOut(code)
     else:
         tlvdata.send(session.connection, {'operation': 'stop',
                                           'path': currconsole})
@@ -795,6 +816,10 @@ parser.add_option("-s", "--server", dest="netserver",
 parser.add_option("-c", "--control", dest="controlpath",
                   help="Path to offer terminal control",
                   metavar="PATH")
+parser.add_option(
+    '-m', '--mintime', default=0,
+    help='Minimum time to run or else pause for input (used to keep a '
+         'terminal from closing quickly on error)')
 opts, shellargs = parser.parse_args()
 
 username = None
@@ -823,45 +848,6 @@ def server_connect():
         passphrase = getpass.getpass("Passphrase: ")
         session.authenticate(username, passphrase)
 
-
-try:
-    server_connect()
-except EOFError, KeyboardInterrupt:
-    sys.exit(0)
-except socket.gaierror:
-    sys.stderr.write('Could not connect to confluent\n')
-    sys.exit(1)
-# clear on start can help with readable of TUI, but it
-# can be annoying, so for now don't do it.
-# sys.stdout.write('\x1b[H\x1b[J')
-# sys.stdout.flush()
-
-if sys.stdout.isatty():
-    import readline
-
-    readline.parse_and_bind("tab: complete")
-    readline.parse_and_bind("set bell-style none")
-    dl = readline.get_completer_delims().replace('-', '')
-    readline.set_completer_delims(dl)
-    readline.set_completer(completer)
-
-doexit = False
-inconsole = False
-pendingcommand = ""
-session_node = get_session_node(shellargs)
-if session_node is not None:
-    consoleonly = True
-    do_command("start /nodes/%s/console/session" % session_node, netserver)
-    doexit = True
-elif shellargs:
-    command = " ".join(shellargs)
-    do_command(command, netserver)
-    quitconfetty(fullexit=True, fixterm=False)
-
-powerstate = None
-powertime = None
-
-
 def check_power_state():
     tlvdata.send(
         session.connection,
@@ -870,86 +856,145 @@ def check_power_state():
     return
 
 
-while inconsole or not doexit:
-    if inconsole:
-        try:
-            rdylist, _, _ = select.select(
-                (sys.stdin, session.connection), (), (), 10)
-        except select.error:
-            rdylist = ()
-        for fh in rdylist:
-            if fh == session.connection:
-                # this only should get called in the
-                # case of a console session
-                # each command should slurp up all relevant
-                # recv potential
-                #fh.read()
-                try:
-                    data = tlvdata.recv(fh)
-                except Exception:
-                    data = None
-                if type(data) == dict:
-                    updatestatus(data)
-                    continue
-                if data is not None:
-                    if clearpowermessage:
-                        sys.stdout.write("\x1b[2J\x1b[;H")
-                        clearpowermessage = False
-                    try:
-                        sys.stdout.write(data)
-                    except IOError:  # Some times circumstances are bad
-                        # resort to byte at a time...
-                        for d in data:
-                            sys.stdout.write(d)
-                    now = time.time()
-                    if ('showtime' not in laststate or
-                            (now // 60) != laststate['showtime'] // 60):
-                        # don't bother churning if minute does not change
-                        laststate['showtime'] = now
-                        updatestatus()
-                    sys.stdout.flush()
-                else:
-                    deadline = 5
-                    connected = False
-                    while not connected and deadline > 0:
-                        try:
-                            server_connect()
-                            connected = True
-                        except (socket.gaierror, socket.error):
-                            pass
-                        if not connected:
-                            time.sleep(1)
-                            deadline -=1
-                    if connected:
-                        do_command(
-                            "start /nodes/%s/console/session skipreplay=True" % consolename,
-                            netserver)
-                    else:
-                        doexit = True
-                        inconsole = False
-                        sys.stdout.write("\r\n[remote disconnected]\r\n")
-                    break
-            else:
-                try:
-                    myinput = fh.read()
-                    myinput = check_escape_seq(myinput, fh)
-                    if myinput:
-                        tlvdata.send(session.connection, myinput)
-                except IOError:
-                    pass
-        if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
-            powertime = time.time()
-            powerstate = True
-            check_power_state()
-    else:
-        currcommand = prompt()
-        try:
-            do_command(currcommand, netserver)
-        except socket.error:
+if sys.stdout.isatty():
+    import readline
+
+
+def main():
+    global inconsole
+    try:
+        server_connect()
+    except EOFError, KeyboardInterrupt:
+        raise BailOut(0)
+    except socket.gaierror:
+        sys.stderr.write('Could not connect to confluent\n')
+        raise BailOut(1)
+    # clear on start can help with readable of TUI, but it
+    # can be annoying, so for now don't do it.
+    # sys.stdout.write('\x1b[H\x1b[J')
+    # sys.stdout.flush()
+    global powerstate, powertime, clearpowermessage
+
+
+    if sys.stdout.isatty():
+
+        readline.parse_and_bind("tab: complete")
+        readline.parse_and_bind("set bell-style none")
+        dl = readline.get_completer_delims().replace('-', '')
+        readline.set_completer_delims(dl)
+        readline.set_completer(completer)
+
+    doexit = False
+    inconsole = False
+    pendingcommand = ""
+    session_node = get_session_node(shellargs)
+    if session_node is not None:
+        consoleonly = True
+        do_command("start /nodes/%s/console/session" % session_node, netserver)
+        doexit = True
+    elif shellargs:
+        command = " ".join(shellargs)
+        do_command(command, netserver)
+        quitconfetty(fullexit=True, fixterm=False)
+
+    powerstate = None
+    powertime = None
+
+    while inconsole or not doexit:
+        if inconsole:
+            try:
+                rdylist, _, _ = select.select(
+                    (sys.stdin, session.connection), (), (), 10)
+            except select.error:
+                rdylist = ()
+            for fh in rdylist:
+                if fh == session.connection:
+                    # this only should get called in the
+                    # case of a console session
+                    # each command should slurp up all relevant
+                    # recv potential
+                    #fh.read()
+                    try:
+                        data = tlvdata.recv(fh)
+                    except Exception:
+                        data = None
+                    if type(data) == dict:
+                        updatestatus(data)
+                        continue
+                    if data is not None:
+                        if clearpowermessage:
+                            sys.stdout.write("\x1b[2J\x1b[;H")
+                            clearpowermessage = False
+                        try:
+                            sys.stdout.write(data)
+                        except IOError:  # Some times circumstances are bad
+                            # resort to byte at a time...
+                            for d in data:
+                                sys.stdout.write(d)
+                        now = time.time()
+                        if ('showtime' not in laststate or
+                                (now // 60) != laststate['showtime'] // 60):
+                            # don't bother churning if minute does not change
+                            laststate['showtime'] = now
+                            updatestatus()
+                        sys.stdout.flush()
+                    else:
+                        deadline = 5
+                        connected = False
+                        while not connected and deadline > 0:
+                            try:
+                                server_connect()
+                                connected = True
+                            except (socket.gaierror, socket.error):
+                                pass
+                            if not connected:
+                                time.sleep(1)
+                                deadline -=1
+                        if connected:
+                            do_command(
+                                "start /nodes/%s/console/session skipreplay=True" % consolename,
+                                netserver)
+                        else:
+                            doexit = True
+                            inconsole = False
+                            sys.stdout.write("\r\n[remote disconnected]\r\n")
+                        break
+                else:
+                    try:
+                        myinput = fh.read()
+                        myinput = check_escape_seq(myinput, fh)
+                        if myinput:
+                            tlvdata.send(session.connection, myinput)
+                    except IOError:
+                        pass
+            if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
+                powertime = time.time()
+                powerstate = True
+                check_power_state()
+        else:
+            currcommand = prompt()
             try:
-                server_connect()
                 do_command(currcommand, netserver)
             except socket.error:
-                doexit = True
-                sys.stdout.write('Lost connection to server')
-quitconfetty(fullexit=True)
+                try:
+                    server_connect()
+                    do_command(currcommand, netserver)
+                except socket.error:
+                    doexit = True
+                    sys.stdout.write('Lost connection to server')
+    quitconfetty(fullexit=True)
+
+if __name__ == '__main__':
+    errcode = 0
+    deadline = 0
+    if opts.mintime:
+        deadline = os.times()[4] + float(opts.mintime)
+    try:
+        main()
+    except BailOut as e:
+        errcode = e.errorcode
+    finally:
+        if deadline and os.times()[4] < deadline:
+            sys.stderr.write('[Exited early, hit enter to continue]')
+            sys.stdin.readline()
+        sys.exit(errcode)

confluent_client/bin/confluent2ansible

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o ansible.hosts
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='Ansible hosts file')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not options.output:
+        sys.stderr.write('Output file must be specified by -o\n')
+        sys.exit(1)
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    nodesbygroup = {}
+    with open(options.output, 'w') as importfile:
+        needempty = False
+        for node in sortutil.natural_sort(databynode):
+            data = databynode[node]
+            if not data.get('groups', []):
+                importfile.write(node + '\n')
+                needempty = True
+            for g in data.get('groups', []):
+                if g not in nodesbygroup:
+                    nodesbygroup[g] = set([node])
+                else:
+                    nodesbygroup[g].add(node)
+        if needempty:
+            importfile.write('\n')
+        for group in sortutil.natural_sort(nodesbygroup):
+            importfile.write('[{0}]\n'.format(group))
+            for node in sortutil.natural_sort(nodesbygroup[group]):
+                importfile.write('{0}\n'.format(node))
+            importfile.write('\n')
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/confluent2lxca

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+import csv
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+headers = '# Type,Serial Number,Current IP,Current username,Current password,New password,Recovery password,Switch enable password,New IPv4,IPv4 Subnet mask,IPv4 Default gateway,IPv4 DNS1,IPv4 DNS2,New IPv6,IPv6 Prefix,IPv6 Default gateway,IPv6 DNS1,IPv6 DNS2,Domain,Host name,Display name,Rack,Lowest Rack Unit,Height,Force,Stored credentials ID,Managed authentication,Group Name'.split(',')
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o bulkimport.csv
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='File to write for bulk import into xClarity Administrator')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not options.output:
+        sys.stderr.write('Output file must be specified by -o\n')
+        sys.exit(1)
+    if 'XCCUSER' not in os.environ or 'XCCPASS' not in os.environ:
+        sys.stderr.write('Must specify XCCUSER and XCCPASS in environment variables\n')
+        sys.exit(1)
+    xccuser = os.environ['XCCUSER']
+    xccpass = os.environ['XCCPASS']
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    for res in sess.read(
+            '/noderange/{0}/configuration/management_controller/net_interfaces/management'.format(
+                    noderange)):
+        for node in res.get('databynode', {}):
+            currip = res['databynode'][node].get('ipv4_address', {}).get(
+                'value', '')
+            if currip:
+                databynode[node]['hardwaremanagement.manager'] = {
+                    'value': currip.split('/', 1)[0]}
+    with open(options.output, 'w') as importfile:
+        bulkimport = csv.writer(importfile)
+        bulkimport.writerow(headers)
+        for node in sortutil.natural_sort(databynode):
+            data = databynode[node]
+            row = ['server', lookupdata(data, 'id.serial'), lookupdata(data, 'hardwaremanagement.manager'), xccuser, xccpass, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', node, lookupdata(data, 'location.rack'), lookupdata(data, 'location.u'), '', '', '','', '']
+            bulkimport.writerow(row)
+
+
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/confluent2xcat

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+import csv
+import optparse
+import signal
+import sys
+import os
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def lookupdata(data, key):
+    ret = data.get(key, {}).get('value', '')
+    if ret is None:
+        ret = ''
+    return ret
+
+def main():
+    argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange -o ansible.hosts
+             \n ''')
+    argparser.add_option('-o', '--output',
+                         help='xCAT stanza file')
+    argparser.add_option('-m', '--macs',
+                         help='xCAT macs.csv to write')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    if not (options.output or options.macs):
+        sys.stderr.write('Output file must be specified by -o or -m\n')
+        sys.exit(1)
+    sess = client.Command()
+    databynode = {}
+    for res in sess.read('/noderange/{0}/attributes/all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            if node not in databynode:
+                databynode[node] = {}
+            databynode[node].update(res['databynode'][node])
+    if options.output:
+        with open(options.output, 'w') as importfile:
+            for node in sortutil.natural_sort(databynode):
+                xcatattrs = collect_attribute_data(databynode, node)
+                importfile.write('{0}:\n'.format(node))
+                importfile.write('   objtype=node\n')
+                importfile.write('   arch=x86_64\n')
+                importfile.write('   netboot=xnba\n')
+                importfile.write('   mgt=ipmi\n')
+                for attr in xcatattrs:
+                    if xcatattrs[attr] is None:
+                        continue
+                    importfile.write('   {0}={1}\n'.format(attr,
+                                                           xcatattrs[attr]))
+                importfile.write('\n')
+    if options.macs:
+        with open(options.macs, 'w') as importfile:
+            macsv = csv.writer(importfile)
+            macsv.writerow(['#node', 'mac'])
+            for node in sortutil.natural_sort(databynode):
+                xcatattrs = collect_attribute_data(databynode, node)
+                macsv.writerow([node, xcatattrs['mac']])
+
+
+
+def collect_attribute_data(databynode, node):
+    data = databynode[node]
+    xcatattrs = {}
+    xcatattrs['groups'] = ','.join(data.get('groups', []))
+    xcatattrs['bmc'] = data.get('hardwaremanagement.manager', {}).get(
+        'value', None)
+    xcatattrs['mpa'] = data.get('enclosure.manager', {}).get(
+        'value', None)
+    xcatattrs['slotid'] = data.get('enclosure.bay', {}).get(
+        'value', None)
+    gotmac = False
+    for key in data:
+        if key.startswith('net.') and 'hwaddr' in key:
+            currmac = data[key].get('value', None)
+            if currmac:
+                if gotmac:
+                    sys.stderr.write(
+                        'Ignoring {0} and using only {1} for mac, '
+                        'multiple macs not supported by '
+                        'confluent2xcat\n'.format(key, gotmac))
+                    continue
+                gotmac = key
+                xcatattrs['mac'] = currmac
+    return xcatattrs
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodeattrib

@@ -17,6 +17,7 @@
 
 __author__ = 'alin37'
 
+from getpass import getpass
 import optparse
 import os
 import signal
@@ -47,6 +48,8 @@ argparser.add_option('-e', '--environment', action='store_true',
                           'same name')
 argparser.add_option('-c', '--clear', action='store_true',
                      help='Clear attributes')
+argparser.add_option('-p', '--prompt', action='store_true',
+                     help='Prompt for attribute values interactively')
 (options, args) = argparser.parse_args()
 
 
@@ -67,12 +70,24 @@ exitcode = 0
 nodetype="noderange"
 
 if len(args) > 1:
-    if "=" in args[1] or options.clear or options.environment:
+    if "=" in args[1] or options.clear or options.environment or options.prompt:
         if "=" in args[1] and options.clear:
             print("Can not clear and set at the same time!")
             argparser.print_help()
             sys.exit(1)
-        exitcode=client.updateattrib(session,args,nodetype, noderange, options)
+        argassign = None
+        if options.prompt:
+            argassign = {}
+            for arg in args[1:]:
+                oneval = 1
+                twoval = 2
+                while oneval != twoval:
+                    oneval = getpass('Enter value for {0}: '.format(arg))
+                    twoval = getpass('Confirm value for {0}: '.format(arg))
+                    if oneval != twoval:
+                        print('Values did not match.')
+                argassign[arg] = twoval
+        exitcode=client.updateattrib(session,args,nodetype, noderange, options, argassign)
     try:
         # setting user output to what the user inputs
         if args[1] == 'all':

confluent_client/bin/nodeboot

@@ -35,6 +35,9 @@ argparser = optparse.OptionParser()
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')
+argparser.add_option('-u', '--uefi', dest='uefimode',
+                     action='store_true', default=False,
+                     help='Request UEFI style boot (rather than BIOS)')
 argparser.add_option('-p', '--persist', dest='persist', action='store_true',
                      default=False,
                      help='Request the boot device be persistent rather than '

confluent_client/bin/nodeconfig

@@ -19,6 +19,7 @@
 import os
 import signal
 import optparse
+import shlex
 import sys
 
 try:
@@ -47,6 +48,8 @@ argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '
                           'that are non default')
+argparser.add_option('-b', '--batch', dest='batch', metavar='settings.batch',
+                     default=False, help='Provide settings in a batch file')
 argparser.add_option('-d', '--detail', dest='detail',
                      action='store_true', default=False,
                      help='Provide verbose information as available, such as '
@@ -55,6 +58,16 @@ argparser.add_option('-x', '--exclude', dest='exclude',
                      action='store_true', default=False,
                      help='Treat positional arguments as items to not '
                           'examine, compare, or restore default')
+argparser.add_option('-a', '--advanced', dest='advanced',
+                     action='store_true', default=False,
+                     help='Include advanced settings, which are normally not '
+                          'intended to be used without direction from the '
+                          'relevant server vendor.')
+argparser.add_option('-r', '--restoredefault', default=False,
+                     dest='restoredefault', metavar="COMPONENT",
+                     help='Restore the configuration of the node '
+                          'to factory default for given component. '
+                          'Currently only uefi is supported')
 (options, args) = argparser.parse_args()
 
 cfgpaths = {
@@ -108,63 +121,88 @@ def _assign_value():
     assignment[key] = value
 
 
-for param in args[1:]:
-    if param == 'show':
-        continue  # forgive muscle memory of pasu users
-    if param == 'set':
-        setmode = True
-        forceset = True
-        continue
-    if needval:
-        key = needval
-        value = param
-        _assign_value()
-        continue
-    if '=' in param or param[-1] == ':' or forceset:
-        if setmode is None:
+def parse_config_line(arguments):
+    global setmode, forceset, key, value, needval, candidate, path, attrib
+    for param in arguments:
+        if param == 'show':
+            continue  # forgive muscle memory of pasu users
+        if param == 'set':
             setmode = True
-        if setmode != True:
-            bailout('Cannot do set and query in same command')
-        if '=' in param:
-            key, _, value = param.partition('=')
+            forceset = True
+            continue
+        if needval:
+            key = needval
+            needval = None
+            value = param
             _assign_value()
-        elif param[-1] == ':':
-            needval = param[:-1]
+            continue
+        if '=' in param or param[-1] == ':' or forceset:
+            if setmode is None:
+                setmode = True
+            if setmode != True:
+                bailout('Cannot do set and query in same command')
+            if '=' in param:
+                key, _, value = param.partition('=')
+                _assign_value()
+            elif param[-1] == ':':
+                needval = param[:-1]
+            else:
+                needval = param
         else:
-            needval = param
-    else:
-        if setmode is None:
-            setmode = False
-        if setmode != False:
-            bailout('Cannot do set and query in same command')
-        if '.' not in param:
-            matchedparms = False
-            for candidate in cfgpaths:
-                if candidate.startswith('{0}.'.format(param)):
-                    matchedparms = True
-                    if not options.exclude:
-                        path, attrib = cfgpaths[candidate]
-                        path = '/noderange/{0}/{1}'.format(noderange, path)
-                        if path not in queryparms:
-                            queryparms[path] = {}
-                        queryparms[path][attrib] = candidate
-                    else:
-                        try:
-                            del queryparms[path]
-                        except KeyError:
-                            pass
-            if not matchedparms:
+            if setmode is None:
+                setmode = False
+            if setmode != False:
+                bailout('Cannot do set and query in same command')
+            if '.' not in param:
+                matchedparms = False
+                for candidate in cfgpaths:
+                    if candidate.startswith('{0}.'.format(param)):
+                        matchedparms = True
+                        if not options.exclude:
+                            path, attrib = cfgpaths[candidate]
+                            path = '/noderange/{0}/{1}'.format(noderange, path)
+                            if path not in queryparms:
+                                queryparms[path] = {}
+                            queryparms[path][attrib] = candidate
+                        else:
+                            try:
+                                del queryparms[path]
+                            except KeyError:
+                                pass
+                if not matchedparms:
+                    printsys.append(param)
+            elif param not in cfgpaths:
                 printsys.append(param)
-        elif param not in cfgpaths:
-            printsys.append(param)
-        else:
-            path, attrib = cfgpaths[param]
-            path = '/noderange/{0}/{1}'.format(noderange, path)
-            if path not in queryparms:
-                queryparms[path] = {}
-            queryparms[path][attrib] = param
+            else:
+                path, attrib = cfgpaths[param]
+                path = '/noderange/{0}/{1}'.format(noderange, path)
+                if path not in queryparms:
+                    queryparms[path] = {}
+                queryparms[path][attrib] = param
+
+if options.batch:
+    printsys = []
+    argfile = open(options.batch, 'r')
+    argset = argfile.readline()
+    while argset:
+        parse_config_line(shlex.split(argset))
+        argset = argfile.readline()
+else:
+    parse_config_line(args[1:])
 session = client.Command()
 rcode = 0
+if options.restoredefault and options.restoredefault.lower() in (
+        'sys', 'system', 'uefi', 'bios'):
+    for fr in session.update(
+            '/noderange/{0}/configuration/system/clear'.format(noderange),
+            {'clear': True}):
+        rcode |= client.printerror(fr)
+    sys.exit(rcode)
+elif options.restoredefault:
+    sys.stderr.write(
+        'Unrecognized component to restore defaults: {0}\n'.format(
+            options.restoredefault))
+    sys.exit(1)
 if setmode:
     if options.exclude:
         sys.stderr.write('Cannot use exclude and assign at the same time\n')
@@ -209,7 +247,11 @@ else:
     if printsys or options.exclude:
         if printsys == 'all':
             printsys = []
-        path = '/noderange/{0}/configuration/system/all'.format(noderange)
+        if (options.comparedefault or printsys == []) and not options.advanced:
+            path = '/noderange/{0}/configuration/system/all'.format(noderange)
+        else:
+            path = '/noderange/{0}/configuration/system/advanced'.format(
+                noderange)
         rcode = client.print_attrib_path(path, session, printsys,
                                          options)
-sys.exit(rcode)
+sys.exit(rcode)

confluent_client/bin/nodeconsole

@@ -17,7 +17,14 @@
 
 import optparse
 import os
+import subprocess
 import sys
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+import confluent.client as client
+import confluent.sortutil as sortutil
 
 confettypath = os.path.join(os.path.dirname(sys.argv[0]), 'confetty')
 argparser = optparse.OptionParser(
@@ -26,9 +33,42 @@ argparser = optparse.OptionParser(
            "ctrl-'e', then release ctrl, then 'c', then '?' for a full list. "
            "For example, ctrl-'e', then 'c', then '.' will exit the current "
            "console")
+argparser.add_option('-t', '--tile', action='store_true', default=False,
+                     help='Tile console windows in the terminal')
 (options, args) = argparser.parse_args()
 if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
-os.execl(confettypath, confettypath, 'start',
-         '/nodes/{0}/console/session'.format(args[0]))
+if options.tile:
+    nodes = []
+    sess = client.Command()
+    for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
+        node = res.get('item', {}).get('href', '/').replace('/', '')
+        if not node:
+            sys.stderr.write(res.get('error', repr(res)) + '\n')
+            sys.exit(1)
+        nodes.append(node)
+    initial = True
+    pane = 0
+    for node in sortutil.natural_sort(nodes):
+        if initial:
+            initial = False
+            subprocess.call(
+                ['tmux', 'new-session', '-d', '-s',
+                 'nodeconsole_{0}'.format(os.getpid()), '-x', '800', '-y',
+                 '800', '{0} -m 5 start /nodes/{1}/console/session'.format(
+                    confettypath, node)])
+        else:
+            subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
+            pane += 1
+            subprocess.call(
+                ['tmux', 'split', '-h',
+                 '{0} -m 5 start /nodes/{1}/console/session'.format(
+                     confettypath, node)])
+            subprocess.call(['tmux', 'select-layout', 'tiled'])
+    subprocess.call(['tmux', 'select-pane', '-t', '0'])
+    os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
+        os.getpid()))
+else:
+    os.execl(confettypath, confettypath, 'start',
+             '/nodes/{0}/console/session'.format(args[0]))

confluent_client/bin/nodediscover

@@ -19,6 +19,7 @@ import csv
 import optparse
 import os
 import sys
+import time
 
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
@@ -26,31 +27,52 @@ if path.startswith('/opt'):
     sys.path.append(path)
 
 import confluent.client as client
+import confluent.sortutil as sortutil
 
-tabformat = '{0:>15}|{1:>15}|{2:>15}|{3:>36}|{4:>17}|{5:>12}|{6:>48}'
-columns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
-           'Current IP Addresses']
-delimit = ['-' * 15, '-' * 15, '-' * 15, '-' * 36, '-' * 17, '-' * 12,
-           '-' * 48]
+defcolumns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
+              'Current IP Addresses']
+columnmapping = {
+    'Node': 'nodename',
+    'Model': 'modelnumber',
+    'Serial': 'serialnumber',
+    'UUID': 'uuid',
+    'Type': 'types',
+    'Current IP Addresses': 'ipaddrs',
+    'IP': 'ipaddrs',
+    'Bay': 'bay',
+    'Mac Address': 'macs',
+    'Mac': 'macs',
+    'Switch': 'switch',
+    'Port': 'port',
+    'Advertised IP': 'otheripaddrs',
+    'Other IP': 'otheripaddrs',
+}
 
 
-def dumpmacs(procinfo):
-    return ','.join(procinfo['macs']) # + procinfo.get('relatedmacs', []))
-
-
-def print_disco(options, session, currmac):
+def print_disco(options, session, currmac, outhandler, columns):
     procinfo = {}
     for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):
 
         procinfo.update(tmpinfo)
-    record = [procinfo['nodename'], procinfo['modelnumber'],
-              procinfo['serialnumber'], procinfo['uuid'], dumpmacs(procinfo),
-              ','.join(procinfo['types']),
-              ','.join(sorted(procinfo['ipaddrs']))]
+    if 'Switch' in columns or 'Port' in columns:
+        for tmpinfo in session.read(
+                '/networking/macs/by-mac/{0}'.format(currmac)):
+            if 'ports' in tmpinfo:
+                # The api sorts so that the most specific available value
+                # is last
+                procinfo.update(tmpinfo['ports'][-1])
+    record = []
+    for col in columns:
+        rawcol = columnmapping[col]
+        rawval = procinfo.get(rawcol, '')
+        if isinstance(rawval, list):
+            record.append(','.join(rawval))
+        else:
+            record.append(str(rawval))
     if options.csv:
         csv.writer(sys.stdout).writerow(record)
     else:
-        print(tabformat.format(*record))
+        outhandler.add_row(record)
 
 
 def process_header(header):
@@ -72,7 +94,7 @@ def process_header(header):
         elif datum in ('bmc', 'imm', 'xcc'):
             fields.append('hardwaremanagement.manager')
         elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
-            fields.append('net.bmc.gateway')
+            fields.append('net.bmc.ipv4_gateway')
         elif datum in ('bmcuser', 'username', 'user'):
             fields.append('secret.hardwaremanagementuser')
         elif datum in ('bmcpass', 'password', 'pass'):
@@ -122,18 +144,35 @@ def datum_to_attrib(datum):
     del datum['node']
     return datum
 
+unique_fields = frozenset(['serial', 'mac', 'uuid'])
+
 def import_csv(options, session):
     nodedata = []
+    unique_data = {}
     with open(options.importfile, 'r') as datasrc:
         records = csv.reader(datasrc)
         fields = process_header(next(records))
+        for field in fields:
+            if field in unique_fields:
+                unique_data[field] = set([])
         for record in records:
             currfields = list(fields)
             nodedatum = {}
             for datum in record:
-                nodedatum[currfields.pop(0)] = datum
+                currfield = currfields.pop(0)
+                if currfield in unique_fields:
+                    if datum in unique_data[currfield]:
+                        sys.stderr.write(
+                            "Import contains duplicate values "
+                            "({0} with value {1}\n".format(currfield, datum)
+                        )
+                        sys.exit(1)
+                    unique_data[currfield].add(datum)
+                nodedatum[currfield] = datum
             if not datum_complete(nodedatum):
                 sys.exit(1)
+            if not search_record(nodedatum, options, session):
+                blocking_scan(session)
             if not search_record(nodedatum, options, session):
                 sys.stderr.write(
                     "Could not match the following data: " +
@@ -165,13 +204,30 @@ def import_csv(options, session):
 
 
 def list_discovery(options, session):
+    outhandler = None
+    orderby = None
+    if options.fields:
+        columns = []
+        for field in options.fields.split(','):
+            for cdt in columnmapping:
+                if cdt.lower().replace(
+                        ' ', '') == field.lower().replace(' ', ''):
+                    columns.append(cdt)
+    else:
+        columns = defcolumns
+    if options.order:
+        for field in columns:
+            if options.order.lower() == field.lower():
+                orderby = field
     if options.csv:
         csv.writer(sys.stdout).writerow(columns)
     else:
-        print(tabformat.format(*columns))
-        print(tabformat.format(*delimit))
+        outhandler = client.Tabulator(columns)
     for mac in list_matching_macs(options, session):
-        print_disco(options, session, mac)
+        print_disco(options, session, mac, outhandler, columns)
+    if outhandler:
+        for row in outhandler.get_table(orderby):
+            print(row)
 
 def clear_discovery(options, session):
     for mac in list_matching_macs(options, session):
@@ -220,6 +276,10 @@ def assign_discovery(options, session):
     if abort:
         sys.exit(1)
     matches = list_matching_macs(options, session)
+    if not matches:
+        # Do a rescan to catch missing requested data
+        blocking_scan(session)
+        matches = list_matching_macs(options, session)
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)
@@ -230,6 +290,10 @@ def assign_discovery(options, session):
         else:
             print(repr(res))
 
+def blocking_scan(session):
+    list(session.update('/discovery/rescan', {'rescan': 'start'}))
+    while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
+        time.sleep(0.5)
 
 
 def main():
@@ -266,6 +330,11 @@ def main():
     parser.add_option('-i', '--import', dest='importfile',
                       help='Import bulk assignment data from given CSV file',
                       metavar='IMPORT.CSV')
+    parser.add_option('-f', '--fields', dest='fields',
+                      help='Select fields for output',
+                      metavar='FIELDS')
+    parser.add_option('-o', '--order', dest='order',
+                      help='Order output by given field', metavar='ORDER')
     (options, args) = parser.parse_args()
     if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
         parser.print_help()
@@ -278,8 +347,8 @@ def main():
     if args[0] == 'assign':
         assign_discovery(options, session)
     if args[0] == 'rescan':
-        list(session.update('/discovery/rescan', {'rescan': 'start'}))
-        print("Rescan initiated")
+        blocking_scan(session)
+        print("Rescan complete")
 
 
 if __name__ == '__main__':

confluent_client/bin/nodefirmware

@@ -37,14 +37,14 @@ exitcode = 0
 
 
 def printfirm(node, prefix, data):
-    if 'model' in data:
+    if 'model' in data and data['model']:
         prefix += ' ' + data['model']
     builddesc = []
-    if 'build' in data:
+    if 'build' in data and data['build']:
         builddesc.append(data['build'])
-    if 'date' in data:
+    if 'date' in data and data['date']:
         builddesc.append(data['date'])
-    if 'version' in data:
+    if 'version' in data and data['version']:
         version = data['version']
         if builddesc:
             version += ' ({0})'.format(' '.join(builddesc))
@@ -55,7 +55,8 @@ def printfirm(node, prefix, data):
 components = ['all']
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog <noderange> [update [--backup <file>]]|[<components>]")
+    usage="Usage: "
+          "%prog <noderange> [list][update [--backup <file>]]|[<components>]")
 argparser.add_option('-b', '--backup', action='store_true',
                      help='Target a backup bank rather than primary')
 (options, args) = argparser.parse_args()
@@ -66,9 +67,15 @@ try:
         if args[1] == 'update':
             upfile = args[2]
         else:
+            if args[1] == 'list':
+                comps = args[2:]
+            else:
+                comps = args[1:]
             components = []
-            for arg in args[1:]:
+            for arg in comps:
                 components += arg.split(',')
+            if not components:
+                components = ['all']
 
 except IndexError:
     argparser.print_help()

confluent_client/bin/nodegroupattrib

@@ -17,6 +17,7 @@
 
 __author__ = 'alin37'
 
+from getpass import getpass
 import optparse
 import os
 import signal
@@ -47,6 +48,8 @@ argparser.add_option('-e', '--environment', action='store_true',
                           'same name')
 argparser.add_option('-c', '--clear', action='store_true',
                      help='Clear variables')
+argparser.add_option('-p', '--prompt', action='store_true',
+                     help='Prompt for attribute values interactively')
 (options, args) = argparser.parse_args()
 
 
@@ -72,7 +75,19 @@ if len(args) > 1:
         print("Can not clear and set at the same time!")
         argparser.print_help()
         sys.exit(1)
-    exitcode=client.updateattrib(session,args,nodetype, nodegroups, options)
+    argassign = None
+    if options.prompt:
+        argassign = {}
+        for arg in args[1:]:
+            oneval = 1
+            twoval = 2
+            while oneval != twoval:
+                oneval = getpass('Enter value for {0}: '.format(arg))
+                twoval = getpass('Confirm value for {0}: '.format(arg))
+                if oneval != twoval:
+                    print('Values did not match.')
+            argassign[arg] = twoval
+    exitcode=client.updateattrib(session,args,nodetype, nodegroups, options, argassign)
     try:
         # setting user output to what the user inputs
         if args[1] == 'all':

confluent_client/bin/nodehealth

@@ -44,6 +44,13 @@ except IndexError:
 client.check_globbing(noderange)
 
 
+codemappings = {
+    'ok': 0,
+    'warning': 1,
+    'critical': 2,
+    'failed': 2,
+}
+
 def main():
     global session, exitcode, healthbynode, healthexplanations, health, node, sensor, explanation
     session = client.Command()
@@ -66,9 +73,11 @@ def main():
             if 'error' in health[node]:
                 sys.stderr.write('{0}: Error: {1}\n'.format(
                     node, health[node]['error']))
-                exitcode |= 1
+                exitcode = 3
             if 'health' in health[node]:
                 healthbynode[node] = health[node]['health']['value']
+                if codemappings[healthbynode[node]] > exitcode:
+                    exitcode = codemappings[healthbynode[node]]
             if 'sensors' in health[node]:
                 healthexplanations[node] = []
                 for sensor in health[node]['sensors']:
@@ -94,5 +103,5 @@ try:
     main()
 except KeyboardInterrupt:
     print('')
-    sys.exit(0)
+sys.exit(exitcode)
 

confluent_client/bin/nodeinventory

@@ -87,6 +87,7 @@ def printerror(res, node=None):
 
 
 url = '/noderange/{0}/inventory/hardware/all/all'
+usedprefixes = set([])
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog <noderange> [serial|model|uuid|mac]")
@@ -129,6 +130,11 @@ try:
                 continue
             for inv in res['databynode'][node]['inventory']:
                 prefix = inv['name']
+                idx = 2
+                while (node, prefix) in usedprefixes:
+                    prefix = '{0} {1}'.format(inv['name'], idx)
+                    idx += 1
+                usedprefixes.add((node, prefix))
                 if not inv['present']:
                     if not filters:
                         if options.json:

confluent_client/bin/nodelicense

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+exitcode = 0
+
+argparser = optparse.OptionParser(
+    usage="Usage: "
+          "%prog <noderange> [list][install <file>]")
+(options, args) = argparser.parse_args()
+upfile = None
+try:
+    noderange = args[0]
+    if len(args) > 1:
+        if args[1] == 'install':
+            upfile = args[2]
+        else:
+            components = ['all']
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+
+def install_license(session, filename):
+    global exitcode
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/licenses/'.format(noderange)
+    filename = os.path.abspath(filename)
+    instargs = {'filename': filename}
+    for res in session.create(resource, instargs):
+        pass # print(repr(res))
+    show_licenses()
+
+def show_licenses(session):
+    global exitcode
+    firmware_shown = False
+    for res in session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange)):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                print('{0}: {1}'.format(node, license.get('feature',
+                                                          'Unknown')))
+
+
+try:
+    session = client.Command()
+    if upfile is None:
+        show_licenses(session)
+    else:
+        install_license(session, upfile)
+except KeyboardInterrupt:
+    print('')
+sys.exit(exitcode)

confluent_client/bin/nodelist

@@ -46,7 +46,10 @@ def main():
         noderange = args[0]
         nodelist = '/noderange/{0}/nodes/'.format(noderange)
     except IndexError:
-        nodelist = '/nodes/'
+        if os.environ.get('TEST_XCAT'):
+            nodelist = "/xcat/nodes/"
+        else:
+            nodelist = '/nodes/'
     client.check_globbing(noderange)
     session = client.Command()
     exitcode = 0

confluent_client/bin/noderun

@@ -85,7 +85,7 @@ def run():
             desc = pipedesc[r]
             node = desc['node']
             data = True
-            while data and select.select([r], [], [], 0):
+            while data and select.select([r], [], [], 0)[0]:
                 data = r.readline()
                 if data:
                     if desc['type'] == 'stdout':

confluent_client/bin/nodesetboot

@@ -53,8 +53,8 @@ except IndexError:
     argparser.print_help()
     sys.exit(1)
 bootdev = None
-if len(sys.argv) > 2:
-    bootdev = sys.argv[2]
+if len(args) > 1:
+    bootdev = args[1]
     if bootdev in ('net', 'pxe'):
         bootdev = 'network'
 session = client.Command()
@@ -65,4 +65,4 @@ else:
     bootmode = 'uefi'
 sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
-                                     persistent=options.persist))
+                                     persistent=options.persist))

confluent_client/bin/nodestorage

@@ -0,0 +1,203 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2018 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+exitcode = 0
+
+class OptParser(optparse.OptionParser):
+
+    def format_epilog(self, formatter):
+        return self.expand_prog_name(self.epilog)
+
+def mbtohuman(mb):
+    if mb > 1000000:
+        return '{0:.3f} TB'.format(mb/1000000.0)
+    if mb > 1000:
+        return '{0:.3f} GB'.format(mb/1000.0)
+    return '{0:.3f} MB'.format(mb)
+
+def showstorage(noderange, options, args):
+    global exitcode
+    session = client.Command()
+    disks = {}
+    arrays = {}
+    volumes = {}
+    scfg = session.read('/noderange/{0}/configuration/storage/all'.format(
+        noderange))
+    _print_cfg(scfg)
+
+
+def _print_cfg(scfg):
+    global exitcode
+    storagebynode = {}
+    for e in scfg:
+        if 'error' in e:
+            sys.stderr.write(e['error'] + '\n')
+            exitcode = e.get('errorcode', 1)
+        for node in e.get('databynode', {}):
+            if node not in storagebynode:
+                storagebynode[node] = {'disks': [], 'arrays': [],
+                                       'volumes': []}
+            curr = e['databynode'][node]
+            storagebynode[node][curr['type'] + 's'].append(curr)
+    for node in storagebynode:
+        for disk in sorted(storagebynode[node]['disks'],
+                           key=lambda x: x['name']):
+            print('{0}: Disk {1} Description: {2}'.format(
+                node, disk['name'], disk['description']))
+            print('{0}: Disk {1} State: {2}'.format(node, disk['name'],
+                                                    disk['state']))
+            print('{0}: Disk {1} FRU: {2}'.format(node, disk['name'],
+                                                  disk['fru']))
+            print('{0}: Disk {1} Serial Number: {2}'.format(node, disk['name'],
+                                                            disk['serial']))
+            if disk['array']:
+                print('{0}: Disk {1} Array: {2}'.format(node, disk['name'],
+                                                        disk['array']))
+        for arr in storagebynode[node]['arrays']:
+            print('{0}: Array {1} Available Capacity: {2}'.format(
+                node, arr['id'], mbtohuman(arr['available'])))
+            print('{0}: Array {1} Total Capacity: {2}'.format(
+                node, arr['id'], mbtohuman(arr['capacity'])))
+            print('{0}: Array {1} RAID: {2}'.format(node, arr['id'],
+                                                    arr['raid']))
+            print('{0}: Array {1} Disks: {2}'.format(node, arr['id'], ','.join(
+                arr['disks'])))
+            print(
+                '{0}: Array {1} Volumes: {2}'.format(node, arr['id'], ','.join(
+                    arr['volumes'])))
+        for vol in storagebynode[node]['volumes']:
+            print('{0}: Volume {1}: Size: {2}'.format(node, vol['name'],
+                                                      mbtohuman(vol['size'])))
+            print('{0}: Volume {1}: State: {2}'.format(node, vol['name'],
+                                                       vol['state']))
+            print('{0}: Volume {1}: Array {2}'.format(node, vol['name'],
+                                                      vol['array']))
+
+
+def createstorage(noderange, options, args):
+    if options.raidlevel is None or options.disks is None:
+        sys.stderr.write('-r and -d are required arguments to create array\n')
+        sys.exit(1)
+    session = client.Command()
+    names = options.name
+    if names is None:
+        names = ''.join(args)
+    parms = {'disks': options.disks, 'raidlevel': options.raidlevel,
+             'name': names}
+    if options.size:
+        parms['size'] = options.size
+    _print_cfg(session.create(
+            '/noderange/{0}/configuration/storage/volumes/{1}'.format(
+                noderange, names), parms))
+
+
+def deletestorage(noderange, options, args):
+    if options.name is None:
+        if len(args) == 1:
+            names = args[0]
+        else:
+            sys.stderr.write('-n is required to indicate volume(s) to delete\n')
+            sys.exit(1)
+    else:
+        names = options.name
+    session = client.Command()
+    for rsp in session.delete(
+            '/noderange/{0}/configuration/storage/volumes/{1}'.format(
+                noderange, names)):
+        if 'deleted' in rsp:
+            print('Deleted: {0}'.format(rsp['deleted']))
+        elif 'databynode' in rsp:
+            for node in rsp['databynode']:
+                if 'error' in rsp['databynode'][node]:
+                    sys.stderr.write('{0}: {1}\n'.format(
+                        node, rsp['databynode'][node]['error']))
+                else:
+                    sys.stderr.write('{0}: {1}\n'.format(
+                        node, repr(rsp['databynode'][node])))
+        else:
+            print(repr(rsp))
+
+
+def setstorage(noderange, options, args):
+    pass
+
+funmap = {
+    'create': createstorage,
+    'show': showstorage,
+    'set': setstorage,
+    'delete': deletestorage,
+    'rm': deletestorage,
+}
+
+
+
+def main():
+    argparser = OptParser(
+        usage='Usage: %prog <noderange> [show|create|set|delete]',
+        epilog='',
+    )
+    argparser.add_option('-r', '--raidlevel', type='int',
+                         help='RAID level to use when creating an array')
+    argparser.add_option('-d', '--disks', type='str',
+                         help='Comma separated list of disks to use, or the '
+                              'word "rest" to indicate use of all available '
+                              'disks')
+    argparser.add_option('-s', '--size', type='str',
+                         help='Comma separated list of sizes to use when '
+                              'creating volumes.  The sizes may be absolute '
+                              'size (e.g. 16gb), percentage (10%) or the word '
+                              '"rest" to use remaining capacity, default '
+                              'behavior is to use all capacity to make a '
+                              'volume')
+    argparser.add_option('-n', '--name', type='str',
+                         help='Comma separated list of names to use when '
+                              'naming volumes, or selecting a volume for '
+                              'delete.  Default behavior is to use '
+                              'implementation provided default names.')
+    (options, args) = argparser.parse_args()
+    try:
+        noderange = args[0]
+        operation = args[1]
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    try:
+        handler = funmap[operation]
+    except KeyError:
+        argparser.print_help()
+        sys.exit(1)
+    handler(noderange, options, args[2:])
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodesupport

@@ -0,0 +1,145 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2018 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+
+exitcode = 0
+
+
+def get_update_progress(session, url):
+    for res in session.read(url):
+        status = res['phase']
+        percent = res['progress']
+        detail = res['detail']
+        if status == 'error':
+            text = 'error!'
+        else:
+            text = '{0}: {1:3.0f}%'.format(status, percent)
+    return text, status, detail
+
+
+def printerror(res, node=None):
+    global exitcode
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+    for node in res.get('databynode', ()):
+        printerror(res['databynode'][node], node)
+
+
+
+def download_servicedata(noderange, media):
+    global exitcode
+    session = client.Command()
+    output = sq.ScreenPrinter(noderange, session)
+    filename = os.path.abspath(media)
+    resource = '/noderange/{0}/support/servicedata/'.format(noderange)
+    upargs = {'filename': filename}
+    noderrs = {}
+    nodeurls = {}
+    for res in session.create(resource, upargs):
+        if 'created' not in res:
+            for nodename in res.get('databynode', ()):
+                output.set_output(nodename, 'error!')
+                noderrs[nodename] = res['databynode'][nodename].get(
+                    'error', 'Unknown Error')
+            continue
+        watchurl = res['created']
+        currnode = watchurl.split('/')[1]
+        nodeurls[currnode] = '/' + watchurl
+    while nodeurls:
+        for node in list(nodeurls):
+            progress, status, err = get_update_progress(
+                session, nodeurls[node])
+            if status == 'error':
+                exitcode = 1
+                noderrs[node] = err
+            if status in ('error', 'complete', 'pending'):
+                list(session.delete(nodeurls[node]))
+                del nodeurls[node]
+            output.set_output(node, progress)
+        time.sleep(2)
+    allerrnodes = ','.join(noderrs)
+    if noderrs:
+        sys.stderr.write(
+            'Nodes had errors retrieving service data ({0})!\n'.format(allerrnodes))
+    for node in noderrs:
+        sys.stderr.write('{0}: {1}\n'.format(node, noderrs[node]))
+
+funmap = {
+    'servicedata': download_servicedata,
+}
+
+
+class OptParser(optparse.OptionParser):
+
+    def format_epilog(self, formatter):
+        return self.expand_prog_name(self.epilog)
+
+def main():
+    argparser = OptParser(
+        usage="Usage: %prog <noderange> [servicedata] "
+              "<filename>",
+        epilog='\nservicedata will save service data to the given '
+               'directory\n'
+               '\n\nSee `man %prog` for more info.\n')
+    (options, args) = argparser.parse_args()
+    media = None
+    try:
+        noderange = args[0]
+        operation = args[1]
+        arglength = 2
+        if operation == 'servicedata':
+            media = args[2]
+            arglength = 3
+        if len(args) > arglength:
+            argparser.print_help()
+            sys.exit(1)
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    try:
+        handler = funmap[operation]
+    except KeyError:
+        argparser.print_help()
+        sys.exit(1)
+    handler(noderange, media)
+if __name__ == '__main__':
+    main()

confluent_client/confluent/client.py

@@ -17,6 +17,7 @@
 
 import anydbm as dbm
 import errno
+import fnmatch
 import hashlib
 import os
 import shlex
@@ -24,6 +25,7 @@ import socket
 import ssl
 import sys
 import confluent.tlvdata as tlvdata
+import confluent.sortutil as sortutil
 
 SO_PASSCRED = 16
 
@@ -34,6 +36,41 @@ _attraliases = {
 }
 
 
+class Tabulator(object):
+    def __init__(self, headers):
+        self.headers = headers
+        self.rows = []
+
+    def add_row(self, row):
+        self.rows.append(row)
+
+    def get_table(self, order=None):
+        i = 0
+        fmtstr = ''
+        separator = []
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            neededlen = len(head)
+            for row in self.rows:
+                if len(row[i]) > neededlen:
+                    neededlen = len(row[i])
+            separator.append('-' * (neededlen + 1))
+            fmtstr += '{{{0}:>{1}}}|'.format(i, neededlen + 1)
+            i = i + 1
+        fmtstr = fmtstr[:-1]
+        yield fmtstr.format(*self.headers)
+        yield fmtstr.format(*separator)
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                yield fmtstr.format(*row)
+        else:
+            for row in self.rows:
+                yield fmtstr.format(*row)
+
+
 def printerror(res, node=None):
     exitcode = 0
     if 'errorcode' in res:
@@ -311,7 +348,7 @@ def attrrequested(attr, attrlist, seenattributes):
             candidate = candidate.replace('hm', 'hardwaremanagement', 1)
         if candidate in _attraliases:
             candidate = _attraliases[candidate]
-        if candidate.lower() == attr.lower():
+        if fnmatch.fnmatch(attr.lower(), candidate.lower()):
             seenattributes.add(truename)
             return True
         elif attr.lower().startswith(candidate.lower() + '.'):
@@ -493,7 +530,7 @@ def printgroupattributes(session, requestargs, showtype, nodetype, noderange, op
                     exitcode = 1
     return exitcode
 
-def updateattrib(session, updateargs, nodetype, noderange, options):
+def updateattrib(session, updateargs, nodetype, noderange, options, dictassign=None):
     # update attribute
     exitcode = 0
     if options.clear:
@@ -526,6 +563,14 @@ def updateattrib(session, updateargs, nodetype, noderange, options):
                                                             'attributes/all',
                                                             value, key)
         sys.exit(exitcode)
+    elif dictassign:
+        for key in dictassign:
+            if nodetype == 'nodegroups':
+                exitcode = session.simple_nodegroups_command(
+                    noderange, 'attributes/all', dictassign[key], key)
+            else:
+                exitcode = session.simple_noderange_command(
+                    noderange, 'attributes/all', dictassign[key], key)
     else:
         if "=" in updateargs[1]:
             try:

confluent_client/confluent_env.sh

@@ -36,4 +36,241 @@ alias nodereseat='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export C
 alias noderun='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; noderun'
 alias nodesensors='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesensors'
 alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesetboot'
+alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
+alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
+
+
+_confluent_get_args()
+{
+    CMPARGS=($COMP_LINE)
+    NUMARGS=${#CMPARGS[@]}
+    if [ "${COMP_WORDS[-1]}" == '' ]; then
+        NUMARGS=$((NUMARGS+1))
+        CMPARGS+=("")
+    fi
+    GENNED=""
+    for CAND in ${COMP_CANDIDATES[@]}; do
+        candarray=(${CAND//,/ })
+        matched=0
+        for c in "${candarray[@]}"; do
+            for arg in "${CMPARGS[@]}"; do
+                if [ "$arg" = "$c" ]; then
+                    matched=1
+                    break
+                fi
+            done
+        done
+        if [ 0 = $matched ]; then
+            for c in "${candarray[@]}"; do
+                GENNED+=" $c"
+            done
+        fi
+    done
+}
+
+function _confluent_generic_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+_confluent_nodeidentify_completion()
+{
+    COMP_CANDIDATES=("on,off -h")
+    _confluent_generic_completion
+}
+
+
+_confluent_nodesetboot_completion()
+{
+    COMP_CANDIDATES=("default,cd,network,setup,hd -h -b -p")
+    _confluent_generic_completion
+}
+
+_confluent_nodepower_completion()
+{
+   COMP_CANDIDATES=("boot,off,on,status -h -p")
+   _confluent_generic_completion
+}
+
+_confluent_nodemedia_completion()
+{
+    COMP_CANDIDATES=("list,upload,attach,detachall -h")
+    _confluent_get_args
+    if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[-2]} == 'upload' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -ge 3 ] && [ ! -z "$GENNED" ]; then
+        COMPREPLY=($(compgen -W "$GENNED" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodefirmware_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "list update" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS -gt 3 ] && [ ${CMPARGS[2]} == 'update' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodeshell_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -c -- ${COMP_WORDS[-1]}))
+        return
+    fi
+    if [ $NUMARGS -gt 3 ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+        _confluent_nr_completion
+        return;
+    fi
+}
+
+_confluent_nodelicense_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "install list" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
+        compopt -o default
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+         _confluent_nr_completion
+         return
+    fi
+}
+
+_confluent_nodesupport_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS == 3 ]; then
+        COMPREPLY=($(compgen -W "servicedata" -- ${COMP_WORDS[-1]}))
+        return;
+    fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'servicedata' ]; then
+        compopt -o  dirnames
+        COMPREPLY=()
+        return
+    fi
+    if [ $NUMARGS -lt 3 ]; then
+         _confluent_nr_completion
+         return
+    fi
+}
+
+_confluent_nn_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    COMPREPLY=($(compgen -W "$(nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+_confluent_nr_completion()
+{
+    CMPARGS=($COMP_LINE)
+    NUMARGS=${#CMPARGS[@]}
+    if [ "${COMP_WORDS[-1]}" == '' ]; then
+        NUMARGS=$((NUMARGS+1))
+    fi
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    #COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/;nodelist | sed -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+_confluent_ng_completion()
+{
+    _confluent_get_args
+    if [ $NUMARGS -gt 2 ]; then
+        return;
+    fi
+    INPUT=${COMP_WORDS[-1]}
+    INPUT=${INPUT##*,-}
+    INPUT=${INPUT##*,}
+    INPUT=${INPUT##*@}
+    PREFIX=""
+    if [ "$INPUT" != "${COMP_WORDS[-1]}" ]; then
+        PREFIX=${COMP_WORDS[-1]}
+        PREFIX=$(echo $PREFIX | sed -e 's/,[^,@-]*$/,/' -e 's/,-[^,@]*$/,-/' -e 's/@[^,@]*/@/')
+    fi
+
+    COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
+}
+complete -F _confluent_nr_completion nodeattrib
+complete -F _confluent_nr_completion nodebmcreset
+complete -F _confluent_nodesetboot_completion nodeboot
+complete -F _confluent_nr_completion nodeconfig
+complete -F _confluent_nn_completion nodeconsole
+complete -F _confluent_nr_completion nodeeventlog
+complete -F _confluent_nodefirmware_completion nodefirmware
+complete -F _confluent_ng_completion nodegroupattrib
+complete -F _confluent_ng_completion nodegroupremove
+complete -F _confluent_nr_completion nodehealth
+complete -F _confluent_nodeidentify_completion nodeidentify
+complete -F _confluent_nr_completion nodeinventory
+complete -F _confluent_nr_completion nodelist
+complete -F _confluent_nodemedia_completion nodemedia
+complete -F _confluent_nodepower_completion nodepower
+complete -F _confluent_nr_completion noderemove
+complete -F _confluent_nr_completion nodereseat
+complete -F _confluent_nodeshell_completion noderun
+complete -F _confluent_nr_completion nodesensors
+complete -F _confluent_nodesetboot_completion nodesetboot
+complete -F _confluent_nodeshell_completion nodeshell
+complete -F _confluent_nodesupport_completion nodesupport
+complete -F _confluent_nodelicense_completion nodelicense
+

confluent_client/doc/man/collate.ronn

@@ -3,7 +3,7 @@ collate(1) -- Organize text input by node
 
 ## SYNOPSIS
 
-`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r]`
+`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r] [-l lognametemplate]`
 
 ## DESCRIPTION
 
@@ -26,6 +26,10 @@ node and group names sorted alphanumerically.
    Express all but the most common result group in terms of diff from
    the most common result group
 
+* `-l`, `--log`:
+   Save output per node to individual log files, replacing {node} in the name
+   with the nodename of each
+
 * `-w`, `--watch`:
    Update results dynamically as data becomes available, rather than
    waiting    for the command to fully complete.
@@ -85,4 +89,4 @@ node and group names sorted alphanumerically.
   `  Processors.ExecuteDisableBit=Enable`  
 
    
-   
+   

confluent_client/doc/man/nodeattrib.ronn

@@ -7,6 +7,7 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
 
 ## DESCRIPTION
 
@@ -40,6 +41,8 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
   Clear specified nodeattributes
 * `-e`, `--environment`:
   Set specified attributes based on exported environment variable of matching name.  Environment variable names may be lower case or all upper case.  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+* `-p`, `--prompt`:
+  Request interactive prompting to provide values rather than the command line or environment variables.
 
 ## EXAMPLES
 * Listing matching nodes of a simple noderange:

confluent_client/doc/man/nodeboot.ronn

@@ -4,12 +4,44 @@ nodeboot(8) -- Reboot a confluent node to a specific device
 ## SYNOPSIS
 
 `nodeboot <noderange>`  
-`nodeboot <noderange>` [net|setup]  
+`nodeboot [options] <noderange>` [default|cd|network|setup|hd]  
 
 ## DESCRIPTION
 
 **nodeboot** reboots nodes in a noderange.  If an additional argument is given,
-it sets the node to specifically boot to that as the next boot.
+it sets the node to specifically boot to that as the next boot.  This 
+performs an immediate reboot without waiting for the OS.  To set the boot
+device without inducing a reboot, see the `nodesetboot` command.
+
+## OPTIONS
+
+* `-b`, `--bios`:
+  For a system that supports both BIOS and UEFI style boot, request BIOS style
+  boot if supported (some platforms will UEFI boot with this flag anyway).
+
+* `-u`, `--uefi`:
+  This flag does nothing, it is for command compatibility with xCAT's rsetboot
+
+* `-p`, `--persist`:
+  For a system that supports it, mark the boot override to persist rather than
+  be a one time change.  Many systems do not support this functionality.
+
+* `default`:
+  Request a normal default boot with no particular device override
+
+* `cd`:
+  Request boot from media.  Note that this can include physical CD,
+  remote media mounted as CD/DVD, and detachable hard disks drives such as usb
+  key devices.
+  
+* `network`:
+  Request boot to network
+  
+* `setup`:
+  Request to enter the firmware configuration menu (e.g. F1 setup) on next boot.
+
+* `hd`:
+  Boot straight to hard disk drive
 
 ## EXAMPLES
 * Booting n3 and n4 to the default boot behavior:

confluent_client/doc/man/nodeconfig.ronn

@@ -30,6 +30,14 @@ given as a node expression, as documented in the man page for nodeattribexpressi
   Provide detailed data as available.  This can include help text and valid
   values for a setting.
 
+* `-r`, `--restoredefault`:
+  Request that the specified component of the targeted nodes will have its
+  configuration reset to default.  Currently the only component implemented
+  is uefi.
+
+* `-b`, '--batch':
+   Provide arguments as lines of a file, rather than the command line.
+
 
 ## EXAMPLES
 * Showing the current IP configuration of noderange BMC/IMM/XCC:

confluent_client/doc/man/nodeconsole.ronn

@@ -2,7 +2,7 @@ nodeconsole(8) -- Open a console to a confluent node
 =====================================================
 
 ## SYNOPSIS
-`nodeconsole <node>`
+`nodeconsole [options] <noderange>`
 
 ## DESCRIPTION
 
@@ -16,6 +16,12 @@ will initiate an automatic retry interval that is randomized between 2 and 4 min
 The reopen escape sequence below requests an immediate retry, as does connecting
 a new session.
 
+## OPTIONS
+
+* `-t`, `--tile`:
+  Use tmux to arrange consoles of the given noderange into a tiled layout on
+  the terminal screen
+
 ## ESCAPE SEQUENCE COMMANDS
 
 While connected to a console, a number of commands may be performed through escape

confluent_client/doc/man/nodedefine.ronn

@@ -1,4 +1,5 @@
 nodedefine(8) -- Define new confluent nodes
+===================================================================
 
 ## SYNOPSIS
 

confluent_client/doc/man/nodediscover.ronn

@@ -75,6 +75,20 @@ the nodes.
 * `-e MAC`, `--ethaddr=MAC`:
   Operate against the system with the specified MAC
   address
+* `-f FIELDS`, `--fields=FIELDS`:
+  Request a custom set of fields.  The available fields are:  
+  Node: The node name if a correlation has been identified  
+  Model: The model number  
+  Serial: The serial number  
+  UUID: The UUID as it should appear in DMI  
+  Type: Device type (e.g. lenovo-xcc, pxe-client, etc)  
+  IP: The confirmed working IP addresses associated with the record  
+  Mac: Mac address of the relevant network interface  
+  Switch: The nearest detected switch to the entry  
+  Port: Port of the switch that most closely connects to the network interface  
+  Advertised IP: IP addresses that may not have been confirmed, but are advertised  
+* `-o ORDER`, `--order=ORDER`:
+  Order output by given field.  Field names are the same as documented in the -f argument.
 * `-t TYPE`, `--type=TYPE`:
   Operate against the system of the specified type
 * `-c`, `--csv`:

confluent_client/doc/man/nodefirmware.ronn

@@ -4,7 +4,7 @@ nodefirmware(8) -- Report firmware information on confluent nodes
 ## SYNOPSIS
 
 `nodefirmware <noderange>`  
-`nodefirmware <noderange> <components>|core`  
+`nodefirmware <noderange> list|<components>|core`  
 `nodefirmware <noderange> update [--backup] <filename>`  
 
 ## DESCRIPTION

confluent_client/doc/man/nodegroupattrib.ronn

@@ -7,6 +7,7 @@ nodegroupattrib(8) -- List or change confluent nodegroup attributes
 `nodegroupattrib <group> [<nodeattribute>...]`  
 `nodegroupattrib <group> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodegroupattrib <group> [-c] [<nodeattribute1> <nodeattribute2=value2> ...]`  
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`  
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodegroupdefine.ronn

@@ -1,4 +1,5 @@
 nodegroupdefine(8) -- Define new confluent node group
+===================================================================
 
 ## SYNOPSIS
 
@@ -6,7 +7,7 @@ nodegroupdefine(8) -- Define new confluent node group
 
 ## DESCRIPTION
 
-`nodegroupdefine` allows the definition of a new node for the confluent management
+`nodegroupdefine` allows the definition of a new nodegroup for the confluent management
 service.  It may only define a single group name at a time.
 It has the same syntax as `nodegroupattrib(8)`, and the commands differ in
 that `nodegroupattrib(8)` will error if a node group does not exist.

confluent_client/doc/man/nodelicense.ronn

@@ -0,0 +1,11 @@
+nodelicense(8) -- Manage license keys on BMC
+=================================================================
+
+## SYNOPSIS
+
+`nodelicense <noderange> [list|install <filename>]`
+
+## DESCRIPTION
+
+`nodelicense` shows and installs license keys on supported BMCs
+

confluent_client/doc/man/nodesetboot.ronn

@@ -30,7 +30,10 @@ control.
 * `-p`, `--persist`:
   For a system that supports it, mark the boot override to persist rather than
   be a one time change.  Many systems do not support this functionality.
-  
+
+* `-u`, `--uefi`:
+  This flag does nothing, it is for command compatibility with xCAT's rsetboot
+
 * `default`:
   Request a normal default boot with no particular device override
 

confluent_client/doc/man/nodesupport.ronn

@@ -0,0 +1,25 @@
+nodesupport(8) -- Utilities for interacting with vendor support
+=================================================================
+
+## SYNOPSIS
+
+`nodesupport <noderange> servicedata <directory or filename>`  
+
+## DESCRIPTION
+
+`nodesupport` provides capabilities associated with interactiong with support.
+Currently it only has the `servicedata` subcommand.  `servicedata` takes
+an argument that is either a directory name (that can be used for a single node
+or multiple nodes) or a file name (only to be used with single node noderange)
+
+## EXAMPLES
+
+* Download support data from a single node to a specific filename  
+`# nodesupport d1 servicedata svcdata.out`  
+`d1:initializing:  15%`  
+
+* Download support data from multiple nodes to a directory  
+`# nodesupport d1-d4 servicedata service/`    
+`d1:initializing:   0% d2:initializing:   0% d3:initializing:   0% d4:initializing:   0%`  
+`# ls service/`  
+`d1.svcdata d2.svcdata  d3.svcdata  d4.svcdata`  

confluent_common/setup.cfg

@@ -1,17 +0,0 @@
-[metadata]
-name = confluent_common
-summary = Confluent Common Libraries
-description-file =
-    README.txt
-author = Jarrod Johnson
-author-email = jjohnson2@lenovo.com
-home-page = http://xcat.sf.net/
-classifier =
-    Intended Audience :: Information Technology
-    Intended Audience :: System Administrators
-    License :: OSI Approved :: Apache Software License
-    Operating System :: POSIX :: Linux
-    Programming Language :: Python :: 2.6
-    Programming Language :: Python :: 2.7
-
-[files]

confluent_server/MANIFEST.in

@@ -1,3 +1,4 @@
+include pam/*
 include sysvinit/*
 include systemd/*
 include sysctl/*

confluent_server/bin/collective

@@ -66,8 +66,9 @@ def join_collective(server, invitation):
                                     'invitation': invitation,
                                     'server': server}})
     res = tlvdata.recv(s)
-    print(res.get('collective',
-                  {'status': 'Unknown response: ' + repr(res)})['status'])
+    res = res.get('collective',
+                  {'status': 'Unknown response: ' + repr(res)})
+    print(res.get('status', res.get('error', repr(res))))
 
 
 def show_collective():

confluent_server/bin/confluentcertutil.py

@@ -0,0 +1,50 @@
+import shutil
+import socket
+import subprocess
+import tempfile
+
+def get_openssl_conf_location():
+    # CentOS/RHAT
+    return '/etc/pki/tls/openssl.cnf'
+
+def get_ip_addresses():
+    lines = subprocess.check_output('ip addr'.split(' '))
+    for line in lines.split('\n'):
+        if line.startswith('    inet6 '):
+            line = line.replace('    inet6 ', '').split('/')[0]
+            if line.startswith('fe80::'):
+                continue
+            if line == '::1':
+                continue
+        elif line.startswith('    inet '):
+            line = line.replace('    inet ', '').split('/')[0]
+            if line == '127.0.0.1':
+                continue
+            if line.startswith('169.254.'):
+                continue
+        else:
+            continue
+        yield line
+
+def create_certificate():
+    shortname = socket.gethostname().split('.')[0]
+    longname = socket.getfqdn()
+    subprocess.check_call(
+        'openssl ecparam -name secp384r1 -genkey -out privkey.pem'.split(' '))
+    san = ['IP:{0}'.format(x) for x in get_ip_addresses()]
+    san.append('DNS:{0}'.format(shortname))
+    san.append('DNS:{0}'.format(longname))
+    san = ','.join(san)
+    sslcfg = get_openssl_conf_location()
+    tmpconfig = tempfile.mktemp()
+    shutil.copy2(sslcfg, tmpconfig)
+    with open(tmpconfig, 'a') as cfgfile:
+        cfgfile.write('\n[SAN]\nsubjectAltName={0}'.format(san))
+    subprocess.check_call(
+        'openssl req -new -x509 -key privkey.pem -days 7300 -out cert.pem '
+        '-subj /CN={0} -extensions SAN '
+        '-config {1}'.format(longname, tmpconfig).split(' ')
+    )
+
+if __name__ == '__main__':
+    create_certificate()

confluent_server/confluent/asynchttp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,6 +83,8 @@ class AsyncSession(object):
         self.reaper = eventlet.spawn_after(15, self.destroy)
 
     def add(self, requestid, rsp):
+        if self.responses is None:
+            return
         self.responses.append((requestid, rsp))
         if self._evt:
             self._evt.send()
@@ -107,6 +109,7 @@ class AsyncSession(object):
         for console in self.consoles:
             _consolesessions[console]['session'].destroy()
         self.consoles = None
+        self.responses = None
         del _asyncsessions[self.asyncid]
 
     def run_handler(self, handler, requestid):
@@ -166,6 +169,9 @@ def handle_async(env, querydict, threadset):
         currsess = AsyncSession()
         yield messages.AsyncSession(currsess.asyncid)
         return
+    if  querydict['asyncid'] not in _asyncsessions:
+        raise exc.InvalidArgumentException(
+                'Invalid or expired async id')
     mythreadid = greenlet.getcurrent()
     threadset.add(mythreadid)
     loggedout = None

confluent_server/confluent/auth.py

@@ -158,20 +158,13 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         # but here there's no need for that
         eventlet.sleep(0.5)
     credobj = Credentials(user, passphrase)
-    try:
-        pammy = PAM.pam()
-        pammy.start(_pamservice, user, credobj.pam_conv)
-        pammy.authenticate()
-        pammy.acct_mgmt()
-        del pammy
-        return authorize(user, element, tenant, skipuserobj=False)
-    except NameError:
-        pass
-    except PAM.error:
-        if credobj.haspam:
-            return None
+    cfm = configmanager.ConfigManager(tenant, username=user)
+    ucfg = cfm.get_user(user)
+    if ucfg is None:
+        eventlet.sleep(0.05)
+        return None
     if (user, tenant) in _passcache:
-        if passphrase == _passcache[(user, tenant)]:
+        if hashlib.sha256(passphrase).digest() == _passcache[(user, tenant)]:
             return authorize(user, element, tenant)
         else:
             # In case of someone trying to guess,
@@ -179,38 +172,48 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
             # invalidate cache and force the slower check
             del _passcache[(user, tenant)]
             return None
-    cfm = configmanager.ConfigManager(tenant, username=user)
-    ucfg = cfm.get_user(user)
-    if ucfg is None or 'cryptpass' not in ucfg:
-        eventlet.sleep(0.05)  # stall even on test for existence of a username
-        return None
-    _passchecking[(user, tenant)] = True
-    # TODO(jbjohnso): WORKERPOOL
-    # PBKDF2 is, by design, cpu intensive
-    # throw it at the worker pool when implemented
-    # maybe a distinct worker pool, wondering about starving out non-auth stuff
-    salt, crypt = ucfg['cryptpass']
-    # execute inside tpool to get greenthreads to give it a special thread
-    # world
-    #TODO(jbjohnso): util function to generically offload a call
-    #such a beast could be passed into pyghmi as a way for pyghmi to
-    #magically get offload of the crypto functions without having
-    #to explicitly get into the eventlet tpool game
-    global authworkers
-    global authcleaner
-    if authworkers is None:
-        authworkers = multiprocessing.Pool(processes=1)
-    else:
-        authcleaner.cancel()
-    authcleaner = eventlet.spawn_after(30, _clean_authworkers)
-    crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
-    del _passchecking[(user, tenant)]
-    eventlet.sleep(0.05)  # either way, we want to stall so that client can't
+    if 'cryptpass' in ucfg:
+        _passchecking[(user, tenant)] = True
+        # TODO(jbjohnso): WORKERPOOL
+        # PBKDF2 is, by design, cpu intensive
+        # throw it at the worker pool when implemented
+        # maybe a distinct worker pool, wondering about starving out non-auth stuff
+        salt, crypt = ucfg['cryptpass']
+        # execute inside tpool to get greenthreads to give it a special thread
+        # world
+        # TODO(jbjohnso): util function to generically offload a call
+        # such a beast could be passed into pyghmi as a way for pyghmi to
+        # magically get offload of the crypto functions without having
+        # to explicitly get into the eventlet tpool game
+        global authworkers
+        global authcleaner
+        if authworkers is None:
+            authworkers = multiprocessing.Pool(processes=1)
+        else:
+            authcleaner.cancel()
+        authcleaner = eventlet.spawn_after(30, _clean_authworkers)
+        crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
+        del _passchecking[(user, tenant)]
+        eventlet.sleep(
+            0.05)  # either way, we want to stall so that client can't
         # determine failure because there is a delay, valid response will
         # delay as well
-    if crypt == crypted:
-        _passcache[(user, tenant)] = passphrase
-        return authorize(user, element, tenant)
+        if crypt == crypted:
+            _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+            return authorize(user, element, tenant)
+    try:
+        pammy = PAM.pam()
+        pammy.start(_pamservice, user, credobj.pam_conv)
+        pammy.authenticate()
+        pammy.acct_mgmt()
+        del pammy
+        _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
+        return authorize(user, element, tenant, skipuserobj=False)
+    except NameError:
+        pass
+    except PAM.error:
+        pass
+    eventlet.sleep(0.05)  # stall even on test for existence of a username
     return None
 
 

confluent_server/confluent/collective/manager.py

@@ -18,12 +18,14 @@ import base64
 import confluent.collective.invites as invites
 import confluent.config.configmanager as cfm
 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.util as util
 import eventlet
 import eventlet.green.socket as socket
 import eventlet.green.ssl as ssl
 import eventlet.green.threading as threading
+import greenlet
 import random
 try:
     import OpenSSL.crypto as crypto
@@ -33,38 +35,41 @@ except ImportError:
     crypto = None
 
 currentleader = None
-cfginitlock = None
 follower = None
 retrythread = None
 
 class ContextBool(object):
     def __init__(self):
         self.active = False
+        self.mylock = threading.RLock()
 
     def __enter__(self):
         self.active = True
+        self.mylock.__enter__()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.active = False
+        self.mylock.__exit__(exc_type, exc_val, exc_tb)
 
 connecting = ContextBool()
 leader_init = ContextBool()
 
-
 def connect_to_leader(cert=None, name=None, leader=None):
     global currentleader
-    global cfginitlock
     global follower
-    if cfginitlock is None:
-        cfginitlock = threading.RLock()
     if leader is None:
         leader = currentleader
+    log.log({'info': 'Attempting connection to leader {0}'.format(leader),
+             'subsystem': 'collective'})
     try:
         remote = connect_to_collective(cert, leader)
-    except socket.error:
+    except socket.error as e:
+        log.log({'error': 'Collective connection attempt to {0} failed: {1}'
+                          ''.format(leader, str(e)),
+                 'subsystem': 'collective'})
         return False
     with connecting:
-        with cfginitlock:
+        with cfm._initlock:
             tlvdata.recv(remote)  # the banner
             tlvdata.recv(remote)  # authpassed... 0..
             if name is None:
@@ -77,10 +82,16 @@ def connect_to_leader(cert=None, name=None, leader=None):
                 return False
             if 'error' in keydata:
                 if 'backoff' in keydata:
-                    eventlet.spawn_after(random.random(), connect_to_leader,
-                                         cert, name, leader)
-                    return True
+                    log.log({
+                        'info': 'Collective initialization in progress on '
+                                '{0}'.format(leader),
+                        'subsystem': 'collective'})
+                    return False
                 if 'leader' in keydata:
+                    log.log(
+                        {'info': 'Prospective leader {0} has redirected this '
+                                 'member to {1}'.format(leader, keydata['leader']),
+                         'subsystem': 'collective'})
                     ldrc = cfm.get_collective_member_by_address(
                         keydata['leader'])
                     if ldrc and ldrc['name'] == name:
@@ -88,13 +99,22 @@ def connect_to_leader(cert=None, name=None, leader=None):
                     return connect_to_leader(name=name,
                                              leader=keydata['leader'])
                 if 'txcount' in keydata:
+                    log.log({'info':
+                                 'Prospective leader {0} has inferior '
+                                 'transaction count, becoming leader'
+                                 ''.format(leader), 'subsystem': 'collective',
+                             'subsystem': 'collective'})
                     return become_leader(remote)
-                print(keydata['error'])
                 return False
-            if follower is not None:
                 follower.kill()
                 cfm.stop_following()
                 follower = None
+            if follower:
+                follower.kill()
+                cfm.stop_following()
+                follower = None
+            log.log({'info': 'Following leader {0}'.format(leader),
+                     'subsystem': 'collective'})
             colldata = tlvdata.recv(remote)
             globaldata = tlvdata.recv(remote)
             dbi = tlvdata.recv(remote)
@@ -134,11 +154,22 @@ def connect_to_leader(cert=None, name=None, leader=None):
 
 def follow_leader(remote):
     global currentleader
-    cfm.follow_channel(remote)
-    # The leader has folded, time to startup again...
-    cfm.stop_following()
-    currentleader = None
-    eventlet.spawn_n(start_collective)
+    cleanexit = False
+    try:
+        cfm.follow_channel(remote)
+    except greenlet.GreenletExit:
+        cleanexit = True
+    finally:
+        if cleanexit:
+            log.log({'info': 'Previous following cleanly closed',
+                     'subsystem': 'collective'})
+            return
+        log.log({'info': 'Current leader has disappeared, restarting '
+                         'collective membership', 'subsystem': 'collective'})
+        # The leader has folded, time to startup again...
+        cfm.stop_following()
+        currentleader = None
+        eventlet.spawn_n(start_collective)
 
 
 def connect_to_collective(cert, member):
@@ -291,6 +322,8 @@ def handle_connection(connection, cert, request, local=False):
             f = open('/etc/confluent/cfg/myname', 'w')
             f.write(name)
             f.close()
+            log.log({'info': 'Connecting to collective due to join',
+                     'subsystem': 'collective'})
             eventlet.spawn_n(connect_to_leader, rsp['collective'][
                 'fingerprint'], name)
     if 'enroll' == operation:
@@ -320,6 +353,11 @@ def handle_connection(connection, cert, request, local=False):
     if 'assimilate' == operation:
         drone = request['name']
         droneinfo = cfm.get_collective_member(drone)
+        if not droneinfo:
+            tlvdata.send(connection,
+                         {'error': 'Unrecognized leader, '
+                                   'redo invitation process'})
+            return
         if not util.cert_matches(droneinfo['fingerprint'], cert):
             tlvdata.send(connection,
                          {'error': 'Invalid certificate, '
@@ -329,8 +367,22 @@ def handle_connection(connection, cert, request, local=False):
             tlvdata.send(connection,
                          {'error': 'Refusing to be assimilated by inferior'
                                    'transaction count',
-                          'txcount': cfm._txcount})
+                          'txcount': cfm._txcount,})
             return
+        if connecting.active:
+            # don't try to connect while actively already trying to connect
+            tlvdata.send(connection, {'status': 0})
+            connection.close()
+            return
+        if (currentleader == connection.getpeername()[0] and
+                follower and follower.isAlive()):
+            # if we are happily following this leader already, don't stir
+            # the pot
+            tlvdata.send(connection, {'status': 0})
+            connection.close()
+            return
+        log.log({'info': 'Connecting in response to assimilation',
+                 'subsystem': 'collective'})
         eventlet.spawn_n(connect_to_leader, None, None,
                          leader=connection.getpeername()[0])
         tlvdata.send(connection, {'status': 0})
@@ -359,6 +411,11 @@ def handle_connection(connection, cert, request, local=False):
             connection.close()
             return
         myself = connection.getsockname()[0]
+        if connecting.active:
+            tlvdata.send(connection, {'error': 'Connecting right now',
+                                      'backoff': True})
+            connection.close()
+            return
         if myself != get_leader(connection):
             tlvdata.send(
                 connection,
@@ -366,17 +423,14 @@ def handle_connection(connection, cert, request, local=False):
                           'in another castle', 'leader': currentleader})
             connection.close()
             return
-        if connecting.active:
-            tlvdata.send(connection, {'error': 'Connecting right now',
-                                      'backoff': True})
-            connection.close()
-            return
         if request['txcount'] > cfm._txcount:
             retire_as_leader()
             tlvdata.send(connection,
                          {'error': 'Client has higher tranasaction count, '
                                    'should assimilate me, connecting..',
                           'txcount': cfm._txcount})
+            log.log({'info': 'Connecting to leader due to superior '
+                             'transaction count', 'subsystem': collective})
             eventlet.spawn_n(connect_to_leader, None, None,
                              connection.getpeername()[0])
             connection.close()
@@ -430,11 +484,36 @@ def try_assimilate(drone):
     tlvdata.recv(remote)  # the banner
     tlvdata.recv(remote)  # authpassed... 0..
     answer = tlvdata.recv(remote)
-    if answer and 'error' in answer:
+    if not answer:
+        log.log(
+            {'error':
+                 'No answer from {0} while trying to assimilate'.format(
+                     drone),
+            'subsystem': 'collective'})
+        return
+    if 'txcount' in answer:
+        log.log({'info': 'Deferring to {0} due to transaction count'.format(
+            drone), 'subsystem': 'collective'})
         connect_to_leader(None, None, leader=remote.getpeername()[0])
+        return
+    if 'error' in answer:
+        log.log({
+            'error': 'Error encountered while attempting to '
+                     'assimilate {0}: {1}'.format(drone, answer['error']),
+            'subsystem': 'collective'})
+        return
+    log.log({'info': 'Assimilated {0} into collective'.format(drone),
+             'subsystem': 'collective'})
+
 
 def get_leader(connection):
     if currentleader is None or connection.getpeername()[0] == currentleader:
+        if currentleader is None:
+            msg = 'Becoming leader as no leader known'
+        else:
+            msg = 'Becoming leader because {0} attempted to connect and it ' \
+                  'is current leader'.format(currentleader)
+        log.log({'info': msg, 'subsystem': 'collective'})
         become_leader(connection)
     return currentleader
 
@@ -446,27 +525,34 @@ def retire_as_leader():
 def become_leader(connection):
     global currentleader
     global follower
+    global retrythread
+    log.log({'info': 'Becoming leader of collective',
+             'subsystem': 'collective'})
     if follower:
         follower.kill()
+        cfm.stop_following()
         follower = None
+    if retrythread:
+        retrythread.cancel()
+        retrythread = None
     currentleader = connection.getsockname()[0]
     skipaddr = connection.getpeername()[0]
     myname = get_myname()
+    skipem = set(cfm.cfgstreams)
+    skipem.add(currentleader)
+    skipem.add(skipaddr)
     for member in cfm.list_collective():
         dronecandidate = cfm.get_collective_member(member)['address']
-        if dronecandidate in (currentleader, skipaddr) or member == myname:
+        if dronecandidate in skipem or member == myname:
             continue
         eventlet.spawn_n(try_assimilate, dronecandidate)
 
 
 def startup():
-    global cfginitlock
     members = list(cfm.list_collective())
     if len(members) < 2:
         # Not in collective mode, return
         return
-    if cfginitlock is None:
-        cfginitlock = threading.RLock()
     eventlet.spawn_n(start_collective)
 
 def start_collective():
@@ -474,7 +560,15 @@ def start_collective():
     global retrythread
     if follower:
         follower.kill()
+        cfm.stop_following()
         follower = None
+    try:
+        if cfm.cfgstreams:
+            cfm.check_quorum()
+            # Do not start if we have quorum and are leader
+            return
+    except exc.DegradedCollective:
+        pass
     if leader_init.active:  # do not start trying to connect if we are
         # xmitting data to a follower
         return
@@ -485,6 +579,8 @@ def start_collective():
         if cfm.cfgleader is None:
             cfm.stop_following(True)
         ldrcandidate = cfm.get_collective_member(member)['address']
+        log.log({'info': 'Performing startup attempt to {0}'.format(
+            ldrcandidate), 'subsystem': 'collective'})
         if connect_to_leader(name=myname, leader=ldrcandidate):
             break
     else:

confluent_server/confluent/config/attributes.py

@@ -156,6 +156,12 @@ node = {
                         'indicates candidate managers, either for '
                         'high availability or load balancing purposes.')
     },
+    'discovery.passwordrules': {
+        'description':  'Any specified rules shall be configured on the BMC '
+                        'upon discovery.  "expiration=no,loginfailures=no" '
+                        'would disable password expiration and login failures '
+                        'triggering a lockout.'
+    },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
                         'and identified nodes. Valid values are "manual", '

confluent_server/confluent/config/configmanager.py

@@ -82,6 +82,7 @@ _dirtylock = threading.RLock()
 _leaderlock = gthread.RLock()
 _synclock = threading.RLock()
 _rpclock = gthread.RLock()
+_initlock = gthread.RLock()
 _followerlocks = {}
 _config_areas = ('nodegroups', 'nodes', 'usergroups', 'users')
 tracelog = None
@@ -94,7 +95,9 @@ _hasquorum = True
 _attraliases = {
     'bmc': 'hardwaremanagement.manager',
     'bmcuser': 'secret.hardwaremanagementuser',
+    'switchuser': 'secret.hardwaremanagementuser',
     'bmcpass': 'secret.hardwaremanagementpassword',
+    'switchpass': 'secret.hardwaremanagementpassword',
 }
 
 def _mkpath(pathname):
@@ -623,7 +626,7 @@ def rollback_clear():
     _cfgstore = _oldcfgstore
     _oldtxcount = 0
     _oldcfgstore = None
-    ConfigManager._bg_sync_to_file()
+    ConfigManager.wait_for_sync(True)
 
 
 def clear_configuration():
@@ -643,12 +646,13 @@ def commit_clear():
     global _oldcfgstore
     _oldcfgstore = None
     _oldtxcount = 0
-    todelete = _config_areas + ('globals', 'collective', 'transactioncount')
-    for cfg in todelete:
-        try:
-            os.remove(os.path.join(ConfigManager._cfgdir, cfg))
-        except OSError as oe:
-            pass
+    with _synclock:
+        todelete = ('transactioncount', 'globals', 'collective') + _config_areas
+        for cfg in todelete:
+            try:
+                os.remove(os.path.join(ConfigManager._cfgdir, cfg))
+            except OSError as oe:
+                pass
     ConfigManager.wait_for_sync(True)
     ConfigManager._bg_sync_to_file()
 
@@ -658,41 +662,43 @@ cfgleader = None
 def follow_channel(channel):
     global _txcount
     global _hasquorum
-    stop_leading()
-    stop_following(channel)
-    lh = StreamHandler(channel)
-    msg = lh.get_next_msg()
-    while msg:
-        sz = struct.unpack('!Q', msg)[0]
-        if sz != 0:
-            rpc = ''
-            while len(rpc) < sz:
-                nrpc = channel.recv(sz - len(rpc))
-                if not nrpc:
-                    raise Exception('Truncated message error')
-                rpc += nrpc
-            rpc = cPickle.loads(rpc)
-            if 'txcount' in rpc:
-                _txcount = rpc['txcount']
-            if 'function' in rpc:
-                try:
-                    globals()[rpc['function']](*rpc['args'])
-                except Exception as e:
-                    print(repr(e))
-            if 'xid' in rpc and rpc['xid']:
-                if rpc.get('exc', None):
-                    _pendingchangesets[rpc['xid']].send_exception(rpc['exc'])
-                else:
-                    _pendingchangesets[rpc['xid']].send()
-            if 'quorum' in rpc:
-                _hasquorum = rpc['quorum']
-            _push_rpc(channel, b'')  # use null as ACK
+    try:
+        stop_leading()
+        stop_following(channel)
+        lh = StreamHandler(channel)
         msg = lh.get_next_msg()
-    # mark the connection as broken
-    if cfgstreams:
-        stop_following(None)
-    else:
-        stop_following(True)
+        while msg:
+            sz = struct.unpack('!Q', msg)[0]
+            if sz != 0:
+                rpc = ''
+                while len(rpc) < sz:
+                    nrpc = channel.recv(sz - len(rpc))
+                    if not nrpc:
+                        raise Exception('Truncated message error')
+                    rpc += nrpc
+                rpc = cPickle.loads(rpc)
+                if 'txcount' in rpc:
+                    _txcount = rpc['txcount']
+                if 'function' in rpc:
+                    try:
+                        globals()[rpc['function']](*rpc['args'])
+                    except Exception as e:
+                        print(repr(e))
+                if 'xid' in rpc and rpc['xid']:
+                    if rpc.get('exc', None):
+                        _pendingchangesets[rpc['xid']].send_exception(rpc['exc'])
+                    else:
+                        _pendingchangesets[rpc['xid']].send()
+                if 'quorum' in rpc:
+                    _hasquorum = rpc['quorum']
+                _push_rpc(channel, b'')  # use null as ACK
+            msg = lh.get_next_msg()
+    finally:
+        # mark the connection as broken
+        if cfgstreams:
+            stop_following(None)
+        else:
+            stop_following(True)
 
 
 def add_collective_member(name, address, fingerprint):
@@ -947,38 +953,46 @@ class ConfigManager(object):
     _nodecollwatchers = {}
     _notifierids = {}
 
+    @property
+    def _cfgstore(self):
+        if self.tenant is None:
+            return _cfgstore['main']
+        return _cfgstore['tenant'][self.tenant]
+
     def __init__(self, tenant, decrypt=False, username=None):
         global _cfgstore
-        if _cfgstore is None:
-            init()
-        self.decrypt = decrypt
-        self.current_user = username
-        if tenant is None:
-            self.tenant = None
-            if 'main' not in _cfgstore:
-                _cfgstore['main'] = {}
+        with _initlock:
+            if _cfgstore is None:
+                init()
+            self.decrypt = decrypt
+            self.current_user = username
+            if tenant is None:
+                self.tenant = None
+                if 'main' not in _cfgstore:
+                    _cfgstore['main'] = {}
+                    self._bg_sync_to_file()
+                if 'nodegroups' not in self._cfgstore:  # This can happen during a clear... it seams... and if so it messes up...
+                    self._cfgstore['nodegroups'] = {'everything': {'nodes': set()}}
+                    _mark_dirtykey('nodegroups', 'everything', self.tenant)
+                    self._bg_sync_to_file()
+                if 'nodes' not in self._cfgstore:
+                    self._cfgstore['nodes'] = {}
+                    self._bg_sync_to_file()
+                return
+            elif 'tenant' not in _cfgstore:
+                _cfgstore['tenant'] = {tenant: {}}
                 self._bg_sync_to_file()
-            self._cfgstore = _cfgstore['main']
+            elif tenant not in _cfgstore['tenant']:
+                _cfgstore['tenant'][tenant] = {}
+                self._bg_sync_to_file()
+            self.tenant = tenant
             if 'nodegroups' not in self._cfgstore:
-                self._cfgstore['nodegroups'] = {'everything': {'nodes': set()}}
-                self._bg_sync_to_file()
+                self._cfgstore['nodegroups'] = {'everything': {}}
+                _mark_dirtykey('nodegroups', 'everything', self.tenant)
             if 'nodes' not in self._cfgstore:
                 self._cfgstore['nodes'] = {}
-                self._bg_sync_to_file()
-            return
-        elif 'tenant' not in _cfgstore:
-            _cfgstore['tenant'] = {tenant: {}}
             self._bg_sync_to_file()
-        elif tenant not in _cfgstore['tenant']:
-            _cfgstore['tenant'][tenant] = {}
-            self._bg_sync_to_file()
-        self.tenant = tenant
-        self._cfgstore = _cfgstore['tenant'][tenant]
-        if 'nodegroups' not in self._cfgstore:
-            self._cfgstore['nodegroups'] = {'everything': {}}
-        if 'nodes' not in self._cfgstore:
-            self._cfgstore['nodes'] = {}
-        self._bg_sync_to_file()
+            self.wait_for_sync()
 
     def get_collective_member(self, name):
         return get_collective_member(name)
@@ -1226,9 +1240,9 @@ class ConfigManager(object):
 
     def del_user(self, name):
         if cfgleader:
-            return exec_on_leader('_rpc_master_del_user', name)
+            return exec_on_leader('_rpc_master_del_user', self.tenant, name)
         if cfgstreams:
-            exec_on_followers('_rpc_del_user', name)
+            exec_on_followers('_rpc_del_user', self.tenant, name)
         self._true_del_user(name)
 
     def _true_del_user(self, name):
@@ -2050,9 +2064,13 @@ class ConfigManager(object):
         if statelessmode:
             return
         with cls._syncstate:
-            if cls._syncrunning:
+            if (cls._syncrunning and cls._cfgwriter is not None and
+                    cls._cfgwriter.isAlive()):
                 cls._writepending = True
                 return
+            if cls._syncrunning:  # This suggests an unclean write attempt,
+                # do a fullsync as a recovery
+                fullsync = True
             cls._syncrunning = True
         # if the thread is exiting, join it to let it close, just in case
         if cls._cfgwriter is not None:
@@ -2065,16 +2083,17 @@ class ConfigManager(object):
         with _synclock:
             if statelessmode:
                 return
+            _mkpath(cls._cfgdir)
             with open(os.path.join(cls._cfgdir, 'transactioncount'), 'w') as f:
                 f.write(struct.pack('!Q', _txcount))
-            if fullsync or 'dirtyglobals' in _cfgstore:
-                if fullsync:
+            if (fullsync or 'dirtyglobals' in _cfgstore and
+                    'globals' in _cfgstore):
+                if fullsync:  # globals is not a given to be set..
                     dirtyglobals = _cfgstore['globals']
                 else:
                     with _dirtylock:
                         dirtyglobals = copy.deepcopy(_cfgstore['dirtyglobals'])
                         del _cfgstore['dirtyglobals']
-                _mkpath(cls._cfgdir)
                 globalf = dbm.open(os.path.join(cls._cfgdir, "globals"), 'c', 384)  # 0600
                 try:
                     for globalkey in dirtyglobals:

confluent_server/confluent/consoleserver.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2017 Lenovo
+# Copyright 2017-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -145,8 +145,9 @@ class ConsoleHandler(object):
     _genwatchattribs = frozenset(('console.method', 'console.logging',
                                   'collective.manager'))
 
-    def __init__(self, node, configmanager):
+    def __init__(self, node, configmanager, width=80, height=24):
         self.clearpending = False
+        self.initsize = (width, height)
         self._dologging = True
         self._is_local = True
         self._isondemand = False
@@ -359,8 +360,11 @@ class ConsoleHandler(object):
         except (exc.NotImplementedException, exc.NotFoundException):
             self._console = None
         except:
-            _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
-                          event=log.Events.stacktrace)
+            if _tracelog:
+                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                              event=log.Events.stacktrace)
+            else:
+                print(traceback.format_exc())
         if not isinstance(self._console, conapi.Console):
             self.clearbuffer()
             self.connectstate = 'unconnected'
@@ -369,6 +373,7 @@ class ConsoleHandler(object):
                               'error': self.error})
             return
         self.send_break = self._console.send_break
+        self.resize = self._console.resize
         if self._attribwatcher:
             self.cfgmgr.remove_watcher(self._attribwatcher)
             self._attribwatcher = None
@@ -380,6 +385,7 @@ class ConsoleHandler(object):
             self._attribwatcher = self.cfgmgr.watch_attributes(
                 (self.node,), attribstowatch, self._attribschanged)
         try:
+            self.resize(width=self.initsize[0], height=self.initsize[1])
             self._console.connect(self.get_console_output)
         except exc.TargetEndpointBadCredentials:
             self.clearbuffer()
@@ -625,17 +631,20 @@ def start_console_sessions():
     configmodule.hook_new_configmanagers(_start_tenant_sessions)
 
 
-def connect_node(node, configmanager, username=None):
+def connect_node(node, configmanager, username=None, direct=True, width=80,
+                 height=24):
     attrval = configmanager.get_node_attributes(node, 'collective.manager')
     myc = attrval.get(node, {}).get('collective.manager', {}).get(
         'value', None)
     myname = collective.get_myname()
-    if myc and myc != collective.get_myname():
+    if myc and myc != collective.get_myname() and direct:
         minfo = configmodule.get_collective_member(myc)
-        return ProxyConsole(node, minfo, myname, configmanager, username)
+        return ProxyConsole(node, minfo, myname, configmanager, username,
+                            width, height)
     consk = (node, configmanager.tenant)
     if consk not in _handled_consoles:
-        _handled_consoles[consk] = ConsoleHandler(node, configmanager)
+        _handled_consoles[consk] = ConsoleHandler(node, configmanager, width,
+                                                  height)
     return _handled_consoles[consk]
 
 # A stub console handler that just passes through to a remote confluent
@@ -644,8 +653,10 @@ def connect_node(node, configmanager, username=None):
 class ProxyConsole(object):
     _genwatchattribs = frozenset(('collective.manager',))
 
-    def __init__(self, node, managerinfo, myname, configmanager, user):
+    def __init__(self, node, managerinfo, myname, configmanager, user, width,
+                 height):
         self.skipreplay = True
+        self.initsize = (width, height)
         self.managerinfo = managerinfo
         self.myname = myname
         self.cfm = configmanager
@@ -656,6 +667,7 @@ class ProxyConsole(object):
         self._attribwatcher = configmanager.watch_attributes(
             (self.node,), self._genwatchattribs, self._attribschanged)
 
+
     def _attribschanged(self, nodeattribs, configmanager, **kwargs):
         if self.clisession:
             self.clisession.detach()
@@ -697,6 +709,8 @@ class ProxyConsole(object):
                 'tenant': self.cfm.tenant,
                 'node': self.node,
                 'skipreplay': self.skipreplay,
+                'width': self.initsize[0],
+                'height': self.initsize[1],
                 #TODO(jjohnson2): declare myself as a proxy,
                 #facilitate redirect rather than relay on manager change
             },
@@ -736,6 +750,10 @@ class ProxyConsole(object):
     def reopen(self):
         tlvdata.send(self.remote, {'operation': 'reopen'})
 
+    def resize(self, width, height):
+        tlvdata.send(self.remote, {'operation': 'resize', 'width': width,
+                                   'height': height})
+
 
 # this represents some api view of a console handler.  This handles things like
 # holding the caller specific queue data, for example, when http api should be
@@ -758,10 +776,9 @@ class ConsoleSession(object):
                          'get_next_output' non-functional
     :param skipreplay: If true, will skip the attempt to redraw the screen
     """
-    connector = connect_node
 
     def __init__(self, node, configmanager, username, datacallback=None,
-                 skipreplay=False):
+                 skipreplay=False, direct=True, width=80, height=24):
         self.registered = False
         self.tenant = configmanager.tenant
         if not configmanager.is_node(node):
@@ -769,6 +786,10 @@ class ConsoleSession(object):
         self.username = username
         self.node = node
         self.configmanager = configmanager
+        self.direct = direct  # true if client is directly connected versus
+                              # relay
+        self.width = width
+        self.height = height
         self.connect_session()
         self.registered = True
         self._evt = None
@@ -797,12 +818,16 @@ class ConsoleSession(object):
         between console and shell.
         """
         self.conshdl = connect_node(self.node, self.configmanager,
-                                    self.username)
+                                    self.username, self.direct, self.width,
+                                    self.height)
     def send_break(self):
         """Send break to remote system
         """
         self.conshdl.send_break()
 
+    def resize(self, width, height):
+        self.conshdl.resize(width, height)
+
     def get_buffer_age(self):
         """Get the age in seconds of the buffered data
 

confluent_server/confluent/core.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -123,7 +123,7 @@ def load_plugins():
 
 
 rootcollections = ['discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'version']
+                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'xcat/', 'version']
 
 
 class PluginRoute(object):
@@ -165,6 +165,10 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'licenses': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                 'net_interfaces': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -196,11 +200,37 @@ def _init_core():
                     }),
                 },
             },
+            'storage': {
+                'all': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'arrays': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'disks': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'volumes': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                })
+            },
             'system': {
                 'all': PluginRoute({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'advanced': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'clear': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                })
             },
         },
         '_console': {
@@ -233,6 +263,10 @@ def _init_core():
                 'default': 'ipmi',
             }),
         },
+        'description': PluginRoute({
+            'pluginattrs': ['hardwaremanagement.method'],
+            'default': 'ipmi',
+        }),
         'events': {
             'hardware': {
                 'log': PluginRoute({
@@ -334,6 +368,12 @@ def _init_core():
             },
 
         },
+        'support': {
+            'servicedata': PluginCollection({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+        },
     }
 
     nodegroupresources = {
@@ -567,6 +607,15 @@ class BadPlugin(object):
             self.node, self.plugin + ' is not a supported plugin')
 
 
+class BadCollective(object):
+    def __init__(self, node):
+        self.node = node
+
+    def error(self, *args, **kwargs):
+        yield msg.ConfluentNodeError(
+            self.node, 'collective mode is active, but collective.manager '
+                       'is not set for this node')
+
 def abbreviate_noderange(configmanager, inputdata, operation):
     if operation != 'create':
         raise exc.InvalidArgumentException('Must be a create with nodes in list')
@@ -759,6 +808,7 @@ def handle_node_request(configmanager, inputdata, operation,
                         continue
                 elif list(cfm.list_collective()):
                     badcollnodes.append(node)
+                    continue
             if plugpath is not None:
                 try:
                     hfunc = getattr(pluginmap[plugpath], operation)
@@ -769,11 +819,8 @@ def handle_node_request(configmanager, inputdata, operation,
                     nodesbyhandler[hfunc].append(node)
                 else:
                     nodesbyhandler[hfunc] = [node]
-        if badcollnodes:
-            raise exc.ConfluentException(
-                'collective management active, '
-                'collective.manager must be set for {0}'.format(
-                    ','.join(badcollnodes)))
+        for bn in badcollnodes:
+            nodesbyhandler[BadCollective(bn).error] = [bn]
         workers = greenpool.GreenPool()
         numworkers = 0
         for hfunc in nodesbyhandler:
@@ -809,6 +856,8 @@ def iterate_queue(numworkers, passvalues, strip=False):
         if nv == 'theend':
             completions += 1
         else:
+            if isinstance(nv, Exception):
+                raise nv
             if strip and not isinstance(nv, console.Console):
                 nv.strip_node(strip)
             yield nv
@@ -822,6 +871,8 @@ def addtoqueue(theq, fun, kwargs):
         else:
             for pv in result:
                 theq.put(pv)
+    except Exception as e:
+        theq.put(e)
     finally:
         theq.put('theend')
 
@@ -837,8 +888,16 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
                                  certfile='/etc/confluent/srvcert.pem')
     except Exception:
         for node in nodes:
-            yield msg.ConfluentResourceUnavailable(
-                node, 'Collective member {0} is unreachable'.format(a['name']))
+            if a:
+                yield msg.ConfluentResourceUnavailable(
+                    node, 'Collective member {0} is unreachable'.format(
+                        a['name']))
+            else:
+                yield msg.ConfluentResourceUnavailable(
+                    node,
+                    '"{0}" is not recognized as a collective member'.format(
+                        manager))
+
         return
     if not util.cert_matches(a['fingerprint'], remote.getpeercert(
             binary_form=True)):
@@ -918,6 +977,16 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
     """
     pathcomponents = path.split('/')
     del pathcomponents[0]  # discard the value from leading /
+    xcat_url = False
+    if pathcomponents and pathcomponents[0] == 'xcat':
+        xcat_url = True
+        if 'xcat/' in rootcollections: 
+            rootcollections.remove('xcat/')
+        del pathcomponents[0]
+    elif 'xcat/' not in rootcollections:
+            rootcollections.insert(-1, 'xcat/')
+    if xcat_url:
+        configmanager.log("Start to deal with xCAT URL: %s" % path)
     if pathcomponents[-1] == '':
         del pathcomponents[-1]
     if not pathcomponents:  # root collection list

confluent_server/confluent/discovery/core.py

@@ -86,6 +86,8 @@ import eventlet.greenpool
 import eventlet.semaphore
 
 autosensors = set()
+scanner = None
+
 class nesteddict(dict):
 
     def __missing__(self, key):
@@ -209,6 +211,8 @@ def send_discovery_datum(info):
         if infotype in servicenames:
             types.append(servicenames[infotype])
     yield msg.KeyValueData({'types': types})
+    if 'otheraddresses' in info:
+        yield msg.KeyValueData({'otheripaddrs': list(info['otheraddresses'])})
 
 
 def _info_matches(info, criteria):
@@ -381,6 +385,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
             raise exc.InvalidArgumentException()
         rescan()
         return (msg.KeyValueData({'rescan': 'started'}),)
+
     elif operation in ('update', 'create'):
         if 'node' not in inputdata:
             raise exc.InvalidArgumentException('Missing node name in input')
@@ -391,8 +396,15 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
                 'Unable to {0} to {1}'.format(operation,
                                               '/'.join(pathcomponents)))
         handler = info['handler'].NodeHandler(info, configmanager)
-        eval_node(configmanager, handler, info, inputdata['node'],
-                  manual=True)
+        try:
+            eval_node(configmanager, handler, info, inputdata['node'],
+                      manual=True)
+        except Exception as e:
+            # or... incorrect passworod provided..
+            if 'Incorrect password' in str(e) or 'Unauthorized name' in str(e):
+                return [msg.ConfluentTargetInvalidCredentials(
+                    inputdata['node'])]
+            raise
         return [msg.AssignedResource(inputdata['node'])]
     elif operation == 'delete':
         mac = _get_mac_from_query(pathcomponents)
@@ -416,6 +428,8 @@ def handle_read_api_request(pathcomponents):
     # TODO(jjohnson2): This should be more generalized...
     #  odd indexes into components are 'by-'*, even indexes
     # starting at 2 are parameters to previous index
+    if pathcomponents == ['discovery', 'rescan']:
+        return (msg.KeyValueData({'scanning': bool(scanner)}),)
     subcats, queryparms, indexof, coll = _parameterize_path(pathcomponents[1:])
     if len(pathcomponents) == 1:
         dirlist = [msg.ChildCollection(x + '/') for x in sorted(list(subcats))]
@@ -458,6 +472,9 @@ def _recheck_nodes_backend(nodeattribs, configmanager):
     _map_unique_ids(nodeattribs)
     # for the nodes whose attributes have changed, consider them as potential
     # strangers
+    if nodeattribs:
+        macmap.vintage = 0  # expire current mac map data, in case
+        # the attributes changed impacted the result
     for node in nodeattribs:
         if node in known_nodes:
             for somemac in known_nodes[node]:
@@ -571,6 +588,11 @@ def detected(info):
             break
     else:  # no nodehandler, ignore for now
         return
+    if (handler and not handler.NodeHandler.adequate(info) and
+            info.get('protocol', None)):
+        eventlet.spawn_after(10, info['protocol'].fix_info, info,
+                             safe_detected)
+        return
     try:
         snum = info['attributes']['enclosure-serial-number'][0].strip()
         if snum:
@@ -617,6 +639,9 @@ def detected(info):
     uuid = info.get('uuid', None)
     if uuid_is_valid(uuid):
         known_uuids[uuid][info['hwaddr']] = info
+    info['otheraddresses'] = set([])
+    for i4addr in info.get('attributes', {}).get('ipv4-address', []):
+        info['otheraddresses'].add(i4addr)
     if handler and handler.https_supported and not handler.https_cert:
         if handler.cert_fail_reason == 'unreachable':
             log.log(
@@ -686,6 +711,10 @@ def get_enclosure_chain_head(nodename, cfg):
         if ne in members:
             raise exc.InvalidArgumentException(
                 'Circular chain that includes ' + nodename)
+        if not cfg.is_node(ne):
+            raise exc.InvalidArgumentException(
+                '{0} is chained to nonexistent node {1} '.format(
+                    nodename, ne))
         nodename = ne
         members.append(nodename)
     return nodename
@@ -1091,6 +1120,7 @@ needaddhandled = False
 def _handle_nodelist_change(configmanager):
     global needaddhandled
     global nodeaddhandler
+    macmap.vintage = 0  # the current mac map is probably inaccurate
     _recheck_nodes((), configmanager)
     if needaddhandled:
         needaddhandled = False
@@ -1142,7 +1172,11 @@ def _periodic_recheck(configmanager):
 
 def rescan():
     _map_unique_ids()
-    eventlet.spawn_n(slp.active_scan, safe_detected)
+    global scanner
+    if scanner:
+        return
+    else:
+        scanner = eventlet.spawn(slp.active_scan, safe_detected, slp)
 
 
 def start_detection():
@@ -1171,8 +1205,8 @@ def stop_autosense():
         autosensors.discard(watcher)
 
 def start_autosense():
-    autosensors.add(eventlet.spawn(slp.snoop, safe_detected))
-    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected))
+    autosensors.add(eventlet.spawn(slp.snoop, safe_detected, slp))
+    autosensors.add(eventlet.spawn(pxe.snoop, safe_detected, pxe))
 
 
 nodes_by_fprint = {}
@@ -1194,7 +1228,7 @@ def _map_unique_ids(nodes=None):
     uuid_by_nodes = {}
     fprint_by_nodes = {}
     for uuid in nodes_by_uuid:
-        if not uuid_is_valid():
+        if not uuid_is_valid(uuid):
             continue
         node = nodes_by_uuid[uuid]
         if node in bigmap:

confluent_server/confluent/discovery/handlers/bmc.py

@@ -45,7 +45,7 @@ class NodeHandler(generic.NodeHandler):
     def config(self, nodename, reset=False):
         self._bmcconfig(nodename, reset)
 
-    def _bmcconfig(self, nodename, reset=False):
+    def _bmcconfig(self, nodename, reset=False, customconfig=None):
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
@@ -74,6 +74,8 @@ class NodeHandler(generic.NodeHandler):
                 ic = self._get_ipmicmd(user, passwd)
             else:
                 raise
+        if customconfig:
+            customconfig(ic)
         currusers = ic.get_users()
         lanchan = ic.get_network_channel()
         userdata = ic.xraw_command(netfn=6, command=0x44, data=(lanchan,
@@ -122,16 +124,18 @@ class NodeHandler(generic.NodeHandler):
             if currusers[uid]['name'] == newuser:
                 # Use existing account that has been created
                 newuserslot = uid
+                if newpass != passwd:  # don't mess with existing if no change
+                    ic.set_user_password(newuserslot, password=newpass)
                 break
         else:
             newuserslot = lockedusers + 1
             if newuserslot < 2:
                 newuserslot = 2
+            if newpass != passwd:  # don't mess with existing if no change
+                ic.set_user_password(newuserslot, password=newpass)
             ic.set_user_name(newuserslot, newuser)
             ic.set_user_access(newuserslot, lanchan,
                                privilege_level='administrator')
-        if newpass != passwd:  # don't mess with existing if no change
-            ic.set_user_password(newuserslot, password=newpass)
         # Now to zap others
         for uid in currusers:
             if uid != newuserslot:

confluent_server/confluent/discovery/handlers/generic.py

@@ -40,6 +40,12 @@ class NodeHandler(object):
                 targsa = info['addresses'][0]
         self.ipaddr = targsa[0]
 
+    @classmethod
+    def adequate(cls, info):
+        # Check if the referenced info is really enough, if false, a rescan
+        # may occur against the target in a short while
+        return True
+
     def scan(self):
         # Do completely passive things to enhance data.
         # Probe is permitted to for example attempt a login

confluent_server/confluent/discovery/handlers/imm.py

@@ -21,6 +21,12 @@ import struct
 class NodeHandler(bmchandler.NodeHandler):
     devname = 'IMM'
 
+    @classmethod
+    def adequate(cls, info):
+        # We can sometimes receive a partially initialized SLP packet
+        # This is not adequate for being satisfied
+        return bool(info['attributes'])
+
     def scan(self):
         slpattrs = self.info.get('attributes', {})
         self.isdense = False

confluent_server/confluent/discovery/handlers/xcc.py

@@ -13,14 +13,22 @@
 # limitations under the License.
 
 import confluent.discovery.handlers.imm as immhandler
+import confluent.util as util
 import pyghmi.exceptions as pygexc
 import pyghmi.ipmi.oem.lenovo.imm as imm
 
 
 
+
 class NodeHandler(immhandler.NodeHandler):
     devname = 'XCC'
 
+    @classmethod
+    def adequate(cls, info):
+        # We can sometimes receive a partially initialized SLP packet
+        # This is not adequate for being satisfied
+        return bool(info.get('attributes', {}))
+
     def preconfig(self):
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
@@ -41,16 +49,45 @@ class NodeHandler(immhandler.NodeHandler):
         #if ipmicmd:
         #    ipmicmd.ipmi_session.logout()
 
+    def validate_cert(self, certificate):
+        # broadly speaking, merely checks consistency moment to moment,
+        # but if https_cert gets stricter, this check means something
+        fprint = util.get_fingerprint(self.https_cert)
+        return util.cert_matches(fprint, certificate)
+
+    def set_password_policy(self, ic):
+        ruleset = {'USER_GlobalMinPassChgInt': '0'}
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                ruleset['USER_GlobalPassExpPeriod'] = value
+                if int(value) < 5:
+                    ruleset['USER_GlobalPassExpWarningPeriod'] = value
+            if name.lower() in ('lockout', 'loginfailures'):
+                if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                    value = '0'
+                ruleset['USER_GlobalMaxLoginFailures'] = value
+        ic.register_key_handler(self.validate_cert)
+        ic.oem_init()
+        ic._oem.immhandler.wc.grab_json_response('/api/dataset', ruleset)
+
     def config(self, nodename, reset=False):
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
-        ic = self._bmcconfig(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        ic = self._bmcconfig(nodename, customconfig=self.set_password_policy)
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return
         # Ok, we can get the enclosure uuid now..
-        ic.oem_init()
         enclosureuuid = ic._oem.immhandler.get_property(
             '/v2/ibmc/smm/chassis/uuid')
         enclosureuuid = ic._oem.immhandler.get_property(

confluent_server/confluent/discovery/protocols/pxe.py

@@ -72,7 +72,7 @@ def find_info_in_options(rq, optidx):
         return uuid, arch
     return uuid, arch
 
-def snoop(handler):
+def snoop(handler, protocol=None):
     #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
     #prominent
     #TODO(jjohnson2): IP_PKTINFO, recvmsg to get the destination ip, per

confluent_server/confluent/discovery/protocols/slp.py

@@ -255,7 +255,7 @@ def _parse_attrlist(attrstr):
                 currattr = currattr.decode('utf-8')
                 attribs[currattr] = None
             else:
-                attrname, attrval = currattr.split('=')
+                attrname, attrval = currattr.split('=', 1)
                 attrname = attrname.decode('utf-8')
                 attribs[attrname] = []
                 for val in attrval.split(','):
@@ -295,9 +295,11 @@ def _parse_attrlist(attrstr):
     return attribs
 
 
-def _parse_attrs(data, parsed):
+def _parse_attrs(data, parsed, xid=None):
     headinfo = _parse_slp_header(data)
-    if headinfo['function'] != 7 or headinfo['xid'] != parsed['xid']:
+    if xid is None:
+        xid = parsed['xid']
+    if headinfo['function'] != 7 or headinfo['xid'] != xid:
         return
     payload = headinfo['payload']
     if struct.unpack('!H', bytes(payload[:2]))[0] != 0:
@@ -307,8 +309,18 @@ def _parse_attrs(data, parsed):
     parsed['attributes'] = _parse_attrlist(attrstr)
 
 
+def fix_info(info, handler):
+    if '_attempts' not in info:
+        info['_attempts'] = 10
+    if info['_attempts'] == 0:
+        return
+    info['_attempts'] -= 1
+    _add_attributes(info)
+    handler(info)
+
 def _add_attributes(parsed):
-    attrq = _generate_attr_request(parsed['services'][0], parsed['xid'])
+    xid = parsed.get('xid', 42)
+    attrq = _generate_attr_request(parsed['services'][0], xid)
     target = None
     # prefer reaching out to an fe80 if present, to be highly robust
     # in face of network changes
@@ -329,7 +341,7 @@ def _add_attributes(parsed):
     net.sendall(attrq)
     rsp = net.recv(8192)
     net.close()
-    _parse_attrs(rsp, parsed)
+    _parse_attrs(rsp, parsed, xid)
 
 
 def query_srvtypes(target):
@@ -383,7 +395,7 @@ def rescan(handler):
             handler(scanned)
 
 
-def snoop(handler):
+def snoop(handler, protocol=None):
     """Watch for SLP activity
 
     handler will be called with a dictionary of relevant attributes
@@ -392,7 +404,7 @@ def snoop(handler):
     :return:
     """
     tracelog = log.Logger('trace')
-    active_scan(handler)
+    active_scan(handler, protocol)
     net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
     net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
     slpg = socket.inet_pton(socket.AF_INET6, 'ff01::123')
@@ -469,13 +481,14 @@ def snoop(handler):
                 peerbymacaddress[mac]['xid'] = 1
                 _add_attributes(peerbymacaddress[mac])
                 peerbymacaddress[mac]['hwaddr'] = mac
+                peerbymacaddress[mac]['protocol'] = protocol
                 handler(peerbymacaddress[mac])
         except Exception as e:
             tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                          event=log.Events.stacktrace)
 
 
-def active_scan(handler):
+def active_scan(handler, protocol=None):
     known_peers = set([])
     for scanned in scan():
         for addr in scanned['addresses']:
@@ -486,6 +499,7 @@ def active_scan(handler):
                 break
             known_peers.add(addr)
         else:
+            scanned['protocol'] = protocol
             handler(scanned)
 
 

confluent_server/confluent/exceptions.py

@@ -106,6 +106,7 @@ class PubkeyInvalid(ConfluentException):
         super(PubkeyInvalid, self).__init__(self, text)
         self.fingerprint = fingerprint
         self.attrname = attribname
+        self.message = text
         bodydata = {'message': text,
                     'event': event,
                     'fingerprint': fingerprint,

confluent_server/confluent/firmwaremanager.py

@@ -22,19 +22,24 @@ import confluent.exceptions as exc
 import confluent.messages as msg
 import eventlet
 import os
+import pwd
 import socket
 
 updatesbytarget = {}
 uploadsbytarget = {}
+downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
 
 
-def execupdate(handler, filename, updateobj, type):
-    if not os.path.exists(filename):
+def execupdate(handler, filename, updateobj, type, owner, node):
+    if type != 'ffdc' and not os.path.exists(filename):
         errstr =  '{0} does not appear to exist on {1}'.format(
             filename, socket.gethostname())
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': errstr})
+        return
+    if type == 'ffdc' and os.path.isdir(filename):
+        filename += '/' + node + '.svcdata'
     try:
         if type == 'firmware':
             completion = handler(filename, progress=updateobj.handle_progress,
@@ -43,6 +48,9 @@ def execupdate(handler, filename, updateobj, type):
             completion = handler(filename, progress=updateobj.handle_progress)
         if completion is None:
             completion = 'complete'
+        if owner:
+            pwent = pwd.getpwnam(owner)
+            os.chown(filename, pwent.pw_uid, pwent.pw_gid)
         updateobj.handle_progress({'phase': completion, 'progress': 100.0})
     except exc.PubkeyInvalid as pi:
         errstr = 'Certificate mismatch detected, does not match value in ' \
@@ -55,18 +63,20 @@ def execupdate(handler, filename, updateobj, type):
 
 class Updater(object):
     def __init__(self, node, handler, filename, tenant=None, name=None,
-                 bank=None, type='firmware'):
+                 bank=None, type='firmware', owner=None):
         self.bank = bank
         self.node = node
         self.phase = 'initializing'
         self.detail = ''
         self.percent = 0.0
         self.updateproc = updatepool.spawn(execupdate, handler, filename,
-                                           self, type)
+                                           self, type, owner, node)
         if type == 'firmware':
             myparty = updatesbytarget
         elif type == 'mediaupload':
             myparty = uploadsbytarget
+        elif type == 'ffdc':
+            myparty = downloadsbytarget
         if (node, tenant) not in myparty:
             myparty[(node, tenant)] = {}
         if name is None:
@@ -96,6 +106,8 @@ def remove_updates(nodes, tenant, element, type='firmware'):
     upid = element[-1]
     if type == 'firmware':
         myparty = updatesbytarget
+    elif type == 'ffdc':
+        myparty = downloadsbytarget
     else:
         myparty = uploadsbytarget
     for node in nodes:
@@ -115,6 +127,9 @@ def list_updates(nodes, tenant, element, type='firmware'):
     if type == 'mediaupload':
         myparty = uploadsbytarget
         verb = 'upload'
+    elif type == 'ffdc':
+        verb = 'download'
+        myparty = downloadsbytarget
     else:
         myparty = updatesbytarget
         verb = 'update'

confluent_server/confluent/httpapi.py

@@ -30,6 +30,7 @@ import confluent.asynchttp
 import confluent.shellserver as shellserver
 import confluent.tlvdata
 import confluent.util as util
+from confluent.xcclient.xcat_manager import xCATConfigManager
 import copy
 import eventlet
 import eventlet.greenthread
@@ -56,7 +57,7 @@ opmap = {
     'PUT': 'update',
     'DELETE': 'delete',
 }
-
+xcat_cfm = None
 
 class RobustCookie(Cookie.SimpleCookie):
     # this is very bad form, but BaseCookie has a terrible flaw
@@ -437,6 +438,11 @@ def resourcehandler_backend(env, start_response):
         ("Set-Cookie", m.OutputString())
         for m in authorized['cookie'].values())
     cfgmgr = authorized['cfgmgr']
+    if (env['PATH_INFO'].startswith('/xcat')):
+        global xcat_cfm
+        if xcat_cfm is None:
+            xcat_cfm = xCATConfigManager()
+        cfgmgr = xcat_cfm
     if (operation == 'create') and env['PATH_INFO'] == '/sessions/current/async':
         pagecontent = ""
         try:
@@ -496,6 +502,8 @@ def resourcehandler_backend(env, start_response):
             skipreplay = False
             if 'skipreplay' in querydict and querydict['skipreplay']:
                 skipreplay = True
+            width = querydict.get('width', 80)
+            height = querydict.get('height', 24)
             datacallback = None
             async = None
             if 'HTTP_CONFLUENTASYNCID' in env:
@@ -507,13 +515,13 @@ def resourcehandler_backend(env, start_response):
                     consession = shellserver.ShellSession(
                         node=nodename, configmanager=cfgmgr,
                         username=authorized['username'], skipreplay=skipreplay,
-                        datacallback=datacallback
+                        datacallback=datacallback, width=width, height=height
                     )
                 else:
                     consession = consoleserver.ConsoleSession(
                         node=nodename, configmanager=cfgmgr,
                         username=authorized['username'], skipreplay=skipreplay,
-                        datacallback=datacallback
+                        datacallback=datacallback, width=width, height=height
                     )
             except exc.NotFoundException:
                 start_response("404 Not found", headers)
@@ -548,6 +556,9 @@ def resourcehandler_backend(env, start_response):
         elif 'action' in querydict:
             if querydict['action'] == 'break':
                 consolesessions[querydict['session']]['session'].send_break()
+            elif querydict['action'] == 'resize':
+                consolesessions[querydict['session']]['session'].resize(
+                    width=querydict['width'], height=querydict['height'])
             elif querydict['action'] == 'reopen':
                 consolesessions[querydict['session']]['session'].reopen()
             else:
@@ -784,6 +795,8 @@ def serve(bind_host, bind_port):
                 'Failed to open HTTP due to busy port, trying again in'
                 ' a second\n')
             eventlet.sleep(1)
+    # TCP_FASTOPEN
+    sock.setsockopt(socket.SOL_TCP, 23, 5)
     eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
                          debug=False, socket_timeout=60)
 

confluent_server/confluent/interface/console.py

@@ -41,6 +41,11 @@ class Console(object):
         the remote console"""
         return
 
+    def resize(self, width, height):
+        """This function is responsible for relaying resize request from
+        client terminal to remote console"""
+        return
+
     def ping(self):
         """This function is a hint to the console plugin that now would be a
         nice time to assess health of console connection.  Plugins that see

confluent_server/confluent/messages.py

@@ -32,6 +32,11 @@ valid_health_values = set([
     'unknown',
 ])
 
+def simplify_name(name):
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
+
+
 def _htmlify_structure(indict):
     ret = "<ul>"
     if isinstance(indict, dict):
@@ -416,27 +421,63 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
     elif (path[:3] == ['configuration', 'system', 'all'] and
             operation != 'retrieve'):
         return InputConfigChangeSet(path, inputdata, nodes, configmanager)
+    elif (path[:3] == ['configuration', 'system', 'clear'] and
+            operation != 'retrieve'):
+        return InputConfigClear(path, inputdata, nodes, configmanager)
+    elif (path[:3] == ['configuration', 'storage', 'disks'] and
+            operation != 'retrieve'):
+        return InputDisk(path, nodes, inputdata)
+    elif (path[:3] == ['configuration', 'storage', 'volumes'] and
+          operation in ('update', 'create')):
+        return InputVolumes(path, nodes, inputdata)
     elif 'inventory/firmware/updates/active' in '/'.join(path) and inputdata:
-        return InputFirmwareUpdate(path, nodes, inputdata)
+        return InputFirmwareUpdate(path, nodes, inputdata, configmanager)
     elif '/'.join(path).startswith('media/detach'):
         return DetachMedia(path, nodes, inputdata)
     elif '/'.join(path).startswith('media/') and inputdata:
-        return InputMedia(path, nodes, inputdata)
+        return InputMedia(path, nodes, inputdata, configmanager)
+    elif '/'.join(path).startswith('support/servicedata') and inputdata:
+        return InputMedia(path, nodes, inputdata, configmanager)
+    elif '/'.join(path).startswith(
+            'configuration/management_controller/licenses') and inputdata:
+        return InputLicense(path, nodes, inputdata, configmanager)
     elif inputdata:
         raise exc.InvalidArgumentException(
             'No known input handler for request')
 
 class InputFirmwareUpdate(ConfluentMessage):
 
-    def __init__(self, path, nodes, inputdata):
-        self.filename = inputdata.get('filename', inputdata.get('url', None))
+    def __init__(self, path, nodes, inputdata, configmanager):
+        self._filename = inputdata.get('filename', inputdata.get('url', None))
         self.bank = inputdata.get('bank', None)
         self.nodes = nodes
+        self.filebynode = {}
+        self._complexname = False
+        for expanded in configmanager.expand_attrib_expression(
+                nodes, self._filename):
+            node, value = expanded
+            if value != self._filename:
+                self._complexname = True
+            self.filebynode[node] = value
+
+    @property
+    def filename(self):
+        if self._complexname:
+            raise Exception('User requested substitutions, but code is '
+                            'written against old api, code must be fixed or '
+                            'skip {} expansion')
+        return self._filename
+
+    def nodefile(self, node):
+        return self.filebynode[node]
 
 class InputMedia(InputFirmwareUpdate):
     # Use InputFirmwareUpdate
     pass
 
+class InputLicense(InputFirmwareUpdate):
+    pass
+
 
 class DetachMedia(ConfluentMessage):
     def __init__(self, path, nodes, inputdata):
@@ -490,6 +531,13 @@ class InputExpression(ConfluentMessage):
         nodeattr = deepcopy(self.nodeattribs[node])
         return nodeattr
 
+class InputConfigClear(ConfluentMessage):
+    def __init__(self, path, inputdata, nodes=None, configmanager=None):
+        if not inputdata:
+            raise exc.InvalidArgumentException('no request data provided')
+        if 'clear' not in inputdata or not inputdata['clear']:
+            raise exc.InvalidArgumentException('Input must be {"clear":true}')
+
 class InputConfigChangeSet(InputExpression):
     # For now, this is identical to InputExpression, later it may
     # internalize formula expansion, but not now..
@@ -511,7 +559,6 @@ class InputConfigChangeSet(InputExpression):
                     [node], attrs[attr]))[0][1]
         return endattrs
 
-
 class InputAttributes(ConfluentMessage):
     # This is particularly designed for attributes, where a simple string
     # should become either a string value or a dict with {'expression':} to
@@ -689,6 +736,70 @@ class InputIdentifyMessage(ConfluentInputMessage):
     keyname = 'identify'
 
 
+class InputDisk(ConfluentInputMessage):
+    valid_values = set([
+        'jbod',
+        'unconfigured',
+        'hotspare',
+    ])
+
+    keyname = 'state'
+
+
+class InputVolumes(ConfluentInputMessage):
+    def __init__(self, path, nodes, inputdata):
+        self.inputbynode = {}
+        self.stripped = False
+        if not inputdata:
+            raise exc.InvalidArgumentException('missing input data')
+        if isinstance(inputdata, dict):
+            volnames = [None]
+            if len(path) == 6:
+                volnames = path[-1]
+            volnames = inputdata.get('name', volnames)
+            if not isinstance(volnames, list):
+                volnames = volnames.split(',')
+            sizes = inputdata.get('size', [None])
+            if not isinstance(sizes, list):
+                sizes = sizes.split(',')
+            disks = inputdata.get('disks', [])
+            if not disks:
+                raise exc.InvalidArgumentException(
+                    'disks are currently required to create a volume')
+            raidlvl = inputdata.get('raidlevel', None)
+            inputdata = []
+            for size in sizes:
+                if volnames:
+                    currname = volnames.pop(0)
+                else:
+                    currname = None
+                inputdata.append(
+                    {'name': currname, 'size': size,
+                     'disks': disks,
+                     'raidlevel': raidlvl})
+        for node in nodes:
+            self.inputbynode[node] = []
+        for input in inputdata:
+            volname = None
+            if len(path) == 6:
+                volname = path[-1]
+            volname = input.get('name', volname)
+            if not volname:
+                volname = None
+            volsize = input.get('size', None)
+            if isinstance(input['disks'], list):
+                disks = input['disks']
+            else:
+                disks = input['disks'].split(',')
+            raidlvl = input.get('raidlevel', None)
+            for node in nodes:
+                self.inputbynode[node].append({'name': volname,
+                                               'size': volsize,
+                                               'disks': disks,
+                                               'raidlevel': raidlvl,
+                                               })
+
+
 class InputPowerMessage(ConfluentInputMessage):
     valid_values = set([
         'on',
@@ -1229,6 +1340,74 @@ class KeyValueData(ConfluentMessage):
         else:
             self.kvpairs = {name: kvdata}
 
+class Array(ConfluentMessage):
+    def __init__(self, name, disks=None, raid=None, volumes=None,
+                 id=None, capacity=None, available=None):
+        self.kvpairs = {
+            name: {
+                'type': 'array',
+                'disks': disks,
+                'raid': raid,
+                'id': id,
+                'volumes': volumes,
+                'capacity': capacity,
+                'available': available,
+            }
+        }
+
+class Volume(ConfluentMessage):
+    def __init__(self, name, volname, size, state, array):
+        self.kvpairs = {
+            name: {
+                'type': 'volume',
+                'name': simplify_name(volname),
+                'label': volname,
+                'size': size,
+                'state': state,
+                'array': array,
+            }
+        }
+
+class Disk(ConfluentMessage):
+    valid_states = set([
+        'jbod',
+        'unconfigured',
+        'hotspare',
+        'online',
+    ])
+    state_aliases = {
+        'unconfigured good': 'unconfigured',
+        'global hot spare': 'hotspare',
+    }
+
+    def _normalize_state(self, instate):
+        newstate = instate.lower()
+        if newstate in self.valid_states:
+            return newstate
+        elif newstate in self.state_aliases:
+            return self.state_aliases[newstate]
+        raise Exception("Unknown state {0}".format(instate))
+
+
+    def __init__(self, name, label=None, description=None,
+                 diskid=None, state=None, serial=None, fru=None,
+                 array=None):
+        state = self._normalize_state(state)
+        self.kvpairs = {
+            name: {
+                'type': 'disk',
+                'name': simplify_name(label),
+                'label': label,
+                'description': description,
+                'diskid': diskid,
+                'state': state,
+                'serial': serial,
+                'fru': fru,
+                'array': array,
+            }
+        }
+
+
 
 class LEDStatus(ConfluentMessage):
     readonly = True
@@ -1376,12 +1555,12 @@ class NTPServer(ConfluentMessage):
 class License(ConfluentMessage):
     readonly = True
 
-    def __init__(self, name=None, kvm=None):
+    def __init__(self, name=None, kvm=None, feature=None):
         self.notnode = name is None
         self.desc = 'License'
 
         kv = []
-        kv.append({'kvm_availability': str(kvm)})
+        kv.append({'kvm_availability': str(kvm), 'feature': feature})
         if self.notnode:
             self.kvpairs = {'License': kv}
         else:
@@ -1396,9 +1575,13 @@ class CryptedAttributes(Attributes):
         self.desc = desc
         nkv = {}
         for key in kv.iterkeys():
-            nkv[key] = {'isset': False}
             try:
-                if kv[key] is not None and kv[key]['cryptvalue'] != '':
+                if kv[key] is None:
+                    continue
+                elif not isinstance(kv[key], dict):
+                    self.defaulttype = 'text'
+                    nkv[key] = kv[key]
+                elif kv[key] is not None and kv[key]['cryptvalue'] != '':
                     nkv[key] = {'isset': True}
                     nkv[key]['inheritedfrom'] = kv[key]['inheritedfrom']
             except KeyError:

confluent_server/confluent/networking/lldp.py

@@ -171,7 +171,7 @@ def _extract_neighbor_data_b(args):
 
     args are carried as a tuple, because of eventlet convenience
     """
-    switch, password, user, force = args
+    switch, password, user, force = args[:4]
     vintage = _neighdata.get(switch, {}).get('!!vintage', 0)
     now = util.monotonic_time()
     if vintage > (now - 60) and not force:
@@ -220,17 +220,19 @@ def _extract_neighbor_data_b(args):
     _neighdata[switch] = lldpdata
 
 
-def update_switch_data(switch, configmanager, force=False):
+def update_switch_data(switch, configmanager, force=False, retexc=False):
     switchcreds = netutil.get_switchcreds(configmanager, (switch,))[0]
-    _extract_neighbor_data(switchcreds + (force,))
+    ndr = _extract_neighbor_data(switchcreds + (force, retexc))
+    if retexc and isinstance(ndr, Exception):
+        raise ndr
     return _neighdata.get(switch, {})
 
 
-def update_neighbors(configmanager, force=False):
-    return _update_neighbors_backend(configmanager, force)
+def update_neighbors(configmanager, force=False, retexc=False):
+    return _update_neighbors_backend(configmanager, force, retexc)
 
 
-def _update_neighbors_backend(configmanager, force):
+def _update_neighbors_backend(configmanager, force, retexc):
     global _neighdata
     global _neighbypeerid
     vintage = _neighdata.get('!!vintage', 0)
@@ -241,7 +243,7 @@ def _update_neighbors_backend(configmanager, force):
     _neighbypeerid = {'!!vintage': now}
     switches = netutil.list_switches(configmanager)
     switchcreds = netutil.get_switchcreds(configmanager, switches)
-    switchcreds = [ x + (force,) for x in switchcreds]
+    switchcreds = [ x + (force, retexc) for x in switchcreds]
     pool = GreenPool(64)
     for ans in pool.imap(_extract_neighbor_data, switchcreds):
         yield ans
@@ -258,9 +260,15 @@ def _extract_neighbor_data(args):
         return
     try:
         with _updatelocks[switch]:
-            _extract_neighbor_data_b(args)
-    except Exception:
-        log.logtrace()
+            return _extract_neighbor_data_b(args)
+    except Exception as e:
+        yieldexc = False
+        if len(args) >= 5:
+            yieldexc = args[4]
+        if yieldexc:
+            return e
+        else:
+            log.logtrace()
 
 if __name__ == '__main__':
     # a quick one-shot test, args are switch and snmpv1 string for now
@@ -327,7 +335,9 @@ def _handle_neighbor_query(pathcomponents, configmanager):
         # guaranteed
         if (parms['by-peerid'] not in _neighbypeerid and
                 _neighbypeerid.get('!!vintage', 0) < util.monotonic_time() - 60):
-            list(update_neighbors(configmanager))
+            for x in update_neighbors(configmanager, retexc=True):
+                if isinstance(x, Exception):
+                    raise x
         if parms['by-peerid'] not in _neighbypeerid:
             raise exc.NotFoundException('No matching peer known')
         return _dump_neighbordatum(_neighbypeerid[parms['by-peerid']])
@@ -336,9 +346,11 @@ def _handle_neighbor_query(pathcomponents, configmanager):
     if listrequested not in multi_selectors | single_selectors:
         raise exc.NotFoundException('{0} is not found'.format(listrequested))
     if 'by-switch' in parms:
-        update_switch_data(parms['by-switch'], configmanager)
+        update_switch_data(parms['by-switch'], configmanager, retexc=True)
     else:
-        list(update_neighbors(configmanager))
+        for x in update_neighbors(configmanager, retexc=True):
+            if isinstance(x, Exception):
+                raise x
     return list_info(parms, listrequested)
 
 

confluent_server/confluent/networking/macmap.py

@@ -47,6 +47,7 @@ import eventlet.semaphore
 import re
 
 _macmap = {}
+_apimacmap = {}
 _macsbyswitch = {}
 _nodesbymac = {}
 _switchportmap = {}
@@ -211,7 +212,7 @@ def _map_switch_backend(args):
                 maccounts[ifname] = 1
             else:
                 maccounts[ifname] += 1
-    _macsbyswitch[switch] = {}
+    newmacs = {}
     for mac in mactobridge:
         # We want to merge it so that when a mac appears in multiple
         # places, it is captured.
@@ -223,10 +224,10 @@ def _map_switch_backend(args):
             _macmap[mac].append((switch, ifname, maccounts[ifname]))
         else:
             _macmap[mac] = [(switch, ifname, maccounts[ifname])]
-        if ifname in _macsbyswitch[switch]:
-            _macsbyswitch[switch][ifname].append(mac)
+        if ifname in newmacs:
+            newmacs[ifname].append(mac)
         else:
-            _macsbyswitch[switch][ifname] = [mac]
+            newmacs[ifname] = [mac]
         nodename = _nodelookup(switch, ifname)
         if nodename is not None:
             if mac in _nodesbymac and _nodesbymac[mac][0] != nodename:
@@ -238,6 +239,7 @@ def _map_switch_backend(args):
                 _nodesbymac[mac] = (None, None)
             else:
                 _nodesbymac[mac] = (nodename, maccounts[ifname])
+    _macsbyswitch[switch] = newmacs
 
 
 switchbackoff = 30
@@ -295,6 +297,7 @@ def _finish_update(completions):
 
 def _full_updatemacmap(configmanager):
     global vintage
+    global _apimacmap
     global _macmap
     global _nodesbymac
     global _switchportmap
@@ -307,7 +310,6 @@ def _full_updatemacmap(configmanager):
         _macmap = {}
         _nodesbymac = {}
         _switchportmap = {}
-        _macsbyswitch = {}
         if configmanager.tenant is not None:
             raise exc.ForbiddenRequest(
                 'Network topology not available to tenants')
@@ -340,11 +342,15 @@ def _full_updatemacmap(configmanager):
                         _switchportmap[curswitch][portname] = None
                     else:
                         _switchportmap[curswitch][portname] = node
+        for switch in _macsbyswitch:
+            if switch not in switches:
+                del _macsbyswitch[switch]
         switchauth = get_switchcreds(configmanager, switches)
         pool = GreenPool(64)
         for ans in pool.imap(_map_switch, switchauth):
             vintage = util.monotonic_time()
             yield ans
+    _apimacmap = _macmap
     endtime = util.monotonic_time()
     duration = endtime - start
     duration = duration * 15  # wait 15 times as long as it takes to walk
@@ -424,7 +430,7 @@ def handle_read_api_request(pathcomponents, configmanager):
     elif pathcomponents[2] == 'by-mac':
         if len(pathcomponents) == 3:
             return [msg.ChildCollection(x.replace(':', '-'))
-                    for x in sorted(list(_macmap))]
+                    for x in sorted(list(_apimacmap))]
         elif len(pathcomponents) == 4:
             return dump_macinfo(pathcomponents[-1])
     elif pathcomponents[2] == 'by-switch':
@@ -457,6 +463,8 @@ def handle_read_api_request(pathcomponents, configmanager):
                     for x in sorted(maclist)]
         if len(pathcomponents) == 8:
             return dump_macinfo(pathcomponents[-1])
+    elif pathcomponents[2] == 'rescan':
+        return [msg.KeyValueData({'scanning': mapupdating.locked()})]
     raise exc.NotFoundException('Unrecognized path {0}'.format(
         '/'.join(pathcomponents)))
 

confluent_server/confluent/plugins/configuration/attributes.py

@@ -100,7 +100,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
 def retrieve_nodes(nodes, element, configmanager, inputdata):
     attributes = configmanager.get_node_attributes(nodes)
     if element[-1] == 'all':
-        for node in nodes:
+        for node in util.natural_sort(nodes):
             theattrs = set(allattributes.node).union(set(attributes[node]))
             for attribute in sorted(theattrs):
                 if attribute in attributes[node]:  # have a setting for it
@@ -125,9 +125,11 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                         allattributes.node.get(
                             attribute, {}).get('description', ''))
     elif element[-1] == 'current':
-        for node in sorted(list(attributes)):
+        for node in util.natural_sort(list(attributes)):
             for attribute in sorted(attributes[node].iterkeys()):
                 currattr = attributes[node][attribute]
+                if currattr is None:
+                    continue
                 try:
                     desc = allattributes.node[attribute]['description']
                 except KeyError:
@@ -140,6 +142,8 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                 elif isinstance(currattr, list):
                     yield msg.ListAttributes(
                         node, {attribute: currattr}, desc)
+                elif isinstance(currattr, str):
+                    yield msg.Attributes(node, {attribute: currattr}, desc)
                 else:
                     print attribute
                     print repr(currattr)

confluent_server/confluent/plugins/hardwaremanagement/enclosure.py

@@ -14,6 +14,7 @@
 import confluent.core as core
 import confluent.messages as msg
 import pyghmi.exceptions as pygexc
+import confluent.exceptions as exc
 
 def update(nodes, element, configmanager, inputdata):
     emebs = configmanager.get_node_attributes(
@@ -36,3 +37,5 @@ def update(nodes, element, configmanager, inputdata):
                 yield rsp
         except pygexc.UnsupportedFunctionality as uf:
             yield msg.ConfluentNodeError(node, str(uf))
+        except exc.TargetEndpointUnreachable as uf:
+            yield msg.ConfluentNodeError(node, str(uf))

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -1,5 +1,5 @@
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,19 +19,55 @@ import confluent.firmwaremanager as firmwaremanager
 import confluent.interface.console as conapi
 import confluent.messages as msg
 import confluent.util as util
+import copy
+import errno
 import eventlet
 import eventlet.event
 import eventlet.green.threading as threading
 import eventlet.greenpool as greenpool
 import eventlet.queue as queue
 import eventlet.support.greendns
+from fnmatch import fnmatch
+import os
 import pyghmi.constants as pygconstants
 import pyghmi.exceptions as pygexc
+import pyghmi.storage as storage
 console = eventlet.import_patched('pyghmi.ipmi.console')
 ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
 import socket
 import ssl
 
+pci_cache = {}
+
+def get_dns_txt(qstring):
+    return eventlet.support.greendns.resolver.query(
+        qstring, 'TXT')[0].strings[0].replace('i=', '')
+
+def get_pci_text_from_ids(subdevice, subvendor, device, vendor):
+    fqpi = '{0}.{1}.{2}.{3}'.format(subdevice, subvendor, device, vendor)
+    if fqpi in pci_cache:
+        return pci_cache[fqpi]
+    vendorstr = None
+    try:
+        vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(subvendor))
+    except Exception:
+        try:
+            vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(vendor))
+        except Exception:
+            pass
+    devstr = None
+    try:
+        devstr = get_dns_txt(fqpi + '.pci.id.ucw.cz')
+    except Exception:
+        try:
+            devstr = get_dns_txt('{0}.{1}.pci.id.ucw.cz'.format(
+                device, vendor))
+        except Exception:
+            pass
+    if vendorstr and devstr:
+        pci_cache[fqpi] = vendorstr, devstr
+    return vendorstr, devstr
+
 
 # There is something not right with the RLocks used in pyghmi when
 # eventlet comes into play.  It seems like sometimes on acquire,
@@ -96,7 +132,8 @@ def hex2bin(hexstring):
 
 
 def simplify_name(name):
-    return name.lower().replace(' ', '_').replace('/', '-')
+    return name.lower().replace(' ', '_').replace('/', '-').replace(
+        '_-_', '-')
 
 
 def sanitize_invdata(indata):
@@ -137,6 +174,17 @@ class IpmiCommandWrapper(ipmicommand.Command):
                       'hardwaremanagement.manager'), self._attribschanged)
         super(self.__class__, self).__init__(**kwargs)
         self.setup_confluent_keyhandler()
+        try:
+            os.makedirs('/var/cache/confluent/ipmi/')
+        except OSError as e:
+            if e.errno != errno.EEXIST or not os.path.isdir(
+                    '/var/cache/confluent/ipmi/'):
+                raise
+        try:
+            self.set_sdr_cachedir('/var/cache/confluent/ipmi/')
+        except Exception:
+            pass
+
 
     def setup_confluent_keyhandler(self):
         self.register_key_handler(util.TLSCertVerifier(
@@ -270,13 +318,17 @@ class IpmiConsole(conapi.Console):
                                                  kg=self.kg, force=True,
                                                  iohandler=self.handle_data)
             self.solconnection.outputlock = NullLock()
-            while not self.solconnection.connected and not self.broken:
+            while (self.solconnection and not self.solconnection.connected and
+                   not (self.broken or self.solconnection.broken or
+                        self.solconnection.ipmi_session.broken)):
                 w = eventlet.event.Event()
                 _ipmiwaiters.append(w)
-                w.wait()
-                if self.broken:
-                    break
-            if self.broken:
+                w.wait(15)
+            if (self.broken or not self.solconnection or
+                    self.solconnection.broken or
+                    self.solconnection.ipmi_session.broken):
+                if not self.error:
+                    self.error = 'Unknown error'
                 if (self.error.startswith('Incorrect password') or
                         self.error.startswith('Unauthorized name')):
                     raise exc.TargetEndpointBadCredentials
@@ -303,28 +355,40 @@ class IpmiConsole(conapi.Console):
         self.solconnection.send_break()
 
 
-def perform_requests(operator, nodes, element, cfg, inputdata):
+def perform_requests(operator, nodes, element, cfg, inputdata, realop):
     cryptit = cfg.decrypt
     cfg.decrypt = True
     configdata = cfg.get_node_attributes(nodes, _configattributes)
     cfg.decrypt = cryptit
     resultdata = queue.LightQueue()
     livingthreads = set([])
+    numnodes = len(nodes)
     for node in nodes:
         livingthreads.add(_ipmiworkers.spawn(
             perform_request, operator, node, element, configdata, inputdata,
-            cfg, resultdata))
+            cfg, resultdata, realop))
     while livingthreads:
         try:
+            bundle = []
             datum = resultdata.get(timeout=10)
             while datum:
                 if datum != 'Done':
                     if isinstance(datum, Exception):
                         raise datum
-                    yield datum
-                datum = resultdata.get_nowait()
+                    if (hasattr(datum, 'kvpairs') and datum.kvpairs and
+                            len(datum.kvpairs) == 1):
+                        bundle.append((datum.kvpairs.keys()[0], datum))
+                        numnodes -= 1
+                    else:
+                        yield datum
+                timeout = 0.1 if numnodes else 0.001
+                datum = resultdata.get(timeout=timeout)
         except queue.Empty:
             pass
+        finally:
+            for datum in sorted(
+                    bundle, key=lambda x: util.naturalize_string(x[0])):
+                yield datum[1]
         for t in list(livingthreads):
             if t.dead:
                 livingthreads.discard(t)
@@ -339,10 +403,10 @@ def perform_requests(operator, nodes, element, cfg, inputdata):
 
 
 def perform_request(operator, node, element,
-                    configdata, inputdata, cfg, results):
+                    configdata, inputdata, cfg, results, realop):
         try:
             return IpmiHandler(operator, node, element, configdata, inputdata,
-                               cfg, results).handle_request()
+                               cfg, results, realop).handle_request()
         except pygexc.IpmiException as ipmiexc:
             excmsg = str(ipmiexc)
             if excmsg in ('Session no longer connected', 'timeout'):
@@ -354,8 +418,8 @@ def perform_request(operator, node, element,
             results.put(msg.ConfluentTargetTimeout(node, str(tu)))
         except ssl.SSLEOFError:
             results.put(msg.ConfluentNodeError(
-                self.node, 'Unable to communicate with the https server on '
-                           'the target BMC'))
+                node, 'Unable to communicate with the https server on '
+                      'the target BMC'))
         except exc.PubkeyInvalid:
             results.put(msg.ConfluentNodeError(
                 node,
@@ -372,7 +436,8 @@ def perform_request(operator, node, element,
 persistent_ipmicmds = {}
 
 class IpmiHandler(object):
-    def __init__(self, operation, node, element, cfd, inputdata, cfg, output):
+    def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
+                 realop):
         self.sensormap = {}
         self.invmap = {}
         self.output = output
@@ -381,17 +446,20 @@ class IpmiHandler(object):
         self.error = None
         eventlet.sleep(0)
         self.cfg = cfd[node]
+        self.current_user = cfg.current_user
         self.loggedin = False
         self.node = node
         self.element = element
         self.op = operation
+        self.realop = realop
         connparams = get_conn_params(node, self.cfg)
         self.ipmicmd = None
         self.inputdata = inputdata
         self.tenant = cfg.tenant
         tenant = cfg.tenant
         if ((node, tenant) not in persistent_ipmicmds or
-                not persistent_ipmicmds[(node, tenant)].ipmi_session.logged):
+                not persistent_ipmicmds[(node, tenant)].ipmi_session.logged or
+                persistent_ipmicmds[(node, tenant)].ipmi_session.broken):
             try:
                 persistent_ipmicmds[(node, tenant)].close_confluent()
             except KeyError:  # was no previous session
@@ -438,6 +506,10 @@ class IpmiHandler(object):
                 self.output.put(msg.ConfluentTargetTimeout(
                     self.node, self.error))
                 return
+            elif 'Invalid Session ID' in self.error:
+                self.output.put(msg.ConfluentTargetTimeout(
+                    self.node, 'Temporary Login Error'))
+                return
             elif ('Unauthorized' in self.error or
                     'Incorrect password' in self.error):
                 self.output.put(
@@ -457,6 +529,8 @@ class IpmiHandler(object):
             self.identify()
         elif self.element[0] == 'sensors':
             self.handle_sensors()
+        elif self.element[:2] == ['configuration', 'storage']:
+            self.handle_storage()
         elif self.element[0] == 'configuration':
             self.handle_configuration()
         elif self.element[:3] == ['inventory', 'firmware', 'updates']:
@@ -477,12 +551,16 @@ class IpmiHandler(object):
             self.decode_alert()
         elif self.element == ['console', 'license']:
             self.handle_license()
+        elif self.element == ['support', 'servicedata']:
+            self.handle_servicedata_fetch()
+        elif self.element == ['description']:
+            self.handle_description()
         else:
             raise Exception('Not Implemented')
 
     def handle_update(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
-                                    self.inputdata.filename, self.tenant,
+                                    self.inputdata.nodefile(self.node), self.tenant,
                                     bank=self.inputdata.bank)
         self.output.put(
             msg.CreatedResource(
@@ -491,14 +569,23 @@ class IpmiHandler(object):
 
     def handle_media_upload(self):
         u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
-                                     self.inputdata.filename, self.tenant,
+                                     self.inputdata.nodefile(self.node), self.tenant,
                                      type='mediaupload')
         self.output.put(msg.CreatedResource(
             'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))
 
+    def handle_servicedata_fetch(self):
+        u = firmwaremanager.Updater(
+            self.node, self.ipmicmd.get_diagnostic_data,
+            self.inputdata.nodefile(self.node), self.tenant, type='ffdc',
+            owner=self.current_user)
+        self.output.put(msg.CreatedResource(
+            'nodes/{0}/support/servicedata/{1}'.format(self.node, u.name)))
+
     def handle_attach_media(self):
         try:
-            self.ipmicmd.attach_remote_media(self.inputdata.filename)
+            self.ipmicmd.attach_remote_media(self.inputdata.nodefile(
+                self.node))
         except pygexc.UnsupportedFunctionality as uf:
             self.output.put(msg.ConfluentNodeError(self.node, str(uf)))
 
@@ -528,6 +615,12 @@ class IpmiHandler(object):
             return self.handle_ntp()
         elif self.element[1:3] == ['system', 'all']:
             return self.handle_sysconfig()
+        elif self.element[1:3] == ['system', 'advanced']:
+            return self.handle_sysconfig(True)
+        elif self.element[1:3] == ['system', 'clear']:
+            return self.handle_sysconfigclear()
+        elif self.element[1:3] == ['management_controller', 'licenses']:
+            return self.handle_licenses()
         raise Exception('Not implemented')
 
     def decode_alert(self):
@@ -814,12 +907,14 @@ class IpmiHandler(object):
             if component == 'all':
                 for invdata in self.ipmicmd.get_inventory():
                     if invdata[1] is None:
-                        newinf = {'present': False, 'information': None}
+                        newinf = {'present': False, 'information': None,
+                                  'name': invdata[0]}
+
                     else:
                         sanitize_invdata(invdata[1])
                         newinf = {'present': True, 'information': invdata[1]}
-                    newinf['name'] = invdata[0]
-                    invitems.append(newinf)
+                        newinf['name'] = invdata[1].get('name', invdata[0])
+                    self.add_invitem(invitems, newinf)
             else:
                 self.make_inventory_map()
                 compname = self.invmap.get(component, None)
@@ -828,12 +923,13 @@ class IpmiHandler(object):
                     return
                 invdata = self.ipmicmd.get_inventory_of_component(compname)
                 if invdata is None:
-                    newinf = {'present': False, 'information': None}
+                    newinf = {'present': False, 'information': None,
+                              'name': compname}
                 else:
                     sanitize_invdata(invdata)
-                    newinf = {'present': True, 'information': invdata}
-                newinf['name'] = compname
-                invitems.append(newinf)
+                    newinf = {'present': True, 'information': invdata,
+                              'name': invdata.get('name', compname)}
+                self.add_invitem(invitems, newinf)
         except ssl.SSLEOFError:
             errorneeded = msg.ConfluentNodeError(
                 self.node, 'Unable to communicate with the https server on '
@@ -850,6 +946,131 @@ class IpmiHandler(object):
         if errorneeded:
             self.output.put(errorneeded)
 
+    def add_invitem(self, invitems, newinf):
+        if newinf.get('information', None) and 'name' in newinf['information']:
+            newinf = copy.deepcopy(newinf)
+            del newinf['information']['name']
+        if fnmatch(newinf['name'], 'Adapter ??:??:??') or fnmatch(
+                newinf['name'], 'PCIeGen? x*'):
+            myinf = newinf.get('information', {})
+            sdid = myinf.get('PCI Subsystem Device ID', None)
+            svid = myinf.get('PCI Subsystem Vendor ID', None)
+            did = myinf.get('PCI Device ID', None)
+            vid = myinf.get('PCI Vendor ID', None)
+            vstr, dstr = get_pci_text_from_ids(sdid, svid, did, vid)
+            if vstr:
+                newinf['information']['PCI Vendor'] = vstr
+            if dstr:
+                newinf['name'] = dstr
+        invitems.append(newinf)
+
+    def handle_storage(self):
+        if self.element[-1] == '':
+            self.element = self.element[:-1]
+        storelem = self.element[2:]
+        if 'read' == self.op:
+            return self._show_storage(storelem)
+        elif 'update' == self.realop:
+            return self._update_storage(storelem)
+        elif 'delete' == self.op:
+            return self._delete_storage(storelem)
+        elif 'create' == self.realop:
+            return self._create_storage(storelem)
+
+    def _delete_storage(self, storelem):
+        if len(storelem) < 2 or storelem[0] != 'volumes':
+            raise exc.InvalidArgumentException('Must target a specific volume')
+        volname = storelem[-1]
+        curr = self.ipmicmd.get_storage_configuration()
+        volumes = []
+        volsfound = False
+        toremove = storage.ConfigSpec(arrays=[storage.Array(volumes=volumes)])
+        for pool in curr.arrays:
+            for vol in pool.volumes:
+                if simplify_name(vol.name) == volname:
+                    volsfound = True
+                    volumes.append(vol)
+        if not volsfound:
+            self.output.put(msg.ConfluentTargetNotFound(
+                self.node, "No volume named '{0}' found".format(volname)))
+            return
+        self.ipmicmd.remove_storage_configuration(toremove)
+        self.output.put(msg.DeletedResource(volname))
+
+    def _create_storage(self, storelem):
+        if 'volumes' not in storelem:
+            raise exc.InvalidArgumentException('Can only create volumes')
+        vols = []
+        thedisks = None
+        currcfg = self.ipmicmd.get_storage_configuration()
+        currnames = []
+        for arr in currcfg.arrays:
+            arrname = '{0}-{1}'.format(*arr.id)
+            for vol in arr.volumes:
+                currnames.append(vol.name)
+        disks = []
+        vols = []
+        vol = self.inputdata.inputbynode[self.node][0]
+        raidlvl = vol['raidlevel']
+        for disk in currcfg.disks:
+            if simplify_name(disk.name) in vol['disks']:
+                disks.append(disk)
+            elif (disk.status == 'Unconfigured Good' and
+                  vol['disks'][0] in ('remainder', 'rest')):
+                disks.append(disk)
+            elif vol['disks'][0] == 'all':
+                disks.append(disk)
+        for vol in self.inputdata.inputbynode[self.node]:
+            if thedisks and thedisks != vol['disks']:
+                    raise exc.InvalidArgumentException(
+                        'Not currently supported to create multiple arrays '
+                        'in a single request')
+            if raidlvl and vol['raidlevel'] != raidlvl:
+                raise exc.InvalidArgumentException('Cannot mix raid levels in '
+                                                   'a single array')
+            vols.append(storage.Volume(name=vol['name'], size=vol['size']))
+        newcfg = storage.ConfigSpec(
+            arrays=(storage.Array(raid=raidlvl, disks=disks, volumes=vols),))
+        self.ipmicmd.apply_storage_configuration(newcfg)
+        for vol in self.inputdata.inputbynode[self.node]:
+            if vol['name'] is None:
+                newcfg = self.ipmicmd.get_storage_configuration()
+                for arr in newcfg.arrays:
+                    arrname = '{0}-{1}'.format(*arr.id)
+                    for vol in arr.volumes:
+                        if vol.name not in currnames:
+                            self.output.put(
+                                msg.Volume(self.node, vol.name, vol.size,
+                                           vol.status, arrname))
+                return
+            else:
+                self._show_storage(storelem[:1] + [vol['name']])
+
+    def _update_storage(self, storelem):
+        if storelem[0] == 'disks':
+            if len(storelem) == 1:
+                raise exc.InvalidArgumentException('Must target a disk')
+            self.set_disk(storelem[-1],
+                          self.inputdata.inputbynode[self.node])
+        self._show_storage(storelem)
+
+    def _show_storage(self, storelem):
+        if storelem[0] == 'disks':
+            if len(storelem) == 1:
+                return self.list_disks()
+            return self.show_disk(storelem[1])
+        elif storelem[0] == 'arrays':
+            if len(storelem) == 1:
+                return self.list_arrays()
+            return self.show_array(storelem[1])
+        elif storelem[0] == 'volumes':
+            if len(storelem) == 1:
+                return self.list_volumes()
+            return self.show_volume(storelem[1])
+        elif storelem[0] == 'all':
+            return self._show_all_storage()
+
+
     def handle_sensors(self):
         if self.element[-1] == '':
             self.element = self.element[:-1]
@@ -873,6 +1094,100 @@ class IpmiHandler(object):
             return True
         return False
 
+    def set_disk(self, name, state):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for disk in scfg.disks:
+            if (name == 'all' or simplify_name(disk.name) == name or
+                    disk == name):
+                disk.status = state
+        self.ipmicmd.apply_storage_configuration(
+            storage.ConfigSpec(disks=scfg.disks))
+
+    def _show_all_storage(self):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for disk in scfg.disks:
+            self.output.put(
+                msg.Disk(self.node, disk.name, disk.description,
+                         disk.id, disk.status, disk.serial,
+                         disk.fru))
+        for arr in scfg.arrays:
+            for disk in arr.disks:
+                self.output.put(
+                    msg.Disk(self.node, disk.name, disk.description,
+                             disk.id, disk.status, disk.serial,
+                             disk.fru, array='{0}-{1}'.format(*arr.id)))
+        for arr in scfg.arrays:
+            arrname = '{0}-{1}'.format(*arr.id)
+            self._detail_array(arr, arrname, True)
+
+    def show_disk(self, name):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for disk in scfg.disks:
+            if simplify_name(disk.name) == name or disk == name:
+                self.output.put(
+                    msg.Disk(self.node, disk.name, disk.description,
+                                         disk.id, disk.status, disk.serial,
+                                         disk.fru))
+        for arr in scfg.arrays:
+            arrname = '{0}-{1}'.format(*arr.id)
+            for disk in arr.disks:
+                if (name == 'all' or simplify_name(disk.name) == name or
+                        disk == name):
+                    self.output.put(
+                        msg.Disk(self.node, disk.name, disk.description,
+                                 disk.id, disk.status, disk.serial,
+                                 disk.fru, arrname))
+
+    def list_disks(self):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for disk in scfg.disks:
+            self.output.put(msg.ChildCollection(simplify_name(disk.name)))
+        for arr in scfg.arrays:
+            for disk in arr.disks:
+                self.output.put(msg.ChildCollection(simplify_name(disk.name)))
+
+    def list_arrays(self):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for arr in scfg.arrays:
+            self.output.put(msg.ChildCollection('{0}-{1}'.format(*arr.id)))
+
+    def show_array(self, name):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for arr in scfg.arrays:
+            arrname = '{0}-{1}'.format(*arr.id)
+            if arrname == name:
+                self._detail_array(arr, arrname)
+
+    def _detail_array(self, arr, arrname, detailvol=False):
+        vols = []
+        for vol in arr.volumes:
+            vols.append(simplify_name(vol.name))
+        disks = []
+        for disk in arr.disks:
+            disks.append(simplify_name(disk.name))
+        self.output.put(msg.Array(self.node, disks, arr.raid,
+                                  vols, arrname, arr.capacity,
+                                  arr.available_capacity))
+        if detailvol:
+            for vol in arr.volumes:
+                self.output.put(msg.Volume(self.node, vol.name, vol.size,
+                                           vol.status, arrname))
+
+    def show_volume(self, name):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for arr in scfg.arrays:
+            arrname = '{0}-{1}'.format(*arr.id)
+            for vol in arr.volumes:
+                if name == simplify_name(vol.name):
+                    self.output.put(msg.Volume(self.node, vol.name, vol.size,
+                                               vol.status, arrname))
+
+    def list_volumes(self):
+        scfg = self.ipmicmd.get_storage_configuration()
+        for arr in scfg.arrays:
+            for vol in arr.volumes:
+                self.output.put(msg.ChildCollection(simplify_name(vol.name)))
+
     def list_sensors(self):
         try:
             sensors = self.ipmicmd.get_sensor_descriptions()
@@ -1020,10 +1335,17 @@ class IpmiHandler(object):
             self.ipmicmd.set_domain_name(dn)
             return
 
-    def handle_sysconfig(self):
+    def handle_sysconfigclear(self):
+        if 'read' == self.op:
+            raise exc.InvalidArgumentException(
+                'Cannot read the "clear" resource')
+        self.ipmicmd.clear_system_configuration()
+
+    def handle_sysconfig(self, advanced=False):
         if 'read' == self.op:
             self.output.put(msg.ConfigSet(
-                self.node, self.ipmicmd.get_system_configuration()))
+                self.node, self.ipmicmd.get_system_configuration(
+                    hideadvanced=not advanced)))
         elif 'update' == self.op:
             self.ipmicmd.set_system_configuration(
                 self.inputdata.get_attributes(self.node))
@@ -1078,6 +1400,32 @@ class IpmiHandler(object):
         self.output.put(msg.License(self.node, available))
         return
 
+    def handle_licenses(self):
+        if self.element[-1] == '':
+            self.element = self.element[:-1]
+        if self.op in ('create', 'update'):
+            self.ipmicmd.apply_license(self.inputdata.nodefile(self.node))
+        if len(self.element) == 3:
+            self.output.put(msg.ChildCollection('all'))
+            i = 1
+            for lic in self.ipmicmd.get_licenses():
+                self.output.put(msg.ChildCollection(str(i)))
+                i += 1
+            return
+        licname = self.element[3]
+        if licname == 'all':
+            for lic in self.ipmicmd.get_licenses():
+                self.output.put(msg.License(self.node, feature=lic['name']))
+        else:
+            index = int(licname)
+            lic = list(self.ipmicmd.get_licenses())[index - 1]
+            self.output.put(msg.License(self.node, feature=lic['name']))
+
+    def handle_description(self):
+        dsc = self.ipmicmd.get_description()
+        self.output.put(msg.KeyValueData(dsc, self.node))
+
+
 def _str_health(health):
     if isinstance(health, str):
         return health
@@ -1098,7 +1446,7 @@ def initthread():
         _ipmithread = eventlet.spawn(_ipmi_evtloop)
 
 
-def create(nodes, element, configmanager, inputdata):
+def create(nodes, element, configmanager, inputdata, realop='create'):
     initthread()
     if element == ['_console', 'session']:
         if len(nodes) > 1:
@@ -1106,12 +1454,12 @@ def create(nodes, element, configmanager, inputdata):
         return IpmiConsole(nodes[0], configmanager)
     else:
         return perform_requests(
-            'update', nodes, element, configmanager, inputdata)
+            'update', nodes, element, configmanager, inputdata, realop)
 
 
 def update(nodes, element, configmanager, inputdata):
     initthread()
-    return create(nodes, element, configmanager, inputdata)
+    return create(nodes, element, configmanager, inputdata, 'update')
 
 
 def retrieve(nodes, element, configmanager, inputdata):
@@ -1122,8 +1470,12 @@ def retrieve(nodes, element, configmanager, inputdata):
     elif '/'.join(element).startswith('media/uploads'):
         return firmwaremanager.list_updates(nodes, configmanager.tenant,
                                             element, 'mediaupload')
+    elif '/'.join(element).startswith('support/servicedata'):
+        return firmwaremanager.list_updates(nodes, configmanager.tenant,
+                                            element, 'ffdc')
     else:
-        return perform_requests('read', nodes, element, configmanager, inputdata)
+        return perform_requests('read', nodes, element, configmanager,
+                                inputdata, 'read')
 
 def delete(nodes, element, configmanager, inputdata):
     initthread()
@@ -1133,5 +1485,9 @@ def delete(nodes, element, configmanager, inputdata):
     elif '/'.join(element).startswith('media/uploads'):
         return firmwaremanager.remove_updates(nodes, configmanager.tenant,
                                               element, type='mediaupload')
+    elif '/'.join(element).startswith('support/servicedata'):
+        return firmwaremanager.remove_updates(nodes, configmanager.tenant,
+                                              element, type='ffdc')
     return perform_requests(
-        'delete', nodes, element, configmanager, inputdata)
+        'delete', nodes, element, configmanager, inputdata, 'delete')
+

confluent_server/confluent/plugins/shell/ssh.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,9 +22,22 @@
 import confluent.exceptions as cexc
 import confluent.interface.console as conapi
 import confluent.log as log
+import cryptography
+
 import eventlet
 import hashlib
+import sys
+sys.modules['gssapi'] = None
 paramiko = eventlet.import_patched('paramiko')
+warnhostkey = False
+if cryptography.__version__.split('.') < ['1', '5']:
+    # older cryptography with paramiko breaks most key support except
+    # ed25519
+    warnhostkey = True
+    paramiko.transport.Transport._preferred_keys = filter(
+        lambda x: 'ed25519' in x,
+        paramiko.transport.Transport._preferred_keys)
+
 
 
 class HostKeyHandler(paramiko.client.MissingHostKeyPolicy):
@@ -67,8 +80,18 @@ class SshShell(conapi.Console):
         self.nodeconfig = config
         self.username = username
         self.password = password
+        self.connected = False
+        self.width = 80
+        self.height = 24
         self.inputmode = 0  # 0 = username, 1 = password...
 
+    def resize(self, width, height):
+        self.width = width
+        self.height = height
+        if not self.connected:
+            return
+        self.shell.resize_pty(width=width, height=height)
+
     def recvdata(self):
         while self.connected:
             pendingdata = self.shell.recv(8192)
@@ -110,14 +133,64 @@ class SshShell(conapi.Console):
             self.username = ''
             self.password = ''
             self.datacallback('\r\nlogin as: ')
+            return
+        except cexc.PubkeyInvalid as pi:
+            self.keyaction = ''
+            self.candidatefprint = pi.fingerprint
+            self.datacallback(pi.message)
+            self.keyattrname = pi.attrname
+            self.datacallback('\r\nNew fingerprint: ' + pi.fingerprint)
+            self.inputmode = -1
+            self.datacallback('\r\nEnter "disconnect" or "accept": ')
+            return
+        except paramiko.SSHException as pi:
+            self.inputmode = -2
+            warn = str(pi)
+            if warnhostkey:
+                warn += ' (Older cryptography package on this host only ' \
+                        'works with ed25519, check ssh startup on target ' \
+                        'and permissions on /etc/ssh/*key)\r\n' \
+                        'Press Enter to close...'
+            self.datacallback('\r\n' + warn)
+
             return
         self.inputmode = 2
         self.connected = True
-        self.shell = self.ssh.invoke_shell()
+        self.shell = self.ssh.invoke_shell(width=self.width,
+                                           height=self.height)
         self.rxthread = eventlet.spawn(self.recvdata)
 
     def write(self, data):
-        if self.inputmode == 0:
+        if self.inputmode == -2:
+            self.datacallback(conapi.ConsoleEvent.Disconnect)
+            return
+        elif self.inputmode == -1:
+            while len(data) and data[0] == b'\x7f' and len(self.keyaction):
+                self.datacallback('\b \b')  # erase previously echoed value
+                self.keyaction = self.keyaction[:-1]
+                data = data[1:]
+            while len(data) and data[0] == b'\x7f':
+                data = data[1:]
+            while b'\x7f' in data:
+                delidx = data.index(b'\x7f')
+                data = data[:delidx - 1] + data[delidx + 1:]
+            self.keyaction += data
+            if '\r' in self.keyaction:
+                action = self.keyaction.split('\r')[0]
+                if action.lower() == 'accept':
+                    self.nodeconfig.set_node_attributes(
+                        {self.node:
+                             {self.keyattrname: self.candidatefprint}})
+                    self.datacallback('\r\n')
+                    self.logon()
+                elif action.lower() == 'disconnect':
+                    self.datacallback(conapi.ConsoleEvent.Disconnect)
+                else:
+                    self.keyaction = ''
+                    self.datacallback('\r\nEnter "disconnect" or "accept": ')
+            elif len(data) > 0:
+                self.datacallback(data)
+        elif self.inputmode == 0:
             while len(data) and data[0] == b'\x7f' and len(self.username):
                 self.datacallback('\b \b')  # erase previously echoed value
                 self.username = self.username[:-1]

confluent_server/confluent/shellserver.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -87,12 +87,13 @@ class ShellSession(consoleserver.ConsoleSession):
     """
 
     def __init__(self, node, configmanager, username, datacallback=None,
-                 skipreplay=False, sessionid=None):
+                 skipreplay=False, sessionid=None, width=80, height=24):
         self.sessionid = sessionid
         self.configmanager = configmanager
         self.node = node
         super(ShellSession, self).__init__(node, configmanager, username,
-                                           datacallback, skipreplay)
+                                           datacallback, skipreplay,
+                                           width=width, height=height)
 
     def connect_session(self):
         global activesessions
@@ -105,7 +106,7 @@ class ShellSession(consoleserver.ConsoleSession):
                 self.sessionid += 1
             self.sessionid = str(self.sessionid)
         if self.sessionid not in activesessions[(tenant, self.node, self.username)]:
-            activesessions[(tenant, self.node, self.username)][self.sessionid] = _ShellHandler(self.node, self.configmanager)
+            activesessions[(tenant, self.node, self.username)][self.sessionid] = _ShellHandler(self.node, self.configmanager, width=self.width, height=self.height)
         self.conshdl = activesessions[(self.configmanager.tenant, self.node, self.username)][self.sessionid]
 
     def destroy(self):

confluent_server/confluent/snmputil.py

@@ -92,12 +92,17 @@ class Session(object):
                 errstr, errnum, erridx, answers = rsp
                 if errstr:
                     errstr = str(errstr)
-                    if errstr in ('unknownUserName', 'wrongDigest'):
-                        raise exc.TargetEndpointBadCredentials(errstr)
+                    finerr = errstr + ' while trying to connect to ' \
+                                      '{0}'.format(self.server)
+                    if errstr in ('Unknown USM user', 'unknownUserName',
+                                  'wrongDigest', 'Wrong SNMP PDU digest'):
+                        raise exc.TargetEndpointBadCredentials(finerr)
                     # need to do bad credential versus timeout
-                    raise exc.TargetEndpointUnreachable(errstr)
+                    raise exc.TargetEndpointUnreachable(finerr)
                 elif errnum:
-                    raise exc.ConfluentException(errnum.prettyPrint())
+                    raise exc.ConfluentException(errnum.prettyPrint() +
+                                                 ' while trying to connect to '
+                                                 '{0}'.format(self.server))
                 for ans in answers:
                     if not obj[0].isPrefixOf(ans[0]):
                         # PySNMP returns leftovers in a bulk command

confluent_server/confluent/sockapi.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,7 +46,9 @@ import confluent.core as pluginapi
 import confluent.shellserver as shellserver
 import confluent.collective.manager as collective
 import confluent.util as util
+from confluent.xcclient.xcat_manager import xCATConfigManager
 
+xcat_cfm = None
 tracelog = None
 auditlog = None
 try:
@@ -115,6 +117,8 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
         if authdata is not None:
             cfm = authdata[1]
             authenticated = True
+    if xcat_cfm is None:
+        xcat_cfm = xCATConfigManager()
     send_data(connection, "Confluent -- v0 --")
     while not authenticated:  # prompt for name and passphrase
         send_data(connection, {'authpassed': 0})
@@ -143,10 +147,22 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             cfm = authdata[1]
     send_data(connection, {'authpassed': 1})
     request = tlvdata.recv(connection)
-    if 'collective' in request and skipauth:
+    if request and 'collective' in request and skipauth:
+        if not libssl:
+            tlvdata.send(
+                connection,
+                {'collective': {'error': 'Server either does not have '
+                                         'python-pyopenssl installed or has an '
+                                         'incorrect version installed '
+                                         '(e.g. pyOpenSSL would need to be '
+                                         'replaced with python-pyopenssl)'}})
+            return
         return collective.handle_connection(connection, None, request['collective'],
                                      local=True)
     while request is not None:
+        if request['path'].startswith('/xcat'):
+            global xcat_cfm
+            cfm = xcat_cfm
         try:
             process_request(
                 connection, request, cfm, authdata, authname, skipauth)
@@ -229,7 +245,9 @@ def start_proxy_term(connection, cert, request):
     ccons = ClientConsole(connection)
     consession = consoleserver.ConsoleSession(
         node=request['node'], configmanager=cfm, username=request['user'],
-        datacallback=ccons.sendall, skipreplay=request['skipreplay'])
+        datacallback=ccons.sendall, skipreplay=request['skipreplay'],
+        direct=False, width=request.get('width', 80), height=request.get(
+            'height', 24))
     term_interact(None, None, ccons, None, connection, consession, None)
 
 def start_term(authname, cfm, connection, params, path, authdata, skipauth):
@@ -253,7 +271,8 @@ def start_term(authname, cfm, connection, params, path, authdata, skipauth):
         consession = shellserver.ShellSession(
             node=node, configmanager=cfm, username=authname,
             datacallback=ccons.sendall, skipreplay=skipreplay,
-            sessionid=sessionid)
+            sessionid=sessionid, width=params.get('width', 80),
+            height=params.get('height', 24))
     else:
         raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
     if consession is None:
@@ -281,6 +300,9 @@ def term_interact(authdata, authname, ccons, cfm, connection, consession,
             elif data['operation'] == 'reopen':
                 consession.reopen()
                 continue
+            elif data['operation'] == 'resize':
+                consession.resize(width=data['width'], height=data['height'])
+                continue
             else:
                 try:
                     process_request(connection, data, cfm, authdata, authname,
@@ -314,6 +336,8 @@ def _tlshandler(bind_host, bind_port):
                 raise
             sys.stderr.write('TLS Socket in use, retrying in 1 second\n')
             eventlet.sleep(1)
+    # Enable TCP_FASTOPEN
+    plainsocket.setsockopt(socket.SOL_TCP, 23, 5)
     plainsocket.listen(5)
     while (1):  # TODO: exithook
         cnn, addr = plainsocket.accept()

confluent_server/confluent/xcclient/attributes.py

@@ -0,0 +1,93 @@
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+# -*- encoding: utf-8 -*-
+
+
+# 'node', which can be considered a 'system' or a 'vm'
+nodeattrs = {
+    'groups': 'nodelist.groups',
+    'collective.manager': None,
+    'discovery.passwordrules': None,
+    'discovery.policy': ['switch', 'mtms', 'sequential'],
+    'info.note': 'nodelist.comments',
+    'location.room': 'nodepos.room',
+    'location.row': 'nodepos.comments', 
+    'location.rack': 'nodepos.rack',
+    'location.u': 'nodepos.u',
+    'console.logging': 'site.consoleondemand',
+    'console.method': 'nodehm.cons',
+    'virtualization.host': 'vm.host',
+    'hardwaremanagement.manager': 'ipmi.bmc',
+    'hardwaremanagement.method': 'nodehm.mgt',
+    'enclosure.bay': None,
+    'enclosure.extends': None,
+    'enclosure.manager': None,
+    'id.model': 'vpd.mtm',
+    'id.serial': 'vpd.serial',
+    'id.uuid': 'vpd.uuid',
+    'net.bootable': None, 
+    'net.ipv4_gateway': None,
+    'net.hwaddr': 'mac.mac',
+    'net.switch': 'switch.switch',
+    'net.switchport': 'switch.port',
+    'secret.snmpcommunity': 'site.snmpc',
+    'secret.ipmikg': None,
+    'secret.hardwaremanagementuser': 'ipmi.username',
+    'secret.hardwaremanagementpassword': 'ipmi.password',
+    'pubkeys.addpolicy': None,
+    'pubkeys.ssh': None,
+}
+
+def conattr2xcatattr(attrs=()):
+    if '*' in attrs or attrs == ():
+        return nodeattrs
+    attr_dict = {}
+    if not isinstance(attrs, list):
+        attr_dict[attrs] = nodeattrs.get(attrs, None)
+    else:
+        for attr in attrs:
+            attr_dict[attr] = nodeattrs.get(attr, None)
+    return attr_dict
+
+def conattr2xcatattrv1(attrs):
+    if '*' in attrs:
+        attrs = nodeattrs.keys()
+    all_attr_set=set(nodeattrs.keys())
+    check_set=set(attrs)
+    unknown_attrs = check_set - all_attr_set
+    known_attrs = check_set & all_attr_set
+    if len(unknown_attrs) > 0:
+        return "The attrs " + str(unknown_attrs) + " are unknown"
+    tab_cols={}
+    error_attrs=[]
+    for attr in known_attrs:
+        xcatattr = nodeattrs[attr]
+        if xcatattr is None:
+            print(attr + " value is None?")
+            error_attrs.append(attr)
+        elif isinstance(xcatattr, list):
+            for k in xcatattr:
+                t,p,c = k.rpartition('.')
+                if t in tab_cols:
+                    tab_cols[t].append(c)
+                else:
+                    tab_cols[t] = [c]
+        else:
+            t,p,c = xcatattr.rpartition('.')
+            if t in tab_cols:
+                tab_cols[t].append(c)
+            else:
+                tab_cols[t] = [c]
+    if len(error_attrs):
+        return "The attrs " + str(error_attrs) + " are not supported yet"
+    return tab_cols
+       
+if __name__ == '__main__':
+    #print(conattr2xcatattrv2(['hardwaremanagement.manager','collective.manager', 'secret.hardwaremanagementuser']))
+    #print(conattr2xcatattr(['hardwaremanagement.manager','collective.manager', 'secret.hardwaremanagementuser']))
+    #print(conattr2xcatattr())
+    #mydict = conattr2xcatattr(['hardwaremanagement.manager','collective.manager', 'secret.hardwaremanagementuser'])
+    mydict = conattr2xcatattr('*')
+    print(get_node_attr(mydict))
+    

confluent_server/confluent/xcclient/dbfactory.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+# -*- coding: utf-8 -*-
+#
+
+from __future__ import print_function
+import sys
+import dbobject
+from dbobject import *
+from dbsession import DBsession
+from sqlalchemy import or_
+#from xcclient.shell import CommandException
+from exceptions import *
+import utils
+
+def create_or_update(session,tabcls,key,newdict,ismatrixtable=True):
+    tabkeys=tabcls.primkeys()
+    #for matrix table, remove the record if all the non-key values are None or blank 
+    delrow=1 
+    #for flat table, keep the record untouch if the non-key values are None
+    skiprow=1
+    
+    objkeys=tabcls.getobjkey()
+  
+    if type(key) not in (list,tuple):
+        key=[key] 
+    keyvals=zip(objkeys,key)
+    for (keyname,keyval)  in keyvals:
+        newdict[keyname]=keyval
+
+    tabcols=tabcls.getcolumns()
+    for item in newdict.keys():
+        if item not in tabcols:
+            if newdict[item] is None:
+                del newdict[item] 
+                continue
+            else:
+                raise BadSchemaException("Error: no column '"+item+"' in table "+tabcls.__tablename__+", might caused by mismatch between schema version and xCAT version!")
+            
+        if item not in objkeys and newdict[item]:
+            skiprow=0        
+            delrow=0
+
+        if  item not in objkeys and newdict[item] is None:
+            newdict[item]=''
+        
+        #delete table rows when (1)the object keys are None or blank (2)the key is not specified in newdict and all non-key values are blank 
+        if item not in objkeys and newdict[item]!='': 
+            delrow=0
+
+        if item == 'disable' and newdict[item]=='':
+            newdict[item]=None
+   
+    if not ismatrixtable:
+        if skiprow:
+            return
+        #do not remove for flat table
+        delrow=0
+
+    #if tabcls.__tablename__=='switch':
+    #    import pdb
+    #    pdb.set_trace()
+    try:
+        query=session.query(tabcls)
+        for tabkey in tabkeys:
+            query=query.filter(getattr(tabcls,tabkey) == newdict[tabkey])
+        record=query.all()
+    except Exception, e:
+        raise DBException("Error: query xCAT table "+tabcls.__tablename__+" failed: "+str(e))
+    if record:
+        if delrow:
+            try:
+                for item in record:
+                    session.delete(item)
+            except Exception, e:
+                raise DBException("Error: delete "+key+" is failed: "+str(e))
+            #else:
+            #    print("delete row in xCAT table "+tabcls.__tablename__+".")
+        else:
+           try:
+               #query=session.query(tabcls)
+               #for tabkey in tabkeys:
+               #    query=query.filter(getattr(tabcls,tabkey) == newdict[tabkey])
+               query.update(newdict)
+           except Exception, e:
+               raise DBException("Error: import object "+key+" is failed: "+str(e))
+    elif delrow == 0:
+        try:
+            session.execute(tabcls.__table__.insert(), newdict)
+        except Exception, e:
+            raise DBException("Error: import object "+key+" is failed: "+str(e)) 
+
+class matrixdbfactory():
+    def __init__(self,dbsession):
+        self._dbsession=dbsession
+
+    def gettab(self,tabs,keys=[]):    
+        ret={}
+        for tabname in tabs:
+           dbsession=self._dbsession.loadSession(tabname); 
+           if hasattr(dbobject,tabname):
+               tab=getattr(dbobject,tabname)
+           else:
+               continue
+           tabobjs=[]
+           tabkeys=tab.primkeys()
+           if not keys or len(keys)==0:
+               tabobjs=dbsession.query(tab).filter(or_(tab.disable == None, tab.disable.notin_(['1','yes']))).all()
+           elif len(tabkeys)==1:
+               tabobjs = dbsession.query(tab).filter(getattr(tab, tabkeys[0]).in_(keys),
+                                                     or_(tab.disable == None, tab.disable.notin_(['1', 'yes']))).all()
+           elif len(tabkeys)>1:
+               for key in keys:
+                   if type(key)!=tuple:
+                       key=[key]
+                   kvdict=dict(zip(tabkeys,key))
+                   query=dbsession.query(tab)
+                   for key,value in kvdict.items():
+                       query=query.filter(getattr(tab,key).in_([value]))
+                   query=query.filter(or_(tab.disable == None, tab.disable.notin_(['1','yes'])))  
+                   tabobj=query.all()
+                   tabobjs.extend(tabobj)
+           if not tabobjs:
+               continue
+
+           objkeyname=tab.getobjkey()
+           dictoftab={}
+           for myobj in tabobjs:
+               mydict=myobj.getdict()
+
+               if len(objkeyname)==1:
+                   mykey=mydict[tab.__tablename__+'.'+objkeyname[0]]
+               elif len(objkeyname)>1:
+                   mykeylist=[]
+                   for key in objkeyname:
+                       mykeylist.append(mydict[tab.__tablename__+'.'+key])  
+                   mykey=tuple(mykeylist)
+
+               if mykey not in dictoftab.keys():
+                   dictoftab[mykey]=mydict
+               else:
+                   if type(dictoftab[mykey])!=list:
+                       dictoftab[mykey]=[dictoftab[mykey]]
+                   dictoftab[mykey].append(mydict)
+
+
+           for mykey in dictoftab.keys():
+               if mykey not in ret.keys():
+                   ret[mykey]={}
+               if type(dictoftab[mykey])==list:
+                   if tabname not in ret[mykey].keys():
+                       ret[mykey][tabname]=[]
+                   ret[mykey][tabname].extend(dictoftab[mykey])
+               else:
+                   ret[mykey].update(dictoftab[mykey])
+
+        for mykey in ret.keys():
+            if len(ret[mykey].keys())==1 and ret[mykey].keys()[0] in tabs :
+                ret[mykey]=ret[mykey][ret[mykey].keys()[0]]          
+
+        return ret 
+
+    def settab(self,tabdict=None):
+        #print("=========matrixdbfactory:settab========")
+        #print(tabdict)
+        #print("\n")
+        if tabdict is None:
+            return None
+        for key in tabdict.keys():
+            utils.verbose("  writting object: "+str(key),file=sys.stdout)
+            #clear any existing table entries before adding new entries
+            df=dbfactory(self._dbsession) 
+            df.cleartab(tabdict[key].keys(),[key])
+            for tab in tabdict[key].keys():
+                dbsession=self._dbsession.loadSession(tab);
+                if hasattr(dbobject,tab):
+                    tabcls=getattr(dbobject,tab)
+                else:
+                    continue
+
+                for record in tabdict[key][tab]:
+                    if tabcls.isValid(key,record):
+                        create_or_update(dbsession,tabcls,key,record)
+
+class flatdbfactory() :
+    def __init__(self,dbsession):
+        self._dbsession=dbsession
+
+    def gettab(self,tabs,keys=None):    
+        ret={}
+        if keys:
+            rootkey=keys[0]  
+        else:
+            rootkey='clustersite'
+        ret[rootkey]={}
+        for tabname in tabs:
+            dbsession=self._dbsession.loadSession(tabname);
+            if hasattr(dbobject,tabname):
+                tab=getattr(dbobject,tabname)
+            else:
+                continue
+            tabobj=dbsession.query(tab).filter(or_(tab.disable == None,tab.disable.notin_(['1','yes']))).all()
+            if not tabobj:
+                continue
+            for myobj in tabobj:
+                mydict=myobj.getdict()
+                ret[rootkey].update(mydict)
+        return  ret
+   
+    def settab(self,tabdict=None):
+        #print("======flatdbfactory:settab======")
+        #print(tabdict)
+        if tabdict is None:
+            return None
+        for key in tabdict.keys():
+            for tab in tabdict[key].keys():
+                if hasattr(dbobject,tab):
+                    tabcls=getattr(dbobject,tab)
+                else:
+                    continue
+                tabkey=tabcls.getobjkey()[0]
+                rowentlist=tabcls.dict2tabentry(tabdict[key][tab][0])
+                dbsession=self._dbsession.loadSession(tab)
+                for rowent in rowentlist:
+                    if tabcls.isValid(key, rowent):
+                        create_or_update(dbsession,tabcls,rowent[tabkey],rowent,False)
+
+    
+
+class dbfactory():
+    
+    def __init__(self,dbsession):
+        self._dbsession=dbsession
+
+    def gettab(self,tabs,keys=None):
+        flattabs=[]
+        matrixtabs=[]
+        mydict={}
+
+        for tab in tabs:
+            if hasattr(dbobject,tab):
+                tabcls=getattr(dbobject,tab)
+            else:
+                continue       
+            if tabcls.getTabtype() == 'flat':
+                flattabs.append(tab)
+            else:
+                matrixtabs.append(tab)
+        if flattabs:
+            df_flat=flatdbfactory(self._dbsession)
+            mydict.update(df_flat.gettab(flattabs,keys))
+        if matrixtabs:
+            df_matrix=matrixdbfactory(self._dbsession)
+            mydict.update(df_matrix.gettab(matrixtabs,keys))
+        return mydict
+    
+    #convert db dict from format {key:{tab.col=value}} to {key:{tab:{col}}}
+    def __tabtransform(self,dbdict):
+        #print("__tabtransform")
+        #print(dbdict)
+        flattabdict={}
+        matrixtabdict={}
+
+        #try:
+        for key in dbdict.keys():
+            dbentlist=[]
+            rawdbents=dbdict[key]
+            if type(rawdbents)==dict:
+                dbentlist.append(rawdbents)
+            else:
+                dbentlist.extend(rawdbents)  
+   
+
+            for dbent in dbentlist:
+                #print("@@@@@@@@@@@@")
+                #print(dbent)
+                #print("@@@@@@@@@@@@")
+                rowdict={}
+                for tabcol in dbent.keys():
+                    (tab,col)=tabcol.split('.')
+
+                    if tab not in rowdict.keys():
+                        rowdict[tab]={}
+                    if col not in rowdict[tab].keys():
+                        rowdict[tab][col]={}
+                    rowdict[tab][col]=dbent[tabcol] 
+
+
+                for tab in rowdict.keys():
+                    if hasattr(dbobject,tab):
+                        tabcls=getattr(dbobject,tab)
+                    else:
+                        continue       
+                    if tabcls.getTabtype() == 'flat':
+                        if key not in flattabdict.keys():
+                            flattabdict[key]={}
+                        if tab not in flattabdict[key].keys():
+                            flattabdict[key][tab]=[]
+                        flattabdict[key][tab].append(rowdict[tab])
+                    else:
+                        if key not in matrixtabdict.keys():
+                            matrixtabdict[key]={}
+                        if tab not in matrixtabdict[key].keys():
+                            matrixtabdict[key][tab]=[]
+                        matrixtabdict[key][tab].append(rowdict[tab])
+        return(matrixtabdict,flattabdict)
+     
+                
+    def settab(self,dbdict=None):                 
+        if dbdict is None:
+            return None
+        (matrixtabdict,flattabdict)=self.__tabtransform(dbdict)
+        if flattabdict:
+            df_flat=flatdbfactory(self._dbsession)
+            mydict=df_flat.settab(flattabdict)
+        if matrixtabdict: 
+            df_matrix=matrixdbfactory(self._dbsession)
+            mydict=df_matrix.settab(matrixtabdict)  
+        #except Exception as e:
+        #  raise ("Error: import object failed.")
+        #else:
+        #    print("import object successfully.")          
+
+    def cleartab(self,tabs,objkey=[]):
+        for tab in tabs:
+            if hasattr(dbobject,tab):
+                tabcls=getattr(dbobject,tab)
+            else:
+                continue
+            tabkey=tabcls.getobjkey()[0]
+            ReservedKeys=tabcls.getReservedKeys()
+            dbsession=self._dbsession.loadSession(tab)
+            try:
+                query=dbsession.query(tabcls)
+                query=query.filter(or_(tabcls.disable == None, tabcls.disable.notin_(['1','yes'])))
+                if ReservedKeys:
+                    query=query.filter(getattr(tabcls,tabkey).notin_(ReservedKeys))        
+                    
+                if objkey:
+                    query=query.filter(getattr(tabcls,tabkey).in_(objkey)) 
+                query.delete(synchronize_session='fetch')
+            except Exception as e:
+                raise DBException("Error: failed to clear table "+str(tab)+": "+str(e))
+        #else:
+        #    print("table "+tab+ "cleared!")
+
+if __name__ == "__main__":
+     pass

confluent_server/confluent/xcclient/dbobject.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+# -*- coding: utf-8 -*-
+#
+
+from dbsession import *
+from copy import *
+from sqlalchemy import inspect
+import pdb
+
+class mixin(object):
+    def getdict(self):
+        mydict={}
+        for mykey in self.__dict__.keys():
+           if mykey in self.__table__.columns:
+              mydict[self.__tablename__+'.'+mykey.encode()]= self.__dict__[mykey] if self.__dict__[mykey] is None else self.__dict__[mykey].encode()
+        try:
+            self.__class__.outprocess(mydict)
+        except:
+            pass 
+        return mydict
+
+    #return a tuple of table primary keys
+    @classmethod
+    def primkeys(cls):
+        ins = inspect(cls)
+        prikeys=[ item.key for item in ins.primary_key ]
+        prikeys.sort(None,None,reverse=False)
+        return tuple(prikeys)
+
+    #return the key of object in table row
+    @classmethod
+    def getobjkey(cls):
+        return cls.primkeys()
+
+    @classmethod
+    def isValid(cls, netname, tabdict):
+        return True
+
+    @classmethod
+    def dict2tabentry(self,objdict):
+        pass      
+    
+    @classmethod
+    def getcolumns(self):
+        return self.__table__.columns.keys()
+
+    @classmethod
+    def getTabtype(self):
+        return 'matrix'
+
+    @classmethod
+    def getReservedKeys(self):
+        return []
+########################################################################
+class passwd(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('passwd')
+    __tablename__ = 'passwd'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def primkeys(cls):
+        return ('key','username')
+
+    @classmethod
+    def getobjkey(cls):
+        return tuple(['key'])
+    
+########################################################################
+class networks(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('networks')
+    __tablename__ = 'networks'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def primkeys(cls):
+        return tuple(['netname'])
+    @classmethod
+    def getobjkey(cls):
+        return tuple(['netname'])
+
+    @classmethod    
+    def isValid(cls, netname, tabdict):
+        eptkey=0
+        if 'net' not in tabdict.keys() or not tabdict['net']:
+            print("Error: net value should not be empty for xCAT network object "+netname)
+            eptkey=1
+        if 'mask' not in tabdict.keys() or not tabdict['mask']:
+            print ("Error: mask value should not be empty for xCAT network object "+netname)
+            eptkey=1
+        if eptkey:
+            return False
+        else:
+            return True
+
+########################################################################
+class routes(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('routes')
+    __tablename__ = 'routes'
+    __table_args__ = {'autoload':True}
+
+
+########################################################################
+class nodetype(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nodetype')
+    __tablename__ = 'nodetype'
+    __table_args__ = {'autoload':True} 
+
+########################################################################
+'''
+class hosts(Base,mixin):
+    """"""
+    __tablename__ = 'hosts'
+    __table_args__ = {'autoload':True}
+'''
+########################################################################
+class noderes(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('noderes')
+    __tablename__ = 'noderes'
+    __table_args__ = {'autoload':True}
+
+########################################################################
+class switch(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('switch')
+    __tablename__ = 'switch'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def getobjkey(cls):
+        return tuple(['node'])
+########################################################################
+class switches(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('switches')
+    __tablename__ = 'switches'
+    __table_args__ = {'autoload':True}
+
+
+########################################################################
+class mac(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('mac')
+    __tablename__ = 'mac'
+    __table_args__ = {'autoload':True}
+########################################################################
+class hwinv(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('hwinv')
+    __tablename__ = 'hwinv'
+    __table_args__ = {'autoload':True}
+########################################################################
+class postscripts(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('postscripts')
+    __tablename__ = 'postscripts'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def getReservedKeys(self):
+        return ('xcatdefaults','service')
+    
+########################################################################
+class bootparams(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('bootparams')
+    __tablename__ = 'bootparams'
+    __table_args__ = {'autoload':True}
+
+########################################################################
+class nodelist(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nodelist')
+    __tablename__ = 'nodelist'
+    __table_args__ = {'autoload':True}
+
+########################################################################
+class vm(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('vm')
+    __tablename__ = 'vm'
+    __table_args__ = {'autoload':True}
+########################################################################
+class policy(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('policy')
+    __tablename__ = 'policy'
+    __table_args__ = {'autoload':True}
+
+########################################################################
+class nodehm(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nodehm')
+    __tablename__ = 'nodehm'
+    __table_args__ = {'autoload':True}
+########################################################################
+class nodegroup(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nodegroup')
+    __tablename__ = 'nodegroup'
+    __table_args__ = {'autoload':True}
+########################################################################
+class vpd(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('vpd')
+    __tablename__ = 'vpd'
+    __table_args__ = {'autoload':True}
+########################################################################
+class servicenode(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('servicenode')
+    __tablename__ = 'servicenode'
+    __table_args__ = {'autoload':True}
+########################################################################
+class hosts(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('hosts')
+    __tablename__ = 'hosts'
+    __table_args__ = {'autoload':True}
+########################################################################
+class nics(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nics')
+    __tablename__ = 'nics'
+    __table_args__ = {'autoload':True}
+########################################################################
+class openbmc(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('openbmc')
+    __tablename__ = 'openbmc'
+    __table_args__ = {'autoload':True}
+########################################################################
+class prodkey(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('prodkey')
+    __tablename__ = 'prodkey'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def getobjkey(cls):
+        return tuple(['node'])
+########################################################################
+class domain(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('domain')
+    __tablename__ = 'domain'
+    __table_args__ = {'autoload':True}
+########################################################################
+class chain(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('chain')
+    __tablename__ = 'chain'
+    __table_args__ = {'autoload':True}
+########################################################################
+class rack(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('rack')
+    __tablename__ = 'rack'
+    __table_args__ = {'autoload':True}
+########################################################################
+class nodepos(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nodepos')
+    __tablename__ = 'nodepos'
+    __table_args__ = {'autoload':True}
+########################################################################
+class ppc(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('ppc')
+    __tablename__ = 'ppc'
+    __table_args__ = {'autoload':True}
+########################################################################
+class ppchcp(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('ppchcp')
+    __tablename__ = 'ppchcp'
+    __table_args__ = {'autoload':True}
+########################################################################
+class mp(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('mp')
+    __tablename__ = 'mp'
+    __table_args__ = {'autoload':True}
+########################################################################
+class zvm(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('zvm')
+    __tablename__ = 'zvm'
+    __table_args__ = {'autoload':True}
+########################################################################
+class mpa(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('mpa')
+    __tablename__ = 'mpa'
+    __table_args__ = {'autoload':True}
+########################################################################
+class pdu(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('pdu')
+    __tablename__ = 'pdu'
+    __table_args__ = {'autoload':True}
+########################################################################
+class pduoutlet(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('pduoutlet')
+    __tablename__ = 'pduoutlet'
+    __table_args__ = {'autoload':True}
+########################################################################
+class cfgmgt(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('cfgmgt')
+    __tablename__ = 'cfgmgt'
+    __table_args__ = {'autoload':True}
+########################################################################
+class hypervisor(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('hypervisor')
+    __tablename__ = 'hypervisor'
+    __table_args__ = {'autoload':True}
+########################################################################
+class iscsi(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('iscsi')
+    __tablename__ = 'iscsi'
+    __table_args__ = {'autoload':True}
+########################################################################
+class mic(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('mic')
+    __tablename__ = 'mic'
+    __table_args__ = {'autoload':True}
+########################################################################
+class ppcdirect(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('ppcdirect')
+    __tablename__ = 'ppcdirect'
+    __table_args__ = {'autoload':True}
+
+    @classmethod
+    def primkeys(cls):
+        return tuple(['hcp'])
+########################################################################
+class storage(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('storage')
+    __tablename__ = 'storage'
+    __table_args__ = {'autoload':True}
+########################################################################
+class websrv(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('websrv')
+    __tablename__ = 'websrv'
+    __table_args__ = {'autoload':True}
+########################################################################
+class prescripts(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('prescripts')
+    __tablename__ = 'prescripts'
+    __table_args__ = {'autoload':True}
+########################################################################
+class ipmi(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('ipmi')
+    __tablename__ = 'ipmi'
+    __table_args__ = {'autoload':True}
+########################################################################
+class osimage(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('osimage')
+    __tablename__ = 'osimage'
+    __table_args__ = {'autoload':True}
+########################################################################
+class linuximage(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('linuximage')
+    __tablename__ = 'linuximage'
+    __table_args__ = {'autoload':True}
+########################################################################
+class winimage(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('winimage')
+    __tablename__ = 'winimage'
+    __table_args__ = {'autoload':True}
+########################################################################
+class nimimage(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('nimimage')
+    __tablename__ = 'nimimage'
+    __table_args__ = {'autoload':True}
+########################################################################
+class zone(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('zone')
+    __tablename__ = 'zone'
+    __table_args__ = {'autoload':True}
+########################################################################
+class osdistro(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('osdistro')
+    __tablename__ = 'osdistro'
+    __table_args__ = {'autoload':True}
+########################################################################
+class site(Base,mixin):
+    """"""
+    Base.metadata.bind = DBsession.getEngine('site')
+    __tablename__ = 'site'
+    __table_args__ = {'autoload':True}
+########################################################################
+    def getdict(self):
+        mydict={}
+        mykey=self.__dict__['key']
+        mydict[self.__tablename__+'.'+mykey]=mykey=self.__dict__['value']
+        return mydict
+
+    @classmethod
+    def dict2tabentry(self,objdict):
+        mydict={}
+        ret=[]
+        for key in objdict.keys():
+            mydict['key']=key
+            mydict['value']=objdict[key]
+            mydict['disable']=None
+            ret.append(deepcopy(mydict))
+        return ret
+       
+    @classmethod
+    def getTabtype(self):
+        return 'flat'
+#----------------------------------------------------------------------
+
+def query_table_by_node(session, tclass, tkey):
+    """"""
+    result=session.query(tclass).filter(tclass.node == tkey).all()
+    if not result:
+       return None 
+    return result[0].getdict()
+
+
+def query_nodelist_by_key(session, nodelist):
+    """"""
+    nodelist_value = {}
+    for node in nodelist:
+        nodelist_value[node]={}
+        classlist = [Bootparams,Nodetype,Hosts,Switch,Mac,Noderes]
+        for eachclass in classlist:
+            clsdict = query_table_by_node(session,eachclass,node)
+            nodelist_value[node].update(clsdict)
+    return nodelist_value
+
+if __name__ == "__main__":
+     pass
+  

confluent_server/confluent/xcclient/dbsession.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+# -*- coding: utf-8 -*-
+#
+
+from sqlalchemy import create_engine,inspect
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+import re
+import os
+import sqlalchemy.exc
+from exceptions import *
+import codecs
+
+codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None)
+
+Base = declarative_base()
+Base.metadata.bind = None;
+
+
+class Singleton(object):
+  def __new__(cls,*args,**kwargs):
+    if not hasattr(cls,'_inst'):
+      cls._inst=super(Singleton,cls).__new__(cls,*args,**kwargs)
+    return cls._inst
+
+class DBsession(Singleton):
+    _dbcfgpath='/etc/xcat/cfgloc'
+    _dbcfgregex=re.compile("^(\S+):dbname=(\S+);host=(\S+)\|(\S+)\|(\S*)$")
+
+    def __init__(self):
+        self._sessions={}
+
+    def __del__(self):
+        self.close()
+
+    #check whether the backend db is sqlite
+    @classmethod
+    def isSqlite(cls):
+        if os.path.exists(cls._dbcfgpath):
+            return False
+        else:
+            return True
+
+    #create DB engine according to the xcat cfgloc file 
+    @classmethod
+    def createEngine(cls,tablename=None):
+        if not cls.isSqlite():
+            dbcfgfile = open(cls._dbcfgpath)
+            dbcfgloc = dbcfgfile.read( )
+            dbcfgfile.close()
+            try:
+                (dbtype,dbname,dbhost,dbusername,dbpasswd)=re.findall(cls._dbcfgregex,dbcfgloc)[0]
+            except:
+                raise BadDBHdlException("Error: invalid cfgloc file: %(o)s", o=cls._dbcfgpath) 
+            if dbtype == 'Pg':
+                conn="postgresql+psycopg2://"
+            elif dbtype == 'mysql':
+                conn="mysql+pymysql://"
+            engine_value = conn+dbusername+':'+dbpasswd+'@'+dbhost+'/'+dbname
+        elif tablename:
+            engine_value = 'sqlite:////etc/xcat/'+tablename+'.sqlite'
+        else:
+            raise BadDBHdlException("Error: table name not specified!") 
+        engine=create_engine(engine_value, echo=False)
+        return engine
+
+    @classmethod
+    def getEngine(cls,tablename=None):
+        """"""
+        if not cls.isSqlite():
+            if not Base.metadata.bind:
+                engine=cls.createEngine()
+            else:
+                return Base.metadata.bind
+        elif tablename:
+            engine=cls.createEngine(tablename)
+        return engine
+
+    #bind Base.metadata to db engine
+    @classmethod
+    def createSession(cls,tablename=None):
+        """"""
+        engine=cls.getEngine(tablename)
+        #metadata = Base.metadata
+        #print metadata.tables.keys()
+        Session = sessionmaker(bind=engine)
+        session = Session()
+        return session
+
+
+    #get db session
+    def loadSession(self,tablename=None):
+        if self.__class__.isSqlite():
+            if tablename not in self._sessions.keys():
+                session=self.__class__.createSession(tablename)
+                if session:
+                    self._sessions[tablename]=session
+                return session
+            else:
+                return self._sessions[tablename]
+        else:    
+            if self._sessions:
+                mykey=self._sessions.keys()[0]
+                return self._sessions[mykey]
+            else:
+                session=self.__class__.createSession(tablename)
+                self._sessions['_globaldbsession']=session
+                return session
+        
+    #commit all the transactions 
+    def commit(self):
+        for session in self._sessions.keys():
+            self._sessions[session].commit()
+    #close all sessions
+    def close(self):
+        for session in self._sessions.keys():
+            self._sessions[session].close()
+            del self._sessions[session]
+        
+    #for debug: print the self._sessions
+    def dumpsession(self):
+        print(self._sessions)
+ 
+if __name__ == "__main__":
+    pass

confluent_server/confluent/xcclient/globalvars.py

@@ -0,0 +1,18 @@
+
+#xcat version string
+xcat_version=""
+
+#xcat version number
+xcat_verno=""
+
+#xcat service running?
+isxcatrunning=0
+
+implicitEnvVars={'OBJNAME':{'description':"the object name to import"},
+                 'GITROOT':{'description':"the root path of the git repo where the inventory file resides in"},
+                 'GITBRANCH':{'description':"the current git branch name of the inventory file to import"},
+                 'GITTAG':{'description':"the current git tag of the inventory file to import"},
+                 'GITCOMMIT':{'description':"the current git commit number of the inventory file to import"}}
+
+#verbose?
+verbose=False

confluent_server/confluent/xcclient/utils.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+# -*- coding: utf-8 -*-
+# common helper subroutines
+#
+from __future__ import print_function
+import os
+import re
+import subprocess
+import json
+import yaml
+import sys
+import globalvars
+from exceptions import *
+from contextlib import contextmanager
+
+def runCommand(cmd, env=None):
+    """
+    Run one command only, when you don't want to bother setting up
+    the Popen stuff.
+        (retcode,out,err)=runCommand('lsxcatd -v')
+    """
+    try:
+        p = subprocess.Popen(cmd,
+        env=env,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+        out, err = p.communicate()
+    except OSError,e:
+        return p.returncode,out, err
+    return p.returncode,out, err
+
+#remove the dict entries whose value is null or ''
+def Util_rmnullindict(mydict):
+    for key in mydict.keys():
+        if isinstance(mydict[key],dict):
+            Util_rmnullindict(mydict[key])
+            if not mydict[key].keys():
+                del mydict[key]
+        else:
+            if not mydict[key]:
+                del mydict[key]
+
+
+# replace the "{{x}}" variables in the value of dict "mydict" 
+# with the values of "vardict[x]"
+def Util_subvarsindict(mydict,vardict):
+    for key in mydict.keys():
+        if isinstance(mydict[key],dict):
+            Util_subvarsindict(mydict[key],vardict)
+        elif isinstance(mydict[key],list):
+            for idx,val in enumerate(mydict[key]):
+                if isinstance(val,str):
+                    mydict[key][idx]=re.sub(r'\{\{(.*)\}\}',lambda m:vardict.get(m.group(1)),val)
+        elif isinstance(mydict[key],str):
+            mydict[key]=re.sub(r'\{\{(.*)\}\}',lambda m:vardict.get(m.group(1)),mydict[key])
+            
+
+# get the dict value mydict[a][b][c] with key path a.b.c
+def Util_getdictval(mydict,keystr):
+    if not  isinstance(mydict,dict):
+        return None
+    dictkeyregex=re.compile("([^\.]+)\.?(\S+)*")
+    result=re.findall(dictkeyregex,keystr)
+    if result:
+        (key,remdkey)=result[0]
+        if key not in mydict.keys():
+            return None
+        if remdkey:
+            return Util_getdictval(mydict[key],remdkey)
+        else:
+            return mydict[key]
+
+# get the dict value mydict[a][b][c] with key path a.b.c
+def Util_setdictval(mydict,keystr,value):
+    dictkeyregex=re.compile("([^\.]+)\.?(\S+)*")
+    result=re.findall(dictkeyregex,keystr)
+    if result:
+        (key,remdkey)=result[0]
+        if remdkey:
+            if key not in mydict.keys():
+                mydict[key]={}
+            Util_setdictval(mydict[key],remdkey,value)
+        else:
+            mydict[key]=value
+
+#remove dict key [a][b][c] with key path a.b.c
+def Util_deldictkey(mydict,keystr):
+    dictkeyregex=re.compile("([^\.]+)\.?(\S+)*")
+    result=re.findall(dictkeyregex,keystr)
+    if result:
+        (key,remdkey)=result[0]
+        if remdkey:
+            if key not in mydict.keys():
+                mydict[key]={}
+            Util_deldictkey(mydict[key],remdkey)
+        else:
+            del mydict[key]
+
+
+def loadfile(filename):
+    if not os.path.exists(filename):
+        raise FileNotExistException("Error: File '%s' does not exist, please check..." % filename)
+
+    contents={}
+    fmt = 'json'
+    with open(filename,"r") as fh:
+        f=fh.read()
+        if not f.startswith('{'):
+            fmt = 'yaml'
+        try:
+            contents=json.loads(f)
+        except ValueError:
+            try:
+                contents = yaml.load(f)
+            except Exception,e:
+                raise InvalidFileException("Error: failed to load file \"%s\", please validate the file with 'yamllint %s'(for yaml format) or 'cat %s|python -mjson.tool'(for json format)!"%(filename,filename,filename))
+        return contents, fmt
+    return None, fmt
+
+#initialize the global vars in globalvars.py
+def initglobal():
+    if os.path.exists("/var/run/xcatd.pid"):
+        globalvars.isxcatrunning=1
+    else:
+        globalvars.isxcatrunning=0
+    if globalvars.isxcatrunning:
+        (retcode,out,err)=runCommand("XCATBYPASS=0 lsxcatd -v")
+    if retcode!=0 or not globalvars.isxcatrunning:
+        (retcode,out,err)=runCommand("XCATBYPASS=1 lsxcatd -v")
+    if retcode!=0:
+        globalvars.xcat_version=""
+        globalvars.xcat_verno=""
+    else:
+        globalvars.xcat_version=out.strip()
+        globalvars.xcat_verno=globalvars.xcat_version.split(' ')[1]
+
+# if "key" of d1 or "key" of d1[key] not in d2, delete it
+def filter_dict_keys(d1, d2):
+    tmp_d1 = d1
+    for key in tmp_d1.keys():
+        if key not in d2:
+            del tmp_d1[key]
+            continue
+        if type(tmp_d1[key]) != dict:
+            continue
+        for subkey in tmp_d1[key].keys():
+            if subkey not in d2[key]:
+                del tmp_d1[key][subkey]
+    return tmp_d1
+
+#print if -v|--verbose specified
+def verbose(message,file=sys.stdout):
+    if globalvars.verbose:
+        print("%s"%(message),file=file)
+
+#get home directory of the current user
+def gethome():
+    home = os.path.expanduser("~")
+    return home
+
+#strip single and double quotes from string
+def stripquotes(instring):
+    return instring.strip('"').strip("'")
+
+#redirect stdout to stream
+@contextmanager
+def stdout_redirector(stream):
+    old_stdout = sys.stdout
+    sys.stdout = stream
+    try:
+        yield
+    finally:
+        sys.stdout = old_stdout
+
+#redirect stderr to stream
+@contextmanager
+def stderr_redirector(stream):
+    old_stderr = sys.stderr
+    sys.stderr = stream
+    try:
+        yield
+    finally:
+        sys.stderr = old_stderr
+
+
+
+#traverse the inventory object directory and get the object type and object names
+def traverseobjdir(path):
+    if not os.path.isdir(path):
+        return None
+    ret={}
+    from manager import InventoryFactory
+    for subdir in os.listdir(path):
+        objpath=os.path.join(path,subdir) 
+        if os.path.isdir(objpath):
+            if os.path.isfile(os.path.join(objpath,'definition.yaml')):
+                objfile=os.path.join(objpath,'definition.yaml')
+                with open(objfile, 'r') as stream:
+                    try:
+                        obj_attr_dict = yaml.load(stream)
+                    except:
+                        continue
+            elif os.path.isfile(os.path.join(objpath,'definition.json')):
+                objfile=os.path.join(objpath,'definition.json')
+                with open(objfile, 'r') as stream:
+                    string=stream.read()
+                    try:
+                        obj_attr_dict = json.loads(string)
+                    except:
+                        continue
+            else:
+                continue
+            objtypes=list(set(obj_attr_dict.keys()) & set(InventoryFactory.getvalidobjtypes(ignorepartial=1)))
+            if objtypes and len(objtypes)==1:
+                objtype=objtypes[0] 
+                if objtype not in ret.keys():
+                    ret[objtype]=[subdir]
+                else:
+                    ret[objtype].append(subdir)
+    return ret
+     
+
+
+
+ 
+            
+            

confluent_server/confluent/xcclient/xcat_manager.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+###############################################################################
+# IBM(c) 2019 EPL license http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import confluent.log as log
+from functools import wraps
+from .dbsession import DBsession
+from .dbfactory import dbfactory
+from .attributes import conattr2xcatattr
+xcat_dbi=None
+data_store=None
+xcfm_logger=None
+def xcat_manager_decorator(function):
+    @wraps(function)
+    def decorator(*args, **kwargs):
+        xcfm_logger.log("In xCAT configManager: run " + function.__name__)
+        return function(*args, **kwargs)
+    return decorator
+
+class xCATConfigManager(object):
+    global data_store
+    data_store={}
+    data_store['nodes']={}
+    data_store['nodegroup']=[]
+
+    def __init__(self):
+        global xcat_dbi
+        if xcat_dbi is None:
+            dbsession = DBsession()
+            xcat_dbi = dbfactory(dbsession)
+        global xcfm_logger
+        if xcfm_logger is None:
+            xcfm_logger = log.Logger("xcat_manager")
+        xcfm_logger.log("Init xCAT configManager")
+        self.decrypt = False
+        self.current_user = 'xcat'
+        self.tenant = None
+        self._list_nodes()
+        self._list_nodegroup() 
+    def _get_data_from_db(self):
+        try:
+            nodelist_set = xcat_dbi.gettab(['nodelist'])
+            nodelist = nodelist_set.keys()
+        except KeyError:
+            data_store['nodes'] = {}
+            return
+        attrs_dict = conattr2xcatattr('*')
+        xcat_dbs=attrs_dict.values() 
+        db_hash = {}
+        for e in xcat_dbs:
+            if e is None:
+                continue
+            elif isinstance(e, list):
+                continue
+            else:
+                t,p,c=e.rpartition('.')
+                if t in db_hash:
+                    db_hash[t].append(c)
+                else:
+                    db_hash[t]=[c]
+        dataset = xcat_dbi.gettab(db_hash.keys())
+        for node in nodelist:
+            nodeinfo = {} 
+            for attr in attrs_dict.keys():
+                self.log("The attr:" + attr)
+                nodeinfo[attr] = {}
+                if attrs_dict[attr] is None:
+                    nodeinfo[attr]['value'] = None
+                elif isinstance(attrs_dict[attr], list):
+                    nodeinfo[attr]['value'] = attrs_dict[attr]
+                elif attrs_dict[attr] in dataset[node]:
+                    nodeinfo[attr]['value'] = dataset[node][attrs_dict[attr]]
+                else:
+                    nodeinfo[attr]['value'] = attrs_dict[attr]
+            data_store['nodes'][node] = nodeinfo
+        
+    def _list_nodes(self):
+        self._get_data_from_db()
+
+    def _list_nodegroup(self):
+        try:
+            dataset = xcat_dbi.gettab(['nodegroup'])
+            data_store['nodegroup'] = dataset.keys() 
+        except KeyError:
+            data_store['nodegroup'] = []
+    @xcat_manager_decorator
+    def list_nodes(self):
+        try:
+            dataset = xcat_dbi.gettab(['nodelist', 'nodegroup'])
+            return iter(dataset.keys())
+        except KeyError:
+            return []
+    @xcat_manager_decorator
+    def get_groups(self, sizesort=False):
+        if len(data_store['nodegroup']) == 0:
+            self._list_nodegroup()
+        return data_store['nodegroup']
+    @xcat_manager_decorator
+    def is_node(self, node):
+        return node in data_store['nodes'].keys()
+
+    @xcat_manager_decorator
+    def is_nodegroup(self, nodegroup):
+        return nodegroup in data_store['nodegroup']
+
+    @xcat_manager_decorator
+    def check_quorum(self):
+        pass
+    @xcat_manager_decorator
+    def get_node_attributes(self, nodelist, attributes=(), decrypt=None):
+        ret_dict = {}
+        if '*' in nodelist:
+            nodelist = data_store['nodes'].keys()
+        if len(nodelist) == 0:
+            return None
+        if len(set(nodelist) - set(data_store['nodes'].keys())):
+            self._get_data_from_db()
+        if '*' in attributes or attributes == ():
+            for node in nodelist:
+                ret_dict[node] = data_store['nodes'][node]
+            self.log(self.__class__.__name__ + "==> get all attributes for: " + str(nodelist))
+            self.log(str(ret_dict))
+            return ret_dict
+        self.log(self.__class__.__name__ + "==> get attributes:" + str(attributes) + "for: " + str(nodelist))
+        for node in nodelist:
+            nodeinfo = {}
+            for attr in attributes:
+                nodeinfo[attr] = data_store['nodes'][node][attr]
+            ret_dict[node] = nodeinfo
+        return ret_dict
+    @xcat_manager_decorator
+    def get_collective_member(self, name):
+        return None
+        #return get_collective_member(name)
+    @xcat_manager_decorator
+    def watch_attributes(self, nodes, attributes, callback): 
+        pass
+
+    def log(self, logmsg):
+        xcfm_logger.log(logmsg)
+if __name__ == '__main__':
+    xcfm = xCATConfigManager()
+    xcfm.get_node_attributes()

confluent_server/confluent_server.spec.tmpl

@@ -40,6 +40,9 @@ true
 %post
 sysctl -p /usr/lib/sysctl.d/confluent.conf >& /dev/null
 if [ -x /usr/bin/systemctl ]; then /usr/bin/systemctl try-restart confluent >& /dev/null; fi
+if [ ! -e /etc/pam.d/confluent ]; then
+    ln -s /etc/pam.d/sshd /etc/pam.d/confluent
+fi
 true
 
 %clean

confluent_server/setup.cfg

@@ -1,22 +0,0 @@
-[metadata]
-name = confluent_server
-summary = Confluent systems management daemon
-description-file =
-    README.txt
-author = Jarrod Johnson
-author-email = jbjohnso@us.ibm.com
-home-page = http://xcat.sf.net/
-classifier =
-    Intended Audience :: Information Technology
-    Intended Audience :: System Administrators
-    License :: OSI Approved :: Apache Software License
-    Operating System :: POSIX :: Linux
-    Programming Language :: Python :: 2.6
-    Programming Language :: Python :: 2.7
-
-[files]
-packages =
-    confluent
-    confluent/interface
-    confluent/config
-

confluent_server/setup.py.tmpl

@@ -16,6 +16,7 @@ setup(
               'confluent/plugins/hardwaremanagement/',
               'confluent/plugins/shell/',
               'confluent/collective/',
+              'confluent/xcclient/',
               'confluent/plugins/configuration/'],
     install_requires=['paramiko', 'pycrypto>=2.6', 'confluent_client>=0.1.0', 'eventlet',
                       'dnspython', 'netifaces', 'pyte', 'pysnmp', 'pyparsing',

confluent_server/systemd/confluent.service

@@ -4,7 +4,7 @@ Description=Confluent hardware manager
 
 [Service]
 Type=forking
-PIDFile=/var/run/confluent/pid
+#PIDFile=/var/run/confluent/pid
 ExecStart=/opt/confluent/bin/confluent
 ExecStop=/opt/confluent/bin/confetty shutdown /
 Restart=on-failure

confluent_web/js/consoles.js

@@ -1,38 +0,0 @@
-function getRequest(url, success) {
-    var request = new XMLHttpRequest();
-    request.open('GET', url, true);
-    request.setRequestHeader('Accept', 'application/json');
-    request.onload = function() {
-        if (this.status >= 200 && this.status <= 400) {
-            success(JSON.parse(this.responseText));
-        }
-    };
-    request.send();
-}
-
-document.addEventListener('DOMContentLoaded', function() {
-
-  getRequest("/confluent-api/nodes/", function( data) {
-	var items = [];
-	var options = [];
-	var nodename = "";
-    data["_links"]["item"].forEach( function( val, key ) {
-        console.log(val);
-        if (typeof(val) == "object") {
-		    nodename = val.href;
-        } else {
-            nodename = val;
-        }
-        console.log(nodename);
-		nodename = nodename.replace('/', '');
-        var myrow = document.createElement('div');
-        myrow.innerHTML = "<button id="+nodename+">"+nodename+"</button><br>";
-        document.getElementById("nodes").appendChild(myrow);
-        document.getElementById(nodename).addEventListener("click", function( event ) {
-            var tname = this.id;
-            var url  = "/confluent-api/nodes/" + tname + "/console/session";
-            new ConsoleWindow(url, tname);
-        });
-	});
-  });
-}); // end document

confluent_web/js/consolewindow.js

@@ -1,742 +0,0 @@
-/**
- * tty.js
- * Copyright (c) 2012-2013, Christopher Jeffrey (MIT License)
- * Copyright 2014, IBM Corporation
- * Copyright 2014, Lenovo
- */
-
-;(function() {
-
-'use strict';
-/**
- * Elements
- */
-
-var document = this.document
-  , window = this
-  , root
-  , body
-  , h1
-  , open
-  , lights;
-
-/**
- * Helpers
- */
-
-var EventEmitter = Terminal.EventEmitter
-  , inherits = Terminal.inherits
-  , on = Terminal.on
-  , off = Terminal.off
-  , cancel = Terminal.cancel;
-
-function postRequest(url, data, success) {
-	var request = new XMLHttpRequest();
-	request.open('POST', url, true);
-        request.setRequestHeader('Content-Type', 'application/json');
-        request.setRequestHeader('Accept', 'application/json');
-	request.onload = function() {
-		if (this.status >= 200 && this.status < 400) {
-			success(JSON.parse(this.responseText));
-		}
-	};
-	if (data) {
-		request.send(JSON.stringify(data));
-	} else {
-		request.send("");
-	}
-        request = null;
-}
-/**
- * Console
- */
-
-function ConsoleWindow(consoleurl, nodename) {
-  var self = this;
-
-  if (!(this instanceof ConsoleWindow)) {
-    return new ConsoleWindow(consoleurl, nodename);
-  }
-
-  EventEmitter.call(this);
-
-  var el
-    , grip
-    , bar
-    , button
-    , title;
-
-  el = document.createElement('div');
-  el.className = 'window';
-
-  grip = document.createElement('div');
-  grip.className = 'grip';
-
-  bar = document.createElement('div');
-  bar.className = 'bar';
-
-  button = document.createElement('div');
-  button.innerHTML = 'x';
-  button.title = 'close';
-  button.className = 'tab';
-
-  title = document.createElement('div');
-  title.className = 'title';
-  title.innerHTML = nodename;
-
-  this.nodename = nodename;
-  this.element = el;
-  this.grip = grip;
-  this.bar = bar;
-  this.button = button;
-  this.title = title;
-  this.consoleurl = consoleurl;
-
-  this.tabs = [];
-  this.focused = null;
-
-  this.cols = 100; //Terminal.geometry[0];
-  this.rows = 30; //Terminal.geometry[1];
-
-  el.appendChild(grip);
-  el.appendChild(bar);
-  bar.appendChild(title);
-  bar.appendChild(button);
-  document.body.appendChild(el);
-
-  //tty.windows.push(this);
-
-  this.createTab();
-  this.focus();
-  this.bind();
-
-  this.tabs[0].once('open', function() {
-    //tty.emit('open window', self);
-    self.emit('open');
-  });
-}
-
-inherits(ConsoleWindow, EventEmitter);
-
-ConsoleWindow.prototype.bind = function() {
-  var self = this
-    , el = this.element
-    , bar = this.bar
-    , grip = this.grip
-    , button = this.button
-    , last = 0;
-
-  on(button, 'click', function(ev) {
-    self.destroy();
-    return cancel(ev);
-  });
-
-  on(grip, 'mousedown', function(ev) {
-    self.focus();
-    self.resizing(ev);
-    return cancel(ev);
-  });
-
-  on(el, 'mousedown', function(ev) {
-    if (ev.target !== el && ev.target !== bar) return;
-
-    self.focus();
-
-    cancel(ev);
-
-    if (new Date - last < 600) {
-      return self.maximize();
-    }
-    last = new Date;
-
-    self.drag(ev);
-
-    return cancel(ev);
-  });
-};
-
-ConsoleWindow.prototype.focus = function() {
-  // Restack
-  var parent = this.element.parentNode;
-  if (parent) {
-    parent.removeChild(this.element);
-    parent.appendChild(this.element);
-  }
-
-  // Focus Foreground Tab
-  this.focused.focus();
-
-  //tty.emit('focus window', this);
-  this.emit('focus');
-};
-
-ConsoleWindow.prototype.destroy = function() {
-  if (this.destroyed) return;
-  this.destroyed = true;
-
-  if (this.minimize) this.minimize();
-
-  //splice(tty.windows, this);
-  //if (tty.windows.length) tty.windows[0].focus();
-
-  this.element.parentNode.removeChild(this.element);
-
-  this.each(function(term) {
-    term.destroy();
-  });
-
-  //tty.emit('close window', this);
-  this.emit('close');
-};
-
-ConsoleWindow.prototype.drag = function(ev) {
-  var self = this
-    , el = this.element;
-
-  if (this.minimize) return;
-
-  var drag = {
-    left: el.offsetLeft,
-    top: el.offsetTop,
-    pageX: ev.pageX,
-    pageY: ev.pageY
-  };
-
-  el.style.opacity = '0.60';
-  el.style.cursor = 'move';
-  document.documentElement.style.cursor = 'move';
-
-  function move(ev) {
-    el.style.left =
-      (drag.left + ev.pageX - drag.pageX) + 'px';
-    var tmptop = (drag.top + ev.pageY - drag.pageY);
-    if (tmptop < 0) {
-       tmptop = 0;
-    }
-    el.style.top = tmptop + 'px';
-  }
-
-  function up() {
-    el.style.opacity = '';
-    el.style.cursor = '';
-    document.documentElement.style.cursor = '';
-
-    off(document, 'mousemove', move);
-    off(document, 'mouseup', up);
-
-    var ev = {
-      left: el.style.left.replace(/\w+/g, ''),
-      top: el.style.top.replace(/\w+/g, '')
-    };
-
-    //tty.emit('drag window', self, ev);
-    self.emit('drag', ev);
-  }
-
-  on(document, 'mousemove', move);
-  on(document, 'mouseup', up);
-};
-
-ConsoleWindow.prototype.resizing = function(ev) {
-  var self = this
-    , el = this.element
-    , term = this.focused;
-
-  if (this.minimize) delete this.minimize;
-
-  var resize = {
-    w: el.clientWidth,
-    h: el.clientHeight
-  };
-
-  el.style.overflow = 'hidden';
-  el.style.opacity = '0.70';
-  el.style.cursor = 'se-resize';
-  document.documentElement.style.cursor = 'se-resize';
-  term.element.style.height = '100%';
-
-  function move(ev) {
-    var x, y;
-    y = el.offsetHeight - term.element.clientHeight;
-    x = ev.pageX - el.offsetLeft;
-    y = (ev.pageY - el.offsetTop) - y;
-    el.style.width = x + 'px';
-    el.style.height = y + 'px';
-  }
-
-  function up() {
-    var x, y;
-
-    x = el.clientWidth / resize.w;
-    y = el.clientHeight / resize.h;
-    x = (x * term.cols) | 0;
-    y = (y * term.rows) | 0;
-
-    self.resize(x, y);
-
-    el.style.width = '';
-    el.style.height = '';
-
-    el.style.overflow = '';
-    el.style.opacity = '';
-    el.style.cursor = '';
-    document.documentElement.style.cursor = '';
-    term.element.style.height = '';
-
-    off(document, 'mousemove', move);
-    off(document, 'mouseup', up);
-  }
-
-  on(document, 'mousemove', move);
-  on(document, 'mouseup', up);
-};
-
-ConsoleWindow.prototype.maximize = function() {
-  if (this.minimize) return this.minimize();
-
-  var self = this
-    , el = this.element
-    , term = this.focused
-    , x
-    , y;
-
-  var m = {
-    cols: term.cols,
-    rows: term.rows,
-    left: el.offsetLeft,
-    top: el.offsetTop,
-    root: root.className
-  };
-
-  this.minimize = function() {
-    delete this.minimize;
-
-    el.style.left = m.left + 'px';
-    el.style.top = m.top + 'px';
-    el.style.width = '';
-    el.style.height = '';
-    term.element.style.width = '';
-    term.element.style.height = '';
-    el.style.boxSizing = '';
-    self.grip.style.display = '';
-    root.className = m.root;
-
-    self.resize(m.cols, m.rows);
-
-    //tty.emit('minimize window', self);
-    self.emit('minimize');
-  };
-
-  window.scrollTo(0, 0);
-
-  x = root.clientWidth / term.element.offsetWidth;
-  y = root.clientHeight / term.element.offsetHeight;
-  x = (x * term.cols) | 0;
-  y = (y * term.rows) | 0;
-
-  el.style.left = '0px';
-  el.style.top = '0px';
-  el.style.width = '100%';
-  el.style.height = '100%';
-  term.element.style.width = '100%';
-  term.element.style.height = '100%';
-  el.style.boxSizing = 'border-box';
-  this.grip.style.display = 'none';
-  root.className = 'maximized';
-
-  this.resize(x, y);
-
-  //tty.emit('maximize window', this);
-  this.emit('maximize');
-};
-
-ConsoleWindow.prototype.resize = function(cols, rows) {
-  this.cols = cols;
-  this.rows = rows;
-
-  this.each(function(term) {
-    term.resize(cols, rows);
-  });
-
-  //tty.emit('resize window', this, cols, rows);
-  this.emit('resize', cols, rows);
-};
-
-ConsoleWindow.prototype.each = function(func) {
-  var i = this.tabs.length;
-  while (i--) {
-    func(this.tabs[i], i);
-  }
-};
-
-ConsoleWindow.prototype.createTab = function() {
-  return new Tab(this, this.consoleurl);
-};
-
-ConsoleWindow.prototype.highlight = function() {
-  var self = this;
-
-  this.element.style.borderColor = 'orange';
-  setTimeout(function() {
-    self.element.style.borderColor = '';
-  }, 200);
-
-  this.focus();
-};
-
-ConsoleWindow.prototype.focusTab = function(next) {
-  var tabs = this.tabs
-    , i = indexOf(tabs, this.focused)
-    , l = tabs.length;
-
-  if (!next) {
-    if (tabs[--i]) return tabs[i].focus();
-    if (tabs[--l]) return tabs[l].focus();
-  } else {
-    if (tabs[++i]) return tabs[i].focus();
-    if (tabs[0]) return tabs[0].focus();
-  }
-
-  return this.focused && this.focused.focus();
-};
-
-ConsoleWindow.prototype.nextTab = function() {
-  return this.focusTab(true);
-};
-
-ConsoleWindow.prototype.previousTab = function() {
-  return this.focusTab(false);
-};
-
-/**
- * Tab
- */
-
-function Tab(win, consoleurl) {
-  var self = this;
-
-  var cols = win.cols
-    , rows = win.rows;
-
-  Terminal.call(this, {
-    cols: cols,
-    rows: rows
-  });
-
-  var button = document.createElement('div');
-  button.className = 'tab';
-  button.innerHTML = '\u2022';
-  //win.bar.appendChild(button);
-
-  on(button, 'click', function(ev) {
-    if (ev.ctrlKey || ev.altKey || ev.metaKey || ev.shiftKey) {
-      self.destroy();
-    } else {
-      self.focus();
-    }
-    return cancel(ev);
-  });
-
-  this.id = '';
-  this.consoleurl = consoleurl;
-  this.clientcount = 0;
-  this.connectstate = 'unconnected';
-  this.lasterror = ''
-  this.window = win;
-  this.button = button;
-  this.element = null;
-  this.process = '';
-  this.open();
-  this.hookKeys();
-  // Now begins the code that will embarass me when I actually know my way
-  // around javascript -jbjohnso
-  this.sessid = '';
-  this.datapending = false;
-  this.waitingdata = false;
-  this.sentdata = function(data, textStatus, jqXHR) {
-    if (this.waitingdata) {
-      postRequest(consoleurl,  { session: this.sessid, bytes: this.waitingdata }, this.sentdata);
-      this.waitingdata = false;
-    } else {
-        this.datapending = false;
-    }
-  }.bind(this);
-  this.on('data', function(data) {
-    // Send data to console from terminal
-    if (this.datapending) {
-      if (!this.waitingdata) {
-        this.waitingdata = data;
-      } else {
-        this.waitingdata = this.waitingdata + data;
-      }
-      return;
-    }
-    this.datapending = true;
-    postRequest(consoleurl,  { session: this.sessid, bytes: data }, this.sentdata);
-  }.bind(this));
-  this.gotdata = function(data, textStatus, jqXHR) {
-    if ("data" in data) {
-       this.write(data.data);
-    }
-    var updatetitle = false;
-    var updateinfo = [];
-    if ("connectstate" in data) {
-        updatetitle = true;
-        this.connectstate = data.connectstate;
-    }
-    if (this.connectstate != "connected") {
-        updateinfo.push(this.connectstate);
-    } else {
-        self.lasterror = '';
-    }
-    if ("error" in data) {
-        updatetitle = true;
-        this.lasterror = data.error
-    }
-    if (this.lasterror != '') {
-        updateinfo.push(this.lasterror);
-    }
-    if ("clientcount" in data) {
-        updatetitle = true;
-        this.clientcount = data.clientcount;
-    }
-    if (this.clientcount > 1) {
-        updateinfo.push("clients: " + this.clientcount.toString());
-    }
-    if (updatetitle == true) {
-        if (updateinfo.length > 0) {
-            this.window.title.innerHTML = this.window.nodename + " [" + updateinfo.join() + "]";
-        } else {
-            this.window.title.innerHTML = this.window.nodename;
-        }
-    }
-    postRequest(consoleurl,  { session: this.sessid }, this.gotdata);
-  }.bind(this);
-  this.gotsession = function(data, textStatus, jqXHR) {
-    this.sessid = data.session
-    postRequest(consoleurl,  { session: this.sessid }, this.gotdata);
-  }.bind(this);
-  postRequest(consoleurl,  false, this.gotsession);
-
-  win.tabs.push(this);
-};
-
-inherits(Tab, Terminal);
-
-Tab.prototype._write = Tab.prototype.write;
-
-Tab.prototype.write = function(data) {
-  if (this.window.focused !== this) this.button.style.color = 'red';
-  return this._write(data);
-};
-
-Tab.prototype._focus = Tab.prototype.focus;
-
-Tab.prototype.focus = function() {
-  if (Terminal.focus === this) return;
-
-  var win = this.window;
-
-  // maybe move to Tab.prototype.switch
-  if (win.focused !== this) {
-    if (win.focused) {
-      if (win.focused.element.parentNode) {
-        win.focused.element.parentNode.removeChild(win.focused.element);
-      }
-      win.focused.button.style.fontWeight = '';
-    }
-
-    win.element.appendChild(this.element);
-    win.focused = this;
-
-    //win.title.innerHTML = this.process;
-    this.button.style.fontWeight = 'bold';
-    this.button.style.color = '';
-  }
-
-  this._focus();
-
-  win.focus();
-
-  //tty.emit('focus tab', this);
-  this.emit('focus');
-};
-
-Tab.prototype._resize = Tab.prototype.resize;
-
-Tab.prototype.resize = function(cols, rows) {
-  //this.socket.emit('resize', this.id, cols, rows);
-  this._resize(cols, rows);
-  //tty.emit('resize tab', this, cols, rows);
-  this.emit('resize', cols, rows);
-};
-
-Tab.prototype.__destroy = Tab.prototype.destroy;
-
-Tab.prototype._destroy = function() {
-  if (this.destroyed) return;
-  this.destroyed = true;
-
-  var win = this.window;
-
-  this.button.parentNode.removeChild(this.button);
-  if (this.element.parentNode) {
-    this.element.parentNode.removeChild(this.element);
-  }
-
-  if (tty.terms[this.id]) delete tty.terms[this.id];
-  splice(win.tabs, this);
-
-  if (win.focused === this) {
-    win.previousTab();
-  }
-
-  if (!win.tabs.length) {
-    win.destroy();
-  }
-
-  this.__destroy();
-};
-
-Tab.prototype.destroy = function() {
-  if (this.destroyed) return;
-  //TODO: politely let server know of client closure
-  this._destroy();
-  //tty.emit('close tab', this);
-  this.emit('close');
-};
-
-Tab.prototype.hookKeys = function() {
-  var self = this;
-
-  // Alt-[jk] to quickly swap between windows.
-  this.on('key', function(key, ev) {
-    if (Terminal.focusKeys === false) {
-      return;
-    }
-
-    var offset
-      , i;
-
-    if (key === '\x1bj') {
-      offset = -1;
-    } else if (key === '\x1bk') {
-      offset = +1;
-    } else {
-      return;
-    }
-
-    i = indexOf(tty.windows, this.window) + offset;
-
-    this._ignoreNext();
-
-    if (tty.windows[i]) return tty.windows[i].highlight();
-
-    if (offset > 0) {
-      if (tty.windows[0]) return tty.windows[0].highlight();
-    } else {
-      i = tty.windows.length - 1;
-      if (tty.windows[i]) return tty.windows[i].highlight();
-    }
-
-    return this.window.highlight();
-  });
-
-  this.on('request paste', function(key) {
-    this.socket.emit('request paste', function(err, text) {
-      if (err) return;
-      self.send(text);
-    });
-  });
-
-  this.on('request create', function() {
-    this.window.createTab();
-  });
-
-  this.on('request term', function(key) {
-    if (this.window.tabs[key]) {
-      this.window.tabs[key].focus();
-    }
-  });
-
-  this.on('request term next', function(key) {
-    this.window.nextTab();
-  });
-
-  this.on('request term previous', function(key) {
-    this.window.previousTab();
-  });
-};
-
-Tab.prototype._ignoreNext = function() {
-  // Don't send the next key.
-  var handler = this.handler;
-  this.handler = function() {
-    this.handler = handler;
-  };
-  var showCursor = this.showCursor;
-  this.showCursor = function() {
-    this.showCursor = showCursor;
-  };
-};
-
-/**
- * Program-specific Features
- */
-
-Tab.prototype._bindMouse = Tab.prototype.bindMouse;
-
-Tab.prototype.bindMouse = function() {
-  if (!Terminal.programFeatures) return this._bindMouse();
-
-  var self = this;
-
-  var wheelEvent = 'onmousewheel' in window
-    ? 'mousewheel'
-    : 'DOMMouseScroll';
-
-  on(self.element, wheelEvent, function(ev) {
-    if (self.mouseEvents) return;
-
-    if ((ev.type === 'mousewheel' && ev.wheelDeltaY > 0)
-        || (ev.type === 'DOMMouseScroll' && ev.detail < 0)) {
-      // page up
-      self.keyDown({keyCode: 33});
-    } else {
-      // page down
-      self.keyDown({keyCode: 34});
-    }
-
-    return cancel(ev);
-  });
-
-  return this._bindMouse();
-};
-
-/**
- * Helpers
- */
-
-function indexOf(obj, el) {
-  var i = obj.length;
-  while (i--) {
-    if (obj[i] === el) return i;
-  }
-  return -1;
-}
-
-function splice(obj, el) {
-  var i = indexOf(obj, el);
-  if (~i) obj.splice(i, 1);
-}
-
-function sanitize(text) {
-  if (!text) return '';
-  return (text + '').replace(/[&<>]/g, '')
-}
-
-this.ConsoleWindow = ConsoleWindow;
-
-}).call(function() {
-    return this;
-}());