Fix inconsistent dict member extension

If two portions of a list come back piecewise from the plugin that
are both lists, extend them rather than making a nested list.

confluent_client/MANIFEST.in

@@ -1 +1,2 @@
-include confluent_env.sh
+include confluent_env.sh
+include confluent_env.csh

confluent_client/bin/collate

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a utility to bring xcoll (plus enhancements) to confluent
+# the core engine is textgroup.py, this simply provides a CLI to use
+# generically
+
+import optparse
+import os
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.textgroup as tg
+import confluent.client
+
+argparser = optparse.OptionParser(usage="Usage: <other command> | %prog [options]")
+argparser.add_option('-a', '--abbreviate', action='store_true',
+                     help='Attempt to use confluent server to shorten noderanges')
+argparser.add_option('-d', '--diff', action='store_true',
+                     help='Show what differs between most common '
+                          'output group and others')
+argparser.add_option('-w', '--watch', action='store_true',
+                     help='Show intermediate results while running')
+argparser.add_option('-s', '--skipcommon', action='store_true',
+                     help='Do not print most common result, only non modal '
+                          'groups, useful when combined with -d')
+argparser.add_option('-c', '--count', action='store_true',
+                     help='Also display count of nodes in a given group')
+argparser.add_option('-r', '--reverse', action='store_true',
+                     help='Reverse sort order to show biggest output group '
+                          'last')
+(options, args) = argparser.parse_args()
+if sys.stdin.isatty():
+    argparser.print_help()
+    sys.exit(1)
+
+grouped = tg.GroupedData()
+
+if options.abbreviate:
+    grouped = tg.GroupedData(confluent.client.Command())
+else:
+    grouped = tg.GroupedData()
+
+def print_current():
+    if options.diff:
+        grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
+                          reverse=options.reverse)
+    else:
+        grouped.print_all(skipmodal=options.skipcommon,
+                               count=options.count,
+                               reverse=options.reverse)
+    sys.stdout.flush()
+
+fullline = sys.stdin.readline()
+while fullline:
+    for line in fullline.split('\n'):
+        if not line:
+            continue
+        if ': ' not in line:
+            line = 'UNKNOWN: ' + line
+        grouped.add_line(*line.split(': ', 1))
+        if options.watch:
+            sys.stdout.write('\x1b[2J\x1b[;H')  # clear screen
+            print_current()
+    fullline = sys.stdin.readline()
+if not options.watch:
+    print_current()
+

confluent_client/bin/confetty

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2016 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,6 +47,7 @@ import optparse
 import os
 import select
 import shlex
+import signal
 import socket
 import sys
 import time
@@ -56,7 +57,10 @@ try:
     import tty
 except ImportError:
     pass
-
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 exitcode = 0
 consoleonly = False
 consolename = ""
@@ -72,6 +76,7 @@ import confluent.tlvdata as tlvdata
 import confluent.client as client
 
 conserversequence = '\x05c'  # ctrl-e, c
+clearpowermessage = False
 
 oldtcattr = None
 fd = sys.stdin
@@ -84,19 +89,57 @@ netserver = None
 laststate = {}
 
 
+def print_help():
+    print("confetty provides a filesystem like interface to confluent.  "
+          "Navigation is done using the same commands as would be used in a "
+          "filesystem.  Tab completion is supported to aid in navigation,"
+          "as is up arrow to recall previous commands and control-r to search"
+          "previous command history, similar to using bash\n\n"
+          "The supported commands are:\n"
+          "cd [location] - Set the current command context, similar to a "
+          "working directory.\n"
+          "show [resource] - Present the information about the specified "
+          "resource, or current context if omitted.\n"
+          "create [resource] attributename=value attributename=value - Create "
+          "a new instance of a resource.\n"
+          "remove [resource] - Remove a resource from a list\n"
+          "set [resource] attributename=value attributename=value - Change "
+          "the specified attributes value for the given resource name\n"
+          "unset [resource] attributename - Clear any value for the given "
+          "attribute names on a resource.\n"
+          "start [resource] - When used on a text session resource, it "
+          "enters remote terminal mode.  In this mode, use 'ctrl-e, c, ?' for "
+          "help"
+          )
+    #TODO(jjohnson2): lookup context help for 'target' variable, perhaps
+    #common with the api document
+
+
 def updatestatus(stateinfo={}):
+    global powerstate, powertime, clearpowermessage
     status = consolename
     info = []
-    for statekey in stateinfo.iterkeys():
+    for statekey in stateinfo:
         laststate[statekey] = stateinfo[statekey]
     if ('connectstate' in laststate and
             laststate['connectstate'] != 'connected'):
         info.append(laststate['connectstate'])
+        if laststate['connectstate'] == 'closed':
+            quitconfetty(fullexit=consoleonly)
     if 'error' in laststate:
         info.append(laststate['error'])
         # error will be repeated if relevant
         # avoid keeping it around as stale
         del laststate['error']
+    if 'state' in stateinfo:  # currently only read power means anything
+        newpowerstate = stateinfo['state']['value']
+        if newpowerstate != powerstate and newpowerstate == 'off':
+            sys.stdout.write("\x1b[2J\x1b[;H[powered off]\r\n")
+            clearpowermessage = True
+        if newpowerstate == 'on' and clearpowermessage:
+            sys.stdout.write("\x1b[2J\x1b[;H")
+            clearpowermessage = False
+        powerstate = newpowerstate
     if 'clientcount' in laststate and laststate['clientcount'] != 1:
         info.append('clients: %d' % laststate['clientcount'])
     if 'bufferage' in stateinfo and stateinfo['bufferage'] is not None:
@@ -104,14 +147,14 @@ def updatestatus(stateinfo={}):
     if 'showtime' in laststate:
         showtime = laststate['showtime']
         age = time.time() - laststate['showtime']
-        if age > 86400: # older than one day
+        if age > 86400:  # older than one day
             # disambiguate by putting date in and time
             info.append(time.strftime('%m-%dT%H:%M', time.localtime(showtime)))
         else:
             info.append(time.strftime('%H:%M', time.localtime(showtime)))
     if info:
         status += ' [' + ','.join(info) + ']'
-    if os.environ['TERM'] not in ('linux'):
+    if os.environ.get('TERM', '') not in ('linux'):
         sys.stdout.write('\x1b]0;console: %s\x07' % status)
     sys.stdout.flush()
 
@@ -143,7 +186,7 @@ def recurse_format(datum, levels=0):
 
 
 def prompt():
-    if os.environ['TERM'] not in ('linux'):
+    if os.environ.get('TERM', '') not in ('linux'):
         sys.stdout.write('\x1b]0;confetty: %s\x07' % target)
     try:
         return raw_input(target + ' -> ')
@@ -167,6 +210,7 @@ valid_commands = [
     'remove',
     'rm',
     'delete',
+    'help',
 ]
 
 candidates = None
@@ -233,7 +277,11 @@ def rcompleter(text, state):
 
 
 def parse_command(command):
-    args = shlex.split(command, posix=True)
+    try:
+        args = shlex.split(command, posix=True)
+    except ValueError as ve:
+        print('Error: ' + str(ve))
+        return []
     return args
 
 
@@ -247,14 +295,17 @@ def print_result(res):
     if 'databynode' in res:
         print_result(res['databynode'])
         return
-    for key in res.iterkeys():
+    for key in sorted(res):
         notes = []
         if res[key] is None:
             attrstr = '%s=""' % key
         elif type(res[key]) == list:
             attrstr = '%s=%s' % (key, recurse_format(res[key]))
         elif not isinstance(res[key], dict):
-            print '{0}: {1}'.format(key, res[key])
+            try:
+                print '{0}: {1}'.format(key, res[key])
+            except UnicodeEncodeError:
+                print '{0}: {1}'.format(key, repr(res[key]))
             continue
         elif 'value' in res[key] and res[key]['value'] is not None:
             attrstr = '%s="%s"' % (key, res[key]['value'])
@@ -300,22 +351,67 @@ def do_command(command, server):
         return
     argv[0] = argv[0].lower()
     if argv[0] == 'exit':
+        if os.environ.get('TERM', '') not in ('linux'):
+            sys.stdout.write('\x1b]0;\x07')
         sys.exit(0)
+    elif argv[0] in ('help', '?'):
+        return print_help()
     elif argv[0] == 'cd':
         otarget = target
         if len(argv) > 1:
             target = fullpath_target(argv[1], forcepath=True)
         else:  # cd by itself, go 'home'
             target = '/'
-        for res in session.read(target, server):
-            if 'errorcode' in res:
-                exitcode = res['errorcode']
-            if 'error' in res:
-                sys.stderr.write(target + ': ' + res['error'] + '\n')
-                target = otarget
+        if target[-1] == '/':
+            parentpath = target[:-1]
+        else:
+            parentpath = target
+        if parentpath:
+            childname = '{0}/'.format(parentpath[parentpath.rindex('/') + 1:])
+            parentpath = parentpath[:parentpath.rindex('/') + 1]
+            if parentpath == '/noderange/':
+                for res in session.read(target, server):
+                    if 'errorcode' in res:
+                        exitcode = res['errorcode']
+                        target = otarget
+                    if 'error' in res:
+                        sys.stderr.write(target + ': ' + res['error'] + '\n')
+                        target = otarget
+            else:
+                foundchild = False
+                for res in session.read(parentpath, server):
+                    try:
+                        if res['item']['href'] == childname:
+                            foundchild = True
+                    except KeyError:
+                        pass
+                    if 'errorcode' in res:
+                        exitcode = res['errorcode']
+                        target = otarget
+                    if 'error' in res:
+                        sys.stderr.write(target + ': ' + res['error'] + '\n')
+                        target = otarget
+                if not foundchild:
+                    sys.stderr.write(target + ': Target not found - \n')
+                    target = otarget
     elif argv[0] in ('cat', 'show', 'ls', 'dir'):
         if len(argv) > 1:
             targpath = fullpath_target(argv[1])
+            if argv[0] in ('ls', 'dir'):
+                if targpath[-1] != '/':
+                    # could still be a directory, fetch the parent..
+                    childname = targpath[targpath.rindex('/') + 1:]
+                    parentpath = targpath[:targpath.rindex('/') + 1]
+                    if parentpath != '/noderange/':
+                        # if it were /noderange/, then it's a directory
+                        # even though parent won't tell us that
+                        for res in session.read(parentpath, server):
+                            try:
+                                if res['item']['href'] == childname:
+                                    print(childname)
+                                    return
+                            except KeyError:
+                                pass
         else:
             targpath = target
         for res in session.read(targpath):
@@ -378,14 +474,21 @@ def createresource(args):
     if keydata is None:
         return
     targpath = fullpath_target(resname)
-    collection, _, resname = targpath.rpartition('/')
-    keydata['name'] = resname
+    if targpath.startswith('/noderange//'):
+        collection = targpath
+    else:
+        collection, _, resname = targpath.rpartition('/')
+        keydata['name'] = resname
     makecall(session.create, (collection, keydata))
 
 
 def makecall(callout, args):
     global exitcode
     for response in callout(*args):
+        if 'deleted' in response:
+            print("Deleted: " + response['deleted'])
+        if 'created' in response:
+            print("Created: " + response['created'])
         if 'error' in response:
             if 'errorcode' in response:
                 exitcode = response['errorcode']
@@ -494,7 +597,11 @@ def quitconfetty(code=0, fullexit=False, fixterm=True):
         fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
         if oldtcattr is not None:
             termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
+    # Request default color scheme, to undo potential weirdness of terminal
+    sys.stdout.write('\x1b[m')
     if fullexit:
+        if os.environ.get('TERM', '') not in ('linux'):
+            sys.stdout.write('\x1b]0;\x07')
         sys.exit(code)
     else:
         tlvdata.send(session.connection, {'operation': 'stop',
@@ -514,7 +621,7 @@ def get_session_node(shellargs):
     return None
 
 
-def conserver_command(filehandle, command):
+def conserver_command(filehandle, localcommand):
     # x - conserver has that as 'show baud', I am inclined to replace that with
     #     'request exclusive'
     # b - conserver has that as 'broadcast message', I'm tempted to use that
@@ -529,33 +636,139 @@ def conserver_command(filehandle, command):
     # d - down a console... never used this...
     # L - toggle logging
     # w - who is on console
-    while not command:
-        ready, _, _ = select.select((filehandle,), (), (), 1)
-        if ready:
-            command += filehandle.read()
-    if command[0] == '.':
+
+    cmdlen = 1
+    localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+    if localcommand[0] == '.':
         print("disconnect]\r")
         quitconfetty(fullexit=consoleonly)
-    elif command[0] == 'o':
+    elif localcommand[0] == 'o':
         tlvdata.send(session.connection, {'operation': 'reopen',
                                           'path': currconsole})
         print('reopen]\r')
-    elif command[0] == 'b':
+    elif localcommand[0] == 'b':
         tlvdata.send(session.connection, {'operation': 'break',
                                           'path': currconsole})
         print("break sent]\r")
-    elif command[0] == '?':
+    elif localcommand[0] == 'p':  # power
+        cmdlen += 1
+        localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+        if localcommand[1] == 'o': # off
+            print("powering off...")
+            session.simple_noderange_command(consolename, '/power/state', 'off')
+            print("complete]\r")
+        elif localcommand[1] == 's':  # shutdown
+            print("shutting down...")
+            session.simple_noderange_command(consolename, '/power/state', 'shutdown')
+            print("complete]\r")
+        elif localcommand[1] == 'b': # boot
+            cmdlen += 1
+            localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+            if localcommand[2] == 's':  # boot to setup
+                print("booting to setup...")
+
+                bootmode = 'uefi'
+                bootdev = 'setup'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            elif localcommand[2] == 'n':  # boot to network
+                print("booting to network...")
+
+                bootmode = 'uefi'
+                bootdev = 'network'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            elif localcommand[2] == '\x0d':  # boot to default
+                print("booting to default...")
+
+                bootmode = 'uefi'
+                bootdev = 'default'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            else:
+                print("Unknown boot state.]\r")
+
+        else:
+            print("Unknown power state.]\r")
+
+        check_power_state()
+    elif localcommand[0] == 'r':
+        sys.stdout.write('\x1b7\x1b[999;999H\x1b[6n')
+        sys.stdout.flush()
+        reply = ''
+        while 'R' not in reply:
+            try:
+                reply += sys.stdin.read(1)
+            except IOError:
+                pass
+        reply = reply.replace('\x1b[', '')
+        reply = reply.replace('R', '')
+        height, width = reply.split(';')
+        sys.stdout.write('\x1b8')
+        sys.stdout.flush()
+        print('sending stty commands]')
+        return 'stty columns {0}\rstty rows {1}\r'.format(width, height)
+    elif localcommand[0] == '?':
         print("help]\r")
-        print(".    disconnect\r")
-        print("b    break\r")
-        print("o    reopen\r")
+        print(".       exit console\r")
+        print("b       break\r")
+        print("o       reopen\r")
+        print("po      power off\r")
+        print("ps      shutdown\r")
+        print("pbs     boot to setup\r")
+        print("pbn     boot to network\r")
+        print("pb<ent> boot to default\r")
+        print("r       send stty command to resize terminal\r")
         print("<cr> abort command\r")
-    elif command[0] == '\x0d':
+    elif localcommand[0] == '\x0d':
         print("ignored]\r")
     else:  # not a command at all..
         print("unknown -- use '?']\r")
 
 
+def get_command_bytes(filehandle, localcommand, cmdlen):
+    while len(localcommand) < cmdlen:
+        ready, _, _ = select.select((filehandle,), (), (), 1)
+        if ready:
+            localcommand += filehandle.read()
+    return localcommand
+
+
 def check_escape_seq(currinput, filehandle):
     while conserversequence.startswith(currinput):
         if currinput.startswith(conserversequence):  # We have full sequence
@@ -622,6 +835,8 @@ if sys.stdout.isatty():
 
     readline.parse_and_bind("tab: complete")
     readline.parse_and_bind("set bell-style none")
+    dl = readline.get_completer_delims().replace('-', '')
+    readline.set_completer_delims(dl)
     readline.set_completer(completer)
 
 doexit = False
@@ -637,10 +852,22 @@ elif shellargs:
     do_command(command, netserver)
     quitconfetty(fullexit=True, fixterm=False)
 
+powerstate = None
+powertime = None
+
+
+def check_power_state():
+    tlvdata.send(
+        session.connection,
+        {'operation': 'retrieve',
+         'path': '/nodes/' + consolename + '/power/state'})
+    return
+
+
 while inconsole or not doexit:
     if inconsole:
         rdylist, _, _ = select.select(
-            (sys.stdin, session.connection), (), (), 60)
+            (sys.stdin, session.connection), (), (), 10)
         for fh in rdylist:
             if fh == session.connection:
                 # this only should get called in the
@@ -656,7 +883,15 @@ while inconsole or not doexit:
                     updatestatus(data)
                     continue
                 if data is not None:
-                    sys.stdout.write(data)
+                    if clearpowermessage:
+                        sys.stdout.write("\x1b[2J\x1b[;H")
+                        clearpowermessage = False
+                    try:
+                        sys.stdout.write(data)
+                    except IOError:  # Some times circumstances are bad
+                        # resort to byte at a time...
+                        for d in data:
+                            sys.stdout.write(d)
                     now = time.time()
                     if ('showtime' not in laststate or
                             (now // 60) != laststate['showtime'] // 60):
@@ -686,10 +921,17 @@ while inconsole or not doexit:
                         sys.stdout.write("\r\n[remote disconnected]\r\n")
                     break
             else:
-                myinput = fh.read()
-                myinput = check_escape_seq(myinput, fh)
-                if myinput:
-                    tlvdata.send(session.connection, myinput)
+                try:
+                    myinput = fh.read()
+                    myinput = check_escape_seq(myinput, fh)
+                    if myinput:
+                        tlvdata.send(session.connection, myinput)
+                except IOError:
+                    pass
+        if powerstate is None or powertime < time.time() - 10:  # Check powerstate every 10 seconds
+            powertime = time.time()
+            powerstate = True
+            check_power_state()
     else:
         currcommand = prompt()
         try:

confluent_client/bin/dir2img

@@ -0,0 +1,19 @@
+#!/bin/sh
+# This will take a given directory and make a 'big floppy image'
+# out of it, suitable for nodemedia upload.
+
+if [ -z "$1" -o -z "$2" ]; then
+        echo "Usage: $0 <directory> <imagefile>"
+        exit 1
+fi
+# Get the needed payload side
+SIZE=$(du -sB 512 $1|awk '{print $1}')
+ENTRIES=$(find $1 |wc -l)
+# Also, each file and directory has overhead, pad size by 32kb per file,
+# which should be overkill but other values proved to be inadequate
+# A deeper understanding of VFAT would probably allow for more precise value
+SIZE=$((SIZE + ENTRIES * 64))
+dd if=/dev/zero of=$2 bs=512 count=$SIZE
+# Make a big single sided floppy with many many tracks, saves on complex math
+mformat -i $2 -d 1 -t $SIZE -s 1 -h 1 ::
+mcopy -i $2 -s $1/* ::

confluent_client/bin/nodeattrib

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'alin37'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog [-b] noderange [list of attributes]  \
+             \n       %prog -c noderange <list of attributes>  \
+             \n       %prog -e noderange <attribute names to set>  \
+             \n       %prog noderange attribute1=value1 attribute2=value,...
+             \n ''')
+argparser.add_option('-b', '--blame', action='store_true',
+                     help='Show information about how attributes inherited')
+argparser.add_option('-e', '--environment', action='store_true',
+                     help='Set attributes, but from environment variable of '
+                          'same name')
+argparser.add_option('-c', '--clear', action='store_true',
+                     help='Clear attributes')
+(options, args) = argparser.parse_args()
+
+
+#setting minimal output to only output current information
+showtype = 'current'
+requestargs=None
+try:
+    noderange = args[0]
+    nodelist = '/noderange/{0}/nodes/'.format(noderange)
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+
+#Sets attributes
+nodetype="noderange"
+
+if len(args) > 1:
+    if "=" in args[1] or options.clear or options.environment:
+        if "=" in args[1] and options.clear:
+            print("Can not clear and set at the same time!")
+            argparser.print_help()
+            sys.exit(1)
+        exitcode=client.updateattrib(session,args,nodetype, noderange, options)
+    try:
+        # setting user output to what the user inputs
+        if args[1] == 'all':
+            showtype = 'all'
+            requestargs=args[2:]
+        elif args[1] == 'current':
+            showtype = 'current'
+            requestargs=args[2:]
+        else:
+            showtype = 'all'
+            requestargs=args[1:]
+    except:
+        pass
+
+if exitcode != 0:
+    sys.exit(exitcode)
+
+# Lists all attributes
+if len(args) > 0:
+    # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank.
+    if requestargs is None:
+        showtype = 'current'
+    elif requestargs == []:
+        #showtype already set
+        pass
+    else:
+        try:
+            requestargs.remove('all')
+            requestargs.remove('current')
+        except ValueError:
+            pass
+    exitcode = client.printattributes(session, requestargs, showtype,nodetype, noderange, options)
+else:
+    for res in session.read(nodelist):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print res['item']['href'].replace('/', '')
+
+sys.exit(exitcode)

confluent_client/bin/nodebmcreset

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+
+errorNodes = set([])
+
+success = session.simple_noderange_command(noderange, 'configuration/management_controller/reset', 'reset', key='state', errnodes=errorNodes)   # = 0 if successful
+
+# Determine which nodes were successful and print them
+
+allNodes = set([])
+
+for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
+    allNodes.add(node['item']['href'].replace("/", ""))
+
+goodNodes = allNodes - errorNodes
+
+for node in goodNodes:
+    print node + ": BMC Reset Successful"
+
+
+sys.exit(success)

confluent_client/bin/nodeboot

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser()
+argparser.add_option('-b', '--bios', dest='biosmode',
+                     action='store_true', default=False,
+                     help='Request BIOS style boot (rather than UEFI)')
+argparser.add_option('-p', '--persist', dest='persist', action='store_true',
+                     default=False,
+                     help='Request the boot device be persistent rather than '
+                          'one time')
+
+(options, args) = argparser.parse_args()
+
+try:
+    noderange = args[0]
+except IndexError:
+    sys.stderr.write(
+        'Usage: {0} <noderange> [default|cd|network|setup|hd]\n'.format(
+            sys.argv[0]))
+    sys.exit(1)
+client.check_globbing(noderange)
+bootdev = None
+if len(sys.argv) > 2:
+    bootdev = sys.argv[2]
+    if bootdev in ('net', 'pxe'):
+        bootdev = 'network'
+session = client.Command()
+exitcode = 0
+if options.biosmode:
+    bootmode = 'bios'
+else:
+    bootmode = 'uefi'
+
+errnodes = set([])
+rc = session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
+                                     bootmode=bootmode,
+                                     persistent=options.persist,
+                                     errnodes=errnodes)
+if errnodes:
+    noderange = noderange + ',-(' + ','.join(errnodes) + ')'
+rc |= session.simple_noderange_command(noderange, '/power/state', 'boot')
+sys.exit(rc)

confluent_client/bin/nodeconfig

@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import signal
+import optparse
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+class NullOpt(object):
+    blame = None
+    clear = None
+
+
+def bailout(msg, code=1):
+    sys.stderr.write(msg + '\n')
+    sys.exit(code)
+
+
+argparser = optparse.OptionParser()
+argparser.add_option('-c', '--comparedefault', dest='comparedefault',
+                     action='store_true', default=False,
+                     help='Compare given settings to default or list settings '
+                          'that are non default')
+argparser.add_option('-d', '--detail', dest='detail',
+                     action='store_true', default=False,
+                     help='Provide verbose information as available, such as '
+                          'help text and possible valid values')
+argparser.add_option('-x', '--exclude', dest='exclude',
+                     action='store_true', default=False,
+                     help='Treat positional arguments as items to not '
+                          'examine, compare, or restore default')
+(options, args) = argparser.parse_args()
+
+cfgpaths = {
+    'bmc.ipv4_address': (
+        'configuration/management_controller/net_interfaces/management',
+        'ipv4_address'),
+    'bmc.ipv4_method': (
+        'configuration/management_controller/net_interfaces/management',
+        'ipv4_configuration'),
+    'bmc.ipv4_gateway': (
+        'configuration/management_controller/net_interfaces/management',
+        'ipv4_gateway'),
+}
+
+autodeps = {
+    'bmc.ipv4_address': (('bmc.ipv4_method', 'static'),)
+}
+
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+setmode = None
+assignment = {}
+queryparms = {}
+printsys = []
+setsys = {}
+forceset = False
+needval = None
+
+if len(args) == 1 or options.exclude:
+    if not options.exclude:
+        printsys = 'all'
+    for candidate in cfgpaths:
+        path, attrib = cfgpaths[candidate]
+        path = '/noderange/{0}/{1}'.format(noderange, path)
+        if path not in queryparms:
+            queryparms[path] = {}
+        queryparms[path][attrib] = candidate
+
+
+def _assign_value():
+    if key not in cfgpaths:
+        setsys[key] = value
+    for depkey, depval in autodeps.get(key, []):
+        assignment[depkey] = depval
+    assignment[key] = value
+
+
+for param in args[1:]:
+    if param == 'show':
+        continue  # forgive muscle memory of pasu users
+    if param == 'set':
+        setmode = True
+        forceset = True
+        continue
+    if needval:
+        key = needval
+        value = param
+        _assign_value()
+        continue
+    if '=' in param or param[-1] == ':' or forceset:
+        if setmode is None:
+            setmode = True
+        if setmode != True:
+            bailout('Cannot do set and query in same command')
+        if '=' in param:
+            key, _, value = param.partition('=')
+            _assign_value()
+        elif param[-1] == ':':
+            needval = param[:-1]
+        else:
+            needval = param
+    else:
+        if setmode is None:
+            setmode = False
+        if setmode != False:
+            bailout('Cannot do set and query in same command')
+        if '.' not in param:
+            matchedparms = False
+            for candidate in cfgpaths:
+                if candidate.startswith('{0}.'.format(param)):
+                    matchedparms = True
+                    if not options.exclude:
+                        path, attrib = cfgpaths[candidate]
+                        path = '/noderange/{0}/{1}'.format(noderange, path)
+                        if path not in queryparms:
+                            queryparms[path] = {}
+                        queryparms[path][attrib] = candidate
+                    else:
+                        try:
+                            del queryparms[path]
+                        except KeyError:
+                            pass
+            if not matchedparms:
+                printsys.append(param)
+        elif param not in cfgpaths:
+            printsys.append(param)
+        else:
+            path, attrib = cfgpaths[param]
+            path = '/noderange/{0}/{1}'.format(noderange, path)
+            if path not in queryparms:
+                queryparms[path] = {}
+            queryparms[path][attrib] = param
+session = client.Command()
+rcode = 0
+if setmode:
+    if options.exclude:
+        sys.stderr.write('Cannot use exclude and assign at the same time\n')
+        sys.exit(1)
+    updatebypath = {}
+    attrnamebypath = {}
+    for key in assignment:
+        if key not in cfgpaths:
+            path = 'configuration/system/all'
+            attrib = key
+        else:
+            path, attrib = cfgpaths[key]
+        if path not in updatebypath:
+            updatebypath[path] = {}
+            attrnamebypath[path] = {}
+        updatebypath[path][attrib] = assignment[key]
+        attrnamebypath[path][attrib] = key
+    # well, we want to expand things..
+    # check ipv4, if requested change method to static
+    for path in updatebypath:
+        for fr in session.update('/noderange/{0}/{1}'.format(noderange, path),
+                                updatebypath[path]):
+            for node in fr['databynode']:
+                r = fr['databynode'][node]
+                if 'error' in r:
+                    sys.stderr.write(node + ': ' + r['error'] + '\n')
+                    if 'errorcode' in r:
+                        rcode |= r['errorcode']
+                if 'value' not in r:
+                    continue
+                keyval = r['value']
+                key, val = keyval.split('=')
+                if key in attrnamebypath[path]:
+                    key = attrnamebypath[path][key]
+                print('{0}: {1}: {2}'.format(node, key, val))
+else:
+    for path in queryparms:
+        if options.comparedefault:
+            continue
+        client.print_attrib_path(path, session, list(queryparms[path]),
+                                 NullOpt(), queryparms[path])
+    if printsys or options.exclude:
+        if printsys == 'all':
+            printsys = []
+        path = '/noderange/{0}/configuration/system/all'.format(noderange)
+        client.print_attrib_path(path, session, printsys,
+                                 options)
+sys.exit(rcode)

confluent_client/bin/nodedefine

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange attribute1=value1 attribute2=value,...
+             \n ''')
+(options, args) = argparser.parse_args()
+requestargs=None
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+attribs = {'name': noderange}
+for arg in args[1:]:
+    key, val = arg.split('=')
+    attribs[key] = val
+for r in session.create('/noderange/', attribs):
+    if 'error' in r:
+        sys.stderr.write(r['error'] + '\n')
+        exitcode |= 1
+    if 'created' in r:
+        print('{0}: created'.format(r['created']))
+sys.exit(exitcode)

confluent_client/bin/nodediscover

@@ -0,0 +1,286 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import optparse
+import os
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+tabformat = '{0:>15}|{1:>15}|{2:>15}|{3:>36}|{4:>17}|{5:>12}|{6:>48}'
+columns = ['Node', 'Model', 'Serial', 'UUID', 'Mac Address', 'Type',
+           'Current IP Addresses']
+delimit = ['-' * 15, '-' * 15, '-' * 15, '-' * 36, '-' * 17, '-' * 12,
+           '-' * 48]
+
+
+def dumpmacs(procinfo):
+    return ','.join(procinfo['macs']) # + procinfo.get('relatedmacs', []))
+
+
+def print_disco(options, session, currmac):
+    procinfo = {}
+    for tmpinfo in session.read('/discovery/by-mac/{0}'.format(currmac)):
+
+        procinfo.update(tmpinfo)
+    record = [procinfo['nodename'], procinfo['modelnumber'],
+              procinfo['serialnumber'], procinfo['uuid'], dumpmacs(procinfo),
+              ','.join(procinfo['types']),
+              ','.join(sorted(procinfo['ipaddrs']))]
+    if options.csv:
+        csv.writer(sys.stdout).writerow(record)
+    else:
+        print(tabformat.format(*record))
+
+
+def process_header(header):
+    # normalize likely header titles
+    fields = []
+    broken = False
+    for datum in header:
+        datum = datum.lower()
+        if datum.startswith('node') or datum.startswith('name'):
+            fields.append('node')
+        elif datum in ('nodegroup', 'nodegroups', 'group', 'groups'):
+            fields.append('groups')
+        elif datum.startswith('mac') or datum.startswith('ether'):
+            fields.append('mac')
+        elif datum.startswith('serial') or datum in ('sn', 's/n'):
+            fields.append('serial')
+        elif datum == 'uuid':
+            fields.append('uuid')
+        elif datum in ('bmc', 'imm', 'xcc'):
+            fields.append('hardwaremanagement.manager')
+        elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
+            fields.append('net.bmc.gateway')
+        elif datum in ('bmcuser', 'username', 'user'):
+            fields.append('secret.hardwaremanagementuser')
+        elif datum in ('bmcpass', 'password', 'pass'):
+            fields.append('secret.hardwaremanagementpassword')
+        else:
+            print("Unrecognized column name {0}".format(datum))
+            broken = True
+    if broken:
+        sys.exit(1)
+    return tuple(fields)
+
+
+def datum_complete(datum):
+    if 'node' not in datum or not datum['node']:
+        sys.stderr.write('Nodename is a required field')
+        return False
+    provided = set(datum)
+    required = set(['serial', 'uuid', 'mac'])
+    for field in provided & required:
+        if datum[field]:
+            break
+    else:
+        sys.stderr.write('One of the fields "Serial Number", "UUID", or '
+                   '"MAC Address" must be provided')
+        return False
+    return True
+
+
+searchkeys = set(['mac', 'serial', 'uuid'])
+
+
+def search_record(datum, options, session):
+    for searchkey in searchkeys:
+        options.__dict__[searchkey] = None
+    for searchkey in searchkeys & set(datum):
+        options.__dict__[searchkey] = datum[searchkey]
+    return list(list_matching_macs(options, session))
+
+
+def datum_to_attrib(datum):
+    for key in ('serial', 'uuid', 'mac'):
+        try:
+            del datum[key]
+        except KeyError:
+            pass
+    datum['name'] = datum['node']
+    del datum['node']
+    return datum
+
+def import_csv(options, session):
+    nodedata = []
+    with open(options.importfile, 'r') as datasrc:
+        records = csv.reader(datasrc)
+        fields = process_header(next(records))
+        for record in records:
+            currfields = list(fields)
+            nodedatum = {}
+            for datum in record:
+                nodedatum[currfields.pop(0)] = datum
+            if not datum_complete(nodedatum):
+                sys.exit(1)
+            if not search_record(nodedatum, options, session):
+                sys.stderr.write(
+                    "Could not match the following data: " +
+                    repr(nodedatum) + '\n')
+                sys.exit(1)
+            nodedata.append(nodedatum)
+    for datum in nodedata:
+        maclist = search_record(datum, options, session)
+        datum = datum_to_attrib(datum)
+        nodename = datum['name']
+        for res in session.create('/nodes/', datum):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                continue
+            elif 'created' in res:
+                print('Defined ' + res['created'])
+            else:
+                print(repr(res))
+        for mac in maclist:
+            for res in session.update('/discovery/by-mac/{0}'.format(mac),
+                                      {'node': nodename}):
+                if 'error' in res:
+                    sys.stderr.write(res['error'] + '\n')
+                    continue
+                elif 'assigned' in res:
+                    print('Discovered ' + res['assigned'])
+                else:
+                    print(repr(res))
+
+
+def list_discovery(options, session):
+    if options.csv:
+        csv.writer(sys.stdout).writerow(columns)
+    else:
+        print(tabformat.format(*columns))
+        print(tabformat.format(*delimit))
+    for mac in list_matching_macs(options, session):
+        print_disco(options, session, mac)
+
+def clear_discovery(options, session):
+    for mac in list_matching_macs(options, session):
+        for res in session.delete('/discovery/by-mac/{0}'.format(mac)):
+            if 'deleted' in res:
+                print('Cleared info for {0}'.format(res['deleted']))
+            else:
+                print(repr(res))
+
+def list_matching_macs(options, session):
+    path = '/discovery/'
+    if options.model:
+        path += 'by-model/{0}/'.format(options.model)
+    if options.serial:
+        path += 'by-serial/{0}/'.format(options.serial)
+    if options.uuid:
+        path += 'by-uuid/{0}/'.format(options.uuid)
+    if options.type:
+        path += 'by-type/{0}/'.format(options.type)
+    if options.state:
+        if options.state == 'unknown':
+            options.state = 'unidentified'
+        path += 'by-state/{0}/'.format(options.state).lower()
+    if options.mac:
+        path += 'by-mac/{0}'.format(options.mac)
+        result = list(session.read(path))[0]
+        if 'error' in result:
+            return []
+        return [options.mac.replace(':', '-')]
+    else:
+        path += 'by-mac/'
+        return [x['item']['href'] for x in session.read(path)]
+
+def assign_discovery(options, session):
+    abort = False
+    if options.importfile:
+        return import_csv(options, session)
+    if not (options.serial or options.uuid or options.mac):
+        sys.stderr.write(
+            "UUID (-u), serial (-s), or ether address (-e) required for "
+            "assignment\n")
+        abort = True
+    if not options.node:
+        sys.stderr.write("Node (-n) must be specified for assignment\n")
+        abort = True
+    if abort:
+        sys.exit(1)
+    matches = list_matching_macs(options, session)
+    if not matches:
+        sys.stderr.write("No matching discovery candidates found\n")
+        sys.exit(1)
+    for res in session.update('/discovery/by-mac/{0}'.format(matches[0]),
+                   {'node': options.node}):
+        if 'assigned' in res:
+            print('Assigned: {0}'.format(res['assigned']))
+        else:
+            print(repr(res))
+
+
+
+def main():
+    parser = optparse.OptionParser(
+        usage='Usage: %prog [list|assign|rescan|clear] [options]')
+    # -a for 'address' maybe?
+    # order by
+    # show state (discovered or..
+    # nodediscover approve?
+    # flush to clear old data out?  (e.g. no good way to age pxe data)
+    # also delete discovery datum... more targeted
+    # defect: -t lenovo-imm returns all
+    parser.add_option('-m', '--model', dest='model',
+                      help='Operate with nodes matching the specified model '
+                       'number', metavar='MODEL')
+    parser.add_option('-d', '--discoverystate', dest='state',
+                      help='The discovery state of the entries (discovered, '
+                           'identified, and unidentified)')
+    parser.add_option('-s', '--serial', dest='serial',
+                      help='Operate against the system matching the specified '
+                           'serial number', metavar='SERIAL')
+    parser.add_option('-u', '--uuid', dest='uuid',
+                      help='Operate against the system matching the specified '
+                           'UUID', metavar='UUID')
+    parser.add_option('-n', '--node', help='Operate with the given nodename')
+    parser.add_option('-e', '--ethaddr', dest='mac',
+                      help='Operate against the system with the specified MAC '
+                           'address', metavar='MAC')
+    parser.add_option('-t', '--type', dest='type',
+                      help='Operate against the system of the specified type',
+                      metavar='TYPE')
+    parser.add_option('-c', '--csv', dest='csv',
+                      help='Use CSV formatted output', action='store_true')
+    parser.add_option('-i', '--import', dest='importfile',
+                      help='Import bulk assignment data from given CSV file',
+                      metavar='IMPORT.CSV')
+    (options, args) = parser.parse_args()
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
+        parser.print_help()
+        sys.exit(1)
+    session = client.Command()
+    if args[0] == 'list':
+        list_discovery(options, session)
+    if args[0] == 'clear':
+        clear_discovery(options, session)
+    if args[0] == 'assign':
+        assign_discovery(options, session)
+    if args[0] == 'rescan':
+        list(session.update('/discovery/rescan', {'rescan': 'start'}))
+        print("Rescan initiated")
+
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,10 +15,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 from datetime import datetime as dt
+import optparse
 import os
+import signal
 import sys
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -26,34 +33,42 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
+sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange [clear]")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [clear]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
-
+client.check_globbing(noderange)
 deletemode = False
+if len(sys.argv) > 3:
+    argparser.print_help()
+    sys.exit(1)
 if len(sys.argv) == 3:
     if sys.argv[2] == 'clear':
         deletemode = True
+    else:
+        argparser.print_help()
+        sys.exit(1)
 
 session = client.Command()
 exitcode = 0
 
 def format_event(evt):
     retparts = []
-    if 'timestamp' in evt:
+    if 'timestamp' in evt and evt['timestamp'] is not None:
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
-    if 'component_type' in evt:
+    if 'component_type' in evt and evt['component_type'] is not None:
         dscparts.append(evt['component_type'])
     if 'component' in evt and evt['component'] is not None:
         dscparts.append(evt['component'])
-    if 'event' in evt:
+    if 'event' in evt and evt['event'] and evt['event'] is not None:
         evttext = evt['event']
         try:
             if evttext.startswith(evt['component'] + ' - '):
@@ -83,4 +98,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
             if 'events' in thisdata:
                evtdata = thisdata['events']
                for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print '{0}: {1}'.format(node, format_event(evt))

confluent_client/bin/nodefirmware

@@ -0,0 +1,170 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+
+exitcode = 0
+
+def printerror(res, node=None):
+    global exitcode
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+
+
+def printfirm(node, prefix, data):
+    if 'model' in data:
+        prefix += ' ' + data['model']
+    builddesc = []
+    if 'build' in data:
+        builddesc.append(data['build'])
+    if 'date' in data:
+        builddesc.append(data['date'])
+    if 'version' in data:
+        version = data['version']
+        if builddesc:
+            version += ' ({0})'.format(' '.join(builddesc))
+    else:
+        version = ' '.join(builddesc)
+    print('{0}: {1}: {2}'.format(node, prefix, version))
+
+components = ['all']
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog <noderange> [update [--backup <file>]]|[<components>]")
+argparser.add_option('-b', '--backup', action='store_true',
+                     help='Target a backup bank rather than primary')
+(options, args) = argparser.parse_args()
+upfile = None
+try:
+    noderange = args[0]
+    if len(args) > 1:
+        if args[1] == 'update':
+            upfile = args[2]
+        else:
+            components = []
+            for arg in args[1:]:
+                components += arg.split(',')
+
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+
+def get_update_progress(session, url):
+    for res in session.read(url):
+        status = res['phase']
+        percent = res['progress']
+        detail = res['detail']
+        if status == 'error':
+            text = 'error!'
+        else:
+            text = '{0}: {1:3.0f}%'.format(status, percent)
+    return text, status, detail
+
+def update_firmware(session, filename):
+    global exitcode
+    output = sq.ScreenPrinter(noderange, session)
+    nodeurls = {}
+    filename = os.path.abspath(filename)
+    resource = '/noderange/{0}/inventory/firmware/updates/active'.format(
+        noderange)
+    upargs = {'filename': filename}
+    if options.backup:
+        upargs['bank'] = 'backup'
+    noderrs = {}
+    for res in session.create(resource, upargs):
+        if 'created' not in res:
+            for nodename in res.get('databynode', ()):
+                output.set_output(nodename, 'error!')
+                noderrs[nodename] = res['databynode'][nodename].get(
+                    'error', 'Unknown Error')
+            continue
+        watchurl = res['created']
+        currnode = watchurl.split('/')[1]
+        nodeurls[currnode] = '/' + watchurl
+    while nodeurls:
+        for node in list(nodeurls):
+            progress, status, err = get_update_progress(
+                session, nodeurls[node])
+            if status == 'error':
+                exitcode = 1
+                noderrs[node] = err
+            if status in ('error', 'complete', 'pending'):
+                list(session.delete(nodeurls[node]))
+                del nodeurls[node]
+            output.set_output(node, progress)
+        time.sleep(2)
+    allerrnodes = ','.join(noderrs)
+    if noderrs:
+        sys.stderr.write(
+            'Nodes had errors updating ({0})!\n'.format(allerrnodes))
+    for node in noderrs:
+        sys.stderr.write('{0}: {1}\n'.format(node, noderrs[node]))
+
+def show_firmware(session):
+    firmware_shown = False
+    for component in components:
+        for res in session.read(
+                '/noderange/{0}/inventory/firmware/all/{1}'.format(
+                    noderange, component)):
+            printerror(res)
+            if 'databynode' not in res:
+                continue
+            for node in res['databynode']:
+                printerror(res['databynode'][node], node)
+                if 'firmware' not in res['databynode'][node]:
+                    continue
+                for inv in res['databynode'][node]['firmware']:
+                    for prefix in inv:
+                        firmware_shown = True
+                        printfirm(node, prefix, inv[prefix])
+    if not firmware_shown:
+        argparser.print_help()
+
+
+try:
+    session = client.Command()
+    if upfile is None:
+        show_firmware(session)
+    else:
+        update_firmware(session, upfile)
+except KeyboardInterrupt:
+    print('')
+sys.exit(exitcode)

confluent_client/bin/nodegroupattrib

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'alin37'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog [options]   \
+             \n       %prog [options] nodegroup [list of attributes]  \
+             \n       %prog [options] nodegroup nodes=value1,value2   \
+             \n       %prog -e nodegroup <attribute names to set>  \             
+             \n       %prog [options] nodegroup nodes=value1,value2
+             \n ''')
+argparser.add_option('-b', '--blame', action='store_true',
+                     help='Show information about how attributes inherited')
+argparser.add_option('-e', '--environment', action='store_true',
+                     help='Set attributes, but from environment variable of '
+                          'same name')
+argparser.add_option('-c', '--clear', action='store_true',
+                     help='Clear variables')
+(options, args) = argparser.parse_args()
+
+
+#setting minimal output to only output current information
+showtype = 'current'
+requestargs=None
+nodetype="nodegroups"
+
+try:
+    nodegroups = args[0]
+    nodelist = '/{0}/{1}/'.format(nodetype,nodegroups)
+except IndexError:
+    nodelist = '/nodegroups/'
+client.check_globbing(nodegroups)
+session = client.Command()
+exitcode = 0
+
+#Sets attributes
+
+if len(args) > 1:
+    showtype = 'all'
+    if "=" in args[1] and options.clear:
+        print("Can not clear and set at the same time!")
+        argparser.print_help()
+        sys.exit(1)
+    exitcode=client.updateattrib(session,args,nodetype, nodegroups, options)
+    try:
+        # setting user output to what the user inputs
+        if args[1] == 'all':
+            showtype = 'all'
+        elif args[1] == 'current':
+            showtype = 'current'
+
+        requestargs=args[1:]
+    except Exception as e:
+        print str(e)
+
+if exitcode != 0:
+    sys.exit(exitcode)
+
+# Lists all attributes
+if len(args) > 0:
+    # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank.
+    if requestargs is None:
+        showtype = 'current'
+    elif requestargs == []:
+        #showtype already set
+        pass
+    else:
+        try:
+            requestargs.remove('all')
+            requestargs.remove('current')
+        except ValueError:
+            pass
+    exitcode = client.printgroupattributes(session, requestargs, showtype,nodetype, nodegroups, options)
+else:
+    for res in session.read(nodelist):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+        else:
+            print res['item']['href'].replace('/', '')
+
+sys.exit(exitcode)

confluent_client/bin/nodegroupdefine

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange attribute1=value1 attribute2=value,...
+             \n ''')
+(options, args) = argparser.parse_args()
+requestargs=None
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+attribs = {'name': noderange}
+for arg in args[1:]:
+    key, val = arg.split('=')
+    attribs[key] = val
+for r in session.create('/nodegroups/', attribs):
+    if 'error' in r:
+        sys.stderr.write(r['error'] + '\n')
+        exitcode |= 1
+    if 'created' in r:
+        print('{0}: created'.format(r['created']))
+sys.exit(exitcode)

confluent_client/bin/nodegroupremove

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange
+             \n ''')
+(options, args) = argparser.parse_args()
+if len(args) != 1:
+    argparser.print_help()
+    sys.exit(1)
+noderange = args[0]
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+for r in session.delete('/nodegroups/{0}'.format(noderange)):
+    if 'error' in r:
+        sys.stderr.write(r['error'] + '\n')
+        exitcode |= 1
+    if 'deleted' in r:
+        print('{0}: deleted'.format(r['deleted']))
+sys.exit(exitcode)

confluent_client/bin/nodehealth

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,9 +16,15 @@
 # limitations under the License.
 
 import codecs
+import optparse
 import os
+import signal
 import sys
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -28,11 +34,14 @@ import confluent.client as client
 
 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange>\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
+client.check_globbing(noderange)
 
 
 def main():

confluent_client/bin/nodeidentify

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,8 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
+import signal
 import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
@@ -25,15 +31,20 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange> [on|off]\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
-
+client.check_globbing(noderange)
 identifystate = None
 if len(sys.argv) > 2:
     identifystate = sys.argv[2]
+else:
+    argparser.print_help()
+    sys.exit(1)
 session = client.Command()
 exitcode = 0
 sys.exit(

confluent_client/bin/nodeinventory

@@ -0,0 +1,155 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import codecs
+import optparse
+import os
+import re
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+
+filters = []
+
+
+def pretty(text):
+    if text == 'pcislot':
+        return 'PCI slot'
+    if text == 'partnumber':
+        return 'part number'
+    return text
+
+def print_mem_info(node, prefix, meminfo):
+    memdescfmt = '{0}GB PC'
+    if meminfo['memory_type'] == 'DDR3 SDRAM':
+        memdescfmt += '3-{1} '
+    elif meminfo['memory_type'] == 'DDR4 SDRAM':
+        memdescfmt += '4-{1} '
+    else:
+        print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
+        return
+    if meminfo['ecc']:
+        memdescfmt += 'ECC '
+    capacity = meminfo['capacity_mb'] / 1024
+    memdescfmt += meminfo['module_type']
+    memdesc = memdescfmt.format(capacity, meminfo['speed'])
+    print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
+    print('{0}: {1} manufacturer: {2}'.format(
+        node, prefix, meminfo['manufacturer']))
+    print('{0}: {1} model: {2}'.format(node, prefix, meminfo['model']))
+    print('{0}: {1} serial number: {2}'.format(node, prefix,
+                                               meminfo['serial']))
+    print('{0}: {1} manufacture date: {2}'.format(node, prefix,
+                                                  meminfo['manufacture_date']))
+    print('{0}: {1} manufacture location: {2}'.format(
+        node, prefix, meminfo['manufacture_location']))
+
+exitcode = 0
+
+def printerror(res, node=None):
+    global exitcode
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+
+
+url = '/noderange/{0}/inventory/hardware/all/all'
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog <noderange> [serial|model|uuid|mac]")
+(options, args) = argparser.parse_args()
+try:
+    noderange = sys.argv[1]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+if len(args) > 1:
+    if args[1] == 'firm':
+        os.execlp('nodefirmware', 'nodefirmware', noderange)
+    else:
+        url = '/noderange/{0}/inventory/hardware/all/system'
+        for arg in args:
+            for arg in arg.split(','):
+                if arg == 'serial':
+                    filters.append(re.compile('serial number'))
+                elif arg == 'model':
+                    filters.append(re.compile('^model'))
+                    filters.append(re.compile('product name'))
+                elif arg == 'uuid':
+                    filters.append(re.compile('uuid'))
+                elif arg == 'mac':
+                    filters.append(re.compile('mac address'))
+                    url = '/noderange/{0}/inventory/hardware/all/all'
+try:
+    session = client.Command()
+    for res in session.read(url.format(noderange)):
+        printerror(res)
+        if 'databynode' not in res:
+            continue
+        for node in res['databynode']:
+            printerror(res['databynode'][node], node)
+            if 'inventory' not in res['databynode'][node]:
+                continue
+            for inv in res['databynode'][node]['inventory']:
+                prefix = inv['name']
+                if not inv['present']:
+                    if not filters:
+                        print '{0}: {1}: Not Present'.format(node, prefix)
+                    continue
+                info = inv['information']
+                info.pop('board_extra', None)
+                info.pop('oem_parser', None)
+                info.pop('chassis_extra', None)
+                info.pop('product_extra', None)
+                if 'memory_type' in info:
+                    if not filters:
+                        print_mem_info(node, prefix, info)
+                    continue
+                for datum in info:
+                    if filters:
+                        for filter in filters:
+                            if filter.match(datum.lower()):
+                                break
+                        else:
+                            continue
+                    if info[datum] is None:
+                        continue
+                    print(u'{0}: {1} {2}: {3}'.format(node, prefix,
+                                                     pretty(datum),
+                                                     info[datum]))
+except KeyboardInterrupt:
+    print('')
+sys.exit(exitcode)

confluent_client/bin/nodelist

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,76 +15,55 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__author__ = 'jjohnson2'
+__author__ = 'jjohnson2,alin37'
 
 import optparse
 import os
+import signal
 import sys
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
     sys.path.append(path)
 
 import confluent.client as client
-argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [list of attributes")
-argparser.add_option('-b', '--blame', action='store_true',
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog noderange\n"
+          "   or: %prog [options] noderange <nodeattribute>...")
+    argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
-(options, args) = argparser.parse_args()
-try:
-    noderange = args[0]
-    nodelist = '/noderange/{0}/nodes/'.format(noderange)
-except IndexError:
-    nodelist = '/nodes/'
-session = client.Command()
-exitcode = 0
-if len(args) > 1:
-    seenattributes = set([])
-    for res in session.read('/noderange/{0}/attributes/all'.format(noderange)):
-        if 'error' in res:
-            sys.stderr.write(res['error'] + '\n')
-            exitcode = 1
-            continue
-        for node in res['databynode']:
-            for attr in res['databynode'][node]:
-                seenattributes.add(attr)
-                currattr = res['databynode'][node][attr]
-                if attr in args[1:]:
-                    if 'value' in currattr:
-                        if currattr['value'] is not None:
-                           attrout = '{0}: {1}: {2}'.format(
-                                node, attr, currattr['value'])
-                        else:
-                            attrout = '{0}: {1}:'.format(node, attr)
-                    elif 'isset' in currattr:
-                        if currattr['isset']:
-                            attrout = '{0}: {1}: ********'.format(node, attr)
-                        else:
-                            attrout = '{0}: {1}:'.format(node, attr)
-                    if options.blame:
-                        blamedata = []
-                        if 'inheritedfrom' in currattr:
-                            blamedata.append('inherited from group {0}'.format(
-                                currattr['inheritedfrom']
-                            ))
-                        if 'expression' in currattr:
-                            blamedata.append(
-                                'derived from expression "{0}"'.format(
-                                    currattr['expression']))
-                        if blamedata:
-                            attrout += ' (' + ', '.join(blamedata) + ')'
-                    print attrout
-    if not exitcode:
-        for attr in args[1:]:
-            if attr not in seenattributes:
-                sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    try:
+        noderange = args[0]
+        nodelist = '/noderange/{0}/nodes/'.format(noderange)
+    except IndexError:
+        nodelist = '/nodes/'
+    client.check_globbing(noderange)
+    session = client.Command()
+    exitcode = 0
+    showtype='all'
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) > 1:
+        exitcode=client.printattributes(session,requestargs,showtype,nodetype,noderange,options)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
-else:
-    for res in session.read(nodelist):
-        if 'error' in res:
-            sys.stderr.write(res['error'] + '\n')
-            exitcode = 1
-        else:
-            print res['item']['href'].replace('/', '')
-sys.exit(exitcode)
+            else:
+                print res['item']['href'].replace('/', '')
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodemedia

@@ -0,0 +1,190 @@
+#!/usr/bin/python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2018 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+import time
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+
+exitcode = 0
+
+
+def get_update_progress(session, url):
+    for res in session.read(url):
+        status = res['phase']
+        percent = res['progress']
+        detail = res['detail']
+        if status == 'error':
+            text = 'error!'
+        else:
+            text = '{0}: {1:3.0f}%'.format(status, percent)
+    return text, status, detail
+
+
+def printerror(res, node=None):
+    global exitcode
+    if 'errorcode' in res:
+        exitcode = res['errorcode']
+    if 'error' in res:
+        if node:
+            sys.stderr.write('{0}: {1}\n'.format(node, res['error']))
+        else:
+            sys.stderr.write('{0}\n'.format(res['error']))
+        if 'errorcode' not in res:
+            exitcode = 1
+    for node in res.get('databynode', ()):
+        printerror(res['databynode'][node], node)
+
+
+def attach_media(noderange, media):
+    global exitcode
+    session = client.Command()
+    if ':' not in media:
+        sys.stderr.write('Full URL required for attach\n')
+        sys.exit(1)
+    resource = '/noderange/{0}/media/attach'.format(noderange)
+    for res in session.update(resource, {'url': media}):
+        printerror(res)
+    list_media(noderange, media)
+
+
+def list_media(noderange, media):
+    session = client.Command()
+    resource = '/noderange/{0}/media/current'.format(noderange)
+    for res in session.read(resource):
+        printerror(res)
+        for node in res.get('databynode', []):
+            url = res['databynode'][node].get('url', None)
+            name =  res['databynode'][node].get('name', None)
+            if url and not res['databynode'][node].get('secure', False):
+                name += ' (insecure)'
+            if not name:
+                continue
+            print('{0}: {1}'.format(node, url + '/' + name if url else name))
+
+
+def detach_media(noderange, media):
+    global exitcode
+    session = client.Command()
+    resource = '/noderange/{0}/media/detach'.format(noderange)
+    for res in session.update(resource, {'detachall': 1}):
+        printerror(res)
+
+
+def upload_media(noderange, media):
+    global exitcode
+    if not os.path.exists(media):
+        sys.stderr.write('Unable to locate requested file {0}\n'.format(
+            media))
+        sys.exit(404)
+    session = client.Command()
+    output = sq.ScreenPrinter(noderange, session)
+    filename = os.path.abspath(media)
+    resource = '/noderange/{0}/media/uploads/'.format(noderange)
+    upargs = {'filename': filename}
+    noderrs = {}
+    nodeurls = {}
+    for res in session.create(resource, upargs):
+        if 'created' not in res:
+            for nodename in res.get('databynode', ()):
+                output.set_output(nodename, 'error!')
+                noderrs[nodename] = res['databynode'][nodename].get(
+                    'error', 'Unknown Error')
+            continue
+        watchurl = res['created']
+        currnode = watchurl.split('/')[1]
+        nodeurls[currnode] = '/' + watchurl
+    while nodeurls:
+        for node in list(nodeurls):
+            progress, status, err = get_update_progress(
+                session, nodeurls[node])
+            if status == 'error':
+                exitcode = 1
+                noderrs[node] = err
+            if status in ('error', 'complete', 'pending'):
+                list(session.delete(nodeurls[node]))
+                del nodeurls[node]
+            output.set_output(node, progress)
+        time.sleep(2)
+    allerrnodes = ','.join(noderrs)
+    if noderrs:
+        sys.stderr.write(
+            'Nodes had errors receiving media ({0})!\n'.format(allerrnodes))
+    for node in noderrs:
+        sys.stderr.write('{0}: {1}\n'.format(node, noderrs[node]))
+    list_media(noderange, media)
+
+funmap = {
+    'upload': upload_media,
+    'attach': attach_media,
+    'detachall': detach_media,
+    'list': list_media,
+}
+
+
+class OptParser(optparse.OptionParser):
+
+    def format_epilog(self, formatter):
+        return self.expand_prog_name(self.epilog)
+
+def main():
+    argparser = OptParser(
+        usage="Usage: %prog <noderange> [list|upload|attach|detachall] "
+              "<filename>|all|<url>",
+        epilog='\nupload will take the specified file and upload it to the '
+               'BMC.\n\n'
+               'attach will instruct the BMC to connect a remote media to the '
+               'specified url.\n\ndetachall will remove *ALL* uploaded and '
+               'attached urls from the BMC\n\nlist shows currently mounted '
+               'media.\n\nSee `man %prog` for more info.\n')
+    (options, args) = argparser.parse_args()
+    media = None
+    try:
+        noderange = args[0]
+        operation = args[1]
+        arglength = 2
+        if operation in ('attach', 'upload'):
+            media = args[2]
+            arglength = 3
+        if len(args) > arglength:
+            argparser.print_help()
+            sys.exit(1)
+    except IndexError:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(noderange)
+    try:
+        handler = funmap[operation]
+    except KeyError:
+        argparser.print_help()
+        sys.exit(1)
+    handler(noderange, media)
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodepower

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,9 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
+import signal
 import sys
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -25,15 +31,19 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange "
+          "([status|on|off|shutdown|boot|reset])")
+argparser.add_option('-p', '--showprevious', dest='previous',
+                     action='store_true', default=False,
+                     help='Show previous power state')
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> ([status|on|off|shutdown|boot|reset]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
-
+client.check_globbing(noderange)
 setstate = None
 if len(sys.argv) > 2:
     if setstate == 'softoff':
@@ -41,7 +51,25 @@ if len(sys.argv) > 2:
     elif not sys.argv[2] in ('stat', 'state', 'status'):
         setstate = sys.argv[2]
 
+if setstate not in (None, 'on', 'off', 'shutdown', 'boot', 'reset'):
+    argparser.print_help()
+    sys.exit(1)
 session = client.Command()
 exitcode = 0
-sys.exit(
-    session.simple_noderange_command(noderange, '/power/state', setstate))
+session.add_precede_key('oldstate')
+
+if options.previous:
+    # get previous states
+    prev = {}
+    for rsp in session.read("/noderange/{0}/power/state".format(noderange)):
+        # gets previous (current) states
+
+        databynode = rsp["databynode"]
+
+        for node in databynode:
+            prev[node] = databynode[node]["state"]["value"]
+
+    # add dictionary to session
+    session.add_precede_dict(prev)
+
+sys.exit(session.simple_noderange_command(noderange, '/power/state', setstate))

confluent_client/bin/noderemove

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(
+    usage='''\n       %prog noderange
+             \n ''')
+(options, args) = argparser.parse_args()
+if len(args) != 1:
+    argparser.print_help()
+    sys.exit(1)
+noderange = args[0]
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+for r in session.delete('/noderange/{0}'.format(noderange)):
+    if 'error' in r:
+        sys.stderr.write(r['error'] + '\n')
+        exitcode |= 1
+    if 'deleted' in r:
+        print('{0}: deleted'.format(r['deleted']))
+sys.exit(exitcode)

confluent_client/bin/nodereseat

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+session = client.Command()
+exitcode = 0
+
+errorNodes = set([])
+
+success = session.simple_noderange_command(noderange, 'power/reseat', 'reseat', key='reseat', errnodes=errorNodes)   # = 0 if successful
+
+# Determine which nodes were successful and print them
+
+allNodes = set([])
+
+for node in session.read('/noderange/{0}/nodes/'.format(noderange)):
+    allNodes.add(node['item']['href'].replace("/", ""))
+
+goodNodes = allNodes - errorNodes
+
+for node in goodNodes:
+    print node + ": Reseat successful"
+
+
+sys.exit(success)

confluent_client/bin/noderun

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def run():
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2:
+        argparser.print_help()
+        sys.exit(1)
+    concurrentprocs = options.count
+    client.check_globbing(args[0])
+    c = client.Command()
+    cmdstr = " ".join(args[1:])
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                        {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value'].encode('utf-8')
+            cmdv = shlex.split(cmd)
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0):
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/nodesensors

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,11 +16,17 @@
 # limitations under the License.
 
 import csv
+import datetime
 import optparse
 import os
+import signal
 import sys
 import time
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -30,6 +36,7 @@ import confluent.client as client
 
 sensorcollections = {
     'all': 'sensors/hardware/all/all',
+    'energy': 'sensors/hardware/energy/all',
     'temperature': 'sensors/hardware/temperature/all',
     'temp': 'sensors/hardware/temperature/all',
     'power': 'sensors/hardware/power/all',
@@ -39,13 +46,15 @@ sensorcollections = {
 
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [sensor(s)")
-argparser.add_option('-i', '--interval', type='int',
+    usage="Usage: %prog [options] noderange ([sensor(s)])")
+argparser.add_option('-i', '--interval', type='float',
                      help='Interval to do repeated samples over')
 argparser.add_option('-n', '--numreadings', type='int',
                      help='Number of readings to gather')
 argparser.add_option('-c', '--csv', action='store_true',
                      help='Output in CSV format')
+argparser.add_option('-s', '--skipnumberless', action='store_true',
+                     help='Output in CSV format')
 (options, args) = argparser.parse_args()
 repeatmode = False
 if options.interval:
@@ -56,8 +65,9 @@ if options.numreadings:
         options.interval = 1
 try:
     noderange = args[0]
+    client.check_globbing(noderange)
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 sensors = []
 for sensorgroup in args[1:]:
@@ -70,7 +80,6 @@ for sensorgroup in args[1:]:
                 sensors.append('sensors/hardware/all/' + sensor)
 if not sensors:
     sensors = ['sensors/hardware/all/all']
-
 session = client.Command()
 exitcode = 0
 sensorheaders = {}
@@ -101,6 +110,8 @@ def sensorpass(showout=True, appendtime=False):
                 if 'sensors' not in reading[node]:
                     continue
                 for sensedata in reading[node]['sensors']:
+                    if sensedata['value'] is None and options.skipnumberless:
+                        continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
                             sensedata['states'].remove(redundant_state)
@@ -115,6 +126,8 @@ def sensorpass(showout=True, appendtime=False):
                     if showout:
                         if sensedata['value'] is None:
                             showval = ''
+                        elif isinstance(sensedata['value'], float):
+                            showval = u' {0:.5f} '.format(sensedata['value'])
                         else:
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):
@@ -136,13 +149,19 @@ def sensorpass(showout=True, appendtime=False):
                                 '%Y-%m-%dT%H:%M:%S')
                         print(u'{0}: {1}:{2}'.format(
                             node, sensedata['name'], showval).encode('utf-8'))
+                        sys.stdout.flush()
     return resultdata
 
 
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            if showtime.is_integer():
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            else:
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
+                           str(datetime.datetime.now().microsecond//1000),
+                           nodekey]
         else:
             rowdata = [nodekey]
         for sensorkey in orderedsensors:
@@ -193,7 +212,8 @@ def main():
             nextstart = os.times()[4] + options.interval
             resdata = sensorpass(linebyline, True)
             if options.csv:
-                format_csv(csvwriter, orderedsensors, resdata)
+                format_csv(csvwriter, orderedsensors, resdata,
+                           showtime=options.interval)
             if options.numreadings:
                 options.numreadings -= 1
                 if options.numreadings <= 0:

confluent_client/bin/nodesetboot

@@ -15,9 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
+import signal
 import sys
 
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -25,13 +31,26 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
+argparser = optparse.OptionParser(
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
+argparser.add_option('-b', '--bios', dest='biosmode',
+                     action='store_true', default=False,
+                     help='Request BIOS style boot (rather than UEFI)')
+argparser.add_option('-p', '--persist', dest='persist', action='store_true',
+                     default=False,
+                     help='Request the boot device be persistent rather than '
+                          'one time')
+argparser.add_option('-u', '--uefi', dest='uefi', action='store_true',
+                     default=True,
+                     help='Request UEFI style boot (rather than BIOS)')
+
+(options, args) = argparser.parse_args()
 
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
+    client.check_globbing(noderange)
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [default|cd|network|setup|hd]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 bootdev = None
 if len(sys.argv) > 2:
@@ -40,5 +59,10 @@ if len(sys.argv) > 2:
         bootdev = 'network'
 session = client.Command()
 exitcode = 0
-sys.exit(
-    session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev))
+if options.biosmode:
+    bootmode = 'bios'
+else:
+    bootmode = 'uefi'
+sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
+                                     bootmode=bootmode,
+                                     persistent=options.persist))

confluent_client/bin/nodeshell

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.sortutil as sortutil
+
+
+def run():
+
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog noderange commandexpression",
+        epilog="Expressions are the same as in attributes, e.g. "
+               "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of commands to run at a time')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2:
+        argparser.print_help()
+        sys.exit(1)
+    client.check_globbing(args[0])
+    concurrentprocs = options.count
+    c = client.Command()
+    cmdstr = " ".join(args[1:])
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                        {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value'].encode('utf-8')
+            cmdv = ['ssh', node, cmd]
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    while all:
+        pernodeout = {}
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0):
+                data = r.readline()
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = []
+                        pernodeout[node].append(data)
+                    else:
+                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        sys.stderr.flush()
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        for node in sortutil.natural_sort(pernodeout):
+            for line in pernodeout[node]:
+                sys.stdout.write('{0}: {1}'.format(node, line))
+        sys.stdout.flush()
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/confluent/client.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import anydbm as dbm
 import errno
 import hashlib
 import os
+import shlex
 import socket
 import ssl
 import sys
@@ -26,6 +27,15 @@ import confluent.tlvdata as tlvdata
 
 SO_PASSCRED = 16
 
+_attraliases = {
+    'bmc': 'hardwaremanagement.manager',
+    'bmcuser': 'secret.hardwaremanagementuser',
+    'bmcpass': 'secret.hardwaremanagementpassword',
+}
+
+def cprint(txt):
+    print(txt)
+    sys.stdout.flush()
 
 def _parseserver(string):
     if ']:' in string:
@@ -42,9 +52,11 @@ def _parseserver(string):
 
 
 class Command(object):
-
     def __init__(self, server=None):
+        self._prevdict = None
+        self._prevkeyname = None
         self.connection = None
+        self._currnoderange = None
         if server is None:
             if 'CONFLUENT_HOST' in os.environ:
                 self.serverloc = os.environ['CONFLUENT_HOST']
@@ -74,8 +86,16 @@ class Command(object):
         if authdata['authpassed'] == 1:
             self.authenticated = True
 
-    def handle_results(self, ikey, rc, res):
+    def add_precede_key(self, keyname):
+        self._prevkeyname = keyname
+
+    def add_precede_dict(self, dict):
+        self._prevdict = dict
+
+    def handle_results(self, ikey, rc, res, errnodes=None):
         if 'error' in res:
+            if errnodes is not None:
+                errnodes.add(self._currnoderange)
             sys.stderr.write('Error: {0}\n'.format(res['error']))
             if 'errorcode' in res:
                 return res['errorcode']
@@ -86,6 +106,8 @@ class Command(object):
         res = res['databynode']
         for node in res:
             if 'error' in res[node]:
+                if errnodes is not None:
+                    errnodes.add(node)
                 sys.stderr.write('{0}: Error: {1}\n'.format(
                     node, res[node]['error']))
                 if 'errorcode' in res[node]:
@@ -93,12 +115,26 @@ class Command(object):
                 else:
                     rc |= 1
             elif ikey in res[node]:
-                print('{0}: {1}'.format(node, res[node][ikey]['value']))
+                if 'value' in res[node][ikey]:
+                    val = res[node][ikey]['value']
+                elif 'isset' in res[node][ikey]:
+                    val = '********' if res[node][ikey] else ''
+                else:
+                    val = repr(res[node][ikey])
+                if self._prevkeyname and self._prevkeyname in res[node]:
+                    cprint('{0}: {2}->{1}'.format(
+                        node, val, res[node][self._prevkeyname]['value']))
+                elif self._prevdict and node in self._prevdict:
+                    cprint('{0}: {2}->{1}'.format(
+                        node, val, self._prevdict[node]))
+                else:
+                    cprint('{0}: {1}'.format(node, val))
         return rc
 
     def simple_noderange_command(self, noderange, resource, input=None,
-                                 key=None):
+                                 key=None, errnodes=None, **kwargs):
         try:
+            self._currnoderange = noderange
             rc = 0
             if resource[0] == '/':
                 resource = resource[1:]
@@ -110,28 +146,60 @@ class Command(object):
             if input is None:
                 for res in self.read('/noderange/{0}/{1}'.format(
                         noderange, resource)):
+                    rc = self.handle_results(ikey, rc, res, errnodes)
+            else:
+                kwargs[ikey] = input
+                for res in self.update('/noderange/{0}/{1}'.format(
+                        noderange, resource), kwargs):
+                    rc = self.handle_results(ikey, rc, res, errnodes)
+            self._currnoderange = None
+            return rc
+        except KeyboardInterrupt:
+            cprint('')
+            return 0
+
+    def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs):
+        try:
+            rc = 0
+            if resource[0] == '/':
+                resource = resource[1:]
+            # The implicit key is the resource basename
+            if key is None:
+                ikey = resource.rpartition('/')[-1]
+            else:
+                ikey = key
+            if input is None:
+                for res in self.read('/nodegroups/{0}/{1}'.format(
+                        noderange, resource)):
                     rc = self.handle_results(ikey, rc, res)
             else:
-                for res in self.update('/noderange/{0}/{1}'.format(
-                        noderange, resource), {ikey: input}):
+                kwargs[ikey] = input
+                for res in self.update('/nodegroups/{0}/{1}'.format(
+                        noderange, resource), kwargs):
                     rc = self.handle_results(ikey, rc, res)
             return rc
         except KeyboardInterrupt:
-            print('')
+            cprint('')
             return 0
 
-
-
     def read(self, path, parameters=None):
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
         return send_request('retrieve', path, self.connection, parameters)
 
     def update(self, path, parameters=None):
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
         return send_request('update', path, self.connection, parameters)
 
     def create(self, path, parameters=None):
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
         return send_request('create', path, self.connection, parameters)
 
     def delete(self, path, parameters=None):
+        if not self.authenticated:
+            raise Exception('Unauthenticated')
         return send_request('delete', path, self.connection, parameters)
 
     def _connect_unix(self):
@@ -183,7 +251,7 @@ class Command(object):
         if knownhosts:
             certdata = self.connection.getpeercert(binary_form=True)
             fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
-            hostid = '@'.join((port,server))
+            hostid = '@'.join((port, server))
             khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
             if hostid in khf:
                 if fingerprint == khf[hostid]:
@@ -193,11 +261,10 @@ class Command(object):
                         "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
-            print 'Adding new key for %s:%s' % (server, port)
+            cprint('Adding new key for %s:%s' % (server, port))
             khf[hostid] = fingerprint
 
 
-
 def send_request(operation, path, server, parameters=None):
     """This function iterates over all the responses
     received from the server.
@@ -214,8 +281,287 @@ def send_request(operation, path, server, parameters=None):
     tlvdata.send(server, payload)
     result = tlvdata.recv(server)
     while '_requestdone' not in result:
-        yield result
+        try:
+            yield result
+        except GeneratorExit:
+            while '_requestdone' not in result:
+                result = tlvdata.recv(server)
+            raise
         result = tlvdata.recv(server)
 
 
+def attrrequested(attr, attrlist, seenattributes):
+    for candidate in attrlist:
+        truename = candidate
+        if candidate.startswith('hm'):
+            candidate = candidate.replace('hm', 'hardwaremanagement', 1)
+        if candidate in _attraliases:
+            candidate = _attraliases[candidate]
+        if candidate.lower() == attr.lower():
+            seenattributes.add(truename)
+            return True
+        elif attr.lower().startswith(candidate.lower() + '.'):
+            seenattributes.add(truename)
+            return True
+    return False
 
+
+def printattributes(session, requestargs, showtype, nodetype, noderange, options):
+    path = '/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)
+    return print_attrib_path(path, session, requestargs, options)
+
+
+def print_attrib_path(path, session, requestargs, options, rename=None):
+    exitcode = 0
+    seenattributes = set([])
+    for res in session.read(path):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for node in sorted(res['databynode']):
+            for attr, val in sorted(
+                    res['databynode'][node].items(),
+                    key=lambda (k, v): v.get('sortid', k) if isinstance(v, dict) else k):
+                seenattributes.add(attr)
+                if rename:
+                    printattr = rename.get(attr, attr)
+                else:
+                    printattr = attr
+                currattr = res['databynode'][node][attr]
+                if show_attr(attr, requestargs, seenattributes, options):
+                    if 'value' in currattr:
+                        if currattr['value'] is not None:
+                            val = currattr['value']
+                            if isinstance(val, list):
+                                val = ','.join(val)
+                            attrout = '{0}: {1}: {2}'.format(
+                                node, printattr, val)
+                        else:
+                            attrout = '{0}: {1}:'.format(node, printattr)
+                    elif 'isset' in currattr:
+                        if currattr['isset']:
+                            attrout = '{0}: {1}: ********'.format(node,
+                                                                  printattr)
+                        else:
+                            attrout = '{0}: {1}:'.format(node, printattr)
+                    elif 'broken' in currattr:
+                        attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                                  '{2}'.format(node, printattr,
+                                               currattr['broken'])
+                    elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                        attrout = '{0}: {1}: {2}'.format(node, attr, ','.join(map(str, currattr)))
+                    elif isinstance(currattr, dict):
+                        dictout = []
+                        for k, v in currattr.items:
+                            dictout.append("{0}={1}".format(k, v))
+                        attrout = '{0}: {1}: {2}'.format(node, printattr, ','.join(map(str, dictout)))
+                    else:
+                        cprint("CODE ERROR" + repr(attr))
+                    try:
+                        blame = options.blame
+                    except AttributeError:
+                        blame = False
+                    if blame or 'broken' in currattr:
+                        blamedata = []
+                        if 'inheritedfrom' in currattr:
+                            blamedata.append('inherited from group {0}'.format(
+                                currattr['inheritedfrom']
+                            ))
+                        if 'expression' in currattr:
+                            blamedata.append(
+                                'derived from expression "{0}"'.format(
+                                    currattr['expression']))
+                        if blamedata:
+                            attrout += ' (' + ', '.join(blamedata) + ')'
+                    try:
+                        comparedefault = options.comparedefault
+                    except AttributeError:
+                        comparedefault = False
+                    if comparedefault:
+                        try:
+                            exclude = options.exclude
+                        except AttributeError:
+                            exclude = False
+                        if ((requestargs and not exclude) or
+                                (currattr.get('default', None) is not None and
+                                currattr.get('value', None) is not None and
+                                currattr['value'] != currattr['default'])):
+                            cprint('{0}: {1}: {2} (Default: {3})'.format(
+                                node, printattr, currattr['value'],
+                                currattr['default']))
+                    else:
+
+                        try:
+                            details = options.detail
+                        except AttributeError:
+                            details = False
+                        if details:
+                            if currattr.get('help', None):
+                                attrout += ' (Help: {0})'.format(
+                                    currattr['help'].encode('utf-8'))
+                            if currattr.get('possible', None):
+                                attrout += ' (Choices: {0})'.format(
+                                    ','.join(currattr['possible']))
+                        cprint(attrout)
+    if not exitcode:
+        if requestargs:
+            for attr in requestargs:
+                if attr not in seenattributes:
+                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+                    exitcode = 1
+    return exitcode
+
+
+def show_attr(attr, requestargs, seenattributes, options):
+    try:
+        reverse = options.exclude
+    except AttributeError:
+        reverse = False
+    if requestargs is None or requestargs == []:
+        return True
+    processattr = attrrequested(attr, requestargs, seenattributes)
+    if reverse:
+        processattr = not processattr
+    return processattr
+
+
+def printgroupattributes(session, requestargs, showtype, nodetype, noderange, options):
+    exitcode = 0
+    seenattributes = set([])
+    for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)):
+        if 'error' in res:
+            sys.stderr.write(res['error'] + '\n')
+            exitcode = 1
+            continue
+        for attr in res:
+            seenattributes.add(attr)
+            currattr = res[attr]
+            if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)):
+                if 'value' in currattr:
+                    if currattr['value'] is not None:
+                        attrout = '{0}: {1}: {2}'.format(
+                            noderange, attr, currattr['value'])
+                    else:
+                        attrout = '{0}: {1}:'.format(noderange, attr)
+                elif 'isset' in currattr:
+                    if currattr['isset']:
+                        attrout = '{0}: {1}: ********'.format(noderange, attr)
+                    else:
+                        attrout = '{0}: {1}:'.format(noderange, attr)
+                elif 'broken' in currattr:
+                    attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \
+                              '{2}'.format(noderange, attr,
+                                           currattr['broken'])
+                elif 'expression' in currattr:
+                    attrout = '{0}: {1}:  (will derive from expression {2})'.format(noderange, attr, currattr['expression'])
+                elif isinstance(currattr, list) or isinstance(currattr, tuple):
+                    attrout = '{0}: {1}: {2}'.format(noderange, attr, ','.join(map(str, currattr)))
+                elif isinstance(currattr, dict):
+                    dictout = []
+                    for k, v in currattr.items:
+                        dictout.append("{0}={1}".format(k, v))
+                    attrout = '{0}: {1}: {2}'.format(noderange, attr, ','.join(map(str, dictout)))
+                else:
+                    cprint("CODE ERROR" + repr(attr))
+                cprint(attrout)
+    if not exitcode:
+        if requestargs:
+            for attr in requestargs:
+                if attr not in seenattributes:
+                    sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr))
+                    exitcode = 1
+    return exitcode
+
+def updateattrib(session, updateargs, nodetype, noderange, options):
+    # update attribute
+    exitcode = 0
+    if options.clear:
+        targpath = '/{0}/{1}/attributes/all'.format(nodetype, noderange)
+        keydata = {}
+        for attrib in updateargs[1:]:
+            keydata[attrib] = None
+        for res in session.update(targpath, keydata):
+            if 'error' in res:
+                if 'errorcode' in res:
+                    exitcode = res['errorcode']
+                sys.stderr.write('Error: ' + res['error'] + '\n')
+        sys.exit(exitcode)
+    elif hasattr(options, 'environment') and options.environment:
+        for key in updateargs[1:]:
+            key = key.replace('.', '_')
+            value = os.environ.get(
+                key, os.environ[key.upper()])
+            # Let's do one pass to make sure that there's not a usage problem
+        for key in updateargs[1:]:
+            key = key.replace('.', '_')
+            value = os.environ.get(
+                key, os.environ[key.upper()])
+            if (nodetype == "nodegroups"):
+                exitcode = session.simple_nodegroups_command(noderange,
+                                                             'attributes/all',
+                                                             value, key)
+            else:
+                exitcode = session.simple_noderange_command(noderange,
+                                                            'attributes/all',
+                                                            value, key)
+        sys.exit(exitcode)
+    else:
+        if "=" in updateargs[1]:
+            try:
+                for val in updateargs[1:]:
+                    val = val.split('=')
+                    if val[0][-1] in (',', '-', '^'):
+                        key = val[0][:-1]
+                        if val[0][-1] == ',':
+                            value = {'prepend': val[1]}
+                        elif val[0][-1] in ('-', '^'):
+                            value = {'remove': val[1]}
+                    else:
+                        key = val[0]
+                        value = val[1]
+                    if (nodetype == "nodegroups"):
+                        exitcode =  session.simple_nodegroups_command(noderange, 'attributes/all',
+                                                                     value, key)
+                    else:
+                        exitcode = session.simple_noderange_command(noderange, 'attributes/all',
+                                                                    value, key)
+            except:
+                sys.stderr.write('Error: {0} not a valid expression\n'.format(str(updateargs[1:])))
+                exitcode = 1
+            sys.exit(exitcode)
+    return exitcode
+
+
+# So we try to prevent bad things from happening when globbing
+# We tried to head this off at the shell, but the various solutions would end
+# up breaking the shell in various ways (breaking pipe capability if using
+# DEBUG, breaking globbing if in pipe, etc)
+# Then we tried to parse the original commandline instead, however shlex isn't
+# going to parse full bourne language (e.g. knowing that '|' and '>' and
+# a world of other things would not be in our command line
+# so finally, just make sure the noderange appears verbatim in the command line
+# if we glob to something, then bash will change noderange and this should
+# detect it and save the user from tragedy
+def check_globbing(noderange):
+    if not os.path.exists(noderange):
+        return True
+    rawargs = os.environ.get('CURRENT_CMDLINE', None)
+    if rawargs:
+        rawargs = shlex.split(rawargs)
+        for arg in rawargs:
+            if arg.startswith('$'):
+                arg = arg[1:]
+                if arg.endswith(';'):
+                    arg = arg[:-1]
+                arg = os.environ.get(arg, '$' + arg)
+            if arg.startswith(noderange):
+                break
+        else:
+            sys.stderr.write(
+                'Shell glob conflict detected, specified target "{0}" '
+                'not in command line, but is a file.  You can use "set -f" in '
+                'bash or change directories such that there is no filename '
+                'that would conflict.'
+                '\n'.format(noderange))
+            sys.exit(1)

confluent_client/confluent/screensqueeze.py

@@ -0,0 +1,85 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fcntl
+import sys
+import struct
+import termios
+
+def get_screenwidth():
+    return struct.unpack('hh', fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ,
+                                           b'....'))[1]
+class ScreenPrinter(object):
+
+    def __init__(self, noderange, client, textlen=4):
+        self.squeeze = sys.stdout.isatty()
+        self.textlen = textlen
+        self.noderange = noderange
+        self.client = client
+        self.nodeoutput = {}
+        self.nodelist = []
+        maxlen = 0
+        for ans in self.client.read('/noderange/{0}/nodes/'.format(noderange)):
+            if 'error' in ans:
+                sys.stderr.write(ans['error'])
+                continue
+            nodename = ans['item']['href'][:-1]
+            if len(nodename) > maxlen:
+                maxlen = len(nodename)
+            self.nodelist.append(nodename)
+            self.nodeoutput[nodename] = ''
+        self.nodenamelen = maxlen
+        self.textlen = textlen
+        self.fieldwidth = maxlen + textlen + 1  # 1 for column
+
+    def set_output(self, node, text):
+        self.nodeoutput[node] = text
+        if len(text) >= self.textlen:
+            self.textlen = len(text) + 1
+            self.fieldwidth = self.textlen + self.nodenamelen + 1
+        self.drawscreen()
+
+    def drawscreen(self):
+        if self.squeeze:
+            currwidth = get_screenwidth()
+            numfields = currwidth // self.fieldwidth
+            fieldformat = '{{0:>{0}}}:{{1:{1}}}'.format(self.nodenamelen,
+                                                        self.textlen)
+            sys.stdout.write('\x1b[2J\x1b[;H')  # clear screen
+        else:
+            numfields = 1
+            fieldformat = '{0}: {1}'
+        currfields = 0
+        for node in self.nodelist:
+            if currfields >= numfields:
+                sys.stdout.write('\n')
+                currfields = 1
+            else:
+                currfields += 1
+            sys.stdout.write(fieldformat.format(node, self.nodeoutput[node]))
+        sys.stdout.write('\n')
+        sys.stdout.flush()
+
+
+
+
+
+if __name__ == '__main__':
+    import confluent.client as client
+    c = client.Command()
+    p = ScreenPrinter('n1-n13', c)
+    p.set_output('n3', 'Upload: 67%')
+
+
+

confluent_client/confluent/sortutil.py

@@ -0,0 +1,43 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2014 IBM Corporation
+# Copyright 2015-2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+numregex = re.compile('([0-9]+)')
+
+
+def naturalize_string(key):
+    """Analyzes string in a human way to enable natural sort
+
+    :param key: The node name to analyze
+    :returns: A structure that can be consumed by 'sorted'
+    """
+    return [int(text) if text.isdigit() else text.lower()
+            for text in re.split(numregex, key)]
+
+
+def natural_sort(iterable):
+    """Return a sort using natural sort if possible
+
+    :param iterable:
+    :return:
+    """
+    try:
+        return sorted(iterable, key=naturalize_string)
+    except TypeError:
+        # The natural sort attempt failed, fallback to ascii sort
+        return sorted(iterable)

confluent_client/confluent/textgroup.py

@@ -0,0 +1,217 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import difflib
+import re
+import sys
+
+try:
+    range = xrange
+except NameError:
+    pass
+
+
+
+numregex = re.compile('([0-9]+)')
+def humanify_nodename(nodename):
+    """Analyzes nodename in a human way to enable natural sort
+
+    :param nodename: The node name to analyze
+    :returns: A structure that can be consumed by 'sorted'
+    """
+    return [int(text) if text.isdigit() else text.lower()
+            for text in re.split(numregex, nodename)]
+
+
+def _colorize_line(orig, mask):
+    highlighted = False
+    newline = orig[0]
+    for i in range(1, len(orig)):
+        if i > len(mask) - 1:
+            if highlighted:
+                newline += '\x1b[0m'
+            newline += orig[i:]
+            break
+        if highlighted and mask[i] == ' ':
+            highlighted = False
+            newline += '\x1b[0m'
+        elif not highlighted and mask[i] != ' ':
+            highlighted = True
+            newline += '\x1b[31m'
+        newline += orig[i]
+    newline += '\x1b[0m'
+    return newline
+
+
+def near_diff(idx, diffdata):
+    maxidx = idx + 3
+    if maxidx > len(diffdata) -1:
+        maxidx = len(diffdata) - 1
+    idx = idx - 3
+    if idx < 0:
+        idx = 0
+    while idx <= maxidx:
+        if len(diffdata[idx]) > 0 and diffdata[idx][0] in ('+', '-'):
+            return True
+        idx += 1
+    return False
+
+
+def colordiff(first, second):
+    diffdata = list(difflib.ndiff(first, second))
+    quiet = True
+    for i in range(len(diffdata)):
+        if i < len(diffdata) - 1 and diffdata[i + 1].startswith('?'):
+            if quiet:
+                quiet = False
+                yield '@@'
+            yield _colorize_line(diffdata[i], diffdata[i + 1])
+        elif not diffdata[i].startswith('?') and near_diff(i, diffdata):
+            if quiet:
+                quiet = False
+                yield '@@'
+            yield diffdata[i]
+        elif not diffdata[i].startswith('?'):
+            quiet = True
+
+
+
+class GroupedData(object):
+    '''A post processor to sort and compare per-node data
+
+    :param confluentconnection: If given, will attempt to use the connection to abbreviate noderanges
+    '''
+
+    def __init__(self, confluentconnection=None):
+        self.bynode = {}
+        self.byoutput = {}
+        self.header = {}
+        self.client = confluentconnection
+
+    def generate_byoutput(self):
+        self.byoutput = {}
+        for n in self.bynode:
+            output = '\n'.join(self.bynode[n])
+            if output not in self.byoutput:
+                self.byoutput[output] = set([n])
+            else:
+                self.byoutput[output].add(n)
+
+    def add_line(self, node, line):
+        if node not in self.bynode:
+            self.bynode[node] = [line]
+        else:
+            self.bynode[node].append(line)
+
+    def get_group_text(self, nodes):
+        if self.client:
+            headerkey = ','.join(sorted(nodes))
+            if headerkey in self.header:
+                return self.header[headerkey]
+            noderange = ''
+            for reply in self.client.create('/noderange//abbreviate',
+                                            {'nodes': sorted(nodes)}):
+                noderange = reply['noderange']
+            self.header[headerkey] = noderange
+            return noderange
+        else:
+            return ','.join(sorted(nodes, key=humanify_nodename))
+
+    def print_all(self, output=sys.stdout, skipmodal=False, reverse=False,
+                  count=False):
+        self.generate_byoutput()
+        modaloutput = None
+        ismodal = True
+
+        if reverse:
+            outdatalist = sorted(
+                self.byoutput, key=lambda x: [len(self.byoutput[x]),
+                                              humanify_nodename(
+                                                  self.get_group_text(
+                                                      self.byoutput[x]
+                                                  ))])
+        else:
+            outdatalist = sorted(
+                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                                              humanify_nodename(
+                                                  self.get_group_text(
+                                                      self.byoutput[x]
+                                                  ))])
+        if reverse and skipmodal:
+            # if reversed, the last is biggest and should be skipped if modal
+            outdatalist = outdatalist[:-1]
+        for outdata in outdatalist:
+            if not reverse and skipmodal:
+                # If big first, this makes skipmodal skip first
+                skipmodal = False
+                continue
+            currout = '====================================\n'
+            currout += self.get_group_text(self.byoutput[outdata])
+            currout += '\n====================================\n'
+            if count:
+                currout += 'Count: {0}'.format(len(list(
+                    self.byoutput[outdata])))
+                currout += '\n====================================\n'
+            currout += outdata
+            currout += '\n\n'
+            output.write(currout)
+        output.flush()
+
+    def print_deviants(self, output=sys.stdout, skipmodal=False, reverse=False,
+                       count=False):
+        self.generate_byoutput()
+        modaloutput = None
+        ismodal = True
+        revoutput = []
+        for outdata in sorted(
+                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                                              humanify_nodename(
+                                                  self.get_group_text(
+                                                      self.byoutput[x]
+                                                  ))]):
+            if modaloutput is None:
+                modaloutput = outdata
+            if skipmodal:
+                skipmodal = False
+                ismodal = False
+                continue
+            currout = '====================================\n'
+            currout += self.get_group_text(self.byoutput[outdata])
+            currout += '\n====================================\n'
+            if count:
+                currout += 'Count: {0}'.format(len(list(
+                    self.byoutput[outdata])))
+                currout += '\n====================================\n'
+            if ismodal:
+                ismodal = False
+                currout += outdata
+            else:
+                currout += '\n'.join(colordiff(modaloutput.split('\n'),
+                                                 outdata.split('\n')))
+            currout += '\n\n'
+            if reverse:
+                revoutput.append(currout)
+            else:
+                output.write(currout)
+        for currout in reversed(revoutput):
+            output.write(currout)
+        output.flush()
+
+if __name__ == '__main__':
+    groupoutput = GroupedData()
+    for line in sys.stdin.read().split('\n'):
+        if not line:
+            continue
+        groupoutput.add_line(*line.split(': ', 1))
+    groupoutput.print_deviants()

confluent_client/confluent/tlvdata.py

@@ -16,18 +16,31 @@
 # limitations under the License.
 
 import confluent.tlv as tlv
+from datetime import datetime
 import json
 import struct
 
+def decodestr(value):
+    ret = None
+    try:
+        ret = value.decode('utf-8')
+    except UnicodeDecodeError:
+        try:
+            ret = value.decode('cp437')
+        except UnicodeDecodeError:
+            ret = value
+    return ret
 
 def unicode_dictvalues(dictdata):
     for key in dictdata:
         if isinstance(dictdata[key], str):
-            dictdata[key] = dictdata[key].decode('utf-8')
+            dictdata[key] = decodestr(dictdata[key])
+        elif isinstance(dictdata[key], datetime):
+            dictdata[key] = dictdata[key].strftime('%Y-%m-%dT%H:%M:%S')
         elif isinstance(dictdata[key], list):
             for i in xrange(len(dictdata[key])):
                 if isinstance(dictdata[key][i], str):
-                    dictdata[key][i] = dictdata[key][i].decode('utf-8')
+                    dictdata[key][i] = decodestr(dictdata[key][i])
                 elif isinstance(dictdata[key][i], dict):
                     unicode_dictvalues(dictdata[key][i])
         elif isinstance(dictdata[key], dict):

confluent_client/confluent_env.csh

@@ -0,0 +1 @@
+setenv PATH /opt/confluent/bin:$PATH

confluent_client/confluent_env.sh

@@ -1,2 +1,39 @@
 PATH=/opt/confluent/bin:$PATH
 export PATH
+MANPATH=/opt/confluent/share/man:$MANPATH
+export MANPATH
+# The aliases below are to signify that file globbing is unwelcome at the shell
+# this avoids a problem if a user does a noderange like 'n[21-33] and there is a file
+# in the directory like 'n3' that causes the parameter to change and target a totally
+# different node
+# Unfortunately in bourne shell, we cannot reliably ensure a prepended set-f
+# and an appended set +f are both run.  alias seems to be the only mechanism
+# that can intervene before glob expansion, but it lacks power.
+# putting it into a function to append is all well and good, *except*  that
+# if doing something like 'nodepower compute|grep' causes set -f to execute
+# in current shell, and the function to be in a subshell and leaves globbing
+# disabled in the parent shell.  Instead, store the current command in a
+# variable and use that to check for misglobbed noderanges, which was the goal
+alias nodeattrib='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeattrib'
+alias nodebmcreset='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodebmcreset'
+alias nodeboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeboot'
+alias nodeconfig='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconfig'
+alias nodeconsole='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeconsole'
+alias nodedefine='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodedefine'
+alias nodeeventlog='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeeventlog'
+alias nodefirmware='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodefirmware'
+alias nodegroupattrib='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodegroupattrib'
+alias nodegroupdefine='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodegroupdefine'
+alias nodegroupremove='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodegroupremove'
+alias nodehealth='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodehealth'
+alias nodeidentify='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeidentify'
+alias nodeinventory='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeinventory'
+alias nodelist='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelist'
+alias nodemedia='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodemedia'
+alias nodepower='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodepower'
+alias noderemove='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; noderemove'
+alias nodereseat='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodereseat'
+alias noderun='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; noderun'
+alias nodesensors='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesensors'
+alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodesetboot'
+alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'

confluent_client/doc/man/buildindex.sh

@@ -0,0 +1 @@
+for i in *.ronn; do echo -n `head -n 1  $i|awk '{print $1}'`; echo " $i"; done > index.txt

confluent_client/doc/man/collate.ronn

@@ -0,0 +1,88 @@
+collate(1) -- Organize text input by node
+==============================
+
+## SYNOPSIS
+
+`<other command> | collate [-a] [-d] [-w] [-s] [-c] [-r]`
+
+## DESCRIPTION
+
+**collate** takes input in the form of <nodename>: <data> and groups
+the output for each <nodename> together, and checks for identical data to further group nodes together.
+Output groups are sorted in descending size such that the topmost group is the largest, and the
+last output group is the smallest.  In the event of equally sized output groups,
+the groups are sorted alphanumerically by their header, and each header has
+node and group names sorted alphanumerically.
+
+## OPTIONS
+
+* `-a`, `--abbreviate`:
+   Use confluent to attempt to shorten the noderange.  This can help identify
+   when output differs along a telling group boundary.  For example, if
+   output suggests a large number of nodes are unreachable, abbreviate
+   showing 'rack1' as being unreachable may make more obvious a possible cause.
+
+* `-d`, `--diff`:
+   Express all but the most common result group in terms of diff from
+   the most common result group
+
+* `-w`, `--watch`:
+   Update results dynamically as data becomes available, rather than
+   waiting    for the command to fully complete.
+
+* `-s`, `--skipcommon`:
+   Suppress printing of the most common result text group.  This is used to
+   focus on stray output against a well known and expected result.
+   
+* `-c`, `--count`:
+   Print a count of the number of nodes in an output group under the 
+  noderange.
+  
+ * `-r`, `--reverse`:
+    Rather than starting with most common to least common, start with
+    the least common and print the most common last.
+   
+## EXAMPLES
+ * Organizing power state of multiple nodes:  
+  `# nodepower n1-n12 | collate`  
+  `====================================`  
+  `n1,n2,n3,n4,n7,n8,n9,n10,n11,n12`  
+  `====================================`  
+  `on`  
+  ` `  
+  `====================================`  
+  `n5,n6`  
+  `====================================`  
+  `off`  
+
+* Using diff to detect distinct UEFI configuration
+
+  `# pasu n1-n4 show Processors|collate -d -s`  
+  `====================================`  
+  `n3`  
+  `====================================`  
+  `@@`  
+  `  Processors.ProcessorPerformanceStates=Enable`  
+  `  Processors.C-States=Enable`  
+  `  Processors.PackageACPIC-StateLimit=ACPI C3`  
+  `- Processors.C1EnhancedMode=Enable`  
+  `+ Processors.C1EnhancedMode=Disable`  
+  `- Processors.Hyper-Threading=Enable`  
+  `+ Processors.Hyper-Threading=Disable`  
+  `  Processors.ExecuteDisableBit=Enable`  
+  `  Processors.IntelVirtualizationTechnology=Enable`  
+  ` `  
+  `====================================`  
+  `n1`  
+  `====================================`  
+  `@@`  
+  `  Processors.ProcessorPerformanceStates=Enable`  
+  `  Processors.C-States=Enable`  
+  `  Processors.PackageACPIC-StateLimit=ACPI C3`  
+  `- Processors.C1EnhancedMode=Enable`  
+  `+ Processors.C1EnhancedMode=Disable`  
+  `  Processors.Hyper-Threading=Enable`  
+  `  Processors.ExecuteDisableBit=Enable`  
+
+   
+   

confluent_client/doc/man/confetty.ronn

@@ -0,0 +1,36 @@
+confetty(8) --- Interactive confluent client
+=================================================
+
+## SYNOPSIS
+
+`confetty`  
+`confetty <confetty command line>`
+
+## DESCRIPTION
+
+**confetty** launches an interactive CLI session to the
+confluent service.  It provides a filesystem-like
+view of the confluent interface.  It is intended to
+be mostly an aid for developing client software, with
+day to day administration generally being easier with
+the various function specific commands.
+
+## COMMANDS
+
+The CLI may be navigated by shell commands and some other
+commands.
+
+* `cd`:
+  Change the location within the tree
+* `ls`:
+  List the elements within the current directory/tree
+* `show` **ELEMENT**, `cat` **ELEMENT**:
+  Display the result of reading a specific element (by full or relative path)
+* `unset` **ELEMENT** **ATTRIBUTE**
+  For an element with attributes, request to clear the value of the attribue
+* `set` **ELEMENT** **ATTRIBUTE**=**VALUE**
+  Set the specified attribute to the given value
+* `start` **ELEMENT**
+  Start a console session indicated by **ELEMENT** (e.g. /nodes/n1/console/session)
+* `rm` **ELEMENT**
+  Request removal of an element.  (e.g. rm events/hardware/log clears log from a node)

confluent_client/doc/man/confluent.ronn

@@ -0,0 +1,14 @@
+confluent(8) -- Start the confluent server
+=========================================================
+
+## SYNOPSIS
+
+`confluent`
+
+## DESCRIPTION
+
+**confluent** is the name of the server daemon.  It is normally run
+through the init subsystem rather than executed directly.  All confluent
+commands connect to confluent daemon.  It provides the web interface, debug,
+and unix socket connectivity.
+

confluent_client/doc/man/confluentdbutil.ronn

@@ -0,0 +1,26 @@
+confluentdbutil(8) -- Backup or restore confluent database
+=========================================================
+
+## SYNOPSIS
+
+
+`confluentdbutil [options] <dump|restore> <path>`
+
+## DESCRIPTION
+
+**confluentdbutil** is a utility to export/import the confluent attributes
+to/from json files.  The path is a directory that holds the json version.
+In order to perform restore, the confluent service must not be running.  It
+is required to indicate how to treat the usernames/passwords are treated in
+the json files (password protected, removed from the files, or unprotected).
+
+## OPTIONS
+
+* `-p`, `--password`:
+  If specified, information such as usernames and passwords will be encrypted
+  using the given password.
+* `-r`, `--redact`:
+  Indicates to replace usernames and passwords with a dummy string rather
+  than included.
+* `-u`, `--unprotected`:
+  The keys.json file will include the encryption keys without any protection.

confluent_client/doc/man/nodeattrib.ronn

@@ -0,0 +1,86 @@
+nodeattrib(8) -- List or change confluent nodes attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodeattrib [-b] <noderange> [<nodeattribute>...]`  
+`nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
+`nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
+`nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
+
+## DESCRIPTION
+
+**nodeattrib** manages the attributes of confluent nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  Attributes can be
+straightforward values, or an expression as documented in nodeattribexpressions(5).
+For a full list of attributes, run `nodeattrib <node> all` against a node.
+If `-c` is specified, this will set the nodeattribute to a null valid.
+This is different from setting the value to an empty string.
+
+For the `groups` attribute, it is possible to add a group by doing
+`groups,=<newgroup>`` and to remove by doing `groups^=<oldgroup>`
+
+Note that `nodeattrib <group>` will likely not provide the expected behavior.
+See nodegroupattrib(8) command on how to manage attributes on a group level.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+* `-c`, `--clear`:
+  Clear specified nodeattributes
+* `-e`, `--environment`:
+  Set specified attributes based on exported environment variable of matching name.  Environment variable names may be lower case or all upper case.  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+    `# nodeattrib n1-n2`  
+    `n1: console.method: ipmi`  
+    `n1: hardwaremanagement.manager: 172.30.3.1`  
+    `n2: console.method: ipmi`  
+    `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Getting an attribute of nodes matching a noderange:
+    `# nodeattrib n1,n2 hardwaremanagement.manager`  
+    `n1: hardwaremanagement.manager: 172.30.3.1`  
+    `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Getting a group of attributes while determining what group defines them:
+    `# nodeattrib n1,n2 hardwaremanagement --blame`  
+    `n1: hardwaremanagement.manager: 172.30.3.1`  
+    `n1: hardwaremanagement.method: ipmi (inherited from group everything)`  
+    `n1: hardwaremanagement.switch: r8e1`  
+    `n1: hardwaremanagement.switchport: 14`  
+    `n2: hardwaremanagement.manager: 172.30.3.2`  
+    `n2: hardwaremanagement.method: ipmi (inherited from group everything)`  
+    `n2: hardwaremanagement.switch: r8e1`  
+    `n2: hardwaremanagement.switchport: 2`  
+
+* Listing matching nodes of a simple noderange that are set:
+    `# nodeattrib n1-n2 current`  
+    `n1: console.method: ipmi`  
+    `n1: hardwaremanagement.manager: 172.30.3.1`  
+    `n2: console.method: ipmi`  
+    `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Change attribute on nodes of a simple noderange:
+    `# nodeattrib n1-n2 console.method=serial`  
+    `n1: console.method: serial`  
+    `n1: hardwaremanagement.manager: 172.30.3.1`  
+    `n2: console.method: serial`  
+    `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Clear attribute on nodes of a simple noderange, if you want to retain the variable set the attribute to "":
+    `# nodeattrib n1-n2 -c console.method`  
+    `# nodeattrib n1-n2 console.method`  
+    `n1: console.method: `  
+    `n2: console.method: `  
+
+## SEE ALSO
+
+nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeattribexpressions.ronn

@@ -0,0 +1,64 @@
+nodeattribexpressions(5) -- Confluent attribute expression syntax
+=================================================================
+
+## DESCRIPTION
+
+In confluent, any attribute may either be a straightforward value, or an
+expression to generate the value.
+
+An expression will contain some directives wrapped in `{}` characters. Within
+`{}` are a number of potential substitute values and operations.
+
+The most common operation is to extract a number from the nodename. These 
+values are available as n1, n2, etc. So for example attributes for a node named 
+b1o2r3u4 would have {n1} as 1, {n2} as 2, {n3} as 3, and {n4} as 4. 
+Additionally, {n0} is special as representing the last number in a name, so in
+the b1o2r3u4 example, {n0} would be 4.
+
+Frequently a value derives from a number in the node name, but must undergo a
+transform to be useful. As an example, if we have a scheme where nodes are
+numbered n1-n512, and they are arranged 1-42 in rack1, 43-84 in rack2, and so
+forth, it is convenient to perform arithmetic on the extracted number. Here is
+an example of codifying the above scheme, and setting the u to the remainder:
+
+`location.rack=rack{(n1-1)/42+1}`  
+`location.u={(n1-1)%42+1}`  
+
+Note how text may be mixed into expressions, only data within {} will receive
+special treatment. Here we also had to adjust by subtracting 1 and adding it
+back to make the math work as expected.
+
+It is sometimes the case that the number must be formatted a different way,
+either specifying 0 padding or converting to hexadecimal. This can be done by a
+number of operators at the end to indicate formatting changes.
+
+`{n1:02x} - Zero pad to two decimal places, and convert to hexadecimal, as mightbe used for generating MAC addresses`  
+`{n1:x} - Hexadecimal without padding, as may be used in a generated IPv6 address`  
+`{n1:X} - Uppercase hexadecimal`  
+`{n1:02d} - Zero pad a normal numeric representation of the number.`  
+
+Another common element to pull into an expression is the node name in whole:
+
+`hardwaremanagement.manager={node}-imm`
+
+Additionally other attributes may be pulled in:
+
+`hardwaremanagement.switchport={location.u}`  
+
+Multiple expressions are permissible within a single attribute:
+
+`hardwaremanagement.manager={node}-{hardwaremanagement.method}`
+
+A note to developers: in general the API layer will automatically recognize a
+generic set attribute to string with expression syntax and import it as an
+expression. For example, submitting the following JSON:
+
+`{ 'location.rack': '{n1}' }`
+
+Will auto-detect {n1} as an expression and assign it normally. If wanting to
+set that value verbatim, it can either be escaped by doubling the {} or by
+explicitly declaring it as a value:
+
+`{ 'location.rack': '{{n1}}' }`  
+
+`{ 'location.rack': { 'value': '{n1}' } }`

confluent_client/doc/man/nodebmcreset.ronn

@@ -0,0 +1,19 @@
+nodebmcreset(8) -- Reset management controller
+=========================================================
+
+## SYNOPSIS
+
+`nodebmcreset <noderange>`  
+
+## DESCRIPTION
+
+`nodebmcreset` allows you to reset the management controller of the specified noderange
+
+## EXAMPLES:
+
+* Reset the management controller for nodes n1 through n4:
+  `# nodebmcreset n1-n4`  
+  `n1: BMC Reset Successful`  
+  `n2: BMC Reset Successful`  
+  `n3: BMC Reset Successful`  
+  `n4: BMC Reset Successful`  

confluent_client/doc/man/nodeboot.ronn

@@ -0,0 +1,34 @@
+nodeboot(8) -- Reboot a confluent node to a specific device
+=========================================================
+
+## SYNOPSIS
+
+`nodeboot <noderange>`  
+`nodeboot <noderange>` [net|setup]  
+
+## DESCRIPTION
+
+**nodeboot** reboots nodes in a noderange.  If an additional argument is given,
+it sets the node to specifically boot to that as the next boot.
+
+## EXAMPLES
+* Booting n3 and n4 to the default boot behavior:
+  `# nodeboot n3-n4`  
+  `n3: default`  
+  `n4: default`  
+  `n3: on->reset`  
+  `n4: on->reset`  
+
+* Booting n1 and n2 to setup menu:
+  `# nodeboot n1-n2 setup`
+  `n2: setup`  
+  `n1: setup`  
+  `n2: on->reset`  
+  `n1: on->reset`  
+
+* Booting n3 and n4 to network:
+  `# nodeboot n3-n4 net`  
+  `n3: network`  
+  `n4: network`  
+  `n4: on->reset`  
+  `n3: off->on`  

confluent_client/doc/man/nodeconfig.ronn

@@ -0,0 +1,50 @@
+nodeconfig(8) -- Show or change node configuration
+==================================================
+
+## SYNOPSIS
+
+`nodeconfig <noderange> [options] [<configuration>..]`
+`nodeconfig <noderange> [options] [<configuration=value>..]`
+
+## DESCRIPTION
+
+**nodeconfig** manages the configuration of nodes managed by confluent.
+Rather than manipulating the confluent database, this actually modifies the
+running configuration on the node firmware.  Calling without '=' will show the
+current value, and '=' will change the value.  Network information can be
+given as a node expression, as documented in the man page for nodeattribexpressions(5).
+
+## OPTIONS
+
+* `-c`, `--comparedefault`:
+  Take the given settings and compare against default value, if available.  If
+  no configuration values are specified, it will show only those that differ.
+  If combined with `-x`, will show all differing values except those indicated
+  by `-x`
+
+* `-x`, `--exclude`:
+  Rather than listing only the specified configuration parameters, list all
+  attributes except for the specified ones
+
+* `-d`, `--detail`:
+  Provide detailed data as available.  This can include help text and valid
+  values for a setting.
+
+
+## EXAMPLES
+* Showing the current IP configuration of noderange BMC/IMM/XCC:
+    `# nodeconfig s3,s4 bmc`  
+    `s3: bmc.ipv4_address: 172.30.254.193/16`  
+    `s3: bmc.ipv4_method: DHCP`  
+    `s3: bmc.ipv4_gateway: 172.30.0.6`  
+    `s4: bmc.ipv4_address: 172.30.254.192/16`  
+    `s4: bmc.ipv4_method: DHCP`  
+    `s4: bmc.ipv4_gateway: 172.30.0.6`  
+
+* Changing nodes `s3` and `s4` to have the ip addressess 10.1.2.3 and 10.1.2.4 with a 16 bit subnet mask:
+    `# nodeconfig s3,s4 bmc.ipv4_address=10.1.2.{n1}/16`  
+
+## SEE ALSO
+
+nodeattribexpressions(5)
+

confluent_client/doc/man/nodeconsole.ronn

@@ -0,0 +1,57 @@
+nodeconsole(8) -- Open a console to a confluent node
+=====================================================
+
+## SYNOPSIS
+`nodeconsole <node>`
+
+## DESCRIPTION
+
+**nodeconsole** opens an interactive console session to a given node.  This is the
+text or serial console of a system.  Exiting is done by hitting `Ctrl-e`, then `c`,
+ then `.`.  Note that console output by default is additionally logged to
+`/var/log/confluent/consoles/`**NODENAME**.
+
+When the console connection to the target is broken, then confluent on backend
+will initiate an automatic retry interval that is randomized between 2 and 4 minutes.
+The reopen escape sequence below requests an immediate retry, as does connecting
+a new session.
+
+## ESCAPE SEQUENCE COMMANDS
+
+While connected to a console, a number of commands may be performed through escape
+sequences.  To begin an command escape sequence, hit `Ctrl-e`, then `c`.  The next
+keystroke will be interpreted as a command.  The following commands are available.
+
+* `.`:
+  Exit the session and return to the command prompt
+* `b`:
+  [send Break]
+  Send a break to the remote console when possible (some console plugins may not support this)
+* `o`:
+  [reOpen]
+  Request confluent to disconnect and reconnect to console.  For example if there is suspicion
+  that the console has gone inoperable, but would work if reconnected.
+* `po`:
+  [Power Off]
+  Power off server immediately, without waiting for OS to shutdown
+* `ps`:
+  [Power Shutdown]
+  Request OS shut down gracefully, and then power off
+* `pb<ent>`:
+  [Power Boot]
+  Cause system to immediately boot, resetting or turning on as appropriate.
+  Hitting enter is required to execute the reboot rather than another pb sequence
+* `pbs`:
+  [Power Boot Setup]
+  Request immediate boot ultimately landing in interactive firmware setup
+* `pbn`:
+  [Power Boot Network]
+  Request immediate boot to network
+* `r`:
+  [send Resize]
+  This queries the current terminal and sends stty commands to advertise the user termineal
+  size to the remote console
+* `?`:
+  Get a list of supported commands
+* `<ent>`:
+  Hit enter to skip entering a command at the escape prompt.

confluent_client/doc/man/nodedefine.ronn

@@ -0,0 +1,28 @@
+nodedefine(8) -- Define new confluent nodes
+
+## SYNOPSIS
+
+`nodedefine <noderange> [nodeattribute1=value1> <nodeattribute2=value2> ...]`  
+
+## DESCRIPTION
+
+`nodedefine` allows the definition of new nodes for the confluent management
+system.  It has the same syntax as `nodeattrib(8)`, and the commands differ in
+that `nodeattrib(8)` will error if a node does not exist.
+
+## EXAMPLES
+
+* Define two racks of nodes, named r{rack}u{u}:
+    `# nodedefine r1u1-r2u4`  
+    `r1u4: created`  
+    `r1u1: created`  
+    `r1u2: created`  
+    `r1u3: created`  
+    `r2u4: created`  
+    `r2u3: created`  
+    `r2u2: created`  
+    `r2u1: created`  
+
+## SEE ALSO
+
+noderange(5), nodeattribexpressions(8)

confluent_client/doc/man/nodediscover.ronn

@@ -0,0 +1,130 @@
+nodediscover(8)  -- List or manage confluent node discovery
+=========================================================
+
+## SYNOPSIS
+
+`nodediscover rescan`  
+`nodediscover [options] list`  
+`nodediscover [options] assign`
+`nodediscover [options] rescan`
+`nodediscover [options] clear`
+
+## DESCRIPTION
+
+**nodediscover** provides streamlined access to the confluent discovery data
+and assignment.  Nodes are detected through either an active scan (as occurs
+at service startup and on request by nodediscover rescan) or through passive
+detection (as a target comes online, it may attempt to register with the
+network).
+
+**nodediscover list** provides the currently known data in tabular format.  The
+data may be filtered by various parameters, as denoted in the options below.
+
+**nodediscover assign** performs manual discovery, assigning an entry to a node
+identity or, using `-i`, using a csv file to assign nodes all at once.  For
+example, a spreadsheet of serial numbers to desired node names could be used.
+
+**nodediscover rescan** requests the server to do an active sweep for new
+devices.  Generally every effort is made to passively detect devices as they
+become available (as they boot or are plugged in), however sometimes an active
+scan is the best approach to catch something that appears to be missing.
+
+**nodedsicover clear** requests the server forget about a selection of
+detected device.  It takes the same arguments as **nodediscover list**.
+
+## CSV FORMAT
+
+The CSV format used by nodediscover consists of one header describing the
+columns followed by the data.  The available columns are:
+
+* node: The name desired for the node in confluent
+* groups: A comma delimited list of groups to put the node into (using normal CSV escape rules for the commas)
+* mac: The mac address of the node
+* serial: The serial number of the node
+* uuid: The uuid of the node
+* bmc: The name or ip address that should be assigned to the BMC, regardless of current address
+* bmc_gateway: IP address of gateway, if desired
+* bmcuser: The desired username for the BMC to have as administrator
+* bmcpass: The desired password for the BMC to require
+
+Note that node is the only mandatory field.  To identify the systems, one of
+mac, serial, or uuid should be specified, it is pointless to provide more than
+one of these columns.  Other attributes if not provided may be defined through
+nodeattrib or group inherited.  It is possible to define nodes without ever
+providing a BMC ip, in which case IPv6 will be used automatically if possible.
+
+One example of a valid CSV file would be:
+node,serial,bmc,bmcuser,bmcpass
+n1,06DPMDF,172.30.204.1,admin,Passw0rd12
+n2,J30002HG,172.30.204.2,admin,Passw0rd12
+
+Which would use the serial number to assign the name and other three values to
+the nodes.
+
+## OPTIONS
+
+* `-m MODEL`, `--model=MODEL`:
+  Operate with nodes matching the specified model number
+* `-s SERIAL`, `--serial=SERIAL`:
+  Operate against the system matching the specified
+  serial number
+* `-u UUID`, `--uuid=UUID`:
+  Operate against the system matching the specified UUID
+* `-n NODE`, `--node=NODE`:
+  Operate with the given nodename
+* `-e MAC`, `--ethaddr=MAC`:
+  Operate against the system with the specified MAC
+  address
+* `-t TYPE`, `--type=TYPE`:
+  Operate against the system of the specified type
+* `-c`, `--csv`:
+  Use CSV formatted output
+* `-i IMPORT.CSV`, `--import=IMPORT.CSV`:
+  Import bulk assignment data from given CSV file
+* `-d STATE`, `--discoverystate=STATE`:
+  Indicate devices with a particular state.  The states are listed below
+* discovered: The device has been identified and has also had discovery
+              activities performed, including any relevant certificate
+              exchanges and deploying user and network configuration.
+* identified:  The device has been identified as to what node it is
+               supposed to be, however no active changes to the attributes
+               or device configuration has been performed.  This is
+               generally discovery awaiting approval due to
+               discovery.policy specifying a strict security model.
+* unidentified:  A device has been sensed, but no node identity has been
+                 established at all.  It provides data that can be used
+                 for nodediscover assign, as well as current IP addresses
+                 that can be used for manual efforts as possible.
+
+## EXAMPLES
+
+* Listing all detected Lenovo IMMv2 systems on a local network:
+  `# nodediscover list -t lenovo-imm2`  
+  `           Node|          Model|         Serial|                                UUID|      Mac Address|        Type|                            Current IP Addresses`  
+  `---------------|---------------|---------------|------------------------------------|-----------------|------------|------------------------------------------------`  
+  `             r2|        5463AC1|        06DPMDF|5f7133b8-c8cb-11e4-99a9-40f2e9b91018|40:f2:e9:b9:10:1d| lenovo-imm2|     172.30.204.1,fe80::42f2:e9ff:feb9:101d%eth1`  
+  `               |        7906AC1|        06PBX15|e98d483d-2759-11e1-8ffd-5cf3fc11249c|5c:f3:fc:11:24:9f| lenovo-imm2|      172.30.3.12,fe80::5ef3:fcff:fe11:249f%eth1`  
+  `             n1|        8737AC1|        23XXH41|14dd3ba6-5c38-11e1-931a-5cf3fc6e4680|5c:f3:fc:6e:13:e1| lenovo-imm2|       172.30.3.1,fe80::5ef3:fcff:fe6e:13e1%eth1`  
+  `             n7|        8737AC1|        23XXH32|79d2ce28-5cd5-11e1-8c86-5cf3fc6e46b0|5c:f3:fc:6e:13:f9| lenovo-imm2|       172.30.3.7,fe80::5ef3:fcff:fe6e:13f9%eth1`  
+  `             n8|        8737AC1|        23XXH49|551a8438-5cd5-11e1-8d6c-5cf3fc6e4708|5c:f3:fc:6e:14:25| lenovo-imm2|       172.30.3.8,fe80::5ef3:fcff:fe6e:1425%eth1`  
+  `             n3|        8737AC1|        23XXH30|1dd7f7b3-5da5-11e1-baf0-5cf3fc6e4738|5c:f3:fc:6e:14:3d| lenovo-imm2|       172.30.3.3,fe80::5ef3:fcff:fe6e:143d%eth1`  
+  `             n4|        8737AC1|        23XXH35|45b81dae-5d9b-11e1-8337-5cf3fc6e4858|5c:f3:fc:6e:14:cd| lenovo-imm2|       172.30.3.4,fe80::5ef3:fcff:fe6e:14cd%eth1`  
+  `            n11|        8737AC1|        23XXH12|31d90128-5c37-11e1-bdb7-5cf3fc6e4920|5c:f3:fc:6e:15:31| lenovo-imm2|      172.30.3.11,fe80::5ef3:fcff:fe6e:1531%eth1`  
+  `            n13|        8737AC1|        23XXH44|e23a138a-5cd3-11e1-8f3d-5cf3fc6e4950|5c:f3:fc:6e:15:49| lenovo-imm2|      172.30.3.13,fe80::5ef3:fcff:fe6e:1549%eth1`  
+  `               |        8737AC1|        23XXH29|5cd1216b-5c37-11e1-ba0c-5cf3fc6e49c8|5c:f3:fc:6e:15:85| lenovo-imm2|       172.30.3.9,fe80::5ef3:fcff:fe6e:1585%eth1`  
+  `               |        8737AC1|        23ZYT44|f4bf48ca-71f0-11e1-b274-5cf3fc6e4f10|5c:f3:fc:6e:18:29| lenovo-imm2|      172.30.3.10,fe80::5ef3:fcff:fe6e:1829%eth1`  
+  `         hpcedr|        7915AC1|        06DRHL5|a64e3014-d7e3-11e1-8d21-6cae8b1dff32|6c:ae:8b:1d:ff:36| lenovo-imm2|   172.30.254.250,fe80::6eae:8bff:fe1d:ff36%eth1`  
+  `               |        8737AC1|        06YRWC3|3af85a51-7efd-11e3-8599-000af7482e00|6c:ae:8b:32:cb:c5| lenovo-imm2|       172.30.3.6,fe80::6eae:8bff:fe32:cbc5%eth1`  
+  `               |        8737AC1|        06YRWB7|b230f62e-7efd-11e3-9773-000af7482980|6c:ae:8b:32:cd:01| lenovo-imm2|       172.30.3.5,fe80::6eae:8bff:fe32:cd01%eth1`  
+  `               |        8737AC1|        06YRWC7|09586005-7efe-11e3-9f03-000af7482df0|6c:ae:8b:32:cd:a5| lenovo-imm2|       172.30.3.2,fe80::6eae:8bff:fe32:cda5%eth1`  
+
+* Manually assign a single node according to serial number:
+  `[root@odin ~]# nodediscover assign -s 06PBX15 -n n12`  
+  `Assigned: n12`  
+
+* Bulk execute discovery based on spreadsheet:
+  `[root@odin ~]# nodediscover assign -i import.csv`  
+  `Defined r2`  
+  `Discovered r2`  
+  `Defined c1`  
+  `Discovered c1`  

confluent_client/doc/man/nodeeventlog.ronn

@@ -0,0 +1,33 @@
+nodeeventlog(8) -- Pull eventlog from confluent nodes
+============================================================
+
+## SYNOPSIS
+
+`nodeeventlog <noderange>`  
+`nodeeventlog <noderange> [clear]`
+
+## DESCRIPTION
+
+`nodeeventlog` pulls and optionally clears the event log from the requested
+noderange.
+
+## EXAMPLES
+* Pull the event log from n2 and n3:
+  `# nodeeventlog n2,n3`  
+  `n2: 05/03/2017 11:44:25 Event Log Disabled - SEL Fullness - Log clear`  
+  `n2: 05/03/2017 11:44:56 System Firmware - Progress - Unspecified`  
+  `n3: 05/03/2017 11:44:39 Event Log Disabled - SEL Fullness - Log clear`  
+  `n3: 05/03/2017 11:45:00 System Firmware - Progress - Unspecified`  
+  `n3: 05/03/2017 11:47:22 System Firmware - Progress - Starting OS boot`  
+
+* Pull and clear the event log from n2 and n3:
+`# nodeeventlog n2,n3 clear`  
+`n2: 05/03/2017 11:44:25 Event Log Disabled - SEL Fullness - Log clear`  
+`n2: 05/03/2017 11:44:56 System Firmware - Progress - Unspecified`  
+`n2: 05/03/2017 11:48:29 System Firmware - Progress - Starting OS boot`  
+`n3: 05/03/2017 11:44:39 Event Log Disabled - SEL Fullness - Log clear`  
+`n3: 05/03/2017 11:45:00 System Firmware - Progress - Unspecified`  
+`n3: 05/03/2017 11:47:22 System Firmware - Progress - Starting OS boot`  
+`# nodeeventlog n2,n3`  
+`n2: 05/03/2017 11:48:48 Event Log Disabled - SEL Fullness - Log clear`  
+`n3: 05/03/2017 11:48:52 Event Log Disabled - SEL Fullness - Log clear`  

confluent_client/doc/man/nodefirmware.ronn

@@ -0,0 +1,43 @@
+nodefirmware(8) -- Report firmware information on confluent nodes
+=================================================================
+
+## SYNOPSIS
+
+`nodefirmware <noderange>`  
+`nodefirmware <noderange> <components>|core`  
+`nodefirmware <noderange> update [--backup] <filename>`  
+
+## DESCRIPTION
+
+`nodefirmware` reports and updates various firmware on nodes.  By default it
+will retrieve all firmware, but can be directed to fetch specific firmware by
+calling out the name of the firmware (e.g. uefi or xcc) or request reading only
+core firmware firmware by using the word 'core', which is generally a quicker
+operation.
+
+In the update form, it accepts a single file and attempts to update it using
+the out of band facilities.  Firmware updates can end in one of three states:
+
+* `error`: The attempted update encountered an error that prevented successful install.  Nodes experiencing this state will be reported at the end and more detail on the error will be given
+* `pending`:  The firmware update process has completed, but the firmware will not be active until the relevant component next resets.  Generally speaking, for UEFI update the system will need a reboot, and for BMC updates, the `nodebmcreset` command will begin the process to activate the firmware.
+* `complete`:  The firmware update process has completed and activation has proceeded.  Note that while the activation process has commenced, the component may still be in the process of rebooting when nodefirmware exits.
+
+## EXAMPLES
+
+* Pull firmware from a node:
+`# nodefirmware r1`  
+`r1: IMM: 3.70 (TCOO26H 2016-11-29T05:09:51)`  
+`r1: IMM Backup: 1.71 (TCOO10D 2015-04-17T00:00:00)`  
+`r1: IMM Trusted Image: TCOO26H`  
+`r1: UEFI: 2.31 (TCE128I 2016-12-13T00:00:00)`  
+`r1: UEFI Backup: 2.20 (TCE126O)`  
+`r1: FPGA: 3.2.0`  
+`r1: Broadcom NetXtreme Gigabit Ethernet Controller Bootcode: 1.38`  
+`r1: Broadcom NetXtreme Gigabit Ethernet Controller MBA: 16.8.0`  
+`r1: Broadcom NetXtreme Gigabit Ethernet Controller Firmware Package: 0.0.0a`  
+`r1: ServeRAID M1215 MegaRAID Controller Firmware: 24.12.0-0038 (2016-10-20T00:00:00)`  
+`r1: ServeRAID M1215 Disk 28 MBF2600RC: SB2C`  
+`r1: ServeRAID M1215 Disk 29 MBF2600RC: SB2C`  
+`r1: ServeRAID M5210 Disk 0 MBF2600RC: SB2C`  
+`r1: ServeRAID M5210 Disk 1 MBF2600RC: SB2C`  
+`r1: ServeRAID M5210 Disk 2 MBF2600RC: SB2C`  

confluent_client/doc/man/nodegroupattrib.ronn

@@ -0,0 +1,42 @@
+nodegroupattrib(8) -- List or change confluent nodegroup attributes
+===================================================================
+
+## SYNOPSIS
+
+`nodegroupattrib <group> [ current | all ]`  
+`nodegroupattrib <group> [<nodeattribute>...]`  
+`nodegroupattrib <group> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
+`nodegroupattrib <group> [-c] [<nodeattribute1> <nodeattribute2=value2> ...]`  
+
+## DESCRIPTION
+
+`nodegroupattrip` queries the confluent server to get information about nodes.
+In the simplest form, it simply takes the given group and lists the attributes of that group.
+
+Contrasted with nodeattrib(8), settings managed by nodegroupattrib will be added
+and removed from a node as it is added or removed from a group.  If an attribute
+is set using nodeattrib(8) against a noderange(5) that happens to be a group name,
+nodeattrib(8) individually sets attributes directly on each individual node that is
+currently a member of that group.  Removing group membership or adding a new
+node after using the nodeattrib(8) command will not have attributes change automatically.
+It's easiest to see by using the `nodeattrib <noderange> -b` to understand how
+the attributes are set on the node versus a group to which a node belongs.
+
+## OPTIONS
+
+* `-c`, `--clear`:
+  Clear specified nodeattributes.
+
+## EXAMPLES
+
+* Show attributes of a group called `demogrp`:
+  `# nodegroupattrib demogrp`  
+  `demogrp: hardwaremanagement.manager:  (will derive from expression 10.30.{n0/255}.{n0%255})`  
+  `demogrp: nodes: n12,n13,n10,n11,n9,n1,n2,n3,n4`  
+
+* Set location.u to be the remainder of first number in node name when divided by 42:  
+  `# nodegroupattrib demogrp location.u={n1%42}`  
+
+## SEE ALSO
+
+nodeattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodegroupdefine.ronn

@@ -0,0 +1,23 @@
+nodegroupdefine(8) -- Define new confluent node group
+
+## SYNOPSIS
+
+`nodegroupdefine <groupname> [nodeattribute1=value1> <nodeattribute2=value2> ...]`  
+
+## DESCRIPTION
+
+`nodegroupdefine` allows the definition of a new node for the confluent management
+service.  It may only define a single group name at a time.
+It has the same syntax as `nodegroupattrib(8)`, and the commands differ in
+that `nodegroupattrib(8)` will error if a node group does not exist.
+
+## EXAMPLES
+
+* Create a group called `compute`:
+    `# nodegroupdefine compute`  
+    `compute: created`  
+
+
+## SEE ALSO
+
+nodeattribexpressions(8), nodegroupattrib(8), nodegroupremove(8)

confluent_client/doc/man/nodegroupremove.ronn

@@ -0,0 +1,18 @@
+nodegroupremove(8) -- Remove a nodegroup from the confluent database
+====================================================================
+
+## SYNOPSIS
+
+`nodegroupremove <noderange>`  
+
+## DESCRIPTION
+
+`nodegroupremove` simply removes the given single nodegroup from the confluent database.
+
+
+## EXAMPLES
+
+* Remove group called testgroup
+    `# nodegroupremove testgroup`  
+    `testgroup: deleted`  
+

confluent_client/doc/man/nodehealth.ronn

@@ -0,0 +1,22 @@
+nodehealth(8) -- Show health summary of confluent nodes
+========================================================
+
+## SYNOPSIS
+
+`nodehealth <noderange>`  
+
+## DESCRIPTION
+
+`nodehealth` reports the current health assessment of a confluent node.  It
+will report either `ok`, `warning`, `critical`, or `failed`, along with
+a string explaining the reason for any result other than `ok`.
+
+## EXAMPLES
+
+* Pull health summary of 5 nodes:
+  `# nodehealth n1-n4,r1`  
+  `n1: critical (Mezz Exp 2 Fault:Critical)`  
+  `n3: ok`  
+  `n2: ok`  
+  `r1: ok`  
+  `n4: ok`

confluent_client/doc/man/nodeidentify.ronn

@@ -0,0 +1,31 @@
+nodeidentify(8) -- Control the identify LED of confluent nodes
+=========================================================
+
+## SYNOPSIS
+
+`nodidentify <noderange> [on|off]`  
+
+## DESCRIPTION
+
+`nodeidentify` allows you to turn on or off the location LED of conflueunt nodes,
+making it easier to determine the physical location of the nodes.  The following
+options are supported:
+
+* `on`: Turn on the identify LED
+* `off`: Turn off the identify LED
+
+## EXAMPLES:
+
+* Turn on the identify LED on nodes n1 through n4:
+  `# nodeidentify n1-n4 on`  
+  `n1: on`  
+  `n2: on`  
+  `n3: on`  
+  `n4: on`  
+
+* Turn off the identify LED on nodes n1 thorugh n4:
+  `# nodeidentify n1-n4 off`  
+  `n1: off`  
+  `n2: off`  
+  `n4: off`  
+  `n3: off`  

confluent_client/doc/man/nodeinventory.ronn

@@ -0,0 +1,121 @@
+nodeinventory(8) -- Get hardware inventory of confluent node
+===============================================================
+
+## SYNOPSIS
+
+`nodeinventory <noderange> [serial|model|uuid|mac]`
+
+## DESCRIPTION
+
+`nodeinventory` pulls information about hardware of a node.  This includes 
+information such as adapters, serial numbers, processors, and memory modules,
+as supported by the platforms hardware management implementation.  It accepts
+arguments such as serial or model or others as listed above to filter
+output to specific data.
+
+## EXAMPLES
+
+* Pulling inventory of a node named r1:
+  `# nodeinventory r1`   
+  `r1: System MAC Address 1: 40:f2:e9:af:45:a0`   
+  `r1: System MAC Address 2: 40:f2:e9:af:45:a1`   
+  `r1: System MAC Address 3: 40:f2:e9:af:45:a2`   
+  `r1: System MAC Address 4: 40:f2:e9:af:45:a3`   
+  `r1: System Board manufacturer: IBM`   
+  `r1: System Product name: System x3650 M5`   
+  `r1: System Device ID: 32`   
+  `r1: System Revision: 9`   
+  `r1: System Product ID: 323`   
+  `r1: System Board model: 00KG915`   
+  `r1: System Device Revision: 0`   
+  `r1: System Serial Number: E2K4831`   
+  `r1: System Board manufacture date: 2014-10-20T12:00`   
+  `r1: System Board serial number: Y010UF4AL0B5`   
+  `r1: System Manufacturer: IBM`   
+  `r1: System FRU Number: 00FK639`   
+  `r1: System Board product name: System Board`   
+  `r1: System Model: 5462AC1`   
+  `r1: System UUID: 1B29CE46-765E-31A3-A3B9-B5FB934F15AB`   
+  `r1: System Hardware Version: 0x0000`   
+  `r1: System Manufacturer ID: 20301`   
+  `r1: System Chassis serial number: E2K4831`   
+  `r1: System Asset Number: `   
+  `r1: System Chassis type: Other`   
+  `r1: Power Supply 1 Board model: 94Y8136`   
+  `r1: Power Supply 1 Board manufacturer: EMER`   
+  `r1: Power Supply 1 FRU Number: 94Y8137`   
+  `r1: Power Supply 1 Board product name: IBM Designed Device`   
+  `r1: Power Supply 1 Board manufacture date: 2014-11-08T00:00`   
+  `r1: Power Supply 1 Board serial number:  K13814B88ED`   
+  `r1: Power Supply 1 Revision: 49`   
+  `r1: Power Supply 2: Not Present`   
+  `r1: DASD Backplane 1 Board model: 00JY139`   
+  `r1: DASD Backplane 1 Board manufacturer: WIST`   
+  `r1: DASD Backplane 1 FRU Number: 00FJ756`   
+  `r1: DASD Backplane 1 Board product name: IBM Designed Device`   
+  `r1: DASD Backplane 1 Board manufacture date: 2014-08-28T00:00`   
+  `r1: DASD Backplane 1 Board serial number: Y011UF48W02U`   
+  `r1: DASD Backplane 1 Revision: 0`   
+  `r1: DASD Backplane 2: Not Present`   
+  `r1: DASD Backplane 3: Not Present`   
+  `r1: DASD Backplane 4: Not Present`   
+  `r1: DASD Backplane 5 Board model: 00YJ530`   
+  `r1: DASD Backplane 5 Board manufacturer: WIST`   
+  `r1: DASD Backplane 5 FRU Number: 00AL953`   
+  `r1: DASD Backplane 5 Board product name: IBM Designed Device`   
+  `r1: DASD Backplane 5 Board manufacture date: 2016-02-04T00:00`   
+  `r1: DASD Backplane 5 Board serial number: Y010UF624024`   
+  `r1: DASD Backplane 5 Revision: 0`   
+  `r1: DASD Backplane 6: Not Present`   
+  `r1: CPU 1 Hardware Version: Intel(R) Xeon(R) CPU E5-2640 v3 @ 2.60GHz`   
+  `r1: CPU 1 Asset Number: Unknown`   
+  `r1: CPU 1 Manufacturer: Intel(R) Corporation`   
+  `r1: CPU 2: Not Present`   
+  `r1: ML2 Card: Not Present`   
+  `r1: DIMM 1: Not Present`   
+  `r1: DIMM 2: Not Present`   
+  `r1: DIMM 3: Not Present`   
+  `r1: DIMM 4: Not Present`   
+  `r1: DIMM 5: Not Present`   
+  `r1: DIMM 6: Not Present`   
+  `r1: DIMM 7: Not Present`   
+  `r1: DIMM 8: Not Present`   
+  `r1: DIMM 9: Not Present`   
+  `r1: DIMM 10: Not Present`   
+  `r1: DIMM 11: Not Present`   
+  `r1: DIMM 12: Not Present`   
+  `r1: DIMM 13: Not Present`   
+  `r1: DIMM 14: Not Present`   
+  `r1: DIMM 15: Not Present`   
+  `r1: DIMM 16: Not Present`   
+  `r1: DIMM 17: Not Present`   
+  `r1: DIMM 18: Not Present`   
+  `r1: DIMM 19: Not Present`   
+  `r1: DIMM 20: Not Present`   
+  `r1: DIMM 21: Not Present`   
+  `r1: DIMM 22: Not Present`   
+  `r1: DIMM 23: Not Present`   
+  `r1: DIMM 24: Not Present`   
+  `r1: X8 PCI 1: Not Present`   
+  `r1: X8 PCI 2: Not Present`   
+  `r1: X8 PCI 6: Not Present`   
+  `r1: X8 PCI 7: Not Present`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller MAC Address 1: 40:f2:e9:af:45:a0`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller MAC Address 2: 40:f2:e9:af:45:a1`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller MAC Address 3: 40:f2:e9:af:45:a2`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller MAC Address 4: 40:f2:e9:af:45:a3`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller PCI slot: 1b:00`   
+  `r1: Broadcom NetXtreme Gigabit Ethernet Controller location: Onboard`   
+
+* Reporting just the mac addresses of a node named r2:
+  `# nodeinventory r2 mac`
+  `r2: System MAC Address 1: 40:f2:e9:b9:10:18`
+  `r2: System MAC Address 2: 40:f2:e9:b9:10:19`
+  `r2: System MAC Address 3: 40:f2:e9:b9:10:1a`
+  `r2: System MAC Address 4: 40:f2:e9:b9:10:1b`
+
+* Getting the model number and serial number of a node named s2:
+  `# nodeinventory stark2 serial model`
+  `s2: System Product name: LENOVO THINKSYSTEM SD530 SERVER`
+  `s2: System Serial Number: DEV0000002`
+  `s2: System Model: 7X2104Z028`

confluent_client/doc/man/nodelist.ronn

@@ -0,0 +1,47 @@
+nodelist(8) -- List confluent nodes and their attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodelist <noderange>`  
+`nodelist <noderange> [-b] <nodeattribute>...`  
+
+## DESCRIPTION
+
+**nodelist** queries the confluent server to get information about nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  There is more
+information on node attributes in nodeattributes(5) man page.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+  `# nodelist n1-n4`  
+  `n1`  
+  `n2`  
+  `n3`  
+  `n4`  
+
+* Getting an attribute of nodes matching a noderange:
+  `# nodelist n1,n2 hardwaremanagement.manager`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Getting a group of attributes while determining what group defines them:
+  `# nodelist n1,n2 hardwaremanegement --blame`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n1: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n1: hardwaremanagement.switch: r8e1`  
+  `n1: hardwaremanagement.switchport: 14`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+  `n2: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n2: hardwaremanagement.switch: r8e1`  
+  `n2: hardwaremanagement.switchport: 2`  

confluent_client/doc/man/nodemedia.ronn

@@ -0,0 +1,56 @@
+nodemedia(8) -- Manage server remote media
+=========================================================
+
+## SYNOPSIS
+
+`nodemedia <noderange> [attach|detachall|list|upload] [options] <media>`
+
+## DESCRIPTION
+
+**nodemedia** manages the remote media functionality of supported BMCs.
+
+`list` shows all the current remote media the BMCs of the noderange are
+providing to the host platform.  The string (insecure) is appended to URLs that
+are mounted in an insecure fashion.  http is insecure, and https is also
+insecure when no meaningful certificate validation is performed.  Currently
+there is no action that can change this, and this is purely informational.  A
+future version of software may provide a means to increase security of attached
+remote media.
+
+`detachall` removes all the currently provided media to the host.  This unlinks
+remote media from urls and deletes uploaded media from the BMC.
+
+`upload` takes the given media image and uploads it to the BMC.  This causes
+the remote media to reside internally to the system without having to go
+to the network after the upload.  This is more constrained, for example the
+Lenovo xClarity Controller has a limit of 50 megabytes, but it has zero ongoing
+load on the media source.
+
+`attach` takes a URL to a remote media as an argument, and has the given
+BMCs map a virtual USB device to that url.  Content is loaded on demand, and
+as such that URL is referenced potentially once for every IO operation that
+the host platform attempts.
+
+## OPTIONS
+
+* `-c`:
+  Specify the maximum number of instances to run concurrently.
+
+## EXAMPLES
+* Listing currently mounted media:
+  `# nodemedia s1-s4 list`  
+  `s1: boot.img`  
+  `s2: boot.img`  
+  `s4: boot.img`  
+
+* Uploading a small boot image to the BMC:
+  `# nodemedia s1-s4 upload boot.img`  
+  `s1:complete: 100%     s2:complete: 100%     s3:complete: 100%     s4:complete: 100%`  
+  `s1: boot.img`  
+  `s4: boot.img`  
+  `s2: boot.img`  
+
+* Attaching a larger ISO for on-demand access:
+  `# nodemedia s1,s4 attach http://172.30.0.6/install/rhel74.iso`  
+  `s4: http://172.30.0.6/install/rhel74.iso (insecure)`  
+  `s1: http://172.30.0.6/install/rhel74.iso (insecure)`

confluent_client/doc/man/nodepower.ronn

@@ -0,0 +1,43 @@
+nodepower(8) -- Check or change power state of confluent nodes
+=========================================================
+
+## SYNOPSIS
+
+`nodepower <noderange>`  
+`nodepower <noderange> [on|off|boot|shutdown|reset|status]`
+
+## DESCRIPTION
+
+**nodepower** with only a noderange will retrieve current power state of nodes
+through confluent.  When given an additional argument, it will request a change
+to the power state of the nodes.  The following arguments are recognized:
+
+* `on`: Turn on the specified noderange.  Nothing will happen to nodes of
+the noderange that are already on.
+* `off`:  Immediately turn off the specified noderange, without waiting for OS
+to shutdown.  Nothing will happen to nodes of the noderange that are already on.
+* `boot`:  Immediately boot a system.  This will power on nodes of the noderange
+that are off, and reset nodes of the noderange that are on.  The previous state
+will be reflected in the output.
+* `shutdown`:  Request the OS gracefully shut down.  Nothing will happen for
+nodes that are off, and nodes will not shutdown if the OS fails to gracefully
+respond.
+* `reset`:  Request immediate reset of nodes of the noderange.  Nodes that are
+off will not react to this request.
+* `status`:  Behave identically to having no argument passed at all.
+
+## EXAMPLES
+* Get power state of nodes n1 through n4:
+  `# nodepower n1-n4`  
+  `n1: on`  
+  `n2: on`  
+  `n3: on`  
+  `n4: off`  
+
+
+* Forcing a reboot of nodes n1-n4:
+    `# nodepower n1-n4 boot`  
+    `n3: on->reset`  
+    `n1: on->reset`  
+    `n2: on->reset`  
+    `n4: off->on`  

confluent_client/doc/man/noderange.ronn

@@ -0,0 +1,57 @@
+noderange(5) -- Indicate a target set of nodes in confluent
+=========================================================
+
+## SYNOPSIS
+
+`<noderange>`  
+
+## DESCRIPTION
+
+<noderange> is a syntax that can be used in most Confluent commands to conveniently specify a list of nodes. The result is that the command will be applied to a range of nodes, often in parallel.
+
+## EXAMPLES:
+
+The simplest noderange is a single node name:
+`n1`
+
+Nodes and groups may be used interchangeably. The following may be used in any context where n1 would be accepted as a noderange:
+`rack1`
+
+Commonly, there is a desire to target a range of elements. There are a few identically behaving syntaxes for the purpose.
+`n1:n20`
+`n1-n20`
+`n[1-20]`
+
+Note that numbers may be zero padded or not, it will automatically detect the padding amount and adjust members of the range accordingly. Ranges also can understand multiple numeric values changing:
+`r[1-3]u[01-10]`
+`r1u01-r3u10`
+
+Ranges can also be applied to group names, and all above syntaxes are compatible:
+`rack1-rack10`
+`rack[1-10]`
+
+Also, regular expressions may be used to indicate nodes with names matching certain patterns:
+`~r1u..`
+
+The other major noderange primitive is indicating nodes by some attribute value:
+`location.rack=7`
+
+Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
+`n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
+
+Exclusions can be done by prepending a ‘-‘ before a portion of a noderange:
+`rack1,-n2`
+`compute,-rack1`
+`compute,-location.row=12`
+
+To indicate nodes that match multiple selections at once (set intersection), the @ symbol may be used:
+`compute@rack1`
+`location.rack=10@compute`
+
+For complex expressions, () may be used to indicate order of expanding the noderange to be explicit
+`rack1,-(console.logging=full@compute)`
+
+Noderange syntax can also indicate ‘pagination’, or separating the nodes into well defined chunks. > is used to indicate how many nodes to display at a time, and < is used to indicate how many nodes to skip into a noderange:
+`rack1>3<6`
+
+The above would show the seventh through ninth nodes of the rack1 group. Like all other noderange operations, this may be combined with any of the above, but must appear as the very last operation. Ordering is done with a natural sort.

confluent_client/doc/man/noderemove.ronn

@@ -0,0 +1,25 @@
+noderemove(8) -- Remove nodes from the confluent management service
+===================================================================
+
+## SYNOPSIS
+
+`noderemove <noderange>`  
+
+## DESCRIPTION
+
+`noderemove` simply removes the given noderange from the confluent database.
+
+
+## EXAMPLES
+
+* Remove two racks each with 4 nodes:
+    `# noderemove r1u1-r2u4`  
+    `r1u4: deleted`  
+    `r1u1: deleted`  
+    `r1u2: deleted`  
+    `r1u3: deleted`  
+    `r2u4: deleted`  
+    `r2u3: deleted`  
+    `r2u2: deleted`  
+    `r2u1: deleted`  
+

confluent_client/doc/man/nodereseat.ronn

@@ -0,0 +1,19 @@
+nodereseat(8) -- Request a reseat of a node
+============================================
+
+## SYNOPSIS
+
+`nodereseat <noderange>`  
+
+## DESCRIPTION
+
+`nodereseat` requests the enclosure manager of the current node to reseat that
+node's slot.  This should be equivalent to removing the system entirely from
+the chassis and putting it back in, but without actually having to do so.
+
+## EXAMPLES
+
+* Reseating the node `s1`:
+    `# nodereseat s1`  
+    `s1: Reseat successful`  
+

confluent_client/doc/man/noderun.ronn

@@ -0,0 +1,53 @@
+noderun(8) -- Run arbitrary commands per node in a noderange
+=============================================================
+
+## SYNOPSIS
+
+`noderun <noderange> <command string>`
+
+## DESCRIPTION
+
+`noderun` will take a given command and execute it in parallel once per node
+in the specified noderange.  Attribute expressions as documented in
+nodeattribexpressions(5) are expanded prior to execution of the command.  For
+noderun, the commands are locally executed.  To execute commands on the nodes
+themselves, see nodeshell(8).
+
+## EXAMPLES
+
+* Run ping against nodes n1 through n4:
+  `# noderun n1-n4 ping -c 1 {nodename}`  
+  `n3: PING n3 (172.30.2.3) 56(84) bytes of data.`  
+  `n3: 64 bytes from n3 (172.30.2.3): icmp_seq=1 ttl=64 time=0.387 ms`  
+  `n3: `  
+  `n3: --- n3 ping statistics ---`  
+  `n3: 1 packets transmitted, 1 received, 0% packet loss, time 0ms`  
+  `n3: rtt min/avg/max/mdev = 0.387/0.387/0.387/0.000 ms`  
+  `n4: PING n4 (172.30.2.4) 56(84) bytes of data.`  
+  `n4: 64 bytes from n4 (172.30.2.4): icmp_seq=1 ttl=64 time=0.325 ms`  
+  `n4: `  
+  `n4: --- n4 ping statistics ---`  
+  `n4: 1 packets transmitted, 1 received, 0% packet loss, time 0ms`  
+  `n4: rtt min/avg/max/mdev = 0.325/0.325/0.325/0.000 ms`  
+  `n2: PING n2 (172.30.2.2) 56(84) bytes of data.`  
+  `n2: From odin (172.30.0.6) icmp_seq=1 Destination Host Unreachable`  
+  `n2: `  
+  `n2: --- n2 ping statistics ---`  
+  `n2: 1 packets transmitted, 0 received, +1 errors, 100% packet loss, time 3000ms`  
+  `n2: `  
+  `n1: PING n1 (172.30.2.1) 56(84) bytes of data.`  
+  `n1: `  
+  `n1: --- n1 ping statistics ---`  
+  `n1: 1 packets transmitted, 0 received, 100% packet loss, time 10000ms`  
+  `n1: `  
+
+* Run an ipmitool raw command against the management controllers of n1 through n4:
+  `# noderun n1-n4 ipmitool -I lanplus -U USERID -E -H {hardwaremanagement.manager} raw 0 1`  
+  `n3:  01 10 00`  
+  `n1:  01 10 00`  
+  `n4:  01 10 00`  
+  `n2:  01 10 00`  
+
+## SEE ALSO
+
+nodeshell(8)

confluent_client/doc/man/nodesensors.ronn

@@ -0,0 +1,55 @@
+nodesensors(8) --- Retrieve telemetry for sensors of confluent nodes
+====================================================================
+
+## SYNOPSIS
+
+`nodesensors <noderange> [-c] [-i <interval>] [-n <samplecount>]  [<sensor name  or category>...]`  
+
+
+## DESCRIPTION
+
+**nodesensors** queries the confluent server to get telemetry from nodes.  Telemetry can include
+data such as temperature, power, and so forth.  Without arguments, it lists all available sensors
+and their current values.  If `-c` is specified, CSV format is used for output.  Normally
+nodesensors outputs once and exits.  Repeated periodic gathering can be done with `-i` to specify
+interval and `-n` to specify number of requests.  If `-i` is specified without `-n`, then it will
+retrieve data at the requested interval indefinitely.  If '-n' is specified without `-i`, an
+interval of 1 second is used.
+
+## OPTIONS
+
+* `-c`, `--csv`:
+  Organize output into CSV format, one sensor per column.
+
+* `-i`, `--interval`=**SECONDS**:
+  Repeat data gathering waiting, waiting the specified time between samples.  Unless `-n` is
+  specified, indefinite retrieval is assumed.
+
+* `-n`, `--numreadings`=**SAMPLES**:
+  Perform the specified number of readings, waiting `-i` indicated interval or 1 second if not
+  otherwise indicated.
+
+## EXAMPLES
+* Retrieving all temperature related sensors from one system  
+  `# nodesensors n1 temperature`  
+  `n1: CPU 1 Overtemp: Ok`  
+  `n1: CPU 2 Overtemp: Ok`  
+  `n1: Inlet Temp: 16.0 °C`  
+  `n1: PCH Overtemp: Ok`  
+  `n1: LOM Temp: Ok`  
+
+* Retrieving a sensor named "Inlet Temp" for 4 systems over a 3 second period of time:  
+   `# nodesensors n1-n4 'Inlet Temp' -c -n 3`  
+   `time,node,Inlet Temp (°C)`  
+   `2016-10-04T15:09:20,n1,19.0`  
+   `2016-10-04T15:09:20,n2,18.0`  
+   `2016-10-04T15:09:20,n3,18.0`  
+   `2016-10-04T15:09:20,n4,17.0`  
+   `2016-10-04T15:09:21,n1,19.0`  
+   `2016-10-04T15:09:21,n2,18.0`  
+   `2016-10-04T15:09:21,n3,18.0`  
+   `2016-10-04T15:09:21,n4,17.0`  
+   `2016-10-04T15:09:22,n1,19.0`  
+   `2016-10-04T15:09:22,n2,18.0`  
+   `2016-10-04T15:09:22,n3,18.0`  
+   `2016-10-04T15:09:22,n4,17.0`  

confluent_client/doc/man/nodesetboot.ronn

@@ -0,0 +1,69 @@
+nodesetboot(8) -- Check or set next boot device for noderange
+====================================================
+
+## SYNOPSIS
+
+`nodesetboot <noderange>`  
+`nodesetboot [options] <noderange> [default|cd|network|setup|hd]`
+
+## DESCRIPTION
+
+Requests that the next boot occur from the specified device.  Unless otherwise
+specified, this is a one time boot option, and does not change the normal boot
+behavior of the system.  This is useful for taking a system that normally boots
+to the hard drive and startking a network install, or to go into the firmware
+setup menu without having to hit a keystroke at the correct time on the console.
+
+Generally, it's a bit more convenient and direct to use the nodeboot(8) command,
+which will follow up the boot device with an immediate power directive to take
+effect.  The `nodesetboot` command is still useful, particularly if you want
+to use `nodesetboot <noderange> setup` and then initiate a reboot from within
+the operating system with ssh or similar rather than using the remote hardware
+control.
+
+## OPTIONS
+
+* `-b`, `--bios`:
+  For a system that supports both BIOS and UEFI style boot, request BIOS style
+  boot if supported (some platforms will UEFI boot with this flag anyway).
+
+* `-p`, `--persist`:
+  For a system that supports it, mark the boot override to persist rather than
+  be a one time change.  Many systems do not support this functionality.
+  
+* `default`:
+  Request a normal default boot with no particular device override
+
+* `cd`:
+  Request boot from media.  Note that this can include physical CD,
+  remote media mounted as CD/DVD, and detachable hard disks drives such as usb
+  key devices.
+  
+* `network`:
+  Request boot to network
+  
+* `setup`:
+  Request to enter the firmware configuration menu (e.g. F1 setup) on next boot.
+
+* `hd`:
+  Boot straight to hard disk drive
+
+## EXAMPLES
+
+* Set next boot to setup for four nodes:
+  `# nodesetboot n1-n4 setup`  
+  `n1: setup`  
+  `n3: setup`  
+  `n2: setup`  
+  `n4: setup`  
+
+* Check boot override settings on four nodes:
+  `# nodesetboot n1-n4`  
+  `n1: setup`  
+  `n2: setup`  
+  `n3: setup`  
+  `n4: setup`  
+  
+## SEE ALSO
+
+nodeboot(8)

confluent_client/doc/man/nodeshell.ronn

@@ -0,0 +1,34 @@
+nodeshell(8) -- Execute command on many nodes in a noderange through ssh
+=========================================================================
+
+## SYNOPSIS
+
+`nodeshell <noderange> [options] <command to execute on each node>`
+
+## DESCRIPTION
+
+Allows execution of a command on many nodes in parallel.  Like noderun(8), it
+accepts and interpolates confluent attribute expressions as documented in 
+nodeattribexpressions(5).  `nodeshell` provides stdout as stdout and stderr
+as stderr, unlike psh which combines all stdout and stderr into stdout.
+
+## OPTIONS
+
+* `-c`:
+  Specify the maximum number of instances to run concurrently.
+
+## EXAMPLES
+
+* Running `echo hi` on for nodes:
+  `# nodeshell n1-n4 echo hi`  
+  `n1: hi`  
+  `n2: hi`  
+  `n3: hi`  
+  `n4: hi`  
+
+* Setting a new static ip address temporarily on secondary interface of four nodes:
+  `# nodeshell n1-n4 ifconfig eth1 172.30.93.{n1}`  
+  
+## SEE ALSO
+
+noderun(8)

confluent_client/makeman

@@ -0,0 +1,10 @@
+#!/bin/sh
+cd `dirname $0`/doc/man
+mkdir -p ../../man/man1
+mkdir -p ../../man/man5
+mkdir -p ../../man/man8
+ronn -r *.ronn
+mv *.1 ../../man/man1/
+mv *.5 ../../man/man5/
+mv *.8 ../../man/man8/
+

confluent_client/setup.py.tmpl

@@ -1,4 +1,14 @@
 from setuptools import setup
+import os
+
+data_files = [('/etc/profile.d', ['confluent_env.sh', 'confluent_env.csh'])]
+try:
+    scriptlist = ['bin/{0}'.format(d) for d in os.listdir('bin/')]
+    data_files.append(('/opt/confluent/share/man/man1', ['man/man1/' + x for x in os.listdir('man/man1')]))
+    data_files.append(('/opt/confluent/share/man/man5', ['man/man5/' + x for x in os.listdir('man/man5')]))
+    data_files.append(('/opt/confluent/share/man/man8', ['man/man8/' + x for x in os.listdir('man/man8')]))
+except OSError:
+    pass
 
 setup(
     name='confluent_client',
@@ -7,6 +17,6 @@ setup(
     author_email='jjohnson2@lenovo.com',
     url='http://xcat.sf.net/',
     packages=['confluent'],
-    scripts=['bin/confetty', 'bin/nodeconsole', 'bin/nodeeventlog', 'bin/nodehealth', 'bin/nodeidentify', 'bin/nodelist', 'bin/nodepower', 'bin/nodesensors', 'bin/nodesetboot'],
-    data_files=[('/etc/profile.d', ['confluent_env.sh'])],
+    scripts=scriptlist,
+    data_files=data_files,
 )

confluent_server/MANIFEST.in

@@ -1,2 +1,3 @@
 include sysvinit/*
 include systemd/*
+include sysctl/*

confluent_server/bin/confluentdbutil

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import optparse
+import sys
+import os
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    # if installed into system path, do not muck with things
+    sys.path.append(path)
+import confluent.config.configmanager as cfm
+import confluent.config.conf as conf
+import confluent.main as main
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] [dump|restore] [path]")
+argparser.add_option('-p', '--password',
+                     help='Password to use to protect/unlock a protected dump')
+argparser.add_option('-r', '--redact', action='store_true',
+                     help='Redact potentially sensitive data rather than store')
+argparser.add_option('-u', '--unprotected', action='store_true',
+                      help='Specify that no password should be used to protect'
+                           ' the key information.  Fields will be encrypted, '
+                           'but keys.json will contain unencrypted decryption'
+                           ' keys that may be used to read the dump')
+(options, args) = argparser.parse_args()
+if len(args) != 2 or args[0] not in ('dump', 'restore'):
+    argparser.print_help()
+    sys.exit(1)
+dumpdir = args[1]
+
+
+if args[0] == 'restore':
+    pid = main.is_running()
+    if pid is not None:
+        print("Confluent is running, must shut down to restore db")
+        sys.exit(1)
+    cfm.restore_db_from_directory(dumpdir, options.password)
+elif args[0] == 'dump':
+    if options.password is None and not (options.unprotected or options.redact):
+        print("Must indicate a password to protect or -u to opt opt of "
+              "secure value protection or -r to skip all protected data")
+        sys.exit(1)
+    os.umask(077)
+    main._initsecurity(conf.get_config())
+    if not os.path.exists(dumpdir):
+        os.makedirs(dumpdir)
+    cfm.dump_db_to_directory(dumpdir, options.password, options.redact)
+
+
+

confluent_server/buildrpm

@@ -1,5 +1,8 @@
 #!/bin/sh
 cd `dirname $0`
+if [ -x ./makeman ]; then
+    ./makeman
+fi
 ./makesetup
 VERSION=`cat VERSION`
 PKGNAME=$(basename $(pwd))

confluent_server/confluent/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.5.0.dev395.ggacb3a20"

confluent_server/confluent/asynchttp.py

@@ -0,0 +1,187 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Overall, the result of this shall be:
+# - Web clients can create the same out-of-order responsiveness as socket
+#   clients (but with more complexity on their end)
+# - Web clients can share single request among console sessions
+# - Web clients can get async notify of things like node add/remove, events
+
+# This provides an async strategy to http clients.  The design is that a http
+# session may have an 'async' resource.  In such a case, any requests are
+# queued and immediately the response is given accepting the queued request.
+# A request flags itself as queue-compatible through an HTTP header indicating
+# the identifier of the async thread.  As responses happen to the queued
+# request, data is dispatched to the first registered poller for data on
+# the session.  This way, a client may elect to provide multiple pollers
+# to mitigate general choppiness of http network pattern.  It may not be
+# worth it, but it's possible.
+
+# Additionally support console session multiplexing, to mitigate needed
+# connection count.
+
+# Also, this should allow a client to register for notifications of things
+# like node add/delete or an event firing, ultimately.
+
+# Much like console sessions, these will be reaped if a client spends too
+# far away.
+
+import collections
+import confluent.exceptions as exc
+import confluent.messages as messages
+import confluent.util as util
+import eventlet
+import greenlet
+import time
+
+_asyncsessions = {}
+_cleanthread = None
+_consolesessions = None
+
+
+def _assign_asyncid(asyncsession):
+    sessid = util.randomstring(32)
+    while sessid in _asyncsessions:
+        sessid = util.randomstring(32)
+    _asyncsessions[sessid] = {'asyncsession': asyncsession}
+    return sessid
+
+
+class AsyncTermRelation(object):
+    # Need to keep an association of term object to async
+    # This allows the async handler to know the context of
+    # outgoing data to provide to calling code
+    def __init__(self, termid, async):
+        self.async = async
+        self.termid = termid
+
+    def got_data(self, data):
+        self.async.add(self.termid, data)
+
+
+class AsyncSession(object):
+
+    def __init__(self):
+        self.asyncid = _assign_asyncid(self)
+        self.responses = collections.deque()
+        self._evt = None
+        self.termrelations = []
+        self.consoles = set([])
+        self.reaper = eventlet.spawn_after(15, self.destroy)
+
+    def add(self, requestid, rsp):
+        self.responses.append((requestid, rsp))
+        if self._evt:
+            self._evt.send()
+            self._evt = None
+
+    def set_term_relation(self, env):
+        # need a term relation to keep track of what data belongs
+        # to what object (since the callback does not provide context
+        # for data, and here ultimately the client is responsible
+        # for sorting out which is which.
+        termrel = AsyncTermRelation(env['HTTP_CONFLUENTREQUESTID'], self)
+        self.termrelations.append(termrel)
+        return termrel
+
+    def add_console_session(self, sessionid):
+        self.consoles.add(sessionid)
+
+    def destroy(self):
+        if self._evt:
+            self._evt.send()
+            self._evt = None
+        for console in self.consoles:
+            _consolesessions[console]['session'].destroy()
+        self.consoles = None
+        del _asyncsessions[self.asyncid]
+
+    def run_handler(self, handler, requestid):
+        try:
+            for rsp in handler:
+                self.add(requestid, rsp)
+            self.add(requestid, messages.AsyncCompletion())
+        except Exception as e:
+            self.add(requestid, e)
+
+    def get_responses(self, timeout=25):
+        self.reaper.cancel()
+        self.reaper = eventlet.spawn_after(timeout + 15, self.destroy)
+        nextexpiry = time.time() + 90
+        for csess in list(self.consoles):
+            try:
+                _consolesessions[csess]['expiry'] = nextexpiry
+            except KeyError:  # session has been closed elsewhere
+                self.consoles.discard(csess)
+        if self._evt:
+            # TODO(jjohnson2): This precludes the goal of 'double barreled'
+            # access....  revisit if this could matter
+            raise Exception('get_responses is not re-entrant')
+        if not self.responses:  # wait to accumulate some
+            self._evt = eventlet.event.Event()
+            with eventlet.Timeout(timeout, False):
+                self._evt.wait()
+            self._evt = None
+        while self.responses:
+            yield self.responses.popleft()
+
+
+def run_handler(hdlr, env):
+    asyncsessid = env['HTTP_CONFLUENTASYNCID']
+    try:
+        asyncsession = _asyncsessions[asyncsessid]['asyncsession']
+        requestid = env['HTTP_CONFLUENTREQUESTID']
+    except KeyError:
+        raise exc.InvalidArgumentException(
+                'Invalid Session ID or missing request id')
+    eventlet.spawn_n(asyncsession.run_handler, hdlr, requestid)
+    return requestid
+
+
+def get_async(env, querydict):
+    global _cleanthread
+    return _asyncsessions[env['HTTP_CONFLUENTASYNCID']]['asyncsession']
+
+
+def handle_async(env, querydict, threadset):
+    global _cleanthread
+    # This may be one of two things, a request for a new async stream
+    # or a request for next data from async stream
+    # httpapi otherwise handles requests an injecting them to queue
+    if 'asyncid' not in querydict or not querydict['asyncid']:
+        # This is a new request, create a new multiplexer
+        currsess = AsyncSession()
+        yield messages.AsyncSession(currsess.asyncid)
+        return
+    mythreadid = greenlet.getcurrent()
+    threadset.add(mythreadid)
+    loggedout = None
+    currsess = None
+    try:
+        currsess = _asyncsessions[querydict['asyncid']]['asyncsession']
+        for rsp in currsess.get_responses():
+            yield messages.AsyncMessage(rsp)
+    except greenlet.GreenletExit as ge:
+        loggedout = ge
+    threadset.discard(mythreadid)
+    if loggedout is not None:
+        currsess.destroy()
+        raise exc.LoggedOut()
+
+
+def set_console_sessions(consolesessions):
+    global _consolesessions
+    _consolesessions = consolesessions

confluent_server/confluent/auth.py

@@ -37,6 +37,7 @@ _passcache = {}
 _passchecking = {}
 
 authworkers = None
+authcleaner = None
 
 
 class Credentials(object):
@@ -116,7 +117,7 @@ def authorize(name, element, tenant=False, operation='create',
     user, tenant = _get_usertenant(name, tenant)
     if tenant is not None and not configmanager.is_tenant(tenant):
         return None
-    manager = configmanager.ConfigManager(tenant)
+    manager = configmanager.ConfigManager(tenant, username=user)
     if skipuserobj:
         return None, manager, user, tenant, skipuserobj
     userobj = manager.get_user(user)
@@ -178,7 +179,7 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
             # invalidate cache and force the slower check
             del _passcache[(user, tenant)]
             return None
-    cfm = configmanager.ConfigManager(tenant)
+    cfm = configmanager.ConfigManager(tenant, username=user)
     ucfg = cfm.get_user(user)
     if ucfg is None or 'cryptpass' not in ucfg:
         eventlet.sleep(0.05)  # stall even on test for existence of a username
@@ -195,6 +196,13 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
     #such a beast could be passed into pyghmi as a way for pyghmi to
     #magically get offload of the crypto functions without having
     #to explicitly get into the eventlet tpool game
+    global authworkers
+    global authcleaner
+    if authworkers is None:
+        authworkers = multiprocessing.Pool(processes=1)
+    else:
+        authcleaner.cancel()
+    authcleaner = eventlet.spawn_after(30, _clean_authworkers)
     crypted = eventlet.tpool.execute(_do_pbkdf, passphrase, salt)
     del _passchecking[(user, tenant)]
     eventlet.sleep(0.05)  # either way, we want to stall so that client can't
@@ -211,19 +219,16 @@ def _apply_pbkdf(passphrase, salt):
                       lambda p, s: hmac.new(p, s, hashlib.sha256).digest())
 
 
+def _clean_authworkers():
+    global authworkers
+    global authcleaner
+    authworkers = None
+    authcleaner = None
+
+
 def _do_pbkdf(passphrase, salt):
     # we must get it over to the authworkers pool or else get blocked in
     # compute.  However, we do want to wait for result, so we have
     # one of the exceedingly rare sort of circumstances where 'apply'
     # actually makes sense
-    return authworkers.apply(_apply_pbkdf, [passphrase, salt])
-
-
-def init_auth():
-    # have a some auth workers available.  Keep them distinct from
-    # the general populace of workers to avoid unauthorized users
-    # starving out productive work
-    global authworkers
-    # for now we'll just have one auth worker and see if there is any
-    # demand for more.  I personally doubt it.
-    authworkers = multiprocessing.Pool(processes=1)
+    return authworkers.apply(_apply_pbkdf, [passphrase, salt])

confluent_server/confluent/config/attributes.py

@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
+# Copyright 2015 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,50 +16,69 @@
 # limitations under the License.
 
 
-#This defines the attributes of variou classes of things
+#This defines the attributes of various classes of things
 
 # 'nic', meant to be a nested structure under node
-nic = {
-    'name': {
-        'description': 'Name in ip/ifconfig as desired by administrator',
-    },
-    'port': {
-        'description': 'Port that this nic connects to',
-    },
-    'switch': {
-        'description': 'Switch that this nic connects to',
-    },
-    'customhardwareaddress': {
-        'description': 'Mac address to push to nic',
-    },
-    'dnssuffix': {
-        'description': ('String to place after nodename, but before'
-                        'Network.Domain to derive FQDN for this NIC'),
-    },
-    'hardwareaddress': {
-        'description': 'Active mac address on this nic (factory or custom)'
-    },
-    'ipaddresses': {
-        'description': 'Set of IPv4 and IPv6 addresses in CIDR format'
-    },
-    'pvid': {
-        'description': 'PVID of port on switch this nic connects to',
-    },
-    'mtu': {
-        'description': 'Requested MTU to configure on this interface',
-    },
-    'vlans': {
-        'description': 'Tagged VLANs to apply to nic/switch',
-    },
-    'dhcpv4enabled': {
-        'description':  ('Whether DHCP should be attempted to acquire IPv4'
-                         'address on this interface'),
-    },
-    'dhcpv6enabled': {
-        'description':  ('Whether DHCP should be attempted to acquire IPv6'
-                         'address on this interface'),
-    },
-}
+# changing mind on design, flattening to a single attribute, a *touch* less
+# flexible at the top end, but much easier on the low end
+# now net.<name>.attribute scheme
+# similarly, leaning toward comma delimited ip addresses, since 99.99% of the
+# time each nic will have one ip address
+# vlan specification will need to be thought about a tad, each ip could be on
+# a distinct vlan, but could have a vlan without an ip for sake of putting
+# to a bridge.  Current thought is
+# vlans attribute would be comma delimited referring to the same index
+# as addresses, with either 'native' or a number for vlan id
+# the 'joinbridge' attribute would have some syntax like @<vlanid> to indicate
+# joining only a vlan of the nic to the bridge
+# 'joinbond' attribute would not support vlans.
+
+#nic = {
+#    'name': {
+#        'description': 'Name in ip/ifconfig as desired by administrator',
+#    },
+#    'biosdevname': {
+#        'description': '"biosdevname" scheme to identify the adapter. If not'
+#                       'mac address match is preferred, then biosdevname, then'
+#                       'name.',
+#    },
+#    'port': {
+#        'description': 'Port that this nic connects to',
+#    },
+#    'switch': {
+#        'description': 'Switch that this nic connects to',
+#    },
+#    'customhardwareaddress': {
+#        'description': 'Mac address to push to nic',
+#    },
+#    'dnssuffix': {
+#        'description': ('String to place after nodename, but before'
+#                        'Network.Domain to derive FQDN for this NIC'),
+#    },
+#    'hardwareaddress': {
+#        'description': 'Active mac address on this nic (factory or custom)'
+#    },
+#    'ipaddresses': {
+#        'description': 'Set of IPv4 and IPv6 addresses in CIDR format'
+#    },
+#    'pvid': {
+#        'description': 'PVID of port on switch this nic connects to',
+#    },
+#    'mtu': {
+#        'description': 'Requested MTU to configure on this interface',
+#    },
+#    'vlans': {
+#        'description': 'Tagged VLANs to apply to nic/switch',
+#    },
+#    'dhcpv4enabled': {
+#        'description':  ('Whether DHCP should be attempted to acquire IPv4'
+#                         'address on this interface'),
+#    },
+#    'dhcpv6enabled': {
+#        'description':  ('Whether DHCP should be attempted to acquire IPv6'
+#                         'address on this interface'),
+#    },
+#}
 
 user = {
     'password': {
@@ -70,7 +90,6 @@ user = {
 node = {
     'groups': {
         'type': list,
-        'default': 'all',
         'description': ('List of static groups for which this node is '
                         'considered a member'),
     },
@@ -80,6 +99,84 @@ node = {
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
+    # autonode is the feature of generating nodes based on connectivity to
+    # current node.  In recursive autonode, for now we just allow endpoint to
+    # either be a server directly *or* a server enclosure.  This precludes
+    # for the moment a concept of nested arbitrarily deep, but for now do this.
+    # hypothetically, one could imagine supporting an array and 'popping'
+    # names until reaching end.  Not worth implementing at this point.  If
+    # a traditional switch is added, it needs some care and feeding anyway.
+    # If a more exciting scheme presents itself, well we won't have to
+#   # own discovering switches anyway.
+#   'autonode.servername': {
+#       'description': ('Template for creating nodenames for automatic '
+#                       'creation of nodes detected as children of '
+#                       'this node.  For example, a node in a server '
+#                       'enclosure bay or a server connected to a switch or '
+#                       'an enclosure manager connected to a switch.  Certain '
+#                       'special template parameters are available and can '
+#                       'be used alongside usual config template directives. '
+#                       '"discovered.nodenumber" will be replaced with the '
+#                       'bay or port number where the child node is connected.'
+#                       ),
+#   },
+#   'autonode.servergroups': {
+#       'type': list,
+#       'description': ('A list of groups to which discovered nodes will '
+#                       'belong to.  As in autonode.servername, "discovered." '
+#                       'variable names will be substituted in special context')
+#   },
+#   'autonode.enclosurename': {
+#       'description': ('Template for creating nodenames when the discovered '
+#                       'node is an enclosure that will in turn generate nodes.'
+#                       )
+#   },
+#   'autonode.enclosuregroups': {
+#       'type': list,
+#       'description': ('A list of groups to which a discovered node will be'
+#                       'placed, presuming that node is an enclosure.')
+#   },
+#For now, we consider this eventuality if needed.  For now emphasize paradigm
+# of group membership and see how far that goes.
+#    'autonode.copyattribs': {
+#        'type': list,
+#        'description': ('A list of attributes to copy from the node generator '
+#                        'to the generated node.  Expressions will be copied '
+#                        'over without evaluation, so will be evaluated '
+#                        'in the context of the generated node, rather than the'
+#                        'parent node.  By default, an enclosure will copy over'
+#                        'autonode.servername, so that would not need to be '
+#                        'copied ')
+#    },
+    'discovery.policy': {
+        'description':  'Policy to use for auto-configuration of discovered '
+                        'and identified nodes. Valid values are "manual", '
+                        '"permissive", or "open". "manual" means nodes are '
+                        'detected, but not autoconfigured until a user '
+                        'approves. "permissive" indicates to allow discovery, '
+                        'so long as the node has no existing public key. '
+                        '"open" allows discovery even if a known public key '
+                        'is already stored',
+    },
+    'info.note': {
+        'description':  'A field used for administrators to make arbitrary '
+                        'notations about nodes. This is meant entirely for '
+                        'human use and not programmatic use, so it can be '
+                        'freeform text data without concern for issues in how '
+                        'the server will process it.',
+    },
+    'location.room': {
+        'description': 'Room description for the node',
+    },
+    'location.row': {
+        'description': 'Row description for the rack the node is in',
+    },
+    'location.rack': {
+        'description': 'Rack number of the rack the node is in',
+    },
+    'location.u': {
+        'description': 'Position in the rack of the node',
+    },
 #    'location.timezone': {
 #        'description': 'POSIX timezone to apply to this node',
 #    },
@@ -175,54 +272,109 @@ node = {
 #        'appliesto': ['vm'],
 #    },
     'hardwaremanagement.manager': {
-        'description': 'The management address dedicated to this node',
+        'description': 'The management address dedicated to this node.  This '
+                       'is the address of, for example, the Lenovo IMM.',
     },
     'hardwaremanagement.method': {
         'description': 'The method used to perform operations such as power '
                        'control, get sensor data, get inventory, and so on. '
     },
-#    'enclosure.manager': {
-#        'description': "The management device for this node's chassis",
+    'enclosure.bay': {
+         'description': 'The bay in the enclosure, if any',
 #        'appliesto': ['system'],
-#    },
-#    'enclosure.bay': {
-#        'description': 'The bay in the enclosure, if any',
+    },
+    'enclosure.extends': {
+        'description': 'When using an extendable enclosure, this is the node '
+                       'representing the manager that is one closer to the '
+                       'uplink.',
+    },
+    'enclosure.manager': {
+        'description': "The management device for this node's chassis",
 #        'appliesto': ['system'],
-#    },
+    },
+
 #    'enclosure.type': {
 #        'description': '''The type of enclosure in use (e.g. IBM BladeCenter,
 #IBM Flex)''',
 #        'appliesto': ['system'],
 #    },
-#    'inventory.serialnumber': {
-#        'description': 'The manufacturer serial number of node',
-#    },
-#    'inventory.uuid': {
-#        'description': 'The UUID of the node as presented in DMI',
-#    },
-#    'inventory.modelnumber': {
+    'id.model': {
+        'description': 'The model number of a node.  In scenarios where there '
+                       'is both a name and a model number, it is generally '
+                       'expected that this would be the generally more '
+                       'specific model number.'
+    },
+    'id.serial': {
+        'description': 'The manufacturer serial number of node',
+    },
+    'id.uuid': {
+        'description': 'The UUID of the node as presented in DMI.',
+    },
+    # For single interface mac collection, net.bootable suffices
+    # For multiple interface mac collection, we perhaps add an address field and use subnet affinity as a clue
+    # to disambiguate multiple addresses for external provisioning
+    # For internal provisioning, the UUID matters rather than the MAC, though subnet affinity may
+    # still be a factor to select the appropriate static config to send down.  In such a case bonding
+    # is hopefully more likely as that's a bit easier.
+    # Start with the first case and only document that, the other thoughts can be future items if they turn up
+    'net.bootable': {
+        'type': bool,
+        'description': 'Whether or not the indicated network interface is to be used for booting.  This is used by '
+                       'the discovery process to decide where to place the mac address of a detected PXE nic.',
+    },
+    'net.ipv4_gateway': {
+        'description':  'The IPv4 gateway to use if applicable.  As is the '
+                        'case for other net attributes, net.eth0.ipv4_gateway '
+                        'and similar is accepted.'
+    },
+    'net.hwaddr': {
+        'description': 'The hardware address, aka MAC address of the interface indicated, generally populated by the '
+                       'PXE discovery mechanism'
+    },
+    # 'net.pxe': { 'description': 'Whether pxe will be used on this interface'
+    # TODO(jjohnson2):  Above being 'true' will control whether mac addresses
+    # are stored in this nics attribute on pxe-client discovery, since
+    # pxe discovery is ambiguous for BMC and system on same subnet,
+    # or even both on the same port and same subnet
+    'net.switch': {
+        'description': 'An ethernet switch the node is connected to.  Note '
+                       'that net.* attributes may be indexed by interface. '
+                       'For example instead of using net.switch, it is '
+                       'possible to use net.eth0.switch and net.eth1.switch '
+                       'or net.0.switch and net.1.switch to define multiple '
+                       'sets of net connectivity associated with each other.'
+    },
+    'net.switchport': {
+        'description': 'The port on the switch that corresponds to this node. '
+                       'See information on net.switch for more on the '
+                       'flexibility of net.* attributes.'
+    },
+#    'id.modelnumber': {
 #        'description': 'The manufacturer dictated  model number for the node',
 #    },
-#    'inventory.snmpengineid': {
+#    'id.modelname': {
+#        'description': 'The manufacturer model label for the node',
+#    },
+#    'id.snmpengineid': {
 #        'description': 'The SNMP Engine id used by this node',
 #    },
 #    'secret.snmpuser': {
 #        'description': 'The user to use for SNMPv3 access to this node',
 #    },
-#    'secret.snmppassphrase': {
-#        'description': 'The passphrase to use for SNMPv3 access to this node',
+#    'secret.snmppassword': {
+#        'description': 'The password to use for SNMPv3 access to this node',
 #    },
+    'secret.snmpcommunity': {
+        'description': ('SNMPv1 community string, it is highly recommended to'
+                        'step up to SNMPv3'),
+    },
 #    'secret.snmplocalizedkey': {
 #        'description': ("SNMPv3 key localized to this node's SNMP Engine id"
 #                        'This can be used in lieu of snmppassphrase to avoid'
 #                        'retaining the passphrase TODO: document procedure'
 #                        'to commit passphrase to localized key'),
 #    },
-#    'secret.snmpcommunity': {
-#        'description': ('SNMPv1 community string, it is highly recommended to'
-#                        'step up to SNMPv3'),
-#    },
-#    'secret.localadminpassphrase': {
+#    'secret.adminpassword': {
 #        'description': ('The passphrase to apply to local root/administrator '
 #                        'account. '
 #                        'If the environment is 100% Linux, the value may be '
@@ -233,34 +385,35 @@ node = {
 #                        'AD')
 #    },
     'secret.ipmikg': {
-        'description': 'Optional Integrity key for IPMI communication'
+        'description': 'Optional Integrity key for IPMI communication.  This '
+                       'should generally be ignored, as mutual authentication '
+                       'is normally done with the password alone (which is a '
+                       'shared secret in IPMI)'
     },
-#    'secret.ipmiuser': {
-#        'description': ('The username to use to log into IPMI device related '
-#                        'to the node.  For setting username, default '
-#                        'behavior is to randomize username, for using '
-#                        'username if not set, USERID is assumed'),
-#    },
-#    'secret.ipmipassphrase': {
-#        'description': ('The key to use to authenticate to IPMI device '
-#                        'related to the node.  For setting passphrase, '
-#                        'default behavior is to randomize passphrase and '
-#                        'store it here.  If going to connect over the '
-#                        'network and value is not set, PASSW0RD is attempted')
-#    },
     'secret.hardwaremanagementuser': {
-        'description': ('Username to be set and used by protocols like SSH '
-                        'and HTTP where client provides passphrase over the '
-                        'network. Given the distinct security models betwen '
-                        'this class of protocols and SNMP and IPMI, snmp and '
-                        'ipmi utilize dedicated values.'),
+        'description': ('The username to use when connecting to the hardware '
+                        'manager'),
     },
     'secret.hardwaremanagementpassword': {
-        'description': ('Passphrase to be set and used by protocols like SSH '
-                        'and HTTP, where client sends passphrase over the '
-                        'network.  Given distinct security models between '
-                        'this class of protocols, SNMP, and IPMI, SNMP and '
-                        'IPMI are given their own settings with distinct '
-                        'behaviors'),
+        'description': ('Password to use when connecting to the hardware '
+                        'manager'),
+    },
+    'pubkeys.addpolicy': {
+        'description': ('Policy to use when encountering unknown public '
+                        'keys.  Choices are "automatic" to accept and '
+                        'store new key if no key known and "manual" '
+                        'to always reject a new key, even if no key known'
+                        'Note that if the trusted CA verifies the certificate,'
+                        ' that is accepted ignoring this policy.  Default '
+                        'policy is "automatic"'),
+        'valid_values': ('automatic', 'manual'),
+    },
+    'pubkeys.tls_hardwaremanager': {
+        'description':  ('Fingerprint of the TLS certificate recognized as'
+                         'belonging to the hardware manager of the server'),
+    },
+    'pubkeys.ssh': {
+        'description': ('Fingerprint of the SSH key of the OS running on the '
+                        'system.'),
     },
 }

confluent_server/confluent/config/configmanager.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2018 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -65,12 +65,16 @@ import anydbm as dbm
 import ast
 import base64
 import confluent.config.attributes as allattributes
+import confluent.config.conf as conf
 import confluent.log
+import confluent.noderange as noderange
 import confluent.util
 import confluent.exceptions as exc
 import copy
 import cPickle
 import errno
+import eventlet
+import fnmatch
 import json
 import operator
 import os
@@ -87,6 +91,14 @@ _masterintegritykey = None
 _dirtylock = threading.RLock()
 _config_areas = ('nodegroups', 'nodes', 'usergroups', 'users')
 tracelog = None
+statelessmode = False
+_cfgstore = None
+
+_attraliases = {
+    'bmc': 'hardwaremanagement.manager',
+    'bmcuser': 'secret.hardwaremanagementuser',
+    'bmcpass': 'secret.hardwaremanagementpassword',
+}
 
 def _mkpath(pathname):
     try:
@@ -128,6 +140,18 @@ def _get_protected_key(keydict, password, paramname):
         raise exc.LockedCredentials("No available decryption key")
 
 
+def _parse_key(keydata, password=None):
+    if keydata.startswith('*unencrypted:'):
+        return base64.b64decode(keydata[13:])
+    elif password:
+        salt, iv, crypt, hmac = [base64.b64decode(x)
+                                 for x in keydata.split('!')]
+        privkey, integkey = _derive_keys(password, salt)
+        return decrypt_value([iv, crypt, hmac], privkey, integkey)
+    raise(exc.LockedCredentials(
+        "Passphrase protected secret requires password"))
+
+
 def _format_key(key, password=None):
     if password is not None:
         salt = os.urandom(32)
@@ -138,6 +162,29 @@ def _format_key(key, password=None):
         return {"unencryptedvalue": key}
 
 
+def _do_notifier(cfg, watcher, callback):
+    try:
+        callback(nodeattribs=watcher['nodeattrs'], configmanager=cfg)
+    except Exception:
+        logException()
+
+
+def logException():
+    global tracelog
+    if tracelog is None:
+        tracelog = confluent.log.Logger('trace')
+    tracelog.log(traceback.format_exc(),
+                 ltype=confluent.log.DataTypes.event,
+                 event=confluent.log.Events.stacktrace)
+
+
+def _do_add_watcher(watcher, added, configmanager):
+    try:
+        watcher(added=added, deleting=[], configmanager=configmanager)
+    except Exception:
+        logException()
+
+
 def init_masterkey(password=None):
     global _masterkey
     global _masterintegritykey
@@ -185,6 +232,49 @@ def decrypt_value(cryptvalue,
     return value[0:-padsize]
 
 
+def fixup_attribute(attrname, attrval):
+    # Normalize some data, for example strings and numbers to bool
+    attrname = _get_valid_attrname(attrname)
+    if attrname not in allattributes.node:  # no fixup possible
+        return attrval
+    if 'type' in allattributes.node[attrname] and not isinstance(attrval, allattributes.node[attrname]['type']):
+        if (allattributes.node[attrname]['type'] == bool and
+                (isinstance(attrval, str) or isinstance(attrval, unicode))):
+            return attrval.lower() in ('true', '1', 'y', 'yes', 'enable', 'enabled')
+    return attrval
+
+
+def attribute_is_invalid(attrname, attrval):
+    if attrname.startswith('custom.'):
+        # No type checking or name checking is provided for custom,
+        # it's not possible
+        return False
+    attrname = _get_valid_attrname(attrname)
+    if attrname not in allattributes.node:
+        # Otherwise, it must be in the allattributes key list
+        return True
+    if 'type' in allattributes.node[attrname]:
+        if not isinstance(attrval, allattributes.node[attrname]['type']):
+            # it is valid if it is {'value': actualvalue}
+            if (isinstance(attrval, dict) and 'value' in attrval and
+                    isinstance(attrval['value'],
+                               allattributes.node[attrname]['type'])):
+                return False
+            # provide type checking for attributes with a specific type
+            return True
+    return False
+
+
+def _get_valid_attrname(attrname):
+    if attrname.startswith('net.'):
+        # For net.* attribtues, split on the dots and put back together
+        # longer term we might want a generic approach, but
+        # right now it's just net. attributes
+        netattrparts = attrname.split('.')
+        attrname = netattrparts[0] + '.' + netattrparts[-1]
+    return attrname
+
+
 def crypt_value(value,
                 key=None,
                 integritykey=None):
@@ -201,7 +291,10 @@ def crypt_value(value,
     neededpad = 16 - (len(value) % 16)
     pad = chr(neededpad) * neededpad
     value += pad
-    cryptval = crypter.encrypt(value)
+    try:
+        cryptval = crypter.encrypt(value)
+    except TypeError:
+        cryptval = crypter.encrypt(value.encode('utf-8'))
     hmac = HMAC.new(integritykey, cryptval, SHA256).digest()
     return iv, cryptval, hmac
 
@@ -214,8 +307,14 @@ def _load_dict_from_dbm(dpath, tdb):
             if elem not in currdict:
                 currdict[elem] = {}
             currdict = currdict[elem]
-        for tk in dbe.iterkeys():
-            currdict[tk] = cPickle.loads(dbe[tk])
+        try:
+            for tk in dbe:
+                currdict[tk] = cPickle.loads(dbe[tk])
+        except AttributeError:
+            tk = dbe.firstkey()
+            while tk != None:
+                currdict[tk] = cPickle.loads(dbe[tk])
+                tk = dbe.nextkey(tk)
     except dbm.error:
         return
 
@@ -233,6 +332,8 @@ def get_global(globalname):
 
     :param globalname: The global parameter name to read
     """
+    if _cfgstore is None:
+        init()
     try:
         return _cfgstore['globals'][globalname]
     except KeyError:
@@ -249,6 +350,8 @@ def set_global(globalname, value):
     :param globalname:  The global parameter name to store
     :param value: The value to set the global parameter to.
     """
+    if _cfgstore is None:
+        init()
     with _dirtylock:
         if 'dirtyglobals' not in _cfgstore:
             _cfgstore['dirtyglobals'] = set()
@@ -319,20 +422,27 @@ class _ExpressionFormat(string.Formatter):
             return node.n
         elif isinstance(node, ast.Attribute):
             #ok, we have something with a dot
-            left = node.value.id
-            right = node.attr
-            key = left + '.' + right
-            if '_expressionkeys' not in self._nodeobj:
-                self._nodeobj['_expressionkeys'] = set([key])
-            else:
-                self._nodeobj['_expressionkeys'].add(key)
-            val = _decode_attribute(key, self._nodeobj,
-                                    formatter=self)
-            return val['value'] if 'value' in val else ""
+            left = node
+            key = ''
+            while isinstance(left, ast.Attribute):
+                # Loop through, to handle multi dot expressions
+                # such as 'net.pxe.hwaddr'
+                key = '.' + left.attr + key
+                left = left.value
+            key = left.id + key
+            if (not key.startswith('custom.') and
+                        _get_valid_attrname(key) not in allattributes.node):
+                raise ValueError(
+                    '{0} is not a valid attribute name'.format(key))
+            val = self._expand_attribute(key)
+            return val['value'] if val and 'value' in val else ""
         elif isinstance(node, ast.Name):
             var = node.id
-            if var == 'nodename':
+            if var in ('node', 'nodename'):
                 return self._nodename
+            if var in _attraliases:
+                val = self._expand_attribute(_attraliases[var])
+                return val['value'] if 'value' in val else ""
             mg = re.match(self.posmatch, var)
             if mg:
                 idx = int(mg.group(1))
@@ -341,20 +451,28 @@ class _ExpressionFormat(string.Formatter):
                 return int(self._numbers[idx - 1])
             else:
                 if var in self._nodeobj:
-                    if '_expressionkeys' not in self._nodeobj:
-                        self._nodeobj['_expressionkeys'] = set([var])
-                    else:
-                        self._nodeobj['_expressionkeys'].add(var)
-                    val = _decode_attribute(var, self._nodeobj,
-                                            formatter=self)
-                    return val['value'] if 'value' in val else ""
+                    val = self._expand_attribute(var)
+                    return val['value'] if val and 'value' in val else ""
+                elif (not var.startswith('custom.') and
+                        _get_valid_attrname(var) not in allattributes.node):
+                    raise ValueError(
+                        '{0} is not a valid attribute name'.format(var))
         elif isinstance(node, ast.BinOp):
             optype = type(node.op)
             if optype not in self._supported_ops:
                 raise Exception("Unsupported operation")
             op = self._supported_ops[optype]
-            return op(self._handle_ast_node(node.left),
-                      self._handle_ast_node(node.right))
+            return op(int(self._handle_ast_node(node.left)),
+                      int(self._handle_ast_node(node.right)))
+
+    def _expand_attribute(self, key):
+        if '_expressionkeys' not in self._nodeobj:
+            self._nodeobj['_expressionkeys'] = set([key])
+        else:
+            self._nodeobj['_expressionkeys'].add(key)
+        val = _decode_attribute(key, self._nodeobj,
+                                formatter=self)
+        return val
 
 
 def _decode_attribute(attribute, nodeobj, formatter=None, decrypt=False):
@@ -409,7 +527,7 @@ def hook_new_configmanagers(callback):
     #TODO(jbjohnso): actually live up to the promise of ongoing callbacks
     callback(ConfigManager(None))
     try:
-        for tenant in _cfgstore['tenant'].iterkeys():
+        for tenant in _cfgstore['tenant']:
             callback(ConfigManager(tenant))
     except KeyError:
         pass
@@ -429,9 +547,12 @@ class ConfigManager(object):
     _nodecollwatchers = {}
     _notifierids = {}
 
-    def __init__(self, tenant, decrypt=False):
+    def __init__(self, tenant, decrypt=False, username=None):
         global _cfgstore
+        if _cfgstore is None:
+            init()
         self.decrypt = decrypt
+        self.current_user = username
         if tenant is None:
             self.tenant = None
             if 'main' not in _cfgstore:
@@ -531,7 +652,9 @@ class ConfigManager(object):
 
     def watch_attributes(self, nodes, attributes, callback):
         """
-        Watch a list of attributes for changes on a list of nodes
+        Watch a list of attributes for changes on a list of nodes.  The
+        attributes may be literal, or a filename style wildcard like
+        'net*.switch'
 
         :param nodes: An iterable of node names to be watching
         :param attributes: An iterable of attribute names to be notified about
@@ -559,6 +682,10 @@ class ConfigManager(object):
                     }
                 else:
                     attribwatchers[node][attribute][notifierid] = callback
+                if '*' in attribute:
+                    currglobs = attribwatchers[node].get('_attrglobs', set([]))
+                    currglobs.add(attribute)
+                    attribwatchers[node]['_attrglobs'] = currglobs
         return notifierid
 
     def watch_nodecollection(self, callback):
@@ -606,7 +733,7 @@ class ConfigManager(object):
 
     def list_users(self):
         try:
-            return self._cfgstore['users'].iterkeys()
+            return list(self._cfgstore['users'])
         except KeyError:
             return []
 
@@ -648,7 +775,7 @@ class ConfigManager(object):
         :param attributemap: The mapping of keys to values to set
         """
 
-        for attribute in attributemap.iterkeys():
+        for attribute in attributemap:
             self._cfgstore['usergroups'][attribute] = attributemap[attribute]
         _mark_dirtykey('usergroups', groupname, self.tenant)
 
@@ -700,6 +827,8 @@ class ConfigManager(object):
         :param uid: Custom identifier number if desired.  Defaults to random.
         :param displayname: Optional long format name for UI consumption
         """
+        if 'idmap' not in _cfgstore['main']:
+            _cfgstore['main']['idmap'] = {}
         if uid is None:
             uid = _generate_new_id()
         else:
@@ -713,8 +842,6 @@ class ConfigManager(object):
         self._cfgstore['users'][name] = {'id': uid}
         if displayname is not None:
             self._cfgstore['users'][name]['displayname'] = displayname
-        if 'idmap' not in _cfgstore['main']:
-            _cfgstore['main']['idmap'] = {}
         _cfgstore['main']['idmap'][uid] = {
             'tenant': self.tenant,
             'username': name
@@ -731,19 +858,23 @@ class ConfigManager(object):
     def is_nodegroup(self, nodegroup):
         return nodegroup in self._cfgstore['nodegroups']
 
-    def get_groups(self):
-        return self._cfgstore['nodegroups'].iterkeys()
+    def get_groups(self, sizesort=False):
+        if sizesort:
+            return reversed(
+                sorted(self._cfgstore['nodegroups'], key=lambda x: len(
+                    self._cfgstore['nodegroups'][x]['nodes'])))
+        return iter(self._cfgstore['nodegroups'])
 
     def list_nodes(self):
         try:
-            return self._cfgstore['nodes'].iterkeys()
+            return iter(self._cfgstore['nodes'])
         except KeyError:
             return []
 
     def get_nodegroup_attributes(self, nodegroup, attributes=()):
         cfgnodeobj = self._cfgstore['nodegroups'][nodegroup]
         if not attributes:
-            attributes = cfgnodeobj.iterkeys()
+            attributes = cfgnodeobj
         nodeobj = {}
         for attribute in attributes:
             if attribute.startswith('_'):
@@ -754,11 +885,23 @@ class ConfigManager(object):
                                                    decrypt=self.decrypt)
         return nodeobj
 
-    def get_node_attributes(self, nodelist, attributes=()):
+    def expand_attrib_expression(self, nodelist, expression):
+        if type(nodelist) in (unicode, str):
+            nodelist = (nodelist,)
+        for node in nodelist:
+            cfgobj = self._cfgstore['nodes'][node]
+            fmt = _ExpressionFormat(cfgobj, node)
+            yield (node, fmt.format(expression))
+
+    def get_node_attributes(self, nodelist, attributes=(), decrypt=None):
+        if decrypt is None:
+            decrypt = self.decrypt
         retdict = {}
-        relattribs = attributes
         if isinstance(nodelist, str) or isinstance(nodelist, unicode):
             nodelist = [nodelist]
+        if isinstance(attributes, str) or isinstance(attributes, unicode):
+            attributes = [attributes]
+        relattribs = attributes
         for node in nodelist:
             if node not in self._cfgstore['nodes']:
                 continue
@@ -770,13 +913,17 @@ class ConfigManager(object):
                 if attribute.startswith('_'):
                     # skip private things
                     continue
+                if '*' in attribute:
+                    for attr in fnmatch.filter(list(cfgnodeobj), attribute):
+                        nodeobj[attr] = _decode_attribute(attr, cfgnodeobj,
+                                                          decrypt=decrypt)
                 if attribute not in cfgnodeobj:
                     continue
                 # since the formatter is not passed in, the calculator is
                 # skipped.  The decryption, however, we want to do only on
                 # demand
                 nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj,
-                                                       decrypt=self.decrypt)
+                                                       decrypt=decrypt)
             retdict[node] = nodeobj
         return retdict
 
@@ -786,7 +933,7 @@ class ConfigManager(object):
             groupcfg = self._cfgstore['nodegroups'][group]
         except KeyError:  # something did not exist, nothing to do
             return
-        for attrib in groupcfg.iterkeys():
+        for attrib in groupcfg:
             self._do_inheritance(nodecfg, attrib, node, changeset)
             _addchange(changeset, node, attrib)
 
@@ -840,7 +987,7 @@ class ConfigManager(object):
                 return
 
     def _sync_groups_to_node(self, groups, node, changeset):
-        for group in self._cfgstore['nodegroups'].iterkeys():
+        for group in self._cfgstore['nodegroups']:
             if group not in groups:
                 if node in self._cfgstore['nodegroups'][group]['nodes']:
                     self._cfgstore['nodegroups'][group]['nodes'].discard(node)
@@ -857,7 +1004,7 @@ class ConfigManager(object):
             self._node_added_to_group(node, group, changeset)
 
     def _sync_nodes_to_group(self, nodes, group, changeset):
-        for node in self._cfgstore['nodes'].iterkeys():
+        for node in self._cfgstore['nodes']:
             if node not in nodes and 'groups' in self._cfgstore['nodes'][node]:
                 if group in self._cfgstore['nodes'][node]['groups']:
                     self._cfgstore['nodes'][node]['groups'].remove(group)
@@ -878,38 +1025,70 @@ class ConfigManager(object):
 
     def set_group_attributes(self, attribmap, autocreate=False):
         changeset = {}
-        for group in attribmap.iterkeys():
+        for group in attribmap:
             if group == '':
                 raise ValueError('"{0}" is not a valid group name'.format(
                     group))
+            if autocreate:
+                try:
+                    noderange._parser.parseString(
+                        '({0})'.format(group)).asList()
+                except noderange.pp.ParseException as pe:
+                    raise ValueError('"{0}" is not a valid group name'.format(
+                        group))
             if not autocreate and group not in self._cfgstore['nodegroups']:
                 raise ValueError("{0} group does not exist".format(group))
-            for attr in attribmap[group].iterkeys():
-                if (attr not in ('nodes', 'noderange') and
-                        (attr not in allattributes.node or
-                         ('type' in allattributes.node[attr] and
-                          not isinstance(attribmap[group][attr],
-                                         allattributes.node[attr]['type'])))):
-                    raise ValueError
+            for attr in attribmap[group]:
+                # first do a pass to normalize out any aliased attribute names
+                if attr in _attraliases:
+                    newattr = _attraliases[attr]
+                    attribmap[group][newattr] = attribmap[group][attr]
+                    del attribmap[group][attr]
+            for attr in attribmap[group]:
+                if attr in _attraliases:
+                    newattr = _attraliases[attr]
+                    attribmap[group][newattr] = attribmap[group][attr]
+                    del attribmap[group][attr]
+                if attr not in ('nodes', 'noderange'):
+                    attrval = fixup_attribute(attr, attribmap[group][attr])
+                    if attribute_is_invalid(attr, attrval):
+                        errstr = "{0} attribute is invalid".format(attr)
+                        raise ValueError(errstr)
+                    attribmap[group][attr] = attrval
                 if attr == 'nodes':
+                    if isinstance(attribmap[group][attr], dict):
+                        currnodes = list(self.get_nodegroup_attributes(
+                            group, ['nodes']).get('nodes', []))
+                        if attribmap[group][attr].get('prepend', False):
+                            newnodes = attribmap[group][attr][
+                                'prepend'].split(',')
+                            attribmap[group][attr] = newnodes + currnodes
+                        elif attribmap[group][attr].get('remove', False):
+                            delnodes = attribmap[group][attr][
+                                'remove'].split(',')
+                            attribmap[group][attr] = [
+                                x for x in currnodes if x not in delnodes]
                     if not isinstance(attribmap[group][attr], list):
-                        raise ValueError(
-                            "nodes attribute on group must be list")
+                        if type(attribmap[group][attr]) is unicode or type(attribmap[group][attr]) is str:
+                            attribmap[group][attr]=attribmap[group][attr].split(",")
+                        else:
+                            raise ValueError("nodes attribute on group must be list")
                     for node in attribmap[group]['nodes']:
                         if node not in self._cfgstore['nodes']:
                             raise ValueError(
                                 "{0} node does not exist to add to {1}".format(
                                     node, group))
-        for group in attribmap.iterkeys():
+        for group in attribmap:
             group = group.encode('utf-8')
             if group not in self._cfgstore['nodegroups']:
                 self._cfgstore['nodegroups'][group] = {'nodes': set()}
             cfgobj = self._cfgstore['nodegroups'][group]
-            for attr in attribmap[group].iterkeys():
+            for attr in attribmap[group]:
                 if attr == 'nodes':
                     newdict = set(attribmap[group][attr])
                 elif (isinstance(attribmap[group][attr], str) or
-                        isinstance(attribmap[group][attr], unicode)):
+                        isinstance(attribmap[group][attr], unicode) or
+                        isinstance(attribmap[group][attr], bool)):
                     newdict = {'value': attribmap[group][attr]}
                 else:
                     newdict = attribmap[group][attr]
@@ -933,6 +1112,13 @@ class ConfigManager(object):
 
     def clear_group_attributes(self, groups, attributes):
         changeset = {}
+        realattributes = []
+        for attrname in list(attributes):
+            if attrname in _attraliases:
+                realattributes.append(_attraliases[attrname])
+            else:
+                realattributes.append(attrname)
+        attributes = realattributes
         if type(groups) in (str, unicode):
             groups = (groups,)
         for group in groups:
@@ -987,7 +1173,7 @@ class ConfigManager(object):
             return
         notifdata = {}
         attribwatchers = self._attribwatchers[self.tenant]
-        for node in nodeattrs.iterkeys():
+        for node in nodeattrs:
             if node not in attribwatchers:
                 continue
             attribwatcher = attribwatchers[node]
@@ -1000,10 +1186,21 @@ class ConfigManager(object):
                 # to deletion, to make all watchers aware of the removed
                 # node and take appropriate action
                 checkattrs = attribwatcher
+            globattrs = {}
+            for attrglob in attribwatcher.get('_attrglobs', []):
+                for matched in fnmatch.filter(list(checkattrs), attrglob):
+                    globattrs[matched] = attrglob
             for attrname in checkattrs:
-                if attrname not in attribwatcher:
+                if attrname == '_attrglobs':
                     continue
-                for notifierid in attribwatcher[attrname].iterkeys():
+                watchkey = attrname
+                # the attrib watcher could still have a glob
+                if attrname not in attribwatcher:
+                    if attrname in globattrs:
+                        watchkey = globattrs[attrname]
+                    else:
+                        continue
+                for notifierid in attribwatcher[watchkey]:
                     if notifierid in notifdata:
                         if node in notifdata[notifierid]['nodeattrs']:
                             notifdata[notifierid]['nodeattrs'][node].append(
@@ -1014,18 +1211,12 @@ class ConfigManager(object):
                     else:
                         notifdata[notifierid] = {
                             'nodeattrs': {node: [attrname]},
-                            'callback': attribwatcher[attrname][notifierid]
+                            'callback': attribwatcher[watchkey][notifierid]
                         }
         for watcher in notifdata.itervalues():
             callback = watcher['callback']
-            try:
-                callback(nodeattribs=watcher['nodeattrs'], configmanager=self)
-            except Exception:
-                global tracelog
-                if tracelog is None:
-                    tracelog = confluent.log.Logger('trace')
-                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
-                             event=log.Events.stacktrace)
+            eventlet.spawn_n(_do_notifier, self, watcher, callback)
+
 
     def del_nodes(self, nodes):
         if self.tenant in self._nodecollwatchers:
@@ -1059,6 +1250,13 @@ class ConfigManager(object):
     def clear_node_attributes(self, nodes, attributes):
         # accumulate all changes into a changeset and push in one go
         changeset = {}
+        realattributes = []
+        for attrname in list(attributes):
+            if attrname in _attraliases:
+                realattributes.append(_attraliases[attrname])
+            else:
+                realattributes.append(attrname)
+        attributes = realattributes
         for node in nodes:
             node = node.encode('utf-8')
             try:
@@ -1085,7 +1283,7 @@ class ConfigManager(object):
         self._bg_sync_to_file()
 
     def add_node_attributes(self, attribmap):
-        for node in attribmap.iterkeys():
+        for node in attribmap:
             if 'groups' not in attribmap[node]:
                 attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
@@ -1098,23 +1296,48 @@ class ConfigManager(object):
         changeset = {}
         # first do a sanity check of the input upfront
         # this mitigates risk of arguments being partially applied
-        for node in attribmap.iterkeys():
+        for node in attribmap:
             node = node.encode('utf-8')
             if node == '':
                 raise ValueError('"{0}" is not a valid node name'.format(node))
+            if autocreate:
+                try:
+                    noderange._parser.parseString(
+                        '({0})'.format(node)).asList()
+                except noderange.pp.ParseException as pe:
+                    raise ValueError(
+                        '"{0}" is not a valid node name'.format(node))
             if autocreate is False and node not in self._cfgstore['nodes']:
                 raise ValueError("node {0} does not exist".format(node))
-            for attrname in attribmap[node].iterkeys():
+            for attrname in list(attribmap[node]):
+                if attrname in _attraliases:
+                    truename = _attraliases[attrname]
+                    attribmap[node][truename] = attribmap[node][attrname]
+                    del attribmap[node][attrname]
+            for attrname in attribmap[node]:
                 attrval = attribmap[node][attrname]
-                if (attrname not in allattributes.node or
-                        ('type' in allattributes.node[attrname] and
-                         not isinstance(
-                             attrval,
-                             allattributes.node[attrname]['type']))):
-                    errstr = "{0} attribute on node {1} is invalid".format(
-                        attrname, node)
-                    raise ValueError(errstr)
+                try:
+                    if (allattributes.node[attrname]['type'] == 'list' and
+                            type(attrval) in (str, unicode)):
+                        attrval = attrval.split(",")
+                except KeyError:
+                    pass
                 if attrname == 'groups':
+                    if isinstance(attribmap[node]['groups'], dict):
+                        currgroups = self.get_node_attributes(
+                            node, 'groups').get(node, {}).get('groups', [])
+                        if attribmap[node]['groups'].get('prepend', False):
+                            newgroups = attribmap[node]['groups'][
+                                'prepend'].split(',')
+                            attribmap[node]['groups'] = newgroups + currgroups
+                        elif attribmap[node]['groups'].get('remove', False):
+                            delgroups = attribmap[node]['groups'][
+                                'remove'].split(',')
+                            newgroups = [
+                                x for x in currgroups if x not in delgroups]
+                            attribmap[node]['groups'] = newgroups
+                    elif type(attribmap[node]['groups']) != list:
+                        attribmap[node]['groups']=attribmap[node]['groups'].split(",")
                     for group in attribmap[node]['groups']:
                         if group not in self._cfgstore['nodegroups']:
                             raise ValueError(
@@ -1122,7 +1345,14 @@ class ConfigManager(object):
                     if ('everything' in self._cfgstore['nodegroups'] and
                             'everything' not in attribmap[node]['groups']):
                         attribmap[node]['groups'].append('everything')
-        for node in attribmap.iterkeys():
+                else:
+                    attrval = fixup_attribute(attrname, attrval)
+                    if attribute_is_invalid(attrname, attrval):
+                        errstr = "{0} attribute on node {1} is invalid".format(
+                            attrname, node)
+                        raise ValueError(errstr)
+                    attribmap[node][attrname] = attrval
+        for node in attribmap:
             node = node.encode('utf-8')
             exprmgr = None
             if node not in self._cfgstore['nodes']:
@@ -1130,9 +1360,10 @@ class ConfigManager(object):
                 self._cfgstore['nodes'][node] = {}
             cfgobj = self._cfgstore['nodes'][node]
             recalcexpressions = False
-            for attrname in attribmap[node].iterkeys():
+            for attrname in attribmap[node]:
                 if (isinstance(attribmap[node][attrname], str) or
-                        isinstance(attribmap[node][attrname], unicode)):
+                        isinstance(attribmap[node][attrname], unicode) or
+                        isinstance(attribmap[node][attrname], bool)):
                     newdict = {'value': attribmap[node][attrname]}
                 else:
                     newdict = attribmap[node][attrname]
@@ -1166,10 +1397,76 @@ class ConfigManager(object):
             if self.tenant in self._nodecollwatchers:
                 nodecollwatchers = self._nodecollwatchers[self.tenant]
                 for watcher in nodecollwatchers.itervalues():
-                    watcher(added=newnodes, deleting=[], configmanager=self)
+                    eventlet.spawn_n(_do_add_watcher, watcher, newnodes, self)
         self._bg_sync_to_file()
         #TODO: wait for synchronization to suceed/fail??)
 
+    def _load_from_json(self, jsondata):
+        """Load fresh configuration data from jsondata
+
+        :param jsondata: String of jsondata
+        :return:
+        """
+        dumpdata = json.loads(jsondata)
+        tmpconfig = {}
+        for confarea in _config_areas:
+            if confarea not in dumpdata:
+                continue
+            tmpconfig[confarea] = {}
+            for element in dumpdata[confarea]:
+                newelement = copy.deepcopy(dumpdata[confarea][element])
+                for attribute in dumpdata[confarea][element]:
+                    if newelement[attribute] == '*REDACTED*':
+                        raise Exception(
+                            "Unable to restore from redacted backup")
+                    elif attribute == 'cryptpass':
+                        passparts = newelement[attribute].split('!')
+                        newelement[attribute] = tuple([base64.b64decode(x)
+                                                       for x in passparts])
+                    elif 'cryptvalue' in newelement[attribute]:
+                        bincrypt = newelement[attribute]['cryptvalue']
+                        bincrypt = tuple([base64.b64decode(x)
+                                          for x in bincrypt.split('!')])
+                        newelement[attribute]['cryptvalue'] = bincrypt
+                    elif attribute in ('nodes', '_expressionkeys'):
+                        # A group with nodes
+                        # delete it and defer until nodes are being added
+                        # which will implicitly fill this up
+                        # Or _expressionkeys attribute, which will similarly
+                        # be rebuilt
+                        del newelement[attribute]
+                tmpconfig[confarea][element] = newelement
+        # We made it through above section without an exception, go ahead and
+        # replace
+        # Start by erasing the dbm files if present
+        for confarea in _config_areas:
+            try:
+                os.unlink(os.path.join(self._cfgdir, confarea))
+            except OSError as e:
+                if e.errno == 2:
+                    pass
+        # Now we have to iterate through each fixed up element, using the
+        # set attribute to flesh out inheritence and expressions
+        _cfgstore['main']['idmap'] = {}
+        for confarea in _config_areas:
+            self._cfgstore[confarea] = {}
+            if confarea not in tmpconfig:
+                continue
+            if confarea == 'nodes':
+                self.set_node_attributes(tmpconfig[confarea], True)
+            elif confarea == 'nodegroups':
+                self.set_group_attributes(tmpconfig[confarea], True)
+            elif confarea == 'users':
+                for user in tmpconfig[confarea]:
+                    uid = tmpconfig[confarea].get('id', None)
+                    displayname = tmpconfig[confarea].get('displayname', None)
+                    self.create_user(user, uid=uid, displayname=displayname)
+                    if 'cryptpass' in tmpconfig[confarea][user]:
+                        self._cfgstore['users'][user]['cryptpass'] = \
+                            tmpconfig[confarea][user]['cryptpass']
+                        _mark_dirtykey('users', user, self.tenant)
+        self._bg_sync_to_file()
+
     def _dump_to_json(self, redact=None):
         """Dump the configuration in json form to output
 
@@ -1188,10 +1485,10 @@ class ConfigManager(object):
             if confarea not in self._cfgstore:
                 continue
             dumpdata[confarea] = {}
-            for element in self._cfgstore[confarea].iterkeys():
+            for element in self._cfgstore[confarea]:
                 dumpdata[confarea][element] = \
                     copy.deepcopy(self._cfgstore[confarea][element])
-                for attribute in self._cfgstore[confarea][element].iterkeys():
+                for attribute in self._cfgstore[confarea][element]:
                     if 'inheritedfrom' in dumpdata[confarea][element][attribute]:
                         del dumpdata[confarea][element][attribute]
                     elif (attribute == 'cryptpass' or
@@ -1237,14 +1534,20 @@ class ConfigManager(object):
             pass
 
     @classmethod
-    def shutdown(cls):
+    def wait_for_sync(cls):
         cls._bg_sync_to_file()
         if cls._cfgwriter is not None:
             cls._cfgwriter.join()
+
+    @classmethod
+    def shutdown(cls):
+        cls.wait_for_sync()
         sys.exit(0)
 
     @classmethod
     def _bg_sync_to_file(cls):
+        if statelessmode:
+            return
         with cls._syncstate:
             if cls._syncrunning:
                 cls._writepending = True
@@ -1258,6 +1561,8 @@ class ConfigManager(object):
 
     @classmethod
     def _sync_to_file(cls):
+        if statelessmode:
+            return
         if 'dirtyglobals' in _cfgstore:
             with _dirtylock:
                 dirtyglobals = copy.deepcopy(_cfgstore['dirtyglobals'])
@@ -1278,7 +1583,7 @@ class ConfigManager(object):
             with _dirtylock:
                 currdirt = copy.deepcopy(_cfgstore['dirtykeys'])
                 del _cfgstore['dirtykeys']
-            for tenant in currdirt.iterkeys():
+            for tenant in currdirt:
                 dkdict = currdirt[tenant]
                 if tenant is None:
                     pathname = cls._cfgdir
@@ -1286,7 +1591,7 @@ class ConfigManager(object):
                 else:
                     pathname = os.path.join(cls._cfgdir, 'tenants', tenant)
                     currdict = _cfgstore['tenant'][tenant]
-                for category in dkdict.iterkeys():
+                for category in dkdict:
                     _mkpath(pathname)
                     dbf = dbm.open(os.path.join(pathname, category), 'c', 384)  # 0600
                     try:
@@ -1309,7 +1614,7 @@ class ConfigManager(object):
             return cls._sync_to_file()
 
     def _recalculate_expressions(self, cfgobj, formatter, node, changeset):
-        for key in cfgobj.iterkeys():
+        for key in cfgobj:
             if not isinstance(cfgobj[key], dict):
                 continue
             if 'expression' in cfgobj[key]:
@@ -1323,39 +1628,95 @@ class ConfigManager(object):
                 self._recalculate_expressions(cfgobj[key], formatter, node,
                                               changeset)
 
+
+def _restore_keys(jsond, password, newpassword=None):
+    # the jsond from the restored file, password (if any) used to protect
+    # the file, and newpassword to use, (also check the service.cfg file)
+    global _masterkey
+    global _masterintegritykey
+    keydata = json.loads(jsond)
+    cryptkey = _parse_key(keydata['cryptkey'], password)
+    integritykey = _parse_key(keydata['integritykey'], password)
+    conf.init_config()
+    cfg = conf.get_config()
+    if cfg.has_option('security', 'externalcfgkey'):
+        keyfilename = cfg.get('security', 'externalcfgkey')
+        with open(keyfilename, 'r') as keyfile:
+            newpassword = keyfile.read()
+    set_global('master_privacy_key', _format_key(cryptkey,
+                                                 password=newpassword))
+    set_global('master_integrity_key', _format_key(integritykey,
+                                                   password=newpassword))
+    _masterkey = cryptkey
+    _masterintegritykey = integritykey
+    ConfigManager.wait_for_sync()
+    # At this point, we should have the key situation all sorted
+
+
 def _dump_keys(password):
     if _masterkey is None or _masterintegritykey is None:
         init_masterkey()
     cryptkey = _format_key(_masterkey, password=password)
-    cryptkey = '!'.join(map(base64.b64encode, cryptkey['passphraseprotected']))
+    if 'passphraseprotected' in cryptkey:
+        cryptkey = '!'.join(map(base64.b64encode,
+                                cryptkey['passphraseprotected']))
+    else:
+        cryptkey = '*unencrypted:{0}'.format(base64.b64encode(
+            cryptkey['unencryptedvalue']))
     integritykey = _format_key(_masterintegritykey, password=password)
-    integritykey = '!'.join(map(base64.b64encode, integritykey['passphraseprotected']))
+    if 'passphraseprotected' in integritykey:
+        integritykey = '!'.join(map(base64.b64encode,
+                                    integritykey['passphraseprotected']))
+    else:
+        integritykey = '*unencrypted:{0}'.format(base64.b64encode(
+            integritykey['unencryptedvalue']))
     return json.dumps({'cryptkey': cryptkey, 'integritykey': integritykey},
                       sort_keys=True, indent=4, separators=(',', ': '))
 
 
+def restore_db_from_directory(location, password):
+    try:
+        with open(os.path.join(location, 'keys.json'), 'r') as cfgfile:
+            keydata = cfgfile.read()
+            json.loads(keydata)
+            _restore_keys(keydata, password)
+    except IOError as e:
+        if e.errno == 2:
+            raise Exception("Cannot restore without keys, this may be a "
+                            "redacted dump")
+    with open(os.path.join(location, 'main.json'), 'r') as cfgfile:
+        cfgdata = cfgfile.read()
+        ConfigManager(tenant=None)._load_from_json(cfgdata)
+
+
 def dump_db_to_directory(location, password, redact=None):
-    with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
-        cfgfile.write(_dump_keys(password))
-        cfgfile.write('\n')
+    if not redact:
+        with open(os.path.join(location, 'keys.json'), 'w') as cfgfile:
+            cfgfile.write(_dump_keys(password))
+            cfgfile.write('\n')
     with open(os.path.join(location, 'main.json'), 'w') as cfgfile:
         cfgfile.write(ConfigManager(tenant=None)._dump_to_json(redact=redact))
         cfgfile.write('\n')
     try:
         for tenant in os.listdir(
                 os.path.join(ConfigManager._cfgdir, '/tenants/')):
-            with open(os.path.join(location, tenant + '.json'), 'w') as cfgfile:
+            with open(os.path.join(location, 'tenants', tenant,
+                                   'main.json'), 'w') as cfgfile:
                 cfgfile.write(ConfigManager(tenant=tenant)._dump_to_json(
                     redact=redact))
                 cfgfile.write('\n')
     except OSError:
         pass
 
-
-try:
-    ConfigManager._read_from_path()
-except IOError:
-    _cfgstore = {}
+def init(stateless=False):
+    global _cfgstore
+    if stateless:
+        _cfgstore = {}
+        return
+    try:
+        ConfigManager._read_from_path()
+    except IOError:
+        _cfgstore = {}
 
 
 # some unit tests worth implementing:

confluent_server/confluent/consoleserver.py

@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
+# Copyright 2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,8 +19,9 @@
 # whatever filehandle is conversing with the client and starts
 # relaying data.  It uses Ctrl-] like telnet for escape back to prompt
 
-#we track nodes that are actively being logged, watched, or have attached
-#there should be no more than one handler per node
+# we track nodes that are actively being logged, watched, or have attached
+# there should be no more than one handler per node
+import codecs
 import collections
 import confluent.config.configmanager as configmodule
 import confluent.exceptions as exc
@@ -29,32 +31,134 @@ import confluent.core as plugin
 import confluent.util as util
 import eventlet
 import eventlet.event
+import pyte
 import random
 import time
 import traceback
 
 _handled_consoles = {}
 
-_genwatchattribs = frozenset(('console.method', 'console.logging'))
-
 _tracelog = None
 
+try:
+    range = xrange
+except NameError:
+    pass
+
+pytecolors2ansi = {
+    'black': 0,
+    'red': 1,
+    'green': 2,
+    'brown': 3,
+    'blue': 4,
+    'magenta': 5,
+    'cyan': 6,
+    'white': 7,
+    'default': 9,
+}
+# might be able to use IBMPC map from pyte charsets,
+# in that case, would have to mask out certain things (like ESC)
+# in the same way that Screen's draw method would do
+# for now at least get some of the arrows in there (note ESC is one
+# of those arrows... so skip it...
+ansichars = dict(zip((0x18, 0x19), u'\u2191\u2193'))
+
+
+def _utf8_normalize(data, shiftin, decoder):
+    # first we give the stateful decoder a crack at the byte stream,
+    # we may come up empty in the event of a partial multibyte
+    try:
+        data = decoder.decode(data)
+    except UnicodeDecodeError:
+        # first order of business is to reset the state of
+        # the decoder to a clean one, so we can switch back to utf-8
+        # when things change, for example going from an F1 setup menu stuck
+        # in the old days to a modern platform using utf-8
+        decoder.setstate(codecs.getincrementaldecoder('utf-8')().getstate())
+        # Ok, so we have something that is not valid UTF-8,
+        # our next stop is to try CP437.  We don't try incremental
+        # decode, since cp437 is single byte
+        # replace is silly here, since there does not exist invalid c437,
+        # but just in case
+        data = data.decode('cp437', 'replace')
+    # Finally, the low part of ascii is valid utf-8, but we are going to be
+    # more interested in the cp437 versions (since this is console *output*
+    # not input
+    if shiftin is None:
+        data = data.translate(ansichars)
+    return data.encode('utf-8')
+
+
+def pytechars2line(chars, maxlen=None):
+    line = '\x1b[m'  # start at default params
+    lb = False  # last bold
+    li = False  # last italic
+    lu = False  # last underline
+    ls = False  # last strikethrough
+    lr = False  # last reverse
+    lfg = 'default'  # last fg color
+    lbg = 'default'   # last bg color
+    hasdata = False
+    len = 1
+    for charidx in range(maxlen):
+        char = chars[charidx]
+        csi = []
+        if char.fg != lfg:
+            csi.append(30 + pytecolors2ansi[char.fg])
+            lfg = char.fg
+        if char.bg != lbg:
+            csi.append(40 + pytecolors2ansi[char.bg])
+            lbg = char.bg
+        if char.bold != lb:
+            lb = char.bold
+            csi.append(1 if lb else 22)
+        if char.italics != li:
+            li = char.italics
+            csi.append(3 if li else 23)
+        if char.underscore != lu:
+            lu = char.underscore
+            csi.append(4 if lu else 24)
+        if char.strikethrough != ls:
+            ls = char.strikethrough
+            csi.append(9 if ls else 29)
+        if char.reverse != lr:
+            lr = char.reverse
+            csi.append(7 if lr else 27)
+        if csi:
+            line += b'\x1b[' + b';'.join(['{0}'.format(x) for x in csi]) + b'm'
+        if not hasdata and char.data.encode('utf-8').rstrip():
+            hasdata = True
+        line += char.data.encode('utf-8')
+        if maxlen and len >= maxlen:
+            break
+        len += 1
+    return line, hasdata
+
+
+class ConsoleHandler(object):
+    _plugin_path = '/nodes/{0}/_console/session'
+    _logtobuffer = True
+    _genwatchattribs = frozenset(('console.method', 'console.logging'))
 
-class _ConsoleHandler(object):
     def __init__(self, node, configmanager):
+        self.clearpending = False
         self._dologging = True
         self._isondemand = False
         self.error = None
-        self.rcpts = {}
+        self._retrytime = 0
         self.cfgmgr = configmanager
         self.node = node
         self.connectstate = 'unconnected'
-        self.clientcount = 0
         self._isalive = True
-        self.logger = log.Logger(node, console=True,
-                                 tenant=configmanager.tenant)
-        self.buffer = bytearray()
-        (text, termstate, timestamp) = self.logger.read_recent_text(8192)
+        self.buffer = pyte.Screen(100, 31)
+        self.termstream = pyte.ByteStream()
+        self.termstream.attach(self.buffer)
+        self.livesessions = set([])
+        self.utf8decoder = codecs.getincrementaldecoder('utf-8')()
+        if self._logtobuffer:
+            self.logger = log.Logger(node, console=True,
+                                     tenant=configmanager.tenant)
+        (text, termstate, timestamp) = (b'', 0, False)
         # when reading from log file, we will use wall clock
         # it should usually match walltime.
         self.lasttime = 0
@@ -66,7 +170,7 @@ class _ConsoleHandler(object):
                 # wall clock has gone backwards, use current time as best
                 # guess
                 self.lasttime = util.monotonic_time()
-        self.buffer += text
+        self.clearbuffer()
         self.appmodedetected = False
         self.shiftin = None
         self.reconnect = None
@@ -79,12 +183,34 @@ class _ConsoleHandler(object):
         self._console = None
         self.connectionthread = None
         self.send_break = None
-        self._attribwatcher = self.cfgmgr.watch_attributes(
-            (self.node,), _genwatchattribs, self._attribschanged)
+        if self._genwatchattribs:
+            self._attribwatcher = self.cfgmgr.watch_attributes(
+                (self.node,), self._genwatchattribs, self._attribschanged)
         self.check_isondemand()
         if not self._isondemand:
+            self.connectstate = 'connecting'
             eventlet.spawn(self._connect)
 
+    def _get_retry_time(self):
+        clustsize = len(self.cfgmgr._cfgstore['nodes'])
+        self._retrytime = self._retrytime * 2 + 1
+        if self._retrytime > 120:
+            self._retrytime = 120
+        retrytime = clustsize * 0.05 * self._retrytime
+        if retrytime > 120:
+            retrytime = 120
+        return retrytime + (retrytime * random.random())
+
+    def feedbuffer(self, data):
+        try:
+            self.termstream.feed(data)
+        except StopIteration:  # corrupt parser state, start over
+            self.termstream = pyte.ByteStream()
+            self.termstream.attach(self.buffer)
+        except Exception:
+            _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                          event=log.Events.stacktrace)
+
     def check_isondemand(self):
         self._dologging = True
         attrvalue = self.cfgmgr.get_node_attributes(
@@ -93,11 +219,12 @@ class _ConsoleHandler(object):
             self._isondemand = False
         elif 'console.logging' not in attrvalue[self.node]:
             self._isondemand = False
-        elif (attrvalue[self.node]['console.logging']['value'] not in (
-                'full', '')):
-            self._isondemand = True
-        elif (attrvalue[self.node]['console.logging']['value']) == 'none':
-            self._dologging = False
+        else:
+            if (attrvalue[self.node]['console.logging']['value'] not in (
+                'full', '', 'buffer')):
+                self._isondemand = True
+            if (attrvalue[self.node]['console.logging']['value']) in ('none', 'memory'):
+                self._dologging = False
 
     def get_buffer_age(self):
         """Return age of buffered data
@@ -132,9 +259,9 @@ class _ConsoleHandler(object):
                     return
             else:
                 self._ondemand()
-                if logvalue == 'none':
+                if logvalue in ('none', 'memory'):
                     self._dologging = False
-        if not self._isondemand or self.clientcount > 0:
+        if not self._isondemand or self.livesessions:
             eventlet.spawn(self._connect)
 
     def log(self, *args, **kwargs):
@@ -151,10 +278,18 @@ class _ConsoleHandler(object):
         else:
             self._console.ping()
 
+    def clearbuffer(self):
+        self.feedbuffer(
+            '\x1bc[no replay buffer due to console.logging attribute set to '
+            'none or interactive,\r\nconnection loss, or service restart]')
+        self.clearpending = True
+
     def _disconnect(self):
         if self.connectionthread:
             self.connectionthread.kill()
             self.connectionthread = None
+        # clear the terminal buffer when disconnected
+        self.clearbuffer()
         if self._console:
             self.log(
                 logdata='console disconnected', ltype=log.DataTypes.event,
@@ -166,7 +301,7 @@ class _ConsoleHandler(object):
 
     def _ondemand(self):
         self._isondemand = True
-        if self.clientcount < 1 and self._console:
+        if not self.livesessions and self._console:
             self._disconnect()
 
     def _connect(self):
@@ -186,12 +321,15 @@ class _ConsoleHandler(object):
             self.reconnect = None
         try:
             self._console = plugin.handle_path(
-                "/nodes/%s/_console/session" % self.node,
+                self._plugin_path.format(self.node),
                 "create", self.cfgmgr)
+        except (exc.NotImplementedException, exc.NotFoundException):
+            self._console = None
         except:
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
         if not isinstance(self._console, conapi.Console):
+            self.clearbuffer()
             self.connectstate = 'unconnected'
             self.error = 'misconfigured'
             self._send_rcpts({'connectstate': self.connectstate,
@@ -202,39 +340,43 @@ class _ConsoleHandler(object):
             self.cfgmgr.remove_watcher(self._attribwatcher)
             self._attribwatcher = None
         if hasattr(self._console, "configattributes"):
-            attribstowatch = self._console.configattributes | _genwatchattribs
+            attribstowatch = self._console.configattributes | self._genwatchattribs
         else:
-            attribstowatch = _genwatchattribs
-        self._attribwatcher = self.cfgmgr.watch_attributes(
-            (self.node,), attribstowatch, self._attribschanged)
+            attribstowatch = self._genwatchattribs
+        if self._genwatchattribs:
+            self._attribwatcher = self.cfgmgr.watch_attributes(
+                (self.node,), attribstowatch, self._attribschanged)
         try:
             self._console.connect(self.get_console_output)
         except exc.TargetEndpointBadCredentials:
+            self.clearbuffer()
             self.error = 'badcredentials'
             self.connectstate = 'unconnected'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
-            retrytime = 30 + (30 * random.random())
+            retrytime = self._get_retry_time()
             if not self.reconnect:
                 self.reconnect = eventlet.spawn_after(retrytime, self._connect)
             return
         except exc.TargetEndpointUnreachable:
+            self.clearbuffer()
             self.error = 'unreachable'
             self.connectstate = 'unconnected'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
-            retrytime = 30 + (30 * random.random())
+            retrytime = self._get_retry_time()
             if not self.reconnect:
                 self.reconnect = eventlet.spawn_after(retrytime, self._connect)
             return
         except Exception:
+            self.clearbuffer()
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
             self.error = 'unknown'
             self.connectstate = 'unconnected'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
-            retrytime = 30 + (30 * random.random())
+            retrytime = self._get_retry_time()
             if not self.reconnect:
                 self.reconnect = eventlet.spawn_after(retrytime, self._connect)
             return
@@ -242,19 +384,23 @@ class _ConsoleHandler(object):
 
     def _got_connected(self):
         self.connectstate = 'connected'
+        self._retrytime = 0
         self.log(
             logdata='console connected', ltype=log.DataTypes.event,
             event=log.Events.consoleconnect)
         self._send_rcpts({'connectstate': self.connectstate})
 
     def _got_disconnected(self):
-        self.connectstate = 'unconnected'
-        self.log(
-            logdata='console disconnected', ltype=log.DataTypes.event,
-            event=log.Events.consoledisconnect)
-        self._send_rcpts({'connectstate': self.connectstate})
+        if self.connectstate != 'unconnected':
+            self.connectstate = 'unconnected'
+            self.log(
+                logdata='console disconnected', ltype=log.DataTypes.event,
+                event=log.Events.consoledisconnect)
+            self._send_rcpts({'connectstate': self.connectstate})
         if self._isalive:
             self._connect()
+        else:
+            self.clearbuffer()
 
     def close(self):
         self._isalive = False
@@ -268,21 +414,22 @@ class _ConsoleHandler(object):
             self.connectionthread.kill()
             self.connectionthread = None
 
-    def unregister_rcpt(self, handle):
-        self.clientcount -= 1
-        if handle in self.rcpts:
-            del self.rcpts[handle]
-        self._send_rcpts({'clientcount': self.clientcount})
-        if self._isondemand and self.clientcount < 1:
-            self._disconnect()
+    def get_console_output(self, data):
+        # Spawn as a greenthread, return control as soon as possible
+        # to the console object
+        eventlet.spawn(self._handle_console_output, data)
 
-    def register_rcpt(self, callback):
-        self.clientcount += 1
-        self._send_rcpts({'clientcount': self.clientcount})
-        hdl = random.random()
-        while hdl in self.rcpts:
-            hdl = random.random()
-        self.rcpts[hdl] = callback
+    def attachsession(self, session):
+        edata = 1
+        for currsession in self.livesessions:
+            if currsession.username == session.username:
+                # indicate that user has multiple connections
+                edata = 2
+        self.livesessions.add(session)
+        self.log(
+            logdata=session.username, ltype=log.DataTypes.event,
+            event=log.Events.clientconnect, eventdata=edata)
+        self._send_rcpts({'clientcount': len(self.livesessions)})
         if self.connectstate == 'unconnected':
             # if console is not connected, take time to try to assert
             # connectivity now.
@@ -292,52 +439,24 @@ class _ConsoleHandler(object):
                 self.reconnect = None
             self.connectstate = 'connecting'
             eventlet.spawn(self._connect)
-        return hdl
 
-    def flushbuffer(self):
-        # Logging is handled in a different stream
-        # this buffer is now just for having screen redraw on
-        # connect
-        self.buffer = bytearray(self.buffer[-8192:])
 
-    def get_console_output(self, data):
-        # Spawn as a greenthread, return control as soon as possible
-        # to the console object
-        eventlet.spawn(self._handle_console_output, data)
 
-    def attachuser(self, username):
-        if username in self.users:
-            self.users[username] += 1
-        else:
-            self.users[username] = 1
-        edata = self.users[username]
-        if edata > 2:  # for log purposes, only need to
-            # clearly indicate redundant connections
-            # not connection count
-            edata = 2
-        if edata < 0:
-            _tracelog.log('client count negative' + traceback.format_exc(),
-                          ltype=log.DataTypes.event,
-                          event=log.Events.stacktrace)
-            edata = 0
+    def detachsession(self, session):
+        edata = 0
+        self.livesessions.discard(session)
+        for currsession in self.livesessions:
+            if currsession.username == session.username:
+                edata += 1
+            if edata > 1:  # don't bother counting beyond 2 in the log
+                break
         self.log(
-            logdata=username, ltype=log.DataTypes.event,
-            event=log.Events.clientconnect, eventdata=edata)
-
-    def detachuser(self, username):
-        self.users[username] -= 1
-        if self.users[username] < 2:
-            edata = self.users[username]
-        else:
-            edata = 2
-        if edata < 0:
-            _tracelog.log('client count negative' + traceback.format_exc(),
-              ltype=log.DataTypes.event,
-              event=log.Events.stacktrace)
-            edata = 0
-        self.log(
-            logdata=username, ltype=log.DataTypes.event,
+            logdata=session.username, ltype=log.DataTypes.event,
             event=log.Events.clientdisconnect, eventdata=edata)
+        self._send_rcpts({'clientcount': len(self.livesessions)})
+        if self._isondemand and not self.livesessions:
+            self._disconnect()
+
 
     def reopen(self):
         self._got_disconnected()
@@ -365,65 +484,73 @@ class _ConsoleHandler(object):
             eventdata |= 2
         self.log(data, eventdata=eventdata)
         self.lasttime = util.monotonic_time()
-        if isinstance(data, bytearray) or isinstance(data, bytes):
-            self.buffer += data
-        else:
-            self.buffer += data.encode('utf-8')
-        #TODO: analyze buffer for registered events, examples:
+        self.feedbuffer(data)
+        # TODO: analyze buffer for registered events, examples:
         #   panics
         #   certificate signing request
-        if len(self.buffer) > 16384:
-            self.flushbuffer()
-        self._send_rcpts(data)
+        if self.clearpending:
+            self.clearpending = False
+            self.feedbuffer(b'\x1bc')
+            self._send_rcpts(b'\x1bc')
+        self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
 
     def _send_rcpts(self, data):
-        for rcpt in self.rcpts.itervalues():
+        for rcpt in list(self.livesessions):
             try:
-                rcpt(data)
+                rcpt.data_handler(data)
             except:  # No matter the reason, advance to next recipient
-                pass
+                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                          event=log.Events.stacktrace)
 
     def get_recent(self):
         """Retrieve 'recent' data
 
         Replay data in the intent to perhaps reproduce the display.
         """
-        #For now, just try to seek back in buffer to find a clear screen
-        #If that fails, just return buffer
-        #a scheme always tracking the last clear screen would be too costly
+        # For now, just try to seek back in buffer to find a clear screen
+        # If that fails, just return buffer
+        # a scheme always tracking the last clear screen would be too costly
         connstate = {
             'connectstate': self.connectstate,
-            'clientcount': self.clientcount,
+            'clientcount': len(self.livesessions),
         }
-        retdata = ''
+        retdata = b'\x1b[H\x1b[J'  # clear screen
+        pendingbl = b''  # pending blank lines
+        maxlen = 0
+        for line in self.buffer.display:
+            line = line.rstrip()
+            if len(line) > maxlen:
+                maxlen = len(line)
+        for line in range(self.buffer.lines):
+            nline, notblank = pytechars2line(self.buffer.buffer[line], maxlen)
+            if notblank:
+                if pendingbl:
+                    retdata += pendingbl
+                    pendingbl = b''
+                retdata += nline + '\r\n'
+            else:
+                pendingbl += nline + '\r\n'
+        if len(retdata) >  6:
+            retdata = retdata[:-2]  # remove the last \r\n
+        retdata += b'\x1b[{0};{1}H'.format(self.buffer.cursor.y + 1,
+                                           self.buffer.cursor.x + 1)
         if self.shiftin is not None:  # detected that terminal requested a
-            #shiftin character set, relay that to the terminal that cannected
+            # shiftin character set, relay that to the terminal that cannected
             retdata += '\x1b)' + self.shiftin
         if self.appmodedetected:
             retdata += '\x1b[?1h'
         else:
             retdata += '\x1b[?1l'
-        #an alternative would be to emulate a VT100 to know what the
-        #whole screen would look like
-        #this is one scheme to clear screen, move cursor then clear
-        bufidx = self.buffer.rfind('\x1b[H\x1b[J')
-        if bufidx >= 0:
-            return retdata + str(self.buffer[bufidx:]), connstate
-        #another scheme is the 2J scheme
-        bufidx = self.buffer.rfind('\x1b[2J')
-        if bufidx >= 0:
-            # there was some sort of clear screen event
-            # somewhere in the buffer, replay from that point
-            # in hopes that it reproduces the screen
-            return retdata + str(self.buffer[bufidx:]), connstate
-        else:
-            #we have no indication of last erase, play back last kibibyte
-            #to give some sense of context anyway
-            return retdata + str(self.buffer[-1024:]), connstate
+        return retdata, connstate
 
     def write(self, data):
         if self.connectstate == 'connected':
-            self._console.write(data)
+            try:
+                self._console.write(data)
+            except Exception:
+                _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
+                              event=log.Events.stacktrace)
+                self._got_disconnected()
 
 
 def disconnect_node(node, configmanager):
@@ -456,13 +583,15 @@ def start_console_sessions():
     configmodule.hook_new_configmanagers(_start_tenant_sessions)
 
 
-def connect_node(node, configmanager):
+def connect_node(node, configmanager, username=None):
     consk = (node, configmanager.tenant)
     if consk not in _handled_consoles:
-        _handled_consoles[consk] = _ConsoleHandler(node, configmanager)
-#this represents some api view of a console handler.  This handles things like
-#holding the caller specific queue data, for example, when http api should be
-#sending data, but there is no outstanding POST request to hold it,
+        _handled_consoles[consk] = ConsoleHandler(node, configmanager)
+    return _handled_consoles[consk]
+
+# this represents some api view of a console handler.  This handles things like
+# holding the caller specific queue data, for example, when http api should be
+# sending data, but there is no outstanding POST request to hold it,
 # this object has the job of holding the data
 
 
@@ -474,7 +603,14 @@ class ConsoleSession(object):
     event watching will all be handled seamlessly
 
     :param node: Name of the node for which this session will be created
+    :param configmanager: A configuration manager object for current context
+    :param username: Username for which this session object will operate
+    :param datacallback: An asynchronous data handler, to be called when data
+                         is available.  Note that if passed, it makes
+                         'get_next_output' non-functional
+    :param skipreplay: If true, will skip the attempt to redraw the screen
     """
+    connector = connect_node
 
     def __init__(self, node, configmanager, username, datacallback=None,
                  skipreplay=False):
@@ -482,30 +618,41 @@ class ConsoleSession(object):
         self.tenant = configmanager.tenant
         if not configmanager.is_node(node):
             raise exc.NotFoundException("Invalid node")
-        consk = (node, self.tenant)
-        self.ckey = consk
         self.username = username
-        connect_node(node, configmanager)
+        self.node = node
+        self.configmanager = configmanager
+        self.connect_session()
         self.registered = True
-        _handled_consoles[consk].attachuser(username)
         self._evt = None
         self.node = node
-        self.conshdl = _handled_consoles[consk]
-        self.write = _handled_consoles[consk].write
+        self.write = self.conshdl.write
         if datacallback is None:
             self.reaper = eventlet.spawn_after(15, self.destroy)
             self.databuffer = collections.deque([])
-            self.reghdl = _handled_consoles[consk].register_rcpt(self.got_data)
+            self.data_handler = self.got_data
             if not skipreplay:
-                self.databuffer.extend(_handled_consoles[consk].get_recent())
+                self.databuffer.extend(self.conshdl.get_recent())
         else:
-            self.reghdl = _handled_consoles[consk].register_rcpt(datacallback)
+            self.data_handler = datacallback
             if not skipreplay:
-                for recdata in _handled_consoles[consk].get_recent():
+                for recdata in self.conshdl.get_recent():
                     if recdata:
                         datacallback(recdata)
+        self.conshdl.attachsession(self)
 
+
+    def connect_session(self):
+        """Connect to the appropriate backend handler
+
+        This is not intended to be called by your usual consumer,
+        it is a hook for confluent to abstract the concept of a terminal
+        between console and shell.
+        """
+        self.conshdl = connect_node(self.node, self.configmanager,
+                                    self.username)
     def send_break(self):
+        """Send break to remote system
+        """
         self.conshdl.send_break()
 
     def get_buffer_age(self):
@@ -515,13 +662,21 @@ class ConsoleSession(object):
         return self.conshdl.get_buffer_age()
 
     def reopen(self):
+        """Reopen the session
+
+        This can be useful if there is suspicion that the remote console is
+        dead.  Note that developers should consider need for this a bug unless
+        there really is some fundamental, unavoidable limitation regarding
+        automatically detecting an unusable console in the underlying
+        technology that cannot be unambiguously autodetected.
+        """
         self.conshdl.reopen()
 
     def destroy(self):
         if self.registered:
-            _handled_consoles[self.ckey].detachuser(self.username)
-            _handled_consoles[self.ckey].unregister_rcpt(self.reghdl)
-        self.databuffer = None
+            self.conshdl.detachsession(self)
+        if self._evt:
+            self._evt.send()
         self._evt = None
         self.reghdl = None
 
@@ -529,19 +684,26 @@ class ConsoleSession(object):
         """Receive data from console and buffer
 
         If the caller does not provide a callback and instead will be polling
-        for data, we must maintain data in a buffer until retrieved
+        for data, we must maintain data in a buffer until retrieved.  This is
+        an internal function used as a means to convert the async behavior to
+        polling for consumers that cannot do the async behavior.
         """
         self.databuffer.append(data)
         if self._evt:
             self._evt.send()
+            self._evt = None
 
     def get_next_output(self, timeout=45):
         """Poll for next available output on this console.
 
         Ideally purely event driven scheme is perfect.  AJAX over HTTP is
-        at least one case where we don't have that luxury
+        at least one case where we don't have that luxury.  This function
+        will not work if the session was initialized with a data callback
+        instead of polling mode.
         """
         self.reaper.cancel()
+        # postpone death to be 15 seconds after this would timeout
+        self.reaper = eventlet.spawn_after(timeout + 15, self.destroy)
         if self._evt:
             raise Exception('get_next_output is not re-entrant')
         if not self.databuffer:
@@ -550,16 +712,12 @@ class ConsoleSession(object):
                 self._evt.wait()
             self._evt = None
         if not self.databuffer:
-            self.reaper = eventlet.spawn_after(15, self.destroy)
             return ""
         currdata = self.databuffer.popleft()
         if isinstance(currdata, dict):
-            self.reaper = eventlet.spawn_after(15, self.destroy)
             return currdata
         retval = currdata
         while self.databuffer and not isinstance(self.databuffer[0], dict):
             retval += self.databuffer.popleft()
-        # the client has 15 seconds to make a new request for data before
-        # they are given up on
-        self.reaper = eventlet.spawn_after(15, self.destroy)
+
         return retval

confluent_server/confluent/core.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,11 +33,14 @@
 # functions.  Console is special and just get's passed through
 # see API.txt
 
+import confluent
 import confluent.alerts as alerts
 import confluent.config.attributes as attrscheme
+import confluent.discovery.core as disco
 import confluent.interface.console as console
 import confluent.exceptions as exc
 import confluent.messages as msg
+import confluent.networking.macmap as macmap
 import confluent.noderange as noderange
 try:
     import confluent.shellmodule as shellmodule
@@ -70,6 +73,7 @@ def nested_lookup(nestdict, key):
 
 def load_plugins():
     # To know our plugins directory, we get the parent path of 'bin'
+    _init_core()
     path = os.path.dirname(os.path.realpath(__file__))
     plugintop = os.path.realpath(os.path.join(path, 'plugins'))
     plugins = set()
@@ -77,7 +81,7 @@ def load_plugins():
         plugindir = os.path.join(plugintop, plugindir)
         if not os.path.isdir(plugindir):
             continue
-        sys.path.append(plugindir)
+        sys.path.insert(1, plugindir)
         # two passes, to avoid adding both py and pyc files
         for plugin in os.listdir(plugindir):
             if plugin.startswith('.'):
@@ -95,9 +99,12 @@ def load_plugins():
                     pluginmap[name] = tmpmod
             else:
                 pluginmap[plugin] = tmpmod
+        # restore path to not include the plugindir
+        sys.path.pop(1)
 
 
-rootcollections = ['noderange/', 'nodes/', 'nodegroups/', 'users/', 'events/']
+rootcollections = ['discovery/', 'events/', 'networking/',
+                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'version']
 
 
 class PluginRoute(object):
@@ -109,147 +116,209 @@ class PluginCollection(object):
     def __init__(self, routedict):
         self.routeinfo = routedict
 
-# _ prefix indicates internal use (e.g. special console scheme) and should not
-# be enumerated in any collection
-noderesources = {
-    'attributes': {
-        'all': PluginRoute({'handler': 'attributes'}),
-        'current': PluginRoute({'handler': 'attributes'}),
-    },
-    'boot': {
-        'nextdevice': PluginRoute({
-            'pluginattrs': ['hardwaremanagement.method'],
-            'default': 'ipmi',
-        }),
-    },
-    'configuration': {
-        'management_controller': {
-            'alerts': {
-                'destinations': PluginCollection({
+def _init_core():
+    global noderesources
+    global nodegroupresources
+    import confluent.shellserver as shellserver
+    # _ prefix indicates internal use (e.g. special console scheme) and should not
+    # be enumerated in any collection
+    noderesources = {
+        'attributes': {
+            'all': PluginRoute({'handler': 'attributes'}),
+            'current': PluginRoute({'handler': 'attributes'}),
+            'expression': PluginRoute({'handler': 'attributes'}),
+        },
+        'boot': {
+            'nextdevice': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+        },
+        'configuration': {
+            'management_controller': {
+                'alerts': {
+                    'destinations': PluginCollection({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
+                'users': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'net_interfaces': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'reset': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'identifier': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'domain_name': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'ntp': {
+                    'enabled': PluginRoute({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                    'servers': PluginCollection({
+                        'pluginattrs': ['hardwaremanagement.method'],
+                        'default': 'ipmi',
+                    }),
+                },
+            },
+            'system': {
+                'all': PluginRoute({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
             },
-            'users': PluginCollection({
+        },
+        '_console': {
+            'session': PluginRoute({
+                'pluginattrs': ['console.method'],
+            }),
+        },
+        '_shell': {
+            'session': PluginRoute({
+                # For now, not configurable, wait until there's demand
+                'handler': 'ssh',
+            }),
+        },
+        '_enclosure': {
+            'reseat_bay': PluginRoute(
+                {'pluginattrs': ['hardwaremanagement.method'],
+                 'default': 'ipmi'}),
+        },
+        'shell': {
+            # another special case similar to console
+            'sessions': PluginCollection({
+                    'handler': shellserver,
+            }),
+        },
+        'console': {
+            # this is a dummy value, http or socket must handle special
+            'session': None,
+            'license': PluginRoute({
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-            'net_interfaces': PluginCollection({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'reset': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'identifier': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'domain_name': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'ntp': {
-                'enabled': PluginRoute({
+        },
+        'events': {
+            'hardware': {
+                'log': PluginRoute({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
-                'servers': PluginCollection({
+                'decode': PluginRoute({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
             },
-        }
-    },
-    '_console': {
-        'session': PluginRoute({
-            'pluginattrs': ['console.method'],
-        }),
-    },
-    'console': {
-        # this is a dummy value, http or socket must handle special
-        'session': PluginRoute({}),
-        'license': PluginRoute({
+        },
+        #'forward': {
+        #    # Another dummy value, currently only for the gui
+        #    'web': None,
+        #},
+        'health': {
+            'hardware': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+        },
+        'identify': PluginRoute({
             'pluginattrs': ['hardwaremanagement.method'],
             'default': 'ipmi',
         }),
-    },
-    'events': {
-        'hardware': {
-            'log': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'decode': PluginRoute({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
+        'inventory': {
+            'hardware': {
+                'all': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+            },
+            'firmware': {
+                'all': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'updates': {
+                    'active': PluginCollection({
+                            'pluginattrs': ['hardwaremanagement.method'],
+                            'default': 'ipmi',
+                    }),
+                },
+            },
         },
-    },
-    'health': {
-        'hardware': PluginRoute({
-            'pluginattrs': ['hardwaremanagement.method'],
-            'default': 'ipmi',
-        }),
-    },
-    'identify': PluginRoute({
-        'pluginattrs': ['hardwaremanagement.method'],
-        'default': 'ipmi',
-    }),
-    'inventory': {
-        'hardware': {
-            'all': PluginCollection({
+        'media': {
+            'uploads': PluginCollection({
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-        },
-        'firmware': {
-            'all': PluginCollection({
+            'attach': PluginRoute({
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-        },
-    },
-    'power': {
-        'state': PluginRoute({
-            'pluginattrs': ['hardwaremanagement.method'],
-            'default': 'ipmi',
-        }),
-    },
-    'sensors': {
-        'hardware': {
-            'all': PluginCollection({
+            'detach': PluginRoute({
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-            'temperature': PluginCollection({
+            'current': PluginRoute({
                 'pluginattrs': ['hardwaremanagement.method'],
                 'default': 'ipmi',
             }),
-            'power': PluginCollection({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'fans': PluginCollection({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-            'leds': PluginCollection({
-                'pluginattrs': ['hardwaremanagement.method'],
-                'default': 'ipmi',
-            }),
-        },
 
-    },
-}
+        },
+        'power': {
+            'state': PluginRoute({
+                'pluginattrs': ['hardwaremanagement.method'],
+                'default': 'ipmi',
+            }),
+            'reseat':  PluginRoute({'handler': 'enclosure'}),
+        },
+        'sensors': {
+            'hardware': {
+                'all': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'energy': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'temperature': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'power': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'fans': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+                'leds': PluginCollection({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
+            },
 
-nodegroupresources = {
-    'attributes': {
-        'all': PluginRoute({'handler': 'attributes'}),
-        'current': PluginRoute({'handler': 'attributes'}),
-    },
-}
+        },
+    }
+
+    nodegroupresources = {
+        'attributes': {
+            'all': PluginRoute({'handler': 'attributes'}),
+            'current': PluginRoute({'handler': 'attributes'}),
+        },
+    }
 
 
 def create_user(inputdata, configmanager):
@@ -302,10 +371,12 @@ def iterate_collections(iterable, forcecollection=True):
 
 
 def iterate_resources(fancydict):
-    for resource in fancydict.iterkeys():
+    for resource in fancydict:
         if resource.startswith("_"):
             continue
-        if not isinstance(fancydict[resource], PluginRoute):  # a resource
+        if resource == 'abbreviate':
+            pass
+        elif not isinstance(fancydict[resource], PluginRoute):  # a resource
             resource += '/'
         yield msg.ChildCollection(resource)
 
@@ -324,11 +395,14 @@ def delete_nodegroup_collection(collectionpath, configmanager):
         raise Exception("Not implemented")
 
 
-def delete_node_collection(collectionpath, configmanager):
+def delete_node_collection(collectionpath, configmanager, isnoderange):
     if len(collectionpath) == 2:  # just node
-        node = collectionpath[-1]
-        configmanager.del_nodes([node])
-        yield msg.DeletedResource(node)
+        nodes = [collectionpath[-1]]
+        if isnoderange:
+            nodes = noderange.NodeRange(nodes[0], configmanager).nodes
+        configmanager.del_nodes(nodes)
+        for node in nodes:
+            yield msg.DeletedResource(node)
     else:
         raise Exception("Not implemented")
 
@@ -336,7 +410,8 @@ def delete_node_collection(collectionpath, configmanager):
 def enumerate_nodegroup_collection(collectionpath, configmanager):
     nodegroup = collectionpath[1]
     if not configmanager.is_nodegroup(nodegroup):
-        raise exc.NotFoundException("Invalid element requested")
+        raise exc.NotFoundException(
+            'Invalid nodegroup: {0} not found'.format(nodegroup))
     del collectionpath[0:2]
     collection = nested_lookup(nodegroupresources, collectionpath)
     return iterate_resources(collection)
@@ -356,6 +431,7 @@ def enumerate_node_collection(collectionpath, configmanager):
     collection = nested_lookup(noderesources, collectionpath[2:])
     if len(collectionpath) == 2 and collectionpath[0] == 'noderange':
         collection['nodes'] = {}
+        collection['abbreviate'] = {}
     if not isinstance(collection, dict):
         raise exc.NotFoundException("Invalid element requested")
     return iterate_resources(collection)
@@ -372,6 +448,7 @@ def create_group(inputdata, configmanager):
         configmanager.add_group_attributes(attribmap)
     except ValueError as e:
         raise exc.InvalidArgumentException(str(e))
+    yield msg.CreatedResource(groupname)
 
 
 def create_node(inputdata, configmanager):
@@ -385,6 +462,25 @@ def create_node(inputdata, configmanager):
         configmanager.add_node_attributes(attribmap)
     except ValueError as e:
         raise exc.InvalidArgumentException(str(e))
+    yield msg.CreatedResource(nodename)
+
+
+def create_noderange(inputdata, configmanager):
+    try:
+        noder = inputdata['name']
+        del inputdata['name']
+        attribmap = {}
+        for node in noderange.NodeRange(noder).nodes:
+            attribmap[node] = inputdata
+    except KeyError:
+        raise exc.InvalidArgumentException('name not specified')
+    try:
+        configmanager.add_node_attributes(attribmap)
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
+    for node in attribmap:
+        yield msg.CreatedResource(node)
+
 
 
 def enumerate_collections(collections):
@@ -399,7 +495,7 @@ def handle_nodegroup_request(configmanager, inputdata,
     if len(pathcomponents) < 2:
         if operation == "create":
             inputdata = msg.InputAttributes(pathcomponents, inputdata)
-            create_group(inputdata.attribs, configmanager)
+            return create_group(inputdata.attribs, configmanager)
         allgroups = list(configmanager.get_groups())
         try:
             allgroups.sort(key=noderange.humanify_nodename)
@@ -438,6 +534,26 @@ def handle_nodegroup_request(configmanager, inputdata,
     raise Exception("unknown case encountered")
 
 
+class BadPlugin(object):
+    def __init__(self, node, plugin):
+        self.node = node
+        self.plugin = plugin
+
+    def error(self, *args, **kwargs):
+        yield msg.ConfluentNodeError(
+            self.node, self.plugin + ' is not a supported plugin')
+
+
+def abbreviate_noderange(configmanager, inputdata, operation):
+    if operation != 'create':
+        raise exc.InvalidArgumentException('Must be a create with nodes in list')
+    if 'nodes' not in inputdata:
+        raise exc.InvalidArgumentException('Must be given list of nodes under key named nodes')
+    if isinstance(inputdata['nodes'], str) or isinstance(inputdata['nodes'], unicode):
+        inputdata['nodes'] = inputdata['nodes'].split(',')
+    return (msg.KeyValueData({'noderange': noderange.ReverseNodeRange(inputdata['nodes'], configmanager).noderange}),)
+
+
 def handle_node_request(configmanager, inputdata, operation,
                         pathcomponents, autostrip=True):
     iscollection = False
@@ -450,6 +566,8 @@ def handle_node_request(configmanager, inputdata, operation,
             # facility anyway
             isnoderange = False
             pathcomponents = pathcomponents[2:]
+        elif len(pathcomponents) == 3 and pathcomponents[2] == 'abbreviate':
+            return abbreviate_noderange(configmanager, inputdata, operation)
         else:
             isnoderange = True
     else:
@@ -458,7 +576,8 @@ def handle_node_request(configmanager, inputdata, operation,
         nodeorrange = pathcomponents[1]
         if not isnoderange and not configmanager.is_node(nodeorrange):
             raise exc.NotFoundException("Invalid Node")
-        if isnoderange:
+        if isnoderange and not (len(pathcomponents) == 3 and
+                                        pathcomponents[2] == 'abbreviate'):
             try:
                 nodes = noderange.NodeRange(nodeorrange, configmanager).nodes
             except Exception as e:
@@ -469,11 +588,14 @@ def handle_node_request(configmanager, inputdata, operation,
         # this is enumerating a list of nodes or just empty noderange
         if isnoderange and operation == "retrieve":
             return iterate_collections([])
+        elif isnoderange and operation == "create":
+            inputdata = msg.InputAttributes(pathcomponents, inputdata)
+            return create_noderange(inputdata.attribs, configmanager)
         elif isnoderange or operation == "delete":
             raise exc.InvalidArgumentException()
         if operation == "create":
             inputdata = msg.InputAttributes(pathcomponents, inputdata)
-            create_node(inputdata.attribs, configmanager)
+            return create_node(inputdata.attribs, configmanager)
         allnodes = list(configmanager.list_nodes())
         try:
             allnodes.sort(key=noderange.humanify_nodename)
@@ -500,9 +622,12 @@ def handle_node_request(configmanager, inputdata, operation,
             iscollection = True
         elif isinstance(routespec, PluginCollection):
             iscollection = False  # it is a collection, but plugin defined
+        elif routespec is None:
+            raise exc.InvalidArgumentException('Custom interface required for resource')
     if iscollection:
         if operation == "delete":
-            return delete_node_collection(pathcomponents, configmanager)
+            return delete_node_collection(pathcomponents, configmanager,
+                                          isnoderange)
         elif operation == "retrieve":
             return enumerate_node_collection(pathcomponents, configmanager)
         else:
@@ -511,15 +636,21 @@ def handle_node_request(configmanager, inputdata, operation,
     passvalues = []
     plugroute = routespec.routeinfo
     inputdata = msg.get_input_message(
-        pathcomponents, operation, inputdata, nodes, isnoderange)
+        pathcomponents, operation, inputdata, nodes, isnoderange,
+        configmanager)
     if 'handler' in plugroute:  # fixed handler definition, easy enough
-        hfunc = getattr(pluginmap[plugroute['handler']], operation)
+        if isinstance(plugroute['handler'], str):
+            hfunc = getattr(pluginmap[plugroute['handler']], operation)
+        else:
+            hfunc = getattr(plugroute['handler'], operation)
         passvalue = hfunc(
             nodes=nodes, element=pathcomponents,
             configmanager=configmanager,
             inputdata=inputdata)
         if isnoderange:
             return passvalue
+        elif isinstance(passvalue, console.Console):
+            return passvalue
         else:
             return stripnode(passvalue, nodes[0])
     elif 'pluginattrs' in plugroute:
@@ -534,7 +665,11 @@ def handle_node_request(configmanager, inputdata, operation,
                 if attrname in nodeattr[node]:
                     plugpath = nodeattr[node][attrname]['value']
             if plugpath is not None:
-                hfunc = getattr(pluginmap[plugpath], operation)
+                try:
+                    hfunc = getattr(pluginmap[plugpath], operation)
+                except KeyError:
+                    nodesbyhandler[BadPlugin(node, plugpath).error] = [node]
+                    continue
                 if hfunc in nodesbyhandler:
                     nodesbyhandler[hfunc].append(node)
                 else:
@@ -546,11 +681,28 @@ def handle_node_request(configmanager, inputdata, operation,
                 inputdata=inputdata))
         if isnoderange or not autostrip:
             return itertools.chain(*passvalues)
-        elif isinstance(passvalues[0], console.Console):
-            return passvalues[0]
         else:
-            return stripnode(passvalues[0], nodes[0])
+            if len(passvalues) > 0:
+                if isinstance(passvalues[0], console.Console):
+                    return passvalues[0]
+                else:
+                    return stripnode(passvalues[0], nodes[0])
+            else:
+                raise exc.NotImplementedException()
 
+        # elif isinstance(passvalues[0], console.Console):
+        #     return passvalues[0]
+        # else:
+        #     return stripnode(passvalues[0], nodes[0])
+
+
+def handle_discovery(pathcomponents, operation, configmanager, inputdata):
+    if pathcomponents[0] == 'detected':
+        pass
+
+def handle_discovery(pathcomponents, operation, configmanager, inputdata):
+    if pathcomponents[0] == 'detected':
+        pass
 
 def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
     """Given a full path request, return an object.
@@ -576,6 +728,14 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
         # single node request of some sort
         return handle_node_request(configmanager, inputdata,
                                    operation, pathcomponents, autostrip)
+    elif pathcomponents[0] == 'discovery':
+        return disco.handle_api_request(
+            configmanager, inputdata, operation, pathcomponents)
+    elif pathcomponents[0] == 'networking':
+        return macmap.handle_api_request(
+            configmanager, inputdata, operation, pathcomponents)
+    elif pathcomponents[0] == 'version':
+        return (msg.Attributes(kv={'version': confluent.__version__}),)
     elif pathcomponents[0] == 'users':
         # TODO: when non-administrator accounts exist,
         # they must only be allowed to see their own user
@@ -584,7 +744,8 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
         except IndexError:  # it's just users/
             if operation == 'create':
                 inputdata = msg.get_input_message(
-                    pathcomponents, operation, inputdata)
+                    pathcomponents, operation, inputdata,
+                    configmanager=configmanager)
                 create_user(inputdata.attribs, configmanager)
             return iterate_collections(configmanager.list_users(),
                                        forcecollection=False)
@@ -596,7 +757,8 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
             return delete_user(user, configmanager)
         elif operation == 'update':
             inputdata = msg.get_input_message(
-                pathcomponents, operation, inputdata)
+                pathcomponents, operation, inputdata,
+                configmanager=configmanager)
             update_user(user, inputdata.attribs, configmanager)
             return show_user(user, configmanager)
     elif pathcomponents[0] == 'events':
@@ -610,5 +772,8 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
             raise exc.NotFoundException()
         if operation == 'update':
             return alerts.decode_alert(inputdata, configmanager)
+    elif pathcomponents[0] == 'discovery':
+        return handle_discovery(pathcomponents[1:], operation, configmanager,
+                                inputdata)
     else:
         raise exc.NotFoundException()

confluent_server/confluent/discovery/handlers/bmc.py

@@ -0,0 +1,157 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.generic as generic
+import confluent.exceptions as exc
+import confluent.netutil as netutil
+import eventlet.support.greendns
+
+# Provide foundation for general IPMI device configuration
+
+import pyghmi.exceptions as pygexc
+ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
+ipmicommand.session.select = eventlet.green.select
+ipmicommand.session.threading = eventlet.green.threading
+ipmicommand.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+DEFAULT_USER = 'USERID'
+DEFAULT_PASS = 'PASSW0RD'
+
+
+class NodeHandler(generic.NodeHandler):
+
+    def _get_ipmicmd(self, user=DEFAULT_USER, password=DEFAULT_PASS):
+        return ipmicommand.Command(self.ipaddr, user, password)
+
+    def __init__(self, info, configmanager):
+        super(NodeHandler, self).__init__(info, configmanager)
+
+    def probe(self):
+        return
+        # TODO(jjohnson2): probe serial number and uuid
+
+    def config(self, nodename, reset=False):
+        self._bmcconfig(nodename, reset)
+
+    def _bmcconfig(self, nodename, reset=False):
+        # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
+        # In general, try to use https automation, to make it consistent
+        # between hypothetical secure path and today.
+        try:
+            ic = self._get_ipmicmd()
+            passwd = DEFAULT_PASS
+        except pygexc.IpmiException as pi:
+            creds = self.configmanager.get_node_attributes(
+                nodename,
+                ['secret.hardwaremanagementuser',
+                 'secret.hardwaremanagementpassword'], decrypt=True)
+            user = creds.get(nodename, {}).get(
+                'secret.hardwaremanagementuser', {}).get('value', None)
+            havecustomcreds = False
+            if user is not None and user != DEFAULT_USER:
+                havecustomcreds = True
+            else:
+                user = DEFAULT_USER
+            passwd = creds.get(nodename, {}).get(
+                'secret.hardwaremanagementpassword', {}).get('value', None)
+            if passwd is not None and passwd != DEFAULT_PASS:
+                havecustomcreds = True
+            else:
+                passwd = DEFAULT_PASS
+            if havecustomcreds:
+                ic = self._get_ipmicmd(user, passwd)
+            else:
+                raise
+        currusers = ic.get_users()
+        lanchan = ic.get_network_channel()
+        userdata = ic.xraw_command(netfn=6, command=0x44, data=(lanchan,
+                                                                1))
+        userdata = bytearray(userdata['data'])
+        maxusers = userdata[0] & 0b111111
+        enabledusers = userdata[1] & 0b111111
+        lockedusers = userdata[2] & 0b111111
+        cfg = self.configmanager
+        cd = cfg.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager'], True)
+        cd = cd.get(nodename, {})
+        if ('secret.hardwaremanagementuser' not in cd or
+                'secret.hardwaremanagementpassword' not in cd):
+            raise exc.TargetEndpointBadCredentials(
+                'secret.hardwaremanagementuser and/or '
+                'secret.hardwaremanagementpassword was not configured')
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            # This getaddrinfo is repeated in get_nic_config, could be
+            # optimized, albeit with a more convoluted api..
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
+            plen = netconfig['prefix']
+            newip = '{0}/{1}'.format(newip, plen)
+            ic.set_net_configuration(ipv4_address=newip,
+                                     ipv4_configuration='static',
+                                     ipv4_gateway=netconfig['ipv4_gateway'])
+        elif self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address')
+        newuser = cd['secret.hardwaremanagementuser']['value']
+        newpass = cd['secret.hardwaremanagementpassword']['value']
+        for uid in currusers:
+            if currusers[uid]['name'] == newuser:
+                # Use existing account that has been created
+                newuserslot = uid
+                break
+        else:
+            newuserslot = lockedusers + 1
+            if newuserslot < 2:
+                newuserslot = 2
+            ic.set_user_name(newuserslot, newuser)
+            ic.set_user_access(newuserslot, lanchan,
+                               privilege_level='administrator')
+        if newpass != passwd:  # don't mess with existing if no change
+            ic.set_user_password(newuserslot, password=newpass)
+        # Now to zap others
+        for uid in currusers:
+            if uid != newuserslot:
+                if uid <= lockedusers:  # we cannot delete, settle for disable
+                    ic.disable_user(uid, 'disable')
+                else:
+                    # lead with the most critical thing, removing user access
+                    ic.set_user_access(uid, channel=None, callback=False,
+                                       link_auth=False, ipmi_msg=False,
+                                       privilege_level='no_access')
+                    # next, try to disable the password
+                    ic.set_user_password(uid, mode='disable', password=None)
+                    # ok, now we can be less paranoid
+                    try:
+                        ic.user_delete(uid)
+                    except pygexc.IpmiException as ie:
+                        if ie.ipmicode != 0xd5:  # some response to the 0xff
+                            # name...
+                            # the user will remain, but that is life
+                            raise
+        if reset:
+            ic.reset_bmc()
+        return ic

confluent_server/confluent/discovery/handlers/generic.py

@@ -0,0 +1,96 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import eventlet
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+
+class NodeHandler(object):
+    https_supported = True
+    is_enclosure = False
+    devname = ''
+    maxmacs = 2  # reasonable default, allowing for common scenario of
+                 # shared nic in theory, but blocking enclosure managers
+                 # and uplink ports
+
+    def __init__(self, info, configmanager):
+        self._certfailreason = None
+        self._fp = None
+        self.info = info
+        self.configmanager = configmanager
+        targsa = [None]
+        # first let us prefer LLA if possible, since that's most stable
+        for sa in info['addresses']:
+            if sa[0].startswith('fe80'):
+                targsa = sa
+                break
+        else:
+            if info.get('addresses', False):
+                targsa = info['addresses'][0]
+        self.ipaddr = targsa[0]
+
+    def scan(self):
+        # Do completely passive things to enhance data.
+        # Probe is permitted to for example attempt a login
+        # scan *only* does what it can without a login attempt
+        return
+
+    def probe(self):
+        # Use appropriate direct strategy to gather data such as
+        # serial number and uuid to flesh out data as needed
+        return
+
+    def preconfig(self):
+        return
+
+    def discoverable_by_switch(self, macs):
+        # Given the number of macs sharing the port, is this handler
+        # appropriate?
+        return macs <= self.maxmacs
+
+    def _savecert(self, certificate):
+        self._fp = certificate
+        return True
+
+    @property
+    def cert_fail_reason(self):
+        if self._certfailreason == 1:
+            return 'refused'
+        elif self._certfailreason == 2:
+            return 'unreachable'
+
+    @property
+    def https_cert(self):
+        if self._fp:
+            return self._fp
+        if ':' in self.ipaddr:
+            ip = '[{0}]'.format(self.ipaddr)
+        else:
+            ip = self.ipaddr
+        wc = webclient.SecureHTTPConnection(ip, verifycallback=self._savecert)
+        try:
+            wc.connect()
+        except IOError as ie:
+            if ie.errno == errno.ECONNREFUSED:
+                self._certfailreason = 1
+                return None
+            elif ie.errno == errno.EHOSTUNREACH:
+                self._certfailreason = 2
+                return None
+            self._certfailreason = 2
+            return None
+        except Exception:
+            self._certfailreason = 2
+            return None
+        return self._fp

confluent_server/confluent/discovery/handlers/imm.py

@@ -0,0 +1,85 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.bmc as bmchandler
+import pyghmi.exceptions as pygexc
+import pyghmi.ipmi.private.util as pygutil
+import string
+import struct
+
+class NodeHandler(bmchandler.NodeHandler):
+    devname = 'IMM'
+
+    def scan(self):
+        slpattrs = self.info.get('attributes', {})
+        self.isdense = False
+        try:
+            ff = slpattrs.get('enclosure-form-factor', [''])[0]
+        except IndexError:
+            return
+        wronguuid = slpattrs.get('node-uuid', [''])[0]
+        if wronguuid:
+            # we need to fix the first three portions of the uuid
+            uuidprefix = wronguuid.split('-')[:3]
+            uuidprefix = struct.pack(
+                '<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+            uuidprefix = uuidprefix[:8] + '-' + uuidprefix[8:12] + '-' + \
+                         uuidprefix[12:16]
+            self.info['uuid'] = uuidprefix + '-' + '-'.join(
+                wronguuid.split('-')[3:])
+            self.info['uuid'] = string.lower(self.info['uuid'])
+        if ff not in ('dense-computing', 'BC2'):
+            # do not probe unless it's a dense platform
+            return
+        self.isdense = True
+        slot = int(slpattrs.get('slot', ['0'])[0])
+        if slot != 0:
+            self.info['enclosure.bay'] = slot
+
+    def probe(self):
+        if self.info.get('enclosure.bay', 0) == 0:
+            self.scan()
+        if self.info.get('enclosure.bay', 0) != 0:
+            # scan has already populated info
+            return
+        ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
+        if ff != 'dense-computing':
+            return
+        try:
+            # we are a dense platform, but the SLP data did not give us slot
+            # attempt to probe using IPMI
+            ipmicmd = self._get_ipmicmd()
+            guiddata = ipmicmd.xraw_command(netfn=6, command=8)
+            self.info['uuid'] = pygutil.decode_wireformat_uuid(
+                guiddata['data']).lower()
+            ipmicmd.oem_init()
+            bayid = ipmicmd._oem.immhandler.get_property(
+                '/v2/cmm/sp/7')
+            if not bayid:
+                return
+            self.info['enclosure.bay'] = int(bayid)
+            smmid = ipmicmd._oem.immhandler.get_property(
+                '/v2/ibmc/smm/chassis/uuid')
+            if not smmid:
+                return
+            smmid = smmid.lower().replace(' ', '')
+            smmid = '{0}-{1}-{2}-{3}-{4}'.format(smmid[:8], smmid[8:12],
+                                                 smmid[12:16], smmid[16:20],
+                                                 smmid[20:])
+            self.info['enclosure.uuid'] = smmid
+            self.info['enclosure.type'] = 'smm'
+        except pygexc.IpmiException as ie:
+            print(repr(ie))
+            raise
+

confluent_server/confluent/discovery/handlers/pxe.py

@@ -0,0 +1,39 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This contains functionality for passive detection and, one day, active
+# response to pxe
+
+
+import confluent.discovery.handlers.generic as generic
+
+class NodeHandler(generic.NodeHandler):
+    https_supported = False
+    is_enclosure = False
+    devname = 'PXE'
+
+    def __init__(self, info, configmanager):
+        self.ipaddr = ''
+        self.cfm = configmanager
+
+    @property
+    def cert_fail_reason(self):
+        return 'unsupported'
+
+    @property
+    def https_cert(self):
+        return None
+
+    def config(self, nodename):
+        return

confluent_server/confluent/discovery/handlers/smm.py

@@ -0,0 +1,56 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.bmc as bmchandler
+import struct
+
+def fixuuid(baduuid):
+    # SMM dumps it out in hex
+    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
+    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode(
+        'hex')
+    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
+    return '-'.join(uuid).lower()
+
+class NodeHandler(bmchandler.NodeHandler):
+    is_enclosure = True
+    devname = 'SMM'
+    maxmacs = 5  # support an enclosure, but try to avoid catching daisy chain
+
+    def scan(self):
+        # the UUID is in a weird order, fix it up to match
+        # ipmi return and property value
+        uuid = self.info.get('attributes', {}).get('uuid', None)
+        if uuid:
+            uuid = fixuuid(uuid[0])
+            self.info['uuid'] = uuid
+
+    def config(self, nodename):
+        # SMM for now has to reset to assure configuration applies
+        super(NodeHandler, self).config(nodename)
+
+# notes for smm:
+# POST to:
+# https://172.30.254.160/data/changepwd
+# oripwd=PASSW0RD&newpwd=Passw0rd!4321
+# got response:
+# <?xml version="1.0" encoding="UTF-8"?><root><statusCode>0-ChangePwd</statusCode><fowardUrl>login.html</fowardUrl><status>ok</status></root>
+# requires relogin
+# https://172.30.254.160/index.html
+# post to:
+# https://172.30.254.160/data/login
+# with body user=USERID&password=Passw0rd!4321
+# yields:
+# <?xml version="1.0" encoding="UTF-8"?><root> <status>ok</status> <authResult>0</authResult> <forwardUrl>index.html</forwardUrl> </root>
+# note forwardUrl, if password change needed, will indicate something else

confluent_server/confluent/discovery/handlers/xcc.py

@@ -0,0 +1,72 @@
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.discovery.handlers.imm as immhandler
+import pyghmi.exceptions as pygexc
+import pyghmi.ipmi.oem.lenovo.imm as imm
+
+
+
+class NodeHandler(immhandler.NodeHandler):
+    devname = 'XCC'
+
+    def preconfig(self):
+        ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
+        if ff not in ('dense-computing', [u'dense-computing']):
+            return
+        # attempt to enable SMM
+        #it's normal to get a 'not supported' (193) for systems without an SMM
+        ipmicmd = None
+        try:
+            ipmicmd = self._get_ipmicmd()
+            ipmicmd.xraw_command(netfn=0x3a, command=0xf1, data=(1,))
+        except pygexc.IpmiException as e:
+            if (e.ipmicode != 193 and 'Unauthorized name' not in str(e) and
+                    'Incorrect password' not in str(e)):
+                # raise an issue if anything other than to be expected
+                raise
+        #TODO: decide how to clean out if important
+        #as it stands, this can step on itself
+        #if ipmicmd:
+        #    ipmicmd.ipmi_session.logout()
+
+    def config(self, nodename, reset=False):
+        # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
+        # In general, try to use https automation, to make it consistent
+        # between hypothetical secure path and today.
+        ic = self._bmcconfig(nodename)
+        ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
+        if ff not in ('dense-computing', [u'dense-computing']):
+            return
+        # Ok, we can get the enclosure uuid now..
+        ic.oem_init()
+        enclosureuuid = ic._oem.immhandler.get_property(
+            '/v2/ibmc/smm/chassis/uuid')
+        enclosureuuid = ic._oem.immhandler.get_property(
+            '/v2/ibmc/smm/chassis/uuid')
+        if enclosureuuid:
+            enclosureuuid = imm.fixup_uuid(enclosureuuid).lower()
+            em = self.configmanager.get_node_attributes(nodename,
+                                                        'enclosure.manager')
+            em = em.get(nodename, {}).get('enclosure.manager', {}).get(
+                'value', None)
+            # ok, set the uuid of the manager...
+            if em:
+                self.configmanager.set_node_attributes(
+                    {em: {'id.uuid': enclosureuuid}})
+
+# TODO(jjohnson2): web based init config for future prevalidated cert scheme
+#    def config(self, nodename):
+#        return
+

confluent_server/confluent/discovery/protocols/pxe.py

@@ -0,0 +1,118 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We can listen to port 69 with SO_REUSEADDR to snoop port 69 *even* if dhcp
+# is running (because the other dhcp servers do it already)
+
+# Goal is to detect and act on a DHCPDISCOVER, without actually having to do
+# any offer
+
+# option 97 = UUID (wireformat)
+
+import eventlet.green.socket as socket
+import struct
+
+pxearchs = {
+    '\x00\x00': 'bios-x86',
+    '\x00\x07': 'uefi-x64',
+    '\x00\x09': 'uefi-x64',
+    '\x00\x0b': 'uefi-aarch64',
+}
+
+
+def decode_uuid(rawguid):
+    lebytes = struct.unpack_from('<IHH', buffer(rawguid[:8]))
+    bebytes = struct.unpack_from('>HHI', buffer(rawguid[8:]))
+    return '{0:08X}-{1:04X}-{2:04X}-{3:04X}-{4:04X}{5:08X}'.format(
+        lebytes[0], lebytes[1], lebytes[2], bebytes[0], bebytes[1], bebytes[2]).lower()
+
+
+def find_info_in_options(rq, optidx):
+    uuid = None
+    arch = None
+    try:
+        while uuid is None or arch is None:
+            if rq[optidx] == 53:  # DHCP message type
+                # we want only length 1 and only discover (type 1)
+                if rq[optidx + 1] != 1 or rq[optidx + 2] != 1:
+                    return uuid, arch
+                optidx += 3
+            elif rq[optidx] == 97:
+                if rq[optidx + 1] != 17:
+                    # 16 bytes of uuid and one reserved byte
+                    return uuid, arch
+                if rq[optidx + 2] != 0:  # the reserved byte should be zero,
+                    # anything else would be a new spec that we don't know yet
+                    return uuid, arch
+                uuid = decode_uuid(rq[optidx + 3:optidx + 19])
+                optidx += 19
+            elif rq[optidx] == 93:
+                if rq[optidx + 1] != 2:
+                    return uuid, arch
+                archraw = bytes(rq[optidx + 2:optidx + 4])
+                if archraw in pxearchs:
+                    arch = pxearchs[archraw]
+                optidx += 4
+            else:
+                optidx += rq[optidx + 1] + 2
+    except IndexError:
+        return uuid, arch
+    return uuid, arch
+
+def snoop(handler):
+    #TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
+    #prominent
+    #TODO(jjohnson2): IP_PKTINFO, recvmsg to get the destination ip, per
+    #proxydhcp.c from xCAT
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4.bind(('', 67))
+    while True:
+        # Just need some delay, picked a prime number so that overlap with other
+        # timers might be reduced, though it really is probably nothing
+        (rq, peer) = net4.recvfrom(9000)
+        # if we have a small packet, just skip, it can't possible hold enough
+        # data and avoids some downstream IndexErrors that would be messy
+        # with try/except
+        if len(rq) < 64:
+            continue
+        rq = bytearray(rq)
+        if rq[0] == 1:  # Boot request
+            addrlen = rq[2]
+            if addrlen > 16 or addrlen == 0:
+                continue
+            netaddr = rq[28:28+addrlen]
+            netaddr = ':'.join(['{0:02x}'.format(x) for x in netaddr])
+            optidx = 0
+            try:
+                optidx = rq.index('\x63\x82\x53\x63') + 4
+            except ValueError:
+                continue
+            uuid, arch = find_info_in_options(rq, optidx)
+            if uuid is None:
+                continue
+            # We will fill out service to have something to byte into,
+            # but the nature of the beast is that we do not have peers,
+            # so that will not be present for a pxe snoop
+            handler({'hwaddr': netaddr, 'uuid': uuid, 'architecture': arch,
+                     'services': ('pxe-client',)})
+
+if __name__ == '__main__':
+    def testsnoop(info):
+        print(repr(info))
+    snoop(testsnoop)
+
+

confluent_server/confluent/discovery/protocols/slp.py

@@ -0,0 +1,545 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.neighutil as neighutil
+import confluent.util as util
+import os
+import random
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import struct
+import subprocess
+
+
+_slp_services = set([
+    'service:management-hardware.IBM:integrated-management-module2',
+    'service:lenovo-smm',
+    'service:management-hardware.Lenovo:lenovo-xclarity-controller',
+    'service:management-hardware.IBM:chassis-management-module',
+    'service:management-hardware.Lenovo:chassis-management-module',
+    'service:io-device.Lenovo:management-module',
+])
+
+# SLP has a lot of ambition that was unfulfilled in practice.
+# So we have a static footer here to always use 'DEFAULT' scope, no LDAP
+# predicates, and no authentication for service requests
+srvreqfooter = b'\x00\x07DEFAULT\x00\x00\x00\x00'
+# An empty instance of the attribute list extension
+# which is defined in RFC 3059, used to indicate support for that capability
+attrlistext = b'\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00'
+
+try:
+    IPPROTO_IPV6 = socket.IPPROTO_IPV6
+except AttributeError:
+    IPPROTO_IPV6 = 41  # Assume Windows value if socket is missing it
+
+
+
+def _parse_slp_header(packet):
+    packet = bytearray(packet)
+    if len(packet) < 16 or packet[0] != 2:
+        # discard packets that are obviously useless
+        return None
+    parsed = {
+        'function': packet[1],
+    }
+    (offset, parsed['xid'], langlen) = struct.unpack('!IHH',
+                                           bytes(b'\x00' + packet[7:14]))
+    parsed['lang'] = packet[14:14 + langlen].decode('utf-8')
+    parsed['payload'] = packet[14 + langlen:]
+    if offset:
+        parsed['offset'] = 14 + langlen
+        parsed['extoffset'] = offset
+    return parsed
+
+
+def _pop_url(payload):
+    urllen = struct.unpack('!H', bytes(payload[3:5]))[0]
+    url = bytes(payload[5:5+urllen]).decode('utf-8')
+    if payload[5+urllen] != 0:
+        raise Exception('Auth blocks unsupported')
+    payload = payload[5+urllen+1:]
+    return url, payload
+
+
+def _parse_SrvRply(parsed):
+    """ Modify passed dictionary to have parsed data
+
+
+    :param parsed:
+    :return:
+    """
+    payload = parsed['payload']
+    ecode, ucount = struct.unpack('!HH', bytes(payload[0:4]))
+    if ecode:
+        parsed['errorcode'] = ecode
+    payload = payload[4:]
+    parsed['urls'] = []
+    while ucount:
+        ucount -= 1
+        url, payload = _pop_url(payload)
+        parsed['urls'].append(url)
+
+
+def _parse_slp_packet(packet, peer, rsps, xidmap):
+    parsed = _parse_slp_header(packet)
+    if not parsed:
+        return
+    addr = peer[0]
+    if '%' in addr:
+        addr = addr[:addr.index('%')]
+    mac = None
+    if addr in neighutil.neightable:
+        identifier = neighutil.neightable[addr]
+        mac = identifier
+    else:
+        identifier = addr
+    if (identifier, parsed['xid']) in rsps:
+        # avoid obviously duplicate entries
+        parsed = rsps[(identifier, parsed['xid'])]
+    else:
+        rsps[(identifier, parsed['xid'])] = parsed
+    if mac and 'hwaddr' not in parsed:
+        parsed['hwaddr'] = mac
+    if parsed['xid'] in xidmap:
+        parsed['services'] = [xidmap[parsed['xid']]]
+    if 'addresses' in parsed:
+        if peer not in parsed['addresses']:
+            parsed['addresses'].append(peer)
+    else:
+        parsed['addresses'] = [peer]
+    if parsed['function'] == 2:  # A service reply
+        _parse_SrvRply(parsed)
+
+
+def _v6mcasthash(srvtype):
+    # The hash algorithm described by RFC 3111
+    nums = bytearray(srvtype.encode('utf-8'))
+    hashval = 0
+    for i in nums:
+        hashval *= 33
+        hashval += i
+        hashval &= 0xffff  # only need to track the lowest 16 bits
+    hashval &= 0x3ff
+    hashval |= 0x1000
+    return '{0:x}'.format(hashval)
+
+
+def _generate_slp_header(payload, multicast, functionid, xid, extoffset=0):
+    if multicast:
+        flags = 0x2000
+    else:
+        flags = 0
+    packetlen = len(payload) + 16  # we have a fixed 16 byte header supported
+    if extoffset:  # if we have an offset, add 16 to account for this function
+        # generating a 16 byte header
+        extoffset += 16
+    if packetlen > 1400:
+        # For now, we aren't intending to support large SLP transmits
+        # raise an exception to help identify if such a requirement emerges
+        raise Exception("TODO: Transmit overflow packets")
+    # We always do SLP v2, and only v2
+    header = bytearray([2, functionid])
+    # SLP uses 24 bit packed integers, so in such places we pack 32 then
+    # discard the high byte
+    header.extend(struct.pack('!IH', packetlen, flags)[1:])
+    # '2' below refers to the length of the language tag
+    header.extend(struct.pack('!IHH', extoffset, xid, 2)[1:])
+    # we only do english (in SLP world, it's not like non-english appears...)
+    header.extend(b'en')
+    return header
+
+def _generate_attr_request(service, xid):
+    service = service.encode('utf-8')
+    payload = bytearray(struct.pack('!HH', 0, len(service)) + service)
+    payload.extend(srvreqfooter)
+    header = _generate_slp_header(payload, False, functionid=6, xid=xid)
+    return header + payload
+
+
+
+def _generate_request_payload(srvtype, multicast, xid, prlist=''):
+    prlist = prlist.encode('utf-8')
+    payload = bytearray(struct.pack('!H', len(prlist)) + prlist)
+    srvtype = srvtype.encode('utf-8')
+    payload.extend(struct.pack('!H', len(srvtype)) + srvtype)
+    payload.extend(srvreqfooter)
+    extoffset = len(payload)
+    payload.extend(attrlistext)
+    header = _generate_slp_header(payload, multicast, functionid=1, xid=xid,
+                                  extoffset=extoffset)
+    return header + payload
+
+
+def _find_srvtype(net, net4, srvtype, addresses, xid):
+    """Internal function to find a single service type
+
+    Helper to do singleton requests to srvtype
+
+    :param net: Socket active
+    :param srvtype: Service type to do now
+    :param addresses:  Pass through of addresses argument from find_targets
+    :return:
+    """
+    if addresses is None:
+        data = _generate_request_payload(srvtype, True, xid)
+        net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+        v6addrs = []
+        v6hash = _v6mcasthash(srvtype)
+        # do 'interface local' and 'link local'
+        # it shouldn't make sense, but some configurations work with interface
+        # local that do not work with link local
+        v6addrs.append(('ff01::1:' + v6hash, 427, 0, 0))
+        v6addrs.append(('ff02::1:' + v6hash, 427, 0, 0))
+        for idx in util.list_interface_indexes():
+            # IPv6 multicast is by index, so lead with that
+            net.setsockopt(IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, idx)
+            for sa in v6addrs:
+                try:
+                    net.sendto(data, sa)
+                except socket.error:
+                    # if we hit an interface without ipv6 multicast,
+                    # this can cause an error, skip such an interface
+                    # case in point, 'lo'
+                    pass
+        for i4 in util.list_ips():
+            if 'broadcast' not in i4:
+                continue
+            addr = i4['addr']
+            bcast = i4['broadcast']
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                           socket.inet_aton(addr))
+            try:
+                net4.sendto(data, ('239.255.255.253', 427))
+            except socket.error as se:
+                # On occasion, multicasting may be disabled
+                # tolerate this scenario and move on
+                if se.errno != 101:
+                    raise
+            net4.sendto(data, (bcast, 427))
+
+
+def _grab_rsps(socks, rsps, interval, xidmap):
+    r, _, _ = select.select(socks, (), (), interval)
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            neighutil.refresh_neigh()
+            _parse_slp_packet(rsp, peer, rsps, xidmap)
+            r, _, _ = select.select(socks, (), (), interval)
+
+
+
+def _parse_attrlist(attrstr):
+    attribs = {}
+    while attrstr:
+        if attrstr[0] == '(':
+            if ')' not in attrstr:
+                attribs['INCOMPLETE'] = True
+                return attribs
+            currattr = attrstr[1:attrstr.index(')')]
+            if '=' not in currattr:  # Not allegedly kosher, but still..
+                currattr = currattr.decode('utf-8')
+                attribs[currattr] = None
+            else:
+                attrname, attrval = currattr.split('=')
+                attrname = attrname.decode('utf-8')
+                attribs[attrname] = []
+                for val in attrval.split(','):
+                    try:
+                        val = val.decode('utf-8')
+                    except UnicodeDecodeError:
+                        val = '*DECODEERROR*'
+                    if val[:3] == '\\FF':  # we should make this bytes
+                        finalval = bytearray([])
+                        for bnum in attrval[3:].split('\\'):
+                            if bnum == '':
+                                continue
+                            finalval.append(int(bnum, 16))
+                        val = finalval
+                        if 'uuid' in attrname and len(val) == 16:
+                            lebytes = struct.unpack_from(
+                                '<IHH', buffer(val[:8]))
+                            bebytes = struct.unpack_from(
+                                '>HHI', buffer(val[8:]))
+                            val = '{0:08X}-{1:04X}-{2:04X}-{3:04X}-' \
+                                  '{4:04X}{5:08X}'.format(
+                                lebytes[0], lebytes[1], lebytes[2], bebytes[0],
+                                bebytes[1], bebytes[2]
+                            ).lower()
+                    attribs[attrname].append(val)
+            attrstr = attrstr[attrstr.index(')'):]
+        elif attrstr[0] == ',':
+            attrstr = attrstr[1:]
+        elif ',' in attrstr:
+            currattr = attrstr[:attrstr.index(',')]
+            attribs[currattr] = None
+            attrstr = attrstr[attrstr.index(','):]
+        else:
+            currattr = attrstr
+            attribs[currattr] = None
+            attrstr = None
+    return attribs
+
+
+def _parse_attrs(data, parsed):
+    headinfo = _parse_slp_header(data)
+    if headinfo['function'] != 7 or headinfo['xid'] != parsed['xid']:
+        return
+    payload = headinfo['payload']
+    if struct.unpack('!H', bytes(payload[:2]))[0] != 0:
+        return
+    length = struct.unpack('!H', bytes(payload[2:4]))[0]
+    attrstr = bytes(payload[4:4+length])
+    parsed['attributes'] = _parse_attrlist(attrstr)
+
+
+def _add_attributes(parsed):
+    attrq = _generate_attr_request(parsed['services'][0], parsed['xid'])
+    target = None
+    # prefer reaching out to an fe80 if present, to be highly robust
+    # in face of network changes
+    for addr in parsed['addresses']:
+        if addr[0].startswith('fe80'):
+            target = addr
+    # however if no fe80 seen, roll with the first available address
+    if not target:
+        target = parsed['addresses'][0]
+    if len(target) == 4:
+        net = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        net.connect(target)
+    except socket.error:
+        return
+    net.sendall(attrq)
+    rsp = net.recv(8192)
+    net.close()
+    _parse_attrs(rsp, parsed)
+
+
+def query_srvtypes(target):
+    """Query the srvtypes advertised by the target
+
+    :param target: A sockaddr tuple (if you get the peer info)
+    """
+    payload = b'\x00\x00\xff\xff\x00\x07DEFAULT'
+    header = _generate_slp_header(payload, False, functionid=9, xid=1)
+    packet = header + payload
+    if len(target) == 2:
+        net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    elif len(target) == 4:
+        net = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+    else:
+        raise Exception('Unrecognized target {0}'.format(repr(target)))
+    tries = 3
+    connected = False
+    while tries and not connected:
+        tries -= 1
+        try:
+            net.connect(target)
+            connected = True
+        except socket.error:
+            pass
+    if not connected:
+        return [u'']
+    net.sendall(packet)
+    rs = net.recv(8192)
+    net.close()
+    parsed = _parse_slp_header(rs)
+    if parsed:
+        payload = parsed['payload']
+        if payload[:2] != '\x00\x00':
+            return
+        stypelen = struct.unpack('!H', bytes(payload[2:4]))[0]
+        stypes = payload[4:4+stypelen].decode('utf-8')
+        return stypes.split(',')
+
+def rescan(handler):
+    known_peers = set([])
+    for scanned in scan():
+        for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                continue
+            if addr in known_peers:
+                break
+            known_peers.add(addr)
+        else:
+            handler(scanned)
+
+
+def snoop(handler):
+    """Watch for SLP activity
+
+    handler will be called with a dictionary of relevant attributes
+
+    :param handler:
+    :return:
+    """
+    active_scan(handler)
+    net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    slpg = socket.inet_pton(socket.AF_INET6, 'ff01::123')
+    slpg2 = socket.inet_pton(socket.AF_INET6, 'ff02::123')
+    for i6idx in util.list_interface_indexes():
+        mreq = slpg + struct.pack('=I', i6idx)
+        net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
+        mreq = slpg2 + struct.pack('=I', i6idx)
+        net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    for i4 in util.list_ips():
+        if 'broadcast' not in i4:
+            continue
+        slpmcast = socket.inet_aton('239.255.255.253') + \
+            socket.inet_aton(i4['addr'])
+        try:
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                            slpmcast)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            # socket in use can occur when aliased ipv4 are encountered
+    net.bind(('', 427))
+    net4.bind(('', 427))
+
+    while True:
+        newmacs = set([])
+        r, _, _ = select.select((net, net4), (), (), 60)
+        # clear known_peers and peerbymacaddress
+        # to avoid stale info getting in...
+        # rely upon the select(0.2) to catch rapid fire and aggregate ip
+        # addresses that come close together
+        # calling code needs to understand deeper context, as snoop
+        # will now yield dupe info over time
+        known_peers = set([])
+        peerbymacaddress = {}
+        neighutil.update_neigh()
+        while r:
+            for s in r:
+                (rsp, peer) = s.recvfrom(9000)
+                ip = peer[0].partition('%')[0]
+                if ip not in neighutil.neightable:
+                    continue
+                if peer in known_peers:
+                    continue
+                known_peers.add(peer)
+                mac = neighutil.neightable[ip]
+                if mac in peerbymacaddress:
+                    peerbymacaddress[mac]['addresses'].append(peer)
+                else:
+                    q = query_srvtypes(peer)
+                    if not q or not q[0]:
+                        # SLP might have started and not ready yet
+                        # ignore for now
+                        known_peers.discard(peer)
+                        continue
+                    # we want to prioritize the very well known services
+                    svcs = []
+                    for svc in q:
+                        if svc in _slp_services:
+                            svcs.insert(0, svc)
+                        else:
+                            svcs.append(svc)
+                    peerbymacaddress[mac] = {
+                        'services': svcs,
+                        'addresses': [peer],
+                    }
+                newmacs.add(mac)
+            r, _, _ = select.select((net, net4), (), (), 0.2)
+        for mac in newmacs:
+            peerbymacaddress[mac]['xid'] = 1
+            _add_attributes(peerbymacaddress[mac])
+            peerbymacaddress[mac]['hwaddr'] = mac
+            handler(peerbymacaddress[mac])
+
+
+def active_scan(handler):
+    known_peers = set([])
+    for scanned in scan():
+        for addr in scanned['addresses']:
+            ip = addr[0].partition('%')[0]  # discard scope if present
+            if ip not in neighutil.neightable:
+                continue
+            if addr in known_peers:
+                break
+            known_peers.add(addr)
+        else:
+            handler(scanned)
+
+
+def scan(srvtypes=_slp_services, addresses=None, localonly=False):
+    """Find targets providing matching requested srvtypes
+
+    This is a generator that will iterate over respondants to the SrvType
+    requested.
+
+    :param srvtypes: An iterable list of the service types to find
+    :param addresses: An iterable of addresses/ranges.  Default is to scan
+                      local network segment using multicast and broadcast.
+                      Each address can be a single address, hyphen-delimited
+                      range, or an IP/CIDR indication of a network.
+    :return: Iterable set of results
+    """
+    net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    # increase RCVBUF to max, mitigate chance of
+    # failure due to full buffer.
+    net.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16777216)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16777216)
+    # SLP is very poor at scanning large counts and managing it, so we
+    # must make the best of it
+    # Some platforms/config default to IPV6ONLY, we are doing IPv4
+    # too, so force it
+    #net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+    # we are going to do broadcast, so allow that...
+    initxid = random.randint(0, 32768)
+    xididx = 0
+    xidmap = {}
+    # First we give fast repsonders of each srvtype individual chances to be
+    # processed, mitigating volume of response traffic
+    rsps = {}
+    for srvtype in srvtypes:
+        xididx += 1
+        _find_srvtype(net, net4, srvtype, addresses, initxid + xididx)
+        xidmap[initxid + xididx] = srvtype
+        _grab_rsps((net, net4), rsps, 0.1, xidmap)
+        # now do a more slow check to work to get stragglers,
+        # but fortunately the above should have taken the brunt of volume, so
+        # reduced chance of many responses overwhelming receive buffer.
+    _grab_rsps((net, net4), rsps, 1, xidmap)
+    # now to analyze and flesh out the responses
+    for id in rsps:
+        if localonly:
+            for addr in rsps[id]['addresses']:
+                if 'fe80' in addr[0]:
+                    break
+            else:
+                continue
+        _add_attributes(rsps[id])
+        del rsps[id]['payload']
+        del rsps[id]['function']
+        del rsps[id]['xid']
+        yield rsps[id]
+
+
+if __name__ == '__main__':
+    def testsnoop(a):
+        print(repr(a))
+    snoop(testsnoop)

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -0,0 +1,232 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Documented somewhat at
+# http://buildingskb.schneider-electric.com/view.php?AID=15197
+
+# Here is the payload of an SSDP 'announce', sent to the multicast v4/v6 1900
+# NOTIFY * HTTP/1.1
+# HOST: 239.255.255.250:1900
+# CACHE-CONTROL: max-age=1800
+# AL: https://172.30.254.151:8080/redfish/v1
+# SERVER: Linux/3.14.28-ltsi Redfish/1.0
+# NT: urn:dmtf-org:service:redfish-rest:1
+# USN: uuid:00000000-0000-0000-0005-000000000001::urn:dmtf-org:service:redfish-rest:1
+# NTS: ssdp:alive
+
+
+import confluent.neighutil as neighutil
+import confluent.util as util
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+import struct
+
+mcastv4addr = '239.255.255.250'
+mcastv6addr = 'ff02::c'
+
+ssdp6mcast = socket.inet_pton(socket.AF_INET6, mcastv6addr)
+smsg = ('M-SEARCH * HTTP/1.1\r\n'
+        'HOST: {0}:1900\r\n'
+        'MAN: "ssdp:discover"\r\n'
+        'ST: {1}\r\n'
+        'MX: 3\r\n\r\n')
+
+
+def scan(services, target=None):
+    for service in services:
+        for rply in _find_service(service, target):
+            yield rply
+
+
+def snoop(handler, byehandler=None):
+    """Watch for SSDP notify messages
+
+    The handler shall be called on any service coming online.
+    byehandler is called whenever a system advertises that it is departing.
+    If no byehandler is specified, byebye messages are ignored.  The handler is
+    given (as possible), the mac address, a list of viable sockaddrs to reference
+    the peer, and the notification type (e.g.
+    'urn:dmtf-org:service:redfish-rest:1'
+
+    :param handler:  A handler for online notifications from network
+    :param byehandler: Optional handler for devices going off the network
+    """
+    # Normally, I like using v6/v4 agnostic socket. However, since we are
+    # dabbling in multicast wizardry here, such sockets can cause big problems,
+    # so we will have two distinct sockets
+    known_peers = set([])
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    for ifidx in util.list_interface_indexes():
+        v6grp = ssdp6mcast + struct.pack('=I', ifidx)
+        net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
+    net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    for i4 in util.list_ips():
+        ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
+                     socket.inet_aton(i4['addr'])
+        net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
+                        ssdp4mcast)
+    net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    net4.bind(('', 1900))
+    net6.bind(('', 1900))
+    peerbymacaddress = {}
+    while True:
+        newmacs = set([])
+        machandlers = {}
+        r, _, _ = select.select((net4, net6), (), (), 60)
+        neighutil.update_neigh()
+        while r:
+            for s in r:
+                (rsp, peer) = s.recvfrom(9000)
+                rsp = rsp.split('\r\n')
+                method, _, _ = rsp[0].split(' ', 2)
+                if method == 'NOTIFY':
+                    ip = peer[0].partition('%')[0]
+                    if ip not in neighutil.neightable:
+                        continue
+                    if peer in known_peers:
+                        continue
+                    mac = neighutil.neightable[ip]
+                    known_peers.add(peer)
+                    newmacs.add(mac)
+                    if mac in peerbymacaddress:
+                        peerbymacaddress[mac]['peers'].append(peer)
+                    else:
+                        peerbymacaddress[mac] = {
+                            'hwaddr': mac,
+                            'peers': [peer],
+                        }
+                        peerdata = peerbymacaddress[mac]
+                        for headline in rsp[1:]:
+                            if not headline:
+                                continue
+                            header, _, value = headline.partition(':')
+                            header = header.strip()
+                            value = value.strip()
+                            if header == 'NT':
+                                peerdata['service'] = value
+                            elif header == 'NTS':
+                                if value == 'ssdp:byebye':
+                                    machandlers[mac] = byehandler
+                                elif value == 'ssdp:alive':
+                                    machandlers[mac] = handler
+            r, _, _ = select.select((net4, net6), (), (), 0.1)
+        for mac in newmacs:
+            thehandler = machandlers.get(mac, None)
+            if thehandler:
+                thehandler(peerbymacaddress[mac])
+
+
+def _find_service(service, target):
+    net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+    net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+    if target:
+        addrs = socket.getaddrinfo(target, 1900, 0, socket.SOCK_DGRAM)
+        for addr in addrs:
+            host = addr[4][0]
+            if addr[0] == socket.AF_INET:
+                net4.sendto(smsg.format(host, service), addr[4])
+            elif addr[0] == socket.AF_INET6:
+                host = '[{0}]'.format(host)
+                net6.sendto(smsg.format(host, service), addr[4])
+    else:
+        net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+        for idx in util.list_interface_indexes():
+            net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF,
+                            idx)
+            try:
+                net6.sendto(smsg.format('[{0}]'.format(mcastv6addr), service
+                                        ), (mcastv6addr, 1900, 0, 0))
+            except socket.error:
+                # ignore interfaces without ipv6 multicast causing error
+                pass
+        for i4 in util.list_ips():
+            if 'broadcast' not in i4:
+                continue
+            addr = i4['addr']
+            bcast = i4['broadcast']
+            net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
+                            socket.inet_aton(addr))
+            net4.sendto(smsg.format(mcastv4addr, service),
+                        (mcastv4addr, 1900))
+            net4.sendto(smsg.format(bcast, service), (bcast, 1900))
+    # SSDP by spec encourages responses to spread out over a 3 second interval
+    # hence we must be a bit more patient
+    r, _, _ = select.select((net4, net6), (), (), 4)
+    peerdata = {}
+    while r:
+        for s in r:
+            (rsp, peer) = s.recvfrom(9000)
+            neighutil.refresh_neigh()
+            _parse_ssdp(peer, rsp, peerdata)
+        r, _, _ = select.select((net4, net6), (), (), 4)
+    for nid in peerdata:
+        yield peerdata[nid]
+
+
+def _parse_ssdp(peer, rsp, peerdata):
+    ip = peer[0].partition('%')[0]
+    nid = ip
+    mac = None
+    if ip in neighutil.neightable:
+        nid = neighutil.neightable[ip]
+        mac = nid
+    headlines = rsp.split('\r\n')
+    try:
+        _, code, _ = headlines[0].split(' ', 2)
+    except ValueError:
+        return
+    myurl = None
+    if code == '200':
+        if nid in peerdata:
+            peerdatum = peerdata[nid]
+        else:
+            peerdatum = {
+                'peers': [peer],
+                'hwaddr': mac,
+            }
+            peerdata[nid] = peerdatum
+        for headline in headlines[1:]:
+            if not headline:
+                continue
+            header, _, value = headline.partition(':')
+            header = header.strip()
+            value = value.strip()
+            if header == 'AL' or header == 'LOCATION':
+                myurl = value
+                if 'urls' not in peerdatum:
+                    peerdatum['urls'] = [value]
+                elif value not in peerdatum['urls']:
+                    peerdatum['urls'].append(value)
+            elif header == 'ST':
+                if 'services' not in peerdatum:
+                    peerdatum['services'] = [value]
+                elif value not in peerdatum['services']:
+                    peerdatum['services'].append(value)
+
+
+
+if __name__ == '__main__':
+
+    for rsp in scan(['urn:dmtf-org:service:redfish-rest:1']):
+        print(repr(rsp))
+    def fun(a):
+        print(repr(a))
+    def byefun(a):
+        print('bye' + repr(a))
+    snoop(fun, byefun)

confluent_server/confluent/exceptions.py

@@ -1,6 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,50 +15,105 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import json
+
 
 class ConfluentException(Exception):
-    pass
+    apierrorcode = 500
+    _apierrorstr = 'Unexpected Error'
+
+    def get_error_body(self):
+        errstr = ' - '.join((self._apierrorstr, str(self)))
+        return json.dumps({'error': errstr })
+
+    @property
+    def apierrorstr(self):
+        if str(self):
+            return self._apierrorstr + ' - ' + str(self)
+        return self._apierrorstr
 
 
 class NotFoundException(ConfluentException):
     # Something that could be construed as a name was not found
     # basically, picture an http error code 404
-    pass
+    apierrorcode = 404
+    _apierrorstr = 'Target not found'
 
 
 class InvalidArgumentException(ConfluentException):
     # Something from the remote client wasn't correct
     # like http code 400
-    pass
+    apierrorcode = 400
+    _apierrorstr = 'Bad Request'
 
 
 class TargetEndpointUnreachable(ConfluentException):
     # A target system was unavailable.  For example, a BMC
     # was unreachable.  http code 504
-    pass
+    apierrorcode = 504
+    _apierrorstr = 'Unreachable Target'
 
 
 class TargetEndpointBadCredentials(ConfluentException):
     # target was reachable, but authentication/authorization
     # failed
-    pass
+    apierrorcode = 502
+    _apierrorstr = 'Bad Credentials'
+
 
 class LockedCredentials(ConfluentException):
     # A request was performed that required a credential, but the credential
     # store is locked
-    pass
+    _apierrorstr = 'Credential store locked'
 
 
 class ForbiddenRequest(ConfluentException):
     # The client request is not allowed by authorization engine
-    pass
+    apierrorcode = 403
+    _apierrorstr = 'Forbidden'
 
 
 class NotImplementedException(ConfluentException):
     # The current configuration/plugin is unable to perform
     # the requested task. http code 501
-    pass
+    apierrorcode = 501
+    _apierrorstr = '501 - Not Implemented'
+
+
 
 class GlobalConfigError(ConfluentException):
     # The configuration in the global config file is not right
-    pass
+    _apierrorstr = 'Global configuration contains an error'
+
+
+class TargetResourceUnavailable(ConfluentException):
+    # This is meant for scenarios like asking to read a sensor that is
+    # currently unavailable.  This may be a persistent or transient state
+    apierrocode = 503
+    _apierrorstr = 'Target Resource Unavailable'
+
+class PubkeyInvalid(ConfluentException):
+    apierrorcode = 502
+    _apierrorstr = '502 - Invalid certificate or key on target'
+
+    def __init__(self, text, certificate, fingerprint, attribname, event):
+        super(PubkeyInvalid, self).__init__(self, text)
+        self.fingerprint = fingerprint
+        self.attrname = attribname
+        bodydata = {'message': text,
+                    'event': event,
+                    'fingerprint': fingerprint,
+                    'fingerprintfield': attribname,
+                    'certificate': base64.b64encode(certificate)}
+        self.errorbody = json.dumps(bodydata)
+
+    def get_error_body(self):
+        return self.errorbody
+
+class LoggedOut(ConfluentException):
+    apierrorcode = 401
+    _apierrorstr = '401 - Logged out'
+
+    def get_error_body(self):
+        return '{"loggedout": 1}'

confluent_server/confluent/firmwaremanager.py

@@ -0,0 +1,131 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# provide managing firmware update process and firmware repository if/when
+# the time comes
+
+import confluent.exceptions as exc
+import confluent.messages as msg
+import eventlet
+
+updatesbytarget = {}
+uploadsbytarget = {}
+updatepool = eventlet.greenpool.GreenPool(256)
+
+
+def execupdate(handler, filename, updateobj, type):
+    try:
+        if type == 'firmware':
+            completion = handler(filename, progress=updateobj.handle_progress,
+                                 bank=updateobj.bank)
+        else:
+            completion = handler(filename, progress=updateobj.handle_progress)
+        if completion is None:
+            completion = 'complete'
+        updateobj.handle_progress({'phase': completion, 'progress': 100.0})
+    except exc.PubkeyInvalid as pi:
+        errstr = 'Certificate mismatch detected, does not match value in ' \
+                 'attribute {0}'.format(pi.attrname)
+        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                   'detail': errstr})
+    except Exception as e:
+        updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
+                                   'detail': str(e)})
+
+class Updater(object):
+    def __init__(self, node, handler, filename, tenant=None, name=None,
+                 bank=None, type='firmware'):
+        self.bank = bank
+        self.node = node
+        self.phase = 'initializing'
+        self.detail = ''
+        self.percent = 0.0
+        self.updateproc = updatepool.spawn(execupdate, handler, filename,
+                                           self, type)
+        if type == 'firmware':
+            myparty = updatesbytarget
+        elif type == 'mediaupload':
+            myparty = uploadsbytarget
+        if (node, tenant) not in myparty:
+            myparty[(node, tenant)] = {}
+        if name is None:
+            name = 1
+            while '{0}'.format(name) in myparty[(node, tenant)]:
+                name += 1
+        self.name = '{0}'.format(name)
+        myparty[(node, tenant)][self.name] = self
+
+    def handle_progress(self, progress):
+        self.phase = progress.get('phase', 'unknown')
+        self.percent = float(progress.get('progress', 100.0))
+        self.detail = progress.get('detail', '')
+
+    def cancel(self):
+        self.updateproc.kill()
+
+    @property
+    def progress(self):
+        return {'phase': self.phase, 'progress': self.percent,
+                'detail': self.detail}
+
+
+def remove_updates(nodes, tenant, element, type='firmware'):
+    if len(element) < 5 and element[:2] != ['media', 'uploads']:
+        raise exc.InvalidArgumentException()
+    upid = element[-1]
+    if type == 'firmware':
+        myparty = updatesbytarget
+    else:
+        myparty = uploadsbytarget
+    for node in nodes:
+        try:
+            upd = myparty[(node, tenant)][upid]
+        except KeyError:
+            raise exc.NotFoundException('No active update matches request')
+        upd.cancel()
+        del myparty[(node, tenant)][upid]
+        yield msg.DeletedResource(
+            'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
+                node, upid))
+
+
+def list_updates(nodes, tenant, element, type='firmware'):
+    showmode = False
+    if type == 'mediaupload':
+        myparty = uploadsbytarget
+        verb = 'upload'
+    else:
+        myparty = updatesbytarget
+        verb = 'update'
+    if type == 'firmware':
+        specificlen = 4
+    else:
+        specificlen = 2
+    if len(element) > specificlen:
+        showmode = True
+        upid = element[-1]
+    for node in nodes:
+        if showmode:
+            try:
+                updater = myparty[(node, tenant)][upid]
+            except KeyError:
+                raise exc.NotFoundException(
+                    'No matching {0} process found'.format(verb))
+            yield msg.KeyValueData(updater.progress, name=node)
+        else:
+            for updateid in myparty.get((node, tenant), {}):
+                yield msg.ChildCollection(updateid)

confluent_server/confluent/forwarder.py

@@ -0,0 +1,119 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#This handles port forwarding for web interfaces on management devices
+#It will also hijack port 3900 and do best effort..
+
+import eventlet
+import eventlet.green.select as select
+import eventlet.green.socket as socket
+forwardersbyclient = {}
+relaysbysession = {}
+sessionsbyip = {}
+ipsbysession = {}
+sockhandler = {}
+vidtargetbypeer = {}
+vidforwarder = None
+
+def handle_connection(incoming, outgoing):
+    while True:
+        r, _, _ = select.select((incoming, outgoing), (), (), 60)
+        for mysock in r:
+            data = mysock.recv(32768)
+            if not data:
+                return
+            if mysock == incoming:
+                outgoing.sendall(data)
+            elif mysock == outgoing:
+                incoming.sendall(data)
+
+
+def forward_port(sock, target, clientip, sessionid):
+    while True:
+        conn, cli = sock.accept()
+        if cli[0] != clientip:
+            conn.close()
+            continue
+        try:
+            client = socket.create_connection((target, 443))
+        except Exception:
+            conn.close()
+            continue
+        if sessionid not in relaysbysession:
+            relaysbysession[sessionid] = {}
+        relaysbysession[sessionid][eventlet.spawn(
+            handle_connection, conn, client)] = conn
+
+
+def forward_video():
+    sock = eventlet.listen(('::', 3900, 0, 0), family=socket.AF_INET6)
+    while True:
+        conn, cli = sock.accept()
+        if cli[0] not in vidtargetbypeer or not sessionsbyip.get(cli[0], None):
+            conn.close()
+            continue
+        try:
+            vidclient = socket.create_connection((vidtargetbypeer[cli[0]],
+                                                  3900))
+        except Exception:
+            conn.close()
+            continue
+        eventlet.spawn_n(handle_connection, conn, vidclient)
+
+
+def close_session(sessionid):
+    for addr in forwardersbyclient.get(sessionid, []):
+        killsock = forwardersbyclient[sessionid][addr]
+        sockhandler[killsock].kill()
+        del sockhandler[killsock]
+        killsock.close()
+    if sessionid in forwardersbyclient:
+        del forwardersbyclient[sessionid]
+    for clip in ipsbysession.get(sessionid, ()):
+        sessionsbyip[clip].discard(sessionid)
+    if sessionid in ipsbysession:
+        del ipsbysession[sessionid]
+    for relay in list(relaysbysession.get(sessionid, ())):
+        conn = relaysbysession[sessionid][relay]
+        relay.kill()
+        conn.close()
+    if sessionid in relaysbysession:
+        del relaysbysession[sessionid]
+
+
+def get_port(addr, clientip, sessionid):
+    global vidforwarder
+    if socket.getaddrinfo(clientip, 0)[0][0] == socket.AF_INET:
+        clientip = '::ffff:' + clientip
+    if sessionid not in ipsbysession:
+        ipsbysession[sessionid] = set([])
+    if clientip not in sessionsbyip:
+        sessionsbyip[clientip] = set([])
+    sessionsbyip[clientip].add(sessionid)
+    ipsbysession[sessionid].add(clientip)
+    if sessionid not in forwardersbyclient:
+        forwardersbyclient[sessionid] = {}
+    if addr not in forwardersbyclient[sessionid]:
+        newsock = eventlet.listen(('::', 0, 0, 0),
+                  family=socket.AF_INET6)
+        forwardersbyclient[sessionid][addr] = newsock
+        sockhandler[newsock] = eventlet.spawn(forward_port, newsock, addr,
+                                              clientip, sessionid)
+        if not vidforwarder:
+            vidforwarder = eventlet.spawn(forward_video)
+    vidtargetbypeer[clientip] = addr
+    return forwardersbyclient[sessionid][addr].getsockname()[1]
+

confluent_server/confluent/httpapi.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2016 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,16 +21,22 @@ import Cookie
 import confluent.auth as auth
 import confluent.config.attributes as attribs
 import confluent.consoleserver as consoleserver
+import confluent.forwarder as forwarder
 import confluent.exceptions as exc
 import confluent.log as log
 import confluent.messages
 import confluent.core as pluginapi
+import confluent.asynchttp
+import confluent.shellserver as shellserver
 import confluent.tlvdata
 import confluent.util as util
 import copy
 import eventlet
+import eventlet.greenthread
+import greenlet
 import json
 import socket
+import sys
 import traceback
 import time
 import urlparse
@@ -42,6 +48,7 @@ tlvdata = confluent.tlvdata
 auditlog = None
 tracelog = None
 consolesessions = {}
+confluent.asynchttp.set_console_sessions(consolesessions)
 httpsessions = {}
 opmap = {
     'POST': 'create',
@@ -140,12 +147,19 @@ create_resource_functions = {
 def _sessioncleaner():
     while True:
         currtime = time.time()
-        for session in httpsessions.keys():
+        targsessions = []
+        for session in httpsessions:
             if httpsessions[session]['expiry'] < currtime:
-                del httpsessions[session]
-        for session in consolesessions.keys():
+                targsessions.append(session)
+        for session in targsessions:
+            forwarder.close_session(session)
+            del httpsessions[session]
+        targsessions = []
+        for session in consolesessions:
             if consolesessions[session]['expiry'] < currtime:
-                del consolesessions[session]
+                targsessions.append(session)
+        for session in targsessions:
+            del consolesessions[session]
         eventlet.sleep(10)
 
 
@@ -183,6 +197,69 @@ def _get_query_dict(env, reqbody, reqtype):
         qdict = nqdict
     return qdict
 
+def _should_skip_authlog(env):
+    if ('/console/session' in env['PATH_INFO'] or
+            '/shell/sessions/' in env['PATH_INFO']):
+        # we should only log starting of a console
+        return True
+    if '/sessions/current/async' in env['PATH_INFO']:
+        # this is effectively invisible
+        return True
+    if (env['REQUEST_METHOD'] == 'GET' and
+            ('/sensors/' in env['PATH_INFO'] or
+             '/health/' in env['PATH_INFO'] or
+             '/power/state' in env['PATH_INFO'] or
+             '/nodes/' == env['PATH_INFO'] or
+             '/sessions/current/info' == env['PATH_INFO'] or
+                 (env['PATH_INFO'].startswith('/noderange/') and
+                  env['PATH_INFO'].endswith('/nodes/')))):
+        # these are pretty innocuous, and noisy to log.
+        return True
+    return False
+
+
+def _csrf_exempt(path):
+    # first a get of info to get CSRF key, also '/forward/web' to enable
+    # the popup ability to just forward
+    return path == '/sessions/current/info' or path.endswith('/forward/web')
+
+
+def _csrf_valid(env, session):
+    # This could be simplified into a statement, but this is more readable
+    # to have it broken out
+    if env['REQUEST_METHOD'] == 'GET' and _csrf_exempt(env['PATH_INFO']):
+        # Provide a web client a safe hook to request the CSRF token
+        # This means that we consider GET of /sessions/current/info to be
+        # a safe thing to inflict via CSRF, since CORS should prevent
+        # hypothetical attacker from reading the data and it has no
+        # side effects to speak of
+        return True
+    if 'csrftoken' not in session:
+        # The client has not (yet) requested CSRF protection
+        # so we return true
+        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
+            # The client has requested CSRF countermeasures,
+            # oblige the request and apply a new token to the
+            # session
+            session['csrftoken'] = util.randomstring(32)
+        elif 'HTTP_REFERER' in env:
+            # If there is a referrer, make sure it stays consistent
+            # across the session.  A change in referer is a bad thing
+            try:
+                referer = env['HTTP_REFERER'].split('/')[2]
+            except IndexError:
+                return False
+            if 'validreferer' not in session:
+                session['validreferer'] = referer
+            elif session['validreferer'] != referer:
+                return False
+        return True
+    # The session has CSRF protection enabled, only mark valid if
+    # the client has provided an auth token and that token matches the
+    # value protecting the session
+    return ('HTTP_CONFLUENTAUTHTOKEN' in env and
+            env['HTTP_CONFLUENTAUTHTOKEN'] == session['csrftoken'])
+
 
 def _authorize_request(env, operation):
     """Grant/Deny access based on data from wsgi env
@@ -190,6 +267,7 @@ def _authorize_request(env, operation):
     """
     authdata = None
     name = ''
+    sessionid = None
     cookie = Cookie.SimpleCookie()
     if 'HTTP_COOKIE' in env:
         #attempt to use the cookie.  If it matches
@@ -197,13 +275,32 @@ def _authorize_request(env, operation):
         cc.load(env['HTTP_COOKIE'])
         if 'confluentsessionid' in cc:
             sessionid = cc['confluentsessionid'].value
+            sessid = sessionid
             if sessionid in httpsessions:
-                httpsessions[sessionid]['expiry'] = time.time() + 90
-                name = httpsessions[sessionid]['name']
-                authdata = auth.authorize(
-                    name, element=None,
-                    skipuserobj=httpsessions[sessionid]['skipuserobject'])
+                if _csrf_valid(env, httpsessions[sessionid]):
+                    if env['PATH_INFO'] == '/sessions/current/logout':
+                        targets = []
+                        for mythread in httpsessions[sessionid]['inflight']:
+                            targets.append(mythread)
+                        for mythread in targets:
+                            eventlet.greenthread.kill(mythread)
+                        forwarder.close_session(sessionid)
+                        del httpsessions[sessionid]
+                        return ('logout',)
+                    httpsessions[sessionid]['expiry'] = time.time() + 90
+                    name = httpsessions[sessionid]['name']
+                    authdata = auth.authorize(
+                        name, element=None,
+                        skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
+        if env['PATH_INFO'] == '/sessions/current/logout':
+            if 'HTTP_REFERER' in env:
+                # note that this doesn't actually do harm
+                # otherwise, but this way do not give appearance
+                # of something having a side effect if it has the smell
+                # of a CSRF
+                return {'code': 401}
+            return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
         authdata = auth.check_user_passphrase(name, passphrase, element=None)
@@ -213,14 +310,15 @@ def _authorize_request(env, operation):
         while sessid in httpsessions:
             sessid = util.randomstring(32)
         httpsessions[sessid] = {'name': name, 'expiry': time.time() + 90,
-                                'skipuserobject': authdata[4]}
+                                'skipuserobject': authdata[4],
+                                'inflight': set([])}
+        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
+            httpsessions[sessid]['csrftoken'] = util.randomstring(32)
         cookie['confluentsessionid'] = sessid
         cookie['confluentsessionid']['secure'] = 1
         cookie['confluentsessionid']['httponly'] = 1
         cookie['confluentsessionid']['path'] = '/'
-    skiplog = False
-    if '/console/session' in env['PATH_INFO']:
-        skiplog = True
+    skiplog = _should_skip_authlog(env)
     if authdata:
         auditmsg = {
             'user': name,
@@ -236,8 +334,12 @@ def _authorize_request(env, operation):
             auditmsg['tenant'] = authdata[3]
             authinfo['tenant'] = authdata[3]
         auditmsg['user'] = authdata[2]
+        if sessid is not None:
+            authinfo['sessionid'] = sessid
         if not skiplog:
             auditlog.log(auditmsg)
+        if 'csrftoken' in httpsessions[sessid]:
+            authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
         return authinfo
     else:
         return {'code': 401}
@@ -263,7 +365,7 @@ def _pick_mimetype(env):
         return 'application/json; charset=utf-8', '.json'
     elif env['PATH_INFO'].endswith('.html'):
         return 'text/html', '.html'
-    elif 'application/json' in env['HTTP_ACCEPT']:
+    elif 'HTTP_ACCEPT' in env and 'application/json' in env['HTTP_ACCEPT']:
         return 'application/json; charset=utf-8', ''
     else:
         return 'text/html', ''
@@ -271,7 +373,7 @@ def _pick_mimetype(env):
 
 def _assign_consessionid(consolesession):
     sessid = util.randomstring(32)
-    while sessid in consolesessions.keys():
+    while sessid in consolesessions:
         sessid = util.randomstring(32)
     consolesessions[sessid] = {'session': consolesession,
                                'expiry': time.time() + 60}
@@ -294,6 +396,13 @@ def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
     mimetype, extension = _pick_mimetype(env)
+    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-store'),
+               ('Pragma', 'no-cache'),
+               ('X-Content-Type-Options', 'nosniff'),
+               ('Content-Security-Policy', "default-src 'self'"),
+               ('X-XSS-Protection', '1; mode=block'), ('X-Frame-Options', 'deny'),
+               ('Strict-Transport-Security', 'max-age=86400'),
+               ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
     reqtype = None
     if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
@@ -305,30 +414,74 @@ def resourcehandler_backend(env, start_response):
         operation = querydict['restexplorerop']
         del querydict['restexplorerop']
     authorized = _authorize_request(env, operation)
+    if 'logout' in authorized:
+        start_response('200 Successful logout', headers)
+        yield('{"result": "200 - Successful logout"}')
+        return
+    if 'HTTP_SUPPRESSAUTHHEADER' in env or 'HTTP_CONFLUENTAUTHTOKEN' in env:
+        badauth = [('Content-type', 'text/plain')]
+    else:
+        badauth = [('Content-type', 'text/plain'),
+                   ('WWW-Authenticate', 'Basic realm="confluent"')]
     if authorized['code'] == 401:
-        start_response(
-            '401 Authentication Required',
-            [('Content-type', 'text/plain'),
-             ('WWW-Authenticate', 'Basic realm="confluent"')])
+        start_response('401 Authentication Required', badauth)
         yield 'authentication required'
         return
     if authorized['code'] == 403:
-        start_response(
-            '403 Forbidden',
-            [('Content-type', 'text/plain'),
-             ('WWW-Authenticate', 'Basic realm="confluent"')])
+        start_response('403 Forbidden', badauth)
         yield 'authorization failed'
         return
     if authorized['code'] != 200:
         raise Exception("Unrecognized code from auth engine")
-    headers = [('Content-Type', mimetype)]
     headers.extend(
         ("Set-Cookie", m.OutputString())
         for m in authorized['cookie'].values())
     cfgmgr = authorized['cfgmgr']
-    if '/console/session' in env['PATH_INFO']:
+    if (operation == 'create') and env['PATH_INFO'] == '/sessions/current/async':
+        pagecontent = ""
+        try:
+            for rsp in _assemble_json(
+                    confluent.asynchttp.handle_async(
+                            env, querydict,
+                            httpsessions[authorized['sessionid']]['inflight'])):
+                pagecontent += rsp
+            start_response("200 OK", headers)
+            yield pagecontent
+            return
+        except exc.ConfluentException as e:
+            if e.apierrorcode == 500:
+                # raise generics to trigger the tracelog
+                raise
+            start_response('{0} {1}'.format(e.apierrorcode, e.apierrorstr),
+                           headers)
+            yield e.get_error_body()
+    elif (env['PATH_INFO'].endswith('/forward/web') and
+              env['PATH_INFO'].startswith('/nodes/')):
+        prefix, _, _ = env['PATH_INFO'].partition('/forward/web')
+        _, _, nodename = prefix.rpartition('/')
+        hm = cfgmgr.get_node_attributes(nodename, 'hardwaremanagement.manager')
+        targip = hm.get(nodename, {}).get(
+            'hardwaremanagement.manager', {}).get('value', None)
+        if not targip:
+            start_response('404 Not Found', headers)
+            yield 'No hardwaremanagemnet.manager defined for node'
+            return
+        funport = forwarder.get_port(targip, env['HTTP_X_FORWARDED_FOR'],
+                                     authorized['sessionid'])
+        host = env['HTTP_X_FORWARDED_HOST']
+        url = 'https://{0}:{1}/'.format(host, funport)
+        start_response('302', [('Location', url)])
+        yield 'Our princess is in another castle!'
+        return
+    elif (operation == 'create' and ('/console/session' in env['PATH_INFO'] or
+            '/shell/sessions/' in env['PATH_INFO'])):
         #hard bake JSON into this path, do not support other incarnations
-        prefix, _, _ = env['PATH_INFO'].partition('/console/session')
+        if '/console/session' in env['PATH_INFO']:
+            prefix, _, _ = env['PATH_INFO'].partition('/console/session')
+            shellsession = False
+        elif '/shell/sessions/' in env['PATH_INFO']:
+            prefix, _, _ = env['PATH_INFO'].partition('/shell/sessions')
+            shellsession = True
         _, _, nodename = prefix.rpartition('/')
         if 'session' not in querydict.keys() or not querydict['session']:
             auditmsg = {
@@ -343,10 +496,25 @@ def resourcehandler_backend(env, start_response):
             skipreplay = False
             if 'skipreplay' in querydict and querydict['skipreplay']:
                 skipreplay = True
+            datacallback = None
+            async = None
+            if 'HTTP_CONFLUENTASYNCID' in env:
+                async = confluent.asynchttp.get_async(env, querydict)
+                termrel = async.set_term_relation(env)
+                datacallback = termrel.got_data
             try:
-                consession = consoleserver.ConsoleSession(
-                    node=nodename, configmanager=cfgmgr,
-                    username=authorized['username'], skipreplay=skipreplay)
+                if shellsession:
+                    consession = shellserver.ShellSession(
+                        node=nodename, configmanager=cfgmgr,
+                        username=authorized['username'], skipreplay=skipreplay,
+                        datacallback=datacallback
+                    )
+                else:
+                    consession = consoleserver.ConsoleSession(
+                        node=nodename, configmanager=cfgmgr,
+                        username=authorized['username'], skipreplay=skipreplay,
+                        datacallback=datacallback
+                    )
             except exc.NotFoundException:
                 start_response("404 Not found", headers)
                 yield "404 - Request Path not recognized"
@@ -355,6 +523,8 @@ def resourcehandler_backend(env, start_response):
                 start_response("500 Internal Server Error", headers)
                 return
             sessid = _assign_consessionid(consession)
+            if async:
+                async.add_console_session(sessid)
             start_response('200 OK', headers)
             yield '{"session":"%s","data":""}' % sessid
             return
@@ -369,14 +539,51 @@ def resourcehandler_backend(env, start_response):
             start_response('200 OK', headers)
             yield json.dumps({'session': querydict['session']})
             return  # client has requests to send or receive, not both...
+        elif 'closesession' in querydict:
+            consolesessions[querydict['session']]['session'].destroy()
+            del consolesessions[querydict['session']]
+            start_response('200 OK', headers)
+            yield '{"sessionclosed": true}'
+            return
+        elif 'action' in querydict:
+            if querydict['action'] == 'break':
+                consolesessions[querydict['session']]['session'].send_break()
+            elif querydict['action'] == 'reopen':
+                consolesessions[querydict['session']]['session'].reopen()
+            else:
+                start_response('400 Bad Request')
+                yield 'Unrecognized action ' + querydict['action']
+                return
+            start_response('200 OK', headers)
+            yield json.dumps({'session': querydict['session']})
         else:  # no keys, but a session, means it's hooking to receive data
             sessid = querydict['session']
             if sessid not in consolesessions:
                 start_response('400 Expired Session', headers)
+                yield ''
                 return
             consolesessions[sessid]['expiry'] = time.time() + 90
-            outdata = consolesessions[sessid]['session'].get_next_output(
-                timeout=45)
+            # add our thread to the 'inflight' to have a hook to terminate
+            # a long polling request
+            loggedout = None
+            mythreadid = greenlet.getcurrent()
+            httpsessions[authorized['sessionid']]['inflight'].add(mythreadid)
+            try:
+                outdata = consolesessions[sessid]['session'].get_next_output(
+                    timeout=25)
+            except greenlet.GreenletExit as ge:
+                loggedout = ge
+            httpsessions[authorized['sessionid']]['inflight'].discard(
+                    mythreadid)
+            if sessid not in consolesessions:
+                start_response('400 Expired Session', headers)
+                yield ''
+                return
+            if loggedout is not None:
+                consolesessions[sessid]['session'].destroy()
+                start_response('401 Logged out', headers)
+                yield '{"loggedout": 1}'
+                return
             bufferage = False
             if 'stampsent' not in consolesessions[sessid]:
                 consolesessions[sessid]['stampsent'] = True
@@ -405,12 +612,23 @@ def resourcehandler_backend(env, start_response):
         url = env['PATH_INFO']
         url = url.replace('.json', '')
         url = url.replace('.html', '')
+        if url == '/sessions/current/info':
+            start_response('200 OK', headers)
+            sessinfo = {'username': authorized['username']}
+            if 'authtoken' in authorized:
+                sessinfo['authtoken'] = authorized['authtoken']
+            yield json.dumps(sessinfo)
+            return
         resource = '.' + url[url.rindex('/'):]
         lquerydict = copy.deepcopy(querydict)
         try:
             hdlr = pluginapi.handle_path(url, operation,
                                          cfgmgr, querydict)
-
+            if 'HTTP_CONFLUENTASYNCID' in env:
+                confluent.asynchttp.run_handler(hdlr, env)
+                start_response('202 Accepted', headers)
+                yield 'Request queued'
+                return
             pagecontent = ""
             if mimetype == 'text/html':
                 for datum in _assemble_html(hdlr, resource, lquerydict, url,
@@ -421,24 +639,14 @@ def resourcehandler_backend(env, start_response):
                     pagecontent += datum
             start_response('200 OK', headers)
             yield pagecontent
-        except exc.NotFoundException as ne:
-            start_response('404 Not found', headers)
-            yield "404 - Request path not recognized - " + str(ne)
-        except exc.InvalidArgumentException as e:
-            start_response('400 Bad Request - ' + str(e), headers)
-            yield '400 - Bad Request - ' + str(e)
-        except exc.TargetEndpointUnreachable as tu:
-            start_response('504 Unreachable Target', headers)
-            yield '504 - Unreachable Target - ' + str(tu)
-        except exc.TargetEndpointBadCredentials:
-            start_response('502 Bad Credentials', headers)
-            yield '502 - Bad Credentials'
-        except exc.LockedCredentials:
-            start_response('500 Locked credential store', headers)
-            yield '500 - Credential store locked'
-        except exc.NotImplementedException:
-            start_response('501 Not Implemented', headers)
-            yield '501 Not Implemented'
+        except exc.ConfluentException as e:
+            if ((not isinstance(e, exc.LockedCredentials)) and
+                    e.apierrorcode == 500):
+                # raise generics to trigger the tracelog
+                raise
+            start_response('{0} {1}'.format(e.apierrorcode, e.apierrorstr),
+                           headers)
+            yield e.get_error_body()
 
 def _assemble_html(responses, resource, querydict, url, extension):
     yield '<html><head><meta charset="UTF-8"><title>' \
@@ -495,21 +703,21 @@ def _assemble_html(responses, resource, querydict, url, extension):
                '</form></body></html>')
 
 
-def _assemble_json(responses, resource, url, extension):
+def _assemble_json(responses, resource=None, url=None, extension=None):
     #NOTE(jbjohnso) I'm considering giving up on yielding bit by bit
     #in json case over http.  Notably, duplicate key values from plugin
     #overwrite, but we'd want to preserve them into an array instead.
     #the downside is that http would just always blurt it ll out at
     #once and hold on to all the data in memory
-    links = {
-        'self': {"href": resource + extension},
-    }
-    if url == '/':
-        pass
-    elif resource[-1] == '/':
-        links['collection'] = {"href": "../" + extension}
-    else:
-        links['collection'] = {"href": "./" + extension}
+    links = {}
+    if resource is not None:
+        links['self'] = {"href": resource + extension}
+        if url == '/':
+            pass
+        elif resource[-1] == '/':
+            links['collection'] = {"href": "../" + extension}
+        else:
+            links['collection'] = {"href": "./" + extension}
     rspdata = {}
     for rsp in responses:
         if isinstance(rsp, confluent.messages.LinkRelation):
@@ -533,11 +741,14 @@ def _assemble_json(responses, resource, url, extension):
             for dk in rsp.iterkeys():
                 if dk in rspdata:
                     if isinstance(rspdata[dk], list):
-                        rspdata[dk].append(rsp[dk])
+                        if isinstance(rsp[dk], list):
+                            rspdata[dk].extend(rsp[dk])
+                        else:
+                            rspdata[dk].append(rsp[dk])
                     else:
                         rspdata[dk] = [rspdata[dk], rsp[dk]]
                 else:
-                    if dk == 'databynode':
+                    if dk == 'databynode' or dk == 'asyncresponse':
                         # a quirk, databynode suggests noderange
                         # multi response.  This should *always* be a list,
                         # even if it will be length 1
@@ -561,9 +772,20 @@ def serve(bind_host, bind_port):
     #but deps are simpler without flup
     #also, the potential for direct http can be handy
     #todo remains unix domain socket for even http
-    eventlet.wsgi.server(
-        eventlet.listen((bind_host, bind_port, 0, 0), family=socket.AF_INET6),
-        resourcehandler, log=False, log_output=False, debug=False)
+    sock = None
+    while not sock:
+        try:
+            sock = eventlet.listen(
+                (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write(
+                'Failed to open HTTP due to busy port, trying again in'
+                ' a second\n')
+            eventlet.sleep(1)
+    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                         debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2016 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -197,7 +197,7 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
             self.when = conf.get_option('log', 'when').upper()
         except (AttributeError):
             self.when = 'D'
-        self.backupCount = conf.get_int_option('log', 'backup_count') or 3
+        self.backupCount = conf.get_int_option('log', 'backup_count') or 0
         self.maxBytes = conf.get_int_option(
             'log','max_bytes') or 4 * 1024 * 1024 * 1024
         if self.maxBytes < 8192:
@@ -379,7 +379,7 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
 
     def _sizeRoll(self):
         self.close()
-        for i in range(self.sizeRollingCount, 0, -1):
+        for i in range(self.backupCount - 1, 0, -1):
             sbfn = "%s.%d" % (self.binpath, i)
             dbfn = "%s.%d" % (self.binpath, i + 1)
             stfn = "%s.%d" % (self.textpath, i)
@@ -392,8 +392,6 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
                 if os.path.exists(dtfn):
                     os.remove(dtfn)
                 os.rename(stfn, dtfn)
-        # size rolling happens, add statistics count
-        self.sizeRollingCount += 1
         dbfn = self.binpath + ".1"
         dtfn = self.textpath + ".1"
         if os.path.exists(dbfn):
@@ -404,8 +402,18 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
             os.rename(self.binpath, dbfn)
         if os.path.exists(self.textpath):
             os.rename(self.textpath, dtfn)
+        self._deleteFilesForSizeRolling()
         return dbfn, dtfn
 
+    def _deleteFilesForSizeRolling(self):
+        for i in range(self.sizeRollingCount, self.backupCount -1, -1):
+            dbfn = "%s.%d" % (self.binpath, i)
+            dtfn = "%s.%d" % (self.textpath, i)
+            if os.path.exists(dbfn):
+                os.remove(dbfn)
+            if os.path.exists(dtfn):
+                os.remove(dtfn)
+
     def _timeRoll(self):
         self.close()
         # get the time that this sequence started at and make it a TimeTuple
@@ -424,8 +432,7 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
                     addend = -3600
                 timeTuple = time.localtime(t + addend)
 
-        # if size rolling files exist
-        for i in range(self.sizeRollingCount, 0, -1):
+        for i in range(self.backupCount - 1, 0, -1):
             sbfn = "%s.%d" % ( self.binpath, i)
             dbfn = "%s.%s.%d" % (
             self.binpath, time.strftime(self.suffix, timeTuple),i)
@@ -441,15 +448,18 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
                     os.remove(dtfn)
                 os.rename(stfn, dtfn)
 
-        # As time rolling happens, reset statistics count
-        self.sizeRollingCount = 0
         dbfn = self.binpath + "." + time.strftime(self.suffix, timeTuple)
+        odbfn = dbfn
         dtfn = self.textpath + "." + time.strftime(self.suffix, timeTuple)
-
-        if os.path.exists(dbfn):
-            os.remove(dbfn)
-        if os.path.exists(dtfn):
-            os.remove(dtfn)
+        odtfn = dtfn
+        append=1
+        while os.path.exists(dbfn):
+            dbfn = odbfn + '.{0}'.format(append)
+            append += 1
+        append=1
+        while os.path.exists(dtfn):
+            dtfn = odtfn + '.{0}'.format(append)
+            append += 1
         if os.path.exists(self.binpath):
             os.rename(self.binpath, dbfn)
         if os.path.exists(self.textpath):
@@ -489,7 +499,7 @@ class Logger(object):
                      False, events will be formatted like syslog:
                      date: message<CR>
     """
-    def __new__(cls, logname, console=False, tenant=None):
+    def __new__(cls, logname, console=False, tenant=None, buffered=True):
         global _loggers
         if console:
             relpath = 'consoles/' + logname
@@ -500,11 +510,12 @@ class Logger(object):
         else:
             return object.__new__(cls)
 
-    def __init__(self, logname, console=False, tenant=None):
+    def __init__(self, logname, console=False, tenant=None, buffered=True):
         if hasattr(self, 'initialized'):
             # we are just a copy of the same object
             return
         self.initialized = True
+        self.buffered = buffered
         self.filepath = confluent.config.configmanager.get_global("logdirectory")
         if self.filepath is None:
             if os.name == 'nt':
@@ -534,6 +545,12 @@ class Logger(object):
             tstamp = entry[1]
             data = entry[2]
             evtdata = entry[3]
+            if len(data) > 65535:
+                # our max log entry is 65k, take only the first 65k and put
+                # rest back on as a continuation
+                entry[2] = data[65535:]
+                self.logentries.appendleft(entry)
+                data = data[:65535]
             textdate = ''
             if self.isconsole and ltype != 2:
                 textdate = time.strftime(
@@ -604,7 +621,6 @@ class Logger(object):
         offsets = []
         termstate = None
         recenttimestamp = 0
-        access_last_rename = False
         flock(textfile, LOCK_SH)
         while binidx > 0 and currsize < size:
             binidx -= 16
@@ -614,23 +630,29 @@ class Logger(object):
                 struct.unpack(">BBIHIBBH", recbytes)
             # rolling events found.
             if ltype == DataTypes.event and evtdata == Events.logrollover:
-                # Now, we can only find the last renamed file which logging
-                # the data.
-                if access_last_rename == False:
-                    access_last_rename = True
-                else:
+                txtpath, bpath = parse_last_rolling_files(textfile, offset,
+                                                          datalen)
+                if txtpath == textpath:
                     break
-                textpath, binpath = parse_last_rolling_files(textfile, offset,
-                                                             datalen)
+                if bpath == binpath:
+                    break
+                textpath = txtpath
+                binpath = bpath
                 # Rolling event detected, close the current bin file, then open
                 # the renamed bin file.
                 flock(binfile, LOCK_UN)
+                flock(textfile, LOCK_UN)
                 binfile.close()
+                textfile.close()
                 try:
                     binfile = open(binpath, mode='r')
+                    textfile = open(textpath, mode='r')
                 except IOError:
-                    return '', 0, 0
+                    binfile = None
+                    textfile = None
+                    break
                 flock(binfile, LOCK_SH)
+                flock(textfile, LOCK_SH)
                 binfile.seek(0, 2)
                 binidx = binfile.tell()
                 # things have been set up for next iteration to dig to
@@ -644,22 +666,36 @@ class Logger(object):
             offsets.append((offset, datalen, textpath))
             if termstate is None:
                 termstate = eventaux
-        flock(binfile, LOCK_UN)
-        binfile.close()
+        try:
+            flock(binfile, LOCK_UN)
+            binfile.close()
+        except:
+            pass
         textdata = ''
         while offsets:
             (offset, length, textpath) = offsets.pop()
+            if textfile is None:
+                textfile = open(textpath, mode='r')
+                flock(textfile, LOCK_SH)
             if textfile.name != textpath:
-                flock(textfile, LOCK_UN)
-                textfile.close()
                 try:
-                    textfile = open(textpath)
-                except IOError:
-                    return '', 0, 0
-            textfile.seek(offset, 0)
+                    flock(textfile, LOCK_UN)
+                    textfile.close()
+                    textfile = open(textpath, mode='r')
+                    flock(textfile, LOCK_SH)
+                except (ValueError, IOError) as e:
+                    break
+            try:
+                textfile.seek(offset, 0)
+            except ValueError:
+                # file was closed, settle with what we have and continue
+                break
             textdata += textfile.read(length)
-        flock(textfile, LOCK_UN)
-        textfile.close()
+        try:
+            flock(textfile, LOCK_UN)
+            textfile.close()
+        except:
+            pass
         if termstate is None:
             termstate = 0
         return textdata, termstate, recenttimestamp
@@ -703,9 +739,29 @@ class Logger(object):
         else:
             self.logentries.append(
                 [ltype, timestamp, logdata, event, eventdata])
-        if self.writer is None:
-            self.writer = eventlet.spawn_after(2, self.writedata)
+        if self.buffered:
+            if self.writer is None:
+                self.writer = eventlet.spawn_after(2, self.writedata)
+        else:
+            self.writedata()
 
     def closelog(self):
         self.handler.close()
         self.closer = None
+
+globaleventlog = None
+tracelog = None
+
+
+def log(logdata=None, ltype=None, event=0, eventdata=None):
+    global globaleventlog
+    if globaleventlog is None:
+        globaleventlog = Logger('events')
+    globaleventlog.log(logdata, ltype, event, eventdata)
+
+def logtrace():
+    global tracelog
+    if tracelog is None:
+        tracelog = Logger('trace', buffered=False)
+    tracelog.log(traceback.format_exc(), ltype=DataTypes.event,
+                 event=Events.stacktrace)

confluent_server/confluent/lookuptools.py

@@ -27,7 +27,7 @@ __author__ = 'jjohnson2'
 
 import confluent.config.configmanager as configmanager
 import itertools
-import socket
+from eventlet.support import greendns
 
 manager_to_nodemap = {}
 
@@ -46,7 +46,7 @@ def node_by_manager(manager):
     """
 
     manageraddresses = []
-    for tmpaddr in socket.getaddrinfo(manager, None):
+    for tmpaddr in greendns.getaddrinfo(manager, None):
         manageraddresses.append(tmpaddr[4][0])
     cfm = configmanager.ConfigManager(None)
     if manager in manager_to_nodemap:
@@ -67,7 +67,7 @@ def node_by_manager(manager):
         if currhm in manageraddresses:
             manager_to_nodemap[manager] = node
             return node
-        for curraddr in socket.getaddrinfo(currhm, None):
+        for curraddr in greendns.getaddrinfo(currhm, None):
             curraddr = curraddr[4][0]
             if curraddr in manageraddresses:
                 manager_to_nodemap[manager] = node

confluent_server/confluent/main.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,17 +39,24 @@ except ImportError:
     #On platforms without pwd, give up on the sockapi in general and be http
     #only for now
     pass
+import confluent.discovery.core as disco
 import eventlet
-#import eventlet.backdoor as backdoor
+dbgif = False
+if  map(int, (eventlet.__version__.split('.'))) > [0, 18]:
+    import eventlet.backdoor as backdoor
+    dbgif = True
 havefcntl = True
 try:
     import fcntl
 except ImportError:
     havefcntl = False
 #import multiprocessing
+import gc
+from greenlet import greenlet
 import sys
 import os
 import signal
+import socket
 import time
 import traceback
 
@@ -71,8 +78,8 @@ def _daemonize():
     os.open(os.devnull, os.O_RDWR)
     os.dup2(0, 1)
     os.dup2(0, 2)
-    sys.stdout = log.Logger('stdout')
-    sys.stderr = log.Logger('stderr')
+    sys.stdout = log.Logger('stdout', buffered=False)
+    sys.stderr = log.Logger('stderr', buffered=False)
 
 
 def _updatepidfile():
@@ -83,15 +90,41 @@ def _updatepidfile():
     pidfile.close()
 
 
+def is_running():
+    # Utility function for utilities to check if confluent is running
+    try:
+        pidfile = open('/var/run/confluent/pid', 'r+')
+        fcntl.flock(pidfile, fcntl.LOCK_SH)
+        pid = pidfile.read()
+        if pid != '':
+            try:
+                os.kill(int(pid), 0)
+                return pid
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        fcntl.flock(pidfile, fcntl.LOCK_UN)
+        pidfile.close()
+    except IOError:
+        pass
+    return None
+
+
 def _checkpidfile():
     try:
         pidfile = open('/var/run/confluent/pid', 'r+')
         fcntl.flock(pidfile, fcntl.LOCK_EX)
         pid = pidfile.read()
         if pid != '':
-            print ('/var/run/confluent/pid exists and indicates %s is still '
-                   'running' % pid)
-            sys.exit(1)
+            try:
+                os.kill(int(pid), 0)
+                print ('/var/run/confluent/pid exists and indicates %s is still '
+                       'running' % pid)
+                sys.exit(1)
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        pidfile.seek(0)
         pidfile.write(str(os.getpid()))
         fcntl.flock(pidfile, fcntl.LOCK_UN)
         pidfile.close()
@@ -125,11 +158,22 @@ def dumptrace(signalname, frame):
     ht = open('/var/log/confluent/hangtraces', 'a')
     ht.write('Dumping active trace on ' + time.strftime('%X %x\n'))
     ht.write(''.join(traceback.format_stack(frame)))
+    for o in gc.get_objects():
+        if not isinstance(o, greenlet):
+            continue
+        if not o:
+            continue
+        ht.write('Thread trace: ({0})\n'.format(id(o)))
+        ht.write(''.join(traceback.format_stack(o.gr_frame)))
     ht.close()
 
 def doexit():
     if not havefcntl:
         return
+    try:
+        os.remove('/var/run/confluent/dbg.sock')
+    except OSError:
+        pass
     pidfile = open('/var/run/confluent/pid')
     pid = pidfile.read()
     if pid == str(os.getpid()):
@@ -142,10 +186,28 @@ def _initsecurity(config):
         with open(keyfile, 'r') as keyhandle:
             key = keyhandle.read()
         configmanager.init_masterkey(key)
+        # We don't want to os._exit() until sync finishes from
+        # init above
+        configmanager.ConfigManager.wait_for_sync()
+
+
+def setlimits():
+    try:
+        import resource
+        currlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if currlimit[0] < currlimit[1]:
+            resource.setrlimit(
+                resource.RLIMIT_NOFILE, (currlimit[1], currlimit[1]))
+    except Exception:
+        pass
 
 
 def run():
-    signal.signal(signal.SIGUSR1, dumptrace)
+    setlimits()
+    try:
+        signal.signal(signal.SIGUSR1, dumptrace)
+    except AttributeError:
+        pass   # silly windows
     if havefcntl:
         _checkpidfile()
     conf.init_config()
@@ -164,25 +226,34 @@ def run():
     _daemonize()
     if havefcntl:
         _updatepidfile()
-    auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
-    #TODO(jbjohnso): eventlet has a bug about unix domain sockets, this code
-    #works with bugs fixed
-    #dbgsock = eventlet.listen("/var/run/confluent/dbg.sock",
-    #                           family=socket.AF_UNIX)
-    #eventlet.spawn_n(backdoor.backdoor_server, dbgsock)
+    if dbgif:
+        oumask = os.umask(0077)
+        try:
+            os.remove('/var/run/confluent/dbg.sock')
+        except OSError:
+            pass  # We are not expecting the file to exist
+        try:
+            dbgsock = eventlet.listen("/var/run/confluent/dbg.sock",
+                                       family=socket.AF_UNIX)
+            eventlet.spawn_n(backdoor.backdoor_server, dbgsock)
+        except AttributeError:
+            pass  # Windows...
+        os.umask(oumask)
     http_bind_host, http_bind_port = _get_connector_config('http')
     sock_bind_host, sock_bind_port = _get_connector_config('socket')
-    consoleserver.start_console_sessions()
     webservice = httpapi.HttpApi(http_bind_host, http_bind_port)
     webservice.start()
+    disco.start_detection()
     try:
         sockservice = sockapi.SockApi(sock_bind_host, sock_bind_port)
         sockservice.start()
     except NameError:
         pass
     atexit.register(doexit)
+    eventlet.sleep(1)
+    consoleserver.start_console_sessions()
     while 1:
         eventlet.sleep(100)
 

confluent_server/confluent/messages.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,9 @@
 # Things are defined here to 'encourage' developers to coordinate information
 # format.  This is also how different data formats are supported
 import confluent.exceptions as exc
+import confluent.config.configmanager as cfm
+from copy import deepcopy
+from datetime import datetime
 import json
 
 valid_health_values = set([
@@ -32,10 +35,12 @@ valid_health_values = set([
 def _htmlify_structure(indict):
     ret = "<ul>"
     if isinstance(indict, dict):
-        for key in indict.iterkeys():
+        for key in sorted(indict):
             ret += "<li>{0}: ".format(key)
             if type(indict[key]) in (str, unicode, float, int):
                 ret += str(indict[key])
+            elif isinstance(indict[key], datetime):
+                ret += indict[key].strftime('%Y-%m-%dT%H:%M:%S')
             else:
                 ret += _htmlify_structure(indict[key])
     elif isinstance(indict, list):
@@ -55,6 +60,7 @@ def _htmlify_structure(indict):
 
 
 class ConfluentMessage(object):
+    apicode = 200
     readonly = False
     defaultvalue = ''
     defaulttype = 'text'
@@ -71,14 +77,15 @@ class ConfluentMessage(object):
             datasource = self.kvpairs
         else:
             datasource = {'databynode': self.kvpairs}
-        jsonsnippet = json.dumps(datasource, separators=(',', ':'))[1:-1]
+        jsonsnippet = json.dumps(datasource, sort_keys=True, separators=(',', ':'))[1:-1]
         return jsonsnippet
 
     def raw(self):
         """Return pythonic representation of the response.
 
         Used by httpapi while assembling data prior to json serialization"""
-        if hasattr(self, 'stripped') and self.stripped:
+        if ((hasattr(self, 'stripped') and self.stripped) or
+                (hasattr(self, 'notnode') and self.notnode)):
             return self.kvpairs
         return {'databynode': self.kvpairs}
 
@@ -132,7 +139,10 @@ class ConfluentMessage(object):
                         '<input type="checkbox" name="restexplorerhonorkey" '
                         'value="{1}">\r').format(valtype, key, self.desc)
                 return snippet
-            if val is not None and 'value' in val:
+            if (isinstance(val, bool) or isinstance(val, str) or
+                    isinstance(val, unicode)):
+                value = str(val)
+            elif val is not None and 'value' in val:
                 value = val['value']
                 if 'inheritedfrom' in val:
                     notes.append('Inherited from %s' % val['inheritedfrom'])
@@ -156,7 +166,7 @@ class ConfluentMessage(object):
                     notes.append('Inherited from %s' % val['inheritedfrom'])
                 value = '********'
             if self.readonly:
-                snippet += "{0}: {1}".format(key, value)
+                snippet += "{0}: {1}<br>".format(key, value)
             else:
                 snippet += (key + ":" +
                             '<input type="{0}" name="{1}" value="{2}" '
@@ -169,12 +179,15 @@ class ConfluentMessage(object):
 
 
 class ConfluentNodeError(object):
+    apicode = 500
+
     def __init__(self, node, errorstr):
         self.node = node
         self.error = errorstr
 
     def raw(self):
-        return {'databynode': {self.node: {'error': self.error}}}
+        return {'databynode': {self.node: {'errorcode': self.apicode,
+                                           'error': self.error}}}
 
     def html(self):
         return self.node + ":" + self.error
@@ -185,16 +198,32 @@ class ConfluentNodeError(object):
         raise Exception(self.error)
 
 
+class ConfluentResourceUnavailable(ConfluentNodeError):
+    apicode = 503
+
+    def __init__(self, node, errstr='Unavailable'):
+        self.node = node
+        self.error = errstr
+
+    def strip_node(self, node):
+        raise exc.TargetResourceUnavailable()
+
+
 class ConfluentTargetTimeout(ConfluentNodeError):
+    apicode = 504
+
     def __init__(self, node, errstr='timeout'):
         self.node = node
         self.error = errstr
 
+
     def strip_node(self, node):
         raise exc.TargetEndpointUnreachable(self.error)
 
 
 class ConfluentTargetNotFound(ConfluentNodeError):
+    apicode = 404
+
     def __init__(self, node, errorstr='not found'):
         self.node = node
         self.error = errorstr
@@ -204,6 +233,7 @@ class ConfluentTargetNotFound(ConfluentNodeError):
 
 
 class ConfluentTargetInvalidCredentials(ConfluentNodeError):
+    apicode = 502
     def __init__(self, node):
         self.node = node
         self.error = 'bad credentials'
@@ -213,9 +243,31 @@ class ConfluentTargetInvalidCredentials(ConfluentNodeError):
 
 
 class DeletedResource(ConfluentMessage):
+    notnode = True
     def __init__(self, resource):
-        self.kvpairs = {}
+        self.kvpairs = {'deleted': resource}
 
+    def strip_node(self, node):
+        pass
+
+
+class CreatedResource(ConfluentMessage):
+    notnode = True
+    readonly = True
+
+    def __init__(self, resource):
+        self.kvpairs = {'created': resource}
+
+    def strip_node(self, node):
+        pass
+
+
+class AssignedResource(ConfluentMessage):
+    notnode = True
+    readonly = True
+
+    def __init__(self, resource):
+        self.kvpairs = {'assigned': resource}
 
 class ConfluentChoiceMessage(ConfluentMessage):
     valid_values = set()
@@ -310,9 +362,20 @@ class ChildCollection(LinkRelation):
                                                                   extension)
 
 
-def get_input_message(path, operation, inputdata, nodes=None, multinode=False):
+# TODO(jjohnson2): enhance the following to support expressions:
+# InputNetworkConfiguration
+# InputMCI
+# InputDomainName
+# InputNTPServer
+def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
+                      configmanager=None):
     if path[0] == 'power' and path[1] == 'state' and operation != 'retrieve':
         return InputPowerMessage(path, nodes, inputdata)
+    elif (path in (['power', 'reseat'], ['_enclosure', 'reseat_bay']) and
+            operation != 'retrieve'):
+        return InputReseatMessage(path, nodes, inputdata)
+    elif path == ['attributes', 'expression']:
+        return InputExpression(path, inputdata, nodes)
     elif path[0] in ('attributes', 'users') and operation != 'retrieve':
         return InputAttributes(path, inputdata, nodes)
     elif path == ['boot', 'nextdevice'] and operation != 'retrieve':
@@ -336,7 +399,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False):
         return InputMCI(path, nodes, inputdata)
     elif (path[:4] == ['configuration', 'management_controller',
             'net_interfaces', 'management'] and operation != 'retrieve'):
-        return InputNetworkConfiguration(path, nodes, inputdata)
+        return InputNetworkConfiguration(path, nodes, inputdata,
+                                         configmanager)
     elif (path[:3] == ['configuration', 'management_controller', 'domain_name']
             and operation != 'retrieve'):
         return InputDomainName(path, nodes, inputdata)
@@ -346,9 +410,41 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False):
     elif (path[:4] == ['configuration', 'management_controller', 'ntp',
             'servers'] and operation != 'retrieve' and len(path) == 5):
         return InputNTPServer(path, nodes, inputdata)
+    elif (path[:3] == ['configuration', 'system', 'all'] and
+            operation != 'retrieve'):
+        return InputConfigChangeSet(path, inputdata, nodes, configmanager)
+    elif 'inventory/firmware/updates/active' in '/'.join(path) and inputdata:
+        return InputFirmwareUpdate(path, nodes, inputdata)
+    elif '/'.join(path).startswith('media/detach'):
+        return DetachMedia(path, nodes, inputdata)
+    elif '/'.join(path).startswith('media/') and inputdata:
+        return InputMedia(path, nodes, inputdata)
     elif inputdata:
-        raise exc.InvalidArgumentException()
+        raise exc.InvalidArgumentException(
+            'No known input handler for request')
 
+class InputFirmwareUpdate(ConfluentMessage):
+
+    def __init__(self, path, nodes, inputdata):
+        self.filename = inputdata.get('filename', inputdata.get('url', None))
+        self.bank = inputdata.get('bank', None)
+        self.nodes = nodes
+
+class InputMedia(InputFirmwareUpdate):
+    # Use InputFirmwareUpdate
+    pass
+
+
+class DetachMedia(ConfluentMessage):
+    def __init__(self, path, nodes, inputdata):
+        if 'detachall' not in inputdata:
+            raise exc.InvalidArgumentException('Currently only supporting'
+                                               '{"detachall": 1}')
+
+
+class Media(ConfluentMessage):
+    def __init__(self, node, media):
+        self.kvpairs = {node: {'name': media.name, 'url': media.url}}
 
 class InputAlertData(ConfluentMessage):
 
@@ -372,10 +468,60 @@ class InputAlertData(ConfluentMessage):
         return self.alertparams
 
 
-class InputAttributes(ConfluentMessage):
+class InputExpression(ConfluentMessage):
+    # This is specifically designed to suppress the expansion of an expression
+    # so that it can make it intact to the pertinent configmanager function
+    def __init__(self, path, inputdata, nodes=None):
+        self.nodeattribs = {}
+        if not inputdata:
+            raise exc.InvalidArgumentException('no request data provided')
+        if nodes is None:
+            self.attribs = inputdata
+            return
+        for node in nodes:
+            self.nodeattribs[node] = inputdata
+
+    def get_attributes(self, node):
+        if node not in self.nodeattribs:
+            return {}
+        nodeattr = deepcopy(self.nodeattribs[node])
+        return nodeattr
+
+class InputConfigChangeSet(InputExpression):
+    # For now, this is identical to InputExpression, later it may
+    # internalize formula expansion, but not now..
+    def __init__(self, path, inputdata, nodes=None, configmanager=None):
+        self.cfm = configmanager
+        super(InputConfigChangeSet, self).__init__(path, inputdata, nodes)
+
+    def get_attributes(self, node):
+        attrs = super(InputConfigChangeSet, self).get_attributes(node)
+        endattrs = {}
+        for attr in attrs:
+            origval = attrs[attr]
+            if isinstance(origval, str) or isinstance(origval, unicode):
+                origval = {'expression': origval}
+            if 'expression' not in origval:
+                endattrs[attr] = attrs[attr]
+            else:
+                endattrs[attr] = list(self.cfm.expand_attrib_expression(
+                    [node], attrs[attr]))[0][1]
+        return endattrs
+
+
+class InputAttributes(ConfluentMessage):
+    # This is particularly designed for attributes, where a simple string
+    # should become either a string value or a dict with {'expression':} to
+    # preserve the client provided expression for posterity, rather than
+    # immediate consumption.
+    # for things like node configuration or similar, a different class is
+    # appropriate since it needs to immediately expand an expression.
+    # with that class, the 'InputExpression' and calling code in attributes.py
+    # might be deprecated in favor of the generic expression expander
+    # and a small function in attributes.py to reflect the expansion back
+    # to the client
     def __init__(self, path, inputdata, nodes=None):
         self.nodeattribs = {}
-        nestedmode = False
         if not inputdata:
             raise exc.InvalidArgumentException('no request data provided')
         if nodes is None:
@@ -400,21 +546,12 @@ class InputAttributes(ConfluentMessage):
                             'expression': self.attribs[attrib]}
             return
         for node in nodes:
-            if node in inputdata:
-                nestedmode = True
-                self.nodeattribs[node] = inputdata[node]
-        if nestedmode:
-            for key in inputdata:
-                if key not in nodes:
-                    raise exc.InvalidArgumentException
-        else:
-            for node in nodes:
-                self.nodeattribs[node] = inputdata
+            self.nodeattribs[node] = inputdata
 
     def get_attributes(self, node):
         if node not in self.nodeattribs:
             return {}
-        nodeattr = self.nodeattribs[node]
+        nodeattr = deepcopy(self.nodeattribs[node])
         for attr in nodeattr:
             if type(nodeattr[attr]) in (str, unicode):
                 try:
@@ -446,19 +583,19 @@ class InputCredential(ConfluentMessage):
 
     def __init__(self, path, inputdata, nodes=None):
         self.credentials = {}
-        nestedmode = False
         if not inputdata:
             raise exc.InvalidArgumentException('no request data provided')
 
         if len(path) == 4:
             inputdata['uid'] = path[-1]
         # if the operation is 'create' check if all fields are present
-        elif ('uid' not in inputdata or 'privilege_level' not in inputdata or
-                'username' not in inputdata or 'password' not in inputdata):
-            raise exc.InvalidArgumentException('all fields are required')
-
-        if 'uid' not in inputdata:
-            raise exc.InvalidArgumentException('uid is missing')
+        missingattrs = []
+        for attrname in ('uid', 'privilege_level', 'username', 'password'):
+            if attrname not in inputdata:
+                missingattrs.append(attrname)
+        if missingattrs:
+            raise exc.InvalidArgumentException(
+                'Required fields missing: {0}'.format(','.join(missingattrs)))
         if (isinstance(inputdata['uid'], str) and
                 not inputdata['uid'].isdigit()):
             raise exc.InvalidArgumentException('uid must be a number')
@@ -488,9 +625,9 @@ class InputCredential(ConfluentMessage):
     def get_attributes(self, node):
         if node not in self.credentials:
             return {}
-        credential = self.credentials[node]
-        for attr in credentials:
-            if type(credentials[attr]) in (str, unicode):
+        credential = deepcopy(self.credentials[node])
+        for attr in credential:
+            if type(credential[attr]) in (str, unicode):
                 try:
                     # as above, use format() to see if string follows
                     # expression, store value back in case of escapes
@@ -520,7 +657,7 @@ class ConfluentInputMessage(ConfluentMessage):
                 if self.keyname not in datum:
                     raise exc.InvalidArgumentException(
                         'missing {0} argument'.format(self.keyname))
-                elif datum[self.keyname] not in self.valid_values:
+                elif not self.is_valid_key(datum[self.keyname]):
                     raise exc.InvalidArgumentException(
                         datum[self.keyname] + ' is not one of ' +
                         ','.join(self.valid_values))
@@ -530,13 +667,15 @@ class ConfluentInputMessage(ConfluentMessage):
             if self.keyname not in datum:
                 raise exc.InvalidArgumentException(
                     'missing {0} argument'.format(self.keyname))
-            elif datum[self.keyname] not in self.valid_values:
+            elif not self.is_valid_key(datum[self.keyname]):
                 raise exc.InvalidArgumentException(datum[self.keyname] +
                                                    ' is not one of ' +
                                                    ','.join(self.valid_values))
             for node in nodes:
                 self.inputbynode[node] = datum[self.keyname]
 
+    def is_valid_key(self, key):
+        return key in self.valid_values
 
 class InputIdentifyMessage(ConfluentInputMessage):
     valid_values = set([
@@ -560,6 +699,16 @@ class InputPowerMessage(ConfluentInputMessage):
     def powerstate(self, node):
         return self.inputbynode[node]
 
+class InputReseatMessage(ConfluentInputMessage):
+    valid_values = set([
+        'reseat',
+    ])
+
+    keyname = 'reseat'
+
+    def is_valid_key(self, key):
+        return key in self.valid_values or isinstance(key, int)
+
 
 class InputBMCReset(ConfluentInputMessage):
     valid_values = set([
@@ -591,7 +740,7 @@ class InputMCI(ConfluentInputMessage):
 
 
 class InputNetworkConfiguration(ConfluentInputMessage):
-    def __init__(self, path, nodes, inputdata):
+    def __init__(self, path, nodes, inputdata, configmanager=None):
         self.inputbynode = {}
         self.stripped = False
         if not inputdata:
@@ -615,8 +764,27 @@ class InputNetworkConfiguration(ConfluentInputMessage):
         if nodes is None:
             raise exc.InvalidArgumentException(
                 'This only supports per-node input')
+        nodeattrmap = {}
+        for attr in inputdata:
+            try:
+                if inputdata[attr] is not None:
+                    inputdata[attr].format()
+            except (KeyError, IndexError):
+                nodeattrmap[attr] = {}
+                for expanded in configmanager.expand_attrib_expression(
+                        nodes, inputdata[attr]):
+                    node, value = expanded
+                    nodeattrmap[attr][node] = value
+        if not nodeattrmap:
+            for node in nodes:
+                self.inputbynode[node] = inputdata
+            return
+        # an expression was encountered
         for node in nodes:
-            self.inputbynode[node] = inputdata
+            self.inputbynode[node] = deepcopy(inputdata)
+            for attr in self.inputbynode[node]:
+                if attr in nodeattrmap:
+                    self.inputbynode[node][attr] = nodeattrmap[attr][node]
 
     def netconfig(self, node):
         return self.inputbynode[node]
@@ -689,10 +857,10 @@ class BootDevice(ConfluentChoiceMessage):
 
     valid_paramset = {
         'bootmode': valid_bootmodes,
+        'persistent': set([True, False]),
         }
 
-
-    def __init__(self, node, device, bootmode='unspecified'):
+    def __init__(self, node, device, bootmode='unspecified', persistent=False):
         if device not in self.valid_values:
             raise Exception("Invalid boot device argument passed in:" +
                             repr(device))
@@ -703,6 +871,7 @@ class BootDevice(ConfluentChoiceMessage):
             node: {
                 'nextdevice': {'value': device},
                 'bootmode': {'value': bootmode},
+                'persistent': {'value': persistent},
             }
         }
 
@@ -711,6 +880,7 @@ class InputBootDevice(BootDevice):
     def __init__(self, path, nodes, inputdata):
         self.bootdevbynode = {}
         self.bootmodebynode = {}
+        self.persistentbynode = {}
         if not inputdata:
             raise exc.InvalidArgumentException()
         if 'nextdevice' not in inputdata:
@@ -732,6 +902,8 @@ class InputBootDevice(BootDevice):
                             datum['bootmode'] + ' is not one of ' +
                             ','.join(self.valid_bootmodes))
                     self.bootmodebynode[key] = datum['bootmode']
+                if 'persistent' in datum:
+                    self.bootmodebynode[key] = datum['persistent']
         else:
             datum = inputdata
             if 'nextdevice' not in datum:
@@ -745,6 +917,8 @@ class InputBootDevice(BootDevice):
                 self.bootdevbynode[node] = datum['nextdevice']
                 if 'bootmode' in datum:
                     self.bootmodebynode[node] = datum['bootmode']
+                if 'persistent' in datum:
+                    self.persistentbynode[node] = datum['persistent']
 
     def bootdevice(self, node):
         return self.bootdevbynode[node]
@@ -752,6 +926,9 @@ class InputBootDevice(BootDevice):
     def bootmode(self, node):
         return self.bootmodebynode.get(node, 'unspecified')
 
+    def persistent(self, node):
+        return self.persistentbynode.get(node, False)
+
 
 class IdentifyState(ConfluentChoiceMessage):
     valid_values = set([
@@ -762,6 +939,13 @@ class IdentifyState(ConfluentChoiceMessage):
     keyname = 'identify'
 
 
+class ReseatResult(ConfluentChoiceMessage):
+    valid_values = set([
+        'success',
+    ])
+    keyname = 'reseat'
+
+
 class PowerState(ConfluentChoiceMessage):
     valid_values = set([
         'on',
@@ -773,6 +957,11 @@ class PowerState(ConfluentChoiceMessage):
     ])
     keyname = 'state'
 
+    def __init__(self, node, state, oldstate=None):
+        super(PowerState, self).__init__(node, state)
+        if oldstate is not None:
+            self.kvpairs[node]['oldstate'] = {'value': oldstate}
+
 
 class BMCReset(ConfluentChoiceMessage):
     valid_values = set([
@@ -835,6 +1024,47 @@ class EventCollection(ConfluentMessage):
             self.kvpairs = {name: {'events': eventdata}}
 
 
+class AsyncCompletion(ConfluentMessage):
+    def __init__(self):
+        self.stripped = True
+        self.notnode = True
+
+    def raw(self):
+        return {'_requestdone': True}
+
+
+class AsyncMessage(ConfluentMessage):
+    def __init__(self, pair):
+        self.stripped = True
+        self.notnode = True
+        self.msgpair = pair
+
+    def raw(self):
+        rsp = self.msgpair[1]
+        rspdict = None
+        if (isinstance(rsp, ConfluentMessage) or
+                isinstance(rsp, ConfluentNodeError)):
+            rspdict = rsp.raw()
+        elif isinstance(rsp, exc.ConfluentException):
+            rspdict = {'exceptioncode': rsp.apierrorcode,
+                       'exception': rsp.get_error_body()}
+        elif isinstance(rsp, Exception):
+            rspdict = {'exceptioncode': 500, 'exception': str(rsp)}
+        elif isinstance(rsp, dict):  # console metadata
+            rspdict = rsp
+        else: # terminal text
+            rspdict = {'data': rsp}
+        return {'asyncresponse':
+                    {'requestid': self.msgpair[0],
+                      'response': rspdict}}
+
+class AsyncSession(ConfluentMessage):
+    def __init__(self, id):
+        self.desc = 'foo'
+        self.notnode = True
+        self.stripped = True
+        self.kvpairs = {'asyncid': id}
+
 class User(ConfluentMessage):
     def __init__(self, uid, username, privilege_level, name=None):
         self.desc = 'foo'
@@ -1031,7 +1261,7 @@ class Attributes(ConfluentMessage):
         self.desc = desc
         nkv = {}
         self.notnode = name is None
-        for key in kv.iterkeys():
+        for key in kv:
             if type(kv[key]) in (str, unicode):
                 nkv[key] = {'value': kv[key]}
             else:
@@ -1043,6 +1273,8 @@ class Attributes(ConfluentMessage):
                 name: nkv
             }
 
+class ConfigSet(Attributes):
+    pass
 
 class ListAttributes(ConfluentMessage):
     def __init__(self, name=None, kv=None, desc=''):

confluent_server/confluent/neighutil.py

@@ -0,0 +1,64 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A consolidated manage of neighbor table information management.
+# Ultimately, this should use AF_NETLINK, but in the interest of time,
+# use ip neigh for the moment
+
+import eventlet.green.subprocess as subprocess
+import os
+
+neightable = {}
+neightime = 0
+
+import re
+
+_validmac = re.compile('..:..:..:..:..:..')
+
+
+def update_neigh():
+    global neightable
+    global neightime
+    neightable = {}
+    if os.name == 'nt':
+        return
+    ipn = subprocess.Popen(['ip', 'neigh'], stdin=subprocess.PIPE,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.PIPE)
+    (neighdata, err) = ipn.communicate()
+    for entry in neighdata.split('\n'):
+        entry = entry.split(' ')
+        if len(entry) < 5 or not entry[4]:
+            continue
+        if entry[0] in ('192.168.0.100', '192.168.70.100', '192.168.70.125'):
+            # Note that these addresses are common static ip addresses
+            # that are hopelessly ambiguous if there are many
+            # so ignore such entries and move on
+            # ideally the system network steers clear of this landmine of
+            # a subnet, but just in case
+            continue
+        if not _validmac.match(entry[4]):
+            continue
+        neightable[entry[0]] = entry[4]
+    neightime = os.times()[4]
+
+
+def refresh_neigh():
+    global neightime
+    if os.name == 'nt':
+        return
+    if os.times()[4] > (neightime + 30):
+        update_neigh()