Rename nodeboot.py to nodeboot

No point in having '.py' extensions

confluent_client/bin/confetty

@@ -2,7 +2,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2016 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -316,21 +316,31 @@ def do_command(command, server):
         if parentpath:
             childname = '{0}/'.format(parentpath[parentpath.rindex('/') + 1:])
             parentpath = parentpath[:parentpath.rindex('/') + 1]
-            foundchild = False
-            for res in session.read(parentpath, server):
-                try:
-                    if res['item']['href'] == childname:
-                        foundchild = True
-                except KeyError:
-                    pass
-                if 'errorcode' in res:
-                    exitcode = res['errorcode']
-                if 'error' in res:
-                    sys.stderr.write(target + ': ' + res['error'] + '\n')
+            if parentpath == '/noderange/':
+                for res in session.read(target, server):
+                    if 'errorcode' in res:
+                        exitcode = res['errorcode']
+                        target = otarget
+                    if 'error' in res:
+                        sys.stderr.write(target + ': ' + res['error'] + '\n')
+                        target = otarget
+            else:
+                foundchild = False
+                for res in session.read(parentpath, server):
+                    try:
+                        if res['item']['href'] == childname:
+                            foundchild = True
+                    except KeyError:
+                        pass
+                    if 'errorcode' in res:
+                        exitcode = res['errorcode']
+                        target = otarget
+                    if 'error' in res:
+                        sys.stderr.write(target + ': ' + res['error'] + '\n')
+                        target = otarget
+                if not foundchild:
+                    sys.stderr.write(target + ': Target not found - \n')
                     target = otarget
-            if not foundchild:
-                sys.stderr.write(target + ': Target not found - \n')
-                target = otarget
     elif argv[0] in ('cat', 'show', 'ls', 'dir'):
         if len(argv) > 1:
             targpath = fullpath_target(argv[1])
@@ -532,7 +542,7 @@ def get_session_node(shellargs):
     return None
 
 
-def conserver_command(filehandle, command):
+def conserver_command(filehandle, localcommand):
     # x - conserver has that as 'show baud', I am inclined to replace that with
     #     'request exclusive'
     # b - conserver has that as 'broadcast message', I'm tempted to use that
@@ -547,33 +557,121 @@ def conserver_command(filehandle, command):
     # d - down a console... never used this...
     # L - toggle logging
     # w - who is on console
-    while not command:
-        ready, _, _ = select.select((filehandle,), (), (), 1)
-        if ready:
-            command += filehandle.read()
-    if command[0] == '.':
+
+    cmdlen = 1
+    localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+    if localcommand[0] == '.':
         print("disconnect]\r")
         quitconfetty(fullexit=consoleonly)
-    elif command[0] == 'o':
+    elif localcommand[0] == 'o':
         tlvdata.send(session.connection, {'operation': 'reopen',
                                           'path': currconsole})
         print('reopen]\r')
-    elif command[0] == 'b':
+    elif localcommand[0] == 'b':
         tlvdata.send(session.connection, {'operation': 'break',
                                           'path': currconsole})
         print("break sent]\r")
-    elif command[0] == '?':
+    elif localcommand[0] == 'p':  # print
+        cmdlen += 1
+        localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+        if localcommand[1] == 'o': # off
+            print("powering off...")
+            session.simple_noderange_command(consolename, '/power/state', 'off')
+            print("complete]\r")
+        elif localcommand[1] == 's':  # shutdown
+            print("shutting down...")
+            session.simple_noderange_command(consolename, '/power/state', 'shutdown')
+            print("complete]\r")
+        elif localcommand[1] == 'b': # boot
+            cmdlen += 1
+            localcommand = get_command_bytes(filehandle, localcommand, cmdlen)
+
+            if localcommand[2] == 's':  # boot to setup
+                print("booting to setup...")
+
+                bootmode = 'uefi'
+                bootdev = 'setup'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            elif localcommand[2] == 'n':  # boot to network
+                print("booting to network...")
+
+                bootmode = 'uefi'
+                bootdev = 'network'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            elif localcommand[2] == '\x0d':  # boot to default
+                print("booting to default...")
+
+                bootmode = 'uefi'
+                bootdev = 'default'
+
+                rc = session.simple_noderange_command(consolename, '/boot/nextdevice', bootdev, bootmode=bootmode)
+
+                if rc:
+                    print("Error]\r")
+                else:
+                    rc = session.simple_noderange_command(consolename, '/power/state', 'boot')
+
+                    if rc:
+                        print("Error]\r")
+                    else:
+                        print("complete]\r")
+
+            else:
+                print("Unknown boot state.]\r")
+
+        else:
+            print("Unknown power state.]\r")
+
+    elif localcommand[0] == '?':
         print("help]\r")
-        print(".    disconnect\r")
-        print("b    break\r")
-        print("o    reopen\r")
+        print(".       disconnect\r")
+        print("b       break\r")
+        print("o       reopen\r")
+        print("po      power off\r")
+        print("ps      shutdown\r")
+        print("pbs     boot to setup\r")
+        print("pbn     boot to network\r")
+        print("pb<ent> boot to default\r")
         print("<cr> abort command\r")
-    elif command[0] == '\x0d':
+    elif localcommand[0] == '\x0d':
         print("ignored]\r")
     else:  # not a command at all..
         print("unknown -- use '?']\r")
 
 
+def get_command_bytes(filehandle, localcommand, cmdlen):
+    while len(localcommand) < cmdlen:
+        ready, _, _ = select.select((filehandle,), (), (), 1)
+        if ready:
+            localcommand += filehandle.read()
+    return localcommand
+
+
 def check_escape_seq(currinput, filehandle):
     while conserversequence.startswith(currinput):
         if currinput.startswith(conserversequence):  # We have full sequence

confluent_client/bin/nodeboot

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser()
+argparser.add_option('-b', '--bios', dest='biosmode',
+                     action='store_true', default=False,
+                     help='Request BIOS style boot (rather than UEFI)')
+argparser.add_option('-p', '--persist', dest='persist', action='store_true',
+                     default=False,
+                     help='Request the boot device be persistent rather than '
+                          'one time')
+
+(options, args) = argparser.parse_args()
+
+try:
+    noderange = args[0]
+except IndexError:
+    sys.stderr.write(
+        'Usage: {0} <noderange> [default|cd|network|setup|hd]\n'.format(
+            sys.argv[0]))
+    sys.exit(1)
+bootdev = None
+if len(sys.argv) > 2:
+    bootdev = sys.argv[2]
+    if bootdev in ('net', 'pxe'):
+        bootdev = 'network'
+session = client.Command()
+exitcode = 0
+if options.biosmode:
+    bootmode = 'bios'
+else:
+    bootmode = 'uefi'
+
+rc = session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
+                                     bootmode=bootmode,
+                                     persistent=options.persist)
+
+if rc:
+    sys.exit(rc)
+else:
+    sys.exit(session.simple_noderange_command(noderange, '/power/state', 'boot'))

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 from datetime import datetime as dt
+import optparse
 import os
 import sys
 
@@ -26,13 +27,13 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange (clear)")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [clear]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 deletemode = False
@@ -45,15 +46,15 @@ exitcode = 0
 
 def format_event(evt):
     retparts = []
-    if 'timestamp' in evt:
+    if 'timestamp' in evt and evt['timestamp'] is not None:
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
-    if 'component_type' in evt:
+    if 'component_type' in evt and evt['component_type'] is not None:
         dscparts.append(evt['component_type'])
     if 'component' in evt and evt['component'] is not None:
         dscparts.append(evt['component'])
-    if 'event' in evt and evt['event']:
+    if 'event' in evt and evt['event'] and evt['event'] is not None:
         evttext = evt['event']
         try:
             if evttext.startswith(evt['component'] + ' - '):
@@ -83,4 +84,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
             if 'events' in thisdata:
                evtdata = thisdata['events']
                for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print '{0}: {1}'.format(node, format_event(evt))

confluent_client/bin/nodefirmware

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 path = os.path.dirname(os.path.realpath(__file__))
@@ -57,12 +58,12 @@ def printfirm(node, prefix, data):
     print('{0}: {1}: {2}'.format(node, prefix, version))
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange>\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()

confluent_client/bin/nodehealth

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import codecs
+import optparse
 import os
 import sys
 
@@ -28,10 +29,12 @@ import confluent.client as client
 
 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange>\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 

confluent_client/bin/nodeidentify

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 
@@ -25,10 +26,12 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write('Usage: {0} <noderange> [on|off]\n'.format(sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 identifystate = None

confluent_client/bin/nodeinventory

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 path = os.path.dirname(os.path.realpath(__file__))
@@ -69,12 +70,12 @@ def printerror(res, node=None):
             exitcode = 1
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
     noderange = sys.argv[1]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange>\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()

confluent_client/bin/nodelist

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ def attrrequested(attr, attrlist, seenattributes):
             return True
     return False
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [list of attributes")
+    usage="Usage: %prog [options] noderange [list of attributes]")
 argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
 (options, args) = argparser.parse_args()

confluent_client/bin/nodepower

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
 import os
 import sys
 
@@ -25,13 +26,14 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-
+argparser = optparse.OptionParser(
+    usage="Usage: %prog [options] noderange "
+          "([status|on|off|shutdown|boot|reset])")
+(options, args) = argparser.parse_args()
 try:
-    noderange = sys.argv[1]
+    noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> ([status|on|off|shutdown|boot|reset]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 
 setstate = None

confluent_client/bin/noderun

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import select
+import shlex
+import subprocess
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog node commandexpression",
+    epilog="Expressions are the same as in attributes, e.g. "
+           "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+argparser.disable_interspersed_args()
+(options, args) = argparser.parse_args()
+if len(args) < 2:
+    argparser.print_help()
+    sys.exit(1)
+c = client.Command()
+cmdstr = " ".join(args[1:])
+
+nodeforpopen = {}
+popens = []
+for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                    {'expression': cmdstr}):
+    ex = exp['databynode']
+    for node in ex:
+        cmd = ex[node]['value'].encode('utf-8')
+        cmdv = shlex.split(cmd)
+        nopen = subprocess.Popen(
+            cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        popens.append(nopen)
+        nodeforpopen[nopen] = node
+
+all = set([])
+pipedesc = {}
+exitcode = 0
+for pop in popens:
+    node = nodeforpopen[pop]
+    pipedesc[pop.stdout] = { 'node': node, 'popen': pop, 'type': 'stdout'}
+    pipedesc[pop.stderr] = {'node': node, 'popen': pop, 'type': 'stderr'}
+    all.add(pop.stdout)
+    all.add(pop.stderr)
+rdy, _, _ = select.select(all, [], [], 10)
+while all and rdy:
+    for r in rdy:
+        data = r.readline()
+        desc = pipedesc[r]
+        if data:
+            node = desc['node']
+            if desc['type'] == 'stdout':
+                sys.stdout.write('{0}: {1}'.format(node,data))
+            else:
+                sys.stderr.write('{0}: {1}'.format(node, data))
+        else:
+            pop = desc['popen']
+            ret = pop.poll()
+            if ret is not None:
+                exitcode = exitcode | ret
+                all.discard(r)
+    if all:
+        rdy, _, _ = select.select(all, [], [], 10)
+sys.exit(exitcode)

confluent_client/bin/nodesensors

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import csv
+import datetime
 import optparse
 import os
 import sys
@@ -39,8 +40,8 @@ sensorcollections = {
 
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [sensor(s)")
-argparser.add_option('-i', '--interval', type='int',
+    usage="Usage: %prog [options] noderange ([sensor(s)])")
+argparser.add_option('-i', '--interval', type='float',
                      help='Interval to do repeated samples over')
 argparser.add_option('-n', '--numreadings', type='int',
                      help='Number of readings to gather')
@@ -59,7 +60,7 @@ if options.numreadings:
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 sensors = []
 for sensorgroup in args[1:]:
@@ -146,7 +147,12 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            if showtime.is_integer():
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            else:
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
+                           str(datetime.datetime.now().microsecond//1000),
+                           nodekey]
         else:
             rowdata = [nodekey]
         for sensorkey in orderedsensors:
@@ -197,7 +203,8 @@ def main():
             nextstart = os.times()[4] + options.interval
             resdata = sensorpass(linebyline, True)
             if options.csv:
-                format_csv(csvwriter, orderedsensors, resdata)
+                format_csv(csvwriter, orderedsensors, resdata,
+                           showtime=options.interval)
             if options.numreadings:
                 options.numreadings -= 1
                 if options.numreadings <= 0:

confluent_client/bin/nodesetboot

@@ -26,7 +26,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')
@@ -40,9 +41,7 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 try:
     noderange = args[0]
 except IndexError:
-    sys.stderr.write(
-        'Usage: {0} <noderange> [default|cd|network|setup|hd]\n'.format(
-            sys.argv[0]))
+    argparser.print_help()
     sys.exit(1)
 bootdev = None
 if len(sys.argv) > 2:
@@ -55,7 +54,6 @@ if options.biosmode:
     bootmode = 'bios'
 else:
     bootmode = 'uefi'
-sys.exit(
-    session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
+sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist))

confluent_client/doc/man/confetty.ronn

@@ -0,0 +1,37 @@
+confetty(1) --- Interactive confluent client
+=================================================
+
+## SYNOPSIS
+
+`confetty`
+
+## DESCRIPTION
+
+**confetty** launches an interactive CLI session to the
+confluent service.  It provides a filesystem-like
+view of the confluent interface.  It is intended to
+be mostly an aid for developing client software, with
+day to day administration generally being easier with
+the various function specific commands.
+
+## COMMANDS
+
+The CLI may be navigated by shell commands and some other
+commands.
+
+* `cd`:
+  Change the location within the tree
+* `ls`:
+  List the elements within the current directory/tree
+* `show` **ELEMENT**, `cat` **ELEMENT**:
+  Display the result of reading a specific element (by full or relative path)
+* `unset` **ELEMENT** **ATTRIBUTE**
+  For an element with attributes, request to clear the value of the attribue
+* `set` **ELEMENT** **ATTRIBUTE**=**VALUE**
+  Set the specified attribute to the given value
+* `start` **ELEMENT**
+  Start a console session indicated by **ELEMENT** (e.g. /nodes/n1/console/session)
+* `rm` **ELEMENT**
+  Request removal of an element.  (e.g. rm events/hardware/log clears log from a node)
+
+

confluent_client/doc/man/nodeconsole.ronn

@@ -0,0 +1,30 @@
+nodeconsole(1) -- Open a console to a confluent node
+=====================================================
+
+## SYNOPSIS
+`nodeconsole` `node`
+
+## DESCRIPTION
+
+**nodeconsole** opens an interactive console session to a given node.  This is the
+text or serial console of a system.  Exiting is done by hitting `Ctrl-e`, then `c`,
+ then `.`.  Note that console output by default is additionally logged to 
+`/var/log/confluent/consoles/`**NODENAME**.
+
+## ESCAPE SEQUENCE COMMANDS
+
+While connected to a console, a number of commands may be performed through escape
+sequences.  To begin an command escape sequence, hit `Ctrl-e`, then `c`.  The next
+keystroke will be interpreted as a command.  The following commands are available.
+
+* `.`:
+  Exit the session and return to the command prompt
+* `b`:
+  Send a break to the remote console when possible (some console plugins may not support this)
+* `o`:
+  Request confluent to disconnect and reconnect to console.  For example if there is suspicion
+  that the console has gone inoperable, but would work if reconnected.
+* `?`:
+  Get a list of supported commands
+* `<enter>`:
+  Abandon entering an escape sequence command

confluent_client/doc/man/nodelist.ronn

@@ -0,0 +1,48 @@
+nodelist(1) -- List confluent nodes and their attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodelist` `noderange`  
+`nodelist` `noderange` [-b] [<nodeattribute>...]  
+
+## DESCRIPTION
+
+**nodelist** queries the confluent server to get information about nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  There is more
+information on node attributes in nodeattributes(5) man page.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+  `# nodelist n1-n4`  
+  `n1`  
+  `n2`  
+  `n3`  
+  `n4`  
+
+* Getting an attribute of nodes matching a noderange:
+  `# nodelist n1,n2 hardwaremanagement.manager`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Getting a group of attributes while determining what group defines them:
+  `# nodelist n1,n2 hardwaremanegement --blame`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n1: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n1: hardwaremanagement.switch: r8e1`  
+  `n1: hardwaremanagement.switchport: 14`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+  `n2: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n2: hardwaremanagement.switch: r8e1`  
+  `n2: hardwaremanagement.switchport: 2`  
+

confluent_client/doc/man/nodesensors.ronn

@@ -0,0 +1,55 @@
+nodesensors(1) --- Retrieve telemetry for sensors of confluent nodes
+====================================================================
+
+## SYNOPSIS
+
+`nodesensors` `noderange` [-c] [-i <interval>] [-n <samplecount>]  [<sensor name  or category>...]  
+
+
+## DESCRIPTION
+
+**nodesensors** queries the confluent server to get telemetry from nodes.  Telemetry can include
+data such as temperature, power, and so forth.  Without arguments, it lists all available sensors
+and their current values.  If `-c` is specified, CSV format is used for output.  Normally
+nodesensors outputs once and exits.  Repeated periodic gathering can be done with `-i` to specify
+interval and `-n` to specify number of requests.  If `-i` is specified without `-n`, then it will
+retrieve data at the requested interval indefinitely.  If '-n' is specified without `-i`, an
+interval of 1 second is used.
+
+## OPTIONS
+
+* `-c`, `--csv`:
+  Organize output into CSV format, one sensor per column.
+
+* `-i`, `--interval`=**SECONDS**:
+  Repeat data gathering waiting, waiting the specified time between samples.  Unless `-n` is
+  specified, indefinite retrieval is assumed.
+
+* `-n`, `--numreadings`=**SAMPLES**:
+  Perform the specified number of readings, waiting `-i` indicated interval or 1 second if not
+  otherwise indicated.
+
+## EXAMPLES
+* Retrieving all temperature related sensors from one system  
+  `# nodesensors n1 temperature`  
+  `n1: CPU 1 Overtemp: Ok`  
+  `n1: CPU 2 Overtemp: Ok`  
+  `n1: Inlet Temp: 16.0 °C`  
+  `n1: PCH Overtemp: Ok`  
+  `n1: LOM Temp: Ok`  
+
+* Retrieving a sensor named "Inlet Temp" for 4 systems over a 3 second period of time:  
+   `# nodesensors n1-n4 'Inlet Temp' -c -n 3`  
+   `time,node,Inlet Temp (°C)`  
+   `2016-10-04T15:09:20,n1,19.0`  
+   `2016-10-04T15:09:20,n2,18.0`  
+   `2016-10-04T15:09:20,n3,18.0`  
+   `2016-10-04T15:09:20,n4,17.0`  
+   `2016-10-04T15:09:21,n1,19.0`  
+   `2016-10-04T15:09:21,n2,18.0`  
+   `2016-10-04T15:09:21,n3,18.0`  
+   `2016-10-04T15:09:21,n4,17.0`  
+   `2016-10-04T15:09:22,n1,19.0`  
+   `2016-10-04T15:09:22,n2,18.0`  
+   `2016-10-04T15:09:22,n3,18.0`  
+   `2016-10-04T15:09:22,n4,17.0`  

confluent_client/setup.py.tmpl

@@ -10,6 +10,6 @@ setup(
     scripts=['bin/confetty', 'bin/nodeconsole', 'bin/nodeeventlog',
              'bin/nodefirmware', 'bin/nodehealth', 'bin/nodeidentify',
              'bin/nodeinventory', 'bin/nodelist', 'bin/nodepower',
-             'bin/nodesensors', 'bin/nodesetboot'],
+             'bin/nodesensors', 'bin/nodesetboot', 'bin/noderun'],
     data_files=[('/etc/profile.d', ['confluent_env.sh'])],
 )

confluent_server/confluent/asynchttp.py

@@ -110,9 +110,12 @@ class AsyncSession(object):
         del _asyncsessions[self.asyncid]
 
     def run_handler(self, handler, requestid):
-        for rsp in handler:
-            self.add(requestid, rsp)
-        self.add(requestid, messages.AsyncCompletion())
+        try:
+            for rsp in handler:
+                self.add(requestid, rsp)
+            self.add(requestid, messages.AsyncCompletion())
+        except Exception as e:
+            self.add(requestid, e)
 
     def get_responses(self, timeout=25):
         self.reaper.cancel()

confluent_server/confluent/config/attributes.py

@@ -188,12 +188,24 @@ node = {
 #        'appliesto': ['vm'],
 #    },
     'hardwaremanagement.manager': {
-        'description': 'The management address dedicated to this node',
+        'description': 'The management address dedicated to this node.  This '
+                       'is the address of, for example, the Lenovo IMM.',
     },
     'hardwaremanagement.method': {
         'description': 'The method used to perform operations such as power '
                        'control, get sensor data, get inventory, and so on. '
     },
+    'hardwaremanagement.switch': {
+        'description': 'The switch to which the hardware manager is connected.'
+                       ' Only relevant if using switch based discovery via the'
+                       ' hardware manager (Lenovo IMMs and CMMs).  Not '
+                       'applicable to Lenovo Flex nodes.'
+    },
+    'hardwaremanagement.switchport': {
+        'description': 'The port of the switch that the hardware manager is '
+                       'connected.  See documentation of '
+                       'hardwaremanagement.switch for more detail.'
+    },
     'enclosure.manager': {
         'description': "The management device for this node's chassis",
 #        'appliesto': ['system'],
@@ -202,40 +214,44 @@ node = {
          'description': 'The bay in the enclosure, if any',
 #        'appliesto': ['system'],
     },
+
 #    'enclosure.type': {
 #        'description': '''The type of enclosure in use (e.g. IBM BladeCenter,
 #IBM Flex)''',
 #        'appliesto': ['system'],
 #    },
-#    'inventory.serialnumber': {
+#    'id.serial': {
 #        'description': 'The manufacturer serial number of node',
 #    },
-#    'inventory.uuid': {
+#    'id.uuid': {
 #        'description': 'The UUID of the node as presented in DMI',
 #    },
-#    'inventory.modelnumber': {
+#    'id.modelnumber': {
 #        'description': 'The manufacturer dictated  model number for the node',
 #    },
-#    'inventory.snmpengineid': {
+#    'id.modelname': {
+#        'description': 'The manufacturer model label for the node',
+#    },
+#    'id.snmpengineid': {
 #        'description': 'The SNMP Engine id used by this node',
 #    },
 #    'secret.snmpuser': {
 #        'description': 'The user to use for SNMPv3 access to this node',
 #    },
-#    'secret.snmppassphrase': {
-#        'description': 'The passphrase to use for SNMPv3 access to this node',
+#    'secret.snmppassword': {
+#        'description': 'The password to use for SNMPv3 access to this node',
 #    },
+    'secret.snmpcommunity': {
+        'description': ('SNMPv1 community string, it is highly recommended to'
+                        'step up to SNMPv3'),
+    },
 #    'secret.snmplocalizedkey': {
 #        'description': ("SNMPv3 key localized to this node's SNMP Engine id"
 #                        'This can be used in lieu of snmppassphrase to avoid'
 #                        'retaining the passphrase TODO: document procedure'
 #                        'to commit passphrase to localized key'),
 #    },
-#    'secret.snmpcommunity': {
-#        'description': ('SNMPv1 community string, it is highly recommended to'
-#                        'step up to SNMPv3'),
-#    },
-#    'secret.localadminpassphrase': {
+#    'secret.adminpassword': {
 #        'description': ('The passphrase to apply to local root/administrator '
 #                        'account. '
 #                        'If the environment is 100% Linux, the value may be '
@@ -246,35 +262,18 @@ node = {
 #                        'AD')
 #    },
     'secret.ipmikg': {
-        'description': 'Optional Integrity key for IPMI communication'
+        'description': 'Optional Integrity key for IPMI communication.  This '
+                       'should generally be ignored, as mutual authentication '
+                       'is normally done with the password alone (which is a '
+                       'shared secret in IPMI)'
     },
-#    'secret.ipmiuser': {
-#        'description': ('The username to use to log into IPMI device related '
-#                        'to the node.  For setting username, default '
-#                        'behavior is to randomize username, for using '
-#                        'username if not set, USERID is assumed'),
-#    },
-#    'secret.ipmipassphrase': {
-#        'description': ('The key to use to authenticate to IPMI device '
-#                        'related to the node.  For setting passphrase, '
-#                        'default behavior is to randomize passphrase and '
-#                        'store it here.  If going to connect over the '
-#                        'network and value is not set, PASSW0RD is attempted')
-#    },
     'secret.hardwaremanagementuser': {
-        'description': ('Username to be set and used by protocols like SSH '
-                        'and HTTP where client provides passphrase over the '
-                        'network. Given the distinct security models betwen '
-                        'this class of protocols and SNMP and IPMI, snmp and '
-                        'ipmi utilize dedicated values.'),
+        'description': ('The username to use when connecting to the hardware '
+                        'manager'),
     },
     'secret.hardwaremanagementpassword': {
-        'description': ('Passphrase to be set and used by protocols like SSH '
-                        'and HTTP, where client sends passphrase over the '
-                        'network.  Given distinct security models between '
-                        'this class of protocols, SNMP, and IPMI, SNMP and '
-                        'IPMI are given their own settings with distinct '
-                        'behaviors'),
+        'description': ('Password to use when connecting to the hardware '
+                        'manager'),
     },
     'pubkeys.addpolicy': {
         'description': ('Policy to use when encountering unknown public '

confluent_server/confluent/config/configmanager.py

@@ -761,7 +761,17 @@ class ConfigManager(object):
                                                    decrypt=self.decrypt)
         return nodeobj
 
-    def get_node_attributes(self, nodelist, attributes=()):
+    def expand_attrib_expression(self, nodelist, expression):
+        if type(nodelist) in (unicode, str):
+            nodelist = (nodelist,)
+        for node in nodelist:
+            cfgobj = self._cfgstore['nodes'][node]
+            fmt = _ExpressionFormat(cfgobj, node)
+            yield (node, fmt.format(expression))
+
+    def get_node_attributes(self, nodelist, attributes=(), decrypt=None):
+        if decrypt is None:
+            decrypt = self.decrypt
         retdict = {}
         relattribs = attributes
         if isinstance(nodelist, str) or isinstance(nodelist, unicode):
@@ -783,7 +793,7 @@ class ConfigManager(object):
                 # skipped.  The decryption, however, we want to do only on
                 # demand
                 nodeobj[attribute] = _decode_attribute(attribute, cfgnodeobj,
-                                                       decrypt=self.decrypt)
+                                                       decrypt=decrypt)
             retdict[node] = nodeobj
         return retdict
 

confluent_server/confluent/consoleserver.py

@@ -194,6 +194,8 @@ class ConsoleHandler(object):
             self._console = plugin.handle_path(
                 self._plugin_path.format(self.node),
                 "create", self.cfgmgr)
+        except exc.NotImplementedException:
+            self._console = None
         except:
             _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                           event=log.Events.stacktrace)
@@ -255,11 +257,12 @@ class ConsoleHandler(object):
         self._send_rcpts({'connectstate': self.connectstate})
 
     def _got_disconnected(self):
-        self.connectstate = 'unconnected'
-        self.log(
-            logdata='console disconnected', ltype=log.DataTypes.event,
-            event=log.Events.consoledisconnect)
-        self._send_rcpts({'connectstate': self.connectstate})
+        if self.connectstate != 'unconnected':
+            self.connectstate = 'unconnected'
+            self.log(
+                logdata='console disconnected', ltype=log.DataTypes.event,
+                event=log.Events.consoledisconnect)
+            self._send_rcpts({'connectstate': self.connectstate})
         if self._isalive:
             self._connect()
 

confluent_server/confluent/core.py

@@ -78,7 +78,7 @@ def load_plugins():
         plugindir = os.path.join(plugintop, plugindir)
         if not os.path.isdir(plugindir):
             continue
-        sys.path.append(plugindir)
+        sys.path.insert(1, plugindir)
         # two passes, to avoid adding both py and pyc files
         for plugin in os.listdir(plugindir):
             if plugin.startswith('.'):
@@ -96,6 +96,8 @@ def load_plugins():
                     pluginmap[name] = tmpmod
             else:
                 pluginmap[plugin] = tmpmod
+        # restore path to not include the plugindir
+        sys.path.pop(1)
 
 
 rootcollections = ['noderange/', 'nodes/', 'nodegroups/', 'users/', 'events/']
@@ -120,6 +122,7 @@ def _init_core():
         'attributes': {
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
+            'expression': PluginRoute({'handler': 'attributes'}),
         },
         'boot': {
             'nextdevice': PluginRoute({
@@ -570,10 +573,19 @@ def handle_node_request(configmanager, inputdata, operation,
                 inputdata=inputdata))
         if isnoderange or not autostrip:
             return itertools.chain(*passvalues)
-        elif isinstance(passvalues[0], console.Console):
-            return passvalues[0]
         else:
-            return stripnode(passvalues[0], nodes[0])
+            if len(passvalues) > 0:
+                if isinstance(passvalues[0], console.Console):
+                    return passvalues[0]
+                else:
+                    return stripnode(passvalues[0], nodes[0])
+            else:
+                raise exc.NotImplementedException()
+
+        # elif isinstance(passvalues[0], console.Console):
+        #     return passvalues[0]
+        # else:
+        #     return stripnode(passvalues[0], nodes[0])
 
 
 def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):

confluent_server/confluent/exceptions.py

@@ -24,53 +24,61 @@ class ConfluentException(Exception):
     apierrorstr = 'Unexpected Error'
 
     def get_error_body(self):
-        return self.apierrorstr
+        errstr = ' - '.join((self.apierrorstr, str(self)))
+        return json.dumps({'error': errstr })
 
 
 class NotFoundException(ConfluentException):
     # Something that could be construed as a name was not found
     # basically, picture an http error code 404
-    pass
+    apierrorcode = 404
+    apierrorstr = 'Request path not recognized'
 
 
 class InvalidArgumentException(ConfluentException):
     # Something from the remote client wasn't correct
     # like http code 400
-    pass
+    apierrorcode = 400
+    apierrorstr = 'Bad Request'
 
 
 class TargetEndpointUnreachable(ConfluentException):
     # A target system was unavailable.  For example, a BMC
     # was unreachable.  http code 504
-    pass
+    apierrorcode = 504
+    apierrorstr = 'Unreachable Target'
 
 
 class TargetEndpointBadCredentials(ConfluentException):
     # target was reachable, but authentication/authorization
     # failed
-    pass
+    apierrorcode = 502
+    apierrorstr = 'Bad Credentials'
 
 
 class LockedCredentials(ConfluentException):
     # A request was performed that required a credential, but the credential
     # store is locked
-    pass
+    apierrorstr = 'Credential store locked'
 
 
 class ForbiddenRequest(ConfluentException):
     # The client request is not allowed by authorization engine
-    pass
+    apierrorcode = 403
+    apierrorstr = 'Forbidden'
 
 
 class NotImplementedException(ConfluentException):
     # The current configuration/plugin is unable to perform
     # the requested task. http code 501
-    pass
+    apierrorcode = 501
+    apierrorstr = '501 - Not Implemented'
+
 
 
 class GlobalConfigError(ConfluentException):
     # The configuration in the global config file is not right
-    pass
+    apierrorstr = 'Global configuration contains an error'
 
 
 class PubkeyInvalid(ConfluentException):

confluent_server/confluent/httpapi.py

@@ -35,6 +35,7 @@ import eventlet.greenthread
 import greenlet
 import json
 import socket
+import sys
 import traceback
 import time
 import urlparse
@@ -214,6 +215,44 @@ def _should_skip_authlog(env):
         return True
     return False
 
+def _csrf_valid(env, session):
+    # This could be simplified into a statement, but this is more readable
+    # to have it broken out
+    if (env['REQUEST_METHOD'] == 'GET' and
+            env['PATH_INFO'] == '/sessions/current/info'):
+        # Provide a web client a safe hook to request the CSRF token
+        # This means that we consider GET of /sessions/current/info to be
+        # a safe thing to inflict via CSRF, since CORS should prevent
+        # hypothetical attacker from reading the data and it has no
+        # side effects to speak of
+        return True
+    if 'csrftoken' not in session:
+        # The client has not (yet) requested CSRF protection
+        # so we return true
+        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
+            # The client has requested CSRF countermeasures,
+            # oblige the request and apply a new token to the
+            # session
+            session['csrftoken'] = util.randomstring(32)
+        elif 'HTTP_REFERER' in env:
+            # If there is a referrer, make sure it stays consistent
+            # across the session.  A change in referer is a bad thing
+            try:
+                referer = env['HTTP_REFERER'].split('/')[2]
+            except IndexError:
+                return False
+            if 'validreferer' not in session:
+                session['validreferer'] = referer
+            elif session['validreferer'] != referer:
+                return False
+        return True
+    # The session has CSRF protection enabled, only mark valid if
+    # the client has provided an auth token and that token matches the
+    # value protecting the session
+    return ('HTTP_CONFLUENTAUTHTOKEN' in env and
+            env['HTTP_CONFLUENTAUTHTOKEN'] == session['csrftoken'])
+
+
 def _authorize_request(env, operation):
     """Grant/Deny access based on data from wsgi env
 
@@ -228,22 +267,30 @@ def _authorize_request(env, operation):
         cc.load(env['HTTP_COOKIE'])
         if 'confluentsessionid' in cc:
             sessionid = cc['confluentsessionid'].value
+            sessid = sessionid
             if sessionid in httpsessions:
-                if env['PATH_INFO'] == '/sessions/current/logout':
-                    targets = []
-                    for mythread in httpsessions[sessionid]['inflight']:
-                        targets.append(mythread)
-                    for mythread in targets:
-                        eventlet.greenthread.kill(mythread)
-                    del httpsessions[sessionid]
-                    return ('logout',)
-                httpsessions[sessionid]['expiry'] = time.time() + 90
-                name = httpsessions[sessionid]['name']
-                authdata = auth.authorize(
-                    name, element=None,
-                    skipuserobj=httpsessions[sessionid]['skipuserobject'])
+                if _csrf_valid(env, httpsessions[sessionid]):
+                    if env['PATH_INFO'] == '/sessions/current/logout':
+                        targets = []
+                        for mythread in httpsessions[sessionid]['inflight']:
+                            targets.append(mythread)
+                        for mythread in targets:
+                            eventlet.greenthread.kill(mythread)
+                        del httpsessions[sessionid]
+                        return ('logout',)
+                    httpsessions[sessionid]['expiry'] = time.time() + 90
+                    name = httpsessions[sessionid]['name']
+                    authdata = auth.authorize(
+                        name, element=None,
+                        skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
         if env['PATH_INFO'] == '/sessions/current/logout':
+            if 'HTTP_REFERER' in env:
+                # note that this doesn't actually do harm
+                # otherwise, but this way do not give appearance
+                # of something having a side effect if it has the smell
+                # of a CSRF
+                return {'code': 401}
             return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
@@ -256,6 +303,8 @@ def _authorize_request(env, operation):
         httpsessions[sessid] = {'name': name, 'expiry': time.time() + 90,
                                 'skipuserobject': authdata[4],
                                 'inflight': set([])}
+        if 'HTTP_CONFLUENTAUTHTOKEN' in env:
+            httpsessions[sessid]['csrftoken'] = util.randomstring(32)
         cookie['confluentsessionid'] = sessid
         cookie['confluentsessionid']['secure'] = 1
         cookie['confluentsessionid']['httponly'] = 1
@@ -276,10 +325,12 @@ def _authorize_request(env, operation):
             auditmsg['tenant'] = authdata[3]
             authinfo['tenant'] = authdata[3]
         auditmsg['user'] = authdata[2]
-        if sessionid is not None:
-            authinfo['sessionid'] = sessionid
+        if sessid is not None:
+            authinfo['sessionid'] = sessid
         if not skiplog:
             auditlog.log(auditmsg)
+        if 'csrftoken' in httpsessions[sessid]:
+            authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
         return authinfo
     else:
         return {'code': 401}
@@ -336,7 +387,13 @@ def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
     mimetype, extension = _pick_mimetype(env)
-    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-cache')]
+    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-store'),
+               ('Pragma', 'no-cache'),
+               ('X-Content-Type-Options', 'nosniff'),
+               ('Content-Security-Policy', "default-src 'self'"),
+               ('X-XSS-Protection', '1'), ('X-Frame-Options', 'deny'),
+               ('Strict-Transport-Security', 'max-age=86400'),
+               ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
     reqtype = None
     if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
@@ -349,10 +406,10 @@ def resourcehandler_backend(env, start_response):
         del querydict['restexplorerop']
     authorized = _authorize_request(env, operation)
     if 'logout' in authorized:
-        start_response('200 Sucessful logout', headers)
+        start_response('200 Successful logout', headers)
         yield('{"result": "200 - Successful logout"}')
         return
-    if 'HTTP_SUPPRESSAUTHHEADER' in env:
+    if 'HTTP_SUPPRESSAUTHHEADER' in env or 'HTTP_CONFLUENTAUTHTOKEN' in env:
         badauth = [('Content-type', 'text/plain')]
     else:
         badauth = [('Content-type', 'text/plain'),
@@ -461,6 +518,17 @@ def resourcehandler_backend(env, start_response):
             start_response('200 OK', headers)
             yield '{"sessionclosed": true}'
             return
+        elif 'action' in querydict:
+            if querydict['action'] == 'break':
+                consolesessions[querydict['session']]['session'].send_break()
+            elif querydict['action'] == 'reopen':
+                consolesessions[querydict['session']]['session'].reopen()
+            else:
+                start_response('400 Bad Request')
+                yield 'Unrecognized action ' + querydict['action']
+                return
+            start_response('200 OK', headers)
+            yield json.dumps({'session': querydict['session']})
         else:  # no keys, but a session, means it's hooking to receive data
             sessid = querydict['session']
             if sessid not in consolesessions:
@@ -519,7 +587,10 @@ def resourcehandler_backend(env, start_response):
         url = url.replace('.html', '')
         if url == '/sessions/current/info':
             start_response('200 OK', headers)
-            yield json.dumps({'username': authorized['username']})
+            sessinfo = {'username': authorized['username']}
+            if 'authtoken' in authorized:
+                sessinfo['authtoken'] = authorized['authtoken']
+            yield json.dumps(sessinfo)
             return
         resource = '.' + url[url.rindex('/'):]
         lquerydict = copy.deepcopy(querydict)
@@ -541,26 +612,9 @@ def resourcehandler_backend(env, start_response):
                     pagecontent += datum
             start_response('200 OK', headers)
             yield pagecontent
-        except exc.NotFoundException as ne:
-            start_response('404 Not found', headers)
-            yield "404 - Request path not recognized - " + str(ne)
-        except exc.InvalidArgumentException as e:
-            start_response('400 Bad Request - ' + str(e), headers)
-            yield '400 - Bad Request - ' + str(e)
-        except exc.TargetEndpointUnreachable as tu:
-            start_response('504 Unreachable Target', headers)
-            yield '504 - Unreachable Target - ' + str(tu)
-        except exc.TargetEndpointBadCredentials:
-            start_response('502 Bad Credentials', headers)
-            yield '502 - Bad Credentials'
-        except exc.LockedCredentials:
-            start_response('500 Locked credential store', headers)
-            yield '500 - Credential store locked'
-        except exc.NotImplementedException:
-            start_response('501 Not Implemented', headers)
-            yield '501 Not Implemented'
         except exc.ConfluentException as e:
-            if e.apierrorcode == 500:
+            if ((not isinstance(e, exc.LockedCredentials)) and
+                    e.apierrorcode == 500):
                 # raise generics to trigger the tracelog
                 raise
             start_response('{0} {1}'.format(e.apierrorcode, e.apierrorstr),
@@ -688,9 +742,20 @@ def serve(bind_host, bind_port):
     #but deps are simpler without flup
     #also, the potential for direct http can be handy
     #todo remains unix domain socket for even http
-    eventlet.wsgi.server(
-        eventlet.listen((bind_host, bind_port, 0, 0), family=socket.AF_INET6),
-        resourcehandler, log=False, log_output=False, debug=False)
+    sock = None
+    while not sock:
+        try:
+            sock = eventlet.listen(
+                (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write(
+                'Failed to open HTTP due to busy port, trying again in'
+                ' a second\n')
+            eventlet.sleep(1)
+    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                         debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -444,12 +444,17 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
         # As time rolling happens, reset statistics count
         self.sizeRollingCount = 0
         dbfn = self.binpath + "." + time.strftime(self.suffix, timeTuple)
+        odbfn = dbfn
         dtfn = self.textpath + "." + time.strftime(self.suffix, timeTuple)
-
-        if os.path.exists(dbfn):
-            os.remove(dbfn)
-        if os.path.exists(dtfn):
-            os.remove(dtfn)
+        odtfn = dtfn
+        append=1
+        while os.path.exists(dbfn):
+            dbfn = odbfn + '.{}'.format(append)
+            append += 1
+        append=1
+        while os.path.exists(dtfn):
+            dtfn = odtfn + '.{}'.format(append)
+            append += 1
         if os.path.exists(self.binpath):
             os.rename(self.binpath, dbfn)
         if os.path.exists(self.textpath):
@@ -613,8 +618,14 @@ class Logger(object):
                 struct.unpack(">BBIHIBBH", recbytes)
             # rolling events found.
             if ltype == DataTypes.event and evtdata == Events.logrollover:
-                textpath, binpath = parse_last_rolling_files(textfile, offset,
-                                                             datalen)
+                txtpath, bpath = parse_last_rolling_files(textfile, offset,
+                                                          datalen)
+                if txtpath == textpath:
+                    break
+                if bpath == binpath:
+                    break
+                textpath = txtpath
+                binpath = bpath
                 # Rolling event detected, close the current bin file, then open
                 # the renamed bin file.
                 flock(binfile, LOCK_UN)
@@ -722,3 +733,19 @@ class Logger(object):
     def closelog(self):
         self.handler.close()
         self.closer = None
+
+globaleventlog = None
+tracelog = None
+
+
+def log(logdata=None, ltype=None, event=0, eventdata=None):
+    if globaleventlog is None:
+        globaleventlog = Logger('events')
+    globaleventlog.log(logdata, ltype, event, eventdata)
+
+def logtrace():
+    global tracelog
+    if tracelog is None:
+        tracelog = Logger('trace')
+    tracelog.log(traceback.format_exc(), ltype=DataTypes.event,
+                 event=Events.stacktrace)

confluent_server/confluent/main.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -95,9 +95,15 @@ def _checkpidfile():
         fcntl.flock(pidfile, fcntl.LOCK_EX)
         pid = pidfile.read()
         if pid != '':
-            print ('/var/run/confluent/pid exists and indicates %s is still '
-                   'running' % pid)
-            sys.exit(1)
+            try:
+                os.kill(int(pid), 0)
+                print ('/var/run/confluent/pid exists and indicates %s is still '
+                       'running' % pid)
+                sys.exit(1)
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        pidfile.seek(0)
         pidfile.write(str(os.getpid()))
         fcntl.flock(pidfile, fcntl.LOCK_UN)
         pidfile.close()
@@ -199,17 +205,18 @@ def run():
     auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
-    #TODO(jbjohnso): eventlet has a bug about unix domain sockets, this code
-    #works with bugs fixed
     if dbgif:
         oumask = os.umask(0077)
+        try:
+            os.remove('/var/run/confluent/dbg.sock')
+        except OSError:
+            pass  # We are not expecting the file to exist
         dbgsock = eventlet.listen("/var/run/confluent/dbg.sock",
                                    family=socket.AF_UNIX)
         eventlet.spawn_n(backdoor.backdoor_server, dbgsock)
         os.umask(oumask)
     http_bind_host, http_bind_port = _get_connector_config('http')
     sock_bind_host, sock_bind_port = _get_connector_config('socket')
-    consoleserver.start_console_sessions()
     webservice = httpapi.HttpApi(http_bind_host, http_bind_port)
     webservice.start()
     try:
@@ -218,6 +225,8 @@ def run():
     except NameError:
         pass
     atexit.register(doexit)
+    eventlet.sleep(1)
+    consoleserver.start_console_sessions()
     while 1:
         eventlet.sleep(100)
 

confluent_server/confluent/messages.py

@@ -19,6 +19,7 @@
 # Things are defined here to 'encourage' developers to coordinate information
 # format.  This is also how different data formats are supported
 import confluent.exceptions as exc
+from copy import deepcopy
 from datetime import datetime
 import json
 
@@ -58,6 +59,7 @@ def _htmlify_structure(indict):
 
 
 class ConfluentMessage(object):
+    apicode = 200
     readonly = False
     defaultvalue = ''
     defaulttype = 'text'
@@ -173,12 +175,15 @@ class ConfluentMessage(object):
 
 
 class ConfluentNodeError(object):
+    apicode = 500
+
     def __init__(self, node, errorstr):
         self.node = node
         self.error = errorstr
 
     def raw(self):
-        return {'databynode': {self.node: {'error': self.error}}}
+        return {'databynode': {self.node: {'errorcode': self.apicode,
+                                           'error': self.error}}}
 
     def html(self):
         return self.node + ":" + self.error
@@ -190,15 +195,20 @@ class ConfluentNodeError(object):
 
 
 class ConfluentTargetTimeout(ConfluentNodeError):
+    apicode = 504
+
     def __init__(self, node, errstr='timeout'):
         self.node = node
         self.error = errstr
 
+
     def strip_node(self, node):
         raise exc.TargetEndpointUnreachable(self.error)
 
 
 class ConfluentTargetNotFound(ConfluentNodeError):
+    apicode = 404
+
     def __init__(self, node, errorstr='not found'):
         self.node = node
         self.error = errorstr
@@ -208,6 +218,7 @@ class ConfluentTargetNotFound(ConfluentNodeError):
 
 
 class ConfluentTargetInvalidCredentials(ConfluentNodeError):
+    apicode = 502
     def __init__(self, node):
         self.node = node
         self.error = 'bad credentials'
@@ -418,7 +429,7 @@ class InputAttributes(ConfluentMessage):
     def get_attributes(self, node):
         if node not in self.nodeattribs:
             return {}
-        nodeattr = self.nodeattribs[node]
+        nodeattr = deepcopy(self.nodeattribs[node])
         for attr in nodeattr:
             if type(nodeattr[attr]) in (str, unicode):
                 try:
@@ -492,7 +503,7 @@ class InputCredential(ConfluentMessage):
     def get_attributes(self, node):
         if node not in self.credentials:
             return {}
-        credential = self.credentials[node]
+        credential = deepcopy(self.credentials[node])
         for attr in credential:
             if type(credential[attr]) in (str, unicode):
                 try:
@@ -869,6 +880,11 @@ class AsyncMessage(ConfluentMessage):
         if (isinstance(rsp, ConfluentMessage) or
                 isinstance(rsp, ConfluentNodeError)):
             rspdict = rsp.raw()
+        elif isinstance(rsp, exc.ConfluentException):
+            rspdict = {'exceptioncode': rsp.apierrorcode,
+                       'exception': rsp.get_error_body()}
+        elif isinstance(rsp, Exception):
+            rspdict = {'exceptioncode': 500, 'exception': str(rsp)}
         elif isinstance(rsp, dict):  # console metadata
             rspdict = rsp
         else: # terminal text

confluent_server/confluent/networking/macmap.py

@@ -0,0 +1,249 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This provides the implementation of locating MAC addresses on ethernet
+# switches.  It is, essentially, a port of 'MacMap.pm' to confluent.
+# However, there are enhancements.
+# For one, each switch interrogation is handled in an eventlet 'thread'
+# For another, MAC addresses are checked in the dictionary on every
+# switch return, rather than waiting for all switches to check in
+# (which makes it more responsive when there is a missing or bad switch)
+# Also, we track the quantity, actual ifName value, and provide a mechanism
+# to detect ambiguous result (e.g. if two matches are found, can log an error
+# rather than doing the wrong one, complete with the detected ifName value).
+# Further, the map shall be available to all facets of the codebase, not just
+# the discovery process, so that the cached data maintenance will pay off
+# for direct queries
+
+# this module will provide mac to switch and full 'ifName' label
+# This functionality is restricted to the null tenant
+
+import confluent.exceptions as exc
+import confluent.log as log
+import confluent.snmputil as snmp
+from eventlet.greenpool import GreenPool
+import re
+
+_macmap = {}
+_macsbyswitch = {}
+_nodesbymac = {}
+_switchportmap = {}
+
+
+_whitelistnames = (
+    # 3com
+    re.compile(r'^RMON Port (\d+) on unit \d+'),
+    # Dell
+    re.compile(r'^Unit \d+ Port (\d+)\Z'),
+)
+
+_blacklistnames = (
+    re.compile(r'vl'),
+    re.compile(r'Nu'),
+    re.compile(r'RMON'),
+    re.compile(r'onsole'),
+    re.compile(r'Stack'),
+    re.compile(r'Trunk'),
+    re.compile(r'po\d'),
+    re.compile(r'XGE'),
+    re.compile(r'LAG'),
+    re.compile(r'CPU'),
+    re.compile(r'Management'),
+)
+
+
+def _namesmatch(switchdesc, userdesc):
+    if switchdesc == userdesc:
+        return True
+    try:
+        portnum = int(userdesc)
+    except ValueError:
+        portnum = None
+    if portnum is not None:
+        for exp in _whitelistnames:
+            match = exp.match(switchdesc)
+            if match:
+                snum = int(match.groups()[0])
+                if snum == portnum:
+                    return True
+    anymatch = re.search(r'[^0123456789]' + userdesc + r'(\.0)?\Z', switchdesc)
+    if anymatch:
+        for blexp in _blacklistnames:
+            if blexp.match(switchdesc):
+                return False
+        return True
+    return False
+
+def _map_switch(args):
+    try:
+        return _map_switch_backend(args)
+    except Exception as e:
+        log.logtrace()
+
+
+def _nodelookup(switch, ifname):
+    """Get a nodename for a given switch and interface name
+    """
+    for portdesc in _switchportmap.get(switch, {}):
+        if _namesmatch(ifname, portdesc):
+            return _switchportmap[switch][portdesc]
+    return None
+
+
+def _map_switch_backend(args):
+    """Manipulate portions of mac address map relevant to a given switch
+    """
+
+    # 1.3.6.1.2.1.17.7.1.2.2.1.2 - mactoindex (qbridge - preferred)
+    #  if not, check for cisco and if cisco, build list of all relevant vlans:
+    #  .1.3.6.1.4.1.9.9.46.1.6.1.1.5 - trunk port vlan map (cisco only)
+    #  .1.3.6.1.4.1.9.9.68.1.2.2.1.2 - access port vlan map (cisco only)
+    # if cisco, vlan community string indexed or snmpv3 contest for:
+    # 1.3.6.1.2.1.17.4.3.1.2 - mactoindx (bridge - low-end switches and cisco)
+    #     .1.3.6.1.2.1.17.1.4.1.2 - bridge index to if index map
+    # no vlan index or context for:
+    #  .1.3.6.1.2.1.31.1.1.1.1 - ifName... but some switches don't do it
+    #  .1.3.6.1.2.1.2.2.1.2 - ifDescr, usually useless, but a
+    #   fallback if ifName is empty
+    #
+    global _macmap
+    switch, password, user = args
+    haveqbridge = False
+    mactobridge = {}
+    conn = snmp.Session(switch, password, user)
+    for vb in conn.walk('1.3.6.1.2.1.17.7.1.2.2.1.2'):
+        haveqbridge = True
+        oid, bridgeport = vb
+        if not bridgeport:
+            continue
+        oid = str(oid).rsplit('.', 6)  # if 7, then oid[1] would be vlan id
+        macaddr = '{0:02x}:{1:02x}:{2:02x}:{3:02x}:{4:02x}:{5:02x}'.format(
+            *([int(x) for x in oid[-6:]])
+        )
+        mactobridge[macaddr] = int(bridgeport)
+    if not haveqbridge:
+        raise exc.NotImplementedException('TODO: Bridge-MIB without QBRIDGE')
+    bridgetoifmap = {}
+    for vb in conn.walk('1.3.6.1.2.1.17.1.4.1.2'):
+        bridgeport, ifidx = vb
+        bridgeport = int(str(bridgeport).rsplit('.', 1)[1])
+        bridgetoifmap[bridgeport] = int(ifidx)
+    ifnamemap = {}
+    havenames = False
+    for vb in conn.walk('1.3.6.1.2.1.31.1.1.1.1'):
+        ifidx, ifname = vb
+        if not ifname:
+            continue
+        havenames = True
+        ifidx = int(str(ifidx).rsplit('.', 1)[1])
+        ifnamemap[ifidx] = str(ifname)
+    if not havenames:
+        for vb in conn.walk( '1.3.6.1.2.1.2.2.1.2'):
+            ifidx, ifname = vb
+            ifidx = int(str(ifidx).rsplit('.', 1)[1])
+            ifnamemap[ifidx] = str(ifname)
+    maccounts = {}
+    for mac in mactobridge:
+        ifname = ifnamemap[bridgetoifmap[mactobridge[mac]]]
+        if ifname not in maccounts:
+            maccounts[ifname] = 1
+        else:
+            maccounts[ifname] += 1
+    _macsbyswitch[switch] = {}
+    for mac in mactobridge:
+        # We want to merge it so that when a mac appears in multiple
+        # places, it is captured.
+        ifname = ifnamemap[bridgetoifmap[mactobridge[mac]]]
+        if mac in _macmap:
+            _macmap[mac].append((switch, ifname, maccounts[ifname]))
+        else:
+            _macmap[mac] = [(switch, ifname, maccounts[ifname])]
+        if ifname in _macsbyswitch[switch]:
+            _macsbyswitch[switch][ifname].append(mac)
+        else:
+            _macsbyswitch[switch][ifname] = [mac]
+        nodename = _nodelookup(switch, ifname)
+        if nodename is not None:
+            if mac in _nodesbymac and _nodesbymac[mac] != nodename:
+                log.log({'warning': '{0} and {1} described by ambiguous'
+                                    ' switch topology values'.format(nodename,
+                                                              _nodesbymac[mac]
+                                                              )})
+            _nodesbymac[mac] = nodename
+
+
+def update_macmap(configmanager):
+    """Interrogate switches to build/update mac table
+
+    Begin a rebuild process.  This process is a generator that will yield
+    as each switch interrogation completes, allowing a caller to
+    recheck the cache as results become possible, rather
+    than having to wait for the process to complete to interrogate.
+    """
+    global _macmap
+    global _nodesbymac
+    global _switchportmap
+    # Clear all existing entries
+    _macmap = {}
+    _nodesbymac = {}
+    _switchportmap = {}
+    if configmanager.tenant is not None:
+        raise exc.ForbiddenRequest('Network topology not available to tenants')
+    nodelocations = configmanager.get_node_attributes(
+        configmanager.list_nodes(), ('hardwaremanagement.switch',
+                                     'hardwaremanagement.switchport'))
+    switches = set([])
+    for node in nodelocations:
+        cfg = nodelocations[node]
+        if 'hardwaremanagement.switch' in cfg:
+            curswitch = cfg['hardwaremanagement.switch']['value']
+            switches.add(curswitch)
+            if 'hardwaremanagement.switchport' in cfg:
+                portname = cfg['hardwaremanagement.switchport']['value']
+                if curswitch not in _switchportmap:
+                    _switchportmap[curswitch] = {}
+                if portname in _switchportmap[curswitch]:
+                    log.log({'warning': 'Duplicate switch topology config for '
+                                        '{0} and {1}'.format(node,
+                                                             _switchportmap[
+                                                                 curswitch][
+                                                                 portname])})
+                _switchportmap[curswitch][portname] = node
+    switchcfg = configmanager.get_node_attributes(
+        switches, ('secret.hardwaremanagementuser',
+                   'secret.hardwaremanagementpassword'), decrypt=True)
+    switchauth = []
+    for switch in switches:
+        password = 'public'
+        user = None
+        if (switch in switchcfg and
+                'secret.hardwaremanagementpassword' in switchcfg[switch]):
+            password = switchcfg[switch]['secret.hardwaremanagementpassword'][
+                'value']
+            if 'secret.hardwaremanagementuser' in switchcfg[switch]:
+                user = switchcfg[switch]['secret.hardwaremanagementuser'][
+                    'value']
+        switchauth.append((switch, password, user))
+    pool = GreenPool()
+    for res in pool.imap(_map_switch, switchauth):
+        yield res
+        print(repr(_macmap))
+
+
+if __name__ == '__main__':
+    # invoke as switch community
+    import sys
+    _map_switch(sys.argv[1], sys.argv[2])

confluent_server/confluent/plugins/configuration/attributes.py

@@ -152,6 +152,20 @@ def update_nodegroup(group, element, configmanager, inputdata):
     return retrieve_nodegroup(group, element, configmanager, inputdata)
 
 
+def _expand_expression(nodes, configmanager, inputdata):
+    expression = inputdata.get_attributes(list(nodes)[0])
+    if type(expression) is dict:
+        expression = expression['expression']
+    if type(expression) is dict:
+        expression = expression['expression']
+    for expanded in configmanager.expand_attrib_expression(nodes, expression):
+        yield msg.KeyValueData({'value': expanded[1]}, expanded[0])
+
+
+def create(nodes, element, configmanager, inputdata):
+    if nodes is not None and element[-1] == 'expression':
+        return _expand_expression(nodes, configmanager, inputdata)
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
     for node in nodes:

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -51,6 +51,7 @@ sensor_categories = {
     'fans': frozenset(['Fan', 'Cooling Device']),
 }
 
+
 def hex2bin(hexstring):
     hexvals = hexstring.split(':')
     if len(hexvals) < 2:
@@ -60,6 +61,7 @@ def hex2bin(hexstring):
     bytedata = [int(i, 16) for i in hexvals]
     return bytearray(bytedata)
 
+
 def simplify_name(name):
     return name.lower().replace(' ', '_').replace('/', '-')
 
@@ -217,6 +219,7 @@ class IpmiConsole(conapi.Console):
                 self.broken = True
                 self.error = data['error']
                 if self.connected:
+                    self.connected = False
                     self.datacallback(conapi.ConsoleEvent.Disconnect)
         else:
             self.datacallback(data)
@@ -341,6 +344,7 @@ class IpmiHandler(object):
             except socket.gaierror as ge:
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
+                raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
 
     bootdevices = {
@@ -352,8 +356,9 @@ class IpmiHandler(object):
             self.broken = True
             self.error = response['error']
         else:
+            self.ipmicmd = ipmicmd
             self.loggedin = True
-            ipmicmd.setup_confluent_keyhandler()
+            self.ipmicmd.setup_confluent_keyhandler()
         self._logevt.set()
 
     def handle_request(self):
@@ -850,7 +855,12 @@ class IpmiHandler(object):
                     else:
                         idx = int(self.element[-1]) - 1
                         servers = self.ipmicmd.get_ntp_servers()
-                        self.output.put(msg.NTPServer(self.node, servers[idx]))
+                        if len(servers) > idx:
+                            self.output.put(msg.NTPServer(self.node, servers[idx]))
+                        else:
+                            self.output.put(
+                                msg.ConfluentTargetNotFound(
+                                    self.node, 'Requested NTP configuration not found'))
                         return
                 elif self.op in ('update', 'create'):
                     if self.element[-1] == 'all':

confluent_server/confluent/plugins/shell/ssh.py

@@ -117,6 +117,15 @@ class SshShell(conapi.Console):
 
     def write(self, data):
         if self.inputmode == 0:
+            while len(data) and data[0] == b'\x7f' and len(self.username):
+                self.datacallback('\b \b')  # erase previously echoed value
+                self.username = self.username[:-1]
+                data = data[1:]
+            while len(data) and data[0] == b'\x7f':
+                data = data[1:]
+            while b'\x7f' in data:
+                delidx = data.index(b'\x7f')
+                data = data[:delidx - 1] + data[delidx + 1:]
             self.username += data
             if '\r' in self.username:
                 self.username, self.password = self.username.split('\r')
@@ -125,10 +134,16 @@ class SshShell(conapi.Console):
                     self.datacallback(lastdata)
                 self.datacallback('\r\nEnter password: ')
                 self.inputmode = 1
-            else:
+            elif len(data) > 0:
                 # echo back typed data
                 self.datacallback(data)
         elif self.inputmode == 1:
+            while len(data) > 0 and data[0] == b'\x7f':
+                self.password = self.password[:-1]
+                data = data[1:]
+            while b'\x7f' in data:
+                delidx = data.index(b'\x7f')
+                data = data[:delidx - 1] + data[delidx + 1:]
             self.password += data
             if '\r' in self.password:
                 self.password = self.password.split('\r')[0]

confluent_server/confluent/snmputil.py

@@ -0,0 +1,103 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This provides a simplified wrapper around snmp implementation roughly
+# mapping to the net-snmp commands
+
+# net-snmp-python was considered as the API is cleaner, but the ability to
+# patch pysnmp to have it be eventlet friendly has caused it's selection
+# This module simplifies the complex hlapi pysnmp interface
+
+import confluent.exceptions as exc
+import eventlet
+from eventlet.support.greendns import getaddrinfo
+import socket
+snmp = eventlet.import_patched('pysnmp.hlapi')
+
+
+def _get_transport(name):
+    # Annoyingly, pysnmp does not automatically determine ipv6 v ipv4
+    res = getaddrinfo(name, 161, 0, socket.SOCK_DGRAM)
+    if res[0][0] == socket.AF_INET6:
+        return snmp.Udp6TransportTarget(res[0][4])
+    else:
+        return snmp.UdpTransportTarget(res[0][4])
+
+
+class Session(object):
+
+    def __init__(self, server, secret, username=None, context=None):
+        """Create a new session to interrogate a switch
+
+        If username is not given, it is assumed that
+        the secret is community string, and v2c is used.  If a username given,
+        it'll assume SHA auth and DES privacy with the secret being the same
+        for both.
+
+        :param server: The network name/address to target
+        :param secret: The community string or password
+        :param username: The username for SNMPv3
+        :param context: The SNMPv3 context or index for community indexing
+        """
+        self.server = server
+        self.context = context
+        if username is None:
+            # SNMP v2c
+            self.authdata = snmp.CommunityData(secret, mpModel=1)
+        else:
+            self.authdata = snmp.UsmUserData(username, authKey=secret,
+                                             privKey=secret)
+        self.eng = snmp.SnmpEngine()
+
+    def walk(self, oid):
+        """Walk over children of a given OID
+
+        This is roughly equivalent to snmpwalk.  It will automatically try to
+        be a snmpbulkwalk if possible.
+
+        :param oid: The SNMP object identifier
+        """
+        # SNMP is a complicated mess of things.  Will endeavor to shield caller
+        # from as much as possible, assuming reasonable defaults when possible.
+        # there may come a time where we add more parameters to override the
+        # automatic behavior (e.g. DES is weak, so it's likely to be
+        # overriden, but some devices only support DES)
+        tp = _get_transport(self.server)
+        ctx = snmp.ContextData(self.context)
+        if '::' in oid:
+            mib, field = oid.split('::')
+            obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field))
+        else:
+            obj = snmp.ObjectType(snmp.ObjectIdentity(oid))
+
+        walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
+                               lexicographicMode=False)
+        for rsp in walking:
+            errstr, errnum, erridx, answers = rsp
+            if errstr:
+                raise exc.TargetEndpointUnreachable(str(errstr))
+            elif errnum:
+                raise exc.ConfluentException(errnum.prettyPrint())
+            for ans in answers:
+                yield ans
+
+
+if __name__ == '__main__':
+    import sys
+    ts = Session(sys.argv[1], 'public')
+    for kp in ts.walk(sys.argv[2]):
+        print(str(kp[0]))
+        print(str(kp[1]))

confluent_server/confluent/sockapi.py

@@ -118,37 +118,9 @@ def sessionhdl(connection, authname, skipauth=False):
         try:
             process_request(
                 connection, request, cfm, authdata, authname, skipauth)
-        except exc.ForbiddenRequest:
-            send_data(connection, {'errorcode': 403,
-                                      'error': 'Forbidden'})
-            send_data(connection, {'_requestdone': 1})
-        except exc.TargetEndpointBadCredentials:
-            send_data(connection, {'errorcode': 502,
-                                      'error': 'Bad Credentials'})
-            send_data(connection, {'_requestdone': 1})
-        except exc.TargetEndpointUnreachable as tu:
-            send_data(connection, {'errorcode': 504,
-                                      'error': 'Unreachable Target - ' + str(
-                                          tu)})
-            send_data(connection, {'_requestdone': 1})
-        except exc.NotImplementedException:
-            send_data(connection, {'errorcode': 501,
-                                      'error': 'Not Implemented'})
-            send_data(connection, {'_requestdone': 1})
-        except exc.NotFoundException as nfe:
-            send_data(connection, {'errorcode': 404,
-                                      'error': str(nfe)})
-            send_data(connection, {'_requestdone': 1})
-        except exc.InvalidArgumentException as iae:
-            send_data(connection, {'errorcode': 400,
-                                      'error': 'Bad Request - ' + str(iae)})
-            send_data(connection, {'_requestdone': 1})
-        except exc.LockedCredentials as lockedcred:
-            send_data(connection, {'errorcode': 500,
-                                      'error': 'Locked Credential Store'})
-            send_data(connection, {'_requestdone': 1})
         except exc.ConfluentException as e:
-            if e.apierrorcode == 500:
+            if ((not isinstance(e, exc.LockedCredentials)) and
+                    e.apierrorcode == 500):
                 tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                          event=log.Events.stacktrace)
             send_data(connection, {'errorcode': e.apierrorcode,
@@ -176,7 +148,7 @@ def send_response(responses, connection):
 
 def process_request(connection, request, cfm, authdata, authname, skipauth):
     if not isinstance(request, dict):
-        raise ValueError
+        raise exc.InvalidArgumentException
     operation = request['operation']
     path = request['path']
     params = request.get('parameters', {})
@@ -198,7 +170,8 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
         auditlog.log(auditmsg)
     try:
         if operation == 'start':
-            return start_term(authname, cfm, connection, params, path)
+            return start_term(authname, cfm, connection, params, path,
+                              authdata, skipauth)
         elif operation == 'shutdown':
             configmanager.ConfigManager.shutdown()
         else:
@@ -215,7 +188,7 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
     return
 
 
-def start_term(authname, cfm, connection, params, path):
+def start_term(authname, cfm, connection, params, path, authdata, skipauth):
     elems = path.split('/')
     if len(elems) < 4 or elems[1] != 'nodes':
         raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
@@ -261,7 +234,9 @@ def start_term(authname, cfm, connection, params, path):
                 consession.reopen()
                 continue
             else:
-                raise Exception("TODO")
+                process_request(connection, data, cfm, authdata, authname,
+                                skipauth)
+                continue
         if not data:
             consession.destroy()
             return
@@ -272,7 +247,16 @@ def _tlshandler(bind_host, bind_port):
     plainsocket = socket.socket(socket.AF_INET6)
     plainsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     plainsocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-    plainsocket.bind((bind_host, bind_port, 0, 0))
+    bound = False
+    while not bound:
+        try:
+            plainsocket.bind((bind_host, bind_port, 0, 0))
+            bound = True
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write('TLS Socket in use, retrying in 1 second\n')
+            eventlet.sleep(1)
     plainsocket.listen(5)
     while (1):  # TODO: exithook
         cnn, addr = plainsocket.accept()

confluent_server/confluent_server.spec.tmpl

@@ -12,7 +12,7 @@ Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
 BuildArch: noarch
-Requires: python-pyghmi, python-eventlet, python-greenlet, python-crypto >= 2.6.1, confluent_client, pyparsing, python-paramiko, python-dnspython
+Requires: python-pyghmi, python-eventlet, python-greenlet, python-crypto >= 2.6.1, confluent_client, pyparsing, python-paramiko, python-dns
 Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
 Url: http://xcat.sf.net/
 
@@ -33,6 +33,9 @@ done
 grep -v confluent/__init__.py INSTALLED_FILES.bare > INSTALLED_FILES
 cat INSTALLED_FILES
 
+%post
+if [ -x /usr/bin/systemctl ]; then /usr/bin/systemctl try-restart confluent; fi
+
 %clean
 rm -rf $RPM_BUILD_ROOT
 

confluent_server/makesetup

@@ -1,8 +1,8 @@
 cd `dirname $0`
-VERSION=`git tag -l|tail -n 1`
-NUMCOMMITS=`git rev-list $VERSION..HEAD|wc -l`
-if [ "$NUMCOMMITS" -ne 0 ]; then
-    VERSION=$VERSION.dev$NUMCOMMITS.g`git rev-parse --short HEAD`
+VERSION=`git describe|cut -d- -f 1`
+NUMCOMMITS=`git describe|cut -d- -f 2`
+if [ "$NUMCOMMITS" != "$VERSION"  ]; then
+    VERSION=$VERSION.dev$NUMCOMMITS.g`git describe|cut -d- -f 3`
 fi
 echo $VERSION > VERSION
 sed -e "s/#VERSION#/$VERSION/" setup.py.tmpl > setup.py