Fix SMM handler when None bmc

This fixes a common scenario for using fe80 collection

confluent_client/bin/collate

@@ -35,6 +35,9 @@ import confluent.client
 argparser = optparse.OptionParser(usage="Usage: <other command> | %prog [options]")
 argparser.add_option('-a', '--abbreviate', action='store_true',
                      help='Attempt to use confluent server to shorten noderanges')
+argparser.add_option('-b', '--base',
+                     help='Use given node as reference for comparison when '
+                          'using -d, instead of using the most common result')
 argparser.add_option('-d', '--diff', action='store_true',
                      help='Show what differs between most common '
                           'output group and others')
@@ -65,7 +68,7 @@ else:
 def print_current():
     if options.diff:
         grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
-                          reverse=options.reverse)
+                          reverse=options.reverse, basenode=options.base)
     else:
         grouped.print_all(skipmodal=options.skipcommon,
                                count=options.count,

confluent_client/bin/confetty

@@ -504,6 +504,13 @@ def makecall(callout, args):
             if 'errorcode' in response:
                 exitcode = response['errorcode']
             sys.stderr.write('Error: ' + response['error'] + '\n')
+        if 'databynode' in response:
+            lresponse = response['databynode']
+            for node in lresponse:
+                if 'errorcode' in lresponse[node]:
+                    exitcode = lresponse[node]['errorcode']
+                if 'error' in lresponse[node]:
+                    sys.stderr.write('{0}: Error - {1}\n'.format(node, lresponse[node]['error']))
 
 
 def clearvalues(resource, attribs):
@@ -997,4 +1004,6 @@ if __name__ == '__main__':
         if deadline and os.times()[4] < deadline:
             sys.stderr.write('[Exited early, hit enter to continue]')
             sys.stdin.readline()
+        if errcode == 0:
+            errcode = exitcode
         sys.exit(errcode)

confluent_client/bin/confluent2xcat

@@ -26,7 +26,7 @@ def lookupdata(data, key):
 
 def main():
     argparser = optparse.OptionParser(
-    usage='''\n       %prog noderange -o ansible.hosts
+    usage='''\n       %prog noderange -o xcatnodes.def
              \n ''')
     argparser.add_option('-o', '--output',
                          help='xCAT stanza file')

confluent_client/bin/nodeattrib

@@ -126,6 +126,6 @@ else:
             sys.stderr.write(res['error'] + '\n')
             exitcode = 1
         else:
-            print res['item']['href'].replace('/', '')
+            print(res['item']['href'].replace('/', ''))
 
 sys.exit(exitcode)

confluent_client/bin/nodeconfig

@@ -43,7 +43,7 @@ def bailout(msg, code=1):
     sys.exit(code)
 
 
-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '
@@ -185,7 +185,13 @@ if options.batch:
     argfile = open(options.batch, 'r')
     argset = argfile.readline()
     while argset:
-        parse_config_line(shlex.split(argset))
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            parse_config_line(shlex.split(argset))
         argset = argfile.readline()
 else:
     parse_config_line(args[1:])

confluent_client/bin/nodeconsole

@@ -40,6 +40,7 @@ if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
 if options.tile:
+    null = open('/dev/null', 'w')
     nodes = []
     sess = client.Command()
     for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
@@ -60,13 +61,15 @@ if options.tile:
                     confettypath, node)])
         else:
             subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
+            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
             pane += 1
             subprocess.call(
                 ['tmux', 'split', '-h',
                  '{0} -m 5 start /nodes/{1}/console/session'.format(
                      confettypath, node)])
-            subprocess.call(['tmux', 'select-layout', 'tiled'])
+            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
     subprocess.call(['tmux', 'select-pane', '-t', '0'])
+    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
     os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
         os.getpid()))
 else:

confluent_client/bin/nodediscover

@@ -69,10 +69,7 @@ def print_disco(options, session, currmac, outhandler, columns):
             record.append(','.join(rawval))
         else:
             record.append(str(rawval))
-    if options.csv:
-        csv.writer(sys.stdout).writerow(record)
-    else:
-        outhandler.add_row(record)
+    outhandler.add_row(record)
 
 
 def process_header(header):
@@ -95,6 +92,8 @@ def process_header(header):
             fields.append('hardwaremanagement.manager')
         elif datum in ('bmc gateway', 'xcc gateway', 'imm gateway'):
             fields.append('net.bmc.ipv4_gateway')
+        elif datum in ('bmc_gateway', 'xcc_gateway', 'imm_gateway'):
+            fields.append('net.bmc.ipv4_gateway')
         elif datum in ('bmcuser', 'username', 'user'):
             fields.append('secret.hardwaremanagementuser')
         elif datum in ('bmcpass', 'password', 'pass'):
@@ -155,6 +154,7 @@ def import_csv(options, session):
         for field in fields:
             if field in unique_fields:
                 unique_data[field] = set([])
+        broken = False
         for record in records:
             currfields = list(fields)
             nodedatum = {}
@@ -171,14 +171,16 @@ def import_csv(options, session):
                 nodedatum[currfield] = datum
             if not datum_complete(nodedatum):
                 sys.exit(1)
-            if not search_record(nodedatum, options, session):
+            if not search_record(nodedatum, options, session) and not broken:
                 blocking_scan(session)
             if not search_record(nodedatum, options, session):
                 sys.stderr.write(
                     "Could not match the following data: " +
                     repr(nodedatum) + '\n')
-                sys.exit(1)
+                broken = True
             nodedata.append(nodedatum)
+    if broken:
+        sys.exit(1)
     for datum in nodedata:
         maclist = search_record(datum, options, session)
         datum = datum_to_attrib(datum)
@@ -204,7 +206,6 @@ def import_csv(options, session):
 
 
 def list_discovery(options, session):
-    outhandler = None
     orderby = None
     if options.fields:
         columns = []
@@ -219,13 +220,12 @@ def list_discovery(options, session):
         for field in columns:
             if options.order.lower() == field.lower():
                 orderby = field
-    if options.csv:
-        csv.writer(sys.stdout).writerow(columns)
-    else:
-        outhandler = client.Tabulator(columns)
+    outhandler = client.Tabulator(columns)
     for mac in list_matching_macs(options, session):
         print_disco(options, session, mac, outhandler, columns)
-    if outhandler:
+    if options.csv:
+        outhandler.write_csv(sys.stdout, orderby)
+    else:
         for row in outhandler.get_table(orderby):
             print(row)
 
@@ -237,8 +237,10 @@ def clear_discovery(options, session):
             else:
                 print(repr(res))
 
-def list_matching_macs(options, session):
+def list_matching_macs(options, session, node=None):
     path = '/discovery/'
+    if node:
+        path += 'by-node/{0}/'.format(node)
     if options.model:
         path += 'by-model/{0}/'.format(options.model)
     if options.serial:
@@ -261,25 +263,25 @@ def list_matching_macs(options, session):
         path += 'by-mac/'
         return [x['item']['href'] for x in session.read(path)]
 
-def assign_discovery(options, session):
+def assign_discovery(options, session, needid=True):
     abort = False
     if options.importfile:
         return import_csv(options, session)
-    if not (options.serial or options.uuid or options.mac):
+    if not options.node:
+        sys.stderr.write("Node (-n) must be specified for assignment\n")
+        abort = True
+    if needid and not (options.serial or options.uuid or options.mac):
         sys.stderr.write(
             "UUID (-u), serial (-s), or ether address (-e) required for "
             "assignment\n")
         abort = True
-    if not options.node:
-        sys.stderr.write("Node (-n) must be specified for assignment\n")
-        abort = True
     if abort:
         sys.exit(1)
-    matches = list_matching_macs(options, session)
+    matches = list_matching_macs(options, session, None if needid else options.node)
     if not matches:
         # Do a rescan to catch missing requested data
         blocking_scan(session)
-        matches = list_matching_macs(options, session)
+        matches = list_matching_macs(options, session, None if needid else options.node)
     if not matches:
         sys.stderr.write("No matching discovery candidates found\n")
         sys.exit(1)
@@ -294,6 +296,7 @@ def blocking_scan(session):
     list(session.update('/discovery/rescan', {'rescan': 'start'}))
     while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
         time.sleep(0.5)
+    list(session.update('/networking/macs/rescan', {'rescan': 'start'}))
 
 
 def main():
@@ -336,7 +339,7 @@ def main():
     parser.add_option('-o', '--order', dest='order',
                       help='Order output by given field', metavar='ORDER')
     (options, args) = parser.parse_args()
-    if len(args) == 0 or args[0] not in ('list', 'assign', 'rescan', 'clear'):
+    if len(args) == 0 or args[0] not in ('list', 'assign', 'reassign', 'rescan', 'clear'):
         parser.print_help()
         sys.exit(1)
     session = client.Command()
@@ -346,10 +349,12 @@ def main():
         clear_discovery(options, session)
     if args[0] == 'assign':
         assign_discovery(options, session)
+    if args[0] == 'reassign':
+        assign_discovery(options, session, False)
     if args[0] == 'rescan':
         blocking_scan(session)
         print("Rescan complete")
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,7 +33,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(
     usage="Usage: %prog [options] noderange [clear]")
@@ -64,6 +65,8 @@ def format_event(evt):
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
+    if evt.get('log_id', None):
+        retparts.append(evt['log_id'] + ':')
     if 'component_type' in evt and evt['component_type'] is not None:
         dscparts.append(evt['component_type'])
     if 'component' in evt and evt['component'] is not None:
@@ -77,7 +80,10 @@ def format_event(evt):
             pass
         dscparts.append(evttext)
     retparts.append(' - '.join(dscparts))
-    return ' '.join(retparts)
+    msg = evt.get('message')
+    if not msg:
+        msg = ''
+    return ' '.join(retparts) + msg
 
 
 if deletemode:
@@ -98,4 +104,4 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
             if 'events' in thisdata:
                evtdata = thisdata['events']
                for evt in evtdata:
-                   print '{0}: {1}'.format(node, format_event(evt))
+                   print('{0}: {1}'.format(node, format_event(evt)))

confluent_client/bin/nodefirmware

@@ -151,7 +151,7 @@ def show_firmware(session):
                     for prefix in inv:
                         firmware_shown = True
                         printfirm(node, prefix, inv[prefix])
-    if not firmware_shown:
+    if not firmware_shown and not exitcode:
         argparser.print_help()
 
 

confluent_client/bin/nodegroupattrib

@@ -36,9 +36,9 @@ import confluent.client as client
 
 argparser = optparse.OptionParser(
     usage='''\n       %prog [options]   \
-             \n       %prog [options] nodegroup [list of attributes]  \
+             \n       %prog [options] nodegroup [list of attributes|all]  \
              \n       %prog [options] nodegroup nodes=value1,value2   \
-             \n       %prog -e nodegroup <attribute names to set>  \             
+             \n       %prog -e nodegroup <attribute names to set>  \
              \n       %prog [options] nodegroup nodes=value1,value2
              \n ''')
 argparser.add_option('-b', '--blame', action='store_true',

confluent_client/bin/nodegrouplist

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    showtype='all'
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) > 0:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print(res['item']['href'].replace('/', ''))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodegrouprename

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog <group> <new group name>\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) != 2:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.update(
+            '/nodegroups/{0}/attributes/rename'.format(args[0]),
+            {'rename': args[1]}):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print('{0}: {1}'.format(res['oldname'], res['newname']))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodehealth

@@ -32,7 +32,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
 (options, args) = argparser.parse_args()

confluent_client/bin/nodeinventory

@@ -34,7 +34,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+if sys.version_info[0] < 3:
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
 filters = []
 
@@ -50,15 +51,17 @@ def print_mem_info(node, prefix, meminfo):
     memdescfmt = '{0}GB PC'
     if meminfo['memory_type'] == 'DDR3 SDRAM':
         memdescfmt += '3-{1} '
-    elif meminfo['memory_type'] == 'DDR4 SDRAM':
+    elif 'DDR4' in meminfo['memory_type']:
         memdescfmt += '4-{1} '
     else:
         print('{0}: {1}: Unrecognized Memory'.format(node, prefix))
         return
-    if meminfo['ecc']:
+    if meminfo.get('ecc', False):
         memdescfmt += 'ECC '
     capacity = meminfo['capacity_mb'] / 1024
-    memdescfmt += meminfo['module_type']
+    modtype = meminfo.get('module_type', None)
+    if modtype:
+        memdescfmt += modtype
     memdesc = memdescfmt.format(capacity, meminfo['speed'])
     print('{0}: {1} description: {2}'.format(node, prefix, memdesc))
     print('{0}: {1} manufacturer: {2}'.format(
@@ -66,10 +69,11 @@ def print_mem_info(node, prefix, meminfo):
     print('{0}: {1} model: {2}'.format(node, prefix, meminfo['model']))
     print('{0}: {1} serial number: {2}'.format(node, prefix,
                                                meminfo['serial']))
-    print('{0}: {1} manufacture date: {2}'.format(node, prefix,
-                                                  meminfo['manufacture_date']))
-    print('{0}: {1} manufacture location: {2}'.format(
-        node, prefix, meminfo['manufacture_location']))
+    if 'manufacture_date' in meminfo:
+        print('{0}: {1} manufacture date: {2}'.format(node, prefix,
+                                                      meminfo['manufacture_date']))
+        print('{0}: {1} manufacture location: {2}'.format(
+            node, prefix, meminfo['manufacture_location']))
 
 exitcode = 0
 
@@ -142,7 +146,7 @@ try:
                                 databynode[node] = {}
                             databynode[node][prefix] = inv
                         else:
-                            print '{0}: {1}: Not Present'.format(node, prefix)
+                            print('{0}: {1}: Not Present'.format(node, prefix))
                     continue
                 info = inv['information']
                 info.pop('board_extra', None)

confluent_client/bin/nodelicense

@@ -36,49 +36,111 @@ exitcode = 0
 
 argparser = optparse.OptionParser(
     usage="Usage: "
-          "%prog <noderange> [list][install <file>]")
+          "%prog <noderange> [list][install <file>|save <directory>|delete <name>]")
 (options, args) = argparser.parse_args()
 upfile = None
+downdir = None
+delete = False
 try:
     noderange = args[0]
     if len(args) > 1:
         if args[1] == 'install':
             upfile = args[2]
-        else:
-            components = ['all']
+        elif args[1] == 'save':
+            downdir = args[2]
+        elif args[1] == 'delete':
+            delete = args[2]
+        elif args[1] != 'list':
+            argparser.print_help()
+            sys.exit(1)
 except IndexError:
     argparser.print_help()
     sys.exit(1)
 client.check_globbing(noderange)
 
+
 def install_license(session, filename):
     global exitcode
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate requested file {0}\n'.format(
+            filename))
+        sys.exit(404)
     resource = '/noderange/{0}/configuration/' \
                'management_controller/licenses/'.format(noderange)
     filename = os.path.abspath(filename)
     instargs = {'filename': filename}
     for res in session.create(resource, instargs):
-        pass # print(repr(res))
-    show_licenses()
+        for node in res.get('databynode', []):
+            if 'error' in res['databynode'][node]:
+                sys.stderr.write('{0}: {1}\n'.format(
+                    node, res['databynode'][node]['error']))
+                sys.exit(res['databynode'][node].get('errorcode', 1))        
+    show_licenses(session)
+
+
+def save_licenses(session, dirname):
+    global exitcode
+    resource = '/noderange/{0}/configuration/' \
+               'management_controller/save_licenses'.format(noderange)
+    filename = os.path.abspath(dirname)
+    if not os.path.exists(filename):
+        sys.stderr.write('Unable to locate specified directory {0}\n'.format(
+            filename))
+        sys.exit(404)
+    instargs = {'dirname': filename}
+    for res in session.create(resource, instargs):
+        for node in res.get('databynode', {}):
+            fname = res['databynode'][node].get('filename', None)
+            if fname:
+                print('{0}: Saved license to {1}'.format(node, fname))
+            else:
+                sys.stderr.write('{0}: {1}', node, repr(res['databynode'][node]))
+
 
 def show_licenses(session):
     global exitcode
-    firmware_shown = False
     for res in session.read(
             '/noderange/{0}/configuration/management_controller/licenses/'
             'all'.format(noderange)):
         for node in res.get('databynode', {}):
             for license in res['databynode'][node].get('License', []):
-                print('{0}: {1}'.format(node, license.get('feature',
-                                                          'Unknown')))
+                msg = '{0}: {1}'.format(node, license.get('feature',
+                                                          'Unknown'))
+                if license.get('state', 'Active') != 'Active':
+                    msg += ' ({0})'.format(license['state'])
+                print(msg)
 
 
+def delete_license(session, licname):
+    global exitcode
+    licstodel = []
+    for res in list(session.read(
+            '/noderange/{0}/configuration/management_controller/licenses/'
+            'all'.format(noderange))):
+        for node in res.get('databynode', {}):
+            for license in res['databynode'][node].get('License', []):
+                if license.get('feature', None) == licname:
+                    prefix = '/nodes/{0}/configuration/management_controller/licenses/'.format(node)
+                    for currlic in list(session.read(prefix)):
+                        currlic = currlic.get('item', {}).get('href', 'all')
+                        if currlic == 'all':
+                            continue
+                        currname = list(session.read(prefix + currlic))[0]
+                        currname = currname.get('License', [{}])[0].get('feature', None)
+                        if currname == licname:
+                            list(session.delete(prefix + currlic))
+    show_licenses(session)
+
 try:
     session = client.Command()
-    if upfile is None:
-        show_licenses(session)
-    else:
+    if upfile:
         install_license(session, upfile)
+    elif downdir:
+        save_licenses(session, downdir)
+    elif delete:
+        delete_license(session, delete)
+    else:
+        show_licenses(session)
 except KeyboardInterrupt:
     print('')
-sys.exit(exitcode)
+sys.exit(exitcode)

confluent_client/bin/nodelist

@@ -61,7 +61,7 @@ def main():
                 sys.stderr.write(res['error'] + '\n')
                 exitcode = 1
             else:
-                print res['item']['href'].replace('/', '')
+                print(res['item']['href'].replace('/', ''))
 
     sys.exit(exitcode)
 

confluent_client/bin/nodemedia

@@ -83,7 +83,8 @@ def list_media(noderange, media):
         for node in res.get('databynode', []):
             url = res['databynode'][node].get('url', None)
             name =  res['databynode'][node].get('name', None)
-            if url and not res['databynode'][node].get('secure', False):
+            if (url and not url.startswith('file:') and
+                    not res['databynode'][node].get('secure', False)):
                 name += ' (insecure)'
             if not name:
                 continue

confluent_client/bin/noderename

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> <newname>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+identifystate = None
+if len(sys.argv) > 2:
+    newname = sys.argv[2]
+else:
+    argparser.print_help()
+    sys.exit(1)
+session = client.Command()
+exitcode = 0
+sys.exit(
+    session.simple_noderange_command(noderange, 'attributes/rename', newname))
+

confluent_client/bin/nodersync

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+import confluent.sortutil as sortutil
+
+
+def run():
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog location noderange:location",
+    )
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of nodes to concurrently rsync')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2 or ':' not in args[-1]:
+        argparser.print_help()
+        sys.exit(1)
+    concurrentprocs = options.count
+    noderange, targpath = args[-1].split(':', 1)
+    client.check_globbing(noderange)
+    c = client.Command()
+    cmdstr = " ".join(args[:-1])
+    cmdstr = 'rsync -av --info=progress2 ' + cmdstr
+    cmdstr += ' {node}:' + targpath
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
+                        {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value'].encode('utf-8')
+            cmdv = shlex.split(cmd)
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    nodeerrs = {}
+    pernodeout = {}
+    pernodefile = {}
+    output = sq.ScreenPrinter(noderange, c)
+
+    while all:
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.read(1)
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = ''
+                        pernodeout[node] += data
+                        if '\n' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\n', 1)
+                            if currout:
+                                pernodefile[node] = os.path.basename(currout)
+                        if '\r' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\r', 1)
+                            if currout:
+                                currout = currout.split()
+                                try:
+                                    currout = currout[1]
+                                    output.set_output(node, '{0}:{1}'.format(pernodefile[node], currout))
+                                except IndexError:
+                                    pernodefile = currout[0]
+                                    pass
+                    else:
+                        output.set_output(node, 'error!')
+                        if node not in nodeerrs:
+                            nodeerrs[node] = ''
+                        nodeerrs[node] += data
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if node not in nodeerrs:
+                            output.set_output(node, 'complete')
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    for node in nodeerrs:
+        for line in nodeerrs[node].split('\n'):
+            sys.stderr.write('{0}: {1}\n'.format(node, line))
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/noderun

@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -93,7 +95,10 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -107,7 +112,10 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodesensors

@@ -114,7 +114,8 @@ def sensorpass(showout=True, appendtime=False):
                         continue
                     for redundant_state in ('Non-Critical', 'Critical'):
                         try:
-                            sensedata['states'].remove(redundant_state)
+                            if sensedata.get('states', False):
+                                sensedata['states'].remove(redundant_state)
                         except ValueError:
                             pass
                     resultdata[node][sensedata['name']] = sensedata
@@ -132,11 +133,12 @@ def sensorpass(showout=True, appendtime=False):
                             showval = u' {0} '.format(sensedata['value'])
                         if sensedata['units'] not in (None, u''):
                             showval += sensedata['units']
-                        if sensedata['health'] != 'ok':
+                        if sensedata.get('health', 'ok') != 'ok':
                             datadescription = [sensedata['health']]
                         else:
                             datadescription = []
-                        datadescription.extend(sensedata['states'])
+                        if sensedata.get('states', False):
+                            datadescription.extend(sensedata['states'])
                         if datadescription:
                             if showval == '':
                                 showval += u' {0}'.format(
@@ -148,7 +150,7 @@ def sensorpass(showout=True, appendtime=False):
                             showval += ' @' + time.strftime(
                                 '%Y-%m-%dT%H:%M:%S')
                         print(u'{0}: {1}:{2}'.format(
-                            node, sensedata['name'], showval).encode('utf-8'))
+                            node, sensedata['name'], showval).encode('utf8'))
                         sys.stdout.flush()
     return resultdata
 
@@ -156,7 +158,7 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            if showtime.is_integer():
+            if isinstance(showtime, int):
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
             else:
                 rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
@@ -183,6 +185,7 @@ def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
             except KeyError:
                 rowdata.append('N/A')
         csvwriter.writerow(rowdata)
+        sys.stdout.flush()
 
 
 def main():
@@ -220,7 +223,7 @@ def main():
                     sys.exit(exitcode)
             sleeptime = nextstart - os.times()[4]
             if sleeptime > 0:
-                time.sleep(nextstart - os.times()[4])
+                time.sleep(sleeptime)
     else:
         if options.csv:
             format_csv(csvwriter, orderedsensors, resdata, showtime=False)

confluent_client/bin/nodeshell

@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil
 def run():
 
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -94,7 +96,10 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -108,7 +113,10 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodestorage

@@ -115,6 +115,8 @@ def createstorage(noderange, options, args):
              'name': names}
     if options.size:
         parms['size'] = options.size
+    if options.stripsizes:
+        parms['stripsizes'] = options.stripsizes
     _print_cfg(session.create(
             '/noderange/{0}/configuration/storage/volumes/{1}'.format(
                 noderange, names), parms))
@@ -147,13 +149,26 @@ def deletestorage(noderange, options, args):
             print(repr(rsp))
 
 
-def setstorage(noderange, options, args):
-    pass
+def setdisk(noderange, options, args):
+    if options.disks is None:
+        if len(args):
+            names = args.pop(0)
+        else:
+            sys.stderr.write('-d is required to indicate disk to modify\n')
+            sys.exit(1)
+    else:
+        names = options.disks
+    if not len(args) or args[0] not in ('hotspare', 'jbod', 'unconfigured'):
+        sys.stderr.write('diskset requires valid state as argument (hotspare, jbod, unconfigured)\n')
+        sys.exit(1)
+    session = client.Command()
+    scfg = session.update('/noderange/{0}/configuration/storage/disks/{1}'.format(noderange, names), {'state': args[0]})
+    _print_cfg(scfg)
 
 funmap = {
     'create': createstorage,
     'show': showstorage,
-    'set': setstorage,
+    'diskset': setdisk,
     'delete': deletestorage,
     'rm': deletestorage,
 }
@@ -162,7 +177,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|set|delete]',
+        usage='Usage: %prog <noderange> [show|create|delete|diskset]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',
@@ -183,7 +198,13 @@ def main():
                               'naming volumes, or selecting a volume for '
                               'delete.  Default behavior is to use '
                               'implementation provided default names.')
+    argparser.add_option('-z', '--stripsizes', type='str',
+                         help='Comma separated list of stripsizes to use when creating volumes. '
+                            'This value is in kilobytes.  The default behavior is to allow the '
+                            'storage controller to decide.')
     (options, args) = argparser.parse_args()
+    if len(args) == 1:
+        args.append('show')
     try:
         noderange = args[0]
         operation = args[1]
@@ -200,4 +221,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodesupport

@@ -114,10 +114,12 @@ class OptParser(optparse.OptionParser):
 
 def main():
     argparser = OptParser(
-        usage="Usage: %prog <noderange> [servicedata] "
+        usage="Usage: %prog <noderange> servicedata "
               "<filename>",
         epilog='\nservicedata will save service data to the given '
-               'directory\n'
+               'directory.  It is saved to the location on the relevant '
+               'management server (the confluent server if running remote, '
+               'and the collective.manager if in collective)\n'
                '\n\nSee `man %prog` for more info.\n')
     (options, args) = argparser.parse_args()
     media = None

confluent_client/bin/stats

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import fcntl
+import io
+import numpy as np
+
+import os
+import sixel
+import subprocess
+import sys
+
+class DumbWriter(sixel.SixelWriter):
+    def restore_position(self, output):
+        return
+
+
+def plot(gui, output, plotdata, bins):
+    import matplotlib as mpl
+    if not gui:
+        mpl.use('Agg')
+    import matplotlib.pyplot as plt
+    n, bins, patches = plt.hist(plotdata, bins)
+    plt.show()
+    if not gui:
+        if output:
+            tdata = output
+        else:
+            tdata = io.BytesIO()
+        plt.savefig(tdata)
+    if not gui and not output:
+        writer = DumbWriter()
+        writer.draw(tdata)
+    return n, bins
+
+def textplot(plotdata, bins):
+    n, bins = np.histogram(plotdata, bins)
+    labels = []
+    for bin in bins:
+        labels.append('{0:0.1f}'.format(bin))
+    width = 80
+    # Since this will be primarily piped into, hard to get 
+    # terminal width
+    labelwidth = 0
+    for lab in labels:
+        if len(lab) > labelwidth:
+            labelwidth = len(lab)
+    width -= (labelwidth) + 1
+    labelfmt = '{{0:>{0}s}}|'.format(labelwidth)
+    maxn = 0.0
+    for lgth in n:
+        if lgth > maxn:
+            maxn = float(lgth)
+    for i in range(len(n)):
+        print(labelfmt.format(labels[i]) + '=' * int(np.round((n[i]/maxn) * width)))
+    return n, bins
+
+histogram = False
+aparser = argparse.ArgumentParser(description='Quick access to common statistics')
+aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
+aparser.add_argument('-d', default=None, help='Value used to separate columns')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
+aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
+aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
+aparser.add_argument('-t', default=False, action='store_true', help='Output a histogram in text format')
+aparser.add_argument('-v', default=False, action='store_true', help='Attempt to list nodes relevant to each histogram bar (requires -s, -o, or -t)')
+aparser.add_argument('-b', type=int, default=10, help='Number of bins to use in histogram (default is 10)')
+args = aparser.parse_args(sys.argv[1:])
+plotdata = []
+headlines = int(args.s)
+while headlines >= 0:
+    data = sys.stdin.readline()
+    headlines -= 1
+if args.d:
+    delimiter = args.d
+else:
+    if '\t' in data:
+        delimiter = '\t'
+    elif ' ' in data:
+        delimiter = ' '
+    elif ',' in data:
+        delimiter = ','
+    else:
+        delimiter = ' '  # handle single column
+data = list(csv.reader([data], delimiter=delimiter))[0]
+nodebydatum = {}
+idx = args.c - 1
+autoidx = False
+while data:
+    node = None
+    if ':' in data[0]:
+        node, data[0] = data[0].split(':', 1)
+    else:
+        node = data[0]
+    if idx == -1 and not autoidx:
+        while not autoidx:
+            try:
+                datum = float(data[idx])
+            except ValueError:
+                idx -= 1
+                continue
+            except IndexError:
+                sys.stderr.write('Unable to identify a numerical column\n')
+                sys.exit(1)
+            autoidx = True
+    else:
+        datum = float(data[idx])
+    if node:
+        if datum in nodebydatum:
+            nodebydatum[datum].add(node)
+        else:
+            nodebydatum[datum] = set([node])
+    plotdata.append(datum)
+    data = sys.stdin.readline()
+    data = list(csv.reader([data], delimiter=delimiter))[0]
+n = None
+if args.g or args.o or args.x:
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+if args.t:
+    n, bins = textplot(plotdata, bins=args.b)
+print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))
+if args.v and n is not None and nodebydatum:
+    print('')
+    currbin = bins[0]
+    bins = bins[1:]
+    currbinmembers = []
+    for datum in sorted(nodebydatum):
+        if datum > bins[0]:
+           nextbin = None
+           endbin = bins[0]
+           while len(bins) and bins[0] < datum:
+                nextbin = bins[0]
+                bins = bins[1:]
+           if not nextbin:
+               nextbin = np.max(plotdata)
+           print('Entries between {0} and {1}'.format(currbin, endbin))
+           currbin = nextbin
+           print('-' * 80)
+           print(','.join(sorted(currbinmembers)))
+           print('')
+           print('')
+           currbinmembers = []
+        for node in nodebydatum[datum]:
+            currbinmembers.append(node)
+    if currbinmembers:
+       print('Entries between {0} and {1}'.format(currbin, np.max(plotdata)))
+       print('-' * 80)
+       print(','.join(sorted(currbinmembers)))
+       print('')
+       print('')

confluent_client/builddeb

@@ -0,0 +1 @@
+../confluent_server/builddeb

confluent_client/confluent/client.py

@@ -15,7 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import anydbm as dbm
+try:
+    import anydbm as dbm
+except ImportError:
+    import dbm
+import csv
 import errno
 import fnmatch
 import hashlib
@@ -70,6 +74,23 @@ class Tabulator(object):
             for row in self.rows:
                 yield fmtstr.format(*row)
 
+    def write_csv(self, output, order=None):
+        output = csv.writer(output)
+        output.writerow(self.headers)
+        i = 0
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            i = i + 1
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                output.writerow(row)
+        else:
+            for row in self.rows:
+                output.writerow(row)
+
 
 def printerror(res, node=None):
     exitcode = 0
@@ -118,6 +139,8 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+        elif self.serverloc == '/var/run/confluent/api.sock':
+            raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
         tlvdata.recv(self.connection)
@@ -302,14 +325,19 @@ class Command(object):
         if knownhosts:
             certdata = self.connection.getpeercert(binary_form=True)
             fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest()
+            fingerprint = fingerprint.encode('utf-8')
             hostid = '@'.join((port, server))
             khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384)
             if hostid in khf:
                 if fingerprint == khf[hostid]:
                     return
                 else:
-                    replace = raw_input(
-                        "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    try:
+                        replace = raw_input(
+                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
+                    except NameError:
+                        replace = input(
+                            "MISMATCHED CERTIFICATE DATA, ACCEPT NEW? (y/n):")
                     if replace not in ('y', 'Y'):
                         raise Exception("BAD CERTIFICATE")
             cprint('Adding new key for %s:%s' % (server, port))
@@ -373,7 +401,7 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
         for node in sorted(res['databynode']):
             for attr, val in sorted(
                     res['databynode'][node].items(),
-                    key=lambda (k, v): v.get('sortid', k) if isinstance(v, dict) else k):
+                    key=lambda k: k[1].get('sortid', k[0]) if isinstance(k[1], dict) else k[0]):
                 if attr == 'error':
                     sys.stderr.write('{0}: Error: {1}\n'.format(node, val))
                     continue
@@ -444,9 +472,12 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                                 (currattr.get('default', None) is not None and
                                 currattr.get('value', None) is not None and
                                 currattr['value'] != currattr['default'])):
+                            cval = ','.join(currattr['value']) if isinstance(
+                                currattr['value'], list) else currattr['value']
+                            dval = ','.join(currattr['default']) if isinstance(
+                                currattr['default'], list) else currattr['default']
                             cprint('{0}: {1}: {2} (Default: {3})'.format(
-                                node, printattr, currattr['value'],
-                                currattr['default']))
+                                node, printattr, cval, dval))
                     else:
 
                         try:
@@ -455,11 +486,14 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                             details = False
                         if details:
                             if currattr.get('help', None):
-                                attrout += ' (Help: {0})'.format(
-                                    currattr['help'].encode('utf-8'))
+                                attrout += u' (Help: {0})'.format(
+                                    currattr['help'])
                             if currattr.get('possible', None):
-                                attrout += ' (Choices: {0})'.format(
-                                    ','.join(currattr['possible']))
+                                try:
+                                    attrout += u' (Choices: {0})'.format(
+                                        ','.join(currattr['possible']))
+                                except TypeError:
+                                    pass
                         cprint(attrout)
     if not exitcode:
         if requestargs:
@@ -629,4 +663,4 @@ def check_globbing(noderange):
                 'bash or change directories such that there is no filename '
                 'that would conflict.'
                 '\n'.format(noderange))
-            sys.exit(1)
+            sys.exit(1)

confluent_client/confluent/screensqueeze.py

@@ -44,6 +44,8 @@ class ScreenPrinter(object):
         self.fieldwidth = maxlen + textlen + 1  # 1 for column
 
     def set_output(self, node, text):
+        if self.nodeoutput[node] == text:
+            return
         self.nodeoutput[node] = text
         if len(text) >= self.textlen:
             self.textlen = len(text) + 1

confluent_client/confluent/textgroup.py

@@ -169,13 +169,18 @@ class GroupedData(object):
         output.flush()
 
     def print_deviants(self, output=sys.stdout, skipmodal=False, reverse=False,
-                       count=False):
+                       count=False, basenode=None):
         self.generate_byoutput()
         modaloutput = None
         ismodal = True
         revoutput = []
+        if basenode:
+            for checkout in self.byoutput:
+                if basenode in self.byoutput[checkout]:
+                    modaloutput = checkout
         for outdata in sorted(
-                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                self.byoutput, key=lambda x: [0 if modaloutput == x else 1,
+                                              0 - len(self.byoutput[x]),
                                               humanify_nodename(
                                                   self.get_group_text(
                                                       self.byoutput[x]

confluent_client/confluent/tlvdata.py

@@ -34,6 +34,8 @@ def decodestr(value):
             ret = value.decode('cp437')
         except UnicodeDecodeError:
             ret = value
+    except AttributeError:
+        return value
     return ret
 
 def unicode_dictvalues(dictdata):

confluent_client/confluent_env.sh

@@ -39,6 +39,8 @@ alias nodesetboot='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export
 alias nodestorage='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodestorage'
 alias nodeshell='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodeshell'
 alias nodelicense='CURRENT_CMDLINE=$(HISTTIMEFORMAT= builtin history 1); export CURRENT_CMDLINE; nodelicense'
+# Do not continue for non-bash shells, the rest of this sets up bash completion functions
+[ -z "$BASH_VERSION" -o -z "$PS1" ] && return
 
 
 _confluent_get_args()
@@ -158,7 +160,7 @@ _confluent_nodelicense_completion()
 {
     _confluent_get_args
     if [ $NUMARGS == 3 ]; then
-        COMPREPLY=($(compgen -W "install list" -- ${COMP_WORDS[-1]}))
+        COMPREPLY=($(compgen -W "install list save delete" -- ${COMP_WORDS[-1]}))
         return;
     fi
     if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'install' ]; then
@@ -166,6 +168,11 @@ _confluent_nodelicense_completion()
         COMPREPLY=()
         return
     fi
+    if [ $NUMARGS == 4 ] && [ ${CMPARGS[2]} == 'save' ]; then
+        compopt -o dirnames
+        COMPREPLY=()
+        return
+    fi
     if [ $NUMARGS -lt 3 ]; then
          _confluent_nr_completion
          return
@@ -190,6 +197,13 @@ _confluent_nodesupport_completion()
     fi
 }
 
+_confluent_nodeattrib_completion()
+{
+    COMP_CANDIDATES=$(nodeattrib '~.>1' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_generic_completion
+}
+
+
 _confluent_nn_completion()
 {
     _confluent_get_args
@@ -250,7 +264,8 @@ _confluent_ng_completion()
 
     COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
 }
-complete -F _confluent_nr_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodegroupattrib
 complete -F _confluent_nr_completion nodebmcreset
 complete -F _confluent_nodesetboot_completion nodeboot
 complete -F _confluent_nr_completion nodeconfig
@@ -262,7 +277,7 @@ complete -F _confluent_ng_completion nodegroupremove
 complete -F _confluent_nr_completion nodehealth
 complete -F _confluent_nodeidentify_completion nodeidentify
 complete -F _confluent_nr_completion nodeinventory
-complete -F _confluent_nr_completion nodelist
+complete -F _confluent_nodeattrib_completion nodelist
 complete -F _confluent_nodemedia_completion nodemedia
 complete -F _confluent_nodepower_completion nodepower
 complete -F _confluent_nr_completion noderemove

confluent_client/doc/man/collective.ronn

@@ -0,0 +1,55 @@
+collective(1) -- Check and manage a confluent collective
+==============================
+
+## SYNOPSIS
+
+`collective invite <server>`  
+`collective join <server> [-i TOKEN]`  
+`collective show`  
+`collective gencert`  
+
+
+## DESCRIPTION
+
+**collective** helps manage the collective mode of confluent, where multiple
+confluent servers are linked together to act as one.  For example, the procedure to set up
+a collective to run on three servers called mgt1, mgt2, and mgt3, first install and start
+confluent as usual on the three servers.  On mgt1, run `collective invite mgt2` and an
+invitation token will be output.  On mgt2, either run `collective join mgt1` to paste
+the token interactively, or `collective join mgt1 -i <token>`.  At this point, either
+mgt1 or mgt2 can bring in mgt3.  For example on mgt2 run `collective invite mgt3` and
+on mgt3 run `collective join mgt2 -i <token>`
+
+This can be linked together in the following manner with ssh:
+on mgt1:
+  `# ssh mgt2 collective join mgt1 -i $(collective invite mgt2)`
+
+Note that a collective is only redundant with 3 or more members.  The collective
+will function so long as more than half of the members are online.  A collective
+of two members is supported, but without redundancy.
+
+Also note that the collective leader role is dynamic, but has no impact on interacting
+with confluent.  It is merely an internal role that can dynamically change depending
+on circumstances.
+
+## OPTIONS
+
+ * `-i`:
+    Provide the token as an argument rather than interactively.
+   
+## EXAMPLES
+ * Inviting a server called mgt2:
+  `# collective invite mgt2`  
+  `bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+
+* On mgt2, joining mgt1:
+  `# collective join mgt1 -i bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+  `Success`  
+
+* Showing the collective state:
+  `# collective show`  
+  `Quorum: True`  
+  `Leader: mgt1`  
+  `Active collective members:`  
+  `  mgt2`  
+

confluent_client/doc/man/nodeattrib.ronn

@@ -23,6 +23,9 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 If the word all is specified, then all available attributes are given.
 Omitting any attribute name or the word 'all' will display only attributes
 that are currently set.
@@ -88,6 +91,12 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
     `n1: console.method: `  
     `n2: console.method: `  
 
+* List all switches that a node is described as connected to:
+    `# nodeattrib d1 net.*switch`  
+    `d1: net.mgt.switch: mgtswitch1`  
+    `d1: net.pxe.switch: pxeswitch1`  
+    `d1: net.switch:`  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeattribexpressions.ronn

@@ -9,6 +9,18 @@ expression to generate the value.
 An expression will contain some directives wrapped in `{}` characters. Within
 `{}` are a number of potential substitute values and operations.
 
+Note that syntax of expressions can have overlap with the shell syntax.
+For example:
+
+`$ echo (n2)`  
+`-bash: syntax error near unexpected token `n2'`  
+
+In such a case, it helps to quote the expression to allow it to be passed:
+
+`$ echo '(n2)'`  
+`(n2)`  
+
+
 The most common operation is to extract a number from the nodename. These 
 values are available as n1, n2, etc. So for example attributes for a node named 
 b1o2r3u4 would have {n1} as 1, {n2} as 2, {n3} as 3, and {n4} as 4. 

confluent_client/doc/man/nodeconfig.ronn

@@ -3,8 +3,8 @@ nodeconfig(8) -- Show or change node configuration
 
 ## SYNOPSIS
 
-`nodeconfig <noderange> [options] [<configuration>..]`
-`nodeconfig <noderange> [options] [<configuration=value>..]`
+`nodeconfig <noderange> [options] [<configuration>..]`  
+`nodeconfig <noderange> [options] [<configuration=value>..]`  
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodegrouplist.ronn

@@ -0,0 +1,14 @@
+nodegrouplist(8) -- List the defined confluent nodegroups
+===================================================================
+
+## SYNOPSIS
+
+`nodegrouplist`  
+
+## DESCRIPTION
+
+`nodegrouplist` lists the currently defined groups in confluent.
+
+## SEE ALSO
+
+nodeattrib(8), nodeattribexpressions(5), nodegroupattrib(8)

confluent_client/doc/man/nodelicense.ronn

@@ -3,9 +3,15 @@ nodelicense(8) -- Manage license keys on BMC
 
 ## SYNOPSIS
 
-`nodelicense <noderange> [list|install <filename>]`
+`nodelicense <noderange> [list|install <filename>|delete <license>|save <directory>]`
 
 ## DESCRIPTION
 
-`nodelicense` shows and installs license keys on supported BMCs
+`nodelicense` manages license keys on supported BMCs. Without an argument, the command
+lists currently installed license.  Using `delete` will remove the specified license name
+from th eBMC.  The `save` subcommand will take the passed directory (which may be in the form
+of /path/to/{node}/ to have the node name substituted for each node) and back up installed licenses
+to that directory. The `install` command will take the specified filename and install.  The filename
+argument may be of the form  xcc_fod_0034_7X21{id.serial}.key to have the serial number substituted
+to allow unique licenses to be specified in a single command.
 

confluent_client/doc/man/nodelist.ronn

@@ -17,6 +17,9 @@ displayed.  If `-b` is specified, it will also display information on
 how inherited and expression based attributes are defined.  There is more
 information on node attributes in nodeattributes(5) man page.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 ## OPTIONS
 
 * `-b`, `--blame`:

confluent_client/doc/man/noderange.ronn

@@ -36,6 +36,9 @@ Also, regular expressions may be used to indicate nodes with names matching cert
 The other major noderange primitive is indicating nodes by some attribute value:
 `location.rack=7`
 
+The attribute name may use a wildcard:
+`net.*switch=switch1`
+
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
 

confluent_client/doc/man/nodestorage.ronn

@@ -0,0 +1,65 @@
+nodestorage(8) -- Examine/Modify storage configuration of a node
+============================================
+
+## SYNOPSIS
+
+`nodestorage <noderange> [show|create|delete] [options]`  
+
+## DESCRIPTION
+
+`nodestorage` provides access to the remote storage configuration of
+the noderange.
+
+## OPTIONS
+* `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
+  RAID level to use when creating an array
+
+* `-d` **DISKS**, `--disks`=**DISKS**:
+   Comma separated list of disks to use, or the word "rest" to
+   indicate use of all available disks
+
+* `-s` **SIZE**, `--size`=**SIZE**:
+  Comma separated list of sizes to use when creating
+  volumes.  The sizes may be absolute size (e.g. 16gb),
+  percentage (10%) or the word "rest" to use remaining
+  capacity, default behavior is to use all capacity to
+  make a volume
+* `-n` **NAME**, `--name`=**NAME**:
+  Comma separated list of names to use when naming
+  volumes, or selecting a volume for delete.  Default
+  behavior is to use implementation provided default
+
+## EXAMPLES
+
+* Deleting the volume `somedata`:
+    `$ nodestorage d5 delete somedata`  
+    `Deleted: somedata`  
+
+* Creating a raid5 of 4 disks and a volume named `somedata`:
+    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata`  
+    `d5: Volume somedata: Size: 1.905 TB`  
+    `d5: Volume somedata: State: Optimal`  
+    `d5: Volume somedata: Array 1-2`  
+
+* Showing current storage configuration of `d3`:
+    `$ nodestorage d3`  
+    `d3: Disk m.2-0 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-0 State: online`  
+    `d3: Disk m.2-0 FRU: 00LF428`  
+    `d3: Disk m.2-0 Serial Number: H6B80054`  
+    `d3: Disk m.2-0 Array: 0-0`  
+    `d3: Disk m.2-1 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-1 State: online`  
+    `d3: Disk m.2-1 FRU: 00LF428`  
+    `d3: Disk m.2-1 Serial Number: H6B80059`  
+    `d3: Disk m.2-1 Array: 0-0`  
+    `d3: Array 0-0 Available Capacity: 0.000 MB`  
+    `d3: Array 0-0 Total Capacity: 131.072 GB`  
+    `d3: Array 0-0 RAID: RAID 1`  
+    `d3: Array 0-0 Disks: m.2-0,m.2-1`  
+    `d3: Array 0-0 Volumes: new_vd`  
+    `d3: Volume new_vd: Size: 122.040 GB`  
+    `d3: Volume new_vd: State: Optimal`  
+    `d3: Volume new_vd: Array 0-0`  
+
+

confluent_client/doc/man/nodesupport.ronn

@@ -10,7 +10,11 @@ nodesupport(8) -- Utilities for interacting with vendor support
 `nodesupport` provides capabilities associated with interactiong with support.
 Currently it only has the `servicedata` subcommand.  `servicedata` takes
 an argument that is either a directory name (that can be used for a single node
-or multiple nodes) or a file name (only to be used with single node noderange)
+or multiple nodes) or a file name (only to be used with single node noderange).
+Note that the file will be downloaded to the confluent server that actually
+connects to the managed system, so it will download to the remote system if running
+remotely and will download to the collective.manager indicated system if
+running in collective mode.
 
 ## EXAMPLES
 

confluent_server/bin/collective

@@ -101,11 +101,11 @@ def main():
     sl = sp.add_parser('show', help='Show information about the collective')
     ic = sp.add_parser('invite', help='Generate a invitation to allow a new '
                                     'confluent instance to join as a '
-                                    'collective member')
+                                    'collective member.  Run collective invite -h for more information')
     ic.add_argument('name', help='Name of server to invite to join the '
                                  'collective')
-    jc = sp.add_parser('join', help='Join a collective')
-    jc.add_argument('server', help='A server currently in the collective')
+    jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
+    jc.add_argument('server', help='Existing collective member that ran invite and generated a token')
     jc.add_argument('-i', help='Invitation provided by runniing invite on an '
                                'existing collective member')
     cmdset = a.parse_args()

confluent_server/builddeb

@@ -0,0 +1,64 @@
+#!/bin/bash
+cd `dirname $0`
+PKGNAME=$(basename $(pwd))
+DPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+OPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+if grep wheezy /etc/os-release; then
+    DPKGNAME=python-$DPKGNAME
+fi
+cd ..
+mkdir -p /tmp/confluent # $DPKGNAME
+cp -a * .git /tmp/confluent # $DPKGNAME
+cd /tmp/confluent/$PKGNAME
+if [ -x ./makeman ]; then
+    ./makeman
+fi
+./makesetup
+VERSION=`cat VERSION`
+cat > setup.cfg << EOF
+[install]
+install-purelib=/opt/confluent/lib/python
+install-scripts=/opt/confluent/bin
+
+[sdist_dsc]
+package=$DPKGNAME
+EOF
+
+python setup.py sdist > /dev/null 2>&1
+py2dsc dist/*.tar.gz
+shopt -s extglob
+cd deb_dist/!(*.orig)/
+if [ "$OPKGNAME" = "confluent-server" ]; then
+    if grep wheezy /etc/os-release; then
+        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex/' debian/control
+    else
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python-lxml, python-eficompressor, python-pycryptodome, python-dateutil/' debian/control
+    fi
+    if grep wheezy /etc/os-release; then
+        echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
+    else
+        echo 'confluent_client confluent-client' >> debian/pydist-overrides
+    fi
+fi
+head -n -1 debian/control > debian/control1
+mv debian/control1 debian/control
+echo 'export PYBUILD_INSTALL_ARGS=--install-lib=/opt/confluent/lib/python' >> debian/rules
+#echo 'Provides: python-'$DPKGNAME >> debian/control
+#echo 'Conflicts: python-'$DPKGNAME >> debian/control
+#echo 'Replaces: python-'$DPKGNAME' (<<2)' >> debian/control
+#echo 'Breaks: python-'$DPKGNAME' (<<2)' >> debian/control
+
+dpkg-buildpackage -rfakeroot -uc  -us -i
+if [ $? -ne 0 ]; then
+    echo "[ERROR] rpmbuild returned non-zero, run: rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec"
+    exit 1
+else
+    cd -
+    # Clean up the generated files in this directory
+    rm -rf $PKGNAME.egg-info dist setup.py
+    rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d)
+    if [ ! -z "$1" ]; then
+        mv deb_dist/* $1/
+    fi
+fi
+exit 0

confluent_server/confluent/auth.py

@@ -23,9 +23,11 @@ import confluent.config.configmanager as configmanager
 import eventlet
 import eventlet.tpool
 import Cryptodome.Protocol.KDF as KDF
+from fnmatch import fnmatch
 import hashlib
 import hmac
 import multiprocessing
+import confluent.userutil as userutil
 try:
     import PAM
 except ImportError:
@@ -39,7 +41,59 @@ _passchecking = {}
 authworkers = None
 authcleaner = None
 
+_allowedbyrole = {
+    'Operator': {
+        'retrieve': ['*'],
+        'create': [
+            '/noderange/',
+            '/nodes/',
+            '/node*/media/uploads/',
+            '/node*/inventory/firmware/updates/*',
+            '/node*/suppport/servicedata*',
+            '/node*/attributes/expression',
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+            '/node*/configuration/*',
+        ],
+        'update': [
+            '/discovery/*', 
+            '/networking/macs/rescan',
+            '/node*/power/state', 
+            '/node*/power/reseat',
+            '/node*/attributes/*',
+            '/node*/media/*tach',
+            '/node*/boot/nextdevice',
+            '/node*/identify',
+            '/node*/configuration/*',
+        ],
+        'start': [
+            '/nodes/*/console/session*',
+            '/nodes/*/shell/sessions*',
+        ],
+        'delete': [
+            '/discovery/*',
+            '/node*',
+        ],
+    },
+    'Monitor': {
+        'retrieve': [
+            '/node*/health/hardware',
+            '/node*/power/state',
+            '/node*/sensors/*',
+            '/nodes/',
+            '/',
+        ],
+    }
+}
 
+_deniedbyrole = {
+    # This supersedes the above and is only consulted after the allowed has happened
+    'Operator': {
+        'update': [
+            '/node*/configuration/management_controller/users/*',
+        ]
+    }
+}
 class Credentials(object):
     def __init__(self, username, passphrase):
         self.username = username
@@ -112,21 +166,37 @@ def authorize(name, element, tenant=False, operation='create',
             and the relevant ConfigManager object for the context of the
             request.
     """
-    if operation not in ('create', 'start', 'update', 'retrieve', 'delete'):
-        return None
+    # skipuserobj is a leftover from the now abandoned plan to use pam session
+    # to do authorization and authentication.  Now confluent always does authorization
+    # even if pam does authentication.
+    if operation not in ('create', 'start', 'update', 'retrieve', 'delete', None):
+        return False
     user, tenant = _get_usertenant(name, tenant)
     if tenant is not None and not configmanager.is_tenant(tenant):
-        return None
+        return False
     manager = configmanager.ConfigManager(tenant, username=user)
-    if skipuserobj:
-        return None, manager, user, tenant, skipuserobj
     userobj = manager.get_user(user)
+    if not userobj:
+        for group in userutil.grouplist(user):
+            userobj = manager.get_usergroup(group)
+            if userobj:
+                break
     if userobj:  # returning
+        role = userobj.get('role', 'Administrator')
+        if element and role != 'Administrator':
+            for rule in _allowedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    break
+            else:
+                return False
+            for rule in _deniedbyrole.get(role, {}).get(operation, []):
+                if fnmatch(element, rule):
+                    return False
         return userobj, manager, user, tenant, skipuserobj
-    return None
+    return False
 
 
-def check_user_passphrase(name, passphrase, element=None, tenant=False):
+def check_user_passphrase(name, passphrase, operation=None, element=None, tenant=False):
     """Check a a login name and passphrase for authenticity and authorization
 
     The function combines authentication and authorization into one function.
@@ -160,12 +230,20 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
     credobj = Credentials(user, passphrase)
     cfm = configmanager.ConfigManager(tenant, username=user)
     ucfg = cfm.get_user(user)
+    if ucfg is None:
+        try:
+            for group in userutil.grouplist(user):
+                ucfg = cfm.get_usergroup(group)
+                if ucfg:
+                    break
+        except KeyError:
+            pass
     if ucfg is None:
         eventlet.sleep(0.05)
         return None
     if (user, tenant) in _passcache:
         if hashlib.sha256(passphrase).digest() == _passcache[(user, tenant)]:
-            return authorize(user, element, tenant)
+            return authorize(user, element, tenant, operation=operation)
         else:
             # In case of someone trying to guess,
             # while someone is legitimately logged in
@@ -200,7 +278,7 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         # delay as well
         if crypt == crypted:
             _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
-            return authorize(user, element, tenant)
+            return authorize(user, element, tenant, operation)
     try:
         pammy = PAM.pam()
         pammy.start(_pamservice, user, credobj.pam_conv)
@@ -208,7 +286,7 @@ def check_user_passphrase(name, passphrase, element=None, tenant=False):
         pammy.acct_mgmt()
         del pammy
         _passcache[(user, tenant)] = hashlib.sha256(passphrase).digest()
-        return authorize(user, element, tenant, skipuserobj=False)
+        return authorize(user, element, tenant, operation, skipuserobj=False)
     except NameError:
         pass
     except PAM.error:

confluent_server/confluent/config/attributes.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -93,9 +93,10 @@ node = {
         'description': ('List of static groups for which this node is '
                         'considered a member'),
     },
-    #'type': {
-    #    'description': ('Classification of node as system, vm, etc')
-    #},
+    'type': {
+        'description': ('Classification of node as server or switch'),
+        'validvalues': ('switch', 'server'),
+    },
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
@@ -158,9 +159,11 @@ node = {
     },
     'discovery.passwordrules': {
         'description':  'Any specified rules shall be configured on the BMC '
-                        'upon discovery.  "expiration=no,loginfailures=no" '
-                        'would disable password expiration and login failures '
-                        'triggering a lockout.'
+                        'upon discovery.  "expiration=no,loginfailures=no,complexity=no,reuse=no" '
+                        'would disable password expiration, login failures '
+                        'triggering a lockout, password complexity requirements,'
+                        'and any restrictions around reusing an old password.',
+        'validlistkeys': ('expiration', 'loginfailures', 'complexity', 'reuse'),
     },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
@@ -171,6 +174,7 @@ node = {
                         'so long as the node has no existing public key. '
                         '"open" allows discovery even if a known public key '
                         'is already stored',
+        'validlist': ('manual', 'permissive', 'pxe', 'open'),
     },
     'info.note': {
         'description':  'A field used for administrators to make arbitrary '
@@ -179,6 +183,9 @@ node = {
                         'freeform text data without concern for issues in how '
                         'the server will process it.',
     },
+    'location.height': {
+        'description': 'Height in RU of the system (defaults to query the systems)',
+    },
     'location.room': {
         'description': 'Room description for the node',
     },
@@ -258,11 +265,13 @@ node = {
     'console.logging': {
         'description': ('Indicate logging level to apply to console.  Valid '
                         'values are currently "full", "interactive", and '
-                        '"none". Defaults to "full".')
+                        '"none". Defaults to "full".'),
+        'validvalues': ('full', 'interactive', 'none'),
     },
     'console.method': {
         'description': ('Indicate the method used to access the console of '
-                        'the managed node.')
+                        'the managed node.'),
+        'validvalues': ('ssh', 'ipmi'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/configmanager.py

@@ -1,7 +1,7 @@
 7# vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -99,6 +99,7 @@ _attraliases = {
     'bmcpass': 'secret.hardwaremanagementpassword',
     'switchpass': 'secret.hardwaremanagementpassword',
 }
+_validroles = ('Administrator', 'Operator', 'Monitor')
 
 def _mkpath(pathname):
     try:
@@ -144,10 +145,13 @@ def _parse_key(keydata, password=None):
     if keydata.startswith('*unencrypted:'):
         return base64.b64decode(keydata[13:])
     elif password:
-        salt, iv, crypt, hmac = [base64.b64decode(x)
+        cryptbits = [base64.b64decode(x)
                                  for x in keydata.split('!')]
+        salt, iv, crypt, hmac = cryptbits[:4]
         privkey, integkey = _derive_keys(password, salt)
-        return decrypt_value([iv, crypt, hmac], privkey, integkey)
+        if len(cryptbits) > 4:
+            integkey = None
+        return decrypt_value(cryptbits[1:], privkey, integkey)
     raise(exc.LockedCredentials(
         "Passphrase protected secret requires password"))
 
@@ -156,7 +160,7 @@ def _format_key(key, password=None):
     if password is not None:
         salt = os.urandom(32)
         privkey, integkey = _derive_keys(password, salt)
-        cval = crypt_value(key, key=privkey, integritykey=integkey)
+        cval = crypt_value(key, key=privkey) # , integritykey=integkey)
         return {"passphraseprotected": (salt,) + cval}
     else:
         return {"unencryptedvalue": key}
@@ -169,6 +173,24 @@ def _do_notifier(cfg, watcher, callback):
         logException()
 
 
+
+def _rpc_master_del_usergroup(tenant, name):
+    ConfigManager(tenant).del_usergroup(name)
+
+
+def _rpc_del_usergroup(tenant, name):
+    ConfigManager(tenant)._true_del_usergroup(name)
+
+
+
+def _rpc_master_set_usergroup(tenant, name, attributemap):
+    ConfigManager(tenant).set_user(name, attributemap)
+
+
+def _rpc_set_usergroup(tenant, name, attributemap):
+    ConfigManager(tenant)._true_set_user(name, attributemap)
+
+
 def _rpc_master_set_user(tenant, name, attributemap):
     ConfigManager(tenant).set_user(name, attributemap)
 
@@ -181,6 +203,14 @@ def _rpc_master_set_node_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant).set_node_attributes(attribmap, autocreate)
 
 
+def _rpc_master_rename_nodes(tenant, renamemap):
+    ConfigManager(tenant).rename_nodes(renamemap)
+
+
+def _rpc_master_rename_nodegroups(tenant, renamemap):
+    ConfigManager(tenant).rename_nodegroups(renamemap)
+
+
 def _rpc_master_clear_node_attributes(tenant, nodes, attributes):
     ConfigManager(tenant).clear_node_attributes(nodes, attributes)
 
@@ -212,9 +242,19 @@ def _rpc_del_user(tenant, name):
 def _rpc_master_create_user(tenant, *args):
     ConfigManager(tenant).create_user(*args)
 
+
+def _rpc_master_create_usergroup(tenant, *args):
+    ConfigManager(tenant).create_usergroup(*args)
+
+
 def _rpc_create_user(tenant, *args):
     ConfigManager(tenant)._true_create_user(*args)
 
+
+def _rpc_create_usergroup(tenant, *args):
+    ConfigManager(tenant)._true_create_usergroup(*args)
+
+
 def _rpc_master_del_groups(tenant, groups):
     ConfigManager(tenant).del_groups(groups)
 
@@ -234,6 +274,14 @@ def _rpc_set_node_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant)._true_set_node_attributes(attribmap, autocreate)
 
 
+def _rpc_rename_nodes(tenant, renamemap):
+    ConfigManager(tenant)._true_rename_nodes(renamemap)
+
+
+def _rpc_rename_nodegroups(tenant, renamemap):
+    ConfigManager(tenant)._true_rename_nodegroups(renamemap)
+
+
 def _rpc_set_group_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant)._true_set_group_attributes(attribmap, autocreate)
 
@@ -288,9 +336,9 @@ def logException():
                  event=confluent.log.Events.stacktrace)
 
 
-def _do_add_watcher(watcher, added, configmanager):
+def _do_add_watcher(watcher, added, configmanager, renamed=()):
     try:
-        watcher(added=added, deleting=[], configmanager=configmanager)
+        watcher(added=added, deleting=(), renamed=renamed, configmanager=configmanager)
     except Exception:
         logException()
 
@@ -311,11 +359,11 @@ def init_masterkey(password=None, autogen=True):
     if cfgn:
         _masterintegritykey = _get_protected_key(cfgn, password,
                                                  'master_integrity_key')
-    elif autogen:
-        _masterintegritykey = os.urandom(64)
-        set_global('master_integrity_key', _format_key(
-            _masterintegritykey,
-            password=password))
+    #elif autogen:
+    #    _masterintegritykey = os.urandom(64)
+    #    set_global('master_integrity_key', _format_key(
+    #        _masterintegritykey,
+    #        password=password))
 
 
 def _push_rpc(stream, payload):
@@ -328,25 +376,34 @@ def _push_rpc(stream, payload):
 def decrypt_value(cryptvalue,
                   key=None,
                   integritykey=None):
-    iv, cipherdata, hmac = cryptvalue
+    # for future reference, if cryptvalue len == 3, then cbc+hmac, 4 includes version
+    iv, cipherdata, hmac = cryptvalue[:3]
     if key is None and integritykey is None:
-        if _masterkey is None or _masterintegritykey is None:
+        if _masterkey is None:
             init_masterkey(autogen=False)
         key = _masterkey
         integritykey = _masterintegritykey
-    check_hmac = HMAC.new(integritykey, cipherdata, SHA256).digest()
-    if hmac != check_hmac:
-        raise Exception("bad HMAC value on crypted value")
-    decrypter = AES.new(key, AES.MODE_CBC, iv)
-    value = decrypter.decrypt(cipherdata)
-    padsize = ord(value[-1])
-    pad = value[-padsize:]
-    # Note that I cannot grasp what could be done with a subliminal
-    # channel in padding in this case, but check the padding anyway
-    for padbyte in pad:
-        if ord(padbyte) != padsize:
-            raise Exception("bad padding in encrypted value")
-    return value[0:-padsize]
+    if len(cryptvalue) == 3:
+        check_hmac = HMAC.new(integritykey, cipherdata, SHA256).digest()
+        if hmac != check_hmac:
+            check_hmac = HMAC.new(integritykey, cipherdata + iv, SHA256).digest()
+        if hmac != check_hmac:
+            raise Exception("bad HMAC value on crypted value")
+        decrypter = AES.new(key, AES.MODE_CBC, iv)
+        value = decrypter.decrypt(cipherdata)
+        padsize = ord(value[-1])
+        pad = value[-padsize:]
+        # Note that I cannot grasp what could be done with a subliminal
+        # channel in padding in this case, but check the padding anyway
+        for padbyte in pad:
+            if ord(padbyte) != padsize:
+                raise Exception("bad padding in encrypted value")
+        return value[0:-padsize]
+    else:
+        decrypter = AES.new(key, AES.MODE_GCM, nonce=iv)
+        value = decrypter.decrypt(cipherdata)
+        decrypter.verify(hmac)
+        return value
 
 
 def fixup_attribute(attrname, attrval):
@@ -398,22 +455,18 @@ def crypt_value(value,
     # encrypt given value
     # PKCS7 is the padding scheme to employ, if no padded needed, pad with 16
     # check HMAC prior to attempting decrypt
-    if key is None or integritykey is None:
-        if _masterkey is None or _masterintegritykey is None:
+    hmac = None
+    if key is None:
+        if _masterkey is None:
             init_masterkey()
         key = _masterkey
-        integritykey = _masterintegritykey
-    iv = os.urandom(16)
-    crypter = AES.new(key, AES.MODE_CBC, iv)
-    neededpad = 16 - (len(value) % 16)
-    pad = chr(neededpad) * neededpad
-    value += pad
+    iv = os.urandom(12)
+    crypter = AES.new(key, AES.MODE_GCM, nonce=iv)
     try:
-        cryptval = crypter.encrypt(value)
+        cryptval, hmac = crypter.encrypt_and_digest(value)
     except TypeError:
-        cryptval = crypter.encrypt(value.encode('utf-8'))
-    hmac = HMAC.new(integritykey, cryptval, SHA256).digest()
-    return iv, cryptval, hmac
+        cryptval, hmac = crypter.encrypt_and_digest(value.encode('utf-8'))
+    return iv, cryptval, hmac, '\x02'
 
 
 def _load_dict_from_dbm(dpath, tdb):
@@ -1039,24 +1092,32 @@ class ConfigManager(object):
             raise Exception('Invalid Expression')
         for node in nodes:
             try:
-                currval = self._cfgstore['nodes'][node][attribute]['value']
+                currvals = [self._cfgstore['nodes'][node][attribute]['value']]
             except KeyError:
                 # Let's treat 'not set' as being an empty string for this path
-                currval = ''
-            if exmatch:
-                if yieldmatches:
-                    if exmatch.search(currval):
-                        yield node
+                currvals = list(
+                    [self._cfgstore['nodes'][node][x].get('value', '') 
+                    for x in fnmatch.filter(self._cfgstore['nodes'][node], attribute)])
+                currvals.append('')
+            for currval in currvals:
+                if exmatch:
+                    if yieldmatches:
+                        if exmatch.search(currval):
+                            yield node
+                            break
+                    else:
+                        if not exmatch.search(currval):
+                            yield node
+                            break
                 else:
-                    if not exmatch.search(currval):
-                        yield node
-            else:
-                if yieldmatches:
-                    if match == currval:
-                        yield node
-                else:
-                    if match != currval:
-                        yield node
+                    if yieldmatches:
+                        if match == currval:
+                            yield node
+                            break
+                    else:
+                        if match != currval:
+                            yield node
+                            break
 
     def filter_nodenames(self, expression, nodes=None):
         """Filter nodenames by regular expression
@@ -1158,6 +1219,12 @@ class ConfigManager(object):
         except KeyError:
             return []
 
+    def list_usergroups(self):
+        try:
+            return list(self._cfgstore['usergroups'])
+        except KeyError:
+            return []
+
     def get_user(self, name):
         """Get user information from DB
 
@@ -1195,12 +1262,46 @@ class ConfigManager(object):
         :param groupname: the name of teh group to modify
         :param attributemap: The mapping of keys to values to set
         """
+        if cfgleader:
+            return exec_on_leader('_rpc_master_set_usergroup', self.tenant,
+                                  groupname, attributemap)
+        if cfgstreams:
+            exec_on_followers('_rpc_set_usergroup', self.tenant, groupname,
+                              attributemap)
+        self._true_set_usergroup(groupname, attributemap)
 
+    def _true_set_usergroup(self, groupname, attributemap):
         for attribute in attributemap:
-            self._cfgstore['usergroups'][attribute] = attributemap[attribute]
+            if attribute == 'role':
+                therole = None
+                for candrole in _validroles:
+                    if candrole.lower().startswith(attributemap[attribute].lower()):
+                        therole = candrole
+                if therole not in _validroles:
+                    raise ValueError(
+                        'Unrecognized role "{0}" (valid roles: {1})'.format(attributemap[attribute], ','.join(_validroles)))
+                attributemap[attribute] = therole
+            self._cfgstore['usergroups'][groupname][attribute] = attributemap[attribute]
         _mark_dirtykey('usergroups', groupname, self.tenant)
+        self._bg_sync_to_file()
 
     def create_usergroup(self, groupname, role="Administrator"):
+        """Create a new user
+
+        :param groupname: The name of the user group
+        :param role: The role the user should be considered.  Can be
+                     "Administrator" or "Technician", defaults to
+                     "Administrator"
+        """
+        if cfgleader:
+            return exec_on_leader('_rpc_master_create_usergroup', self.tenant,
+                                  groupname, role)
+        if cfgstreams:
+            exec_on_followers('_rpc_create_usergroup', self.tenant, groupname,
+                              role)
+        self._true_create_usergroup(groupname, role)
+
+    def _true_create_usergroup(self, groupname, role="Administrator"):
         if 'usergroups' not in self._cfgstore:
             self._cfgstore['usergroups'] = {}
         groupname = groupname.encode('utf-8')
@@ -1208,6 +1309,20 @@ class ConfigManager(object):
             raise Exception("Duplicate groupname requested")
         self._cfgstore['usergroups'][groupname] = {'role': role}
         _mark_dirtykey('usergroups', groupname, self.tenant)
+        self._bg_sync_to_file()
+
+    def del_usergroup(self, name):
+        if cfgleader:
+            return exec_on_leader('_rpc_master_del_usergroup', self.tenant, name)
+        if cfgstreams:
+            exec_on_followers('_rpc_del_usergroup', self.tenant, name)
+        self._true_del_usergroup(name)
+
+    def _true_del_usergroup(self, name):
+        if name in self._cfgstore['usergroups']:
+            del self._cfgstore['usergroups'][name]
+            _mark_dirtykey('usergroups', name, self.tenant)
+        self._bg_sync_to_file()
 
     def set_user(self, name, attributemap):
         """Set user attribute(s)
@@ -1225,6 +1340,15 @@ class ConfigManager(object):
     def _true_set_user(self, name, attributemap):
         user = self._cfgstore['users'][name]
         for attribute in attributemap:
+            if attribute == 'role':
+                therole = None
+                for candrole in _validroles:
+                    if candrole.lower().startswith(attributemap[attribute].lower()):
+                        therole = candrole
+                if therole not in _validroles:
+                    raise ValueError(
+                        'Unrecognized role "{0}" (valid roles: {1})'.format(attributemap[attribute], ','.join(_validroles)))
+                attributemap[attribute] = therole
             if attribute == 'password':
                 salt = os.urandom(8)
                 #TODO: WORKERPOOL, offload password set to a worker
@@ -1499,6 +1623,9 @@ class ConfigManager(object):
                     newattr = _attraliases[attr]
                     attribmap[group][newattr] = attribmap[group][attr]
                     del attribmap[group][attr]
+            if 'noderange' in attribmap[group]:
+                if len(attribmap[group]) > 1:
+                    raise ValueError('noderange attribute must be set by itself')
             for attr in attribmap[group]:
                 if attr in _attraliases:
                     newattr = _attraliases[attr]
@@ -1515,17 +1642,20 @@ class ConfigManager(object):
                         currnodes = list(self.get_nodegroup_attributes(
                             group, ['nodes']).get('nodes', []))
                         if attribmap[group][attr].get('prepend', False):
-                            newnodes = attribmap[group][attr][
-                                'prepend'].split(',')
-                            attribmap[group][attr] = newnodes + currnodes
+                            newnodes = noderange.NodeRange(attribmap[group][attr][
+                                'prepend'], config=self).nodes
+                            attribmap[group][attr] = list(
+                                newnodes) + currnodes
                         elif attribmap[group][attr].get('remove', False):
-                            delnodes = attribmap[group][attr][
-                                'remove'].split(',')
+                            delnodes = noderange.NodeRange(
+                                attribmap[group][attr]['remove'],
+                                config=self).nodes
                             attribmap[group][attr] = [
                                 x for x in currnodes if x not in delnodes]
                     if not isinstance(attribmap[group][attr], list):
                         if type(attribmap[group][attr]) is unicode or type(attribmap[group][attr]) is str:
-                            attribmap[group][attr]=attribmap[group][attr].split(",")
+                            attribmap[group][attr] = noderange.NodeRange(
+                                attribmap[group][attr], config=self).nodes
                         else:
                             raise ValueError("nodes attribute on group must be list")
                     for node in attribmap[group]['nodes']:
@@ -1538,6 +1668,13 @@ class ConfigManager(object):
             if group not in self._cfgstore['nodegroups']:
                 self._cfgstore['nodegroups'][group] = {'nodes': set()}
             cfgobj = self._cfgstore['nodegroups'][group]
+            if 'noderange' in attribmap[group] and attribmap[group]['noderange']:
+                if cfgobj['nodes']:
+                    raise ValueError('Cannot set both nodes and noderange on group')
+                if set(cfgobj) - set(['noderange', 'nodes']):
+                    raise ValueError('Cannot set noderange on a group with attributes')
+            elif 'noderange' in cfgobj and cfgobj['noderange']:
+                raise ValueError('Attributes cannot be set on a group with a noderange')
             for attr in attribmap[group]:
                 if attr == 'nodes':
                     newdict = set(attribmap[group][attr])
@@ -1692,7 +1829,7 @@ class ConfigManager(object):
     def _true_del_nodes(self, nodes):
         if self.tenant in self._nodecollwatchers:
             for watcher in self._nodecollwatchers[self.tenant].itervalues():
-                watcher(added=[], deleting=nodes, configmanager=self)
+                watcher(added=(), deleting=nodes, renamed=(), configmanager=self)
         changeset = {}
         for node in nodes:
             # set a reserved attribute for the sake of the change notification
@@ -1776,6 +1913,86 @@ class ConfigManager(object):
                 attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
 
+    def rename_nodes(self, renamemap):
+        if cfgleader:
+            return exec_on_leader('_rpc_master_rename_nodes', self.tenant,
+            renamemap)
+        if cfgstreams:
+            exec_on_followers('_rpc_rename_nodes', self.tenant, renamemap)
+        self._true_rename_nodes(renamemap)
+
+    def _true_rename_nodes(self, renamemap):
+        oldnames = set(renamemap)
+        exprmgr = None
+        currnodes = set(self._cfgstore['nodes'])
+        missingnodes = oldnames - currnodes
+        if missingnodes:
+            raise ValueError(
+                'The following nodes to rename do not exist: {0}'.format(
+                    ','.join(missingnodes)))
+        newnames = set([])
+        for name in renamemap:
+            newnames.add(renamemap[name])
+        if newnames & currnodes:
+            raise ValueError(
+                'The following requested new names conflict with existing nodes: {0}'.format(
+                    ','.join(newnames & currnodes)))
+        for name in renamemap:
+            self._cfgstore['nodes'][renamemap[name]] = self._cfgstore['nodes'][name]
+            del self._cfgstore['nodes'][name]
+            _mark_dirtykey('nodes', name, self.tenant)
+            _mark_dirtykey('nodes', renamemap[name], self.tenant)
+            for group in self._cfgstore['nodes'][renamemap[name]].get('groups', []):
+                self._cfgstore['nodegroups'][group]['nodes'].discard(name)
+                self._cfgstore['nodegroups'][group]['nodes'].add(renamemap[name])
+                _mark_dirtykey('nodegroups', group, self.tenant)
+            cfgobj = self._cfgstore['nodes'][renamemap[name]]
+            node = renamemap[name]
+            changeset = {}
+            if exprmgr is None:
+                exprmgr = _ExpressionFormat(cfgobj, node)
+            self._recalculate_expressions(cfgobj, formatter=exprmgr, node=renamemap[name], changeset=changeset)
+        if self.tenant in self._nodecollwatchers:
+            nodecollwatchers = self._nodecollwatchers[self.tenant]
+            for watcher in nodecollwatchers.itervalues():
+                eventlet.spawn_n(_do_add_watcher, watcher, (), self, renamemap)
+        self._bg_sync_to_file()
+
+    def rename_nodegroups(self, renamemap):
+        if cfgleader:
+            return exec_on_leader('_rpc_master_rename_nodegroups', self.tenant, renamemap)
+        if cfgstreams:
+            exec_on_followers('_rpc_rename_nodegroups', self.tenant, renamemap)
+        self._true_rename_groups(renamemap)
+
+    def _true_rename_groups(self, renamemap):
+        oldnames = set(renamemap)
+        currgroups = set(self._cfgstore['nodegroups'])
+        missinggroups = oldnames - currgroups
+        if missinggroups:
+            raise ValueError(
+                'The following groups to rename do not exist: {0}'.format(
+                    ','.join(missinggroups)))
+        newnames = set([])
+        for name in renamemap:
+            newnames.add(renamemap[name])
+        if newnames & currgroups:
+            raise ValueError(
+                'The following requested new names conflict with existing groups: {0}'.format(
+                    ','.join(newnames & currgroups)))
+        for name in renamemap:
+            self._cfgstore['nodegroups'][renamemap[name]] = self._cfgstore['nodegroups'][name]
+            del self._cfgstore['nodegroups'][name]
+            _mark_dirtykey('nodegroups', name, self.tenant)
+            _mark_dirtykey('nodegroups', renamemap[name], self.tenant)
+            for node in self._cfgstore['nodegroups'][renamemap[name]].get('nodes', []):
+                lidx = self._cfgstore['nodes'][node]['groups'].index(name)
+                self._cfgstore['nodes'][node]['groups'][lidx] = renamemap[name]
+                _mark_dirtykey('nodes', node, self.tenant)
+        self._bg_sync_to_file()
+
+
+
     def set_node_attributes(self, attribmap, autocreate=False):
         if cfgleader:  # currently config slave to another
             return exec_on_leader('_rpc_master_set_node_attributes',
@@ -1960,6 +2177,9 @@ class ConfigManager(object):
                 self.set_node_attributes(tmpconfig[confarea], True)
             elif confarea == 'nodegroups':
                 self.set_group_attributes(tmpconfig[confarea], True)
+            elif confarea == 'usergroups':
+                for usergroup in tmpconfig[confarea]:
+                    self.create_usergroup(usergroup)
             elif confarea == 'users':
                 for user in tmpconfig[confarea]:
                     uid = tmpconfig[confarea].get('id', None)
@@ -2029,7 +2249,9 @@ class ConfigManager(object):
         rootpath = cls._cfgdir
         try:
             with open(os.path.join(rootpath, 'transactioncount'), 'r') as f:
-                _txcount = struct.unpack('!Q', f.read())[0]
+                txbytes = f.read()
+                if len(txbytes) == 8:
+                    _txcount = struct.unpack('!Q', txbytes)[0]
         except IOError:
             pass
         _load_dict_from_dbm(['collective'], os.path.join(rootpath,
@@ -2213,7 +2435,7 @@ def _restore_keys(jsond, password, newpassword=None, sync=True):
 
 
 def _dump_keys(password, dojson=True):
-    if _masterkey is None or _masterintegritykey is None:
+    if _masterkey is None:
         init_masterkey()
     cryptkey = _format_key(_masterkey, password=password)
     if 'passphraseprotected' in cryptkey:
@@ -2222,14 +2444,16 @@ def _dump_keys(password, dojson=True):
     else:
         cryptkey = '*unencrypted:{0}'.format(base64.b64encode(
             cryptkey['unencryptedvalue']))
-    integritykey = _format_key(_masterintegritykey, password=password)
-    if 'passphraseprotected' in integritykey:
-        integritykey = '!'.join(map(base64.b64encode,
-                                    integritykey['passphraseprotected']))
-    else:
-        integritykey = '*unencrypted:{0}'.format(base64.b64encode(
-            integritykey['unencryptedvalue']))
-    keydata = {'cryptkey': cryptkey, 'integritykey': integritykey}
+    keydata = {'cryptkey': cryptkey}
+    if _masterintegritykey is not None:
+        integritykey = _format_key(_masterintegritykey, password=password)
+        if 'passphraseprotected' in integritykey:
+            integritykey = '!'.join(map(base64.b64encode,
+                                        integritykey['passphraseprotected']))
+        else:
+            integritykey = '*unencrypted:{0}'.format(base64.b64encode(
+                integritykey['unencryptedvalue']))
+        keydata['integritykey'] = integritykey
     if dojson:
         return json.dumps(keydata, sort_keys=True, indent=4, separators=(',', ': '))
     return keydata

confluent_server/confluent/consoleserver.py

@@ -147,6 +147,7 @@ class ConsoleHandler(object):
 
     def __init__(self, node, configmanager, width=80, height=24):
         self.clearpending = False
+        self.clearerror = False
         self.initsize = (width, height)
         self._dologging = True
         self._is_local = True
@@ -308,8 +309,11 @@ class ConsoleHandler(object):
 
     def clearbuffer(self):
         self.feedbuffer(
-            '\x1bc[no replay buffer due to console.logging attribute set to '
-            'none or interactive,\r\nconnection loss, or service restart]')
+            '\x1bc[No data has been received from the remote console since ' \
+            'connecting.  This could\r\nbe due to having the console.logging ' \
+            'attribute set to none or interactive,\r\nserial console not '  \
+            'being enabled or incorrectly configured in the OS or\r\nfirmware, ' \
+            'or the console simply not having any output since last connection]')
         self.clearpending = True
 
     def _detach(self):
@@ -371,7 +375,20 @@ class ConsoleHandler(object):
             self.error = 'misconfigured'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
+            self.feedbuffer(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self._send_rcpts(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self.clearerror = True
             return
+        if self.clearerror:
+            self.clearerror = False
+            self.clearbuffer()
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self.send_break = self._console.send_break
         self.resize = self._console.resize
         if self._attribwatcher:
@@ -527,10 +544,11 @@ class ConsoleHandler(object):
         # TODO: analyze buffer for registered events, examples:
         #   panics
         #   certificate signing request
-        if self.clearpending:
+        if self.clearpending or self.clearerror:
             self.clearpending = False
-            self.feedbuffer(b'\x1bc')
-            self._send_rcpts(b'\x1bc')
+            self.clearerror = False
+            self.feedbuffer(b'\x1bc\x1b[2J\x1b[1;1H')
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
         self.log(data, eventdata=eventdata)
         self.lasttime = util.monotonic_time()
@@ -603,11 +621,14 @@ def disconnect_node(node, configmanager):
         del _handled_consoles[consk]
 
 
-def _nodechange(added, deleting, configmanager):
-    for node in added:
-        connect_node(node, configmanager)
+def _nodechange(added, deleting, renamed, configmanager):
     for node in deleting:
         disconnect_node(node, configmanager)
+    for node in renamed:
+        disconnect_node(node, configmanager)
+        connect_node(renamed[node], configmanager)
+    for node in added:
+        connect_node(node, configmanager)
 
 
 def _start_tenant_sessions(cfm):

confluent_server/confluent/core.py

@@ -35,6 +35,7 @@
 
 import confluent
 import confluent.alerts as alerts
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.config.attributes as attrscheme
 import confluent.config.configmanager as cfm
@@ -123,7 +124,7 @@ def load_plugins():
 
 
 rootcollections = ['discovery/', 'events/', 'networking/',
-                   'noderange/', 'nodes/', 'nodegroups/', 'users/', 'version']
+                   'noderange/', 'nodes/', 'nodegroups/', 'usergroups/' , 'users/', 'version']
 
 
 class PluginRoute(object):
@@ -143,6 +144,7 @@ def _init_core():
     # be enumerated in any collection
     noderesources = {
         'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
             'expression': PluginRoute({'handler': 'attributes'}),
@@ -169,6 +171,10 @@ def _init_core():
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
                 }),
+                'save_licenses': PluginRoute({
+                    'pluginattrs': ['hardwaremanagement.method'],
+                    'default': 'ipmi',
+                }),
                 'net_interfaces': PluginCollection({
                     'pluginattrs': ['hardwaremanagement.method'],
                     'default': 'ipmi',
@@ -378,6 +384,7 @@ def _init_core():
 
     nodegroupresources = {
         'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
         },
@@ -393,13 +400,33 @@ def create_user(inputdata, configmanager):
     configmanager.create_user(username, attributemap=inputdata)
 
 
+def create_usergroup(inputdata, configmanager):
+    try:
+        groupname = inputdata['name']
+        del inputdata['name']
+    except (KeyError, ValueError):
+        raise exc.InvalidArgumentException()
+    configmanager.create_usergroup(groupname)
+
+
+def update_usergroup(groupname, attribmap, configmanager):
+    try:
+        configmanager.set_usergroup(groupname, attribmap)
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
+
 def update_user(name, attribmap, configmanager):
     try:
         configmanager.set_user(name, attribmap)
-    except ValueError:
-        raise exc.InvalidArgumentException()
+    except ValueError as e:
+        raise exc.InvalidArgumentException(str(e))
 
 
+def show_usergroup(groupname, configmanager):
+    groupinfo = configmanager.get_usergroup(groupname)
+    for attr in groupinfo:
+        yield msg.Attributes(kv={attr: groupinfo[attr]})
+
 def show_user(name, configmanager):
     userobj = configmanager.get_user(name)
     rv = {}
@@ -416,6 +443,10 @@ def show_user(name, configmanager):
                 rv[attr] = userobj[attr]
             yield msg.Attributes(kv={attr: rv[attr]},
                                  desc=attrscheme.user[attr]['description'])
+    if 'role' in userobj:
+        yield msg.Attributes(kv={'role': userobj['role']})
+
+
 
 
 def stripnode(iterablersp, node):
@@ -448,6 +479,10 @@ def delete_user(user, configmanager):
     configmanager.del_user(user)
     yield msg.DeletedResource(user)
 
+def delete_usergroup(usergroup, configmanager):
+    configmanager.del_usergroup(usergroup)
+    yield msg.DeletedResource(usergroup)
+
 
 def delete_nodegroup_collection(collectionpath, configmanager):
     if len(collectionpath) == 2:  # just the nodegroup
@@ -651,7 +686,7 @@ def handle_dispatch(connection, cert, dispatch, peername):
                 plugpath = nodeattr[node][attrname]['value']
             elif 'default' in plugroute:
                 plugpath = plugroute['default']
-        if plugpath is not None:
+        if plugpath:
             try:
                 hfunc = getattr(pluginmap[plugpath], operation)
             except KeyError:
@@ -675,7 +710,11 @@ def handle_dispatch(connection, cert, dispatch, peername):
 
 
 def _forward_rsp(connection, res):
-    r = pickle.dumps(res)
+    try:
+        r = pickle.dumps(res)
+    except TypeError:
+        r = pickle.dumps(Exception(
+            'Cannot serialize error, check collective.manager error logs for details' + str(res)))
     rlen = len(r)
     if not rlen:
         return
@@ -685,6 +724,8 @@ def _forward_rsp(connection, res):
 
 def handle_node_request(configmanager, inputdata, operation,
                         pathcomponents, autostrip=True):
+    if log.logfull:
+        raise exc.TargetResourceUnavailable('Filesystem full, free up space and restart confluent service')
     iscollection = False
     routespec = None
     if pathcomponents[0] == 'noderange':
@@ -809,7 +850,7 @@ def handle_node_request(configmanager, inputdata, operation,
                 elif list(cfm.list_collective()):
                     badcollnodes.append(node)
                     continue
-            if plugpath is not None:
+            if plugpath:
                 try:
                     hfunc = getattr(pluginmap[plugpath], operation)
                 except KeyError:
@@ -882,7 +923,7 @@ def dispatch_request(nodes, manager, element, configmanager, inputdata,
     a = configmanager.get_collective_member(manager)
     try:
         remote = socket.create_connection((a['address'], 13001))
-        remote.settimeout(90)
+        remote.settimeout(180)
         remote = ssl.wrap_socket(remote, cert_reqs=ssl.CERT_NONE,
                                  keyfile='/etc/confluent/privkey.pem',
                                  certfile='/etc/confluent/srvcert.pem')
@@ -1000,6 +1041,31 @@ def handle_path(path, operation, configmanager, inputdata=None, autostrip=True):
             configmanager, inputdata, operation, pathcomponents)
     elif pathcomponents[0] == 'version':
         return (msg.Attributes(kv={'version': confluent.__version__}),)
+    elif pathcomponents[0] == 'usergroups':
+        # TODO: when non-administrator accounts exist,
+        # they must only be allowed to see their own user
+        try:
+            usergroup = pathcomponents[1]
+        except IndexError:  # it's just users/
+            if operation == 'create':
+                inputdata = msg.get_input_message(
+                    pathcomponents, operation, inputdata,
+                    configmanager=configmanager)
+                create_usergroup(inputdata.attribs, configmanager)
+            return iterate_collections(configmanager.list_usergroups(),
+                                       forcecollection=False)
+        if usergroup not in configmanager.list_usergroups():
+            raise exc.NotFoundException("Invalid usergroup %s" % usergroup)
+        if operation == 'retrieve':
+            return show_usergroup(usergroup, configmanager)
+        elif operation == 'delete':
+            return delete_usergroup(usergroup, configmanager)
+        elif operation == 'update':
+            inputdata = msg.get_input_message(
+                pathcomponents, operation, inputdata,
+                configmanager=configmanager)
+            update_usergroup(usergroup, inputdata.attribs, configmanager)
+            return show_usergroup(usergroup, configmanager)
     elif pathcomponents[0] == 'users':
         # TODO: when non-administrator accounts exist,
         # they must only be allowed to see their own user

confluent_server/confluent/discovery/core.py

@@ -63,6 +63,7 @@
 
 import base64
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 #import confluent.discovery.protocols.ssdp as ssdp
 import confluent.discovery.protocols.slp as slp
@@ -1055,6 +1056,14 @@ def discover_node(cfg, handler, info, nodename, manual):
                 traceback.print_exc()
                 return False
             newnodeattribs = {}
+            if cfm.list_collective():
+                # We are in a collective, check collective.manager
+                cmc = cfg.get_node_attributes(nodename, 'collective.manager')
+                cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
+                if not cm:
+                    # Node is being discovered in collective, but no collective.manager, default
+                    # to the collective member actually able to execute the discovery
+                    newnodeattribs['collective.manager'] = collective.get_myname()
             if 'uuid' in info:
                 newnodeattribs['id.uuid'] = info['uuid']
             if 'serialnumber' in info:
@@ -1129,10 +1138,18 @@ def _handle_nodelist_change(configmanager):
         nodeaddhandler = None
 
 
-def newnodes(added, deleting, configmanager):
+def newnodes(added, deleting, renamed, configmanager):
     global attribwatcher
     global needaddhandled
     global nodeaddhandler
+    alldeleting = set(deleting) | set(renamed)
+    for node in alldeleting:
+        if node not in known_nodes:
+            continue
+        for mac in known_nodes[node]:
+            if mac in known_info:
+                del known_info[mac]
+        del known_nodes[node]
     _map_unique_ids()
     configmanager.remove_watcher(attribwatcher)
     allnodes = configmanager.list_nodes()

confluent_server/confluent/discovery/handlers/bmc.py

@@ -32,10 +32,21 @@ DEFAULT_PASS = 'PASSW0RD'
 
 class NodeHandler(generic.NodeHandler):
 
-    def _get_ipmicmd(self, user=DEFAULT_USER, password=DEFAULT_PASS):
-        return ipmicommand.Command(self.ipaddr, user, password)
+    def _get_ipmicmd(self, user=None, password=None):
+        priv = None
+        if user is None or password is None:
+            if self.trieddefault:
+                raise pygexc.IpmiException()
+            priv = 4  # manually indicate priv to avoid double-attempt
+        if user is None:
+            user = DEFAULT_USER
+        if password is None:
+            password = DEFAULT_PASS
+        return ipmicommand.Command(self.ipaddr, user, password,
+                                   privlevel=priv, keepalive=False)
 
     def __init__(self, info, configmanager):
+        self.trieddefault = None
         super(NodeHandler, self).__init__(info, configmanager)
 
     def probe(self):
@@ -45,27 +56,27 @@ class NodeHandler(generic.NodeHandler):
     def config(self, nodename, reset=False):
         self._bmcconfig(nodename, reset)
 
-    def _bmcconfig(self, nodename, reset=False, customconfig=None):
+    def _bmcconfig(self, nodename, reset=False, customconfig=None, vc=None):
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
         try:
             ic = self._get_ipmicmd()
             passwd = DEFAULT_PASS
         except pygexc.IpmiException as pi:
-            creds = self.configmanager.get_node_attributes(
-                nodename,
-                ['secret.hardwaremanagementuser',
-                 'secret.hardwaremanagementpassword'], decrypt=True)
-            user = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementuser', {}).get('value', None)
             havecustomcreds = False
             if user is not None and user != DEFAULT_USER:
                 havecustomcreds = True
             else:
                 user = DEFAULT_USER
-            passwd = creds.get(nodename, {}).get(
-                'secret.hardwaremanagementpassword', {}).get('value', None)
             if passwd is not None and passwd != DEFAULT_PASS:
                 havecustomcreds = True
             else:
@@ -74,8 +85,8 @@ class NodeHandler(generic.NodeHandler):
                 ic = self._get_ipmicmd(user, passwd)
             else:
                 raise
-        if customconfig:
-            customconfig(ic)
+        if vc:
+            ic.register_key_handler(vc)
         currusers = ic.get_users()
         lanchan = ic.get_network_channel()
         userdata = ic.xraw_command(netfn=6, command=0x44, data=(lanchan,
@@ -95,29 +106,6 @@ class NodeHandler(generic.NodeHandler):
             raise exc.TargetEndpointBadCredentials(
                 'secret.hardwaremanagementuser and/or '
                 'secret.hardwaremanagementpassword was not configured')
-        if ('hardwaremanagement.manager' in cd and
-                cd['hardwaremanagement.manager']['value'] and
-                not cd['hardwaremanagement.manager']['value'].startswith(
-                    'fe80::')):
-            newip = cd['hardwaremanagement.manager']['value']
-            newipinfo = getaddrinfo(newip, 0)[0]
-            # This getaddrinfo is repeated in get_nic_config, could be
-            # optimized, albeit with a more convoluted api..
-            newip = newipinfo[-1][0]
-            if ':' in newip:
-                raise exc.NotImplementedException('IPv6 remote config TODO')
-            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
-            plen = netconfig['prefix']
-            newip = '{0}/{1}'.format(newip, plen)
-            ic.set_net_configuration(ipv4_address=newip,
-                                     ipv4_configuration='static',
-                                     ipv4_gateway=netconfig['ipv4_gateway'])
-        elif self.ipaddr.startswith('fe80::'):
-            cfg.set_node_attributes(
-                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
-        else:
-            raise exc.TargetEndpointUnreachable(
-                'hardwaremanagement.manager must be set to desired address')
         newuser = cd['secret.hardwaremanagementuser']['value']
         newpass = cd['secret.hardwaremanagementpassword']['value']
         for uid in currusers:
@@ -126,6 +114,9 @@ class NodeHandler(generic.NodeHandler):
                 newuserslot = uid
                 if newpass != passwd:  # don't mess with existing if no change
                     ic.set_user_password(newuserslot, password=newpass)
+                    ic = self._get_ipmicmd(user, passwd)
+                    if vc:
+                        ic.register_key_handler(vc)
                 break
         else:
             newuserslot = lockedusers + 1
@@ -134,8 +125,14 @@ class NodeHandler(generic.NodeHandler):
             if newpass != passwd:  # don't mess with existing if no change
                 ic.set_user_password(newuserslot, password=newpass)
             ic.set_user_name(newuserslot, newuser)
-            ic.set_user_access(newuserslot, lanchan,
-                               privilege_level='administrator')
+            if havecustomcreds:
+                ic = self._get_ipmicmd(user, passwd)
+                if vc:
+                    ic.register_key_handler(vc)
+            #We are remote operating on the account we are
+            #using, no need to try to set user access
+            #ic.set_user_access(newuserslot, lanchan,
+            #                   privilege_level='administrator')
         # Now to zap others
         for uid in currusers:
             if uid != newuserslot:
@@ -156,6 +153,36 @@ class NodeHandler(generic.NodeHandler):
                             # name...
                             # the user will remain, but that is life
                             raise
+        if customconfig:
+            customconfig(ic)
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            # This getaddrinfo is repeated in get_nic_config, could be
+            # optimized, albeit with a more convoluted api..
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
+            plen = netconfig['prefix']
+            newip = '{0}/{1}'.format(newip, plen)
+            currcfg = ic.get_net_configuration()
+            if currcfg['ipv4_address'] != newip:
+                # do not change the ipv4_config if the current config looks
+                # like it is already accurate
+                ic.set_net_configuration(ipv4_address=newip,
+                                         ipv4_configuration='static',
+                                         ipv4_gateway=netconfig[
+                                             'ipv4_gateway'])
+        elif self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address')
         if reset:
             ic.reset_bmc()
         return ic

confluent_server/confluent/discovery/handlers/generic.py

@@ -68,6 +68,23 @@ class NodeHandler(object):
     def _savecert(self, certificate):
         self._fp = certificate
         return True
+    
+    def get_node_credentials(self, nodename, creds, defuser, defpass):
+        user = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', None)
+        havecustomcreds = False
+        if user is not None and user != defuser:
+            havecustomcreds = True
+        else:
+            user = defuser
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', None)
+        if passwd is not None and passwd != defpass:
+            havecustomcreds = True
+        else:
+            passwd = defpass
+        return user, passwd, not havecustomcreds
+
 
     @property
     def cert_fail_reason(self):

confluent_server/confluent/discovery/handlers/imm.py

@@ -15,7 +15,6 @@
 import confluent.discovery.handlers.bmc as bmchandler
 import pyghmi.exceptions as pygexc
 import pyghmi.ipmi.private.util as pygutil
-import string
 import struct
 
 class NodeHandler(bmchandler.NodeHandler):
@@ -25,7 +24,7 @@ class NodeHandler(bmchandler.NodeHandler):
     def adequate(cls, info):
         # We can sometimes receive a partially initialized SLP packet
         # This is not adequate for being satisfied
-        return bool(info['attributes'])
+        return bool(info.get('attributes', {}))
 
     def scan(self):
         slpattrs = self.info.get('attributes', {})
@@ -44,7 +43,7 @@ class NodeHandler(bmchandler.NodeHandler):
                          uuidprefix[12:16]
             self.info['uuid'] = uuidprefix + '-' + '-'.join(
                 wronguuid.split('-')[3:])
-            self.info['uuid'] = string.lower(self.info['uuid'])
+            self.info['uuid'] = self.info['uuid'].lower()
         if ff not in ('dense-computing', 'BC2'):
             # do not probe unless it's a dense platform
             return

confluent_server/confluent/discovery/handlers/smm.py

@@ -13,7 +13,15 @@
 # limitations under the License.
 
 import confluent.discovery.handlers.bmc as bmchandler
+import confluent.exceptions as exc
+import pyghmi.util.webclient as webclient
 import struct
+import urllib
+import eventlet.support.greendns
+import confluent.netutil as netutil
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+from xml.etree.ElementTree import fromstring
 
 def fixuuid(baduuid):
     # SMM dumps it out in hex
@@ -26,7 +34,7 @@ def fixuuid(baduuid):
 class NodeHandler(bmchandler.NodeHandler):
     is_enclosure = True
     devname = 'SMM'
-    maxmacs = 5  # support an enclosure, but try to avoid catching daisy chain
+    maxmacs = 6  # support an enclosure, but try to avoid catching daisy chain
 
     def scan(self):
         # the UUID is in a weird order, fix it up to match
@@ -36,9 +44,147 @@ class NodeHandler(bmchandler.NodeHandler):
             uuid = fixuuid(uuid[0])
             self.info['uuid'] = uuid
 
+    def _validate_cert(self, certificate):
+        # Assumption is by the time we call config, that discovery core has
+        # vetted self._fp.  Our job here then is just to make sure that
+        # the currect connection matches the previously saved cert
+        if not self._fp:  # circumstances are that we haven't validated yet
+            self._fp = certificate
+        return certificate == self._fp
+
+    def _webconfigrules(self, wc):
+        rules = []
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                rules.append('passwordDurationDays:' + value)
+                warndays = '5' if int(value) > 5 else value
+                rules.append('passwordExpireWarningDays:' + warndays)
+            if name.lower() in ('lockout', 'loginfailures'):
+                rules.append('passwordFailAllowdNum:' + value)
+            if name.lower() == 'reuse':
+                rules.append('passwordReuseCheckNum:' + value)
+        if rules:
+            apirequest = 'set={0}'.format(','.join(rules))
+            wc.request('POST', '/data', apirequest)
+            wc.getresponse().read()
+
+    def _webconfignet(self, wc, nodename):
+        cfg = self.configmanager
+        cd = cfg.get_node_attributes(
+            nodename, ['hardwaremanagement.manager'])
+        smmip = cd.get(nodename, {}).get('hardwaremanagement.manager', {}).get('value', None)
+        if smmip and ':' not in smmip:
+            smmip = getaddrinfo(smmip, 0)[0]
+            smmip = smmip[-1][0]
+            if smmip and ':' in smmip:
+                raise exc.NotImplementedException('IPv6 not supported')
+            netconfig = netutil.get_nic_config(cfg, nodename, ip=smmip)
+            netmask = netutil.cidr_to_mask(netconfig['prefix'])
+            setdata = 'set=ifIndex:0,v4DHCPEnabled:0,v4IPAddr:{0},v4NetMask:{1}'.format(smmip, netmask)
+            gateway = netconfig.get('ipv4_gateway', None)
+            if gateway:
+                setdata += ',v4Gateway:{0}'.format(gateway)
+            wc.request('POST', '/data', setdata)
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            if '<statusCode>0' not in rspdata:
+                raise Exception("Error configuring SMM Network")
+            return
+        if smmip and ':' in smmip and not smmip.startswith('fe80::'):
+            raise exc.NotImplementedException('IPv6 configuration TODO')
+        if self.ipaddr.startswith('fe80::'):
+            cfg.set_node_attributes(
+                    {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+
+    def _webconfigcreds(self, username, password):
+        wc = webclient.SecureHTTPConnection(self.ipaddr, 443, verifycallback=self._validate_cert)
+        wc.connect()
+        authdata = {  # start by trying factory defaults
+            'user': 'USERID',
+            'password': 'PASSW0RD',
+        }
+        headers = {'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded'}
+        wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+        rsp = wc.getresponse()
+        rspdata = rsp.read()
+        if 'authResult>0' not in rspdata:
+            # default credentials are refused, try with the actual
+            authdata['user'] = username
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            if 'renew_account' in rspdata:
+                raise Exception('Configured password has expired')
+            if 'authResult>0' not in rspdata:
+                raise Exception('Unknown username/password on SMM')
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+        if 'renew_account' in rspdata:
+            passwdchange = {'oripwd': 'PASSW0RD', 'newpwd': password}
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            wc.request('POST', '/data/changepwd', urllib.urlencode(passwdchange))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['password'] = password
+            wc.request('POST', '/data/login', urllib.urlencode(authdata), headers)
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+        if 'authResult>0' in rspdata:
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            if username == 'USERID':
+                return wc
+            wc.request('POST', '/data', 'set=user(2,1,{0},511,,4,15,0)'.format(username))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            wc.request('POST', '/data/logout')
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            authdata['user'] = username
+            wc.request('POST', '/data/login', urllib.urlencode(authdata, headers))
+            rsp = wc.getresponse()
+            rspdata = rsp.read()
+            tokens = fromstring(rspdata)
+            st2 = tokens.findall('st2')[0].text
+            wc.set_header('ST2', st2)
+            return wc
+
     def config(self, nodename):
         # SMM for now has to reset to assure configuration applies
-        super(NodeHandler, self).config(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        creds = self.configmanager.get_node_attributes(
+            nodename,
+            ['secret.hardwaremanagementuser',
+             'secret.hardwaremanagementpassword'], decrypt=True)
+        username = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementuser', {}).get('value', 'USERID')
+        passwd = creds.get(nodename, {}).get(
+            'secret.hardwaremanagementpassword', {}).get('value', 'PASSW0RD')
+        if passwd == 'PASSW0RD' and self.ruleset:
+            raise Exception('Cannot support default password and setting password rules at same time')
+        if passwd == 'PASSW0RD':
+            # We must avoid hitting the web interface due to forced password change, best effert
+            self._bmcconfig(nodename)
+        else:
+            # Switch to full web based configuration, to mitigate risks with the SMM
+            wc = self._webconfigcreds(username, passwd)
+            self._webconfigrules(wc)
+            self._webconfignet(wc, nodename)
+
 
 # notes for smm:
 # POST to:
@@ -53,4 +199,4 @@ class NodeHandler(bmchandler.NodeHandler):
 # with body user=USERID&password=Passw0rd!4321
 # yields:
 # <?xml version="1.0" encoding="UTF-8"?><root> <status>ok</status> <authResult>0</authResult> <forwardUrl>index.html</forwardUrl> </root>
-# note forwardUrl, if password change needed, will indicate something else
+# note forwardUrl, if password change needed, will indicate something else

confluent_server/confluent/discovery/handlers/xcc.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,10 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import confluent.discovery.handlers.imm as immhandler
+import confluent.netutil as netutil
 import confluent.util as util
+import eventlet
+import eventlet.support.greendns
+import json
+import os
 import pyghmi.exceptions as pygexc
-import pyghmi.ipmi.oem.lenovo.imm as imm
+xcc = eventlet.import_patched('pyghmi.redfish.oem.lenovo.xcc')
+import pyghmi.util.webclient as webclient
+import struct
+getaddrinfo = eventlet.support.greendns.getaddrinfo
+
+
+def fixup_uuid(uuidprop):
+    baduuid = ''.join(uuidprop.split())
+    uuidprefix = (baduuid[:8], baduuid[8:12], baduuid[12:16])
+    a = struct.pack('<IHH', *[int(x, 16) for x in uuidprefix]).encode('hex')
+    uuid = (a[:8], a[8:12], a[12:16], baduuid[16:20], baduuid[20:])
+    return '-'.join(uuid).upper()
 
 
 
@@ -23,6 +40,13 @@ import pyghmi.ipmi.oem.lenovo.imm as imm
 class NodeHandler(immhandler.NodeHandler):
     devname = 'XCC'
 
+    def __init__(self, info, configmanager):
+        self._xcchdlr = None
+        self._wc = None
+        self.nodename = None
+        self._atdefaultcreds = True
+        super(NodeHandler, self).__init__(info, configmanager)
+
     @classmethod
     def adequate(cls, info):
         # We can sometimes receive a partially initialized SLP packet
@@ -33,6 +57,7 @@ class NodeHandler(immhandler.NodeHandler):
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return
+        self.trieddefault = None  # Reset state on a preconfig attempt
         # attempt to enable SMM
         #it's normal to get a 'not supported' (193) for systems without an SMM
         ipmicmd = None
@@ -44,6 +69,7 @@ class NodeHandler(immhandler.NodeHandler):
                     'Incorrect password' not in str(e)):
                 # raise an issue if anything other than to be expected
                 raise
+            self.trieddefault = True
         #TODO: decide how to clean out if important
         #as it stands, this can step on itself
         #if ipmicmd:
@@ -55,7 +81,33 @@ class NodeHandler(immhandler.NodeHandler):
         fprint = util.get_fingerprint(self.https_cert)
         return util.cert_matches(fprint, certificate)
 
-    def set_password_policy(self, ic):
+    @property
+    def wc(self):
+        if self._wc is None:
+            self._wc = webclient.SecureHTTPConnection(
+                self.ipaddr, 443, verifycallback=self.validate_cert)
+            self._wc.connect()
+            self._xcchdlr = xcc.OEMHandler(None, None, self._wc, False)
+        if not self.trieddefault:
+            self._xcchdlr.set_credentials('USERID', 'PASSW0RD')
+            wc = self._xcchdlr.get_webclient()
+            if wc:
+                return wc
+        self.trieddefault = True
+        creds = self.configmanager.get_node_attributes(
+            self.nodename, ['secret.hardwaremanagementuser',
+            'secret.hardwaremanagementpassword'], decrypt=True)
+        user, passwd, isdefault = self.get_node_credentials(
+            self.nodename, creds, 'USERID', 'PASSW0RD')
+        if isdefault:
+            return
+        self._atdefaultcreds = False
+        self._xcchdlr.set_credentials(user, passwd)
+        wc = self._xcchdlr.get_webclient()
+        if wc:
+            return wc
+
+    def set_password_policy(self):
         ruleset = {'USER_GlobalMinPassChgInt': '0'}
         for rule in self.ruleset.split(','):
             if '=' not in rule:
@@ -71,11 +123,92 @@ class NodeHandler(immhandler.NodeHandler):
                 if value.lower() in ('no', 'none', 'disable', 'disabled'):
                     value = '0'
                 ruleset['USER_GlobalMaxLoginFailures'] = value
-        ic.register_key_handler(self.validate_cert)
-        ic.oem_init()
-        ic._oem.immhandler.wc.grab_json_response('/api/dataset', ruleset)
+            if name.lower() == 'complexity':
+                ruleset['USER_GlobalPassComplexRequired'] = value
+            if name.lower() == 'reuse':
+                ruleset['USER_GlobalMinPassReuseCycle'] = value
+        try:
+            self.wc.grab_json_response('/api/dataset', ruleset)
+        except Exception as e:
+            print(repr(e))
+            pass
+
+    def _get_next_userid(self, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        userinfo = userinfo['items'][0]['users']
+        for user in userinfo:
+            if user['users_user_name'] == '':
+                return user['users_user_id']
+
+    def _setup_xcc_account(self, username, passwd, wc):
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        uid = None
+        for user in userinfo['items'][0]['users']:
+            if user['users_user_name'] == username:
+                uid = user['users_user_id']
+                break
+        else:
+            for user in userinfo['items'][0]['users']:
+                if user['users_user_name'] == 'USERID':
+                    uid = user['users_user_id']
+                    break
+        if not uid:
+            raise Exception("XCC has neither the default user nor configured user")
+        # The following will work if the password is force change or normal..
+        wc.grab_json_response('/api/function',
+                             {'USER_UserPassChange': '{0},{1}'.format(uid, passwd)})
+        if username != 'USERID':
+            wc.grab_json_response(
+                '/api/function',
+                {'USER_UserModify': '{0},{1},,1,4,0,0,0,0,,8,'.format(uid, username)})
+
+    def _convert_sha256account(self, user, passwd, wc):
+        # First check if the specified user is sha256...
+        userinfo = wc.grab_json_response('/api/dataset/imm_users')
+        curruser = None
+        uid = None
+        for userent in userinfo['items'][0]['users']:
+            if userent['users_user_name'] == user:
+                curruser = userent
+                break
+        if curruser.get('users_pass_is_sha256', 0):
+            self._wc = None
+            wc = self.wc
+            nwc = wc.dupe()
+            # Have to convert it for being useful with most Lenovo automation tools
+            # This requires deleting the account entirely and trying again
+            tmpuid = self._get_next_userid(wc)
+            try:
+                tpass = base64.b64encode(os.urandom(9)) + 'Iw47$'
+                userparams = "{0},6pmu0ezczzcp,{1},1,4,0,0,0,0,,8,".format(tmpuid, tpass)
+                result = wc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+                adata = json.dumps({
+                    'username': '6pmu0ezczzcp',
+                    'password': tpass,
+                })
+                headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+                nwc.request('POST', '/api/login', adata, headers)
+                rsp = nwc.getresponse()
+                if rsp.status == 200:
+                    rspdata = json.loads(rsp.read())
+                    nwc.set_header('Content-Type', 'application/json')
+                    nwc.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+                    if '_csrf_token' in wc.cookies:
+                        nwc.set_header('X-XSRF-TOKEN', wc.cookies['_csrf_token'])
+                    if rspdata.get('reason', False):
+                        newpass = base64.b64encode(os.urandom(9)) + 'q4J$'
+                        nwc.grab_json_response(
+                            '/api/function',
+                            {'USER_UserPassChange': '{0},{1}'.format(tmpuid, newpass)})
+                    nwc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(curruser['users_user_id'], user)})
+                    userparams = "{0},{1},{2},1,4,0,0,0,0,,8,".format(curruser['users_user_id'], user, passwd)
+                    nwc.grab_json_response('/api/function', {'USER_UserCreate': userparams})
+            finally:
+                self._wc = None
+                self.wc.grab_json_response('/api/function', {'USER_UserDelete': "{0},{1}".format(tmpuid, '6pmu0ezczzcp')})
 
     def config(self, nodename, reset=False):
+        self.nodename = nodename
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe
         # In general, try to use https automation, to make it consistent
         # between hypothetical secure path and today.
@@ -83,17 +216,49 @@ class NodeHandler(immhandler.NodeHandler):
             nodename, 'discovery.passwordrules')
         self.ruleset = dpp.get(nodename, {}).get(
             'discovery.passwordrules', {}).get('value', '')
-        ic = self._bmcconfig(nodename, customconfig=self.set_password_policy)
+        wc = self.wc
+        creds = self.configmanager.get_node_attributes(
+            self.nodename, ['secret.hardwaremanagementuser',
+            'secret.hardwaremanagementpassword'], decrypt=True)
+        user, passwd, isdefault = self.get_node_credentials(nodename, creds, 'USERID', 'PASSW0RD')
+        self.set_password_policy()
+        if self._atdefaultcreds:
+            if not isdefault:
+                self._setup_xcc_account(user, passwd, wc)
+        self._convert_sha256account(user, passwd, wc)
+        cd = self.configmanager.get_node_attributes(
+            nodename, ['secret.hardwaremanagementuser',
+                       'secret.hardwaremanagementpassword',
+                       'hardwaremanagement.manager'], True)
+        cd = cd.get(nodename, {})
+        if ('hardwaremanagement.manager' in cd and
+                cd['hardwaremanagement.manager']['value'] and
+                not cd['hardwaremanagement.manager']['value'].startswith(
+                    'fe80::')):
+            newip = cd['hardwaremanagement.manager']['value']
+            newipinfo = getaddrinfo(newip, 0)[0]
+            newip = newipinfo[-1][0]
+            if ':' in newip:
+                raise exc.NotImplementedException('IPv6 remote config TODO')
+            netconfig = netutil.get_nic_config(self.configmanager, nodename, ip=newip)
+            newmask = netutil.cidr_to_mask(netconfig['prefix'])
+            # do not change the ipv4_config if the current config looks
+            statargs = {'ENET_IPv4Ena': '1', 'ENET_IPv4AddrSource': '0', 'ENET_IPv4StaticIPAddr': newip, 'ENET_IPv4StaticIPNetMask': newmask}
+            if netconfig['ipv4_gateway']:
+                statargs['ENET_IPv4GatewayIPAddr'] = netconfig['ipv4_gateway']
+            wc.grab_json_response('/api/dataset', statargs)
+        elif self.ipaddr.startswith('fe80::'):
+            self.configmanager.set_node_attributes(
+                {nodename: {'hardwaremanagement.manager': self.ipaddr}})
+        else:
+            raise exc.TargetEndpointUnreachable(
+                'hardwaremanagement.manager must be set to desired address (No IPv6 Link Local detected)')
         ff = self.info.get('attributes', {}).get('enclosure-form-factor', '')
         if ff not in ('dense-computing', [u'dense-computing']):
             return
-        # Ok, we can get the enclosure uuid now..
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
-        enclosureuuid = ic._oem.immhandler.get_property(
-            '/v2/ibmc/smm/chassis/uuid')
+        enclosureuuid = self.info.get('attributes', {}).get('chassis-uuid', [None])[0]
         if enclosureuuid:
-            enclosureuuid = imm.fixup_uuid(enclosureuuid).lower()
+            enclosureuuid = enclosureuuid.lower()
             em = self.configmanager.get_node_attributes(nodename,
                                                         'enclosure.manager')
             em = em.get(nodename, {}).get('enclosure.manager', {}).get(
@@ -102,8 +267,3 @@ class NodeHandler(immhandler.NodeHandler):
             if em:
                 self.configmanager.set_node_attributes(
                     {em: {'id.uuid': enclosureuuid}})
-
-# TODO(jjohnson2): web based init config for future prevalidated cert scheme
-#    def config(self, nodename):
-#        return
-

confluent_server/confluent/discovery/protocols/slp.py

@@ -335,6 +335,7 @@ def _add_attributes(parsed):
     else:
         net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
+        net.settimeout(1.0)
         net.connect(target)
     except socket.error:
         return
@@ -363,6 +364,7 @@ def query_srvtypes(target):
     while tries and not connected:
         tries -= 1
         try:
+            net.settimeout(1.0)
             net.connect(target)
             connected = True
         except socket.error:

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -167,6 +167,7 @@ def _find_service(service, target):
             net4.sendto(smsg.format(bcast, service), (bcast, 1900))
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
+    deadline = util.monotonic_time() + 4
     r, _, _ = select.select((net4, net6), (), (), 4)
     peerdata = {}
     while r:
@@ -174,7 +175,10 @@ def _find_service(service, target):
             (rsp, peer) = s.recvfrom(9000)
             neighutil.refresh_neigh()
             _parse_ssdp(peer, rsp, peerdata)
-        r, _, _ = select.select((net4, net6), (), (), 4)
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
     for nid in peerdata:
         yield peerdata[nid]
 
@@ -194,6 +198,8 @@ def _parse_ssdp(peer, rsp, peerdata):
     if code == '200':
         if nid in peerdata:
             peerdatum = peerdata[nid]
+            if peer not in peerdatum['peers']:
+                peerdatum['peers'].append(peer)
         else:
             peerdatum = {
                 'peers': [peer],

confluent_server/confluent/firmwaremanager.py

@@ -19,19 +19,23 @@
 # the time comes
 
 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.messages as msg
 import eventlet
 import os
 import pwd
 import socket
+import traceback
 
 updatesbytarget = {}
 uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
+_tracelog = None
 
 
 def execupdate(handler, filename, updateobj, type, owner, node):
+    global _tracelog
     if type != 'ffdc' and not os.path.exists(filename):
         errstr =  '{0} does not appear to exist on {1}'.format(
             filename, socket.gethostname())
@@ -58,6 +62,9 @@ def execupdate(handler, filename, updateobj, type, owner, node):
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': errstr})
     except Exception as e:
+        if _tracelog is None:
+            _tracelog = log.Logger('trace')
+        _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': str(e)})
 

confluent_server/confluent/httpapi.py

@@ -269,6 +269,9 @@ def _authorize_request(env, operation):
     name = ''
     sessionid = None
     cookie = Cookie.SimpleCookie()
+    element = env['PATH_INFO']
+    if element.startswith('/sessions/current/'):
+        element = None
     if 'HTTP_COOKIE' in env:
         #attempt to use the cookie.  If it matches
         cc = RobustCookie()
@@ -290,7 +293,7 @@ def _authorize_request(env, operation):
                     httpsessions[sessionid]['expiry'] = time.time() + 90
                     name = httpsessions[sessionid]['name']
                     authdata = auth.authorize(
-                        name, element=None,
+                        name, element=element, operation=operation,
                         skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
         if env['PATH_INFO'] == '/sessions/current/logout':
@@ -303,8 +306,10 @@ def _authorize_request(env, operation):
             return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
-        authdata = auth.check_user_passphrase(name, passphrase, element=None)
-        if not authdata:
+        authdata = auth.check_user_passphrase(name, passphrase, operation=operation, element=element)
+        if authdata is False:
+            return {'code': 403}
+        elif not authdata:
             return {'code': 401}
         sessid = util.randomstring(32)
         while sessid in httpsessions:
@@ -341,15 +346,10 @@ def _authorize_request(env, operation):
         if 'csrftoken' in httpsessions[sessid]:
             authinfo['authtoken'] = httpsessions[sessid]['csrftoken']
         return authinfo
-    else:
+    elif authdata is None:
         return {'code': 401}
-        # TODO(jbjohnso): actually evaluate the request for authorization
-        # In theory, the x509 or http auth stuff will get translated and then
-        # passed on to the core authorization function in an appropriate form
-        # expresses return in the form of http code
-        # 401 if there is no known identity
-        # 403 if valid identity, but no access
-        # going to run 200 just to get going for now
+    else:
+        return {'code': 403}
 
 
 def _pick_mimetype(env):
@@ -384,11 +384,11 @@ def resourcehandler(env, start_response):
     try:
         for rsp in resourcehandler_backend(env, start_response):
             yield rsp
-    except:
+    except Exception as e:
         tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                      event=log.Events.stacktrace)
-        start_response('500 - Internal Server Error', [])
-        yield '500 - Internal Server Error'
+        start_response('500 - ' + str(e), [])
+        yield '500 - ' + str(e)
         return
 
 
@@ -429,7 +429,7 @@ def resourcehandler_backend(env, start_response):
         return
     if authorized['code'] == 403:
         start_response('403 Forbidden', badauth)
-        yield 'authorization failed'
+        yield 'Forbidden'
         return
     if authorized['code'] != 200:
         raise Exception("Unrecognized code from auth engine")
@@ -469,6 +469,10 @@ def resourcehandler_backend(env, start_response):
         funport = forwarder.get_port(targip, env['HTTP_X_FORWARDED_FOR'],
                                      authorized['sessionid'])
         host = env['HTTP_X_FORWARDED_HOST']
+        if ']' in host:
+            host = host.split(']')[0] + ']'
+        elif ':' in host:
+            host = host.rsplit(':', 1)[0]
         url = 'https://{0}:{1}/'.format(host, funport)
         start_response('302', [('Location', url)])
         yield 'Our princess is in another castle!'
@@ -790,9 +794,16 @@ def serve(bind_host, bind_port):
                 ' a second\n')
             eventlet.sleep(1)
     # TCP_FASTOPEN
-    sock.setsockopt(socket.SOL_TCP, 23, 5)
-    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                         debug=False, socket_timeout=60)
+    try:
+        sock.setsockopt(socket.SOL_TCP, 23, 5)
+    except Exception:
+        pass  # we gave it our best shot there
+    try:
+        eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                             debug=False, socket_timeout=60)
+    except TypeError:
+        # Older eventlet in place, skip arguments it does not understand
+        eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -51,11 +51,15 @@
 #    - leading bit reserved, 0 for now
 #    - length of metadata record 7 bits
 #    - type of data referenced by this entry (one byte), currently:
-#       0=text event, 1=json, 2=console data
+#       0=text event, 1=json, 2=console data, 3=event
 #    - offset into the text log to begin (4 bytes)
 #    - length of data referenced by this entry (2 bytes)
-#    - UTC timestamp of this entry in seconds since epoch (unsigned 32 bit?)
-#    - CRC32 over the record
+#    - UTC timestamp of this entry in seconds since epoch (unsigned 32 bit)
+#    - Event type (per 'Events' class below)
+#    - Event data (per event, currently used by connect/disconnect to represent
+#      single or multiple connections by user and for 'appmode' and 'shiftin'
+#      status for console
+#    - 2 reserved bytes
 #    (a future extended version might include suport for Forward Secure Sealing
 #    or other fields)
 
@@ -73,6 +77,8 @@ import struct
 import time
 import traceback
 
+daemonized = False
+logfull = False
 try:
     from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_SH
 except ImportError:
@@ -150,21 +156,34 @@ class BaseRotatingHandler(object):
         Output the record to the file, catering for rollover as described
         in doRollover().
         """
-        rolling_type = self.shouldRollover(binrecord, textrecord)
-        if rolling_type:
-            flock(self.textfile, LOCK_UN)
-            return self.doRollover(rolling_type)
-        return None
+        global logfull
+        try:
+            rolling_type = self.shouldRollover(binrecord, textrecord)
+            if rolling_type:
+                flock(self.textfile, LOCK_UN)
+                return self.doRollover(rolling_type)
+            return None
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
+
 
     def emit(self, binrecord, textrecord):
-        if self.textfile is None:
-            self.textfile = open(self.textpath, mode='ab')
-        if self.binfile is None:
-            self.binfile = open(self.binpath, mode='ab')
-        self.textfile.write(textrecord)
-        self.binfile.write(binrecord)
-        self.textfile.flush()
-        self.binfile.flush()
+        global logfull
+        try:
+            if self.textfile is None:
+                self.textfile = open(self.textpath, mode='ab')
+            if self.binfile is None:
+                self.binfile = open(self.binpath, mode='ab')
+            self.textfile.write(textrecord)
+            self.binfile.write(binrecord)
+            self.textfile.flush()
+            self.binfile.flush()
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
 
     def get_textfile_offset(self, data_len):
         if self.textfile is None:
@@ -561,28 +580,35 @@ class Logger(object):
                 textdate = time.strftime(
                     '%b %d %H:%M:%S ', time.localtime(tstamp))
             flock(textfile, LOCK_EX)
-            offset = textfile.tell() + len(textdate)
-            datalen = len(data)
-            eventaux = entry[4]
-            if eventaux is None:
-                eventaux = 0
-            # metadata length is always 16 for this code at the moment
-            binrecord = struct.pack(
-                ">BBIHIBBH", 16, ltype, offset, datalen, tstamp, evtdata,
-                eventaux, 0)
-            if self.isconsole:
-                if ltype == 2:
-                    textrecord = data
+            try:
+                offset = textfile.tell() + len(textdate)
+                datalen = len(data)
+                eventaux = entry[4]
+                if eventaux is None:
+                    eventaux = 0
+                # metadata length is always 16 for this code at the moment
+                binrecord = struct.pack(
+                    ">BBIHIBBH", 16, ltype, offset, datalen, tstamp, evtdata,
+                    eventaux, 0)
+                if self.isconsole:
+                    if ltype == 2:
+                        textrecord = data
+                    else:
+                        textrecord = textdate + data + ']'
                 else:
-                    textrecord = textdate + data + ']'
-            else:
-                textrecord = textdate + data
-                if not textrecord.endswith('\n'):
-                    textrecord += '\n'
-            files = self.handler.try_emit(binrecord, textrecord)
+                    textrecord = textdate + data
+                    if not textrecord.endswith('\n'):
+                        textrecord += '\n'
+                files = self.handler.try_emit(binrecord, textrecord)
+            except struct.error:
+                files = self.handler.doRollover(RollingTypes.size_rolling)
+            finally:
+                try:
+                    flock(textfile, LOCK_UN)
+                except Exception:
+                    pass
             if not files:
                 self.handler.emit(binrecord, textrecord)
-                flock(textfile, LOCK_UN)
             else:
                 # Log the rolling event at first, then log the last data
                 # which cause the rolling event.
@@ -753,11 +779,13 @@ globaleventlog = None
 tracelog = None
 
 
-def log(logdata=None, ltype=None, event=0, eventdata=None):
+def log(logdata=None, ltype=None, event=0, eventdata=None, flush=False):
     global globaleventlog
     if globaleventlog is None:
         globaleventlog = Logger('events')
     globaleventlog.log(logdata, ltype, event, eventdata)
+    if flush:
+        globaleventlog.writedata()
 
 def logtrace():
     global tracelog

confluent_server/confluent/main.py

@@ -72,7 +72,7 @@ def _daemonize():
     os.setsid()
     thispid = os.fork()
     if thispid > 0:
-        print 'confluent server starting as pid %d' % thispid
+        print('confluent server starting as pid {0}'.format(thispid))
         os._exit(0)
     os.closerange(0, 2)
     os.umask(63)
@@ -81,6 +81,7 @@ def _daemonize():
     os.dup2(0, 2)
     sys.stdout = log.Logger('stdout', buffered=False)
     sys.stderr = log.Logger('stderr', buffered=False)
+    log.daemonized = True
 
 
 def _updatepidfile():
@@ -224,6 +225,11 @@ def run():
     except:
         doexit()
         raise
+    try:
+        log.log({'info': 'Confluent management service starting'}, flush=True)
+    except (OSError, IOError) as e:
+        print(repr(e))
+        sys.exit(1)
     _daemonize()
     if havefcntl:
         _updatepidfile()
@@ -231,7 +237,7 @@ def run():
     signal.signal(signal.SIGTERM, terminate)
     collective.startup()
     if dbgif:
-        oumask = os.umask(0077)
+        oumask = os.umask(0o077)
         try:
             os.remove('/var/run/confluent/dbg.sock')
         except OSError:

confluent_server/confluent/messages.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2017 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@
 # format.  This is also how different data formats are supported
 import confluent.exceptions as exc
 import confluent.config.configmanager as cfm
+import confluent.config.conf as cfgfile
 from copy import deepcopy
 from datetime import datetime
 import json
@@ -32,6 +33,17 @@ valid_health_values = set([
     'unknown',
 ])
 
+passcomplexity = cfgfile.get_option('policy', 'passwordcomplexity')
+passminlength = cfgfile.get_option('policy', 'passwordminlength')
+if passminlength:
+    passminlength = int(passminlength)
+else:
+    passminlength = 0
+if passcomplexity:
+    passcomplexity = int(passcomplexity)
+else:
+    passcomplexity = 0
+
 def simplify_name(name):
     return name.lower().replace(' ', '_').replace('/', '-').replace(
         '_-_', '-')
@@ -267,6 +279,24 @@ class CreatedResource(ConfluentMessage):
         pass
 
 
+class RenamedResource(ConfluentMessage):
+    notnode = True
+    readonly = True
+
+    def __init__(self, oldname, newname):
+        self.kvpairs = {'oldname': oldname, 'newname': newname}
+
+    def strip_node(self, node):
+        pass
+
+
+class RenamedNode(ConfluentMessage):
+    def __init__(self, name, rename):
+        self.desc = 'New Name'
+        kv = {'rename': {'value': rename}}
+        self.kvpairs = {name: kv}
+
+
 class AssignedResource(ConfluentMessage):
     notnode = True
     readonly = True
@@ -381,7 +411,9 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputReseatMessage(path, nodes, inputdata)
     elif path == ['attributes', 'expression']:
         return InputExpression(path, inputdata, nodes)
-    elif path[0] in ('attributes', 'users') and operation != 'retrieve':
+    elif path == ['attributes', 'rename']:
+        return InputConfigChangeSet(path, inputdata, nodes, configmanager)
+    elif path[0] in ('attributes', 'users', 'usergroups') and operation != 'retrieve':
         return InputAttributes(path, inputdata, nodes)
     elif path == ['boot', 'nextdevice'] and operation != 'retrieve':
         return InputBootDevice(path, nodes, inputdata)
@@ -438,6 +470,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputMedia(path, nodes, inputdata, configmanager)
     elif '/'.join(path).startswith('support/servicedata') and inputdata:
         return InputMedia(path, nodes, inputdata, configmanager)
+    elif '/'.join(path).startswith('configuration/management_controller/save_licenses') and inputdata:
+        return InputMedia(path, nodes, inputdata, configmanager)
     elif '/'.join(path).startswith(
             'configuration/management_controller/licenses') and inputdata:
         return InputLicense(path, nodes, inputdata, configmanager)
@@ -448,7 +482,7 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
 class InputFirmwareUpdate(ConfluentMessage):
 
     def __init__(self, path, nodes, inputdata, configmanager):
-        self._filename = inputdata.get('filename', inputdata.get('url', None))
+        self._filename = inputdata.get('filename', inputdata.get('url', inputdata.get('dirname', None)))
         self.bank = inputdata.get('bank', None)
         self.nodes = nodes
         self.filebynode = {}
@@ -490,6 +524,10 @@ class Media(ConfluentMessage):
     def __init__(self, node, media):
         self.kvpairs = {node: {'name': media.name, 'url': media.url}}
 
+class SavedFile(ConfluentMessage):
+    def __init__(self, node, file):
+        self.kvpairs = {node: {'filename': file}}
+
 class InputAlertData(ConfluentMessage):
 
     def __init__(self, path, inputdata, nodes=None):
@@ -539,8 +577,6 @@ class InputConfigClear(ConfluentMessage):
             raise exc.InvalidArgumentException('Input must be {"clear":true}')
 
 class InputConfigChangeSet(InputExpression):
-    # For now, this is identical to InputExpression, later it may
-    # internalize formula expansion, but not now..
     def __init__(self, path, inputdata, nodes=None, configmanager=None):
         self.cfm = configmanager
         super(InputConfigChangeSet, self).__init__(path, inputdata, nodes)
@@ -598,7 +634,7 @@ class InputAttributes(ConfluentMessage):
         for node in nodes:
             self.nodeattribs[node] = inputdata
 
-    def get_attributes(self, node):
+    def get_attributes(self, node, validattrs=None):
         if node not in self.nodeattribs:
             return {}
         nodeattr = deepcopy(self.nodeattribs[node])
@@ -613,15 +649,82 @@ class InputAttributes(ConfluentMessage):
                     # an expression string will error if format() done
                     # use that as cue to put it into config as an expr
                     nodeattr[attr] = {'expression': nodeattr[attr]}
+            if validattrs and 'validvalues' in validattrs.get(attr, []):
+                if (nodeattr[attr] and
+                        nodeattr[attr] not in validattrs[attr]['validvalues']):
+                    raise exc.InvalidArgumentException(
+                        'Attribute {0} does not accept value {1} (valid values would be {2})'.format(
+                            attr, nodeattr[attr], ','.join(validattrs[attr]['validvalues'])))
+            elif validattrs and 'validlist' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if v and v not in validattrs[attr]['validlist']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept list member '
+                            '{1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlist'])))
+            elif validattrs and 'validlistkeys' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if '=' not in v:
+                        raise exc.InvalidArgumentException(
+                            'Passed key {0} requires a parameter'.format(v))
+                    v = v.split('=', 1)[0]
+                    if v and v not in validattrs[attr]['validlistkeys']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept key {1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlistkeys'])
+                            )
+                        )
         return nodeattr
 
+def checkPassword(password, username):
+    lowercase = set('abcdefghijklmnopqrstuvwxyz')
+    uppercase = set('abcdefghijklmnopqrstuvwxyz'.upper())
+    numbers = set('0123456789')
+    special = set('`~!@#$%^&*()-_=+[{]};:"/?.>,<' + "'")
+    if len(password) < passminlength:
+        raise exc.InvalidArgumentException('Password must be at least {0} characters long'.format(passminlength))
+    if not isinstance(passcomplexity, int) or passcomplexity < 1:
+        return
+    if not bool(set(password.lower()) & lowercase):  # rule 1
+        raise exc.InvalidArgumentException('Password must contain at least one letter')
+    if passcomplexity < 2:
+        return
+    thepass = set(password)
+    if not bool(thepass & numbers):  # rule 2
+        raise exc.InvalidArgumentException('Password must contain at least one number')
+    if passcomplexity < 3:
+        return
+    classes = 0
+    for charclass in (lowercase, uppercase, special):
+        if bool(thepass & charclass):
+            classes += 1
+    if classes < 2:
+        raise exc.InvalidArgumentException('Password must contain at least two of upper case letter, lower case letter, and/or special character')
+    if passcomplexity < 4:
+        return
+    if username and password in (username, username[::-1]): # rule 4
+        raise exc.InvalidArgumentException('Password must not be similar to username')
+    if passcomplexity < 5:
+        return
+    for char in thepass:
+        if char * 3 in password:
+            raise exc.InvalidArgumentException('Password must not contain any of the same character repeated 3 times')
+
+
 
 class InputCredential(ConfluentMessage):
     valid_privilege_levels = set([
         'callback',
         'user',
+        'ReadOnly',
         'operator',
+        'Operator',
         'administrator',
+        'Administrator',
         'proprietary',
         'no_access',
     ])
@@ -639,33 +742,21 @@ class InputCredential(ConfluentMessage):
         if len(path) == 4:
             inputdata['uid'] = path[-1]
         # if the operation is 'create' check if all fields are present
-        missingattrs = []
-        for attrname in ('uid', 'privilege_level', 'username', 'password'):
-            if attrname not in inputdata:
-                missingattrs.append(attrname)
-        if missingattrs:
-            raise exc.InvalidArgumentException(
-                'Required fields missing: {0}'.format(','.join(missingattrs)))
         if (isinstance(inputdata['uid'], str) and
                 not inputdata['uid'].isdigit()):
-            raise exc.InvalidArgumentException('uid must be a number')
+            inputdata['uid'] = inputdata['uid']
         else:
             inputdata['uid'] = int(inputdata['uid'])
         if ('privilege_level' in inputdata and
               inputdata['privilege_level'] not in self.valid_privilege_levels):
             raise exc.InvalidArgumentException('privilege_level is not one of '
                                         + ','.join(self.valid_privilege_levels))
-        if 'username' in inputdata and len(inputdata['username']) > 16:
-            raise exc.InvalidArgumentException(
-                                        'name must be less than or = 16 chars')
-        if 'password' in inputdata and len(inputdata['password']) > 20:
-            raise exc.InvalidArgumentException('password has limit of 20 chars')
-
         if ('enabled' in inputdata and
             inputdata['enabled'] not in self.valid_enabled_values):
             raise exc.InvalidArgumentException('valid values for enabled are '
                                                             + 'yes and no')
-
+        if 'password' in inputdata and (passcomplexity or passminlength):
+            checkPassword(inputdata['password'], inputdata.get('username', None))
         if nodes is None:
             raise exc.InvalidArgumentException(
                 'This only supports per-node input')
@@ -762,6 +853,9 @@ class InputVolumes(ConfluentInputMessage):
             sizes = inputdata.get('size', [None])
             if not isinstance(sizes, list):
                 sizes = sizes.split(',')
+            stripsizes = inputdata.get('stripsizes', [None])
+            if not isinstance(stripsizes, list):
+                stripsizes = stripsizes.split(',')
             disks = inputdata.get('disks', [])
             if not disks:
                 raise exc.InvalidArgumentException(
@@ -773,8 +867,15 @@ class InputVolumes(ConfluentInputMessage):
                     currname = volnames.pop(0)
                 else:
                     currname = None
+                if stripsizes:
+                    currstripsize = stripsizes.pop(0)
+                    if currstripsize:
+                        currstripsize = int(currstripsize)
+                else:
+                    currstripsize = None
                 inputdata.append(
                     {'name': currname, 'size': size,
+                     'stripsize': currstripsize,
                      'disks': disks,
                      'raidlevel': raidlvl})
         for node in nodes:
@@ -796,6 +897,7 @@ class InputVolumes(ConfluentInputMessage):
                 self.inputbynode[node].append({'name': volname,
                                                'size': volsize,
                                                'disks': disks,
+                                               'stripsize': input.get('stripsize', None),
                                                'raidlevel': raidlvl,
                                                })
 
@@ -1145,7 +1247,9 @@ class EventCollection(ConfluentMessage):
                 'event': event.get('event', None),
                 'severity': event['severity'],
                 'timestamp': event.get('timestamp', None),
+                'message': event.get('message', None),
                 'record_id': event.get('record_id', None),
+                'log_id': event.get('log_id', None),
             }
             if event['severity'] not in valid_health_values:
                 raise exc.NotImplementedException(
@@ -1199,14 +1303,15 @@ class AsyncSession(ConfluentMessage):
         self.kvpairs = {'asyncid': id}
 
 class User(ConfluentMessage):
-    def __init__(self, uid, username, privilege_level, name=None):
+    def __init__(self, uid, username, privilege_level, name=None, expiration=None):
         self.desc = 'foo'
         self.stripped = False
         self.notnode = name is None
         kvpairs = {'username': {'value': username},
                    'password': {'value': '', 'type': 'password'},
                    'privilege_level': {'value': privilege_level},
-                   'enabled': {'value': ''}
+                   'enabled': {'value': ''},
+                   'expiration': {'value': expiration},
                    }
         if self.notnode:
             self.kvpairs = kvpairs
@@ -1225,6 +1330,7 @@ class UserCollection(ConfluentMessage):
             entry = {
                 'uid': user['uid'],
                 'username': user['name'],
+                'expiration': user.get('expiration', None),
                 'privilege_level': user['access']['privilege_level']
             }
             userlist.append(entry)
@@ -1356,12 +1462,13 @@ class Array(ConfluentMessage):
         }
 
 class Volume(ConfluentMessage):
-    def __init__(self, name, volname, size, state, array):
+    def __init__(self, name, volname, size, state, array, stripsize=None):
         self.kvpairs = {
             name: {
                 'type': 'volume',
                 'name': simplify_name(volname),
                 'label': volname,
+                'stripsize': stripsize,
                 'size': size,
                 'state': state,
                 'array': array,
@@ -1378,6 +1485,7 @@ class Disk(ConfluentMessage):
     state_aliases = {
         'unconfigured good': 'unconfigured',
         'global hot spare': 'hotspare',
+        'dedicated hot spare': 'hotspare',
     }
 
     def _normalize_state(self, instate):
@@ -1555,12 +1663,12 @@ class NTPServer(ConfluentMessage):
 class License(ConfluentMessage):
     readonly = True
 
-    def __init__(self, name=None, kvm=None, feature=None):
+    def __init__(self, name=None, kvm=None, feature=None, state=None):
         self.notnode = name is None
         self.desc = 'License'
 
         kv = []
-        kv.append({'kvm_availability': str(kvm), 'feature': feature})
+        kv.append({'kvm_availability': str(kvm), 'feature': feature, 'state': state})
         if self.notnode:
             self.kvpairs = {'License': kv}
         else:

confluent_server/confluent/netutil.py

@@ -24,6 +24,19 @@ import eventlet.support.greendns
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
 
+def mask_to_cidr(mask):
+    maskn = socket.inet_pton(socket.AF_INET, mask)
+    maskn = struct.unpack('!I', maskn)[0]
+    cidr = 32
+    while maskn & 0b1 == 0 and cidr > 0:
+        cidr -= 1
+        maskn >>= 1
+    return cidr
+
+def cidr_to_mask(cidr):
+    return socket.inet_ntop(
+        socket.AF_INET, struct.pack('!I', (2**32 - 1) ^ (2**(32 - cidr) - 1)))
+
 def ip_on_same_subnet(first, second, prefix):
     addrinf = socket.getaddrinfo(first, None, 0, socket.SOCK_STREAM)[0]
     fam = addrinf[0]

confluent_server/confluent/networking/lldp.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016, 2017 Lenovo
+# Copyright 2016-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -105,7 +105,11 @@ def close_enough(fuzz, literal):
     if fuzz == literal:
         return True
     fuzz = '^' + fuzz.replace('-', '[/: -]') + '$'
-    matcher = re.compile(fuzz)
+    try:
+        matcher = re.compile(fuzz)
+    except Exception:
+        raise exc.InvalidArgumentException(
+            'Invalid regular expression specified')
     return bool(matcher.match(literal))
 
 

confluent_server/confluent/networking/macmap.py

@@ -1,6 +1,6 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016-2017 Lenovo
+# Copyright 2016-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -315,10 +315,12 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
         switches = set([])
         for node in nodelocations:
             cfg = nodelocations[node]
+            if cfg.get('type', {}).get('value', None) == 'switch':
+                switches.add(node)
             for attr in cfg:
                 if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                     continue
@@ -342,7 +344,7 @@ def _full_updatemacmap(configmanager):
                         _switchportmap[curswitch][portname] = None
                     else:
                         _switchportmap[curswitch][portname] = node
-        for switch in _macsbyswitch:
+        for switch in list(_macsbyswitch):
             if switch not in switches:
                 del _macsbyswitch[switch]
         switchauth = get_switchcreds(configmanager, switches)
@@ -378,7 +380,7 @@ def handle_api_request(configmanager, inputdata, operation, pathcomponents):
     if (operation in ('update', 'create') and
             pathcomponents == ['networking', 'macs', 'rescan']):
         if inputdata != {'rescan': 'start'}:
-            raise exc.InvalidArgumentException()
+            raise exc.InvalidArgumentException('Input must be rescan=start')
         eventlet.spawn_n(rescan, configmanager)
         return [msg.KeyValueData({'rescan': 'started'})]
     raise exc.NotImplementedException(
@@ -456,9 +458,21 @@ def handle_read_api_request(pathcomponents, configmanager):
                     portname = portname.replace('-', '/')
                 maclist = _macsbyswitch[switchname][portname]
             except KeyError:
-                raise exc.NotFoundException('No known macs for switch {0} '
-                                            'port {1}'.format(switchname,
-                                                              portname))
+                foundsomemacs = False
+                if switchname in _macsbyswitch:
+                    try:
+                        matcher = re.compile(portname)
+                    except Exception:
+                        raise exc.InvalidArgumentException('Invalid regular expression specified')
+                    maclist = []
+                    for actualport in _macsbyswitch[switchname]:
+                        if bool(matcher.match(actualport)):
+                            foundsomemacs = True
+                            maclist = maclist + _macsbyswitch[switchname][actualport]
+                if not foundsomemacs:
+                    raise exc.NotFoundException('No known macs for switch {0} '
+                                                'port {1}'.format(switchname,
+                                                                  portname))
             return [msg.ChildCollection(x.replace(':', '-'))
                     for x in sorted(maclist)]
         if len(pathcomponents) == 8:

confluent_server/confluent/networking/netutil.py

@@ -42,10 +42,12 @@ def get_switchcreds(configmanager, switches):
 
 def list_switches(configmanager):
     nodelocations = configmanager.get_node_attributes(
-        configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+        configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
     switches = set([])
     for node in nodelocations:
         cfg = nodelocations[node]
+        if cfg.get('type', {}).get('value', None) == 'switch':
+            switches.add(node)
         for attr in cfg:
             if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                 continue

confluent_server/confluent/plugins/configuration/attributes.py

@@ -1,5 +1,5 @@
 # Copyright 2014 IBM Corporation
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ import confluent.exceptions as exc
 import confluent.messages as msg
 import confluent.config.attributes as allattributes
 import confluent.util as util
+from fnmatch import fnmatch
 
 
 def retrieve(nodes, element, configmanager, inputdata):
@@ -39,6 +40,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
     if element == 'all':
         theattrs = set(allattributes.node).union(set(grpcfg))
         theattrs.add('nodes')
+        theattrs.add('noderange')
         for attribute in sorted(theattrs):
             if attribute == 'groups':
                 continue
@@ -51,6 +53,10 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                 val = grpcfg[attribute]
             else:
                 val = {'value': None}
+            if attribute == 'noderange':
+                val['desc'] = 'The noderange this group is expanded ' \
+                    'to when used in noderange, exclusive with static ' \
+                    'nodes'
             if attribute.startswith('secret.'):
                 yield msg.CryptedAttributes(
                     kv={attribute: val},
@@ -69,6 +75,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
         for attribute in sorted(list(grpcfg)):
             currattr = grpcfg[attribute]
             if attribute == 'nodes':
+                if not currattr:
+                    continue
                 desc = 'The nodes belonging to this group'
             elif attribute == 'noderange':
                 desc = 'A dynamic noderange that this group refers to in noderange expansion'
@@ -92,8 +100,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                     kv={attribute: currattr},
                     desc=desc)
             else:
-                print attribute
-                print repr(currattr)
+                print(attribute)
+                print(repr(currattr))
                 raise Exception("BUGGY ATTRIBUTE FOR NODEGROUP")
 
 
@@ -141,8 +149,8 @@ def retrieve_nodes(nodes, element, configmanager, inputdata):
                     yield msg.ListAttributes(
                         node, {attribute: currattr}, desc)
                 else:
-                    print attribute
-                    print repr(currattr)
+                    print(attribute)
+                    print(repr(currattr))
                     raise Exception("BUGGY ATTRIBUTE FOR NODE")
 
 
@@ -156,6 +164,11 @@ def update(nodes, element, configmanager, inputdata):
 
 
 def update_nodegroup(group, element, configmanager, inputdata):
+    if 'rename' in element:
+        namemap = {}
+        namemap[group] = inputdata.attribs['rename']
+        configmanager.rename_nodegroups(namemap)
+        return yield_rename_resources(namemap, isnode=False)
     try:
         clearattribs = []
         for attrib in inputdata.attribs.iterkeys():
@@ -195,18 +208,44 @@ def create(nodes, element, configmanager, inputdata):
     if nodes is not None and element[-1] == 'expression':
         return _expand_expression(nodes, configmanager, inputdata)
 
+def yield_rename_resources(namemap, isnode):
+    for node in namemap:
+        if isnode:
+            yield msg.RenamedNode(node, namemap[node])
+        else:
+            yield msg.RenamedResource(node, namemap[node])
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
+    if not nodes:
+        raise exc.InvalidArgumentException(
+            'No action to take, noderange is empty (if trying to define '
+            'group attributes, use nodegroupattrib)')
+    if 'rename' in element:
+        namemap = {}
+        for node in nodes:
+            rename = inputdata.get_attributes(node)
+            namemap[node] = rename['rename']
+        configmanager.rename_nodes(namemap)
+        return yield_rename_resources(namemap, isnode=True)
     for node in nodes:
-        updatenode = inputdata.get_attributes(node)
+        updatenode = inputdata.get_attributes(node, allattributes.node)
         clearattribs = []
         if updatenode:
-            for attrib in updatenode.iterkeys():
+            for attrib in list(updatenode):
                 if updatenode[attrib] is None:
-                    clearattribs.append(attrib)
-            if len(clearattribs) > 0:
-                for attrib in clearattribs:
                     del updatenode[attrib]
+                    if attrib in allattributes.node  or attrib.startswith('custom.') or attrib.startswith('net.'):
+                        clearattribs.append(attrib)
+                    else:
+                        foundattrib = False
+                        for candattrib in allattributes.node:
+                            if fnmatch(candattrib, attrib):
+                                clearattribs.append(candattrib)
+                                foundattrib = True
+                        if not foundattrib:
+                            raise exc.InvalidArgumentException("No attribute matches '" + attrib + "' (try wildcard if trying to clear a group)")
+            if len(clearattribs) > 0:
                 configmanager.clear_node_attributes([node], clearattribs)
             updatedict[node] = updatenode
     try:

confluent_server/confluent/plugins/hardwaremanagement/cnos.py

@@ -0,0 +1,147 @@
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#Noncritical:
+#  - One or more temperature sensors is in the warning range;
+#  - A panic dump exists in flash.
+#Critical:
+#  - One or more temperature sensors is in the failure range;
+#  - One or more fans are running < 100 RPM;
+#  - One power supply is off.
+
+
+import eventlet
+import eventlet.queue as queue
+import confluent.exceptions as exc
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import confluent.messages as msg
+import confluent.util as util
+
+class SwitchSensor(object):
+    def __init__(self, name, states, value=None, health=None):
+        self.name = name
+        self.value = value
+        self.states = states
+        self.health = health
+
+
+def cnos_login(node, configmanager, creds):
+    wc = webclient.SecureHTTPConnection(node, port=443, verifycallback=util.TLSCertVerifier(
+        configmanager, node, 'pubkeys.tls_hardwaremanager').verify_cert)
+    wc.set_basic_credentials(creds[node]['secret.hardwaremanagementuser']['value'], creds[node]['secret.hardwaremanagementpassword']['value'])
+    wc.request('GET', '/nos/api/login/')
+    rsp = wc.getresponse()
+    body = rsp.read()
+    if rsp.status == 401:  # CNOS gives 401 on first attempt...
+        wc.request('GET', '/nos/api/login/')
+        rsp = wc.getresponse()
+        body = rsp.read()
+    if rsp.status >= 200 and rsp.status < 300:
+        return wc
+    raise exc.TargetEndpointBadCredentials('Unable to authenticate')
+
+def update(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def delete(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def create(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def retrieve(nodes, element, configmanager, inputdata):
+    results = queue.LightQueue()
+    workers = set([])
+    if element == ['power', 'state']:
+        for node in nodes:
+            yield msg.PowerState(node=node, state='on')
+        return
+    elif element == ['health', 'hardware']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_health, configmanager, creds,
+                                       node, results))
+    else:
+        for node in nodes:
+            yield msg.ConfluentNodeError(node, 'Not Implemented')
+        return
+    currtimeout = 10
+    while workers:
+        try:
+            datum = results.get(10)
+            while datum:
+                if datum:
+                    yield datum
+                datum = results.get_nowait()
+        except queue.Empty:
+            pass
+        eventlet.sleep(0.001)
+        for t in list(workers):
+            if t.dead:
+                workers.discard(t)
+    try:
+        while True:
+            datum = results.get_nowait()
+            if datum:
+                yield datum
+    except queue.Empty:
+        pass
+
+
+def retrieve_health(configmanager, creds, node, results):
+    wc = cnos_login(node, configmanager, creds)
+    hinfo = wc.grab_json_response('/nos/api/sysinfo/globalhealthstatus')
+    summary = hinfo['status'].lower()
+    if summary == 'noncritical':
+        summary = 'warning'
+    results.put(msg.HealthSummary(summary, name=node))
+    state = None
+    badreadings = []
+    if summary != 'ok':  # temperature or dump or fans or psu
+        wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        if switchinfo:
+            badreadings.append(
+                SwitchSensor('Panicdump', ['Present'], health='warning'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/temperatures')
+        for temp in switchinfo:
+            if temp == 'Temperature threshold':
+                continue
+            if switchinfo[temp]['State'] != 'OK':
+                temphealth = switchinfo[temp]['State'].lower()
+                if temphealth == 'noncritical':
+                    temphealth = 'warning'
+                tempval = switchinfo[temp]['Temp']
+                badreadings.append(
+                    SwitchSensor(temp, [], value=tempval, health=temphealth))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/fans')
+        for fan in switchinfo:
+            if switchinfo[fan]['speed-rpm'] < 100:
+                badreadings.append(
+                    SwitchSensor(fan, [], value=switchinfo[fan]['speed-rpm'],
+                                 health='critical'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/power')
+        for psu in switchinfo:
+            if switchinfo[psu]['State'] != 'Normal ON':
+                psuname = switchinfo[psu]['Name']
+                badreadings.append(
+                    SwitchSensor(psuname, states=[switchinfo[psu]['State']],
+                                 health='critical'))
+    results.put(msg.SensorReadings(badreadings, name=node))

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -36,6 +36,11 @@ console = eventlet.import_patched('pyghmi.ipmi.console')
 ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
 import socket
 import ssl
+import traceback
+
+
+if not hasattr(ssl, 'SSLEOFError'):
+    ssl.SSLEOFError = None
 
 pci_cache = {}
 
@@ -166,8 +171,10 @@ class IpmiCommandWrapper(ipmicommand.Command):
     def __init__(self, node, cfm, **kwargs):
         self.cfm = cfm
         self.node = node
+        self.sensormap = {}
         self._inhealth = False
         self._lasthealth = None
+        kwargs['keepalive'] = False
         self._attribwatcher = cfm.watch_attributes(
             (node,), ('secret.hardwaremanagementuser', 'collective.manager',
                       'secret.hardwaremanagementpassword', 'secret.ipmikg',
@@ -226,7 +233,6 @@ def _ipmi_evtloop():
                 waiter = _ipmiwaiters.pop()
                 waiter.send()
         except:  # TODO(jbjohnso): log the trace into the log
-            import traceback
 
             traceback.print_exc()
 
@@ -428,8 +434,8 @@ def perform_request(operator, node, element,
         except pygexc.InvalidParameterValue as e:
             results.put(msg.ConfluentNodeError(node, str(e)))
         except Exception as e:
-            results.put(e)
-            raise
+            results.put(msg.ConfluentNodeError(node, 'Unexpected Error: {0}'.format(str(e))))
+            traceback.print_exc()
         finally:
             results.put('Done')
 
@@ -438,7 +444,6 @@ persistent_ipmicmds = {}
 class IpmiHandler(object):
     def __init__(self, operation, node, element, cfd, inputdata, cfg, output,
                  realop):
-        self.sensormap = {}
         self.invmap = {}
         self.output = output
         self.sensorcategory = None
@@ -457,11 +462,12 @@ class IpmiHandler(object):
         self.inputdata = inputdata
         self.tenant = cfg.tenant
         tenant = cfg.tenant
-        if ((node, tenant) not in persistent_ipmicmds or
+        while ((node, tenant) not in persistent_ipmicmds or
                 not persistent_ipmicmds[(node, tenant)].ipmi_session.logged or
                 persistent_ipmicmds[(node, tenant)].ipmi_session.broken):
             try:
                 persistent_ipmicmds[(node, tenant)].close_confluent()
+                persistent_ipmicmds[(node, tenant)].ipmi_session._mark_broken()
             except KeyError:  # was no previous session
                 pass
             try:
@@ -473,13 +479,17 @@ class IpmiHandler(object):
 
                 ipmisess = persistent_ipmicmds[(node, tenant)].ipmi_session
                 begin = util.monotonic_time()
-                while ((not (self.broken or self.loggedin)) and
+                while ((not (ipmisess.broken or self.loggedin)) and
                                (util.monotonic_time() - begin) < 30):
                     ipmisess.wait_for_rsp(31 - (util.monotonic_time() - begin))
-                if not (self.broken or self.loggedin):
-                    ipmisess._mark_broken()
-                    raise exc.TargetEndpointUnreachable(
-                        "Login process to " + connparams['bmc'] + " died")
+                if self.broken or self.loggedin:
+                    break
+                cfd = cfg.get_node_attributes(node, _configattributes, decrypt=True)
+                self.cfg = cfd[node]
+                connparams = get_conn_params(node, self.cfg)
+                ipmisess._mark_broken()
+                #    raise exc.TargetEndpointUnreachable(
+                #        "Login process to " + connparams['bmc'] + " died")
             except socket.gaierror as ge:
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
@@ -621,6 +631,8 @@ class IpmiHandler(object):
             return self.handle_sysconfigclear()
         elif self.element[1:3] == ['management_controller', 'licenses']:
             return self.handle_licenses()
+        elif self.element[1:3] == ['management_controller', 'save_licenses']:
+            return self.save_licenses()
         raise Exception('Not implemented')
 
     def decode_alert(self):
@@ -737,6 +749,7 @@ class IpmiHandler(object):
                     uid=data['uid'],
                     username=data['name'],
                     privilege_level=data['access']['privilege_level'],
+                    expiration=data['expiration'],
                     name=self.node))
                 return
             elif self.op == 'update':
@@ -745,14 +758,16 @@ class IpmiHandler(object):
                 if 'username' in user:
                     self.ipmicmd.set_user_name(uid=user['uid'],
                                                name=user['username'])
-                if 'privilege_level' in user:
-                    self.ipmicmd.set_user_access(uid=user['uid'],
-                                    privilege_level=user['privilege_level'])
+
                 if 'password' in user:
                     self.ipmicmd.set_user_password(uid=user['uid'],
                                                    password=user['password'])
                     self.ipmicmd.set_user_password(uid=user['uid'],
                                     mode='enable', password=user['password'])
+                    if 'privilege_level' in user:
+                        self.ipmicmd.set_user_access(uid=user['uid'],
+                                                     privilege_level=user[
+                                                         'privilege_level'])
                 if 'enabled' in user:
                     if user['enabled'] == 'yes':
                         mode = 'enable'
@@ -793,7 +808,7 @@ class IpmiHandler(object):
             sensors = self.ipmicmd.get_sensor_descriptions()
         for sensor in sensors:
             resourcename = sensor['name']
-            self.sensormap[simplify_name(resourcename)] = resourcename
+            self.ipmicmd.sensormap[simplify_name(resourcename)] = resourcename
 
     def read_sensors(self, sensorname):
         if sensorname == 'all':
@@ -814,15 +829,16 @@ class IpmiHandler(object):
                 readings.append(reading)
             self.output.put(msg.SensorReadings(readings, name=self.node))
         else:
-            self.make_sensor_map()
-            if sensorname not in self.sensormap:
+            if sensorname not in self.ipmicmd.sensormap:
+                self.make_sensor_map()
+            if sensorname not in self.ipmicmd.sensormap:
                 self.output.put(
                     msg.ConfluentTargetNotFound(self.node,
                                                 'Sensor not found'))
                 return
             try:
                 reading = self.ipmicmd.get_sensor_reading(
-                    self.sensormap[sensorname])
+                    self.ipmicmd.sensormap[sensorname])
                 if hasattr(reading, 'health'):
                     reading.health = _str_health(reading.health)
                 self.output.put(
@@ -871,6 +887,9 @@ class IpmiHandler(object):
                 'Extended information unavailable, mismatch detected between '
                 'target certificate fingerprint and '
                 'pubkeys.tls_hardwaremanager attribute')
+        except pygexc.TemporaryError as e:
+                errorneeded = msg.ConfluentNodeError(
+                self.node, str(e))
         self.output.put(msg.Firmware(items, self.node))
         if errorneeded:
             self.output.put(errorneeded)
@@ -950,8 +969,8 @@ class IpmiHandler(object):
         if newinf.get('information', None) and 'name' in newinf['information']:
             newinf = copy.deepcopy(newinf)
             del newinf['information']['name']
-        if fnmatch(newinf['name'], 'Adapter ??:??:??') or fnmatch(
-                newinf['name'], 'PCIeGen? x*'):
+        if (fnmatch(newinf['name'], 'Adapter ??:??:??') or fnmatch(
+                newinf['name'], 'PCIeGen? x*') or not newinf['name']):
             myinf = newinf.get('information', {})
             sdid = myinf.get('PCI Subsystem Device ID', None)
             svid = myinf.get('PCI Subsystem Vendor ID', None)
@@ -1028,7 +1047,7 @@ class IpmiHandler(object):
             if raidlvl and vol['raidlevel'] != raidlvl:
                 raise exc.InvalidArgumentException('Cannot mix raid levels in '
                                                    'a single array')
-            vols.append(storage.Volume(name=vol['name'], size=vol['size']))
+            vols.append(storage.Volume(name=vol['name'], size=vol['size'], stripsize=vol['stripsize']))
         newcfg = storage.ConfigSpec(
             arrays=(storage.Array(raid=raidlvl, disks=disks, volumes=vols),))
         self.ipmicmd.apply_storage_configuration(newcfg)
@@ -1116,6 +1135,11 @@ class IpmiHandler(object):
                     msg.Disk(self.node, disk.name, disk.description,
                              disk.id, disk.status, disk.serial,
                              disk.fru, array='{0}-{1}'.format(*arr.id)))
+            for disk in arr.hotspares:
+                self.output.put(
+                    msg.Disk(self.node, disk.name, disk.description,
+                             disk.id, disk.status, disk.serial,
+                             disk.fru, array='{0}-{1}'.format(*arr.id)))
         for arr in scfg.arrays:
             arrname = '{0}-{1}'.format(*arr.id)
             self._detail_array(arr, arrname, True)
@@ -1137,6 +1161,13 @@ class IpmiHandler(object):
                         msg.Disk(self.node, disk.name, disk.description,
                                  disk.id, disk.status, disk.serial,
                                  disk.fru, arrname))
+            for disk in arr.hotspares:
+                if (name == 'all' or simplify_name(disk.name) == name or
+                        disk == name):
+                    self.output.put(
+                        msg.Disk(self.node, disk.name, disk.description,
+                                 disk.id, disk.status, disk.serial,
+                                 disk.fru, arrname))
 
     def list_disks(self):
         scfg = self.ipmicmd.get_storage_configuration()
@@ -1145,6 +1176,8 @@ class IpmiHandler(object):
         for arr in scfg.arrays:
             for disk in arr.disks:
                 self.output.put(msg.ChildCollection(simplify_name(disk.name)))
+            for disk in arr.hotspares:
+                self.output.put(msg.ChildCollection(simplify_name(disk.name)))
 
     def list_arrays(self):
         scfg = self.ipmicmd.get_storage_configuration()
@@ -1165,6 +1198,8 @@ class IpmiHandler(object):
         disks = []
         for disk in arr.disks:
             disks.append(simplify_name(disk.name))
+        for disk in arr.hotspares:
+            disks.append(simplify_name(disk.name))
         self.output.put(msg.Array(self.node, disks, arr.raid,
                                   vols, arrname, arr.capacity,
                                   arr.available_capacity))
@@ -1343,9 +1378,13 @@ class IpmiHandler(object):
 
     def handle_sysconfig(self, advanced=False):
         if 'read' == self.op:
-            self.output.put(msg.ConfigSet(
-                self.node, self.ipmicmd.get_system_configuration(
-                    hideadvanced=not advanced)))
+            try:
+                self.output.put(msg.ConfigSet(
+                    self.node, self.ipmicmd.get_system_configuration(
+                        hideadvanced=not advanced)))
+            except Exception as e:
+                self.output.put(
+                    msg.ConfluentNodeError(self.node, str(e)))
         elif 'update' == self.op:
             self.ipmicmd.set_system_configuration(
                 self.inputdata.get_attributes(self.node))
@@ -1400,6 +1439,11 @@ class IpmiHandler(object):
         self.output.put(msg.License(self.node, available))
         return
 
+    def save_licenses(self):
+        directory = self.inputdata.nodefile(self.node)
+        for saved in self.ipmicmd.save_licenses(directory):
+            self.output.put(msg.SavedFile(self.node, saved))
+
     def handle_licenses(self):
         if self.element[-1] == '':
             self.element = self.element[:-1]
@@ -1415,12 +1459,17 @@ class IpmiHandler(object):
         licname = self.element[3]
         if licname == 'all':
             for lic in self.ipmicmd.get_licenses():
-                self.output.put(msg.License(self.node, feature=lic['name']))
+                if self.op == 'delete':
+                    self.ipmicmd.delete_license(lic['name'])
+                else:
+                    self.output.put(msg.License(self.node, feature=lic['name'], state=lic.get('state', 'Active')))
         else:
             index = int(licname)
             lic = list(self.ipmicmd.get_licenses())[index - 1]
-            self.output.put(msg.License(self.node, feature=lic['name']))
-
+            if self.op == 'delete':
+                self.ipmicmd.delete_license(lic['name'])
+            else:
+                self.output.put(msg.License(self.node, feature=lic['name'], state=lic.get('state', 'Active')))
     def handle_description(self):
         dsc = self.ipmicmd.get_description()
         self.output.put(msg.KeyValueData(dsc, self.node))

confluent_server/confluent/plugins/shell/ssh.py

@@ -22,7 +22,11 @@
 import confluent.exceptions as cexc
 import confluent.interface.console as conapi
 import confluent.log as log
-import cryptography
+try:
+    import cryptography
+except ImportError:
+    # Using older, non-crypography based paramiko
+    cryptography = None
 
 import eventlet
 import hashlib
@@ -30,7 +34,7 @@ import sys
 sys.modules['gssapi'] = None
 paramiko = eventlet.import_patched('paramiko')
 warnhostkey = False
-if cryptography.__version__.split('.') < ['1', '5']:
+if cryptography and cryptography.__version__.split('.') < ['1', '5']:
     # older cryptography with paramiko breaks most key support except
     # ed25519
     warnhostkey = True

confluent_server/confluent/snmputil.py

@@ -79,14 +79,16 @@ class Session(object):
         # overriden, but some devices only support DES)
         tp = _get_transport(self.server)
         ctx = snmp.ContextData(self.context)
+        resolvemib = False
         if '::' in oid:
+            resolvemib = True
             mib, field = oid.split('::')
             obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field))
         else:
             obj = snmp.ObjectType(snmp.ObjectIdentity(oid))
 
         walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
-                               lexicographicMode=False)
+                               lexicographicMode=False, lookupMib=resolvemib)
         try:
             for rsp in walking:
                 errstr, errnum, erridx, answers = rsp

confluent_server/confluent/sockapi.py

@@ -75,6 +75,14 @@ except ImportError:
 
 plainsocket = None
 
+def _should_authlog(path, operation):
+    if (operation == 'retrieve' and
+            ('/sensors/' in path or '/health/' in path or
+             '/power/state' in path or '/nodes/' == path or
+             (path.startswith('/noderange/') and path.endswith('/nodes/')))):
+        return False
+    return True
+
 class ClientConsole(object):
     def __init__(self, client):
         self.client = client
@@ -112,13 +120,15 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
         cfm = configmanager.ConfigManager(tenant=None, username=authname)
     elif authname:
         authdata = auth.authorize(authname, element=None)
-        if authdata is not None:
+        if authdata:
             cfm = authdata[1]
             authenticated = True
     send_data(connection, "Confluent -- v0 --")
     while not authenticated:  # prompt for name and passphrase
         send_data(connection, {'authpassed': 0})
         response = tlvdata.recv(connection)
+        if not response:
+            return
         if 'collective' in response:
             return collective.handle_connection(connection, cert,
                                            response['collective'])
@@ -135,7 +145,7 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
         # element path, that authorization will need to be called
         # per request the user makes
         authdata = auth.check_user_passphrase(authname, passphrase)
-        if authdata is None:
+        if not authdata:
             auditlog.log(
                 {'operation': 'connect', 'user': authname, 'allowed': False})
         else:
@@ -151,7 +161,9 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
                                          'python-pyopenssl installed or has an '
                                          'incorrect version installed '
                                          '(e.g. pyOpenSSL would need to be '
-                                         'replaced with python-pyopenssl)'}})
+                                         'replaced with python-pyopenssl). '
+                                         'Restart confluent after updating '
+                                         'the dependency.'}})
             return
         return collective.handle_connection(connection, None, request['collective'],
                                      local=True)
@@ -170,11 +182,11 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
             send_data(connection, {'_requestdone': 1})
         except SystemExit:
             sys.exit(0)
-        except:
+        except Exception as e:
             tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                          event=log.Events.stacktrace)
             send_data(connection, {'errorcode': 500,
-                                      'error': 'Unexpected error'})
+                                      'error': 'Unexpected error - ' + str(e)})
             send_data(connection, {'_requestdone': 1})
         request = tlvdata.recv(connection)
 
@@ -194,20 +206,21 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
     path = request['path']
     params = request.get('parameters', {})
     hdlr = None
+    auditmsg = {
+        'operation': operation,
+        'target': path,
+    }
     if not skipauth:
         authdata = auth.authorize(authdata[2], path, authdata[3], operation)
-        auditmsg = {
-            'operation': operation,
-            'target': path,
-        }
-        if authdata is None:
+        if not authdata:
             auditmsg['allowed'] = False
             auditlog.log(auditmsg)
             raise exc.ForbiddenRequest()
         auditmsg['user'] = authdata[2]
         if authdata[3] is not None:
             auditmsg['tenant'] = authdata[3]
-        auditmsg['allowed'] = True
+    auditmsg['allowed'] = True
+    if _should_authlog(path, operation):
         auditlog.log(auditmsg)
     try:
         if operation == 'start':
@@ -300,12 +313,12 @@ def term_interact(authdata, authname, ccons, cfm, connection, consession,
                 try:
                     process_request(connection, data, cfm, authdata, authname,
                                     skipauth)
-                except Exception:
+                except Exception as e:
                     tracelog.log(traceback.format_exc(),
                                  ltype=log.DataTypes.event,
                                  event=log.Events.stacktrace)
                     send_data(connection, {'errorcode': 500,
-                                           'error': 'Unexpected error'})
+                                           'error': 'Unexpected error - ' + str(e)})
                     send_data(connection, {'_requestdone': 1})
                 continue
         if not data:

confluent_server/confluent/userutil.py

@@ -0,0 +1,40 @@
+from ctypes import *
+from ctypes.util import find_library
+import grp
+import pwd
+import os
+libc = cdll.LoadLibrary(find_library('libc'))
+_getgrouplist = libc.getgrouplist
+_getgrouplist.restype = c_int32
+
+
+class TooSmallException(Exception):
+    def __init__(self, count):
+        self.count = count
+        super(TooSmallException, self).__init__()
+
+
+def getgrouplist(name, gid, ng=32):
+    _getgrouplist.argtypes = [c_char_p, c_uint, POINTER(c_uint * ng), POINTER(c_int)]
+    glist = (c_uint * ng)()
+    nglist = c_int(ng)
+    count = _getgrouplist(name, gid, byref(glist), byref(nglist))
+    if count < 0:
+        raise TooSmallException(nglist.value)
+    for gidx in range(count):
+        gent = glist[gidx]
+        yield grp.getgrgid(gent).gr_name
+
+
+def grouplist(username):
+    pent = pwd.getpwnam(username)
+    try:
+        groups = getgrouplist(pent.pw_name, pent.pw_gid)
+    except TooSmallException as e:
+        groups = getgrouplist(pent.pw_name, pent.pw_gid, e.count)
+    return list(groups)
+
+if __name__ == '__main__':
+    import sys
+    print(repr(grouplist(sys.argv[1])))
+

confluent_server/confluent_server.spec.tmpl

@@ -12,7 +12,7 @@ Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
 BuildArch: noarch
-Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client, python-pyparsing, python-paramiko, python-dns, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-pyte, python-lxml, python-eficompressor, python-setuptools
+Requires: python-pyghmi >= 1.0.34, python-eventlet, python-greenlet, python-pycryptodomex >= 3.4.7, confluent_client, python-pyparsing, python-paramiko, python-dns, python-netifaces, python2-pyasn1 >= 0.2.3, python-pysnmp >= 4.3.4, python-pyte, python-lxml, python-eficompressor, python-setuptools, python-dateutil
 Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
 Url: http://xcat.sf.net/
 

misc/autocons.c

@@ -0,0 +1,76 @@
+#include <termios.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define COM1 0x3f8
+#define COM2 0x2f8
+#define COM3 0x3e8
+#define COM4 0x2e8
+
+#define SPEEDNOOP 0
+#define SPEED9600 3
+#define SPEED19200 4
+#define SPEED57600 6
+#define SPEED115200 7
+
+int main(int argc, char* argv[]) {
+    struct termios tty;
+    int ttyf;
+    int spcr;
+    int currspeed;
+    speed_t cspeed;
+    char buff[128];
+    uint64_t address;
+    spcr = open("/sys/firmware/acpi/tables/SPCR", O_RDONLY);
+    if (spcr < 0) {
+        exit(0);
+    }
+    if (read(spcr, buff, 80) < 80) {
+        exit(0);
+    }
+    if (buff[8] != 2) exit(0); //revision 2
+    if (buff[36] != 0) exit(0); //16550 only
+    if (buff[40] != 1) exit(0); //IO only
+    address = *(uint64_t *)(buff + 44);
+    currspeed = buff[58];
+    if (address == COM1) {
+        strncpy(buff, "/dev/ttyS0", 128);
+    } else if (address == COM2) {
+        strncpy(buff, "/dev/ttyS1", 128);
+    } else if (address == COM3) {
+        strncpy(buff, "/dev/ttyS2", 128);
+    } else if (address == COM4) {
+        strncpy(buff, "/dev/ttyS3", 128);
+    } else {
+        exit(0);
+    }
+    if (currspeed == SPEED9600) {
+        cspeed = B9600;
+    } else if (currspeed == SPEED19200) {
+        cspeed = B19200;
+    } else if (currspeed == SPEED57600) {
+        cspeed = B57600;
+    } else if (currspeed == SPEED115200) {
+        cspeed = B115200;
+    } else if (currspeed == SPEED115200) {
+        cspeed = 0;
+    } else {
+        exit(0);
+    }
+    printf("%s\n", buff);
+    ttyf = open(buff, O_RDWR | O_NOCTTY);
+    tcgetattr(ttyf, &tty);
+    if (cspeed) {
+        cfsetospeed(&tty, B115200);
+        cfsetispeed(&tty, B115200);
+    }
+    tcsetattr(ttyf, TCSANOW, &tty);
+    ioctl(ttyf, TIOCCONS, 0);
+
+}
+

misc/autocons.py

@@ -0,0 +1,90 @@
+import fcntl
+import os
+import signal
+import struct
+import subprocess
+import termios
+import time
+
+addrtoname = {
+    0x3f8: '/dev/ttyS0',
+    0x2f8: '/dev/ttyS1',
+    0x3e8: '/dev/ttyS2',
+    0x2e8: '/dev/ttyS3',
+}
+speedmap = {
+    0: None,
+    3: 9600,
+    4: 19200,
+    6: 57600,
+    7: 115200,
+}
+
+termiobaud = {
+    9600: termios.B9600,
+    19200: termios.B19200,
+    57600: termios.B57600,
+    115200: termios.B115200,
+}
+
+def do_serial_config():
+    if 'console=ttyS' in open('/proc/cmdline').read():
+        return None # Do not do autoconsole if manually configured
+    spcr = open("/sys/firmware/acpi/tables/SPCR", "rb")
+    spcr = bytearray(spcr.read())
+    if spcr[8] != 2 or spcr[36] != 0 or spcr[40] != 1:
+        return None
+    address = struct.unpack('<Q', spcr[44:52])[0]
+    tty = None
+    try:
+        tty = addrtoname[address]
+    except KeyError:
+        return None
+    retval = { 'tty': tty }
+    try:
+        retval['speed'] = speedmap[spcr[58]]
+    except KeyError:
+        return None
+    if retval['speed']:
+        ttyf = os.open(tty, os.O_RDWR | os.O_NOCTTY)
+        currattr = termios.tcgetattr(ttyf)
+        currattr[4:6] = [0, termiobaud[retval['speed']]]
+        termios.tcsetattr(ttyf, termios.TCSANOW, currattr)
+    retval['connected'] = bool(struct.unpack('<I', fcntl.ioctl(
+        ttyf, termios.TIOCMGET, '\x00\x00\x00\x00'))[0] & termios.TIOCM_CAR)
+    os.close(ttyf)
+    return retval
+
+def is_connected(tty):
+    ttyf = os.open(tty, os.O_RDWR | os.O_NOCTTY)
+    retval = bool(struct.unpack('<I', fcntl.ioctl(
+        ttyf, termios.TIOCMGET, '\x00\x00\x00\x00'))[0] & termios.TIOCM_CAR)
+    os.close(ttyf)
+    return retval
+
+
+if __name__ == '__main__':
+    serialinfo = do_serial_config()
+    if serialinfo:
+        running = False
+        while True:
+            if running and running.poll() is not None:
+                running = False
+            if running and not is_connected(serialinfo['tty']):
+                try:
+                    running.terminate()
+                    running.wait()
+                except Exception:
+                    pass
+                time.sleep(0.5)
+                running = subprocess.Popen(['/bin/sh', '-c',  'exec screen -x console <> {0} >&0 2>&1'.format(serialinfo['tty'])])
+                time.sleep(0.5)
+                try:
+                    running.terminate()
+                    running.wait()
+                except Exception:
+                    pass
+                running = False
+            elif not running and is_connected(serialinfo['tty']):
+                running = subprocess.Popen(['/bin/sh', '-c',  'exec screen -x console <> {0} >&0 2>&1'.format(serialinfo['tty'])])
+            time.sleep(0.5)

misc/copernicus.c

@@ -0,0 +1,167 @@
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <ifaddrs.h>
+#include <net/if_arp.h>
+#include <linux/if_packet.h>
+#include <arpa/inet.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <net/if.h>
+
+int add_uuid(char* destination, int maxsize) {
+    int uuidf;
+    int uuidsize;
+    uuidf = open("/sys/devices/virtual/dmi/id/product_uuid", O_RDONLY);
+    if (uuidf < 1) { return 0; }
+    strncpy(destination, "/uuid=", maxsize);
+    uuidsize = read(uuidf, destination + 6, maxsize - 6);
+    close(uuidf);
+    if (destination[uuidsize + 5] == '\n') {
+        destination[uuidsize + 5 ] = 0;
+    }
+    return  uuidsize + 6;
+}
+
+int add_macs(char* destination, int maxsize) {
+    struct ifaddrs *ifc, *ifa;
+    struct sockaddr_ll *lla;
+    int offset;
+    char macaddr[32];
+
+    offset = 0;
+    getifaddrs(&ifa);
+    for (ifc = ifa; ifc != NULL; ifc = ifc->ifa_next) {
+        if (ifc->ifa_addr->sa_family != AF_PACKET)
+            continue;
+        lla = (struct sockaddr_ll *)ifc->ifa_addr;
+        if (lla->sll_hatype == ARPHRD_INFINIBAND) {
+            snprintf(macaddr, 32, "/mac=%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+                lla->sll_addr[12], lla->sll_addr[13], lla->sll_addr[14],
+                lla->sll_addr[15], lla->sll_addr[16], lla->sll_addr[17],
+                lla->sll_addr[18], lla->sll_addr[19]
+            );
+        } else if (lla->sll_hatype == ARPHRD_ETHER) {
+            snprintf(macaddr, 32, "/mac=%02x:%02x:%02x:%02x:%02x:%02x",
+                lla->sll_addr[0], lla->sll_addr[1], lla->sll_addr[2],
+                lla->sll_addr[3], lla->sll_addr[4], lla->sll_addr[5],
+                lla->sll_addr[6]
+            );
+        } else {
+            continue;
+        }
+        strncpy(destination + offset, macaddr, maxsize - offset);
+        offset += strnlen(macaddr, 32);
+    }
+    freeifaddrs(ifa);
+}
+
+int main(int argc, char* argv[]) {
+    struct ifaddrs *ifc, *ifa;
+    struct sockaddr_in6 *in6;
+    struct sockaddr_in *in, *bin;
+    int ns, n4;
+    struct sockaddr_in6 addr, dst;
+    struct sockaddr_in addr4, dst4;
+    char msg[1024];
+    char lastmsg[1024];
+    int ifidx, offset;
+    fd_set rfds;
+    struct timeval tv;
+    socklen_t dstsize, dst4size;
+    dstsize = sizeof(dst);
+    dst4size = sizeof(dst4);
+
+    memset(msg, 0, 1024);
+    memset(&addr, 0, sizeof(addr));
+    memset(&dst, 0, sizeof(dst));
+    memset(&dst4, 0, sizeof(dst4));
+    addr.sin6_family = AF_INET6;
+    addr.sin6_addr = in6addr_any;
+    addr.sin6_port = htons(190);
+    addr4.sin_family = AF_INET;
+    addr4.sin_addr.s_addr = htonl(INADDR_ANY);
+    addr4.sin_port = htons(190);
+    dst.sin6_family = AF_INET6;
+    dst.sin6_port = htons(1900);
+    inet_pton(AF_INET6, "ff02::c", &dst.sin6_addr);
+    dst4.sin_family = AF_INET;
+    dst4.sin_port = htons(1900);
+    inet_pton(AF_INET, "239.255.255.250", &dst4.sin_addr);
+    strncpy(msg,  "M-SEARCH * HTTP/1.1\r\nST: urn:xcat.org:service:confluent:", 1024);
+    offset = strnlen(msg, 1024);
+    if (argc > 1) {
+        snprintf(msg + offset, 1024 - offset, "/node=%s", argv[1]);
+        offset = strnlen(msg, 1024);
+    }
+    add_uuid(msg + offset, 1024 - offset);
+    offset = strnlen(msg, 1024);
+    add_macs(msg + offset, 1024 - offset);
+    offset = strnlen(msg, 1024);
+    ns = socket(AF_INET6, SOCK_DGRAM, 0);
+    n4 = socket(AF_INET, SOCK_DGRAM, 0);
+    ifidx = 1; /* reuse ifidx because it's an unused int here */
+    setsockopt(n4, SOL_SOCKET, SO_BROADCAST, &ifidx, sizeof(ifidx));
+    setsockopt(ns, IPPROTO_IPV6, IPV6_V6ONLY, &ifidx, sizeof(ifidx));
+    /* For now, bind to 190 to prove we are a privileged process */
+    bind(n4, (const struct sockaddr *)&addr4, sizeof(addr4));
+    bind(ns, (const struct sockaddr *)&addr, sizeof(addr));
+    getifaddrs(&ifa);
+    for (ifc = ifa; ifc != NULL; ifc = ifc->ifa_next) {
+        if (!ifc->ifa_addr) continue;
+        if (ifc->ifa_flags & IFF_LOOPBACK) continue;
+        if (ifc->ifa_flags & IFF_MULTICAST != IFF_MULTICAST) continue;
+        if (ifc->ifa_addr->sa_family == AF_INET6) {
+            in6 = (struct sockaddr_in6 *)ifc->ifa_addr;
+            if (in6->sin6_scope_id == 0)
+                continue;
+            ifidx = in6->sin6_scope_id;
+            setsockopt(ns, IPPROTO_IPV6, IPV6_MULTICAST_IF, &ifidx, sizeof(ifidx));
+            sendto(ns, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst, sizeof(dst));
+        } else if (ifc->ifa_addr->sa_family == AF_INET) {
+            in = (struct sockaddr_in *)ifc->ifa_addr;
+            bin = (struct sockaddr_in *)ifc->ifa_ifu.ifu_broadaddr;
+            bin->sin_port = htons(1900);
+            setsockopt(n4, IPPROTO_IP, IP_MULTICAST_IF, &in->sin_addr, sizeof(in->sin_addr));
+            sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)&dst4, sizeof(dst4));
+            sendto(n4, msg, strnlen(msg, 1024), 0, (const struct sockaddr *)bin, sizeof(*bin));
+        }
+    }
+    FD_ZERO(&rfds);
+    FD_SET(n4, &rfds);
+    FD_SET(ns, &rfds);
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    ifidx = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
+    while (ifidx) {
+        if (ifidx == -1) perror("Unable to select");
+        if (ifidx) {
+            if (FD_ISSET(n4, &rfds)) {
+                recvfrom(n4, msg, 1024, 0, (struct sockaddr *)&dst4, &dst4size);
+                inet_ntop(dst4.sin_family, &dst4.sin_addr, msg, dst4size);
+                /* Take measure from printing out the same ip twice in a row */
+                if (strncmp(lastmsg, msg, 1024) != 0) {
+                    printf("%s\n", msg);
+                    strncpy(lastmsg, msg, 1024);
+                }
+            }
+            if (FD_ISSET(ns, &rfds)) {
+                recvfrom(ns, msg, 1024, 0, (struct sockaddr *)&dst, &dstsize);
+                inet_ntop(dst.sin6_family, &dst.sin6_addr, msg, dstsize);
+                if (strncmp(lastmsg, msg, 1024) != 0) {
+                    printf("%s\n", msg);
+                    strncpy(lastmsg, msg, 1024);
+                }
+            }
+        }
+        tv.tv_sec = 0;
+        tv.tv_usec = 500000;
+        FD_SET(n4, &rfds);
+        FD_SET(ns, &rfds);
+        ifidx = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
+    }
+}

misc/disablepasscomplexity.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import pyghmi.util.webclient as webclient
+import json
+import os
+import sys
+
+missingargs = False
+if 'XCCUSER' not in os.environ:
+    print('Must set XCCUSER environment variable')
+    missingargs = True
+if 'XCCPASS' not in os.environ:
+    print('Must set XCCPASS environment variable')
+    missingargs = True
+if missingargs:
+    sys.exit(1)
+
+w = webclient.SecureHTTPConnection(sys.argv[1], 443, verifycallback=lambda x: True)
+w.connect()
+adata = json.dumps({'username': os.environ['XCCUSER'], 'password': os.environ['XCCPASS']})
+headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+w.request('POST', '/api/login', adata, headers)
+rsp = w.getresponse()
+if rsp.status == 200:
+     rspdata = json.loads(rsp.read())
+     w.set_header('Content-Type', 'application/json')
+     w.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+     if '_csrf_token' in w.cookies:
+         w.set_header('X-XSRF-TOKEN', w.cookies['_csrf_token'])
+     print(repr(w.grab_json_response('/api/dataset', {
+         'USER_GlobalPassComplexRequired': '0',
+     })))
+     

misc/fixexpiry.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import pyghmi.util.webclient as webclient
+import json
+import os
+import sys
+
+tmppassword = 'to3BdS91ABrd'
+missingargs = False
+if 'XCCUSER' not in os.environ:
+    print('Must set XCCUSER environment variable')
+    missingargs = True
+if 'XCCPASS' not in os.environ:
+    print('Must set XCCPASS environment variable')
+    missingargs = True
+if missingargs:
+    sys.exit(1)
+
+w = webclient.SecureHTTPConnection(sys.argv[1], 443, verifycallback=lambda x: True)
+w.connect()
+adata = json.dumps({'username': os.environ['XCCUSER'], 'password': os.environ['XCCPASS']})
+headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+w.request('POST', '/api/login', adata, headers)
+rsp = w.getresponse()
+if rsp.status != 200:
+     rsp.read()
+     adata = json.dumps({'username': os.environ['XCCUSER'], 'password': tmppassword})
+     headers = {'Connection': 'keep-alive', 'Content-Type': 'application/json'}
+     w.request('POST', '/api/login', adata, headers)
+     rsp = w.getresponse()
+if rsp.status == 200:
+     rspdata = json.loads(rsp.read())
+     w.set_header('Content-Type', 'application/json')
+     w.set_header('Authorization', 'Bearer ' + rspdata['access_token'])
+     if '_csrf_token' in w.cookies:
+         w.set_header('X-XSRF-TOKEN', w.cookies['_csrf_token'])
+     if rspdata.get('pwchg_required', False):
+         print(repr(w.grab_json_response('/api/function', {'USER_UserPassChange': '1,to3BdS91ABrd'})))
+     print(repr(w.grab_json_response('/api/dataset', {
+         'USER_GlobalPassExpWarningPeriod': '0', 
+         'USER_GlobalPassExpPeriod': '0', 
+         'USER_GlobalMinPassReuseCycle': '0', 
+         'USER_GlobalMinPassReuseCycle': '0', 
+         'USER_GlobalMinPassChgInt': '0', 
+     })))
+     print(repr(w.grab_json_response('/api/function', {'USER_UserPassChange': '1,' + os.environ['XCCPASS']})))
+     

misc/fixsmmexpiry.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+import pyghmi.util.webclient as webclient
+from xml.etree.ElementTree import fromstring
+import os
+import sys
+
+tmppassword = 'to3BdS91ABrd'
+missingargs = False
+if 'SMMUSER' not in os.environ:
+    print('Must set SMMUSER environment variable')
+    missingargs = True
+if 'SMMPASS' not in os.environ:
+    print('Must set SMMPASS environment variable')
+    missingargs = True
+if missingargs:
+    sys.exit(1)
+
+w = webclient.SecureHTTPConnection(sys.argv[1], 443, verifycallback=lambda x: True)
+w.connect()
+adata = 'user={0}&password={1}'.format(os.environ['SMMUSER'], os.environ['SMMPASS'])
+bdata = 'user={0}&password={1}'.format(os.environ['SMMUSER'], tmppassword)
+headers = {'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded'}
+w.request('POST', '/data/login', adata, headers)
+rsp = w.getresponse()
+rspdata = rsp.read()
+restorepwd = False
+if 'authResult>1' in rspdata:
+     restorepwd = True
+     w.request('POST', '/data/login', bdata, headers)
+     rsp = w.getresponse()
+     rspdata = rsp.read()
+if 'renew_account' in rspdata:
+     restorepwd = True
+     tokens = fromstring(rspdata)
+     st2 = tokens.findall('st2')[0].text
+     w.set_header('ST2', st2)
+     w.request('POST', '/data/changepwd', 'oripwd={0}&newpwd={1}'.format(os.environ['SMMPASS'], tmppassword))
+     rsp = w.getresponse()
+     rspdata = rsp.read()
+     w.request('POST', '/data/login', bdata, headers)
+     rsp = w.getresponse()
+     rspdata = rsp.read()
+if 'authResult>0' in rspdata:
+     tokens = fromstring(rspdata)
+     st2 = tokens.findall('st2')[0].text
+     w.set_header('ST2', st2)
+     rules = 'set=passwordDurationDays:0,passwordExpireWarningDays:0,passwordChangeInterval:0'
+     w.request('POST', '/data', rules)
+     rsp = w.getresponse()
+     print(repr(rsp.read()))
+     if restorepwd:
+         w.request('POST', '/data/changepwd', 'oripwd={1}&newpwd={0}'.format(os.environ['SMMPASS'], tmppassword))
+         rsp = w.getresponse()
+         print(repr(rsp.read()))

misc/logreader.py

@@ -0,0 +1,199 @@
+#!/usr/bin/python
+import collections
+import os
+import struct
+import sys
+import time
+import fcntl
+import select
+import termios
+import tty
+
+def writeout(data):
+    done = False
+    try:
+        sys.stdout.write(data)
+        done = True
+    except IOError:
+        time.sleep(0.1)
+        pass
+
+
+class LogReplay(object):
+    def __init__(self, logfile, cblfile):
+        self.bin = open(cblfile, 'r')
+        self.txt = open(logfile, 'r')
+        self.cleardata = []
+        self.clearidx = 0
+        self.pendingdata = collections.deque([])
+        self.priordata = collections.deque([])
+        self.laststamp = None
+        self.needclear = False
+
+    def _rewind(self, datasize=None):
+        curroffset = self.bin.tell() - 16
+        if self.cleardata and self.clearidx > 1:
+            self.clearidx -= 1
+            priordata = self.cleardata[self.clearidx - 1]
+            return curroffset, priordata
+        self.cleardata = []
+        self.clearidx = 0
+        newoffset = curroffset - 32
+        if newoffset < 0:  #TODO: Follow a log roll
+            newoffset = 0
+        if datasize:
+            while datasize > 0 and newoffset > 0:
+                self.bin.seek(newoffset)
+                tmprec = self.bin.read(16)
+                newoffset -= 32
+                tmprec = struct.unpack('!BBIHIBBH', tmprec)
+                if tmprec[1] == 2:
+                    datasize -= tmprec[3]
+        if newoffset >= 0:
+            self.bin.seek(newoffset)
+        return curroffset, None
+
+    def debuginfo(self):
+        return '{0}, {1}'.format(self.bin.tell(), self.clearidx)
+
+    def get_output(self, reverse=False):
+        endoffset = None
+        output = ''
+        if reverse:  # Forget the uncommited future, if present
+            output += '\x1b[2J\x1b[H'
+            endoffset, priordata = self._rewind(4096)
+            if priordata is not None:
+                return priordata, 1
+        elif self.needclear:
+            output += '\x1b[2J\x1b[H'
+            self.needclear = False
+        if self.cleardata and self.clearidx < len(self.cleardata):
+            datachunk = self.cleardata[self.clearidx]
+            self.clearidx += 1
+            return datachunk, 1
+        self.cleardata = []
+        self.clearidx = 0
+        while (not reverse) or (self.bin.tell() < endoffset):
+            record = self.bin.read(16)
+            if not record:
+                return '', 0
+            record = struct.unpack('!BBIHIBBH', record)
+            if record[0] > 16:
+                # Unsupported record, skip
+                self.bin.seek(record[0] - 16, 1)
+                continue
+            type = record[1]
+            offset = record[2]
+            size = record[3]
+            evtdata = record[5]
+            auxdata = record[6]
+            if type == 3:
+                #TODO: provide data for status bar
+                continue
+            elif type == 2:
+                self.laststamp = record[4]
+                self.txt.seek(offset)
+                txtout = self.txt.read(size)
+                if reverse and self.bin.tell() < endoffset:
+                    output += txtout
+                    continue
+                if '\x1b[2J' in txtout:
+                    self.cleardata = txtout.split('\x1b[2J')
+                    for idx in range(1, len(self.cleardata)):
+                        self.cleardata[idx] = '\x1b[2J' + self.cleardata[idx]
+                    self.clearidx = 0
+                    if not self.cleardata[0]:
+                        self.cleardata = self.cleardata[1:]
+                    if self.cleardata:
+                        if reverse:
+                            output = self.cleardata[-1]
+                            self.clearidx = len(self.cleardata)
+                        else:
+                            output += self.cleardata[0]
+                            self.clearidx = 1
+                else:
+                    output += txtout
+                break
+        if endoffset is not None and endoffset >= 0:
+            self.bin.seek(endoffset)
+        return output, 1
+
+    def begin(self):
+        self.needclear = True
+        self.bin.seek(0)
+
+    def end(self):
+        self.bin.seek(0, 2)
+
+
+def main(txtfile, binfile):
+    replay = LogReplay(txtfile, binfile)
+    oldtcattr = termios.tcgetattr(sys.stdin.fileno())
+    tty.setraw(sys.stdin.fileno())
+    currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl | os.O_NONBLOCK)
+    reverse = False
+    skipnext = False
+    quitit = False
+    writeout('\x1b[2J\x1b[;H')
+    try:
+        while not quitit:
+            if not skipnext:
+                newdata, delay = replay.get_output(reverse)
+            skipnext = False
+            reverse = False
+            if newdata:
+                writeout(newdata)
+                writeout('\x1b]0;[Time: {0}]\x07'.format(
+                    time.strftime('%m/%d %H:%M:%S', time.localtime(replay.laststamp))))
+                sys.stdout.flush()
+            while True:
+                select.select((sys.stdin,), (), (), 86400)
+                myinput = sys.stdin.read()
+                if myinput.startswith('\x1b[C') or myinput.startswith('\x1bOC') or myinput == '\r':  # right
+                    break
+                elif myinput.startswith('\x1b[D') or myinput.startswith('\x1bOD') or myinput == 'y':  # left
+                    writeout('\x1b[2J\x1b[;H')
+                    reverse = True
+                    break
+                elif myinput == 'G' or myinput.startswith('\x1b[F'):
+                    replay.end()
+                    reverse = True
+                    break
+                elif myinput == 'g' or myinput.startswith('\x1b[H'):
+                    replay.begin()
+                    break
+                elif myinput.lower() == 'q' or myinput == '\x03':
+                    quitit = True
+                    break
+                elif myinput.lower() == 'd':
+                    writeout('\x1b];{0}\x07'.format(replay.debuginfo()))
+                    sys.stdout.flush()
+                else:
+                    pass # print(repr(myinput))
+    except Exception:
+        currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+        fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
+        termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
+        writeout('\x1b[m')
+        raise
+    currfl = fcntl.fcntl(sys.stdin.fileno(), fcntl.F_GETFL)
+    fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFL, currfl ^ os.O_NONBLOCK)
+    termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, oldtcattr)
+    writeout('\x1b[m')
+
+if __name__ == '__main__':
+    txtfile = sys.argv[1]
+    if len(sys.argv) > 2:
+        binfile = sys.argv[2]
+    else:
+        if os.path.exists(txtfile + '.cbl'):
+            binfile = txtfile + '.cbl'
+        else:
+            fileparts = txtfile.split('.')
+            prefix = '.'.join(fileparts[:-1])
+            binfile = prefix + '.cbl.' + fileparts[-1]
+            if not os.path.exists(binfile):
+                sys.stderr.write('Unable to locate cbl file\n')
+                sys.exit(1)
+    main(txtfile, binfile)