Add a prototype stats command for CLI commands

MIB resolution turns out to be rather CPU intensive, and the
current SNMP consumers don't want the resolution anyway.

confluent_client/bin/collate

@@ -35,6 +35,9 @@ import confluent.client
 argparser = optparse.OptionParser(usage="Usage: <other command> | %prog [options]")
 argparser.add_option('-a', '--abbreviate', action='store_true',
                      help='Attempt to use confluent server to shorten noderanges')
+argparser.add_option('-b', '--base',
+                     help='Use given node as reference for comparison when '
+                          'using -d, instead of using the most common result')
 argparser.add_option('-d', '--diff', action='store_true',
                      help='Show what differs between most common '
                           'output group and others')
@@ -65,7 +68,7 @@ else:
 def print_current():
     if options.diff:
         grouped.print_deviants(skipmodal=options.skipcommon, count=options.count,
-                          reverse=options.reverse)
+                          reverse=options.reverse, basenode=options.base)
     else:
         grouped.print_all(skipmodal=options.skipcommon,
                                count=options.count,

confluent_client/bin/nodeconfig

@@ -43,7 +43,7 @@ def bailout(msg, code=1):
     sys.exit(code)
 
 
-argparser = optparse.OptionParser()
+argparser = optparse.OptionParser(usage="Usage: %prog [options] noderange [option|option=value]")
 argparser.add_option('-c', '--comparedefault', dest='comparedefault',
                      action='store_true', default=False,
                      help='Compare given settings to default or list settings '
@@ -185,7 +185,13 @@ if options.batch:
     argfile = open(options.batch, 'r')
     argset = argfile.readline()
     while argset:
-        parse_config_line(shlex.split(argset))
+        try:
+            argset = argset[:argset.index('#')]
+        except ValueError:
+            pass
+        argset = argset.strip()
+        if argset:
+            parse_config_line(shlex.split(argset))
         argset = argfile.readline()
 else:
     parse_config_line(args[1:])

confluent_client/bin/nodeconsole

@@ -40,6 +40,7 @@ if len(args) != 1:
     argparser.print_help()
     sys.exit(1)
 if options.tile:
+    null = open('/dev/null', 'w')
     nodes = []
     sess = client.Command()
     for res in sess.read('/noderange/{0}/nodes/'.format(args[0])):
@@ -60,13 +61,15 @@ if options.tile:
                     confettypath, node)])
         else:
             subprocess.call(['tmux', 'select-pane', '-t', str(pane)])
+            subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
             pane += 1
             subprocess.call(
                 ['tmux', 'split', '-h',
                  '{0} -m 5 start /nodes/{1}/console/session'.format(
                      confettypath, node)])
-            subprocess.call(['tmux', 'select-layout', 'tiled'])
+            subprocess.call(['tmux', 'select-layout', 'tiled'], stdout=null)
     subprocess.call(['tmux', 'select-pane', '-t', '0'])
+    subprocess.call(['tmux', 'set-option', 'pane-border-status', 'top'], stderr=null)
     os.execlp('tmux', 'tmux', 'attach', '-t', 'nodeconsole_{0}'.format(
         os.getpid()))
 else:

confluent_client/bin/nodediscover

@@ -69,10 +69,7 @@ def print_disco(options, session, currmac, outhandler, columns):
             record.append(','.join(rawval))
         else:
             record.append(str(rawval))
-    if options.csv:
-        csv.writer(sys.stdout).writerow(record)
-    else:
-        outhandler.add_row(record)
+    outhandler.add_row(record)
 
 
 def process_header(header):
@@ -204,7 +201,6 @@ def import_csv(options, session):
 
 
 def list_discovery(options, session):
-    outhandler = None
     orderby = None
     if options.fields:
         columns = []
@@ -219,13 +215,12 @@ def list_discovery(options, session):
         for field in columns:
             if options.order.lower() == field.lower():
                 orderby = field
-    if options.csv:
-        csv.writer(sys.stdout).writerow(columns)
-    else:
-        outhandler = client.Tabulator(columns)
+    outhandler = client.Tabulator(columns)
     for mac in list_matching_macs(options, session):
         print_disco(options, session, mac, outhandler, columns)
-    if outhandler:
+    if options.csv:
+        outhandler.write_csv(sys.stdout, orderby)
+    else:
         for row in outhandler.get_table(orderby):
             print(row)
 
@@ -294,6 +289,7 @@ def blocking_scan(session):
     list(session.update('/discovery/rescan', {'rescan': 'start'}))
     while(list(session.read('/discovery/rescan'))[0].get('scanning', False)):
         time.sleep(0.5)
+    list(session.update('/networking/macs/rescan', {'rescan': 'start'}))
 
 
 def main():
@@ -352,4 +348,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodegroupattrib

@@ -36,9 +36,9 @@ import confluent.client as client
 
 argparser = optparse.OptionParser(
     usage='''\n       %prog [options]   \
-             \n       %prog [options] nodegroup [list of attributes]  \
+             \n       %prog [options] nodegroup [list of attributes|all]  \
              \n       %prog [options] nodegroup nodes=value1,value2   \
-             \n       %prog -e nodegroup <attribute names to set>  \             
+             \n       %prog -e nodegroup <attribute names to set>  \
              \n       %prog [options] nodegroup nodes=value1,value2
              \n ''')
 argparser.add_option('-b', '--blame', action='store_true',

confluent_client/bin/nodegrouplist

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2015-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    showtype='all'
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) > 0:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.read(nodelist):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print(res['item']['href'].replace('/', ''))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/nodegrouprename

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'jjohnson2,alin37,andywray'
+
+import optparse
+import os
+import signal
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+def main():
+    argparser = optparse.OptionParser(
+    usage="Usage: %prog <group> <new group name>\n")
+    (options, args) = argparser.parse_args()
+    noderange=""
+    nodelist=""
+    nodelist = '/nodegroups/'
+    session = client.Command()
+    exitcode = 0
+    requestargs=args[1:]
+    nodetype='noderange'
+    if len(args) != 2:
+        argparser.print_help()
+        sys.exit(1)
+    else:
+        for res in session.update(
+            '/nodegroups/{0}/attributes/rename'.format(args[0]),
+            {'rename': args[1]}):
+            if 'error' in res:
+                sys.stderr.write(res['error'] + '\n')
+                exitcode = 1
+            else:
+                print('{0}: {1}'.format(res['oldname'], res['newname']))
+
+    sys.exit(exitcode)
+
+if __name__ == '__main__':
+    main()

confluent_client/bin/noderename

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import signal
+import sys
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> <newname>")
+(options, args) = argparser.parse_args()
+try:
+    noderange = args[0]
+except IndexError:
+    argparser.print_help()
+    sys.exit(1)
+client.check_globbing(noderange)
+identifystate = None
+if len(sys.argv) > 2:
+    newname = sys.argv[2]
+else:
+    argparser.print_help()
+    sys.exit(1)
+session = client.Command()
+exitcode = 0
+sys.exit(
+    session.simple_noderange_command(noderange, 'attributes/rename', newname))
+

confluent_client/bin/nodersync

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import deque
+import optparse
+import os
+import select
+import shlex
+import signal
+import subprocess
+import sys
+
+try:
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+except AttributeError:
+    pass
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+import confluent.screensqueeze as sq
+import confluent.sortutil as sortutil
+
+
+def run():
+    argparser = optparse.OptionParser(
+        usage="Usage: %prog location noderange:location",
+    )
+    argparser.add_option('-f', '-c', '--count', type='int', default=168,
+                         help='Number of nodes to concurrently rsync')
+    # among other things, FD_SETSIZE limits.  Besides, spawning too many
+    # processes can be unkind for the unaware on memory pressure and such...
+    argparser.disable_interspersed_args()
+    (options, args) = argparser.parse_args()
+    if len(args) < 2 or ':' not in args[-1]:
+        argparser.print_help()
+        sys.exit(1)
+    concurrentprocs = options.count
+    noderange, targpath = args[-1].split(':', 1)
+    client.check_globbing(noderange)
+    c = client.Command()
+    cmdstr = " ".join(args[:-1])
+    cmdstr = 'rsync -av --info=progress2 ' + cmdstr
+    cmdstr += ' {node}:' + targpath
+
+    currprocs = 0
+    all = set([])
+    pipedesc = {}
+    pendingexecs = deque()
+    exitcode = 0
+
+    for exp in c.create('/noderange/{0}/attributes/expression'.format(noderange),
+                        {'expression': cmdstr}):
+        if 'error' in exp:
+            sys.stderr.write(exp['error'] + '\n')
+            exitcode |= exp.get('errorcode', 1)
+        ex = exp.get('databynode', ())
+        for node in ex:
+            cmd = ex[node]['value'].encode('utf-8')
+            cmdv = shlex.split(cmd)
+            if currprocs < concurrentprocs:
+                currprocs += 1
+                run_cmdv(node, cmdv, all, pipedesc)
+            else:
+                pendingexecs.append((node, cmdv))
+    if not all or exitcode:
+        sys.exit(exitcode)
+    rdy, _, _ = select.select(all, [], [], 10)
+    nodeerrs = {}
+    pernodeout = {}
+    pernodefile = {}
+    output = sq.ScreenPrinter(noderange, c)
+
+    while all:
+        for r in rdy:
+            desc = pipedesc[r]
+            node = desc['node']
+            data = True
+            while data and select.select([r], [], [], 0)[0]:
+                data = r.read(1)
+                if data:
+                    if desc['type'] == 'stdout':
+                        if node not in pernodeout:
+                            pernodeout[node] = ''
+                        pernodeout[node] += data
+                        if '\n' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\n', 1)
+                            if currout:
+                                pernodefile[node] = os.path.basename(currout)
+                        if '\r' in pernodeout[node]:
+                            currout, pernodeout[node] = pernodeout[node].split('\r', 1)
+                            if currout:
+                                currout = currout.split()
+                                try:
+                                    currout = currout[1]
+                                    output.set_output(node, '{0}:{1}'.format(pernodefile[node], currout))
+                                except IndexError:
+                                    pernodefile = currout[0]
+                                    pass
+                    else:
+                        output.set_output(node, 'error!')
+                        if node not in nodeerrs:
+                            nodeerrs[node] = ''
+                        nodeerrs[node] += data
+                else:
+                    pop = desc['popen']
+                    ret = pop.poll()
+                    if ret is not None:
+                        exitcode = exitcode | ret
+                        all.discard(r)
+                        r.close()
+                        if node not in nodeerrs:
+                            output.set_output(node, 'complete')
+                        if desc['type'] == 'stdout' and pendingexecs:
+                            node, cmdv = pendingexecs.popleft()
+                            run_cmdv(node, cmdv, all, pipedesc)
+        if all:
+            rdy, _, _ = select.select(all, [], [], 10)
+    for node in nodeerrs:
+        for line in nodeerrs[node].split('\n'):
+            sys.stderr.write('{0}: {1}\n'.format(node, line))
+    sys.exit(exitcode)
+
+
+def run_cmdv(node, cmdv, all, pipedesc):
+    nopen = subprocess.Popen(
+        cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
+                              'type': 'stdout'}
+    pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
+                              'type': 'stderr'}
+    all.add(nopen.stdout)
+    all.add(nopen.stderr)
+
+
+if __name__ == '__main__':
+    run()

confluent_client/bin/noderun

@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil
 
 def run():
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -93,7 +95,10 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -107,7 +112,10 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodeshell

@@ -39,11 +39,13 @@ import confluent.sortutil as sortutil
 def run():
 
     argparser = optparse.OptionParser(
-        usage="Usage: %prog noderange commandexpression",
+        usage="Usage: %prog [options] noderange commandexpression",
         epilog="Expressions are the same as in attributes, e.g. "
                "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of commands to run at a time')
+    argparser.add_option('-n', '--nonodeprefix', action='store_true',
+                         help='Do not prefix output with node names')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
     # processes can be unkind for the unaware on memory pressure and such...
     argparser.disable_interspersed_args()
@@ -94,7 +96,10 @@ def run():
                             pernodeout[node] = []
                         pernodeout[node].append(data)
                     else:
-                        sys.stderr.write('{0}: {1}'.format(node, data))
+                        if options.nonodeprefix:
+                            sys.stderr.write(data)
+                        else:
+                            sys.stderr.write('{0}: {1}'.format(node, data))
                         sys.stderr.flush()
                 else:
                     pop = desc['popen']
@@ -108,7 +113,10 @@ def run():
                             run_cmdv(node, cmdv, all, pipedesc)
         for node in sortutil.natural_sort(pernodeout):
             for line in pernodeout[node]:
-                sys.stdout.write('{0}: {1}'.format(node, line))
+                if options.nonodeprefix:
+                    sys.stdout.write(line)
+                else:
+                    sys.stdout.write('{0}: {1}'.format(node, line))
         sys.stdout.flush()
         if all:
             rdy, _, _ = select.select(all, [], [], 10)

confluent_client/bin/nodestorage

@@ -147,13 +147,13 @@ def deletestorage(noderange, options, args):
             print(repr(rsp))
 
 
-def setstorage(noderange, options, args):
-    pass
+# def setstorage(noderange, options, args):
+#     pass
 
 funmap = {
     'create': createstorage,
     'show': showstorage,
-    'set': setstorage,
+#     'set': setstorage,
     'delete': deletestorage,
     'rm': deletestorage,
 }
@@ -162,7 +162,7 @@ funmap = {
 
 def main():
     argparser = OptParser(
-        usage='Usage: %prog <noderange> [show|create|set|delete]',
+        usage='Usage: %prog <noderange> [show|create|delete]',
         epilog='',
     )
     argparser.add_option('-r', '--raidlevel', type='int',
@@ -184,6 +184,8 @@ def main():
                               'delete.  Default behavior is to use '
                               'implementation provided default names.')
     (options, args) = argparser.parse_args()
+    if len(args) == 1:
+        args.append('show')
     try:
         noderange = args[0]
         operation = args[1]
@@ -200,4 +202,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

confluent_client/bin/nodesupport

@@ -114,10 +114,12 @@ class OptParser(optparse.OptionParser):
 
 def main():
     argparser = OptParser(
-        usage="Usage: %prog <noderange> [servicedata] "
+        usage="Usage: %prog <noderange> servicedata "
               "<filename>",
         epilog='\nservicedata will save service data to the given '
-               'directory\n'
+               'directory.  It is saved to the location on the relevant '
+               'management server (the confluent server if running remote, '
+               'and the collective.manager if in collective)\n'
                '\n\nSee `man %prog` for more info.\n')
     (options, args) = argparser.parse_args()
     media = None

confluent_client/builddeb

@@ -0,0 +1 @@
+../confluent_server/builddeb

confluent_client/confluent/client.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import anydbm as dbm
+import csv
 import errno
 import fnmatch
 import hashlib
@@ -70,6 +71,23 @@ class Tabulator(object):
             for row in self.rows:
                 yield fmtstr.format(*row)
 
+    def write_csv(self, output, order=None):
+        output = csv.writer(output)
+        output.writerow(self.headers)
+        i = 0
+        for head in self.headers:
+            if order and order == head:
+                order = i
+            i = i + 1
+        if order is not None:
+            for row in sorted(
+                    self.rows,
+                    key=lambda x: sortutil.naturalize_string(x[order])):
+                output.writerow(row)
+        else:
+            for row in self.rows:
+                output.writerow(row)
+
 
 def printerror(res, node=None):
     exitcode = 0
@@ -118,6 +136,8 @@ class Command(object):
             self.serverloc = server
         if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
             self._connect_unix()
+        elif self.serverloc == '/var/run/confluent/api.sock':
+            raise Exception('Confluent service is not available')
         else:
             self._connect_tls()
         tlvdata.recv(self.connection)
@@ -444,9 +464,12 @@ def print_attrib_path(path, session, requestargs, options, rename=None):
                                 (currattr.get('default', None) is not None and
                                 currattr.get('value', None) is not None and
                                 currattr['value'] != currattr['default'])):
+                            cval = ','.join(currattr['value']) if isinstance(
+                                currattr['value'], list) else currattr['value']
+                            dval = ','.join(currattr['default']) if isinstance(
+                                currattr['default'], list) else currattr['default']
                             cprint('{0}: {1}: {2} (Default: {3})'.format(
-                                node, printattr, currattr['value'],
-                                currattr['default']))
+                                node, printattr, cval, dval))
                     else:
 
                         try:

confluent_client/confluent/screensqueeze.py

@@ -44,6 +44,8 @@ class ScreenPrinter(object):
         self.fieldwidth = maxlen + textlen + 1  # 1 for column
 
     def set_output(self, node, text):
+        if self.nodeoutput[node] == text:
+            return
         self.nodeoutput[node] = text
         if len(text) >= self.textlen:
             self.textlen = len(text) + 1

confluent_client/confluent/textgroup.py

@@ -169,13 +169,18 @@ class GroupedData(object):
         output.flush()
 
     def print_deviants(self, output=sys.stdout, skipmodal=False, reverse=False,
-                       count=False):
+                       count=False, basenode=None):
         self.generate_byoutput()
         modaloutput = None
         ismodal = True
         revoutput = []
+        if basenode:
+            for checkout in self.byoutput:
+                if basenode in self.byoutput[checkout]:
+                    modaloutput = checkout
         for outdata in sorted(
-                self.byoutput, key=lambda x: [0 - len(self.byoutput[x]),
+                self.byoutput, key=lambda x: [0 if modaloutput == x else 1,
+                                              0 - len(self.byoutput[x]),
                                               humanify_nodename(
                                                   self.get_group_text(
                                                       self.byoutput[x]

confluent_client/confluent_env.sh

@@ -190,6 +190,13 @@ _confluent_nodesupport_completion()
     fi
 }
 
+_confluent_nodeattrib_completion()
+{
+    COMP_CANDIDATES=$(nodeattrib '~.>1' all | awk '{print $2}'|sed -e 's/://')
+    _confluent_generic_completion
+}
+
+
 _confluent_nn_completion()
 {
     _confluent_get_args
@@ -250,7 +257,8 @@ _confluent_ng_completion()
 
     COMPREPLY=($(compgen -W "$(confetty show /nodegroups|sed -e 's/\///' -e s/^/$PREFIX/)" -- "${COMP_WORDS[-1]}"))
 }
-complete -F _confluent_nr_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodeattrib
+complete -F _confluent_nodeattrib_completion nodegroupattrib
 complete -F _confluent_nr_completion nodebmcreset
 complete -F _confluent_nodesetboot_completion nodeboot
 complete -F _confluent_nr_completion nodeconfig

confluent_client/doc/man/collective.ronn

@@ -0,0 +1,55 @@
+collective(1) -- Check and manage a confluent collective
+==============================
+
+## SYNOPSIS
+
+`collective invite <server>`  
+`collective join <server> [-i TOKEN]`  
+`collective show`  
+`collective gencert`  
+
+
+## DESCRIPTION
+
+**collective** helps manage the collective mode of confluent, where multiple
+confluent servers are linked together to act as one.  For example, the procedure to set up
+a collective to run on three servers called mgt1, mgt2, and mgt3, first install and start
+confluent as usual on the three servers.  On mgt1, run `collective invite mgt2` and an
+invitation token will be output.  On mgt2, either run `collective join mgt1` to paste
+the token interactively, or `collective join mgt1 -i <token>`.  At this point, either
+mgt1 or mgt2 can bring in mgt3.  For example on mgt2 run `collective invite mgt3` and
+on mgt3 run `collective join mgt2 -i <token>`
+
+This can be linked together in the following manner with ssh:
+on mgt1:
+  `# ssh mgt2 collective join mgt1 -i $(collective invite mgt2)`
+
+Note that a collective is only redundant with 3 or more members.  The collective
+will function so long as more than half of the members are online.  A collective
+of two members is supported, but without redundancy.
+
+Also note that the collective leader role is dynamic, but has no impact on interacting
+with confluent.  It is merely an internal role that can dynamically change depending
+on circumstances.
+
+## OPTIONS
+
+ * `-i`:
+    Provide the token as an argument rather than interactively.
+   
+## EXAMPLES
+ * Inviting a server called mgt2:
+  `# collective invite mgt2`  
+  `bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+
+* On mgt2, joining mgt1:
+  `# collective join mgt1 -i bWd0MkA+BNQ6XAxMXlqJJa+EQRlihL/k9xCXnasgSQXZr989Pa1/ln7G3e1Ncxx6BMzMqqreHJVkPr2FrzjNit/UgHlg`  
+  `Success`  
+
+* Showing the collective state:
+  `# collective show`  
+  `Quorum: True`  
+  `Leader: mgt1`  
+  `Active collective members:`  
+  `  mgt2`  
+

confluent_client/doc/man/nodeattrib.ronn

@@ -23,6 +23,9 @@ For a full list of attributes, run `nodeattrib <node> all` against a node.
 If `-c` is specified, this will set the nodeattribute to a null value.
 This is different from setting the value to an empty string.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 If the word all is specified, then all available attributes are given.
 Omitting any attribute name or the word 'all' will display only attributes
 that are currently set.
@@ -88,6 +91,12 @@ See nodegroupattrib(8) command on how to manage attributes on a group level.
     `n1: console.method: `  
     `n2: console.method: `  
 
+* List all switches that a node is described as connected to:
+    `# nodeattrib d1 net.*switch`  
+    `d1: net.mgt.switch: mgtswitch1`  
+    `d1: net.pxe.switch: pxeswitch1`  
+    `d1: net.switch:`  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodeattribexpressions.ronn

@@ -9,6 +9,18 @@ expression to generate the value.
 An expression will contain some directives wrapped in `{}` characters. Within
 `{}` are a number of potential substitute values and operations.
 
+Note that syntax of expressions can have overlap with the shell syntax.
+For example:
+
+`$ echo (n2)`  
+`-bash: syntax error near unexpected token `n2'`  
+
+In such a case, it helps to quote the expression to allow it to be passed:
+
+`$ echo '(n2)'`  
+`(n2)`  
+
+
 The most common operation is to extract a number from the nodename. These 
 values are available as n1, n2, etc. So for example attributes for a node named 
 b1o2r3u4 would have {n1} as 1, {n2} as 2, {n3} as 3, and {n4} as 4. 

confluent_client/doc/man/nodeconfig.ronn

@@ -3,8 +3,8 @@ nodeconfig(8) -- Show or change node configuration
 
 ## SYNOPSIS
 
-`nodeconfig <noderange> [options] [<configuration>..]`
-`nodeconfig <noderange> [options] [<configuration=value>..]`
+`nodeconfig <noderange> [options] [<configuration>..]`  
+`nodeconfig <noderange> [options] [<configuration=value>..]`  
 
 ## DESCRIPTION
 

confluent_client/doc/man/nodegrouplist.ronn

@@ -0,0 +1,14 @@
+nodegrouplist(8) -- List the defined confluent nodegroups
+===================================================================
+
+## SYNOPSIS
+
+`nodegrouplist`  
+
+## DESCRIPTION
+
+`nodegrouplist` lists the currently defined groups in confluent.
+
+## SEE ALSO
+
+nodeattrib(8), nodeattribexpressions(5), nodegroupattrib(8)

confluent_client/doc/man/nodelist.ronn

@@ -17,6 +17,9 @@ displayed.  If `-b` is specified, it will also display information on
 how inherited and expression based attributes are defined.  There is more
 information on node attributes in nodeattributes(5) man page.
 
+Attributes may be specified by wildcard, for example `net.*switch` will report
+all attributes that begin with `net.` and end with `switch`.
+
 ## OPTIONS
 
 * `-b`, `--blame`:

confluent_client/doc/man/noderange.ronn

@@ -36,6 +36,9 @@ Also, regular expressions may be used to indicate nodes with names matching cert
 The other major noderange primitive is indicating nodes by some attribute value:
 `location.rack=7`
 
+The attribute name may use a wildcard:
+`net.*switch=switch1`
+
 Commas can be used to indicate multiple nodes, and can mix and match any of the above primitives. The following can be a valid single noderange, combining any and all members of each comma separated component
 `n1,n2,rack1,storage,location.rack=9,~s1..,n20-n30`
 

confluent_client/doc/man/nodestorage.ronn

@@ -0,0 +1,65 @@
+nodestorage(8) -- Examine/Modify storage configuration of a node
+============================================
+
+## SYNOPSIS
+
+`nodestorage <noderange> [show|create|delete] [options]`  
+
+## DESCRIPTION
+
+`nodestorage` provides access to the remote storage configuration of
+the noderange.
+
+## OPTIONS
+* `-r` **RAIDLEVEL**, `--raidlevel`=**RAIDLEVEL**:
+  RAID level to use when creating an array
+
+* `-d` **DISKS**, `--disks`=**DISKS**:
+   Comma separated list of disks to use, or the word "rest" to
+   indicate use of all available disks
+
+* `-s` **SIZE**, `--size`=**SIZE**:
+  Comma separated list of sizes to use when creating
+  volumes.  The sizes may be absolute size (e.g. 16gb),
+  percentage (10%) or the word "rest" to use remaining
+  capacity, default behavior is to use all capacity to
+  make a volume
+* `-n` **NAME**, `--name`=**NAME**:
+  Comma separated list of names to use when naming
+  volumes, or selecting a volume for delete.  Default
+  behavior is to use implementation provided default
+
+## EXAMPLES
+
+* Deleting the volume `somedata`:
+    `$ nodestorage d5 delete somedata`  
+    `Deleted: somedata`  
+
+* Creating a raid5 of 4 disks and a volume named `somedata`:  
+    `$ nodestorage d5 create -r 5 -d drive0,drive_1,drive_2,drive_3 -n somedata
+    `d5: Volume somedata: Size: 1.905 TB`  
+    `d5: Volume somedata: State: Optimal`  
+    `d5: Volume somedata: Array 1-2`  
+
+* Showing current storage configuration of `d3`:
+    `$ nodestorage d3`  
+    `d3: Disk m.2-0 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-0 State: online`  
+    `d3: Disk m.2-0 FRU: 00LF428`  
+    `d3: Disk m.2-0 Serial Number: H6B80054`  
+    `d3: Disk m.2-0 Array: 0-0`  
+    `d3: Disk m.2-1 Description: 128GB M.2 SATA SSD`  
+    `d3: Disk m.2-1 State: online`  
+    `d3: Disk m.2-1 FRU: 00LF428`  
+    `d3: Disk m.2-1 Serial Number: H6B80059`  
+    `d3: Disk m.2-1 Array: 0-0`  
+    `d3: Array 0-0 Available Capacity: 0.000 MB`  
+    `d3: Array 0-0 Total Capacity: 131.072 GB`  
+    `d3: Array 0-0 RAID: RAID 1`  
+    `d3: Array 0-0 Disks: m.2-0,m.2-1`  
+    `d3: Array 0-0 Volumes: new_vd`  
+    `d3: Volume new_vd: Size: 122.040 GB`  
+    `d3: Volume new_vd: State: Optimal`  
+    `d3: Volume new_vd: Array 0-0`  
+
+

confluent_client/doc/man/nodesupport.ronn

@@ -10,7 +10,11 @@ nodesupport(8) -- Utilities for interacting with vendor support
 `nodesupport` provides capabilities associated with interactiong with support.
 Currently it only has the `servicedata` subcommand.  `servicedata` takes
 an argument that is either a directory name (that can be used for a single node
-or multiple nodes) or a file name (only to be used with single node noderange)
+or multiple nodes) or a file name (only to be used with single node noderange).
+Note that the file will be downloaded to the confluent server that actually
+connects to the managed system, so it will download to the remote system if running
+remotely and will download to the collective.manager indicated system if
+running in collective mode.
 
 ## EXAMPLES
 

confluent_server/bin/collective

@@ -101,11 +101,11 @@ def main():
     sl = sp.add_parser('show', help='Show information about the collective')
     ic = sp.add_parser('invite', help='Generate a invitation to allow a new '
                                     'confluent instance to join as a '
-                                    'collective member')
+                                    'collective member.  Run collective invite -h for more information')
     ic.add_argument('name', help='Name of server to invite to join the '
                                  'collective')
-    jc = sp.add_parser('join', help='Join a collective')
-    jc.add_argument('server', help='A server currently in the collective')
+    jc = sp.add_parser('join', help='Join a collective.  Run collective join -h for more information')
+    jc.add_argument('server', help='Existing collective member that ran invite and generated a token')
     jc.add_argument('-i', help='Invitation provided by runniing invite on an '
                                'existing collective member')
     cmdset = a.parse_args()

confluent_server/builddeb

@@ -0,0 +1,63 @@
+#!/bin/bash
+cd `dirname $0`
+PKGNAME=$(basename $(pwd))
+DPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+OPKGNAME=$(basename $(pwd) | sed -e s/_/-/)
+if grep wheezy /etc/os-release; then
+    DPKGNAME=python-$DPKGNAME
+fi
+cd ..
+mkdir -p /tmp/confluent # $DPKGNAME
+cp -a * .git /tmp/confluent # $DPKGNAME
+cd /tmp/confluent/$PKGNAME
+if [ -x ./makeman ]; then
+    ./makeman
+fi
+./makesetup
+VERSION=`cat VERSION`
+cat > setup.cfg << EOF
+[install]
+install-purelib=/opt/confluent/lib/python
+install-scripts=/opt/confluent/bin
+
+[sdist_dsc]
+package=$DPKGNAME
+EOF
+
+python setup.py sdist > /dev/null 2>&1
+py2dsc dist/*.tar.gz
+shopt -s extglob
+cd deb_dist/!(*.orig)/
+if [ "$OPKGNAME" = "confluent-server" ]; then
+    if grep wheezy /etc/os-release; then
+        sed -i 's/^\(Depends:.*\)/\1, python-confluent-client, python-lxml, python-eficompressor, python-pycryptodomex/' debian/control
+    else
+        sed -i 's/^\(Depends:.*\)/\1, confluent-client, python-lxml, python-eficompressor, python-pycryptodome/' debian/control
+    fi
+    if grep wheezy /etc/os-release; then
+        echo 'confluent_client python-confluent-client' >> debian/pydist-overrides
+    else
+        echo 'confluent_client confluent-client' >> debian/pydist-overrides
+    fi
+fi
+head -n -1 debian/control > debian/control1
+mv debian/control1 debian/control
+#echo 'Provides: python-'$DPKGNAME >> debian/control
+#echo 'Conflicts: python-'$DPKGNAME >> debian/control
+#echo 'Replaces: python-'$DPKGNAME' (<<2)' >> debian/control
+#echo 'Breaks: python-'$DPKGNAME' (<<2)' >> debian/control
+
+dpkg-buildpackage -rfakeroot -uc  -us -i
+if [ $? -ne 0 ]; then
+    echo "[ERROR] rpmbuild returned non-zero, run: rpmbuild -ba ~/rpmbuild/SPECS/$PKGNAME.spec"
+    exit 1
+else
+    cd -
+    # Clean up the generated files in this directory
+    rm -rf $PKGNAME.egg-info dist setup.py
+    rm -rf $(find deb_dist -mindepth 1 -maxdepth 1 -type d)
+    if [ ! -z "$1" ]; then
+        mv deb_dist/* $1/
+    fi
+fi
+exit 0

confluent_server/confluent/config/attributes.py

@@ -93,9 +93,10 @@ node = {
         'description': ('List of static groups for which this node is '
                         'considered a member'),
     },
-    #'type': {
-    #    'description': ('Classification of node as system, vm, etc')
-    #},
+    'type': {
+        'description': ('Classification of node as server or switch'),
+        'validvalues': ('switch', 'server'),
+    },
     #'id': {
     #    'description': ('Numeric identifier for node')
     #},
@@ -158,9 +159,11 @@ node = {
     },
     'discovery.passwordrules': {
         'description':  'Any specified rules shall be configured on the BMC '
-                        'upon discovery.  "expiration=no,loginfailures=no" '
-                        'would disable password expiration and login failures '
-                        'triggering a lockout.'
+                        'upon discovery.  "expiration=no,loginfailures=no,complexity=no,reuse=no" '
+                        'would disable password expiration, login failures '
+                        'triggering a lockout, password complexity requirements,'
+                        'and any restrictions around reusing an old password.',
+        'validlistkeys': ('expiration', 'loginfailures', 'complexity', 'reuse'),
     },
     'discovery.policy': {
         'description':  'Policy to use for auto-configuration of discovered '
@@ -171,6 +174,7 @@ node = {
                         'so long as the node has no existing public key. '
                         '"open" allows discovery even if a known public key '
                         'is already stored',
+        'validlist': ('manual', 'permissive', 'pxe', 'open'),
     },
     'info.note': {
         'description':  'A field used for administrators to make arbitrary '
@@ -258,11 +262,13 @@ node = {
     'console.logging': {
         'description': ('Indicate logging level to apply to console.  Valid '
                         'values are currently "full", "interactive", and '
-                        '"none". Defaults to "full".')
+                        '"none". Defaults to "full".'),
+        'validvalues': ('full', 'interactive', 'none'),
     },
     'console.method': {
         'description': ('Indicate the method used to access the console of '
-                        'the managed node.')
+                        'the managed node.'),
+        'validvalues': ('ssh', 'ipmi'),
     },
 #    'virtualization.host': {
 #        'description': ('Hypervisor where this node does/should reside'),

confluent_server/confluent/config/configmanager.py

@@ -1,7 +1,7 @@
 7# vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015-2018 Lenovo
+# Copyright 2015-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -181,6 +181,14 @@ def _rpc_master_set_node_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant).set_node_attributes(attribmap, autocreate)
 
 
+def _rpc_master_rename_nodes(tenant, renamemap):
+    ConfigManager(tenant).rename_nodes(renamemap)
+
+
+def _rpc_master_rename_nodegroups(tenant, renamemap):
+    ConfigManager(tenant).rename_nodegroups(renamemap)
+
+
 def _rpc_master_clear_node_attributes(tenant, nodes, attributes):
     ConfigManager(tenant).clear_node_attributes(nodes, attributes)
 
@@ -234,6 +242,14 @@ def _rpc_set_node_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant)._true_set_node_attributes(attribmap, autocreate)
 
 
+def _rpc_rename_nodes(tenant, renamemap):
+    ConfigManager(tenant)._true_rename_nodes(renamemap)
+
+
+def _rpc_rename_nodegroups(tenant, renamemap):
+    ConfigManager(tenant)._true_rename_nodegroups(renamemap)
+
+
 def _rpc_set_group_attributes(tenant, attribmap, autocreate):
     ConfigManager(tenant)._true_set_group_attributes(attribmap, autocreate)
 
@@ -288,9 +304,9 @@ def logException():
                  event=confluent.log.Events.stacktrace)
 
 
-def _do_add_watcher(watcher, added, configmanager):
+def _do_add_watcher(watcher, added, configmanager, renamed=()):
     try:
-        watcher(added=added, deleting=[], configmanager=configmanager)
+        watcher(added=added, deleting=(), renamed=renamed, configmanager=configmanager)
     except Exception:
         logException()
 
@@ -1039,24 +1055,32 @@ class ConfigManager(object):
             raise Exception('Invalid Expression')
         for node in nodes:
             try:
-                currval = self._cfgstore['nodes'][node][attribute]['value']
+                currvals = [self._cfgstore['nodes'][node][attribute]['value']]
             except KeyError:
                 # Let's treat 'not set' as being an empty string for this path
-                currval = ''
-            if exmatch:
-                if yieldmatches:
-                    if exmatch.search(currval):
-                        yield node
+                currvals = list(
+                    [self._cfgstore['nodes'][node][x].get('value', '') 
+                    for x in fnmatch.filter(self._cfgstore['nodes'][node], attribute)])
+                currvals.append('')
+            for currval in currvals:
+                if exmatch:
+                    if yieldmatches:
+                        if exmatch.search(currval):
+                            yield node
+                            break
+                    else:
+                        if not exmatch.search(currval):
+                            yield node
+                            break
                 else:
-                    if not exmatch.search(currval):
-                        yield node
-            else:
-                if yieldmatches:
-                    if match == currval:
-                        yield node
-                else:
-                    if match != currval:
-                        yield node
+                    if yieldmatches:
+                        if match == currval:
+                            yield node
+                            break
+                    else:
+                        if match != currval:
+                            yield node
+                            break
 
     def filter_nodenames(self, expression, nodes=None):
         """Filter nodenames by regular expression
@@ -1499,6 +1523,9 @@ class ConfigManager(object):
                     newattr = _attraliases[attr]
                     attribmap[group][newattr] = attribmap[group][attr]
                     del attribmap[group][attr]
+            if 'noderange' in attribmap[group]:
+                if len(attribmap[group]) > 1:
+                    raise ValueErorr('noderange attribute must be set by itself')            
             for attr in attribmap[group]:
                 if attr in _attraliases:
                     newattr = _attraliases[attr]
@@ -1515,17 +1542,20 @@ class ConfigManager(object):
                         currnodes = list(self.get_nodegroup_attributes(
                             group, ['nodes']).get('nodes', []))
                         if attribmap[group][attr].get('prepend', False):
-                            newnodes = attribmap[group][attr][
-                                'prepend'].split(',')
-                            attribmap[group][attr] = newnodes + currnodes
+                            newnodes = noderange.NodeRange(attribmap[group][attr][
+                                'prepend'], config=self).nodes
+                            attribmap[group][attr] = list(
+                                newnodes) + currnodes
                         elif attribmap[group][attr].get('remove', False):
-                            delnodes = attribmap[group][attr][
-                                'remove'].split(',')
+                            delnodes = noderange.NodeRange(
+                                attribmap[group][attr]['remove'],
+                                config=self).nodes
                             attribmap[group][attr] = [
                                 x for x in currnodes if x not in delnodes]
                     if not isinstance(attribmap[group][attr], list):
                         if type(attribmap[group][attr]) is unicode or type(attribmap[group][attr]) is str:
-                            attribmap[group][attr]=attribmap[group][attr].split(",")
+                            attribmap[group][attr] = noderange.NodeRange(
+                                attribmap[group][attr], config=self).nodes
                         else:
                             raise ValueError("nodes attribute on group must be list")
                     for node in attribmap[group]['nodes']:
@@ -1538,6 +1568,13 @@ class ConfigManager(object):
             if group not in self._cfgstore['nodegroups']:
                 self._cfgstore['nodegroups'][group] = {'nodes': set()}
             cfgobj = self._cfgstore['nodegroups'][group]
+            if 'noderange' in attribmap[group] and attribmap[group]['noderange']:
+                if cfgobj['nodes']:
+                    raise ValueError('Cannot set both nodes and noderange on group')
+                if set(cfgobj) - set(['noderange', 'nodes']):
+                    raise ValueError('Cannot set noderange on a group with attributes')
+            elif 'noderange' in cfgobj and cfgobj['noderange']:
+                raise ValueError('Attributes cannot be set on a group with a noderange')
             for attr in attribmap[group]:
                 if attr == 'nodes':
                     newdict = set(attribmap[group][attr])
@@ -1692,7 +1729,7 @@ class ConfigManager(object):
     def _true_del_nodes(self, nodes):
         if self.tenant in self._nodecollwatchers:
             for watcher in self._nodecollwatchers[self.tenant].itervalues():
-                watcher(added=[], deleting=nodes, configmanager=self)
+                watcher(added=(), deleting=nodes, renamed=(), configmanager=self)
         changeset = {}
         for node in nodes:
             # set a reserved attribute for the sake of the change notification
@@ -1776,6 +1813,86 @@ class ConfigManager(object):
                 attribmap[node]['groups'] = []
         self.set_node_attributes(attribmap, autocreate=True)
 
+    def rename_nodes(self, renamemap):
+        if cfgleader:
+            return exec_on_leader('_rpc_master_rename_nodes', self.tenant,
+            renamemap)
+        if cfgstreams:
+            exec_on_followers('_rpc_rename_nodes', self.tenant, renamemap)
+        self._true_rename_nodes(renamemap)
+
+    def _true_rename_nodes(self, renamemap):
+        oldnames = set(renamemap)
+        exprmgr = None
+        currnodes = set(self._cfgstore['nodes'])
+        missingnodes = oldnames - currnodes
+        if missingnodes:
+            raise ValueError(
+                'The following nodes to rename do not exist: {0}'.format(
+                    ','.join(missingnodes)))
+        newnames = set([])
+        for name in renamemap:
+            newnames.add(renamemap[name])
+        if newnames & currnodes:
+            raise ValueError(
+                'The following requested new names conflict with existing nodes: {0}'.format(
+                    ','.join(newnames & currnodes)))
+        for name in renamemap:
+            self._cfgstore['nodes'][renamemap[name]] = self._cfgstore['nodes'][name]
+            del self._cfgstore['nodes'][name]
+            _mark_dirtykey('nodes', name, self.tenant)
+            _mark_dirtykey('nodes', renamemap[name], self.tenant)
+            for group in self._cfgstore['nodes'][renamemap[name]].get('groups', []):
+                self._cfgstore['nodegroups'][group]['nodes'].discard(name)
+                self._cfgstore['nodegroups'][group]['nodes'].add(renamemap[name])
+                _mark_dirtykey('nodegroups', group, self.tenant)
+            cfgobj = self._cfgstore['nodes'][renamemap[name]]
+            node = renamemap[name]
+            changeset = {}
+            if exprmgr is None:
+                exprmgr = _ExpressionFormat(cfgobj, node)
+            self._recalculate_expressions(cfgobj, formatter=exprmgr, node=renamemap[name], changeset=changeset)
+        if self.tenant in self._nodecollwatchers:
+            nodecollwatchers = self._nodecollwatchers[self.tenant]
+            for watcher in nodecollwatchers.itervalues():
+                eventlet.spawn_n(_do_add_watcher, watcher, (), self, renamemap)
+        self._bg_sync_to_file()
+
+    def rename_nodegroups(self, renamemap):
+        if cfgleader:
+            return exec_on_leader('_rpc_master_rename_nodegroups', self.tenant, renamemap)
+        if cfgstreams:
+            exec_on_followers('_rpc_rename_nodegroups', self.tenant, renamemap)
+        self._true_rename_groups(renamemap)
+
+    def _true_rename_groups(self, renamemap):
+        oldnames = set(renamemap)
+        currgroups = set(self._cfgstore['nodegroups'])
+        missinggroups = oldnames - currgroups
+        if missinggroups:
+            raise ValueError(
+                'The following groups to rename do not exist: {0}'.format(
+                    ','.join(missinggroups)))
+        newnames = set([])
+        for name in renamemap:
+            newnames.add(renamemap[name])
+        if newnames & currgroups:
+            raise ValueError(
+                'The following requested new names conflict with existing groups: {0}'.format(
+                    ','.join(newnames & currgroups)))
+        for name in renamemap:
+            self._cfgstore['nodegroups'][renamemap[name]] = self._cfgstore['nodegroups'][name]
+            del self._cfgstore['nodegroups'][name]
+            _mark_dirtykey('nodegroups', name, self.tenant)
+            _mark_dirtykey('nodegroups', renamemap[name], self.tenant)
+            for node in self._cfgstore['nodegroups'][renamemap[name]].get('nodes', []):
+                lidx = self._cfgstore['nodes'][node]['groups'].index(name)
+                self._cfgstore['nodes'][node]['groups'][lidx] = renamemap[name]
+                _mark_dirtykey('nodes', node, self.tenant)
+        self._bg_sync_to_file()
+
+
+
     def set_node_attributes(self, attribmap, autocreate=False):
         if cfgleader:  # currently config slave to another
             return exec_on_leader('_rpc_master_set_node_attributes',
@@ -2029,7 +2146,9 @@ class ConfigManager(object):
         rootpath = cls._cfgdir
         try:
             with open(os.path.join(rootpath, 'transactioncount'), 'r') as f:
-                _txcount = struct.unpack('!Q', f.read())[0]
+                txbytes = f.read()
+                if len(txbytes) == 8:
+                    _txcount = struct.unpack('!Q', txbytes)[0]
         except IOError:
             pass
         _load_dict_from_dbm(['collective'], os.path.join(rootpath,

confluent_server/confluent/consoleserver.py

@@ -147,6 +147,7 @@ class ConsoleHandler(object):
 
     def __init__(self, node, configmanager, width=80, height=24):
         self.clearpending = False
+        self.clearerror = False
         self.initsize = (width, height)
         self._dologging = True
         self._is_local = True
@@ -308,8 +309,11 @@ class ConsoleHandler(object):
 
     def clearbuffer(self):
         self.feedbuffer(
-            '\x1bc[no replay buffer due to console.logging attribute set to '
-            'none or interactive,\r\nconnection loss, or service restart]')
+            '\x1bc[No data has been received from the remote console since ' \
+            'connecting.  This could\r\nbe due to having the console.logging ' \
+            'attribute set to none or interactive,\r\nserial console not '  \
+            'being enabled or incorrectly configured in the OS or\r\nfirmware, ' \
+            'or the console simply not having any output since last connection]')
         self.clearpending = True
 
     def _detach(self):
@@ -371,7 +375,20 @@ class ConsoleHandler(object):
             self.error = 'misconfigured'
             self._send_rcpts({'connectstate': self.connectstate,
                               'error': self.error})
+            self.feedbuffer(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self._send_rcpts(
+                '\x1bc\x1b[2J\x1b[1;1H[The console.method attribute for this node is '
+                'not configured,\r\nset it to a valid value for console '
+                'function]')
+            self.clearerror = True
             return
+        if self.clearerror:
+            self.clearerror = False
+            self.clearbuffer()
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self.send_break = self._console.send_break
         self.resize = self._console.resize
         if self._attribwatcher:
@@ -527,10 +544,11 @@ class ConsoleHandler(object):
         # TODO: analyze buffer for registered events, examples:
         #   panics
         #   certificate signing request
-        if self.clearpending:
+        if self.clearpending or self.clearerror:
             self.clearpending = False
-            self.feedbuffer(b'\x1bc')
-            self._send_rcpts(b'\x1bc')
+            self.clearerror = False
+            self.feedbuffer(b'\x1bc\x1b[2J\x1b[1;1H')
+            self._send_rcpts(b'\x1bc\x1b[2J\x1b[1;1H')
         self._send_rcpts(_utf8_normalize(data, self.shiftin, self.utf8decoder))
         self.log(data, eventdata=eventdata)
         self.lasttime = util.monotonic_time()
@@ -603,11 +621,14 @@ def disconnect_node(node, configmanager):
         del _handled_consoles[consk]
 
 
-def _nodechange(added, deleting, configmanager):
-    for node in added:
-        connect_node(node, configmanager)
+def _nodechange(added, deleting, renamed, configmanager):
     for node in deleting:
         disconnect_node(node, configmanager)
+    for node in renamed:
+        disconnect_node(node, configmanager)
+        connect_node(renamed[node], configmanager)
+    for node in added:
+        connect_node(node, configmanager)
 
 
 def _start_tenant_sessions(cfm):

confluent_server/confluent/core.py

@@ -35,6 +35,7 @@
 
 import confluent
 import confluent.alerts as alerts
+import confluent.log as log
 import confluent.tlvdata as tlvdata
 import confluent.config.attributes as attrscheme
 import confluent.config.configmanager as cfm
@@ -143,6 +144,7 @@ def _init_core():
     # be enumerated in any collection
     noderesources = {
         'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
             'expression': PluginRoute({'handler': 'attributes'}),
@@ -378,6 +380,7 @@ def _init_core():
 
     nodegroupresources = {
         'attributes': {
+            'rename': PluginRoute({'handler': 'attributes'}),
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
         },
@@ -651,7 +654,7 @@ def handle_dispatch(connection, cert, dispatch, peername):
                 plugpath = nodeattr[node][attrname]['value']
             elif 'default' in plugroute:
                 plugpath = plugroute['default']
-        if plugpath is not None:
+        if plugpath:
             try:
                 hfunc = getattr(pluginmap[plugpath], operation)
             except KeyError:
@@ -685,6 +688,8 @@ def _forward_rsp(connection, res):
 
 def handle_node_request(configmanager, inputdata, operation,
                         pathcomponents, autostrip=True):
+    if log.logfull:
+        raise exc.TargetResourceUnavailable('Filesystem full, free up space and restart confluent service')
     iscollection = False
     routespec = None
     if pathcomponents[0] == 'noderange':
@@ -809,7 +814,7 @@ def handle_node_request(configmanager, inputdata, operation,
                 elif list(cfm.list_collective()):
                     badcollnodes.append(node)
                     continue
-            if plugpath is not None:
+            if plugpath:
                 try:
                     hfunc = getattr(pluginmap[plugpath], operation)
                 except KeyError:

confluent_server/confluent/discovery/core.py

@@ -63,6 +63,7 @@
 
 import base64
 import confluent.config.configmanager as cfm
+import confluent.collective.manager as collective
 import confluent.discovery.protocols.pxe as pxe
 #import confluent.discovery.protocols.ssdp as ssdp
 import confluent.discovery.protocols.slp as slp
@@ -1055,6 +1056,14 @@ def discover_node(cfg, handler, info, nodename, manual):
                 traceback.print_exc()
                 return False
             newnodeattribs = {}
+            if cfm.list_collective():
+                # We are in a collective, check collective.manager
+                cmc = cfg.get_node_attributes(nodename, 'collective.manager')
+                cm = cmc.get(nodename, {}).get('collective.manager', {}).get('value', None)
+                if not cm:
+                    # Node is being discovered in collective, but no collective.manager, default
+                    # to the collective member actually able to execute the discovery
+                    newnodeattribs['collective.manager'] = collective.get_myname()
             if 'uuid' in info:
                 newnodeattribs['id.uuid'] = info['uuid']
             if 'serialnumber' in info:
@@ -1129,10 +1138,18 @@ def _handle_nodelist_change(configmanager):
         nodeaddhandler = None
 
 
-def newnodes(added, deleting, configmanager):
+def newnodes(added, deleting, renamed, configmanager):
     global attribwatcher
     global needaddhandled
     global nodeaddhandler
+    alldeleting = set(deleting) | set(renamed)
+    for node in alldeleting:
+        if node not in known_nodes:
+            continue
+        for mac in known_nodes[node]:
+            if mac in known_info:
+                del known_info[mac]
+        del known_nodes[node]
     _map_unique_ids()
     configmanager.remove_watcher(attribwatcher)
     allnodes = configmanager.list_nodes()

confluent_server/confluent/discovery/handlers/bmc.py

@@ -109,9 +109,14 @@ class NodeHandler(generic.NodeHandler):
             netconfig = netutil.get_nic_config(cfg, nodename, ip=newip)
             plen = netconfig['prefix']
             newip = '{0}/{1}'.format(newip, plen)
-            ic.set_net_configuration(ipv4_address=newip,
-                                     ipv4_configuration='static',
-                                     ipv4_gateway=netconfig['ipv4_gateway'])
+            currcfg = ic.get_net_configuration()
+            if currcfg['ipv4_address'] != newip:
+                # do not change the ipv4_config if the current config looks
+                # like it is already accurate
+                ic.set_net_configuration(ipv4_address=newip,
+                                         ipv4_configuration='static',
+                                         ipv4_gateway=netconfig[
+                                             'ipv4_gateway'])
         elif self.ipaddr.startswith('fe80::'):
             cfg.set_node_attributes(
                 {nodename: {'hardwaremanagement.manager': self.ipaddr}})

confluent_server/confluent/discovery/handlers/imm.py

@@ -25,7 +25,7 @@ class NodeHandler(bmchandler.NodeHandler):
     def adequate(cls, info):
         # We can sometimes receive a partially initialized SLP packet
         # This is not adequate for being satisfied
-        return bool(info['attributes'])
+        return bool(info.get('attributes', {}))
 
     def scan(self):
         slpattrs = self.info.get('attributes', {})

confluent_server/confluent/discovery/handlers/smm.py

@@ -36,9 +36,42 @@ class NodeHandler(bmchandler.NodeHandler):
             uuid = fixuuid(uuid[0])
             self.info['uuid'] = uuid
 
+    def _validate_cert(self, certificate):
+        # Assumption is by the time we call config, that discovery core has
+        # vetted self._fp.  Our job here then is just to make sure that
+        # the currect connection matches the previously saved cert
+        return certificate == self._fp
+
+    def set_password_policy(self, ic):
+        rules = []
+        for rule in self.ruleset.split(','):
+            if '=' not in rule:
+                continue
+            name, value = rule.split('=')
+            if value.lower() in ('no', 'none', 'disable', 'disabled'):
+                value = '0'
+            if name.lower() in ('expiry', 'expiration'):
+                rules.append('passwordDurationDays:' + value)
+                warndays = '5' if int(value) > 5 else value
+                rules.append('passwordExpireWarningDays:' + warndays)
+            if name.lower() in ('lockout', 'loginfailures'):
+                rules.append('passwordFailAllowdNum:' + value)
+            if name.lower() == 'reuse':
+                rules.append('passwordReuseCheckNum:' + value)
+        if rules:
+            apirequest = 'set={0}'.format(','.join(rules))
+            ic.register_key_handler(self._validate_cert)
+            ic.oem_init()
+            ic._oem.smmhandler.wc.request('POST', '/data', apirequest)
+            ic._oem.smmhandler.wc.getresponse().read()
+
     def config(self, nodename):
         # SMM for now has to reset to assure configuration applies
-        super(NodeHandler, self).config(nodename)
+        dpp = self.configmanager.get_node_attributes(
+            nodename, 'discovery.passwordrules')
+        self.ruleset = dpp.get(nodename, {}).get(
+            'discovery.passwordrules', {}).get('value', '')
+        ic = self._bmcconfig(nodename, customconfig=self.set_password_policy)
 
 # notes for smm:
 # POST to:

confluent_server/confluent/discovery/handlers/xcc.py

@@ -1,4 +1,4 @@
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -71,9 +71,17 @@ class NodeHandler(immhandler.NodeHandler):
                 if value.lower() in ('no', 'none', 'disable', 'disabled'):
                     value = '0'
                 ruleset['USER_GlobalMaxLoginFailures'] = value
+            if name.lower() == 'complexity':
+                ruleset['USER_GlobalPassComplexRequired'] = value
+            if name.lower() == 'reuse':
+                ruleset['USER_GlobalMinPassReuseCycle'] = value
         ic.register_key_handler(self.validate_cert)
         ic.oem_init()
-        ic._oem.immhandler.wc.grab_json_response('/api/dataset', ruleset)
+        try:
+            ic._oem.immhandler.wc.grab_json_response('/api/dataset', ruleset)
+        except Exception as e:
+            print(repr(e))
+            pass
 
     def config(self, nodename, reset=False):
         # TODO(jjohnson2): set ip parameters, user/pass, alert cfg maybe

confluent_server/confluent/discovery/protocols/slp.py

@@ -335,6 +335,7 @@ def _add_attributes(parsed):
     else:
         net = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
+        net.settimeout(1.0)
         net.connect(target)
     except socket.error:
         return
@@ -363,6 +364,7 @@ def query_srvtypes(target):
     while tries and not connected:
         tries -= 1
         try:
+            net.settimeout(1.0)
             net.connect(target)
             connected = True
         except socket.error:

confluent_server/confluent/discovery/protocols/ssdp.py

@@ -167,6 +167,7 @@ def _find_service(service, target):
             net4.sendto(smsg.format(bcast, service), (bcast, 1900))
     # SSDP by spec encourages responses to spread out over a 3 second interval
     # hence we must be a bit more patient
+    deadline = util.monotonic_time() + 4
     r, _, _ = select.select((net4, net6), (), (), 4)
     peerdata = {}
     while r:
@@ -174,7 +175,10 @@ def _find_service(service, target):
             (rsp, peer) = s.recvfrom(9000)
             neighutil.refresh_neigh()
             _parse_ssdp(peer, rsp, peerdata)
-        r, _, _ = select.select((net4, net6), (), (), 4)
+        timeout = deadline - util.monotonic_time()
+        if timeout < 0:
+            timeout = 0
+        r, _, _ = select.select((net4, net6), (), (), timeout)
     for nid in peerdata:
         yield peerdata[nid]
 
@@ -194,6 +198,8 @@ def _parse_ssdp(peer, rsp, peerdata):
     if code == '200':
         if nid in peerdata:
             peerdatum = peerdata[nid]
+            if peer not in peerdatum['peers']:
+                peerdatum['peers'].append(peer)
         else:
             peerdatum = {
                 'peers': [peer],

confluent_server/confluent/firmwaremanager.py

@@ -19,19 +19,23 @@
 # the time comes
 
 import confluent.exceptions as exc
+import confluent.log as log
 import confluent.messages as msg
 import eventlet
 import os
 import pwd
 import socket
+import traceback
 
 updatesbytarget = {}
 uploadsbytarget = {}
 downloadsbytarget = {}
 updatepool = eventlet.greenpool.GreenPool(256)
+_tracelog = None
 
 
 def execupdate(handler, filename, updateobj, type, owner, node):
+    global _tracelog
     if type != 'ffdc' and not os.path.exists(filename):
         errstr =  '{0} does not appear to exist on {1}'.format(
             filename, socket.gethostname())
@@ -58,6 +62,9 @@ def execupdate(handler, filename, updateobj, type, owner, node):
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': errstr})
     except Exception as e:
+        if _tracelog is None:
+            _tracelog = log.Logger('trace')
+        _tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event, event=log.Events.stacktrace)
         updateobj.handle_progress({'phase': 'error', 'progress': 0.0,
                                    'detail': str(e)})
 

confluent_server/confluent/httpapi.py

@@ -791,8 +791,12 @@ def serve(bind_host, bind_port):
             eventlet.sleep(1)
     # TCP_FASTOPEN
     sock.setsockopt(socket.SOL_TCP, 23, 5)
-    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                         debug=False, socket_timeout=60)
+    try:
+        eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                             debug=False, socket_timeout=60)
+    except TypeError:
+        # Older eventlet in place, skip arguments it does not understand
+        eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -73,6 +73,8 @@ import struct
 import time
 import traceback
 
+daemonized = False
+logfull = False
 try:
     from fcntl import flock, LOCK_EX, LOCK_UN, LOCK_SH
 except ImportError:
@@ -150,21 +152,34 @@ class BaseRotatingHandler(object):
         Output the record to the file, catering for rollover as described
         in doRollover().
         """
-        rolling_type = self.shouldRollover(binrecord, textrecord)
-        if rolling_type:
-            flock(self.textfile, LOCK_UN)
-            return self.doRollover(rolling_type)
-        return None
+        global logfull
+        try:
+            rolling_type = self.shouldRollover(binrecord, textrecord)
+            if rolling_type:
+                flock(self.textfile, LOCK_UN)
+                return self.doRollover(rolling_type)
+            return None
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
+
 
     def emit(self, binrecord, textrecord):
-        if self.textfile is None:
-            self.textfile = open(self.textpath, mode='ab')
-        if self.binfile is None:
-            self.binfile = open(self.binpath, mode='ab')
-        self.textfile.write(textrecord)
-        self.binfile.write(binrecord)
-        self.textfile.flush()
-        self.binfile.flush()
+        global logfull
+        try:
+            if self.textfile is None:
+                self.textfile = open(self.textpath, mode='ab')
+            if self.binfile is None:
+                self.binfile = open(self.binpath, mode='ab')
+            self.textfile.write(textrecord)
+            self.binfile.write(binrecord)
+            self.textfile.flush()
+            self.binfile.flush()
+        except (IOError, OSError) as e:
+            if not daemonized:
+                raise
+            logfull = True
 
     def get_textfile_offset(self, data_len):
         if self.textfile is None:
@@ -753,11 +768,13 @@ globaleventlog = None
 tracelog = None
 
 
-def log(logdata=None, ltype=None, event=0, eventdata=None):
+def log(logdata=None, ltype=None, event=0, eventdata=None, flush=False):
     global globaleventlog
     if globaleventlog is None:
         globaleventlog = Logger('events')
     globaleventlog.log(logdata, ltype, event, eventdata)
+    if flush:
+        globaleventlog.writedata()
 
 def logtrace():
     global tracelog

confluent_server/confluent/main.py

@@ -72,7 +72,7 @@ def _daemonize():
     os.setsid()
     thispid = os.fork()
     if thispid > 0:
-        print 'confluent server starting as pid %d' % thispid
+        print('confluent server starting as pid {0}'.format(thispid))
         os._exit(0)
     os.closerange(0, 2)
     os.umask(63)
@@ -81,6 +81,7 @@ def _daemonize():
     os.dup2(0, 2)
     sys.stdout = log.Logger('stdout', buffered=False)
     sys.stderr = log.Logger('stderr', buffered=False)
+    log.daemonized = True
 
 
 def _updatepidfile():
@@ -224,6 +225,11 @@ def run():
     except:
         doexit()
         raise
+    try:
+        log.log({'info': 'Confluent management service starting'}, flush=True)
+    except (OSError, IOError) as e:
+        print(repr(e))
+        sys.exit(1)
     _daemonize()
     if havefcntl:
         _updatepidfile()

confluent_server/confluent/messages.py

@@ -267,6 +267,24 @@ class CreatedResource(ConfluentMessage):
         pass
 
 
+class RenamedResource(ConfluentMessage):
+    notnode = True
+    readonly = True
+
+    def __init__(self, oldname, newname):
+        self.kvpairs = {'oldname': oldname, 'newname': newname}
+
+    def strip_node(self, node):
+        pass
+
+
+class RenamedNode(ConfluentMessage):
+    def __init__(self, name, rename):
+        self.desc = 'New Name'
+        kv = {'rename': {'value': rename}}
+        self.kvpairs = {name: kv}
+
+
 class AssignedResource(ConfluentMessage):
     notnode = True
     readonly = True
@@ -381,6 +399,8 @@ def get_input_message(path, operation, inputdata, nodes=None, multinode=False,
         return InputReseatMessage(path, nodes, inputdata)
     elif path == ['attributes', 'expression']:
         return InputExpression(path, inputdata, nodes)
+    elif path == ['attributes', 'rename']:
+        return InputConfigChangeSet(path, inputdata, nodes, configmanager)
     elif path[0] in ('attributes', 'users') and operation != 'retrieve':
         return InputAttributes(path, inputdata, nodes)
     elif path == ['boot', 'nextdevice'] and operation != 'retrieve':
@@ -539,8 +559,6 @@ class InputConfigClear(ConfluentMessage):
             raise exc.InvalidArgumentException('Input must be {"clear":true}')
 
 class InputConfigChangeSet(InputExpression):
-    # For now, this is identical to InputExpression, later it may
-    # internalize formula expansion, but not now..
     def __init__(self, path, inputdata, nodes=None, configmanager=None):
         self.cfm = configmanager
         super(InputConfigChangeSet, self).__init__(path, inputdata, nodes)
@@ -598,7 +616,7 @@ class InputAttributes(ConfluentMessage):
         for node in nodes:
             self.nodeattribs[node] = inputdata
 
-    def get_attributes(self, node):
+    def get_attributes(self, node, validattrs=None):
         if node not in self.nodeattribs:
             return {}
         nodeattr = deepcopy(self.nodeattribs[node])
@@ -613,6 +631,35 @@ class InputAttributes(ConfluentMessage):
                     # an expression string will error if format() done
                     # use that as cue to put it into config as an expr
                     nodeattr[attr] = {'expression': nodeattr[attr]}
+            if validattrs and 'validvalues' in validattrs.get(attr, []):
+                if (nodeattr[attr] and
+                        nodeattr[attr] not in validattrs[attr]['validvalues']):
+                    raise exc.InvalidArgumentException(
+                        'Attribute {0} does not accept value {1} (valid values would be {2})'.format(
+                            attr, nodeattr[attr], ','.join(validattrs[attr]['validvalues'])))
+            elif validattrs and 'validlist' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if v and v not in validattrs[attr]['validlist']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept list member '
+                            '{1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlist'])))
+            elif validattrs and 'validlistkeys' in validattrs.get(attr, []) and nodeattr[attr]:
+                req = nodeattr[attr].split(',')
+                for v in req:
+                    if '=' not in v:
+                        raise exc.InvalidArgumentException(
+                            'Passed key {0} requires a parameter'.format(v))
+                    v = v.split('=', 1)[0]
+                    if v and v not in validattrs[attr]['validlistkeys']:
+                        raise exc.InvalidArgumentException(
+                            'Attribute {0} does not accept key {1} (valid values would be {2})'.format(
+                                attr, v, ','.join(
+                                    validattrs[attr]['validlistkeys'])
+                            )
+                        )
         return nodeattr
 
 

confluent_server/confluent/networking/macmap.py

@@ -315,10 +315,12 @@ def _full_updatemacmap(configmanager):
                 'Network topology not available to tenants')
         # here's a list of switches... need to add nodes that are switches
         nodelocations = configmanager.get_node_attributes(
-            configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+            configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
         switches = set([])
         for node in nodelocations:
             cfg = nodelocations[node]
+            if cfg.get('type', {}).get('value', None) == 'switch':
+                switches.add(node)
             for attr in cfg:
                 if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                     continue
@@ -342,7 +344,7 @@ def _full_updatemacmap(configmanager):
                         _switchportmap[curswitch][portname] = None
                     else:
                         _switchportmap[curswitch][portname] = node
-        for switch in _macsbyswitch:
+        for switch in list(_macsbyswitch):
             if switch not in switches:
                 del _macsbyswitch[switch]
         switchauth = get_switchcreds(configmanager, switches)

confluent_server/confluent/networking/netutil.py

@@ -42,10 +42,12 @@ def get_switchcreds(configmanager, switches):
 
 def list_switches(configmanager):
     nodelocations = configmanager.get_node_attributes(
-        configmanager.list_nodes(), ('net*.switch', 'net*.switchport'))
+        configmanager.list_nodes(), ('type', 'net*.switch', 'net*.switchport'))
     switches = set([])
     for node in nodelocations:
         cfg = nodelocations[node]
+        if cfg.get('type', {}).get('value', None) == 'switch':
+            switches.add(node)
         for attr in cfg:
             if not attr.endswith('.switch') or 'value' not in cfg[attr]:
                 continue

confluent_server/confluent/plugins/configuration/attributes.py

@@ -1,5 +1,5 @@
 # Copyright 2014 IBM Corporation
-# Copyright 2017 Lenovo
+# Copyright 2017-2019 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,6 +39,7 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
     if element == 'all':
         theattrs = set(allattributes.node).union(set(grpcfg))
         theattrs.add('nodes')
+        theattrs.add('noderange')
         for attribute in sorted(theattrs):
             if attribute == 'groups':
                 continue
@@ -51,6 +52,10 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                 val = grpcfg[attribute]
             else:
                 val = {'value': None}
+            if attribute == 'noderange':
+                val['desc'] = 'The noderange this group is expanded ' \
+                    'to when used in noderange, exclusive with static ' \
+                    'nodes'
             if attribute.startswith('secret.'):
                 yield msg.CryptedAttributes(
                     kv={attribute: val},
@@ -69,6 +74,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
         for attribute in sorted(list(grpcfg)):
             currattr = grpcfg[attribute]
             if attribute == 'nodes':
+                if not currattr:
+                    continue
                 desc = 'The nodes belonging to this group'
             elif attribute == 'noderange':
                 desc = 'A dynamic noderange that this group refers to in noderange expansion'
@@ -92,8 +99,8 @@ def retrieve_nodegroup(nodegroup, element, configmanager, inputdata):
                     kv={attribute: currattr},
                     desc=desc)
             else:
-                print attribute
-                print repr(currattr)
+                print(attribute)
+                print(repr(currattr))
                 raise Exception("BUGGY ATTRIBUTE FOR NODEGROUP")
 
 
@@ -156,6 +163,11 @@ def update(nodes, element, configmanager, inputdata):
 
 
 def update_nodegroup(group, element, configmanager, inputdata):
+    if 'rename' in element:
+        namemap = {}
+        namemap[group] = inputdata.attribs['rename']
+        configmanager.rename_nodegroups(namemap)
+        return yield_rename_resources(namemap, isnode=False)
     try:
         clearattribs = []
         for attrib in inputdata.attribs.iterkeys():
@@ -195,10 +207,28 @@ def create(nodes, element, configmanager, inputdata):
     if nodes is not None and element[-1] == 'expression':
         return _expand_expression(nodes, configmanager, inputdata)
 
+def yield_rename_resources(namemap, isnode):
+    for node in namemap:
+        if isnode:
+            yield msg.RenamedNode(node, namemap[node])
+        else:
+            yield msg.RenamedResource(node, namemap[node])
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
+    if not nodes:
+        raise exc.InvalidArgumentException(
+            'No action to take, noderange is empty (if trying to define '
+            'group attributes, use nodegroupattrib)')
+    if 'rename' in element:
+        namemap = {}
+        for node in nodes:
+            rename = inputdata.get_attributes(node)
+            namemap[node] = rename['rename']
+        configmanager.rename_nodes(namemap)
+        return yield_rename_resources(namemap, isnode=True)
     for node in nodes:
-        updatenode = inputdata.get_attributes(node)
+        updatenode = inputdata.get_attributes(node, allattributes.node)
         clearattribs = []
         if updatenode:
             for attrib in updatenode.iterkeys():

confluent_server/confluent/plugins/hardwaremanagement/cnos.py

@@ -0,0 +1,147 @@
+
+# Copyright 2019 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#Noncritical:
+#  - One or more temperature sensors is in the warning range;
+#  - A panic dump exists in flash.
+#Critical:
+#  - One or more temperature sensors is in the failure range;
+#  - One or more fans are running < 100 RPM;
+#  - One power supply is off.
+
+
+import eventlet
+import eventlet.queue as queue
+import confluent.exceptions as exc
+webclient = eventlet.import_patched('pyghmi.util.webclient')
+import confluent.messages as msg
+import confluent.util as util
+
+class SwitchSensor(object):
+    def __init__(self, name, states, value=None, health=None):
+        self.name = name
+        self.value = value
+        self.states = states
+        self.health = health
+
+
+def cnos_login(node, configmanager, creds):
+    wc = webclient.SecureHTTPConnection(node, port=443, verifycallback=util.TLSCertVerifier(
+        configmanager, node, 'pubkeys.tls_hardwaremanager').verify_cert)
+    wc.set_basic_credentials(creds[node]['secret.hardwaremanagementuser']['value'], creds[node]['secret.hardwaremanagementpassword']['value'])
+    wc.request('GET', '/nos/api/login/')
+    rsp = wc.getresponse()
+    body = rsp.read()
+    if rsp.status == 401:  # CNOS gives 401 on first attempt...
+        wc.request('GET', '/nos/api/login/')
+        rsp = wc.getresponse()
+        body = rsp.read()
+    if rsp.status >= 200 and rsp.status < 300:
+        return wc
+    raise exc.TargetEndpointBadCredentials('Unable to authenticate')
+
+def update(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def delete(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def create(nodes, element, configmanager, inputdata):
+    for node in nodes:
+        yield msg.ConfluentNodeError(node, 'Not Implemented')
+
+def retrieve(nodes, element, configmanager, inputdata):
+    results = queue.LightQueue()
+    workers = set([])
+    if element == ['power', 'state']:
+        for node in nodes:
+            yield msg.PowerState(node=node, state='on')
+        return
+    elif element == ['health', 'hardware']:
+        creds = configmanager.get_node_attributes(
+                nodes, ['secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'], decrypt=True)
+        for node in nodes:
+            workers.add(eventlet.spawn(retrieve_health, configmanager, creds,
+                                       node, results))
+    else:
+        for node in nodes:
+            yield msg.ConfluentNodeError(node, 'Not Implemented')
+        return
+    currtimeout = 10
+    while workers:
+        try:
+            datum = results.get(10)
+            while datum:
+                if datum:
+                    yield datum
+                datum = results.get_nowait()
+        except queue.Empty:
+            pass
+        eventlet.sleep(0.001)
+        for t in list(workers):
+            if t.dead:
+                workers.discard(t)
+    try:
+        while True:
+            datum = results.get_nowait()
+            if datum:
+                yield datum
+    except queue.Empty:
+        pass
+
+
+def retrieve_health(configmanager, creds, node, results):
+    wc = cnos_login(node, configmanager, creds)
+    hinfo = wc.grab_json_response('/nos/api/sysinfo/globalhealthstatus')
+    summary = hinfo['status'].lower()
+    if summary == 'noncritical':
+        summary = 'warning'
+    results.put(msg.HealthSummary(summary, name=node))
+    state = None
+    badreadings = []
+    if summary != 'ok':  # temperature or dump or fans or psu
+        wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/panic_dump')
+        if switchinfo:
+            badreadings.append(
+                SwitchSensor('Panicdump', ['Present'], health='warning'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/temperatures')
+        for temp in switchinfo:
+            if temp == 'Temperature threshold':
+                continue
+            if switchinfo[temp]['State'] != 'OK':
+                temphealth = switchinfo[temp]['State'].lower()
+                if temphealth == 'noncritical':
+                    temphealth = 'warning'
+                tempval = switchinfo[temp]['Temp']
+                badreadings.append(
+                    SwitchSensor(temp, [], value=tempval, health=temphealth))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/fans')
+        for fan in switchinfo:
+            if switchinfo[fan]['speed-rpm'] < 100:
+                badreadings.append(
+                    SwitchSensor(fan, [], value=switchinfo[fan]['speed-rpm'],
+                                 health='critical'))
+        switchinfo = wc.grab_json_response('/nos/api/sysinfo/power')
+        for psu in switchinfo:
+            if switchinfo[psu]['State'] != 'Normal ON':
+                psuname = switchinfo[psu]['Name']
+                badreadings.append(
+                    SwitchSensor(psuname, states=[switchinfo[psu]['State']],
+                                 health='critical'))
+    results.put(msg.SensorReadings(badreadings, name=node))

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -37,6 +37,9 @@ ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
 import socket
 import ssl
 
+if not hasattr(ssl, 'SSLEOFError'):
+    ssl.SSLEOFError = None
+
 pci_cache = {}
 
 def get_dns_txt(qstring):
@@ -745,14 +748,16 @@ class IpmiHandler(object):
                 if 'username' in user:
                     self.ipmicmd.set_user_name(uid=user['uid'],
                                                name=user['username'])
-                if 'privilege_level' in user:
-                    self.ipmicmd.set_user_access(uid=user['uid'],
-                                    privilege_level=user['privilege_level'])
+
                 if 'password' in user:
                     self.ipmicmd.set_user_password(uid=user['uid'],
                                                    password=user['password'])
                     self.ipmicmd.set_user_password(uid=user['uid'],
                                     mode='enable', password=user['password'])
+                    if 'privilege_level' in user:
+                        self.ipmicmd.set_user_access(uid=user['uid'],
+                                                     privilege_level=user[
+                                                         'privilege_level'])
                 if 'enabled' in user:
                     if user['enabled'] == 'yes':
                         mode = 'enable'
@@ -871,6 +876,9 @@ class IpmiHandler(object):
                 'Extended information unavailable, mismatch detected between '
                 'target certificate fingerprint and '
                 'pubkeys.tls_hardwaremanager attribute')
+        except pygexc.TemporaryError as e:
+                errorneeded = msg.ConfluentNodeError(
+                self.node, str(e))
         self.output.put(msg.Firmware(items, self.node))
         if errorneeded:
             self.output.put(errorneeded)

confluent_server/confluent/plugins/shell/ssh.py

@@ -22,7 +22,11 @@
 import confluent.exceptions as cexc
 import confluent.interface.console as conapi
 import confluent.log as log
-import cryptography
+try:
+    import cryptography
+except ImportError:
+    # Using older, non-crypography based paramiko
+    cryptography = None
 
 import eventlet
 import hashlib
@@ -30,7 +34,7 @@ import sys
 sys.modules['gssapi'] = None
 paramiko = eventlet.import_patched('paramiko')
 warnhostkey = False
-if cryptography.__version__.split('.') < ['1', '5']:
+if cryptography and cryptography.__version__.split('.') < ['1', '5']:
     # older cryptography with paramiko breaks most key support except
     # ed25519
     warnhostkey = True

confluent_server/confluent/snmputil.py

@@ -79,14 +79,16 @@ class Session(object):
         # overriden, but some devices only support DES)
         tp = _get_transport(self.server)
         ctx = snmp.ContextData(self.context)
+        resolvemib = False
         if '::' in oid:
+            resolvemib = True
             mib, field = oid.split('::')
             obj = snmp.ObjectType(snmp.ObjectIdentity(mib, field))
         else:
             obj = snmp.ObjectType(snmp.ObjectIdentity(oid))
 
         walking = snmp.bulkCmd(self.eng, self.authdata, tp, ctx, 0, 10, obj,
-                               lexicographicMode=False)
+                               lexicographicMode=False, lookupMib=resolvemib)
         try:
             for rsp in walking:
                 errstr, errnum, erridx, answers = rsp

confluent_server/confluent/sockapi.py

@@ -75,6 +75,14 @@ except ImportError:
 
 plainsocket = None
 
+def _should_authlog(path, operation):
+    if (operation == 'retrieve' and
+            ('/sensors/' in path or '/health/' in path or
+             '/power/state' in path or '/nodes/' == path or
+             (path.startswith('/noderange/') and path.endswith('/nodes/')))):
+        return False
+    return True
+
 class ClientConsole(object):
     def __init__(self, client):
         self.client = client
@@ -151,7 +159,9 @@ def sessionhdl(connection, authname, skipauth=False, cert=None):
                                          'python-pyopenssl installed or has an '
                                          'incorrect version installed '
                                          '(e.g. pyOpenSSL would need to be '
-                                         'replaced with python-pyopenssl)'}})
+                                         'replaced with python-pyopenssl). '
+                                         'Restart confluent after updating '
+                                         'the dependency.'}})
             return
         return collective.handle_connection(connection, None, request['collective'],
                                      local=True)
@@ -194,12 +204,12 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
     path = request['path']
     params = request.get('parameters', {})
     hdlr = None
+    auditmsg = {
+        'operation': operation,
+        'target': path,
+    }
     if not skipauth:
         authdata = auth.authorize(authdata[2], path, authdata[3], operation)
-        auditmsg = {
-            'operation': operation,
-            'target': path,
-        }
         if authdata is None:
             auditmsg['allowed'] = False
             auditlog.log(auditmsg)
@@ -207,7 +217,8 @@ def process_request(connection, request, cfm, authdata, authname, skipauth):
         auditmsg['user'] = authdata[2]
         if authdata[3] is not None:
             auditmsg['tenant'] = authdata[3]
-        auditmsg['allowed'] = True
+    auditmsg['allowed'] = True
+    if _should_authlog(path, operation):
         auditlog.log(auditmsg)
     try:
         if operation == 'start':

misc/stats

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+
+import numpy as np
+import matplotlib as mpl
+mpl.use('Agg')
+import matplotlib.pyplot as plt
+import os
+import sixel
+import subprocess
+import sys
+import tempfile
+
+class DumbWriter(sixel.SixelWriter):
+    def restore_position(self, output):
+        return
+
+
+plotdata = []
+data = sys.stdin.readline()
+while data:
+    datum = float(data.split()[-1])
+    plotdata.append(datum)
+    data = sys.stdin.readline()
+n, bins, patches = plt.hist(plotdata, 20)
+plt.show()
+path = tempfile.mkdtemp()
+plt.savefig('{0}/histo.png'.format(path))
+writer = DumbWriter()
+writer.draw('{0}/histo.png'.format(path))
+print('Min: {3} Median: {0} Mean: {1} Max: {4} Standard Deviation: {2} Samples: {5}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata)))
+os.remove('{0}/histo.png'.format(path))
+os.rmdir(path)
+
+
+
+
+