From cfd8ac4c75f2f9038592510f24b96fde2a230e93 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 6 Apr 2017 11:30:08 -0400 Subject: [PATCH 01/13] refactoring nodelist and nodeattrib to use common code --- confluent_client/bin/nodeattrib | 120 +++++---------------------- confluent_client/bin/nodelist | 116 +++++++------------------- confluent_client/confluent/client.py | 101 ++++++++++++++++++++++ 3 files changed, 153 insertions(+), 184 deletions(-) diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index a05915c6..4d6678eb 100644 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -28,18 +28,6 @@ if path.startswith('/opt'): import confluent.client as client -def attrrequested(attr, attrlist, seenattributes): - for candidate in attrlist: - truename = candidate - if candidate.startswith('hm'): - candidate = candidate.replace('hm', 'hardwaremanagement', 1) - if candidate == attr: - seenattributes.add(truename) - return True - elif '.' not in candidate and attr.startswith(candidate + '.'): - seenattributes.add(truename) - return True - return False argparser = optparse.OptionParser( usage='''\n %prog [options] noderange [list of attributes] \ \n %prog [options] noderange attribute1=value1,attribute2=value,... @@ -50,6 +38,8 @@ argparser.add_option('-c', '--clear', action='store_true', help='Clear variables') (options, args) = argparser.parse_args() + +#setting minimal output to only output current information showtype = 'current' requestargs=None try: @@ -60,101 +50,34 @@ except IndexError: session = client.Command() exitcode = 0 - #Sets attributes +nodetype="noderange" + if len(args) > 1: - #clears attribute - if options.clear: - targpath = '/noderange/{0}/attributes/all'.format(noderange) - keydata = {} - for attrib in args[1:]: - keydata[attrib] = None - for res in session.update(targpath, keydata): - if 'error' in res: - if 'errorcode' in res: - exitcode = res['errorcode'] - sys.stderr.write('Error: ' + res['error'] + '\n') - sys.exit(exitcode) - else: - if args[1] == 'all': + exitcode=client.updateattrib(session,args,nodetype, noderange, options) + try: + # setting user output to what the user inputs + if args[1] == 'all': showtype = 'all' elif args[1] == 'current': showtype = 'current' - elif "=" in args[1]: - try: - if len(args[1:]) > 1: - for val in args[1:]: - val = val.split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0]) - else: - val=args[1].split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0]) - except: - sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:]))) - exitcode = 1 - sys.exit(exitcode) - else: - requestargs = args[1:] + requestargs=args[2:] + except: + pass + +if exitcode != 0: + sys.exit(exitcode) # Lists all attributes if len(args) > 0: - seenattributes = set([]) - for res in session.read('/noderange/{0}/attributes/{1}'.format(noderange,showtype)): - if 'error' in res: - print "found error" - sys.stderr.write(res['error'] + '\n') - exitcode = 1 - continue - for node in res['databynode']: - for attr in res['databynode'][node]: - seenattributes.add(attr) - currattr = res['databynode'][node][attr] - if requestargs is None or attrrequested(attr, args[1:], seenattributes): - if 'value' in currattr: - if currattr['value'] is not None: - attrout = '{0}: {1}: {2}'.format( - node, attr, currattr['value']) - else: - attrout = '{0}: {1}:'.format(node, attr) - elif 'isset' in currattr: - if currattr['isset']: - attrout = '{0}: {1}: ********'.format(node, attr) - else: - attrout = '{0}: {1}:'.format(node, attr) - elif 'broken' in currattr: - attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \ - '{2}'.format(node, attr, - currattr['broken']) - elif isinstance(currattr, list) or isinstance(currattr, tuple): - attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, currattr))) - elif isinstance(currattr, dict): - dictout = [] - for k,v in currattr.items: - dictout.append("{0}={1}".format(k,v)) - attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, dictout))) - else: - print ("CODE ERROR" + repr(attr)) + # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank. + if requestargs is None: + showtype = 'current' + elif requestargs == []: + #showtype already set + pass - if options.blame or 'broken' in currattr: - blamedata = [] - if 'inheritedfrom' in currattr: - blamedata.append('inherited from group {0}'.format( - currattr['inheritedfrom'] - )) - if 'expression' in currattr: - blamedata.append( - 'derived from expression "{0}"'.format( - currattr['expression'])) - if blamedata: - attrout += ' (' + ', '.join(blamedata) + ')' - print attrout - - if not exitcode: - if requestargs: - for attr in args[1:]: - if attr not in seenattributes: - sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) - exitcode = 1 + exitcode = client.printattributes(session, args, requestargs, showtype,nodetype, noderange, options) else: for res in session.read(nodelist): if 'error' in res: @@ -162,4 +85,5 @@ else: exitcode = 1 else: print res['item']['href'].replace('/', '') + sys.exit(exitcode) \ No newline at end of file diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist index 92ea8320..89c3fe06 100755 --- a/confluent_client/bin/nodelist +++ b/confluent_client/bin/nodelist @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__author__ = 'jjohnson2' +__author__ = 'jjohnson2,alin37' import optparse import os @@ -28,91 +28,35 @@ if path.startswith('/opt'): import confluent.client as client - -def attrrequested(attr, attrlist, seenattributes): - for candidate in attrlist: - truename = candidate - if candidate.startswith('hm'): - candidate = candidate.replace('hm', 'hardwaremanagement', 1) - if candidate == attr: - seenattributes.add(truename) - return True - elif '.' not in candidate and attr.startswith(candidate + '.'): - seenattributes.add(truename) - return True - return False -argparser = optparse.OptionParser( +def main(): + argparser = optparse.OptionParser( usage="Usage: %prog [options] noderange [list of attributes]") -argparser.add_option('-b', '--blame', action='store_true', + argparser.add_option('-b', '--blame', action='store_true', help='Show information about how attributes inherited') -(options, args) = argparser.parse_args() -try: - noderange = args[0] - nodelist = '/noderange/{0}/nodes/'.format(noderange) -except IndexError: - nodelist = '/nodes/' -session = client.Command() -exitcode = 0 -if len(args) > 1: - seenattributes = set([]) - for res in session.read('/noderange/{0}/attributes/all'.format(noderange)): - if 'error' in res: - sys.stderr.write(res['error'] + '\n') - exitcode = 1 - continue - for node in res['databynode']: - for attr in res['databynode'][node]: - seenattributes.add(attr) - currattr = res['databynode'][node][attr] - if attrrequested(attr, args[1:], seenattributes): - if 'value' in currattr: - if currattr['value'] is not None: - attrout = '{0}: {1}: {2}'.format( - node, attr, currattr['value']) - else: - attrout = '{0}: {1}:'.format(node, attr) - elif 'isset' in currattr: - if currattr['isset']: - attrout = '{0}: {1}: ********'.format(node, attr) - else: - attrout = '{0}: {1}:'.format(node, attr) - elif 'broken' in currattr: - attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \ - '{2}'.format(node, attr, - currattr['broken']) - elif isinstance(currattr, list) or isinstance(currattr, tuple): - attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, currattr))) - elif isinstance(currattr, dict): - dictout = [] - for k, v in currattr.items: - dictout.append("{0}={1}".format(k, v)) - attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, dictout))) - else: - print ("CODE ERROR" + repr(attr)) - - if options.blame or 'broken' in currattr: - blamedata = [] - if 'inheritedfrom' in currattr: - blamedata.append('inherited from group {0}'.format( - currattr['inheritedfrom'] - )) - if 'expression' in currattr: - blamedata.append( - 'derived from expression "{0}"'.format( - currattr['expression'])) - if blamedata: - attrout += ' (' + ', '.join(blamedata) + ')' - print attrout - if not exitcode: - for attr in args[1:]: - if attr not in seenattributes: - sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) + (options, args) = argparser.parse_args() + noderange="" + nodelist="" + try: + noderange = args[0] + nodelist = '/noderange/{0}/nodes/'.format(noderange) + except IndexError: + nodelist = '/nodes/' + session = client.Command() + exitcode = 0 + showtype='all' + requestargs=args[1:] + nodetype='noderange' + if len(args) > 1: + exitcode=client.printattributes(session,args,requestargs,showtype,nodetype,noderange,options) + else: + for res in session.read(nodelist): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') exitcode = 1 -else: - for res in session.read(nodelist): - if 'error' in res: - sys.stderr.write(res['error'] + '\n') - exitcode = 1 - else: - print res['item']['href'].replace('/', '') -sys.exit(exitcode) \ No newline at end of file + else: + print res['item']['href'].replace('/', '') + + sys.exit(exitcode) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index 5d213003..cfa9d663 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -245,5 +245,106 @@ def send_request(operation, path, server, parameters=None): raise result = tlvdata.recv(server) +def attrrequested(attr, attrlist, seenattributes): + for candidate in attrlist: + truename = candidate + if candidate.startswith('hm'): + candidate = candidate.replace('hm', 'hardwaremanagement', 1) + if candidate == attr: + seenattributes.add(truename) + return True + elif '.' not in candidate and attr.startswith(candidate + '.'): + seenattributes.add(truename) + return True + return False + +def printattributes(session,args,requestargs,showtype,nodetype, noderange,options): + exitcode=0 + seenattributes = set([]) + for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype,noderange,showtype)): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + continue + for node in res['databynode']: + for attr in res['databynode'][node]: + seenattributes.add(attr) + currattr = res['databynode'][node][attr] + if (requestargs is None or requestargs == [] or attrrequested(attr, args[1:], seenattributes)): + if 'value' in currattr: + if currattr['value'] is not None: + attrout = '{0}: {1}: {2}'.format( + node, attr, currattr['value']) + else: + attrout = '{0}: {1}:'.format(node, attr) + elif 'isset' in currattr: + if currattr['isset']: + attrout = '{0}: {1}: ********'.format(node, attr) + else: + attrout = '{0}: {1}:'.format(node, attr) + elif 'broken' in currattr: + attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \ + '{2}'.format(node, attr, + currattr['broken']) + elif isinstance(currattr, list) or isinstance(currattr, tuple): + attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, currattr))) + elif isinstance(currattr, dict): + dictout = [] + for k, v in currattr.items: + dictout.append("{0}={1}".format(k, v)) + attrout = '{0}: {1}: {2}'.format(node, attr, ', '.join(map(str, dictout))) + else: + print ("CODE ERROR" + repr(attr)) + + if options.blame or 'broken' in currattr: + blamedata = [] + if 'inheritedfrom' in currattr: + blamedata.append('inherited from group {0}'.format( + currattr['inheritedfrom'] + )) + if 'expression' in currattr: + blamedata.append( + 'derived from expression "{0}"'.format( + currattr['expression'])) + if blamedata: + attrout += ' (' + ', '.join(blamedata) + ')' + print attrout + if not exitcode: + if requestargs: + for attr in args[1:]: + if attr not in seenattributes: + sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) + exitcode = 1 + return exitcode +def updateattrib(session,args,nodetype, noderange, options ): + #update attribute + exitcode=0 + if options.clear: + targpath = '/{0}/{1}/attributes/all'.format(nodetype,noderange) + keydata = {} + for attrib in args[1:]: + keydata[attrib] = None + for res in session.update(targpath, keydata): + if 'error' in res: + if 'errorcode' in res: + exitcode = res['errorcode'] + sys.stderr.write('Error: ' + res['error'] + '\n') + sys.exit(exitcode) + else: + if "=" in args[1]: + try: + if len(args[1:]) > 1: + for val in args[1:]: + val = val.split('=') + exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0]) + else: + val=args[1].split('=') + exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0]) + except: + sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:]))) + exitcode = 1 + sys.exit(exitcode) + return exitcode + From 563a2b11c3e6984b639d84309f008edc42e76813 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Apr 2017 14:39:29 -0400 Subject: [PATCH 02/13] Change the stdout/stderr/trace logs to not buffer things over time If something went completely off the rails, it could easily fill up lots of memory with log entries in the 2 seconds it would buffer. For now disable the buffering on key debug logs, as the main purpose was reducing IOPs in the per-node console logs anyway. A future behavior may be to also limit the size and/or number of outstanding log entries before committing to disk. --- confluent_server/confluent/log.py | 12 ++++++++---- confluent_server/confluent/main.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/log.py b/confluent_server/confluent/log.py index 5e7c33a1..c916dcfd 100644 --- a/confluent_server/confluent/log.py +++ b/confluent_server/confluent/log.py @@ -505,11 +505,12 @@ class Logger(object): else: return object.__new__(cls) - def __init__(self, logname, console=False, tenant=None): + def __init__(self, logname, console=False, tenant=None, buffered=True): if hasattr(self, 'initialized'): # we are just a copy of the same object return self.initialized = True + self.buffered = buffered self.filepath = confluent.config.configmanager.get_global("logdirectory") if self.filepath is None: if os.name == 'nt': @@ -727,8 +728,11 @@ class Logger(object): else: self.logentries.append( [ltype, timestamp, logdata, event, eventdata]) - if self.writer is None: - self.writer = eventlet.spawn_after(2, self.writedata) + if self.buffered: + if self.writer is None: + self.writer = eventlet.spawn_after(2, self.writedata) + else: + self.writedata() def closelog(self): self.handler.close() @@ -746,6 +750,6 @@ def log(logdata=None, ltype=None, event=0, eventdata=None): def logtrace(): global tracelog if tracelog is None: - tracelog = Logger('trace') + tracelog = Logger('trace', buffered=False) tracelog.log(traceback.format_exc(), ltype=DataTypes.event, event=Events.stacktrace) \ No newline at end of file diff --git a/confluent_server/confluent/main.py b/confluent_server/confluent/main.py index cd26fba5..7457dbdf 100644 --- a/confluent_server/confluent/main.py +++ b/confluent_server/confluent/main.py @@ -77,8 +77,8 @@ def _daemonize(): os.open(os.devnull, os.O_RDWR) os.dup2(0, 1) os.dup2(0, 2) - sys.stdout = log.Logger('stdout') - sys.stderr = log.Logger('stderr') + sys.stdout = log.Logger('stdout', buffered=False) + sys.stderr = log.Logger('stderr', buffered=False) def _updatepidfile(): From e230f803ce85be23415bb3fdaedb403686e08888 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 11 Apr 2017 15:41:26 -0400 Subject: [PATCH 03/13] Correct missing argument __new__ was missed in the last commit. --- confluent_server/confluent/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/log.py b/confluent_server/confluent/log.py index c916dcfd..c63ddf12 100644 --- a/confluent_server/confluent/log.py +++ b/confluent_server/confluent/log.py @@ -494,7 +494,7 @@ class Logger(object): False, events will be formatted like syslog: date: message """ - def __new__(cls, logname, console=False, tenant=None): + def __new__(cls, logname, console=False, tenant=None, buffered=True): global _loggers if console: relpath = 'consoles/' + logname From db15992894d30b487525a8151f061e4d1b70d493 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Wed, 12 Apr 2017 10:36:27 -0400 Subject: [PATCH 04/13] Adding nodegroupattrib to update nodegroups --- confluent_client/bin/nodeattrib | 17 ++- confluent_client/bin/nodegroupattrib | 98 +++++++++++++ confluent_client/confluent/client.py | 133 +++++++++++++++--- .../confluent/config/configmanager.py | 32 +++-- 4 files changed, 242 insertions(+), 38 deletions(-) create mode 100644 confluent_client/bin/nodegroupattrib diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index 4d6678eb..c36942d2 100644 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -54,14 +54,18 @@ exitcode = 0 nodetype="noderange" if len(args) > 1: - exitcode=client.updateattrib(session,args,nodetype, noderange, options) + if "=" in args[1]: + exitcode=client.updateattrib(session,args,nodetype, noderange, options) try: # setting user output to what the user inputs if args[1] == 'all': showtype = 'all' + requestargs=args[2:] elif args[1] == 'current': showtype = 'current' - requestargs=args[2:] + requestargs=args[2:] + else: + requestargs=args[1:] except: pass @@ -76,8 +80,13 @@ if len(args) > 0: elif requestargs == []: #showtype already set pass - - exitcode = client.printattributes(session, args, requestargs, showtype,nodetype, noderange, options) + else: + try: + requestargs.remove('all') + requestargs.remove('current') + except ValueError: + pass + exitcode = client.printattributes(session, requestargs, showtype,nodetype, noderange, options) else: for res in session.read(nodelist): if 'error' in res: diff --git a/confluent_client/bin/nodegroupattrib b/confluent_client/bin/nodegroupattrib new file mode 100644 index 00000000..f0fa7051 --- /dev/null +++ b/confluent_client/bin/nodegroupattrib @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2017 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'alin37' + +import optparse +import os +import sys + +path = os.path.dirname(os.path.realpath(__file__)) +path = os.path.realpath(os.path.join(path, '..', 'lib', 'python')) +if path.startswith('/opt'): + sys.path.append(path) + +import confluent.client as client + +argparser = optparse.OptionParser( + usage='''\n %prog [options] \ + \n %prog [options] nodegroup [list of attributes] \ + \n %prog [options] nodegroup group=value1,value2 \ + \n %prog [options] nodegroup group=value1,value2 + \n ''') +argparser.add_option('-b', '--blame', action='store_true', + help='Show information about how attributes inherited') +argparser.add_option('-c', '--clear', action='store_true', + help='Clear variables') +(options, args) = argparser.parse_args() + + +#setting minimal output to only output current information +showtype = 'current' +requestargs=None +nodetype="nodegroups" + +try: + nodegroups = args[0] + nodelist = '/{0}/{1}/'.format(nodetype,nodegroups) +except IndexError: + nodelist = '/nodegroups/' +session = client.Command() +exitcode = 0 + +#Sets attributes + +if len(args) > 1: + exitcode=client.updateattrib(session,args,nodetype, nodegroups, options) + try: + # setting user output to what the user inputs + if args[1] == 'all': + showtype = 'all' + elif args[1] == 'current': + showtype = 'current' + + requestargs=args[1:] + except Exception as e: + print str(e) + +if exitcode != 0: + sys.exit(exitcode) + +# Lists all attributes +if len(args) > 0: + # setting output to all so it can search since if we do have something to search, we want to show all outputs even if it is blank. + if requestargs is None: + showtype = 'current' + elif requestargs == []: + #showtype already set + pass + else: + try: + requestargs.remove('all') + requestargs.remove('current') + except ValueError: + pass + exitcode = client.printgroupattributes(session, requestargs, showtype,nodetype, nodegroups, options) +else: + for res in session.read(nodelist): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + else: + print res['item']['href'].replace('/', '') + +sys.exit(exitcode) \ No newline at end of file diff --git a/confluent_client/confluent/client.py b/confluent_client/confluent/client.py index cfa9d663..3d14ef90 100644 --- a/confluent_client/confluent/client.py +++ b/confluent_client/confluent/client.py @@ -42,7 +42,6 @@ def _parseserver(string): class Command(object): - def __init__(self, server=None): self._prevkeyname = None self.connection = None @@ -135,7 +134,29 @@ class Command(object): print('') return 0 - + def simple_nodegroups_command(self, noderange, resource, input=None, key=None, **kwargs): + try: + rc = 0 + if resource[0] == '/': + resource = resource[1:] + # The implicit key is the resource basename + if key is None: + ikey = resource.rpartition('/')[-1] + else: + ikey = key + if input is None: + for res in self.read('/nodegroups/{0}/{1}'.format( + noderange, resource)): + rc = self.handle_results(ikey, rc, res) + else: + kwargs[ikey] = input + for res in self.update('/nodegroups/{0}/{1}'.format( + noderange, resource), kwargs): + rc = self.handle_results(ikey, rc, res) + return rc + except KeyboardInterrupt: + print('') + return 0 def read(self, path, parameters=None): if not self.authenticated: @@ -206,7 +227,7 @@ class Command(object): if knownhosts: certdata = self.connection.getpeercert(binary_form=True) fingerprint = 'sha512$' + hashlib.sha512(certdata).hexdigest() - hostid = '@'.join((port,server)) + hostid = '@'.join((port, server)) khf = dbm.open(os.path.join(clientcfgdir, "knownhosts"), 'c', 384) if hostid in khf: if fingerprint == khf[hostid]: @@ -220,7 +241,6 @@ class Command(object): khf[hostid] = fingerprint - def send_request(operation, path, server, parameters=None): """This function iterates over all the responses received from the server. @@ -245,6 +265,7 @@ def send_request(operation, path, server, parameters=None): raise result = tlvdata.recv(server) + def attrrequested(attr, attrlist, seenattributes): for candidate in attrlist: truename = candidate @@ -258,10 +279,11 @@ def attrrequested(attr, attrlist, seenattributes): return True return False -def printattributes(session,args,requestargs,showtype,nodetype, noderange,options): - exitcode=0 + +def printattributes(session, requestargs, showtype, nodetype, noderange, options): + exitcode = 0 seenattributes = set([]) - for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype,noderange,showtype)): + for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)): if 'error' in res: sys.stderr.write(res['error'] + '\n') exitcode = 1 @@ -270,7 +292,7 @@ def printattributes(session,args,requestargs,showtype,nodetype, noderange,option for attr in res['databynode'][node]: seenattributes.add(attr) currattr = res['databynode'][node][attr] - if (requestargs is None or requestargs == [] or attrrequested(attr, args[1:], seenattributes)): + if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)): if 'value' in currattr: if currattr['value'] is not None: attrout = '{0}: {1}: {2}'.format( @@ -311,20 +333,78 @@ def printattributes(session,args,requestargs,showtype,nodetype, noderange,option print attrout if not exitcode: if requestargs: - for attr in args[1:]: + for attr in requestargs: if attr not in seenattributes: sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) exitcode = 1 return exitcode -def updateattrib(session,args,nodetype, noderange, options ): - #update attribute - exitcode=0 +def printgroupattributes(session, requestargs, showtype, nodetype, noderange, options): + exitcode = 0 + seenattributes = set([]) + for res in session.read('/{0}/{1}/attributes/{2}'.format(nodetype, noderange, showtype)): + if 'error' in res: + sys.stderr.write(res['error'] + '\n') + exitcode = 1 + continue + for attr in res: + seenattributes.add(attr) + currattr = res[attr] + if (requestargs is None or requestargs == [] or attrrequested(attr, requestargs, seenattributes)): + if 'value' in currattr: + if currattr['value'] is not None: + attrout = '{0}: {1}: {2}'.format( + noderange, attr, currattr['value']) + else: + attrout = '{0}: {1}:'.format(noderange, attr) + elif 'isset' in currattr: + if currattr['isset']: + attrout = '{0}: {1}: ********'.format(noderange, attr) + else: + attrout = '{0}: {1}:'.format(noderange, attr) + elif 'broken' in currattr: + attrout = '{0}: {1}: *ERROR* BROKEN EXPRESSION: ' \ + '{2}'.format(noderange, attr, + currattr['broken']) + elif isinstance(currattr, list) or isinstance(currattr, tuple): + attrout = '{0}: {1}: {2}'.format(noderange, attr, ', '.join(map(str, currattr))) + elif isinstance(currattr, dict): + dictout = [] + for k, v in currattr.items: + dictout.append("{0}={1}".format(k, v)) + attrout = '{0}: {1}: {2}'.format(noderange, attr, ', '.join(map(str, dictout))) + else: + print ("CODE ERROR" + repr(attr)) + + if options.blame or 'broken' in currattr: + blamedata = [] + if 'inheritedfrom' in currattr: + blamedata.append('inherited from group {0}'.format( + currattr['inheritedfrom'] + )) + if 'expression' in currattr: + blamedata.append( + 'derived from expression "{0}"'.format( + currattr['expression'])) + if blamedata: + attrout += ' (' + ', '.join(blamedata) + ')' + print attrout + if not exitcode: + if requestargs: + for attr in requestargs: + if attr not in seenattributes: + sys.stderr.write('Error: {0} not a valid attribute\n'.format(attr)) + exitcode = 1 + return exitcode + +def updateattrib(session, updateargs, nodetype, noderange, options): + # update attribute + exitcode = 0 if options.clear: - targpath = '/{0}/{1}/attributes/all'.format(nodetype,noderange) + targpath = '/{0}/{1}/attributes/all'.format(nodetype, noderange) keydata = {} - for attrib in args[1:]: + for attrib in updateargs[1:]: keydata[attrib] = None for res in session.update(targpath, keydata): if 'error' in res: @@ -333,18 +413,27 @@ def updateattrib(session,args,nodetype, noderange, options ): sys.stderr.write('Error: ' + res['error'] + '\n') sys.exit(exitcode) else: - if "=" in args[1]: + if "=" in updateargs[1]: try: - if len(args[1:]) > 1: - for val in args[1:]: + if len(updateargs[1:]) > 1: + for val in updateargs[1:]: val = val.split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), val[1], val[0]) + if (nodetype == "nodegroups"): + exitcode = session.simple_nodegroups_command(noderange, 'attributes/all'.format(noderange), + val[1],val[0]) + else: + exitcode = session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) else: - val=args[1].split('=') - exitcode=session.simple_noderange_command(noderange, 'attributes/all'.format(noderange),val[1],val[0]) + val = updateargs[1].split('=') + if nodetype == "nodegroups" : + exitcode = session.simple_nodegroups_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) + else: + exitcode = session.simple_noderange_command(noderange, 'attributes/all'.format(noderange), + val[1], val[0]) except: - sys.stderr.write('Error: {0} not a valid expression\n'.format(str (args[1:]))) + sys.stderr.write('Error: {0} not a valid expression\n'.format(str(updateargs[1:]))) exitcode = 1 sys.exit(exitcode) return exitcode - diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 4a36a97d..329b8320 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -920,11 +920,13 @@ class ConfigManager(object): ('type' in allattributes.node[attr] and not isinstance(attribmap[group][attr], allattributes.node[attr]['type'])))): - raise ValueError + raise ValueError("nodes attribute is invalid") if attr == 'nodes': if not isinstance(attribmap[group][attr], list): - raise ValueError( - "nodes attribute on group must be list") + if type(attribmap[group][attr]) is unicode or type(attribmap[group][attr]) is str: + attribmap[group][attr]=attribmap[group][attr].split(",") + else: + raise ValueError("nodes attribute on group must be list") for node in attribmap[group]['nodes']: if node not in self._cfgstore['nodes']: raise ValueError( @@ -1136,7 +1138,21 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] - if (attrname not in allattributes.node or + try: + if (allattributes.node[attrname]['type'] == 'list' and + type(attrval) in (str, unicode)): + attrval = attrval.split(",") + except KeyError: + pass + if attrname == 'groups': + for group in attribmap[node]['groups'].split(","): + if group not in self._cfgstore['nodegroups']: + raise ValueError( + "group {0} does not exist".format(group)) + if ('everything' in self._cfgstore['nodegroups'] and + 'everything' not in attribmap[node]['groups']): + attribmap[node]['groups'].append('everything') + elif (attrname not in allattributes.node or ('type' in allattributes.node[attrname] and not isinstance( attrval, @@ -1144,14 +1160,6 @@ class ConfigManager(object): errstr = "{0} attribute on node {1} is invalid".format( attrname, node) raise ValueError(errstr) - if attrname == 'groups': - for group in attribmap[node]['groups']: - if group not in self._cfgstore['nodegroups']: - raise ValueError( - "group {0} does not exist".format(group)) - if ('everything' in self._cfgstore['nodegroups'] and - 'everything' not in attribmap[node]['groups']): - attribmap[node]['groups'].append('everything') for node in attribmap.iterkeys(): node = node.encode('utf-8') exprmgr = None From 43211cf9a7fd040e4ac070e70f12331ce9c96e8a Mon Sep 17 00:00:00 2001 From: Allen Lin Date: Wed, 12 Apr 2017 10:50:44 -0400 Subject: [PATCH 05/13] Updating file permissions on nodegroupattrib and nodeattrib --- confluent_client/bin/nodeattrib | 0 confluent_client/bin/nodegroupattrib | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 confluent_client/bin/nodeattrib mode change 100644 => 100755 confluent_client/bin/nodegroupattrib diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib old mode 100644 new mode 100755 diff --git a/confluent_client/bin/nodegroupattrib b/confluent_client/bin/nodegroupattrib old mode 100644 new mode 100755 From 2042bcb8f096c762acd1dfd2178177a06831fa45 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 13 Apr 2017 17:36:41 -0400 Subject: [PATCH 06/13] updating server side for use with nodegroupattrib --- confluent_server/confluent/config/configmanager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 329b8320..7a54895a 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -896,6 +896,11 @@ class ConfigManager(object): if node not in self._cfgstore['nodes']: self._cfgstore['nodes'][node] = {'groups': [group]} _mark_dirtykey('nodes', node, self.tenant) + elif 'value' in self._cfgstore['nodes'][node]['groups']: + tmplist=self._cfgstore['nodes'][node]['groups']['value'].split(",") + tmplist.insert(0, unicode(group)) + self._cfgstore['nodes'][node]['groups']['value']=",".join(tmplist) + _mark_dirtykey('nodes', node, self.tenant) elif group not in self._cfgstore['nodes'][node]['groups']: self._cfgstore['nodes'][node]['groups'].insert(0, group) _mark_dirtykey('nodes', node, self.tenant) @@ -1138,14 +1143,18 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] + print attrval + print allattributes.node[attrname] try: if (allattributes.node[attrname]['type'] == 'list' and - type(attrval) in (str, unicode)): + type(attrval) in (str, unicode)) and type(attrval)==list: attrval = attrval.split(",") except KeyError: pass if attrname == 'groups': - for group in attribmap[node]['groups'].split(","): + if type(attribmap[node]['groups']) != list: + attribmap[node]['groups']=attribmap[node]['groups'].split(",") + for group in attribmap[node]['groups']: if group not in self._cfgstore['nodegroups']: raise ValueError( "group {0} does not exist".format(group)) From 86e502a47a87cb6ddc2062c8467fa471af7acde2 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 13 Apr 2017 22:07:47 -0400 Subject: [PATCH 07/13] remove debug and fixing issue checking attribute --- confluent_server/confluent/config/configmanager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 7a54895a..71efda93 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -1143,11 +1143,9 @@ class ConfigManager(object): raise ValueError("node {0} does not exist".format(node)) for attrname in attribmap[node].iterkeys(): attrval = attribmap[node][attrname] - print attrval - print allattributes.node[attrname] try: if (allattributes.node[attrname]['type'] == 'list' and - type(attrval) in (str, unicode)) and type(attrval)==list: + type(attrval) in (str, unicode)): attrval = attrval.split(",") except KeyError: pass From 27294103cfe0199123f43216c6aa7f08cc16d043 Mon Sep 17 00:00:00 2001 From: Amanda Duffy Date: Thu, 20 Apr 2017 14:30:33 -0400 Subject: [PATCH 08/13] Complain if not enough args --- confluent_client/bin/nodeidentify | 3 +++ 1 file changed, 3 insertions(+) diff --git a/confluent_client/bin/nodeidentify b/confluent_client/bin/nodeidentify index 17c397d9..2cbf573f 100755 --- a/confluent_client/bin/nodeidentify +++ b/confluent_client/bin/nodeidentify @@ -37,6 +37,9 @@ except IndexError: identifystate = None if len(sys.argv) > 2: identifystate = sys.argv[2] +else: + argparser.print_help() + sys.exit(1) session = client.Command() exitcode = 0 sys.exit( From b64858e0b3c9d971cd829593294f4a8340ccc242 Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Thu, 20 Apr 2017 15:41:11 -0400 Subject: [PATCH 09/13] Removing unnecessary case if groups ever has a key of 'value' --- confluent_server/confluent/config/configmanager.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/confluent_server/confluent/config/configmanager.py b/confluent_server/confluent/config/configmanager.py index 71efda93..d9600a3a 100644 --- a/confluent_server/confluent/config/configmanager.py +++ b/confluent_server/confluent/config/configmanager.py @@ -896,11 +896,6 @@ class ConfigManager(object): if node not in self._cfgstore['nodes']: self._cfgstore['nodes'][node] = {'groups': [group]} _mark_dirtykey('nodes', node, self.tenant) - elif 'value' in self._cfgstore['nodes'][node]['groups']: - tmplist=self._cfgstore['nodes'][node]['groups']['value'].split(",") - tmplist.insert(0, unicode(group)) - self._cfgstore['nodes'][node]['groups']['value']=",".join(tmplist) - _mark_dirtykey('nodes', node, self.tenant) elif group not in self._cfgstore['nodes'][node]['groups']: self._cfgstore['nodes'][node]['groups'].insert(0, group) _mark_dirtykey('nodes', node, self.tenant) From ca25e9fa783c0179ae8378e925d58f39a97d0c5f Mon Sep 17 00:00:00 2001 From: Allen Lin37 Date: Tue, 25 Apr 2017 13:38:01 -0400 Subject: [PATCH 10/13] Fix nodelist to use correct number of arguments --- confluent_client/bin/nodelist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/bin/nodelist b/confluent_client/bin/nodelist index 89c3fe06..ef3816e6 100755 --- a/confluent_client/bin/nodelist +++ b/confluent_client/bin/nodelist @@ -47,7 +47,7 @@ def main(): requestargs=args[1:] nodetype='noderange' if len(args) > 1: - exitcode=client.printattributes(session,args,requestargs,showtype,nodetype,noderange,options) + exitcode=client.printattributes(session,requestargs,showtype,nodetype,noderange,options) else: for res in session.read(nodelist): if 'error' in res: From cf97bbe2996d1f413876c10cc9f3ca9a552aa37a Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 25 Apr 2017 15:26:56 -0400 Subject: [PATCH 11/13] Rework ipmi worker management Provide a more concrete measurement of children, rather than relying upon a sentinel value on the queue. It seems that even using 'finally' didn't assure that we always get that sentinel value before a worker dies. Sentinel value still used to avoid a long wait in the usual case. --- .../plugins/hardwaremanagement/ipmi.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index ce92b869..ff8adbb2 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -273,17 +273,23 @@ def perform_requests(operator, nodes, element, cfg, inputdata): configdata = cfg.get_node_attributes(nodes, _configattributes) cfg.decrypt = cryptit resultdata = queue.LightQueue() - pendingnum = len(nodes) + livingthreads = set([]) for node in nodes: - _ipmiworkers.spawn_n( + livingthreads.add(_ipmiworkers.spawn( perform_request, operator, node, element, configdata, inputdata, - cfg, resultdata) - while pendingnum: - datum = resultdata.get() - if datum == 'Done': - pendingnum -= 1 - else: - yield datum + cfg, resultdata)) + while livingthreads: + try: + datum = resultdata.get(timeout=10) + while datum: + if datum != 'Done': + yield datum + datum = resultdata.get_nowait() + except queue.Empty: + pass + for t in list(livingthreads): + if t.dead: + livingthreads.discard(t) def perform_request(operator, node, element, From 6ab4d3c88658b8bce1772901b2e139957019ba0f Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 25 Apr 2017 16:48:35 -0400 Subject: [PATCH 12/13] Add a recourse to drain queue While it may not have been possible in eventlet for this to happen, strictly speaking if it were a thread, it could exit during check for liveness and leave data on the queue. To be careful, also drain the queue after all children dead. --- .../confluent/plugins/hardwaremanagement/ipmi.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index ff8adbb2..f4278a90 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -290,6 +290,15 @@ def perform_requests(operator, nodes, element, cfg, inputdata): for t in list(livingthreads): if t.dead: livingthreads.discard(t) + try: + # drain queue if a thread put something on the queue and died + while True: + datum = resultdata.get_nowait() + if datum != 'Done': + yield datum + except queue.Empty: + pass + def perform_request(operator, node, element, From 68037473d0b9d2bf18a06e5803fb0fae382e4a04 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 26 Apr 2017 14:26:00 -0400 Subject: [PATCH 13/13] Replace login event mechanism Provide a different scheme that does not involve a wait(), if by chance the flow dies without getting back to our thread. wait() has no timeout so this is a strategy to cope by making sure we hang for no longer than 3 minutes, which is well beyond any time a login should possibly take. --- .../confluent/plugins/hardwaremanagement/ipmi.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py index f4278a90..ff956a9a 100644 --- a/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py +++ b/confluent_server/confluent/plugins/hardwaremanagement/ipmi.py @@ -342,10 +342,8 @@ class IpmiHandler(object): self.ipmicmd = None self.inputdata = inputdata tenant = cfg.tenant - self._logevt = None if ((node, tenant) not in persistent_ipmicmds or not persistent_ipmicmds[(node, tenant)].ipmi_session.logged): - self._logevt = threading.Event() try: persistent_ipmicmds[(node, tenant)].close_confluent() except KeyError: # was no previous session @@ -356,6 +354,14 @@ class IpmiHandler(object): userid=connparams['username'], password=connparams['passphrase'], kg=connparams['kg'], port=connparams['port'], onlogon=self.logged) + ipmisess = persistent_ipmicmds[(node, tenant)].ipmi_session + begin = util.monotonic_time() + while ((not (self.broken or self.loggedin)) and + (util.monotonic_time() - begin) < 180): + ipmisess.wait_for_rsp(180) + if not (self.broken or self.loggedin): + raise exc.TargetEndpointUnreachable( + "Login process to " + bmc + " died") except socket.gaierror as ge: if ge[0] == -2: raise exc.TargetEndpointUnreachable(ge[1]) @@ -374,12 +380,8 @@ class IpmiHandler(object): self.ipmicmd = ipmicmd self.loggedin = True self.ipmicmd.setup_confluent_keyhandler() - self._logevt.set() def handle_request(self): - if self._logevt is not None: - self._logevt.wait() - self._logevt = None if self.broken: if (self.error == 'timeout' or 'Insufficient resources' in self.error):