diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index 8c6f078d..f4b0331f 100755 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -22,6 +22,7 @@ import optparse import os import signal import sys +import shlex try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) diff --git a/confluent_client/bin/nodersync b/confluent_client/bin/nodersync index ba95662c..8d316bab 100755 --- a/confluent_client/bin/nodersync +++ b/confluent_client/bin/nodersync @@ -48,6 +48,8 @@ def run(): 'prompting if over the threshold') argparser.add_option('-l', '--loginname', type='str', help='Username to use when connecting, defaults to current user.') + argparser.add_option('-s', '--substitutename', + help='Use a different name other than the nodename for rsync') argparser.add_option('-f', '-c', '--count', type='int', default=168, help='Number of nodes to concurrently rsync') # among other things, FD_SETSIZE limits. Besides, spawning too many @@ -63,10 +65,20 @@ def run(): c = client.Command() cmdstr = ' '.join(args[:-1]) cmdstr = 'rsync -av --info=progress2 ' + cmdstr - if options.loginname: - cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath + + targname = options.substitutename + if targname and '{' in targname: + targname = targname + ':' + elif targname: + targname = '{node}' + targname + ':' else: - cmdstr += ' {node}:' + targpath + targname = '{node}:' + + if options.loginname: + cmdstr += ' {}@'.format(options.loginname) + targname + targpath + else: + cmdstr += ' {}'.format(targname) + targpath + currprocs = 0 all = set([]) diff --git a/confluent_client/bin/stats b/confluent_client/bin/stats index 7158e7a7..94af75db 100755 --- a/confluent_client/bin/stats +++ b/confluent_client/bin/stats @@ -16,13 +16,10 @@ # limitations under the License. import argparse +import base64 import csv -import fcntl import io import numpy as np - -import os -import subprocess import sys try: @@ -35,7 +32,31 @@ except ImportError: pass -def plot(gui, output, plotdata, bins): +def iterm_draw(data): + databuf = data.getbuffer() + datalen = len(databuf) + data = base64.b64encode(databuf).decode('utf8') + sys.stdout.write( + '\x1b]1337;File=inline=1;size={}:'.format(datalen)) + sys.stdout.write(data) + sys.stdout.write('\a') + sys.stdout.write('\n') + sys.stdout.flush() + + +def kitty_draw(data): + data = base64.b64encode(data.getbuffer()) + while data: + chunk, data = data[:4096], data[4096:] + m = 1 if data else 0 + sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m)) + sys.stdout.write(chunk.decode('utf8')) + sys.stdout.write('\x1b\\') + sys.stdout.flush() + sys.stdout.write('\n') + + +def plot(gui, output, plotdata, bins, fmt): import matplotlib as mpl if gui and mpl.get_backend() == 'agg': sys.stderr.write('Error: No GUI backend available and -g specified!\n') @@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins): tdata = io.BytesIO() plt.savefig(tdata) if not gui and not output: - writer = DumbWriter() - writer.draw(tdata) + if fmt == 'sixel': + writer = DumbWriter() + writer.draw(tdata) + elif fmt == 'kitty': + kitty_draw(tdata) + elif fmt == 'iterm': + iterm_draw(tdata) return n, bins def textplot(plotdata, bins): @@ -81,7 +107,8 @@ histogram = False aparser = argparse.ArgumentParser(description='Quick access to common statistics') aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)') aparser.add_argument('-d', default=None, help='Value used to separate columns') -aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format') +aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format') +aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)') aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing') aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window') aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format') @@ -138,7 +165,7 @@ while data: data = list(csv.reader([data], delimiter=delimiter))[0] n = None if args.g or args.o or args.x: - n, bins = plot(args.g, args.o, plotdata, bins=args.b) + n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f) if args.t: n, bins = textplot(plotdata, bins=args.b) print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata))) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index b1330198..c8127ad8 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -7,8 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes `nodeattrib [ ...]` `nodeattrib -c ...` `nodeattrib -e ...` -`nodeattrib -p ...` -`nodeattrib -s ` +`nodeattrib -p ...` +`nodeattrib -s ...` ## DESCRIPTION @@ -54,14 +54,17 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-e`, `--environment`: Set specified attributes based on exported environment variable of matching name. Environment variable names may be lower case or all upper case. - Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE + Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE) * `-p`, `--prompt`: Request interactive prompting to provide values rather than the command line or environment variables. * `-s`, `--set`: - Set attributes using a batch file + Set attributes using a batch file rather than the command line. The attributes in the batch file + can be specified as one line of key=value pairs simmilar to command line or each attribute can + be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch + file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: Prompt if trying to set attributes on more than @@ -120,6 +123,25 @@ to a blank value will allow masking a group defined attribute with an empty valu `d1: net.pxe.switch: pxeswitch1` `d1: net.switch:` +* Setting attributes using a batch file with syntax similar to command line: + `# cat nodeattributes.batch` + `# power` + `power.psu1.outlet=3 power.psu1.pdu=pdu2` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: 3` + `n41: pdu2` + +* Setting attributes using a batch file with syntax where each attribute is in its own line: + `# cat nodeattributes.batch` + `# management` + `custom.mgt.switch=switch_main` + `custom.mgt.switch.port=swp4` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: switch_main` + `n41: swp4` + + + ## SEE ALSO nodegroupattrib(8), nodeattribexpressions(5) diff --git a/confluent_client/doc/man/nodersync.ronn b/confluent_client/doc/man/nodersync.ronn index 582f79d9..c187215e 100644 --- a/confluent_client/doc/man/nodersync.ronn +++ b/confluent_client/doc/man/nodersync.ronn @@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes. Specify how many rsync executions to do concurrently. If noderange exceeds the count, then excess nodes will wait until one of the active count completes. + +* `-s`, `--substitutename`: + 'Use a different name other than the nodename for rsync' * `-m MAXNODES`, `--maxnodes=MAXNODES`: Specify a maximum number of nodes to run rsync to, prompting if over the diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 867c0102..83ffc519 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -1,3 +1,4 @@ +cd $(dirname $0) VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then @@ -29,4 +30,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/ mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/ rm -rf el9bin rm -rf el8bin -rpmbuild -ba confluent_osdeploy-aarch64.spec +podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec + diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index dec1808d..f2a2edff 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -435,10 +435,26 @@ if __name__ == '__main__': curridx = addr[-1] if curridx in doneidxs: continue - status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + for tries in (1, 2 3): + try: + status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue nc = json.loads(nc) if not dc: - status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + for tries in (1, 2 3): + try: + status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue dc = json.loads(dc) iname = get_interface_name(idxmap[curridx], nc.get('default', {})) if iname: diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 2f671d38..5db222a7 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -35,6 +35,7 @@ echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.con echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf systemctl restart sshd mkdir -p /etc/confluent +export nodename confluent_profile confluent_mgr curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions . /etc/confluent/functions run_remote_parts pre.d diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index a9ee1dba..f70bc6ae 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -446,6 +446,7 @@ def _init_core(): }, }, }, + 'layout': PluginRoute({'handler': 'layout'}), 'media': { 'uploads': PluginCollection({ 'pluginattrs': ['hardwaremanagement.method'], diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index 5a145a0c..f36f2c73 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -1084,7 +1084,7 @@ def serve(bind_host, bind_port): pass # we gave it our best shot there try: eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False, - debug=False, socket_timeout=60) + debug=False, socket_timeout=60, keepalive=False) except TypeError: # Older eventlet in place, skip arguments it does not understand eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False) diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py index a24a4d78..ce36344d 100644 --- a/confluent_server/confluent/messages.py +++ b/confluent_server/confluent/messages.py @@ -92,6 +92,7 @@ def msg_deserialize(packed): return cls(*m[1:]) raise Exception("Unknown shenanigans") + class ConfluentMessage(object): apicode = 200 readonly = False @@ -254,6 +255,21 @@ class ConfluentNodeError(object): raise Exception('{0}: {1}'.format(self.node, self.error)) +class Generic(ConfluentMessage): + + def __init__(self, data): + self.data = data + + def json(self): + return json.dumps(self.data) + + def raw(self): + return self.data + + def html(self): + return json.dumps(self.data) + + class ConfluentResourceUnavailable(ConfluentNodeError): apicode = 503 diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index 37e8d198..9e9fd597 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -25,6 +25,9 @@ import eventlet.support.greendns import os getaddrinfo = eventlet.support.greendns.getaddrinfo +eventlet.support.greendns.resolver.clear() +eventlet.support.greendns.resolver._resolver.lifetime = 1 + def msg_align(len): return (len + 3) & ~3 @@ -333,11 +336,13 @@ def get_full_net_config(configmanager, node, serverip=None): myaddrs = get_addresses_by_serverip(serverip) nm = NetManager(myaddrs, node, configmanager) defaultnic = {} + ppool = eventlet.greenpool.GreenPool(64) if None in attribs: - nm.process_attribs(None, attribs[None]) + ppool.spawn(nm.process_attribs, None, attribs[None]) del attribs[None] for netname in sorted(attribs): - nm.process_attribs(netname, attribs[netname]) + ppool.spawn(nm.process_attribs, netname, attribs[netname]) + ppool.waitall() retattrs = {} if None in nm.myattribs: retattrs['default'] = nm.myattribs[None] diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index b59bf7c6..df4552b8 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -55,6 +55,127 @@ def humanify_nodename(nodename): return [int(text) if text.isdigit() else text.lower() for text in re.split(numregex, nodename)] +def unnumber_nodename(nodename): + # stub out numbers + chunked = ["{}" if text.isdigit() else text.lower() + for text in re.split(numregex, nodename)] + return chunked + +def getnumbers_nodename(nodename): + return [x for x in re.split(numregex, nodename) if x.isdigit()] + + +class Bracketer(object): + __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens'] + + def __init__(self, nodename): + self.sequences = [] + self.numlens = [] + realnodename = nodename + if ':' in nodename: + realnodename = nodename.split(':', 1)[0] + self.count = len(getnumbers_nodename(realnodename)) + self.nametmpl = unnumber_nodename(realnodename) + for n in range(self.count): + self.sequences.append(None) + self.numlens.append([0, 0]) + self.diffn = None + self.tokens = [] + self.extend(nodename) + if self.count == 0: + self.tokens = [nodename] + + def extend(self, nodeorseq): + # can only differentiate a single number + endname = None + endnums = None + if ':' in nodeorseq: + nodename, endname = nodeorseq.split(':', 1) + else: + nodename = nodeorseq + txtnums = getnumbers_nodename(nodename) + nums = [int(x) for x in txtnums] + for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: + continue # new nodename has no new number, keep going + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or + (needpad and padto != self.numlens[n][1])): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [padto, padto] + self.diffn = None + else: + self.diffn = n + if self.sequences[n][2] == (nums[n] - 1): + self.sequences[n][2] = nums[n] + self.numlens[n][1] = padto + elif self.sequences[n][2] < (nums[n] - 1): + if self.sequences[n][2] != self.sequences[n][1]: + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) + else: + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) + self.sequences[n][1] = nums[n] + self.numlens[n][0] = padto + self.sequences[n][2] = nums[n] + self.numlens[n][1] = padto + + def flush_current(self): + txtfields = [] + if self.sequences and self.sequences[0] is not None: + for n in range(self.count): + if self.sequences[n][1] == self.sequences[n][2]: + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) + else: + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) + txtfield = ','.join(self.sequences[n][0]) + if txtfield.isdigit(): + txtfields.append(txtfield) + else: + txtfields.append('[{}]'.format(txtfield)) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) + self.sequences = [] + for n in range(self.count): + self.sequences.append(None) + + @property + def range(self): + if self.sequences: + self.flush_current() + return ','.join(self.tokens) + + +def group_elements(elems): + """ Take the specefied elements and chunk them according to text similarity + """ + prev = None + currchunk = [] + chunked_elems = [currchunk] + for elem in elems: + elemtxt = unnumber_nodename(elem) + if not prev: + prev = elemtxt + currchunk.append(elem) + continue + if prev == elemtxt: + currchunk.append(elem) + else: + currchunk = [elem] + chunked_elems.append(currchunk) + prev = elemtxt + return chunked_elems + class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -71,7 +192,8 @@ class ReverseNodeRange(object): @property def noderange(self): subsetgroups = [] - for group in self.cfm.get_groups(sizesort=True): + allgroups = self.cfm.get_groups(sizesort=True) + for group in allgroups: if lastnoderange: for nr in lastnoderange: if lastnoderange[nr] - self.nodes: @@ -88,7 +210,39 @@ class ReverseNodeRange(object): self.nodes -= nl if not self.nodes: break - return ','.join(sorted(subsetgroups) + sorted(self.nodes)) + # then, analyze sequentially identifying matching alpha subsections + # then try out noderange from beginning to end + # we need to know discontinuities, which are either: + # nodes that appear in the noderange that are not in the nodes + # nodes that do not exist at all (we need a noderange modification + # that returns non existing nodes) + ranges = [] + try: + subsetgroups.sort(key=humanify_nodename) + groupchunks = group_elements(subsetgroups) + for gc in groupchunks: + if not gc: + continue + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) + except Exception: + subsetgroups.sort() + ranges.extend(subsetgroups) + try: + nodes = sorted(self.nodes, key=humanify_nodename) + nodechunks = group_elements(nodes) + for nc in nodechunks: + if not nc: + continue + bracketer = Bracketer(nc[0]) + for chnk in nc[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) + except Exception: + ranges.extend(sorted(self.nodes)) + return ','.join(ranges) diff --git a/confluent_server/confluent/plugins/info/layout.py b/confluent_server/confluent/plugins/info/layout.py new file mode 100644 index 00000000..8397af7f --- /dev/null +++ b/confluent_server/confluent/plugins/info/layout.py @@ -0,0 +1,100 @@ +# Copyright 2023 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.core as core +import confluent.messages as msg + +def retrieve(nodes, element, configmanager, inputdata): + locationinfo = configmanager.get_node_attributes(nodes, + (u'enclosure.manager', u'enclosure.bay', u'location.rack', + u'location.row', u'location.u', u'location.height')) + enclosuremap = {} + rackmap = {} + allnodedata = {} + needenclosures = set([]) + locatednodes = set([]) + for node in locationinfo: + nodeinfo = locationinfo[node] + rack = nodeinfo.get(u'location.rack', {}).get('value', '') + u = nodeinfo.get(u'location.u', {}).get('value', None) + row = nodeinfo.get(u'location.row', {}).get('value', '') + enclosure = nodeinfo.get(u'enclosure.manager', {}).get('value', None) + bay = nodeinfo.get(u'enclosure.bay', {}).get('value', None) + height = nodeinfo.get(u'location.height', {}).get('value', None) + if enclosure: + if enclosure not in enclosuremap: + enclosuremap[enclosure] = {} + enclosuremap[enclosure][bay] = node + if u: + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = {'node': enclosure, 'children': enclosuremap[enclosure]} + allnodedata[enclosure] = rackmap[row][rack][u] + if height: + allnodedata[enclosure]['height'] = height + else: # need to see if enclosure lands in the map naturally or need to pull it + needenclosures.add(enclosure) + elif u: + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = {'node': node} + allnodedata[node] = rackmap[row][rack][u] + if height: + allnodedata[node]['height'] = height + locatednodes.add(node) + cfgenc = needenclosures - locatednodes + locationinfo = configmanager.get_node_attributes(cfgenc, (u'location.rack', u'location.row', u'location.u', u'location.height')) + for enclosure in locationinfo: + nodeinfo = locationinfo[enclosure] + rack = nodeinfo.get(u'location.rack', {}).get('value', '') + u = nodeinfo.get(u'location.u', {}).get('value', None) + row = nodeinfo.get(u'location.row', {}).get('value', '') + height = nodeinfo.get(u'location.height', {}).get('value', None) + if u: + allnodedata[enclosure] = {'node': enclosure, 'children': enclosuremap[enclosure]} + if height: + allnodedata[enclosure]['height'] = height + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = allnodedata[enclosure] + results = { + 'errors': [], + 'locations': rackmap, + } + for enclosure in enclosuremap: + if enclosure not in allnodedata: + results['errors'].append('Enclosure {} is missing required location information'.format(enclosure)) + else: + allnodedata[enclosure]['children'] = enclosuremap[enclosure] + needheight = set([]) + for node in allnodedata: + if 'height' not in allnodedata[node]: + needheight.add(node) + needheight = ','.join(needheight) + if needheight: + for rsp in core.handle_path( + '/noderange/{0}/description'.format(needheight), + 'retrieve', configmanager, + inputdata=None): + kvp = rsp.kvpairs + for node in kvp: + allnodedata[node]['height'] = kvp[node]['height'] + yield msg.Generic(results) + diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 04030491..cd4180c7 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -19,6 +19,12 @@ import json import os import time import yaml +try: + from yaml import CSafeDumper as SafeDumper + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from yaml import SafeDumper import confluent.discovery.protocols.ssdp as ssdp import eventlet webclient = eventlet.import_patched('pyghmi.util.webclient') @@ -31,7 +37,20 @@ currtzvintage = None def yamldump(input): - return yaml.safe_dump(input, default_flow_style=False) + return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False) + +def yamlload(input): + return yaml.load(input, Loader=SafeLoader) + +def listdump(input): + # special case yaml for flat dumb list + # this is about 25x faster than doing full yaml dump even with CSafeDumper + # with a 17,000 element list + retval = '' + for entry in input: + retval += '- ' + entry + '\n' + return retval + def get_extra_names(nodename, cfg, myip=None): names = set([]) @@ -402,10 +421,13 @@ def handle_request(env, start_response): yield node + '\n' else: start_response('200 OK', (('Content-Type', retype),)) - yield dumper(list(util.natural_sort(nodes))) + if retype == 'application/yaml': + yield listdump(list(util.natural_sort(nodes))) + else: + yield dumper(list(util.natural_sort(nodes))) elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody: try: - reqbody = yaml.safe_load(reqbody) + reqbody = yamlload(reqbody) except Exception: reqbody = None cfgmod = reqbody.get('configmod', 'unspecified') @@ -419,7 +441,7 @@ def handle_request(env, start_response): start_response('200 Ok', ()) yield 'complete' elif env['PATH_INFO'] == '/self/updatestatus' and reqbody: - update = yaml.safe_load(reqbody) + update = yamlload(reqbody) statusstr = update.get('state', None) statusdetail = update.get('state_detail', None) didstateupdate = False @@ -522,7 +544,7 @@ def handle_request(env, start_response): '/var/lib/confluent/public/os/{0}/scripts/{1}') if slist: start_response('200 OK', (('Content-Type', 'application/yaml'),)) - yield yaml.safe_dump(util.natural_sort(slist), default_flow_style=False) + yield yamldump(util.natural_sort(slist)) else: start_response('200 OK', ()) yield '' diff --git a/misc/swraid b/misc/swraid index 3e234dc8..836f1fb1 100644 --- a/misc/swraid +++ b/misc/swraid @@ -1,6 +1,6 @@ DEVICES="/dev/sda /dev/sdb" RAIDLEVEL=1 -mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null && exit 0 +mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null || ( lvm vgchange -a n mdadm -S -s NUMDEVS=$(for dev in $DEVICES; do @@ -14,5 +14,6 @@ mdadm -C /dev/md/raid $DEVICES -n $NUMDEVS -e 1.0 -l $RAIDLEVEL # shut and restart array to prime things for anaconda mdadm -S -s mdadm --assemble --scan +) readlink /dev/md/raid|sed -e 's/.*\///' > /tmp/installdisk