Merge branch 'lenovo:master' into master

confluent_client/bin/nodeattrib

@@ -22,6 +22,7 @@ import optparse
 import os
 import signal
 import sys
+import shlex
 
 try:
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)

confluent_client/bin/nodersync

@@ -48,6 +48,8 @@ def run():
                           'prompting if over the threshold')   
     argparser.add_option('-l', '--loginname', type='str',
                          help='Username to use when connecting, defaults to current user.')
+    argparser.add_option('-s', '--substitutename',
+                         help='Use a different name other than the nodename for rsync')
     argparser.add_option('-f', '-c', '--count', type='int', default=168,
                          help='Number of nodes to concurrently rsync')
     # among other things, FD_SETSIZE limits.  Besides, spawning too many
@@ -63,10 +65,20 @@ def run():
     c = client.Command()
     cmdstr = ' '.join(args[:-1])
     cmdstr = 'rsync -av --info=progress2 ' + cmdstr
-    if options.loginname:
-        cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath
+    
+    targname = options.substitutename
+    if targname and '{' in targname:
+        targname =  targname + ':'
+    elif targname:
+        targname = '{node}' + targname + ':'
     else:
-        cmdstr += ' {node}:' + targpath
+        targname = '{node}:'
+
+    if options.loginname:
+        cmdstr += ' {}@'.format(options.loginname) + targname + targpath
+    else:
+        cmdstr += ' {}'.format(targname) + targpath
+    
 
     currprocs = 0
     all = set([])

confluent_client/bin/stats

@@ -16,13 +16,10 @@
 # limitations under the License.
 
 import argparse
+import base64
 import csv
-import fcntl
 import io
 import numpy as np
-
-import os
-import subprocess
 import sys
 
 try:
@@ -35,7 +32,31 @@ except ImportError:
     pass
 
 
-def plot(gui, output, plotdata, bins):
+def iterm_draw(data):
+    databuf = data.getbuffer()
+    datalen = len(databuf)
+    data = base64.b64encode(databuf).decode('utf8')
+    sys.stdout.write(
+        '\x1b]1337;File=inline=1;size={}:'.format(datalen))
+    sys.stdout.write(data)
+    sys.stdout.write('\a')
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+
+def kitty_draw(data):
+    data = base64.b64encode(data.getbuffer())
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m))
+        sys.stdout.write(chunk.decode('utf8'))
+        sys.stdout.write('\x1b\\')
+        sys.stdout.flush()
+    sys.stdout.write('\n')
+
+
+def plot(gui, output, plotdata, bins, fmt):
     import matplotlib as mpl
     if gui and mpl.get_backend() == 'agg':
         sys.stderr.write('Error: No GUI backend available and -g specified!\n')
@@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins):
             tdata = io.BytesIO()
         plt.savefig(tdata)
     if not gui and not output:
-        writer = DumbWriter()
-        writer.draw(tdata)
+        if fmt == 'sixel':
+            writer = DumbWriter()
+            writer.draw(tdata)
+        elif fmt == 'kitty':
+            kitty_draw(tdata)
+        elif fmt == 'iterm':
+            iterm_draw(tdata)
     return n, bins
 
 def textplot(plotdata, bins):
@@ -81,7 +107,8 @@ histogram = False
 aparser = argparse.ArgumentParser(description='Quick access to common statistics')
 aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)')
 aparser.add_argument('-d', default=None, help='Value used to separate columns')
-aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format')
+aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format')
+aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)')
 aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing')
 aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window')
 aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format')
@@ -138,7 +165,7 @@ while data:
     data = list(csv.reader([data], delimiter=delimiter))[0]
 n = None
 if args.g or args.o or args.x:
-    n, bins = plot(args.g, args.o, plotdata, bins=args.b)
+    n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f)
 if args.t:
     n, bins = textplot(plotdata, bins=args.b)
 print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata)))

confluent_client/doc/man/nodeattrib.ronn.tmpl

@@ -7,8 +7,8 @@ nodeattrib(8) -- List or change confluent nodes attributes
 `nodeattrib <noderange> [<nodeattribute1=value1> <nodeattribute2=value2> ...]`  
 `nodeattrib -c <noderange> <nodeattribute1> <nodeattribute2> ...`  
 `nodeattrib -e <noderange> <nodeattribute1> <nodeattribute2> ...`  
-`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...` 
-`nodeattrib <noderange> -s <attributes.batch>` 
+`nodeattrib -p <noderange> <nodeattribute1> <nodeattribute2> ...`
+`nodeattrib <noderange> -s <attributes.batch> ...` 
 
 ## DESCRIPTION
 
@@ -54,14 +54,17 @@ to a blank value will allow masking a group defined attribute with an empty valu
 * `-e`, `--environment`:
   Set specified attributes based on exported environment variable of matching name. 
   Environment variable names may be lower case or all upper case. 
-  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE
+  Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE)
   
 * `-p`, `--prompt`:
   Request interactive prompting to provide values rather than the command line
   or environment variables.
 
 * `-s`, `--set`:
-  Set attributes using a batch file
+  Set attributes using a batch file rather than the command line. The attributes in the batch file 
+  can be specified as one line of key=value pairs simmilar to command line or each attribute can
+  be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch
+  file syntax.   
  
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Prompt if trying to set attributes on more than
@@ -120,6 +123,25 @@ to a blank value will allow masking a group defined attribute with an empty valu
     `d1: net.pxe.switch: pxeswitch1`  
     `d1: net.switch:`  
 
+* Setting attributes using a batch file with syntax similar to command line:
+    `# cat nodeattributes.batch`  
+    `# power`
+    `power.psu1.outlet=3 power.psu1.pdu=pdu2`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: 3`
+    `n41: pdu2`
+
+* Setting attributes using a batch file with syntax where each attribute is in its own line:
+    `# cat nodeattributes.batch`  
+    `# management`
+    `custom.mgt.switch=switch_main`
+    `custom.mgt.switch.port=swp4`
+    `# nodeattrib n41 -s nodeattributes.batch`
+    `n41: switch_main`
+    `n41: swp4`
+
+  
+
 ## SEE ALSO
 
 nodegroupattrib(8), nodeattribexpressions(5)

confluent_client/doc/man/nodersync.ronn

@@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes.
   Specify how many rsync executions to do concurrently.  If noderange
   exceeds the count, then excess nodes will wait until one of the
   active count completes. 
+
+* `-s`, `--substitutename`:
+  'Use a different name other than the nodename for rsync'
   
 * `-m MAXNODES`, `--maxnodes=MAXNODES`:
   Specify a maximum number of nodes to run rsync to, prompting if over the

confluent_osdeploy/buildrpm-aarch64

@@ -1,3 +1,4 @@
+cd $(dirname $0)
 VERSION=`git describe|cut -d- -f 1`
 NUMCOMMITS=`git describe|cut -d- -f 2`
 if [ "$NUMCOMMITS" != "$VERSION"  ]; then
@@ -29,4 +30,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/
 mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/
 rm -rf el9bin
 rm -rf el8bin
-rpmbuild -ba confluent_osdeploy-aarch64.spec
+podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec
+

confluent_osdeploy/common/profile/scripts/confignet

@@ -435,10 +435,26 @@ if __name__ == '__main__':
                     curridx = addr[-1]
         if curridx in doneidxs:
             continue
-        status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+        for tries in (1, 2 3):
+            try:
+                status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg')
+                break
+            except Exception:
+                if tries == 3:
+                    raise
+                time.sleep(1)
+                continue
         nc = json.loads(nc)
         if not dc:
-            status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+            for tries in (1, 2 3):
+                try:
+                    status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2')
+                    break
+                except Exception:
+                    if tries == 3:
+                        raise
+                    time.sleep(1)
+                    continue
             dc = json.loads(dc)
         iname = get_interface_name(idxmap[curridx], nc.get('default', {}))
         if iname:

confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh

@@ -35,6 +35,7 @@ echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.con
 echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf
 systemctl restart sshd
 mkdir -p /etc/confluent
+export nodename confluent_profile confluent_mgr
 curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions
 . /etc/confluent/functions
 run_remote_parts pre.d

confluent_server/confluent/core.py

@@ -446,6 +446,7 @@ def _init_core():
                 },
             },
         },
+        'layout': PluginRoute({'handler': 'layout'}),
         'media': {
             'uploads': PluginCollection({
                 'pluginattrs': ['hardwaremanagement.method'],

confluent_server/confluent/httpapi.py

@@ -1084,7 +1084,7 @@ def serve(bind_host, bind_port):
         pass  # we gave it our best shot there
     try:
         eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
-                             debug=False, socket_timeout=60)
+                             debug=False, socket_timeout=60, keepalive=False)
     except TypeError:
         # Older eventlet in place, skip arguments it does not understand
         eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)

confluent_server/confluent/messages.py

@@ -92,6 +92,7 @@ def msg_deserialize(packed):
         return cls(*m[1:])
     raise Exception("Unknown shenanigans")
 
+
 class ConfluentMessage(object):
     apicode = 200
     readonly = False
@@ -254,6 +255,21 @@ class ConfluentNodeError(object):
         raise Exception('{0}: {1}'.format(self.node, self.error))
 
 
+class Generic(ConfluentMessage):
+
+    def __init__(self, data):
+        self.data = data
+
+    def json(self):
+        return json.dumps(self.data)
+    
+    def raw(self):
+        return self.data
+    
+    def html(self):
+        return json.dumps(self.data)
+
+
 class ConfluentResourceUnavailable(ConfluentNodeError):
     apicode = 503
 

confluent_server/confluent/netutil.py

@@ -25,6 +25,9 @@ import eventlet.support.greendns
 import os
 getaddrinfo = eventlet.support.greendns.getaddrinfo
 
+eventlet.support.greendns.resolver.clear()
+eventlet.support.greendns.resolver._resolver.lifetime = 1
+
 def msg_align(len):
     return (len + 3) & ~3
 
@@ -333,11 +336,13 @@ def get_full_net_config(configmanager, node, serverip=None):
         myaddrs = get_addresses_by_serverip(serverip)
     nm = NetManager(myaddrs, node, configmanager)
     defaultnic = {}
+    ppool = eventlet.greenpool.GreenPool(64)
     if None in attribs:
-        nm.process_attribs(None, attribs[None])
+        ppool.spawn(nm.process_attribs, None, attribs[None])
         del attribs[None]
     for netname in sorted(attribs):
-        nm.process_attribs(netname, attribs[netname])
+        ppool.spawn(nm.process_attribs, netname, attribs[netname])
+    ppool.waitall()
     retattrs = {}
     if None in nm.myattribs:
         retattrs['default'] = nm.myattribs[None]

confluent_server/confluent/noderange.py

@@ -55,6 +55,127 @@ def humanify_nodename(nodename):
     return [int(text) if text.isdigit() else text.lower()
             for text in re.split(numregex, nodename)]
 
+def unnumber_nodename(nodename):
+    # stub out numbers
+    chunked = ["{}" if text.isdigit() else text.lower()
+            for text in re.split(numregex, nodename)]
+    return chunked
+
+def getnumbers_nodename(nodename):
+    return [x for x in re.split(numregex, nodename) if x.isdigit()]
+    
+
+class Bracketer(object):
+    __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens']
+
+    def __init__(self, nodename):
+        self.sequences = []
+        self.numlens = []
+        realnodename = nodename
+        if ':' in nodename:
+            realnodename = nodename.split(':', 1)[0]
+        self.count = len(getnumbers_nodename(realnodename))
+        self.nametmpl = unnumber_nodename(realnodename)
+        for n in range(self.count):
+            self.sequences.append(None)
+            self.numlens.append([0, 0])
+        self.diffn = None
+        self.tokens = []
+        self.extend(nodename)
+        if self.count == 0:
+            self.tokens = [nodename]
+
+    def extend(self, nodeorseq):
+        # can only differentiate a single number
+        endname = None
+        endnums = None
+        if ':' in nodeorseq:
+            nodename, endname = nodeorseq.split(':', 1)
+        else:
+            nodename = nodeorseq
+        txtnums = getnumbers_nodename(nodename)
+        nums = [int(x) for x in txtnums]
+        for n in range(self.count):
+            padto = len(txtnums[n])
+            needpad = (padto != len('{}'.format(nums[n])))
+            if self.sequences[n] is None:
+                # We initialize to text pieces, 'currstart', and 'prev' number
+                self.sequences[n] = [[], nums[n], nums[n]]
+                self.numlens[n] = [len(txtnums[n]), len(txtnums[n])]
+            elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto:
+                continue  # new nodename has no new number, keep going
+            else: # if self.sequences[n][2] != nums[n] or :
+                if self.diffn is not None and (n != self.diffn or 
+                        (padto < self.numlens[n][1]) or
+                        (needpad and padto != self.numlens[n][1])):
+                    self.flush_current()
+                    self.sequences[n] = [[], nums[n], nums[n]]
+                    self.numlens[n] = [padto, padto]
+                    self.diffn = None
+                else:
+                    self.diffn = n
+                if self.sequences[n][2] == (nums[n] - 1):
+                    self.sequences[n][2] = nums[n]
+                    self.numlens[n][1] = padto
+                elif self.sequences[n][2] < (nums[n] - 1):
+                    if self.sequences[n][2] != self.sequences[n][1]:
+                        fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                    else:
+                        fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                        self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                    self.sequences[n][1] = nums[n]
+                    self.numlens[n][0] = padto
+                self.sequences[n][2] = nums[n]
+                self.numlens[n][1] = padto
+
+    def flush_current(self):
+        txtfields = []
+        if self.sequences and self.sequences[0] is not None:
+            for n in range(self.count):
+                if self.sequences[n][1] == self.sequences[n][2]:
+                    fmtstr = '{{:0{}d}}'.format(self.numlens[n][0])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1]))
+                else:
+                    fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n])
+                    self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2]))
+                txtfield = ','.join(self.sequences[n][0])
+                if txtfield.isdigit():
+                    txtfields.append(txtfield)
+                else:
+                    txtfields.append('[{}]'.format(txtfield))
+            self.tokens.append(''.join(self.nametmpl).format(*txtfields))
+        self.sequences = []
+        for n in range(self.count):
+            self.sequences.append(None)
+
+    @property
+    def range(self):
+        if self.sequences:
+            self.flush_current()
+        return ','.join(self.tokens)
+
+
+def group_elements(elems):
+    """ Take the specefied elements and chunk them according to text similarity
+    """
+    prev = None
+    currchunk = []
+    chunked_elems = [currchunk]
+    for elem in elems:
+        elemtxt = unnumber_nodename(elem)
+        if not prev:
+            prev = elemtxt
+            currchunk.append(elem)
+            continue
+        if prev == elemtxt:
+            currchunk.append(elem)
+        else:
+            currchunk = [elem]
+            chunked_elems.append(currchunk)
+            prev = elemtxt
+    return chunked_elems
+
 
 class ReverseNodeRange(object):
     """Abbreviate a set of nodes to a shorter noderange representation
@@ -71,7 +192,8 @@ class ReverseNodeRange(object):
     @property
     def noderange(self):
         subsetgroups = []
-        for group in self.cfm.get_groups(sizesort=True):
+        allgroups = self.cfm.get_groups(sizesort=True)
+        for group in allgroups:
             if lastnoderange:
                 for nr in lastnoderange:
                     if lastnoderange[nr] - self.nodes:
@@ -88,7 +210,39 @@ class ReverseNodeRange(object):
                 self.nodes -= nl
                 if not self.nodes:
                     break
-        return ','.join(sorted(subsetgroups) + sorted(self.nodes))
+        # then, analyze sequentially identifying matching alpha subsections
+        # then try out noderange from beginning to end
+        # we need to know discontinuities, which are either:
+        # nodes that appear in the noderange that are not in the nodes
+        # nodes that do not exist at all (we need a noderange modification 
+        # that returns non existing nodes)
+        ranges = []
+        try:
+            subsetgroups.sort(key=humanify_nodename)
+            groupchunks = group_elements(subsetgroups)
+            for gc in groupchunks:
+                if not gc:
+                    continue
+                bracketer = Bracketer(gc[0])
+                for chnk in gc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            subsetgroups.sort()
+            ranges.extend(subsetgroups)
+        try:
+            nodes = sorted(self.nodes, key=humanify_nodename)
+            nodechunks = group_elements(nodes)
+            for nc in nodechunks:
+                if not nc:
+                    continue
+                bracketer = Bracketer(nc[0])
+                for chnk in nc[1:]:
+                    bracketer.extend(chnk)
+                ranges.append(bracketer.range)
+        except Exception:
+            ranges.extend(sorted(self.nodes))
+        return ','.join(ranges)
 
 
 

confluent_server/confluent/plugins/info/layout.py

@@ -0,0 +1,100 @@
+# Copyright 2023 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import confluent.core as core
+import confluent.messages as msg
+
+def retrieve(nodes, element, configmanager, inputdata):
+    locationinfo = configmanager.get_node_attributes(nodes,
+            (u'enclosure.manager', u'enclosure.bay', u'location.rack',
+             u'location.row', u'location.u', u'location.height'))
+    enclosuremap = {}
+    rackmap = {}
+    allnodedata = {}
+    needenclosures = set([])
+    locatednodes = set([])
+    for node in locationinfo:
+        nodeinfo = locationinfo[node]
+        rack = nodeinfo.get(u'location.rack', {}).get('value', '')
+        u = nodeinfo.get(u'location.u', {}).get('value', None)
+        row = nodeinfo.get(u'location.row', {}).get('value', '')
+        enclosure = nodeinfo.get(u'enclosure.manager', {}).get('value', None)
+        bay = nodeinfo.get(u'enclosure.bay', {}).get('value', None)
+        height = nodeinfo.get(u'location.height', {}).get('value', None)
+        if enclosure:
+            if enclosure not in enclosuremap:
+                enclosuremap[enclosure] = {}
+            enclosuremap[enclosure][bay] = node
+            if u:
+                if row not in rackmap:
+                    rackmap[row] = {}
+                if rack not in rackmap[row]:
+                    rackmap[row][rack] = {}
+                rackmap[row][rack][u] = {'node': enclosure, 'children': enclosuremap[enclosure]}
+                allnodedata[enclosure] = rackmap[row][rack][u]
+                if height:
+                    allnodedata[enclosure]['height'] = height
+            else: # need to see if enclosure lands in the map naturally or need to pull it
+                needenclosures.add(enclosure)
+        elif u:
+            if row not in rackmap:
+                rackmap[row] = {}
+            if rack not in rackmap[row]:
+                rackmap[row][rack] = {}
+            rackmap[row][rack][u] = {'node': node}
+            allnodedata[node] = rackmap[row][rack][u]
+            if height:
+                allnodedata[node]['height'] = height
+            locatednodes.add(node)
+    cfgenc = needenclosures - locatednodes
+    locationinfo = configmanager.get_node_attributes(cfgenc, (u'location.rack', u'location.row', u'location.u', u'location.height'))
+    for enclosure in locationinfo:
+        nodeinfo = locationinfo[enclosure]
+        rack = nodeinfo.get(u'location.rack', {}).get('value', '')
+        u = nodeinfo.get(u'location.u', {}).get('value', None)
+        row = nodeinfo.get(u'location.row', {}).get('value', '')
+        height = nodeinfo.get(u'location.height', {}).get('value', None)
+        if u:
+            allnodedata[enclosure] = {'node': enclosure, 'children': enclosuremap[enclosure]}
+            if height:
+                allnodedata[enclosure]['height'] = height
+            if row not in rackmap:
+                rackmap[row] = {}
+            if rack not in rackmap[row]:
+                rackmap[row][rack] = {}
+            rackmap[row][rack][u] = allnodedata[enclosure]
+    results = {
+        'errors': [],
+        'locations': rackmap,
+    }
+    for enclosure in enclosuremap:
+        if enclosure not in allnodedata:
+            results['errors'].append('Enclosure {} is missing required location information'.format(enclosure))
+        else:
+            allnodedata[enclosure]['children'] = enclosuremap[enclosure]
+    needheight = set([])
+    for node in allnodedata:
+        if 'height' not in allnodedata[node]:
+            needheight.add(node)
+    needheight = ','.join(needheight)
+    if needheight:
+        for rsp in core.handle_path(
+            '/noderange/{0}/description'.format(needheight),
+            'retrieve', configmanager,
+            inputdata=None):
+                kvp = rsp.kvpairs
+                for node in kvp:
+                    allnodedata[node]['height'] = kvp[node]['height']
+    yield msg.Generic(results)
+

confluent_server/confluent/selfservice.py

@@ -19,6 +19,12 @@ import json
 import os
 import time
 import yaml
+try:
+    from yaml import CSafeDumper as SafeDumper
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+    from yaml import SafeDumper
 import confluent.discovery.protocols.ssdp as ssdp
 import eventlet
 webclient = eventlet.import_patched('pyghmi.util.webclient')
@@ -31,7 +37,20 @@ currtzvintage = None
 
 
 def yamldump(input):
-    return yaml.safe_dump(input, default_flow_style=False)
+    return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False)
+
+def yamlload(input):
+    return yaml.load(input, Loader=SafeLoader)
+
+def listdump(input):
+    # special case yaml for flat dumb list
+    # this is about 25x faster than doing full yaml dump even with CSafeDumper
+    # with a 17,000 element list
+    retval = ''
+    for entry in input:
+        retval += '- ' + entry + '\n'
+    return retval
+
 
 def get_extra_names(nodename, cfg, myip=None):
     names = set([])
@@ -402,10 +421,13 @@ def handle_request(env, start_response):
                 yield node + '\n'
         else:
             start_response('200 OK', (('Content-Type', retype),))
-            yield dumper(list(util.natural_sort(nodes)))
+            if retype == 'application/yaml':
+                yield listdump(list(util.natural_sort(nodes)))
+            else:
+                yield dumper(list(util.natural_sort(nodes)))
     elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody:
         try:
-            reqbody = yaml.safe_load(reqbody)
+            reqbody = yamlload(reqbody)
         except Exception:
             reqbody = None
         cfgmod = reqbody.get('configmod', 'unspecified')
@@ -419,7 +441,7 @@ def handle_request(env, start_response):
         start_response('200 Ok', ())
         yield 'complete'
     elif env['PATH_INFO'] == '/self/updatestatus' and reqbody:
-        update = yaml.safe_load(reqbody)
+        update = yamlload(reqbody)
         statusstr = update.get('state', None)
         statusdetail = update.get('state_detail', None)
         didstateupdate = False
@@ -522,7 +544,7 @@ def handle_request(env, start_response):
             '/var/lib/confluent/public/os/{0}/scripts/{1}')
         if slist:
             start_response('200 OK', (('Content-Type', 'application/yaml'),))
-            yield yaml.safe_dump(util.natural_sort(slist), default_flow_style=False)
+            yield yamldump(util.natural_sort(slist))
         else:
             start_response('200 OK', ())
             yield ''

misc/swraid

@@ -1,6 +1,6 @@
 DEVICES="/dev/sda /dev/sdb"
 RAIDLEVEL=1
-mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null && exit 0
+mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null || (
 lvm vgchange -a n
 mdadm -S -s
 NUMDEVS=$(for dev in $DEVICES; do
@@ -14,5 +14,6 @@ mdadm -C /dev/md/raid $DEVICES -n $NUMDEVS -e 1.0 -l $RAIDLEVEL
 # shut and restart array to prime things for anaconda
 mdadm -S -s
 mdadm --assemble --scan
+)
 readlink /dev/md/raid|sed -e 's/.*\///' > /tmp/installdisk