From a00fd325aa4a5b0d9a9a55b4b9b01e8426da045d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 27 Sep 2023 13:09:23 -0400 Subject: [PATCH 01/30] Export variables for ubuntu pre.d run --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 2f671d38..4ff1878e 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -35,6 +35,7 @@ echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.con echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf systemctl restart sshd mkdir -p /etc/confluent +export confluent_profile confluent_mgr curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions . /etc/confluent/functions run_remote_parts pre.d From eee8bbb498dad16058d3ceeaba54e19beba16fe6 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 27 Sep 2023 17:52:15 -0400 Subject: [PATCH 02/30] node rsync -s switch --- confluent_client/bin/nodersync | 12 ++++++++++-- confluent_client/doc/man/nodersync.ronn | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/confluent_client/bin/nodersync b/confluent_client/bin/nodersync index ba95662c..8e125ee8 100755 --- a/confluent_client/bin/nodersync +++ b/confluent_client/bin/nodersync @@ -48,6 +48,8 @@ def run(): 'prompting if over the threshold') argparser.add_option('-l', '--loginname', type='str', help='Username to use when connecting, defaults to current user.') + argparser.add_option('-s', '--substitutename', + help='Use a different name other than the nodename for rsync') argparser.add_option('-f', '-c', '--count', type='int', default=168, help='Number of nodes to concurrently rsync') # among other things, FD_SETSIZE limits. Besides, spawning too many @@ -63,8 +65,14 @@ def run(): c = client.Command() cmdstr = ' '.join(args[:-1]) cmdstr = 'rsync -av --info=progress2 ' + cmdstr - if options.loginname: - cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath + if options.loginname and options.substitutename: + cmdstr += ' {}@'.format(options.loginname) + '{node}' + '{}:'.format(options.substitutename) + targpath + elif options.loginname and not options.substitutename: + cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath + elif options.substitutename and not options.loginname: + subname = options.substitutename + if '{' not in subname: + cmdstr += ' {node}' + '{}:'.format(options.substitutename) + targpath else: cmdstr += ' {node}:' + targpath diff --git a/confluent_client/doc/man/nodersync.ronn b/confluent_client/doc/man/nodersync.ronn index 582f79d9..c187215e 100644 --- a/confluent_client/doc/man/nodersync.ronn +++ b/confluent_client/doc/man/nodersync.ronn @@ -16,6 +16,9 @@ noderange. This will present progress as percentage for all nodes. Specify how many rsync executions to do concurrently. If noderange exceeds the count, then excess nodes will wait until one of the active count completes. + +* `-s`, `--substitutename`: + 'Use a different name other than the nodename for rsync' * `-m MAXNODES`, `--maxnodes=MAXNODES`: Specify a maximum number of nodes to run rsync to, prompting if over the From 378929579f2fb36a738e12e5c232d998fbdaca34 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 28 Sep 2023 08:56:52 -0400 Subject: [PATCH 03/30] Allow to be able to specify prefix as well --- confluent_client/bin/nodersync | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/confluent_client/bin/nodersync b/confluent_client/bin/nodersync index 8e125ee8..b53b6f34 100755 --- a/confluent_client/bin/nodersync +++ b/confluent_client/bin/nodersync @@ -72,7 +72,9 @@ def run(): elif options.substitutename and not options.loginname: subname = options.substitutename if '{' not in subname: - cmdstr += ' {node}' + '{}:'.format(options.substitutename) + targpath + cmdstr += ' {node}' + '{}:'.format(subname) + targpath + else: + cmdstr += ' {}:'.format(subname) + targpath else: cmdstr += ' {node}:' + targpath From b63d75f2bb454410558bd0843e7e756143e93796 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Fri, 29 Sep 2023 12:07:38 -0400 Subject: [PATCH 04/30] change to remove to many conditionals --- confluent_client/bin/nodersync | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/confluent_client/bin/nodersync b/confluent_client/bin/nodersync index b53b6f34..8d316bab 100755 --- a/confluent_client/bin/nodersync +++ b/confluent_client/bin/nodersync @@ -65,18 +65,20 @@ def run(): c = client.Command() cmdstr = ' '.join(args[:-1]) cmdstr = 'rsync -av --info=progress2 ' + cmdstr - if options.loginname and options.substitutename: - cmdstr += ' {}@'.format(options.loginname) + '{node}' + '{}:'.format(options.substitutename) + targpath - elif options.loginname and not options.substitutename: - cmdstr += ' {}@'.format(options.loginname) + '{node}:' + targpath - elif options.substitutename and not options.loginname: - subname = options.substitutename - if '{' not in subname: - cmdstr += ' {node}' + '{}:'.format(subname) + targpath - else: - cmdstr += ' {}:'.format(subname) + targpath + + targname = options.substitutename + if targname and '{' in targname: + targname = targname + ':' + elif targname: + targname = '{node}' + targname + ':' else: - cmdstr += ' {node}:' + targpath + targname = '{node}:' + + if options.loginname: + cmdstr += ' {}@'.format(options.loginname) + targname + targpath + else: + cmdstr += ' {}'.format(targname) + targpath + currprocs = 0 all = set([]) From 79e3ad53f880a92041da6dab5d13b27c0d58c60b Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 29 Sep 2023 16:23:59 -0400 Subject: [PATCH 05/30] Add server side rack layout organization The info is hard to put together client side, but supremely easy server side. Provide a nice call to get the layout for a noderange, similar to (but better than) current GUI code. Now GUI can get a nice canned JSON description of the layout. --- confluent_server/confluent/core.py | 1 + confluent_server/confluent/messages.py | 16 +++ .../confluent/plugins/info/layout.py | 100 ++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 confluent_server/confluent/plugins/info/layout.py diff --git a/confluent_server/confluent/core.py b/confluent_server/confluent/core.py index a9ee1dba..f70bc6ae 100644 --- a/confluent_server/confluent/core.py +++ b/confluent_server/confluent/core.py @@ -446,6 +446,7 @@ def _init_core(): }, }, }, + 'layout': PluginRoute({'handler': 'layout'}), 'media': { 'uploads': PluginCollection({ 'pluginattrs': ['hardwaremanagement.method'], diff --git a/confluent_server/confluent/messages.py b/confluent_server/confluent/messages.py index a24a4d78..ce36344d 100644 --- a/confluent_server/confluent/messages.py +++ b/confluent_server/confluent/messages.py @@ -92,6 +92,7 @@ def msg_deserialize(packed): return cls(*m[1:]) raise Exception("Unknown shenanigans") + class ConfluentMessage(object): apicode = 200 readonly = False @@ -254,6 +255,21 @@ class ConfluentNodeError(object): raise Exception('{0}: {1}'.format(self.node, self.error)) +class Generic(ConfluentMessage): + + def __init__(self, data): + self.data = data + + def json(self): + return json.dumps(self.data) + + def raw(self): + return self.data + + def html(self): + return json.dumps(self.data) + + class ConfluentResourceUnavailable(ConfluentNodeError): apicode = 503 diff --git a/confluent_server/confluent/plugins/info/layout.py b/confluent_server/confluent/plugins/info/layout.py new file mode 100644 index 00000000..8397af7f --- /dev/null +++ b/confluent_server/confluent/plugins/info/layout.py @@ -0,0 +1,100 @@ +# Copyright 2023 Lenovo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import confluent.core as core +import confluent.messages as msg + +def retrieve(nodes, element, configmanager, inputdata): + locationinfo = configmanager.get_node_attributes(nodes, + (u'enclosure.manager', u'enclosure.bay', u'location.rack', + u'location.row', u'location.u', u'location.height')) + enclosuremap = {} + rackmap = {} + allnodedata = {} + needenclosures = set([]) + locatednodes = set([]) + for node in locationinfo: + nodeinfo = locationinfo[node] + rack = nodeinfo.get(u'location.rack', {}).get('value', '') + u = nodeinfo.get(u'location.u', {}).get('value', None) + row = nodeinfo.get(u'location.row', {}).get('value', '') + enclosure = nodeinfo.get(u'enclosure.manager', {}).get('value', None) + bay = nodeinfo.get(u'enclosure.bay', {}).get('value', None) + height = nodeinfo.get(u'location.height', {}).get('value', None) + if enclosure: + if enclosure not in enclosuremap: + enclosuremap[enclosure] = {} + enclosuremap[enclosure][bay] = node + if u: + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = {'node': enclosure, 'children': enclosuremap[enclosure]} + allnodedata[enclosure] = rackmap[row][rack][u] + if height: + allnodedata[enclosure]['height'] = height + else: # need to see if enclosure lands in the map naturally or need to pull it + needenclosures.add(enclosure) + elif u: + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = {'node': node} + allnodedata[node] = rackmap[row][rack][u] + if height: + allnodedata[node]['height'] = height + locatednodes.add(node) + cfgenc = needenclosures - locatednodes + locationinfo = configmanager.get_node_attributes(cfgenc, (u'location.rack', u'location.row', u'location.u', u'location.height')) + for enclosure in locationinfo: + nodeinfo = locationinfo[enclosure] + rack = nodeinfo.get(u'location.rack', {}).get('value', '') + u = nodeinfo.get(u'location.u', {}).get('value', None) + row = nodeinfo.get(u'location.row', {}).get('value', '') + height = nodeinfo.get(u'location.height', {}).get('value', None) + if u: + allnodedata[enclosure] = {'node': enclosure, 'children': enclosuremap[enclosure]} + if height: + allnodedata[enclosure]['height'] = height + if row not in rackmap: + rackmap[row] = {} + if rack not in rackmap[row]: + rackmap[row][rack] = {} + rackmap[row][rack][u] = allnodedata[enclosure] + results = { + 'errors': [], + 'locations': rackmap, + } + for enclosure in enclosuremap: + if enclosure not in allnodedata: + results['errors'].append('Enclosure {} is missing required location information'.format(enclosure)) + else: + allnodedata[enclosure]['children'] = enclosuremap[enclosure] + needheight = set([]) + for node in allnodedata: + if 'height' not in allnodedata[node]: + needheight.add(node) + needheight = ','.join(needheight) + if needheight: + for rsp in core.handle_path( + '/noderange/{0}/description'.format(needheight), + 'retrieve', configmanager, + inputdata=None): + kvp = rsp.kvpairs + for node in kvp: + allnodedata[node]['height'] = kvp[node]['height'] + yield msg.Generic(results) + From ef9083062bbb75e4e96c6206f921ce63b7247307 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 3 Oct 2023 10:13:53 -0400 Subject: [PATCH 06/30] Make multiple attempts to fetch networking configuration Since confignet runs early in startup, the networking can be a bit fickle. Tolerate outages during early use. --- .../common/profile/scripts/confignet | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/confluent_osdeploy/common/profile/scripts/confignet b/confluent_osdeploy/common/profile/scripts/confignet index dec1808d..f2a2edff 100644 --- a/confluent_osdeploy/common/profile/scripts/confignet +++ b/confluent_osdeploy/common/profile/scripts/confignet @@ -435,10 +435,26 @@ if __name__ == '__main__': curridx = addr[-1] if curridx in doneidxs: continue - status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + for tries in (1, 2 3): + try: + status, nc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/netcfg') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue nc = json.loads(nc) if not dc: - status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + for tries in (1, 2 3): + try: + status, dc = apiclient.HTTPSClient(usejson=True, host=srv).grab_url_with_status('/confluent-api/self/deploycfg2') + break + except Exception: + if tries == 3: + raise + time.sleep(1) + continue dc = json.loads(dc) iname = get_interface_name(idxmap[curridx], nc.get('default', {})) if iname: From ee19386d8c1cb4e4a33b7ec79b2d7dc0fe632976 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 4 Oct 2023 09:49:09 -0400 Subject: [PATCH 07/30] Export nodename in ubuntu pre --- confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh index 4ff1878e..5db222a7 100755 --- a/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh +++ b/confluent_osdeploy/ubuntu22.04/profiles/default/scripts/pre.sh @@ -35,7 +35,7 @@ echo HostbasedUsesNameFromPacketOnly yes >> /etc/ssh/sshd_config.d/confluent.con echo IgnoreRhosts no >> /etc/ssh/sshd_config.d/confluent.conf systemctl restart sshd mkdir -p /etc/confluent -export confluent_profile confluent_mgr +export nodename confluent_profile confluent_mgr curl -f https://$confluent_mgr/confluent-public/os/$confluent_profile/scripts/functions > /etc/confluent/functions . /etc/confluent/functions run_remote_parts pre.d From 9f168aee7302412f91aaa297000645b4f02162fe Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:28:16 -0400 Subject: [PATCH 08/30] docs- batch file systax --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index b1330198..927ce5ae 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -61,7 +61,9 @@ to a blank value will allow masking a group defined attribute with an empty valu or environment variables. * `-s`, `--set`: - Set attributes using a batch file + Set attributes using a batch file rather than the command line. The attributes in the batch file + can be one line of key=value pairs or each attribute can be in its own line. Lines that start with + # sign will be read as commend. See EXAMPLES for batch file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: Prompt if trying to set attributes on more than @@ -120,6 +122,25 @@ to a blank value will allow masking a group defined attribute with an empty valu `d1: net.pxe.switch: pxeswitch1` `d1: net.switch:` +* Setting Attributes using a batch file with syntax similar to command line: + `# cat nodeattributes.batch` + `# power` + `power.psu1.outlet=3 power.psu1.pdu=pdu2` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: 3` + `n41: pdu2` + +* Setting Attributes using a batch file with syntax where each attribute is in its own line: + `# cat nodeattributes.batch` + `# management` + `custom.mgt.switch=switch_main` + `custom.mgt.switch.port=swp4` + `# nodeattrib n41 -s nodeattributes.batch` + `n41: switch_main` + `n41: swp4` + + + ## SEE ALSO nodegroupattrib(8), nodeattribexpressions(5) From d299db3442f3341f348c4e560397c7a493f9eaf7 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:31:24 -0400 Subject: [PATCH 09/30] doc --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 927ce5ae..28e37a5c 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -122,7 +122,7 @@ to a blank value will allow masking a group defined attribute with an empty valu `d1: net.pxe.switch: pxeswitch1` `d1: net.switch:` -* Setting Attributes using a batch file with syntax similar to command line: +* Setting attributes using a batch file with syntax similar to command line: `# cat nodeattributes.batch` `# power` `power.psu1.outlet=3 power.psu1.pdu=pdu2` @@ -130,7 +130,7 @@ to a blank value will allow masking a group defined attribute with an empty valu `n41: 3` `n41: pdu2` -* Setting Attributes using a batch file with syntax where each attribute is in its own line: +* Setting attributes using a batch file with syntax where each attribute is in its own line: `# cat nodeattributes.batch` `# management` `custom.mgt.switch=switch_main` From c8094276d0f9dc5129c740b32285adf66746cd11 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:34:07 -0400 Subject: [PATCH 10/30] typ0_fix --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 28e37a5c..cacfc80f 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -62,8 +62,9 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-s`, `--set`: Set attributes using a batch file rather than the command line. The attributes in the batch file - can be one line of key=value pairs or each attribute can be in its own line. Lines that start with - # sign will be read as commend. See EXAMPLES for batch file syntax. + can be specified as one line of key=value pairs line command line or each attribute can be in + its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch + file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: Prompt if trying to set attributes on more than From ba90609f3b4893f258ac8f66b83edcd166dccaba Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:36:58 -0400 Subject: [PATCH 11/30] documentation for nodeattrib -s --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index cacfc80f..3d66a65f 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -62,8 +62,8 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-s`, `--set`: Set attributes using a batch file rather than the command line. The attributes in the batch file - can be specified as one line of key=value pairs line command line or each attribute can be in - its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch + can be specified as one line of key=value pairs simmilar to command line or each attribute can + be in its own line. Lines that start with # sign will be read as a comment. See EXAMPLES for batch file syntax. * `-m MAXNODES`, `--maxnodes=MAXNODES`: From eca1854d563e718c0bc4030d7b0076c22d12bcf1 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 10:37:41 -0400 Subject: [PATCH 12/30] fix to env doc --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index 3d66a65f..c71e59e7 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -54,7 +54,7 @@ to a blank value will allow masking a group defined attribute with an empty valu * `-e`, `--environment`: Set specified attributes based on exported environment variable of matching name. Environment variable names may be lower case or all upper case. - Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE + Replace . with _ as needed (e.g. info.note may be specified as either $info_note or $INFO_NOTE) * `-p`, `--prompt`: Request interactive prompting to provide values rather than the command line From 67f607a8f16bb534ecc1a001611b112cb55c7cf7 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 16:26:20 -0400 Subject: [PATCH 13/30] fix to synopsis --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index c71e59e7..c0316a7d 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -8,7 +8,7 @@ nodeattrib(8) -- List or change confluent nodes attributes `nodeattrib -c ...` `nodeattrib -e ...` `nodeattrib -p ...` -`nodeattrib -s ` +`nodeattrib -s ...` ## DESCRIPTION From 2e84c73baaa05cfd3840d3f18afc78926bb55bfd Mon Sep 17 00:00:00 2001 From: tkucherera Date: Wed, 4 Oct 2023 16:27:05 -0400 Subject: [PATCH 14/30] '' --- confluent_client/doc/man/nodeattrib.ronn.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_client/doc/man/nodeattrib.ronn.tmpl b/confluent_client/doc/man/nodeattrib.ronn.tmpl index c0316a7d..c8127ad8 100644 --- a/confluent_client/doc/man/nodeattrib.ronn.tmpl +++ b/confluent_client/doc/man/nodeattrib.ronn.tmpl @@ -7,7 +7,7 @@ nodeattrib(8) -- List or change confluent nodes attributes `nodeattrib [ ...]` `nodeattrib -c ...` `nodeattrib -e ...` -`nodeattrib -p ...` +`nodeattrib -p ...` `nodeattrib -s ...` ## DESCRIPTION From 77eec1a791ee37ba15e4e2316dc9ed1369647c44 Mon Sep 17 00:00:00 2001 From: tkucherera Date: Thu, 5 Oct 2023 11:44:04 -0400 Subject: [PATCH 15/30] missing_shlex import in nodeattrib --- confluent_client/bin/nodeattrib | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_client/bin/nodeattrib b/confluent_client/bin/nodeattrib index 8c6f078d..f4b0331f 100755 --- a/confluent_client/bin/nodeattrib +++ b/confluent_client/bin/nodeattrib @@ -22,6 +22,7 @@ import optparse import os import signal import sys +import shlex try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) From a4ea5e5c4b2dda518af813acdd3de61f1fdb5148 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Sat, 7 Oct 2023 09:51:32 -0400 Subject: [PATCH 16/30] Abbreviate sequential nodes When we have sequential nodes, collapse to ':' delimited range. --- confluent_server/confluent/noderange.py | 112 +++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index b59bf7c6..5021d592 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -55,6 +55,92 @@ def humanify_nodename(nodename): return [int(text) if text.isdigit() else text.lower() for text in re.split(numregex, nodename)] +def unnumber_nodename(nodename): + # stub out numbers + chunked = ["{}" if text.isdigit() else text.lower() + for text in re.split(numregex, nodename)] + return chunked + +def getnumbers_nodename(nodename): + return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + +def group_elements(elems): + """ Take the specefied elements and chunk them according to text similarity + """ + prev = None + currchunk = [] + chunked_elems = [currchunk] + for elem in elems: + elemtxt = unnumber_nodename(elem) + if not prev: + prev = elemtxt + currchunk.append(elem) + continue + if prev == elemtxt: + currchunk.append(elem) + else: + currchunk = [elem] + chunked_elems.append(currchunk) + prev = elemtxt + return chunked_elems + +def abbreviate_chunk(chunk, validset): + if len(chunk) < 3: + return sorted(chunk, key=humanify_nodename) + #chunk = sorted(chunk, key=humanify_nodename) + vset = set(validset) + cset = set(chunk) + mins = None + maxs = None + for name in chunk: + currns = getnumbers_nodename(name) + if mins is None: + mins = list(currns) + maxs = list(currns) + continue + for n in range(len(currns)): + if currns[n] < mins[n]: + mins[n] = currns[n] + if currns[n] > maxs[n]: + maxs[n] = currns[n] + tmplt = ''.join(unnumber_nodename(chunk[0])) + bgnr = tmplt.format(*mins) + endr = tmplt.format(*maxs) + nr = '{}:{}'.format(bgnr, endr) + prospect = NodeRange(nr).nodes + ranges = [] + discontinuities = (prospect - vset).union(prospect - cset) + currstart = None + prevnode = None + chunksize = 0 + prospect = sorted(prospect, key=humanify_nodename) + currstart = prospect[0] + while prospect: + currnode = prospect.pop(0) + if currnode in discontinuities: + if chunksize == 0: + continue + elif chunksize == 1: + ranges.append(prevnode) + elif chunksize == 2: + ranges.append(','.join([currstart, prevnode])) + else: + ranges.append(':'.join([currstart, prevnode])) + chunksize = 0 + currstart = None + continue + elif not currstart: + currstart = currnode + chunksize += 1 + prevnode = currnode + if chunksize == 1: + ranges.append(prevnode) + elif chunksize == 2: + ranges.append(','.join([currstart, prevnode])) + elif chunksize != 0: + ranges.append(':'.join([currstart, prevnode])) + return ranges + class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -71,7 +157,8 @@ class ReverseNodeRange(object): @property def noderange(self): subsetgroups = [] - for group in self.cfm.get_groups(sizesort=True): + allgroups = self.cfm.get_groups(sizesort=True) + for group in allgroups: if lastnoderange: for nr in lastnoderange: if lastnoderange[nr] - self.nodes: @@ -88,7 +175,28 @@ class ReverseNodeRange(object): self.nodes -= nl if not self.nodes: break - return ','.join(sorted(subsetgroups) + sorted(self.nodes)) + # then, analyze sequentially identifying matching alpha subsections + # then try out noderange from beginning to end + # we need to know discontinuities, which are either: + # nodes that appear in the noderange that are not in the nodes + # nodes that do not exist at all (we need a noderange modification + # that returns non existing nodes) + ranges = [] + try: + subsetgroups.sort(key=humanify_nodename) + groupchunks = group_elements(subsetgroups) + for gc in groupchunks: + ranges.extend(abbreviate_chunk(gc, allgroups)) + except Exception: + subsetgroups.sort() + try: + nodes = sorted(self.nodes, key=humanify_nodename) + nodechunks = group_elements(nodes) + for nc in nodechunks: + ranges.extend(abbreviate_chunk(nc, self.cfm.list_nodes())) + except Exception: + ranges = sorted(self.nodes) + return ','.join(ranges) From fe27cdea4a8df9bd10de5a6dd340a3716413a9ce Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 9 Oct 2023 17:18:44 -0400 Subject: [PATCH 17/30] Abbreviate harder, using brackets Add a round that collapses as is convenient to bracketed range. --- confluent_server/confluent/noderange.py | 109 +++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 5021d592..e70fa6fe 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -64,6 +64,105 @@ def unnumber_nodename(nodename): def getnumbers_nodename(nodename): return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + +class Bracketer(object): + __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens'] + + def __init__(self, nodename): + self.sequences = [] + realnodename = nodename + if ':' in nodename: + realnodename = nodename.split(':', 1)[0] + self.count = len(getnumbers_nodename(realnodename)) + self.nametmpl = unnumber_nodename(realnodename) + for n in range(self.count): + self.sequences.append(None) + self.diffn = None + self.tokens = [] + self.extend(nodename) + + def extend(self, nodeorseq): + # can only differentiate a single number + endname = None + endnums = None + enddifn = self.diffn + if ':' in nodeorseq: + nodename, endname = nodeorseq.split(':', 1) + else: + nodename = nodeorseq + nums = getnumbers_nodename(nodename) + if endname: + diffcount = 0 + endnums = getnumbers_nodename(endname) + ecount = len(endnums) + if ecount != self.count: + raise Exception("mismatched names passed") + for n in range(ecount): + if endnums[n] != nums[n]: + enddifn = n + diffcount += 1 + if diffcount > 1: + if self.sequences: + self.flush_current() + self.tokens.append(nodeorseq) # TODO: could just abbreviate this with multiple []... + return + for n in range(self.count): + if endnums and endnums[n] != nums[n]: + outval = '{}:{}'.format(nums[n], endnums[n]) + else: + outval = '{}'.format(nums[n]) + if self.sequences[n] is None: + # We initialize to text pieces, 'currstart', and 'prev' number + self.sequences[n] = [[outval], nums[n], nums[n]] + elif self.sequences[n][2] == nums[n]: + continue # new nodename has no new number, keep going + elif self.sequences[n][2] != nums[n]: + if self.diffn is not None and (n != self.diffn or enddifn != n): + self.flush_current() + self.sequences[n] = [[], nums[n], nums[n]] + self.diffn = n + self.sequences[n][0].append(outval) + self.sequences[n][2] = nums[n] + elif False: # previous attempt + # A discontinuity, need to close off previous chunk + currstart = self.sequences[n][1] + prevnum = self.sequences[n][2] + if currstart == prevnum: + self.sequences[n][0].append('{}'.format(currstart)) + elif prevnum == currstart + 1: + self.sequences[n][0].append('{},{}'.format(currstart, prevnum)) + else: + self.sequences[n][0].append('{}:{}'.format(currstart, prevnum)) + self.sequences[n][1] = nums[n] + self.sequences[n][2] = nums[n] + elif False: # self.sequences[n][2] == nums[n] - 1: # sequential, increment prev + self.sequences[n][2] = nums[n] + else: + raise Exception('Decreasing node in extend call, not supported') + + def flush_current(self): + txtfields = [] + for n in range(self.count): + txtfield = ','.join(self.sequences[n][0]) + #if self.sequences[n][1] == self.sequences[n][2]: + # txtfield.append('{}'.format(self.sequences[n][1])) + #else: + # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + if txtfield.isdigit(): + txtfields.append(txtfield) + else: + txtfields.append('[{}]'.format(txtfield)) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) + self.sequences = [] + for n in range(self.count): + self.sequences.append(None) + + @property + def range(self): + if self.sequences: + self.flush_current() + return ','.join(self.tokens) + def group_elements(elems): """ Take the specefied elements and chunk them according to text similarity """ @@ -123,7 +222,7 @@ def abbreviate_chunk(chunk, validset): elif chunksize == 1: ranges.append(prevnode) elif chunksize == 2: - ranges.append(','.join([currstart, prevnode])) + ranges.extend([currstart, prevnode]) else: ranges.append(':'.join([currstart, prevnode])) chunksize = 0 @@ -136,7 +235,7 @@ def abbreviate_chunk(chunk, validset): if chunksize == 1: ranges.append(prevnode) elif chunksize == 2: - ranges.append(','.join([currstart, prevnode])) + ranges.extend([currstart, prevnode]) elif chunksize != 0: ranges.append(':'.join([currstart, prevnode])) return ranges @@ -193,7 +292,11 @@ class ReverseNodeRange(object): nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: - ranges.extend(abbreviate_chunk(nc, self.cfm.list_nodes())) + currchunks = abbreviate_chunk(nc, self.cfm.list_nodes()) + bracketer = Bracketer(currchunks[0]) + for chnk in currchunks[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) except Exception: ranges = sorted(self.nodes) return ','.join(ranges) From c254564f021264b7de200507882f8a57715a6926 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 10 Oct 2023 12:47:19 -0400 Subject: [PATCH 18/30] Fully give up on multi-iterator abbreviation There's too many cases that can go wrong. Note that with this lower ambition, it would be possible to significantly streamline the implementation. Notably, the 'find discontinuities' approach was selected to *try* to support multiple iterators, but since that didn't pan out, a more straightforward numerical strategy can be used from the onset. --- confluent_server/confluent/noderange.py | 78 ++++++++++++++++--------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e70fa6fe..e2f88b74 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -100,11 +100,19 @@ class Bracketer(object): for n in range(ecount): if endnums[n] != nums[n]: enddifn = n + if self.diffn is None: + self.diffn = enddifn diffcount += 1 - if diffcount > 1: + if diffcount > 1 or enddifn != self.diffn: if self.sequences: self.flush_current() - self.tokens.append(nodeorseq) # TODO: could just abbreviate this with multiple []... + txtfields = [] + for idx in range(len(nums)): + if endnums[idx] == nums[idx]: + txtfields.append(nums[idx]) + else: + txtfields.append('[{}:{}]'.format(nums[idx], endnums[idx])) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) return for n in range(self.count): if endnums and endnums[n] != nums[n]: @@ -142,17 +150,18 @@ class Bracketer(object): def flush_current(self): txtfields = [] - for n in range(self.count): - txtfield = ','.join(self.sequences[n][0]) - #if self.sequences[n][1] == self.sequences[n][2]: - # txtfield.append('{}'.format(self.sequences[n][1])) - #else: - # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) - if txtfield.isdigit(): - txtfields.append(txtfield) - else: - txtfields.append('[{}]'.format(txtfield)) - self.tokens.append(''.join(self.nametmpl).format(*txtfields)) + if self.sequences and self.sequences[0] is not None: + for n in range(self.count): + txtfield = ','.join(self.sequences[n][0]) + #if self.sequences[n][1] == self.sequences[n][2]: + # txtfield.append('{}'.format(self.sequences[n][1])) + #else: + # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + if txtfield.isdigit(): + txtfields.append(txtfield) + else: + txtfields.append('[{}]'.format(txtfield)) + self.tokens.append(''.join(self.nametmpl).format(*txtfields)) self.sequences = [] for n in range(self.count): self.sequences.append(None) @@ -186,28 +195,41 @@ def group_elements(elems): def abbreviate_chunk(chunk, validset): if len(chunk) < 3: return sorted(chunk, key=humanify_nodename) - #chunk = sorted(chunk, key=humanify_nodename) vset = set(validset) cset = set(chunk) - mins = None - maxs = None + minmaxes = [None] + diffn = None + prevns = None for name in chunk: currns = getnumbers_nodename(name) - if mins is None: - mins = list(currns) - maxs = list(currns) + if minmaxes[-1] is None: + minmaxes[-1] = [list(currns), list(currns)] continue + if prevns is None: + prevns = currns for n in range(len(currns)): - if currns[n] < mins[n]: - mins[n] = currns[n] - if currns[n] > maxs[n]: - maxs[n] = currns[n] + if prevns[n] != currns[n]: + if diffn is None: + diffn = n + elif diffn != n: + minmaxes.append([list(currns), list(currns)]) + continue + if currns[n] < minmaxes[-1][0][n]: + minmaxes.append([list(currns), list(currns)]) + if currns[n] > minmaxes[-1][1][n]: + minmaxes[-1][1][n] = currns[n] + prevns = currns tmplt = ''.join(unnumber_nodename(chunk[0])) + ranges = [] + for x in minmaxes: + process_abbreviation(vset, cset, x[0], x[1], tmplt, ranges) + return ranges + +def process_abbreviation(vset, cset, mins, maxs, tmplt, ranges): bgnr = tmplt.format(*mins) endr = tmplt.format(*maxs) nr = '{}:{}'.format(bgnr, endr) prospect = NodeRange(nr).nodes - ranges = [] discontinuities = (prospect - vset).union(prospect - cset) currstart = None prevnode = None @@ -238,8 +260,6 @@ def abbreviate_chunk(chunk, validset): ranges.extend([currstart, prevnode]) elif chunksize != 0: ranges.append(':'.join([currstart, prevnode])) - return ranges - class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -285,7 +305,11 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: - ranges.extend(abbreviate_chunk(gc, allgroups)) + currchunks = abbreviate_chunk(gc, allgroups) + bracketer = Bracketer(currchunks[0]) + for chnk in currchunks[1:]: + bracketer.extend(chnk) + ranges.append(bracketer.range) except Exception: subsetgroups.sort() try: From e9a2f57ad8d262dae5da54bdddfc1906a0e652a4 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 10 Oct 2023 16:56:32 -0400 Subject: [PATCH 19/30] Simplify the noderange abbreviation Since the multi-iterator ambition is out, ditch the expensive set wrangling step. Now the procedure is: -Suck nodes into groups, as possible -Separately for groups and nodes: -Sort the elements -Chunk the elements based on 'non-numberical' situation matching -analyze the iterators to apply [] to shorten the name -Multi-iterator will cause a discontinuity, and a new ',' delimited name gets constructed --- confluent_server/confluent/noderange.py | 146 ++++-------------------- 1 file changed, 23 insertions(+), 123 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e2f88b74..9a7ed44b 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -80,70 +80,39 @@ class Bracketer(object): self.diffn = None self.tokens = [] self.extend(nodename) + if self.count == 0: + self.tokens = [nodename] def extend(self, nodeorseq): # can only differentiate a single number endname = None endnums = None - enddifn = self.diffn if ':' in nodeorseq: nodename, endname = nodeorseq.split(':', 1) else: nodename = nodeorseq nums = getnumbers_nodename(nodename) - if endname: - diffcount = 0 - endnums = getnumbers_nodename(endname) - ecount = len(endnums) - if ecount != self.count: - raise Exception("mismatched names passed") - for n in range(ecount): - if endnums[n] != nums[n]: - enddifn = n - if self.diffn is None: - self.diffn = enddifn - diffcount += 1 - if diffcount > 1 or enddifn != self.diffn: - if self.sequences: - self.flush_current() - txtfields = [] - for idx in range(len(nums)): - if endnums[idx] == nums[idx]: - txtfields.append(nums[idx]) - else: - txtfields.append('[{}:{}]'.format(nums[idx], endnums[idx])) - self.tokens.append(''.join(self.nametmpl).format(*txtfields)) - return for n in range(self.count): - if endnums and endnums[n] != nums[n]: - outval = '{}:{}'.format(nums[n], endnums[n]) - else: - outval = '{}'.format(nums[n]) if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number - self.sequences[n] = [[outval], nums[n], nums[n]] + self.sequences[n] = [[], nums[n], nums[n]] elif self.sequences[n][2] == nums[n]: continue # new nodename has no new number, keep going elif self.sequences[n][2] != nums[n]: - if self.diffn is not None and (n != self.diffn or enddifn != n): + if self.diffn is not None and n != self.diffn: self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] - self.diffn = n - self.sequences[n][0].append(outval) - self.sequences[n][2] = nums[n] - elif False: # previous attempt - # A discontinuity, need to close off previous chunk - currstart = self.sequences[n][1] - prevnum = self.sequences[n][2] - if currstart == prevnum: - self.sequences[n][0].append('{}'.format(currstart)) - elif prevnum == currstart + 1: - self.sequences[n][0].append('{},{}'.format(currstart, prevnum)) + self.diffn = None else: - self.sequences[n][0].append('{}:{}'.format(currstart, prevnum)) - self.sequences[n][1] = nums[n] - self.sequences[n][2] = nums[n] - elif False: # self.sequences[n][2] == nums[n] - 1: # sequential, increment prev + self.diffn = n + if self.sequences[n][2] == (nums[n] - 1): + self.sequences[n][2] = nums[n] + elif self.sequences[n][2] < (nums[n] - 1): + if self.sequences[n][2] != self.sequences[n][1]: + self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + else: + self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + self.sequences[n][1] = nums[n] self.sequences[n][2] = nums[n] else: raise Exception('Decreasing node in extend call, not supported') @@ -152,11 +121,11 @@ class Bracketer(object): txtfields = [] if self.sequences and self.sequences[0] is not None: for n in range(self.count): + if self.sequences[n][1] == self.sequences[n][2]: + self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + else: + self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) txtfield = ','.join(self.sequences[n][0]) - #if self.sequences[n][1] == self.sequences[n][2]: - # txtfield.append('{}'.format(self.sequences[n][1])) - #else: - # txtfield.append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) if txtfield.isdigit(): txtfields.append(txtfield) else: @@ -172,6 +141,7 @@ class Bracketer(object): self.flush_current() return ','.join(self.tokens) + def group_elements(elems): """ Take the specefied elements and chunk them according to text similarity """ @@ -192,74 +162,6 @@ def group_elements(elems): prev = elemtxt return chunked_elems -def abbreviate_chunk(chunk, validset): - if len(chunk) < 3: - return sorted(chunk, key=humanify_nodename) - vset = set(validset) - cset = set(chunk) - minmaxes = [None] - diffn = None - prevns = None - for name in chunk: - currns = getnumbers_nodename(name) - if minmaxes[-1] is None: - minmaxes[-1] = [list(currns), list(currns)] - continue - if prevns is None: - prevns = currns - for n in range(len(currns)): - if prevns[n] != currns[n]: - if diffn is None: - diffn = n - elif diffn != n: - minmaxes.append([list(currns), list(currns)]) - continue - if currns[n] < minmaxes[-1][0][n]: - minmaxes.append([list(currns), list(currns)]) - if currns[n] > minmaxes[-1][1][n]: - minmaxes[-1][1][n] = currns[n] - prevns = currns - tmplt = ''.join(unnumber_nodename(chunk[0])) - ranges = [] - for x in minmaxes: - process_abbreviation(vset, cset, x[0], x[1], tmplt, ranges) - return ranges - -def process_abbreviation(vset, cset, mins, maxs, tmplt, ranges): - bgnr = tmplt.format(*mins) - endr = tmplt.format(*maxs) - nr = '{}:{}'.format(bgnr, endr) - prospect = NodeRange(nr).nodes - discontinuities = (prospect - vset).union(prospect - cset) - currstart = None - prevnode = None - chunksize = 0 - prospect = sorted(prospect, key=humanify_nodename) - currstart = prospect[0] - while prospect: - currnode = prospect.pop(0) - if currnode in discontinuities: - if chunksize == 0: - continue - elif chunksize == 1: - ranges.append(prevnode) - elif chunksize == 2: - ranges.extend([currstart, prevnode]) - else: - ranges.append(':'.join([currstart, prevnode])) - chunksize = 0 - currstart = None - continue - elif not currstart: - currstart = currnode - chunksize += 1 - prevnode = currnode - if chunksize == 1: - ranges.append(prevnode) - elif chunksize == 2: - ranges.extend([currstart, prevnode]) - elif chunksize != 0: - ranges.append(':'.join([currstart, prevnode])) class ReverseNodeRange(object): """Abbreviate a set of nodes to a shorter noderange representation @@ -305,9 +207,8 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: - currchunks = abbreviate_chunk(gc, allgroups) - bracketer = Bracketer(currchunks[0]) - for chnk in currchunks[1:]: + bracketer = Bracketer(gc[0]) + for chnk in gc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: @@ -316,9 +217,8 @@ class ReverseNodeRange(object): nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: - currchunks = abbreviate_chunk(nc, self.cfm.list_nodes()) - bracketer = Bracketer(currchunks[0]) - for chnk in currchunks[1:]: + bracketer = Bracketer(nc[0]) + for chnk in nc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: From 2d906e188628aeb3255915a86864135b8d96de7d Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Wed, 11 Oct 2023 10:15:24 -0400 Subject: [PATCH 20/30] Fix handling of pre-existing array --- misc/swraid | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/swraid b/misc/swraid index 3e234dc8..836f1fb1 100644 --- a/misc/swraid +++ b/misc/swraid @@ -1,6 +1,6 @@ DEVICES="/dev/sda /dev/sdb" RAIDLEVEL=1 -mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null && exit 0 +mdadm --detail /dev/md*|grep 'Version : 1.0' >& /dev/null || ( lvm vgchange -a n mdadm -S -s NUMDEVS=$(for dev in $DEVICES; do @@ -14,5 +14,6 @@ mdadm -C /dev/md/raid $DEVICES -n $NUMDEVS -e 1.0 -l $RAIDLEVEL # shut and restart array to prime things for anaconda mdadm -S -s mdadm --assemble --scan +) readlink /dev/md/raid|sed -e 's/.*\///' > /tmp/installdisk From 6e4d9d9eb485107f327eedf6477d7c5d7308f4ab Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 14:46:09 -0400 Subject: [PATCH 21/30] Address potential slowdowns by misbehaving DNS For one, shorten the DNS timeout, if the DNS server is completely out, give up quickly. For another, if a host has a large number of net.X.hostnames, the sequential nature was intolerable. Have each network be evaluated in a greenthread concurrently to serve the DNS latency concurrently. --- confluent_server/confluent/netutil.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/confluent_server/confluent/netutil.py b/confluent_server/confluent/netutil.py index 37e8d198..9e9fd597 100644 --- a/confluent_server/confluent/netutil.py +++ b/confluent_server/confluent/netutil.py @@ -25,6 +25,9 @@ import eventlet.support.greendns import os getaddrinfo = eventlet.support.greendns.getaddrinfo +eventlet.support.greendns.resolver.clear() +eventlet.support.greendns.resolver._resolver.lifetime = 1 + def msg_align(len): return (len + 3) & ~3 @@ -333,11 +336,13 @@ def get_full_net_config(configmanager, node, serverip=None): myaddrs = get_addresses_by_serverip(serverip) nm = NetManager(myaddrs, node, configmanager) defaultnic = {} + ppool = eventlet.greenpool.GreenPool(64) if None in attribs: - nm.process_attribs(None, attribs[None]) + ppool.spawn(nm.process_attribs, None, attribs[None]) del attribs[None] for netname in sorted(attribs): - nm.process_attribs(netname, attribs[netname]) + ppool.spawn(nm.process_attribs, netname, attribs[netname]) + ppool.waitall() retattrs = {} if None in nm.myattribs: retattrs['default'] = nm.myattribs[None] From 3a6932ea6dab02a69c03a2a8f30e3a4e7623e5e2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 15:28:54 -0400 Subject: [PATCH 22/30] Start tracking padding during abbreviation This will take care of padding when padding is consistent across a range. However, we still have a problem with a progression like: 01 02 ... 98 099 100 Where numbers in the middle start getting padding unexpectedly without a leading digit. --- confluent_server/confluent/noderange.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 9a7ed44b..1c023707 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -62,14 +62,15 @@ def unnumber_nodename(nodename): return chunked def getnumbers_nodename(nodename): - return [int(x) for x in re.split(numregex, nodename) if x.isdigit()] + return [x for x in re.split(numregex, nodename) if x.isdigit()] class Bracketer(object): - __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens'] + __slots__ = ['sequences', 'count', 'nametmpl', 'diffn', 'tokens', 'numlens'] def __init__(self, nodename): self.sequences = [] + self.numlens = [] realnodename = nodename if ':' in nodename: realnodename = nodename.split(':', 1)[0] @@ -77,6 +78,7 @@ class Bracketer(object): self.nametmpl = unnumber_nodename(realnodename) for n in range(self.count): self.sequences.append(None) + self.numlens.append([0, 0]) self.diffn = None self.tokens = [] self.extend(nodename) @@ -84,6 +86,7 @@ class Bracketer(object): self.tokens = [nodename] def extend(self, nodeorseq): + # crap... failed to preserve 0 padding foro fixe width # can only differentiate a single number endname = None endnums = None @@ -91,29 +94,37 @@ class Bracketer(object): nodename, endname = nodeorseq.split(':', 1) else: nodename = nodeorseq - nums = getnumbers_nodename(nodename) + txtnums = getnumbers_nodename(nodename) + nums = [int(x) for x in txtnums] for n in range(self.count): if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] elif self.sequences[n][2] == nums[n]: continue # new nodename has no new number, keep going elif self.sequences[n][2] != nums[n]: if self.diffn is not None and n != self.diffn: self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] + self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] self.diffn = None else: self.diffn = n if self.sequences[n][2] == (nums[n] - 1): self.sequences[n][2] = nums[n] + self.numlens[n][1] = len(txtnums[n]) elif self.sequences[n][2] < (nums[n] - 1): if self.sequences[n][2] != self.sequences[n][1]: - self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) else: - self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) self.sequences[n][1] = nums[n] + self.numlens[n][0] = len(txtnums[n]) self.sequences[n][2] = nums[n] + self.numlens[n][1] = len(txtnums[n]) else: raise Exception('Decreasing node in extend call, not supported') From bfbb7c2843e4353c0342b0b7bb1a4bc40beda534 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 12 Oct 2023 16:09:40 -0400 Subject: [PATCH 23/30] Handle mid-range pad changing, and identical names with only pad difference This would be painful to operate, but if done at least reverse noderange will faithfully honor it now. --- confluent_server/confluent/noderange.py | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index 1c023707..e76391e8 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -86,7 +86,6 @@ class Bracketer(object): self.tokens = [nodename] def extend(self, nodeorseq): - # crap... failed to preserve 0 padding foro fixe width # can only differentiate a single number endname = None endnums = None @@ -97,23 +96,26 @@ class Bracketer(object): txtnums = getnumbers_nodename(nodename) nums = [int(x) for x in txtnums] for n in range(self.count): + padto = len(txtnums[n]) + needpad = (padto != len('{}'.format(nums[n]))) if self.sequences[n] is None: # We initialize to text pieces, 'currstart', and 'prev' number self.sequences[n] = [[], nums[n], nums[n]] self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] - elif self.sequences[n][2] == nums[n]: + elif self.sequences[n][2] == nums[n] and self.numlens[n][1] == padto: continue # new nodename has no new number, keep going - elif self.sequences[n][2] != nums[n]: - if self.diffn is not None and n != self.diffn: + else: # if self.sequences[n][2] != nums[n] or : + if self.diffn is not None and (n != self.diffn or + (needpad and padto != self.numlens[n][1])): self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] - self.numlens[n] = [len(txtnums[n]), len(txtnums[n])] + self.numlens[n] = [padto, padto] self.diffn = None else: self.diffn = n if self.sequences[n][2] == (nums[n] - 1): self.sequences[n][2] = nums[n] - self.numlens[n][1] = len(txtnums[n]) + self.numlens[n][1] = padto elif self.sequences[n][2] < (nums[n] - 1): if self.sequences[n][2] != self.sequences[n][1]: fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) @@ -122,20 +124,20 @@ class Bracketer(object): fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) self.sequences[n][1] = nums[n] - self.numlens[n][0] = len(txtnums[n]) + self.numlens[n][0] = padto self.sequences[n][2] = nums[n] - self.numlens[n][1] = len(txtnums[n]) - else: - raise Exception('Decreasing node in extend call, not supported') + self.numlens[n][1] = padto def flush_current(self): txtfields = [] if self.sequences and self.sequences[0] is not None: for n in range(self.count): if self.sequences[n][1] == self.sequences[n][2]: - self.sequences[n][0].append('{}'.format(self.sequences[n][1])) + fmtstr = '{{:0{}d}}'.format(self.numlens[n][0]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1])) else: - self.sequences[n][0].append('{}:{}'.format(self.sequences[n][1], self.sequences[n][2])) + fmtstr = '{{:0{}d}}:{{:0{}d}}'.format(*self.numlens[n]) + self.sequences[n][0].append(fmtstr.format(self.sequences[n][1], self.sequences[n][2])) txtfield = ','.join(self.sequences[n][0]) if txtfield.isdigit(): txtfields.append(txtfield) From 0434f38ea12035b98e6997fa06c755bb55694bc9 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Fri, 13 Oct 2023 15:25:08 -0400 Subject: [PATCH 24/30] Add iterm and kitty image support to stats This delivers improved graphics speed and quality for selected terminals. --- confluent_client/bin/stats | 45 ++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/confluent_client/bin/stats b/confluent_client/bin/stats index 7158e7a7..94af75db 100755 --- a/confluent_client/bin/stats +++ b/confluent_client/bin/stats @@ -16,13 +16,10 @@ # limitations under the License. import argparse +import base64 import csv -import fcntl import io import numpy as np - -import os -import subprocess import sys try: @@ -35,7 +32,31 @@ except ImportError: pass -def plot(gui, output, plotdata, bins): +def iterm_draw(data): + databuf = data.getbuffer() + datalen = len(databuf) + data = base64.b64encode(databuf).decode('utf8') + sys.stdout.write( + '\x1b]1337;File=inline=1;size={}:'.format(datalen)) + sys.stdout.write(data) + sys.stdout.write('\a') + sys.stdout.write('\n') + sys.stdout.flush() + + +def kitty_draw(data): + data = base64.b64encode(data.getbuffer()) + while data: + chunk, data = data[:4096], data[4096:] + m = 1 if data else 0 + sys.stdout.write('\x1b_Ga=T,f=100,m={};'.format(m)) + sys.stdout.write(chunk.decode('utf8')) + sys.stdout.write('\x1b\\') + sys.stdout.flush() + sys.stdout.write('\n') + + +def plot(gui, output, plotdata, bins, fmt): import matplotlib as mpl if gui and mpl.get_backend() == 'agg': sys.stderr.write('Error: No GUI backend available and -g specified!\n') @@ -51,8 +72,13 @@ def plot(gui, output, plotdata, bins): tdata = io.BytesIO() plt.savefig(tdata) if not gui and not output: - writer = DumbWriter() - writer.draw(tdata) + if fmt == 'sixel': + writer = DumbWriter() + writer.draw(tdata) + elif fmt == 'kitty': + kitty_draw(tdata) + elif fmt == 'iterm': + iterm_draw(tdata) return n, bins def textplot(plotdata, bins): @@ -81,7 +107,8 @@ histogram = False aparser = argparse.ArgumentParser(description='Quick access to common statistics') aparser.add_argument('-c', type=int, default=0, help='Column number to analyze (default is last column)') aparser.add_argument('-d', default=None, help='Value used to separate columns') -aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in sixel format') +aparser.add_argument('-x', default=False, action='store_true', help='Output histogram in graphical format') +aparser.add_argument('-f', default='sixel', help='Format for histogram output (sixel/iterm/kitty)') aparser.add_argument('-s', default=0, help='Number of header lines to skip before processing') aparser.add_argument('-g', default=False, action='store_true', help='Open histogram in separate graphical window') aparser.add_argument('-o', default=None, help='Output histogram to the specified filename in PNG format') @@ -138,7 +165,7 @@ while data: data = list(csv.reader([data], delimiter=delimiter))[0] n = None if args.g or args.o or args.x: - n, bins = plot(args.g, args.o, plotdata, bins=args.b) + n, bins = plot(args.g, args.o, plotdata, bins=args.b, fmt=args.f) if args.t: n, bins = textplot(plotdata, bins=args.b) print('Samples: {5} Min: {3} Median: {0} Mean: {1} Max: {4} StandardDeviation: {2} Sum: {6}'.format(np.median(plotdata), np.mean(plotdata), np.std(plotdata), np.min(plotdata), np.max(plotdata), len(plotdata), np.sum(plotdata))) From 06d18cec63e2da6ddf3b0fe3f3bdc4bf0d0412aa Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Mon, 16 Oct 2023 08:29:45 -0400 Subject: [PATCH 25/30] Fix abbreviation when pad decreases This is a bizarre way to work, but it should be valid. --- confluent_server/confluent/noderange.py | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index e76391e8..dcf0a1cb 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -106,6 +106,7 @@ class Bracketer(object): continue # new nodename has no new number, keep going else: # if self.sequences[n][2] != nums[n] or : if self.diffn is not None and (n != self.diffn or + (padto < self.numlens[n][1]) or (needpad and padto != self.numlens[n][1])): self.flush_current() self.sequences[n] = [[], nums[n], nums[n]] From b91a19418453b902c4b066e30fd89194c069d6d2 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Tue, 17 Oct 2023 16:29:30 -0400 Subject: [PATCH 26/30] Improve selfselfice performance with yaml The yaml python default behavior is 'pure python' and is tortuously slow. As a test, yaml dump of a 17,000 element list took 70 seconds in default configuration. Opting into the C functions, that time comes down to 10 seconds, a nice and easy improvement for generic yaml. For dumping a simple dumb list (e.g. the nodelist for ssh), a special case yaml-looking result is done, which hits 0.4 seconds on that same test. So this special case is added to nodelist, which can be very long and very in demand at the same time. --- confluent_server/confluent/selfservice.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/confluent_server/confluent/selfservice.py b/confluent_server/confluent/selfservice.py index 04030491..cd4180c7 100644 --- a/confluent_server/confluent/selfservice.py +++ b/confluent_server/confluent/selfservice.py @@ -19,6 +19,12 @@ import json import os import time import yaml +try: + from yaml import CSafeDumper as SafeDumper + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + from yaml import SafeDumper import confluent.discovery.protocols.ssdp as ssdp import eventlet webclient = eventlet.import_patched('pyghmi.util.webclient') @@ -31,7 +37,20 @@ currtzvintage = None def yamldump(input): - return yaml.safe_dump(input, default_flow_style=False) + return yaml.dump_all([input], Dumper=SafeDumper, default_flow_style=False) + +def yamlload(input): + return yaml.load(input, Loader=SafeLoader) + +def listdump(input): + # special case yaml for flat dumb list + # this is about 25x faster than doing full yaml dump even with CSafeDumper + # with a 17,000 element list + retval = '' + for entry in input: + retval += '- ' + entry + '\n' + return retval + def get_extra_names(nodename, cfg, myip=None): names = set([]) @@ -402,10 +421,13 @@ def handle_request(env, start_response): yield node + '\n' else: start_response('200 OK', (('Content-Type', retype),)) - yield dumper(list(util.natural_sort(nodes))) + if retype == 'application/yaml': + yield listdump(list(util.natural_sort(nodes))) + else: + yield dumper(list(util.natural_sort(nodes))) elif env['PATH_INFO'] == '/self/remoteconfigbmc' and reqbody: try: - reqbody = yaml.safe_load(reqbody) + reqbody = yamlload(reqbody) except Exception: reqbody = None cfgmod = reqbody.get('configmod', 'unspecified') @@ -419,7 +441,7 @@ def handle_request(env, start_response): start_response('200 Ok', ()) yield 'complete' elif env['PATH_INFO'] == '/self/updatestatus' and reqbody: - update = yaml.safe_load(reqbody) + update = yamlload(reqbody) statusstr = update.get('state', None) statusdetail = update.get('state_detail', None) didstateupdate = False @@ -522,7 +544,7 @@ def handle_request(env, start_response): '/var/lib/confluent/public/os/{0}/scripts/{1}') if slist: start_response('200 OK', (('Content-Type', 'application/yaml'),)) - yield yaml.safe_dump(util.natural_sort(slist), default_flow_style=False) + yield yamldump(util.natural_sort(slist)) else: start_response('200 OK', ()) yield '' From 8b150a904765f79fbe99d848a952b8513723f42e Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 09:25:57 -0400 Subject: [PATCH 27/30] Fix for post group failures A node failure after group failure would erase the group from range. Further, correct an issue where an empty nodeset would trigger a bad behavior. --- confluent_server/confluent/noderange.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/confluent_server/confluent/noderange.py b/confluent_server/confluent/noderange.py index dcf0a1cb..df4552b8 100644 --- a/confluent_server/confluent/noderange.py +++ b/confluent_server/confluent/noderange.py @@ -221,22 +221,27 @@ class ReverseNodeRange(object): subsetgroups.sort(key=humanify_nodename) groupchunks = group_elements(subsetgroups) for gc in groupchunks: + if not gc: + continue bracketer = Bracketer(gc[0]) for chnk in gc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: subsetgroups.sort() + ranges.extend(subsetgroups) try: nodes = sorted(self.nodes, key=humanify_nodename) nodechunks = group_elements(nodes) for nc in nodechunks: + if not nc: + continue bracketer = Bracketer(nc[0]) for chnk in nc[1:]: bracketer.extend(chnk) ranges.append(bracketer.range) except Exception: - ranges = sorted(self.nodes) + ranges.extend(sorted(self.nodes)) return ','.join(ranges) From 063bfc17a57cad22e5fbc3f42d29458a5d271790 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 10:40:57 -0400 Subject: [PATCH 28/30] Start using container for final build process Makes supporting the base platform easier by largely ignoring the base platform. --- confluent_osdeploy/buildrpm-aarch64 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 867c0102..7f95852b 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -29,4 +29,5 @@ mv confluent_el8bin.tar.xz ~/rpmbuild/SOURCES/ mv confluent_el9bin.tar.xz ~/rpmbuild/SOURCES/ rm -rf el9bin rm -rf el8bin -rpmbuild -ba confluent_osdeploy-aarch64.spec +podman run --privileged --rm -v $HOME:/root el8builder rpmbuild -ba /root/confluent/confluent_osdeploy/confluent_osdeploy-aarch64.spec + From 913a26aec93b3c0d55dea1963e3c5a8fe46dae14 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 10:42:39 -0400 Subject: [PATCH 29/30] Change to consistent CWD for osdeploy arm build --- confluent_osdeploy/buildrpm-aarch64 | 1 + 1 file changed, 1 insertion(+) diff --git a/confluent_osdeploy/buildrpm-aarch64 b/confluent_osdeploy/buildrpm-aarch64 index 7f95852b..83ffc519 100644 --- a/confluent_osdeploy/buildrpm-aarch64 +++ b/confluent_osdeploy/buildrpm-aarch64 @@ -1,3 +1,4 @@ +cd $(dirname $0) VERSION=`git describe|cut -d- -f 1` NUMCOMMITS=`git describe|cut -d- -f 2` if [ "$NUMCOMMITS" != "$VERSION" ]; then From 9c9d71882c76bbf4bf989b8c8dacced5d7878794 Mon Sep 17 00:00:00 2001 From: Jarrod Johnson Date: Thu, 19 Oct 2023 15:51:40 -0400 Subject: [PATCH 30/30] Disable keepalive Unfortunately, apache can get a bit odd over how it reports a non-viable open socket for keepalive, which can happen in certain windows. Disable the keepalive feature and take some performance penalty in browsers for the sake of more consistent return behavior and fewer idle greenthreads doing nothing. --- confluent_server/confluent/httpapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/confluent_server/confluent/httpapi.py b/confluent_server/confluent/httpapi.py index 5a145a0c..f36f2c73 100644 --- a/confluent_server/confluent/httpapi.py +++ b/confluent_server/confluent/httpapi.py @@ -1084,7 +1084,7 @@ def serve(bind_host, bind_port): pass # we gave it our best shot there try: eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False, - debug=False, socket_timeout=60) + debug=False, socket_timeout=60, keepalive=False) except TypeError: # Older eventlet in place, skip arguments it does not understand eventlet.wsgi.server(sock, resourcehandler, log=False, debug=False)