xCAT/confluent
2
0
Fork 0
mirror of https://github.com/xcat2/confluent.git synced 2026-09-29 00:31:09 +00:00
Code Issues Projects Releases Wiki Activity

Limit children of noderun/nodeshell

Browse Source 
Implement throttling of concurrent processes.  While 64 might be a bit conservative in this day and
age, we were exceeding FD_SETSIZE, and maybe we were also lucky.  Throttle to 168 for the moment.
This commit is contained in:
Jarrod Johnson
2017-06-08 10:03:56 -04:00
parent 0b30b92bc9
commit b22073e106
2 changed files with 140 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
confluent_client/bin/noderun
+70 -51
View File
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import deque
import optparse
import os
import select
@@ -35,57 +36,75 @@ if path.startswith('/opt'):
import confluent.client as client
argparser = optparse.OptionParser(
usage="Usage: %prog node commandexpression",
epilog="Expressions are the same as in attributes, e.g. "
"'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
argparser.disable_interspersed_args()
(options, args) = argparser.parse_args()
if len(args) < 2:
argparser.print_help()
sys.exit(1)
c = client.Command()
cmdstr = " ".join(args[1:])
def run():
concurrentprocs = 168
# among other things, FD_SETSIZE limits. Besides, spawning too many
# processes can be unkind for the unaware on memory pressure and such...
argparser = optparse.OptionParser(
usage="Usage: %prog node commandexpression",
epilog="Expressions are the same as in attributes, e.g. "
"'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
argparser.disable_interspersed_args()
(options, args) = argparser.parse_args()
if len(args) < 2:
argparser.print_help()
sys.exit(1)
c = client.Command()
cmdstr = " ".join(args[1:])
nodeforpopen = {}
popens = []
for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
{'expression': cmdstr}):
ex = exp['databynode']
for node in ex:
cmd = ex[node]['value'].encode('utf-8')
cmdv = shlex.split(cmd)
nopen = subprocess.Popen(
cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
popens.append(nopen)
nodeforpopen[nopen] = node
currprocs = 0
all = set([])
pipedesc = {}
pendingexecs = deque()
all = set([])
pipedesc = {}
exitcode = 0
for pop in popens:
node = nodeforpopen[pop]
pipedesc[pop.stdout] = { 'node': node, 'popen': pop, 'type': 'stdout'}
pipedesc[pop.stderr] = {'node': node, 'popen': pop, 'type': 'stderr'}
all.add(pop.stdout)
all.add(pop.stderr)
rdy, _, _ = select.select(all, [], [], 10)
while all:
for r in rdy:
data = r.readline()
desc = pipedesc[r]
if data:
node = desc['node']
if desc['type'] == 'stdout':
sys.stdout.write('{0}: {1}'.format(node,data))
for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
{'expression': cmdstr}):
ex = exp['databynode']
for node in ex:
cmd = ex[node]['value'].encode('utf-8')
cmdv = shlex.split(cmd)
if currprocs < concurrentprocs:
currprocs += 1
run_cmdv(node, cmdv, all, pipedesc)
else:
sys.stderr.write('{0}: {1}'.format(node, data))
else:
pop = desc['popen']
ret = pop.poll()
if ret is not None:
exitcode = exitcode | ret
all.discard(r)
if all:
rdy, _, _ = select.select(all, [], [], 10)
sys.exit(exitcode)
pendingexecs.append((node, cmdv))
exitcode = 0
rdy, _, _ = select.select(all, [], [], 10)
while all:
for r in rdy:
data = r.readline()
desc = pipedesc[r]
if data:
node = desc['node']
if desc['type'] == 'stdout':
sys.stdout.write('{0}: {1}'.format(node,data))
else:
sys.stderr.write('{0}: {1}'.format(node, data))
else:
pop = desc['popen']
ret = pop.poll()
if ret is not None:
exitcode = exitcode | ret
all.discard(r)
if desc['type'] == 'stdout' and pendingexecs:
node, cmdv = pendingexecs.popleft()
run_cmdv(node, cmdv, all, pipedesc)
if all:
rdy, _, _ = select.select(all, [], [], 10)
sys.exit(exitcode)
def run_cmdv(node, cmdv, all, pipedesc):
nopen = subprocess.Popen(
cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
'type': 'stdout'}
pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
'type': 'stderr'}
all.add(nopen.stdout)
all.add(nopen.stderr)
if __name__ == '__main__':
run()
confluent_client/bin/nodeshell
+70 -51
View File
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import deque
import optparse
import os
import select
@@ -35,57 +36,75 @@ if path.startswith('/opt'):
import confluent.client as client
argparser = optparse.OptionParser(
usage="Usage: %prog node commandexpression",
epilog="Expressions are the same as in attributes, e.g. "
"'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
argparser.disable_interspersed_args()
(options, args) = argparser.parse_args()
if len(args) < 2:
argparser.print_help()
sys.exit(1)
c = client.Command()
cmdstr = " ".join(args[1:])
def run():
concurrentprocs = 168
# among other things, FD_SETSIZE limits. Besides, spawning too many
# processes can be unkind for the unaware on memory pressure and such...
argparser = optparse.OptionParser(
usage="Usage: %prog node commandexpression",
epilog="Expressions are the same as in attributes, e.g. "
"'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
argparser.disable_interspersed_args()
(options, args) = argparser.parse_args()
if len(args) < 2:
argparser.print_help()
sys.exit(1)
c = client.Command()
cmdstr = " ".join(args[1:])
nodeforpopen = {}
popens = []
for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
{'expression': cmdstr}):
ex = exp['databynode']
for node in ex:
cmd = ex[node]['value'].encode('utf-8')
cmdv = ['ssh', node] + shlex.split(cmd)
nopen = subprocess.Popen(
cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
popens.append(nopen)
nodeforpopen[nopen] = node
currprocs = 0
all = set([])
pipedesc = {}
pendingexecs = deque()
all = set([])
pipedesc = {}
exitcode = 0
for pop in popens:
node = nodeforpopen[pop]
pipedesc[pop.stdout] = { 'node': node, 'popen': pop, 'type': 'stdout'}
pipedesc[pop.stderr] = {'node': node, 'popen': pop, 'type': 'stderr'}
all.add(pop.stdout)
all.add(pop.stderr)
rdy, _, _ = select.select(all, [], [], 10)
while all:
for r in rdy:
data = r.readline()
desc = pipedesc[r]
if data:
node = desc['node']
if desc['type'] == 'stdout':
sys.stdout.write('{0}: {1}'.format(node,data))
for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
{'expression': cmdstr}):
ex = exp['databynode']
for node in ex:
cmd = ex[node]['value'].encode('utf-8')
cmdv = ['ssh', node] + shlex.split(cmd)
if currprocs < concurrentprocs:
currprocs += 1
run_cmdv(node, cmdv, all, pipedesc)
else:
sys.stderr.write('{0}: {1}'.format(node, data))
else:
pop = desc['popen']
ret = pop.poll()
if ret is not None:
exitcode = exitcode | ret
all.discard(r)
if all:
rdy, _, _ = select.select(all, [], [], 10)
sys.exit(exitcode)
pendingexecs.append((node, cmdv))
exitcode = 0
rdy, _, _ = select.select(all, [], [], 10)
while all:
for r in rdy:
data = r.readline()
desc = pipedesc[r]
if data:
node = desc['node']
if desc['type'] == 'stdout':
sys.stdout.write('{0}: {1}'.format(node,data))
else:
sys.stderr.write('{0}: {1}'.format(node, data))
else:
pop = desc['popen']
ret = pop.poll()
if ret is not None:
exitcode = exitcode | ret
all.discard(r)
if desc['type'] == 'stdout' and pendingexecs:
node, cmdv = pendingexecs.popleft()
run_cmdv(node, cmdv, all, pipedesc)
if all:
rdy, _, _ = select.select(all, [], [], 10)
sys.exit(exitcode)
def run_cmdv(node, cmdv, all, pipedesc):
nopen = subprocess.Popen(
cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pipedesc[nopen.stdout] = {'node': node, 'popen': nopen,
'type': 'stdout'}
pipedesc[nopen.stderr] = {'node': node, 'popen': nopen,
'type': 'stderr'}
all.add(nopen.stdout)
all.add(nopen.stderr)
if __name__ == '__main__':
run()