Try restarting confluent if running on update

This mitigates chance of confluent being stale.

confluent_client/bin/nodeeventlog

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015-2016 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,29 +28,25 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [clear]")
+    usage="Usage: %prog [options] noderange (clear)")
 (options, args) = argparser.parse_args()
-
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 
 deletemode = False
-if len(args) == 2:
-    if args[1] == 'clear':
+if len(sys.argv) == 3:
+    if sys.argv[2] == 'clear':
         deletemode = True
-    else:
-        argparser.print_usage()
-        sys.exit(1)
 
 session = client.Command()
 exitcode = 0
 
 def format_event(evt):
     retparts = []
-    if 'timestamp' in evt:
+    if 'timestamp' in evt and evt['timestamp'] is not None:
         display = dt.strptime(evt['timestamp'], '%Y-%m-%dT%H:%M:%S')
         retparts.append(display.strftime('%m/%d/%Y %H:%M:%S'))
     dscparts = []
@@ -86,10 +82,6 @@ for rsp in func('/noderange/{0}/events/hardware/log'.format(noderange)):
                sys.stderr.write('{0}: {1}\n'.format(node, thisdata['error']))
                exitcode |= 1
             if 'events' in thisdata:
-                evtdata = thisdata['events']
-                for evt in evtdata:
-                    try:
-                        print('{0}: {1}'.format(node, format_event(evt)))
-                    except IOError:
-                        sys.exit(0)
-
+               evtdata = thisdata['events']
+               for evt in evtdata:
+                   print '{0}: {1}'.format(node, format_event(evt))

confluent_client/bin/nodefirmware

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,9 +25,6 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange")
-(options, args) = argparser.parse_args()
 
 exitcode = 0
 
@@ -61,10 +58,12 @@ def printfirm(node, prefix, data):
     print('{0}: {1}: {2}'.format(node, prefix, version))
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()

confluent_client/bin/nodehealth

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,16 +27,14 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange")
-(options, args) = argparser.parse_args()
-
 sys.stdout = codecs.getwriter('utf8')(sys.stdout)
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 
 

confluent_client/bin/nodeidentify

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,22 +26,17 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [on|off]")
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange> [on|off]")
 (options, args) = argparser.parse_args()
-
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 
 identifystate = None
-if len(args) > 1:
-    identifystate = args[1]
-    if identifystate not in ('on', 'off'):
-        argparser.print_usage()
-        sys.exit(1)
+if len(sys.argv) > 2:
+    identifystate = sys.argv[2]
 session = client.Command()
 exitcode = 0
 sys.exit(

confluent_client/bin/nodeinventory

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2016 Lenovo
+# Copyright 2016-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,9 +25,6 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange")
-(options, args) = argparser.parse_args()
 
 def pretty(text):
     if text == 'pcislot':
@@ -73,10 +70,12 @@ def printerror(res, node=None):
             exitcode = 1
 
 
+argparser = optparse.OptionParser(usage="Usage: %prog <noderange>")
+(options, args) = argparser.parse_args()
 try:
-    noderange = args[0]
+    noderange = sys.argv[1]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 try:
     session = client.Command()

confluent_client/bin/nodelist

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ def attrrequested(attr, attrlist, seenattributes):
             return True
     return False
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [attributes...]")
+    usage="Usage: %prog [options] noderange [list of attributes]")
 argparser.add_option('-b', '--blame', action='store_true',
                      help='Show information about how attributes inherited')
 (options, args) = argparser.parse_args()

confluent_client/bin/nodepower

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ import optparse
 import os
 import sys
 
-valid_states = ('on', 'off', 'boot', 'reset', 'shutdown')
 path = os.path.dirname(os.path.realpath(__file__))
 path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
 if path.startswith('/opt'):
@@ -28,25 +27,21 @@ if path.startswith('/opt'):
 import confluent.client as client
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [{0}]".format(
-        '|'.join(valid_states)))
+    usage="Usage: %prog [options] noderange "
+          "([status|on|off|shutdown|boot|reset])")
 (options, args) = argparser.parse_args()
-
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 
 setstate = None
-if len(args) > 1:
-    if not args[1] in ('stat', 'state', 'status'):
-        setstate = args[1]
+if len(sys.argv) > 2:
     if setstate == 'softoff':
         setstate = 'shutdown'
-    if setstate is not None and setstate not in valid_states:
-        argparser.print_usage()
-        sys.exit(1)
+    elif not sys.argv[2] in ('stat', 'state', 'status'):
+        setstate = sys.argv[2]
 
 session = client.Command()
 exitcode = 0

confluent_client/bin/noderun

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2016-2017 Lenovo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os
+import select
+import shlex
+import subprocess
+import sys
+
+path = os.path.dirname(os.path.realpath(__file__))
+path = os.path.realpath(os.path.join(path, '..', 'lib', 'python'))
+if path.startswith('/opt'):
+    sys.path.append(path)
+
+import confluent.client as client
+
+
+argparser = optparse.OptionParser(
+    usage="Usage: %prog node commandexpression",
+    epilog="Expressions are the same as in attributes, e.g. "
+           "'ipmitool -H {hardwaremanagement.manager}' will be expanded.")
+argparser.disable_interspersed_args()
+(options, args) = argparser.parse_args()
+if len(args) < 2:
+    argparser.print_help()
+    sys.exit(1)
+c = client.Command()
+cmdstr = " ".join(args[1:])
+
+nodeforpopen = {}
+popens = []
+for exp in c.create('/noderange/{0}/attributes/expression'.format(args[0]),
+                    {'expression': cmdstr}):
+    ex = exp['databynode']
+    for node in ex:
+        cmd = ex[node]['value'].encode('utf-8')
+        cmdv = shlex.split(cmd)
+        nopen = subprocess.Popen(
+            cmdv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        popens.append(nopen)
+        nodeforpopen[nopen] = node
+
+all = set([])
+pipedesc = {}
+exitcode = 0
+for pop in popens:
+    node = nodeforpopen[pop]
+    pipedesc[pop.stdout] = { 'node': node, 'popen': pop, 'type': 'stdout'}
+    pipedesc[pop.stderr] = {'node': node, 'popen': pop, 'type': 'stderr'}
+    all.add(pop.stdout)
+    all.add(pop.stderr)
+rdy, _, _ = select.select(all, [], [], 10)
+while all and rdy:
+    for r in rdy:
+        data = r.readline()
+        desc = pipedesc[r]
+        if data:
+            node = desc['node']
+            if desc['type'] == 'stdout':
+                sys.stdout.write('{0}: {1}'.format(node,data))
+            else:
+                sys.stderr.write('{0}: {1}'.format(node, data))
+        else:
+            pop = desc['popen']
+            ret = pop.poll()
+            if ret is not None:
+                exitcode = exitcode | ret
+                all.discard(r)
+    if all:
+        rdy, _, _ = select.select(all, [], [], 10)
+sys.exit(exitcode)

confluent_client/bin/nodesensors

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import csv
+import datetime
 import optparse
 import os
 import sys
@@ -39,8 +40,8 @@ sensorcollections = {
 
 
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [sensor(s)]")
-argparser.add_option('-i', '--interval', type='int',
+    usage="Usage: %prog [options] noderange ([sensor(s)])")
+argparser.add_option('-i', '--interval', type='float',
                      help='Interval to do repeated samples over')
 argparser.add_option('-n', '--numreadings', type='int',
                      help='Number of readings to gather')
@@ -59,7 +60,7 @@ if options.numreadings:
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 sensors = []
 for sensorgroup in args[1:]:
@@ -146,7 +147,12 @@ def sensorpass(showout=True, appendtime=False):
 def format_csv(csvwriter, orderedsensors, resdata, showtime=True):
     for nodekey in resdata:
         if showtime:
-            rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            if showtime.is_integer():
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S'), nodekey]
+            else:
+                rowdata = [time.strftime('%Y-%m-%dT%H:%M:%S.') +
+                           str(datetime.datetime.now().microsecond//1000),
+                           nodekey]
         else:
             rowdata = [nodekey]
         for sensorkey in orderedsensors:
@@ -197,7 +203,8 @@ def main():
             nextstart = os.times()[4] + options.interval
             resdata = sensorpass(linebyline, True)
             if options.csv:
-                format_csv(csvwriter, orderedsensors, resdata)
+                format_csv(csvwriter, orderedsensors, resdata,
+                           showtime=options.interval)
             if options.numreadings:
                 options.numreadings -= 1
                 if options.numreadings <= 0:

confluent_client/bin/nodesetboot

@@ -26,10 +26,8 @@ if path.startswith('/opt'):
 
 import confluent.client as client
 
-valid_targets = ('default', 'network', 'cd', 'setup', 'hd')
 argparser = optparse.OptionParser(
-    usage="Usage: %prog [options] noderange [{0}]".format(
-        '|'.join(valid_targets)))
+    usage='Usage: %prog [options] noderange [default|cd|network|setup|hd]')
 argparser.add_option('-b', '--bios', dest='biosmode',
                      action='store_true', default=False,
                      help='Request BIOS style boot (rather than UEFI)')
@@ -43,23 +41,19 @@ argparser.add_option('-p', '--persist', dest='persist', action='store_true',
 try:
     noderange = args[0]
 except IndexError:
-    argparser.print_usage()
+    argparser.print_help()
     sys.exit(1)
 bootdev = None
-if len(args) > 1:
-    bootdev = args[1]
+if len(sys.argv) > 2:
+    bootdev = sys.argv[2]
     if bootdev in ('net', 'pxe'):
         bootdev = 'network'
-    if bootdev not in valid_targets:
-        argparser.print_usage()
-        sys.exit(1)
 session = client.Command()
 exitcode = 0
 if options.biosmode:
     bootmode = 'bios'
 else:
     bootmode = 'uefi'
-sys.exit(
-    session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
+sys.exit(session.simple_noderange_command(noderange, '/boot/nextdevice', bootdev,
                                      bootmode=bootmode,
                                      persistent=options.persist))

confluent_client/doc/man/confetty.ronn

@@ -0,0 +1,37 @@
+confetty(1) --- Interactive confluent client
+=================================================
+
+## SYNOPSIS
+
+`confetty`
+
+## DESCRIPTION
+
+**confetty** launches an interactive CLI session to the
+confluent service.  It provides a filesystem-like
+view of the confluent interface.  It is intended to
+be mostly an aid for developing client software, with
+day to day administration generally being easier with
+the various function specific commands.
+
+## COMMANDS
+
+The CLI may be navigated by shell commands and some other
+commands.
+
+* `cd`:
+  Change the location within the tree
+* `ls`:
+  List the elements within the current directory/tree
+* `show` **ELEMENT**, `cat` **ELEMENT**:
+  Display the result of reading a specific element (by full or relative path)
+* `unset` **ELEMENT** **ATTRIBUTE**
+  For an element with attributes, request to clear the value of the attribue
+* `set` **ELEMENT** **ATTRIBUTE**=**VALUE**
+  Set the specified attribute to the given value
+* `start` **ELEMENT**
+  Start a console session indicated by **ELEMENT** (e.g. /nodes/n1/console/session)
+* `rm` **ELEMENT**
+  Request removal of an element.  (e.g. rm events/hardware/log clears log from a node)
+
+

confluent_client/doc/man/nodeconsole.ronn

@@ -0,0 +1,30 @@
+nodeconsole(1) -- Open a console to a confluent node
+=====================================================
+
+## SYNOPSIS
+`nodeconsole` `node`
+
+## DESCRIPTION
+
+**nodeconsole** opens an interactive console session to a given node.  This is the
+text or serial console of a system.  Exiting is done by hitting `Ctrl-e`, then `c`,
+ then `.`.  Note that console output by default is additionally logged to 
+`/var/log/confluent/consoles/`**NODENAME**.
+
+## ESCAPE SEQUENCE COMMANDS
+
+While connected to a console, a number of commands may be performed through escape
+sequences.  To begin an command escape sequence, hit `Ctrl-e`, then `c`.  The next
+keystroke will be interpreted as a command.  The following commands are available.
+
+* `.`:
+  Exit the session and return to the command prompt
+* `b`:
+  Send a break to the remote console when possible (some console plugins may not support this)
+* `o`:
+  Request confluent to disconnect and reconnect to console.  For example if there is suspicion
+  that the console has gone inoperable, but would work if reconnected.
+* `?`:
+  Get a list of supported commands
+* `<enter>`:
+  Abandon entering an escape sequence command

confluent_client/doc/man/nodelist.ronn

@@ -0,0 +1,48 @@
+nodelist(1) -- List confluent nodes and their attributes
+=========================================================
+
+## SYNOPSIS
+
+`nodelist` `noderange`  
+`nodelist` `noderange` [-b] [<nodeattribute>...]  
+
+## DESCRIPTION
+
+**nodelist** queries the confluent server to get information about nodes.  In
+the simplest form, it simply takes the given noderange(5) and lists the
+matching nodes, one line at a time.
+
+If a list of node attribute names are given, the value of those are also
+displayed.  If `-b` is specified, it will also display information on
+how inherited and expression based attributes are defined.  There is more
+information on node attributes in nodeattributes(5) man page.
+
+## OPTIONS
+
+* `-b`, `--blame`:
+  Annotate inherited and expression based attributes to show their base value.
+
+## EXAMPLES
+* Listing matching nodes of a simple noderange:
+  `# nodelist n1-n4`  
+  `n1`  
+  `n2`  
+  `n3`  
+  `n4`  
+
+* Getting an attribute of nodes matching a noderange:
+  `# nodelist n1,n2 hardwaremanagement.manager`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+
+* Getting a group of attributes while determining what group defines them:
+  `# nodelist n1,n2 hardwaremanegement --blame`  
+  `n1: hardwaremanagement.manager: 172.30.3.1`  
+  `n1: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n1: hardwaremanagement.switch: r8e1`  
+  `n1: hardwaremanagement.switchport: 14`  
+  `n2: hardwaremanagement.manager: 172.30.3.2`  
+  `n2: hardwaremanagement.method: ipmi (inherited from group everything)`  
+  `n2: hardwaremanagement.switch: r8e1`  
+  `n2: hardwaremanagement.switchport: 2`  
+

confluent_client/doc/man/nodesensors.ronn

@@ -0,0 +1,55 @@
+nodesensors(1) --- Retrieve telemetry for sensors of confluent nodes
+====================================================================
+
+## SYNOPSIS
+
+`nodesensors` `noderange` [-c] [-i <interval>] [-n <samplecount>]  [<sensor name  or category>...]  
+
+
+## DESCRIPTION
+
+**nodesensors** queries the confluent server to get telemetry from nodes.  Telemetry can include
+data such as temperature, power, and so forth.  Without arguments, it lists all available sensors
+and their current values.  If `-c` is specified, CSV format is used for output.  Normally
+nodesensors outputs once and exits.  Repeated periodic gathering can be done with `-i` to specify
+interval and `-n` to specify number of requests.  If `-i` is specified without `-n`, then it will
+retrieve data at the requested interval indefinitely.  If '-n' is specified without `-i`, an
+interval of 1 second is used.
+
+## OPTIONS
+
+* `-c`, `--csv`:
+  Organize output into CSV format, one sensor per column.
+
+* `-i`, `--interval`=**SECONDS**:
+  Repeat data gathering waiting, waiting the specified time between samples.  Unless `-n` is
+  specified, indefinite retrieval is assumed.
+
+* `-n`, `--numreadings`=**SAMPLES**:
+  Perform the specified number of readings, waiting `-i` indicated interval or 1 second if not
+  otherwise indicated.
+
+## EXAMPLES
+* Retrieving all temperature related sensors from one system  
+  `# nodesensors n1 temperature`  
+  `n1: CPU 1 Overtemp: Ok`  
+  `n1: CPU 2 Overtemp: Ok`  
+  `n1: Inlet Temp: 16.0 °C`  
+  `n1: PCH Overtemp: Ok`  
+  `n1: LOM Temp: Ok`  
+
+* Retrieving a sensor named "Inlet Temp" for 4 systems over a 3 second period of time:  
+   `# nodesensors n1-n4 'Inlet Temp' -c -n 3`  
+   `time,node,Inlet Temp (°C)`  
+   `2016-10-04T15:09:20,n1,19.0`  
+   `2016-10-04T15:09:20,n2,18.0`  
+   `2016-10-04T15:09:20,n3,18.0`  
+   `2016-10-04T15:09:20,n4,17.0`  
+   `2016-10-04T15:09:21,n1,19.0`  
+   `2016-10-04T15:09:21,n2,18.0`  
+   `2016-10-04T15:09:21,n3,18.0`  
+   `2016-10-04T15:09:21,n4,17.0`  
+   `2016-10-04T15:09:22,n1,19.0`  
+   `2016-10-04T15:09:22,n2,18.0`  
+   `2016-10-04T15:09:22,n3,18.0`  
+   `2016-10-04T15:09:22,n4,17.0`  

confluent_client/setup.py.tmpl

@@ -10,6 +10,6 @@ setup(
     scripts=['bin/confetty', 'bin/nodeconsole', 'bin/nodeeventlog',
              'bin/nodefirmware', 'bin/nodehealth', 'bin/nodeidentify',
              'bin/nodeinventory', 'bin/nodelist', 'bin/nodepower',
-             'bin/nodesensors', 'bin/nodesetboot'],
+             'bin/nodesensors', 'bin/nodesetboot', 'bin/noderun'],
     data_files=[('/etc/profile.d', ['confluent_env.sh'])],
 )

confluent_server/confluent/config/configmanager.py

@@ -761,6 +761,14 @@ class ConfigManager(object):
                                                    decrypt=self.decrypt)
         return nodeobj
 
+    def expand_attrib_expression(self, nodelist, expression):
+        if type(nodelist) in (unicode, str):
+            nodelist = (nodelist,)
+        for node in nodelist:
+            cfgobj = self._cfgstore['nodes'][node]
+            fmt = _ExpressionFormat(cfgobj, node)
+            yield (node, fmt.format(expression))
+
     def get_node_attributes(self, nodelist, attributes=(), decrypt=None):
         if decrypt is None:
             decrypt = self.decrypt

confluent_server/confluent/core.py

@@ -122,6 +122,7 @@ def _init_core():
         'attributes': {
             'all': PluginRoute({'handler': 'attributes'}),
             'current': PluginRoute({'handler': 'attributes'}),
+            'expression': PluginRoute({'handler': 'attributes'}),
         },
         'boot': {
             'nextdevice': PluginRoute({

confluent_server/confluent/httpapi.py

@@ -35,6 +35,7 @@ import eventlet.greenthread
 import greenlet
 import json
 import socket
+import sys
 import traceback
 import time
 import urlparse
@@ -233,6 +234,17 @@ def _csrf_valid(env, session):
             # oblige the request and apply a new token to the
             # session
             session['csrftoken'] = util.randomstring(32)
+        elif 'HTTP_REFERER' in env:
+            # If there is a referrer, make sure it stays consistent
+            # across the session.  A change in referer is a bad thing
+            try:
+                referer = env['HTTP_REFERER'].split('/')[2]
+            except IndexError:
+                return False
+            if 'validreferer' not in session:
+                session['validreferer'] = referer
+            elif session['validreferer'] != referer:
+                return False
         return True
     # The session has CSRF protection enabled, only mark valid if
     # the client has provided an auth token and that token matches the
@@ -257,15 +269,15 @@ def _authorize_request(env, operation):
             sessionid = cc['confluentsessionid'].value
             sessid = sessionid
             if sessionid in httpsessions:
-                if env['PATH_INFO'] == '/sessions/current/logout':
-                    targets = []
-                    for mythread in httpsessions[sessionid]['inflight']:
-                        targets.append(mythread)
-                    for mythread in targets:
-                        eventlet.greenthread.kill(mythread)
-                    del httpsessions[sessionid]
-                    return ('logout',)
                 if _csrf_valid(env, httpsessions[sessionid]):
+                    if env['PATH_INFO'] == '/sessions/current/logout':
+                        targets = []
+                        for mythread in httpsessions[sessionid]['inflight']:
+                            targets.append(mythread)
+                        for mythread in targets:
+                            eventlet.greenthread.kill(mythread)
+                        del httpsessions[sessionid]
+                        return ('logout',)
                     httpsessions[sessionid]['expiry'] = time.time() + 90
                     name = httpsessions[sessionid]['name']
                     authdata = auth.authorize(
@@ -273,6 +285,12 @@ def _authorize_request(env, operation):
                         skipuserobj=httpsessions[sessionid]['skipuserobject'])
     if (not authdata) and 'HTTP_AUTHORIZATION' in env:
         if env['PATH_INFO'] == '/sessions/current/logout':
+            if 'HTTP_REFERER' in env:
+                # note that this doesn't actually do harm
+                # otherwise, but this way do not give appearance
+                # of something having a side effect if it has the smell
+                # of a CSRF
+                return {'code': 401}
             return ('logout',)
         name, passphrase = base64.b64decode(
             env['HTTP_AUTHORIZATION'].replace('Basic ', '')).split(':', 1)
@@ -290,6 +308,7 @@ def _authorize_request(env, operation):
         cookie['confluentsessionid'] = sessid
         cookie['confluentsessionid']['secure'] = 1
         cookie['confluentsessionid']['httponly'] = 1
+        cookie['confluentsessionid']['path'] = '/'
     skiplog = _should_skip_authlog(env)
     if authdata:
         auditmsg = {
@@ -306,8 +325,8 @@ def _authorize_request(env, operation):
             auditmsg['tenant'] = authdata[3]
             authinfo['tenant'] = authdata[3]
         auditmsg['user'] = authdata[2]
-        if sessionid is not None:
-            authinfo['sessionid'] = sessionid
+        if sessid is not None:
+            authinfo['sessionid'] = sessid
         if not skiplog:
             auditlog.log(auditmsg)
         if 'csrftoken' in httpsessions[sessid]:
@@ -368,7 +387,13 @@ def resourcehandler_backend(env, start_response):
     """Function to handle new wsgi requests
     """
     mimetype, extension = _pick_mimetype(env)
-    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-cache')]
+    headers = [('Content-Type', mimetype), ('Cache-Control', 'no-store'),
+               ('Pragma', 'no-cache'),
+               ('X-Content-Type-Options', 'nosniff'),
+               ('Content-Security-Policy', "default-src 'self'"),
+               ('X-XSS-Protection', '1'), ('X-Frame-Options', 'deny'),
+               ('Strict-Transport-Security', 'max-age=86400'),
+               ('X-Permitted-Cross-Domain-Policies', 'none')]
     reqbody = None
     reqtype = None
     if 'CONTENT_LENGTH' in env and int(env['CONTENT_LENGTH']) > 0:
@@ -717,9 +742,20 @@ def serve(bind_host, bind_port):
     #but deps are simpler without flup
     #also, the potential for direct http can be handy
     #todo remains unix domain socket for even http
-    eventlet.wsgi.server(
-        eventlet.listen((bind_host, bind_port, 0, 0), family=socket.AF_INET6),
-        resourcehandler, log=False, log_output=False, debug=False)
+    sock = None
+    while not sock:
+        try:
+            sock = eventlet.listen(
+                (bind_host, bind_port, 0, 0), family=socket.AF_INET6)
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write(
+                'Failed to open HTTP due to busy port, trying again in'
+                ' a second\n')
+            eventlet.sleep(1)
+    eventlet.wsgi.server(sock, resourcehandler, log=False, log_output=False,
+                         debug=False)
 
 
 class HttpApi(object):

confluent_server/confluent/log.py

@@ -444,12 +444,17 @@ class TimedAndSizeRotatingFileHandler(BaseRotatingHandler):
         # As time rolling happens, reset statistics count
         self.sizeRollingCount = 0
         dbfn = self.binpath + "." + time.strftime(self.suffix, timeTuple)
+        odbfn = dbfn
         dtfn = self.textpath + "." + time.strftime(self.suffix, timeTuple)
-
-        if os.path.exists(dbfn):
-            os.remove(dbfn)
-        if os.path.exists(dtfn):
-            os.remove(dtfn)
+        odtfn = dtfn
+        append=1
+        while os.path.exists(dbfn):
+            dbfn = odbfn + '.{}'.format(append)
+            append += 1
+        append=1
+        while os.path.exists(dtfn):
+            dtfn = odtfn + '.{}'.format(append)
+            append += 1
         if os.path.exists(self.binpath):
             os.rename(self.binpath, dbfn)
         if os.path.exists(self.textpath):
@@ -613,8 +618,14 @@ class Logger(object):
                 struct.unpack(">BBIHIBBH", recbytes)
             # rolling events found.
             if ltype == DataTypes.event and evtdata == Events.logrollover:
-                textpath, binpath = parse_last_rolling_files(textfile, offset,
-                                                             datalen)
+                txtpath, bpath = parse_last_rolling_files(textfile, offset,
+                                                          datalen)
+                if txtpath == textpath:
+                    break
+                if bpath == binpath:
+                    break
+                textpath = txtpath
+                binpath = bpath
                 # Rolling event detected, close the current bin file, then open
                 # the renamed bin file.
                 flock(binfile, LOCK_UN)

confluent_server/confluent/main.py

@@ -1,7 +1,7 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2014 IBM Corporation
-# Copyright 2015 Lenovo
+# Copyright 2015-2017 Lenovo
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -95,9 +95,15 @@ def _checkpidfile():
         fcntl.flock(pidfile, fcntl.LOCK_EX)
         pid = pidfile.read()
         if pid != '':
-            print ('/var/run/confluent/pid exists and indicates %s is still '
-                   'running' % pid)
-            sys.exit(1)
+            try:
+                os.kill(int(pid), 0)
+                print ('/var/run/confluent/pid exists and indicates %s is still '
+                       'running' % pid)
+                sys.exit(1)
+            except OSError:
+                # There is no process running by that pid, must be stale
+                pass
+        pidfile.seek(0)
         pidfile.write(str(os.getpid()))
         fcntl.flock(pidfile, fcntl.LOCK_UN)
         pidfile.close()
@@ -199,10 +205,12 @@ def run():
     auth.init_auth()
     signal.signal(signal.SIGINT, terminate)
     signal.signal(signal.SIGTERM, terminate)
-    #TODO(jbjohnso): eventlet has a bug about unix domain sockets, this code
-    #works with bugs fixed
     if dbgif:
         oumask = os.umask(0077)
+        try:
+            os.remove('/var/run/confluent/dbg.sock')
+        except OSError:
+            pass  # We are not expecting the file to exist
         dbgsock = eventlet.listen("/var/run/confluent/dbg.sock",
                                    family=socket.AF_UNIX)
         eventlet.spawn_n(backdoor.backdoor_server, dbgsock)

confluent_server/confluent/plugins/configuration/attributes.py

@@ -152,6 +152,20 @@ def update_nodegroup(group, element, configmanager, inputdata):
     return retrieve_nodegroup(group, element, configmanager, inputdata)
 
 
+def _expand_expression(nodes, configmanager, inputdata):
+    expression = inputdata.get_attributes(list(nodes)[0])
+    if type(expression) is dict:
+        expression = expression['expression']
+    if type(expression) is dict:
+        expression = expression['expression']
+    for expanded in configmanager.expand_attrib_expression(nodes, expression):
+        yield msg.KeyValueData({'value': expanded[1]}, expanded[0])
+
+
+def create(nodes, element, configmanager, inputdata):
+    if nodes is not None and element[-1] == 'expression':
+        return _expand_expression(nodes, configmanager, inputdata)
+
 def update_nodes(nodes, element, configmanager, inputdata):
     updatedict = {}
     for node in nodes:

confluent_server/confluent/plugins/hardwaremanagement/ipmi.py

@@ -344,8 +344,8 @@ class IpmiHandler(object):
             except socket.gaierror as ge:
                 if ge[0] == -2:
                     raise exc.TargetEndpointUnreachable(ge[1])
+                raise
         self.ipmicmd = persistent_ipmicmds[(node, tenant)]
-        self.ipmicmd.setup_confluent_keyhandler()
 
     bootdevices = {
         'optical': 'cd'
@@ -356,7 +356,9 @@ class IpmiHandler(object):
             self.broken = True
             self.error = response['error']
         else:
+            self.ipmicmd = ipmicmd
             self.loggedin = True
+            self.ipmicmd.setup_confluent_keyhandler()
         self._logevt.set()
 
     def handle_request(self):

confluent_server/confluent/sockapi.py

@@ -244,7 +244,16 @@ def _tlshandler(bind_host, bind_port):
     plainsocket = socket.socket(socket.AF_INET6)
     plainsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     plainsocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-    plainsocket.bind((bind_host, bind_port, 0, 0))
+    bound = False
+    while not bound:
+        try:
+            plainsocket.bind((bind_host, bind_port, 0, 0))
+            bound = True
+        except socket.error as e:
+            if e.errno != 98:
+                raise
+            sys.stderr.write('TLS Socket in use, retrying in 1 second\n')
+            eventlet.sleep(1)
     plainsocket.listen(5)
     while (1):  # TODO: exithook
         cnn, addr = plainsocket.accept()

confluent_server/confluent_server.spec.tmpl

@@ -12,7 +12,7 @@ Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Prefix: %{_prefix}
 BuildArch: noarch
-Requires: python-pyghmi, python-eventlet, python-greenlet, python-crypto >= 2.6.1, confluent_client, pyparsing, python-paramiko, python-dnspython
+Requires: python-pyghmi, python-eventlet, python-greenlet, python-crypto >= 2.6.1, confluent_client, pyparsing, python-paramiko, python-dns
 Vendor: Jarrod Johnson <jjohnson2@lenovo.com>
 Url: http://xcat.sf.net/
 
@@ -33,6 +33,9 @@ done
 grep -v confluent/__init__.py INSTALLED_FILES.bare > INSTALLED_FILES
 cat INSTALLED_FILES
 
+%post
+if [ -x /usr/bin/systemctl ]; then /usr/bin/systemctl try-restart confluent; fi
+
 %clean
 rm -rf $RPM_BUILD_ROOT